diff --git a/app/bin/service/dartdoc.dart b/app/bin/service/dartdoc.dart index 39580a2daa..3412dfa7c3 100644 --- a/app/bin/service/dartdoc.dart +++ b/app/bin/service/dartdoc.dart @@ -68,6 +68,8 @@ Future _workerMain(WorkerEntryMessage message) async { final jobProcessor = new DartdocJobProcessor(lockDuration: const Duration(minutes: 30)); + await jobProcessor.generateDocsForSdk(); + final jobMaintenance = new JobMaintenance(dbService, jobProcessor); new Timer.periodic(const Duration(minutes: 15), (_) async { diff --git a/app/bin/service/search.dart b/app/bin/service/search.dart index 69dfe067e5..98c9218e97 100644 --- a/app/bin/service/search.dart +++ b/app/bin/service/search.dart @@ -11,6 +11,7 @@ import 'package:gcloud/service_scope.dart'; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; +import 'package:pub_dartlang_org/dartdoc/backend.dart'; import 'package:pub_dartlang_org/shared/analyzer_client.dart'; import 'package:pub_dartlang_org/shared/analyzer_memcache.dart'; import 'package:pub_dartlang_org/shared/configuration.dart'; @@ -23,6 +24,8 @@ import 'package:pub_dartlang_org/shared/service_utils.dart'; import 'package:pub_dartlang_org/shared/task_client.dart'; import 'package:pub_dartlang_org/shared/task_scheduler.dart'; import 'package:pub_dartlang_org/shared/task_sources.dart'; +import 'package:pub_dartlang_org/shared/versions.dart'; +import 'package:pub_dartlang_org/shared/urls.dart'; import 'package:pub_dartlang_org/search/backend.dart'; import 'package:pub_dartlang_org/search/handlers.dart'; @@ -56,6 +59,9 @@ Future _main(FrontendEntryMessage message) async { registerAnalyzerClient(analyzerClient); registerScopeExitCallback(analyzerClient.close); + final Bucket dartdocBucket = await getOrCreateBucket( + storageService, activeConfiguration.dartdocStorageBucketName); + registerDartdocBackend(new DartdocBackend(db.dbService, dartdocBucket)); registerDartdocMemcache(new DartdocMemcache(memcacheService)); final DartdocClient dartdocClient = new DartdocClient(); registerDartdocClient(dartdocClient); @@ -69,9 +75,15 @@ Future _main(FrontendEntryMessage message) async { new SnapshotStorage(storageService, snapshotBucket)); final ReceivePort taskReceivePort = new ReceivePort(); + registerDartSdkIndex(new SimplePackageIndex.sdk( + urlPrefix: dartSdkMainUrl(toolEnvSdkVersion))); registerPackageIndex(new SimplePackageIndex()); registerTaskSendPort(taskReceivePort.sendPort); + // Don't block on SDK index updates, as it may take several minutes before + // the dartdoc service produces the required output. + _updateDartSdkIndex().whenComplete(() {}); + final BatchIndexUpdater batchIndexUpdater = new BatchIndexUpdater(); await batchIndexUpdater.initSnapshot(); @@ -92,3 +104,26 @@ Future _main(FrontendEntryMessage message) async { await runHandler(_logger, searchServiceHandler); }); } + +Future _updateDartSdkIndex() async { + for (int i = 0;; i++) { + try { + _logger.info('Trying to load SDK index.'); + final data = await dartdocBackend.getDartSdkDartdocData(); + if (data != null) { + final docs = + splitLibraries(data).map((lib) => createSdkDocument(lib)).toList(); + await dartSdkIndex.addPackages(docs); + await dartSdkIndex.merge(); + _logger.info('Dart SDK index loaded successfully.'); + return; + } + } catch (e, st) { + _logger.warning('Error loading Dart SDK index.', e, st); + } + if (i % 10 == 0) { + _logger.warning('Unable to load Dart SDK index. Attempt: $i'); + } + await new Future.delayed(const Duration(minutes: 1)); + } +} diff --git a/app/lib/dartdoc/backend.dart b/app/lib/dartdoc/backend.dart index b07f102a00..056a3e1f0c 100644 --- a/app/lib/dartdoc/backend.dart +++ b/app/lib/dartdoc/backend.dart @@ -3,6 +3,7 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:gcloud/db.dart'; @@ -21,9 +22,11 @@ import '../shared/utils.dart' show contentType; import '../shared/versions.dart' as shared_versions; import 'models.dart'; +import 'pub_dartdoc_data.dart'; import 'storage_path.dart' as storage_path; final Logger _logger = new Logger('pub.dartdoc.backend'); +final _gzip = new GZipCodec(); final Duration _contentDeleteThreshold = const Duration(days: 1); final int _concurrentUploads = 8; @@ -43,6 +46,46 @@ class DartdocBackend { DartdocBackend(this._db, this._storage); + /// Whether the storage bucket has a usable extracted data file. + /// Only the existence of the file is checked. + // TODO: decide whether we should re-generate the file after a certain age + Future hasValidDartSdkDartdocData() async { + final objectName = + storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); + try { + final info = await _storage.info(objectName); + return info != null; + } catch (_) { + return false; + } + } + + /// Upload the generated dartdoc data file for the Dart SDK to the storage bucket. + Future uploadDartSdkDartdocData(File file) async { + final objectName = + storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); + try { + final sink = _storage.write(objectName); + await file.openRead().transform(_gzip.encoder).pipe(sink); + } catch (e, st) { + _logger.warning( + 'Unable to upload SDK pub dartdoc data file: $objectName', e, st); + } + } + + /// Read the generated dartdoc data file for the Dart SDK. + Future getDartSdkDartdocData() async { + final objectName = + storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); + final Map map = await _storage + .read(objectName) + .transform(_gzip.decoder) + .transform(utf8.decoder) + .transform(json.decoder) + .single; + return new PubDartdocData.fromJson(map); + } + /// Returns the latest stable version of a package. Future getLatestVersion(String package) async { final list = await _db.lookup([_db.emptyKey.append(Package, id: package)]); diff --git a/app/lib/dartdoc/dartdoc_runner.dart b/app/lib/dartdoc/dartdoc_runner.dart index 706b7ce574..9433a8d538 100644 --- a/app/lib/dartdoc/dartdoc_runner.dart +++ b/app/lib/dartdoc/dartdoc_runner.dart @@ -18,6 +18,7 @@ import '../job/job.dart'; import '../scorecard/backend.dart'; import '../scorecard/models.dart'; import '../shared/analyzer_client.dart'; +import '../shared/configuration.dart' show envConfig; import '../shared/tool_env.dart'; import '../shared/urls.dart'; import '../shared/versions.dart' as versions; @@ -32,7 +33,8 @@ final Uuid _uuid = new Uuid(); const statusFilePath = 'status.json'; const _archiveFilePath = 'package.tar.gz'; const _buildLogFilePath = 'log.txt'; -const _dartdocTimeout = const Duration(minutes: 10); +const _packageTimeout = const Duration(minutes: 10); +const _sdkTimeout = const Duration(minutes: 20); final Duration _twoYears = const Duration(days: 2 * 365); final _pkgPubDartdocDir = @@ -42,6 +44,57 @@ class DartdocJobProcessor extends JobProcessor { DartdocJobProcessor({Duration lockDuration}) : super(service: JobService.dartdoc, lockDuration: lockDuration); + /// Uses the tool environment's SDK (the one that is used for analysis too) to + /// generate dartdoc documentation and extracted data file for SDK API indexing. + /// Only the extracted data file will be used and uploaded. + Future generateDocsForSdk() async { + if (await dartdocBackend.hasValidDartSdkDartdocData()) return; + final tempDir = + await Directory.systemTemp.createTemp('pub-dartlang-dartdoc'); + try { + final tempDirPath = tempDir.resolveSymbolicLinksSync(); + final outputDir = tempDirPath; + final args = [ + '--sdk-docs', + '--output', + outputDir, + '--hosted-url', + siteRoot, + '--link-to-remote', + '--no-validate-links', + ]; + if (envConfig.toolEnvDartSdkDir != null) { + args.addAll(['--sdk-dir', envConfig.toolEnvDartSdkDir]); + } + final pr = await runProc( + 'dart', + ['bin/pub_dartdoc.dart']..addAll(args), + workingDirectory: _pkgPubDartdocDir, + timeout: _sdkTimeout, + ); + + final pubDataFile = new File(p.join(outputDir, 'pub-data.json')); + final hasPubData = await pubDataFile.exists(); + final isOk = pr.exitCode == 0 && hasPubData; + if (!isOk) { + _logger.warning( + 'Error while generating SDK docs.\n\n${pr.stdout}\n\n${pr.stderr}'); + throw new Exception( + 'Error while generating SDK docs (hasPubData: $hasPubData).'); + } + + // prevent close races updating the same content in close succession + if (await dartdocBackend.hasValidDartSdkDartdocData()) return; + + // upload only the pub dartdoc data file + await dartdocBackend.uploadDartSdkDartdocData(pubDataFile); + } catch (e, st) { + _logger.warning('Error while generating SDK docs.', e, st); + } finally { + await tempDir.delete(recursive: true); + } + } + @override Future shouldProcess( String package, String version, DateTime updated) async { @@ -231,7 +284,7 @@ class DartdocJobProcessor extends JobProcessor { 'dart', ['bin/pub_dartdoc.dart']..addAll(args), workingDirectory: _pkgPubDartdocDir, - timeout: _dartdocTimeout, + timeout: _packageTimeout, ); final hasIndexHtml = await new File(p.join(outputDir, 'index.html')).exists(); diff --git a/app/lib/dartdoc/storage_path.dart b/app/lib/dartdoc/storage_path.dart index 1faf38f71d..76b5b17a34 100644 --- a/app/lib/dartdoc/storage_path.dart +++ b/app/lib/dartdoc/storage_path.dart @@ -9,6 +9,7 @@ const _generatedStaticAssetPaths = const ['static-assets']; // Storage contains package data in a form of /package/version/... // This path contains '-' and is an invalid package name, safe of conflicts. const _storageSharedAssetPrefix = 'shared-assets'; +const _sdkAssetPrefix = 'sdk-assets'; /// Whether the generated file can be moved to the shared assets. bool isSharedAsset(String relativePath) { @@ -49,3 +50,13 @@ String contentObjectName(String packageName, String packageVersion, String uuid, String relativePath) { return p.join(contentPrefix(packageName, packageVersion, uuid), relativePath); } + +/// ObjectName of an SDK asset. +String sdkObjectName(String relativePath) { + return p.join(_sdkAssetPrefix, relativePath); +} + +/// The ObjectName for the Dart SDK's extracted dartdoc content. +String dartSdkDartdocDataName(String runtimeVersion) { + return sdkObjectName('dart/pub-dartdoc-data/$runtimeVersion.json.gz'); +} diff --git a/app/lib/frontend/models.dart b/app/lib/frontend/models.dart index 9b29ed0567..3efc8cbe2b 100644 --- a/app/lib/frontend/models.dart +++ b/app/lib/frontend/models.dart @@ -217,6 +217,8 @@ class PrivateKey extends db.Model { /// An extract of [Package] and [PackageVersion] and [AnalysisView], for /// display-only uses. class PackageView { + final bool isExternal; + final String url; final String name; final String version; // Not null only if there is a difference compared to the [version]. @@ -231,6 +233,8 @@ class PackageView { final List apiPages; PackageView({ + this.isExternal: false, + this.url, this.name, this.version, this.devVersion, diff --git a/app/lib/frontend/search_service.dart b/app/lib/frontend/search_service.dart index 02b0ba18cb..4e74950ecb 100644 --- a/app/lib/frontend/search_service.dart +++ b/app/lib/frontend/search_service.dart @@ -36,12 +36,14 @@ class SearchService { Future _loadResultForPackages( SearchQuery query, int totalCount, List packageScores) async { final List packageKeys = packageScores + .where((ps) => !ps.isExternal) .map((ps) => ps.package) .map((package) => dbService.emptyKey.append(Package, id: package)) .toList(); final packageEntries = (await dbService.lookup(packageKeys)).cast(); packageEntries.removeWhere((p) => p == null); + final pubPackages = {}; final List versionKeys = packageEntries.map((p) => p.latestVersionKey).toList(); if (versionKeys.isNotEmpty) { @@ -57,18 +59,37 @@ Future _loadResultForPackages( final List versions = (await batchResults[1]).cast(); - final List resultPackages = new List.generate( - versions.length, - (i) => new PackageView.fromModel( - package: packageEntries[i], - version: versions[i], - analysis: analysisExtracts[i], - apiPages: packageScores[i].apiPages, - )); - return new SearchResultPage(query, totalCount, resultPackages); - } else { - return new SearchResultPage.empty(query); + for (int i = 0; i < versions.length; i++) { + final pv = new PackageView.fromModel( + package: packageEntries[i], + version: versions[i], + analysis: analysisExtracts[i], + apiPages: packageScores[i].apiPages, + ); + pubPackages[pv.name] = pv; + } } + + final resultPackages = packageScores + .map((ps) { + if (pubPackages.containsKey(ps.package)) { + return pubPackages[ps.package]; + } + if (ps.isExternal) { + return new PackageView( + isExternal: true, + url: ps.url, + version: ps.version, + name: ps.package, + ellipsizedDescription: ps.description, + apiPages: ps.apiPages, + ); + } + return null; + }) + .where((pv) => pv != null) + .toList(); + return new SearchResultPage(query, totalCount, resultPackages); } /// The results of a search via the Custom Search API. diff --git a/app/lib/frontend/templates.dart b/app/lib/frontend/templates.dart index 1daa2b8c3f..3599cb248d 100644 --- a/app/lib/frontend/templates.dart +++ b/app/lib/frontend/templates.dart @@ -119,9 +119,25 @@ class TemplateService { for (int i = 0; i < packages.length; i++) { final view = packages[i]; final overallScore = view.overallScore; + String externalType; + bool isSdk = false; + if (view.isExternal && view.url.startsWith('https://api.dartlang.org/')) { + externalType = 'Dart core library'; + isSdk = true; + } + String scoreBoxHtml; + if (isSdk) { + scoreBoxHtml = _renderSdkScoreBox(); + } else if (!view.isExternal) { + scoreBoxHtml = _renderScoreBox(view.analysisStatus, overallScore, + isNewPackage: view.isNewPackage, package: view.name); + } packagesJson.add({ - 'url': urls.pkgPageUrl(view.name), + 'url': view.url ?? urls.pkgPageUrl(view.name), 'name': view.name, + 'is_external': view.isExternal, + 'external_type': externalType, + 'show_metadata': !view.isExternal, 'version': view.version, 'show_dev_version': view.devVersion != null, 'dev_version': view.devVersion, @@ -130,17 +146,14 @@ class TemplateService { 'desc': view.ellipsizedDescription, 'tags_html': _renderTags(view.analysisStatus, view.platforms, package: view.name), - 'score_box_html': _renderScoreBox(view.analysisStatus, overallScore, - isNewPackage: view.isNewPackage, package: view.name), + 'score_box_html': scoreBoxHtml, 'has_api_pages': view.apiPages != null && view.apiPages.isNotEmpty, 'api_pages': view.apiPages ?.map((page) => { 'title': page.title ?? page.path, - 'href': urls.pkgDocUrl( - view.name, - isLatest: true, - relativePath: page.path, - ) + 'href': page.url ?? + urls.pkgDocUrl(view.name, + isLatest: true, relativePath: page.path), }) ?.toList(), }); @@ -879,6 +892,10 @@ String _getAuthorsHtml(List authors) { bool _isAnalysisSkipped(AnalysisStatus status) => status == AnalysisStatus.outdated || status == AnalysisStatus.discontinued; +String _renderSdkScoreBox() { + return '
sdk
'; +} + String _renderScoreBox(AnalysisStatus status, double overallScore, {bool isNewPackage, String package}) { final skippedAnalysis = _isAnalysisSkipped(status); diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index b2a88e8e50..88cffcb636 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -22,6 +22,7 @@ import '../shared/dartdoc_client.dart'; import '../shared/popularity_storage.dart'; import '../shared/search_service.dart'; import '../shared/utils.dart'; +import '../shared/versions.dart' as versions; import 'text_utils.dart'; @@ -95,7 +96,7 @@ class SearchBackend { List apiDocPages; if (pubDataContent != null) { try { - apiDocPages = _apiDocPagesFromPubData(pubDataContent); + apiDocPages = _apiDocPagesFromPubDataBytes(pubDataContent); } catch (e, st) { _logger.severe('Parsing pub-data.json failed.', e, st); } @@ -143,57 +144,94 @@ class SearchBackend { return emails.toList()..sort(); } - List _apiDocPagesFromPubData(List bytes) { + List _apiDocPagesFromPubDataBytes(List bytes) { final decodedMap = json.decode(utf8.decode(bytes)) as Map; final pubData = new PubDartdocData.fromJson(decodedMap.cast()); + return apiDocPagesFromPubData(pubData); + } +} - final nameToKindMap = {}; - pubData.apiElements.forEach((e) { - nameToKindMap[e.name] = e.kind; - }); +/// Creates the index-related API data structure from the extracted dartdoc data. +List apiDocPagesFromPubData(PubDartdocData pubData) { + final nameToKindMap = {}; + pubData.apiElements.forEach((e) { + nameToKindMap[e.name] = e.kind; + }); - final pathMap = {}; - final symbolMap = >{}; - final docMap = >{}; + final pathMap = {}; + final symbolMap = >{}; + final docMap = >{}; - bool isTopLevel(String kind) => kind == 'library' || kind == 'class'; + bool isTopLevel(String kind) => kind == 'library' || kind == 'class'; - void update(String key, String name, String documentation) { - final set = symbolMap.putIfAbsent(key, () => new Set()); - set.addAll(name.split('.')); + void update(String key, String name, String documentation) { + final set = symbolMap.putIfAbsent(key, () => new Set()); + set.addAll(name.split('.')); - documentation = documentation?.trim(); - if (documentation != null && documentation.isNotEmpty) { - final list = docMap.putIfAbsent(key, () => []); - list.add(compactReadme(documentation)); - } + documentation = documentation?.trim(); + if (documentation != null && documentation.isNotEmpty) { + final list = docMap.putIfAbsent(key, () => []); + list.add(compactReadme(documentation)); } + } - pubData.apiElements.forEach((apiElement) { - if (isTopLevel(apiElement.kind)) { - pathMap[apiElement.name] = apiElement.href; - update(apiElement.name, apiElement.name, apiElement.documentation); - } + pubData.apiElements.forEach((apiElement) { + if (isTopLevel(apiElement.kind)) { + pathMap[apiElement.name] = apiElement.href; + update(apiElement.name, apiElement.name, apiElement.documentation); + } - if (!isTopLevel(apiElement.kind) && - apiElement.parent != null && - isTopLevel(nameToKindMap[apiElement.parent])) { - update(apiElement.parent, apiElement.name, apiElement.documentation); - } - }); + if (!isTopLevel(apiElement.kind) && + apiElement.parent != null && + isTopLevel(nameToKindMap[apiElement.parent])) { + update(apiElement.parent, apiElement.name, apiElement.documentation); + } + }); + + final results = pathMap.keys.map((key) { + final path = pathMap[key]; + final symbols = symbolMap[key].toList()..sort(); + return new ApiDocPage( + relativePath: path, + symbols: symbols, + textBlocks: docMap[key], + ); + }).toList(); + results.sort((a, b) => a.relativePath.compareTo(b.relativePath)); + return results; +} - final results = pathMap.keys.map((key) { - final path = pathMap[key]; - final symbols = symbolMap[key].toList()..sort(); - return new ApiDocPage( - relativePath: path, - symbols: symbols, - textBlocks: docMap[key], - ); - }).toList(); - results.sort((a, b) => a.relativePath.compareTo(b.relativePath)); - return results; - } +/// Splits the flat SDK data into per-library data (in the same data format). +List splitLibraries(PubDartdocData data) { + final librariesMap = >{}; + final rootMap = {}; + data.apiElements?.forEach((elem) { + String library; + if (elem.parent == null) { + library = elem.name; + } else { + library = rootMap[elem.parent] ?? elem.parent; + rootMap[elem.name] = library; + } + librariesMap.putIfAbsent(library, () => []).add(elem); + }); + return librariesMap.values + .map((list) => new PubDartdocData(apiElements: list)) + .toList(); +} + +/// Creates the index-related data structure for an SDK library. +PackageDocument createSdkDocument(PubDartdocData lib) { + final apiDocPages = apiDocPagesFromPubData(lib); + final package = lib.apiElements.first.name; + final documentation = lib.apiElements.first.documentation ?? ''; + final description = documentation.split('\n\n').first.trim(); + return new PackageDocument( + package: package, + version: versions.toolEnvSdkVersion, + description: description, + apiDocPages: apiDocPages, + ); } class SnapshotStorage { diff --git a/app/lib/search/handlers.dart b/app/lib/search/handlers.dart index 296d1a2fd3..d5d705b178 100644 --- a/app/lib/search/handlers.dart +++ b/app/lib/search/handlers.dart @@ -50,12 +50,35 @@ Future _searchHandler(shelf.Request request) async { final bool indent = request.url.queryParameters['indent'] == 'true'; final Stopwatch sw = new Stopwatch()..start(); final SearchQuery query = new SearchQuery.fromServiceUrl(request.url); - final PackageSearchResult result = await packageIndex.search(query); + final PackageSearchResult pkgResult = await packageIndex.search(query); final Duration elapsed = sw.elapsed; if (elapsed > _slowSearchThreshold) { _logger.info( 'Slow search: handler exceeded ${_slowSearchThreshold.inMilliseconds}ms: ' '${query.toServiceQueryParameters()}'); } + + PackageSearchResult result = pkgResult; + final includeSdkResults = query.offset == 0 && + query.hasQuery && + query.parsedQuery.text != null && + query.parsedQuery.text.isNotEmpty; + if (includeSdkResults) { + final dartSdkResult = + await dartSdkIndex.search(query.change(order: SearchOrder.text)); + final threshold = + pkgResult.packages.isEmpty ? 0.0 : pkgResult.packages.first.score / 2; + final selected = dartSdkResult.packages + .where((ps) => ps.score > threshold) + .take(3) + .toList(); + if (selected.isNotEmpty) { + result = new PackageSearchResult( + indexUpdated: pkgResult.indexUpdated, + packages: selected..addAll(pkgResult.packages), + totalCount: pkgResult.totalCount, + ); + } + } return jsonResponse(result.toJson(), indent: indent); } diff --git a/app/lib/search/index_simple.dart b/app/lib/search/index_simple.dart index 8efedbcd85..8df605563f 100644 --- a/app/lib/search/index_simple.dart +++ b/app/lib/search/index_simple.dart @@ -5,8 +5,10 @@ import 'dart:async'; import 'dart:math' as math; +import 'package:meta/meta.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:pana/pana.dart' show DependencyTypes; +import 'package:path/path.dart' as p; import '../shared/search_service.dart'; import '../shared/utils.dart' show StringInternPool; @@ -15,6 +17,13 @@ import 'platform_specificity.dart'; import 'scoring.dart'; import 'text_utils.dart'; +/// The [PackageIndex] for Dart SDK API. +PackageIndex get dartSdkIndex => ss.lookup(#_dartSdkIndex) as PackageIndex; + +/// Register a new [PackageIndex] for Dart SDK API. +void registerDartSdkIndex(PackageIndex index) => + ss.register(#_dartSdkIndex, index); + /// The [PackageIndex] registered in the current service scope. PackageIndex get packageIndex => ss.lookup(#packageIndexService) as PackageIndex; @@ -24,6 +33,8 @@ void registerPackageIndex(PackageIndex index) => ss.register(#packageIndexService, index); class SimplePackageIndex implements PackageIndex { + final bool _isSdkIndex; + final String _urlPrefix; final Map _packages = {}; final Map _normalizedPackageText = {}; final TokenIndex _nameIndex = new TokenIndex(minLength: 2); @@ -34,6 +45,14 @@ class SimplePackageIndex implements PackageIndex { DateTime _lastUpdated; bool _isReady = false; + SimplePackageIndex() + : _isSdkIndex = false, + _urlPrefix = null; + + SimplePackageIndex.sdk({@required String urlPrefix}) + : _isSdkIndex = true, + _urlPrefix = urlPrefix; + @override bool get isReady => _isReady; @@ -261,9 +280,33 @@ class SimplePackageIndex implements PackageIndex { }).toList(); } + if (_isSdkIndex) { + results = results.map((ps) { + String url = _urlPrefix; + final doc = _packages[ps.package]; + String description = doc.description ?? ps.package; + if (doc.apiDocPages != null && doc.apiDocPages.isNotEmpty) { + final libPage = doc.apiDocPages.firstWhere( + (dp) => dp.relativePath.endsWith('-library.html'), + orElse: () => doc.apiDocPages.first, + ); + url = p.join(_urlPrefix, libPage.relativePath); + } + final apiPages = ps.apiPages + ?.map((ref) => ref.change(url: p.join(_urlPrefix, ref.path))) + ?.toList(); + return ps.change( + url: url, + version: doc.version, + description: description, + apiPages: apiPages, + ); + }).toList(); + } + return new PackageSearchResult( totalCount: totalCount, - indexUpdated: _lastUpdated.toIso8601String(), + indexUpdated: _lastUpdated?.toIso8601String(), packages: results, ); } @@ -435,15 +478,15 @@ class SimplePackageIndex implements PackageIndex { } String _apiDocPageId(String package, ApiDocPage page) { - return '$package:${page.relativePath}'; + return '$package::${page.relativePath}'; } String _apiDocPkg(String id) { - return id.split(':').first; + return id.split('::').first; } String _apiDocPath(String id) { - return id.split(':').last; + return id.split('::').last; } } diff --git a/app/lib/shared/search_service.dart b/app/lib/shared/search_service.dart index d1d9403868..d288450787 100644 --- a/app/lib/shared/search_service.dart +++ b/app/lib/shared/search_service.dart @@ -455,12 +455,24 @@ class PackageScore { @JsonKey(includeIfNull: false) final double score; + @JsonKey(includeIfNull: false) + final String url; + + @JsonKey(includeIfNull: false) + final String version; + + @JsonKey(includeIfNull: false) + final String description; + @JsonKey(includeIfNull: false) final List apiPages; PackageScore({ this.package, this.score, + this.url, + this.version, + this.description, this.apiPages, }); @@ -469,15 +481,23 @@ class PackageScore { PackageScore change({ double score, + String url, + String version, + String description, List apiPages, }) { return new PackageScore( package: package, score: score ?? this.score, + url: url ?? this.url, + version: version ?? this.version, + description: description ?? this.description, apiPages: apiPages ?? this.apiPages, ); } + bool get isExternal => url != null && version != null && description != null; + Map toJson() => _$PackageScoreToJson(this); } @@ -486,10 +506,21 @@ class ApiPageRef { final String title; final String path; - ApiPageRef({this.title, this.path}); + @JsonKey(includeIfNull: false) + final String url; + + ApiPageRef({this.title, this.path, this.url}); factory ApiPageRef.fromJson(Map json) => _$ApiPageRefFromJson(json); + ApiPageRef change({String title, String url}) { + return new ApiPageRef( + title: title ?? this.title, + path: path, + url: url ?? this.url, + ); + } + Map toJson() => _$ApiPageRefToJson(this); } diff --git a/app/lib/shared/search_service.g.dart b/app/lib/shared/search_service.g.dart index 1cdbaf247c..9f72845a5b 100644 --- a/app/lib/shared/search_service.g.dart +++ b/app/lib/shared/search_service.g.dart @@ -96,6 +96,9 @@ PackageScore _$PackageScoreFromJson(Map json) { return PackageScore( package: json['package'] as String, score: (json['score'] as num)?.toDouble(), + url: json['url'] as String, + version: json['version'] as String, + description: json['description'] as String, apiPages: (json['apiPages'] as List) ?.map((e) => e == null ? null : ApiPageRef.fromJson(e as Map)) @@ -114,14 +117,32 @@ Map _$PackageScoreToJson(PackageScore instance) { } writeNotNull('score', instance.score); + writeNotNull('url', instance.url); + writeNotNull('version', instance.version); + writeNotNull('description', instance.description); writeNotNull('apiPages', instance.apiPages); return val; } ApiPageRef _$ApiPageRefFromJson(Map json) { return ApiPageRef( - title: json['title'] as String, path: json['path'] as String); + title: json['title'] as String, + path: json['path'] as String, + url: json['url'] as String); } -Map _$ApiPageRefToJson(ApiPageRef instance) => - {'title': instance.title, 'path': instance.path}; +Map _$ApiPageRefToJson(ApiPageRef instance) { + var val = { + 'title': instance.title, + 'path': instance.path, + }; + + void writeNotNull(String key, dynamic value) { + if (value != null) { + val[key] = value; + } + } + + writeNotNull('url', instance.url); + return val; +} diff --git a/app/lib/shared/urls.dart b/app/lib/shared/urls.dart index 4e0b56a9b8..1ae87c5ad9 100644 --- a/app/lib/shared/urls.dart +++ b/app/lib/shared/urls.dart @@ -7,6 +7,7 @@ import 'package:path/path.dart' as p; const pubHostedDomain = 'pub.dartlang.org'; const siteRoot = 'https://$pubHostedDomain'; +const _apiDartlangOrg = 'https://api.dartlang.org/'; String pkgPageUrl(String package, {String version, bool includeHost: false, String fragment}) { @@ -108,3 +109,10 @@ void syntaxCheckHomepageUrl(String url) { throw new Exception('Homepage URL has no valid host: $url'); } } + +String dartSdkMainUrl(String version) { + final isDev = version.contains('dev'); + final channel = isDev ? 'dev' : 'stable'; + final url = p.join(_apiDartlangOrg, channel, version); + return '$url/'; +} diff --git a/app/test/dartdoc/handlers_test.dart b/app/test/dartdoc/handlers_test.dart index 99683b8048..40df350bc4 100644 --- a/app/test/dartdoc/handlers_test.dart +++ b/app/test/dartdoc/handlers_test.dart @@ -5,8 +5,10 @@ library pub_dartlang_org.handlers_test; import 'dart:async'; +import 'dart:io'; import 'package:pub_dartlang_org/dartdoc/models.dart'; +import 'package:pub_dartlang_org/dartdoc/pub_dartdoc_data.dart'; import 'package:pub_dartlang_org/shared/task_scheduler.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:test/test.dart'; @@ -195,4 +197,19 @@ class DartdocBackendMock implements DartdocBackend { Future isLegacy(String package, String version) { throw new UnimplementedError(); } + + @override + Future hasValidDartSdkDartdocData() { + throw new UnimplementedError(); + } + + @override + Future getDartSdkDartdocData() { + throw new UnimplementedError(); + } + + @override + Future uploadDartSdkDartdocData(File file) { + throw new UnimplementedError(); + } } diff --git a/app/test/search/handlers_test.dart b/app/test/search/handlers_test.dart index 05ca44ae3e..289f9f4395 100644 --- a/app/test/search/handlers_test.dart +++ b/app/test/search/handlers_test.dart @@ -47,6 +47,7 @@ void main() { Future setUpInServiceScope() async { registerSearchBackend(new MockSearchBackend()); registerPackageIndex(new SimplePackageIndex()); + registerDartSdkIndex(new SimplePackageIndex()); await packageIndex .addPackages(await searchBackend.loadDocuments(['pkg_foo'])); await packageIndex.merge(); diff --git a/app/test/shared/urls_test.dart b/app/test/shared/urls_test.dart index 6d45ef1909..14b5402299 100644 --- a/app/test/shared/urls_test.dart +++ b/app/test/shared/urls_test.dart @@ -69,4 +69,15 @@ void main() { expect(() => syntaxCheckHomepageUrl('http://.../x/'), throwsException); }); }); + + group('SDK urls', () { + test('dev', () { + expect(dartSdkMainUrl('2.1.0-dev.3.1'), + 'https://api.dartlang.org/dev/2.1.0-dev.3.1/'); + }); + + test('stable', () { + expect(dartSdkMainUrl('2.0.0'), 'https://api.dartlang.org/stable/2.0.0/'); + }); + }); } diff --git a/app/views/pkg/index.mustache b/app/views/pkg/index.mustache index b3da3df699..f14c545e0e 100644 --- a/app/views/pkg/index.mustache +++ b/app/views/pkg/index.mustache @@ -28,12 +28,17 @@ {{& score_box_html }}

{{name}}

{{desc}}

+ {{#is_external}} + + {{/is_external}} + {{#show_metadata}} + {{/show_metadata}} {{#has_api_pages}}
API results: diff --git a/static/css/style.css b/static/css/style.css index 02aca27815..7ff1e745e2 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -516,6 +516,10 @@ pre > code { background: #ccc; } +.score-box > .number.-small { + font-size: 12px; +} + .score-box > .new { position: absolute; top: -30%;