From 7c34c90cead439ba368049cd0c553019e5ebf88e Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 5 Sep 2018 10:00:04 +0200 Subject: [PATCH 01/12] Generate Dart SDK doc URLs. --- app/lib/shared/urls.dart | 8 ++++++++ app/test/shared/urls_test.dart | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/app/lib/shared/urls.dart b/app/lib/shared/urls.dart index 4e0b56a9b8..1ae87c5ad9 100644 --- a/app/lib/shared/urls.dart +++ b/app/lib/shared/urls.dart @@ -7,6 +7,7 @@ import 'package:path/path.dart' as p; const pubHostedDomain = 'pub.dartlang.org'; const siteRoot = 'https://$pubHostedDomain'; +const _apiDartlangOrg = 'https://api.dartlang.org/'; String pkgPageUrl(String package, {String version, bool includeHost: false, String fragment}) { @@ -108,3 +109,10 @@ void syntaxCheckHomepageUrl(String url) { throw new Exception('Homepage URL has no valid host: $url'); } } + +String dartSdkMainUrl(String version) { + final isDev = version.contains('dev'); + final channel = isDev ? 'dev' : 'stable'; + final url = p.join(_apiDartlangOrg, channel, version); + return '$url/'; +} diff --git a/app/test/shared/urls_test.dart b/app/test/shared/urls_test.dart index 6d45ef1909..14b5402299 100644 --- a/app/test/shared/urls_test.dart +++ b/app/test/shared/urls_test.dart @@ -69,4 +69,15 @@ void main() { expect(() => syntaxCheckHomepageUrl('http://.../x/'), throwsException); }); }); + + group('SDK urls', () { + test('dev', () { + expect(dartSdkMainUrl('2.1.0-dev.3.1'), + 'https://api.dartlang.org/dev/2.1.0-dev.3.1/'); + }); + + test('stable', () { + expect(dartSdkMainUrl('2.0.0'), 'https://api.dartlang.org/stable/2.0.0/'); + }); + }); } From 6977b11ac1c9192c3a766efe47176095ede3ca59 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 5 Sep 2018 10:01:09 +0200 Subject: [PATCH 02/12] Extend search wire data with new fields. --- app/lib/shared/search_service.dart | 33 +++++++++++++++++++++++++++- app/lib/shared/search_service.g.dart | 27 ++++++++++++++++++++--- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/app/lib/shared/search_service.dart b/app/lib/shared/search_service.dart index d1d9403868..d288450787 100644 --- a/app/lib/shared/search_service.dart +++ b/app/lib/shared/search_service.dart @@ -455,12 +455,24 @@ class PackageScore { @JsonKey(includeIfNull: false) final double score; + @JsonKey(includeIfNull: false) + final String url; + + @JsonKey(includeIfNull: false) + final String version; + + @JsonKey(includeIfNull: false) + final String description; + @JsonKey(includeIfNull: false) final List apiPages; PackageScore({ this.package, this.score, + this.url, + this.version, + this.description, this.apiPages, }); @@ -469,15 +481,23 @@ class PackageScore { PackageScore change({ double score, + String url, + String version, + String description, List apiPages, }) { return new PackageScore( package: package, score: score ?? this.score, + url: url ?? this.url, + version: version ?? this.version, + description: description ?? this.description, apiPages: apiPages ?? this.apiPages, ); } + bool get isExternal => url != null && version != null && description != null; + Map toJson() => _$PackageScoreToJson(this); } @@ -486,10 +506,21 @@ class ApiPageRef { final String title; final String path; - ApiPageRef({this.title, this.path}); + @JsonKey(includeIfNull: false) + final String url; + + ApiPageRef({this.title, this.path, this.url}); factory ApiPageRef.fromJson(Map json) => _$ApiPageRefFromJson(json); + ApiPageRef change({String title, String url}) { + return new ApiPageRef( + title: title ?? this.title, + path: path, + url: url ?? this.url, + ); + } + Map toJson() => _$ApiPageRefToJson(this); } diff --git a/app/lib/shared/search_service.g.dart b/app/lib/shared/search_service.g.dart index 1cdbaf247c..9f72845a5b 100644 --- a/app/lib/shared/search_service.g.dart +++ b/app/lib/shared/search_service.g.dart @@ -96,6 +96,9 @@ PackageScore _$PackageScoreFromJson(Map json) { return PackageScore( package: json['package'] as String, score: (json['score'] as num)?.toDouble(), + url: json['url'] as String, + version: json['version'] as String, + description: json['description'] as String, apiPages: (json['apiPages'] as List) ?.map((e) => e == null ? null : ApiPageRef.fromJson(e as Map)) @@ -114,14 +117,32 @@ Map _$PackageScoreToJson(PackageScore instance) { } writeNotNull('score', instance.score); + writeNotNull('url', instance.url); + writeNotNull('version', instance.version); + writeNotNull('description', instance.description); writeNotNull('apiPages', instance.apiPages); return val; } ApiPageRef _$ApiPageRefFromJson(Map json) { return ApiPageRef( - title: json['title'] as String, path: json['path'] as String); + title: json['title'] as String, + path: json['path'] as String, + url: json['url'] as String); } -Map _$ApiPageRefToJson(ApiPageRef instance) => - {'title': instance.title, 'path': instance.path}; +Map _$ApiPageRefToJson(ApiPageRef instance) { + var val = { + 'title': instance.title, + 'path': instance.path, + }; + + void writeNotNull(String key, dynamic value) { + if (value != null) { + val[key] = value; + } + } + + writeNotNull('url', instance.url); + return val; +} From 56d571305bcf01feaacfb7a5b91e2b9bcd869192 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 5 Sep 2018 10:20:37 +0200 Subject: [PATCH 03/12] Generate extracted data file from Dart SDK. --- app/bin/service/dartdoc.dart | 2 + app/lib/dartdoc/backend.dart | 57 +++++++++++++++++++++++++++++ app/lib/dartdoc/dartdoc_runner.dart | 51 ++++++++++++++++++++++++++ app/lib/dartdoc/storage_path.dart | 11 ++++++ app/test/dartdoc/handlers_test.dart | 17 +++++++++ 5 files changed, 138 insertions(+) diff --git a/app/bin/service/dartdoc.dart b/app/bin/service/dartdoc.dart index 39580a2daa..3412dfa7c3 100644 --- a/app/bin/service/dartdoc.dart +++ b/app/bin/service/dartdoc.dart @@ -68,6 +68,8 @@ Future _workerMain(WorkerEntryMessage message) async { final jobProcessor = new DartdocJobProcessor(lockDuration: const Duration(minutes: 30)); + await jobProcessor.generateDocsForSdk(); + final jobMaintenance = new JobMaintenance(dbService, jobProcessor); new Timer.periodic(const Duration(minutes: 15), (_) async { diff --git a/app/lib/dartdoc/backend.dart b/app/lib/dartdoc/backend.dart index b07f102a00..66b415fca5 100644 --- a/app/lib/dartdoc/backend.dart +++ b/app/lib/dartdoc/backend.dart @@ -3,6 +3,7 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:gcloud/db.dart'; @@ -21,9 +22,11 @@ import '../shared/utils.dart' show contentType; import '../shared/versions.dart' as shared_versions; import 'models.dart'; +import 'pub_dartdoc_data.dart'; import 'storage_path.dart' as storage_path; final Logger _logger = new Logger('pub.dartdoc.backend'); +final _gzip = new GZipCodec(); final Duration _contentDeleteThreshold = const Duration(days: 1); final int _concurrentUploads = 8; @@ -43,6 +46,60 @@ class DartdocBackend { DartdocBackend(this._db, this._storage); + /// Whether the storage bucket has a useable extracted data file. + /// Only the existence of the file is checked. + // TODO: decide whether we should re-generate the file after a certain age + Future hasValidDartSdkDartdocData() async { + final objectName = + storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); + try { + final info = await _storage.info(objectName); + return info != null; + } catch (_) { + return false; + } + } + + /// Upload the generated dartdoc data file for the Dart SDK to the storage bucket. + Future uploadDartSdkDartdocData(File file) async { + final objectName = + storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); + try { + final contentBytes = await file.readAsBytes(); + await _storage.writeBytes(objectName, _gzip.encode(contentBytes)); + } catch (e, st) { + _logger.warning( + 'Unable to upload SDK pub dartdoc data file: $objectName', e, st); + } + } + + /// Read the generated dartdoc data file for the Dart SDK. + Future getDartSdkDartdocData() async { + final objectName = + storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); + Future load() async { + final Map map = await _storage + .read(objectName) + .transform(_gzip.decoder) + .transform(utf8.decoder) + .transform(json.decoder) + .single; + return new PubDartdocData.fromJson(map); + } + + for (int i = 0; i < 3; i++) { + try { + return await load(); + } catch (e, st) { + final message = + 'Unable to read SDK pub dartdoc data file (attempt #${i + 1}): $objectName '; + _logger.info(message, e, st); + } + } + + return null; + } + /// Returns the latest stable version of a package. Future getLatestVersion(String package) async { final list = await _db.lookup([_db.emptyKey.append(Package, id: package)]); diff --git a/app/lib/dartdoc/dartdoc_runner.dart b/app/lib/dartdoc/dartdoc_runner.dart index 706b7ce574..20c43cf57e 100644 --- a/app/lib/dartdoc/dartdoc_runner.dart +++ b/app/lib/dartdoc/dartdoc_runner.dart @@ -18,6 +18,7 @@ import '../job/job.dart'; import '../scorecard/backend.dart'; import '../scorecard/models.dart'; import '../shared/analyzer_client.dart'; +import '../shared/configuration.dart' show envConfig; import '../shared/tool_env.dart'; import '../shared/urls.dart'; import '../shared/versions.dart' as versions; @@ -42,6 +43,56 @@ class DartdocJobProcessor extends JobProcessor { DartdocJobProcessor({Duration lockDuration}) : super(service: JobService.dartdoc, lockDuration: lockDuration); + /// Uses the tool environment's SDK (the one that is used for analysis too) to + /// generate dartdoc documentation and extracted data file for SDK API indexing. + /// Only the extracted data file will be used and uploaded. + Future generateDocsForSdk() async { + if (await dartdocBackend.hasValidDartSdkDartdocData()) return; + final tempDir = + await Directory.systemTemp.createTemp('pub-dartlang-dartdoc'); + try { + final tempDirPath = tempDir.resolveSymbolicLinksSync(); + final outputDir = tempDirPath; + final args = [ + '--sdk-docs', + '--output', + outputDir, + '--hosted-url', + siteRoot, + '--link-to-remote', + '--no-validate-links', + ]; + if (envConfig.toolEnvDartSdkDir != null) { + args.addAll(['--sdk-dir', envConfig.toolEnvDartSdkDir]); + } + final pr = await runProc( + 'dart', + ['bin/pub_dartdoc.dart']..addAll(args), + workingDirectory: _pkgPubDartdocDir, + timeout: _dartdocTimeout * 2, + ); + + final pubDataFile = new File(p.join(outputDir, 'pub-data.json')); + final hasPubData = await pubDataFile.exists(); + final isOk = pr.exitCode == 0 && hasPubData; + if (!isOk) { + _logger.warning( + 'Error while generating SDK docs.\n\n${pr.stdout}\n\n${pr.stderr}'); + return; + } + + // prevent close races updating the same content in close succession + if (await dartdocBackend.hasValidDartSdkDartdocData()) return; + + // upload only the pub dartdoc data file + await dartdocBackend.uploadDartSdkDartdocData(pubDataFile); + } catch (e, st) { + _logger.warning('Error while generating SDK docs.', e, st); + } finally { + await tempDir.delete(recursive: true); + } + } + @override Future shouldProcess( String package, String version, DateTime updated) async { diff --git a/app/lib/dartdoc/storage_path.dart b/app/lib/dartdoc/storage_path.dart index 1faf38f71d..76b5b17a34 100644 --- a/app/lib/dartdoc/storage_path.dart +++ b/app/lib/dartdoc/storage_path.dart @@ -9,6 +9,7 @@ const _generatedStaticAssetPaths = const ['static-assets']; // Storage contains package data in a form of /package/version/... // This path contains '-' and is an invalid package name, safe of conflicts. const _storageSharedAssetPrefix = 'shared-assets'; +const _sdkAssetPrefix = 'sdk-assets'; /// Whether the generated file can be moved to the shared assets. bool isSharedAsset(String relativePath) { @@ -49,3 +50,13 @@ String contentObjectName(String packageName, String packageVersion, String uuid, String relativePath) { return p.join(contentPrefix(packageName, packageVersion, uuid), relativePath); } + +/// ObjectName of an SDK asset. +String sdkObjectName(String relativePath) { + return p.join(_sdkAssetPrefix, relativePath); +} + +/// The ObjectName for the Dart SDK's extracted dartdoc content. +String dartSdkDartdocDataName(String runtimeVersion) { + return sdkObjectName('dart/pub-dartdoc-data/$runtimeVersion.json.gz'); +} diff --git a/app/test/dartdoc/handlers_test.dart b/app/test/dartdoc/handlers_test.dart index 99683b8048..40df350bc4 100644 --- a/app/test/dartdoc/handlers_test.dart +++ b/app/test/dartdoc/handlers_test.dart @@ -5,8 +5,10 @@ library pub_dartlang_org.handlers_test; import 'dart:async'; +import 'dart:io'; import 'package:pub_dartlang_org/dartdoc/models.dart'; +import 'package:pub_dartlang_org/dartdoc/pub_dartdoc_data.dart'; import 'package:pub_dartlang_org/shared/task_scheduler.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:test/test.dart'; @@ -195,4 +197,19 @@ class DartdocBackendMock implements DartdocBackend { Future isLegacy(String package, String version) { throw new UnimplementedError(); } + + @override + Future hasValidDartSdkDartdocData() { + throw new UnimplementedError(); + } + + @override + Future getDartSdkDartdocData() { + throw new UnimplementedError(); + } + + @override + Future uploadDartSdkDartdocData(File file) { + throw new UnimplementedError(); + } } From 5737c57d8734c0b1d441e48afb8aa80340478cb2 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 5 Sep 2018 10:37:31 +0200 Subject: [PATCH 04/12] Put SDK API documents into a separate index and serve the top 3 in search results. --- app/bin/service/search.dart | 35 +++++++++ app/lib/search/backend.dart | 120 +++++++++++++++++++---------- app/lib/search/handlers.dart | 25 +++++- app/lib/search/index_simple.dart | 51 +++++++++++- app/test/search/handlers_test.dart | 1 + 5 files changed, 186 insertions(+), 46 deletions(-) diff --git a/app/bin/service/search.dart b/app/bin/service/search.dart index 69dfe067e5..b46947d1fc 100644 --- a/app/bin/service/search.dart +++ b/app/bin/service/search.dart @@ -11,6 +11,7 @@ import 'package:gcloud/service_scope.dart'; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; +import 'package:pub_dartlang_org/dartdoc/backend.dart'; import 'package:pub_dartlang_org/shared/analyzer_client.dart'; import 'package:pub_dartlang_org/shared/analyzer_memcache.dart'; import 'package:pub_dartlang_org/shared/configuration.dart'; @@ -23,6 +24,8 @@ import 'package:pub_dartlang_org/shared/service_utils.dart'; import 'package:pub_dartlang_org/shared/task_client.dart'; import 'package:pub_dartlang_org/shared/task_scheduler.dart'; import 'package:pub_dartlang_org/shared/task_sources.dart'; +import 'package:pub_dartlang_org/shared/versions.dart'; +import 'package:pub_dartlang_org/shared/urls.dart'; import 'package:pub_dartlang_org/search/backend.dart'; import 'package:pub_dartlang_org/search/handlers.dart'; @@ -56,6 +59,9 @@ Future _main(FrontendEntryMessage message) async { registerAnalyzerClient(analyzerClient); registerScopeExitCallback(analyzerClient.close); + final Bucket dartdocBucket = await getOrCreateBucket( + storageService, activeConfiguration.dartdocStorageBucketName); + registerDartdocBackend(new DartdocBackend(db.dbService, dartdocBucket)); registerDartdocMemcache(new DartdocMemcache(memcacheService)); final DartdocClient dartdocClient = new DartdocClient(); registerDartdocClient(dartdocClient); @@ -69,9 +75,15 @@ Future _main(FrontendEntryMessage message) async { new SnapshotStorage(storageService, snapshotBucket)); final ReceivePort taskReceivePort = new ReceivePort(); + registerDartSdkIndex(new SimplePackageIndex.sdk( + urlPrefix: dartSdkMainUrl(toolEnvSdkVersion))); registerPackageIndex(new SimplePackageIndex()); registerTaskSendPort(taskReceivePort.sendPort); + // don't block on SDK index updates, as it may take several minutes before + // the dartdoc service produces the required output. + _updateDartSdkIndex(); + final BatchIndexUpdater batchIndexUpdater = new BatchIndexUpdater(); await batchIndexUpdater.initSnapshot(); @@ -92,3 +104,26 @@ Future _main(FrontendEntryMessage message) async { await runHandler(_logger, searchServiceHandler); }); } + +Future _updateDartSdkIndex() async { + for (int i = 0;; i++) { + try { + _logger.info('Trying to load SDK index.'); + final data = await dartdocBackend.getDartSdkDartdocData(); + if (data != null) { + final docs = + splitLibraries(data).map((lib) => createSdkDocument(lib)).toList(); + await dartSdkIndex.addPackages(docs); + await dartSdkIndex.merge(); + _logger.info('Dart SDK index loaded successfully.'); + return; + } + } catch (e, st) { + _logger.warning('Error loading Dart SDK index.', e, st); + } + if (i % 10 == 0) { + _logger.warning('Unable to load Dart SDK index. Cycle: $i'); + } + await new Future.delayed(const Duration(minutes: 1)); + } +} diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index b2a88e8e50..88cffcb636 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -22,6 +22,7 @@ import '../shared/dartdoc_client.dart'; import '../shared/popularity_storage.dart'; import '../shared/search_service.dart'; import '../shared/utils.dart'; +import '../shared/versions.dart' as versions; import 'text_utils.dart'; @@ -95,7 +96,7 @@ class SearchBackend { List apiDocPages; if (pubDataContent != null) { try { - apiDocPages = _apiDocPagesFromPubData(pubDataContent); + apiDocPages = _apiDocPagesFromPubDataBytes(pubDataContent); } catch (e, st) { _logger.severe('Parsing pub-data.json failed.', e, st); } @@ -143,57 +144,94 @@ class SearchBackend { return emails.toList()..sort(); } - List _apiDocPagesFromPubData(List bytes) { + List _apiDocPagesFromPubDataBytes(List bytes) { final decodedMap = json.decode(utf8.decode(bytes)) as Map; final pubData = new PubDartdocData.fromJson(decodedMap.cast()); + return apiDocPagesFromPubData(pubData); + } +} - final nameToKindMap = {}; - pubData.apiElements.forEach((e) { - nameToKindMap[e.name] = e.kind; - }); +/// Creates the index-related API data structure from the extracted dartdoc data. +List apiDocPagesFromPubData(PubDartdocData pubData) { + final nameToKindMap = {}; + pubData.apiElements.forEach((e) { + nameToKindMap[e.name] = e.kind; + }); - final pathMap = {}; - final symbolMap = >{}; - final docMap = >{}; + final pathMap = {}; + final symbolMap = >{}; + final docMap = >{}; - bool isTopLevel(String kind) => kind == 'library' || kind == 'class'; + bool isTopLevel(String kind) => kind == 'library' || kind == 'class'; - void update(String key, String name, String documentation) { - final set = symbolMap.putIfAbsent(key, () => new Set()); - set.addAll(name.split('.')); + void update(String key, String name, String documentation) { + final set = symbolMap.putIfAbsent(key, () => new Set()); + set.addAll(name.split('.')); - documentation = documentation?.trim(); - if (documentation != null && documentation.isNotEmpty) { - final list = docMap.putIfAbsent(key, () => []); - list.add(compactReadme(documentation)); - } + documentation = documentation?.trim(); + if (documentation != null && documentation.isNotEmpty) { + final list = docMap.putIfAbsent(key, () => []); + list.add(compactReadme(documentation)); } + } - pubData.apiElements.forEach((apiElement) { - if (isTopLevel(apiElement.kind)) { - pathMap[apiElement.name] = apiElement.href; - update(apiElement.name, apiElement.name, apiElement.documentation); - } + pubData.apiElements.forEach((apiElement) { + if (isTopLevel(apiElement.kind)) { + pathMap[apiElement.name] = apiElement.href; + update(apiElement.name, apiElement.name, apiElement.documentation); + } - if (!isTopLevel(apiElement.kind) && - apiElement.parent != null && - isTopLevel(nameToKindMap[apiElement.parent])) { - update(apiElement.parent, apiElement.name, apiElement.documentation); - } - }); + if (!isTopLevel(apiElement.kind) && + apiElement.parent != null && + isTopLevel(nameToKindMap[apiElement.parent])) { + update(apiElement.parent, apiElement.name, apiElement.documentation); + } + }); + + final results = pathMap.keys.map((key) { + final path = pathMap[key]; + final symbols = symbolMap[key].toList()..sort(); + return new ApiDocPage( + relativePath: path, + symbols: symbols, + textBlocks: docMap[key], + ); + }).toList(); + results.sort((a, b) => a.relativePath.compareTo(b.relativePath)); + return results; +} - final results = pathMap.keys.map((key) { - final path = pathMap[key]; - final symbols = symbolMap[key].toList()..sort(); - return new ApiDocPage( - relativePath: path, - symbols: symbols, - textBlocks: docMap[key], - ); - }).toList(); - results.sort((a, b) => a.relativePath.compareTo(b.relativePath)); - return results; - } +/// Splits the flat SDK data into per-library data (in the same data format). +List splitLibraries(PubDartdocData data) { + final librariesMap = >{}; + final rootMap = {}; + data.apiElements?.forEach((elem) { + String library; + if (elem.parent == null) { + library = elem.name; + } else { + library = rootMap[elem.parent] ?? elem.parent; + rootMap[elem.name] = library; + } + librariesMap.putIfAbsent(library, () => []).add(elem); + }); + return librariesMap.values + .map((list) => new PubDartdocData(apiElements: list)) + .toList(); +} + +/// Creates the index-related data structure for an SDK library. +PackageDocument createSdkDocument(PubDartdocData lib) { + final apiDocPages = apiDocPagesFromPubData(lib); + final package = lib.apiElements.first.name; + final documentation = lib.apiElements.first.documentation ?? ''; + final description = documentation.split('\n\n').first.trim(); + return new PackageDocument( + package: package, + version: versions.toolEnvSdkVersion, + description: description, + apiDocPages: apiDocPages, + ); } class SnapshotStorage { diff --git a/app/lib/search/handlers.dart b/app/lib/search/handlers.dart index 296d1a2fd3..d5d705b178 100644 --- a/app/lib/search/handlers.dart +++ b/app/lib/search/handlers.dart @@ -50,12 +50,35 @@ Future _searchHandler(shelf.Request request) async { final bool indent = request.url.queryParameters['indent'] == 'true'; final Stopwatch sw = new Stopwatch()..start(); final SearchQuery query = new SearchQuery.fromServiceUrl(request.url); - final PackageSearchResult result = await packageIndex.search(query); + final PackageSearchResult pkgResult = await packageIndex.search(query); final Duration elapsed = sw.elapsed; if (elapsed > _slowSearchThreshold) { _logger.info( 'Slow search: handler exceeded ${_slowSearchThreshold.inMilliseconds}ms: ' '${query.toServiceQueryParameters()}'); } + + PackageSearchResult result = pkgResult; + final includeSdkResults = query.offset == 0 && + query.hasQuery && + query.parsedQuery.text != null && + query.parsedQuery.text.isNotEmpty; + if (includeSdkResults) { + final dartSdkResult = + await dartSdkIndex.search(query.change(order: SearchOrder.text)); + final threshold = + pkgResult.packages.isEmpty ? 0.0 : pkgResult.packages.first.score / 2; + final selected = dartSdkResult.packages + .where((ps) => ps.score > threshold) + .take(3) + .toList(); + if (selected.isNotEmpty) { + result = new PackageSearchResult( + indexUpdated: pkgResult.indexUpdated, + packages: selected..addAll(pkgResult.packages), + totalCount: pkgResult.totalCount, + ); + } + } return jsonResponse(result.toJson(), indent: indent); } diff --git a/app/lib/search/index_simple.dart b/app/lib/search/index_simple.dart index 8efedbcd85..8df605563f 100644 --- a/app/lib/search/index_simple.dart +++ b/app/lib/search/index_simple.dart @@ -5,8 +5,10 @@ import 'dart:async'; import 'dart:math' as math; +import 'package:meta/meta.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:pana/pana.dart' show DependencyTypes; +import 'package:path/path.dart' as p; import '../shared/search_service.dart'; import '../shared/utils.dart' show StringInternPool; @@ -15,6 +17,13 @@ import 'platform_specificity.dart'; import 'scoring.dart'; import 'text_utils.dart'; +/// The [PackageIndex] for Dart SDK API. +PackageIndex get dartSdkIndex => ss.lookup(#_dartSdkIndex) as PackageIndex; + +/// Register a new [PackageIndex] for Dart SDK API. +void registerDartSdkIndex(PackageIndex index) => + ss.register(#_dartSdkIndex, index); + /// The [PackageIndex] registered in the current service scope. PackageIndex get packageIndex => ss.lookup(#packageIndexService) as PackageIndex; @@ -24,6 +33,8 @@ void registerPackageIndex(PackageIndex index) => ss.register(#packageIndexService, index); class SimplePackageIndex implements PackageIndex { + final bool _isSdkIndex; + final String _urlPrefix; final Map _packages = {}; final Map _normalizedPackageText = {}; final TokenIndex _nameIndex = new TokenIndex(minLength: 2); @@ -34,6 +45,14 @@ class SimplePackageIndex implements PackageIndex { DateTime _lastUpdated; bool _isReady = false; + SimplePackageIndex() + : _isSdkIndex = false, + _urlPrefix = null; + + SimplePackageIndex.sdk({@required String urlPrefix}) + : _isSdkIndex = true, + _urlPrefix = urlPrefix; + @override bool get isReady => _isReady; @@ -261,9 +280,33 @@ class SimplePackageIndex implements PackageIndex { }).toList(); } + if (_isSdkIndex) { + results = results.map((ps) { + String url = _urlPrefix; + final doc = _packages[ps.package]; + String description = doc.description ?? ps.package; + if (doc.apiDocPages != null && doc.apiDocPages.isNotEmpty) { + final libPage = doc.apiDocPages.firstWhere( + (dp) => dp.relativePath.endsWith('-library.html'), + orElse: () => doc.apiDocPages.first, + ); + url = p.join(_urlPrefix, libPage.relativePath); + } + final apiPages = ps.apiPages + ?.map((ref) => ref.change(url: p.join(_urlPrefix, ref.path))) + ?.toList(); + return ps.change( + url: url, + version: doc.version, + description: description, + apiPages: apiPages, + ); + }).toList(); + } + return new PackageSearchResult( totalCount: totalCount, - indexUpdated: _lastUpdated.toIso8601String(), + indexUpdated: _lastUpdated?.toIso8601String(), packages: results, ); } @@ -435,15 +478,15 @@ class SimplePackageIndex implements PackageIndex { } String _apiDocPageId(String package, ApiDocPage page) { - return '$package:${page.relativePath}'; + return '$package::${page.relativePath}'; } String _apiDocPkg(String id) { - return id.split(':').first; + return id.split('::').first; } String _apiDocPath(String id) { - return id.split(':').last; + return id.split('::').last; } } diff --git a/app/test/search/handlers_test.dart b/app/test/search/handlers_test.dart index 05ca44ae3e..289f9f4395 100644 --- a/app/test/search/handlers_test.dart +++ b/app/test/search/handlers_test.dart @@ -47,6 +47,7 @@ void main() { Future setUpInServiceScope() async { registerSearchBackend(new MockSearchBackend()); registerPackageIndex(new SimplePackageIndex()); + registerDartSdkIndex(new SimplePackageIndex()); await packageIndex .addPackages(await searchBackend.loadDocuments(['pkg_foo'])); await packageIndex.merge(); From e88a4f886d4fccd22db57583ac8e7992b9d8d8de Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 5 Sep 2018 10:43:49 +0200 Subject: [PATCH 05/12] Display SDK API results slightly differently than usual results. --- app/lib/frontend/models.dart | 4 +++ app/lib/frontend/search_service.dart | 43 +++++++++++++++++++++------- app/lib/frontend/templates.dart | 17 +++++++---- app/views/pkg/index.mustache | 7 ++++- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/app/lib/frontend/models.dart b/app/lib/frontend/models.dart index 9b29ed0567..3efc8cbe2b 100644 --- a/app/lib/frontend/models.dart +++ b/app/lib/frontend/models.dart @@ -217,6 +217,8 @@ class PrivateKey extends db.Model { /// An extract of [Package] and [PackageVersion] and [AnalysisView], for /// display-only uses. class PackageView { + final bool isExternal; + final String url; final String name; final String version; // Not null only if there is a difference compared to the [version]. @@ -231,6 +233,8 @@ class PackageView { final List apiPages; PackageView({ + this.isExternal: false, + this.url, this.name, this.version, this.devVersion, diff --git a/app/lib/frontend/search_service.dart b/app/lib/frontend/search_service.dart index 02b0ba18cb..4e74950ecb 100644 --- a/app/lib/frontend/search_service.dart +++ b/app/lib/frontend/search_service.dart @@ -36,12 +36,14 @@ class SearchService { Future _loadResultForPackages( SearchQuery query, int totalCount, List packageScores) async { final List packageKeys = packageScores + .where((ps) => !ps.isExternal) .map((ps) => ps.package) .map((package) => dbService.emptyKey.append(Package, id: package)) .toList(); final packageEntries = (await dbService.lookup(packageKeys)).cast(); packageEntries.removeWhere((p) => p == null); + final pubPackages = {}; final List versionKeys = packageEntries.map((p) => p.latestVersionKey).toList(); if (versionKeys.isNotEmpty) { @@ -57,18 +59,37 @@ Future _loadResultForPackages( final List versions = (await batchResults[1]).cast(); - final List resultPackages = new List.generate( - versions.length, - (i) => new PackageView.fromModel( - package: packageEntries[i], - version: versions[i], - analysis: analysisExtracts[i], - apiPages: packageScores[i].apiPages, - )); - return new SearchResultPage(query, totalCount, resultPackages); - } else { - return new SearchResultPage.empty(query); + for (int i = 0; i < versions.length; i++) { + final pv = new PackageView.fromModel( + package: packageEntries[i], + version: versions[i], + analysis: analysisExtracts[i], + apiPages: packageScores[i].apiPages, + ); + pubPackages[pv.name] = pv; + } } + + final resultPackages = packageScores + .map((ps) { + if (pubPackages.containsKey(ps.package)) { + return pubPackages[ps.package]; + } + if (ps.isExternal) { + return new PackageView( + isExternal: true, + url: ps.url, + version: ps.version, + name: ps.package, + ellipsizedDescription: ps.description, + apiPages: ps.apiPages, + ); + } + return null; + }) + .where((pv) => pv != null) + .toList(); + return new SearchResultPage(query, totalCount, resultPackages); } /// The results of a search via the Custom Search API. diff --git a/app/lib/frontend/templates.dart b/app/lib/frontend/templates.dart index 1daa2b8c3f..8e66d01391 100644 --- a/app/lib/frontend/templates.dart +++ b/app/lib/frontend/templates.dart @@ -119,9 +119,16 @@ class TemplateService { for (int i = 0; i < packages.length; i++) { final view = packages[i]; final overallScore = view.overallScore; + String externalType; + if (view.isExternal && view.url.startsWith('https://api.dartlang.org/')) { + externalType = 'Dart core library'; + } packagesJson.add({ - 'url': urls.pkgPageUrl(view.name), + 'url': view.url ?? urls.pkgPageUrl(view.name), 'name': view.name, + 'is_external': view.isExternal, + 'external_type': externalType, + 'show_metadata': !view.isExternal, 'version': view.version, 'show_dev_version': view.devVersion != null, 'dev_version': view.devVersion, @@ -136,11 +143,9 @@ class TemplateService { 'api_pages': view.apiPages ?.map((page) => { 'title': page.title ?? page.path, - 'href': urls.pkgDocUrl( - view.name, - isLatest: true, - relativePath: page.path, - ) + 'href': page.url ?? + urls.pkgDocUrl(view.name, + isLatest: true, relativePath: page.path), }) ?.toList(), }); diff --git a/app/views/pkg/index.mustache b/app/views/pkg/index.mustache index b3da3df699..d9cbd981e9 100644 --- a/app/views/pkg/index.mustache +++ b/app/views/pkg/index.mustache @@ -25,15 +25,20 @@
    {{#packages}}
  • - {{& score_box_html }} + {{#show_metadata}}{{& score_box_html }}{{/show_metadata}}

    {{name}}

    {{desc}}

    + {{#is_external}} +

    v {{version}} • {{external_type}}

    + {{/is_external}} + {{#show_metadata}}

    v {{version}} {{#show_dev_version}} / {{dev_version}}{{/show_dev_version}} • Updated: {{last_uploaded}} {{& tags_html}}

    + {{/show_metadata}} {{#has_api_pages}}
    API results: From f31213b0fff5eeb9a27fde686493d5387a491e57 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Fri, 7 Sep 2018 19:38:58 +0200 Subject: [PATCH 06/12] SDK score box with small text --- app/lib/frontend/templates.dart | 16 ++++++++++++++-- app/views/pkg/index.mustache | 2 +- static/css/style.css | 4 ++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/app/lib/frontend/templates.dart b/app/lib/frontend/templates.dart index 8e66d01391..3599cb248d 100644 --- a/app/lib/frontend/templates.dart +++ b/app/lib/frontend/templates.dart @@ -120,8 +120,17 @@ class TemplateService { final view = packages[i]; final overallScore = view.overallScore; String externalType; + bool isSdk = false; if (view.isExternal && view.url.startsWith('https://api.dartlang.org/')) { externalType = 'Dart core library'; + isSdk = true; + } + String scoreBoxHtml; + if (isSdk) { + scoreBoxHtml = _renderSdkScoreBox(); + } else if (!view.isExternal) { + scoreBoxHtml = _renderScoreBox(view.analysisStatus, overallScore, + isNewPackage: view.isNewPackage, package: view.name); } packagesJson.add({ 'url': view.url ?? urls.pkgPageUrl(view.name), @@ -137,8 +146,7 @@ class TemplateService { 'desc': view.ellipsizedDescription, 'tags_html': _renderTags(view.analysisStatus, view.platforms, package: view.name), - 'score_box_html': _renderScoreBox(view.analysisStatus, overallScore, - isNewPackage: view.isNewPackage, package: view.name), + 'score_box_html': scoreBoxHtml, 'has_api_pages': view.apiPages != null && view.apiPages.isNotEmpty, 'api_pages': view.apiPages ?.map((page) => { @@ -884,6 +892,10 @@ String _getAuthorsHtml(List authors) { bool _isAnalysisSkipped(AnalysisStatus status) => status == AnalysisStatus.outdated || status == AnalysisStatus.discontinued; +String _renderSdkScoreBox() { + return '
    sdk
    '; +} + String _renderScoreBox(AnalysisStatus status, double overallScore, {bool isNewPackage, String package}) { final skippedAnalysis = _isAnalysisSkipped(status); diff --git a/app/views/pkg/index.mustache b/app/views/pkg/index.mustache index d9cbd981e9..f14c545e0e 100644 --- a/app/views/pkg/index.mustache +++ b/app/views/pkg/index.mustache @@ -25,7 +25,7 @@
      {{#packages}}
    • - {{#show_metadata}}{{& score_box_html }}{{/show_metadata}} + {{& score_box_html }}

      {{name}}

      {{desc}}

      {{#is_external}} diff --git a/static/css/style.css b/static/css/style.css index 02aca27815..7ff1e745e2 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -516,6 +516,10 @@ pre > code { background: #ccc; } +.score-box > .number.-small { + font-size: 12px; +} + .score-box > .new { position: absolute; top: -30%; From a9d892f59b8bfe0a8287c9c724ee0811b0a9d695 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 11 Sep 2018 09:29:10 +0200 Subject: [PATCH 07/12] Update wording. --- app/bin/service/search.dart | 2 +- app/lib/dartdoc/backend.dart | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/bin/service/search.dart b/app/bin/service/search.dart index b46947d1fc..69d7685731 100644 --- a/app/bin/service/search.dart +++ b/app/bin/service/search.dart @@ -122,7 +122,7 @@ Future _updateDartSdkIndex() async { _logger.warning('Error loading Dart SDK index.', e, st); } if (i % 10 == 0) { - _logger.warning('Unable to load Dart SDK index. Cycle: $i'); + _logger.warning('Unable to load Dart SDK index. Attempt: $i'); } await new Future.delayed(const Duration(minutes: 1)); } diff --git a/app/lib/dartdoc/backend.dart b/app/lib/dartdoc/backend.dart index 66b415fca5..88371d319c 100644 --- a/app/lib/dartdoc/backend.dart +++ b/app/lib/dartdoc/backend.dart @@ -46,7 +46,7 @@ class DartdocBackend { DartdocBackend(this._db, this._storage); - /// Whether the storage bucket has a useable extracted data file. + /// Whether the storage bucket has a usable extracted data file. /// Only the existence of the file is checked. // TODO: decide whether we should re-generate the file after a certain age Future hasValidDartSdkDartdocData() async { From 82b37f8b5db88fafcfb6deae7a271f2718cf2982 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 11 Sep 2018 09:30:35 +0200 Subject: [PATCH 08/12] Make unawaited Future more visible. --- app/bin/service/search.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/bin/service/search.dart b/app/bin/service/search.dart index 69d7685731..98c9218e97 100644 --- a/app/bin/service/search.dart +++ b/app/bin/service/search.dart @@ -80,9 +80,9 @@ Future _main(FrontendEntryMessage message) async { registerPackageIndex(new SimplePackageIndex()); registerTaskSendPort(taskReceivePort.sendPort); - // don't block on SDK index updates, as it may take several minutes before + // Don't block on SDK index updates, as it may take several minutes before // the dartdoc service produces the required output. - _updateDartSdkIndex(); + _updateDartSdkIndex().whenComplete(() {}); final BatchIndexUpdater batchIndexUpdater = new BatchIndexUpdater(); await batchIndexUpdater.initSnapshot(); From 9ca974d6787426955ca7147ead8c04c4ab542afd Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 11 Sep 2018 09:43:55 +0200 Subject: [PATCH 09/12] Use streaming storage.write --- app/lib/dartdoc/backend.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/dartdoc/backend.dart b/app/lib/dartdoc/backend.dart index 88371d319c..7007dd8b59 100644 --- a/app/lib/dartdoc/backend.dart +++ b/app/lib/dartdoc/backend.dart @@ -65,8 +65,8 @@ class DartdocBackend { final objectName = storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); try { - final contentBytes = await file.readAsBytes(); - await _storage.writeBytes(objectName, _gzip.encode(contentBytes)); + final sink = _storage.write(objectName); + await file.openRead().transform(_gzip.encoder).pipe(sink); } catch (e, st) { _logger.warning( 'Unable to upload SDK pub dartdoc data file: $objectName', e, st); From 0ec82a7e6fb5231c9d77837a325cc0645432e6d3 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 11 Sep 2018 09:47:27 +0200 Subject: [PATCH 10/12] Separate const field for SDK timeout --- app/lib/dartdoc/dartdoc_runner.dart | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/lib/dartdoc/dartdoc_runner.dart b/app/lib/dartdoc/dartdoc_runner.dart index 20c43cf57e..2677b93e13 100644 --- a/app/lib/dartdoc/dartdoc_runner.dart +++ b/app/lib/dartdoc/dartdoc_runner.dart @@ -33,7 +33,8 @@ final Uuid _uuid = new Uuid(); const statusFilePath = 'status.json'; const _archiveFilePath = 'package.tar.gz'; const _buildLogFilePath = 'log.txt'; -const _dartdocTimeout = const Duration(minutes: 10); +const _packageTimeout = const Duration(minutes: 10); +const _sdkTimeout = const Duration(minutes: 20); final Duration _twoYears = const Duration(days: 2 * 365); final _pkgPubDartdocDir = @@ -69,7 +70,7 @@ class DartdocJobProcessor extends JobProcessor { 'dart', ['bin/pub_dartdoc.dart']..addAll(args), workingDirectory: _pkgPubDartdocDir, - timeout: _dartdocTimeout * 2, + timeout: _sdkTimeout, ); final pubDataFile = new File(p.join(outputDir, 'pub-data.json')); @@ -282,7 +283,7 @@ class DartdocJobProcessor extends JobProcessor { 'dart', ['bin/pub_dartdoc.dart']..addAll(args), workingDirectory: _pkgPubDartdocDir, - timeout: _dartdocTimeout, + timeout: _packageTimeout, ); final hasIndexHtml = await new File(p.join(outputDir, 'index.html')).exists(); From b53d45ca0b798656dcb78f2e1337a20c67407f6a Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 11 Sep 2018 09:53:02 +0200 Subject: [PATCH 11/12] Throw exception when SDK doc generation is not looking right. --- app/lib/dartdoc/dartdoc_runner.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/lib/dartdoc/dartdoc_runner.dart b/app/lib/dartdoc/dartdoc_runner.dart index 2677b93e13..9433a8d538 100644 --- a/app/lib/dartdoc/dartdoc_runner.dart +++ b/app/lib/dartdoc/dartdoc_runner.dart @@ -79,7 +79,8 @@ class DartdocJobProcessor extends JobProcessor { if (!isOk) { _logger.warning( 'Error while generating SDK docs.\n\n${pr.stdout}\n\n${pr.stderr}'); - return; + throw new Exception( + 'Error while generating SDK docs (hasPubData: $hasPubData).'); } // prevent close races updating the same content in close succession From a7b2cf2af11d44a46cdf9a1705a85b9c615a6a04 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 11 Sep 2018 13:17:16 +0200 Subject: [PATCH 12/12] Remove retry of loading the SDK data, will be retried one level up. --- app/lib/dartdoc/backend.dart | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/app/lib/dartdoc/backend.dart b/app/lib/dartdoc/backend.dart index 7007dd8b59..056a3e1f0c 100644 --- a/app/lib/dartdoc/backend.dart +++ b/app/lib/dartdoc/backend.dart @@ -77,27 +77,13 @@ class DartdocBackend { Future getDartSdkDartdocData() async { final objectName = storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion); - Future load() async { - final Map map = await _storage - .read(objectName) - .transform(_gzip.decoder) - .transform(utf8.decoder) - .transform(json.decoder) - .single; - return new PubDartdocData.fromJson(map); - } - - for (int i = 0; i < 3; i++) { - try { - return await load(); - } catch (e, st) { - final message = - 'Unable to read SDK pub dartdoc data file (attempt #${i + 1}): $objectName '; - _logger.info(message, e, st); - } - } - - return null; + final Map map = await _storage + .read(objectName) + .transform(_gzip.decoder) + .transform(utf8.decoder) + .transform(json.decoder) + .single; + return new PubDartdocData.fromJson(map); } /// Returns the latest stable version of a package.