Generate, index and display Dart SDK API results in pub search.

app/bin/service/dartdoc.dart

@@ -68,6 +68,8 @@ Future _workerMain(WorkerEntryMessage message) async {
 
     final jobProcessor =
         new DartdocJobProcessor(lockDuration: const Duration(minutes: 30));
+    await jobProcessor.generateDocsForSdk();
+
     final jobMaintenance = new JobMaintenance(dbService, jobProcessor);
 
     new Timer.periodic(const Duration(minutes: 15), (_) async {

app/bin/service/search.dart

@@ -11,6 +11,7 @@ import 'package:gcloud/service_scope.dart';
 import 'package:gcloud/storage.dart';
 import 'package:logging/logging.dart';
 
+import 'package:pub_dartlang_org/dartdoc/backend.dart';
 import 'package:pub_dartlang_org/shared/analyzer_client.dart';
 import 'package:pub_dartlang_org/shared/analyzer_memcache.dart';
 import 'package:pub_dartlang_org/shared/configuration.dart';
@@ -23,6 +24,8 @@ import 'package:pub_dartlang_org/shared/service_utils.dart';
 import 'package:pub_dartlang_org/shared/task_client.dart';
 import 'package:pub_dartlang_org/shared/task_scheduler.dart';
 import 'package:pub_dartlang_org/shared/task_sources.dart';
+import 'package:pub_dartlang_org/shared/versions.dart';
+import 'package:pub_dartlang_org/shared/urls.dart';
 
 import 'package:pub_dartlang_org/search/backend.dart';
 import 'package:pub_dartlang_org/search/handlers.dart';
@@ -56,6 +59,9 @@ Future _main(FrontendEntryMessage message) async {
     registerAnalyzerClient(analyzerClient);
     registerScopeExitCallback(analyzerClient.close);
 
+    final Bucket dartdocBucket = await getOrCreateBucket(
+        storageService, activeConfiguration.dartdocStorageBucketName);
+    registerDartdocBackend(new DartdocBackend(db.dbService, dartdocBucket));
     registerDartdocMemcache(new DartdocMemcache(memcacheService));
     final DartdocClient dartdocClient = new DartdocClient();
     registerDartdocClient(dartdocClient);
@@ -69,9 +75,15 @@ Future _main(FrontendEntryMessage message) async {
         new SnapshotStorage(storageService, snapshotBucket));
 
     final ReceivePort taskReceivePort = new ReceivePort();
+    registerDartSdkIndex(new SimplePackageIndex.sdk(
+        urlPrefix: dartSdkMainUrl(toolEnvSdkVersion)));
     registerPackageIndex(new SimplePackageIndex());
     registerTaskSendPort(taskReceivePort.sendPort);
 
+    // Don't block on SDK index updates, as it may take several minutes before
+    // the dartdoc service produces the required output.
+    _updateDartSdkIndex().whenComplete(() {});
+
     final BatchIndexUpdater batchIndexUpdater = new BatchIndexUpdater();
     await batchIndexUpdater.initSnapshot();
 
@@ -92,3 +104,26 @@ Future _main(FrontendEntryMessage message) async {
     await runHandler(_logger, searchServiceHandler);
   });
 }
+
+Future _updateDartSdkIndex() async {
+  for (int i = 0;; i++) {
+    try {
+      _logger.info('Trying to load SDK index.');
+      final data = await dartdocBackend.getDartSdkDartdocData();
+      if (data != null) {
+        final docs =
+            splitLibraries(data).map((lib) => createSdkDocument(lib)).toList();
+        await dartSdkIndex.addPackages(docs);
+        await dartSdkIndex.merge();
+        _logger.info('Dart SDK index loaded successfully.');
+        return;
+      }
+    } catch (e, st) {
+      _logger.warning('Error loading Dart SDK index.', e, st);
+    }
+    if (i % 10 == 0) {
+      _logger.warning('Unable to load Dart SDK index. Attempt: $i');
+    }
+    await new Future.delayed(const Duration(minutes: 1));
+  }
+}

app/lib/dartdoc/backend.dart

@@ -3,6 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:gcloud/db.dart';
@@ -21,9 +22,11 @@ import '../shared/utils.dart' show contentType;
 import '../shared/versions.dart' as shared_versions;
 
 import 'models.dart';
+import 'pub_dartdoc_data.dart';
 import 'storage_path.dart' as storage_path;
 
 final Logger _logger = new Logger('pub.dartdoc.backend');
+final _gzip = new GZipCodec();
 
 final Duration _contentDeleteThreshold = const Duration(days: 1);
 final int _concurrentUploads = 8;
@@ -43,6 +46,46 @@ class DartdocBackend {
 
   DartdocBackend(this._db, this._storage);
 
+  /// Whether the storage bucket has a usable extracted data file.
+  /// Only the existence of the file is checked.
+  // TODO: decide whether we should re-generate the file after a certain age
+  Future<bool> hasValidDartSdkDartdocData() async {
+    final objectName =
+        storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion);
+    try {
+      final info = await _storage.info(objectName);
+      return info != null;
+    } catch (_) {
+      return false;
+    }
+  }
+
+  /// Upload the generated dartdoc data file for the Dart SDK to the storage bucket.
+  Future uploadDartSdkDartdocData(File file) async {
+    final objectName =
+        storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion);
+    try {
+      final sink = _storage.write(objectName);
+      await file.openRead().transform(_gzip.encoder).pipe(sink);
+    } catch (e, st) {
+      _logger.warning(
+          'Unable to upload SDK pub dartdoc data file: $objectName', e, st);
+    }
+  }
+
+  /// Read the generated dartdoc data file for the Dart SDK.
+  Future<PubDartdocData> getDartSdkDartdocData() async {
+    final objectName =
+        storage_path.dartSdkDartdocDataName(shared_versions.runtimeVersion);
+    final Map<String, dynamic> map = await _storage
+        .read(objectName)
+        .transform(_gzip.decoder)
+        .transform(utf8.decoder)
+        .transform(json.decoder)
+        .single;
+    return new PubDartdocData.fromJson(map);
+  }
+
   /// Returns the latest stable version of a package.
   Future<String> getLatestVersion(String package) async {
     final list = await _db.lookup([_db.emptyKey.append(Package, id: package)]);

app/lib/dartdoc/dartdoc_runner.dart

@@ -18,6 +18,7 @@ import '../job/job.dart';
 import '../scorecard/backend.dart';
 import '../scorecard/models.dart';
 import '../shared/analyzer_client.dart';
+import '../shared/configuration.dart' show envConfig;
 import '../shared/tool_env.dart';
 import '../shared/urls.dart';
 import '../shared/versions.dart' as versions;
@@ -32,7 +33,8 @@ final Uuid _uuid = new Uuid();
 const statusFilePath = 'status.json';
 const _archiveFilePath = 'package.tar.gz';
 const _buildLogFilePath = 'log.txt';
-const _dartdocTimeout = const Duration(minutes: 10);
+const _packageTimeout = const Duration(minutes: 10);
+const _sdkTimeout = const Duration(minutes: 20);
 final Duration _twoYears = const Duration(days: 2 * 365);
 
 final _pkgPubDartdocDir =
@@ -42,6 +44,57 @@ class DartdocJobProcessor extends JobProcessor {
   DartdocJobProcessor({Duration lockDuration})
       : super(service: JobService.dartdoc, lockDuration: lockDuration);
 
+  /// Uses the tool environment's SDK (the one that is used for analysis too) to
+  /// generate dartdoc documentation and extracted data file for SDK API indexing.
+  /// Only the extracted data file will be used and uploaded.
+  Future generateDocsForSdk() async {
+    if (await dartdocBackend.hasValidDartSdkDartdocData()) return;
+    final tempDir =
+        await Directory.systemTemp.createTemp('pub-dartlang-dartdoc');
+    try {
+      final tempDirPath = tempDir.resolveSymbolicLinksSync();
+      final outputDir = tempDirPath;
+      final args = [
+        '--sdk-docs',
+        '--output',
+        outputDir,
+        '--hosted-url',
+        siteRoot,
+        '--link-to-remote',
+        '--no-validate-links',
+      ];
+      if (envConfig.toolEnvDartSdkDir != null) {
+        args.addAll(['--sdk-dir', envConfig.toolEnvDartSdkDir]);
+      }
+      final pr = await runProc(
+        'dart',
+        ['bin/pub_dartdoc.dart']..addAll(args),
+        workingDirectory: _pkgPubDartdocDir,
+        timeout: _sdkTimeout,
+      );
+
+      final pubDataFile = new File(p.join(outputDir, 'pub-data.json'));
+      final hasPubData = await pubDataFile.exists();
+      final isOk = pr.exitCode == 0 && hasPubData;
+      if (!isOk) {
+        _logger.warning(
+            'Error while generating SDK docs.\n\n${pr.stdout}\n\n${pr.stderr}');
+        throw new Exception(
+            'Error while generating SDK docs (hasPubData: $hasPubData).');
+      }
+
+      // prevent close races updating the same content in close succession
+      if (await dartdocBackend.hasValidDartSdkDartdocData()) return;
+
+      // upload only the pub dartdoc data file
+      await dartdocBackend.uploadDartSdkDartdocData(pubDataFile);
+    } catch (e, st) {
+      _logger.warning('Error while generating SDK docs.', e, st);
+    } finally {
+      await tempDir.delete(recursive: true);
+    }
+  }
+
   @override
   Future<bool> shouldProcess(
       String package, String version, DateTime updated) async {
@@ -231,7 +284,7 @@ class DartdocJobProcessor extends JobProcessor {
         'dart',
         ['bin/pub_dartdoc.dart']..addAll(args),
         workingDirectory: _pkgPubDartdocDir,
-        timeout: _dartdocTimeout,
+        timeout: _packageTimeout,
       );
       final hasIndexHtml =
           await new File(p.join(outputDir, 'index.html')).exists();

app/lib/dartdoc/storage_path.dart

@@ -9,6 +9,7 @@ const _generatedStaticAssetPaths = const <String>['static-assets'];
 // Storage contains package data in a form of /package/version/...
 // This path contains '-' and is an invalid package name, safe of conflicts.
 const _storageSharedAssetPrefix = 'shared-assets';
+const _sdkAssetPrefix = 'sdk-assets';
 
 /// Whether the generated file can be moved to the shared assets.
 bool isSharedAsset(String relativePath) {
@@ -49,3 +50,13 @@ String contentObjectName(String packageName, String packageVersion, String uuid,
     String relativePath) {
   return p.join(contentPrefix(packageName, packageVersion, uuid), relativePath);
 }
+
+/// ObjectName of an SDK asset.
+String sdkObjectName(String relativePath) {
+  return p.join(_sdkAssetPrefix, relativePath);
+}
+
+/// The ObjectName for the Dart SDK's extracted dartdoc content.
+String dartSdkDartdocDataName(String runtimeVersion) {
+  return sdkObjectName('dart/pub-dartdoc-data/$runtimeVersion.json.gz');
+}

app/lib/frontend/models.dart

@@ -217,6 +217,8 @@ class PrivateKey extends db.Model {
 /// An extract of [Package] and [PackageVersion] and [AnalysisView], for
 /// display-only uses.
 class PackageView {
+  final bool isExternal;
+  final String url;
   final String name;
   final String version;
   // Not null only if there is a difference compared to the [version].
@@ -231,6 +233,8 @@ class PackageView {
   final List<ApiPageRef> apiPages;
 
   PackageView({
+    this.isExternal: false,
+    this.url,
     this.name,
     this.version,
     this.devVersion,

app/lib/frontend/search_service.dart

@@ -36,12 +36,14 @@ class SearchService {
 Future<SearchResultPage> _loadResultForPackages(
     SearchQuery query, int totalCount, List<PackageScore> packageScores) async {
   final List<Key> packageKeys = packageScores
+      .where((ps) => !ps.isExternal)
       .map((ps) => ps.package)
       .map((package) => dbService.emptyKey.append(Package, id: package))
       .toList();
   final packageEntries = (await dbService.lookup(packageKeys)).cast<Package>();
   packageEntries.removeWhere((p) => p == null);
 
+  final pubPackages = <String, PackageView>{};
   final List<Key> versionKeys =
       packageEntries.map((p) => p.latestVersionKey).toList();
   if (versionKeys.isNotEmpty) {
@@ -57,18 +59,37 @@ Future<SearchResultPage> _loadResultForPackages(
     final List<PackageVersion> versions =
         (await batchResults[1]).cast<PackageVersion>();
 
-    final List<PackageView> resultPackages = new List.generate(
-        versions.length,
-        (i) => new PackageView.fromModel(
-              package: packageEntries[i],
-              version: versions[i],
-              analysis: analysisExtracts[i],
-              apiPages: packageScores[i].apiPages,
-            ));
-    return new SearchResultPage(query, totalCount, resultPackages);
-  } else {
-    return new SearchResultPage.empty(query);
+    for (int i = 0; i < versions.length; i++) {
+      final pv = new PackageView.fromModel(
+        package: packageEntries[i],
+        version: versions[i],
+        analysis: analysisExtracts[i],
+        apiPages: packageScores[i].apiPages,
+      );
+      pubPackages[pv.name] = pv;
+    }
   }
+
+  final resultPackages = packageScores
+      .map((ps) {
+        if (pubPackages.containsKey(ps.package)) {
+          return pubPackages[ps.package];
+        }
+        if (ps.isExternal) {
+          return new PackageView(
+            isExternal: true,
+            url: ps.url,
+            version: ps.version,
+            name: ps.package,
+            ellipsizedDescription: ps.description,
+            apiPages: ps.apiPages,
+          );
+        }
+        return null;
+      })
+      .where((pv) => pv != null)
+      .toList();
+  return new SearchResultPage(query, totalCount, resultPackages);
 }
 
 /// The results of a search via the Custom Search API.

app/lib/frontend/templates.dart

@@ -119,9 +119,25 @@ class TemplateService {
     for (int i = 0; i < packages.length; i++) {
       final view = packages[i];
       final overallScore = view.overallScore;
+      String externalType;
+      bool isSdk = false;
+      if (view.isExternal && view.url.startsWith('https://api.dartlang.org/')) {
+        externalType = 'Dart core library';
+        isSdk = true;
+      }
+      String scoreBoxHtml;
+      if (isSdk) {
+        scoreBoxHtml = _renderSdkScoreBox();
+      } else if (!view.isExternal) {
+        scoreBoxHtml = _renderScoreBox(view.analysisStatus, overallScore,
+            isNewPackage: view.isNewPackage, package: view.name);
+      }
       packagesJson.add({
-        'url': urls.pkgPageUrl(view.name),
+        'url': view.url ?? urls.pkgPageUrl(view.name),
         'name': view.name,
+        'is_external': view.isExternal,
+        'external_type': externalType,
+        'show_metadata': !view.isExternal,
         'version': view.version,
         'show_dev_version': view.devVersion != null,
         'dev_version': view.devVersion,
@@ -130,17 +146,14 @@ class TemplateService {
         'desc': view.ellipsizedDescription,
         'tags_html': _renderTags(view.analysisStatus, view.platforms,
             package: view.name),
-        'score_box_html': _renderScoreBox(view.analysisStatus, overallScore,
-            isNewPackage: view.isNewPackage, package: view.name),
+        'score_box_html': scoreBoxHtml,
         'has_api_pages': view.apiPages != null && view.apiPages.isNotEmpty,
         'api_pages': view.apiPages
             ?.map((page) => {
                   'title': page.title ?? page.path,
-                  'href': urls.pkgDocUrl(
-                    view.name,
-                    isLatest: true,
-                    relativePath: page.path,
-                  )
+                  'href': page.url ??
+                      urls.pkgDocUrl(view.name,
+                          isLatest: true, relativePath: page.path),
                 })
             ?.toList(),
       });
@@ -879,6 +892,10 @@ String _getAuthorsHtml(List<String> authors) {
 bool _isAnalysisSkipped(AnalysisStatus status) =>
     status == AnalysisStatus.outdated || status == AnalysisStatus.discontinued;
 
+String _renderSdkScoreBox() {
+  return '<div class="score-box"><span class="number -small -solid">sdk</span></div>';
+}
+
 String _renderScoreBox(AnalysisStatus status, double overallScore,
     {bool isNewPackage, String package}) {
   final skippedAnalysis = _isAnalysisSkipped(status);