Index and search public API symbols.

app/bin/service/search.dart

@@ -14,6 +14,8 @@ import 'package:logging/logging.dart';
 import 'package:pub_dartlang_org/shared/analyzer_client.dart';
 import 'package:pub_dartlang_org/shared/analyzer_memcache.dart';
 import 'package:pub_dartlang_org/shared/configuration.dart';
+import 'package:pub_dartlang_org/shared/dartdoc_client.dart';
+import 'package:pub_dartlang_org/shared/dartdoc_memcache.dart';
 import 'package:pub_dartlang_org/shared/handler_helpers.dart';
 import 'package:pub_dartlang_org/shared/popularity_storage.dart';
 import 'package:pub_dartlang_org/shared/scheduler_stats.dart';
@@ -55,6 +57,11 @@ void _main(FrontendEntryMessage message) {
     registerAnalyzerClient(analyzerClient);
     registerScopeExitCallback(analyzerClient.close);
 
+    registerDartdocMemcache(new DartdocMemcache(memcacheService));
+    final DartdocClient dartdocClient = new DartdocClient();
+    registerDartdocClient(dartdocClient);
+    registerScopeExitCallback(dartdocClient.close);
+
     registerSearchBackend(new SearchBackend(db.dbService));
 
     final Bucket snapshotBucket = await getOrCreateBucket(

app/lib/search/backend.dart

@@ -17,6 +17,7 @@ import 'package:json_annotation/json_annotation.dart';
 import '../frontend/model_properties.dart';
 import '../frontend/models.dart';
 import '../shared/analyzer_client.dart';
+import '../shared/dartdoc_client.dart';
 import '../shared/popularity_storage.dart';
 import '../shared/search_service.dart';
 import '../shared/utils.dart';
@@ -68,10 +69,15 @@ class SearchBackend {
         versionList.where((pv) => pv != null),
         key: (pv) => (pv as PackageVersion).package);
 
+    final indexJsonFutures = Future.wait(packages.map(
+        (p) => dartdocClient.getContentBytes(p.name, 'latest', 'index.json')));
+
     final List<AnalysisView> analysisViews =
         await analyzerClient.getAnalysisViews(packages.map((p) =>
             p == null ? null : new AnalysisKey(p.name, p.latestVersion)));
 
+    final indexJsonContents = await indexJsonFutures;
+
     final List<PackageDocument> results = new List(packages.length);
     for (int i = 0; i < packages.length; i++) {
       final Package p = packages[i];
@@ -82,6 +88,9 @@ class SearchBackend {
       final analysisView = analysisViews[i];
       final double popularity = popularityStorage.lookup(pv.package) ?? 0.0;
 
+      final List<int> indexJsonContent = indexJsonContents[i];
+      final publicApiSymbols = _publicApisFromIndexJson(indexJsonContent);
+
       results[i] = new PackageDocument(
         package: pv.package,
         version: p.latestVersion,
@@ -97,6 +106,7 @@ class SearchBackend {
         maintenance: analysisView.maintenanceScore,
         dependencies: _buildDependencies(analysisView),
         emails: _buildEmails(p, pv),
+        publicApiSymbols: publicApiSymbols,
         timestamp: new DateTime.now().toUtc(),
       );
     }
@@ -121,6 +131,21 @@ class SearchBackend {
     }
     return emails.toList()..sort();
   }
+
+  List<String> _publicApisFromIndexJson(List<int> bytes) {
+    if (bytes == null) return null;
+    try {
+      final set = new Set<String>();
+      final list = json.decode(utf8.decode(bytes));
+      for (Map map in list) {
+        set.add(map['name']);
+      }
+      return set.toList()..sort();
+    } catch (e, st) {
+      _logger.warning('Parsing dartdoc index.json failed.', e, st);
+    }
+    return null;
+  }
 }
 
 class SnapshotStorage {

app/lib/search/index_simple.dart

@@ -28,6 +28,7 @@ class SimplePackageIndex implements PackageIndex {
   final TokenIndex _nameIndex = new TokenIndex(minLength: 2);
   final TokenIndex _descrIndex = new TokenIndex(minLength: 3);
   final TokenIndex _readmeIndex = new TokenIndex(minLength: 3);
+  final TokenIndex _apiIndex = new TokenIndex(minLength: 3);
   final StringInternPool _internPool = new StringInternPool();
   DateTime _lastUpdated;
   bool _isReady = false;
@@ -73,6 +74,7 @@ class SimplePackageIndex implements PackageIndex {
     _nameIndex.add(doc.package, doc.package);
     _descrIndex.add(doc.package, doc.description);
     _readmeIndex.add(doc.package, doc.readme);
+    _apiIndex.add(doc.package, doc.publicApiSymbols?.join(' '));
     final String allText = [doc.package, doc.description, doc.readme]
         .where((s) => s != null)
         .join(' ');
@@ -93,6 +95,7 @@ class SimplePackageIndex implements PackageIndex {
     _nameIndex.remove(package);
     _descrIndex.remove(package);
     _readmeIndex.remove(package);
+    _apiIndex.remove(package);
     _normalizedPackageText.remove(package);
   }
 
@@ -280,20 +283,28 @@ class SimplePackageIndex implements PackageIndex {
         final nameTokens = _nameIndex.lookupTokens(word);
         final descrTokens = _descrIndex.lookupTokens(word);
         final readmeTokens = _readmeIndex.lookupTokens(word);
-
-        final maxTokenLength = math.max(nameTokens.maxLength,
-            math.max(descrTokens.maxLength, readmeTokens.maxLength));
+        final apiTokens = _apiIndex.lookupTokens(word);
+
+        final maxTokenLength = [
+          nameTokens.maxLength,
+          descrTokens.maxLength,
+          readmeTokens.maxLength,
+          apiTokens.maxLength
+        ].fold(0, math.max);
         nameTokens.removeShortTokens(maxTokenLength);
         descrTokens.removeShortTokens(maxTokenLength);
         readmeTokens.removeShortTokens(maxTokenLength);
+        apiTokens.removeShortTokens(maxTokenLength);
 
         final name = new Score(_nameIndex.scoreDocs(nameTokens,
             weight: 1.00, wordCount: wordCount));
         final descr = new Score(_descrIndex.scoreDocs(descrTokens,
             weight: 0.95, wordCount: wordCount));
         final readme = new Score(_readmeIndex.scoreDocs(readmeTokens,
             weight: 0.90, wordCount: wordCount));
-        return Score.max([name, descr, readme]).removeLowValues(
+        final api = new Score(
+            _apiIndex.scoreDocs(apiTokens, weight: 0.80, wordCount: wordCount));
+        return Score.max([name, descr, readme, api]).removeLowValues(
             fraction: 0.01, minValue: 0.001);
       }).toList();
       Score score = Score.multiply(wordScores);

app/lib/shared/dartdoc_client.dart

@@ -7,6 +7,7 @@ import 'dart:async';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:http/http.dart' as http;
 import 'package:logging/logging.dart';
+import 'package:path/path.dart' as p;
 import 'package:pool/pool.dart';
 
 import '../dartdoc/dartdoc_runner.dart' show statusFilePath;
@@ -15,6 +16,7 @@ import '../dartdoc/models.dart' show DartdocEntry;
 import 'configuration.dart';
 import 'dartdoc_memcache.dart';
 import 'notification.dart' show notifyService;
+import 'utils.dart' show getUrlWithRetry;
 
 export '../dartdoc/models.dart' show DartdocEntry;
 
@@ -57,25 +59,33 @@ class DartdocClient {
     _client.close();
   }
 
-  Future<DartdocEntry> _getEntry(String package, String version) async {
-    final cachedContent =
-        await dartdocMemcache?.getEntryBytes(package, version, true);
-    if (cachedContent != null) {
-      return new DartdocEntry.fromBytes(cachedContent);
-    }
-    final url =
-        '$_dartdocServiceHttpHostPort/documentation/$package/$version/$statusFilePath';
+  Future<List<int>> getContentBytes(
+      String package, String version, String relativePath) async {
+    final url = p.join(_dartdocServiceHttpHostPort, 'documentation', package,
+        version, relativePath);
     try {
-      final rs = await _client.get(url);
+      final rs = await getUrlWithRetry(_client, url);
       if (rs.statusCode != 200) {
         return null;
       }
-      await dartdocMemcache?.setEntryBytes(
-          package, version, true, rs.bodyBytes);
-      return new DartdocEntry.fromBytes(rs.bodyBytes);
+      return rs.bodyBytes;
     } catch (e) {
       _logger.info('Error requesting entry for: $package $version');
     }
     return null;
   }
+
+  Future<DartdocEntry> _getEntry(String package, String version) async {
+    final cachedContent =
+        await dartdocMemcache?.getEntryBytes(package, version, true);
+    if (cachedContent != null) {
+      return new DartdocEntry.fromBytes(cachedContent);
+    }
+    final content = await getContentBytes(package, version, statusFilePath);
+    if (content != null) {
+      await dartdocMemcache?.setEntryBytes(package, version, true, content);
+      return new DartdocEntry.fromBytes(content);
+    }
+    return null;
+  }
 }

app/lib/shared/search_service.dart

@@ -58,6 +58,8 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin {
   final Map<String, String> dependencies;
   final List<String> emails;
 
+  final List<String> publicApiSymbols;
+
   /// The creation timestamp of this document.
   final DateTime timestamp;
 
@@ -76,6 +78,7 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin {
     this.maintenance,
     this.dependencies,
     this.emails,
+    this.publicApiSymbols,
     this.timestamp,
   });
 
@@ -104,6 +107,7 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin {
               value: (key) => internFn(dependencies[key]),
             ),
       emails: emails?.map(internFn)?.toList(),
+      publicApiSymbols: publicApiSymbols?.map(internFn)?.toList(),
       timestamp: timestamp,
     );
   }

app/lib/shared/search_service.g.dart

@@ -35,6 +35,9 @@ PackageDocument _$PackageDocumentFromJson(Map<String, dynamic> json) =>
             ? null
             : new Map<String, String>.from(json['dependencies'] as Map),
         emails: (json['emails'] as List)?.map((e) => e as String)?.toList(),
+        publicApiSymbols: (json['publicApiSymbols'] as List)
+            ?.map((e) => e as String)
+            ?.toList(),
         timestamp: json['timestamp'] == null
             ? null
             : DateTime.parse(json['timestamp'] as String));
@@ -54,6 +57,7 @@ abstract class _$PackageDocumentSerializerMixin {
   double get maintenance;
   Map<String, String> get dependencies;
   List<String> get emails;
+  List<String> get publicApiSymbols;
   DateTime get timestamp;
   Map<String, dynamic> toJson() => <String, dynamic>{
         'package': package,
@@ -70,6 +74,7 @@ abstract class _$PackageDocumentSerializerMixin {
         'maintenance': maintenance,
         'dependencies': dependencies,
         'emails': emails,
+        'publicApiSymbols': publicApiSymbols,
         'timestamp': timestamp?.toIso8601String()
       };
 }

app/test/frontend/handlers_test_utils.dart

@@ -319,4 +319,10 @@ class DartdocClientMock implements DartdocClient {
 
   @override
   Future close() async {}
+
+  @override
+  Future<List<int>> getContentBytes(
+      String package, String version, String relativePath) async {
+    return null;
+  }
 }

app/test/search/index_api_test.dart

@@ -0,0 +1,118 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:test/test.dart';
+
+import 'package:pub_dartlang_org/search/index_simple.dart';
+import 'package:pub_dartlang_org/search/text_utils.dart';
+import 'package:pub_dartlang_org/shared/search_service.dart';
+
+void main() {
+  group('api', () {
+    SimplePackageIndex index;
+
+    setUpAll(() async {
+      index = new SimplePackageIndex();
+      await index.addPackage(new PackageDocument(
+        package: 'foo',
+        version: '1.0.0',
+        description: compactDescription('Yet another web framework.'),
+        publicApiSymbols: [
+          'generateWebPage',
+          'WebPageGenerator',
+        ],
+      ));
+      await index.addPackage(new PackageDocument(
+        package: 'other_with_api',
+        version: '2.0.0',
+        description: compactDescription('Unrelated package'),
+        publicApiSymbols: [
+          'foo',
+          'serveWebPages',
+        ],
+      ));
+      await index.addPackage(new PackageDocument(
+        package: 'other_without_api',
+        version: '2.0.0',
+        description: compactDescription('Unrelated package'),
+      ));
+      await index.merge();
+    });
+
+    test('foo', () async {
+      final PackageSearchResult result = await index
+          .search(new SearchQuery.parse(query: 'foo', order: SearchOrder.text));
+      expect(json.decode(json.encode(result)), {
+        'indexUpdated': isNotNull,
+        'totalCount': 2,
+        'packages': [
+          {
+            'package': 'foo',
+            'score': closeTo(0.986, 0.001), // finds package name
+          },
+          {
+            'package': 'other_with_api',
+            'score': closeTo(0.772, 0.001), // finds foo method
+          },
+          // should not contain `other_without_api`
+        ],
+      });
+    });
+
+    test('server', () async {
+      final PackageSearchResult result = await index.search(
+          new SearchQuery.parse(query: 'server', order: SearchOrder.text));
+      expect(json.decode(json.encode(result)), {
+        'indexUpdated': isNotNull,
+        'totalCount': 1,
+        'packages': [
+          {
+            'package': 'other_with_api',
+            'score': closeTo(0.772, 0.001), // find serveWebPages
+          },
+          // should not contain `other_without_api`
+        ],
+      });
+    });
+
+    test('page generator', () async {
+      final PackageSearchResult result = await index.search(
+          new SearchQuery.parse(
+              query: 'page generator', order: SearchOrder.text));
+      expect(json.decode(json.encode(result)), {
+        'indexUpdated': isNotNull,
+        'totalCount': 1,
+        'packages': [
+          {
+            'package': 'foo',
+            'score': closeTo(0.614, 0.001), // find WebPageGenerator
+          },
+          // should not contain `other_without_api`
+        ],
+      });
+    });
+
+    test('web page', () async {
+      final PackageSearchResult result = await index.search(
+          new SearchQuery.parse(query: 'web page', order: SearchOrder.text));
+      expect(json.decode(json.encode(result)), {
+        'indexUpdated': isNotNull,
+        'totalCount': 2,
+        'packages': [
+          {
+            'package': 'foo',
+            'score': closeTo(0.731, 0.001), // find WebPageGenerator
+          },
+          {
+            'package': 'other_with_api',
+            'score': closeTo(0.617, 0.001), // find serveWebPages
+          },
+          // should not contain `other_without_api`
+        ],
+      });
+    });
+  });
+}