diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 621e848831..1e0cc9d380 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -14,6 +14,7 @@ import 'package:gcloud/storage.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:json_annotation/json_annotation.dart'; +import '../dartdoc/pub_dartdoc_data.dart'; import '../frontend/model_properties.dart'; import '../frontend/models.dart'; import '../shared/analyzer_client.dart'; @@ -70,15 +71,15 @@ class SearchBackend { versionList.where((pv) => pv != null), key: (pv) => (pv as PackageVersion).package); - final indexJsonFutures = Future.wait(packages.map((p) => - dartdocClient.getContentBytes(p.name, 'latest', 'index.json', + final pubDataFutures = Future.wait(packages.map((p) => + dartdocClient.getContentBytes(p.name, 'latest', 'pub-data.json', timeout: const Duration(seconds: 10)))); final List analysisViews = await analyzerClient.getAnalysisViews(packages.map((p) => p == null ? null : new AnalysisKey(p.name, p.latestVersion))); - final indexJsonContents = await indexJsonFutures; + final pubDataContents = await pubDataFutures; final List results = new List(packages.length); for (int i = 0; i < packages.length; i++) { @@ -90,8 +91,15 @@ class SearchBackend { final analysisView = analysisViews[i]; final double popularity = popularityStorage.lookup(pv.package) ?? 0.0; - final List indexJsonContent = indexJsonContents[i]; - final apiDocPages = _apiDocPagesFromIndexJson(indexJsonContent); + final List pubDataContent = pubDataContents[i]; + List apiDocPages; + if (pubDataContent != null) { + try { + apiDocPages = _apiDocPagesFromPubData(pubDataContent); + } catch (e, st) { + _logger.severe('Parsing pub-data.json failed.', e, st); + } + } results[i] = new PackageDocument( package: pv.package, @@ -135,43 +143,56 @@ class SearchBackend { return emails.toList()..sort(); } - List _apiDocPagesFromIndexJson(List bytes) { - if (bytes == null) return null; - try { - final list = json.decode(utf8.decode(bytes)); - - final pathMap = {}; - final symbolMap = >{}; - for (Map map in list) { - final String name = map['name']; - final type = map['type']; - if (isCommonApiSymbol(name) && type != 'library') { - continue; - } + List _apiDocPagesFromPubData(List bytes) { + final decodedMap = json.decode(utf8.decode(bytes)) as Map; + final pubData = new PubDartdocData.fromJson(decodedMap.cast()); + + final nameToKindMap = {}; + pubData.apiElements.forEach((e) { + nameToKindMap[e.name] = e.kind; + }); - final String qualifiedName = map['qualifiedName']; - final enclosedBy = map['enclosedBy']; - final enclosedByType = enclosedBy is Map ? enclosedBy['type'] : null; - final parentLevel = enclosedByType == 'class' ? 2 : 1; - final String key = qualifiedName.split('.').take(parentLevel).join('.'); + final pathMap = {}; + final symbolMap = >{}; + final docMap = >{}; - if (key == qualifiedName) { - pathMap[key] = map['href'] as String; - } - symbolMap.putIfAbsent(key, () => new Set()).add(name); - } + bool isTopLevel(String kind) => kind == 'library' || kind == 'class'; - final results = pathMap.keys.map((key) { - final path = pathMap[key]; - final symbols = symbolMap[key].toList()..sort(); - return new ApiDocPage(relativePath: path, symbols: symbols); - }).toList(); - results.sort((a, b) => a.relativePath.compareTo(b.relativePath)); - return results; - } catch (e, st) { - _logger.warning('Parsing dartdoc index.json failed.', e, st); + void update(String key, String name, String documentation) { + final set = symbolMap.putIfAbsent(key, () => new Set()); + set.addAll(name.split('.')); + + documentation = documentation?.trim(); + if (documentation != null && documentation.isNotEmpty) { + final list = docMap.putIfAbsent(key, () => []); + list.add(compactReadme(documentation)); + } } - return null; + + pubData.apiElements.forEach((apiElement) { + if (isTopLevel(apiElement.kind)) { + pathMap[apiElement.name] = apiElement.href; + update(apiElement.name, apiElement.name, apiElement.documentation); + } + + if (!isTopLevel(apiElement.kind) && + apiElement.parent != null && + isTopLevel(nameToKindMap[apiElement.parent])) { + update(apiElement.parent, apiElement.name, apiElement.documentation); + } + }); + + final results = pathMap.keys.map((key) { + final path = pathMap[key]; + final symbols = symbolMap[key].toList()..sort(); + return new ApiDocPage( + relativePath: path, + symbols: symbols, + textBlocks: docMap[key], + ); + }).toList(); + results.sort((a, b) => a.relativePath.compareTo(b.relativePath)); + return results; } } diff --git a/app/lib/search/index_simple.dart b/app/lib/search/index_simple.dart index 8d215357df..b44e419a5f 100644 --- a/app/lib/search/index_simple.dart +++ b/app/lib/search/index_simple.dart @@ -85,8 +85,11 @@ class SimplePackageIndex implements PackageIndex { _descrIndex.add(doc.package, doc.description); _readmeIndex.add(doc.package, doc.readme); for (ApiDocPage page in doc.apiDocPages ?? const []) { - _apiDocIndex.add( - _apiDocPageId(doc.package, page), page.symbols?.join(' ')); + final text = [page.symbols, page.textBlocks] + .where((list) => list != null && list.isNotEmpty) + .expand((list) => list) + .join(' '); + _apiDocIndex.add(_apiDocPageId(doc.package, page), text); } final String allText = [doc.package, doc.description, doc.readme] .where((s) => s != null) diff --git a/app/lib/shared/search_service.dart b/app/lib/shared/search_service.dart index 1874008a27..4cd8369100 100644 --- a/app/lib/shared/search_service.dart +++ b/app/lib/shared/search_service.dart @@ -116,8 +116,9 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin { class ApiDocPage extends Object with _$ApiDocPageSerializerMixin { final String relativePath; final List symbols; + final List textBlocks; - ApiDocPage({this.relativePath, this.symbols}); + ApiDocPage({this.relativePath, this.symbols, this.textBlocks}); factory ApiDocPage.fromJson(Map json) => _$ApiDocPageFromJson(json); @@ -126,6 +127,7 @@ class ApiDocPage extends Object with _$ApiDocPageSerializerMixin { return new ApiDocPage( relativePath: internFn(relativePath), symbols: symbols?.map(internFn)?.toList(), + textBlocks: textBlocks, ); } } diff --git a/app/lib/shared/search_service.g.dart b/app/lib/shared/search_service.g.dart index 4bd492d239..81d7a92d0c 100644 --- a/app/lib/shared/search_service.g.dart +++ b/app/lib/shared/search_service.g.dart @@ -86,14 +86,20 @@ abstract class _$PackageDocumentSerializerMixin { ApiDocPage _$ApiDocPageFromJson(Map json) { return new ApiDocPage( relativePath: json['relativePath'] as String, - symbols: (json['symbols'] as List)?.map((e) => e as String)?.toList()); + symbols: (json['symbols'] as List)?.map((e) => e as String)?.toList(), + textBlocks: + (json['textBlocks'] as List)?.map((e) => e as String)?.toList()); } abstract class _$ApiDocPageSerializerMixin { String get relativePath; List get symbols; - Map toJson() => - {'relativePath': relativePath, 'symbols': symbols}; + List get textBlocks; + Map toJson() => { + 'relativePath': relativePath, + 'symbols': symbols, + 'textBlocks': textBlocks + }; } PackageSearchResult _$PackageSearchResultFromJson(Map json) { diff --git a/app/test/search/api_doc_page_test.dart b/app/test/search/api_doc_page_test.dart index 670994aa9c..fb622c6818 100644 --- a/app/test/search/api_doc_page_test.dart +++ b/app/test/search/api_doc_page_test.dart @@ -27,6 +27,9 @@ void main() { 'generateWebPage', 'WebPageGenerator', ], + textBlocks: [ + 'Some fancy goal is described here.', + ], ), ], )); @@ -100,7 +103,7 @@ void main() { 'packages': [ { 'package': 'foo', - 'score': closeTo(0.481, 0.001), // find WebPageGenerator + 'score': closeTo(0.478, 0.001), // find WebPageGenerator 'apiPages': [ {'title': null, 'path': 'generator.html'}, ], @@ -119,7 +122,7 @@ void main() { 'packages': [ { 'package': 'foo', - 'score': closeTo(0.572, 0.001), // find WebPageGenerator + 'score': closeTo(0.570, 0.001), // find WebPageGenerator 'apiPages': [ {'title': null, 'path': 'generator.html'}, ], @@ -135,5 +138,23 @@ void main() { ], }); }); + + test('text block', () async { + final PackageSearchResult result = await index.search( + new SearchQuery.parse(query: 'goal fancy', order: SearchOrder.text)); + expect(json.decode(json.encode(result)), { + 'indexUpdated': isNotNull, + 'totalCount': 1, + 'packages': [ + { + 'package': 'foo', + 'score': closeTo(0.624, 0.001), + 'apiPages': [ + {'title': null, 'path': 'generator.html'}, + ], + }, + ], + }); + }); }); }