Skip to content

Use extracted dartdoc texts in search. #1428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 5, 2018
Merged

Use extracted dartdoc texts in search. #1428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd28edf
Use extracted dartdoc texts in search.
isoos Jul 3, 2018
baa15a2
Fix code review notes: wrapping and log level.
isoos Jul 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 59 additions & 38 deletions app/lib/search/backend.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import 'package:gcloud/storage.dart';
import 'package:gcloud/service_scope.dart' as ss;
import 'package:json_annotation/json_annotation.dart';

import '../dartdoc/pub_dartdoc_data.dart';
import '../frontend/model_properties.dart';
import '../frontend/models.dart';
import '../shared/analyzer_client.dart';
Expand Down Expand Up @@ -70,15 +71,15 @@ class SearchBackend {
versionList.where((pv) => pv != null),
key: (pv) => (pv as PackageVersion).package);

final indexJsonFutures = Future.wait(packages.map((p) =>
dartdocClient.getContentBytes(p.name, 'latest', 'index.json',
final pubDataFutures = Future.wait(packages.map((p) =>
dartdocClient.getContentBytes(p.name, 'latest', 'pub-data.json',
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does pub-data.json already exist?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prod: not yet, but it will with the next release.

timeout: const Duration(seconds: 10))));

final List<AnalysisView> analysisViews =
await analyzerClient.getAnalysisViews(packages.map((p) =>
p == null ? null : new AnalysisKey(p.name, p.latestVersion)));

final indexJsonContents = await indexJsonFutures;
final pubDataContents = await pubDataFutures;

final List<PackageDocument> results = new List(packages.length);
for (int i = 0; i < packages.length; i++) {
Expand All @@ -90,8 +91,15 @@ class SearchBackend {
final analysisView = analysisViews[i];
final double popularity = popularityStorage.lookup(pv.package) ?? 0.0;

final List<int> indexJsonContent = indexJsonContents[i];
final apiDocPages = _apiDocPagesFromIndexJson(indexJsonContent);
final List<int> pubDataContent = pubDataContents[i];
List<ApiDocPage> apiDocPages;
if (pubDataContent != null) {
try {
apiDocPages = _apiDocPagesFromPubData(pubDataContent);
} catch (e, st) {
_logger.severe('Parsing pub-data.json failed.', e, st);
}
}

results[i] = new PackageDocument(
package: pv.package,
Expand Down Expand Up @@ -135,43 +143,56 @@ class SearchBackend {
return emails.toList()..sort();
}

List<ApiDocPage> _apiDocPagesFromIndexJson(List<int> bytes) {
if (bytes == null) return null;
try {
final list = json.decode(utf8.decode(bytes));

final pathMap = <String, String>{};
final symbolMap = <String, Set<String>>{};
for (Map map in list) {
final String name = map['name'];
final type = map['type'];
if (isCommonApiSymbol(name) && type != 'library') {
continue;
}
List<ApiDocPage> _apiDocPagesFromPubData(List<int> bytes) {
final decodedMap = json.decode(utf8.decode(bytes)) as Map;
final pubData = new PubDartdocData.fromJson(decodedMap.cast());

final nameToKindMap = <String, String>{};
pubData.apiElements.forEach((e) {
nameToKindMap[e.name] = e.kind;
});

final String qualifiedName = map['qualifiedName'];
final enclosedBy = map['enclosedBy'];
final enclosedByType = enclosedBy is Map ? enclosedBy['type'] : null;
final parentLevel = enclosedByType == 'class' ? 2 : 1;
final String key = qualifiedName.split('.').take(parentLevel).join('.');
final pathMap = <String, String>{};
final symbolMap = <String, Set<String>>{};
final docMap = <String, List<String>>{};

if (key == qualifiedName) {
pathMap[key] = map['href'] as String;
}
symbolMap.putIfAbsent(key, () => new Set()).add(name);
}
bool isTopLevel(String kind) => kind == 'library' || kind == 'class';

final results = pathMap.keys.map((key) {
final path = pathMap[key];
final symbols = symbolMap[key].toList()..sort();
return new ApiDocPage(relativePath: path, symbols: symbols);
}).toList();
results.sort((a, b) => a.relativePath.compareTo(b.relativePath));
return results;
} catch (e, st) {
_logger.warning('Parsing dartdoc index.json failed.', e, st);
void update(String key, String name, String documentation) {
final set = symbolMap.putIfAbsent(key, () => new Set<String>());
set.addAll(name.split('.'));

documentation = documentation?.trim();
if (documentation != null && documentation.isNotEmpty) {
final list = docMap.putIfAbsent(key, () => []);
list.add(compactReadme(documentation));
}
}
return null;

pubData.apiElements.forEach((apiElement) {
if (isTopLevel(apiElement.kind)) {
pathMap[apiElement.name] = apiElement.href;
update(apiElement.name, apiElement.name, apiElement.documentation);
}

if (!isTopLevel(apiElement.kind) &&
apiElement.parent != null &&
isTopLevel(nameToKindMap[apiElement.parent])) {
update(apiElement.parent, apiElement.name, apiElement.documentation);
}
});

final results = pathMap.keys.map((key) {
final path = pathMap[key];
final symbols = symbolMap[key].toList()..sort();
return new ApiDocPage(
relativePath: path,
symbols: symbols,
textBlocks: docMap[key],
);
}).toList();
results.sort((a, b) => a.relativePath.compareTo(b.relativePath));
return results;
}
}

Expand Down
7 changes: 5 additions & 2 deletions app/lib/search/index_simple.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,11 @@ class SimplePackageIndex implements PackageIndex {
_descrIndex.add(doc.package, doc.description);
_readmeIndex.add(doc.package, doc.readme);
for (ApiDocPage page in doc.apiDocPages ?? const []) {
_apiDocIndex.add(
_apiDocPageId(doc.package, page), page.symbols?.join(' '));
final text = [page.symbols, page.textBlocks]
.where((list) => list != null && list.isNotEmpty)
.expand((list) => list)
.join(' ');
_apiDocIndex.add(_apiDocPageId(doc.package, page), text);
}
final String allText = [doc.package, doc.description, doc.readme]
.where((s) => s != null)
Expand Down
4 changes: 3 additions & 1 deletion app/lib/shared/search_service.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,9 @@ class PackageDocument extends Object with _$PackageDocumentSerializerMixin {
class ApiDocPage extends Object with _$ApiDocPageSerializerMixin {
final String relativePath;
final List<String> symbols;
final List<String> textBlocks;

ApiDocPage({this.relativePath, this.symbols});
ApiDocPage({this.relativePath, this.symbols, this.textBlocks});

factory ApiDocPage.fromJson(Map<String, dynamic> json) =>
_$ApiDocPageFromJson(json);
Expand All @@ -126,6 +127,7 @@ class ApiDocPage extends Object with _$ApiDocPageSerializerMixin {
return new ApiDocPage(
relativePath: internFn(relativePath),
symbols: symbols?.map(internFn)?.toList(),
textBlocks: textBlocks,
);
}
}
Expand Down
12 changes: 9 additions & 3 deletions app/lib/shared/search_service.g.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,20 @@ abstract class _$PackageDocumentSerializerMixin {
ApiDocPage _$ApiDocPageFromJson(Map<String, dynamic> json) {
return new ApiDocPage(
relativePath: json['relativePath'] as String,
symbols: (json['symbols'] as List)?.map((e) => e as String)?.toList());
symbols: (json['symbols'] as List)?.map((e) => e as String)?.toList(),
textBlocks:
(json['textBlocks'] as List)?.map((e) => e as String)?.toList());
}

abstract class _$ApiDocPageSerializerMixin {
String get relativePath;
List<String> get symbols;
Map<String, dynamic> toJson() =>
<String, dynamic>{'relativePath': relativePath, 'symbols': symbols};
List<String> get textBlocks;
Map<String, dynamic> toJson() => <String, dynamic>{
'relativePath': relativePath,
'symbols': symbols,
'textBlocks': textBlocks
};
}

PackageSearchResult _$PackageSearchResultFromJson(Map<String, dynamic> json) {
Expand Down
25 changes: 23 additions & 2 deletions app/test/search/api_doc_page_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ void main() {
'generateWebPage',
'WebPageGenerator',
],
textBlocks: [
'Some fancy goal is described here.',
],
),
],
));
Expand Down Expand Up @@ -100,7 +103,7 @@ void main() {
'packages': [
{
'package': 'foo',
'score': closeTo(0.481, 0.001), // find WebPageGenerator
'score': closeTo(0.478, 0.001), // find WebPageGenerator
'apiPages': [
{'title': null, 'path': 'generator.html'},
],
Expand All @@ -119,7 +122,7 @@ void main() {
'packages': [
{
'package': 'foo',
'score': closeTo(0.572, 0.001), // find WebPageGenerator
'score': closeTo(0.570, 0.001), // find WebPageGenerator
'apiPages': [
{'title': null, 'path': 'generator.html'},
],
Expand All @@ -135,5 +138,23 @@ void main() {
],
});
});

test('text block', () async {
final PackageSearchResult result = await index.search(
new SearchQuery.parse(query: 'goal fancy', order: SearchOrder.text));
expect(json.decode(json.encode(result)), {
'indexUpdated': isNotNull,
'totalCount': 1,
'packages': [
{
'package': 'foo',
'score': closeTo(0.624, 0.001),
'apiPages': [
{'title': null, 'path': 'generator.html'},
],
},
],
});
});
});
}