Skip to content

Commit 003f527

Browse files
isoosisoos
authored
Search order implementation with additional tests. (#366)
1 parent 9ec2b69 commit 003f527

File tree

6 files changed

+265
-95
lines changed

6 files changed

+265
-95
lines changed

‎app/lib/search/index_simple.dart

Lines changed: 120 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -71,66 +71,59 @@ class SimplePackageIndex implements PackageIndex {
7171

7272
@override
7373
Future<PackageSearchResult> search(SearchQuery query) async {
74-
final Map<String, double> total = <String, double>{};
75-
void addAll(Map<String, double> scores, double weight) {
76-
scores?.forEach((String url, double score) {
77-
if (score != null) {
78-
final double prev = total[url] ?? 0.0;
79-
total[url] = prev + score * weight;
80-
}
81-
});
82-
}
83-
84-
addAll(_nameIndex.search(query.text), 0.70);
85-
addAll(_descrIndex.search(query.text), 0.10);
86-
addAll(_readmeIndex.search(query.text), 0.05);
87-
88-
if ((query.text == null || query.text.isEmpty) &&
89-
query.packagePrefix != null) {
90-
addAll(_nameIndex.search(query.packagePrefix), 0.8);
74+
// do text matching
75+
final Score textScore = _searchText(query.text, query.packagePrefix);
76+
77+
// The set of urls to filter on.
78+
final Set<String> urls =
79+
textScore?.getKeys()?.toSet() ?? _documents.keys.toSet();
80+
81+
// filter on package prefix
82+
if (query.packagePrefix != null) {
83+
urls.removeWhere(
84+
(url) => !_documents[url]
85+
.package
86+
.toLowerCase()
87+
.startsWith(query.packagePrefix.toLowerCase()),
88+
);
9189
}
9290

93-
addAll(getHealthScore(total.keys), 0.05);
94-
addAll(getPopularityScore(total.keys), 0.10);
95-
96-
List<PackageScore> results = <PackageScore>[];
97-
for (String url in total.keys) {
98-
final PackageDocument doc = _documents[url];
99-
100-
// filter on platform
101-
if (query.platformPredicate != null &&
102-
!query.platformPredicate.matches(doc.platforms)) {
103-
continue;
104-
}
105-
106-
// filter on package prefix
107-
if (query.packagePrefix != null &&
108-
!doc.package
109-
.toLowerCase()
110-
.startsWith(query.packagePrefix.toLowerCase())) {
111-
continue;
112-
}
113-
114-
results.add(new PackageScore(
115-
url: doc.url,
116-
package: doc.package,
117-
score: total[url],
118-
));
91+
// filter on platform
92+
if (query.platformPredicate != null) {
93+
urls.removeWhere(
94+
(url) => !query.platformPredicate.matches(_documents[url].platforms));
11995
}
12096

121-
results.sort((a, b) => -a.score.compareTo(b.score));
122-
123-
// filter out the noise (maybe a single matching ngram)
124-
if (results.isNotEmpty) {
125-
final double bestScore = results.first.score;
126-
final double scoreTreshold = bestScore / 25;
127-
results.removeWhere((pr) => pr.score < scoreTreshold);
97+
// reduce text results if filter did remove an url
98+
textScore?.removeWhere((key) => !urls.contains(key));
99+
100+
List<PackageScore> results;
101+
switch (query.order ?? SearchOrder.overall) {
102+
case SearchOrder.overall:
103+
final Score overallScore = new Score()
104+
..addValues(textScore?.values, 0.85)
105+
..addValues(getPopularityScore(urls), 0.10)
106+
..addValues(getHealthScore(urls), 0.05);
107+
results = _rankWithValues(overallScore.values);
108+
break;
109+
case SearchOrder.text:
110+
results = _rankWithValues(textScore.values);
111+
break;
112+
case SearchOrder.updated:
113+
results = _rankWithComparator(urls, _compareUpdated);
114+
break;
115+
case SearchOrder.popularity:
116+
results = _rankWithValues(getPopularityScore(urls));
117+
break;
118+
case SearchOrder.health:
119+
results = _rankWithValues(getHealthScore(urls));
120+
break;
128121
}
129122

130123
// bound by offset and limit
131-
final int totalCount = min(maxSearchResults, results.length);
124+
final int totalCount = results.length;
132125
if (query.offset != null && query.offset > 0) {
133-
if (query.offset > totalCount) {
126+
if (query.offset >= results.length) {
134127
results = <PackageScore>[];
135128
} else {
136129
results = results.sublist(query.offset);
@@ -168,6 +161,81 @@ class SimplePackageIndex implements PackageIndex {
168161
value: (String url) => _documents[url].popularity * 100,
169162
);
170163
}
164+
165+
Score _searchText(String text, String packagePrefix) {
166+
if (text != null && text.isNotEmpty) {
167+
final Score textScore = new Score()
168+
..addValues(_nameIndex.search(text), 0.82)
169+
..addValues(_descrIndex.search(text), 0.12)
170+
..addValues(_readmeIndex.search(text), 0.06);
171+
// removes scores that are less than 5% of the best
172+
textScore.removeLowScores(0.05);
173+
// removes scores that are low
174+
textScore.removeWhere((url) => textScore.values[url] < 1.0);
175+
return textScore;
176+
}
177+
return null;
178+
}
179+
180+
List<PackageScore> _rankWithValues(Map<String, double> values) {
181+
final List<PackageScore> list = values.keys
182+
.map((url) => new PackageScore(
183+
url: url,
184+
package: _documents[url].package,
185+
score: values[url],
186+
))
187+
.toList();
188+
list.sort((a, b) {
189+
final int scoreCompare = -a.score.compareTo(b.score);
190+
if (scoreCompare != 0) return scoreCompare;
191+
// if two packages got the same score, order by last updated
192+
return _compareUpdated(_documents[a.url], _documents[b.url]);
193+
});
194+
return list;
195+
}
196+
197+
List<PackageScore> _rankWithComparator(
198+
Set<String> urls, int compare(PackageDocument a, PackageDocument b)) {
199+
final List<PackageScore> list = urls
200+
.map((url) =>
201+
new PackageScore(url: url, package: _documents[url].package))
202+
.toList();
203+
list.sort((a, b) => compare(_documents[a.url], _documents[b.url]));
204+
return list;
205+
}
206+
207+
int _compareUpdated(PackageDocument a, PackageDocument b) {
208+
if (a.updated == null) return -1;
209+
if (b.updated == null) return 1;
210+
return -a.updated.compareTo(b.updated);
211+
}
212+
}
213+
214+
class Score {
215+
final Map<String, double> values = <String, double>{};
216+
217+
Iterable<String> getKeys() => values.keys;
218+
219+
void addValues(Map<String, double> newValues, double weight) {
220+
if (newValues == null) return;
221+
newValues.forEach((String key, double score) {
222+
if (score != null) {
223+
final double prev = values[key] ?? 0.0;
224+
values[key] = prev + score * weight;
225+
}
226+
});
227+
}
228+
229+
void removeWhere(bool keyCondition(String key)) {
230+
final Set<String> keysToRemove = values.keys.where(keyCondition).toSet();
231+
keysToRemove.forEach(values.remove);
232+
}
233+
234+
void removeLowScores(double fraction) {
235+
final double maxValue = values.values.fold(0.0, max);
236+
final double cutoff = maxValue * fraction;
237+
removeWhere((key) => values[key] < cutoff);
238+
}
171239
}
172240

173241
class TokenIndex {

‎app/lib/shared/search_service.dart

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,14 @@ class SearchQuery {
184184
/// Sanity check, whether the query object is to be expected a valid result.
185185
bool get isValid {
186186
final bool hasText = text != null && text.isNotEmpty;
187-
final bool hasPackagePrefix =
188-
packagePrefix != null && packagePrefix.isNotEmpty;
189-
final bool hasNonTextOrdering = order != null &&
190-
order != SearchOrder.overall &&
191-
order != SearchOrder.text;
192-
return hasText || hasPackagePrefix || hasNonTextOrdering;
187+
final bool hasNonTextOrdering = order != SearchOrder.text;
188+
final bool isEmpty = !hasText &&
189+
order == null &&
190+
packagePrefix == null &&
191+
(platformPredicate == null || !platformPredicate.isNotEmpty);
192+
if (isEmpty) return false;
193+
194+
return hasText || hasNonTextOrdering;
193195
}
194196
}
195197

@@ -221,6 +223,8 @@ class PackageSearchResult extends Object
221223
class PackageScore extends Object with _$PackageScoreSerializerMixin {
222224
final String url;
223225
final String package;
226+
227+
@JsonKey(includeIfNull: false)
224228
final double score;
225229

226230
PackageScore({

‎app/lib/shared/search_service.g.dart

Lines changed: 15 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/test/search/handlers_test.dart

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ void main() {
5959
{
6060
'url': 'https://pub.domain/packages/pkg_foo',
6161
'package': 'pkg_foo',
62-
'score': closeTo(71.1, 0.1),
62+
'score': closeTo(70.9, 0.1),
6363
}
6464
],
6565
});
@@ -105,7 +105,7 @@ void main() {
105105
{
106106
'url': 'https://pub.domain/packages/pkg_foo',
107107
'package': 'pkg_foo',
108-
'score': closeTo(13.3, 0.1),
108+
'score': closeTo(1.0, 0.1),
109109
}
110110
],
111111
});

0 commit comments

Comments
 (0)