API for name completion data.

app/lib/frontend/handlers/custom_api.dart

@@ -3,6 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:io';
 
 import 'package:client_data/package_api.dart';
 import 'package:shelf/shelf.dart' as shelf;
@@ -21,6 +22,7 @@ import '../../shared/exceptions.dart';
 import '../../shared/handlers.dart';
 import '../../shared/redis_cache.dart' show cache;
 import '../../shared/urls.dart' as urls;
+import '../../shared/utils.dart' show jsonUtf8Encoder;
 
 /// Handles requests for /api/documentation/<package>
 Future<shelf.Response> apiDocumentationHandler(
@@ -89,6 +91,38 @@ Future<shelf.Response> apiPackageNamesHandler(shelf.Request request) async {
   });
 }
 
+/// Handles requests for
+/// - /api/package-name-completion-data
+Future<shelf.Response> apiPackageNameCompletionDataHandler(
+    shelf.Request request) async {
+  // only accept requests which allow gzip content encoding
+  if (!request.acceptsEncoding('gzip')) {
+    throw NotAcceptableException('Client must accept gzip content.');
+  }
+
+  final bytes = await cache.packageNameCompletitionDataJsonGz().get(() async {
+    final rs = await searchClient.search(
+      ServiceSearchQuery.parse(
+        tagsPredicate: TagsPredicate.regularSearch(),
+        limit: 20000,
+      ),
+      // Do not cache response at the search client level, as we'll be caching
+      // it in a processed form much longer.
+      skipCache: true,
+    );
+
+    return gzip.encode(jsonUtf8Encoder.convert({
+      'packages': rs.packages.map((p) => p.package).toList(),
+    }));
+  });
+
+  return shelf.Response(200, body: bytes, headers: {
+    ...jsonResponseHeaders,
+    'Content-Encoding': 'gzip',
+    'Cache-Control': 'public, max-age=28800', // 8 hours caching
+  });
+}
+
 /// Handles request for /api/packages?page=<num>
 Future<shelf.Response> apiPackagesHandler(shelf.Request request) async {
   final int pageSize = 100;

app/lib/frontend/handlers/pubapi.dart

@@ -316,6 +316,11 @@ class PubApi {
     return apiPackageNamesHandler(request);
   }
 
+  @EndPoint.get('/api/package-name-completion-data')
+  Future<Response> packageNameCompletionData(Request request) async {
+    return apiPackageNameCompletionDataHandler(request);
+  }
+
   @EndPoint.get('/api/packages/<package>/metrics')
   Future<Response> packageMetrics(Request request, String package) =>
       apiPackageMetricsHandler(request, package);

app/lib/search/search_client.dart

@@ -43,6 +43,7 @@ class SearchClient {
     ServiceSearchQuery query, {
     Duration ttl,
     Duration updateCacheAfter,
+    bool skipCache = false,
   }) async {
     // check validity first
     final validity = query.evaluateValidity();
@@ -76,17 +77,23 @@ class SearchClient {
       return result;
     }
 
-    final cacheEntry = cache.packageSearchResult(serviceUrl, ttl: ttl);
-    var result = await cacheEntry.get(searchFn);
-
-    if (updateCacheAfter != null &&
-        result?.timestamp != null &&
-        result.age > updateCacheAfter) {
-      _logger.info('Updating stale cache entry.');
-      final value = await searchFn();
-      if (value != null) {
-        await cacheEntry.set(value);
-        result = value;
+    PackageSearchResult result;
+
+    if (skipCache) {
+      result = await searchFn();
+    } else {
+      final cacheEntry = cache.packageSearchResult(serviceUrl, ttl: ttl);
+      result = await cacheEntry.get(searchFn);
+
+      if (updateCacheAfter != null &&
+          result?.timestamp != null &&
+          result.age > updateCacheAfter) {
+        _logger.info('Updating stale cache entry.');
+        final value = await searchFn();
+        if (value != null) {
+          await cacheEntry.set(value);
+          result = value;
+        }
       }
     }
 

app/lib/shared/exceptions.dart

@@ -32,6 +32,12 @@ class NotFoundException extends ResponseException {
       : super._(404, 'NotFound', 'Could not find `$resource`.');
 }
 
+/// Thrown when request is not acceptable.
+class NotAcceptableException extends ResponseException {
+  NotAcceptableException(String message)
+      : super._(406, 'NotAcceptable', message);
+}
+
 /// Thrown when request input is invalid, bad payload, wrong querystring, etc.
 class InvalidInputException extends ResponseException {
   InvalidInputException._(String message)

app/lib/shared/handlers.dart

@@ -27,6 +27,12 @@ const staticShortCache = Duration(minutes: 5);
 /// and it matches the etag.
 const staticLongCache = Duration(days: 7);
 
+/// The default header values for JSON responses.
+const jsonResponseHeaders = <String, String>{
+  'content-type': 'application/json; charset="utf-8"',
+  'x-content-type-options': 'nosniff',
+};
+
 final _logger = Logger('pub.shared.handler');
 final _prettyJson = JsonUtf8Encoder('  ');
 
@@ -49,9 +55,8 @@ shelf.Response jsonResponse(
     status,
     body: body,
     headers: {
+      ...jsonResponseHeaders,
       if (headers != null) ...headers,
-      'content-type': 'application/json; charset="utf-8"',
-      'x-content-type-options': 'nosniff',
     },
   );
 }
@@ -192,3 +197,16 @@ bool isNotModified(shelf.Request request, DateTime lastModified, String etag) {
 
   return false;
 }
+
+extension RequestExt on shelf.Request {
+  /// Returns true if the current request declares that it accepts the [encoding].
+  ///
+  /// NOTE: the method does not parses the header, only checks whether the String
+  ///       value is present (or everything is accepted).
+  bool acceptsEncoding(String encoding) {
+    final accepting = headers[HttpHeaders.acceptEncodingHeader];
+    return accepting == null ||
+        accepting.contains('*') ||
+        accepting.contains(encoding);
+  }
+}

app/lib/shared/redis_cache.dart

@@ -221,6 +221,10 @@ class CachePatterns {
       .withTTL(Duration(hours: 12))
       .withCodec(utf8)[requestedUri];
 
+  Entry<List<int>> packageNameCompletitionDataJsonGz() => _cache
+      .withPrefix('api-package-name-completition-data-json-gz')
+      .withTTL(Duration(hours: 8))['-'];
+
   Entry<PublisherPage> allPublishersPage() => publisherPage('-');
 
   Entry<PublisherPage> publisherPage(String userId) => _cache

pkg/pub_integration/lib/script/public_pages.dart

@@ -23,6 +23,9 @@ class PublicPagesScript {
     assert(_pubClient == null);
     _pubClient = PubHttpClient(pubHostedUrl);
     try {
+      if (!expectLiveSite) {
+        await _pubClient.forceSearchUpdate();
+      }
       await _landingPage();
       await _helpPages();
       await _securityPage();
@@ -85,6 +88,11 @@ class PublicPagesScript {
     if (packageNames == null || !packageNames.contains('retry')) {
       throw Exception('Expected "retry" in the list of package names.');
     }
+
+    final completitionData = await _pubClient.apiPackageNameCompletionData();
+    if (completitionData == null || !completitionData.contains('retry')) {
+      throw Exception('Expected "retry" in the package name completion data.');
+    }
   }
 
   Future<void> _badRequest() async {

pkg/pub_integration/lib/src/pub_http_client.dart

@@ -224,6 +224,18 @@ class PubHttpClient {
     return packages;
   }
 
+  /// Returns the list of packages from `/api/package-name-completion-data` endpoint.
+  Future<List<String>> apiPackageNameCompletionData() async {
+    final rs =
+        await _http.get('$pubHostedUrl/api/package-name-completion-data');
+    if (rs.statusCode != 200) {
+      throw Exception('Unexpected status code: ${rs.statusCode}');
+    }
+    final map = json.decode(rs.body) as Map<String, dynamic>;
+    final packages = (map['packages'] as List).cast<String>();
+    return packages;
+  }
+
   /// Free resources.
   Future<void> close() async {
     _http.close();