diff --git a/lib/src/crc32c.dart b/lib/src/crc32c.dart new file mode 100644 index 000000000..e346a5943 --- /dev/null +++ b/lib/src/crc32c.dart @@ -0,0 +1,103 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Computes a crc32c checksum. +class Crc32c { + int _current = mask; + static const mask = 0xFFFFFFFF; + + // Algorithm based on https://en.wikipedia.org/wiki/Cyclic_redundancy_check + void update(List data) { + for (var i = 0; i < data.length; i++) { + final lookupIndex = (_current ^ data[i]) & 0xff; + _current = (_current >> 8) ^ _crcTable[lookupIndex]; + } + } + + int finalize() { + // Finalize the CRC-32 value by inverting all the bits + return _current ^ mask & mask; + } + + /// Consumes the entirety of "stream" and returns the CRC32C checksum of its + /// data once the stream is finished. + static Future computeByConsumingStream(Stream> stream) async { + final checksumComputer = Crc32c(); + + await for (final chunk in stream) { + checksumComputer.update(chunk); + } + + return checksumComputer.finalize(); + } +} + +// Generated by ./pycrc.py --algorithm=table-driven --model=crc-32c --generate=c +// See: https://pycrc.org/ +const _crcTable = [ + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, // + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +]; diff --git a/lib/src/exceptions.dart b/lib/src/exceptions.dart index fcde747d8..ec9f87659 100644 --- a/lib/src/exceptions.dart +++ b/lib/src/exceptions.dart @@ -104,6 +104,15 @@ class PackageNotFoundException extends WrappedException { String toString() => 'Package not available ($message).'; } +/// A class for exceptions where a package's checksum could not be validated. +class PackageIntegrityException extends WrappedException { + PackageIntegrityException( + String message, { + Object? innerError, + StackTrace? innerTrace, + }) : super(message, innerError, innerTrace); +} + /// Returns whether [error] is a user-facing error object. /// /// This includes both [ApplicationException] and any dart:io errors. diff --git a/lib/src/io.dart b/lib/src/io.dart index 50f7eb9d3..2b5d0cdc5 100644 --- a/lib/src/io.dart +++ b/lib/src/io.dart @@ -172,7 +172,7 @@ List readBinaryFile(String file) { } /// Reads the contents of the binary file [file] as a [Stream]. -Stream> readBinaryFileAsSream(String file) { +Stream> readBinaryFileAsStream(String file) { log.io('Reading binary file $file.'); var contents = File(file).openRead(); return contents; diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index e7dc87063..de90a64dd 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -5,15 +5,19 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io' as io; +import 'dart:math' as math; +import 'dart:typed_data'; import 'package:collection/collection.dart' show maxBy, IterableNullableExtension; import 'package:http/http.dart' as http; +import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; import 'package:stack_trace/stack_trace.dart'; import '../authentication/client.dart'; +import '../crc32c.dart'; import '../exceptions.dart'; import '../http.dart'; import '../io.dart'; @@ -851,27 +855,53 @@ class HostedSource extends CachedSource { 'Package $packageName has no version $version'); } - var url = versionInfo.archiveUrl; - log.io('Get package from $url.'); + final archiveUrl = versionInfo.archiveUrl; + log.io('Get package from $archiveUrl.'); log.message('Downloading ${log.bold(id.name)} ${id.version}...'); // Download and extract the archive to a temp directory. await withTempDir((tempDirForArchive) async { - var archivePath = - p.join(tempDirForArchive, '$packageName-$version.tar.gz'); - var response = await withAuthenticatedClient( - cache, - Uri.parse(description.url), - (client) => client.send(http.Request('GET', url))); - - // We download the archive to disk instead of streaming it directly into - // the tar unpacking. This simplifies stream handling. - // Package:tar cancels the stream when it reaches end-of-archive, and - // cancelling a http stream makes it not reusable. - // There are ways around this, and we might revisit this later. - await createFileFromStream(response.stream, archivePath); + var fileName = '$packageName-$version.tar.gz'; + var archivePath = p.join(tempDirForArchive, fileName); + + // The client from `withAuthenticatedClient` will retry HTTP requests. + // This wrapper is one layer up and will retry checksum validation errors. + await retry( + // Attempt to download archive and validate its checksum. + () async { + final request = http.Request('GET', archiveUrl); + final response = await withAuthenticatedClient(cache, + Uri.parse(description.url), (client) => client.send(request)); + final expectedChecksum = _parseCrc32c(response.headers, fileName); + + Stream> stream = response.stream; + if (expectedChecksum != null) { + stream = _validateStream( + response.stream, expectedChecksum, id, archiveUrl); + } + + // We download the archive to disk instead of streaming it directly + // into the tar unpacking. This simplifies stream handling. + // Package:tar cancels the stream when it reaches end-of-archive, and + // cancelling a http stream makes it not reusable. + // There are ways around this, and we might revisit this later. + await createFileFromStream(stream, archivePath); + }, + // Retry if the checksum response header was malformed or the actual + // checksum did not match the expected checksum. + retryIf: (e) => e is PackageIntegrityException, + onRetry: (e, retryCount) => log + .io('Retry #${retryCount + 1} because of checksum error with GET ' + '$archiveUrl...'), + maxAttempts: math.max( + 1, // Having less than 1 attempt doesn't make sense. + int.tryParse(io.Platform.environment['PUB_MAX_HTTP_RETRIES'] ?? '') ?? + 7, + ), + ); + var tempDir = cache.createTempDir(); - await extractTarGz(readBinaryFileAsSream(archivePath), tempDir); + await extractTarGz(readBinaryFileAsStream(archivePath), tempDir); // Now that the get has succeeded, move it to the real location in the // cache. @@ -1100,3 +1130,84 @@ class _RefAndCache { @override bool operator ==(Object other) => other is _RefAndCache && other.ref == ref; } + +@visibleForTesting +const checksumHeaderName = 'x-goog-hash'; + +/// Adds a checksum validation "tap" to the response stream and returns a +/// wrapped `Stream` object, which should be used to consume the incoming data. +/// +/// As chunks are received, a CRC32C checksum is updated. +/// Once the download is completed, the final checksum is compared with +/// the one present in the checksum response header. +/// +/// Throws [PackageIntegrityException] if there is a checksum mismatch. +Stream> _validateStream(Stream> stream, + int expectedChecksum, PackageId id, Uri archiveUrl) async* { + final crc32c = Crc32c(); + + await for (final chunk in stream) { + crc32c.update(chunk); + yield chunk; + } + + final actualChecksum = crc32c.finalize(); + + log.fine( + 'Computed checksum $actualChecksum for ${id.name} ${id.version} with ' + 'expected CRC32C of $expectedChecksum.'); + + if (actualChecksum != expectedChecksum) { + throw PackageIntegrityException( + 'Package archive for ${id.name} ${id.version} downloaded from ' + '"$archiveUrl" has "x-goog-hash: crc32c=$expectedChecksum", which ' + 'doesn\'t match the checksum of the archive downloaded.'); + } +} + +/// Parses response [headers] and returns the archive's CRC32C checksum. +/// +/// In most cases, GCS provides both MD5 and CRC32C checksums in its response +/// headers. It uses the header name "x-goog-hash" for these values. It has +/// been documented and observed that GCS will send multiple response headers +/// with the same "x-goog-hash" token as the key. +/// https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooghash +/// +/// Additionally, when the Dart http client encounters multiple response +/// headers with the same key, it concatenates their values with a comma +/// before inserting a single item with that key and concatenated value into +/// its response "headers" Map. +/// See https://github.com/dart-lang/http/issues/24 +/// https://github.com/dart-lang/http/blob/06649afbb5847dbb0293816ba8348766b116e419/pkgs/http/lib/src/base_response.dart#L29 +/// +/// Throws [PackageIntegrityException] if the CRC32C checksum cannot be parsed. +int? _parseCrc32c(Map headers, String fileName) { + final checksumHeader = headers[checksumHeaderName]; + if (checksumHeader == null) return null; + + final parts = checksumHeader.split(','); + for (final part in parts) { + if (part.startsWith('crc32c=')) { + final undecoded = part.substring('crc32c='.length); + + try { + final bytes = base64Decode(undecoded); + + // CRC32C must be 32 bits, or 4 bytes. + if (bytes.length != 4) { + throw FormatException('CRC32C checksum has invalid length', bytes); + } + + return ByteData.view(bytes.buffer).getUint32(0); + } on FormatException catch (e, s) { + throw PackageIntegrityException( + 'Package archive "$fileName" has a malformed CRC32C checksum in ' + 'its response headers', + innerError: e, + innerTrace: s); + } + } + } + + return null; +} diff --git a/lib/src/utils.dart b/lib/src/utils.dart index e4451dbdf..dbc762255 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -638,3 +638,63 @@ Map mapMap( key(entry.key, entry.value): value(entry.key, entry.value), }; } + +/// Call [fn] retrying so long as [retryIf] return `true` for the exception +/// thrown, up-to [maxAttempts] times. +/// +/// Defaults to 8 attempts, sleeping as following after 1st, 2nd, 3rd, ..., +/// 7th attempt: +/// 1. 400 ms +/- 25% +/// 2. 800 ms +/- 25% +/// 3. 1600 ms +/- 25% +/// 4. 3200 ms +/- 25% +/// 5. 6400 ms +/- 25% +/// 6. 12800 ms +/- 25% +/// 7. 25600 ms +/- 25% +/// +/// ```dart +/// final response = await retry( +/// // Make a GET request +/// () => http.get('https://google.com').timeout(Duration(seconds: 5)), +/// // Retry on SocketException or TimeoutException +/// retryIf: (e) => e is SocketException || e is TimeoutException, +/// ); +/// print(response.body); +/// ``` +/// +/// If no [retryIf] function is given this will retry any for any [Exception] +/// thrown. To retry on an [Error], the error must be caught and _rethrown_ +/// as an [Exception]. +/// +/// See https://github.com/google/dart-neats/blob/master/retry/lib/retry.dart +Future retry( + FutureOr Function() fn, { + Duration delayFactor = const Duration(milliseconds: 200), + double randomizationFactor = 0.25, + Duration maxDelay = const Duration(seconds: 30), + int maxAttempts = 8, + FutureOr Function(Exception)? retryIf, + FutureOr Function(Exception, int retryCount)? onRetry, +}) async { + var attempt = 0; + // ignore: literal_only_boolean_expressions + while (true) { + attempt++; // first invocation is the first attempt + try { + return await fn(); + } on Exception catch (e) { + if (attempt >= maxAttempts || (retryIf != null && !(await retryIf(e)))) { + rethrow; + } + if (onRetry != null) { + await onRetry(e, attempt); + } + } + + // Sleep for a delay + final rf = randomizationFactor * (random.nextDouble() * 2 - 1) + 1; + final exp = math.min(attempt, 31); // prevent overflows. + final delay = delayFactor * math.pow(2.0, exp) * rf; + await Future.delayed(delay < maxDelay ? delay : maxDelay); + } +} diff --git a/test/embedding/embedding_test.dart b/test/embedding/embedding_test.dart index 73370d7f6..9f74405d4 100644 --- a/test/embedding/embedding_test.dart +++ b/test/embedding/embedding_test.dart @@ -357,6 +357,18 @@ String _filter(String input) { RegExp(r'Writing \d+ characters', multiLine: true), r'Writing $N characters', ) + .replaceAll( + RegExp(r'x-goog-hash(.*)$', multiLine: true), + r'x-goog-hash: $CHECKSUM_HEADER', + ) + .replaceAll( + RegExp( + r'Computed checksum \d+ for foo 1.0.0 with expected CRC32C of ' + r'\d+\.', + multiLine: true), + r'Computed checksum $CRC32C for foo 1.0.0 with expected CRC32C of ' + r'$CRC32C.', + ) /// TODO(sigurdm): This hack suppresses differences in stack-traces /// between dart 2.17 and 2.18. Remove when 2.18 is stable. diff --git a/test/get/hosted/get_test.dart b/test/get/hosted/get_test.dart index f0270a3c7..81d484558 100644 --- a/test/get/hosted/get_test.dart +++ b/test/get/hosted/get_test.dart @@ -12,10 +12,13 @@ import '../../descriptor.dart' as d; import '../../test_pub.dart'; void main() { - test('gets a package from a pub server', () async { + test('gets a package from a pub server and validates its CRC32C checksum', + () async { final server = await servePackages(); server.serve('foo', '1.2.3'); + expect(await server.peekArchiveChecksumHeader('foo', '1.2.3'), isNotNull); + await d.appDir({'foo': '1.2.3'}).create(); await pubGet(); @@ -26,6 +29,62 @@ void main() { ]).validate(); }); + group('gets a package from a pub server without validating its checksum', () { + late PackageServer server; + + setUp(() async { + server = await servePackages() + ..serveChecksums = false + ..serve('foo', '1.2.3') + ..serve('bar', '1.2.3', headers: { + 'x-goog-hash': [''] + }) + ..serve('baz', '1.2.3', headers: { + 'x-goog-hash': ['md5=loremipsum'] + }); + }); + + test('because of omitted checksum header', () async { + expect(await server.peekArchiveChecksumHeader('foo', '1.2.3'), isNull); + + await d.appDir({'foo': '1.2.3'}).create(); + + await pubGet(); + + await d.cacheDir({'foo': '1.2.3'}).validate(); + await d.appPackageConfigFile([ + d.packageConfigEntry(name: 'foo', version: '1.2.3'), + ]).validate(); + }); + + test('because of empty checksum header', () async { + expect(await server.peekArchiveChecksumHeader('bar', '1.2.3'), ''); + + await d.appDir({'bar': '1.2.3'}).create(); + + await pubGet(); + + await d.cacheDir({'bar': '1.2.3'}).validate(); + await d.appPackageConfigFile([ + d.packageConfigEntry(name: 'bar', version: '1.2.3'), + ]).validate(); + }); + + test('because of missing CRC32C in checksum header', () async { + expect(await server.peekArchiveChecksumHeader('baz', '1.2.3'), + 'md5=loremipsum'); + + await d.appDir({'baz': '1.2.3'}).create(); + + await pubGet(); + + await d.cacheDir({'baz': '1.2.3'}).validate(); + await d.appPackageConfigFile([ + d.packageConfigEntry(name: 'baz', version: '1.2.3'), + ]).validate(); + }); + }); + test('URL encodes the package name', () async { await servePackages(); @@ -64,6 +123,148 @@ void main() { ]).validate(); }); + test('recognizes and retries a package with a CRC32C checksum mismatch', + () async { + var server = await startPackageServer(); + + server.serve('foo', '1.2.3', headers: { + 'x-goog-hash': PackageServer.composeChecksumHeader(crc32c: 3381945770) + }); + + await d.appDir({ + 'foo': { + 'version': '1.2.3', + 'hosted': {'name': 'foo', 'url': 'http://localhost:${server.port}'} + } + }).create(); + + await pubGet( + error: RegExp( + r'''Package archive for foo 1.2.3 downloaded from "(.+)" has ''' + r'''"x-goog-hash: crc32c=(\d+)", which doesn't match the checksum ''' + r'''of the archive downloaded\.'''), + silent: contains('Retry #2 because of checksum error'), + environment: { + 'PUB_MAX_HTTP_RETRIES': '2', + }, + ); + }); + + group('recognizes bad checksum header and retries', () { + late PackageServer server; + + setUp(() async { + server = await servePackages() + ..serve('foo', '1.2.3', headers: { + 'x-goog-hash': ['crc32c=,md5='] + }) + ..serve('bar', '1.2.3', headers: { + 'x-goog-hash': ['crc32c=loremipsum,md5=loremipsum'] + }) + ..serve('baz', '1.2.3', headers: { + 'x-goog-hash': ['crc32c=MTIzNDU=,md5=NTQzMjE='] + }); + }); + + test('when the CRC32C checksum is empty', () async { + await d.appDir({ + 'foo': { + 'version': '1.2.3', + 'hosted': {'name': 'foo', 'url': 'http://localhost:${server.port}'} + } + }).create(); + + await pubGet( + exitCode: exit_codes.DATA, + error: contains( + 'Package archive "foo-1.2.3.tar.gz" has a malformed CRC32C ' + 'checksum in its response headers'), + silent: contains('Retry #2 because of checksum error'), + environment: { + 'PUB_MAX_HTTP_RETRIES': '2', + }, + ); + }); + + test('when the CRC32C checksum has bad encoding', () async { + await d.appDir({ + 'bar': { + 'version': '1.2.3', + 'hosted': {'name': 'bar', 'url': 'http://localhost:${server.port}'} + } + }).create(); + + await pubGet( + exitCode: exit_codes.DATA, + error: contains( + 'Package archive "bar-1.2.3.tar.gz" has a malformed CRC32C ' + 'checksum in its response headers'), + silent: contains('Retry #2 because of checksum error'), + environment: { + 'PUB_MAX_HTTP_RETRIES': '2', + }, + ); + }); + + test('when the CRC32C checksum is malformed', () async { + await d.appDir({ + 'baz': { + 'version': '1.2.3', + 'hosted': {'name': 'baz', 'url': 'http://localhost:${server.port}'} + } + }).create(); + + await pubGet( + exitCode: exit_codes.DATA, + error: contains( + 'Package archive "baz-1.2.3.tar.gz" has a malformed CRC32C ' + 'checksum in its response headers'), + silent: contains('Retry #2 because of checksum error'), + environment: { + 'PUB_MAX_HTTP_RETRIES': '2', + }, + ); + }); + }); + + test('gets a package from a pub server that uses gzip response compression', + () async { + final server = await servePackages(); + server.autoCompress = true; + server.serveChecksums = false; + server.serve('foo', '1.2.3'); + + expect(await server.peekArchiveChecksumHeader('foo', '1.2.3'), isNull); + + await d.appDir({'foo': '1.2.3'}).create(); + + await pubGet(); + + await d.cacheDir({'foo': '1.2.3'}).validate(); + await d.appPackageConfigFile([ + d.packageConfigEntry(name: 'foo', version: '1.2.3'), + ]).validate(); + }); + + test( + 'gets a package from a pub server that uses gzip response compression ' + 'and validates its CRC32C checksum', () async { + final server = await servePackages(); + server.autoCompress = true; + server.serve('foo', '1.2.3'); + + expect(await server.peekArchiveChecksumHeader('foo', '1.2.3'), isNotNull); + + await d.appDir({'foo': '1.2.3'}).create(); + + await pubGet(); + + await d.cacheDir({'foo': '1.2.3'}).validate(); + await d.appPackageConfigFile([ + d.packageConfigEntry(name: 'foo', version: '1.2.3'), + ]).validate(); + }); + group('categorizes dependency types in the lockfile', () { setUp(() async { await servePackages() diff --git a/test/package_server.dart b/test/package_server.dart index 6aaea6893..b7587e73d 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -5,8 +5,11 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:path/path.dart' as p; +import 'package:pub/src/crc32c.dart'; +import 'package:pub/src/source/hosted.dart'; import 'package:pub/src/third_party/tar/tar.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:shelf/shelf.dart' as shelf; @@ -18,8 +21,8 @@ import 'descriptor.dart' as d; import 'test_pub.dart'; class PackageServer { - /// The inner [DescriptorServer] that this uses to serve its descriptors. - final shelf.Server _inner; + /// The inner [IOServer] that this uses to serve its descriptors. + final shelf_io.IOServer _inner; /// Handlers of requests. Last matching handler will be used. final List<_PatternAndHandler> _handlers = []; @@ -27,6 +30,16 @@ class PackageServer { // A list of all the requests recieved up till now. final List requestedPaths = []; + /// Whether the [IOServer] should compress the content, if possible. + /// The default value is `false` (compression disabled). + /// See [HttpServer.autoCompress] for details. + bool get autoCompress => _inner.server.autoCompress; + set autoCompress(bool shouldAutoCompress) => + _inner.server.autoCompress = shouldAutoCompress; + + // Setting this to false will disable automatic calculation of checksums. + bool serveChecksums = true; + PackageServer._(this._inner) { _inner.mount((request) { final path = request.url.path; @@ -63,26 +76,30 @@ class PackageServer { if (package == null) { return shelf.Response.notFound('No package named $name'); } - return shelf.Response.ok(jsonEncode({ - 'name': name, - 'uploaders': ['nweiz@google.com'], - 'versions': package.versions.values - .map((version) => packageVersionApiMap( - server._inner.url.toString(), - version.pubspec, - retracted: version.isRetracted, - )) - .toList(), - if (package.isDiscontinued) 'isDiscontinued': true, - if (package.discontinuedReplacementText != null) - 'replacedBy': package.discontinuedReplacementText, - })); + return shelf.Response.ok( + jsonEncode({ + 'name': name, + 'uploaders': ['nweiz@google.com'], + 'versions': package.versions.values + .map((version) => packageVersionApiMap( + server._inner.url.toString(), + version.pubspec, + retracted: version.isRetracted, + )) + .toList(), + if (package.isDiscontinued) 'isDiscontinued': true, + if (package.discontinuedReplacementText != null) + 'replacedBy': package.discontinuedReplacementText, + }), + headers: { + HttpHeaders.contentTypeHeader: 'application/vnd.pub.v2+json' + }); }, ); server.handle( _downloadPattern, - (shelf.Request request) { + (shelf.Request request) async { final parts = request.url.pathSegments; assert(parts[0] == 'packages'); final name = parts[1]; @@ -98,7 +115,21 @@ class PackageServer { for (final packageVersion in package.versions.values) { if (packageVersion.version == version) { - return shelf.Response.ok(packageVersion.contents()); + final headers = packageVersion.headers ?? {}; + headers[HttpHeaders.contentTypeHeader] ??= [ + 'application/octet-stream' + ]; + + // This gate enables tests to validate the CRC32C parser by + // passing in arbitrary values for the checksum header. + if (server.serveChecksums && + !headers.containsKey(checksumHeaderName)) { + headers[checksumHeaderName] = composeChecksumHeader( + crc32c: await packageVersion.computeArchiveCrc32c()); + } + + return shelf.Response.ok(packageVersion.contents(), + headers: headers); } } return shelf.Response.notFound('No version $version of $name'); @@ -178,7 +209,8 @@ class PackageServer { void serve(String name, String version, {Map? deps, Map? pubspec, - List? contents}) { + List? contents, + Map>? headers}) { var pubspecFields = {'name': name, 'version': version}; if (pubspec != null) pubspecFields.addAll(pubspec); if (deps != null) pubspecFields['dependencies'] = deps; @@ -189,6 +221,7 @@ class PackageServer { var package = _packages.putIfAbsent(name, _ServedPackage.new); package.versions[version] = _ServedPackageVersion( pubspecFields, + headers: headers, contents: () { final entries = []; @@ -243,6 +276,37 @@ class PackageServer { void retractPackageVersion(String name, String version) { _packages[name]!.versions[version]!.isRetracted = true; } + + Future peekArchiveChecksumHeader(String name, String version) async { + final v = _packages[name]!.versions[version]!; + + // If the test configured an overriding header value. + var checksumHeader = v.headers?[checksumHeaderName]; + + // Otherwise, compute from package contents. + if (serveChecksums) { + checksumHeader ??= + composeChecksumHeader(crc32c: await v.computeArchiveCrc32c()); + } + + return checksumHeader?.join(','); + } + + static List composeChecksumHeader( + {int? crc32c, String? md5 = '5f4dcc3b5aa765d61d8327deb882cf99'}) { + List header = []; + + if (crc32c != null) { + final bytes = Uint8List(4)..buffer.asByteData().setUint32(0, crc32c); + header.add('crc32c=${base64.encode(bytes)}'); + } + + if (md5 != null) { + header.add('md5=${base64.encode(utf8.encode(md5))}'); + } + + return header; + } } class _ServedPackage { @@ -255,11 +319,16 @@ class _ServedPackage { class _ServedPackageVersion { final Map pubspec; final Stream> Function() contents; + final Map>? headers; bool isRetracted = false; Version get version => Version.parse(pubspec['version']); - _ServedPackageVersion(this.pubspec, {required this.contents}); + _ServedPackageVersion(this.pubspec, {required this.contents, this.headers}); + + Future computeArchiveCrc32c() async { + return await Crc32c.computeByConsumingStream(contents()); + } } class _PatternAndHandler { diff --git a/test/test_pub.dart b/test/test_pub.dart index 99f56554f..acf222ac7 100644 --- a/test/test_pub.dart +++ b/test/test_pub.dart @@ -182,6 +182,7 @@ Future pubGet({ Iterable? args, Object? output, Object? error, + Object? silent, Object? warning, int? exitCode, Map? environment, @@ -193,6 +194,7 @@ Future pubGet({ args: args, output: output, error: error, + silent: silent, warning: warning, exitCode: exitCode, environment: environment, diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index 14ea76f59..dc09988f5 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -27,7 +27,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | date: $TIME [E] | content-length: 197 [E] | x-frame-options: SAMEORIGIN -[E] | content-type: text/plain; charset=utf-8 +[E] | content-type: application/vnd.pub.v2+json [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff [E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.cache/foo-versions.json. @@ -49,13 +49,15 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz [E] | took: $TIME [E] | x-powered-by: Dart with package:shelf -[E] | transfer-encoding: chunked [E] | date: $TIME +[E] | transfer-encoding: chunked +[E] | x-goog-hash: $CHECKSUM_HEADER [E] | x-frame-options: SAMEORIGIN -[E] | content-type: text/plain; charset=utf-8 +[E] | content-type: application/octet-stream [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff [E] IO : Creating $FILE from stream +[E] FINE: Computed checksum $CRC32C for foo 1.0.0 with expected CRC32C of $CRC32C. [E] FINE: Created $FILE from stream [E] IO : Created temp directory $DIR [E] IO : Reading binary file $FILE. @@ -163,7 +165,7 @@ IO : HTTP response 200 OK for GET http://localhost:$PORT/api/packages/foo | date: $TIME | content-length: 197 | x-frame-options: SAMEORIGIN - | content-type: text/plain; charset=utf-8 + | content-type: application/vnd.pub.v2+json | x-xss-protection: 1; mode=block | x-content-type-options: nosniff IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.cache/foo-versions.json. @@ -187,13 +189,15 @@ IO : HTTP GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz | took: $TIME | x-powered-by: Dart with package:shelf - | transfer-encoding: chunked | date: $TIME + | transfer-encoding: chunked + | x-goog-hash: $CHECKSUM_HEADER | x-frame-options: SAMEORIGIN - | content-type: text/plain; charset=utf-8 + | content-type: application/octet-stream | x-xss-protection: 1; mode=block | x-content-type-options: nosniff IO : Creating $FILE from stream +FINE: Computed checksum $CRC32C for foo 1.0.0 with expected CRC32C of $CRC32C. FINE: Created $FILE from stream IO : Created temp directory $DIR IO : Reading binary file $FILE.