diff --git a/pkgs/native_assets_builder/lib/src/build_runner/build_runner.dart b/pkgs/native_assets_builder/lib/src/build_runner/build_runner.dart index 1eea17597..449fe9a2c 100644 --- a/pkgs/native_assets_builder/lib/src/build_runner/build_runner.dart +++ b/pkgs/native_assets_builder/lib/src/build_runner/build_runner.dart @@ -10,15 +10,14 @@ import 'package:logging/logging.dart'; import 'package:native_assets_cli/native_assets_cli_internal.dart'; import 'package:package_config/package_config.dart'; +import '../dependencies_hash_file/dependencies_hash_file.dart'; import '../locking/locking.dart'; import '../model/build_dry_run_result.dart'; import '../model/build_result.dart'; import '../model/hook_result.dart'; import '../model/link_result.dart'; import '../package_layout/package_layout.dart'; -import '../utils/file.dart'; import '../utils/run_process.dart'; -import '../utils/uri.dart'; import 'build_planner.dart'; typedef DependencyMetadata = Map; @@ -451,6 +450,8 @@ class NativeAssetsBuildRunner { return hookResult; } + // TODO(https://github.com/dart-lang/native/issues/32): Rerun hook if + // environment variables change. Future _runHookForPackageCached( Hook hook, HookConfig config, @@ -473,7 +474,7 @@ class NativeAssetsBuildRunner { final ( compileSuccess, hookKernelFile, - hookLastSourceChange, + hookHashesFile, ) = await _compileHookForPackageCached( config.packageName, config.outputDirectory, @@ -488,7 +489,14 @@ class NativeAssetsBuildRunner { final buildOutputFile = File.fromUri(config.outputDirectory.resolve(hook.outputName)); - if (buildOutputFile.existsSync()) { + final dependenciesHashFile = File.fromUri( + config.outputDirectory + .resolve('../dependencies.dependencies_hash_file.json'), + ); + final dependenciesHashes = + DependenciesHashFile(file: dependenciesHashFile); + final lastModifiedCutoffTime = DateTime.now(); + if (buildOutputFile.existsSync() && dependenciesHashFile.existsSync()) { late final HookOutput output; try { output = _readHookOutputFromUri(hook, buildOutputFile); @@ -503,17 +511,13 @@ ${e.message} return null; } - final lastBuilt = output.timestamp.roundDownToSeconds(); - final dependenciesLastChange = - await Dependencies(output.dependencies).lastModified(); - if (lastBuilt.isAfter(dependenciesLastChange) && - lastBuilt.isAfter(hookLastSourceChange)) { + final outdated = + (await dependenciesHashes.findOutdatedFileSystemEntity()) != null; + if (!outdated) { logger.info( [ 'Skipping ${hook.name} for ${config.packageName} in $outDir.', - 'Last build on $lastBuilt.', - 'Last dependencies change on $dependenciesLastChange.', - 'Last hook change on $hookLastSourceChange.', + 'Last build on ${output.timestamp}.', ].join(' '), ); // All build flags go into [outDir]. Therefore we do not have to @@ -522,7 +526,7 @@ ${e.message} } } - return await _runHookForPackage( + final result = await _runHookForPackage( hook, config, validator, @@ -533,6 +537,24 @@ ${e.message} hookKernelFile, packageLayout, ); + if (result == null) { + if (await dependenciesHashFile.exists()) { + await dependenciesHashFile.delete(); + } + } else { + final modifiedDuringBuild = await dependenciesHashes.hashFiles( + [ + ...result.dependencies, + // Also depend on the hook source code. + hookHashesFile.uri, + ], + validBeforeLastModified: lastModifiedCutoffTime, + ); + if (modifiedDuringBuild != null) { + logger.severe('File modified during build. Build must be rerun.'); + } + } + return result; }, ); } @@ -644,7 +666,10 @@ ${e.message} /// It does not reuse the cached kernel for different configs due to /// reentrancy requirements. For more info see: /// https://github.com/dart-lang/native/issues/1319 - Future<(bool success, File kernelFile, DateTime lastSourceChange)> + /// + /// TODO(https://github.com/dart-lang/native/issues/1578): Compile only once + /// instead of per config. This requires more locking. + Future<(bool success, File kernelFile, File cacheFile)> _compileHookForPackageCached( String packageName, Uri outputDirectory, @@ -659,29 +684,17 @@ ${e.message} final depFile = File.fromUri( outputDirectory.resolve('../hook.dill.d'), ); + final dependenciesHashFile = File.fromUri( + outputDirectory.resolve('../hook.dependencies_hash_file.json'), + ); + final dependenciesHashes = DependenciesHashFile(file: dependenciesHashFile); + final lastModifiedCutoffTime = DateTime.now(); final bool mustCompile; - final DateTime sourceLastChange; - if (!await depFile.exists()) { + if (!await dependenciesHashFile.exists()) { mustCompile = true; - sourceLastChange = DateTime.now(); } else { - // Format: `path/to/my.dill: path/to/my.dart, path/to/more.dart` - final depFileContents = await depFile.readAsString(); - final dartSourceFiles = depFileContents - .trim() - .split(' ') - .skip(1) // ':' - .map((u) => Uri.file(u).fileSystemEntity) - .toList(); - final dartFilesLastChange = await dartSourceFiles.lastModified(); - final packageConfigLastChange = - await packageConfigUri.fileSystemEntity.lastModified(); - sourceLastChange = packageConfigLastChange.isAfter(dartFilesLastChange) - ? packageConfigLastChange - : dartFilesLastChange; - final kernelLastChange = await kernelFile.lastModified(); - mustCompile = sourceLastChange == kernelLastChange || - sourceLastChange.isAfter(kernelLastChange); + mustCompile = + (await dependenciesHashes.findOutdatedFileSystemEntity()) != null; } final bool success; if (!mustCompile) { @@ -696,8 +709,28 @@ ${e.message} kernelFile, depFile, ); + + if (success) { + // Format: `path/to/my.dill: path/to/my.dart, path/to/more.dart` + final depFileContents = await depFile.readAsString(); + final dartSources = depFileContents + .trim() + .split(' ') + .skip(1) // ':' + .map(Uri.file) + .toList(); + final modifiedDuringBuild = await dependenciesHashes.hashFiles( + dartSources, + validBeforeLastModified: lastModifiedCutoffTime, + ); + if (modifiedDuringBuild != null) { + logger.severe('File modified during build. Build must be rerun.'); + } + } else { + await dependenciesHashFile.delete(); + } } - return (success, kernelFile, sourceLastChange); + return (success, kernelFile, dependenciesHashFile); } Future _compileHookForPackage( @@ -859,12 +892,6 @@ ${compileResult.stdout} } } -extension on DateTime { - DateTime roundDownToSeconds() => - DateTime.fromMillisecondsSinceEpoch(millisecondsSinceEpoch - - millisecondsSinceEpoch % const Duration(seconds: 1).inMilliseconds); -} - extension on Uri { Uri get parent => File(toFilePath()).parent.uri; } diff --git a/pkgs/native_assets_builder/lib/src/dependencies_hash_file/dependencies_hash_file.dart b/pkgs/native_assets_builder/lib/src/dependencies_hash_file/dependencies_hash_file.dart new file mode 100644 index 000000000..6e23e319c --- /dev/null +++ b/pkgs/native_assets_builder/lib/src/dependencies_hash_file/dependencies_hash_file.dart @@ -0,0 +1,211 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; + +import '../utils/file.dart'; +import '../utils/uri.dart'; + +class DependenciesHashFile { + DependenciesHashFile({ + required File file, + }) : _file = file; + + final File _file; + FileSystemHashes _hashes = FileSystemHashes(); + + Future _readFile() async { + if (!await _file.exists()) { + _hashes = FileSystemHashes(); + return; + } + final jsonObject = + (json.decode(utf8.decode(await _file.readAsBytes())) as Map) + .cast(); + _hashes = FileSystemHashes.fromJson(jsonObject); + } + + void _reset() => _hashes = FileSystemHashes(); + + /// Populate the hashes and persist file with entries from + /// [fileSystemEntities]. + /// + /// If [validBeforeLastModified] is provided, any entities that were modified + /// after [validBeforeLastModified] will get a dummy hash so that they will + /// show up as outdated. If any such entity exists, its uri will be returned. + Future hashFiles( + List fileSystemEntities, { + DateTime? validBeforeLastModified, + }) async { + _reset(); + + Uri? modifiedAfterTimeStamp; + for (final uri in fileSystemEntities) { + int hash; + if (validBeforeLastModified != null && + (await uri.fileSystemEntity.lastModified()) + .isAfter(validBeforeLastModified)) { + hash = _hashLastModifiedAfterCutoff; + modifiedAfterTimeStamp = uri; + } else { + if (_isDirectoryPath(uri.path)) { + hash = await _hashDirectory(uri); + } else { + hash = await _hashFile(uri); + } + } + _hashes.files.add(FilesystemEntityHash(uri, hash)); + } + await _persist(); + return modifiedAfterTimeStamp; + } + + Future _persist() => _file.writeAsString(json.encode(_hashes.toJson())); + + /// Reads the file with hashes and finds an outdated file or directory if it + /// exists. + Future findOutdatedFileSystemEntity() async { + await _readFile(); + + for (final savedHash in _hashes.files) { + final uri = savedHash.path; + final savedHashValue = savedHash.hash; + final int hashValue; + if (_isDirectoryPath(uri.path)) { + hashValue = await _hashDirectory(uri); + } else { + hashValue = await _hashFile(uri); + } + if (savedHashValue != hashValue) { + return uri; + } + } + return null; + } + + // A 64 bit hash from an md5 hash. + int _md5int64(Uint8List bytes) { + final md5bytes = md5.convert(bytes); + final md5ints = (md5bytes.bytes as Uint8List).buffer.asUint64List(); + return md5ints[0]; + } + + Future _hashFile(Uri uri) async { + final file = File.fromUri(uri); + if (!await file.exists()) { + return _hashNotExists; + } + return _md5int64(await file.readAsBytes()); + } + + Future _hashDirectory(Uri uri) async { + final directory = Directory.fromUri(uri); + if (!await directory.exists()) { + return _hashNotExists; + } + final children = directory.listSync(followLinks: true, recursive: false); + final childrenNames = children.map((e) => _pathBaseName(e.path)).join(';'); + return _md5int64(utf8.encode(childrenNames)); + } + + /// Predefined hash for files and directories that do not exist. + /// + /// There are two predefined hash values. The chance that a predefined hash + /// collides with a real hash is 2/2^64. + static const _hashNotExists = 0; + + /// Predefined hash for files and directories that were modified after the + /// time that the hashes file was created. + /// + /// There are two predefined hash values. The chance that a predefined hash + /// collides with a real hash is 2/2^64. + static const _hashLastModifiedAfterCutoff = 1; +} + +/// Storage format for file system entity hashes. +/// +/// [File] hashes are a hash of the file. +/// +/// [Directory] hashes are a hash of the names of the direct children. +class FileSystemHashes { + FileSystemHashes({ + this.version = 1, + List? files, + }) : files = files ?? []; + + factory FileSystemHashes.fromJson(Map json) { + final version = json[_versionKey] as int; + final rawEntries = + (json[_entitiesKey] as List).cast>(); + final files = [ + for (final Map rawEntry in rawEntries) + FilesystemEntityHash._fromJson(rawEntry), + ]; + return FileSystemHashes( + version: version, + files: files, + ); + } + + final int version; + final List files; + + static const _versionKey = 'version'; + static const _entitiesKey = 'entities'; + + Map toJson() => { + _versionKey: version, + _entitiesKey: [ + for (final FilesystemEntityHash file in files) file.toJson(), + ], + }; +} + +/// A stored file or directory hash and path. +/// +/// [File] hashes are a hash of the file. +/// +/// [Directory] hashes are a hash of the names of the direct children. +class FilesystemEntityHash { + FilesystemEntityHash( + this.path, + this.hash, + ); + + factory FilesystemEntityHash._fromJson(Map json) => + FilesystemEntityHash( + _fileSystemPathToUri(json[_pathKey] as String), + json[_hashKey] as int, + ); + + static const _pathKey = 'path'; + static const _hashKey = 'hash'; + + final Uri path; + + /// A 64 bit hash. + final int hash; + + Object toJson() => { + _pathKey: path.toFilePath(), + _hashKey: hash, + }; +} + +bool _isDirectoryPath(String path) => + path.endsWith(Platform.pathSeparator) || path.endsWith('/'); + +Uri _fileSystemPathToUri(String path) { + if (_isDirectoryPath(path)) { + return Uri.directory(path); + } + return Uri.file(path); +} + +String _pathBaseName(String path) => + path.split(Platform.pathSeparator).where((e) => e.isNotEmpty).last; diff --git a/pkgs/native_assets_builder/pubspec.yaml b/pkgs/native_assets_builder/pubspec.yaml index a70c6ceb0..1eaaee1a1 100644 --- a/pkgs/native_assets_builder/pubspec.yaml +++ b/pkgs/native_assets_builder/pubspec.yaml @@ -11,6 +11,7 @@ environment: dependencies: collection: ^1.18.0 + crypto: ^3.0.6 graphs: ^2.3.1 logging: ^1.2.0 # native_assets_cli: ^0.9.0 diff --git a/pkgs/native_assets_builder/test/build_runner/build_runner_caching_test.dart b/pkgs/native_assets_builder/test/build_runner/build_runner_caching_test.dart index e83948421..0df4b2478 100644 --- a/pkgs/native_assets_builder/test/build_runner/build_runner_caching_test.dart +++ b/pkgs/native_assets_builder/test/build_runner/build_runner_caching_test.dart @@ -21,10 +21,6 @@ void main() async { workingDirectory: packageUri, logger: logger, ); - // Make sure the first compile is at least one second after the - // package_config.json is written, otherwise dill compilation isn't - // cached. - await Future.delayed(const Duration(seconds: 1)); { final logMessages = []; @@ -95,10 +91,6 @@ void main() async { workingDirectory: packageUri, logger: logger, ); - // Make sure the first compile is at least one second after the - // package_config.json is written, otherwise dill compilation isn't - // cached. - await Future.delayed(const Duration(seconds: 1)); { final result = (await build( @@ -151,10 +143,6 @@ void main() async { await runPubGet(workingDirectory: packageUri, logger: logger); logMessages.clear(); - // Make sure the first compile is at least one second after the - // package_config.json is written, otherwise dill compilation isn't - // cached. - await Future.delayed(const Duration(seconds: 1)); final result = (await build( packageUri, diff --git a/pkgs/native_assets_builder/test/build_runner/build_runner_test.dart b/pkgs/native_assets_builder/test/build_runner/build_runner_test.dart index 51a5d68f2..239542042 100644 --- a/pkgs/native_assets_builder/test/build_runner/build_runner_test.dart +++ b/pkgs/native_assets_builder/test/build_runner/build_runner_test.dart @@ -24,11 +24,6 @@ void main() async { logger: logger, ); - // Make sure the first compile is at least one second after the - // package_config.json is written, otherwise dill compilation isn't - // cached. - await Future.delayed(const Duration(seconds: 1)); - // Trigger a build, should invoke build for libraries with native assets. { final logMessages = []; diff --git a/pkgs/native_assets_builder/test/build_runner/link_caching_test.dart b/pkgs/native_assets_builder/test/build_runner/link_caching_test.dart index 36e02c2cd..ded8c8a2a 100644 --- a/pkgs/native_assets_builder/test/build_runner/link_caching_test.dart +++ b/pkgs/native_assets_builder/test/build_runner/link_caching_test.dart @@ -23,10 +23,6 @@ void main() async { workingDirectory: packageUri, logger: logger, ); - // Make sure the first compile is at least one second after the - // package_config.json is written, otherwise dill compilation isn't - // cached. - await Future.delayed(const Duration(seconds: 1)); final logMessages = []; late BuildResult buildResult; @@ -107,9 +103,6 @@ void main() async { sourceUri: testDataUri.resolve('simple_link_change_asset/'), targetUri: packageUri, ); - // Make sure the first hook is at least one second after the last - // change, or caching will not work. - await Future.delayed(const Duration(seconds: 1)); await runBuild(); expect(buildResult, isNotNull); diff --git a/pkgs/native_assets_builder/test/dependencies_hash_file/dependencies_hash_file_test.dart b/pkgs/native_assets_builder/test/dependencies_hash_file/dependencies_hash_file_test.dart new file mode 100644 index 000000000..2865bb4c4 --- /dev/null +++ b/pkgs/native_assets_builder/test/dependencies_hash_file/dependencies_hash_file_test.dart @@ -0,0 +1,108 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:native_assets_builder/src/dependencies_hash_file/dependencies_hash_file.dart'; +import 'package:test/test.dart'; + +import '../helpers.dart'; + +void main() async { + test('json format', () async { + await inTempDir((tempUri) async { + final hashes = FileSystemHashes( + files: [ + FilesystemEntityHash( + tempUri.resolve('foo.dll'), + 1337, + ), + ], + ); + final hashes2 = FileSystemHashes.fromJson(hashes.toJson()); + expect(hashes.files.single.path, equals(hashes2.files.single.path)); + expect(hashes.files.single.hash, equals(hashes2.files.single.hash)); + }); + }); + + test('dependencies hash file', () async { + await inTempDir((tempUri) async { + final tempFile = File.fromUri(tempUri.resolve('foo.txt')); + final tempSubDir = Directory.fromUri(tempUri.resolve('subdir/')); + final subFile = File.fromUri(tempSubDir.uri.resolve('bar.txt')); + + final hashesFile = File.fromUri(tempUri.resolve('hashes.json')); + final hashes = DependenciesHashFile(file: hashesFile); + + Future reset() async { + await tempFile.create(recursive: true); + await tempSubDir.create(recursive: true); + await subFile.create(recursive: true); + await tempFile.writeAsString('hello'); + await subFile.writeAsString('world'); + + await hashes.hashFiles([ + tempFile.uri, + tempSubDir.uri, + ]); + } + + await reset(); + + // No changes + expect(await hashes.findOutdatedFileSystemEntity(), isNull); + + // Change file contents. + await tempFile.writeAsString('asdf'); + expect(await hashes.findOutdatedFileSystemEntity(), tempFile.uri); + await reset(); + + // Delete file. + await tempFile.delete(); + expect(await hashes.findOutdatedFileSystemEntity(), tempFile.uri); + await reset(); + + // Add file to tracked directory. + final subFile2 = File.fromUri(tempSubDir.uri.resolve('baz.txt')); + await subFile2.create(recursive: true); + await subFile2.writeAsString('hello'); + expect(await hashes.findOutdatedFileSystemEntity(), tempSubDir.uri); + await reset(); + + // Delete file from tracked directory. + await subFile.delete(); + expect(await hashes.findOutdatedFileSystemEntity(), tempSubDir.uri); + await reset(); + + // Delete tracked directory. + await tempSubDir.delete(recursive: true); + expect(await hashes.findOutdatedFileSystemEntity(), tempSubDir.uri); + await reset(); + + // Add directory to tracked directory. + final subDir2 = Directory.fromUri(tempSubDir.uri.resolve('baz/')); + await subDir2.create(recursive: true); + expect(await hashes.findOutdatedFileSystemEntity(), tempSubDir.uri); + await reset(); + + // Overwriting a file with identical contents. + await tempFile.writeAsString('something something'); + await tempFile.writeAsString('hello'); + expect(await hashes.findOutdatedFileSystemEntity(), isNull); + await reset(); + + // If a file is modified after the valid timestamp, it should be marked + // as changed. + await hashes.hashFiles( + [ + tempFile.uri, + ], + validBeforeLastModified: (await tempFile.lastModified()) + .subtract(const Duration(seconds: 1)), + ); + expect(await hashes.findOutdatedFileSystemEntity(), tempFile.uri); + }); + }); +}