diff --git a/lib/src/generate/build.dart b/lib/src/generate/build.dart index 1946c84a7..784a5181a 100644 --- a/lib/src/generate/build.dart +++ b/lib/src/generate/build.dart @@ -2,26 +2,17 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; import 'package:logging/logging.dart'; -import 'package:path/path.dart' as path; -import '../asset/asset.dart'; -import '../asset/exceptions.dart'; import '../asset/cache.dart'; import '../asset/file_based.dart'; -import '../asset/id.dart'; import '../asset/reader.dart'; import '../asset/writer.dart'; import '../asset_graph/graph.dart'; -import '../asset_graph/node.dart'; -import '../builder/builder.dart'; -import '../builder/build_step_impl.dart'; import '../package_graph/package_graph.dart'; +import 'build_impl.dart'; import 'build_result.dart'; -import 'input_set.dart'; import 'phase.dart'; /// Runs all of the [Phases] in [phaseGroups]. @@ -54,261 +45,12 @@ Future build(List> phaseGroups, writer ??= new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph)); - /// Run the build in a zone. - var result = await runZoned(() async { - try { - /// Delete all previous outputs! - await _deletePreviousOutputs(phaseGroups); - - /// Run a fresh build. - var result = await _runPhases(phaseGroups); - - // Write out the new build_outputs file. - var buildOutputsAsset = new Asset( - _buildOutputsId, - JSON.encode( - result.outputs.map((output) => output.id.serialize()).toList())); - await writer.writeAsString(buildOutputsAsset); - - return result; - } catch (e, s) { - return new BuildResult(BuildStatus.Failure, BuildType.Full, [], - exception: e, stackTrace: s); - } - }, zoneValues: { - _assetGraphKey: new AssetGraph(), - _assetReaderKey: reader, - _assetWriterKey: writer, - _packageGraphKey: packageGraph, - }); + /// Run the build! + var result = await new BuildImpl( + new AssetGraph(), reader, writer, packageGraph, phaseGroups) + .runBuild(); await logListener.cancel(); return result; } - -/// Keys for reading zone local values. -Symbol _assetGraphKey = #buildAssetGraph; -Symbol _assetReaderKey = #buildAssetReader; -Symbol _assetWriterKey = #buildAssetWriter; -Symbol _packageGraphKey = #buildPackageGraph; - -/// Getters for zone local values. -AssetGraph get _assetGraph => Zone.current[_assetGraphKey]; -AssetReader get _reader => Zone.current[_assetReaderKey]; -AssetWriter get _writer => Zone.current[_assetWriterKey]; -PackageGraph get _packageGraph => Zone.current[_packageGraphKey]; - -/// Asset containing previous build outputs. -AssetId get _buildOutputsId => - new AssetId(_packageGraph.root.name, '.build/build_outputs.json'); - -/// Deletes all previous output files. -Future _deletePreviousOutputs(List> phaseGroups) async { - if (await _reader.hasInput(_buildOutputsId)) { - // Cache file exists, just delete all outputs contained in it. - var previousOutputs = - JSON.decode(await _reader.readAsString(_buildOutputsId)); - await _writer.delete(_buildOutputsId); - await Future.wait(previousOutputs.map((output) { - return _writer.delete(new AssetId.deserialize(output)); - })); - return; - } - - // No cache file exists, run `declareOutputs` on all phases and collect all - // outputs which conflict with existing assets. - final allInputs = await _allInputs(phaseGroups); - final conflictingOutputs = new Set(); - for (var group in phaseGroups) { - final groupOutputIds = []; - for (var phase in group) { - var inputs = _matchingInputs(allInputs, phase.inputSets); - for (var input in inputs) { - for (var builder in phase.builders) { - var outputs = builder.declareOutputs(input); - - groupOutputIds.addAll(outputs); - for (var output in outputs) { - if (allInputs[output.package]?.contains(output) == true) { - conflictingOutputs.add(output); - } - } - } - } - } - - /// Once the group is done, add all outputs so they can be used in the next - /// phase. - for (var outputId in groupOutputIds) { - allInputs.putIfAbsent(outputId.package, () => new Set()); - allInputs[outputId.package].add(outputId); - } - } - - // Check conflictingOuputs, prompt user to delete files. - if (conflictingOutputs.isEmpty) return; - - stdout.writeln('Found ${conflictingOutputs.length} declared outputs ' - 'which already exist on disk. This is likely because the `.build` ' - 'folder was deleted.'); - var done = false; - while (!done) { - stdout.write('Delete these files (y/n) (or list them (l))?: '); - var input = stdin.readLineSync(); - switch (input) { - case 'y': - stdout.writeln('Deleting files...'); - await Future.wait(conflictingOutputs.map((output) { - return _writer.delete(output); - })); - done = true; - break; - case 'n': - stdout.writeln('Exiting...'); - exit(1); - break; - case 'l': - for (var output in conflictingOutputs) { - stdout.writeln(output); - } - break; - default: - stdout.writeln('Unrecognized option $input, (y/n/l) expected.'); - } - } -} - -/// Runs the [phaseGroups] and returns a [Future] which completes -/// once all [Phase]s are done. -Future _runPhases(List> phaseGroups) async { - final allInputs = await _allInputs(phaseGroups); - final outputs = []; - int phaseGroupNum = 0; - for (var group in phaseGroups) { - final groupOutputs = []; - for (var phase in group) { - var inputs = _matchingInputs(allInputs, phase.inputSets); - for (var builder in phase.builders) { - // TODO(jakemac): Optimize, we can run all the builders in a phase - // at the same time instead of sequentially. - await for (var output - in _runBuilder(builder, inputs, allInputs, phaseGroupNum)) { - groupOutputs.add(output); - outputs.add(output); - } - } - } - - /// Once the group is done, add all outputs so they can be used in the next - /// phase. - for (var output in groupOutputs) { - allInputs.putIfAbsent(output.id.package, () => new Set()); - allInputs[output.id.package].add(output.id); - } - phaseGroupNum++; - } - return new BuildResult(BuildStatus.Success, BuildType.Full, outputs); -} - -/// Returns a map of all the available inputs by package. -Future>> _allInputs( - List> phaseGroups) async { - final packages = new Set(); - for (var group in phaseGroups) { - for (var phase in group) { - for (var inputSet in phase.inputSets) { - packages.add(inputSet.package); - } - } - } - - var inputSets = packages.map((package) => new InputSet(package)); - var allInputs = await _reader.listAssetIds(inputSets).toList(); - var inputsByPackage = {}; - for (var input in allInputs) { - inputsByPackage.putIfAbsent(input.package, () => new Set()); - - if (_isValidInput(input)) { - inputsByPackage[input.package].add(input); - } - } - return inputsByPackage; -} - -/// Gets a list of all inputs matching [inputSets] given [allInputs]. -Set _matchingInputs( - Map> inputsByPackage, Iterable inputSets) { - var inputs = new Set(); - for (var inputSet in inputSets) { - assert(inputsByPackage.containsKey(inputSet.package)); - for (var input in inputsByPackage[inputSet.package]) { - if (inputSet.globs.any((g) => g.matches(input.path))) { - inputs.add(input); - } - } - } - return inputs; -} - -/// Checks if an [input] is valid. -bool _isValidInput(AssetId input) { - var parts = path.split(input.path); - // Files must be in a top level directory. - if (parts.length == 1) return false; - if (input.package != _packageGraph.root.name) return parts[0] == 'lib'; - return true; -} - -/// Runs [builder] with [inputs] as inputs. -Stream _runBuilder(Builder builder, Iterable primaryInputs, - Map> allInputs, int phaseGroupNum) async* { - for (var input in primaryInputs) { - var expectedOutputs = builder.declareOutputs(input); - - /// Validate [expectedOutputs]. - for (var output in expectedOutputs) { - if (output.package != _packageGraph.root.name) { - throw new InvalidOutputException(new Asset(output, '')); - } - if (allInputs[output.package]?.contains(output) == true) { - throw new InvalidOutputException(new Asset(output, '')); - } - } - - /// Add nodes to the [AssetGraph] for [expectedOutputs] and [input]. - var inputNode = _assetGraph.addIfAbsent(input, () => new AssetNode(input)); - for (var output in expectedOutputs) { - inputNode.outputs.add(output); - _assetGraph.addIfAbsent( - output, - () => new GeneratedAssetNode( - builder, input, phaseGroupNum, true, output)); - } - - /// Skip the build step if none of the outputs need updating. - var skipBuild = !expectedOutputs.any((output) => - (_assetGraph.get(output) as GeneratedAssetNode).needsUpdate); - if (skipBuild) continue; - - var inputAsset = new Asset(input, await _reader.readAsString(input)); - var buildStep = new BuildStepImpl(inputAsset, expectedOutputs, _reader, - _writer, _packageGraph.root.name); - await builder.build(buildStep); - await buildStep.complete(); - - /// Update the asset graph based on the dependencies discovered. - for (var dependency in buildStep.dependencies) { - var dependencyNode = - _assetGraph.addIfAbsent(dependency, () => new AssetNode(dependency)); - - /// We care about all [expectedOutputs], not just real outputs. - dependencyNode.outputs.addAll(expectedOutputs); - } - - /// Yield the outputs. - for (var output in buildStep.outputs) { - yield output; - } - } -} diff --git a/lib/src/generate/build_impl.dart b/lib/src/generate/build_impl.dart new file mode 100644 index 000000000..9f9edff86 --- /dev/null +++ b/lib/src/generate/build_impl.dart @@ -0,0 +1,296 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import '../asset/asset.dart'; +import '../asset/exceptions.dart'; +import '../asset/id.dart'; +import '../asset/reader.dart'; +import '../asset/writer.dart'; +import '../asset_graph/graph.dart'; +import '../asset_graph/node.dart'; +import '../builder/builder.dart'; +import '../builder/build_step_impl.dart'; +import '../package_graph/package_graph.dart'; +import 'build_result.dart'; +import 'exceptions.dart'; +import 'input_set.dart'; +import 'phase.dart'; + +/// Class which manages running builds. +class BuildImpl { + final AssetGraph _assetGraph; + final AssetReader _reader; + final AssetWriter _writer; + final PackageGraph _packageGraph; + final List> _phaseGroups; + final _inputsByPackage = >{}; + bool _buildRunning = false; + + BuildImpl(this._assetGraph, this._reader, this._writer, this._packageGraph, + this._phaseGroups); + + /// Runs a build + /// + /// The returned [Future] is guaranteed to complete with a [BuildResult]. If + /// an exception is thrown by any phase, [BuildResult#status] will be set to + /// [BuildStatus.Failure]. The exception and stack trace that caused the failure + /// will be available as [BuildResult#exception] and [BuildResult#stackTrace] + /// respectively. + Future runBuild() async { + try { + if (_buildRunning) throw const ConcurrentBuildException(); + _buildRunning = true; + + /// Wait while all inputs are collected. + await _initializeInputsByPackage(); + + /// Delete all previous outputs! + await _deletePreviousOutputs(); + + /// Run a fresh build. + var result = await _runPhases(); + + // Write out the new build_outputs file. + var buildOutputsAsset = new Asset( + _buildOutputsId, + JSON.encode( + result.outputs.map((output) => output.id.serialize()).toList())); + await _writer.writeAsString(buildOutputsAsset); + + return result; + } catch (e, s) { + return new BuildResult(BuildStatus.Failure, BuildType.Full, [], + exception: e, stackTrace: s); + } finally { + _buildRunning = false; + } + } + + /// Asset containing previous build outputs. + AssetId get _buildOutputsId => + new AssetId(_packageGraph.root.name, '.build/build_outputs.json'); + + /// Deletes all previous output files. + Future _deletePreviousOutputs() async { + if (await _reader.hasInput(_buildOutputsId)) { + // Cache file exists, just delete all outputs contained in it. + var previousOutputs = + JSON.decode(await _reader.readAsString(_buildOutputsId)); + await _writer.delete(_buildOutputsId); + _inputsByPackage[_buildOutputsId.package]?.remove(_buildOutputsId); + await Future.wait(previousOutputs.map((output) { + var outputId = new AssetId.deserialize(output); + _inputsByPackage[outputId.package]?.remove(outputId); + return _writer.delete(outputId); + })); + return; + } + + // Deep copy _inputsByPackage, we don't want to actually modify the real one + // as this is just a dry run to determine potential conflicts. + final tempInputsByPackage = {}; + _inputsByPackage.forEach((package, inputs) { + tempInputsByPackage[package] = new Set.from(inputs); + }); + + // No cache file exists, run `declareOutputs` on all phases and collect all + // outputs which conflict with existing assets. + final conflictingOutputs = new Set(); + for (var group in _phaseGroups) { + final groupOutputIds = []; + for (var phase in group) { + var inputs = _matchingInputs(phase.inputSets); + for (var input in inputs) { + for (var builder in phase.builders) { + var outputs = builder.declareOutputs(input); + + groupOutputIds.addAll(outputs); + for (var output in outputs) { + if (tempInputsByPackage[output.package]?.contains(output) == true) { + conflictingOutputs.add(output); + } + } + } + } + } + + /// Once the group is done, add all outputs so they can be used in the next + /// phase. + for (var outputId in groupOutputIds) { + tempInputsByPackage.putIfAbsent( + outputId.package, () => new Set()); + tempInputsByPackage[outputId.package].add(outputId); + } + } + + // Check conflictingOuputs, prompt user to delete files. + if (conflictingOutputs.isEmpty) return; + + stdout.writeln('Found ${conflictingOutputs.length} declared outputs ' + 'which already exist on disk. This is likely because the `.build` ' + 'folder was deleted.'); + var done = false; + while (!done) { + stdout.write('Delete these files (y/n) (or list them (l))?: '); + var input = stdin.readLineSync(); + switch (input) { + case 'y': + stdout.writeln('Deleting files...'); + await Future.wait(conflictingOutputs.map((output) { + return _writer.delete(output); + })); + done = true; + break; + case 'n': + stdout.writeln('Exiting...'); + exit(1); + break; + case 'l': + for (var output in conflictingOutputs) { + stdout.writeln(output); + } + break; + default: + stdout.writeln('Unrecognized option $input, (y/n/l) expected.'); + } + } + } + + /// Runs the [_phaseGroups] and returns a [Future] which + /// completes once all [Phase]s are done. + Future _runPhases() async { + final outputs = []; + int phaseGroupNum = 0; + for (var group in _phaseGroups) { + final groupOutputs = []; + for (var phase in group) { + var inputs = _matchingInputs(phase.inputSets); + for (var builder in phase.builders) { + // TODO(jakemac): Optimize, we can run all the builders in a phase + // at the same time instead of sequentially. + await for (var output + in _runBuilder(builder, inputs, phaseGroupNum)) { + groupOutputs.add(output); + outputs.add(output); + } + } + } + + /// Once the group is done, add all outputs so they can be used in the next + /// phase. + for (var output in groupOutputs) { + _inputsByPackage.putIfAbsent( + output.id.package, () => new Set()); + _inputsByPackage[output.id.package].add(output.id); + } + phaseGroupNum++; + } + return new BuildResult(BuildStatus.Success, BuildType.Full, outputs); + } + + /// Initializes the map of all the available inputs by package. + Future _initializeInputsByPackage() async { + final packages = new Set(); + for (var group in _phaseGroups) { + for (var phase in group) { + for (var inputSet in phase.inputSets) { + packages.add(inputSet.package); + } + } + } + + var inputSets = packages.map((package) => new InputSet(package)); + var allInputs = await _reader.listAssetIds(inputSets).toList(); + _inputsByPackage.clear(); + for (var input in allInputs) { + _inputsByPackage.putIfAbsent(input.package, () => new Set()); + + if (_isValidInput(input)) { + _inputsByPackage[input.package].add(input); + } + } + } + + /// Gets a list of all inputs matching [inputSets]. + Set _matchingInputs(Iterable inputSets) { + var inputs = new Set(); + for (var inputSet in inputSets) { + assert(_inputsByPackage.containsKey(inputSet.package)); + for (var input in _inputsByPackage[inputSet.package]) { + if (inputSet.globs.any((g) => g.matches(input.path))) { + inputs.add(input); + } + } + } + return inputs; + } + + /// Checks if an [input] is valid. + bool _isValidInput(AssetId input) { + var parts = path.split(input.path); + // Files must be in a top level directory. + if (parts.length == 1) return false; + if (input.package != _packageGraph.root.name) return parts[0] == 'lib'; + return true; + } + + /// Runs [builder] with [inputs] as inputs. + Stream _runBuilder(Builder builder, Iterable primaryInputs, + int phaseGroupNum) async* { + for (var input in primaryInputs) { + var expectedOutputs = builder.declareOutputs(input); + + /// Validate [expectedOutputs]. + for (var output in expectedOutputs) { + if (output.package != _packageGraph.root.name) { + throw new InvalidOutputException(new Asset(output, '')); + } + if (_inputsByPackage[output.package]?.contains(output) == true) { + throw new InvalidOutputException(new Asset(output, '')); + } + } + + /// Add nodes to the [AssetGraph] for [expectedOutputs] and [input]. + var inputNode = + _assetGraph.addIfAbsent(input, () => new AssetNode(input)); + for (var output in expectedOutputs) { + inputNode.outputs.add(output); + _assetGraph.addIfAbsent( + output, + () => new GeneratedAssetNode( + builder, input, phaseGroupNum, true, output)); + } + + /// Skip the build step if none of the outputs need updating. + var skipBuild = !expectedOutputs.any((output) => + (_assetGraph.get(output) as GeneratedAssetNode).needsUpdate); + if (skipBuild) continue; + + var inputAsset = new Asset(input, await _reader.readAsString(input)); + var buildStep = new BuildStepImpl(inputAsset, expectedOutputs, _reader, + _writer, _packageGraph.root.name); + await builder.build(buildStep); + await buildStep.complete(); + + /// Update the asset graph based on the dependencies discovered. + for (var dependency in buildStep.dependencies) { + var dependencyNode = _assetGraph.addIfAbsent( + dependency, () => new AssetNode(dependency)); + + /// We care about all [expectedOutputs], not just real outputs. + dependencyNode.outputs.addAll(expectedOutputs); + } + + /// Yield the outputs. + for (var output in buildStep.outputs) { + yield output; + } + } + } +} diff --git a/lib/src/generate/exceptions.dart b/lib/src/generate/exceptions.dart new file mode 100644 index 000000000..d7afda3e6 --- /dev/null +++ b/lib/src/generate/exceptions.dart @@ -0,0 +1,11 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +class ConcurrentBuildException implements Exception { + + const ConcurrentBuildException(); + + @override + String toString() => + 'ConcurrentBuildException: Only one build may be running at a time.'; +}