Skip to content

Commit bd94ab5

Browse files
jensjohacommit-bot@chromium.org
authored andcommitted
[kernel] Test bootstapping from dill with deferred library
This CL adds a test that bootstraps from a dill with a deferred library in it. Before this CL it crashed, see bug #32245. This CL "double fixes" the problem: a) When lazy loading a dill file the _currentLibrary variable wasn't set properly. This has been fixed. b) When bootstrapping from a dill file, (almost) everything in the dill will be included in the output so lazy loading it doesn't make sense. Bug: 32245. Change-Id: I94980608e19633fd31b522651a4a05b9cf7beabd Reviewed-on: https://dart-review.googlesource.com/42600 Commit-Queue: Jens Johansen <[email protected]> Reviewed-by: Kevin Millikin <[email protected]>
1 parent 3809d92 commit bd94ab5

File tree

6 files changed

+140
-73
lines changed

6 files changed

+140
-73
lines changed

pkg/front_end/lib/src/base/processed_options.dart

+2-1
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,8 @@ class ProcessedOptions {
346346
Program program = new Program(nameRoot: nameRoot);
347347
// TODO(ahe): Pass file name to BinaryBuilder.
348348
// TODO(ahe): Control lazy loading via an option.
349-
new BinaryBuilder(bytes, null, false).readProgram(program);
349+
new BinaryBuilder(bytes, filename: null, disableLazyReading: false)
350+
.readProgram(program);
350351
return program;
351352
}
352353

pkg/front_end/lib/src/fasta/incremental_compiler.dart

+13-5
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ library fasta.incremental_compiler;
66

77
import 'dart:async' show Future;
88

9-
import 'package:kernel/kernel.dart'
10-
show loadProgramFromBytes, Library, Procedure, Program, Source;
9+
import 'package:kernel/binary/ast_from_binary.dart' show BinaryBuilder;
10+
11+
import 'package:kernel/kernel.dart' show Library, Procedure, Program, Source;
1112

1213
import '../api_prototype/incremental_kernel_generator.dart'
1314
show IncrementalKernelGenerator;
@@ -71,7 +72,9 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
7172
Program program;
7273
if (summaryBytes != null) {
7374
ticker.logMs("Read ${c.options.sdkSummary}");
74-
program = loadProgramFromBytes(summaryBytes);
75+
program = new Program();
76+
new BinaryBuilder(summaryBytes, disableLazyReading: false)
77+
.readProgram(program);
7578
ticker.logMs("Deserialized ${c.options.sdkSummary}");
7679
bytesLength += summaryBytes.length;
7780
}
@@ -87,10 +90,15 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
8790
ticker.logMs("Read $bootstrapDill");
8891
bool bootstrapFailed = false;
8992
try {
90-
loadProgramFromBytes(bootstrapBytes, program);
93+
// We're going to output all we read here so lazy loading it
94+
// doesn't make sense.
95+
new BinaryBuilder(bootstrapBytes, disableLazyReading: true)
96+
.readProgram(program);
9197
} catch (e) {
9298
bootstrapFailed = true;
93-
program = loadProgramFromBytes(summaryBytes);
99+
program = new Program();
100+
new BinaryBuilder(summaryBytes, disableLazyReading: false)
101+
.readProgram(program);
94102
}
95103
if (!bootstrapFailed) {
96104
bootstrapSuccess = true;

pkg/front_end/test/fasta/bootstrap_test.dart

+2-2
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,8 @@ Future compare(Uri a, Uri b, {bool silent: false}) async {
8080
}
8181
Program programA = new Program();
8282
Program programB = new Program();
83-
new BinaryBuilder(bytesA, a.toFilePath()).readProgram(programA);
84-
new BinaryBuilder(bytesB, b.toFilePath()).readProgram(programB);
83+
new BinaryBuilder(bytesA, filename: a.toFilePath()).readProgram(programA);
84+
new BinaryBuilder(bytesB, filename: b.toFilePath()).readProgram(programB);
8585
RegExp splitLines = new RegExp('^', multiLine: true);
8686
List<String> linesA = programToString(programA).split(splitLines);
8787
List<String> linesB = programToString(programB).split(splitLines);

pkg/front_end/test/incremental_load_from_dill_test.dart

+115-62
Original file line numberDiff line numberDiff line change
@@ -19,67 +19,114 @@ import 'package:front_end/src/fasta/kernel/utils.dart'
1919
import "package:front_end/src/api_prototype/memory_file_system.dart"
2020
show MemoryFileSystem;
2121

22-
final Uri dart2jsUrl = Uri.base.resolve("pkg/compiler/bin/dart2js.dart");
23-
final Uri invalidateUri = Uri.parse("package:compiler/src/filenames.dart");
2422
Directory outDir;
25-
Uri normalDill;
26-
Uri bootstrappedDill;
2723

2824
main() async {
2925
outDir =
3026
Directory.systemTemp.createTempSync("incremental_load_from_dill_test");
31-
normalDill = outDir.uri.resolve("dart2js.full.dill");
32-
bootstrappedDill = outDir.uri.resolve("dart2js.bootstrap.dill");
33-
Uri nonexisting = outDir.uri.resolve("dart2js.nonexisting.dill");
34-
Uri nonLoadable = outDir.uri.resolve("dart2js.nonloadable.dill");
3527
try {
36-
// Compile dart2js without bootstrapping.
37-
Stopwatch stopwatch = new Stopwatch()..start();
38-
await normalCompile();
39-
print("Normal compile took ${stopwatch.elapsedMilliseconds} ms");
40-
41-
// Create a file that cannot be (fully) loaded as a dill file.
42-
List<int> corruptData = new File.fromUri(normalDill).readAsBytesSync();
43-
for (int i = 10 * (corruptData.length ~/ 16);
44-
i < 15 * (corruptData.length ~/ 16);
45-
++i) {
46-
corruptData[i] = 42;
47-
}
48-
new File.fromUri(nonLoadable).writeAsBytesSync(corruptData);
49-
50-
// Compile dart2js, bootstrapping from the just-compiled dill,
51-
// a nonexisting file and a dill file that isn't valid.
52-
for (List<Object> bootstrapData in [
53-
[normalDill, true],
54-
[nonexisting, false],
55-
// [nonLoadable, false] // disabled for now
56-
]) {
57-
Uri bootstrapWith = bootstrapData[0];
58-
bool bootstrapExpect = bootstrapData[1];
59-
stopwatch.reset();
60-
bool bootstrapResult = await bootstrapCompile(bootstrapWith);
61-
Expect.equals(bootstrapExpect, bootstrapResult);
62-
print("Bootstrapped compile(s) from ${bootstrapWith.pathSegments.last} "
63-
"took ${stopwatch.elapsedMilliseconds} ms");
64-
65-
// Compare the two files.
66-
List<int> normalDillData = new File.fromUri(normalDill).readAsBytesSync();
67-
List<int> bootstrappedDillData =
68-
new File.fromUri(bootstrappedDill).readAsBytesSync();
69-
Expect.equals(normalDillData.length, bootstrappedDillData.length);
70-
for (int i = 0; i < normalDillData.length; ++i) {
71-
if (normalDillData[i] != bootstrappedDillData[i]) {
72-
Expect.fail("Normally compiled and bootstrapped compile differs.");
73-
}
74-
}
75-
}
76-
28+
await testDart2jsCompile();
7729
await testDisappearingLibrary();
30+
await testDeferredLibrary();
7831
} finally {
7932
outDir.deleteSync(recursive: true);
8033
}
8134
}
8235

36+
/// Test loading from a dill file with a deferred library.
37+
/// This is done by bootstrapping with no changes.
38+
void testDeferredLibrary() async {
39+
final Uri input = outDir.uri.resolve("testDeferredLibrary_main.dart");
40+
final Uri b = outDir.uri.resolve("testDeferredLibrary_b.dart");
41+
Uri output = outDir.uri.resolve("testDeferredLibrary_full.dill");
42+
Uri bootstrappedOutput =
43+
outDir.uri.resolve("testDeferredLibrary_full_from_bootstrap.dill");
44+
new File.fromUri(input).writeAsStringSync("""
45+
import 'testDeferredLibrary_b.dart' deferred as b;
46+
47+
void main() {
48+
print(b.foo());
49+
}
50+
""");
51+
new File.fromUri(b).writeAsStringSync("""
52+
String foo() => "hello from foo in b";
53+
""");
54+
55+
Stopwatch stopwatch = new Stopwatch()..start();
56+
await normalCompile(input, output);
57+
print("Normal compile took ${stopwatch.elapsedMilliseconds} ms");
58+
59+
stopwatch.reset();
60+
bool bootstrapResult = await bootstrapCompile(
61+
input, bootstrappedOutput, output, [],
62+
performSizeTests: false);
63+
print("Bootstrapped compile(s) from ${output.pathSegments.last} "
64+
"took ${stopwatch.elapsedMilliseconds} ms");
65+
Expect.isTrue(bootstrapResult);
66+
67+
// Compare the two files.
68+
List<int> normalDillData = new File.fromUri(output).readAsBytesSync();
69+
List<int> bootstrappedDillData =
70+
new File.fromUri(bootstrappedOutput).readAsBytesSync();
71+
checkBootstrappedIsEqual(normalDillData, bootstrappedDillData);
72+
}
73+
74+
void testDart2jsCompile() async {
75+
final Uri dart2jsUrl = Uri.base.resolve("pkg/compiler/bin/dart2js.dart");
76+
final Uri invalidateUri = Uri.parse("package:compiler/src/filenames.dart");
77+
Uri normalDill = outDir.uri.resolve("dart2js.full.dill");
78+
Uri bootstrappedDill = outDir.uri.resolve("dart2js.bootstrap.dill");
79+
Uri nonexisting = outDir.uri.resolve("dart2js.nonexisting.dill");
80+
Uri nonLoadable = outDir.uri.resolve("dart2js.nonloadable.dill");
81+
82+
// Compile dart2js without bootstrapping.
83+
Stopwatch stopwatch = new Stopwatch()..start();
84+
await normalCompile(dart2jsUrl, normalDill);
85+
print("Normal compile took ${stopwatch.elapsedMilliseconds} ms");
86+
87+
// Create a file that cannot be (fully) loaded as a dill file.
88+
List<int> corruptData = new File.fromUri(normalDill).readAsBytesSync();
89+
for (int i = 10 * (corruptData.length ~/ 16);
90+
i < 15 * (corruptData.length ~/ 16);
91+
++i) {
92+
corruptData[i] = 42;
93+
}
94+
new File.fromUri(nonLoadable).writeAsBytesSync(corruptData);
95+
96+
// Compile dart2js, bootstrapping from the just-compiled dill,
97+
// a nonexisting file and a dill file that isn't valid.
98+
for (List<Object> bootstrapData in [
99+
[normalDill, true],
100+
[nonexisting, false],
101+
// [nonLoadable, false] // disabled for now
102+
]) {
103+
Uri bootstrapWith = bootstrapData[0];
104+
bool bootstrapExpect = bootstrapData[1];
105+
stopwatch.reset();
106+
bool bootstrapResult = await bootstrapCompile(
107+
dart2jsUrl, bootstrappedDill, bootstrapWith, [invalidateUri]);
108+
Expect.equals(bootstrapExpect, bootstrapResult);
109+
print("Bootstrapped compile(s) from ${bootstrapWith.pathSegments.last} "
110+
"took ${stopwatch.elapsedMilliseconds} ms");
111+
112+
// Compare the two files.
113+
List<int> normalDillData = new File.fromUri(normalDill).readAsBytesSync();
114+
List<int> bootstrappedDillData =
115+
new File.fromUri(bootstrappedDill).readAsBytesSync();
116+
checkBootstrappedIsEqual(normalDillData, bootstrappedDillData);
117+
}
118+
}
119+
120+
void checkBootstrappedIsEqual(
121+
List<int> normalDillData, List<int> bootstrappedDillData) {
122+
Expect.equals(normalDillData.length, bootstrappedDillData.length);
123+
for (int i = 0; i < normalDillData.length; ++i) {
124+
if (normalDillData[i] != bootstrappedDillData[i]) {
125+
Expect.fail("Normally compiled and bootstrapped compile differs.");
126+
}
127+
}
128+
}
129+
83130
/// Compile an application with n libraries, then
84131
/// compile "the same" application, but with m < n libraries,
85132
/// where (at least one) of the missing libraries are "in the middle"
@@ -123,7 +170,6 @@ void testDisappearingLibrary() async {
123170
IncrementalCompiler compiler =
124171
new IncrementalKernelGenerator(options, main);
125172
var program = await compiler.computeDelta();
126-
print(program);
127173
libCount2 = serializeProgram(program);
128174
if (program.libraries.length != 2) {
129175
throw "Expected 2 libraries, got ${program.libraries.length}";
@@ -169,24 +215,29 @@ CompilerOptions getOptions() {
169215
return options;
170216
}
171217

172-
Future<bool> normalCompile() async {
218+
Future<bool> normalCompile(Uri input, Uri output) async {
173219
CompilerOptions options = getOptions();
174-
IncrementalCompiler compiler =
175-
new IncrementalKernelGenerator(options, dart2jsUrl);
220+
IncrementalCompiler compiler = new IncrementalKernelGenerator(options, input);
176221
var y = await compiler.computeDelta();
177-
await writeProgramToFile(y, normalDill);
222+
await writeProgramToFile(y, output);
178223
return compiler.bootstrapSuccess;
179224
}
180225

181-
Future<bool> bootstrapCompile(Uri bootstrapWith) async {
226+
Future<bool> bootstrapCompile(
227+
Uri input, Uri output, Uri bootstrapWith, List<Uri> invalidateUris,
228+
{bool performSizeTests: true}) async {
182229
CompilerOptions options = getOptions();
183230
IncrementalCompiler compiler =
184-
new IncrementalKernelGenerator(options, dart2jsUrl, bootstrapWith);
185-
compiler.invalidate(invalidateUri);
231+
new IncrementalKernelGenerator(options, input, bootstrapWith);
232+
for (Uri invalidateUri in invalidateUris) {
233+
compiler.invalidate(invalidateUri);
234+
}
186235
var bootstrappedProgram = await compiler.computeDelta();
187236
bool result = compiler.bootstrapSuccess;
188-
await writeProgramToFile(bootstrappedProgram, bootstrappedDill);
189-
compiler.invalidate(invalidateUri);
237+
await writeProgramToFile(bootstrappedProgram, output);
238+
for (Uri invalidateUri in invalidateUris) {
239+
compiler.invalidate(invalidateUri);
240+
}
190241

191242
var partialProgram = await compiler.computeDelta();
192243
var emptyProgram = await compiler.computeDelta();
@@ -198,9 +249,11 @@ Future<bool> bootstrapCompile(Uri bootstrapWith) async {
198249
var emptyLibUris =
199250
emptyProgram.libraries.map((lib) => lib.importUri).toList();
200251

201-
Expect.isTrue(fullLibUris.length > partialLibUris.length);
202-
Expect.isTrue(partialLibUris.isNotEmpty);
203-
Expect.isTrue(emptyLibUris.isEmpty);
252+
if (performSizeTests) {
253+
Expect.isTrue(fullLibUris.length > partialLibUris.length);
254+
Expect.isTrue(partialLibUris.isNotEmpty);
255+
Expect.isTrue(emptyLibUris.isEmpty);
256+
}
204257

205258
return result;
206259
}

pkg/kernel/lib/binary/ast_from_binary.dart

+6-2
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ class BinaryBuilder {
6262
/// will not be resolved correctly.
6363
bool _disableLazyReading = false;
6464

65-
BinaryBuilder(this._bytes, [this.filename, this._disableLazyReading = false]);
65+
BinaryBuilder(this._bytes, {this.filename, disableLazyReading = false})
66+
: _disableLazyReading = disableLazyReading;
6667

6768
fail(String message) {
6869
throw new ParseError(message,
@@ -1107,8 +1108,10 @@ class BinaryBuilder {
11071108
final int programStartOffset = _programStartOffset;
11081109
final List<TypeParameter> typeParameters = typeParameterStack.toList();
11091110
final List<VariableDeclaration> variables = variableStack.toList();
1111+
final Library currentLibrary = _currentLibrary;
11101112
result.lazyBuilder = () {
11111113
_byteOffset = savedByteOffset;
1114+
_currentLibrary = currentLibrary;
11121115
typeParameterStack.clear();
11131116
typeParameterStack.addAll(typeParameters);
11141117
variableStack.clear();
@@ -1812,7 +1815,8 @@ class BinaryBuilderWithMetadata extends BinaryBuilder implements BinarySource {
18121815
/// Note: each metadata subsection has its own mapping.
18131816
List<Node> _referencedNodes;
18141817

1815-
BinaryBuilderWithMetadata(bytes, [filename]) : super(bytes, filename);
1818+
BinaryBuilderWithMetadata(bytes, [filename])
1819+
: super(bytes, filename: filename);
18161820

18171821
@override
18181822
bool _readMetadataSection(Program program) {

pkg/kernel/test/binary_bench.dart

+2-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ bool _parseArgs(List<String> args) {
162162

163163
Program _fromBinary(List<int> bytes, {eager: true}) {
164164
var program = new Program();
165-
new BinaryBuilder(bytes, 'filename', eager).readSingleFileProgram(program);
165+
new BinaryBuilder(bytes, filename: 'filename', disableLazyReading: eager)
166+
.readSingleFileProgram(program);
166167
return program;
167168
}
168169

0 commit comments

Comments
 (0)