Skip to content

Commit fcc72ad

Browse files
sjindel-googlecommit-bot@chromium.org
sjindel-google
authored and
[email protected]
committed
[vm/ffi] Support FFI in AOT (excluding callbacks).
Move generation of Function objects for native trampolines to the Precompiler, so they can be generated during AOT and tree-shaken if possible. Issue dartbug.com/35765 Change-Id: I0e69b7e0b22db73e3a40f2fe445660e57ddb6fa9 Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-bare-linux-release-simarm64-try, vm-kernel-precomp-bare-linux-release-x64-try, vm-kernel-precomp-linux-debug-x64-try, vm-dartkb-linux-debug-x64-try Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/107407 Commit-Queue: Samir Jindel <[email protected]> Reviewed-by: Alexander Markov <[email protected]>
1 parent f0d6f11 commit fcc72ad

29 files changed

+407
-163
lines changed

‎pkg/test_runner/lib/src/command.dart

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -76,20 +76,21 @@ class Command {
7676
}
7777

7878
static Command adbPrecompiled(
79-
String precompiledRunner,
79+
String buildPath,
8080
String processTest,
8181
String testDirectory,
8282
List<String> arguments,
8383
bool useBlobs,
84-
bool useElf) {
85-
return AdbPrecompilationCommand._(precompiledRunner, processTest,
86-
testDirectory, arguments, useBlobs, useElf);
84+
bool useElf,
85+
List<String> extraLibs) {
86+
return AdbPrecompilationCommand._(buildPath, processTest, testDirectory,
87+
arguments, useBlobs, useElf, extraLibs);
8788
}
8889

89-
static Command adbDartk(String precompiledRunner, String processTest,
90-
String script, List<String> arguments, List<String> extraLibraries) {
90+
static Command adbDartk(String buildPath, String processTest, String script,
91+
List<String> arguments, List<String> extraLibraries) {
9192
return AdbDartkCommand._(
92-
precompiledRunner, processTest, script, arguments, extraLibraries);
93+
buildPath, processTest, script, arguments, extraLibraries);
9394
}
9495

9596
static Command jsCommandLine(
@@ -587,54 +588,72 @@ class VmBatchCommand extends ProcessCommand implements VmCommand {
587588
}
588589
}
589590

590-
class AdbPrecompilationCommand extends Command {
591-
final String precompiledRunnerFilename;
591+
abstract class AdbCommand {
592+
String get buildPath;
593+
List<String> get extraLibraries;
594+
}
595+
596+
class AdbPrecompilationCommand extends Command implements AdbCommand {
597+
final String buildPath; // Path to the output directory of the build.
592598
final String processTestFilename;
593599
final String precompiledTestDirectory;
594600
final List<String> arguments;
595601
final bool useBlobs;
596602
final bool useElf;
603+
final List<String> extraLibraries;
597604

598605
AdbPrecompilationCommand._(
599-
this.precompiledRunnerFilename,
606+
this.buildPath,
600607
this.processTestFilename,
601608
this.precompiledTestDirectory,
602609
this.arguments,
603610
this.useBlobs,
604611
this.useElf,
612+
this.extraLibraries,
605613
{int index = 0})
606614
: super._("adb_precompilation", index: index);
607615

608616
AdbPrecompilationCommand indexedCopy(int index) => AdbPrecompilationCommand._(
609-
precompiledRunnerFilename,
617+
buildPath,
610618
processTestFilename,
611619
precompiledTestDirectory,
612620
arguments,
613621
useBlobs,
614622
useElf,
623+
extraLibraries,
615624
index: index);
616625
_buildHashCode(HashCodeBuilder builder) {
617626
super._buildHashCode(builder);
618-
builder.add(precompiledRunnerFilename);
627+
builder.add(buildPath);
619628
builder.add(precompiledTestDirectory);
620629
builder.add(arguments);
621630
builder.add(useBlobs);
622631
builder.add(useElf);
632+
extraLibraries.forEach(builder.add);
633+
}
634+
635+
static bool _listEquals(List<String> x, List<String> y) {
636+
if (x.length != y.length) return false;
637+
for (int i = 0; i < x.length; ++i) {
638+
if (x[i] != y[i]) return false;
639+
}
640+
return true;
623641
}
624642

625643
bool _equal(AdbPrecompilationCommand other) =>
626644
super._equal(other) &&
627-
precompiledRunnerFilename == other.precompiledRunnerFilename &&
645+
buildPath == other.buildPath &&
628646
useBlobs == other.useBlobs &&
629647
useElf == other.useElf &&
630648
arguments == other.arguments &&
631-
precompiledTestDirectory == other.precompiledTestDirectory;
649+
precompiledTestDirectory == other.precompiledTestDirectory &&
650+
_listEquals(extraLibraries, other.extraLibraries);
632651

633652
String toString() => 'Steps to push precompiled runner and precompiled code '
634653
'to an attached device. Uses (and requires) adb.';
635654
}
636655

637-
class AdbDartkCommand extends Command {
656+
class AdbDartkCommand extends Command implements AdbCommand {
638657
final String buildPath;
639658
final String processTestFilename;
640659
final String kernelFile;

‎pkg/test_runner/lib/src/process_queue.dart

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -629,9 +629,23 @@ class CommandExecutorImpl implements CommandExecutor {
629629
}
630630
}
631631

632+
List<_StepFunction> _pushLibraries(AdbCommand command, AdbDevice device,
633+
String deviceDir, String deviceTestDir) {
634+
final List<_StepFunction> steps = [];
635+
for (var lib in command.extraLibraries) {
636+
var libName = "lib${lib}.so";
637+
steps.add(() => device.runAdbCommand([
638+
'push',
639+
'${command.buildPath}/$libName',
640+
'$deviceTestDir/$libName'
641+
]));
642+
}
643+
return steps;
644+
}
645+
632646
Future<CommandOutput> _runAdbPrecompilationCommand(
633647
AdbDevice device, AdbPrecompilationCommand command, int timeout) async {
634-
var runner = command.precompiledRunnerFilename;
648+
final String buildPath = command.buildPath;
635649
var processTest = command.processTestFilename;
636650
var testdir = command.precompiledTestDirectory;
637651
var arguments = command.arguments;
@@ -652,8 +666,8 @@ class CommandExecutorImpl implements CommandExecutor {
652666

653667
steps.add(() => device.runAdbShellCommand(['rm', '-Rf', deviceTestDir]));
654668
steps.add(() => device.runAdbShellCommand(['mkdir', '-p', deviceTestDir]));
655-
steps.add(() =>
656-
device.pushCachedData(runner, '$devicedir/dart_precompiled_runtime'));
669+
steps.add(() => device.pushCachedData('$buildPath/dart_precompiled_runtime',
670+
'$devicedir/dart_precompiled_runtime'));
657671
steps.add(
658672
() => device.pushCachedData(processTest, '$devicedir/process_test'));
659673
steps.add(() => device.runAdbShellCommand([
@@ -662,14 +676,17 @@ class CommandExecutorImpl implements CommandExecutor {
662676
'$devicedir/dart_precompiled_runtime $devicedir/process_test'
663677
]));
664678

679+
steps.addAll(_pushLibraries(command, device, devicedir, deviceTestDir));
680+
665681
for (var file in files) {
666682
steps.add(() => device
667683
.runAdbCommand(['push', '$testdir/$file', '$deviceTestDir/$file']));
668684
}
669685

670686
steps.add(() => device.runAdbShellCommand(
671687
[
672-
'$devicedir/dart_precompiled_runtime',
688+
'export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$deviceTestDir;'
689+
'$devicedir/dart_precompiled_runtime',
673690
]..addAll(arguments),
674691
timeout: timeoutDuration));
675692

@@ -723,11 +740,7 @@ class CommandExecutorImpl implements CommandExecutor {
723740
steps.add(() => device
724741
.runAdbCommand(['push', hostKernelFile, '$deviceTestDir/out.dill']));
725742

726-
for (var lib in command.extraLibraries) {
727-
var libName = "lib${lib}.so";
728-
steps.add(() => device.runAdbCommand(
729-
['push', '${buildPath}/$libName', '$deviceTestDir/$libName']));
730-
}
743+
steps.addAll(_pushLibraries(command, device, devicedir, deviceTestDir));
731744

732745
steps.add(() => device.runAdbShellCommand(
733746
[

‎pkg/test_runner/lib/src/runtime_configuration.dart

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -369,11 +369,10 @@ class DartPrecompiledAdbRuntimeConfiguration
369369
throw "dart_precompiled cannot run files of type '$type'.";
370370
}
371371

372-
String precompiledRunner = dartPrecompiledBinaryFileName;
373372
String processTest = processTestBinaryFileName;
374373
return [
375374
Command.adbPrecompiled(
376-
precompiledRunner, processTest, script, arguments, useBlobs, useElf)
375+
buildDir, processTest, script, arguments, useBlobs, useElf, extraLibs)
377376
];
378377
}
379378
}

‎pkg/vm/lib/transformations/ffi.dart

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ const List<int> nativeTypeSizes = [
9090
/// _FfiUseSiteTransformer and _FfiDefinitionTransformer.
9191
class FfiTransformer extends Transformer {
9292
final TypeEnvironment env;
93+
final CoreTypes coreTypes;
9394
final LibraryIndex index;
9495
final ClassHierarchy hierarchy;
9596
final DiagnosticReporter diagnosticReporter;
@@ -107,16 +108,18 @@ class FfiTransformer extends Transformer {
107108
final Procedure storeMethod;
108109
final Procedure offsetByMethod;
109110
final Procedure asFunctionMethod;
111+
final Procedure asFunctionInternal;
110112
final Procedure lookupFunctionMethod;
111113
final Procedure fromFunctionMethod;
112114
final Field addressOfField;
113115
final Constructor structFromPointer;
116+
final Procedure libraryLookupMethod;
114117

115118
/// Classes corresponding to [NativeType], indexed by [NativeType].
116119
final List<Class> nativeTypesClasses;
117120

118121
FfiTransformer(
119-
this.index, CoreTypes coreTypes, this.hierarchy, this.diagnosticReporter)
122+
this.index, this.coreTypes, this.hierarchy, this.diagnosticReporter)
120123
: env = new TypeEnvironment(coreTypes, hierarchy),
121124
intClass = coreTypes.intClass,
122125
doubleClass = coreTypes.doubleClass,
@@ -133,10 +136,14 @@ class FfiTransformer extends Transformer {
133136
structFromPointer =
134137
index.getMember('dart:ffi', 'Struct', 'fromPointer'),
135138
asFunctionMethod = index.getMember('dart:ffi', 'Pointer', 'asFunction'),
139+
asFunctionInternal =
140+
index.getTopLevelMember('dart:ffi', '_asFunctionInternal'),
136141
lookupFunctionMethod =
137142
index.getMember('dart:ffi', 'DynamicLibrary', 'lookupFunction'),
138143
fromFunctionMethod =
139144
index.getMember('dart:ffi', 'Pointer', 'fromFunction'),
145+
libraryLookupMethod =
146+
index.getMember('dart:ffi', 'DynamicLibrary', 'lookup'),
140147
nativeTypesClasses = nativeTypeClassNames
141148
.map((name) => index.getClass('dart:ffi', name))
142149
.toList();

‎pkg/vm/lib/transformations/ffi_use_sites.dart

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -144,34 +144,69 @@ class _FfiUseSiteTransformer extends FfiTransformer {
144144
return node;
145145
}
146146

147+
// We need to replace calls to 'DynamicLibrary.lookupFunction' with explicit
148+
// Kernel, because we cannot have a generic call to 'asFunction' in its body.
149+
//
150+
// Below, in 'visitMethodInvocation', we ensure that the type arguments to
151+
// 'lookupFunction' are constants, so by inlining the call to 'asFunction' at
152+
// the call-site, we ensure that there are no generic calls to 'asFunction'.
153+
//
154+
// We will not detect dynamic invocations of 'asFunction' -- these are handled
155+
// by the stub in 'dynamic_library_patch.dart'. Dynamic invocations of
156+
// 'lookupFunction' (and 'asFunction') are not legal and throw a runtime
157+
// exception.
158+
Expression _replaceLookupFunction(MethodInvocation node) {
159+
// The generated code looks like:
160+
//
161+
// _asFunctionInternal<DS, NS>(lookup<NativeFunction<NS>>(symbolName))
162+
163+
final DartType nativeSignature = node.arguments.types[0];
164+
final DartType dartSignature = node.arguments.types[1];
165+
166+
final Arguments args = Arguments([
167+
node.arguments.positional.single
168+
], types: [
169+
InterfaceType(nativeFunctionClass, [nativeSignature])
170+
]);
171+
172+
final Expression lookupResult = MethodInvocation(
173+
node.receiver, Name("lookup"), args, libraryLookupMethod);
174+
175+
return StaticInvocation(asFunctionInternal,
176+
Arguments([lookupResult], types: [dartSignature, nativeSignature]));
177+
}
178+
147179
@override
148180
visitMethodInvocation(MethodInvocation node) {
149181
super.visitMethodInvocation(node);
150182

151183
Member target = node.interfaceTarget;
152184
try {
185+
// We will not detect dynamic invocations of 'asFunction' and
186+
// 'lookupFunction' -- these are handled by the 'asFunctionInternal' stub
187+
// in 'dynamic_library_patch.dart'. Dynamic invocations of 'asFunction'
188+
// and 'lookupFunction' are not legal and throw a runtime exception.
153189
if (target == lookupFunctionMethod) {
154190
DartType nativeType =
155191
InterfaceType(nativeFunctionClass, [node.arguments.types[0]]);
156192
DartType dartType = node.arguments.types[1];
157193

158194
_ensureNativeTypeValid(nativeType, node);
159195
_ensureNativeTypeToDartType(nativeType, dartType, node);
196+
return _replaceLookupFunction(node);
160197
} else if (target == asFunctionMethod) {
161-
if (isFfiLibrary) {
162-
// Library code of dart:ffi uses asFunction to implement
163-
// lookupFunction. Since we treat lookupFunction as well, this call
164-
// can be generic and still support AOT.
165-
return node;
166-
}
167-
168198
DartType dartType = node.arguments.types[0];
169199
DartType pointerType = node.receiver.getStaticType(env);
170200
DartType nativeType = _pointerTypeGetTypeArg(pointerType);
171201

172202
_ensureNativeTypeValid(pointerType, node);
173203
_ensureNativeTypeValid(nativeType, node);
174204
_ensureNativeTypeToDartType(nativeType, dartType, node);
205+
206+
final DartType nativeSignature =
207+
(nativeType as InterfaceType).typeArguments[0];
208+
return StaticInvocation(asFunctionInternal,
209+
Arguments([node.receiver], types: [dartType, nativeSignature]));
175210
} else if (target == loadMethod) {
176211
// TODO(dacoharkes): should load and store be generic?
177212
// https://github.com/dart-lang/sdk/issues/35902

0 commit comments

Comments
 (0)