[vm/ffi] Support FFI in AOT (excluding callbacks).

Move generation of Function objects for native trampolines to the Precompiler, so they can be generated during AOT and tree-shaken if possible.

Issue dartbug.com/35765

Change-Id: I0e69b7e0b22db73e3a40f2fe445660e57ddb6fa9
Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-bare-linux-release-simarm64-try, vm-kernel-precomp-bare-linux-release-x64-try, vm-kernel-precomp-linux-debug-x64-try, vm-dartkb-linux-debug-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/107407
Commit-Queue: Samir Jindel <[email protected]>
Reviewed-by: Alexander Markov <[email protected]>

Author: sjindel-google

pkg/test_runner/lib/src/command.dart

@@ -76,20 +76,21 @@ class Command {
   }
 
   static Command adbPrecompiled(
-      String precompiledRunner,
+      String buildPath,
       String processTest,
       String testDirectory,
       List<String> arguments,
       bool useBlobs,
-      bool useElf) {
-    return AdbPrecompilationCommand._(precompiledRunner, processTest,
-        testDirectory, arguments, useBlobs, useElf);
+      bool useElf,
+      List<String> extraLibs) {
+    return AdbPrecompilationCommand._(buildPath, processTest, testDirectory,
+        arguments, useBlobs, useElf, extraLibs);
   }
 
-  static Command adbDartk(String precompiledRunner, String processTest,
-      String script, List<String> arguments, List<String> extraLibraries) {
+  static Command adbDartk(String buildPath, String processTest, String script,
+      List<String> arguments, List<String> extraLibraries) {
     return AdbDartkCommand._(
-        precompiledRunner, processTest, script, arguments, extraLibraries);
+        buildPath, processTest, script, arguments, extraLibraries);
   }
 
   static Command jsCommandLine(
@@ -587,54 +588,72 @@ class VmBatchCommand extends ProcessCommand implements VmCommand {
   }
 }
 
-class AdbPrecompilationCommand extends Command {
-  final String precompiledRunnerFilename;
+abstract class AdbCommand {
+  String get buildPath;
+  List<String> get extraLibraries;
+}
+
+class AdbPrecompilationCommand extends Command implements AdbCommand {
+  final String buildPath; // Path to the output directory of the build.
   final String processTestFilename;
   final String precompiledTestDirectory;
   final List<String> arguments;
   final bool useBlobs;
   final bool useElf;
+  final List<String> extraLibraries;
 
   AdbPrecompilationCommand._(
-      this.precompiledRunnerFilename,
+      this.buildPath,
       this.processTestFilename,
       this.precompiledTestDirectory,
       this.arguments,
       this.useBlobs,
       this.useElf,
+      this.extraLibraries,
       {int index = 0})
       : super._("adb_precompilation", index: index);
 
   AdbPrecompilationCommand indexedCopy(int index) => AdbPrecompilationCommand._(
-      precompiledRunnerFilename,
+      buildPath,
       processTestFilename,
       precompiledTestDirectory,
       arguments,
       useBlobs,
       useElf,
+      extraLibraries,
       index: index);
   _buildHashCode(HashCodeBuilder builder) {
     super._buildHashCode(builder);
-    builder.add(precompiledRunnerFilename);
+    builder.add(buildPath);
     builder.add(precompiledTestDirectory);
     builder.add(arguments);
     builder.add(useBlobs);
     builder.add(useElf);
+    extraLibraries.forEach(builder.add);
+  }
+
+  static bool _listEquals(List<String> x, List<String> y) {
+    if (x.length != y.length) return false;
+    for (int i = 0; i < x.length; ++i) {
+      if (x[i] != y[i]) return false;
+    }
+    return true;
   }
 
   bool _equal(AdbPrecompilationCommand other) =>
       super._equal(other) &&
-      precompiledRunnerFilename == other.precompiledRunnerFilename &&
+      buildPath == other.buildPath &&
       useBlobs == other.useBlobs &&
       useElf == other.useElf &&
       arguments == other.arguments &&
-      precompiledTestDirectory == other.precompiledTestDirectory;
+      precompiledTestDirectory == other.precompiledTestDirectory &&
+      _listEquals(extraLibraries, other.extraLibraries);
 
   String toString() => 'Steps to push precompiled runner and precompiled code '
       'to an attached device. Uses (and requires) adb.';
 }
 
-class AdbDartkCommand extends Command {
+class AdbDartkCommand extends Command implements AdbCommand {
   final String buildPath;
   final String processTestFilename;
   final String kernelFile;

pkg/test_runner/lib/src/process_queue.dart

@@ -629,9 +629,23 @@ class CommandExecutorImpl implements CommandExecutor {
     }
   }
 
+  List<_StepFunction> _pushLibraries(AdbCommand command, AdbDevice device,
+      String deviceDir, String deviceTestDir) {
+    final List<_StepFunction> steps = [];
+    for (var lib in command.extraLibraries) {
+      var libName = "lib${lib}.so";
+      steps.add(() => device.runAdbCommand([
+            'push',
+            '${command.buildPath}/$libName',
+            '$deviceTestDir/$libName'
+          ]));
+    }
+    return steps;
+  }
+
   Future<CommandOutput> _runAdbPrecompilationCommand(
       AdbDevice device, AdbPrecompilationCommand command, int timeout) async {
-    var runner = command.precompiledRunnerFilename;
+    final String buildPath = command.buildPath;
     var processTest = command.processTestFilename;
     var testdir = command.precompiledTestDirectory;
     var arguments = command.arguments;
@@ -652,8 +666,8 @@ class CommandExecutorImpl implements CommandExecutor {
 
     steps.add(() => device.runAdbShellCommand(['rm', '-Rf', deviceTestDir]));
     steps.add(() => device.runAdbShellCommand(['mkdir', '-p', deviceTestDir]));
-    steps.add(() =>
-        device.pushCachedData(runner, '$devicedir/dart_precompiled_runtime'));
+    steps.add(() => device.pushCachedData('$buildPath/dart_precompiled_runtime',
+        '$devicedir/dart_precompiled_runtime'));
     steps.add(
         () => device.pushCachedData(processTest, '$devicedir/process_test'));
     steps.add(() => device.runAdbShellCommand([
@@ -662,14 +676,17 @@ class CommandExecutorImpl implements CommandExecutor {
           '$devicedir/dart_precompiled_runtime $devicedir/process_test'
         ]));
 
+    steps.addAll(_pushLibraries(command, device, devicedir, deviceTestDir));
+
     for (var file in files) {
       steps.add(() => device
           .runAdbCommand(['push', '$testdir/$file', '$deviceTestDir/$file']));
     }
 
     steps.add(() => device.runAdbShellCommand(
         [
-          '$devicedir/dart_precompiled_runtime',
+          'export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$deviceTestDir;'
+              '$devicedir/dart_precompiled_runtime',
         ]..addAll(arguments),
         timeout: timeoutDuration));
 
@@ -723,11 +740,7 @@ class CommandExecutorImpl implements CommandExecutor {
     steps.add(() => device
         .runAdbCommand(['push', hostKernelFile, '$deviceTestDir/out.dill']));
 
-    for (var lib in command.extraLibraries) {
-      var libName = "lib${lib}.so";
-      steps.add(() => device.runAdbCommand(
-          ['push', '${buildPath}/$libName', '$deviceTestDir/$libName']));
-    }
+    steps.addAll(_pushLibraries(command, device, devicedir, deviceTestDir));
 
     steps.add(() => device.runAdbShellCommand(
         [

pkg/test_runner/lib/src/runtime_configuration.dart

@@ -369,11 +369,10 @@ class DartPrecompiledAdbRuntimeConfiguration
       throw "dart_precompiled cannot run files of type '$type'.";
     }
 
-    String precompiledRunner = dartPrecompiledBinaryFileName;
     String processTest = processTestBinaryFileName;
     return [
       Command.adbPrecompiled(
-          precompiledRunner, processTest, script, arguments, useBlobs, useElf)
+          buildDir, processTest, script, arguments, useBlobs, useElf, extraLibs)
     ];
   }
 }

pkg/vm/lib/transformations/ffi.dart

@@ -90,6 +90,7 @@ const List<int> nativeTypeSizes = [
 /// _FfiUseSiteTransformer and _FfiDefinitionTransformer.
 class FfiTransformer extends Transformer {
   final TypeEnvironment env;
+  final CoreTypes coreTypes;
   final LibraryIndex index;
   final ClassHierarchy hierarchy;
   final DiagnosticReporter diagnosticReporter;
@@ -107,16 +108,18 @@ class FfiTransformer extends Transformer {
   final Procedure storeMethod;
   final Procedure offsetByMethod;
   final Procedure asFunctionMethod;
+  final Procedure asFunctionInternal;
   final Procedure lookupFunctionMethod;
   final Procedure fromFunctionMethod;
   final Field addressOfField;
   final Constructor structFromPointer;
+  final Procedure libraryLookupMethod;
 
   /// Classes corresponding to [NativeType], indexed by [NativeType].
   final List<Class> nativeTypesClasses;
 
   FfiTransformer(
-      this.index, CoreTypes coreTypes, this.hierarchy, this.diagnosticReporter)
+      this.index, this.coreTypes, this.hierarchy, this.diagnosticReporter)
       : env = new TypeEnvironment(coreTypes, hierarchy),
         intClass = coreTypes.intClass,
         doubleClass = coreTypes.doubleClass,
@@ -133,10 +136,14 @@ class FfiTransformer extends Transformer {
         structFromPointer =
             index.getMember('dart:ffi', 'Struct', 'fromPointer'),
         asFunctionMethod = index.getMember('dart:ffi', 'Pointer', 'asFunction'),
+        asFunctionInternal =
+            index.getTopLevelMember('dart:ffi', '_asFunctionInternal'),
         lookupFunctionMethod =
             index.getMember('dart:ffi', 'DynamicLibrary', 'lookupFunction'),
         fromFunctionMethod =
             index.getMember('dart:ffi', 'Pointer', 'fromFunction'),
+        libraryLookupMethod =
+            index.getMember('dart:ffi', 'DynamicLibrary', 'lookup'),
         nativeTypesClasses = nativeTypeClassNames
             .map((name) => index.getClass('dart:ffi', name))
             .toList();

pkg/vm/lib/transformations/ffi_use_sites.dart

@@ -144,34 +144,69 @@ class _FfiUseSiteTransformer extends FfiTransformer {
     return node;
   }
 
+  // We need to replace calls to 'DynamicLibrary.lookupFunction' with explicit
+  // Kernel, because we cannot have a generic call to 'asFunction' in its body.
+  //
+  // Below, in 'visitMethodInvocation', we ensure that the type arguments to
+  // 'lookupFunction' are constants, so by inlining the call to 'asFunction' at
+  // the call-site, we ensure that there are no generic calls to 'asFunction'.
+  //
+  // We will not detect dynamic invocations of 'asFunction' -- these are handled
+  // by the stub in 'dynamic_library_patch.dart'. Dynamic invocations of
+  // 'lookupFunction' (and 'asFunction') are not legal and throw a runtime
+  // exception.
+  Expression _replaceLookupFunction(MethodInvocation node) {
+    // The generated code looks like:
+    //
+    // _asFunctionInternal<DS, NS>(lookup<NativeFunction<NS>>(symbolName))
+
+    final DartType nativeSignature = node.arguments.types[0];
+    final DartType dartSignature = node.arguments.types[1];
+
+    final Arguments args = Arguments([
+      node.arguments.positional.single
+    ], types: [
+      InterfaceType(nativeFunctionClass, [nativeSignature])
+    ]);
+
+    final Expression lookupResult = MethodInvocation(
+        node.receiver, Name("lookup"), args, libraryLookupMethod);
+
+    return StaticInvocation(asFunctionInternal,
+        Arguments([lookupResult], types: [dartSignature, nativeSignature]));
+  }
+
   @override
   visitMethodInvocation(MethodInvocation node) {
     super.visitMethodInvocation(node);
 
     Member target = node.interfaceTarget;
     try {
+      // We will not detect dynamic invocations of 'asFunction' and
+      // 'lookupFunction' -- these are handled by the 'asFunctionInternal' stub
+      // in 'dynamic_library_patch.dart'. Dynamic invocations of 'asFunction'
+      // and 'lookupFunction' are not legal and throw a runtime exception.
       if (target == lookupFunctionMethod) {
         DartType nativeType =
             InterfaceType(nativeFunctionClass, [node.arguments.types[0]]);
         DartType dartType = node.arguments.types[1];
 
         _ensureNativeTypeValid(nativeType, node);
         _ensureNativeTypeToDartType(nativeType, dartType, node);
+        return _replaceLookupFunction(node);
       } else if (target == asFunctionMethod) {
-        if (isFfiLibrary) {
-          // Library code of dart:ffi uses asFunction to implement
-          // lookupFunction. Since we treat lookupFunction as well, this call
-          // can be generic and still support AOT.
-          return node;
-        }
-
         DartType dartType = node.arguments.types[0];
         DartType pointerType = node.receiver.getStaticType(env);
         DartType nativeType = _pointerTypeGetTypeArg(pointerType);
 
         _ensureNativeTypeValid(pointerType, node);
         _ensureNativeTypeValid(nativeType, node);
         _ensureNativeTypeToDartType(nativeType, dartType, node);
+
+        final DartType nativeSignature =
+            (nativeType as InterfaceType).typeArguments[0];
+        return StaticInvocation(asFunctionInternal,
+            Arguments([node.receiver], types: [dartType, nativeSignature]));
       } else if (target == loadMethod) {
         // TODO(dacoharkes): should load and store be generic?
         // https://github.com/dart-lang/sdk/issues/35902