Disable sandboxing for macOS tests in CI (#6866)

macOS 14 added new requirements that un-codesigned sandbox apps must be granted access when changed. Waiting for this UI caused macOS tests to fail on macOS 14. Additionally, adding codesigning is not sufficient, since it must still be approved before codesigning is enough to pass the check. As a workaround, this PR disables sandboxing for macOS tests in CI.

![Screenshot 2024-05-30 at 2 41 33�PM](https://github.com/flutter/flutter/assets/682784/1bc32620-5edb-420a-866c-5cc529b2ac55)

https://developer.apple.com/documentation/updates/security#June-2023)
> App Sandbox now associates your macOS app with its sandbox container using its code signature. The operating system asks the person using your app to grant permission if it tries to access a sandbox container associated with a different app. For more information, see [Accessing files from the macOS App Sandbox](https://developer.apple.com/documentation/security/app_sandbox/accessing_files_from_the_macos_app_sandbox).

And that link explains why this is happening on a macOS 14 update:

> In macOS 14 and later, the operating system uses your app�s code signature to associate it with its sandbox container. If your app tries to access the sandbox container owned by another app, the system asks the person using your app whether to grant access. If the person denies access and your app is already running, then it can�t read or write the files in the other app�s sandbox container. If the person denies access while your app is launching and trying to enter the other app�s sandbox container, your app fails to launch.
> 
> The operating system also tracks the association between an app�s code signing identity and its sandbox container for helper tools, including launch agents. If a person denies permission for a launch agent to enter its sandbox container and the app fails to start, launchd starts the launch agent again and the operating system re-requests access.

Fixes packages part of flutter/flutter#149264.

Verified tests pass:
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac_arm64%20macos_platform_tests%20master%20-%20packages/6/overview

Author: vashworth

packages/pigeon/platform_tests/test_plugin/example/macos/Runner/DebugProfile.entitlements

@@ -3,7 +3,7 @@
 <plist version="1.0">
 <dict>
 	<key>com.apple.security.app-sandbox</key>
-	<true/>
+	<false/>
 	<key>com.apple.security.cs.allow-jit</key>
 	<true/>
 	<key>com.apple.security.network.server</key>

packages/pigeon/platform_tests/test_plugin/example/macos/Runner/Release.entitlements

@@ -3,6 +3,6 @@
 <plist version="1.0">
 <dict>
 	<key>com.apple.security.app-sandbox</key>
-	<true/>
+	<false/>
 </dict>
 </plist>

script/tool/lib/src/common/xcode.dart

@@ -32,27 +32,37 @@ class Xcode {
 
   /// Runs an `xcodebuild` in [directory] with the given parameters.
   Future<int> runXcodeBuild(
-    Directory directory, {
+    Directory exampleDirectory,
+    String platform, {
     List<String> actions = const <String>['build'],
     required String workspace,
     required String scheme,
     String? configuration,
     List<String> extraFlags = const <String>[],
   }) {
+    File? disabledSandboxEntitlementFile;
+    if (actions.contains('test') && platform.toLowerCase() == 'macos') {
+      disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
+        exampleDirectory.childDirectory(platform.toLowerCase()),
+        configuration ?? 'Debug',
+      );
+    }
     final List<String> args = <String>[
       _xcodeBuildCommand,
       ...actions,
       ...<String>['-workspace', workspace],
       ...<String>['-scheme', scheme],
       if (configuration != null) ...<String>['-configuration', configuration],
       ...extraFlags,
+      if (disabledSandboxEntitlementFile != null)
+        'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
     ];
     final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}';
     if (log) {
       print(completeTestCommand);
     }
     return processRunner.runAndStream(_xcRunCommand, args,
-        workingDir: directory);
+        workingDir: exampleDirectory);
   }
 
   /// Returns true if [project], which should be an .xcodeproj directory,
@@ -156,4 +166,48 @@ class Xcode {
     }
     return null;
   }
+
+  /// Finds and copies macOS entitlements file. In the copy, disables sandboxing.
+  /// If entitlements file is not found, returns null.
+  ///
+  /// As of macOS 14, testing a macOS sandbox app may prompt the user to grant
+  /// access to the app. To workaround this in CI, we create and use a entitlements
+  /// file with sandboxing disabled. See
+  /// https://developer.apple.com/documentation/security/app_sandbox/accessing_files_from_the_macos_app_sandbox.
+  File? _createDisabledSandboxEntitlementFile(
+    Directory macOSDirectory,
+    String configuration,
+  ) {
+    final String entitlementDefaultFileName =
+        configuration == 'Release' ? 'Release' : 'DebugProfile';
+
+    final File entitlementFile = macOSDirectory
+        .childDirectory('Runner')
+        .childFile('$entitlementDefaultFileName.entitlements');
+
+    if (!entitlementFile.existsSync()) {
+      print('Unable to find entitlements file at ${entitlementFile.path}');
+      return null;
+    }
+
+    final String originalEntitlementFileContents =
+        entitlementFile.readAsStringSync();
+    final File disabledSandboxEntitlementFile = macOSDirectory
+        .fileSystem.systemTempDirectory
+        .createTempSync('flutter_disable_sandbox_entitlement.')
+        .childFile(
+            '${entitlementDefaultFileName}WithDisabledSandboxing.entitlements');
+    disabledSandboxEntitlementFile.createSync(recursive: true);
+    disabledSandboxEntitlementFile.writeAsStringSync(
+      originalEntitlementFileContents.replaceAll(
+        RegExp(
+            r'<key>com\.apple\.security\.app-sandbox<\/key>[\S\s]*?<true\/>'),
+        '''
+<key>com.apple.security.app-sandbox</key>
+	<false/>''',
+      ),
+    );
+
+    return disabledSandboxEntitlementFile;
+  }
 }

script/tool/lib/src/native_test_command.dart

@@ -465,6 +465,7 @@ this command.
       _printRunningExampleTestsMessage(example, platform);
       final int exitCode = await _xcode.runXcodeBuild(
         example.directory,
+        platform,
         // Clean before testing to remove cached swiftmodules from previous
         // runs, which can cause conflicts.
         actions: <String>['clean', 'test'],

script/tool/lib/src/xcode_analyze_command.dart

@@ -111,6 +111,7 @@ class XcodeAnalyzeCommand extends PackageLoopingCommand {
       print('Running $platform tests and analyzer for $examplePath...');
       final int exitCode = await _xcode.runXcodeBuild(
         example.directory,
+        platform,
         // Clean before analyzing to remove cached swiftmodules from previous
         // runs, which can cause conflicts.
         actions: <String>['clean', 'analyze'],

script/tool/test/common/xcode_test.dart

@@ -6,6 +6,7 @@ import 'dart:convert';
 
 import 'package:file/file.dart';
 import 'package:file/local.dart';
+import 'package:file/memory.dart';
 import 'package:flutter_plugin_tools/src/common/xcode.dart';
 import 'package:test/test.dart';
 
@@ -161,6 +162,7 @@ void main() {
 
       final int exitCode = await xcode.runXcodeBuild(
         directory,
+        'ios',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
       );
@@ -186,7 +188,7 @@ void main() {
     test('handles all arguments', () async {
       final Directory directory = const LocalFileSystem().currentDirectory;
 
-      final int exitCode = await xcode.runXcodeBuild(directory,
+      final int exitCode = await xcode.runXcodeBuild(directory, 'ios',
           actions: <String>['action1', 'action2'],
           workspace: 'A.xcworkspace',
           scheme: 'AScheme',
@@ -225,6 +227,7 @@ void main() {
 
       final int exitCode = await xcode.runXcodeBuild(
         directory,
+        'ios',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
       );
@@ -246,6 +249,42 @@ void main() {
                 directory.path),
           ]));
     });
+
+    test('sets CODE_SIGN_ENTITLEMENTS for macos tests', () async {
+      final FileSystem fileSystem = MemoryFileSystem();
+      final Directory directory = fileSystem.currentDirectory;
+      directory
+          .childDirectory('macos')
+          .childDirectory('Runner')
+          .childFile('DebugProfile.entitlements')
+          .createSync(recursive: true);
+
+      final int exitCode = await xcode.runXcodeBuild(
+        directory,
+        'macos',
+        workspace: 'A.xcworkspace',
+        scheme: 'AScheme',
+        actions: <String>['test'],
+      );
+
+      expect(exitCode, 0);
+      expect(
+          processRunner.recordedCalls,
+          orderedEquals(<ProcessCall>[
+            ProcessCall(
+                'xcrun',
+                const <String>[
+                  'xcodebuild',
+                  'test',
+                  '-workspace',
+                  'A.xcworkspace',
+                  '-scheme',
+                  'AScheme',
+                  'CODE_SIGN_ENTITLEMENTS=/.tmp_rand0/flutter_disable_sandbox_entitlement.rand0/DebugProfileWithDisabledSandboxing.entitlements'
+                ],
+                directory.path),
+          ]));
+    });
   });
 
   group('projectHasTarget', () {