Skip to content

[ci] Add LUCI version of Android FTL tests in bringup mode #4571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jul 28, 2023
65 changes: 65 additions & 0 deletions .ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,71 @@ targets:
{"dependency": "android_virtual_device", "version": "33"}
]

# Device versions of Android integration tests, run via FTL.
# TODO(stuartmorgan): Revisit whether physical device tests are redundant once
# we have more data about emulator tests; see
# https://github.com/flutter/flutter/issues/131429.

- name: Linux_android android_device_tests_shard_1 master
bringup: true # New target
recipe: packages/packages
timeout: 60
properties:
target_file: android_device_tests.yaml
channel: master
version_file: flutter_master.version
package_sharding: "--shardIndex 0 --shardCount 6"

- name: Linux_android android_device_tests_shard_2 master
bringup: true # New target
recipe: packages/packages
timeout: 60
properties:
target_file: android_device_tests.yaml
channel: master
version_file: flutter_master.version
package_sharding: "--shardIndex 1 --shardCount 6"

- name: Linux_android android_device_tests_shard_3 master
bringup: true # New target
recipe: packages/packages
timeout: 60
properties:
target_file: android_device_tests.yaml
channel: master
version_file: flutter_master.version
package_sharding: "--shardIndex 2 --shardCount 6"

- name: Linux_android android_device_tests_shard_4 master
bringup: true # New target
recipe: packages/packages
timeout: 60
properties:
target_file: android_device_tests.yaml
channel: master
version_file: flutter_master.version
package_sharding: "--shardIndex 3 --shardCount 6"

- name: Linux_android android_device_tests_shard_5 master
bringup: true # New target
recipe: packages/packages
timeout: 60
properties:
target_file: android_device_tests.yaml
channel: master
version_file: flutter_master.version
package_sharding: "--shardIndex 4 --shardCount 6"

- name: Linux_android android_device_tests_shard_6 master
bringup: true # New target
recipe: packages/packages
timeout: 60
properties:
target_file: android_device_tests.yaml
channel: master
version_file: flutter_master.version
package_sharding: "--shardIndex 5 --shardCount 6"

### Web tasks ###
- name: Linux_web web_build_all_packages master
recipe: packages/packages
Expand Down
17 changes: 17 additions & 0 deletions .ci/targets/android_device_tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
tasks:
- name: prepare tool
script: .ci/scripts/prepare_tool.sh
infra_step: true # Note infra steps failing prevents "always" from running.
- name: download Dart and Android deps
script: script/tool_runner.sh
infra_step: true
args: ["fetch-deps", "--android", "--supporting-target-platforms-only"]
- name: Firebase Test Lab
script: script/tool_runner.sh
args:
- "firebase-test-lab"
- "--device"
- "model=redfin,version=30"
- "--exclude=script/configs/exclude_integration_android.yaml"
- "--project=flutter-infra-staging"
- "--results-bucket=flutter_firebase_testlab_staging"
5 changes: 3 additions & 2 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,13 @@ task:
matrix:
CHANNEL: "master"
CHANNEL: "stable"
GCLOUD_KEY_PATH: $HOME/gcloud-service-key.json
MAPS_API_KEY: ENCRYPTED[d6583b08f79f91ea4844c77460f04539965e46ad2fd97fb7c062b4dfe88016228b86ebe8c220ab4187e0c4bd773dc1e7]
GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1a2eebf9367197bbe812d9a0ea83a53a05aeba4bb5e4964fe6a69727883cd87e51238d39237b1f80b0894c48419ac268]
firebase_test_lab_script:
- if [[ -n "$GCLOUD_FIREBASE_TESTLAB_KEY" ]]; then
- echo $GCLOUD_FIREBASE_TESTLAB_KEY > ${HOME}/gcloud-service-key.json
- ./script/tool_runner.sh firebase-test-lab --device model=redfin,version=30 --exclude=script/configs/exclude_integration_android.yaml
- echo $GCLOUD_FIREBASE_TESTLAB_KEY > "${GCLOUD_KEY_PATH}"
- ./script/tool_runner.sh firebase-test-lab --device model=redfin,version=30 --exclude=script/configs/exclude_integration_android.yaml --project=flutter-cirrus --results-bucket=flutter_cirrus_testlab --service-key="${GCLOUD_KEY_PATH}"
- else
- echo "This user does not have permission to run Firebase Test Lab tests."
- fi
36 changes: 20 additions & 16 deletions script/tool/lib/src/firebase_test_lab_command.dart
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/repository_package.dart';

const int _exitGcloudAuthFailed = 2;
const int _exitGcloudAuthFailed = 3;

/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PackageLoopingCommand {
Expand All @@ -25,18 +25,13 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
super.platform,
}) {
argParser.addOption(
'project',
defaultsTo: 'flutter-cirrus',
_gCloudProjectArg,
help: 'The Firebase project name.',
);
final String? homeDir = io.Platform.environment['HOME'];
argParser.addOption('service-key',
defaultsTo: homeDir == null
? null
: path.join(homeDir, 'gcloud-service-key.json'),
argParser.addOption(_gCloudServiceKeyArg,
help: 'The path to the service key for gcloud authentication.\n'
r'If not provided, \$HOME/gcloud-service-key.json will be '
r'assumed if $HOME is set.');
'If not provided, setup will be skipped, so testing will fail '
'unless gcloud is already configured.');
argParser.addOption('test-run-id',
defaultsTo: const Uuid().v4(),
help:
Expand All @@ -57,15 +52,18 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
],
help:
'Device model(s) to test. See https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run for more info');
argParser.addOption('results-bucket',
defaultsTo: 'gs://flutter_cirrus_testlab');
argParser.addOption(_gCloudResultsBucketArg, mandatory: true);
argParser.addOption(
kEnableExperiment,
defaultsTo: '',
help: 'Enables the given Dart SDK experiments.',
);
}

static const String _gCloudServiceKeyArg = 'service-key';
static const String _gCloudProjectArg = 'project';
static const String _gCloudResultsBucketArg = 'results-bucket';

@override
final String name = 'firebase-test-lab';

Expand All @@ -82,9 +80,10 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
return;
}

final String serviceKey = getStringArg('service-key');
final String serviceKey = getStringArg(_gCloudServiceKeyArg);
if (serviceKey.isEmpty) {
print('No --service-key provided; skipping gcloud authorization');
print(
'No --$_gCloudServiceKeyArg provided; skipping gcloud authorization');
} else {
final io.ProcessResult result = await processRunner.run(
'gcloud',
Expand All @@ -99,11 +98,16 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
printError('Unable to activate gcloud account.');
throw ToolExit(_exitGcloudAuthFailed);
}
}
final String project = getStringArg(_gCloudProjectArg);
if (project.isEmpty) {
print('No --$_gCloudProjectArg provided; skipping gcloud config');
} else {
final int exitCode = await processRunner.runAndStream('gcloud', <String>[
'config',
'set',
'project',
getStringArg('project'),
project,
]);
print('');
if (exitCode == 0) {
Expand Down Expand Up @@ -284,7 +288,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
'--timeout',
'7m',
'--results-bucket=${getStringArg('results-bucket')}',
'--results-bucket=gs://${getStringArg(_gCloudResultsBucketArg)}',
'--results-dir=$resultsDir',
for (final String device in getStringListArg('device')) ...<String>[
'--device',
Expand Down
Loading