Skip to content
Merged
70 changes: 69 additions & 1 deletion scripts/run_nightly_tests_in_devicefarm_pool.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,79 @@ function stopDuplicates {
}
stopDuplicates

# Select range of devices from version 7 and above.
device1=$(aws devicefarm list-devices \
--region="us-west-2" \
--filters '[
{"attribute":"AVAILABILITY","operator":"EQUALS","values":["HIGHLY_AVAILABLE"]},
{"attribute":"PLATFORM","operator":"EQUALS","values":["ANDROID"]},
{"attribute":"OS_VERSION","operator":"GREATER_THAN_OR_EQUALS","values":["7"]},
{"attribute":"OS_VERSION","operator":"LESS_THAN","values":["7.1"]},
{"attribute":"MANUFACTURER","operator":"IN","values":["Google", "Pixel", "Samsung"]}
]' \
| jq -r '.devices[0].arn')

device2=$(aws devicefarm list-devices \
--region="us-west-2" \
--filters '[
{"attribute":"AVAILABILITY","operator":"EQUALS","values":["HIGHLY_AVAILABLE"]},
{"attribute":"PLATFORM","operator":"EQUALS","values":["ANDROID"]},
{"attribute":"OS_VERSION","operator":"GREATER_THAN_OR_EQUALS","values":["8"]},
{"attribute":"OS_VERSION","operator":"LESS_THAN","values":["9"]},
{"attribute":"MANUFACTURER","operator":"IN","values":["Samsung"]}
]' \
| jq -r '.devices[0].arn')

device3=$(aws devicefarm list-devices \
--region="us-west-2" \
--filters '[
{"attribute":"ARN","operator":"NOT_IN","values":["'$device2'"]},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like these might not be necessary given that OS_VERSION steps through numbers with no overlap?

{"attribute":"AVAILABILITY","operator":"EQUALS","values":["HIGHLY_AVAILABLE"]},
{"attribute":"PLATFORM","operator":"EQUALS","values":["ANDROID"]},
{"attribute":"OS_VERSION","operator":"GREATER_THAN_OR_EQUALS","values":["9"]},
{"attribute":"OS_VERSION","operator":"LESS_THAN","values":["10"]},
{"attribute":"MANUFACTURER","operator":"IN","values":["Samsung"]}
]' \
| jq -r '.devices[0].arn')

device4=$(aws devicefarm list-devices \
--region="us-west-2" \
--filters '[
{"attribute":"ARN","operator":"NOT_IN","values":["'$device2'", "'$device3'"]},
{"attribute":"AVAILABILITY","operator":"EQUALS","values":["HIGHLY_AVAILABLE"]},
{"attribute":"PLATFORM","operator":"EQUALS","values":["ANDROID"]},
{"attribute":"OS_VERSION","operator":"GREATER_THAN_OR_EQUALS","values":["10"]},
{"attribute":"OS_VERSION","operator":"LESS_THAN","values":["11"]},
{"attribute":"MANUFACTURER","operator":"IN","values":["Samsung"]}
]' \
| jq -r '.devices[0].arn')

device5=$(aws devicefarm list-devices \
--region="us-west-2" \
--filters '[
{"attribute":"AVAILABILITY","operator":"EQUALS","values":["HIGHLY_AVAILABLE"]},
{"attribute":"PLATFORM","operator":"EQUALS","values":["ANDROID"]},
{"attribute":"OS_VERSION","operator":"GREATER_THAN_OR_EQUALS","values":["12"]},
{"attribute":"MANUFACTURER","operator":"IN","values":["Google", "Pixel"]}
]' \
| jq -r '.devices[0].arn')

# IF we fail to find our required test devices, fail.
if [[ -z "${device1}" || -z "${device2}" || -z "${device3}" || -z "${device4}" || -z "${device5}" ]]; then
echo "Failed to grab 5 required devices for integration tests."
exit 1
fi

# Schedule the test run in device farm
echo "Scheduling test run"
run_arn=`aws devicefarm schedule-run --project-arn=$project_arn \
--app-arn="$app_package_upload_arn" \
--device-pool-arn=$device_pool_arn \
--device-selection-configuration='{
"filters": [
{"attribute": "ARN", "operator":"IN", "values":["'$device1'", "'$device2'", "'$device3'", "'$device4'", "'$device5'"]}
],
"maxDevices": '5'
}' \
Comment on lines +136 to +141
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

couldn't find this information online, but I'm guessing that this really does select all 5 devices and never picks from the same pool again

--name="$run_name" \
--test="type=INSTRUMENTATION,testPackageArn=$test_package_upload_arn" \
--execution-configuration="jobTimeoutMinutes=30,videoCapture=false" \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package com.amplifyframework.testutils

/*
* Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

import org.junit.rules.TestRule
import org.junit.runner.Description
import org.junit.runners.model.Statement

/**
* Rule to repeatedly run a test
* usage:
* ```
* @get:Rule
* val repeatRule = RepeatRule()
*
* @Test
* @Repeat(100)
* fun testToBeRepeated() {
* ...
* }
* ```
Comment on lines +24 to +34
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is just setup to turn this on for flaky tests in the future?

*/
class RepeatRule : TestRule {
private class RepeatStatement(
private val statement: Statement,
private val repeat: Int
) :
Statement() {
@Throws(Throwable::class)
override fun evaluate() {
for (i in 0 until repeat) {
statement.evaluate()
}
}
}

override fun apply(
statement: Statement,
description: Description
): Statement {
var result = statement
val repeat: Repeat = description.getAnnotation(Repeat::class.java) as Repeat
val times: Int = repeat.value
result = RepeatStatement(statement, times)
return result
}
}

@kotlin.annotation.Retention(AnnotationRetention.RUNTIME)
@Target(
AnnotationTarget.FUNCTION,
AnnotationTarget.PROPERTY_GETTER,
AnnotationTarget.PROPERTY_SETTER,
AnnotationTarget.ANNOTATION_CLASS
)
annotation class Repeat(val value: Int = 1)