diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc51f3f1..01bf47682 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,24 @@ env: PACKAGE_NAME: aws-iot-device-sdk-cpp-v2 LINUX_BASE_IMAGE: ubuntu-16-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 + DA_TOPIC: test/da + DA_SHADOW_PROPERTY: datest + DA_SHADOW_VALUE_SET: ON + DA_SHADOW_VALUE_DEFAULT: OFF + CI_FOLDER: "D:/a/work" + CI_UTILS_FOLDER: "./aws-iot-device-sdk-cpp-v2/utils" + CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-cpp-v2/build/samples" + CI_IOT_CONTAINERS: ${{ secrets.AWS_CI_IOT_CONTAINERS }} + CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }} + CI_CYCLEPUBSUB_ROLE: ${{ secrets.AWS_CI_CYCLEPUBSUB_ROLE }} + CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }} + CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }} + CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }} + CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} + CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} + CI_DEVICE_DEFENDER: ${{ secrets.AWS_CI_DEVICE_DEFENDER_ROLE }} + CI_SECURE_TUNNEL: ${{ secrets.AWS_CI_SECURE_TUNNEL_ROLE }} jobs: linux-compat: @@ -26,8 +41,14 @@ jobs: - manylinux2014-x64 - manylinux2014-x86 - al2-x64 - + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | @@ -50,7 +71,14 @@ jobs: - gcc-6 - gcc-7 - gcc-8 + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | @@ -59,7 +87,14 @@ jobs: byo-crypto: runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | @@ -68,7 +103,14 @@ jobs: linux-no-cpu-extensions: runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | @@ -77,61 +119,308 @@ jobs: windows: runs-on: windows-latest + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | - md D:\a\work - cd D:\a\work + md ${{ env.CI_FOLDER }} + cd ${{ env.CI_FOLDER }} python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + cd ${{ env.CI_FOLDER }} + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file '.\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\basic_pub_sub\RelWithDebInfo\basic-pub-sub.exe' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run Windows Certificate Connect sample + run: | + cd ${{ env.CI_FOLDER }} + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}\mqtt\windows_cert_connect\RelWithDebInfo\windows-cert-connect.exe" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ${{ env.CI_FOLDER }}\aws-iot-device-sdk-cpp-v2 + python ./deviceadvisor/script/DATestRun.py windows-vs14: runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0) strategy: matrix: arch: [Win32, x64] + fail-fast: false + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | - md D:\a\work - cd D:\a\work + md ${{ env.CI_FOLDER }} + cd ${{ env.CI_FOLDER }} python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-Tv140 --cmake-extra=-A${{ matrix.arch }} + - name: Running samples in CI setup + run: | + python -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + cd ${{ env.CI_FOLDER }} + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file '.\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\basic_pub_sub\RelWithDebInfo\basic-pub-sub.exe' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run Windows Certificate Connect sample + run: | + cd ${{ env.CI_FOLDER }} + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}\mqtt\windows_cert_connect\RelWithDebInfo\windows-cert-connect.exe" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ${{ env.CI_FOLDER }}\aws-iot-device-sdk-cpp-v2 + python ./deviceadvisor/script/DATestRun.py windows-no-cpu-extensions: runs-on: windows-latest + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | - md D:\a\work - cd D:\a\work + md ${{ env.CI_FOLDER }} + cd ${{ env.CI_FOLDER }} python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF - + - name: Running samples in CI setup + run: | + python -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + cd ${{ env.CI_FOLDER }} + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}\pub_sub\basic_pub_sub\RelWithDebInfo\basic-pub-sub.exe" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run Windows Certificate Connect sample + run: | + cd ${{ env.CI_FOLDER }} + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}\mqtt\windows_cert_connect\RelWithDebInfo\windows-cert-connect.exe" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ${{ env.CI_FOLDER }}\aws-iot-device-sdk-cpp-v2 + python ./deviceadvisor/script/DATestRun.py + windows-app-verifier: runs-on: windows-2022 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | - md D:\a\work - cd D:\a\work + md ${{ env.CI_FOLDER }} + cd ${{ env.CI_FOLDER }} python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF + - name: Running samples in CI setup + run: | + python -m pip install boto3 + - name: configure AWS credentials (CyclePubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_CYCLEPUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Run and check AppVerifier run: | - cd D:\a\work + cd ${{ env.CI_FOLDER }} echo "Starting to run AppVerifier with cycle pub-sub sample" - python -m pip install boto3 - python .\aws-iot-device-sdk-cpp-v2\utils\appverifier_launch_sample.py --sample_file .\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\cycle_pub_sub\RelWithDebInfo\cycle-pub-sub.exe + python ${{ env.CI_UTILS_FOLDER }}\appverifier_launch_sample.py --sample_file "${{ env.CI_SAMPLES_FOLDER }}\pub_sub\cycle_pub_sub\RelWithDebInfo\cycle-pub-sub.exe" --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/CyclePubSub/cert' --sample_secret_private_key 'ci/CyclePubSub/key' osx: runs-on: macos-latest + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + python3 ./aws-iot-device-sdk-cpp-v2/utils/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pub_sub/basic_pub_sub/basic-pub-sub" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ./aws-iot-device-sdk-cpp-v2 + python3 ./deviceadvisor/script/DATestRun.py + + # Not strictly needed, but allows us to run Device Advisor and PubSub on Linux without needing to run all samples + linux-github-actions: + runs-on: ubuntu-latest + strategy: + fail-fast: false + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pub_sub/basic_pub_sub/basic-pub-sub" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ./aws-iot-device-sdk-cpp-v2 + python3 ./deviceadvisor/script/DATestRun.py + - name: configure AWS credentials (Device Defender) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_DEFENDER }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceDefemder + run: | + cd ./aws-iot-device-sdk-cpp-v2 + python3 ./devicedefender/script/DDTestRun.py + + # Runs the samples and ensures that everything is working + linux-smoke-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: Setup C++ + run: | + sudo apt-get -qq update -y + sudo apt-get -qq install -y build-essential + sudo apt install cmake + gcc --version + cmake --version + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + echo "Downloading source" + git clone --recursive https://github.com/aws/aws-iot-device-sdk-cpp-v2.git --branch ${{ github.head_ref || github.ref_name }} + echo "Running builder" + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + sudo apt-get update -y + sudo apt-get install softhsm -y + softhsm2-util --version + - name: configure AWS credentials (Connect and PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Basic Connect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt/basic_connect/basic-connect" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run Websocket Connect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt/websocket_connect/websocket-connect" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_arguments '--signing_region us-east-1' + - name: run PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pub_sub/basic_pub_sub/basic-pub-sub" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run PKCS11 Connect sample + run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt/pkcs11_connect/pkcs11-connect" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + - name: configure AWS credentials (Custom Authorizer) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run CustomAuthorizerConnect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt/custom_authorizer_connect/custom-authorizer-connect" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password' + - name: configure AWS credentials (Shadow) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_SHADOW_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Shadow sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/shadow/shadow_sync/shadow-sync" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Shadow/cert' --sample_secret_private_key 'ci/Shadow/key' --sample_arguments '--thing_name CI_Shadow_Thing --shadow_property color --is_ci true' + - name: configure AWS credentials (Jobs) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_JOBS_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Jobs sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/jobs/describe_job_execution/describe-job-execution" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Jobs/cert' --sample_secret_private_key 'ci/Jobs/key' --sample_arguments '--thing_name CI_Jobs_Thing --job_id CI_Jobs_Thing_Job_1' + - name: configure AWS credentials (Fleet provisioning) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Fleet Provisioning sample + run: | + echo "Generating UUID for IoT thing" + Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())") + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language CPP --sample_file "${{ env.CI_SAMPLES_FOLDER }}/identity/fleet_provisioning/fleet-provisioning" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}" + python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" + - name: configure AWS credentials (Secure tunneling) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_SECURE_TUNNEL }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Secure Tunneling sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_secure_tunnel_ci.py --sample_file "${{ env.CI_SAMPLES_FOLDER }}/secure_tunneling/secure_tunnel/secure-tunnel" --sample_region ${{ env.AWS_DEFAULT_REGION }} # check that docs can still build check-docs: diff --git a/builder.json b/builder.json index ecd85dd2a..2a014ee07 100644 --- a/builder.json +++ b/builder.json @@ -8,11 +8,6 @@ "crt" ], "env": { - "DA_TOPIC": "test/da", - "DA_SHADOW_PROPERTY": "datest", - "DA_SHADOW_VALUE_SET": "ON", - "DA_SHADOW_VALUE_DEFAULT": "OFF", - "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs" }, "hosts": { "manylinux": { @@ -28,13 +23,8 @@ "build", "build-samples" ], - "test_steps": [ - "python3 -m pip install boto3", - "python3 deviceadvisor/script/DATestRun.py", - "python3 devicedefender/script/DDTestRun.py"], "variants" : { "skip_sample": { - "!test_steps": [], "!build_steps": [ "build" ] diff --git a/codebuild/samples/connect-custom-authorizer-linux.sh b/codebuild/samples/connect-custom-authorizer-linux.sh deleted file mode 100755 index d1c81c0b8..000000000 --- a/codebuild/samples/connect-custom-authorizer-linux.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e - -env - -pushd $CODEBUILD_SRC_DIR/samples/mqtt/custom_authorizer_connect - -mkdir _build -cd _build -cmake -DCMAKE_PREFIX_PATH=/tmp/install .. -make -j - -ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') -AUTH_NAME=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-name" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') -AUTH_PASSWORD=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-password" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') - -echo "Custom authorizer connect test" -./custom-authorizer-connect --endpoint $ENDPOINT --custom_auth_authorizer_name $AUTH_NAME --custom_auth_password $AUTH_PASSWORD - -popd diff --git a/codebuild/samples/connect-websocket-linux.sh b/codebuild/samples/connect-websocket-linux.sh deleted file mode 100755 index e3fbaa7f0..000000000 --- a/codebuild/samples/connect-websocket-linux.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -env - -pushd $CODEBUILD_SRC_DIR/samples/mqtt/websocket_connect - -mkdir _build -cd _build -cmake -DCMAKE_PREFIX_PATH=/tmp/install .. -make -j - -ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') - -echo "Websocket test" -./websocket-connect --endpoint $ENDPOINT --use_websocket --signing_region us-east-1 - -popd diff --git a/codebuild/samples/linux-smoke-tests.yml b/codebuild/samples/linux-smoke-tests.yml index 6a8fe3a67..087f32d76 100644 --- a/codebuild/samples/linux-smoke-tests.yml +++ b/codebuild/samples/linux-smoke-tests.yml @@ -16,10 +16,6 @@ phases: - echo Build started on `date` - $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/connect-websocket-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/securetunnel-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/connect-custom-authorizer-linux.sh post_build: commands: - echo Build completed on `date` diff --git a/codebuild/samples/pkcs11-connect-linux.sh b/codebuild/samples/pkcs11-connect-linux.sh deleted file mode 100755 index 43d1c51bb..000000000 --- a/codebuild/samples/pkcs11-connect-linux.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e -set -o pipefail - -ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') - -# from hereon commands are echoed. don't leak secrets -set -x - -softhsm2-util --version - -# SoftHSM2's default tokendir path might be invalid on this machine -# so set up a conf file that specifies a known good tokendir path -mkdir -p /tmp/tokens -export SOFTHSM2_CONF=/tmp/softhsm2.conf -echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - -# create token -softhsm2-util --init-token --free --label my-token --pin 0000 --so-pin 0000 - -# add private key to token (must be in PKCS#8 format) -openssl pkcs8 -topk8 -in /tmp/privatekey.pem -out /tmp/privatekey.p8.pem -nocrypt -softhsm2-util --import /tmp/privatekey.p8.pem --token my-token --label my-key --id BEEFCAFE --pin 0000 - -# build and run sample -pushd $CODEBUILD_SRC_DIR/samples/mqtt/pkcs11_connect - -mkdir _build -cd _build -cmake -DCMAKE_PREFIX_PATH=/tmp/install .. -make -j - -./pkcs11-connect \ - --endpoint $ENDPOINT \ - --cert /tmp/certificate.pem \ - --pkcs11_lib /usr/lib/softhsm/libsofthsm2.so \ - --pin 0000 \ - --token_label my-token \ - --key_label my-key - -popd diff --git a/codebuild/samples/securetunnel-linux.sh b/codebuild/samples/securetunnel-linux.sh deleted file mode 100755 index e6e34e58c..000000000 --- a/codebuild/samples/securetunnel-linux.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -e - -env - -pushd $CODEBUILD_SRC_DIR/samples/secure_tunneling/secure_tunnel - -mkdir _build -cd _build -cmake -DCMAKE_PREFIX_PATH=/tmp/install .. -make -j - -echo "aws iotsecuretunneling open-tunnel" -RESPONSE=$(aws iotsecuretunneling open-tunnel) -array=(${RESPONSE//:/ }) - -echo "Assigning SOURCETOKEN" -SOURCETOKEN="${array[11]}" - -echo "Assigning DESTINATIONTOKEN" -DESTINATIONTOKEN="${array[13]}" - -echo "Secure Tunnel Destination test" -./secure-tunnel --test --region us-east-1 --access_token $DESTINATIONTOKEN & -sleep 10 & - -echo "Secure Tunnel Source test" -./secure-tunnel --localProxyModeSource --region us-east-1 --access_token $SOURCETOKEN & diff --git a/deviceadvisor/script/DATestConfig.json b/deviceadvisor/script/DATestConfig.json index dacca0597..6982c6fe9 100644 --- a/deviceadvisor/script/DATestConfig.json +++ b/deviceadvisor/script/DATestConfig.json @@ -2,11 +2,11 @@ "tests" :[ "MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"], "test_suite_ids" : { - "MQTT Connect" : "ejbdzmo3hf3v", - "MQTT Publish" : "euw7favf6an4", - "MQTT Subscribe" : "01o8vo6no7sd", - "Shadow Publish" : "elztm2jebc1q", - "Shadow Update" : "vuydgrbbbfce" + "MQTT Connect" : "mxn32qkm8npn", + "MQTT Publish" : "gcjhujhhz50p", + "MQTT Subscribe" : "nyiuiwx5yxtj", + "Shadow Publish" : "fttdr8ufljnf", + "Shadow Update" : "ng9t8am2jnry" }, "test_exe_path" : { diff --git a/deviceadvisor/script/DATestRun.py b/deviceadvisor/script/DATestRun.py index 9afd85c40..2e504e033 100644 --- a/deviceadvisor/script/DATestRun.py +++ b/deviceadvisor/script/DATestRun.py @@ -28,15 +28,26 @@ def process_logs(log_group, log_stream, thing_name): logGroupName=log_group, logStreamName=log_stream ) - log_file = thing_name + ".log" + log_file = "DA_Log_CPP_" + thing_name + ".log" f = open(log_file, 'w') for event in response["events"]: f.write(event['message']) f.close() - s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file) + + try: + secrets_client = boto3.client( + "secretsmanager", region_name=os.environ["AWS_DEFAULT_REGION"]) + s3_bucket_name = secrets_client.get_secret_value(SecretId="ci/DeviceAdvisor/s3bucket")["SecretString"] + s3.Bucket(s3_bucket_name).upload_file(log_file, log_file) + print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file) + + except Exception: + print ("[Device Advisor] Error: could not store log in S3 bucket!") + os.remove(log_file) print("[Device Advisor] Issues on test " + test_name + ". Please check out the logs at "+thing_name+".log on S3.") +# Sleep for a random time def sleep_with_backoff(base, max): sleep(random.randint(base,max)) @@ -44,14 +55,23 @@ def sleep_with_backoff(base, max): ############################################## # Initialize variables # create aws clients -client = boto3.client('iot') -dataClient = boto3.client('iot-data') -deviceAdvisor = boto3.client('iotdeviceadvisor') -s3 = boto3.resource('s3') +try: + client = boto3.client('iot') + dataClient = boto3.client('iot-data') + deviceAdvisor = boto3.client('iotdeviceadvisor') + s3 = boto3.resource('s3') +except Exception: + print ("[Device Advisor] Error: could not create boto3 clients") + exit(-1) # const BACKOFF_BASE = 5 BACKOFF_MAX = 10 +# 60 minutes divided by the maximum back-off = longest time a DA run can last with this script. +MAXIMUM_CYCLE_COUNT = (3600 / BACKOFF_MAX) + +# Did Device Advisor fail a test? If so, this should be true +did_at_least_one_test_fail = False # load test config f = open('deviceadvisor/script/DATestConfig.json') @@ -80,21 +100,21 @@ def sleep_with_backoff(base, max): # 'thingArn': 'string', # 'thingId': 'string' # } - print("[Device Advisor]Info: Started to create thing...") + print("[Device Advisor] Info: Started to create thing...") create_thing_response = client.create_thing( thingName=thing_name ) os.environ["DA_THING_NAME"] = thing_name - except Exception as e: - print("[Device Advisor]Error: Failed to create thing: " + thing_name) + except Exception: + print("[Device Advisor] Error: Failed to create thing: " + thing_name) exit(-1) ############################################## # create certificate and keys used for testing try: - print("[Device Advisor]Info: Started to create certificate...") + print("[Device Advisor] Info: Started to create certificate...") # create_cert_response: # { # 'certificateArn': 'string', @@ -123,25 +143,44 @@ def sleep_with_backoff(base, max): os.environ["DA_CERTI"] = certificate_path os.environ["DA_KEY"] = key_path - except: - client.delete_thing(thingName = thing_name) - print("[Device Advisor]Error: Failed to create certificate.") + except Exception: + try: + client.delete_thing(thingName = thing_name) + except Exception: + print ("[Device Advisor] Error: Could not delete thing.") + print("[Device Advisor] Error: Failed to create certificate.") + exit(-1) + + certificate_arn = create_cert_response['certificateArn'] + certificate_id = create_cert_response['certificateId'] + + ############################################## + # attach policy to certificate + try: + secrets_client = boto3.client( + "secretsmanager", region_name=os.environ["AWS_DEFAULT_REGION"]) + policy_name = secrets_client.get_secret_value(SecretId="ci/DeviceAdvisor/policy_name")["SecretString"] + client.attach_policy ( + policyName= policy_name, + target = certificate_arn + ) + except Exception as ex: + print (ex) + delete_thing_with_certi(thing_name, certificate_id, certificate_arn ) + print("[Device Advisor] Error: Failed to attach policy.") exit(-1) ############################################## # attach certification to thing try: - print("[Device Advisor]Info: Attach certificate to test thing...") + print("[Device Advisor] Info: Attach certificate to test thing...") # attache the certificate to thing client.attach_thing_principal( thingName = thing_name, principal = create_cert_response['certificateArn'] ) - certificate_arn = create_cert_response['certificateArn'] - certificate_id = create_cert_response['certificateId'] - - except: + except Exception: delete_thing_with_certi(thing_name, certificate_id ,certificate_arn ) print("[Device Advisor]Error: Failed to attach certificate.") exit(-1) @@ -153,6 +192,7 @@ def sleep_with_backoff(base, max): ###################################### # set default shadow, for shadow update, if the # shadow does not exists, update will fail + print("[Device Advisor] Info: About to update shadow.") payload_shadow = json.dumps( { "state": { @@ -169,6 +209,7 @@ def sleep_with_backoff(base, max): payload = payload_shadow) get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name) # make sure shadow is created before we go to next step + print("[Device Advisor] Info: About to wait for shadow update.") while(get_shadow_response is None): get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name) @@ -179,24 +220,31 @@ def sleep_with_backoff(base, max): # 'suiteRunArn': 'string', # 'createdAt': datetime(2015, 1, 1) # } - print("[Device Advisor]Info: Start device advisor test: " + test_name) + print("[Device Advisor] Info: Start device advisor test: " + test_name) sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_start_response = deviceAdvisor.start_suite_run( - suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], - suiteRunConfiguration={ - 'primaryDevice': { - 'thingArn': create_thing_response['thingArn'], - }, - 'parallelRun': True + suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], + suiteRunConfiguration={ + 'primaryDevice': { + 'thingArn': create_thing_response['thingArn'], + }, + 'parallelRun': True }) # get DA endpoint + print("[Device Advisor] Info: Getting Device Advisor endpoint.") endpoint_response = deviceAdvisor.get_endpoint( thingArn = create_thing_response['thingArn'] ) os.environ['DA_ENDPOINT'] = endpoint_response['endpoint'] + cycle_number = 0 while True: + cycle_number += 1 + if (cycle_number >= MAXIMUM_CYCLE_COUNT): + print(f"[Device Advisor] Error: {cycle_number} of cycles lasting {BACKOFF_BASE} to {BACKOFF_MAX} seconds have passed.") + raise Exception(f"ERROR - {cycle_number} of cycles lasting {BACKOFF_BASE} to {BACKOFF_MAX} seconds have passed.") + # sleep for 1s every loop to avoid TooManyRequestsException sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_result_responds = deviceAdvisor.get_suite_run( @@ -214,6 +262,7 @@ def sleep_with_backoff(base, max): elif (test_result_responds['status'] == 'RUNNING' and test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'RUNNING'): try: + print ("[Device Advisor] Info: About to get start Device Advisor companion test application.") exe_path = os.path.join("build/deviceadvisor/tests/",DATestConfig['test_exe_path'][test_name]) # Windows and MAC/LINUX has a different build folder structure if platform.system() == 'Windows': @@ -230,6 +279,7 @@ def sleep_with_backoff(base, max): test_result[test_name] = test_result_responds['status'] # If the test failed, upload the logs to S3 before clean up if(test_result[test_name] != "PASS"): + print ("[Device Advisor] Info: About to upload log to S3.") log_url = test_result_responds['testResult']['groups'][0]['tests'][0]['logUrl'] group_string = re.search('group=(.*);', log_url) log_group = group_string.group(1) @@ -238,9 +288,11 @@ def sleep_with_backoff(base, max): process_logs(log_group, log_stream, thing_name) delete_thing_with_certi(thing_name, certificate_id ,certificate_arn ) break - except Exception as e: + except Exception: delete_thing_with_certi(thing_name, certificate_id ,certificate_arn ) print("[Device Advisor]Error: Failed to test: "+ test_name) + did_at_least_one_test_fail = True + sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) ############################################## # print result and cleanup things @@ -255,4 +307,8 @@ def sleep_with_backoff(base, max): # if the test failed, we dont clean the Thing so that we can track the error exit(-1) +if (did_at_least_one_test_fail == True): + print("[Device Advisor] At least one test failed!") + exit(-1) + exit(0) diff --git a/deviceadvisor/tests/mqtt_connect/main.cpp b/deviceadvisor/tests/mqtt_connect/main.cpp index 57c98421f..c2e72f82c 100644 --- a/deviceadvisor/tests/mqtt_connect/main.cpp +++ b/deviceadvisor/tests/mqtt_connect/main.cpp @@ -25,7 +25,6 @@ int main() String clientId(String("test-") + Aws::Crt::UUID().ToString()); /*********************** Parse Arguments ***************************/ - DeviceAdvisorEnvironment daVars; if (!daVars.init(TestType::CONNECT)) { @@ -48,7 +47,6 @@ int main() /* * Setup up mqttClients */ - Aws::Iot::MqttClient mqttClient; if (!mqttClient) { diff --git a/deviceadvisor/tests/mqtt_publish/main.cpp b/deviceadvisor/tests/mqtt_publish/main.cpp index e7dc90948..1ef9dbfc5 100644 --- a/deviceadvisor/tests/mqtt_publish/main.cpp +++ b/deviceadvisor/tests/mqtt_publish/main.cpp @@ -35,7 +35,6 @@ int main() /* * Setup client configuration with the MqttClientConnectionConfigBuilder. */ - Aws::Iot::MqttClientConnectionConfigBuilder builder = Aws::Iot::MqttClientConnectionConfigBuilder(daVars.certificatePath.c_str(), daVars.keyPath.c_str()); builder.WithEndpoint(daVars.endpoint); @@ -48,7 +47,6 @@ int main() /* * Setup up mqttClients */ - Aws::Iot::MqttClient mqttClient; if (!mqttClient) { diff --git a/deviceadvisor/tests/mqtt_subscribe/main.cpp b/deviceadvisor/tests/mqtt_subscribe/main.cpp index 7e02c6670..5259a828b 100644 --- a/deviceadvisor/tests/mqtt_subscribe/main.cpp +++ b/deviceadvisor/tests/mqtt_subscribe/main.cpp @@ -35,7 +35,6 @@ int main() /* * Setup client configuration with the MqttClientConnectionConfigBuilder. */ - Aws::Iot::MqttClientConnectionConfigBuilder builder = Aws::Iot::MqttClientConnectionConfigBuilder(daVars.certificatePath.c_str(), daVars.keyPath.c_str()); builder.WithEndpoint(daVars.endpoint); @@ -48,7 +47,6 @@ int main() /* * Setup up mqttClients */ - Aws::Iot::MqttClient mqttClient; if (!mqttClient) { diff --git a/deviceadvisor/tests/shadow_update/main.cpp b/deviceadvisor/tests/shadow_update/main.cpp index fd118a1d1..1b03a6b72 100644 --- a/deviceadvisor/tests/shadow_update/main.cpp +++ b/deviceadvisor/tests/shadow_update/main.cpp @@ -49,7 +49,6 @@ int main() /* * Setup client configuration with the MqttClientConnectionConfigBuilder. */ - Aws::Iot::MqttClientConnectionConfigBuilder builder = Aws::Iot::MqttClientConnectionConfigBuilder(daEnv.certificatePath.c_str(), daEnv.keyPath.c_str()); builder.WithEndpoint(daEnv.endpoint); @@ -62,7 +61,6 @@ int main() /* * Setup up mqttClients */ - Aws::Iot::MqttClient mqttClient; if (!mqttClient) { diff --git a/devicedefender/script/DDTestRun.py b/devicedefender/script/DDTestRun.py index 685c5a9ca..601cd88a1 100644 --- a/devicedefender/script/DDTestRun.py +++ b/devicedefender/script/DDTestRun.py @@ -1,10 +1,8 @@ - import boto3 import uuid import os import subprocess import platform -from time import sleep # On something other than Linux? Pass the test instantly since Device Defender is only supported on Linux if platform.system() != "Linux": @@ -54,7 +52,6 @@ def delete_thing_with_certi(thingName, certiId, certiArn): create_thing_response = client.create_thing( thingName=thing_name ) - #print(create_thing_response) thing_arn = create_thing_response["thingArn"] client_made_thing = True @@ -147,6 +144,8 @@ def delete_thing_with_certi(thingName, certiId, certiArn): ############################################## # attach certification to thing +certificate_id = None +certificate_arn = None try: print("[Device Defender]Info: Attach policy to certificate...") # attach policy to thing @@ -170,10 +169,10 @@ def delete_thing_with_certi(thingName, certiId, certiArn): delete_thing_with_certi(thing_name, certificate_id, certificate_arn) else: client.delete_thing(thingName=thing_name) - + if client_made_policy: client.delete_policy(policyName=thing_name + "_policy") - + print("[Device Defender]Error: Failed to attach certificate.") exit(-1) diff --git a/samples/README.md b/samples/README.md index 3b44b3c3e..634d5c333 100644 --- a/samples/README.md +++ b/samples/README.md @@ -552,50 +552,42 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- { "Version": "2012-10-17", "Statement": [ - { - "Effect": "Allow", - "Action": "iot:Connect", - "Resource": [ - "arn:aws:iot:region:account:client/thingname", - "arn:aws:iot:region:account:client/test-*" - ] - }, { "Effect": "Allow", "Action": "iot:Publish", "Resource": [ - "arn:aws:iot:region:account:topic/test/dc/pubtopic", - "arn:aws:iot:region:account:topic/$aws/events/job/*", - "arn:aws:iot:region:account:topic/$aws/events/jobExecution/*", - "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*" + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/get", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/get" ] }, { "Effect": "Allow", - "Action": "iot:Subscribe", + "Action": "iot:Receive", "Resource": [ - "arn:aws:iot:region:account:topicfilter/test/dc/subtopic", - "arn:aws:iot:region:account:topic/$aws/events/jobExecution/*", - "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*" + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/notify-next", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next/*", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update/*", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/get/*", + "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/get/*" ] }, { "Effect": "Allow", - "Action": "iot:Receive", + "Action": "iot:Subscribe", "Resource": [ - "arn:aws:iot:region:account:topic/test/dc/subtopic", - "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*" + "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/notify-next", + "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/start-next/*", + "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/update/*", + "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/get/*", + "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/get/*" ] }, { "Effect": "Allow", - "Action": [ - "iot:DescribeJobExecution", - "iot:GetPendingJobExecutions", - "iot:StartNextPendingJobExecution", - "iot:UpdateJobExecution" - ], - "Resource": "arn:aws:iot:region:account:topic/$aws/things/thingname" + "Action": "iot:Connect", + "Resource": "arn:aws:iot:region:account:client/test-*" } ] } @@ -618,7 +610,7 @@ Note that if you get a `Service Error 4 occurred` error, you may have incorrectl This sample uses the AWS IoT [Fleet provisioning](https://docs.aws.amazon.com/iot/latest/developerguide/provision-wo-cert.html) -to provision devices using either a CSR or KeysAndcertificate and subsequently calls RegisterThing. +to provision devices using either a CSR or Keys-And-Certificate and subsequently calls RegisterThing. On startup, the script subscribes to topics based on the request type of either CSR or Keys topics, publishes the request to corresponding topic and calls RegisterThing. @@ -651,9 +643,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- "Statement": [ { "Effect": "Allow", - "Action": [ - "iot:Publish" - ], + "Action": "iot:Publish", "Resource": [ "arn:aws:iot:region:account:topic/$aws/certificates/create/json", "arn:aws:iot:region:account:topic/$aws/certificates/create-from-csr/json", @@ -663,8 +653,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- { "Effect": "Allow", "Action": [ - "iot:Receive", - "iot:Subscribe" + "iot:Receive" ], "Resource": [ "arn:aws:iot:region:account:topic/$aws/certificates/create/json/accepted", @@ -675,6 +664,20 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- "arn:aws:iot:region:account:topic/$aws/provisioning-templates/templatename/provision/json/rejected" ] }, + { + "Effect": "Allow", + "Action": [ + "iot:Subscribe" + ], + "Resource": [ + "arn:aws:iot:region:account:topicfilter/$aws/certificates/create/json/accepted", + "arn:aws:iot:region:account:topicfilter/$aws/certificates/create/json/rejected", + "arn:aws:iot:region:account:topicfilter/$aws/certificates/create-from-csr/json/accepted", + "arn:aws:iot:region:account:topicfilter/$aws/certificates/create-from-csr/json/rejected", + "arn:aws:iot:region:account:topicfilter/$aws/provisioning-templates/templatename/provision/json/accepted", + "arn:aws:iot:region:account:topicfilter/$aws/provisioning-templates/templatename/provision/json/rejected" + ] + }, { "Effect": "Allow", "Action": "iot:Connect", @@ -719,9 +722,103 @@ aws iot create-provisioning-template \ --enabled ``` The rest of the instructions assume you have used the following for the template body: +
+(see template body) ``` sh -{\"Parameters\":{\"DeviceLocation\":{\"Type\":\"String\"},\"AWS::IoT::Certificate::Id\":{\"Type\":\"String\"},\"SerialNumber\":{\"Type\":\"String\"}},\"Mappings\":{\"LocationTable\":{\"Seattle\":{\"LocationUrl\":\"https://example.aws\"}}},\"Resources\":{\"thing\":{\"Type\":\"AWS::IoT::Thing\",\"Properties\":{\"ThingName\":{\"Fn::Join\":[\"\",[\"ThingPrefix_\",{\"Ref\":\"SerialNumber\"}]]},\"AttributePayload\":{\"version\":\"v1\",\"serialNumber\":\"serialNumber\"}},\"OverrideSettings\":{\"AttributePayload\":\"MERGE\",\"ThingTypeName\":\"REPLACE\",\"ThingGroups\":\"DO_NOTHING\"}},\"certificate\":{\"Type\":\"AWS::IoT::Certificate\",\"Properties\":{\"CertificateId\":{\"Ref\":\"AWS::IoT::Certificate::Id\"},\"Status\":\"Active\"},\"OverrideSettings\":{\"Status\":\"REPLACE\"}},\"policy\":{\"Type\":\"AWS::IoT::Policy\",\"Properties\":{\"PolicyDocument\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iot:Connect\",\"iot:Subscribe\",\"iot:Publish\",\"iot:Receive\"],\"Resource\":\"*\"}]}}}},\"DeviceConfiguration\":{\"FallbackUrl\":\"https://www.example.com/test-site\",\"LocationUrl\":{\"Fn::FindInMap\":[\"LocationTable\",{\"Ref\":\"DeviceLocation\"},\"LocationUrl\"]}}} +{ + "Parameters": { + "DeviceLocation": { + "Type": "String" + }, + "AWS::IoT::Certificate::Id": { + "Type": "String" + }, + "SerialNumber": { + "Type": "String" + } + }, + "Mappings": { + "LocationTable": { + "Seattle": { + "LocationUrl": "https://example.aws" + } + } + }, + "Resources": { + "thing": { + "Type": "AWS::IoT::Thing", + "Properties": { + "ThingName": { + "Fn::Join": [ + "", + [ + "ThingPrefix_", + { + "Ref": "SerialNumber" + } + ] + ] + }, + "AttributePayload": { + "version": "v1", + "serialNumber": "serialNumber" + } + }, + "OverrideSettings": { + "AttributePayload": "MERGE", + "ThingTypeName": "REPLACE", + "ThingGroups": "DO_NOTHING" + } + }, + "certificate": { + "Type": "AWS::IoT::Certificate", + "Properties": { + "CertificateId": { + "Ref": "AWS::IoT::Certificate::Id" + }, + "Status": "Active" + }, + "OverrideSettings": { + "Status": "REPLACE" + } + }, + "policy": { + "Type": "AWS::IoT::Policy", + "Properties": { + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iot:Connect", + "iot:Subscribe", + "iot:Publish", + "iot:Receive" + ], + "Resource": "*" + } + ] + } + } + } + }, + "DeviceConfiguration": { + "FallbackUrl": "https://www.example.com/test-site", + "LocationUrl": { + "Fn::FindInMap": [ + "LocationTable", + { + "Ref": "DeviceLocation" + }, + "LocationUrl" + ] + } + } +} ``` +
+ If you use a different body, you may need to pass in different template parameters. #### Running the sample and provisioning using a certificate-key set from a provisioning claim @@ -811,7 +908,7 @@ Provide the necessary arguments along with the destination access token and star ``` sh ./secure_tunnel --endpoint --ca_file --cert --key ---thing_name --region --access_token_file +--thing_name --region --access_token_file ``` Provide the necessary arguments along with the source access token and start a second sample in source mode by using the flag --localProxyModeSource. @@ -819,7 +916,7 @@ Provide the necessary arguments along with the source access token and start a s ``` sh ./secure_tunnel --endpoint --ca_file --cert --key ---thing_name --region --access_token_file +--thing_name --region --access_token_file --localProxyModeSource ``` @@ -898,7 +995,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- { "Effect": "Allow", "Action": "iot:Connect", - "Resource": "arn:aws:iot:region:account:client/*" + "Resource": "arn:aws:iot:region:account:client-*" } ] } @@ -935,7 +1032,8 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- ], "Resource": [ "arn:aws:iot:region:account:topic/test/shared_topic", - "arn:aws:iot:region:account:topic/test-client-*" + "arn:aws:iot:region:account:topic/test-*", + "arn:aws:iot:region:account:topic/topic1" ] }, { @@ -946,7 +1044,8 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- ], "Resource": [ "arn:aws:iot:region:account:topicfilter/test/shared_topic", - "arn:aws:iot:region:account:topic/test-client-*" + "arn:aws:iot:region:account:topicfilter/test-*", + "arn:aws:iot:region:account:topicfilter/topic1" ] }, { @@ -955,7 +1054,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- "iot:Connect" ], "Resource": [ - "arn:aws:iot:region:account:client/test-client-*" + "arn:aws:iot:region:account:client/test-*" ] } ] diff --git a/samples/device_defender/basic_report/main.cpp b/samples/device_defender/basic_report/main.cpp index fb8225a9e..037ed4fac 100644 --- a/samples/device_defender/basic_report/main.cpp +++ b/samples/device_defender/basic_report/main.cpp @@ -27,8 +27,6 @@ using namespace Aws::Crt; int s_getCustomMetricNumber(double *output) { - /** Set to a random number between -50 and 50 */ - //*output = (double)((rand() % 100 + 1) - 50); *output = 100; return AWS_OP_SUCCESS; } diff --git a/samples/greengrass/basic_discovery/main.cpp b/samples/greengrass/basic_discovery/main.cpp index 098594f86..1c5d69f45 100644 --- a/samples/greengrass/basic_discovery/main.cpp +++ b/samples/greengrass/basic_discovery/main.cpp @@ -65,7 +65,7 @@ int main(int argc, char *argv[]) } /* - * We're using Mutual TLS for Mqtt, so we need to load our client certificates + * We're using Mutual TLS for MQTT, so we need to load our client certificates */ Io::TlsContextOptions tlsCtxOptions = Io::TlsContextOptions::InitClientWithMtls(certificatePath.c_str(), keyPath.c_str()); @@ -119,7 +119,7 @@ int main(int argc, char *argv[]) { proxyOptions.HostName = proxyHost; proxyOptions.Port = proxyPort; - clientConfig.ProxyOptions = proxyOptions; // + clientConfig.ProxyOptions = proxyOptions; } auto discoveryClient = DiscoveryClient::CreateClient(clientConfig); diff --git a/samples/greengrass/ipc/main.cpp b/samples/greengrass/ipc/main.cpp index d9683ad56..c52a75aa0 100644 --- a/samples/greengrass/ipc/main.cpp +++ b/samples/greengrass/ipc/main.cpp @@ -130,15 +130,6 @@ int main(int argc, char *argv[]) } auto subscribeResultFuture = subscribeOperation->GetResult(); - /* - // To avoid throwing exceptions, wait on the result for a specified timeout: - if (subscribeResultFuture.wait_for(std::chrono::seconds(10)) == std::future_status::timeout) - { - fprintf(stderr, "Timed out while waiting for response from Greengrass Core\n"); - exit(-1); - } - */ - auto subscribeResult = subscribeResultFuture.get(); if (subscribeResult) { @@ -188,14 +179,6 @@ int main(int argc, char *argv[]) } auto publishResultFuture = publishOperation->GetResult(); - /* - // To avoid throwing exceptions, wait on the result for a specified timeout: - if (publishResultFuture.wait_for(std::chrono::seconds(10)) == std::future_status::timeout) - { - fprintf(stderr, "Timed out while waiting for response from Greengrass Core\n"); - exit(-1); - } - */ auto publishResult = publishResultFuture.get(); if (publishResult) diff --git a/samples/identity/fleet_provisioning/main.cpp b/samples/identity/fleet_provisioning/main.cpp index ba65d776e..fa84c41f7 100644 --- a/samples/identity/fleet_provisioning/main.cpp +++ b/samples/identity/fleet_provisioning/main.cpp @@ -98,7 +98,7 @@ int main(int argc, char *argv[]) std::promise connectionClosedPromise; /* - * This will execute when an mqtt connect has completed or failed. + * This will execute when an MQTT connect has completed or failed. */ auto onConnectionCompleted = [&](Mqtt::MqttConnection &, int errorCode, Mqtt::ReturnCode returnCode, bool) { if (errorCode) diff --git a/samples/mqtt/websocket_connect/main.cpp b/samples/mqtt/websocket_connect/main.cpp index 7af6a1673..7ab11d712 100644 --- a/samples/mqtt/websocket_connect/main.cpp +++ b/samples/mqtt/websocket_connect/main.cpp @@ -25,7 +25,6 @@ int main(int argc, char *argv[]) * Do the global initialization for the API. */ ApiHandle apiHandle; - // uint16_t proxyPort(8080); /*********************** Parse Arguments ***************************/ Utils::CommandLineUtils cmdUtils = Utils::CommandLineUtils(); diff --git a/samples/mqtt/windows_cert_connect/CMakeLists.txt b/samples/mqtt/windows_cert_connect/CMakeLists.txt index 6358611cd..551a5dcc1 100644 --- a/samples/mqtt/windows_cert_connect/CMakeLists.txt +++ b/samples/mqtt/windows_cert_connect/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.1) # note: cxx-17 requires cmake 3.8, cxx-20 requires cmake 3.12 -project(windows-cert-pub-sub CXX) +project(windows-cert-connect CXX) file(GLOB SRC_FILES "*.cpp" diff --git a/samples/pub_sub/basic_pub_sub/main.cpp b/samples/pub_sub/basic_pub_sub/main.cpp index ed16418be..ccb6f8f62 100644 --- a/samples/pub_sub/basic_pub_sub/main.cpp +++ b/samples/pub_sub/basic_pub_sub/main.cpp @@ -69,7 +69,7 @@ int main(int argc, char *argv[]) std::promise connectionClosedPromise; /* - * This will execute when an mqtt connect has completed or failed. + * This will execute when an MQTT connect has completed or failed. */ auto onConnectionCompleted = [&](Mqtt::MqttConnection &, int errorCode, Mqtt::ReturnCode returnCode, bool) { if (errorCode) diff --git a/samples/pub_sub/cycle_pub_sub/main.cpp b/samples/pub_sub/cycle_pub_sub/main.cpp index 06d8c20dd..488996203 100644 --- a/samples/pub_sub/cycle_pub_sub/main.cpp +++ b/samples/pub_sub/cycle_pub_sub/main.cpp @@ -143,7 +143,7 @@ enum OPERATIONS /** * The null operation - does nothing. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationNull(CycleClient *current_client, int index) @@ -156,7 +156,7 @@ void operationNull(CycleClient *current_client, int index) /** * The start operation will connect the passed-in client, if it is not already connected. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationStart(CycleClient *current_client, int index) @@ -197,7 +197,7 @@ void operationStart(CycleClient *current_client, int index) /** * The stop operation will disconnect the passed-in client, if it is not already disconnected. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationStop(CycleClient *current_client, int index) @@ -235,7 +235,7 @@ void operationStop(CycleClient *current_client, int index) * The subscribe operation will subscribe to the shared topic and the client Id * topic of the passed-in client, if not already subscribed. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationSubscribe(CycleClient *current_client, int index) @@ -301,7 +301,7 @@ void operationSubscribe(CycleClient *current_client, int index) * The unsubscribe operation will unsubscribe to the shared topic and the client Id * topic of the passed-in client, if not already unsubscribed. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationUnsubscribe(CycleClient *current_client, int index) @@ -338,7 +338,7 @@ void operationUnsubscribe(CycleClient *current_client, int index) /** * The publish operation will publish a payload to the passed-in topic using the passed-in QOS. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed * @param QOS The QOS to use with the publish * @param topic The topic to publish the message to @@ -364,7 +364,7 @@ void operationPublish(CycleClient *current_client, int index, Aws::Crt::Mqtt::QO /** * The publish QOS 0 operation will publish a QOS 0 message to the "topic1" topic * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationPublishQOS0(CycleClient *current_client, int index) @@ -375,7 +375,7 @@ void operationPublishQOS0(CycleClient *current_client, int index) /** * The publish QOS 1 operation will publish a QOS 1 message to the "topic1" topic * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationPublishQOS1(CycleClient *current_client, int index) @@ -387,7 +387,7 @@ void operationPublishQOS1(CycleClient *current_client, int index) * The publish to subscribed QOS 0 operation will publish a QOS 0 message to the * client Id topic. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationPublishToSubscribedTopicQOS0(CycleClient *current_client, int index) @@ -399,7 +399,7 @@ void operationPublishToSubscribedTopicQOS0(CycleClient *current_client, int inde * The publish to subscribed QOS 1 operation will publish a QOS 1 message to the * client Id topic. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationPublishToSubscribedTopicQOS1(CycleClient *current_client, int index) @@ -411,7 +411,7 @@ void operationPublishToSubscribedTopicQOS1(CycleClient *current_client, int inde * The publish to shared QOS 0 operation will publish a QOS 0 message to the * shared topic that all clients can be subscribed to. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationPublishToSharedTopicQOS0(CycleClient *current_client, int index) @@ -423,7 +423,7 @@ void operationPublishToSharedTopicQOS0(CycleClient *current_client, int index) * The publish to shared QOS 1 operation will publish a QOS 1 message to the * shared topic that all clients can be subscribed to. * - * @param current_client The client to operform the operation on + * @param current_client The client to perform the operation on * @param index The index of the client passed */ void operationPublishToSharedTopicQOS1(CycleClient *current_client, int index) diff --git a/samples/secure_tunneling/secure_tunnel/main.cpp b/samples/secure_tunneling/secure_tunnel/main.cpp index 2c6da45ea..dd133cc1d 100644 --- a/samples/secure_tunneling/secure_tunnel/main.cpp +++ b/samples/secure_tunneling/secure_tunnel/main.cpp @@ -203,7 +203,7 @@ int main(int argc, char *argv[]) if (isTest) { expectedMessageCount--; - if (expectedMessageCount == 0) + if (expectedMessageCount <= 0) { exit(0); } @@ -307,7 +307,7 @@ int main(int argc, char *argv[]) if (connectionCompletedPromise.get_future().get()) { - while (1) + while (true) { std::this_thread::sleep_for(3000ms); diff --git a/samples/shadow/shadow_sync/main.cpp b/samples/shadow/shadow_sync/main.cpp index f8d982bd5..0d9675829 100644 --- a/samples/shadow/shadow_sync/main.cpp +++ b/samples/shadow/shadow_sync/main.cpp @@ -103,6 +103,8 @@ int main(int argc, char *argv[]) cmdUtils.RegisterCommand("cert", "", "Path to your client certificate in PEM format."); cmdUtils.RegisterCommand("thing_name", "", "The name of your IOT thing."); cmdUtils.RegisterCommand("shadow_property", "", "The name of the shadow property you want to change."); + cmdUtils.RegisterCommand( + "is_ci", "", "If present the sample will run in CI mode (will publish to shadow automatically)."); cmdUtils.AddLoggingCommands(); const char **const_argv = (const char **)argv; cmdUtils.SendArguments(const_argv, const_argv + argc); @@ -110,6 +112,7 @@ int main(int argc, char *argv[]) String thingName = cmdUtils.GetCommandRequired("thing_name"); String shadowProperty = cmdUtils.GetCommandRequired("shadow_property"); + bool isCI = cmdUtils.HasCommand("is_ci"); /* Get a MQTT client connection from the command parser */ auto connection = cmdUtils.BuildMQTTConnection(); @@ -143,7 +146,7 @@ int main(int argc, char *argv[]) auto onDisconnect = [&](Mqtt::MqttConnection & /*conn*/) { { fprintf(stdout, "Disconnect completed\n"); - connectionCompletedPromise.set_value(true); + connectionClosedPromise.set_value(); } }; @@ -258,7 +261,11 @@ int main(int argc, char *argv[]) fprintf(stdout, "Finished clearing shadow properties\n"); currentShadowValue = ""; } - fprintf(stdout, "Enter Desired state of %s:\n", shadowProperty.c_str()); + + if (isCI == false) + { + fprintf(stdout, "Enter Desired state of %s:\n", shadowProperty.c_str()); + } }; auto onUpdateShadowRejected = [&](ErrorResponse *error, int ioErr) { @@ -412,29 +419,47 @@ int main(int argc, char *argv[]) gotInitialShadowPromise.get_future().wait(); // ==================== Shadow change value input loop ==================== - // This section is to getting user input and changing the shadow value pased on that input. + // This section is to getting user input and changing the shadow value passed to that input. + // If in CI, then input is automatically passed - fprintf(stdout, "Enter Desired state of %s:\n", shadowProperty.c_str()); - while (true) + if (isCI == false) { - String input; - std::cin >> input; - - if (input == "exit" || input == "quit") + fprintf(stdout, "Enter Desired state of %s:\n", shadowProperty.c_str()); + while (true) { - fprintf(stdout, "Exiting..."); - break; - } + String input; + std::cin >> input; - if (input == currentShadowValue) - { - fprintf(stdout, "Shadow is already set to \"%s\"\n", currentShadowValue.c_str()); - fprintf(stdout, "Enter Desired state of %s:\n", shadowProperty.c_str()); + if (input == "exit" || input == "quit") + { + fprintf(stdout, "Exiting..."); + break; + } + + if (input == currentShadowValue) + { + fprintf(stdout, "Shadow is already set to \"%s\"\n", currentShadowValue.c_str()); + fprintf(stdout, "Enter Desired state of %s:\n", shadowProperty.c_str()); + } + else + { + s_changeShadowValue(shadowClient, thingName, shadowProperty, input); + } } - else + } + else + { + int messagesSent = 0; + while (messagesSent < 5) { + String input = "Shadow_Value_"; + input.append(std::to_string(messagesSent).c_str()); s_changeShadowValue(shadowClient, thingName, shadowProperty, input); + // Sleep so there is a gap between shadow updates + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + messagesSent += 1; } + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } } diff --git a/utils/delete_iot_thing_ci.py b/utils/delete_iot_thing_ci.py new file mode 100644 index 000000000..b3d30a358 --- /dev/null +++ b/utils/delete_iot_thing_ci.py @@ -0,0 +1,67 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +# Built-in +import argparse +import sys +# Needs to be installed via pip +import boto3 # - for launching sample + +def DeleteIoTThing(parsed_commands): + try: + iot_client = boto3.client('iot', region_name=parsed_commands.region) + except Exception: + print("Error - could not make Boto3 client. Credentials likely could not be sourced") + return -1 + + thing_principals = None + try: + thing_principals = iot_client.list_thing_principals(thingName=parsed_commands.thing_name) + except Exception: + print ("Could not get thing principals!") + return -1 + + try: + if (thing_principals != None): + if (thing_principals["principals"] != None): + if (len(thing_principals["principals"]) > 0 and parsed_commands.delete_certificate == "true"): + for principal in thing_principals["principals"]: + certificate_id = principal.split("/")[1] + iot_client.detach_thing_principal(thingName=parsed_commands.thing_name, principal=principal) + iot_client.update_certificate(certificateId=certificate_id, newStatus ='INACTIVE') + iot_client.delete_certificate(certificateId=certificate_id, forceDelete=True) + except Exception as exception: + print (exception) + print ("Could not delete certificate!") + return -1 + + try: + iot_client.delete_thing(thingName=parsed_commands.thing_name) + except Exception as exception: + print (exception) + print ("Could not delete IoT thing!") + return -1 + + print ("IoT thing deleted successfully") + return 0 + + + +def main(): + argument_parser = argparse.ArgumentParser( + description="Delete IoT Thing") + argument_parser.add_argument("--thing_name", metavar="", required=True, + help="The name of the IoT thing to delete") + argument_parser.add_argument("--region", metavar="", + required=True, default="us-east-1", help="The name of the region to use") + argument_parser.add_argument("--delete_certificate", metavar="", + required=False, default="true", help="Will delete the certificate after detaching it from the IoT thing") + parsed_commands = argument_parser.parse_args() + + print ("Deleting IoT thing...") + delete_result = DeleteIoTThing(parsed_commands) + sys.exit(delete_result) + + +if __name__ == "__main__": + main() diff --git a/utils/run_sample_ci.py b/utils/run_sample_ci.py new file mode 100644 index 000000000..34d389290 --- /dev/null +++ b/utils/run_sample_ci.py @@ -0,0 +1,348 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +# Built-in +import argparse +import os +import subprocess +import pathlib +import sys +# Needs to be installed via pip +import boto3 # - for launching sample + +current_folder = os.path.dirname(pathlib.Path(__file__).resolve()) +if sys.platform == "win32" or sys.platform == "cygwin": + current_folder += "\\" +else: + current_folder += "/" +tmp_certificate_file_path = str(current_folder) + "tmp_certificate.pem" +tmp_private_key_path = str(current_folder) + "tmp_privatekey.pem.key" +tmp_pfx_file_path = str(current_folder) + "tmp_pfx_certificate.pfx" +tmp_pfx_certificate_path = "" +tmp_pfx_certificate_store_location = "CurrentUser\\My" +tmp_pfx_password = "" # Setting a password causes issues, but an empty string is valid so we use that + + +def get_secrets_and_launch(parsed_commands): + global tmp_certificate_file_path + global tmp_private_key_path + global tmp_pfx_file_path + global tmp_pfx_certificate_path + exit_code = 0 + sample_endpoint = "" + sample_certificate = "" + sample_private_key = "" + sample_custom_authorizer_name = "" + sample_custom_authorizer_password = "" + + print("Attempting to get credentials from secrets using Boto3...") + secrets_client = boto3.client( + "secretsmanager", region_name=parsed_commands.sample_region) + try: + if (parsed_commands.sample_secret_endpoint != ""): + sample_endpoint = secrets_client.get_secret_value( + SecretId=parsed_commands.sample_secret_endpoint)["SecretString"] + if (parsed_commands.sample_secret_certificate != ""): + secret_data = secrets_client.get_secret_value( + SecretId=parsed_commands.sample_secret_certificate) + with open(tmp_certificate_file_path, "w") as file: + # lgtm [py/clear-text-storage-sensitive-data] + file.write(secret_data["SecretString"]) + sample_certificate = tmp_certificate_file_path + if (parsed_commands.sample_secret_private_key != ""): + secret_data = secrets_client.get_secret_value( + SecretId=parsed_commands.sample_secret_private_key) + with open(tmp_private_key_path, "w") as file: + # lgtm [py/clear-text-storage-sensitive-data] + file.write(secret_data["SecretString"]) + sample_private_key = tmp_private_key_path + if (parsed_commands.sample_secret_custom_authorizer_name != ""): + sample_custom_authorizer_name = secrets_client.get_secret_value( + SecretId=parsed_commands.sample_secret_custom_authorizer_name)["SecretString"] + if (parsed_commands.sample_secret_custom_authorizer_password != ""): + sample_custom_authorizer_password = secrets_client.get_secret_value( + SecretId=parsed_commands.sample_secret_custom_authorizer_password)["SecretString"] + + except Exception: + sys.exit("ERROR: Could not get secrets to launch sample!") + + extra_step_return = 0 + if (parsed_commands.sample_run_softhsm != ""): + extra_step_return = make_softhsm_key() + sample_private_key = "" # Do not use the private key + if (parsed_commands.sample_run_certutil != ""): + extra_step_return = make_windows_pfx_file() + sample_private_key = "" # Do not use the private key + sample_certificate = tmp_pfx_certificate_path # use the Windows certificate path + + exit_code = extra_step_return + if (extra_step_return == 0): + print("Launching sample...") + exit_code = launch_sample(parsed_commands, sample_endpoint, sample_certificate, + sample_private_key, sample_custom_authorizer_name, sample_custom_authorizer_password) + + if (exit_code == 0): + print("SUCCESS: Finished running sample! Exiting with success") + else: + print("ERROR: Sample did not return success! Exit code " + str(exit_code)) + else: + print ("ERROR: Could not run extra step (SoftHSM, CertUtil, etc)") + + print("Deleting files...") + if (os.path.isfile(tmp_certificate_file_path)): + os.remove(tmp_certificate_file_path) + if (os.path.isfile(tmp_private_key_path)): + os.remove(tmp_private_key_path) + if (os.path.isfile(tmp_pfx_file_path)): + os.remove(tmp_pfx_file_path) + + return exit_code + + +def make_softhsm_key(): + print ("Setting up private key via SoftHSM") + softhsm_run = subprocess.run("softhsm2-util --init-token --free --label my-token --pin 0000 --so-pin 0000", shell=True) + if (softhsm_run.returncode != 0): + print ("ERROR: SoftHSM could not initialize a new token") + return softhsm_run.returncode + softhsm_run = subprocess.run(f"softhsm2-util --import {tmp_private_key_path} --token my-token --label my-key --id BEEFCAFE --pin 0000", shell=True) + if (softhsm_run.returncode != 0): + print ("ERROR: SoftHSM could not import token") + print ("Finished setting up private key in SoftHSM") + return 0 + + +def make_windows_pfx_file(): + global tmp_certificate_file_path + global tmp_private_key_path + global tmp_pfx_file_path + global tmp_pfx_certificate_path + + if sys.platform == "win32" or sys.platform == "cygwin": + if os.path.isfile(tmp_certificate_file_path) != True: + print (tmp_certificate_file_path) + print("ERROR: Certificate file not found!") + return 1 + if os.path.isfile(tmp_private_key_path) != True: + print("ERROR: Private key file not found!") + return 1 + + # Delete old PFX file if it exists + if os.path.isfile(tmp_pfx_file_path): + os.remove(tmp_pfx_file_path) + + # Make a key copy + copy_path = os.path.splitext(tmp_certificate_file_path) + with open(copy_path[0] + ".key", 'w') as file: + key_file = open(tmp_private_key_path) + file.write(key_file.read()) + key_file.close() + + # Make a PFX file + certutil_error_occurred = False + arguments = ["certutil", "-mergePFX", tmp_certificate_file_path, tmp_pfx_file_path] + certutil_run = subprocess.run(args=arguments, shell=True, input=f"{tmp_pfx_password}\n{tmp_pfx_password}", encoding='ascii') + if (certutil_run.returncode != 0): + print ("ERROR: Could not make PFX file") + certutil_error_occurred = True + return 1 + else: + print ("PFX file created successfully") + + # Remove the temporary key copy + if os.path.isfile(copy_path[0] + ".key"): + os.remove(copy_path[0] + ".key") + if (certutil_error_occurred == True): + return 1 + + # Import the PFX into the Windows Certificate Store + # (Passing '$mypwd' is required even though it is empty and our certificate has no password. It fails CI otherwise) + import_pfx_arguments = ["powershell.exe", "Import-PfxCertificate", "-FilePath", tmp_pfx_file_path, "-CertStoreLocation", "Cert:\\" + tmp_pfx_certificate_store_location, "-Password", "$mypwd"] + import_pfx_run = subprocess.run(args=import_pfx_arguments, shell=True, stdout=subprocess.PIPE) + if (import_pfx_run.returncode != 0): + print ("ERROR: Could not import PFX certificate into Windows store!") + return 1 + else: + print ("Certificate imported to Windows Certificate Store successfully") + + # Get the certificate thumbprint from the output: + import_pfx_output = str(import_pfx_run.stdout) + # We know the Thumbprint will always be 40 characters long, so we can find it using that + # TODO: Extract this using a better method + thumbprint = "" + current_str = "" + # The input comes as a string with some special characters still included, so we need to remove them! + import_pfx_output = import_pfx_output.replace("\\r", " ") + import_pfx_output = import_pfx_output.replace("\\n", "\n") + for i in range(0, len(import_pfx_output)): + if (import_pfx_output[i] == " " or import_pfx_output[i] == "\n"): + if (len(current_str) == 40): + thumbprint = current_str + break + current_str = "" + else: + current_str += import_pfx_output[i] + + # Did we get a thumbprint? + if (thumbprint == ""): + print ("ERROR: Could not find certificate thumbprint") + return 1 + + # Construct the certificate path + tmp_pfx_certificate_path = tmp_pfx_certificate_store_location + "\\" + thumbprint + + # Return success + print ("PFX certificate created and imported successfully!") + return 0 + + else: + print("ERROR - Windows PFX file can only be created on a Windows platform!") + return 1 + + +def launch_sample(parsed_commands, sample_endpoint, sample_certificate, sample_private_key, sample_custom_authorizer_name, sample_custom_authorizer_password): + global tmp_certificate_file_path + global tmp_private_key_path + global tmp_pfx_file_path + exit_code = 0 + + print("Processing arguments...") + launch_arguments = [] + launch_arguments.append("--endpoint") + launch_arguments.append(sample_endpoint) + + if (sample_certificate != ""): + launch_arguments.append("--cert") + launch_arguments.append(sample_certificate) + if (sample_private_key != ""): + launch_arguments.append("--key") + launch_arguments.append(sample_private_key) + if (sample_custom_authorizer_name != ""): + launch_arguments.append("--custom_auth_authorizer_name") + launch_arguments.append(sample_custom_authorizer_name) + if (sample_custom_authorizer_password != ""): + launch_arguments.append("--custom_auth_password") + launch_arguments.append(sample_custom_authorizer_password) + if (parsed_commands.sample_arguments != ""): + sample_arguments_split = parsed_commands.sample_arguments.split(" ") + for arg in sample_arguments_split: + launch_arguments.append(arg) + + print("Launching sample...") + # Based on the programming language, we have to run it a different way + if (parsed_commands.language == "Java"): + arguments_as_string = "" + for i in range(0, len(launch_arguments)): + arguments_as_string += str(launch_arguments[i]) + if (i+1 < len(launch_arguments)): + arguments_as_string += " " + arguments = ["mvn", "compile", "exec:java"] + arguments.append("-pl") + arguments.append(parsed_commands.sample_file) + arguments.append("-Dexec.mainClass=" + + parsed_commands.sample_main_class) + arguments.append("-Daws.crt.ci=True") + + # We have to do this as a string, unfortunately, due to how -Dexec.args= works... + argument_string = subprocess.list2cmdline( + arguments) + " -Dexec.args=\"" + arguments_as_string + "\"" + sample_return = subprocess.run(argument_string, shell=True) + exit_code = sample_return.returncode + + elif (parsed_commands.language == "CPP"): + sample_return = subprocess.run( + args=launch_arguments, executable=parsed_commands.sample_file) + exit_code = sample_return.returncode + + elif (parsed_commands.language == "Python"): + launch_arguments.append("--is_ci") + launch_arguments.append("True") + + sample_return = subprocess.run( + args=[sys.executable, parsed_commands.sample_file] + launch_arguments) + exit_code = sample_return.returncode + + elif (parsed_commands.language == "Javascript"): + os.chdir(parsed_commands.sample_file) + + launch_arguments.append("--is_ci") + launch_arguments.append("true") + + sample_return_one = None + if sys.platform == "win32" or sys.platform == "cygwin": + sample_return_one = subprocess.run(args=["npm", "install"], shell=True) + else: + sample_return_one = subprocess.run(args=["npm", "install"]) + + if (sample_return_one == None or sample_return_one.returncode != 0): + exit_code = sample_return_one.returncode + else: + sample_return_two = None + arguments = [] + if (parsed_commands.node_cmd == "" or parsed_commands.node_cmd == None): + arguments = ["node", "dist/index.js"] + else: + arguments = parsed_commands.node_cmd.split(" ") + + if sys.platform == "win32" or sys.platform == "cygwin": + sample_return_two = subprocess.run( + args=arguments + launch_arguments, shell=True) + else: + sample_return_two = subprocess.run( + args=arguments + launch_arguments) + + if (sample_return_two != None): + exit_code = sample_return_two.returncode + else: + exit_code = 1 + + else: + print("ERROR - unknown programming language! Supported programming languages are 'Java', 'CPP', 'Python', and 'Javascript'") + return -1 + + # finish! + return exit_code + + +def main(): + argument_parser = argparse.ArgumentParser( + description="Run Sample in CI") + argument_parser.add_argument("--language", metavar="", required=True, + help="The name of the programming language. Used to determine how to launch the sample") + argument_parser.add_argument("--sample_file", + metavar="", + required=True, default="", help="Sample to launch. Format varies based on programming language") + argument_parser.add_argument("--sample_region", metavar="", + required=True, default="us-east-1", help="The name of the region to use for accessing secrets") + argument_parser.add_argument("--sample_secret_endpoint", metavar="", + required=False, default="", help="The name of the secret containing the endpoint") + argument_parser.add_argument("--sample_secret_certificate", metavar="", required=False, + default="", help="The name of the secret containing the certificate PEM file") + argument_parser.add_argument("--sample_secret_private_key", metavar="", required=False, + default="", help="The name of the secret containing the private key PEM file") + argument_parser.add_argument("--sample_secret_custom_authorizer_name", metavar="", required=False, + default="", help="The name of the secret containing the custom authorizer name") + argument_parser.add_argument("--sample_secret_custom_authorizer_password", metavar="", required=False, + default="", help="The name of the secret containing the custom authorizer password") + argument_parser.add_argument("--sample_run_softhsm", metavar="", required=False, + default="", help="Runs SoftHSM on the private key passed, storing it, rather than passing it directly to the sample. Used for PKCS11 sample") + argument_parser.add_argument("--sample_run_certutil", metavar="", required=False, + default="", help="Runs CertUtil on the private key and certificate passed and makes a certificate.pfx file, " + "which is used automatically in the --cert argument. Used for Windows Certificate Connect sample") + argument_parser.add_argument("--sample_arguments", metavar="", + required=False, default="", + help="Arguments to pass to sample. In Java, these arguments will be in a double quote (\") string") + argument_parser.add_argument("--sample_main_class", metavar="", + required=False, default="", help="Java only: The main class to run") + argument_parser.add_argument("--node_cmd", metavar="", required=False, default="", + help="Javascript only: Overrides the default 'npm dist/index.js' with whatever you pass. Useful for launching pure Javascript samples") + + parsed_commands = argument_parser.parse_args() + + print("Starting to launch sample...") + sample_result = get_secrets_and_launch(parsed_commands) + sys.exit(sample_result) + + +if __name__ == "__main__": + main() diff --git a/utils/run_secure_tunnel_ci.py b/utils/run_secure_tunnel_ci.py new file mode 100644 index 000000000..baa33997f --- /dev/null +++ b/utils/run_secure_tunnel_ci.py @@ -0,0 +1,85 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +# Built-in +import argparse +import subprocess +import pathlib +import sys +from time import sleep +# Needs to be installed via pip +import boto3 # - for launching sample + + +current_folder = pathlib.Path(__file__).resolve() + + +def getSecretsAndLaunch(parsed_commands): + exit_code = 0 + + print ("Creating secure tunnel client using Boto3") + tunnel_client = None + try: + tunnel_client = boto3.client("iotsecuretunneling", region_name=parsed_commands.sample_region) + except Exception: + print ("Could not create tunnel client!") + exit(-1) + + tunnel_data = None + try: + tunnel_data = tunnel_client.open_tunnel() + except Exception: + print ("Could not open tunnel!") + exit(-1) + + print ("Launching Secure Tunnel samples...") + exit_code = launch_samples(parsed_commands, tunnel_data) + + print ("Closing tunnel...") + try: + tunnel_client.close_tunnel(tunnelId=tunnel_data["tunnelId"], delete=True) + except Exception: + print ("Could not close tunnel!") + exit(-1) + + return exit_code + + +def launch_samples(parsed_commands, tunnel_data): + exit_code = 0 + + # Right now secure tunneling is only in C++, so we only support launching the sample in the C++ way + launch_arguments_destination = ["--test", "--region", parsed_commands.sample_region, "--access_token", tunnel_data["destinationAccessToken"]] + launch_arguments_source = ["--local_proxy_mode_source", "--region", parsed_commands.sample_region, "--access_token", tunnel_data["sourceAccessToken"]] + + destination_run = subprocess.Popen(args=launch_arguments_destination, executable=parsed_commands.sample_file) + print ("About to sleep before running source part of sample...") + sleep(10) # Sleep to give the destination some time to run + source_run = subprocess.Popen(args=launch_arguments_source, executable=parsed_commands.sample_file) + + # Wait for the source to finish + source_run.wait() + # Once finished, stop the destination run + destination_run.kill() + + # finish! + return exit_code + + +def main(): + argument_parser = argparse.ArgumentParser( + description="Utility to run Secure Tunneling sample in CI") + argument_parser.add_argument("--sample_file", + metavar="", + required=True, default="", help="Sample to launch. Format varies based on programming language") + argument_parser.add_argument("--sample_region", metavar="", + required=True, default="us-east-1", help="The name of the region to use for accessing secrets") + parsed_commands = argument_parser.parse_args() + + print("Starting to launch Secure Tunneling sample...") + sample_result = getSecretsAndLaunch(parsed_commands) + sys.exit(sample_result) + + +if __name__ == "__main__": + main()