From 7ea3a881026f2b05b2c0c1756789f3a583796801 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 6 Mar 2025 09:55:03 +0900 Subject: [PATCH] Revert "[Benchmark] fail test if model artifact does not exist (#8482)" This reverts commit 24671a96d6936836461810649839aa0fd1058cc3. --- .github/workflows/android-perf.yml | 120 ++++++++++++++--------------- .github/workflows/apple-perf.yml | 119 ++++++++++++++-------------- 2 files changed, 114 insertions(+), 125 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 8c0ba752259..f21ed849d03 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -96,6 +96,63 @@ jobs: PYTHONPATH="${PWD}" python .ci/scripts/gather_benchmark_configs.py $ARGS + prepare-test-specs: + runs-on: linux.2xlarge + needs: set-parameters + strategy: + matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Prepare the spec + id: prepare + shell: bash + env: + BENCHMARK_CONFIG: ${{ toJSON(matrix) }} + working-directory: extension/benchmark/android/benchmark + run: | + set -eux + + # The model will be exported in the next step to this S3 path + MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" + # We could write a script to properly use jinja here, but there is only one variable, + # so let's just sed it + sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 + + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') + # The config for this benchmark runs, we save it in the test spec so that it can be fetched + # later by the upload script + sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 + + cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml + # Just print the test spec for debugging + cat android-llm-device-farm-test-spec.yml + + # Save the benchmark configs so that we can use it later in the dashboard + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT + + - name: Upload the spec + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml + + - name: Update the benchmark configs + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json + export-models: name: export-models uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main @@ -278,69 +335,6 @@ jobs: fi echo "::endgroup::" - prepare-test-specs: - runs-on: linux.2xlarge - needs: - - set-parameters - - export-models - strategy: - matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} - fail-fast: false - steps: - - uses: actions/checkout@v3 - - - name: Prepare the spec - id: prepare - shell: bash - env: - BENCHMARK_CONFIG: ${{ toJSON(matrix) }} - working-directory: extension/benchmark/android/benchmark - run: | - set -eux - - # The model will be exported in the next step to this S3 path - MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" - - # Check if the model artifact exists, fail this step skip generating test-spec. - curl -s --head -f ${MODEL_PATH} - - # We could write a script to properly use jinja here, but there is only one variable, - # so let's just sed it - sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 - - BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') - # The config for this benchmark runs, we save it in the test spec so that it can be fetched - # later by the upload script - sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 - - cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml - # Just print the test spec for debugging - cat android-llm-device-farm-test-spec.yml - - # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" - echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT - - - name: Upload the spec - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml - - - name: Update the benchmark configs - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json - build-benchmark-app: name: build-benchmark-app uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index cc7f85e9386..44aa645d16d 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -98,6 +98,63 @@ jobs: echo "benchmark_configs is: ${{ steps.set-parameters.outputs.benchmark_configs }}" + prepare-test-specs: + runs-on: linux.2xlarge + needs: set-parameters + strategy: + matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Prepare the spec + id: prepare + shell: bash + env: + BENCHMARK_CONFIG: ${{ toJSON(matrix) }} + working-directory: extension/benchmark/apple/Benchmark + run: | + set -eux + + # The model will be exported in the next step to this S3 path + MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" + # We could write a script to properly use jinja here, but there is only one variable, + # so let's just sed it + sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 + + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') + # The config for this benchmark runs, we save it in the test spec so that it can be fetched + # later by the upload script + sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 + + cp default-ios-device-farm-appium-test-spec.yml.j2 default-ios-device-farm-appium-test-spec.yml + # Just print the test spec for debugging + cat default-ios-device-farm-appium-test-spec.yml + + # Save the benchmark configs so that we can use it later in the dashboard + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT + + - name: Upload the spec + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml + + - name: Update the benchmark configs + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json + export-models: name: export-models uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -287,68 +344,6 @@ jobs: fi echo "::endgroup::" - prepare-test-specs: - runs-on: linux.2xlarge - needs: - - set-parameters - - export-models - strategy: - matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} - fail-fast: false - steps: - - uses: actions/checkout@v3 - - - name: Prepare the spec - id: prepare - shell: bash - env: - BENCHMARK_CONFIG: ${{ toJSON(matrix) }} - working-directory: extension/benchmark/apple/Benchmark - run: | - set -eux - - # The model will be exported in the next step to this S3 path - MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" - # Check if the model artifact exists, fail this step skip generating test-spec. - curl -s --head -f ${MODEL_PATH} - # We could write a script to properly use jinja here, but there is only one variable, - # so let's just sed it - sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - - BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') - # The config for this benchmark runs, we save it in the test spec so that it can be fetched - # later by the upload script - sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - - cp default-ios-device-farm-appium-test-spec.yml.j2 default-ios-device-farm-appium-test-spec.yml - # Just print the test spec for debugging - cat default-ios-device-farm-appium-test-spec.yml - - # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" - echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT - - - name: Upload the spec - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml - - - name: Update the benchmark configs - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json - - build-benchmark-app: name: build-benchmark-app uses: pytorch/test-infra/.github/workflows/macos_job.yml@main