diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 9cc416c..6022b46 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -18,6 +18,7 @@ on: env: SKETCHES_REPORTS_PATH: sketches-reports TESTDATA_SKETCHES_PATH: .github/workflows/testdata/sketches + TESTDATA_REPORTS_PATH: .github/workflows/testdata/reports jobs: default-inputs: @@ -87,6 +88,84 @@ jobs: sketches-report-path: ${{ env.SKETCHES_REPORTS_PATH }} verbose: true + - name: Set report artifact name + id: report-artifact-name + run: | + echo "::set-output name=report-artifact-name::${{ github.job }}" + + - name: Save sketches report as workflow artifact + uses: actions/upload-artifact@v2 + with: + path: sketches-reports + name: ${{ steps.report-artifact-name.outputs.report-artifact-name }} + + outputs: + report-artifact-name: ${{ steps.report-artifact-name.outputs.report-artifact-name }} + + + check-sketches-reports: + needs: all-inputs + runs-on: ubuntu-latest + + steps: + # Checkout is needed to get the golden reports + - name: Checkout local repo + uses: actions/checkout@v2 + + - name: Download sketches reports artifact + uses: actions/download-artifact@v2 + with: + name: ${{ needs.all-inputs.outputs.report-artifact-name }} + path: ${{ env.SKETCHES_REPORTS_PATH }} + + - name: Compare generated sketches report to golden report + run: | + # The commit hash changes on every push, so it can't be compared against the golden report + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + COMMIT_HASH="${{ github.event.pull_request.head.sha }}" + else + COMMIT_HASH="$(git rev-parse HEAD)" + fi + + set +o errexit # Unset errexit so the results of all report comparisons are shown before exiting + EXIT_STATUS=0 + while read -r reportPath; do + echo "Sketches report $reportPath matches expected:" + + ( + # Check static report components against golden reports + jq \ + --null-input \ + --exit-status \ + --slurpfile generated "$reportPath" \ + --slurpfile golden "${{ env.TESTDATA_REPORTS_PATH }}/${{ needs.all-inputs.outputs.report-artifact-name }}/$(basename "$reportPath")" \ + '($generated|.[0].boards) == ($golden|.[0].boards)' + ) && ( + # Check the commit_hash value + jq \ + --null-input \ + --exit-status \ + --slurpfile generated "$reportPath" \ + --arg commit_hash "$COMMIT_HASH" \ + '$generated|.[0].commit_hash == $commit_hash|.' + ) && ( + # Check the commit_url value + jq \ + --null-input \ + --exit-status \ + --slurpfile generated "$reportPath" \ + --arg commit_url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${COMMIT_HASH}" \ + '$generated|.[0].commit_url == $commit_url|.' + ) + + if [[ $? -ne 0 ]]; then + echo "::error::Sketches report $reportPath mismatch." + EXIT_STATUS=1 + fi + done <<<"$(find "${{ env.SKETCHES_REPORTS_PATH }}" -type f -and -name '*.json')" + + exit $EXIT_STATUS + expected-failed-compilation: runs-on: ubuntu-latest diff --git a/.github/workflows/testdata/reports/all-inputs/arduino-avr-uno.json b/.github/workflows/testdata/reports/all-inputs/arduino-avr-uno.json new file mode 100644 index 0000000..ebf7b42 --- /dev/null +++ b/.github/workflows/testdata/reports/all-inputs/arduino-avr-uno.json @@ -0,0 +1,147 @@ +{ + "commit_hash": "d4343a2e7d8123726f2c53d2ac186df41b59e946", + "commit_url": "https://github.com/arduino/compile-sketches/commit/d4343a2e7d8123726f2c53d2ac186df41b59e946", + "boards": [ + { + "board": "arduino:avr:uno", + "sketches": [ + { + "name": ".github/workflows/testdata/sketches/BareMinimum", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 32256, + "current": { + "absolute": 444, + "relative": 1.38 + }, + "previous": { + "absolute": 444, + "relative": 1.38 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 2048, + "current": { + "absolute": 9, + "relative": 0.44 + }, + "previous": { + "absolute": 9, + "relative": 0.44 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 0 + }, + "previous": { + "absolute": 0 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": ".github/workflows/testdata/sketches/ServoLibrary", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 32256, + "current": { + "absolute": 888, + "relative": 2.75 + }, + "previous": { + "absolute": 888, + "relative": 2.75 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 2048, + "current": { + "absolute": 46, + "relative": 2.25 + }, + "previous": { + "absolute": 46, + "relative": 2.25 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 0 + }, + "previous": { + "absolute": 0 + }, + "delta": { + "absolute": 0 + } + } + } + ], + "sizes": [ + { + "name": "flash", + "maximum": 32256, + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + }, + "relative": { + "minimum": 0.0, + "maximum": 0.0 + } + } + }, + { + "name": "RAM for global variables", + "maximum": 2048, + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + }, + "relative": { + "minimum": 0.0, + "maximum": 0.0 + } + } + } + ], + "warnings": { + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/.github/workflows/testdata/reports/all-inputs/esp8266-esp8266-huzzah.json b/.github/workflows/testdata/reports/all-inputs/esp8266-esp8266-huzzah.json new file mode 100644 index 0000000..6876eef --- /dev/null +++ b/.github/workflows/testdata/reports/all-inputs/esp8266-esp8266-huzzah.json @@ -0,0 +1,147 @@ +{ + "commit_hash": "d4343a2e7d8123726f2c53d2ac186df41b59e946", + "commit_url": "https://github.com/arduino/compile-sketches/commit/d4343a2e7d8123726f2c53d2ac186df41b59e946", + "boards": [ + { + "board": "esp8266:esp8266:huzzah", + "sketches": [ + { + "name": ".github/workflows/testdata/sketches/BareMinimum", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 1044464, + "current": { + "absolute": 256684, + "relative": 24.58 + }, + "previous": { + "absolute": 256684, + "relative": 24.58 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 81920, + "current": { + "absolute": 26776, + "relative": 32.69 + }, + "previous": { + "absolute": 26776, + "relative": 32.69 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 0 + }, + "previous": { + "absolute": 0 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": ".github/workflows/testdata/sketches/ServoLibrary", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 1044464, + "current": { + "absolute": 256684, + "relative": 24.58 + }, + "previous": { + "absolute": 256684, + "relative": 24.58 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 81920, + "current": { + "absolute": 26776, + "relative": 32.69 + }, + "previous": { + "absolute": 26776, + "relative": 32.69 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 0 + }, + "previous": { + "absolute": 0 + }, + "delta": { + "absolute": 0 + } + } + } + ], + "sizes": [ + { + "name": "flash", + "maximum": 1044464, + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + }, + "relative": { + "minimum": 0.0, + "maximum": 0.0 + } + } + }, + { + "name": "RAM for global variables", + "maximum": 81920, + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + }, + "relative": { + "minimum": 0.0, + "maximum": 0.0 + } + } + } + ], + "warnings": { + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + } + } + } + } + ] +} \ No newline at end of file