From c54e18de792f660f13a7e14049d1f982bf9d3118 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 15:29:45 +0400 Subject: [PATCH 01/14] Use microseconds instead of milliseconds --- .../optimumcode/json/schema/benchmark/CommonBenchmarks.kt | 7 +++---- .../json/schema/benchmark/ComparisonBenchmark.kt | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/benchmark/src/commonMain/kotlin/io/github/optimumcode/json/schema/benchmark/CommonBenchmarks.kt b/benchmark/src/commonMain/kotlin/io/github/optimumcode/json/schema/benchmark/CommonBenchmarks.kt index 853ca4b8..791f6032 100644 --- a/benchmark/src/commonMain/kotlin/io/github/optimumcode/json/schema/benchmark/CommonBenchmarks.kt +++ b/benchmark/src/commonMain/kotlin/io/github/optimumcode/json/schema/benchmark/CommonBenchmarks.kt @@ -1,8 +1,7 @@ package io.github.optimumcode.json.schema.benchmark import kotlinx.benchmark.BenchmarkMode -import kotlinx.benchmark.BenchmarkTimeUnit.MILLISECONDS -import kotlinx.benchmark.BenchmarkTimeUnit.SECONDS +import kotlinx.benchmark.BenchmarkTimeUnit import kotlinx.benchmark.Mode.AverageTime import kotlinx.benchmark.Mode.Throughput import kotlinx.benchmark.OutputTimeUnit @@ -11,7 +10,7 @@ import kotlinx.benchmark.Scope import kotlinx.benchmark.State @State(Scope.Benchmark) -@OutputTimeUnit(SECONDS) +@OutputTimeUnit(BenchmarkTimeUnit.SECONDS) @BenchmarkMode(Throughput) class CommonThroughputBench : AbstractCommonBenchmark() { @Param("object") @@ -22,7 +21,7 @@ class CommonThroughputBench : AbstractCommonBenchmark() { } @State(Scope.Benchmark) -@OutputTimeUnit(MILLISECONDS) +@OutputTimeUnit(BenchmarkTimeUnit.MICROSECONDS) @BenchmarkMode(AverageTime) class CommonAvgTimeBench : AbstractCommonBenchmark() { @Param("object") diff --git a/benchmark/src/jvmMain/kotlin/io/github/optimumcode/json/schema/benchmark/ComparisonBenchmark.kt b/benchmark/src/jvmMain/kotlin/io/github/optimumcode/json/schema/benchmark/ComparisonBenchmark.kt index 5e8e089f..a6a4eb1b 100644 --- a/benchmark/src/jvmMain/kotlin/io/github/optimumcode/json/schema/benchmark/ComparisonBenchmark.kt +++ b/benchmark/src/jvmMain/kotlin/io/github/optimumcode/json/schema/benchmark/ComparisonBenchmark.kt @@ -6,7 +6,7 @@ import kotlinx.benchmark.OutputTimeUnit import kotlinx.benchmark.Param import kotlinx.benchmark.Scope import kotlinx.benchmark.State -import java.util.concurrent.TimeUnit.MILLISECONDS +import java.util.concurrent.TimeUnit.MICROSECONDS import java.util.concurrent.TimeUnit.SECONDS @State(Scope.Benchmark) @@ -21,7 +21,7 @@ class ComparisonThroughputBenchmark : AbstractComparisonBenchmark() { } @State(Scope.Benchmark) -@OutputTimeUnit(MILLISECONDS) +@OutputTimeUnit(MICROSECONDS) @BenchmarkMode(Mode.AverageTime) class ComparisonAvgTimeBenchmark : AbstractComparisonBenchmark() { @Param("object") From c15e13b638b84a1634a22f66c5bf9ede1248423b Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 15:31:59 +0400 Subject: [PATCH 02/14] Add benchmark alerting --- .github/workflows/benchmark.yml | 166 +++++++++++++++++++---- .github/workflows/platform-benchmark.yml | 103 -------------- 2 files changed, 139 insertions(+), 130 deletions(-) delete mode 100644 .github/workflows/platform-benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 83edda62..a5cc16f8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,7 +11,7 @@ on: iterations: description: 'number of iterations in the benchmark' type: number - default: 3 + default: 10 required: false iteration-time: description: 'duration of individual integration in benchmark' @@ -25,31 +25,143 @@ on: required: false schedule: - cron: "0 2 * * 1" + push: + branches: + - main + +env: + REPORT_FORMAT: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' ) && 'json' || 'csv' }} jobs: - check-linux: - uses: ./.github/workflows/platform-benchmark.yml - with: - run-on: ubuntu-latest - warmups: ${{ inputs.warmups }} - iterations: ${{ inputs.iterations }} - iteration-time: ${{ inputs.iteration-time }} - iteration-time-unit: ${{ inputs.iteration-time-unit }} - check-macos: - uses: ./.github/workflows/platform-benchmark.yml - with: - run-on: macos-latest - additional-task: "-x :benchmark:jvmBenchmark" - warmups: ${{ inputs.warmups }} - iterations: ${{ inputs.iterations }} - iteration-time: ${{ inputs.iteration-time }} - iteration-time-unit: ${{ inputs.iteration-time-unit }} - check-windows: - uses: ./.github/workflows/platform-benchmark.yml - with: - run-on: windows-latest - additional-task: "-x :benchmark:jvmBenchmark" - warmups: ${{ inputs.warmups }} - iterations: ${{ inputs.iterations }} - iteration-time: ${{ inputs.iteration-time }} - iteration-time-unit: ${{ inputs.iteration-time-unit }} + benchmark-matrix: + strategy: + matrix: + include: + - os: ubuntu-latest + additional-task: '' + - os: macos-latest + additional-task: '-x :benchmark:jvmBenchmark' + - os: windows-latest + additional-task: '-x :benchmark:jvmBenchmark' + runs-on: ${{ matrix.os }} + steps: + - name: 'Install native dependencies' + run: sudo apt-get install -y libunistring-dev + if: matrix.os == 'ubuntu-latest' + - name: 'Checkout Repository' + uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version-file: .java-version + - uses: actions/setup-python@v5 + with: + python-version-file: .python-version + - name: Validate Gradle Wrapper + uses: gradle/actions/wrapper-validation@v3 + - name: Cache konan + uses: actions/cache@v4 + with: + path: ~/.konan + key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Cache unicode data + uses: actions/cache@v4 + with: + path: unicode_dump + key: unicode-dump-${{ hashFiles('unicode_dump/*') }} + restore-keys: | + unicode-dump- + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + with: + gradle-version: wrapper + - name: Run benchmarks + run: > + ./gradlew --no-daemon :benchmark:benchmark ${{ matrix.additional-task }} + -Pbenchmark_warmups=${{ inputs.warmups }} + -Pbenchmark_iterations=${{ inputs.iterations }} + -Pbenchmark_iteration_time=${{ inputs.iteration-time }} + -Pbenchmark_iteration_time_unit=${{ inputs.iteration-time-unit }} + -Pbenchmark_report_format=${{ env.REPORT_FORMAT }} + - name: Install CSV to MD converter + run: pip install csv2md + - name: Add benchmark results to summary + shell: bash + if: env.REPORT_FORMAT == 'csv' + run: | + for report in $(find ./benchmark/build/reports/benchmarks/main -type f -name "*.csv") + do + file_name=$(basename "$report") + platform="${file_name%.*}" + echo "File $file_name" + # remove empty lines + sed -i -e '/^[[:space:]]*$/d' $report + echo "::group::Report CSV" + cat "$report" + echo "::endgroup::" + markdown_table=$(csv2md "$report") + echo "::group::Report Markdown" + echo "$markdown_table" + echo "::endgroup::" + echo "# Platform ${platform}" >> $GITHUB_STEP_SUMMARY + echo "$markdown_table" >> $GITHUB_STEP_SUMMARY + done + - name: Store results as artifact + if: env.REPORT_FORMAT == 'json' + uses: actions/upload-artifact@v4 + with: + name: bench-result-${{ matrix.os }} + path: benchmark/build/reports/benchmarks/main/**/*.json + + upload-benchmark-results: + if: (github.event_name == 'push' || github.event_name == 'pull_request') && github.repository == 'OptimumCode/json-schema-validator' + needs: + - benchmark-matrix + runs-on: ubuntu-latest + env: + RESULTS_DIR: bench-results + permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + # pull-requests permission to create comments on PR in case of alert + pull-requests: write + steps: + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + path: ${{ env.RESULTS_DIR }} + merge-multiple: true + - name: Show downloaded artifacts + run: tree ${{ env.RESULTS_DIR }} + - name: Prepare and join benchmark reports + id: prep + run: | + for report in $(find ./${{ env.RESULTS_DIR }}/build/reports/benchmarks/main -type f -name "*.json") + do + file_name=$(basename "$report") + platform="${file_name%.*}" + jq '.[] | .benchmark |= "$platform." + ltrimstr("io.github.optimumcode.json.schema.benchmark.")' $report > $report + done + AGGREGATED_REPORT=aggregated.json + # Joinded reports looks like this: [[{},{}], [{},{}]] + # We need to transform them into this: [{},{}] + jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/build/reports/benchmarks/main/**/*.json > $AGGREGATED_REPORT + echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: KMP JSON schema validator + tool: 'jmh' + output-file-path: ${{ steps.prep.outputs.report }} + alert-comment-cc-users: "@OptimumCode" + comment-on-alert: true + summary-always: true + alert-threshold: '50%' + fail-threshold: '100%' + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically only if run in main repo and not in PR + auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/platform-benchmark.yml b/.github/workflows/platform-benchmark.yml deleted file mode 100644 index de464485..00000000 --- a/.github/workflows/platform-benchmark.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Reusable workflow to run a benchmark on the platform - -on: - workflow_call: - inputs: - run-on: - type: string - required: true - description: "runner to check the project" - additional-task: - type: string - description: additional task to add to gradle call - required: false - default: "" - warmups: - description: 'number of warmups run before the actual benchmark' - type: string - required: false - iterations: - description: 'number of iterations in the benchmark' - type: string - required: false - iteration-time: - description: 'duration of individual integration in benchmark' - type: string - required: false - iteration-time-unit: - description: 'timeunit for iteration-time parameter' - default: 's' - type: string - required: false - report-format: - description: 'format of the report' - default: 'csv' - type: string - required: false - -jobs: - benchmark: - runs-on: ${{ inputs.run-on }} - steps: - - name: 'Install native dependencies' - run: sudo apt-get install -y libunistring-dev - if: runner.os == 'Linux' - - name: 'Checkout Repository' - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version-file: .java-version - - uses: actions/setup-python@v5 - with: - python-version-file: .python-version - - name: Validate Gradle Wrapper - uses: gradle/actions/wrapper-validation@v3 - - name: Cache konan - uses: actions/cache@v4 - with: - path: ~/.konan - key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Cache unicode data - uses: actions/cache@v4 - with: - path: unicode_dump - key: unicode-dump-${{ hashFiles('unicode_dump/*') }} - restore-keys: | - unicode-dump- - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - with: - gradle-version: wrapper - - name: Run benchmarks - run: > - ./gradlew --no-daemon :benchmark:benchmark ${{ inputs.additional-task }} - -Pbenchmark_warmups=${{ inputs.warmups }} - -Pbenchmark_iterations=${{ inputs.iterations }} - -Pbenchmark_iteration_time=${{ inputs.iteration-time }} - -Pbenchmark_iteration_time_unit=${{ inputs.iteration-time-unit }} - -Pbenchmark_report_format=${{ inputs.report-format }} - - name: Install CSV to MD converter - run: pip install csv2md - - name: Add benchmark results to summary - shell: bash - run: | - for report in $(find ./benchmark/build/reports/benchmarks/main -type f -name "*.csv") - do - file_name=$(basename "$report") - platform="${file_name%.*}" - echo "File $file_name" - # remove empty lines - sed -i -e '/^[[:space:]]*$/d' $report - echo "::group::Report CSV" - cat "$report" - echo "::endgroup::" - markdown_table=$(csv2md "$report") - echo "::group::Report Markdown" - echo "$markdown_table" - echo "::endgroup::" - echo "# Platform ${platform}" >> $GITHUB_STEP_SUMMARY - echo "$markdown_table" >> $GITHUB_STEP_SUMMARY - done From 490899b0676e17842af33cb8c62f67483a5f3371 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 15:36:43 +0400 Subject: [PATCH 03/14] Add PR trigger. Add concurrency group --- .github/workflows/benchmark.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a5cc16f8..64942a45 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -28,10 +28,15 @@ on: push: branches: - main + pull_request: env: REPORT_FORMAT: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' ) && 'json' || 'csv' }} +concurrency: + cancel-in-progress: true + group: bench-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.after }} + jobs: benchmark-matrix: strategy: From 532356525480fb42a58024871b0c9715fd6629fa Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 15:37:55 +0400 Subject: [PATCH 04/14] Add name for bench job --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 64942a45..119f18d2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -49,6 +49,7 @@ jobs: - os: windows-latest additional-task: '-x :benchmark:jvmBenchmark' runs-on: ${{ matrix.os }} + name: Run benchmarks on ${{ matrix.os }} steps: - name: 'Install native dependencies' run: sudo apt-get install -y libunistring-dev From 944101b151631b09db6acf3c4e2544841783a67b Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 15:38:31 +0400 Subject: [PATCH 05/14] Temporarly exclude linux to speedup test run --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 119f18d2..e1d26ea1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -42,8 +42,8 @@ jobs: strategy: matrix: include: - - os: ubuntu-latest - additional-task: '' + # - os: ubuntu-latest + # additional-task: '' - os: macos-latest additional-task: '-x :benchmark:jvmBenchmark' - os: windows-latest From 13e4134239c9758b0c8cf7e95f5aff9adc21adb1 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 15:52:20 +0400 Subject: [PATCH 06/14] Add macosX64. Correct paths to reports --- .github/workflows/benchmark.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e1d26ea1..bb976c06 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,6 +46,8 @@ jobs: # additional-task: '' - os: macos-latest additional-task: '-x :benchmark:jvmBenchmark' + - os: macos-13 # for macosX64 + additional-task: '-x :benchmark:jvmBenchmark' - os: windows-latest additional-task: '-x :benchmark:jvmBenchmark' runs-on: ${{ matrix.os }} @@ -146,7 +148,7 @@ jobs: - name: Prepare and join benchmark reports id: prep run: | - for report in $(find ./${{ env.RESULTS_DIR }}/build/reports/benchmarks/main -type f -name "*.json") + for report in $(find ./${{ env.RESULTS_DIR }} -type f -name "*.json") do file_name=$(basename "$report") platform="${file_name%.*}" @@ -155,7 +157,7 @@ jobs: AGGREGATED_REPORT=aggregated.json # Joinded reports looks like this: [[{},{}], [{},{}]] # We need to transform them into this: [{},{}] - jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/build/reports/benchmarks/main/**/*.json > $AGGREGATED_REPORT + jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/**/*.json > $AGGREGATED_REPORT echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 From b6f4524c616d0789b92e7a32d1fd85923208c241 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 16:04:51 +0400 Subject: [PATCH 07/14] Add condition for python csv to md converter --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bb976c06..3df0323e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -94,6 +94,7 @@ jobs: -Pbenchmark_iteration_time_unit=${{ inputs.iteration-time-unit }} -Pbenchmark_report_format=${{ env.REPORT_FORMAT }} - name: Install CSV to MD converter + if: env.REPORT_FORMAT == 'csv' run: pip install csv2md - name: Add benchmark results to summary shell: bash From 25a9ebc293f3a8f96eeb4c4ef73507db2ba5e1bb Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 16:22:28 +0400 Subject: [PATCH 08/14] Correct post-processing for benchmark results --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3df0323e..bc8416a7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -153,12 +153,12 @@ jobs: do file_name=$(basename "$report") platform="${file_name%.*}" - jq '.[] | .benchmark |= "$platform." + ltrimstr("io.github.optimumcode.json.schema.benchmark.")' $report > $report + jq ".[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\")" $report > $platfrom.json done AGGREGATED_REPORT=aggregated.json # Joinded reports looks like this: [[{},{}], [{},{}]] # We need to transform them into this: [{},{}] - jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/**/*.json > $AGGREGATED_REPORT + jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 From 7fa3ddad972f37b406d1a701f7693d9a2df43077 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 16:41:19 +0400 Subject: [PATCH 09/14] Correct path for updated results per platform --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bc8416a7..31898655 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -153,7 +153,7 @@ jobs: do file_name=$(basename "$report") platform="${file_name%.*}" - jq ".[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\")" $report > $platfrom.json + jq ".[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\")" $report > ${{ env.RESULTS_DIR }}/$platfrom.json done AGGREGATED_REPORT=aggregated.json # Joinded reports looks like this: [[{},{}], [{},{}]] From 17cd6c5f6276a638d8b50fd0a5f790f957f5e2cf Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 17:00:40 +0400 Subject: [PATCH 10/14] Fix typo. Add ls call befor jq --- .github/workflows/benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 31898655..49b1f480 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -153,11 +153,12 @@ jobs: do file_name=$(basename "$report") platform="${file_name%.*}" - jq ".[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\")" $report > ${{ env.RESULTS_DIR }}/$platfrom.json + jq ".[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\")" $report > ${{ env.RESULTS_DIR }}/$platform.json done AGGREGATED_REPORT=aggregated.json # Joinded reports looks like this: [[{},{}], [{},{}]] # We need to transform them into this: [{},{}] + ls ${{ env.RESULTS_DIR }}/*.json jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT - name: Store benchmark result From 98f77797aa33fc0335fcaf93ab96a1ff8e7464ca Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 17:33:24 +0400 Subject: [PATCH 11/14] Correct jq transformation --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 49b1f480..8ce362e9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -153,7 +153,7 @@ jobs: do file_name=$(basename "$report") platform="${file_name%.*}" - jq ".[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\")" $report > ${{ env.RESULTS_DIR }}/$platform.json + jq "[ .[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\") ]" $report > ${{ env.RESULTS_DIR }}/$platform.json done AGGREGATED_REPORT=aggregated.json # Joinded reports looks like this: [[{},{}], [{},{}]] From 667dcd4af69684a0ded8457a7ed50a748bd44113 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 17:38:55 +0400 Subject: [PATCH 12/14] Fix typo in comment --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8ce362e9..2d43b0f3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -156,7 +156,7 @@ jobs: jq "[ .[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\") ]" $report > ${{ env.RESULTS_DIR }}/$platform.json done AGGREGATED_REPORT=aggregated.json - # Joinded reports looks like this: [[{},{}], [{},{}]] + # Joined reports looks like this: [[{},{}], [{},{}]] # We need to transform them into this: [{},{}] ls ${{ env.RESULTS_DIR }}/*.json jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT From 686ec855bade4dfeb6b9d3cc2cd9cc48db60fca1 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 18:01:38 +0400 Subject: [PATCH 13/14] Try top-level permissions --- .github/workflows/benchmark.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2d43b0f3..1c134b8f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,6 +33,14 @@ on: env: REPORT_FORMAT: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' ) && 'json' || 'csv' }} +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + # pull-requests permission to create comments on PR in case of alert + pull-requests: write + concurrency: cancel-in-progress: true group: bench-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.after }} @@ -131,13 +139,6 @@ jobs: runs-on: ubuntu-latest env: RESULTS_DIR: bench-results - permissions: - # deployments permission to deploy GitHub pages website - deployments: write - # contents permission to update benchmark contents in gh-pages branch - contents: write - # pull-requests permission to create comments on PR in case of alert - pull-requests: write steps: - name: Download benchmark results uses: actions/download-artifact@v4 From 1d284233826e50e78e155030deacd61dfc825396 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 30 Jun 2024 18:14:46 +0400 Subject: [PATCH 14/14] Add checkout step --- .github/workflows/benchmark.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1c134b8f..1a8ce34b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,14 +33,6 @@ on: env: REPORT_FORMAT: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' ) && 'json' || 'csv' }} -permissions: - # deployments permission to deploy GitHub pages website - deployments: write - # contents permission to update benchmark contents in gh-pages branch - contents: write - # pull-requests permission to create comments on PR in case of alert - pull-requests: write - concurrency: cancel-in-progress: true group: bench-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.after }} @@ -139,7 +131,16 @@ jobs: runs-on: ubuntu-latest env: RESULTS_DIR: bench-results + permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + # pull-requests permission to create comments on PR in case of alert + pull-requests: write steps: + - name: 'Checkout Repository' + uses: actions/checkout@v4 - name: Download benchmark results uses: actions/download-artifact@v4 with: