From 76468baf50d6109c3d22424d334c5b1ddf351c47 Mon Sep 17 00:00:00 2001 From: Martin Paulucci Date: Tue, 23 Sep 2025 16:13:45 +0200 Subject: [PATCH 1/5] ci(l1): Update hive runner to use `ethpandaops/hive-github-action`. --- .github/workflows/common_hive_reports.yaml | 19 +--- .github/workflows/pr-main_l1.yaml | 124 +++++++++++---------- 2 files changed, 67 insertions(+), 76 deletions(-) diff --git a/.github/workflows/common_hive_reports.yaml b/.github/workflows/common_hive_reports.yaml index 83d7526029..ad3e70769e 100644 --- a/.github/workflows/common_hive_reports.yaml +++ b/.github/workflows/common_hive_reports.yaml @@ -78,22 +78,6 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 - - name: Pull image - run: | - docker pull ghcr.io/lambdaclass/ethrex:main - # Tests use ethrex:ci so we retag it - docker tag ghcr.io/lambdaclass/ethrex:main ethrex:ci - - - name: Load hive client config - id: client-config - shell: bash - run: | - { - echo "config<>"$GITHUB_OUTPUT" - # Set custom args defined in Dockerfile to pin execution-spec-tests versions # See: https://github.com/ethereum/hive/blob/c2dab60f898b94afe8eeac505f60dcde59205e77/simulators/ethereum/eest/consume-rlp/Dockerfile#L4-L8 - name: Determine hive flags @@ -116,7 +100,7 @@ jobs: - name: Run Hive Simulation id: run-hive-action - uses: ethpandaops/hive-github-action@a9ec89442df18ee579d3179b76c47f5f93954307 + uses: ethpandaops/hive-github-action@v0.5.0 continue-on-error: true with: # This uses ethereum/hive as default @@ -125,7 +109,6 @@ jobs: # hive_version: master simulator: ${{ matrix.test.simulation }} client: ethrex - client_config: ${{ steps.client-config.outputs.config }} extra_flags: ${{ steps.hive-flags.outputs.flags }} workflow_artifact_upload: true workflow_artifact_prefix: ${{ matrix.test.file_name }}_${{ inputs.job_type }} diff --git a/.github/workflows/pr-main_l1.yaml b/.github/workflows/pr-main_l1.yaml index 30d4138f45..60ed3ba537 100644 --- a/.github/workflows/pr-main_l1.yaml +++ b/.github/workflows/pr-main_l1.yaml @@ -72,44 +72,6 @@ jobs: name: ethrex_image path: /tmp/ethrex_image.tar - setup-hive-fork: - name: "Setup Hive" - runs-on: ubuntu-latest - env: - HIVE_COMMIT_HASH: 115f4d6ef1bdd2bfcabe29ec60424f6327e92f43 # commit from our fork - steps: - - uses: actions/checkout@v4 - - name: Setup Hive - run: | - git clone --single-branch --branch update_dockerfile https://github.com/lambdaclass/hive - cd hive - git checkout --detach ${{ env.HIVE_COMMIT_HASH }} - go build . - - name: Upload hive artifacts - uses: actions/upload-artifact@v4 - with: - name: hive-fork - path: hive - - setup-hive-upstream: - name: "Setup Hive" - runs-on: ubuntu-latest - env: - HIVE_COMMIT_HASH: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 - steps: - - uses: actions/checkout@v4 - - name: Setup Hive - run: | - git clone --single-branch --branch master https://github.com/lambdaclass/hive - cd hive - git checkout --detach ${{ env.HIVE_COMMIT_HASH }} - go build . - - name: Upload hive artifacts - uses: actions/upload-artifact@v4 - with: - name: hive-upstream - path: hive - run-assertoor: name: Assertoor - ${{ matrix.name }} runs-on: ubuntu-latest @@ -155,44 +117,63 @@ jobs: run-hive: name: Hive - ${{ matrix.name }} runs-on: ubuntu-latest - needs: [docker_build, setup-hive-fork, setup-hive-upstream] + needs: [docker_build] if: ${{ github.event_name != 'merge_group' }} strategy: fail-fast: true matrix: include: - name: "Rpc Compat tests" - hive_version: "fork" simulation: ethereum/rpc-compat - test_pattern: "" + limit: "" + hive_repository: lambdaclass/hive + hive_version: 115f4d6ef1bdd2bfcabe29ec60424f6327e92f43 + artifact_prefix: rpc_compat - name: "Devp2p tests" - hive_version: "fork" simulation: devp2p - test_pattern: discv4|eth|snap/Ping|Findnode/WithoutEndpointProof|Findnode/PastExpiration|Amplification|Status|StorageRanges|ByteCodes|GetBlockHeaders|SimultaneousRequests|SameRequestID|ZeroRequestID|GetBlockBodies|MaliciousHandshake|MaliciousStatus|Transaction|NewPooledTxs|GetBlockReceipts|LargeTxRequest|InvalidTxs|BlockRangeUpdate + limit: discv4|eth|snap/Ping|Findnode/WithoutEndpointProof|Findnode/PastExpiration|Amplification|Status|StorageRanges|ByteCodes|GetBlockHeaders|SimultaneousRequests|SameRequestID|ZeroRequestID|GetBlockBodies|MaliciousHandshake|MaliciousStatus|Transaction|NewPooledTxs|GetBlockReceipts|LargeTxRequest|InvalidTxs|BlockRangeUpdate # AccountRange and GetTrieNodes don't pass anymore. - #|BlobViolations # Findnode/BasicFindnode fails due to packets being processed out of order # Findnode/UnsolicitedNeighbors flaky in CI very occasionally. When fixed replace all "Findnode/" with "Findnode" + hive_repository: lambdaclass/hive + hive_version: 115f4d6ef1bdd2bfcabe29ec60424f6327e92f43 + artifact_prefix: devp2p - name: "Engine Auth and EC tests" simulation: ethereum/engine - test_pattern: engine-(auth|exchange-capabilities)/ + limit: engine-(auth|exchange-capabilities)/ + hive_repository: ethereum/hive + hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + artifact_prefix: engine_auth_ec - name: "Cancun Engine tests" simulation: ethereum/engine - test_pattern: "engine-cancun/Blob Transactions On Block 1|Blob Transaction Ordering|Parallel Blob Transactions|ForkchoiceUpdatedV3|ForkchoiceUpdatedV2|ForkchoiceUpdated Version|GetPayload|NewPayloadV3 After Cancun|NewPayloadV3 Before Cancun|NewPayloadV3 Versioned Hashes|Incorrect BlobGasUsed|ParentHash equals BlockHash|RPC:|in ForkchoiceState|Unknown SafeBlockHash|Unknown FinalizedBlockHash|Unique|Re-Execute Payload|Multiple New Payloads|NewPayload with|Build Payload with|Re-org to Previously|Safe Re-Org to Side Chain|Transaction Re-Org|Re-Org Back into Canonical Chain|Suggested Fee Recipient Test|PrevRandao Opcode|Fork ID: *|Request Blob Pooled Transactions Single|Invalid NewPayload, Incomplete Transactions|Re-Org Back to Canonical Chain*|Invalid PayloadAttributes*|Invalid NewPayload, VersionedHashes|Invalid NewPayload, Incomplete VersionedHashes|Invalid NewPayload, Extra VersionedHashes|Bad Hash on NewPayload|Unknown HeadBlockHash|In-Order Consecutive Payload Execution|Valid NewPayload->ForkchoiceUpdated|Invalid NewPayload, ParentHash|Syncing=False|Payload Build after New Invalid Payload|Invalid NewPayload|Invalid Missing Ancestor ReOrg|Invalid Missing Ancestor Syncing ReOrG" + limit: "engine-cancun" + hive_repository: lambdaclass/hive + hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + artifact_prefix: engine_cancun - name: "Paris Engine tests" simulation: ethereum/engine - test_pattern: "engine-api/RPC|Bad Hash on NewPayload|Build Payload|Fork ID|In-Order Consecutive Payload Execution|Inconsistent|Invalid Missing Ancestor ReOrg|Invalid NewPayload|Invalid PayloadAttributes|Multiple New Payloads|NewPayload with|ParentHash equals BlockHash on NewPayload|Payload Build|PrevRandao Opcode Transactions|Re-Execute Payload|Re-Org Back|Re-org to Previously Validated Sidechain Payload|RPC:|Safe Re-Org|Suggested Fee|Transaction Re-Org|Unique Payload ID|Unknown|Valid NewPayload->ForkchoiceUpdated" # |Invalid P9 -> flaky - ethrex_flags: "" + limit: "engine-api" + hive_repository: lambdaclass/hive + hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + artifact_prefix: engine_paris - name: "Engine withdrawal tests" simulation: ethereum/engine - test_pattern: "engine-withdrawals/Corrupted Block Hash Payload|Empty Withdrawals|engine-withdrawals test loader|GetPayloadBodies|GetPayloadV2 Block Value|Max Initcode Size|Sync after 2 blocks - Withdrawals on Genesis|Withdraw many accounts|Withdraw to a single account|Withdraw to two accounts|Withdraw zero amount|Withdraw many accounts|Withdrawals Fork on Block 1 - 1 Block Re-Org|Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload|Withdrawals Fork on Block 2|Withdrawals Fork on Block 3|Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload|Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org [^S]|Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org [^S]" + limit: "engine-withdrawals/Corrupted Block Hash Payload|Empty Withdrawals|engine-withdrawals test loader|GetPayloadBodies|GetPayloadV2 Block Value|Max Initcode Size|Sync after 2 blocks - Withdrawals on Genesis|Withdraw many accounts|Withdraw to a single account|Withdraw to two accounts|Withdraw zero amount|Withdraw many accounts|Withdrawals Fork on Block 1 - 1 Block Re-Org|Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload|Withdrawals Fork on Block 2|Withdrawals Fork on Block 3|Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload|Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org [^S]|Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org [^S]" + hive_repository: lambdaclass/hive + hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + artifact_prefix: engine_withdrawals - name: "Sync full" simulation: ethereum/sync - test_pattern: "" + limit: "" + hive_repository: lambdaclass/hive + hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + artifact_prefix: sync_full - name: "Sync snap" simulation: ethereum/sync - test_pattern: "" - ethrex_flags: "--syncmode snap" + limit: "" + hive_repository: lambdaclass/hive + hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + artifact_prefix: sync_snap steps: - name: Checkout sources uses: actions/checkout@v4 @@ -203,17 +184,44 @@ jobs: name: ethrex_image path: /tmp - - name: Download hive artifacts - uses: actions/download-artifact@v4 - with: - name: hive-${{ matrix.hive_version || 'upstream' }} - - name: Load image run: | docker load --input /tmp/ethrex_image.tar + - name: Load hive client config + id: client-config + shell: bash + run: | + { + echo "config<>"$GITHUB_OUTPUT" + + - name: Determine hive flags + id: hive-flags + shell: bash + env: + SIM_LIMIT: ${{ matrix.limit }} + run: | + FLAGS='--sim.parallelism 16 --sim.loglevel 1' + if [[ -n "$SIM_LIMIT" ]]; then + FLAGS+=" --sim.limit \"$SIM_LIMIT\"" + fi + echo "flags=$FLAGS" >> "$GITHUB_OUTPUT" + - name: Run Hive Simulation - run: chmod +x hive && ./hive --client-file .github/config/hive/clients.yaml --client ethrex --sim ${{ matrix.simulation }} --sim.limit "${{ matrix.test_pattern }}" --sim.parallelism 16 --sim.loglevel 1 + id: run-hive-action + uses: ethpandaops/hive-github-action@v0.5.0 + with: + hive_repository: ${{ matrix.hive_repository }} + hive_version: ${{ matrix.hive_version }} + simulator: ${{ matrix.simulation }} + client: ethrex + client_config: ${{ steps.client-config.outputs.config }} + extra_flags: ${{ steps.hive-flags.outputs.flags }} + workflow_artifact_upload: true + workflow_artifact_prefix: ${{ matrix.artifact_prefix }} # The purpose of this job is to add it as a required check in GitHub so that we don't have to add every individual job as a required check all-tests: From 9043ffb734f7eecabd7b675864e707d5a2747565 Mon Sep 17 00:00:00 2001 From: Martin Paulucci Date: Tue, 23 Sep 2025 17:05:09 +0200 Subject: [PATCH 2/5] Fix escape characters. --- .github/workflows/pr-main_l1.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-main_l1.yaml b/.github/workflows/pr-main_l1.yaml index 60ed3ba537..7128484b8b 100644 --- a/.github/workflows/pr-main_l1.yaml +++ b/.github/workflows/pr-main_l1.yaml @@ -120,7 +120,7 @@ jobs: needs: [docker_build] if: ${{ github.event_name != 'merge_group' }} strategy: - fail-fast: true + fail-fast: false matrix: include: - name: "Rpc Compat tests" @@ -206,7 +206,8 @@ jobs: run: | FLAGS='--sim.parallelism 16 --sim.loglevel 1' if [[ -n "$SIM_LIMIT" ]]; then - FLAGS+=" --sim.limit \"$SIM_LIMIT\"" + escaped_limit=${SIM_LIMIT//\'/\'\\\'\'} + FLAGS+=" --sim.limit '$escaped_limit'" fi echo "flags=$FLAGS" >> "$GITHUB_OUTPUT" From 12bd53b79f1385282536a63c6380df7e8b40ddf7 Mon Sep 17 00:00:00 2001 From: Martin Paulucci Date: Thu, 25 Sep 2025 16:07:05 +0200 Subject: [PATCH 3/5] Remove docker output. --- .github/workflows/pr-main_l1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-main_l1.yaml b/.github/workflows/pr-main_l1.yaml index 6e93644cf7..7128484b8b 100644 --- a/.github/workflows/pr-main_l1.yaml +++ b/.github/workflows/pr-main_l1.yaml @@ -204,7 +204,7 @@ jobs: env: SIM_LIMIT: ${{ matrix.limit }} run: | - FLAGS='--sim.parallelism 16 --sim.loglevel 1 --docker.output' + FLAGS='--sim.parallelism 16 --sim.loglevel 1' if [[ -n "$SIM_LIMIT" ]]; then escaped_limit=${SIM_LIMIT//\'/\'\\\'\'} FLAGS+=" --sim.limit '$escaped_limit'" From 8145bd4674fc14d9868a00628e5153dd888ea35b Mon Sep 17 00:00:00 2001 From: Martin Paulucci Date: Tue, 7 Oct 2025 14:34:15 +0200 Subject: [PATCH 4/5] Add back tests. --- .github/workflows/pr-main_l1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-main_l1.yaml b/.github/workflows/pr-main_l1.yaml index b8ff4aa199..1690a43b10 100644 --- a/.github/workflows/pr-main_l1.yaml +++ b/.github/workflows/pr-main_l1.yaml @@ -176,7 +176,7 @@ jobs: artifact_prefix: engine_paris - name: "Engine withdrawal tests" simulation: ethereum/engine - limit: "engine-withdrawals" + limit: "engine-withdrawals/Corrupted Block Hash Payload|Empty Withdrawals|engine-withdrawals test loader|GetPayloadBodies|GetPayloadV2 Block Value|Max Initcode Size|Sync after 2 blocks - Withdrawals on Genesis|Withdraw many accounts|Withdraw to a single account|Withdraw to two accounts|Withdraw zero amount|Withdraw many accounts|Withdrawals Fork on Block 1 - 1 Block Re-Org|Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload|Withdrawals Fork on Block 2|Withdrawals Fork on Block 3|Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload|Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org [^S]|Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org [^S]" hive_repository: lambdaclass/hive hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 artifact_prefix: engine_withdrawals From 2163b94744994c52e57dab174f743c98932cebe7 Mon Sep 17 00:00:00 2001 From: Martin Paulucci Date: Tue, 7 Oct 2025 14:35:28 +0200 Subject: [PATCH 5/5] Use ethereum/hive for some tests. --- .github/workflows/pr-main_l1.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-main_l1.yaml b/.github/workflows/pr-main_l1.yaml index 1690a43b10..c352ab07c6 100644 --- a/.github/workflows/pr-main_l1.yaml +++ b/.github/workflows/pr-main_l1.yaml @@ -160,37 +160,37 @@ jobs: simulation: ethereum/engine limit: engine-(auth|exchange-capabilities)/ hive_repository: ethereum/hive - hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + hive_version: 2b7a9c007770b10cb1a242a6c6de88c87a383e5a artifact_prefix: engine_auth_ec - name: "Cancun Engine tests" simulation: ethereum/engine limit: "engine-cancun" - hive_repository: lambdaclass/hive - hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + hive_repository: ethereum/hive + hive_version: 2b7a9c007770b10cb1a242a6c6de88c87a383e5a artifact_prefix: engine_cancun - name: "Paris Engine tests" simulation: ethereum/engine limit: "engine-api" - hive_repository: lambdaclass/hive - hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + hive_repository: ethereum/hive + hive_version: 2b7a9c007770b10cb1a242a6c6de88c87a383e5a artifact_prefix: engine_paris - name: "Engine withdrawal tests" simulation: ethereum/engine limit: "engine-withdrawals/Corrupted Block Hash Payload|Empty Withdrawals|engine-withdrawals test loader|GetPayloadBodies|GetPayloadV2 Block Value|Max Initcode Size|Sync after 2 blocks - Withdrawals on Genesis|Withdraw many accounts|Withdraw to a single account|Withdraw to two accounts|Withdraw zero amount|Withdraw many accounts|Withdrawals Fork on Block 1 - 1 Block Re-Org|Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload|Withdrawals Fork on Block 2|Withdrawals Fork on Block 3|Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload|Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org [^S]|Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org [^S]" - hive_repository: lambdaclass/hive - hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + hive_repository: ethereum/hive + hive_version: 2b7a9c007770b10cb1a242a6c6de88c87a383e5a artifact_prefix: engine_withdrawals - name: "Sync full" simulation: ethereum/sync limit: "" - hive_repository: lambdaclass/hive - hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + hive_repository: ethereum/hive + hive_version: 2b7a9c007770b10cb1a242a6c6de88c87a383e5a artifact_prefix: sync_full - name: "Sync snap" simulation: ethereum/sync limit: "" - hive_repository: lambdaclass/hive - hive_version: 2d0e4fefb814d6815430c0f9e201fe8a045cf486 + hive_repository: ethereum/hive + hive_version: 2b7a9c007770b10cb1a242a6c6de88c87a383e5a artifact_prefix: sync_snap steps: - name: Free Disk Space (Ubuntu)