feat(L1 follower and rollup verifier): support CodecV7 in L1 follower mode and rollup verifier (#1105)

* port changes from #1013

* port changes from #1068

* go.mod tidy

* fix compile error

* fix goimports

* fix log

* address review comments

* upgrade golang.org/x/net to 0.23.0

* port changes from #1018

* fix tests and linter errors

* address review comments

* refactor rollup sync service / verifier to use CalldataBlobSource to retrieve data from L1

* add configuration and initialize blob clients

* fix unit tests

* remove unused code

* address review comments

* address more review comments

* implement first version of new da-codec and to handle multiple batches submitted in one transaction

* add CommitBatchDAV7 and handle multiple commit events submitted in a single transactions

* fix bug due to previous batch being empty when processing the first batch within a set of batches

* Allow using MPT

* update to latest da-codec

* add field to CommittedBatchMeta to store LastL1MessageQueueHash for CodecV7 batches

* adjust rollup verifier to support CodecV7 batches

* address review comments

* fix issues after merge

* go mod tidy

* fix unit tests

* update da-codec

* add test TestValidateBatchCodecV7

* go mod tidy

* do not log error on shutdown

* add sanity check for version to deserialization of committedBatchMetaV7

* chore: auto version bump [bot]

* address review comments

* chore: auto version bump [bot]

---------

Co-authored-by: Ömer Faruk Irmak <[email protected]>
Co-authored-by: Thegaram <[email protected]>
Co-authored-by: Péter Garamvölgyi <[email protected]>

Author: 3 people

core/rawdb/accessors_rollup_event.go

@@ -2,8 +2,11 @@ package rawdb
 
 import (
 	"bytes"
+	"fmt"
 	"math/big"
 
+	"github.com/scroll-tech/da-codec/encoding"
+
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/ethdb"
 	"github.com/scroll-tech/go-ethereum/log"
@@ -18,12 +21,26 @@ type ChunkBlockRange struct {
 
 // CommittedBatchMeta holds metadata for committed batches.
 type CommittedBatchMeta struct {
+	Version          uint8
+	ChunkBlockRanges []*ChunkBlockRange
+
+	// introduced with CodecV7
+	LastL1MessageQueueHash common.Hash
+}
+
+type committedBatchMetaV0 struct {
 	Version uint8
 	// BlobVersionedHashes are the versioned hashes of the blobs in the batch. Currently unused. Left for compatibility.
 	BlobVersionedHashes []common.Hash
 	ChunkBlockRanges    []*ChunkBlockRange
 }
 
+type committedBatchMetaV7 struct {
+	Version                uint8
+	ChunkBlockRanges       []*ChunkBlockRange
+	LastL1MessageQueueHash common.Hash
+}
+
 // FinalizedBatchMeta holds metadata for finalized batches.
 type FinalizedBatchMeta struct {
 	BatchHash            common.Hash
@@ -143,30 +160,62 @@ func ReadLastFinalizedBatchIndex(db ethdb.Reader) *uint64 {
 
 // WriteCommittedBatchMeta stores the CommittedBatchMeta for a specific batch in the database.
 func WriteCommittedBatchMeta(db ethdb.KeyValueWriter, batchIndex uint64, committedBatchMeta *CommittedBatchMeta) {
-	value, err := rlp.EncodeToBytes(committedBatchMeta)
+	var committedBatchMetaToStore any
+	if encoding.CodecVersion(committedBatchMeta.Version) < encoding.CodecV7 {
+		committedBatchMetaToStore = &committedBatchMetaV0{
+			Version:          committedBatchMeta.Version,
+			ChunkBlockRanges: committedBatchMeta.ChunkBlockRanges,
+		}
+	} else {
+		committedBatchMetaToStore = &committedBatchMetaV7{
+			Version:                committedBatchMeta.Version,
+			ChunkBlockRanges:       committedBatchMeta.ChunkBlockRanges,
+			LastL1MessageQueueHash: committedBatchMeta.LastL1MessageQueueHash,
+		}
+	}
+
+	value, err := rlp.EncodeToBytes(committedBatchMetaToStore)
 	if err != nil {
-		log.Crit("failed to RLP encode committed batch metadata", "batch index", batchIndex, "committed batch meta", committedBatchMeta, "err", err)
+		log.Crit("failed to RLP encode committed batch metadata", "batch index", batchIndex, "committed batch meta", committedBatchMetaToStore, "err", err)
 	}
 	if err := db.Put(committedBatchMetaKey(batchIndex), value); err != nil {
 		log.Crit("failed to store committed batch metadata", "batch index", batchIndex, "value", value, "err", err)
 	}
 }
 
 // ReadCommittedBatchMeta fetches the CommittedBatchMeta for a specific batch from the database.
-func ReadCommittedBatchMeta(db ethdb.Reader, batchIndex uint64) *CommittedBatchMeta {
+func ReadCommittedBatchMeta(db ethdb.Reader, batchIndex uint64) (*CommittedBatchMeta, error) {
 	data, err := db.Get(committedBatchMetaKey(batchIndex))
 	if err != nil && isNotFoundErr(err) {
-		return nil
+		return nil, nil
 	}
 	if err != nil {
-		log.Crit("failed to read committed batch metadata from database", "batch index", batchIndex, "err", err)
+		return nil, fmt.Errorf("failed to read committed batch metadata from database: batch index %d, err: %w", batchIndex, err)
 	}
 
-	cbm := new(CommittedBatchMeta)
-	if err := rlp.Decode(bytes.NewReader(data), cbm); err != nil {
-		log.Crit("Invalid CommittedBatchMeta RLP", "batch index", batchIndex, "data", data, "err", err)
+	// Try decoding from the newest format for future proofness, then the older one for old data.
+	cbm7 := new(committedBatchMetaV7)
+	if err = rlp.Decode(bytes.NewReader(data), cbm7); err == nil {
+		if encoding.CodecVersion(cbm7.Version) < encoding.CodecV7 {
+			return nil, fmt.Errorf("unexpected committed batch metadata version: batch index %d, version %d", batchIndex, cbm7.Version)
+		}
+		return &CommittedBatchMeta{
+			Version:                cbm7.Version,
+			ChunkBlockRanges:       cbm7.ChunkBlockRanges,
+			LastL1MessageQueueHash: cbm7.LastL1MessageQueueHash,
+		}, nil
 	}
-	return cbm
+
+	cbm0 := new(committedBatchMetaV0)
+	if err = rlp.Decode(bytes.NewReader(data), cbm0); err != nil {
+		return nil, fmt.Errorf("failed to decode committed batch metadata: batch index %d, err: %w", batchIndex, err)
+	}
+
+	return &CommittedBatchMeta{
+		Version:                cbm0.Version,
+		ChunkBlockRanges:       cbm0.ChunkBlockRanges,
+		LastL1MessageQueueHash: common.Hash{},
+	}, nil
 }
 
 // DeleteCommittedBatchMeta removes the block ranges of all chunks associated with a specific batch from the database.

core/rawdb/accessors_rollup_event_test.go

@@ -3,6 +3,8 @@ package rawdb
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/scroll-tech/go-ethereum/common"
 )
 
@@ -157,55 +159,67 @@ func TestWriteReadDeleteCommittedBatchMeta(t *testing.T) {
 		{
 			batchIndex: 0,
 			meta: &CommittedBatchMeta{
-				Version:             0,
-				BlobVersionedHashes: []common.Hash{},
-				ChunkBlockRanges:    []*ChunkBlockRange{},
+				Version:          0,
+				ChunkBlockRanges: []*ChunkBlockRange{},
+			},
+		},
+		{
+			batchIndex: 1,
+			meta: &CommittedBatchMeta{
+				Version:          1,
+				ChunkBlockRanges: []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
+			},
+		},
+		{
+			batchIndex: 1,
+			meta: &CommittedBatchMeta{
+				Version:          2,
+				ChunkBlockRanges: []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
 			},
 		},
 		{
 			batchIndex: 1,
 			meta: &CommittedBatchMeta{
-				Version:             1,
-				BlobVersionedHashes: []common.Hash{common.HexToHash("0x1234")},
-				ChunkBlockRanges:    []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
+				Version:                7,
+				ChunkBlockRanges:       []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
+				LastL1MessageQueueHash: common.Hash{1, 2, 3, 4, 5, 6, 7},
 			},
 		},
 		{
 			batchIndex: 255,
 			meta: &CommittedBatchMeta{
-				Version:             255,
-				BlobVersionedHashes: []common.Hash{common.HexToHash("0xabcd"), common.HexToHash("0xef01")},
-				ChunkBlockRanges:    []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}, {StartBlockNumber: 11, EndBlockNumber: 20}},
+				Version:                255,
+				ChunkBlockRanges:       []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}, {StartBlockNumber: 11, EndBlockNumber: 20}},
+				LastL1MessageQueueHash: common.Hash{255},
 			},
 		},
 	}
 
 	for _, tc := range testCases {
 		WriteCommittedBatchMeta(db, tc.batchIndex, tc.meta)
-		got := ReadCommittedBatchMeta(db, tc.batchIndex)
-
-		if got == nil {
-			t.Fatalf("Expected non-nil value for batch index %d", tc.batchIndex)
-		}
+		got, err := ReadCommittedBatchMeta(db, tc.batchIndex)
+		require.NoError(t, err)
+		require.NotNil(t, got)
 
 		if !compareCommittedBatchMeta(tc.meta, got) {
 			t.Fatalf("CommittedBatchMeta mismatch for batch index %d, expected %+v, got %+v", tc.batchIndex, tc.meta, got)
 		}
 	}
 
 	// reading a non-existing value
-	if got := ReadCommittedBatchMeta(db, 256); got != nil {
+	got, err := ReadCommittedBatchMeta(db, 256)
+	require.NoError(t, err)
+	if got != nil {
 		t.Fatalf("Expected nil for non-existing value, got %+v", got)
 	}
 
 	// delete: revert batch
 	for _, tc := range testCases {
 		DeleteCommittedBatchMeta(db, tc.batchIndex)
 
-		readChunkRange := ReadCommittedBatchMeta(db, tc.batchIndex)
-		if readChunkRange != nil {
-			t.Fatal("Committed batch metadata was not deleted", "batch index", tc.batchIndex)
-		}
+		readChunkRange, err := ReadCommittedBatchMeta(db, tc.batchIndex)
+		require.NoError(t, err)
+		require.Nil(t, readChunkRange, "Committed batch metadata was not deleted", "batch index", tc.batchIndex)
 	}
 
 	// delete non-existing value: ensure the delete operation handles non-existing values without errors.
@@ -217,35 +231,37 @@ func TestOverwriteCommittedBatchMeta(t *testing.T) {
 
 	batchIndex := uint64(42)
 	initialMeta := &CommittedBatchMeta{
-		Version:             1,
-		BlobVersionedHashes: []common.Hash{common.HexToHash("0x1234")},
-		ChunkBlockRanges:    []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
+		Version:          1,
+		ChunkBlockRanges: []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
 	}
 	newMeta := &CommittedBatchMeta{
-		Version:             2,
-		BlobVersionedHashes: []common.Hash{common.HexToHash("0x5678"), common.HexToHash("0x9abc")},
-		ChunkBlockRanges:    []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 20}, {StartBlockNumber: 21, EndBlockNumber: 30}},
+		Version:                255,
+		ChunkBlockRanges:       []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 20}, {StartBlockNumber: 21, EndBlockNumber: 30}},
+		LastL1MessageQueueHash: common.Hash{255},
 	}
 
 	// write initial meta
 	WriteCommittedBatchMeta(db, batchIndex, initialMeta)
-	got := ReadCommittedBatchMeta(db, batchIndex)
+	got, err := ReadCommittedBatchMeta(db, batchIndex)
+	require.NoError(t, err)
 
 	if !compareCommittedBatchMeta(initialMeta, got) {
 		t.Fatalf("Initial write failed, expected %+v, got %+v", initialMeta, got)
 	}
 
 	// overwrite with new meta
 	WriteCommittedBatchMeta(db, batchIndex, newMeta)
-	got = ReadCommittedBatchMeta(db, batchIndex)
+	got, err = ReadCommittedBatchMeta(db, batchIndex)
+	require.NoError(t, err)
 
 	if !compareCommittedBatchMeta(newMeta, got) {
 		t.Fatalf("Overwrite failed, expected %+v, got %+v", newMeta, got)
 	}
 
 	// read non-existing batch index
 	nonExistingIndex := uint64(999)
-	got = ReadCommittedBatchMeta(db, nonExistingIndex)
+	got, err = ReadCommittedBatchMeta(db, nonExistingIndex)
+	require.NoError(t, err)
 
 	if got != nil {
 		t.Fatalf("Expected nil for non-existing batch index, got %+v", got)
@@ -256,14 +272,7 @@ func compareCommittedBatchMeta(a, b *CommittedBatchMeta) bool {
 	if a.Version != b.Version {
 		return false
 	}
-	if len(a.BlobVersionedHashes) != len(b.BlobVersionedHashes) {
-		return false
-	}
-	for i := range a.BlobVersionedHashes {
-		if a.BlobVersionedHashes[i] != b.BlobVersionedHashes[i] {
-			return false
-		}
-	}
+
 	if len(a.ChunkBlockRanges) != len(b.ChunkBlockRanges) {
 		return false
 	}
@@ -272,5 +281,6 @@ func compareCommittedBatchMeta(a, b *CommittedBatchMeta) bool {
 			return false
 		}
 	}
-	return true
+
+	return a.LastL1MessageQueueHash == b.LastL1MessageQueueHash
 }

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/prometheus/tsdb v0.7.1
 	github.com/rjeczalik/notify v0.9.1
 	github.com/rs/cors v1.7.0
-	github.com/scroll-tech/da-codec v0.1.3-0.20241218102542-9852fa4e1be5
+	github.com/scroll-tech/da-codec v0.1.3-0.20250210041951-d028c537b995
 	github.com/scroll-tech/zktrie v0.8.4
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sourcegraph/conc v0.3.0

go.sum

@@ -396,8 +396,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/da-codec v0.1.3-0.20241218102542-9852fa4e1be5 h1:vZ75srkZCStjDWq/kqZGLoucf7Y7qXC13nKjQVZ0zp8=
-github.com/scroll-tech/da-codec v0.1.3-0.20241218102542-9852fa4e1be5/go.mod h1:XfQhUl3msmE6dpZEbR/LIwiMxywPQcUQsch9URgXDzs=
+github.com/scroll-tech/da-codec v0.1.3-0.20250210041951-d028c537b995 h1:Zo1p42CUS9pADSKoDD0ZoDxf4dQ3gttqWZlV+RSeImk=
+github.com/scroll-tech/da-codec v0.1.3-0.20250210041951-d028c537b995/go.mod h1:UZhhjzqYsyEhcvY0Y+SP+oMdeOUqFn/UXpbAYuPGzg0=
 github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE=
 github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=

params/version.go

@@ -24,7 +24,7 @@ import (
 const (
 	VersionMajor = 5         // Major version component of the current release
 	VersionMinor = 8         // Minor version component of the current release
-	VersionPatch = 6         // Patch version component of the current release
+	VersionPatch = 7         // Patch version component of the current release
 	VersionMeta  = "mainnet" // Version metadata to append to the version string
 )