feat(rollup-relayer): add a tool to analyze chunk/batch/bundle proposing (#1645)

Co-authored-by: colinlyguo <[email protected]>

Author: colinlyguo

common/utils/flags.go

@@ -22,6 +22,10 @@ var (
 	RollupRelayerFlags = []cli.Flag{
 		&MinCodecVersionFlag,
 	}
+	// ProposerToolFlags contains flags only used in proposer tool
+	ProposerToolFlags = []cli.Flag{
+		&StartL2BlockFlag,
+	}
 	// ConfigFileFlag load json type config file.
 	ConfigFileFlag = cli.StringFlag{
 		Name:  "config",
@@ -90,4 +94,10 @@ var (
 		Usage:    "Minimum required codec version for the chunk/batch/bundle proposers",
 		Required: true,
 	}
+	// StartL2BlockFlag indicates the start L2 block number for proposer tool
+	StartL2BlockFlag = cli.Uint64Flag{
+		Name:  "start-l2-block",
+		Usage: "Start L2 block number for proposer tool",
+		Value: 0,
+	}
 )

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.5.5"
+var tag = "v4.5.6"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

rollup/README.md

@@ -33,3 +33,45 @@ make rollup_bins
 ./build/bin/gas_oracle --config ./conf/config.json
 ./build/bin/rollup_relayer --config ./conf/config.json
 ```
+
+## Proposer Tool
+
+The Proposer Tool replays historical blocks with custom configurations (e.g., future hardfork configs, custom chunk/batch/bundle proposer configs) to generate chunks/batches/bundles, helping test parameter changes before protocol upgrade.
+
+You can:
+
+1. Enable different hardforks in the genesis configuration.
+2. Set custom chunk-proposer, batch-proposer, and bundle-proposer parameters.
+3. Analyze resulting metrics (blob size, block count, transaction count, gas usage).
+
+## How to run the proposer tool?
+
+### Set the configs
+
+1. Set genesis config to enable desired hardforks in [`proposer-tool-genesis.json`](./proposer-tool-genesis.json).
+2. Set proposer config in [`proposer-tool-config.json`](./proposer-tool-config.json) for data analysis.
+3. Set `start-l2-block` in the launch command of proposer-tool in [`docker-compose-proposer-tool.yml`](./docker-compose-proposer-tool.yml) to the block number you want to start from. The default is `0`, which means starting from the genesis block.
+
+### Start the proposer tool using docker-compose
+
+Prerequisite: an RPC URL to an archive L2 node. The default url in [`proposer-tool-config.json`](./proposer-tool-config.json) is `https://rpc.scroll.io`.
+
+```
+cd rollup
+DOCKER_BUILDKIT=1 docker-compose -f docker-compose-proposer-tool.yml up -d
+```
+
+> Note: The port 5432 of database is mapped to the host machine. You can use `psql` or any db clients to connect to the database.
+
+> The DSN for the database is `postgres://postgres:postgres@db:5432/scroll?sslmode=disable`.
+
+
+### Reset env
+```
+docker-compose -f docker-compose-proposer-tool.yml down -v
+```
+
+If you need to rebuild the images, removing the old images is necessary. You can do this by running the following command:
+```
+docker images | grep rollup | awk '{print $3}' | xargs docker rmi -f
+```

rollup/cmd/proposer_tool/app/app.go

@@ -0,0 +1,93 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+
+	"github.com/scroll-tech/da-codec/encoding"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	"scroll-tech/common/utils"
+	"scroll-tech/common/version"
+
+	"scroll-tech/rollup/internal/config"
+	"scroll-tech/rollup/internal/controller/watcher"
+)
+
+var app *cli.App
+
+func init() {
+	// Set up proposer-tool app info.
+	app = cli.NewApp()
+	app.Action = action
+	app.Name = "proposer-tool"
+	app.Usage = "The Scroll Proposer Tool"
+	app.Version = version.Version
+	app.Flags = append(app.Flags, utils.CommonFlags...)
+	app.Flags = append(app.Flags, utils.RollupRelayerFlags...)
+	app.Flags = append(app.Flags, utils.ProposerToolFlags...)
+	app.Commands = []*cli.Command{}
+	app.Before = func(ctx *cli.Context) error {
+		return utils.LogSetup(ctx)
+	}
+}
+
+func action(ctx *cli.Context) error {
+	// Load config file.
+	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
+	cfg, err := config.NewConfigForReplay(cfgFile)
+	if err != nil {
+		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
+	}
+
+	subCtx, cancel := context.WithCancel(ctx.Context)
+
+	startL2BlockHeight := ctx.Uint64(utils.StartL2BlockFlag.Name)
+
+	genesisPath := ctx.String(utils.Genesis.Name)
+	genesis, err := utils.ReadGenesis(genesisPath)
+	if err != nil {
+		log.Crit("failed to read genesis", "genesis file", genesisPath, "error", err)
+	}
+
+	minCodecVersion := encoding.CodecVersion(ctx.Uint(utils.MinCodecVersionFlag.Name))
+
+	// sanity check config
+	if cfg.L2Config.BatchProposerConfig.MaxChunksPerBatch <= 0 {
+		log.Crit("cfg.L2Config.BatchProposerConfig.MaxChunksPerBatch must be greater than 0")
+	}
+	if cfg.L2Config.ChunkProposerConfig.MaxL2GasPerChunk <= 0 {
+		log.Crit("cfg.L2Config.ChunkProposerConfig.MaxL2GasPerChunk must be greater than 0")
+	}
+
+	proposerTool, err := watcher.NewProposerTool(subCtx, cancel, cfg, startL2BlockHeight, minCodecVersion, genesis.Config)
+	if err != nil {
+		log.Crit("failed to create proposer tool", "startL2BlockHeight", startL2BlockHeight, "minCodecVersion", minCodecVersion, "error", err)
+	}
+	proposerTool.Start()
+
+	log.Info("Start proposer-tool successfully", "version", version.Version)
+
+	// Catch CTRL-C to ensure a graceful shutdown.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	// Wait until the interrupt signal is received from an OS signal.
+	<-interrupt
+
+	cancel()
+	proposerTool.Stop()
+
+	return nil
+}
+
+// Run proposer tool cmd instance.
+func Run() {
+	if err := app.Run(os.Args); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

rollup/cmd/proposer_tool/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/rollup/cmd/proposer_tool/app"
+
+func main() {
+	app.Run()
+}

rollup/docker-compose-proposer-tool.yml

@@ -0,0 +1,40 @@
+version: '3'
+
+services:
+  db:
+    image: postgres:14
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=scroll
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  proposer-tool:
+    build:
+      context: ..
+      dockerfile: ./rollup/proposer_tool.Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+    command: [
+      "--config", "/app/conf/proposer-tool-config.json",
+      "--genesis", "/app/conf/proposer-tool-genesis.json",
+      "--min-codec-version", "4",
+      "--start-l2-block", "10000",
+      "--log.debug", "--verbosity", "3"
+    ]
+    volumes:
+      - ./proposer-tool-config.json:/app/conf/proposer-tool-config.json
+      - ./proposer-tool-genesis.json:/app/conf/proposer-tool-genesis.json
+    restart: unless-stopped
+
+volumes:
+  postgres_data:

rollup/internal/config/config.go

@@ -1,7 +1,10 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 
@@ -20,6 +23,11 @@ type Config struct {
 	DBConfig *database.Config `json:"db_config"`
 }
 
+type ConfigForReplay struct {
+	Config
+	DBConfigForReplay *database.Config `json:"db_config_for_replay"`
+}
+
 // NewConfig returns a new instance of Config.
 func NewConfig(file string) (*Config, error) {
 	v := viper.New()
@@ -87,3 +95,19 @@ func NewConfig(file string) (*Config, error) {
 
 	return cfg, nil
 }
+
+// NewConfigForReplay returns a new instance of ConfigForReplay.
+func NewConfigForReplay(file string) (*ConfigForReplay, error) {
+	buf, err := os.ReadFile(filepath.Clean(file))
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &ConfigForReplay{}
+	err = json.Unmarshal(buf, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}

rollup/internal/controller/watcher/batch_proposer.go

@@ -13,6 +13,8 @@ import (
 	"github.com/scroll-tech/go-ethereum/params"
 	"gorm.io/gorm"
 
+	"scroll-tech/common/types"
+
 	"scroll-tech/rollup/internal/config"
 	"scroll-tech/rollup/internal/orm"
 	"scroll-tech/rollup/internal/utils"
@@ -34,6 +36,7 @@ type BatchProposer struct {
 	maxUncompressedBatchBytesSize   uint64
 	maxChunksPerBatch               int
 
+	replayMode      bool
 	minCodecVersion encoding.CodecVersion
 	chainCfg        *params.ChainConfig
 
@@ -80,6 +83,7 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minC
 		gasCostIncreaseMultiplier:       cfg.GasCostIncreaseMultiplier,
 		maxUncompressedBatchBytesSize:   cfg.MaxUncompressedBatchBytesSize,
 		maxChunksPerBatch:               cfg.MaxChunksPerBatch,
+		replayMode:                      false,
 		minCodecVersion:                 minCodecVersion,
 		chainCfg:                        chainCfg,
 
@@ -152,6 +156,14 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minC
 	return p
 }
 
+// SetReplayDB sets the replay database for the BatchProposer.
+// This is used for the proposer tool only, to change the l2_block data source.
+// This function is not thread-safe and should be called after initializing the BatchProposer and before starting to propose chunks.
+func (p *BatchProposer) SetReplayDB(replayDB *gorm.DB) {
+	p.l2BlockOrm = orm.NewL2Block(replayDB)
+	p.replayMode = true
+}
+
 // TryProposeBatch tries to propose a new batches.
 func (p *BatchProposer) TryProposeBatch() {
 	p.batchProposerCircleTotal.Inc()
@@ -226,6 +238,15 @@ func (p *BatchProposer) updateDBBatchInfo(batch *encoding.Batch, codecVersion en
 			log.Warn("BatchProposer.UpdateBatchHashInRange update the chunk's batch hash failure", "hash", dbBatch.Hash, "error", dbErr)
 			return dbErr
 		}
+		if p.replayMode {
+			// If replayMode is true, meaning the batch was proposed by the proposer tool,
+			// set batch status to types.RollupCommitted and assign a unique commit tx hash to enable new bundle proposals.
+			if dbErr = p.batchOrm.UpdateCommitTxHashAndRollupStatus(p.ctx, dbBatch.Hash, dbBatch.Hash, types.RollupCommitted, dbTX); dbErr != nil {
+				log.Warn("BatchProposer.UpdateCommitTxHashAndRollupStatus update the batch's commit tx hash failure", "hash", dbBatch.Hash, "error", dbErr)
+				return dbErr
+			}
+		}
+
 		return nil
 	})
 	if err != nil {

rollup/internal/controller/watcher/bundle_proposer.go

@@ -199,7 +199,7 @@ func (p *BundleProposer) proposeBundle() error {
 
 	currentTimeSec := uint64(time.Now().Unix())
 	if firstChunk.StartBlockTime+p.bundleTimeoutSec < currentTimeSec {
-		log.Info("first block timeout", "batch count", len(batches), "start block number", firstChunk.StartBlockNumber, "start block timestamp", firstChunk.StartBlockTime, "current time", currentTimeSec)
+		log.Info("first block timeout", "batch count", len(batches), "start block number", firstChunk.StartBlockNumber, "start block timestamp", firstChunk.StartBlockTime, "bundle timeout", p.bundleTimeoutSec, "current time", currentTimeSec)
 
 		batches, err = p.allBatchesCommittedInSameTXIncluded(batches)
 		if err != nil {

rollup/internal/controller/watcher/chunk_proposer.go

@@ -36,6 +36,7 @@ type ChunkProposer struct {
 	gasCostIncreaseMultiplier       float64
 	maxUncompressedBatchBytesSize   uint64
 
+	replayMode      bool
 	minCodecVersion encoding.CodecVersion
 	chainCfg        *params.ChainConfig
 
@@ -91,6 +92,7 @@ func NewChunkProposer(ctx context.Context, cfg *config.ChunkProposerConfig, minC
 		chunkTimeoutSec:                 cfg.ChunkTimeoutSec,
 		gasCostIncreaseMultiplier:       cfg.GasCostIncreaseMultiplier,
 		maxUncompressedBatchBytesSize:   cfg.MaxUncompressedBatchBytesSize,
+		replayMode:                      false,
 		minCodecVersion:                 minCodecVersion,
 		chainCfg:                        chainCfg,
 
@@ -175,6 +177,14 @@ func NewChunkProposer(ctx context.Context, cfg *config.ChunkProposerConfig, minC
 	return p
 }
 
+// SetReplayDB sets the replay database for the ChunkProposer.
+// This is used for the proposer tool only, to change the l2_block data source.
+// This function is not thread-safe and should be called after initializing the ChunkProposer and before starting to propose chunks.
+func (p *ChunkProposer) SetReplayDB(replayDB *gorm.DB) {
+	p.l2BlockOrm = orm.NewL2Block(replayDB)
+	p.replayMode = true
+}
+
 // TryProposeChunk tries to propose a new chunk.
 func (p *ChunkProposer) TryProposeChunk() {
 	p.chunkProposerCircleTotal.Inc()
@@ -241,9 +251,12 @@ func (p *ChunkProposer) updateDBChunkInfo(chunk *encoding.Chunk, codecVersion en
 			log.Warn("ChunkProposer.InsertChunk failed", "codec version", codecVersion, "err", err)
 			return err
 		}
-		if err := p.l2BlockOrm.UpdateChunkHashInRange(p.ctx, dbChunk.StartBlockNumber, dbChunk.EndBlockNumber, dbChunk.Hash, dbTX); err != nil {
-			log.Error("failed to update chunk_hash for l2_blocks", "chunk hash", dbChunk.Hash, "start block", dbChunk.StartBlockNumber, "end block", dbChunk.EndBlockNumber, "err", err)
-			return err
+		// In replayMode we don't need to update chunk_hash in l2_block table.
+		if !p.replayMode {
+			if err := p.l2BlockOrm.UpdateChunkHashInRange(p.ctx, dbChunk.StartBlockNumber, dbChunk.EndBlockNumber, dbChunk.Hash, dbTX); err != nil {
+				log.Error("failed to update chunk_hash for l2_block", "chunk hash", dbChunk.Hash, "start block", dbChunk.StartBlockNumber, "end block", dbChunk.EndBlockNumber, "err", err)
+				return err
+			}
 		}
 		return nil
 	})
@@ -436,6 +449,12 @@ func (p *ChunkProposer) recordTimerChunkMetrics(metrics *utils.ChunkMetrics) {
 }
 
 func (p *ChunkProposer) tryProposeEuclidTransitionChunk(blocks []*encoding.Block) (bool, error) {
+	// If we are in replay mode, there is a corner case when StartL2Block is set as 0 in this check,
+	// it needs to get genesis block, but in mainnet db there is no genesis block, so we need to bypass this check.
+	if p.replayMode {
+		return false, nil
+	}
+
 	if !p.chainCfg.IsEuclid(blocks[0].Header.Time) {
 		return false, nil
 	}