Merge branch 'alpha' into feat/new_bridge_contracts

Author: colinlyguo

.github/workflows/common.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
     - uses: actions-rs/toolchain@v1
       with:
-        toolchain: nightly-2022-08-23
+        toolchain: nightly-2022-12-10
         override: true
         components: rustfmt, clippy
     - name: Install Go

.github/workflows/coordinator.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
     - uses: actions-rs/toolchain@v1
       with:
-        toolchain: nightly-2022-08-23
+        toolchain: nightly-2022-12-10
         override: true
         components: rustfmt, clippy
     - name: Install Go

.github/workflows/roller.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
     - uses: actions-rs/toolchain@v1
       with:
-        toolchain: nightly-2022-08-23
+        toolchain: nightly-2022-12-10
         override: true
         components: rustfmt, clippy
     - name: Install Go
@@ -42,6 +42,8 @@ jobs:
     - name: Test
       run: |
         make roller
+        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./prover/lib
+        export CHAIN_ID=534353
         go test -v ./...
   check:
     runs-on: ubuntu-latest

Jenkinsfile

@@ -13,6 +13,8 @@ pipeline {
     environment {
         GO111MODULE = 'on'
         PATH="/home/ubuntu/.cargo/bin:$PATH"
+        LD_LIBRARY_PATH="$LD_LIBRARY_PATH:./coordinator/verifier/lib"
+        CHAIN_ID='534353'
         // LOG_DOCKER = 'true'
     }
     stages {

Makefile

@@ -1,5 +1,7 @@
 .PHONY: check update dev_docker clean
 
+ZKP_VERSION=release-1220
+
 help: ## Display this help message
 	@grep -h \
 		-E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
@@ -29,5 +31,15 @@ dev_docker: ## build docker images for development/testing usages
 	docker build -t scroll_l1geth ./common/docker/l1geth/
 	docker build -t scroll_l2geth ./common/docker/l2geth/
 
+test_zkp: ## Test zkp prove and verify, roller/prover generates the proof and coordinator/verifier verifies it
+	mkdir -p test_params
+	wget https://circuit-release.s3.us-west-2.amazonaws.com/circuit-release/${ZKP_VERSION}/test_params/params19 -O ./test_params/params19
+	wget https://circuit-release.s3.us-west-2.amazonaws.com/circuit-release/${ZKP_VERSION}/test_params/params26 -O ./test_params/params26
+	wget https://circuit-release.s3.us-west-2.amazonaws.com/circuit-release/${ZKP_VERSION}/test_seed -O test_seed
+	rm -rf ./roller/assets/test_params && mv test_params ./roller/assets/ && mv test_seed ./roller/assets/
+	cd ./roller && make test-gpu-prover
+	rm -rf ./coordinator/assets/test_params && mv ./roller/assets/test_params ./coordinator/assets/ && mv ./roller/assets/agg_proof ./coordinator/assets/
+	cd ./coordinator && make test-gpu-verifier
+
 clean: ## Empty out the bin folder
 	@rm -rf build/bin

bridge/l2/backend.go

@@ -27,13 +27,15 @@ func New(ctx context.Context, cfg *config.L2Config, orm database.OrmFactory) (*B
 		return nil, err
 	}
 
+	// Note: initialize watcher before relayer to keep DB consistent.
+	// Otherwise, there will be a race condition between watcher.initializeGenesis and relayer.ProcessPendingBatches.
+	l2Watcher := NewL2WatcherClient(ctx, client, cfg.Confirmations, cfg.BatchProposerConfig, cfg.L2MessengerAddress, orm)
+
 	relayer, err := NewLayer2Relayer(ctx, orm, cfg.RelayerConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	l2Watcher := NewL2WatcherClient(ctx, client, cfg.Confirmations, cfg.BatchProposerConfig, cfg.L2MessengerAddress, orm)
-
 	return &Backend{
 		cfg:       cfg,
 		l2Watcher: l2Watcher,

bridge/l2/batch_proposer.go

@@ -58,15 +58,15 @@ func (w *batchProposer) tryProposeBatch() {
 
 	if blocks[0].GasUsed > w.batchGasThreshold {
 		log.Warn("gas overflow even for only 1 block", "height", blocks[0].Number, "gas", blocks[0].GasUsed)
-		if err = w.createBatchForBlocks(blocks[:1]); err != nil {
+		if _, err = w.createBatchForBlocks(blocks[:1]); err != nil {
 			log.Error("failed to create batch", "number", blocks[0].Number, "err", err)
 		}
 		return
 	}
 
 	if blocks[0].TxNum > w.batchTxNumThreshold {
 		log.Warn("too many txs even for only 1 block", "height", blocks[0].Number, "tx_num", blocks[0].TxNum)
-		if err = w.createBatchForBlocks(blocks[:1]); err != nil {
+		if _, err = w.createBatchForBlocks(blocks[:1]); err != nil {
 			log.Error("failed to create batch", "number", blocks[0].Number, "err", err)
 		}
 		return
@@ -93,15 +93,15 @@ func (w *batchProposer) tryProposeBatch() {
 		return
 	}
 
-	if err = w.createBatchForBlocks(blocks); err != nil {
+	if _, err = w.createBatchForBlocks(blocks); err != nil {
 		log.Error("failed to create batch", "from", blocks[0].Number, "to", blocks[len(blocks)-1].Number, "err", err)
 	}
 }
 
-func (w *batchProposer) createBatchForBlocks(blocks []*orm.BlockInfo) error {
+func (w *batchProposer) createBatchForBlocks(blocks []*orm.BlockInfo) (string, error) {
 	dbTx, err := w.orm.Beginx()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	var dbTxErr error
@@ -128,13 +128,13 @@ func (w *batchProposer) createBatchForBlocks(blocks []*orm.BlockInfo) error {
 
 	batchID, dbTxErr = w.orm.NewBatchInDBTx(dbTx, startBlock, endBlock, startBlock.ParentHash, txNum, gasUsed)
 	if dbTxErr != nil {
-		return dbTxErr
+		return "", dbTxErr
 	}
 
 	if dbTxErr = w.orm.SetBatchIDForBlocksInDBTx(dbTx, blockIDs, batchID); dbTxErr != nil {
-		return dbTxErr
+		return "", dbTxErr
 	}
 
 	dbTxErr = dbTx.Commit()
-	return dbTxErr
+	return batchID, dbTxErr
 }

bridge/l2/batch_proposer_test.go

@@ -40,7 +40,7 @@ func testBatchProposer(t *testing.T) {
 	// Insert traces into db.
 	assert.NoError(t, db.InsertBlockTraces([]*types.BlockTrace{trace2, trace3}))
 
-	id := utils.ComputeBatchID(trace3.Header.Hash(), trace2.Header.ParentHash, big.NewInt(1))
+	id := utils.ComputeBatchID(trace3.Header.Hash(), trace2.Header.ParentHash, big.NewInt(0))
 
 	proposer := newBatchProposer(&config.BatchProposerConfig{
 		ProofGenerationFreq: 1,

bridge/l2/watcher.go

@@ -67,7 +67,7 @@ func NewL2WatcherClient(ctx context.Context, client *ethclient.Client, confirmat
 		savedHeight = 0
 	}
 
-	return &WatcherClient{
+	w := WatcherClient{
 		ctx:                ctx,
 		Client:             client,
 		orm:                orm,
@@ -79,6 +79,75 @@ func NewL2WatcherClient(ctx context.Context, client *ethclient.Client, confirmat
 		stopped:            0,
 		batchProposer:      newBatchProposer(bpCfg, orm),
 	}
+
+	// Initialize genesis before we do anything else
+	if err := w.initializeGenesis(); err != nil {
+		panic(fmt.Sprintf("failed to initialize L2 genesis batch, err: %v", err))
+	}
+
+	return &w
+}
+
+func (w *WatcherClient) initializeGenesis() error {
+	if count, err := w.orm.GetBatchCount(); err != nil {
+		return fmt.Errorf("failed to get batch count: %v", err)
+	} else if count > 0 {
+		log.Info("genesis already imported")
+		return nil
+	}
+
+	genesis, err := w.HeaderByNumber(w.ctx, big.NewInt(0))
+	if err != nil {
+		return fmt.Errorf("failed to retrieve L2 genesis header: %v", err)
+	}
+
+	// EIP1559 is disabled so the RPC won't return baseFeePerGas. However, l2geth
+	// still uses BaseFee when calculating the block hash. If we keep it as <nil>
+	// here the genesis hash will not match.
+	genesis.BaseFee = big.NewInt(0)
+
+	log.Info("retrieved L2 genesis header", "hash", genesis.Hash().String())
+
+	trace := &types.BlockTrace{
+		Coinbase:         nil,
+		Header:           genesis,
+		Transactions:     []*types.TransactionData{},
+		StorageTrace:     nil,
+		ExecutionResults: []*types.ExecutionResult{},
+		MPTWitness:       nil,
+	}
+
+	if err := w.orm.InsertBlockTraces([]*types.BlockTrace{trace}); err != nil {
+		return fmt.Errorf("failed to insert block traces: %v", err)
+	}
+
+	blocks, err := w.orm.GetUnbatchedBlocks(map[string]interface{}{})
+	if err != nil {
+		return err
+	}
+
+	if len(blocks) != 1 {
+		return fmt.Errorf("unexpected number of unbatched blocks in db, expected: 1, actual: %v", len(blocks))
+	}
+
+	batchID, err := w.batchProposer.createBatchForBlocks(blocks)
+	if err != nil {
+		return fmt.Errorf("failed to create batch: %v", err)
+	}
+
+	err = w.orm.UpdateProvingStatus(batchID, orm.ProvingTaskProved)
+	if err != nil {
+		return fmt.Errorf("failed to update genesis batch proving status: %v", err)
+	}
+
+	err = w.orm.UpdateRollupStatus(w.ctx, batchID, orm.RollupFinalized)
+	if err != nil {
+		return fmt.Errorf("failed to update genesis batch rollup status: %v", err)
+	}
+
+	log.Info("successfully imported genesis batch")
+
+	return nil
 }
 
 // Start the Listening process

build/dockerfiles/coordinator.Dockerfile

@@ -1,5 +1,5 @@
 # Build libzkp dependency
-FROM scrolltech/go-rust-builder:go-1.18-rust-nightly-2022-08-23 as chef
+FROM scrolltech/go-rust-builder:go-1.18-rust-nightly-2022-12-10 as chef
 WORKDIR app
 
 FROM chef as planner
@@ -13,10 +13,11 @@ RUN cargo chef cook --release --recipe-path recipe.json
 
 COPY ./common/libzkp/impl .
 RUN cargo build --release
+RUN find ./ | grep libzktrie.so | xargs -i cp {} /app/target/release/
 
 
 # Download Go dependencies
-FROM scrolltech/go-rust-builder:go-1.18-rust-nightly-2022-08-23 as base
+FROM scrolltech/go-rust-builder:go-1.18-rust-nightly-2022-12-10 as base
 WORKDIR /src
 COPY go.work* ./
 COPY ./bridge/go.* ./bridge/
@@ -32,12 +33,14 @@ RUN go mod download -x
 FROM base as builder
 COPY . .
 RUN cp -r ./common/libzkp/interface ./coordinator/verifier/lib
-COPY --from=zkp-builder /app/target/release/libzkp.so ./coordinator/verifier/lib/
+COPY --from=zkp-builder /app/target/release/libzkp.a ./coordinator/verifier/lib/
+COPY --from=zkp-builder /app/target/release/libzktrie.so ./coordinator/verifier/lib/
 RUN cd ./coordinator && go build -v -p 4 -o /bin/coordinator ./cmd && mv verifier/lib /bin/
 
 # Pull coordinator into a second stage deploy alpine container
 FROM ubuntu:20.04
-
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/src/coordinator/verifier/lib
+# ENV CHAIN_ID=534353
 RUN mkdir -p /src/coordinator/verifier/lib
 COPY --from=builder /bin/lib /src/coordinator/verifier/lib
 COPY --from=builder /bin/coordinator /bin/