core, core/bloombits: fix some issues in the bloombits chain indexer

cmd/geth/main.go

@@ -1,20 +1,3 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// geth is a command-line client for Ethereum.
 package main
 
 import (

core/bloom_indexer.go

@@ -1,23 +1,8 @@
-// Copyright 2021 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package core
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -39,7 +24,7 @@ const (
 type BloomIndexer struct {
 	size    uint64               // section size to generate bloombits for
 	db      ethdb.Database       // database instance to write index data and metadata into
-	gen     *bloombits.Generator // generator to rotate the bloom bits crating the bloom index
+	gen     *bloombits.Generator // generator to rotate the bloom bits creating the bloom index
 	section uint64               // Section is the section number being processed currently
 	head    common.Hash          // Head is the hash of the last header processed
 }
@@ -60,28 +45,41 @@ func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *ChainIndexer {
 // section.
 func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error {
 	gen, err := bloombits.NewGenerator(uint(b.size))
+	if err != nil {
+		return err
+	}
 	b.gen, b.section, b.head = gen, section, common.Hash{}
-	return err
+	return nil
 }
 
 // Process implements core.ChainIndexerBackend, adding a new header's bloom into
 // the index.
 func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error {
-	b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom)
+	if b.gen == nil {
+		return errors.New("bloom generator is not initialized")
+	}
+	if err := b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom); err != nil {
+		return err
+	}
 	b.head = header.Hash()
 	return nil
 }
 
 // Commit implements core.ChainIndexerBackend, finalizing the bloom section and
 // writing it out into the database.
 func (b *BloomIndexer) Commit() error {
+	if b.gen == nil {
+		return errors.New("bloom generator is not initialized")
+	}
 	batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength)
 	for i := 0; i < types.BloomBitLength; i++ {
 		bits, err := b.gen.Bitset(uint(i))
 		if err != nil {
 			return err
 		}
-		rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits))
+		if err := rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)); err != nil {
+			return err
+		}
 	}
 	return batch.Write()
 }

core/bloombits/generator_test.go

@@ -1,19 +1,3 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package bloombits
 
 import (
@@ -98,3 +82,43 @@ func BenchmarkGenerator(b *testing.B) {
 		}
 	})
 }
+
+// TestGeneratorEdgeCases tests edge cases for the bloom generator.
+func TestGeneratorEdgeCases(t *testing.T) {
+	// Test with zero sections
+	_, err := NewGenerator(0)
+	if err == nil {
+		t.Fatal("expected error for zero sections, got nil")
+	}
+
+	// Test with non-multiple of 8 sections
+	_, err = NewGenerator(7)
+	if err == nil {
+		t.Fatal("expected error for non-multiple of 8 sections, got nil")
+	}
+
+	// Test with valid sections
+	gen, err := NewGenerator(8)
+	if err != nil {
+		t.Fatalf("failed to create bloombit generator: %v", err)
+	}
+
+	// Test adding bloom with unexpected index
+	err = gen.AddBloom(1, types.Bloom{})
+	if err == nil {
+		t.Fatal("expected error for unexpected index, got nil")
+	}
+
+	// Test retrieving bitset before fully generated
+	_, err = gen.Bitset(0)
+	if err == nil {
+		t.Fatal("expected error for bloom not fully generated, got nil")
+	}
+
+	// Test retrieving bitset with out of bounds index
+	gen.AddBloom(0, types.Bloom{})
+	_, err = gen.Bitset(types.BloomBitLength)
+	if err == nil {
+		t.Fatal("expected error for bloom bit out of bounds, got nil")
+	}
+}

core/bloombits/matcher_test.go

@@ -1,19 +1,3 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package bloombits
 
 import (
@@ -290,3 +274,39 @@ func expMatch3(filter [][]bloomIndexes, i uint64) bool {
 	}
 	return true
 }
+
+// TestMatcherEdgeCases tests edge cases for the matcher.
+func TestMatcherEdgeCases(t *testing.T) {
+	t.Parallel()
+
+	// Test with zero sections
+	_, err := NewMatcher(0, nil)
+	if err == nil {
+		t.Fatal("expected error for zero sections, got nil")
+	}
+
+	// Test with non-multiple of 8 sections
+	_, err = NewMatcher(7, nil)
+	if err == nil {
+		t.Fatal("expected error for non-multiple of 8 sections, got nil")
+	}
+
+	// Test with valid sections
+	matcher := NewMatcher(8, nil)
+	if matcher == nil {
+		t.Fatal("failed to create matcher with valid sections")
+	}
+
+	// Test starting a session with invalid range
+	_, err = matcher.Start(context.Background(), 10, 5, make(chan uint64))
+	if err == nil {
+		t.Fatal("expected error for invalid range, got nil")
+	}
+
+	// Test starting a session with valid range
+	session, err := matcher.Start(context.Background(), 0, 7, make(chan uint64))
+	if err != nil {
+		t.Fatalf("failed to start matcher session: %v", err)
+	}
+	session.Close()
+}

core/bloombits/scheduler_test.go

@@ -1,19 +1,3 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package bloombits
 
 import (
@@ -101,3 +85,42 @@ func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
 		t.Errorf("request count mismatch: have %v, want %v", have, requests)
 	}
 }
+
+// TestSchedulerEdgeCases tests edge cases for the scheduler.
+func TestSchedulerEdgeCases(t *testing.T) {
+	t.Parallel()
+
+	// Test with zero sections
+	_, err := newScheduler(0)
+	if err == nil {
+		t.Fatal("expected error for zero sections, got nil")
+	}
+
+	// Test with non-multiple of 8 sections
+	_, err = newScheduler(7)
+	if err == nil {
+		t.Fatal("expected error for non-multiple of 8 sections, got nil")
+	}
+
+	// Test with valid sections
+	scheduler := newScheduler(8)
+	if scheduler == nil {
+		t.Fatal("failed to create scheduler with valid sections")
+	}
+
+	// Test running the scheduler with invalid channels
+	in := make(chan uint64, 16)
+	out := make(chan []byte, 16)
+	quit := make(chan struct{})
+	var pend sync.WaitGroup
+	pend.Add(1)
+	go func() {
+		defer pend.Done()
+		scheduler.run(in, nil, out, quit, &pend)
+	}()
+
+	// Test sending invalid data to the scheduler
+	in <- 1
+	close(in)
+	pend.Wait()
+}

core/chain_indexer_test.go

@@ -1,19 +1,3 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package core
 
 import (
@@ -244,3 +228,85 @@ func (b *testChainIndexBackend) Commit() error {
 func (b *testChainIndexBackend) Prune(threshold uint64) error {
 	return nil
 }
+
+// TestChainIndexerReorg tests the chain indexer for handling reorgs correctly.
+func TestChainIndexerReorg(t *testing.T) {
+	db := rawdb.NewMemoryDatabase()
+	defer db.Close()
+
+	// Create a chain indexer
+	sectionSize := uint64(10)
+	confirmsReq := uint64(5)
+	backend := &testChainIndexBackend{t: t, processCh: make(chan uint64)}
+	indexer := NewChainIndexer(db, rawdb.NewTable(db, "test"), backend, sectionSize, confirmsReq, 0, "indexer")
+	defer indexer.Close()
+
+	// Function to inject headers into the database
+	inject := func(number uint64) {
+		header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()}
+		if number > 0 {
+			header.ParentHash = rawdb.ReadCanonicalHash(db, number-1)
+		}
+		rawdb.WriteHeader(db, header)
+		rawdb.WriteCanonicalHash(db, header.Hash(), number)
+	}
+
+	// Function to notify the indexer about new heads
+	notify := func(headNum uint64, reorg bool) {
+		indexer.newHead(headNum, reorg)
+	}
+
+	// Inject initial chain
+	for i := uint64(0); i <= 50; i++ {
+		inject(i)
+	}
+	notify(50, false)
+
+	// Reorg the chain
+	notify(25, true)
+
+	// Create new fork
+	for i := uint64(26); i <= 50; i++ {
+		inject(i)
+		notify(i, false)
+	}
+}
+
+// TestChainIndexerPrune tests the chain indexer for pruning old sections correctly.
+func TestChainIndexerPrune(t *testing.T) {
+	db := rawdb.NewMemoryDatabase()
+	defer db.Close()
+
+	// Create a chain indexer
+	sectionSize := uint64(10)
+	confirmsReq := uint64(5)
+	backend := &testChainIndexBackend{t: t, processCh: make(chan uint64)}
+	indexer := NewChainIndexer(db, rawdb.NewTable(db, "test"), backend, sectionSize, confirmsReq, 0, "indexer")
+	defer indexer.Close()
+
+	// Function to inject headers into the database
+	inject := func(number uint64) {
+		header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()}
+		if number > 0 {
+			header.ParentHash = rawdb.ReadCanonicalHash(db, number-1)
+		}
+		rawdb.WriteHeader(db, header)
+		rawdb.WriteCanonicalHash(db, header.Hash(), number)
+	}
+
+	// Function to notify the indexer about new heads
+	notify := func(headNum uint64, reorg bool) {
+		indexer.newHead(headNum, reorg)
+	}
+
+	// Inject initial chain
+	for i := uint64(0); i <= 50; i++ {
+		inject(i)
+	}
+	notify(50, false)
+
+	// Prune old sections
+	if err := indexer.Prune(20); err != nil {
+		t.Fatalf("Failed to prune sections: %v", err)
+	}
+}