From 7180d2653068e8a43d776ecb6fdd629facd44f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 29 Oct 2024 10:31:04 +0200 Subject: [PATCH] core, eth, node: break rawdb -> {leveldb, pebble} dependency (#30689) --- core/bench_test.go | 26 +++-- core/blockchain_repair_test.go | 45 ++++---- core/blockchain_sethead_test.go | 12 +- core/blockchain_snapshot_test.go | 23 ++-- core/blockchain_test.go | 46 ++------ core/rawdb/database.go | 101 +---------------- eth/filters/bench_test.go | 189 ------------------------------- eth/filters/filter_test.go | 2 +- node/database.go | 111 ++++++++++++++++++ node/node.go | 6 +- 10 files changed, 184 insertions(+), 377 deletions(-) delete mode 100644 eth/filters/bench_test.go create mode 100644 node/database.go diff --git a/core/bench_test.go b/core/bench_test.go index 639d36e9ae..d0305e268a 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" ) @@ -173,18 +174,16 @@ func genUncles(i int, gen *BlockGen) { func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Create the database in memory or in a temporary directory. var db ethdb.Database - var err error if !disk { db = rawdb.NewMemoryDatabase() } else { - dir := b.TempDir() - db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false) + pdb, err := pebble.New(b.TempDir(), 128, 128, "", false) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } + db = rawdb.NewDatabase(pdb) defer db.Close() } - // Generate a chain of b.N blocks using the supplied block // generator function. gspec := &Genesis{ @@ -281,11 +280,11 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin func benchWriteChain(b *testing.B, full bool, count uint64) { genesis := &Genesis{Config: params.AllEthashProtocolChanges} for i := 0; i < b.N; i++ { - dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false) if err != nil { - b.Fatalf("error opening database at %v: %v", dir, err) + b.Fatalf("error opening database: %v", err) } + db := rawdb.NewDatabase(pdb) makeChainForBench(db, genesis, full, count) db.Close() } @@ -294,10 +293,12 @@ func benchWriteChain(b *testing.B, full bool, count uint64) { func benchReadChain(b *testing.B, full bool, count uint64) { dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + pdb, err := pebble.New(dir, 1024, 128, "", false) if err != nil { - b.Fatalf("error opening database at %v: %v", dir, err) + b.Fatalf("error opening database: %v", err) } + db := rawdb.NewDatabase(pdb) + genesis := &Genesis{Config: params.AllEthashProtocolChanges} makeChainForBench(db, genesis, full, count) db.Close() @@ -308,15 +309,16 @@ func benchReadChain(b *testing.B, full bool, count uint64) { b.ResetTimer() for i := 0; i < b.N; i++ { - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + pdb, err = pebble.New(dir, 1024, 128, "", false) if err != nil { - b.Fatalf("error opening database at %v: %v", dir, err) + b.Fatalf("error opening database: %v", err) } + db = rawdb.NewDatabase(pdb) + chain, err := NewBlockChain(db, &cacheConfig, genesis, nil, ethash.NewFaker(), vm.Config{}, nil) if err != nil { b.Fatalf("error creating chain: %v", err) } - for n := uint64(0); n < count; n++ { header := chain.GetHeaderByNumber(n) if full { diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 8a2dfe9f11..6c52d057ad 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" ) @@ -1764,12 +1765,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - db, err := rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err := pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to create persistent key-value database: %v", err) + } + db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) if err != nil { - t.Fatalf("Failed to create persistent database: %v", err) + t.Fatalf("Failed to create persistent freezer database: %v", err) } defer db.Close() // Might double close, should be fine @@ -1848,12 +1850,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us - db, err = rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err = pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to reopen persistent key-value database: %v", err) + } + db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) if err != nil { - t.Fatalf("Failed to reopen persistent database: %v", err) + t.Fatalf("Failed to reopen persistent freezer database: %v", err) } defer db.Close() @@ -1912,12 +1915,13 @@ func testIssue23496(t *testing.T, scheme string) { datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - db, err := rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err := pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to create persistent key-value database: %v", err) + } + db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) if err != nil { - t.Fatalf("Failed to create persistent database: %v", err) + t.Fatalf("Failed to create persistent freezer database: %v", err) } defer db.Close() // Might double close, should be fine @@ -1969,12 +1973,13 @@ func testIssue23496(t *testing.T, scheme string) { chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us - db, err = rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err = pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to reopen persistent key-value database: %v", err) + } + db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) if err != nil { - t.Fatalf("Failed to reopen persistent database: %v", err) + t.Fatalf("Failed to reopen persistent freezer database: %v", err) } defer db.Close() diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index b72de33896..424854b2bf 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/hashdb" @@ -1968,12 +1969,13 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - db, err := rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err := pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to create persistent key-value database: %v", err) + } + db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) if err != nil { - t.Fatalf("Failed to create persistent database: %v", err) + t.Fatalf("Failed to create persistent freezer database: %v", err) } defer db.Close() diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 120977f222..1a6fe38af6 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" ) @@ -65,12 +66,13 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - db, err := rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err := pebble.New(datadir, 0, 0, "", false) if err != nil { - t.Fatalf("Failed to create persistent database: %v", err) + t.Fatalf("Failed to create persistent key-value database: %v", err) + } + db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) + if err != nil { + t.Fatalf("Failed to create persistent freezer database: %v", err) } // Initialize a fresh chain var ( @@ -255,12 +257,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us - newdb, err := rawdb.Open(rawdb.OpenOptions{ - Directory: snaptest.datadir, - AncientsDirectory: snaptest.ancient, - }) + pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to create persistent key-value database: %v", err) + } + newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false) if err != nil { - t.Fatalf("Failed to reopen persistent database: %v", err) + t.Fatalf("Failed to create persistent freezer database: %v", err) } defer newdb.Close() diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 05629cd19f..dc391bb520 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -29,7 +29,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -40,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" @@ -2663,12 +2663,13 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) { datadir := t.TempDir() ancient := path.Join(datadir, "ancient") - db, err := rawdb.Open(rawdb.OpenOptions{ - Directory: datadir, - AncientsDirectory: ancient, - }) + pdb, err := pebble.New(datadir, 0, 0, "", false) + if err != nil { + t.Fatalf("Failed to create persistent key-value database: %v", err) + } + db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false) if err != nil { - t.Fatalf("Failed to create persistent database: %v", err) + t.Fatalf("Failed to create persistent freezer database: %v", err) } defer db.Close() @@ -4231,36 +4232,3 @@ func TestPragueRequests(t *testing.T) { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } } - -func BenchmarkReorg(b *testing.B) { - chainLength := b.N - - dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 128, "", false) - if err != nil { - b.Fatalf("cannot create temporary database: %v", err) - } - defer db.Close() - gspec := &Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{benchRootAddr: {Balance: math.BigPow(2, 254)}}, - } - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil) - defer blockchain.Stop() - - // Insert an easy and a difficult chain afterwards - easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.GetBlockByHash(blockchain.CurrentBlock().Hash()), ethash.NewFaker(), db, chainLength, genValueTx(50000)) - diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.GetBlockByHash(blockchain.CurrentBlock().Hash()), ethash.NewFaker(), db, chainLength, genValueTx(50000)) - - if _, err := blockchain.InsertChain(easyBlocks); err != nil { - b.Fatalf("failed to insert easy chain: %v", err) - } - b.ResetTimer() - if _, err := blockchain.InsertChain(diffBlocks); err != nil { - b.Fatalf("failed to insert difficult chain: %v", err) - } -} - -// Master: BenchmarkReorg-8 10000 899591 ns/op 820154 B/op 1440 allocs/op 1549443072 bytes of heap used -// WithoutOldChain: BenchmarkReorg-8 10000 1147281 ns/op 943163 B/op 1564 allocs/op 1163870208 bytes of heap used -// WithoutNewChain: BenchmarkReorg-8 10000 1018922 ns/op 943580 B/op 1564 allocs/op 1171890176 bytes of heap used diff --git a/core/rawdb/database.go b/core/rawdb/database.go index e48e523f9e..53418d1646 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -27,9 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" - "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/log" "github.com/olekukonko/tablewriter" ) @@ -299,37 +297,9 @@ func NewMemoryDatabase() ethdb.Database { return NewDatabase(memorydb.New()) } -// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database -// with an initial starting capacity, but without a freezer moving immutable -// chain segments into cold storage. -func NewMemoryDatabaseWithCap(size int) ethdb.Database { - return NewDatabase(memorydb.NewWithCap(size)) -} - -// NewLevelDBDatabase creates a persistent key-value database without a freezer -// moving immutable chain segments into cold storage. -func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := leveldb.New(file, cache, handles, namespace, readonly) - if err != nil { - return nil, err - } - log.Info("Using LevelDB as the backing database") - return NewDatabase(db), nil -} - -// NewPebbleDBDatabase creates a persistent key-value database without a freezer -// moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := pebble.New(file, cache, handles, namespace, readonly) - if err != nil { - return nil, err - } - return NewDatabase(db), nil -} - const ( - dbPebble = "pebble" - dbLeveldb = "leveldb" + DBPebble = "pebble" + DBLeveldb = "leveldb" ) // PreexistingDatabase checks the given data directory whether a database is already @@ -343,72 +313,9 @@ func PreexistingDatabase(path string) string { if err != nil { panic(err) // only possible if the pattern is malformed } - return dbPebble - } - return dbLeveldb -} - -// OpenOptions contains the options to apply when opening a database. -// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used. -type OpenOptions struct { - Type string // "leveldb" | "pebble" - Directory string // the datadir - AncientsDirectory string // the ancients-dir - Namespace string // the namespace for database relevant metrics - Cache int // the capacity(in megabytes) of the data caching - Handles int // number of files to be open simultaneously - ReadOnly bool -} - -// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. -// -// type == null type != null -// +---------------------------------------- -// db is non-existent | pebble default | specified type -// db is existent | from db | specified type (if compatible) -func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { - // Reject any unsupported database type - if len(o.Type) != 0 && o.Type != dbLeveldb && o.Type != dbPebble { - return nil, fmt.Errorf("unknown db.engine %v", o.Type) - } - // Retrieve any pre-existing database's type and use that or the requested one - // as long as there's no conflict between the two types - existingDb := PreexistingDatabase(o.Directory) - if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { - return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) - } - if o.Type == dbPebble || existingDb == dbPebble { - log.Info("Using pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) - } - if o.Type == dbLeveldb || existingDb == dbLeveldb { - log.Info("Using leveldb as the backing database") - return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) - } - // No pre-existing database, no user-requested one either. Default to Pebble. - log.Info("Defaulting to pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) -} - -// Open opens both a disk-based key-value database such as leveldb or pebble, but also -// integrates it with a freezer database -- if the AncientDir option has been -// set on the provided OpenOptions. -// The passed o.AncientDir indicates the path of root ancient directory where -// the chain freezer can be opened. -func Open(o OpenOptions) (ethdb.Database, error) { - kvdb, err := openKeyValueDatabase(o) - if err != nil { - return nil, err - } - if len(o.AncientsDirectory) == 0 { - return kvdb, nil - } - frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly) - if err != nil { - kvdb.Close() - return nil, err + return DBPebble } - return frdb, nil + return DBLeveldb } type counter uint64 diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go deleted file mode 100644 index 73b96b77af..0000000000 --- a/eth/filters/bench_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filters - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/node" -) - -func BenchmarkBloomBits512(b *testing.B) { - benchmarkBloomBits(b, 512) -} - -func BenchmarkBloomBits1k(b *testing.B) { - benchmarkBloomBits(b, 1024) -} - -func BenchmarkBloomBits2k(b *testing.B) { - benchmarkBloomBits(b, 2048) -} - -func BenchmarkBloomBits4k(b *testing.B) { - benchmarkBloomBits(b, 4096) -} - -func BenchmarkBloomBits8k(b *testing.B) { - benchmarkBloomBits(b, 8192) -} - -func BenchmarkBloomBits16k(b *testing.B) { - benchmarkBloomBits(b, 16384) -} - -func BenchmarkBloomBits32k(b *testing.B) { - benchmarkBloomBits(b, 32768) -} - -const benchFilterCnt = 2000 - -func benchmarkBloomBits(b *testing.B, sectionSize uint64) { - b.Skip("test disabled: this tests presume (and modify) an existing datadir.") - benchDataDir := node.DefaultDataDir() + "/geth/chaindata" - b.Log("Running bloombits benchmark section size:", sectionSize) - - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) - if err != nil { - b.Fatalf("error opening database at %v: %v", benchDataDir, err) - } - head := rawdb.ReadHeadBlockHash(db) - if head == (common.Hash{}) { - b.Fatalf("chain data not found at %v", benchDataDir) - } - - clearBloomBits(db) - b.Log("Generating bloombits data...") - headNum := rawdb.ReadHeaderNumber(db, head) - if headNum == nil || *headNum < sectionSize+512 { - b.Fatalf("not enough blocks for running a benchmark") - } - - start := time.Now() - cnt := (*headNum - 512) / sectionSize - var dataSize, compSize uint64 - for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { - bc, err := bloombits.NewGenerator(uint(sectionSize)) - if err != nil { - b.Fatalf("failed to create generator: %v", err) - } - var header *types.Header - for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { - hash := rawdb.ReadCanonicalHash(db, i) - if header = rawdb.ReadHeader(db, hash, i); header == nil { - b.Fatalf("Error creating bloomBits data") - return - } - bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) - } - sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*sectionSize-1) - for i := 0; i < types.BloomBitLength; i++ { - data, err := bc.Bitset(uint(i)) - if err != nil { - b.Fatalf("failed to retrieve bitset: %v", err) - } - comp := bitutil.CompressBytes(data) - dataSize += uint64(len(data)) - compSize += uint64(len(comp)) - rawdb.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) - } - //if sectionIdx%50 == 0 { - // b.Log(" section", sectionIdx, "/", cnt) - //} - } - - d := time.Since(start) - b.Log("Finished generating bloombits data") - b.Log(" ", d, "total ", d/time.Duration(cnt*sectionSize), "per block") - b.Log(" data size:", dataSize, " compressed size:", compSize, " compression ratio:", float64(compSize)/float64(dataSize)) - - b.Log("Running filter benchmarks...") - start = time.Now() - - var ( - backend *testBackend - sys *FilterSystem - ) - for i := 0; i < benchFilterCnt; i++ { - if i%20 == 0 { - db.Close() - db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) - backend = &testBackend{db: db, sections: cnt} - sys = NewFilterSystem(backend, Config{}) - } - var addr common.Address - addr[0] = byte(i) - addr[1] = byte(i / 256) - filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) - if _, err := filter.Logs(context.Background()); err != nil { - b.Error("filter.Logs error:", err) - } - } - - d = time.Since(start) - b.Log("Finished running filter benchmarks") - b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") - db.Close() -} - -//nolint:unused -func clearBloomBits(db ethdb.Database) { - var bloomBitsPrefix = []byte("bloomBits-") - fmt.Println("Clearing bloombits data...") - it := db.NewIterator(bloomBitsPrefix, nil) - for it.Next() { - db.Delete(it.Key()) - } - it.Release() -} - -func BenchmarkNoBloomBits(b *testing.B) { - b.Skip("test disabled: this tests presume (and modify) an existing datadir.") - benchDataDir := node.DefaultDataDir() + "/geth/chaindata" - b.Log("Running benchmark without bloombits") - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) - if err != nil { - b.Fatalf("error opening database at %v: %v", benchDataDir, err) - } - head := rawdb.ReadHeadBlockHash(db) - if head == (common.Hash{}) { - b.Fatalf("chain data not found at %v", benchDataDir) - } - headNum := rawdb.ReadHeaderNumber(db, head) - - clearBloomBits(db) - - _, sys := newTestFilterSystem(b, db, Config{}) - - b.Log("Running filter benchmarks...") - start := time.Now() - filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil) - filter.Logs(context.Background()) - d := time.Since(start) - b.Log("Finished running filter benchmarks") - b.Log(" ", d, "total ", d*time.Duration(1000000)/time.Duration(*headNum+1), "per million blocks") - db.Close() -} diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index d8b703fee4..6a3057326d 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -48,7 +48,7 @@ func makeReceipt(addr common.Address) *types.Receipt { func BenchmarkFilters(b *testing.B) { var ( - db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false) + db = rawdb.NewMemoryDatabase() _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) diff --git a/node/database.go b/node/database.go new file mode 100644 index 0000000000..e3ccb91066 --- /dev/null +++ b/node/database.go @@ -0,0 +1,111 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package node + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/pebble" + "github.com/ethereum/go-ethereum/log" +) + +// openOptions contains the options to apply when opening a database. +// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used. +type openOptions struct { + Type string // "leveldb" | "pebble" + Directory string // the datadir + AncientsDirectory string // the ancients-dir + Namespace string // the namespace for database relevant metrics + Cache int // the capacity(in megabytes) of the data caching + Handles int // number of files to be open simultaneously + ReadOnly bool +} + +// openDatabase opens both a disk-based key-value database such as leveldb or pebble, but also +// integrates it with a freezer database -- if the AncientDir option has been +// set on the provided OpenOptions. +// The passed o.AncientDir indicates the path of root ancient directory where +// the chain freezer can be opened. +func openDatabase(o openOptions) (ethdb.Database, error) { + kvdb, err := openKeyValueDatabase(o) + if err != nil { + return nil, err + } + if len(o.AncientsDirectory) == 0 { + return kvdb, nil + } + frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly) + if err != nil { + kvdb.Close() + return nil, err + } + return frdb, nil +} + +// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. +// +// type == null type != null +// +---------------------------------------- +// db is non-existent | pebble default | specified type +// db is existent | from db | specified type (if compatible) +func openKeyValueDatabase(o openOptions) (ethdb.Database, error) { + // Reject any unsupported database type + if len(o.Type) != 0 && o.Type != rawdb.DBLeveldb && o.Type != rawdb.DBPebble { + return nil, fmt.Errorf("unknown db.engine %v", o.Type) + } + // Retrieve any pre-existing database's type and use that or the requested one + // as long as there's no conflict between the two types + existingDb := rawdb.PreexistingDatabase(o.Directory) + if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { + return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) + } + if o.Type == rawdb.DBPebble || existingDb == rawdb.DBPebble { + log.Info("Using pebble as the backing database") + return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + } + if o.Type == rawdb.DBLeveldb || existingDb == rawdb.DBLeveldb { + log.Info("Using leveldb as the backing database") + return newLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + } + // No pre-existing database, no user-requested one either. Default to Pebble. + log.Info("Defaulting to pebble as the backing database") + return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) +} + +// newLevelDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func newLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + db, err := leveldb.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + log.Info("Using LevelDB as the backing database") + return rawdb.NewDatabase(db), nil +} + +// newPebbleDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + return rawdb.NewDatabase(db), nil +} diff --git a/node/node.go b/node/node.go index 633f88f058..e23425dfbd 100644 --- a/node/node.go +++ b/node/node.go @@ -723,7 +723,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r if n.config.DataDir == "" { db = rawdb.NewMemoryDatabase() } else { - db, err = rawdb.Open(rawdb.OpenOptions{ + db, err = openDatabase(openOptions{ Type: n.config.DBEngine, Directory: n.ResolvePath(name), Namespace: namespace, @@ -732,7 +732,6 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r ReadOnly: readonly, }) } - if err == nil { db = n.wrapDatabase(db) } @@ -755,7 +754,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient if n.config.DataDir == "" { db, err = rawdb.NewDatabaseWithFreezer(memorydb.New(), "", namespace, readonly) } else { - db, err = rawdb.Open(rawdb.OpenOptions{ + db, err = openDatabase(openOptions{ Type: n.config.DBEngine, Directory: n.ResolvePath(name), AncientsDirectory: n.ResolveAncient(name, ancient), @@ -765,7 +764,6 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient ReadOnly: readonly, }) } - if err == nil { db = n.wrapDatabase(db) }