mirror of https://github.com/ethereum/go-ethereum
core, eth, node: break rawdb -> {leveldb, pebble} dependency (#30689)
Péter Szilágyi
3 weeks ago
committed by
GitHub
parent
98056e1ef2
commit
7180d26530
@ -1,189 +0,0 @@ |
|||||||
// Copyright 2017 The go-ethereum Authors
|
|
||||||
// This file is part of the go-ethereum library.
|
|
||||||
//
|
|
||||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Lesser General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Lesser General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Lesser General Public License
|
|
||||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package filters |
|
||||||
|
|
||||||
import ( |
|
||||||
"context" |
|
||||||
"fmt" |
|
||||||
"testing" |
|
||||||
"time" |
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common" |
|
||||||
"github.com/ethereum/go-ethereum/common/bitutil" |
|
||||||
"github.com/ethereum/go-ethereum/core/bloombits" |
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb" |
|
||||||
"github.com/ethereum/go-ethereum/core/types" |
|
||||||
"github.com/ethereum/go-ethereum/ethdb" |
|
||||||
"github.com/ethereum/go-ethereum/node" |
|
||||||
) |
|
||||||
|
|
||||||
func BenchmarkBloomBits512(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 512) |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkBloomBits1k(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 1024) |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkBloomBits2k(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 2048) |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkBloomBits4k(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 4096) |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkBloomBits8k(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 8192) |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkBloomBits16k(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 16384) |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkBloomBits32k(b *testing.B) { |
|
||||||
benchmarkBloomBits(b, 32768) |
|
||||||
} |
|
||||||
|
|
||||||
const benchFilterCnt = 2000 |
|
||||||
|
|
||||||
func benchmarkBloomBits(b *testing.B, sectionSize uint64) { |
|
||||||
b.Skip("test disabled: this tests presume (and modify) an existing datadir.") |
|
||||||
benchDataDir := node.DefaultDataDir() + "/geth/chaindata" |
|
||||||
b.Log("Running bloombits benchmark section size:", sectionSize) |
|
||||||
|
|
||||||
db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) |
|
||||||
if err != nil { |
|
||||||
b.Fatalf("error opening database at %v: %v", benchDataDir, err) |
|
||||||
} |
|
||||||
head := rawdb.ReadHeadBlockHash(db) |
|
||||||
if head == (common.Hash{}) { |
|
||||||
b.Fatalf("chain data not found at %v", benchDataDir) |
|
||||||
} |
|
||||||
|
|
||||||
clearBloomBits(db) |
|
||||||
b.Log("Generating bloombits data...") |
|
||||||
headNum := rawdb.ReadHeaderNumber(db, head) |
|
||||||
if headNum == nil || *headNum < sectionSize+512 { |
|
||||||
b.Fatalf("not enough blocks for running a benchmark") |
|
||||||
} |
|
||||||
|
|
||||||
start := time.Now() |
|
||||||
cnt := (*headNum - 512) / sectionSize |
|
||||||
var dataSize, compSize uint64 |
|
||||||
for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { |
|
||||||
bc, err := bloombits.NewGenerator(uint(sectionSize)) |
|
||||||
if err != nil { |
|
||||||
b.Fatalf("failed to create generator: %v", err) |
|
||||||
} |
|
||||||
var header *types.Header |
|
||||||
for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { |
|
||||||
hash := rawdb.ReadCanonicalHash(db, i) |
|
||||||
if header = rawdb.ReadHeader(db, hash, i); header == nil { |
|
||||||
b.Fatalf("Error creating bloomBits data") |
|
||||||
return |
|
||||||
} |
|
||||||
bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) |
|
||||||
} |
|
||||||
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*sectionSize-1) |
|
||||||
for i := 0; i < types.BloomBitLength; i++ { |
|
||||||
data, err := bc.Bitset(uint(i)) |
|
||||||
if err != nil { |
|
||||||
b.Fatalf("failed to retrieve bitset: %v", err) |
|
||||||
} |
|
||||||
comp := bitutil.CompressBytes(data) |
|
||||||
dataSize += uint64(len(data)) |
|
||||||
compSize += uint64(len(comp)) |
|
||||||
rawdb.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) |
|
||||||
} |
|
||||||
//if sectionIdx%50 == 0 {
|
|
||||||
// b.Log(" section", sectionIdx, "/", cnt)
|
|
||||||
//}
|
|
||||||
} |
|
||||||
|
|
||||||
d := time.Since(start) |
|
||||||
b.Log("Finished generating bloombits data") |
|
||||||
b.Log(" ", d, "total ", d/time.Duration(cnt*sectionSize), "per block") |
|
||||||
b.Log(" data size:", dataSize, " compressed size:", compSize, " compression ratio:", float64(compSize)/float64(dataSize)) |
|
||||||
|
|
||||||
b.Log("Running filter benchmarks...") |
|
||||||
start = time.Now() |
|
||||||
|
|
||||||
var ( |
|
||||||
backend *testBackend |
|
||||||
sys *FilterSystem |
|
||||||
) |
|
||||||
for i := 0; i < benchFilterCnt; i++ { |
|
||||||
if i%20 == 0 { |
|
||||||
db.Close() |
|
||||||
db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) |
|
||||||
backend = &testBackend{db: db, sections: cnt} |
|
||||||
sys = NewFilterSystem(backend, Config{}) |
|
||||||
} |
|
||||||
var addr common.Address |
|
||||||
addr[0] = byte(i) |
|
||||||
addr[1] = byte(i / 256) |
|
||||||
filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) |
|
||||||
if _, err := filter.Logs(context.Background()); err != nil { |
|
||||||
b.Error("filter.Logs error:", err) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
d = time.Since(start) |
|
||||||
b.Log("Finished running filter benchmarks") |
|
||||||
b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") |
|
||||||
db.Close() |
|
||||||
} |
|
||||||
|
|
||||||
//nolint:unused
|
|
||||||
func clearBloomBits(db ethdb.Database) { |
|
||||||
var bloomBitsPrefix = []byte("bloomBits-") |
|
||||||
fmt.Println("Clearing bloombits data...") |
|
||||||
it := db.NewIterator(bloomBitsPrefix, nil) |
|
||||||
for it.Next() { |
|
||||||
db.Delete(it.Key()) |
|
||||||
} |
|
||||||
it.Release() |
|
||||||
} |
|
||||||
|
|
||||||
func BenchmarkNoBloomBits(b *testing.B) { |
|
||||||
b.Skip("test disabled: this tests presume (and modify) an existing datadir.") |
|
||||||
benchDataDir := node.DefaultDataDir() + "/geth/chaindata" |
|
||||||
b.Log("Running benchmark without bloombits") |
|
||||||
db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) |
|
||||||
if err != nil { |
|
||||||
b.Fatalf("error opening database at %v: %v", benchDataDir, err) |
|
||||||
} |
|
||||||
head := rawdb.ReadHeadBlockHash(db) |
|
||||||
if head == (common.Hash{}) { |
|
||||||
b.Fatalf("chain data not found at %v", benchDataDir) |
|
||||||
} |
|
||||||
headNum := rawdb.ReadHeaderNumber(db, head) |
|
||||||
|
|
||||||
clearBloomBits(db) |
|
||||||
|
|
||||||
_, sys := newTestFilterSystem(b, db, Config{}) |
|
||||||
|
|
||||||
b.Log("Running filter benchmarks...") |
|
||||||
start := time.Now() |
|
||||||
filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil) |
|
||||||
filter.Logs(context.Background()) |
|
||||||
d := time.Since(start) |
|
||||||
b.Log("Finished running filter benchmarks") |
|
||||||
b.Log(" ", d, "total ", d*time.Duration(1000000)/time.Duration(*headNum+1), "per million blocks") |
|
||||||
db.Close() |
|
||||||
} |
|
||||||
@ -0,0 +1,111 @@ |
|||||||
|
// Copyright 2024 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package node |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/core/rawdb" |
||||||
|
"github.com/ethereum/go-ethereum/ethdb" |
||||||
|
"github.com/ethereum/go-ethereum/ethdb/leveldb" |
||||||
|
"github.com/ethereum/go-ethereum/ethdb/pebble" |
||||||
|
"github.com/ethereum/go-ethereum/log" |
||||||
|
) |
||||||
|
|
||||||
|
// openOptions contains the options to apply when opening a database.
|
||||||
|
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
|
||||||
|
type openOptions struct { |
||||||
|
Type string // "leveldb" | "pebble"
|
||||||
|
Directory string // the datadir
|
||||||
|
AncientsDirectory string // the ancients-dir
|
||||||
|
Namespace string // the namespace for database relevant metrics
|
||||||
|
Cache int // the capacity(in megabytes) of the data caching
|
||||||
|
Handles int // number of files to be open simultaneously
|
||||||
|
ReadOnly bool |
||||||
|
} |
||||||
|
|
||||||
|
// openDatabase opens both a disk-based key-value database such as leveldb or pebble, but also
|
||||||
|
// integrates it with a freezer database -- if the AncientDir option has been
|
||||||
|
// set on the provided OpenOptions.
|
||||||
|
// The passed o.AncientDir indicates the path of root ancient directory where
|
||||||
|
// the chain freezer can be opened.
|
||||||
|
func openDatabase(o openOptions) (ethdb.Database, error) { |
||||||
|
kvdb, err := openKeyValueDatabase(o) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
if len(o.AncientsDirectory) == 0 { |
||||||
|
return kvdb, nil |
||||||
|
} |
||||||
|
frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly) |
||||||
|
if err != nil { |
||||||
|
kvdb.Close() |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
return frdb, nil |
||||||
|
} |
||||||
|
|
||||||
|
// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
|
||||||
|
//
|
||||||
|
// type == null type != null
|
||||||
|
// +----------------------------------------
|
||||||
|
// db is non-existent | pebble default | specified type
|
||||||
|
// db is existent | from db | specified type (if compatible)
|
||||||
|
func openKeyValueDatabase(o openOptions) (ethdb.Database, error) { |
||||||
|
// Reject any unsupported database type
|
||||||
|
if len(o.Type) != 0 && o.Type != rawdb.DBLeveldb && o.Type != rawdb.DBPebble { |
||||||
|
return nil, fmt.Errorf("unknown db.engine %v", o.Type) |
||||||
|
} |
||||||
|
// Retrieve any pre-existing database's type and use that or the requested one
|
||||||
|
// as long as there's no conflict between the two types
|
||||||
|
existingDb := rawdb.PreexistingDatabase(o.Directory) |
||||||
|
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { |
||||||
|
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) |
||||||
|
} |
||||||
|
if o.Type == rawdb.DBPebble || existingDb == rawdb.DBPebble { |
||||||
|
log.Info("Using pebble as the backing database") |
||||||
|
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) |
||||||
|
} |
||||||
|
if o.Type == rawdb.DBLeveldb || existingDb == rawdb.DBLeveldb { |
||||||
|
log.Info("Using leveldb as the backing database") |
||||||
|
return newLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) |
||||||
|
} |
||||||
|
// No pre-existing database, no user-requested one either. Default to Pebble.
|
||||||
|
log.Info("Defaulting to pebble as the backing database") |
||||||
|
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) |
||||||
|
} |
||||||
|
|
||||||
|
// newLevelDBDatabase creates a persistent key-value database without a freezer
|
||||||
|
// moving immutable chain segments into cold storage.
|
||||||
|
func newLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { |
||||||
|
db, err := leveldb.New(file, cache, handles, namespace, readonly) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
log.Info("Using LevelDB as the backing database") |
||||||
|
return rawdb.NewDatabase(db), nil |
||||||
|
} |
||||||
|
|
||||||
|
// newPebbleDBDatabase creates a persistent key-value database without a freezer
|
||||||
|
// moving immutable chain segments into cold storage.
|
||||||
|
func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { |
||||||
|
db, err := pebble.New(file, cache, handles, namespace, readonly) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
return rawdb.NewDatabase(db), nil |
||||||
|
} |
||||||
Loading…
Reference in new issue