mirror of https://github.com/ethereum/go-ethereum
all: port boring changes from pbss (#27176)
* all: port boring changes from pbss * core, trie: address comments from martin * trie: minor fixes * core/rawdb: update comment * core, eth, tests, trie: address comments * tests, trie: add extra check when update trie database * trie/triedb/hashdb: degrade the error to warningpull/27230/head
rjl493456442
1 year ago
committed by
GitHub
parent
81d328a73e
commit
5021d36d35
@ -0,0 +1,267 @@ |
|||||||
|
// Copyright 2022 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package trie |
||||||
|
|
||||||
|
import ( |
||||||
|
"errors" |
||||||
|
"runtime" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/VictoriaMetrics/fastcache" |
||||||
|
"github.com/ethereum/go-ethereum/common" |
||||||
|
"github.com/ethereum/go-ethereum/ethdb" |
||||||
|
"github.com/ethereum/go-ethereum/log" |
||||||
|
"github.com/ethereum/go-ethereum/trie/triedb/hashdb" |
||||||
|
"github.com/ethereum/go-ethereum/trie/trienode" |
||||||
|
) |
||||||
|
|
||||||
|
// Config defines all necessary options for database.
|
||||||
|
type Config struct { |
||||||
|
Cache int // Memory allowance (MB) to use for caching trie nodes in memory
|
||||||
|
Journal string // Journal of clean cache to survive node restarts
|
||||||
|
Preimages bool // Flag whether the preimage of trie key is recorded
|
||||||
|
} |
||||||
|
|
||||||
|
// backend defines the methods needed to access/update trie nodes in different
|
||||||
|
// state scheme.
|
||||||
|
type backend interface { |
||||||
|
// Scheme returns the identifier of used storage scheme.
|
||||||
|
Scheme() string |
||||||
|
|
||||||
|
// Initialized returns an indicator if the state data is already initialized
|
||||||
|
// according to the state scheme.
|
||||||
|
Initialized(genesisRoot common.Hash) bool |
||||||
|
|
||||||
|
// Size returns the current storage size of the memory cache in front of the
|
||||||
|
// persistent database layer.
|
||||||
|
Size() common.StorageSize |
||||||
|
|
||||||
|
// Update performs a state transition by committing dirty nodes contained
|
||||||
|
// in the given set in order to update state from the specified parent to
|
||||||
|
// the specified root.
|
||||||
|
Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error |
||||||
|
|
||||||
|
// Commit writes all relevant trie nodes belonging to the specified state
|
||||||
|
// to disk. Report specifies whether logs will be displayed in info level.
|
||||||
|
Commit(root common.Hash, report bool) error |
||||||
|
|
||||||
|
// Close closes the trie database backend and releases all held resources.
|
||||||
|
Close() error |
||||||
|
} |
||||||
|
|
||||||
|
// Database is the wrapper of the underlying backend which is shared by different
|
||||||
|
// types of node backend as an entrypoint. It's responsible for all interactions
|
||||||
|
// relevant with trie nodes and node preimages.
|
||||||
|
type Database struct { |
||||||
|
config *Config // Configuration for trie database
|
||||||
|
diskdb ethdb.Database // Persistent database to store the snapshot
|
||||||
|
cleans *fastcache.Cache // Megabytes permitted using for read caches
|
||||||
|
preimages *preimageStore // The store for caching preimages
|
||||||
|
backend backend // The backend for managing trie nodes
|
||||||
|
} |
||||||
|
|
||||||
|
// prepare initializes the database with provided configs, but the
|
||||||
|
// database backend is still left as nil.
|
||||||
|
func prepare(diskdb ethdb.Database, config *Config) *Database { |
||||||
|
var cleans *fastcache.Cache |
||||||
|
if config != nil && config.Cache > 0 { |
||||||
|
if config.Journal == "" { |
||||||
|
cleans = fastcache.New(config.Cache * 1024 * 1024) |
||||||
|
} else { |
||||||
|
cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) |
||||||
|
} |
||||||
|
} |
||||||
|
var preimages *preimageStore |
||||||
|
if config != nil && config.Preimages { |
||||||
|
preimages = newPreimageStore(diskdb) |
||||||
|
} |
||||||
|
return &Database{ |
||||||
|
config: config, |
||||||
|
diskdb: diskdb, |
||||||
|
cleans: cleans, |
||||||
|
preimages: preimages, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// NewDatabase initializes the trie database with default settings, namely
|
||||||
|
// the legacy hash-based scheme is used by default.
|
||||||
|
func NewDatabase(diskdb ethdb.Database) *Database { |
||||||
|
return NewDatabaseWithConfig(diskdb, nil) |
||||||
|
} |
||||||
|
|
||||||
|
// NewDatabaseWithConfig initializes the trie database with provided configs.
|
||||||
|
// The path-based scheme is not activated yet, always initialized with legacy
|
||||||
|
// hash-based scheme by default.
|
||||||
|
func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database { |
||||||
|
db := prepare(diskdb, config) |
||||||
|
db.backend = hashdb.New(diskdb, db.cleans, mptResolver{}) |
||||||
|
return db |
||||||
|
} |
||||||
|
|
||||||
|
// Reader returns a reader for accessing all trie nodes with provided state root.
|
||||||
|
// Nil is returned in case the state is not available.
|
||||||
|
func (db *Database) Reader(blockRoot common.Hash) Reader { |
||||||
|
return db.backend.(*hashdb.Database).Reader(blockRoot) |
||||||
|
} |
||||||
|
|
||||||
|
// Update performs a state transition by committing dirty nodes contained in the
|
||||||
|
// given set in order to update state from the specified parent to the specified
|
||||||
|
// root. The held pre-images accumulated up to this point will be flushed in case
|
||||||
|
// the size exceeds the threshold.
|
||||||
|
func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error { |
||||||
|
if db.preimages != nil { |
||||||
|
db.preimages.commit(false) |
||||||
|
} |
||||||
|
return db.backend.Update(root, parent, nodes) |
||||||
|
} |
||||||
|
|
||||||
|
// Commit iterates over all the children of a particular node, writes them out
|
||||||
|
// to disk. As a side effect, all pre-images accumulated up to this point are
|
||||||
|
// also written.
|
||||||
|
func (db *Database) Commit(root common.Hash, report bool) error { |
||||||
|
if db.preimages != nil { |
||||||
|
db.preimages.commit(true) |
||||||
|
} |
||||||
|
return db.backend.Commit(root, report) |
||||||
|
} |
||||||
|
|
||||||
|
// Size returns the storage size of dirty trie nodes in front of the persistent
|
||||||
|
// database and the size of cached preimages.
|
||||||
|
func (db *Database) Size() (common.StorageSize, common.StorageSize) { |
||||||
|
var ( |
||||||
|
storages common.StorageSize |
||||||
|
preimages common.StorageSize |
||||||
|
) |
||||||
|
storages = db.backend.Size() |
||||||
|
if db.preimages != nil { |
||||||
|
preimages = db.preimages.size() |
||||||
|
} |
||||||
|
return storages, preimages |
||||||
|
} |
||||||
|
|
||||||
|
// Initialized returns an indicator if the state data is already initialized
|
||||||
|
// according to the state scheme.
|
||||||
|
func (db *Database) Initialized(genesisRoot common.Hash) bool { |
||||||
|
return db.backend.Initialized(genesisRoot) |
||||||
|
} |
||||||
|
|
||||||
|
// Scheme returns the node scheme used in the database.
|
||||||
|
func (db *Database) Scheme() string { |
||||||
|
return db.backend.Scheme() |
||||||
|
} |
||||||
|
|
||||||
|
// Close flushes the dangling preimages to disk and closes the trie database.
|
||||||
|
// It is meant to be called when closing the blockchain object, so that all
|
||||||
|
// resources held can be released correctly.
|
||||||
|
func (db *Database) Close() error { |
||||||
|
if db.preimages != nil { |
||||||
|
db.preimages.commit(true) |
||||||
|
} |
||||||
|
return db.backend.Close() |
||||||
|
} |
||||||
|
|
||||||
|
// saveCache saves clean state cache to given directory path
|
||||||
|
// using specified CPU cores.
|
||||||
|
func (db *Database) saveCache(dir string, threads int) error { |
||||||
|
if db.cleans == nil { |
||||||
|
return nil |
||||||
|
} |
||||||
|
log.Info("Writing clean trie cache to disk", "path", dir, "threads", threads) |
||||||
|
|
||||||
|
start := time.Now() |
||||||
|
err := db.cleans.SaveToFileConcurrent(dir, threads) |
||||||
|
if err != nil { |
||||||
|
log.Error("Failed to persist clean trie cache", "error", err) |
||||||
|
return err |
||||||
|
} |
||||||
|
log.Info("Persisted the clean trie cache", "path", dir, "elapsed", common.PrettyDuration(time.Since(start))) |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
// SaveCache atomically saves fast cache data to the given dir using all
|
||||||
|
// available CPU cores.
|
||||||
|
func (db *Database) SaveCache(dir string) error { |
||||||
|
return db.saveCache(dir, runtime.GOMAXPROCS(0)) |
||||||
|
} |
||||||
|
|
||||||
|
// SaveCachePeriodically atomically saves fast cache data to the given dir with
|
||||||
|
// the specified interval. All dump operation will only use a single CPU core.
|
||||||
|
func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, stopCh <-chan struct{}) { |
||||||
|
ticker := time.NewTicker(interval) |
||||||
|
defer ticker.Stop() |
||||||
|
|
||||||
|
for { |
||||||
|
select { |
||||||
|
case <-ticker.C: |
||||||
|
db.saveCache(dir, 1) |
||||||
|
case <-stopCh: |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Cap iteratively flushes old but still referenced trie nodes until the total
|
||||||
|
// memory usage goes below the given threshold. The held pre-images accumulated
|
||||||
|
// up to this point will be flushed in case the size exceeds the threshold.
|
||||||
|
//
|
||||||
|
// It's only supported by hash-based database and will return an error for others.
|
||||||
|
func (db *Database) Cap(limit common.StorageSize) error { |
||||||
|
hdb, ok := db.backend.(*hashdb.Database) |
||||||
|
if !ok { |
||||||
|
return errors.New("not supported") |
||||||
|
} |
||||||
|
if db.preimages != nil { |
||||||
|
db.preimages.commit(false) |
||||||
|
} |
||||||
|
return hdb.Cap(limit) |
||||||
|
} |
||||||
|
|
||||||
|
// Reference adds a new reference from a parent node to a child node. This function
|
||||||
|
// is used to add reference between internal trie node and external node(e.g. storage
|
||||||
|
// trie root), all internal trie nodes are referenced together by database itself.
|
||||||
|
//
|
||||||
|
// It's only supported by hash-based database and will return an error for others.
|
||||||
|
func (db *Database) Reference(root common.Hash, parent common.Hash) error { |
||||||
|
hdb, ok := db.backend.(*hashdb.Database) |
||||||
|
if !ok { |
||||||
|
return errors.New("not supported") |
||||||
|
} |
||||||
|
hdb.Reference(root, parent) |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
// Dereference removes an existing reference from a root node. It's only
|
||||||
|
// supported by hash-based database and will return an error for others.
|
||||||
|
func (db *Database) Dereference(root common.Hash) error { |
||||||
|
hdb, ok := db.backend.(*hashdb.Database) |
||||||
|
if !ok { |
||||||
|
return errors.New("not supported") |
||||||
|
} |
||||||
|
hdb.Dereference(root) |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
// Node retrieves the rlp-encoded node blob with provided node hash. It's
|
||||||
|
// only supported by hash-based database and will return an error for others.
|
||||||
|
// Note, this function should be deprecated once ETH66 is deprecated.
|
||||||
|
func (db *Database) Node(hash common.Hash) ([]byte, error) { |
||||||
|
hdb, ok := db.backend.(*hashdb.Database) |
||||||
|
if !ok { |
||||||
|
return nil, errors.New("not supported") |
||||||
|
} |
||||||
|
return hdb.Node(hash) |
||||||
|
} |
||||||
@ -1,149 +0,0 @@ |
|||||||
// Copyright 2022 The go-ethereum Authors
|
|
||||||
// This file is part of the go-ethereum library.
|
|
||||||
//
|
|
||||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Lesser General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Lesser General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Lesser General Public License
|
|
||||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package trie |
|
||||||
|
|
||||||
import ( |
|
||||||
"fmt" |
|
||||||
"sort" |
|
||||||
"strings" |
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common" |
|
||||||
"github.com/ethereum/go-ethereum/trie/trienode" |
|
||||||
) |
|
||||||
|
|
||||||
// NodeSet contains all dirty nodes collected during the commit operation.
|
|
||||||
// Each node is keyed by path. It's not thread-safe to use.
|
|
||||||
type NodeSet struct { |
|
||||||
owner common.Hash // the identifier of the trie
|
|
||||||
leaves []*leaf // the list of dirty leaves
|
|
||||||
updates int // the count of updated and inserted nodes
|
|
||||||
deletes int // the count of deleted nodes
|
|
||||||
|
|
||||||
// The set of all dirty nodes. Dirty nodes include newly inserted nodes,
|
|
||||||
// deleted nodes and updated nodes. The original value of the newly
|
|
||||||
// inserted node must be nil, and the original value of the other two
|
|
||||||
// types must be non-nil.
|
|
||||||
nodes map[string]*trienode.WithPrev |
|
||||||
} |
|
||||||
|
|
||||||
// NewNodeSet initializes an empty node set to be used for tracking dirty nodes
|
|
||||||
// from a specific account or storage trie. The owner is zero for the account
|
|
||||||
// trie and the owning account address hash for storage tries.
|
|
||||||
func NewNodeSet(owner common.Hash) *NodeSet { |
|
||||||
return &NodeSet{ |
|
||||||
owner: owner, |
|
||||||
nodes: make(map[string]*trienode.WithPrev), |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// forEachWithOrder iterates the dirty nodes with the order from bottom to top,
|
|
||||||
// right to left, nodes with the longest path will be iterated first.
|
|
||||||
func (set *NodeSet) forEachWithOrder(callback func(path string, n *trienode.Node)) { |
|
||||||
var paths sort.StringSlice |
|
||||||
for path := range set.nodes { |
|
||||||
paths = append(paths, path) |
|
||||||
} |
|
||||||
// Bottom-up, longest path first
|
|
||||||
sort.Sort(sort.Reverse(paths)) |
|
||||||
for _, path := range paths { |
|
||||||
callback(path, set.nodes[path].Unwrap()) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// addNode adds the provided dirty node into set.
|
|
||||||
func (set *NodeSet) addNode(path []byte, n *trienode.WithPrev) { |
|
||||||
if n.IsDeleted() { |
|
||||||
set.deletes += 1 |
|
||||||
} else { |
|
||||||
set.updates += 1 |
|
||||||
} |
|
||||||
set.nodes[string(path)] = n |
|
||||||
} |
|
||||||
|
|
||||||
// addLeaf adds the provided leaf node into set.
|
|
||||||
func (set *NodeSet) addLeaf(node *leaf) { |
|
||||||
set.leaves = append(set.leaves, node) |
|
||||||
} |
|
||||||
|
|
||||||
// Size returns the number of dirty nodes in set.
|
|
||||||
func (set *NodeSet) Size() (int, int) { |
|
||||||
return set.updates, set.deletes |
|
||||||
} |
|
||||||
|
|
||||||
// Hashes returns the hashes of all updated nodes. TODO(rjl493456442) how can
|
|
||||||
// we get rid of it?
|
|
||||||
func (set *NodeSet) Hashes() []common.Hash { |
|
||||||
var ret []common.Hash |
|
||||||
for _, node := range set.nodes { |
|
||||||
ret = append(ret, node.Hash) |
|
||||||
} |
|
||||||
return ret |
|
||||||
} |
|
||||||
|
|
||||||
// Summary returns a string-representation of the NodeSet.
|
|
||||||
func (set *NodeSet) Summary() string { |
|
||||||
var out = new(strings.Builder) |
|
||||||
fmt.Fprintf(out, "nodeset owner: %v\n", set.owner) |
|
||||||
if set.nodes != nil { |
|
||||||
for path, n := range set.nodes { |
|
||||||
// Deletion
|
|
||||||
if n.IsDeleted() { |
|
||||||
fmt.Fprintf(out, " [-]: %x prev: %x\n", path, n.Prev) |
|
||||||
continue |
|
||||||
} |
|
||||||
// Insertion
|
|
||||||
if len(n.Prev) == 0 { |
|
||||||
fmt.Fprintf(out, " [+]: %x -> %v\n", path, n.Hash) |
|
||||||
continue |
|
||||||
} |
|
||||||
// Update
|
|
||||||
fmt.Fprintf(out, " [*]: %x -> %v prev: %x\n", path, n.Hash, n.Prev) |
|
||||||
} |
|
||||||
} |
|
||||||
for _, n := range set.leaves { |
|
||||||
fmt.Fprintf(out, "[leaf]: %v\n", n) |
|
||||||
} |
|
||||||
return out.String() |
|
||||||
} |
|
||||||
|
|
||||||
// MergedNodeSet represents a merged dirty node set for a group of tries.
|
|
||||||
type MergedNodeSet struct { |
|
||||||
sets map[common.Hash]*NodeSet |
|
||||||
} |
|
||||||
|
|
||||||
// NewMergedNodeSet initializes an empty merged set.
|
|
||||||
func NewMergedNodeSet() *MergedNodeSet { |
|
||||||
return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)} |
|
||||||
} |
|
||||||
|
|
||||||
// NewWithNodeSet constructs a merged nodeset with the provided single set.
|
|
||||||
func NewWithNodeSet(set *NodeSet) *MergedNodeSet { |
|
||||||
merged := NewMergedNodeSet() |
|
||||||
merged.Merge(set) |
|
||||||
return merged |
|
||||||
} |
|
||||||
|
|
||||||
// Merge merges the provided dirty nodes of a trie into the set. The assumption
|
|
||||||
// is held that no duplicated set belonging to the same trie will be merged twice.
|
|
||||||
func (set *MergedNodeSet) Merge(other *NodeSet) error { |
|
||||||
_, present := set.sets[other.owner] |
|
||||||
if present { |
|
||||||
return fmt.Errorf("duplicate trie for owner %#x", other.owner) |
|
||||||
} |
|
||||||
set.sets[other.owner] = other |
|
||||||
return nil |
|
||||||
} |
|
||||||
Loading…
Reference in new issue