core/rawdb: refactor db inspector for extending multiple ancient store (#25896)

This PR ports a few changes from PBSS:

- Fix the snapshot generator waiter in case the generation is not even initialized
- Refactor db inspector for ancient store
pull/26066/head
rjl493456442 2 years ago committed by GitHub
parent a1fc0d8144
commit 60e30a940b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      core/blockchain.go
  2. 86
      core/blockchain_test.go
  3. 33
      core/rawdb/ancient_scheme.go
  4. 121
      core/rawdb/ancient_utils.go
  5. 45
      core/rawdb/database.go
  6. 13
      core/state/snapshot/snapshot.go
  7. 8
      core/state/state_object.go
  8. 8
      core/state/statedb.go
  9. 2
      trie/nodeset.go

@ -123,7 +123,7 @@ const (
BlockChainVersion uint64 = 8
)
// CacheConfig contains the configuration values for the trie caching/pruning
// CacheConfig contains the configuration values for the trie database
// that's resident in a blockchain.
type CacheConfig struct {
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
@ -1408,7 +1408,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
if len(logs) > 0 {
bc.logsFeed.Send(logs)
}
// In theory we should fire a ChainHeadEvent when we inject
// In theory, we should fire a ChainHeadEvent when we inject
// a canonical block, but sometimes we can insert a batch of
// canonical blocks. Avoid firing too many ChainHeadEvents,
// we will fire an accumulated ChainHeadEvent and disable fire

@ -4007,3 +4007,89 @@ func TestTxIndexer(t *testing.T) {
os.RemoveAll(frdir)
}
}
func TestCreateThenDeletePreByzantium(t *testing.T) {
// We use Ropsten chain config instead of Testchain config, this is
// deliberate: we want to use pre-byz rules where we have intermediate state roots
// between transactions.
testCreateThenDelete(t, params.RopstenChainConfig)
}
func TestCreateThenDeletePostByzantium(t *testing.T) {
testCreateThenDelete(t, params.TestChainConfig)
}
// testCreateThenDelete tests a creation and subsequent deletion of a contract, happening
// within the same block.
func testCreateThenDelete(t *testing.T, config *params.ChainConfig) {
var (
engine = ethash.NewFaker()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
destAddress = crypto.CreateAddress(address, 0)
funds = big.NewInt(1000000000000000)
)
// runtime code is 0x60ffff : PUSH1 0xFF SELFDESTRUCT, a.k.a SELFDESTRUCT(0xFF)
code := append([]byte{0x60, 0xff, 0xff}, make([]byte, 32-3)...)
initCode := []byte{
// SSTORE 1:1
byte(vm.PUSH1), 0x1,
byte(vm.PUSH1), 0x1,
byte(vm.SSTORE),
// Get the runtime-code on the stack
byte(vm.PUSH32)}
initCode = append(initCode, code...)
initCode = append(initCode, []byte{
byte(vm.PUSH1), 0x0, // offset
byte(vm.MSTORE),
byte(vm.PUSH1), 0x3, // size
byte(vm.PUSH1), 0x0, // offset
byte(vm.RETURN), // return 3 bytes of zero-code
}...)
gspec := &Genesis{
Config: config,
Alloc: GenesisAlloc{
address: {Balance: funds},
},
}
nonce := uint64(0)
signer := types.HomesteadSigner{}
_, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2, func(i int, b *BlockGen) {
fee := big.NewInt(1)
if b.header.BaseFee != nil {
fee = b.header.BaseFee
}
b.SetCoinbase(common.Address{1})
tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{
Nonce: nonce,
GasPrice: new(big.Int).Set(fee),
Gas: 100000,
Data: initCode,
})
nonce++
b.AddTx(tx)
tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{
Nonce: nonce,
GasPrice: new(big.Int).Set(fee),
Gas: 100000,
To: &destAddress,
})
b.AddTx(tx)
nonce++
})
// Import the canonical chain
chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{
//Debug: true,
//Tracer: logger.NewJSONLogger(nil, os.Stdout),
}, nil, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
// Import the blocks
for _, block := range blocks {
if _, err := chain.InsertChain([]*types.Block{block}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
}
}
}

@ -16,8 +16,6 @@
package rawdb
import "fmt"
// The list of table names of chain freezer.
const (
// chainFreezerHeaderTable indicates the name of the freezer header table.
@ -53,34 +51,3 @@ var (
// freezers the collections of all builtin freezers.
var freezers = []string{chainFreezerName}
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
// ancient indicates the path of root ancient directory where the chain freezer can
// be opened. Start and end specify the range for dumping out indexes.
// Note this function can only be used for debugging purposes.
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
var (
path string
tables map[string]bool
)
switch freezerName {
case chainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
noSnappy, exist := tables[tableName]
if !exist {
var names []string
for name := range tables {
names = append(names, name)
}
return fmt.Errorf("unknown table, supported ones: %v", names)
}
table, err := newFreezerTable(path, tableName, noSnappy, true)
if err != nil {
return err
}
table.dumpIndexStdout(start, end)
return nil
}

@ -0,0 +1,121 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rawdb
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
)
type tableSize struct {
name string
size common.StorageSize
}
// freezerInfo contains the basic information of the freezer.
type freezerInfo struct {
name string // The identifier of freezer
head uint64 // The number of last stored item in the freezer
tail uint64 // The number of first stored item in the freezer
sizes []tableSize // The storage size per table
}
// count returns the number of stored items in the freezer.
func (info *freezerInfo) count() uint64 {
return info.head - info.tail + 1
}
// size returns the storage size of the entire freezer.
func (info *freezerInfo) size() common.StorageSize {
var total common.StorageSize
for _, table := range info.sizes {
total += table.size
}
return total
}
// inspectFreezers inspects all freezers registered in the system.
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
var infos []freezerInfo
for _, freezer := range freezers {
switch freezer {
case chainFreezerName:
// Chain ancient store is a bit special. It's always opened along
// with the key-value store, inspect the chain store directly.
info := freezerInfo{name: freezer}
// Retrieve storage size of every contained table.
for table := range chainFreezerNoSnappy {
size, err := db.AncientSize(table)
if err != nil {
return nil, err
}
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := db.Ancients()
if err != nil {
return nil, err
}
info.head = ancients - 1
// Retrieve the number of first stored item
tail, err := db.Tail()
if err != nil {
return nil, err
}
info.tail = tail
infos = append(infos, info)
default:
return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
}
return infos, nil
}
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
// ancient indicates the path of root ancient directory where the chain freezer can
// be opened. Start and end specify the range for dumping out indexes.
// Note this function can only be used for debugging purposes.
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
var (
path string
tables map[string]bool
)
switch freezerName {
case chainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
noSnappy, exist := tables[tableName]
if !exist {
var names []string
for name := range tables {
names = append(names, name)
}
return fmt.Errorf("unknown table, supported ones: %v", names)
}
table, err := newFreezerTable(path, tableName, noSnappy, true)
if err != nil {
return err
}
table.dumpIndexStdout(start, end)
return nil
}

@ -22,6 +22,7 @@ import (
"fmt"
"os"
"path"
"strings"
"sync/atomic"
"time"
@ -379,13 +380,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
beaconHeaders stat
cliqueSnaps stat
// Ancient store statistics
ancientHeadersSize common.StorageSize
ancientBodiesSize common.StorageSize
ancientReceiptsSize common.StorageSize
ancientTdsSize common.StorageSize
ancientHashesSize common.StorageSize
// Les statistic
chtTrieNodes stat
bloomTrieNodes stat
@ -473,20 +467,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
logged = time.Now()
}
}
// Inspect append-only file store then.
ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
if size, err := db.AncientSize(category); err == nil {
*ancientSizes[i] += common.StorageSize(size)
total += common.StorageSize(size)
}
}
// Get number of ancient rows inside the freezer
ancients := counter(0)
if count, err := db.Ancients(); err == nil {
ancients = counter(count)
}
// Display the database statistic.
// Display the database statistic of key-value store.
stats := [][]string{
{"Key-Value store", "Headers", headers.Size(), headers.Count()},
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
@ -504,14 +485,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()},
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
{"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
{"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
{"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
{"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
{"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
}
// Inspect all registered append-only file store then.
ancients, err := inspectFreezers(db)
if err != nil {
return err
}
for _, ancient := range ancients {
for _, table := range ancient.sizes {
stats = append(stats, []string{
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
strings.Title(table.name),
table.size.String(),
fmt.Sprintf("%d", ancient.count()),
})
}
total += ancient.size()
}
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Database", "Category", "Size", "Items"})
table.SetFooter([]string{"", "Total", total.String(), " "})
@ -521,6 +513,5 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
if unaccounted.size > 0 {
log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
}
return nil
}

@ -187,8 +187,9 @@ type Tree struct {
// If the memory layers in the journal do not match the disk layer (e.g. there is
// a gap) or the journal is missing, there are two repair cases:
//
// - if the 'recovery' parameter is true, all memory diff-layers will be discarded.
// This case happens when the snapshot is 'ahead' of the state trie.
// - if the 'recovery' parameter is true, memory diff-layers and the disk-layer
// will all be kept. This case happens when the snapshot is 'ahead' of the
// state trie.
// - otherwise, the entire snapshot is considered invalid and will be recreated on
// a background thread.
func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) {
@ -199,16 +200,16 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root
triedb: triedb,
layers: make(map[common.Hash]snapshot),
}
// Create the building waiter iff the background generation is allowed
if !config.NoBuild && !config.AsyncBuild {
defer snap.waitBuild()
}
// Attempt to load a previously persisted snapshot and rebuild one if failed
head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild)
if disabled {
log.Warn("Snapshot maintenance disabled (syncing)")
return snap, nil
}
// Create the building waiter iff the background generation is allowed
if !config.NoBuild && !config.AsyncBuild {
defer snap.waitBuild()
}
if err != nil {
log.Warn("Failed to load snapshot", "err", err)
if !config.NoBuild {

@ -63,7 +63,7 @@ func (s Storage) Copy() Storage {
// The usage pattern is as follows:
// First you need to obtain a state object.
// Account values can be accessed and modified through the object.
// Finally, call CommitTrie to write the modified storage trie into a database.
// Finally, call commitTrie to write the modified storage trie into a database.
type stateObject struct {
address common.Address
addrHash common.Hash // hash of ethereum address of the account
@ -374,9 +374,9 @@ func (s *stateObject) updateRoot(db Database) {
s.data.Root = s.trie.Hash()
}
// CommitTrie the storage trie of the object to db.
// This updates the trie root.
func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
// commitTrie submits the storage changes into the storage trie and re-computes
// the root. Besides, all trie changes will be collected in a nodeset and returned.
func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
// If nothing changed, don't bother with hashing anything
if s.updateTrie(db) == nil {
return nil, nil

@ -920,7 +920,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
obj.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie
set, err := obj.CommitTrie(s.db)
set, err := obj.commitTrie(s.db)
if err != nil {
return common.Hash{}, err
}
@ -934,6 +934,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
storageTrieNodesDeleted += deleted
}
}
// If the contract is destructed, the storage is still left in the
// database as dangling data. Theoretically it's should be wiped from
// database as well, but in hash-based-scheme it's extremely hard to
// determine that if the trie nodes are also referenced by other storage,
// and in path-based-scheme some technical challenges are still unsolved.
// Although it won't affect the correctness but please fix it TODO(rjl493456442).
}
if len(s.stateObjectsDirty) > 0 {
s.stateObjectsDirty = make(map[common.Address]struct{})

@ -113,6 +113,7 @@ func NewNodeSet(owner common.Hash) *NodeSet {
}
}
/*
// NewNodeSetWithDeletion initializes the nodeset with provided deletion set.
func NewNodeSetWithDeletion(owner common.Hash, paths [][]byte, prev [][]byte) *NodeSet {
set := NewNodeSet(owner)
@ -121,6 +122,7 @@ func NewNodeSetWithDeletion(owner common.Hash, paths [][]byte, prev [][]byte) *N
}
return set
}
*/
// markUpdated marks the node as dirty(newly-inserted or updated) with provided
// node path, node object along with its previous value.

Loading…
Cancel
Save