From 76eed9e50d65e014acf0828588ed8f6ce1777f8d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 31 Mar 2020 10:25:41 +0200 Subject: [PATCH] snapshotter/tests: verify snapdb post-state against trie (#20812) * core/state/snapshot: basic trie-to-hash implementation * tests: validate snapshot after test * core/state/snapshot: fix review concerns --- core/blockchain.go | 9 +++ core/state/snapshot/conversion.go | 114 ++++++++++++++++++++++++++++++ tests/block_test_util.go | 14 ++++ 3 files changed, 137 insertions(+) create mode 100644 core/state/snapshot/conversion.go diff --git a/core/blockchain.go b/core/blockchain.go index d6f732194e..0d1c27f95e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -520,6 +520,15 @@ func (bc *BlockChain) CurrentBlock() *types.Block { return bc.currentBlock.Load().(*types.Block) } +// Snapshot returns the blockchain snapshot tree. This method is mainly used for +// testing, to make it possible to verify the snapshot after execution. +// +// Warning: There are no guarantees about the safety of using the returned 'snap' if the +// blockchain is simultaneously importing blocks, so take care. +func (bc *BlockChain) Snapshot() *snapshot.Tree { + return bc.snaps +} + // CurrentFastBlock retrieves the current fast-sync head block of the canonical // chain. The block is retrieved from the blockchain's internal cache. func (bc *BlockChain) CurrentFastBlock() *types.Block { diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go new file mode 100644 index 0000000000..d9c86e5163 --- /dev/null +++ b/core/state/snapshot/conversion.go @@ -0,0 +1,114 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// conversionAccount is used for converting between full and slim format. When +// doing this, we can consider 'balance' as a byte array, as it has already +// been converted from big.Int into an rlp-byteslice. +type conversionAccount struct { + Nonce uint64 + Balance []byte + Root []byte + CodeHash []byte +} + +// SlimToFull converts data on the 'slim RLP' format into the full RLP-format +func SlimToFull(data []byte) ([]byte, error) { + acc := &conversionAccount{} + if err := rlp.DecodeBytes(data, acc); err != nil { + return nil, err + } + if len(acc.Root) == 0 { + acc.Root = emptyRoot[:] + } + if len(acc.CodeHash) == 0 { + acc.CodeHash = emptyCode[:] + } + fullData, err := rlp.EncodeToBytes(acc) + if err != nil { + return nil, err + } + return fullData, nil +} + +// trieKV represents a trie key-value pair +type trieKV struct { + key common.Hash + value []byte +} + +type trieGeneratorFn func(in chan (trieKV), out chan (common.Hash)) + +// GenerateTrieRoot takes an account iterator and reproduces the root hash. +func GenerateTrieRoot(it AccountIterator) common.Hash { + return generateTrieRoot(it, stdGenerate) +} + +func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Hash { + var ( + in = make(chan trieKV) // chan to pass leaves + out = make(chan common.Hash) // chan to collect result + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + generatorFn(in, out) + wg.Done() + }() + // Feed leaves + start := time.Now() + logged := time.Now() + accounts := 0 + for it.Next() { + slimData := it.Account() + fullData, _ := SlimToFull(slimData) + l := trieKV{it.Hash(), fullData} + in <- l + if time.Since(logged) > 8*time.Second { + log.Info("Generating trie hash from snapshot", + "at", l.key, "accounts", accounts, "elapsed", time.Since(start)) + logged = time.Now() + } + accounts++ + } + close(in) + result := <-out + log.Info("Generated trie hash from snapshot", "accounts", accounts, "elapsed", time.Since(start)) + wg.Wait() + return result +} + +// stdGenerate is a very basic hexary trie builder which uses the same Trie +// as the rest of geth, with no enhancements or optimizations +func stdGenerate(in chan (trieKV), out chan (common.Hash)) { + t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New())) + for leaf := range in { + t.TryUpdate(leaf.key[:], leaf.value) + } + out <- t.Hash() +} diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 1ae986e3ca..37f63f538a 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" @@ -144,6 +145,19 @@ func (t *BlockTest) Run(snapshotter bool) error { if err = t.validatePostState(newDB); err != nil { return fmt.Errorf("post state validation failed: %v", err) } + // Cross-check the snapshot-to-hash against the trie hash + if snapshotter { + snapTree := chain.Snapshot() + root := chain.CurrentBlock().Root() + it, err := snapTree.AccountIterator(root, common.Hash{}) + if err != nil { + return fmt.Errorf("Could not create iterator for root %x: %v", root, err) + } + generatedRoot := snapshot.GenerateTrieRoot(it) + if generatedRoot != root { + return fmt.Errorf("Snapshot corruption, got %d exp %d", generatedRoot, root) + } + } return t.validateImportedHeaders(chain, validBlocks) }