From 15eb9773f9b99c29f3cd17be4e4bbd1bf1b48bb7 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 19 Mar 2024 10:50:08 +0800 Subject: [PATCH] triedb/pathdb: improve tests (#29278) --- eth/protocols/snap/sync_test.go | 6 +- .../utils.go => internal/testrand/rand.go | 28 +++---- trie/stacktrie_test.go | 8 +- triedb/pathdb/database.go | 8 +- triedb/pathdb/database_test.go | 73 +++++++++++++++---- triedb/pathdb/difflayer_test.go | 18 +++-- triedb/pathdb/history_test.go | 12 +-- triedb/pathdb/testutils.go | 7 +- 8 files changed, 104 insertions(+), 56 deletions(-) rename trie/testutil/utils.go => internal/testrand/rand.go (61%) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index cea83aa2bc..87e186633b 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -32,10 +32,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/pathdb" @@ -1816,8 +1816,8 @@ func makeUnevenStorageTrie(owner common.Hash, slots int, db *triedb.Database) (c break } for j := 0; j < slots/3; j++ { - key := append([]byte{byte(n)}, testutil.RandBytes(31)...) - val, _ := rlp.EncodeToBytes(testutil.RandBytes(32)) + key := append([]byte{byte(n)}, testrand.Bytes(31)...) + val, _ := rlp.EncodeToBytes(testrand.Bytes(32)) elem := &kv{key, val} tr.MustUpdate(elem.k, elem.v) diff --git a/trie/testutil/utils.go b/internal/testrand/rand.go similarity index 61% rename from trie/testutil/utils.go rename to internal/testrand/rand.go index a75d0431b0..690993de05 100644 --- a/trie/testutil/utils.go +++ b/internal/testrand/rand.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package testutil +package testrand import ( crand "crypto/rand" @@ -22,11 +22,9 @@ import ( mrand "math/rand" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/trienode" ) -// Prng is a pseudo random number generator seeded by strong randomness. +// prng is a pseudo random number generator seeded by strong randomness. // The randomness is printed on startup in order to make failures reproducible. var prng = initRand() @@ -37,25 +35,19 @@ func initRand() *mrand.Rand { return rnd } -// RandBytes generates a random byte slice with specified length. -func RandBytes(n int) []byte { +// Bytes generates a random byte slice with specified length. +func Bytes(n int) []byte { r := make([]byte, n) prng.Read(r) return r } -// RandomHash generates a random blob of data and returns it as a hash. -func RandomHash() common.Hash { - return common.BytesToHash(RandBytes(common.HashLength)) +// Hash generates a random hash. +func Hash() common.Hash { + return common.BytesToHash(Bytes(common.HashLength)) } -// RandomAddress generates a random blob of data and returns it as an address. -func RandomAddress() common.Address { - return common.BytesToAddress(RandBytes(common.AddressLength)) -} - -// RandomNode generates a random node. -func RandomNode() *trienode.Node { - val := RandBytes(100) - return trienode.New(crypto.Keccak256Hash(val), val) +// Address generates a random address. +func Address() common.Address { + return common.BytesToAddress(Bytes(common.AddressLength)) } diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 3a0e1cb260..203ebd99a9 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/testutil" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" ) @@ -431,12 +431,12 @@ func TestPartialStackTrie(t *testing.T) { for i := 0; i < n; i++ { var val []byte if rand.Intn(3) == 0 { - val = testutil.RandBytes(3) + val = testrand.Bytes(3) } else { - val = testutil.RandBytes(32) + val = testrand.Bytes(32) } entries = append(entries, &kv{ - k: testutil.RandBytes(32), + k: testrand.Bytes(32), v: val, }) } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index b1e01abac4..7bdb6132bb 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -34,9 +34,6 @@ import ( ) const ( - // maxDiffLayers is the maximum diff layers allowed in the layer tree. - maxDiffLayers = 128 - // defaultCleanSize is the default memory allowance of clean cache. defaultCleanSize = 16 * 1024 * 1024 @@ -54,6 +51,11 @@ const ( DefaultBufferSize = 64 * 1024 * 1024 ) +var ( + // maxDiffLayers is the maximum diff layers allowed in the layer tree. + maxDiffLayers = 128 +) + // layer is the interface implemented by all state layers which includes some // public methods and some additional methods for internal usage. type layer interface { diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 2e7e1bef05..a41cf4268a 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -27,8 +27,8 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" "github.com/holiman/uint256" @@ -46,7 +46,10 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm h.Update(key.Bytes(), val) } } - root, nodes, _ := h.Commit(false) + root, nodes, err := h.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit hasher, err: %w", err)) + } return root, nodes } @@ -54,7 +57,7 @@ func generateAccount(storageRoot common.Hash) types.StateAccount { return types.StateAccount{ Nonce: uint64(rand.Intn(100)), Balance: uint256.NewInt(rand.Uint64()), - CodeHash: testutil.RandBytes(32), + CodeHash: testrand.Bytes(32), Root: storageRoot, } } @@ -101,8 +104,8 @@ func newTester(t *testing.T, historyLimit uint64) *tester { disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) db = New(disk, &Config{ StateHistory: historyLimit, - CleanCacheSize: 256 * 1024, - DirtyCacheSize: 256 * 1024, + CleanCacheSize: 16 * 1024, + DirtyCacheSize: 16 * 1024, }) obj = &tester{ db: db, @@ -113,7 +116,7 @@ func newTester(t *testing.T, historyLimit uint64) *tester { snapStorages: make(map[common.Hash]map[common.Hash]map[common.Hash][]byte), } ) - for i := 0; i < 2*128; i++ { + for i := 0; i < 8; i++ { var parent = types.EmptyRootHash if len(obj.roots) != 0 { parent = obj.roots[len(obj.roots)-1] @@ -146,8 +149,8 @@ func (t *tester) generateStorage(ctx *genctx, addr common.Address) common.Hash { origin = make(map[common.Hash][]byte) ) for i := 0; i < 10; i++ { - v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testutil.RandBytes(32))) - hash := testutil.RandomHash() + v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testrand.Bytes(32))) + hash := testrand.Hash() storage[hash] = v origin[hash] = nil @@ -175,8 +178,8 @@ func (t *tester) mutateStorage(ctx *genctx, addr common.Address, root common.Has } } for i := 0; i < 3; i++ { - v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testutil.RandBytes(32))) - hash := testutil.RandomHash() + v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testrand.Bytes(32))) + hash := testrand.Hash() storage[hash] = v origin[hash] = nil @@ -218,7 +221,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode switch rand.Intn(opLen) { case createAccountOp: // account creation - addr := testutil.RandomAddress() + addr := testrand.Address() addrHash := crypto.Keccak256Hash(addr.Bytes()) if _, ok := t.accounts[addrHash]; ok { continue @@ -320,14 +323,16 @@ func (t *tester) verifyState(root common.Hash) error { return errors.New("root node is not available") } for addrHash, account := range t.snapAccounts[root] { - blob, err := reader.Node(common.Hash{}, addrHash.Bytes(), crypto.Keccak256Hash(account)) + path := crypto.Keccak256(addrHash.Bytes()) + blob, err := reader.Node(common.Hash{}, path, crypto.Keccak256Hash(account)) if err != nil || !bytes.Equal(blob, account) { return fmt.Errorf("account is mismatched: %w", err) } } for addrHash, slots := range t.snapStorages[root] { for hash, slot := range slots { - blob, err := reader.Node(addrHash, hash.Bytes(), crypto.Keccak256Hash(slot)) + path := crypto.Keccak256(hash.Bytes()) + blob, err := reader.Node(addrHash, path, crypto.Keccak256Hash(slot)) if err != nil || !bytes.Equal(blob, slot) { return fmt.Errorf("slot is mismatched: %w", err) } @@ -379,6 +384,12 @@ func (t *tester) bottomIndex() int { } func TestDatabaseRollback(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + // Verify state histories tester := newTester(t, 0) defer tester.release() @@ -409,6 +420,12 @@ func TestDatabaseRollback(t *testing.T) { } func TestDatabaseRecoverable(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + var ( tester = newTester(t, 0) index = tester.bottomIndex() @@ -448,6 +465,12 @@ func TestDatabaseRecoverable(t *testing.T) { } func TestDisable(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + tester := newTester(t, 0) defer tester.release() @@ -484,6 +507,12 @@ func TestDisable(t *testing.T) { } func TestCommit(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + tester := newTester(t, 0) defer tester.release() @@ -508,6 +537,12 @@ func TestCommit(t *testing.T) { } func TestJournal(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + tester := newTester(t, 0) defer tester.release() @@ -532,6 +567,12 @@ func TestJournal(t *testing.T) { } func TestCorruptedJournal(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + tester := newTester(t, 0) defer tester.release() @@ -574,6 +615,12 @@ func TestCorruptedJournal(t *testing.T) { // truncating the tail histories. This ensures that the ID of the persistent state // always falls within the range of [oldest-history-id, latest-history-id]. func TestTailTruncateHistory(t *testing.T) { + // Redefine the diff layer depth allowance for faster testing. + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + tester := newTester(t, 10) defer tester.release() diff --git a/triedb/pathdb/difflayer_test.go b/triedb/pathdb/difflayer_test.go index 9b5907c3c5..75890b8a83 100644 --- a/triedb/pathdb/difflayer_test.go +++ b/triedb/pathdb/difflayer_test.go @@ -22,7 +22,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/trie/testutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -66,8 +67,9 @@ func benchmarkSearch(b *testing.B, depth int, total int) { nodes[common.Hash{}] = make(map[string]*trienode.Node) for i := 0; i < 3000; i++ { var ( - path = testutil.RandBytes(32) - node = testutil.RandomNode() + path = testrand.Bytes(32) + blob = testrand.Bytes(100) + node = trienode.New(crypto.Keccak256Hash(blob), blob) ) nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) if npath == nil && depth == index { @@ -112,8 +114,9 @@ func BenchmarkPersist(b *testing.B) { nodes[common.Hash{}] = make(map[string]*trienode.Node) for i := 0; i < 3000; i++ { var ( - path = testutil.RandBytes(32) - node = testutil.RandomNode() + path = testrand.Bytes(32) + blob = testrand.Bytes(100) + node = trienode.New(crypto.Keccak256Hash(blob), blob) ) nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) } @@ -149,8 +152,9 @@ func BenchmarkJournal(b *testing.B) { nodes[common.Hash{}] = make(map[string]*trienode.Node) for i := 0; i < 3000; i++ { var ( - path = testutil.RandBytes(32) - node = testutil.RandomNode() + path = testrand.Bytes(32) + blob = testrand.Bytes(100) + node = trienode.New(crypto.Keccak256Hash(blob), blob) ) nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) } diff --git a/triedb/pathdb/history_test.go b/triedb/pathdb/history_test.go index ab0d44777d..81ac768acd 100644 --- a/triedb/pathdb/history_test.go +++ b/triedb/pathdb/history_test.go @@ -26,8 +26,8 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/triestate" ) @@ -38,11 +38,11 @@ func randomStateSet(n int) *triestate.Set { storages = make(map[common.Address]map[common.Hash][]byte) ) for i := 0; i < n; i++ { - addr := testutil.RandomAddress() + addr := testrand.Address() storages[addr] = make(map[common.Hash][]byte) for j := 0; j < 3; j++ { - v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testutil.RandBytes(32))) - storages[addr][testutil.RandomHash()] = v + v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testrand.Bytes(32))) + storages[addr][testrand.Hash()] = v } account := generateAccount(types.EmptyRootHash) accounts[addr] = types.SlimAccountRLP(account) @@ -51,7 +51,7 @@ func randomStateSet(n int) *triestate.Set { } func makeHistory() *history { - return newHistory(testutil.RandomHash(), types.EmptyRootHash, 0, randomStateSet(3)) + return newHistory(testrand.Hash(), types.EmptyRootHash, 0, randomStateSet(3)) } func makeHistories(n int) []*history { @@ -60,7 +60,7 @@ func makeHistories(n int) []*history { result []*history ) for i := 0; i < n; i++ { - root := testutil.RandomHash() + root := testrand.Hash() h := newHistory(root, parent, uint64(i), randomStateSet(3)) parent = root result = append(result, h) diff --git a/triedb/pathdb/testutils.go b/triedb/pathdb/testutils.go index d6fdacb421..546cb819b8 100644 --- a/triedb/pathdb/testutils.go +++ b/triedb/pathdb/testutils.go @@ -93,10 +93,13 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, e if bytes.Equal(val, h.cleans[hash]) { continue } + // Utilize the hash of the state key as the node path to mitigate + // potential collisions within the path. + path := crypto.Keccak256(hash.Bytes()) if len(val) == 0 { - set.AddNode(hash.Bytes(), trienode.NewDeleted()) + set.AddNode(path, trienode.NewDeleted()) } else { - set.AddNode(hash.Bytes(), trienode.New(crypto.Keccak256Hash(val), val)) + set.AddNode(path, trienode.New(crypto.Keccak256Hash(val), val)) } } root, blob := hash(nodes)