triedb/pathdb: improve tests (#29278)

eth/protocols/snap/sync_test.go

@@ -32,10 +32,10 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/internal/testrand"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/ethereum/go-ethereum/trie/testutil"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/triedb"
 	"github.com/ethereum/go-ethereum/triedb/pathdb"
@@ -1816,8 +1816,8 @@ func makeUnevenStorageTrie(owner common.Hash, slots int, db *triedb.Database) (c
 			break
 		}
 		for j := 0; j < slots/3; j++ {
-			key := append([]byte{byte(n)}, testutil.RandBytes(31)...)
+			key := append([]byte{byte(n)}, testrand.Bytes(31)...)
-			val, _ := rlp.EncodeToBytes(testutil.RandBytes(32))
+			val, _ := rlp.EncodeToBytes(testrand.Bytes(32))
 
 			elem := &kv{key, val}
 			tr.MustUpdate(elem.k, elem.v)

internal/testrand/rand.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package testutil
+package testrand
 
 import (
 	crand "crypto/rand"
@@ -22,11 +22,9 @@ import (
 	mrand "math/rand"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/trie/trienode"
 )
 
-// Prng is a pseudo random number generator seeded by strong randomness.
+// prng is a pseudo random number generator seeded by strong randomness.
 // The randomness is printed on startup in order to make failures reproducible.
 var prng = initRand()
 
@@ -37,25 +35,19 @@ func initRand() *mrand.Rand {
 	return rnd
 }
 
-// RandBytes generates a random byte slice with specified length.
+// Bytes generates a random byte slice with specified length.
-func RandBytes(n int) []byte {
+func Bytes(n int) []byte {
 	r := make([]byte, n)
 	prng.Read(r)
 	return r
 }
 
-// RandomHash generates a random blob of data and returns it as a hash.
+// Hash generates a random hash.
-func RandomHash() common.Hash {
+func Hash() common.Hash {
-	return common.BytesToHash(RandBytes(common.HashLength))
+	return common.BytesToHash(Bytes(common.HashLength))
 }
 
-// RandomAddress generates a random blob of data and returns it as an address.
+// Address generates a random address.
-func RandomAddress() common.Address {
+func Address() common.Address {
-	return common.BytesToAddress(RandBytes(common.AddressLength))
+	return common.BytesToAddress(Bytes(common.AddressLength))
-}
-
-// RandomNode generates a random node.
-func RandomNode() *trienode.Node {
-	val := RandBytes(100)
-	return trienode.New(crypto.Keccak256Hash(val), val)
 }

trie/stacktrie_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/trie/testutil"
+	"github.com/ethereum/go-ethereum/internal/testrand"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/exp/slices"
 )
@@ -431,12 +431,12 @@ func TestPartialStackTrie(t *testing.T) {
 		for i := 0; i < n; i++ {
 			var val []byte
 			if rand.Intn(3) == 0 {
-				val = testutil.RandBytes(3)
+				val = testrand.Bytes(3)
 			} else {
-				val = testutil.RandBytes(32)
+				val = testrand.Bytes(32)
 			}
 			entries = append(entries, &kv{
-				k: testutil.RandBytes(32),
+				k: testrand.Bytes(32),
 				v: val,
 			})
 		}

triedb/pathdb/database.go

@@ -34,9 +34,6 @@ import (
 )
 
 const (
-	// maxDiffLayers is the maximum diff layers allowed in the layer tree.
-	maxDiffLayers = 128
-
 	// defaultCleanSize is the default memory allowance of clean cache.
 	defaultCleanSize = 16 * 1024 * 1024
 
@@ -54,6 +51,11 @@ const (
 	DefaultBufferSize = 64 * 1024 * 1024
 )
 
+var (
+	// maxDiffLayers is the maximum diff layers allowed in the layer tree.
+	maxDiffLayers = 128
+)
+
 // layer is the interface implemented by all state layers which includes some
 // public methods and some additional methods for internal usage.
 type layer interface {

triedb/pathdb/database_test.go

@@ -27,8 +27,8 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/testrand"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/trie/testutil"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/trie/triestate"
 	"github.com/holiman/uint256"
@@ -46,7 +46,10 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm
 			h.Update(key.Bytes(), val)
 		}
 	}
-	root, nodes, _ := h.Commit(false)
+	root, nodes, err := h.Commit(false)
+	if err != nil {
+		panic(fmt.Errorf("failed to commit hasher, err: %w", err))
+	}
 	return root, nodes
 }
 
@@ -54,7 +57,7 @@ func generateAccount(storageRoot common.Hash) types.StateAccount {
 	return types.StateAccount{
 		Nonce:    uint64(rand.Intn(100)),
 		Balance:  uint256.NewInt(rand.Uint64()),
-		CodeHash: testutil.RandBytes(32),
+		CodeHash: testrand.Bytes(32),
 		Root:     storageRoot,
 	}
 }
@@ -101,8 +104,8 @@ func newTester(t *testing.T, historyLimit uint64) *tester {
 		disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
 		db      = New(disk, &Config{
 			StateHistory:   historyLimit,
-			CleanCacheSize: 256 * 1024,
+			CleanCacheSize: 16 * 1024,
-			DirtyCacheSize: 256 * 1024,
+			DirtyCacheSize: 16 * 1024,
 		})
 		obj = &tester{
 			db:           db,
@@ -113,7 +116,7 @@ func newTester(t *testing.T, historyLimit uint64) *tester {
 			snapStorages: make(map[common.Hash]map[common.Hash]map[common.Hash][]byte),
 		}
 	)
-	for i := 0; i < 2*128; i++ {
+	for i := 0; i < 8; i++ {
 		var parent = types.EmptyRootHash
 		if len(obj.roots) != 0 {
 			parent = obj.roots[len(obj.roots)-1]
@@ -146,8 +149,8 @@ func (t *tester) generateStorage(ctx *genctx, addr common.Address) common.Hash {
 		origin   = make(map[common.Hash][]byte)
 	)
 	for i := 0; i < 10; i++ {
-		v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testutil.RandBytes(32)))
+		v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testrand.Bytes(32)))
-		hash := testutil.RandomHash()
+		hash := testrand.Hash()
 
 		storage[hash] = v
 		origin[hash] = nil
@@ -175,8 +178,8 @@ func (t *tester) mutateStorage(ctx *genctx, addr common.Address, root common.Has
 		}
 	}
 	for i := 0; i < 3; i++ {
-		v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testutil.RandBytes(32)))
+		v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testrand.Bytes(32)))
-		hash := testutil.RandomHash()
+		hash := testrand.Hash()
 
 		storage[hash] = v
 		origin[hash] = nil
@@ -218,7 +221,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode
 		switch rand.Intn(opLen) {
 		case createAccountOp:
 			// account creation
-			addr := testutil.RandomAddress()
+			addr := testrand.Address()
 			addrHash := crypto.Keccak256Hash(addr.Bytes())
 			if _, ok := t.accounts[addrHash]; ok {
 				continue
@@ -320,14 +323,16 @@ func (t *tester) verifyState(root common.Hash) error {
 		return errors.New("root node is not available")
 	}
 	for addrHash, account := range t.snapAccounts[root] {
-		blob, err := reader.Node(common.Hash{}, addrHash.Bytes(), crypto.Keccak256Hash(account))
+		path := crypto.Keccak256(addrHash.Bytes())
+		blob, err := reader.Node(common.Hash{}, path, crypto.Keccak256Hash(account))
 		if err != nil || !bytes.Equal(blob, account) {
 			return fmt.Errorf("account is mismatched: %w", err)
 		}
 	}
 	for addrHash, slots := range t.snapStorages[root] {
 		for hash, slot := range slots {
-			blob, err := reader.Node(addrHash, hash.Bytes(), crypto.Keccak256Hash(slot))
+			path := crypto.Keccak256(hash.Bytes())
+			blob, err := reader.Node(addrHash, path, crypto.Keccak256Hash(slot))
 			if err != nil || !bytes.Equal(blob, slot) {
 				return fmt.Errorf("slot is mismatched: %w", err)
 			}
@@ -379,6 +384,12 @@ func (t *tester) bottomIndex() int {
 }
 
 func TestDatabaseRollback(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	// Verify state histories
 	tester := newTester(t, 0)
 	defer tester.release()
@@ -409,6 +420,12 @@ func TestDatabaseRollback(t *testing.T) {
 }
 
 func TestDatabaseRecoverable(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	var (
 		tester = newTester(t, 0)
 		index  = tester.bottomIndex()
@@ -448,6 +465,12 @@ func TestDatabaseRecoverable(t *testing.T) {
 }
 
 func TestDisable(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	tester := newTester(t, 0)
 	defer tester.release()
 
@@ -484,6 +507,12 @@ func TestDisable(t *testing.T) {
 }
 
 func TestCommit(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	tester := newTester(t, 0)
 	defer tester.release()
 
@@ -508,6 +537,12 @@ func TestCommit(t *testing.T) {
 }
 
 func TestJournal(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	tester := newTester(t, 0)
 	defer tester.release()
 
@@ -532,6 +567,12 @@ func TestJournal(t *testing.T) {
 }
 
 func TestCorruptedJournal(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	tester := newTester(t, 0)
 	defer tester.release()
 
@@ -574,6 +615,12 @@ func TestCorruptedJournal(t *testing.T) {
 // truncating the tail histories. This ensures that the ID of the persistent state
 // always falls within the range of [oldest-history-id, latest-history-id].
 func TestTailTruncateHistory(t *testing.T) {
+	// Redefine the diff layer depth allowance for faster testing.
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
 	tester := newTester(t, 10)
 	defer tester.release()
 

triedb/pathdb/difflayer_test.go

@@ -22,7 +22,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/trie/testutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/testrand"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 )
 
@@ -66,8 +67,9 @@ func benchmarkSearch(b *testing.B, depth int, total int) {
 		nodes[common.Hash{}] = make(map[string]*trienode.Node)
 		for i := 0; i < 3000; i++ {
 			var (
-				path = testutil.RandBytes(32)
+				path = testrand.Bytes(32)
-				node = testutil.RandomNode()
+				blob = testrand.Bytes(100)
+				node = trienode.New(crypto.Keccak256Hash(blob), blob)
 			)
 			nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob)
 			if npath == nil && depth == index {
@@ -112,8 +114,9 @@ func BenchmarkPersist(b *testing.B) {
 		nodes[common.Hash{}] = make(map[string]*trienode.Node)
 		for i := 0; i < 3000; i++ {
 			var (
-				path = testutil.RandBytes(32)
+				path = testrand.Bytes(32)
-				node = testutil.RandomNode()
+				blob = testrand.Bytes(100)
+				node = trienode.New(crypto.Keccak256Hash(blob), blob)
 			)
 			nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob)
 		}
@@ -149,8 +152,9 @@ func BenchmarkJournal(b *testing.B) {
 		nodes[common.Hash{}] = make(map[string]*trienode.Node)
 		for i := 0; i < 3000; i++ {
 			var (
-				path = testutil.RandBytes(32)
+				path = testrand.Bytes(32)
-				node = testutil.RandomNode()
+				blob = testrand.Bytes(100)
+				node = trienode.New(crypto.Keccak256Hash(blob), blob)
 			)
 			nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob)
 		}

triedb/pathdb/history_test.go

@@ -26,8 +26,8 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/internal/testrand"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/trie/testutil"
 	"github.com/ethereum/go-ethereum/trie/triestate"
 )
 
@@ -38,11 +38,11 @@ func randomStateSet(n int) *triestate.Set {
 		storages = make(map[common.Address]map[common.Hash][]byte)
 	)
 	for i := 0; i < n; i++ {
-		addr := testutil.RandomAddress()
+		addr := testrand.Address()
 		storages[addr] = make(map[common.Hash][]byte)
 		for j := 0; j < 3; j++ {
-			v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testutil.RandBytes(32)))
+			v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(testrand.Bytes(32)))
-			storages[addr][testutil.RandomHash()] = v
+			storages[addr][testrand.Hash()] = v
 		}
 		account := generateAccount(types.EmptyRootHash)
 		accounts[addr] = types.SlimAccountRLP(account)
@@ -51,7 +51,7 @@ func randomStateSet(n int) *triestate.Set {
 }
 
 func makeHistory() *history {
-	return newHistory(testutil.RandomHash(), types.EmptyRootHash, 0, randomStateSet(3))
+	return newHistory(testrand.Hash(), types.EmptyRootHash, 0, randomStateSet(3))
 }
 
 func makeHistories(n int) []*history {
@@ -60,7 +60,7 @@ func makeHistories(n int) []*history {
 		result []*history
 	)
 	for i := 0; i < n; i++ {
-		root := testutil.RandomHash()
+		root := testrand.Hash()
 		h := newHistory(root, parent, uint64(i), randomStateSet(3))
 		parent = root
 		result = append(result, h)

triedb/pathdb/testutils.go

@@ -93,10 +93,13 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, e
 		if bytes.Equal(val, h.cleans[hash]) {
 			continue
 		}
+		// Utilize the hash of the state key as the node path to mitigate
+		// potential collisions within the path.
+		path := crypto.Keccak256(hash.Bytes())
 		if len(val) == 0 {
-			set.AddNode(hash.Bytes(), trienode.NewDeleted())
+			set.AddNode(path, trienode.NewDeleted())
 		} else {
-			set.AddNode(hash.Bytes(), trienode.New(crypto.Keccak256Hash(val), val))
+			set.AddNode(path, trienode.New(crypto.Keccak256Hash(val), val))
 		}
 	}
 	root, blob := hash(nodes)