@ -22,6 +22,7 @@ import (
"encoding/binary"
"encoding/binary"
"fmt"
"fmt"
"math/big"
"math/big"
mrand "math/rand"
"sync"
"sync"
"testing"
"testing"
"time"
"time"
@ -34,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/testutil"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/trienode"
"golang.org/x/crypto/sha3"
"golang.org/x/crypto/sha3"
@ -253,7 +255,7 @@ func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, orig
func createAccountRequestResponse ( t * testPeer , root common . Hash , origin common . Hash , limit common . Hash , cap uint64 ) ( keys [ ] common . Hash , vals [ ] [ ] byte , proofs [ ] [ ] byte ) {
func createAccountRequestResponse ( t * testPeer , root common . Hash , origin common . Hash , limit common . Hash , cap uint64 ) ( keys [ ] common . Hash , vals [ ] [ ] byte , proofs [ ] [ ] byte ) {
var size uint64
var size uint64
if limit == ( common . Hash { } ) {
if limit == ( common . Hash { } ) {
limit = common . HexToHash ( "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" )
limit = common . MaxHash
}
}
for _ , entry := range t . accountValues {
for _ , entry := range t . accountValues {
if size > cap {
if size > cap {
@ -318,7 +320,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm
if len ( origin ) > 0 {
if len ( origin ) > 0 {
originHash = common . BytesToHash ( origin )
originHash = common . BytesToHash ( origin )
}
}
var limitHash = common . HexToHash ( "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" )
var limitHash = common . MaxHash
if len ( limit ) > 0 {
if len ( limit ) > 0 {
limitHash = common . BytesToHash ( limit )
limitHash = common . BytesToHash ( limit )
}
}
@ -762,7 +764,7 @@ func testSyncWithStorage(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 3 , 3000 , true , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 3 , 3000 , true , fals e , false )
mkSource := func ( name string ) * testPeer {
mkSource := func ( name string ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -772,7 +774,7 @@ func testSyncWithStorage(t *testing.T, scheme string) {
source . storageValues = storageElems
source . storageValues = storageElems
return source
return source
}
}
syncer := setupSyncer ( nodeS cheme, mkSource ( "sourceA" ) )
syncer := setupSyncer ( s cheme, mkSource ( "sourceA" ) )
done := checkStall ( t , term )
done := checkStall ( t , term )
if err := syncer . Sync ( sourceAccountTrie . Hash ( ) , cancel ) ; err != nil {
if err := syncer . Sync ( sourceAccountTrie . Hash ( ) , cancel ) ; err != nil {
t . Fatalf ( "sync failed: %v" , err )
t . Fatalf ( "sync failed: %v" , err )
@ -799,7 +801,7 @@ func testMultiSyncManyUseless(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , fals e , false )
mkSource := func ( name string , noAccount , noStorage , noTrieNode bool ) * testPeer {
mkSource := func ( name string , noAccount , noStorage , noTrieNode bool ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -821,7 +823,7 @@ func testMultiSyncManyUseless(t *testing.T, scheme string) {
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "full" , true , true , true ) ,
mkSource ( "full" , true , true , true ) ,
mkSource ( "noAccounts" , false , true , true ) ,
mkSource ( "noAccounts" , false , true , true ) ,
mkSource ( "noStorage" , true , false , true ) ,
mkSource ( "noStorage" , true , false , true ) ,
@ -853,7 +855,7 @@ func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , fals e , false )
mkSource := func ( name string , noAccount , noStorage , noTrieNode bool ) * testPeer {
mkSource := func ( name string , noAccount , noStorage , noTrieNode bool ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -875,7 +877,7 @@ func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) {
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "full" , true , true , true ) ,
mkSource ( "full" , true , true , true ) ,
mkSource ( "noAccounts" , false , true , true ) ,
mkSource ( "noAccounts" , false , true , true ) ,
mkSource ( "noStorage" , true , false , true ) ,
mkSource ( "noStorage" , true , false , true ) ,
@ -912,7 +914,7 @@ func testMultiSyncManyUnresponsive(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , fals e , false )
mkSource := func ( name string , noAccount , noStorage , noTrieNode bool ) * testPeer {
mkSource := func ( name string , noAccount , noStorage , noTrieNode bool ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -934,7 +936,7 @@ func testMultiSyncManyUnresponsive(t *testing.T, scheme string) {
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "full" , true , true , true ) ,
mkSource ( "full" , true , true , true ) ,
mkSource ( "noAccounts" , false , true , true ) ,
mkSource ( "noAccounts" , false , true , true ) ,
mkSource ( "noStorage" , true , false , true ) ,
mkSource ( "noStorage" , true , false , true ) ,
@ -1215,7 +1217,7 @@ func testSyncBoundaryStorageTrie(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 10 , 1000 , false , true )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 10 , 1000 , false , true , fals e )
mkSource := func ( name string ) * testPeer {
mkSource := func ( name string ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -1226,7 +1228,7 @@ func testSyncBoundaryStorageTrie(t *testing.T, scheme string) {
return source
return source
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "peer-a" ) ,
mkSource ( "peer-a" ) ,
mkSource ( "peer-b" ) ,
mkSource ( "peer-b" ) ,
)
)
@ -1257,7 +1259,7 @@ func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 300 , 1000 , false , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 300 , 1000 , false , false , false )
mkSource := func ( name string , slow bool ) * testPeer {
mkSource := func ( name string , slow bool ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -1273,7 +1275,7 @@ func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) {
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "nice-a" , false ) ,
mkSource ( "nice-a" , false ) ,
mkSource ( "slow" , true ) ,
mkSource ( "slow" , true ) ,
)
)
@ -1304,7 +1306,7 @@ func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , fals e , false )
mkSource := func ( name string , handler storageHandlerFunc ) * testPeer {
mkSource := func ( name string , handler storageHandlerFunc ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -1317,7 +1319,7 @@ func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) {
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "nice-a" , defaultStorageRequestHandler ) ,
mkSource ( "nice-a" , defaultStorageRequestHandler ) ,
mkSource ( "nice-b" , defaultStorageRequestHandler ) ,
mkSource ( "nice-b" , defaultStorageRequestHandler ) ,
mkSource ( "nice-c" , defaultStorageRequestHandler ) ,
mkSource ( "nice-c" , defaultStorageRequestHandler ) ,
@ -1348,7 +1350,7 @@ func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) {
} )
} )
}
}
)
)
nodeScheme , sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , false )
sourceAccountTrie , elems , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 100 , 3000 , true , fals e , false )
mkSource := func ( name string , handler storageHandlerFunc ) * testPeer {
mkSource := func ( name string , handler storageHandlerFunc ) * testPeer {
source := newTestPeer ( name , t , term )
source := newTestPeer ( name , t , term )
@ -1360,7 +1362,7 @@ func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) {
return source
return source
}
}
syncer := setupSyncer (
syncer := setupSyncer (
nodeS cheme,
s cheme,
mkSource ( "nice-a" , defaultStorageRequestHandler ) ,
mkSource ( "nice-a" , defaultStorageRequestHandler ) ,
mkSource ( "nice-b" , defaultStorageRequestHandler ) ,
mkSource ( "nice-b" , defaultStorageRequestHandler ) ,
mkSource ( "nice-c" , defaultStorageRequestHandler ) ,
mkSource ( "nice-c" , defaultStorageRequestHandler ) ,
@ -1413,6 +1415,45 @@ func testSyncWithStorageMisbehavingProve(t *testing.T, scheme string) {
verifyTrie ( scheme , syncer . db , sourceAccountTrie . Hash ( ) , t )
verifyTrie ( scheme , syncer . db , sourceAccountTrie . Hash ( ) , t )
}
}
// TestSyncWithUnevenStorage tests sync where the storage trie is not even
// and with a few empty ranges.
func TestSyncWithUnevenStorage ( t * testing . T ) {
t . Parallel ( )
testSyncWithUnevenStorage ( t , rawdb . HashScheme )
testSyncWithUnevenStorage ( t , rawdb . PathScheme )
}
func testSyncWithUnevenStorage ( t * testing . T , scheme string ) {
var (
once sync . Once
cancel = make ( chan struct { } )
term = func ( ) {
once . Do ( func ( ) {
close ( cancel )
} )
}
)
accountTrie , accounts , storageTries , storageElems := makeAccountTrieWithStorage ( scheme , 3 , 256 , false , false , true )
mkSource := func ( name string ) * testPeer {
source := newTestPeer ( name , t , term )
source . accountTrie = accountTrie . Copy ( )
source . accountValues = accounts
source . setStorageTries ( storageTries )
source . storageValues = storageElems
source . storageRequestHandler = func ( t * testPeer , reqId uint64 , root common . Hash , accounts [ ] common . Hash , origin , limit [ ] byte , max uint64 ) error {
return defaultStorageRequestHandler ( t , reqId , root , accounts , origin , limit , 128 ) // retrieve storage in large mode
}
return source
}
syncer := setupSyncer ( scheme , mkSource ( "source" ) )
if err := syncer . Sync ( accountTrie . Hash ( ) , cancel ) ; err != nil {
t . Fatalf ( "sync failed: %v" , err )
}
verifyTrie ( scheme , syncer . db , accountTrie . Hash ( ) , t )
}
type kv struct {
type kv struct {
k , v [ ] byte
k , v [ ] byte
}
}
@ -1511,7 +1552,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) {
for i := 0 ; i < accountConcurrency ; i ++ {
for i := 0 ; i < accountConcurrency ; i ++ {
last := common . BigToHash ( new ( big . Int ) . Add ( next . Big ( ) , step ) )
last := common . BigToHash ( new ( big . Int ) . Add ( next . Big ( ) , step ) )
if i == accountConcurrency - 1 {
if i == accountConcurrency - 1 {
last = common . HexToHash ( "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" )
last = common . MaxHash
}
}
boundaries = append ( boundaries , last )
boundaries = append ( boundaries , last )
next = common . BigToHash ( new ( big . Int ) . Add ( last . Big ( ) , common . Big1 ) )
next = common . BigToHash ( new ( big . Int ) . Add ( last . Big ( ) , common . Big1 ) )
@ -1608,7 +1649,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots
}
}
// makeAccountTrieWithStorage spits out a trie, along with the leafs
// makeAccountTrieWithStorage spits out a trie, along with the leafs
func makeAccountTrieWithStorage ( scheme string , accounts , slots int , code , boundary bool ) ( string , * trie . Trie , [ ] * kv , map [ common . Hash ] * trie . Trie , map [ common . Hash ] [ ] * kv ) {
func makeAccountTrieWithStorage ( scheme string , accounts , slots int , code , boundary bool , uneven bool ) ( * trie . Trie , [ ] * kv , map [ common . Hash ] * trie . Trie , map [ common . Hash ] [ ] * kv ) {
var (
var (
db = trie . NewDatabase ( rawdb . NewMemoryDatabase ( ) , newDbConfig ( scheme ) )
db = trie . NewDatabase ( rawdb . NewMemoryDatabase ( ) , newDbConfig ( scheme ) )
accTrie = trie . NewEmpty ( db )
accTrie = trie . NewEmpty ( db )
@ -1633,6 +1674,8 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda
)
)
if boundary {
if boundary {
stRoot , stNodes , stEntries = makeBoundaryStorageTrie ( common . BytesToHash ( key ) , slots , db )
stRoot , stNodes , stEntries = makeBoundaryStorageTrie ( common . BytesToHash ( key ) , slots , db )
} else if uneven {
stRoot , stNodes , stEntries = makeUnevenStorageTrie ( common . BytesToHash ( key ) , slots , db )
} else {
} else {
stRoot , stNodes , stEntries = makeStorageTrieWithSeed ( common . BytesToHash ( key ) , uint64 ( slots ) , 0 , db )
stRoot , stNodes , stEntries = makeStorageTrieWithSeed ( common . BytesToHash ( key ) , uint64 ( slots ) , 0 , db )
}
}
@ -1675,7 +1718,7 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda
}
}
storageTries [ common . BytesToHash ( key ) ] = trie
storageTries [ common . BytesToHash ( key ) ] = trie
}
}
return db . Scheme ( ) , accTrie , entries , storageTries , storageEntries
return accTrie , entries , storageTries , storageEntries
}
}
// makeStorageTrieWithSeed fills a storage trie with n items, returning the
// makeStorageTrieWithSeed fills a storage trie with n items, returning the
@ -1721,7 +1764,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo
for i := 0 ; i < accountConcurrency ; i ++ {
for i := 0 ; i < accountConcurrency ; i ++ {
last := common . BigToHash ( new ( big . Int ) . Add ( next . Big ( ) , step ) )
last := common . BigToHash ( new ( big . Int ) . Add ( next . Big ( ) , step ) )
if i == accountConcurrency - 1 {
if i == accountConcurrency - 1 {
last = common . HexToHash ( "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" )
last = common . MaxHash
}
}
boundaries = append ( boundaries , last )
boundaries = append ( boundaries , last )
next = common . BigToHash ( new ( big . Int ) . Add ( last . Big ( ) , common . Big1 ) )
next = common . BigToHash ( new ( big . Int ) . Add ( last . Big ( ) , common . Big1 ) )
@ -1752,6 +1795,38 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo
return root , nodes , entries
return root , nodes , entries
}
}
// makeUnevenStorageTrie constructs a storage tries will states distributed in
// different range unevenly.
func makeUnevenStorageTrie ( owner common . Hash , slots int , db * trie . Database ) ( common . Hash , * trienode . NodeSet , [ ] * kv ) {
var (
entries [ ] * kv
tr , _ = trie . New ( trie . StorageTrieID ( types . EmptyRootHash , owner , types . EmptyRootHash ) , db )
chosen = make ( map [ byte ] struct { } )
)
for i := 0 ; i < 3 ; i ++ {
var n int
for {
n = mrand . Intn ( 15 ) // the last range is set empty deliberately
if _ , ok := chosen [ byte ( n ) ] ; ok {
continue
}
chosen [ byte ( n ) ] = struct { } { }
break
}
for j := 0 ; j < slots / 3 ; j ++ {
key := append ( [ ] byte { byte ( n ) } , testutil . RandBytes ( 31 ) ... )
val , _ := rlp . EncodeToBytes ( testutil . RandBytes ( 32 ) )
elem := & kv { key , val }
tr . MustUpdate ( elem . k , elem . v )
entries = append ( entries , elem )
}
}
slices . SortFunc ( entries , ( * kv ) . cmp )
root , nodes , _ := tr . Commit ( false )
return root , nodes , entries
}
func verifyTrie ( scheme string , db ethdb . KeyValueStore , root common . Hash , t * testing . T ) {
func verifyTrie ( scheme string , db ethdb . KeyValueStore , root common . Hash , t * testing . T ) {
t . Helper ( )
t . Helper ( )
triedb := trie . NewDatabase ( rawdb . NewDatabase ( db ) , newDbConfig ( scheme ) )
triedb := trie . NewDatabase ( rawdb . NewDatabase ( db ) , newDbConfig ( scheme ) )