diff --git a/core/state/dump.go b/core/state/dump.go index a742cf85f5..ded9298ee2 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -27,7 +27,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// DumpAccount represents an account in the state +// DumpAccount represents an account in the state. type DumpAccount struct { Balance string `json:"balance"` Nonce uint64 `json:"nonce"` @@ -40,17 +40,24 @@ type DumpAccount struct { } -// Dump represents the full dump in a collected format, as one large map +// Dump represents the full dump in a collected format, as one large map. type Dump struct { Root string `json:"root"` Accounts map[common.Address]DumpAccount `json:"accounts"` } -// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively +// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively. type iterativeDump struct { *json.Encoder } +// IteratorDump is an implementation for iterating over data. +type IteratorDump struct { + Root string `json:"root"` + Accounts map[common.Address]DumpAccount `json:"accounts"` + Next []byte `json:"next,omitempty"` // nil if no more accounts +} + // Collector interface which the state trie calls during iteration type collector interface { onRoot(common.Hash) @@ -64,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) { func (d *Dump) onAccount(addr common.Address, account DumpAccount) { d.Accounts[addr] = account } +func (d *IteratorDump) onRoot(root common.Hash) { + d.Root = fmt.Sprintf("%x", root) +} + +func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) { + d.Accounts[addr] = account +} func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) { dumpAccount := &DumpAccount{ @@ -88,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) { }{root}) } -func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) { +func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) { emptyAddress := (common.Address{}) missingPreimages := 0 c.onRoot(s.trie.Hash()) - it := trie.NewIterator(s.trie.NodeIterator(nil)) + + var count int + it := trie.NewIterator(s.trie.NodeIterator(start)) for it.Next() { var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { @@ -130,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP } } c.onAccount(addr, account) + count++ + if maxResults > 0 && count >= maxResults { + if it.Next() { + nextKey = it.Key + } + break + } } if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } + + return nextKey } // RawDump returns the entire state an a single large object @@ -141,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b dump := &Dump{ Accounts: make(map[common.Address]DumpAccount), } - s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages) + s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) return *dump } @@ -157,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { - s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages) + s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) +} + +// IteratorDump dumps out a batch of accounts starts with the given start key +func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump { + iterator := &IteratorDump{ + Accounts: make(map[common.Address]DumpAccount), + } + iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults) + return *iterator } diff --git a/eth/api.go b/eth/api.go index a874582e19..b3415f923c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -351,70 +351,50 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, return results, nil } -// AccountRangeResult returns a mapping from the hash of an account addresses -// to its preimage. It will return the JSON null if no preimage is found. -// Since a query can return a limited amount of results, a "next" field is -// also present for paging. -type AccountRangeResult struct { - Accounts map[common.Hash]*common.Address `json:"accounts"` - Next common.Hash `json:"next"` -} - -func accountRange(st state.Trie, start *common.Hash, maxResults int) (AccountRangeResult, error) { - if start == nil { - start = &common.Hash{0} - } - it := trie.NewIterator(st.NodeIterator(start.Bytes())) - result := AccountRangeResult{Accounts: make(map[common.Hash]*common.Address), Next: common.Hash{}} - - if maxResults > AccountRangeMaxResults { - maxResults = AccountRangeMaxResults - } - - for i := 0; i < maxResults && it.Next(); i++ { - if preimage := st.GetKey(it.Key); preimage != nil { - addr := &common.Address{} - addr.SetBytes(preimage) - result.Accounts[common.BytesToHash(it.Key)] = addr - } else { - result.Accounts[common.BytesToHash(it.Key)] = nil - } - } - - if it.Next() { - result.Next = common.BytesToHash(it.Key) - } - - return result, nil -} - // AccountRangeMaxResults is the maximum number of results to be returned per call const AccountRangeMaxResults = 256 -// AccountRange enumerates all accounts in the latest state -func (api *PrivateDebugAPI) AccountRange(ctx context.Context, start *common.Hash, maxResults int) (AccountRangeResult, error) { - var statedb *state.StateDB +// AccountRangeAt enumerates all accounts in the given block and start point in paging request +func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { + var stateDb *state.StateDB var err error - block := api.eth.blockchain.CurrentBlock() - if len(block.Transactions()) == 0 { - statedb, err = api.computeStateDB(block, defaultTraceReexec) - if err != nil { - return AccountRangeResult{}, err + if number, ok := blockNrOrHash.Number(); ok { + if number == rpc.PendingBlockNumber { + // If we're dumping the pending state, we need to request + // both the pending block as well as the pending state from + // the miner and operate on those + _, stateDb = api.eth.miner.Pending() + } else { + var block *types.Block + if number == rpc.LatestBlockNumber { + block = api.eth.blockchain.CurrentBlock() + } else { + block = api.eth.blockchain.GetBlockByNumber(uint64(number)) + } + if block == nil { + return state.IteratorDump{}, fmt.Errorf("block #%d not found", number) + } + stateDb, err = api.eth.BlockChain().StateAt(block.Root()) + if err != nil { + return state.IteratorDump{}, err + } } - } else { - _, _, statedb, err = api.computeTxEnv(block.Hash(), len(block.Transactions())-1, 0) + } else if hash, ok := blockNrOrHash.Hash(); ok { + block := api.eth.blockchain.GetBlockByHash(hash) + if block == nil { + return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex()) + } + stateDb, err = api.eth.BlockChain().StateAt(block.Root()) if err != nil { - return AccountRangeResult{}, err + return state.IteratorDump{}, err } } - trie, err := statedb.Database().OpenTrie(block.Header().Root) - if err != nil { - return AccountRangeResult{}, err + if maxResults > AccountRangeMaxResults || maxResults <= 0 { + maxResults = AccountRangeMaxResults } - - return accountRange(trie, start, maxResults) + return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. @@ -431,7 +411,7 @@ type storageEntry struct { } // StorageRangeAt returns the storage at the given block height and transaction index. -func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { +func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { _, _, statedb, err := api.computeTxEnv(blockHash, txIndex, 0) if err != nil { return StorageRangeResult{}, err diff --git a/eth/api_test.go b/eth/api_test.go index ab846db3ea..42f71e261e 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -33,29 +33,24 @@ import ( var dumper = spew.ConfigState{Indent: " "} -func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start *common.Hash, requestedNum int, expectedNum int) AccountRangeResult { - result, err := accountRange(*trie, start, requestedNum) - if err != nil { - t.Fatal(err) - } +func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { + result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum) if len(result.Accounts) != expectedNum { - t.Fatalf("expected %d results. Got %d", expectedNum, len(result.Accounts)) + t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts)) } - - for _, address := range result.Accounts { - if address == nil { - t.Fatalf("null address returned") + for address := range result.Accounts { + if address == (common.Address{}) { + t.Fatalf("empty address returned") } - if !statedb.Exist(*address) { + if !statedb.Exist(address) { t.Fatalf("account not found in state %s", address.Hex()) } } - return result } -type resultHash []*common.Hash +type resultHash []common.Hash func (h resultHash) Len() int { return len(h) } func (h resultHash) Swap(i, j int) { h[i], h[j] = h[j], h[i] } @@ -80,7 +75,6 @@ func TestAccountRange(t *testing.T) { m[addr] = true } } - state.Commit(true) root := state.IntermediateRoot(true) @@ -88,68 +82,40 @@ func TestAccountRange(t *testing.T) { if err != nil { t.Fatal(err) } - - t.Logf("test getting number of results less than max") - accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) - - t.Logf("test getting number of results greater than max %d", AccountRangeMaxResults) - accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults*2, AccountRangeMaxResults) - - t.Logf("test with empty 'start' hash") - accountRangeTest(t, &trie, state, nil, AccountRangeMaxResults, AccountRangeMaxResults) - - t.Logf("test pagination") - + accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) // test pagination - firstResult := accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults, AccountRangeMaxResults) - - t.Logf("test pagination 2") - secondResult := accountRangeTest(t, &trie, state, &firstResult.Next, AccountRangeMaxResults, AccountRangeMaxResults) + firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) + secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) hList := make(resultHash, 0) - for h1, addr1 := range firstResult.Accounts { - h := &common.Hash{} - h.SetBytes(h1.Bytes()) - hList = append(hList, h) - for h2, addr2 := range secondResult.Accounts { - // Make sure that the hashes aren't the same - if bytes.Equal(h1.Bytes(), h2.Bytes()) { - t.Fatalf("pagination test failed: results should not overlap") - } - - // If either address is nil, then it makes no sense to compare - // them as they might be two different accounts. - if addr1 == nil || addr2 == nil { - continue - } - - // Since the two hashes are different, they should not have - // the same preimage, but let's check anyway in case there - // is a bug in the (hash, addr) map generation code. - if bytes.Equal(addr1.Bytes(), addr2.Bytes()) { - t.Fatalf("pagination test failed: addresses should not repeat") - } + for addr1 := range firstResult.Accounts { + // If address is empty, then it makes no sense to compare + // them as they might be two different accounts. + if addr1 == (common.Address{}) { + continue + } + if _, duplicate := secondResult.Accounts[addr1]; duplicate { + t.Fatalf("pagination test failed: results should not overlap") } + hList = append(hList, crypto.Keccak256Hash(addr1.Bytes())) } - // Test to see if it's possible to recover from the middle of the previous // set and get an even split between the first and second sets. - t.Logf("test random access pagination") sort.Sort(hList) middleH := hList[AccountRangeMaxResults/2] middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults) - innone, infirst, insecond := 0, 0, 0 + missing, infirst, insecond := 0, 0, 0 for h := range middleResult.Accounts { if _, ok := firstResult.Accounts[h]; ok { infirst++ } else if _, ok := secondResult.Accounts[h]; ok { insecond++ } else { - innone++ + missing++ } } - if innone != 0 { - t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", innone) + if missing != 0 { + t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", missing) } if infirst != AccountRangeMaxResults/2 { t.Fatalf("Imbalance in the number of first-test results: %d != %d", infirst, AccountRangeMaxResults/2) @@ -164,20 +130,10 @@ func TestEmptyAccountRange(t *testing.T) { statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) state, _ = state.New(common.Hash{}, statedb, nil) ) - state.Commit(true) - root := state.IntermediateRoot(true) - - trie, err := statedb.OpenTrie(root) - if err != nil { - t.Fatal(err) - } - - results, err := accountRange(trie, &common.Hash{0x0}, AccountRangeMaxResults) - if err != nil { - t.Fatalf("Empty results should not trigger an error: %v", err) - } - if results.Next != common.HexToHash("0") { + state.IntermediateRoot(true) + results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults) + if bytes.Equal(results.Next, (common.Hash{}).Bytes()) { t.Fatalf("Empty results should not return a second page") } if len(results.Accounts) != 0 {