mirror
/
go-ethereum
mirror of https://github.com/ethereum/go-ethereum
2
0
Fork 1
Code Issues Releases Wiki Activity

core/state, cmd/geth: streaming json output for dump command (#15475)

Browse Source 
* core/state, cmd/geth: streaming json output dump cmd + optional code+storage

* dump: add option to continue even if preimages are missing

* core, evm: lint nits

* cmd: use local flags for dump, omit empty code/storage

* core/state: fix state dump test
pull/19753/head
Martin Holst Swende 5 years ago committed by Péter Szilágyi
parent e4a1488b2f
commit 1da5e0ebb0
7 changed files with 142 additions and 44 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      cmd/evm/runner.go
  2. 2
      cmd/evm/staterunner.go
  3. 21
      cmd/geth/chaincmd.go
  4. 16
      cmd/utils/flags.go
  5. 122
      core/state/dump.go
  6. 19
      core/state/state_test.go
  7. 4
      eth/api.go

2
cmd/evm/runner.go
Unescape View File

@ -209,7 +209,7 @@ func runCmd(ctx *cli.Context) error {
if ctx.GlobalBool(DumpFlag.Name) {
statedb.Commit(true)
statedb.IntermediateRoot(true)
fmt.Println(string(statedb.Dump()))
fmt.Println(string(statedb.Dump(false, false, true)))
}
if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {

2
cmd/evm/staterunner.go
Unescape View File

@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error {
// Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error()
if ctx.GlobalBool(DumpFlag.Name) && state != nil {
dump := state.RawDump()
dump := state.RawDump(false, false, true)
result.State = &dump
}
}

21
cmd/geth/chaincmd.go
Unescape View File

@ -162,6 +162,10 @@ Remove blockchain and state databases`,
utils.DataDirFlag,
utils.CacheFlag,
utils.SyncModeFlag,
utils.IterativeOutputFlag,
utils.ExcludeCodeFlag,
utils.ExcludeStorageFlag,
utils.IncludeIncompletesFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -287,7 +291,7 @@ func importChain(ctx *cli.Context) error {
fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000)
fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs))
if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
if ctx.GlobalBool(utils.NoCompactionFlag.Name) {
return nil
}
@ -504,6 +508,7 @@ func dump(ctx *cli.Context) error {
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack)
defer chainDb.Close()
for _, arg := range ctx.Args() {
var block *types.Block
if hashish(arg) {
@ -520,10 +525,20 @@ func dump(ctx *cli.Context) error {
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}
fmt.Printf("%s\n", state.Dump())
excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name)
excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name)
includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name)
if ctx.Bool(utils.IterativeOutputFlag.Name) {
state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
} else {
if includeMissing {
fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
" otherwise the accounts will overwrite each other in the resulting mapping.")
}
fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
}
}
}
chainDb.Close()
return nil
}

16
cmd/utils/flags.go
Unescape View File

@ -196,6 +196,22 @@ var (
Name: "ulc.trusted",
Usage: "List of trusted ULC servers",
}
IterativeOutputFlag = cli.BoolFlag{
Name: "iterative",
Usage: "Print streaming JSON iteratively, delimited by newlines",
}
ExcludeStorageFlag = cli.BoolFlag{
Name: "nostorage",
Usage: "Exclude storage entries (save db lookups)",
}
IncludeIncompletesFlag = cli.BoolFlag{
Name: "incompletes",
Usage: "Include accounts for which we don't have the address (missing preimage)",
}
ExcludeCodeFlag = cli.BoolFlag{
Name: "nocode",
Usage: "Exclude contract code (save db lookups)",
}
defaultSyncMode = eth.DefaultConfig.SyncMode
SyncModeFlag = TextMarshalerFlag{
Name: "syncmode",

122
core/state/dump.go
Unescape View File

@ -21,61 +21,133 @@ import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// DumpAccount represents an account in the state
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
Root string `json:"root"`
CodeHash string `json:"codeHash"`
Code string `json:"code"`
Storage map[string]string `json:"storage"`
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
Root string `json:"root"`
CodeHash string `json:"codeHash"`
Code string `json:"code,omitempty"`
Storage map[common.Hash]string `json:"storage,omitempty"`
Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key
}
// Dump represents the full dump in a collected format, as one large map
type Dump struct {
Root string `json:"root"`
Accounts map[string]DumpAccount `json:"accounts"`
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
}
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
type iterativeDump json.Encoder
// Collector interface which the state trie calls during iteration
type collector interface {
onRoot(common.Hash)
onAccount(common.Address, DumpAccount)
}
func (self *Dump) onRoot(root common.Hash) {
self.Root = fmt.Sprintf("%x", root)
}
func (self *StateDB) RawDump() Dump {
dump := Dump{
Root: fmt.Sprintf("%x", self.trie.Hash()),
Accounts: make(map[string]DumpAccount),
func (self *Dump) onAccount(addr common.Address, account DumpAccount) {
self.Accounts[addr] = account
}
func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) {
dumpAccount := &DumpAccount{
Balance: account.Balance,
Nonce: account.Nonce,
Root: account.Root,
CodeHash: account.CodeHash,
Code: account.Code,
Storage: account.Storage,
SecureKey: account.SecureKey,
Address: nil,
}
if addr != (common.Address{}) {
dumpAccount.Address = &addr
}
(*json.Encoder)(&self).Encode(dumpAccount)
}
func (self iterativeDump) onRoot(root common.Hash) {
(*json.Encoder)(&self).Encode(struct {
Root common.Hash `json:"root"`
}{root})
}
func (self *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
emptyAddress := (common.Address{})
missingPreimages := 0
c.onRoot(self.trie.Hash())
it := trie.NewIterator(self.trie.NodeIterator(nil))
for it.Next() {
addr := self.trie.GetKey(it.Key)
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
panic(err)
}
obj := newObject(nil, common.BytesToAddress(addr), data)
addr := common.BytesToAddress(self.trie.GetKey(it.Key))
obj := newObject(nil, addr, data)
account := DumpAccount{
Balance: data.Balance.String(),
Nonce: data.Nonce,
Root: common.Bytes2Hex(data.Root[:]),
CodeHash: common.Bytes2Hex(data.CodeHash),
Code: common.Bytes2Hex(obj.Code(self.db)),
Storage: make(map[string]string),
}
storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
for storageIt.Next() {
account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
if emptyAddress == addr {
// Preimage missing
missingPreimages++
if excludeMissingPreimages {
continue
}
account.SecureKey = it.Key
}
if !excludeCode {
account.Code = common.Bytes2Hex(obj.Code(self.db))
}
if !excludeStorage {
account.Storage = make(map[common.Hash]string)
storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
for storageIt.Next() {
account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
}
}
dump.Accounts[common.Bytes2Hex(addr)] = account
c.onAccount(addr, account)
}
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
}
// RawDump returns the entire state an a single large object
func (self *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
return dump
self.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
return *dump
}
func (self *StateDB) Dump() []byte {
json, err := json.MarshalIndent(self.RawDump(), "", " ")
// Dump returns a JSON string representing the entire state as a single json-object
func (self *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
dump := self.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
json, err := json.MarshalIndent(dump, "", " ")
if err != nil {
fmt.Println("dump err", err)
}
return json
}
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (self *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
self.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages)
}

19
core/state/state_test.go
Unescape View File

@ -52,33 +52,28 @@ func (s *StateSuite) TestDump(c *checker.C) {
s.state.Commit(false)
// check that dump contains the state objects that are in trie
got := string(s.state.Dump())
got := string(s.state.Dump(false, false, true))
want := `{
"root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
"accounts": {
"0000000000000000000000000000000000000001": {
"0x0000000000000000000000000000000000000001": {
"balance": "22",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
"code": "",
"storage": {}
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
},
"0000000000000000000000000000000000000002": {
"0x0000000000000000000000000000000000000002": {
"balance": "44",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
"code": "",
"storage": {}
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
},
"0000000000000000000000000000000000000102": {
"0x0000000000000000000000000000000000000102": {
"balance": "0",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3",
"code": "03030303030303",
"storage": {}
"code": "03030303030303"
}
}
}`

4
eth/api.go
Unescape View File

@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
// both the pending block as well as the pending state from
// the miner and operate on those
_, stateDb := api.eth.miner.Pending()
return stateDb.RawDump(), nil
return stateDb.RawDump(false, false, true), nil
}
var block *types.Block
if blockNr == rpc.LatestBlockNumber {
@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
if err != nil {
return state.Dump{}, err
}
return stateDb.RawDump(), nil
return stateDb.RawDump(false, false, true), nil
}
// PrivateDebugAPI is the collection of Ethereum full node APIs exposed over

Write Preview
Loading…
Cancel
Save