@ -63,6 +63,10 @@ const (
defaultTracechainMemLimit = common . StorageSize ( 500 * 1024 * 1024 )
defaultTracechainMemLimit = common . StorageSize ( 500 * 1024 * 1024 )
)
)
// StateReleaseFunc is used to deallocate resources held by constructing a
// historical state for tracing purposes.
type StateReleaseFunc func ( )
// Backend interface provides the common API services (that are provided by
// Backend interface provides the common API services (that are provided by
// both full and light clients) with access to necessary functions.
// both full and light clients) with access to necessary functions.
type Backend interface {
type Backend interface {
@ -75,11 +79,8 @@ type Backend interface {
ChainConfig ( ) * params . ChainConfig
ChainConfig ( ) * params . ChainConfig
Engine ( ) consensus . Engine
Engine ( ) consensus . Engine
ChainDb ( ) ethdb . Database
ChainDb ( ) ethdb . Database
// StateAtBlock returns the state corresponding to the stateroot of the block.
StateAtBlock ( ctx context . Context , block * types . Block , reexec uint64 , base * state . StateDB , readOnly bool , preferDisk bool ) ( * state . StateDB , StateReleaseFunc , error )
// N.B: For executing transactions on block N, the required stateRoot is block N-1,
StateAtTransaction ( ctx context . Context , block * types . Block , txIndex int , reexec uint64 ) ( core . Message , vm . BlockContext , * state . StateDB , StateReleaseFunc , error )
// so this method should be called with the parent.
StateAtBlock ( ctx context . Context , block * types . Block , reexec uint64 , base * state . StateDB , checkLive , preferDisk bool ) ( * state . StateDB , error )
StateAtTransaction ( ctx context . Context , block * types . Block , txIndex int , reexec uint64 ) ( core . Message , vm . BlockContext , * state . StateDB , error )
}
}
// API is the collection of tracing APIs exposed over the private debugging endpoint.
// API is the collection of tracing APIs exposed over the private debugging endpoint.
@ -201,7 +202,7 @@ type txTraceResult struct {
type blockTraceTask struct {
type blockTraceTask struct {
statedb * state . StateDB // Intermediate state prepped for tracing
statedb * state . StateDB // Intermediate state prepped for tracing
block * types . Block // Block to trace the transactions from
block * types . Block // Block to trace the transactions from
rootref common . Hash // Trie root reference held for this task
release StateReleaseFunc // The function to release the held resource for this task
results [ ] * txTraceResult // Trace results produced by the task
results [ ] * txTraceResult // Trace results produced by the task
}
}
@ -234,13 +235,6 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf
if from . Number ( ) . Cmp ( to . Number ( ) ) >= 0 {
if from . Number ( ) . Cmp ( to . Number ( ) ) >= 0 {
return nil , fmt . Errorf ( "end block (#%d) needs to come after start block (#%d)" , end , start )
return nil , fmt . Errorf ( "end block (#%d) needs to come after start block (#%d)" , end , start )
}
}
return api . traceChain ( ctx , from , to , config )
}
// traceChain configures a new tracer according to the provided configuration, and
// executes all the transactions contained within. The return value will be one item
// per transaction, dependent on the requested tracer.
func ( api * API ) traceChain ( ctx context . Context , start , end * types . Block , config * TraceConfig ) ( * rpc . Subscription , error ) {
// Tracing a chain is a **long** operation, only do with subscriptions
// Tracing a chain is a **long** operation, only do with subscriptions
notifier , supported := rpc . NotifierFromContext ( ctx )
notifier , supported := rpc . NotifierFromContext ( ctx )
if ! supported {
if ! supported {
@ -248,8 +242,45 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config
}
}
sub := notifier . CreateSubscription ( )
sub := notifier . CreateSubscription ( )
// Prepare all the states for tracing. Note this procedure can take very
resCh := api . traceChain ( from , to , config , notifier . Closed ( ) )
// long time. Timeout mechanism is necessary.
go func ( ) {
for result := range resCh {
notifier . Notify ( sub . ID , result )
}
} ( )
return sub , nil
}
// releaser is a helper tool responsible for caching the release
// callbacks of tracing state.
type releaser struct {
releases [ ] StateReleaseFunc
lock sync . Mutex
}
func ( r * releaser ) add ( release StateReleaseFunc ) {
r . lock . Lock ( )
defer r . lock . Unlock ( )
r . releases = append ( r . releases , release )
}
func ( r * releaser ) call ( ) {
r . lock . Lock ( )
defer r . lock . Unlock ( )
for _ , release := range r . releases {
release ( )
}
r . releases = r . releases [ : 0 ]
}
// traceChain configures a new tracer according to the provided configuration, and
// executes all the transactions contained within. The tracing chain range includes
// the end block but excludes the start one. The return value will be one item per
// transaction, dependent on the requested tracer.
// The tracing procedure should be aborted in case the closed signal is received.
func ( api * API ) traceChain ( start , end * types . Block , config * TraceConfig , closed <- chan interface { } ) chan * blockTraceResult {
reexec := defaultTraceReexec
reexec := defaultTraceReexec
if config != nil && config . Reexec != nil {
if config != nil && config . Reexec != nil {
reexec = * config . Reexec
reexec = * config . Reexec
@ -260,20 +291,23 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config
threads = blocks
threads = blocks
}
}
var (
var (
pend = new ( sync . WaitGroup )
pend = new ( sync . WaitGroup )
tasks = make ( chan * blockTraceTask , threads )
ctx = context . Background ( )
results = make ( chan * blockTraceTask , threads )
taskCh = make ( chan * blockTraceTask , threads )
localctx = context . Background ( )
resCh = make ( chan * blockTraceTask , threads )
reler = new ( releaser )
)
)
for th := 0 ; th < threads ; th ++ {
for th := 0 ; th < threads ; th ++ {
pend . Add ( 1 )
pend . Add ( 1 )
go func ( ) {
go func ( ) {
defer pend . Done ( )
defer pend . Done ( )
// Fetch and execute the next block trace tasks
// Fetch and execute the block trace taskCh
for task := range tasks {
for task := range taskCh {
signer := types . MakeSigner ( api . backend . ChainConfig ( ) , task . block . Number ( ) )
var (
blockCtx := core . NewEVMBlockContext ( task . block . Header ( ) , api . chainContext ( localctx ) , nil )
signer = types . MakeSigner ( api . backend . ChainConfig ( ) , task . block . Number ( ) )
blockCtx = core . NewEVMBlockContext ( task . block . Header ( ) , api . chainContext ( ctx ) , nil )
)
// Trace all the transactions contained within
// Trace all the transactions contained within
for i , tx := range task . block . Transactions ( ) {
for i , tx := range task . block . Transactions ( ) {
msg , _ := tx . AsMessage ( signer , task . block . BaseFee ( ) )
msg , _ := tx . AsMessage ( signer , task . block . BaseFee ( ) )
@ -282,7 +316,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config
TxIndex : i ,
TxIndex : i ,
TxHash : tx . Hash ( ) ,
TxHash : tx . Hash ( ) ,
}
}
res , err := api . traceTx ( local ctx, msg , txctx , blockCtx , task . statedb , config )
res , err := api . traceTx ( ctx , msg , txctx , blockCtx , task . statedb , config )
if err != nil {
if err != nil {
task . results [ i ] = & txTraceResult { Error : err . Error ( ) }
task . results [ i ] = & txTraceResult { Error : err . Error ( ) }
log . Warn ( "Tracing failed" , "hash" , tx . Hash ( ) , "block" , task . block . NumberU64 ( ) , "err" , err )
log . Warn ( "Tracing failed" , "hash" , tx . Hash ( ) , "block" , task . block . NumberU64 ( ) , "err" , err )
@ -292,36 +326,38 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config
task . statedb . Finalise ( api . backend . ChainConfig ( ) . IsEIP158 ( task . block . Number ( ) ) )
task . statedb . Finalise ( api . backend . ChainConfig ( ) . IsEIP158 ( task . block . Number ( ) ) )
task . results [ i ] = & txTraceResult { Result : res }
task . results [ i ] = & txTraceResult { Result : res }
}
}
// Stream the result back to the user or abort on teardown
// Tracing state is used up, queue it for de-referencing
reler . add ( task . release )
// Stream the result back to the result catcher or abort on teardown
select {
select {
case results <- task :
case resCh <- task :
case <- notifier . Closed ( ) :
case <- cl osed:
return
return
}
}
}
}
} ( )
} ( )
}
}
// Start a goroutine to feed all the blocks into the tracers
// Start a goroutine to feed all the blocks into the tracers
var (
begin = time . Now ( )
derefTodo [ ] common . Hash // list of hashes to dereference from the db
derefsMu sync . Mutex // mutex for the derefs
)
go func ( ) {
go func ( ) {
var (
var (
logged time . Time
logged time . Time
begin = time . Now ( )
number uint64
number uint64
traced uint64
traced uint64
failed error
failed error
parent common . Hash
statedb * state . StateDB
statedb * state . StateDB
release StateReleaseFunc
)
)
// Ensure everything is properly cleaned up on any exit path
// Ensure everything is properly cleaned up on any exit path
defer func ( ) {
defer func ( ) {
close ( tasks )
close ( taskCh )
pend . Wait ( )
pend . Wait ( )
// Clean out any pending derefs.
reler . call ( )
// Log the chain result
switch {
switch {
case failed != nil :
case failed != nil :
log . Warn ( "Chain tracing failed" , "start" , start . NumberU64 ( ) , "end" , end . NumberU64 ( ) , "transactions" , traced , "elapsed" , time . Since ( begin ) , "err" , failed )
log . Warn ( "Chain tracing failed" , "start" , start . NumberU64 ( ) , "end" , end . NumberU64 ( ) , "transactions" , traced , "elapsed" , time . Since ( begin ) , "err" , failed )
@ -330,105 +366,97 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config
default :
default :
log . Info ( "Chain tracing finished" , "start" , start . NumberU64 ( ) , "end" , end . NumberU64 ( ) , "transactions" , traced , "elapsed" , time . Since ( begin ) )
log . Info ( "Chain tracing finished" , "start" , start . NumberU64 ( ) , "end" , end . NumberU64 ( ) , "transactions" , traced , "elapsed" , time . Since ( begin ) )
}
}
close ( results )
close ( resCh )
} ( )
} ( )
var preferDisk bool
// Feed all the blocks both into the tracer, as well as fast process concurrently
// Feed all the blocks both into the tracer, as well as fast process concurrently
for number = start . NumberU64 ( ) ; number < end . NumberU64 ( ) ; number ++ {
for number = start . NumberU64 ( ) ; number < end . NumberU64 ( ) ; number ++ {
// Stop tracing if interruption was requested
// Stop tracing if interruption was requested
select {
select {
case <- notifier . Cl osed( ) :
case <- cl osed:
return
return
default :
default :
}
}
// clean out any derefs
derefsMu . Lock ( )
for _ , h := range derefTodo {
statedb . Database ( ) . TrieDB ( ) . Dereference ( h )
}
derefTodo = derefTodo [ : 0 ]
derefsMu . Unlock ( )
// Print progress logs if long enough time elapsed
// Print progress logs if long enough time elapsed
if time . Since ( logged ) > 8 * time . Second {
if time . Since ( logged ) > 8 * time . Second {
logged = time . Now ( )
logged = time . Now ( )
log . Info ( "Tracing chain segment" , "start" , start . NumberU64 ( ) , "end" , end . NumberU64 ( ) , "current" , number , "transactions" , traced , "elapsed" , time . Since ( begin ) )
log . Info ( "Tracing chain segment" , "start" , start . NumberU64 ( ) , "end" , end . NumberU64 ( ) , "current" , number , "transactions" , traced , "elapsed" , time . Since ( begin ) )
}
}
// Retrieve the parent state to trace on top
// Retrieve the parent block and target block for tracing.
block , err := api . blockByNumber ( local ctx, rpc . BlockNumber ( number ) )
block , err := api . blockByNumber ( ctx , rpc . BlockNumber ( number ) )
if err != nil {
if err != nil {
failed = err
failed = err
break
break
}
}
// Prepare the statedb for tracing. Don't use the live database for
next , err := api . blockByNumber ( ctx , rpc . BlockNumber ( number + 1 ) )
// tracing to avoid persisting state junks into the database.
statedb , err = api . backend . StateAtBlock ( localctx , block , reexec , statedb , false , preferDisk )
if err != nil {
if err != nil {
failed = err
failed = err
break
break
}
}
if trieDb := statedb . Database ( ) . TrieDB ( ) ; trieDb != nil {
// Prepare the statedb for tracing. Don't use the live database for
// Hold the reference for tracer, will be released at the final stage
// tracing to avoid persisting state junks into the database. Switch
trieDb . Reference ( block . Root ( ) , common . Hash { } )
// over to `preferDisk` mode only if the memory usage exceeds the
// limit, the trie database will be reconstructed from scratch only
// Release the parent state because it's already held by the tracer
// if the relevant state is available in disk.
if parent != ( common . Hash { } ) {
var preferDisk bool
trieDb . Dereference ( parent )
if statedb != nil {
}
s1 , s2 := statedb . Database ( ) . TrieDB ( ) . Size ( )
// Prefer disk if the trie db memory grows too much
preferDisk = s1 + s2 > defaultTracechainMemLimit
s1 , s2 := trieDb . Size ( )
if ! preferDisk && ( s1 + s2 ) > defaultTracechainMemLimit {
log . Info ( "Switching to prefer-disk mode for tracing" , "size" , s1 + s2 )
preferDisk = true
}
}
}
parent = block . Root ( )
statedb , release , err = api . backend . StateAtBlock ( ctx , block , reexec , statedb , false , preferDisk )
next , err := api . blockByNumber ( localctx , rpc . BlockNumber ( number + 1 ) )
if err != nil {
if err != nil {
failed = err
failed = err
break
break
}
}
// Clean out any pending derefs. Note this step must be done after
// constructing tracing state, because the tracing state of block
// next depends on the parent state and construction may fail if
// we release too early.
reler . call ( )
// Send the block over to the concurrent tracers (if not in the fast-forward phase)
// Send the block over to the concurrent tracers (if not in the fast-forward phase)
txs := next . Transactions ( )
txs := next . Transactions ( )
select {
select {
case tasks <- & blockTraceTask { statedb : statedb . Copy ( ) , block : next , rootref : block . Root ( ) , results : make ( [ ] * txTraceResult , len ( txs ) ) } :
case taskCh <- & blockTraceTask { statedb : statedb . Copy ( ) , block : next , release : release , results : make ( [ ] * txTraceResult , len ( txs ) ) } :
case <- notifier . Closed ( ) :
case <- closed :
reler . add ( release )
return
return
}
}
traced += uint64 ( len ( txs ) )
traced += uint64 ( len ( txs ) )
}
}
} ( )
} ( )
// Keep reading the trace results and stream the to the user
// Keep reading the trace results and stream them to result channel.
retCh := make ( chan * blockTraceResult )
go func ( ) {
go func ( ) {
defer close ( retCh )
var (
var (
done = make ( map [ uint64 ] * blockTraceResult )
next = start . NumberU64 ( ) + 1
next = start . NumberU64 ( ) + 1
done = make ( map [ uint64 ] * blockTraceResult )
)
)
for res := range results {
for res := range resCh {
// Queue up next received result
// Queue up next received result
result := & blockTraceResult {
result := & blockTraceResult {
Block : hexutil . Uint64 ( res . block . NumberU64 ( ) ) ,
Block : hexutil . Uint64 ( res . block . NumberU64 ( ) ) ,
Hash : res . block . Hash ( ) ,
Hash : res . block . Hash ( ) ,
Traces : res . results ,
Traces : res . results ,
}
}
// Schedule any parent tries held in memory by this task for dereferencing
done [ uint64 ( result . Block ) ] = result
done [ uint64 ( result . Block ) ] = result
derefsMu . Lock ( )
derefTodo = append ( derefTodo , res . rootref )
// Stream completed traces to the result channel
derefsMu . Unlock ( )
// Stream completed traces to the user, aborting on the first error
for result , ok := done [ next ] ; ok ; result , ok = done [ next ] {
for result , ok := done [ next ] ; ok ; result , ok = done [ next ] {
if len ( result . Traces ) > 0 || next == end . NumberU64 ( ) {
if len ( result . Traces ) > 0 || next == end . NumberU64 ( ) {
notifier . Notify ( sub . ID , result )
// It will be blocked in case the channel consumer doesn't take the
// tracing result in time(e.g. the websocket connect is not stable)
// which will eventually block the entire chain tracer. It's the
// expected behavior to not waste node resources for a non-active user.
retCh <- result
}
}
delete ( done , next )
delete ( done , next )
next ++
next ++
}
}
}
}
} ( )
} ( )
return sub , nil
return retCh
}
}
// TraceBlockByNumber returns the structured logs created during the execution of
// TraceBlockByNumber returns the structured logs created during the execution of
@ -515,10 +543,12 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config
if config != nil && config . Reexec != nil {
if config != nil && config . Reexec != nil {
reexec = * config . Reexec
reexec = * config . Reexec
}
}
statedb , err := api . backend . StateAtBlock ( ctx , parent , reexec , nil , true , false )
statedb , release , err := api . backend . StateAtBlock ( ctx , parent , reexec , nil , true , false )
if err != nil {
if err != nil {
return nil , err
return nil , err
}
}
defer release ( )
var (
var (
roots [ ] common . Hash
roots [ ] common . Hash
signer = types . MakeSigner ( api . backend . ChainConfig ( ) , block . Number ( ) )
signer = types . MakeSigner ( api . backend . ChainConfig ( ) , block . Number ( ) )
@ -576,10 +606,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac
if config != nil && config . Reexec != nil {
if config != nil && config . Reexec != nil {
reexec = * config . Reexec
reexec = * config . Reexec
}
}
statedb , err := api . backend . StateAtBlock ( ctx , parent , reexec , nil , true , false )
statedb , release , err := api . backend . StateAtBlock ( ctx , parent , reexec , nil , true , false )
if err != nil {
if err != nil {
return nil , err
return nil , err
}
}
defer release ( )
// Execute all the transaction contained within the block concurrently
// Execute all the transaction contained within the block concurrently
var (
var (
signer = types . MakeSigner ( api . backend . ChainConfig ( ) , block . Number ( ) )
signer = types . MakeSigner ( api . backend . ChainConfig ( ) , block . Number ( ) )
@ -666,10 +698,12 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block
if config != nil && config . Reexec != nil {
if config != nil && config . Reexec != nil {
reexec = * config . Reexec
reexec = * config . Reexec
}
}
statedb , err := api . backend . StateAtBlock ( ctx , parent , reexec , nil , true , false )
statedb , release , err := api . backend . StateAtBlock ( ctx , parent , reexec , nil , true , false )
if err != nil {
if err != nil {
return nil , err
return nil , err
}
}
defer release ( )
// Retrieve the tracing configurations, or use default values
// Retrieve the tracing configurations, or use default values
var (
var (
logConfig logger . Config
logConfig logger . Config
@ -793,10 +827,12 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *
if err != nil {
if err != nil {
return nil , err
return nil , err
}
}
msg , vmctx , statedb , err := api . backend . StateAtTransaction ( ctx , block , int ( index ) , reexec )
msg , vmctx , statedb , release , err := api . backend . StateAtTransaction ( ctx , block , int ( index ) , reexec )
if err != nil {
if err != nil {
return nil , err
return nil , err
}
}
defer release ( )
txctx := & Context {
txctx := & Context {
BlockHash : blockHash ,
BlockHash : blockHash ,
TxIndex : int ( index ) ,
TxIndex : int ( index ) ,
@ -837,10 +873,12 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc
if config != nil && config . Reexec != nil {
if config != nil && config . Reexec != nil {
reexec = * config . Reexec
reexec = * config . Reexec
}
}
statedb , err := api . backend . StateAtBlock ( ctx , block , reexec , nil , true , false )
statedb , release , err := api . backend . StateAtBlock ( ctx , block , reexec , nil , true , false )
if err != nil {
if err != nil {
return nil , err
return nil , err
}
}
defer release ( )
vmctx := core . NewEVMBlockContext ( block . Header ( ) , api . chainContext ( ctx ) , nil )
vmctx := core . NewEVMBlockContext ( block . Header ( ) , api . chainContext ( ctx ) , nil )
// Apply the customization rules if required.
// Apply the customization rules if required.
if config != nil {
if config != nil {