save the state upton reloads

apps/remix-ide/src/blockchain/blockchain.tsx

@@ -1,7 +1,7 @@
 import React from 'react' // eslint-disable-line
 import {fromWei, toBigInt, toWei} from 'web3-utils'
 import {Plugin} from '@remixproject/engine'
-import {toBuffer, addHexPrefix} from '@ethereumjs/util'
+import {toBuffer, addHexPrefix, bufferToHex} from '@ethereumjs/util'
 import {EventEmitter} from 'events'
 import {format} from 'util'
 import {ExecutionContext} from './execution-context'
@@ -164,7 +164,7 @@ export class Blockchain extends Plugin {
   }
 
   setupProviders() {
-    const vmProvider = new VMProvider(this.executionContext)
+    const vmProvider = new VMProvider(this.executionContext, this)
     this.providers = {}
     this.providers['vm'] = vmProvider
     this.providers.injected = new InjectedProvider(this.executionContext)
@@ -677,7 +677,7 @@ export class Blockchain extends Plugin {
               view on etherscan
             </a>
           )
-        }
+        } 
       })
     })
     this.txRunner = new TxRunner(web3Runner, {})
@@ -889,8 +889,31 @@ export class Blockchain extends Plugin {
       let execResult
       let returnValue = null
       if (isVM) {
-        const hhlogs = await this.web3().remix.getHHLogsForTx(txResult.transactionHash)
+        if (!tx.useCall) {
+          // TODO: this won't save the state for transactions executed outside of the UI (dor instande from a script execution).
+          setTimeout(async() => {
+            const root = await this.web3().remix.getStateTrieRoot()
+            const db = await this.web3().remix.getStateDb()
+            const state = {
+              root,
+              db: Object.fromEntries(db._database)
+            }
+            console.log('saving', state)
+            const stringifyed = JSON.stringify(state, (key, value) => {
+              if (key === 'root') {
+                return bufferToHex(value)
+              } else if (key === 'db') {
+                return value
+              } else {
+                return bufferToHex(value)           
+              }
+              return value
+          }, '\t')
+            this.call('fileManager', 'writeFile', '.states/state.json', stringifyed)
+          }, 500)
+        }
 
+        const hhlogs = await this.web3().remix.getHHLogsForTx(txResult.transactionHash)
         if (hhlogs && hhlogs.length) {
           const finalLogs = (
             <div>

apps/remix-ide/src/blockchain/providers/vm.ts

@@ -1,8 +1,10 @@
 import Web3, { FMT_BYTES, FMT_NUMBER, LegacySendAsyncProvider } from 'web3'
 import { fromWei, toBigInt } from 'web3-utils'
+import { Plugin } from '@remixproject/engine'
 import { privateToAddress, hashPersonalMessage, isHexString } from '@ethereumjs/util'
 import { extend, JSONRPCRequestPayload, JSONRPCResponseCallback } from '@remix-project/remix-simulator'
 import { ExecutionContext } from '../execution-context'
+import { Blockchain } from '../blockchain'
 
 export class VMProvider {
   executionContext: ExecutionContext
@@ -12,9 +14,9 @@ export class VMProvider {
     sendAsync: (query: JSONRPCRequestPayload, callback: JSONRPCResponseCallback) => void
   }
   newAccountCallback: {[stamp: number]: (error: Error, address: string) => void}
-
-  constructor (executionContext: ExecutionContext) {
-
+  plugin: Plugin
+  constructor (executionContext: ExecutionContext, plugin: Plugin) {
+    this.plugin = plugin
     this.executionContext = executionContext
     this.worker = null
     this.provider = null
@@ -37,7 +39,7 @@ export class VMProvider {
     let incr = 0
     const stamps = {}
 
-    return new Promise((resolve, reject) => {
+    return new Promise(async (resolve, reject) => {
       this.worker.addEventListener('message', (msg) => {
         if (msg.data.cmd === 'sendAsyncResult' && stamps[msg.data.stamp]) {
           if (stamps[msg.data.stamp].callback) {
@@ -76,7 +78,17 @@ export class VMProvider {
           }
         }
       })
-      this.worker.postMessage({ cmd: 'init', fork: this.executionContext.getCurrentFork(), nodeUrl: provider?.options['nodeUrl'], blockNumber: provider?.options['blockNumber']})
+      let stateDb
+      if (await this.plugin.call('fileManager', 'exists', '.states/state.json')) {
+        stateDb = await this.plugin.call('fileManager', 'readFile', '.states/state.json')
+      }
+      this.worker.postMessage({
+        cmd: 'init',
+        fork: this.executionContext.getCurrentFork(),
+        nodeUrl: provider?.options['nodeUrl'],
+        blockNumber: provider?.options['blockNumber'],
+        stateDb
+      })
     })
   }
 

apps/remix-ide/src/blockchain/providers/worker-vm.ts

@@ -1,4 +1,5 @@
 import { Provider } from '@remix-project/remix-simulator'
+import {toBuffer} from '@ethereumjs/util'
 
 let provider: Provider = null
 self.onmessage = (e: MessageEvent) => {
@@ -6,7 +7,12 @@ self.onmessage = (e: MessageEvent) => {
   switch (data.cmd) {
   case 'init': 
   {
-    provider = new Provider({ fork: data.fork, nodeUrl: data.nodeUrl, blockNumber: data.blockNumber })
+    if (data.stateDb) {
+      data.stateDb = JSON.parse(data.stateDb)
+      data.stateDb.root = toBuffer(data.stateDb.root)
+      data.stateDb.db = new Map(Object.entries(data.stateDb.db))
+    }
+    provider = new Provider({ fork: data.fork, nodeUrl: data.nodeUrl, blockNumber: data.blockNumber, stateDb: data.stateDb })
     provider.init().then(() => {
       self.postMessage({
         cmd: 'initiateResult',

libs/remix-simulator/src/methods/transactions.ts

@@ -4,7 +4,7 @@ import { processTx } from './txProcess'
 import { execution } from '@remix-project/remix-lib'
 import { ethers } from 'ethers'
 import { VMexecutionResult } from '@remix-project/remix-lib'
-import { RunTxResult } from '@ethereumjs/vm'
+import { VMContext } from '../vm-context'
 import { Log, EvmError } from '@ethereumjs/evm'
 const TxRunnerVM = execution.TxRunnerVM
 const TxRunner = execution.TxRunner
@@ -19,7 +19,7 @@ export type VMExecResult = {
 }
 
 export class Transactions {
-  vmContext
+  vmContext: VMContext
   accounts
   tags
   txRunnerVMInstance
@@ -74,7 +74,9 @@ export class Transactions {
       eth_getExecutionResultFromSimulator: this.eth_getExecutionResultFromSimulator.bind(this),
       eth_getHHLogsForTx: this.eth_getHHLogsForTx.bind(this),
       eth_getHashFromTagBySimulator: this.eth_getHashFromTagBySimulator.bind(this),
-      eth_registerCallId: this.eth_registerCallId.bind(this)
+      eth_registerCallId: this.eth_registerCallId.bind(this),
+      eth_getStateTrieRoot: this.eth_getStateTrieRoot.bind(this),
+      eth_getStateDb: this.eth_getStateDb.bind(this)
     }
   }
 
@@ -198,6 +200,14 @@ export class Transactions {
     cb()
   }
 
+  eth_getStateTrieRoot (_, cb) {
+    cb(null, this.vmContext.currentVm.stateManager.getTrie().root())
+  }
+
+  eth_getStateDb (_, cb) {
+    cb(null, this.vmContext.currentVm.stateManager.getDb())
+  }
+
   eth_call (payload, cb) {
     // from might be lowercased address (web3)
     if (payload.params && payload.params.length > 0 && payload.params[0].from) {

libs/remix-simulator/src/provider.ts

@@ -27,8 +27,13 @@ export interface JSONRPCResponsePayload {
 
 export type JSONRPCResponseCallback = (err: Error, result?: JSONRPCResponsePayload) =>  void
 
+export type State = {
+  db: Map<string, Buffer>,
+  root: Buffer
+}
+
 export class Provider {
-  options: Record<string, string | number>
+  options: Record<string, string | number | State>
   vmContext
   Accounts
   Transactions
@@ -37,10 +42,10 @@ export class Provider {
   initialized: boolean
   pendingRequests: Array<any>
 
-  constructor (options: Record<string, string | number> = {}) {
+  constructor (options: Record<string, string | number | State> = {}) {
     this.options = options
     this.connected = true
-    this.vmContext = new VMContext(options['fork'] as string, options['nodeUrl'] as string, options['blockNumber'] as (number | 'latest'))
+    this.vmContext = new VMContext(options['fork'] as string, options['nodeUrl'] as string, options['blockNumber'] as (number | 'latest'), options['stateDb'] as State)
 
     this.Accounts = new Web3Accounts(this.vmContext)
     this.Transactions = new Transactions(this.vmContext)
@@ -168,4 +173,18 @@ class Web3TestPlugin extends Web3PluginBase {
       params: [id]
     })
   }
+
+  public getStateTrieRoot() {
+    return this.requestManager.send({
+      method: 'eth_getStateTrieRoot',
+      params: []
+    })
+  }
+
+  public getStateDb() {
+    return this.requestManager.send({
+      method: 'eth_getStateDb',
+      params: []
+    })
+  }
 }

libs/remix-simulator/src/vm-context.ts

@@ -2,7 +2,7 @@
 'use strict'
 import { Cache } from '@ethereumjs/statemanager/dist/cache'
 import { hash } from '@remix-project/remix-lib'
-import { bufferToHex, Account, toBuffer, bufferToBigInt} from '@ethereumjs/util'
+import { bufferToHex, Account, toBuffer, bufferToBigInt, bigIntToHex } from '@ethereumjs/util'
 import { keccak256 } from 'ethereum-cryptography/keccak'
 import type { Address } from '@ethereumjs/util'
 import { decode } from 'rlp'
@@ -13,7 +13,7 @@ import { VmProxy } from './VmProxy'
 import { VM } from '@ethereumjs/vm'
 import type { BigIntLike } from '@ethereumjs/util'
 import { Common, ConsensusType } from '@ethereumjs/common'
-import { Trie } from '@ethereumjs/trie'
+import { Trie, MapDB, DB } from '@ethereumjs/trie'
 import { DefaultStateManager, StateManager, EthersStateManager, EthersStateManagerOpts } from '@ethereumjs/statemanager'
 import { StorageDump } from '@ethereumjs/statemanager/dist/interface'
 import { EVM } from '@ethereumjs/evm'
@@ -21,7 +21,7 @@ import { EEI } from '@ethereumjs/vm'
 import { Blockchain } from '@ethereumjs/blockchain'
 import { Block } from '@ethereumjs/block'
 import { Transaction } from '@ethereumjs/tx'
-import { bigIntToHex } from '@ethereumjs/util'
+import { State } from './provider'
 
 /**
  * Options for constructing a {@link StateManager}.
@@ -40,16 +40,49 @@ export interface DefaultStateManagerOpts {
   prefixCodeHashes?: boolean
 }
 
+class RemixMapDb extends MapDB {
+  async get(key: Buffer): Promise<Buffer | null> {
+    // the remix db contains stringified values (in order to save space),
+    // that's why we need to convert the hex string to the native type that the Trie understands.
+    let value = await super.get(key)
+    if (typeof value === 'string') {
+      value = toBuffer(value)
+    }
+    return value
+  }
+
+  copy(): DB {
+    return new RemixMapDb(this._database)
+  }
+}
+
 /*
   extend vm state manager and instantiate VM
 */
 class StateManagerCommonStorageDump extends DefaultStateManager {
   keyHashes: { [key: string]: string }
-  constructor (opts: DefaultStateManagerOpts = {}) {
+  internalTree: Trie
+  db: MapDB
+  stateDb: State
+  constructor (opts: DefaultStateManagerOpts = {}, stateDb?: State) {
+    const db = new RemixMapDb(stateDb ? stateDb.db: null)
+    const trie = new Trie({ useKeyHashing: true, db, root: stateDb ? stateDb.root : null })
+    opts = { trie, ...opts }
     super(opts)
+    this.stateDb = stateDb
+    this.internalTree = trie
+    this.db = db
     this.keyHashes = {}
   }
 
+  getTrie () {
+    return this.internalTree
+  }
+
+  getDb () {
+    return this.db
+  }
+
   putContractStorage (address, key, value) {
     this.keyHashes[hash.keccak(key).toString('hex')] = bufferToHex(key)
     return super.putContractStorage(address, key, value)
@@ -58,7 +91,7 @@ class StateManagerCommonStorageDump extends DefaultStateManager {
   copy(): StateManagerCommonStorageDump {
     const copyState =  new StateManagerCommonStorageDump({
       trie: this._trie.copy(false),
-    })
+    }, this.stateDb)
     copyState.keyHashes = this.keyHashes
     return copyState
   }
@@ -100,9 +133,9 @@ export interface CustomEthersStateManagerOpts {
 class CustomEthersStateManager extends StateManagerCommonStorageDump {
   private provider: ethers.providers.StaticJsonRpcProvider | ethers.providers.JsonRpcProvider
   private blockTag: string
-
-  constructor(opts: CustomEthersStateManagerOpts) {
-    super(opts)
+  constructor(opts: CustomEthersStateManagerOpts, stateDb?: State) {
+    super(opts, stateDb)
+    this.stateDb = stateDb
     if (typeof opts.provider === 'string') {
       this.provider = new ethers.providers.StaticJsonRpcProvider(opts.provider)
     } else if (opts.provider instanceof ethers.providers.JsonRpcProvider) {
@@ -154,7 +187,7 @@ class CustomEthersStateManager extends StateManagerCommonStorageDump {
       provider: this.provider,
       blockTag: this.blockTag,
       trie: this._trie.copy(false),
-    })
+    }, this.stateDb)
     return newState
   }
 
@@ -258,7 +291,7 @@ class CustomEthersStateManager extends StateManagerCommonStorageDump {
 export type CurrentVm = {
   vm: VM,
   web3vm: VmProxy,
-  stateManager: StateManager,
+  stateManager: StateManagerCommonStorageDump,
   common: Common
 }
 
@@ -298,12 +331,14 @@ export class VMContext {
   exeResults: Record<string, Transaction>
   nodeUrl: string
   blockNumber: number | 'latest'
+  stateDb: any
 
-  constructor (fork?: string, nodeUrl?: string, blockNumber?: number | 'latest') {
+  constructor (fork?: string, nodeUrl?: string, blockNumber?: number | 'latest', stateDb?: any) {
     this.blockGasLimitDefault = 4300000
     this.blockGasLimit = this.blockGasLimitDefault
     this.currentFork = fork || 'merge'
     this.nodeUrl = nodeUrl
+    this.stateDb = stateDb
     this.blockNumber = blockNumber
     this.blocks = {}
     this.latestBlockNumber = "0x0"
@@ -318,7 +353,7 @@ export class VMContext {
   }
 
   async createVm (hardfork) {
-    let stateManager: StateManager
+    let stateManager: StateManagerCommonStorageDump
     if (this.nodeUrl) {
       let block = this.blockNumber
       if (this.blockNumber === 'latest') {
@@ -327,17 +362,17 @@ export class VMContext {
         stateManager = new CustomEthersStateManager({
           provider: this.nodeUrl,
           blockTag: '0x' + block.toString(16)
-        })
+        }, this.stateDb)
         this.blockNumber = block
       } else {
         stateManager = new CustomEthersStateManager({
           provider: this.nodeUrl,
           blockTag: '0x' + this.blockNumber.toString(16)
-        })
+        }, this.stateDb)
       }
       
     } else
-      stateManager = new StateManagerCommonStorageDump()
+      stateManager = new StateManagerCommonStorageDump(null, this.stateDb)
 
     const consensusType = hardfork === 'berlin' || hardfork === 'london' ? ConsensusType.ProofOfWork : ConsensusType.ProofOfStake
     const difficulty = consensusType === ConsensusType.ProofOfStake ? 0 : 69762765929000