From c13f0469c27894e760c0a39309734e4b9c959225 Mon Sep 17 00:00:00 2001 From: yann300 Date: Wed, 10 Jan 2024 13:15:19 +0100 Subject: [PATCH] save the state upton reloads --- apps/remix-ide/src/blockchain/blockchain.tsx | 31 +++++++-- apps/remix-ide/src/blockchain/providers/vm.ts | 22 +++++-- .../src/blockchain/providers/worker-vm.ts | 8 ++- .../src/methods/transactions.ts | 16 ++++- libs/remix-simulator/src/provider.ts | 25 ++++++- libs/remix-simulator/src/vm-context.ts | 65 ++++++++++++++----- 6 files changed, 136 insertions(+), 31 deletions(-) diff --git a/apps/remix-ide/src/blockchain/blockchain.tsx b/apps/remix-ide/src/blockchain/blockchain.tsx index fe9b6598f1..cb9491f8af 100644 --- a/apps/remix-ide/src/blockchain/blockchain.tsx +++ b/apps/remix-ide/src/blockchain/blockchain.tsx @@ -1,7 +1,7 @@ import React from 'react' // eslint-disable-line import {fromWei, toBigInt, toWei} from 'web3-utils' import {Plugin} from '@remixproject/engine' -import {toBuffer, addHexPrefix} from '@ethereumjs/util' +import {toBuffer, addHexPrefix, bufferToHex} from '@ethereumjs/util' import {EventEmitter} from 'events' import {format} from 'util' import {ExecutionContext} from './execution-context' @@ -164,7 +164,7 @@ export class Blockchain extends Plugin { } setupProviders() { - const vmProvider = new VMProvider(this.executionContext) + const vmProvider = new VMProvider(this.executionContext, this) this.providers = {} this.providers['vm'] = vmProvider this.providers.injected = new InjectedProvider(this.executionContext) @@ -677,7 +677,7 @@ export class Blockchain extends Plugin { view on etherscan ) - } + } }) }) this.txRunner = new TxRunner(web3Runner, {}) @@ -889,8 +889,31 @@ export class Blockchain extends Plugin { let execResult let returnValue = null if (isVM) { - const hhlogs = await this.web3().remix.getHHLogsForTx(txResult.transactionHash) + if (!tx.useCall) { + // TODO: this won't save the state for transactions executed outside of the UI (dor instande from a script execution). + setTimeout(async() => { + const root = await this.web3().remix.getStateTrieRoot() + const db = await this.web3().remix.getStateDb() + const state = { + root, + db: Object.fromEntries(db._database) + } + console.log('saving', state) + const stringifyed = JSON.stringify(state, (key, value) => { + if (key === 'root') { + return bufferToHex(value) + } else if (key === 'db') { + return value + } else { + return bufferToHex(value) + } + return value + }, '\t') + this.call('fileManager', 'writeFile', '.states/state.json', stringifyed) + }, 500) + } + const hhlogs = await this.web3().remix.getHHLogsForTx(txResult.transactionHash) if (hhlogs && hhlogs.length) { const finalLogs = (
diff --git a/apps/remix-ide/src/blockchain/providers/vm.ts b/apps/remix-ide/src/blockchain/providers/vm.ts index 67357b347e..af699bed9e 100644 --- a/apps/remix-ide/src/blockchain/providers/vm.ts +++ b/apps/remix-ide/src/blockchain/providers/vm.ts @@ -1,8 +1,10 @@ import Web3, { FMT_BYTES, FMT_NUMBER, LegacySendAsyncProvider } from 'web3' import { fromWei, toBigInt } from 'web3-utils' +import { Plugin } from '@remixproject/engine' import { privateToAddress, hashPersonalMessage, isHexString } from '@ethereumjs/util' import { extend, JSONRPCRequestPayload, JSONRPCResponseCallback } from '@remix-project/remix-simulator' import { ExecutionContext } from '../execution-context' +import { Blockchain } from '../blockchain' export class VMProvider { executionContext: ExecutionContext @@ -12,9 +14,9 @@ export class VMProvider { sendAsync: (query: JSONRPCRequestPayload, callback: JSONRPCResponseCallback) => void } newAccountCallback: {[stamp: number]: (error: Error, address: string) => void} - - constructor (executionContext: ExecutionContext) { - + plugin: Plugin + constructor (executionContext: ExecutionContext, plugin: Plugin) { + this.plugin = plugin this.executionContext = executionContext this.worker = null this.provider = null @@ -37,7 +39,7 @@ export class VMProvider { let incr = 0 const stamps = {} - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { this.worker.addEventListener('message', (msg) => { if (msg.data.cmd === 'sendAsyncResult' && stamps[msg.data.stamp]) { if (stamps[msg.data.stamp].callback) { @@ -76,7 +78,17 @@ export class VMProvider { } } }) - this.worker.postMessage({ cmd: 'init', fork: this.executionContext.getCurrentFork(), nodeUrl: provider?.options['nodeUrl'], blockNumber: provider?.options['blockNumber']}) + let stateDb + if (await this.plugin.call('fileManager', 'exists', '.states/state.json')) { + stateDb = await this.plugin.call('fileManager', 'readFile', '.states/state.json') + } + this.worker.postMessage({ + cmd: 'init', + fork: this.executionContext.getCurrentFork(), + nodeUrl: provider?.options['nodeUrl'], + blockNumber: provider?.options['blockNumber'], + stateDb + }) }) } diff --git a/apps/remix-ide/src/blockchain/providers/worker-vm.ts b/apps/remix-ide/src/blockchain/providers/worker-vm.ts index 64a8d0255b..657fe8c223 100644 --- a/apps/remix-ide/src/blockchain/providers/worker-vm.ts +++ b/apps/remix-ide/src/blockchain/providers/worker-vm.ts @@ -1,4 +1,5 @@ import { Provider } from '@remix-project/remix-simulator' +import {toBuffer} from '@ethereumjs/util' let provider: Provider = null self.onmessage = (e: MessageEvent) => { @@ -6,7 +7,12 @@ self.onmessage = (e: MessageEvent) => { switch (data.cmd) { case 'init': { - provider = new Provider({ fork: data.fork, nodeUrl: data.nodeUrl, blockNumber: data.blockNumber }) + if (data.stateDb) { + data.stateDb = JSON.parse(data.stateDb) + data.stateDb.root = toBuffer(data.stateDb.root) + data.stateDb.db = new Map(Object.entries(data.stateDb.db)) + } + provider = new Provider({ fork: data.fork, nodeUrl: data.nodeUrl, blockNumber: data.blockNumber, stateDb: data.stateDb }) provider.init().then(() => { self.postMessage({ cmd: 'initiateResult', diff --git a/libs/remix-simulator/src/methods/transactions.ts b/libs/remix-simulator/src/methods/transactions.ts index 6018f9fc99..9a47aa0135 100644 --- a/libs/remix-simulator/src/methods/transactions.ts +++ b/libs/remix-simulator/src/methods/transactions.ts @@ -4,7 +4,7 @@ import { processTx } from './txProcess' import { execution } from '@remix-project/remix-lib' import { ethers } from 'ethers' import { VMexecutionResult } from '@remix-project/remix-lib' -import { RunTxResult } from '@ethereumjs/vm' +import { VMContext } from '../vm-context' import { Log, EvmError } from '@ethereumjs/evm' const TxRunnerVM = execution.TxRunnerVM const TxRunner = execution.TxRunner @@ -19,7 +19,7 @@ export type VMExecResult = { } export class Transactions { - vmContext + vmContext: VMContext accounts tags txRunnerVMInstance @@ -74,7 +74,9 @@ export class Transactions { eth_getExecutionResultFromSimulator: this.eth_getExecutionResultFromSimulator.bind(this), eth_getHHLogsForTx: this.eth_getHHLogsForTx.bind(this), eth_getHashFromTagBySimulator: this.eth_getHashFromTagBySimulator.bind(this), - eth_registerCallId: this.eth_registerCallId.bind(this) + eth_registerCallId: this.eth_registerCallId.bind(this), + eth_getStateTrieRoot: this.eth_getStateTrieRoot.bind(this), + eth_getStateDb: this.eth_getStateDb.bind(this) } } @@ -198,6 +200,14 @@ export class Transactions { cb() } + eth_getStateTrieRoot (_, cb) { + cb(null, this.vmContext.currentVm.stateManager.getTrie().root()) + } + + eth_getStateDb (_, cb) { + cb(null, this.vmContext.currentVm.stateManager.getDb()) + } + eth_call (payload, cb) { // from might be lowercased address (web3) if (payload.params && payload.params.length > 0 && payload.params[0].from) { diff --git a/libs/remix-simulator/src/provider.ts b/libs/remix-simulator/src/provider.ts index 59066c0ab6..8ad22b35ec 100644 --- a/libs/remix-simulator/src/provider.ts +++ b/libs/remix-simulator/src/provider.ts @@ -27,8 +27,13 @@ export interface JSONRPCResponsePayload { export type JSONRPCResponseCallback = (err: Error, result?: JSONRPCResponsePayload) => void +export type State = { + db: Map, + root: Buffer +} + export class Provider { - options: Record + options: Record vmContext Accounts Transactions @@ -37,10 +42,10 @@ export class Provider { initialized: boolean pendingRequests: Array - constructor (options: Record = {}) { + constructor (options: Record = {}) { this.options = options this.connected = true - this.vmContext = new VMContext(options['fork'] as string, options['nodeUrl'] as string, options['blockNumber'] as (number | 'latest')) + this.vmContext = new VMContext(options['fork'] as string, options['nodeUrl'] as string, options['blockNumber'] as (number | 'latest'), options['stateDb'] as State) this.Accounts = new Web3Accounts(this.vmContext) this.Transactions = new Transactions(this.vmContext) @@ -168,4 +173,18 @@ class Web3TestPlugin extends Web3PluginBase { params: [id] }) } + + public getStateTrieRoot() { + return this.requestManager.send({ + method: 'eth_getStateTrieRoot', + params: [] + }) + } + + public getStateDb() { + return this.requestManager.send({ + method: 'eth_getStateDb', + params: [] + }) + } } diff --git a/libs/remix-simulator/src/vm-context.ts b/libs/remix-simulator/src/vm-context.ts index 0bf58c3ad8..405ad3eed2 100644 --- a/libs/remix-simulator/src/vm-context.ts +++ b/libs/remix-simulator/src/vm-context.ts @@ -2,7 +2,7 @@ 'use strict' import { Cache } from '@ethereumjs/statemanager/dist/cache' import { hash } from '@remix-project/remix-lib' -import { bufferToHex, Account, toBuffer, bufferToBigInt} from '@ethereumjs/util' +import { bufferToHex, Account, toBuffer, bufferToBigInt, bigIntToHex } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import type { Address } from '@ethereumjs/util' import { decode } from 'rlp' @@ -13,7 +13,7 @@ import { VmProxy } from './VmProxy' import { VM } from '@ethereumjs/vm' import type { BigIntLike } from '@ethereumjs/util' import { Common, ConsensusType } from '@ethereumjs/common' -import { Trie } from '@ethereumjs/trie' +import { Trie, MapDB, DB } from '@ethereumjs/trie' import { DefaultStateManager, StateManager, EthersStateManager, EthersStateManagerOpts } from '@ethereumjs/statemanager' import { StorageDump } from '@ethereumjs/statemanager/dist/interface' import { EVM } from '@ethereumjs/evm' @@ -21,7 +21,7 @@ import { EEI } from '@ethereumjs/vm' import { Blockchain } from '@ethereumjs/blockchain' import { Block } from '@ethereumjs/block' import { Transaction } from '@ethereumjs/tx' -import { bigIntToHex } from '@ethereumjs/util' +import { State } from './provider' /** * Options for constructing a {@link StateManager}. @@ -40,16 +40,49 @@ export interface DefaultStateManagerOpts { prefixCodeHashes?: boolean } +class RemixMapDb extends MapDB { + async get(key: Buffer): Promise { + // the remix db contains stringified values (in order to save space), + // that's why we need to convert the hex string to the native type that the Trie understands. + let value = await super.get(key) + if (typeof value === 'string') { + value = toBuffer(value) + } + return value + } + + copy(): DB { + return new RemixMapDb(this._database) + } +} + /* extend vm state manager and instantiate VM */ class StateManagerCommonStorageDump extends DefaultStateManager { keyHashes: { [key: string]: string } - constructor (opts: DefaultStateManagerOpts = {}) { + internalTree: Trie + db: MapDB + stateDb: State + constructor (opts: DefaultStateManagerOpts = {}, stateDb?: State) { + const db = new RemixMapDb(stateDb ? stateDb.db: null) + const trie = new Trie({ useKeyHashing: true, db, root: stateDb ? stateDb.root : null }) + opts = { trie, ...opts } super(opts) + this.stateDb = stateDb + this.internalTree = trie + this.db = db this.keyHashes = {} } + getTrie () { + return this.internalTree + } + + getDb () { + return this.db + } + putContractStorage (address, key, value) { this.keyHashes[hash.keccak(key).toString('hex')] = bufferToHex(key) return super.putContractStorage(address, key, value) @@ -58,7 +91,7 @@ class StateManagerCommonStorageDump extends DefaultStateManager { copy(): StateManagerCommonStorageDump { const copyState = new StateManagerCommonStorageDump({ trie: this._trie.copy(false), - }) + }, this.stateDb) copyState.keyHashes = this.keyHashes return copyState } @@ -100,9 +133,9 @@ export interface CustomEthersStateManagerOpts { class CustomEthersStateManager extends StateManagerCommonStorageDump { private provider: ethers.providers.StaticJsonRpcProvider | ethers.providers.JsonRpcProvider private blockTag: string - - constructor(opts: CustomEthersStateManagerOpts) { - super(opts) + constructor(opts: CustomEthersStateManagerOpts, stateDb?: State) { + super(opts, stateDb) + this.stateDb = stateDb if (typeof opts.provider === 'string') { this.provider = new ethers.providers.StaticJsonRpcProvider(opts.provider) } else if (opts.provider instanceof ethers.providers.JsonRpcProvider) { @@ -154,7 +187,7 @@ class CustomEthersStateManager extends StateManagerCommonStorageDump { provider: this.provider, blockTag: this.blockTag, trie: this._trie.copy(false), - }) + }, this.stateDb) return newState } @@ -258,7 +291,7 @@ class CustomEthersStateManager extends StateManagerCommonStorageDump { export type CurrentVm = { vm: VM, web3vm: VmProxy, - stateManager: StateManager, + stateManager: StateManagerCommonStorageDump, common: Common } @@ -298,12 +331,14 @@ export class VMContext { exeResults: Record nodeUrl: string blockNumber: number | 'latest' + stateDb: any - constructor (fork?: string, nodeUrl?: string, blockNumber?: number | 'latest') { + constructor (fork?: string, nodeUrl?: string, blockNumber?: number | 'latest', stateDb?: any) { this.blockGasLimitDefault = 4300000 this.blockGasLimit = this.blockGasLimitDefault this.currentFork = fork || 'merge' this.nodeUrl = nodeUrl + this.stateDb = stateDb this.blockNumber = blockNumber this.blocks = {} this.latestBlockNumber = "0x0" @@ -318,7 +353,7 @@ export class VMContext { } async createVm (hardfork) { - let stateManager: StateManager + let stateManager: StateManagerCommonStorageDump if (this.nodeUrl) { let block = this.blockNumber if (this.blockNumber === 'latest') { @@ -327,17 +362,17 @@ export class VMContext { stateManager = new CustomEthersStateManager({ provider: this.nodeUrl, blockTag: '0x' + block.toString(16) - }) + }, this.stateDb) this.blockNumber = block } else { stateManager = new CustomEthersStateManager({ provider: this.nodeUrl, blockTag: '0x' + this.blockNumber.toString(16) - }) + }, this.stateDb) } } else - stateManager = new StateManagerCommonStorageDump() + stateManager = new StateManagerCommonStorageDump(null, this.stateDb) const consensusType = hardfork === 'berlin' || hardfork === 'london' ? ConsensusType.ProofOfWork : ConsensusType.ProofOfStake const difficulty = consensusType === ConsensusType.ProofOfStake ? 0 : 69762765929000