move sha3 mapping from traceChache to web3VMProvider

- add we3.debug.preimage
- move mapping resolver from soliditystate to storageViewer

src/solidity/decodeInfo.js

@@ -190,7 +190,7 @@ function struct (type, stateDefinitions, contractName, location) {
     if (!location) {
       location = match[2].trim()
     }
-    var memberDetails = getStructMembers(match[1], stateDefinitions, contractName) // type is used to extract the ast struct definition
+    var memberDetails = getStructMembers(match[1], stateDefinitions, contractName, location) // type is used to extract the ast struct definition
     if (!memberDetails) return null
     return new StructType(memberDetails, location, match[1])
   } else {
@@ -233,7 +233,7 @@ function getEnum (type, stateDefinitions, contractName) {
   * @param {String} location - location of the data (storage ref| storage pointer| memory| calldata)
   * @return {Array} containing all members of the current struct type
   */
-function getStructMembers (type, stateDefinitions, contractName) {
+function getStructMembers (type, stateDefinitions, contractName, location) {
   var split = type.split('.')
   if (!split.length) {
     type = contractName + '.' + type

src/solidity/types/Mapping.js

@@ -1,5 +1,6 @@
 'use strict'
 var RefType = require('./RefType')
+var util = require('./util')
 var ethutil = require('ethereumjs-util')
 
 class Mapping extends RefType {
@@ -9,22 +10,26 @@ class Mapping extends RefType {
     this.valueType = underlyingTypes.valueType
   }
 
-  setMappingElements (mappingKeyPreimages) {
-    this.preimages = mappingKeyPreimages
-  }
-
   async decodeFromStorage (location, storageResolver) {
-    // location.offset should always be 0 for a mapping (?? double check)
-
+    try {
+      var mappingsPreimages = await storageResolver.mappingPreimages()
+    } catch (e) {
+      return {
+        value: '<error> ' + e.message,
+        type: this.type
+      }
+    }
+    var mapSlot = util.toBN(location.slot).toString(16)
+    mapSlot = ethutil.setLengthLeft('0x' + mapSlot, 32).toString('hex')
+    var mappingPreimages = mappingsPreimages[mapSlot]
     var ret = {}
-    for (var i in this.preimages) {
-      var preimage = this.preimages[i]
-      var mapLocation = getMappingLocation(preimage, location.slot)
+    for (var i in mappingPreimages) {
+      var mapLocation = getMappingLocation(i, location.slot)
       var globalLocation = {
         offset: location.offset,
         slot: mapLocation
       }
-      ret[preimage] = await this.valueType.decodeFromStorage(globalLocation, storageResolver)
+      ret[i] = await this.valueType.decodeFromStorage(globalLocation, storageResolver)
     }
 
     return {

src/solidity/types/StringType.js

@@ -40,7 +40,7 @@ function format (decoded) {
   var value = decoded.value
   value = value.replace('0x', '').replace(/(..)/g, '%$1')
   var ret = {
-    // length: decoded.length, // unneeded, only dynamicBytes uses length
+    length: decoded.length,
     raw: decoded.value,
     type: 'string'
   }

src/storage/mappingPreimages.js

@@ -0,0 +1,58 @@
+var global = require('../helpers/global')
+
+module.exports = {
+  extractMappingPreimages: extractMappingPreimages
+}
+
+async function extractMappingPreimages (storageViewer) {
+  return new Promise((resolve, reject) => {
+    storageViewer.storageRange(function (error, storage) {
+      if (!error) {
+        decodeMappingsKeys(storage, (error, mappings) => {
+          if (error) {
+            reject(error)
+          } else {
+            resolve(mappings)
+          }
+        })
+      } else {
+        reject(error)
+      }
+    })
+  })
+}
+
+async function decodeMappingsKeys (storage, callback) {
+  var ret = {}
+  for (var hashedLoc in storage) {
+    var preimage
+    try {
+      preimage = await getPreimage(storage[hashedLoc].key)
+    } catch (e) {
+    }
+    if (preimage) {
+      // got preimage!
+      // get mapping position (i.e. storage slot), its the last 32 bytes
+      var slotByteOffset = preimage.length - 64
+      var mappingSlot = preimage.substr(slotByteOffset)
+      var mappingKey = preimage.substr(0, slotByteOffset)
+      if (!ret[mappingSlot]) {
+        ret[mappingSlot] = {}
+      }
+      ret[mappingSlot][mappingKey] = preimage
+    }
+  }
+  callback(null, ret)
+}
+
+function getPreimage (key) {
+  return new Promise((resolve, reject) => {
+    global.web3.debug.preimage(key, function (error, preimage) {
+      if (error) {
+        reject(error)
+      } else {
+        resolve(preimage)
+      }
+    })
+  })
+}

src/storage/storageViewer.js

@@ -1,10 +1,15 @@
 'use strict'
 var helper = require('../helpers/util')
+var mappingPreimagesExtractor = require('./mappingPreimages')
 
 class StorageViewer {
   constructor (_context, _storageResolver, _traceManager) {
     this.context = _context
     this.storageResolver = _storageResolver
+    // contains [mappingSlot][mappingkey] = preimage
+    // this map is renewed for each execution step
+    // this map is shared among all the mapping types
+    this.mappingsPreimages = null
     _traceManager.accumulateStorageChanges(this.context.stepIndex, this.context.address, {}, (error, storageChanges) => {
       if (!error) {
         this.storageChanges = storageChanges
@@ -58,6 +63,13 @@ class StorageViewer {
   isComplete (address) {
     return this.storageResolver.isComplete(address)
   }
+
+  async mappingPreimages () {
+    if (!this.mappingsPreimages) {
+      this.mappingsPreimages = await mappingPreimagesExtractor.extractMappingPreimages(this)
+    }
+    return this.mappingsPreimages
+  }
 }
 
 module.exports = StorageViewer

src/trace/traceAnalyser.js

@@ -1,6 +1,5 @@
 'use strict'
 var traceHelper = require('../helpers/traceHelper')
-var ethutil = require('ethereumjs-util')
 
 function TraceAnalyser (_cache) {
   this.traceCache = _cache
@@ -72,18 +71,6 @@ TraceAnalyser.prototype.buildMemory = function (index, step) {
   }
 }
 
-function getSha3Input (stack, memory) {
-  var memoryStart = stack[stack.length - 1]
-  var memoryLength = stack[stack.length - 2]
-  var memStartDec = (new ethutil.BN(memoryStart.replace('0x', ''), 16)).toString(10)
-  memoryStart = parseInt(memStartDec) * 2
-  var memLengthDec = (new ethutil.BN(memoryLength.replace('0x', ''), 16).toString(10))
-  memoryLength = parseInt(memLengthDec) * 2
-  var memoryHex = memory.join('')
-  var sha3Input = memoryHex.substr(memoryStart, memoryLength)
-  return sha3Input
-}
-
 TraceAnalyser.prototype.buildStorage = function (index, step, context) {
   if (traceHelper.newContextStorage(step) && !traceHelper.isCallToPrecompiledContract(index, this.trace)) {
     var calledAddress = traceHelper.resolveCalledAddress(index, this.trace)
@@ -93,9 +80,6 @@ TraceAnalyser.prototype.buildStorage = function (index, step, context) {
       console.log('unable to build storage changes. ' + index + ' does not match with a CALL. storage changes will be corrupted')
     }
     this.traceCache.pushStoreChanges(index + 1, context.storageContext[context.storageContext.length - 1])
-  } else if (traceHelper.isSHA3Instruction(step)) {
-    var sha3Input = getSha3Input(step.stack, step.memory)
-    this.traceCache.pushSha3Preimage(sha3Input, context.storageContext[context.storageContext.length - 1])
   } else if (traceHelper.isSSTOREInstruction(step)) {
     this.traceCache.pushStoreChanges(index + 1, context.storageContext[context.storageContext.length - 1], step.stack[step.stack.length - 1], step.stack[step.stack.length - 2])
   } else if (traceHelper.isReturnInstruction(step)) {

src/trace/traceCache.js

@@ -1,6 +1,5 @@
 'use strict'
 var helper = require('../helpers/util')
-var ethutil = require('ethereumjs-util')
 
 function TraceCache () {
   this.init()
@@ -21,9 +20,6 @@ TraceCache.prototype.init = function () {
   this.memoryChanges = []
   this.storageChanges = []
   this.sstore = {} // all sstore occurence in the trace
-  if (!this.sha3Preimages) { // need to accumulate the preimages over multiple tx's, so dont clear
-    this.sha3Preimages = {}
-  }
 }
 
 TraceCache.prototype.pushSteps = function (index, currentCallIndex) {
@@ -96,16 +92,6 @@ TraceCache.prototype.pushStoreChanges = function (index, address, key, value) {
   this.storageChanges.push(index)
 }
 
-TraceCache.prototype.pushSha3Preimage = function (sha3Input, address) {
-  console.log('pushSha3Preimage sha3Input:', sha3Input)
-  var preimage = sha3Input
-  var imageHash = ethutil.sha3('0x' + sha3Input).toString('hex')
-  this.sha3Preimages[imageHash] = {
-    'address': address,
-    'preimage': preimage
-  }
-}
-
 TraceCache.prototype.accumulateStorageChanges = function (index, address, storage) {
   var ret = Object.assign({}, storage)
   for (var k in this.storageChanges) {

src/ui/SolidityState.js

@@ -3,8 +3,6 @@ var DropdownPanel = require('./DropdownPanel')
 var stateDecoder = require('../solidity/stateDecoder')
 var solidityTypeFormatter = require('./SolidityTypeFormatter')
 var StorageViewer = require('../storage/storageViewer')
-var util = require('../solidity/types/util')
-var ethutil = require('ethereumjs-util')
 var yo = require('yo-yo')
 
 function SolidityState (_parent, _traceManager, _codeManager, _solidityProxy) {
@@ -66,28 +64,9 @@ SolidityState.prototype.init = function () {
               tx: self.parent.tx,
               address: address
             }, self.storageResolver, self.traceManager)
-
-            var storageJSON = {}
-            storageViewer.storageRange(function (error, result) {
-              if (!error) {
-                storageJSON = result
-                var sha3Preimages = self.traceManager.traceCache.sha3Preimages
-                var mappingPreimages = getMappingPreimages(stateVars, storageJSON, sha3Preimages)
-
-                for (var k in stateVars) {
-                  var stateVar = stateVars[k]
-                  if (stateVar.type.typeName.indexOf('mapping') === 0) {
-                    var mapSlot = util.toBN(stateVar.storagelocation.slot).toString(16)
-                    mapSlot = ethutil.setLengthLeft('0x' + mapSlot, 32).toString('hex')
-                    stateVar.type.setMappingElements(mappingPreimages[mapSlot])
-                  }
-                }
-
-                stateDecoder.decodeState(stateVars, storageViewer).then((result) => {
-                  if (!result.error) {
-                    self.basicPanel.update(result)
-                  }
-                })
+            stateDecoder.decodeState(stateVars, storageViewer).then((result) => {
+              if (!result.error) {
+                self.basicPanel.update(result)
               }
             })
           }
@@ -97,45 +76,4 @@ SolidityState.prototype.init = function () {
   })
 }
 
-function getMappingPreimages (stateVars, storage, preimages) {
-  // loop over stateVars and get the locations of all the mappings
-  // then on each mapping, pass its specific preimage keys
-
-  // first filter out all non-mapping storage slots
-
-  var ignoreSlots = []
-  for (var k in stateVars) {
-    var stateVar = stateVars[k]
-    if (stateVar.type.typeName.indexOf('mapping') !== 0) {
-      ignoreSlots.push(stateVar.storagelocation.slot.toString())
-    }
-  }
-
-  var possibleMappings = []
-  for (var hashedLoc in storage) {
-    var slotNum = util.toBN(storage[hashedLoc].key).toString(10)
-    if (ignoreSlots.indexOf(slotNum) === -1) {
-      possibleMappings.push(storage[hashedLoc].key)
-    }
-  }
-
-  var storageMappings = {}
-  for (var pk in possibleMappings) {
-    var possMapKey = possibleMappings[pk].replace('0x', '')
-    if (preimages[possMapKey]) {
-      // got preimage!
-      var preimage = preimages[possMapKey].preimage
-      // get mapping position (i.e. storage slot), its the last 32 bytes
-      var slotByteOffset = preimage.length - 64
-      var mappingSlot = preimage.substr(slotByteOffset)
-      var mappingKey = preimage.substr(0, slotByteOffset)
-      if (!storageMappings[mappingSlot]) {
-        storageMappings[mappingSlot] = []
-      }
-      storageMappings[mappingSlot].push(mappingKey)
-    }
-  }
-  return storageMappings
-}
-
 module.exports = SolidityState

src/util/web3Admin.js

@@ -6,6 +6,15 @@ module.exports = {
     }
     // DEBUG
     var methods = []
+    if (!(web3.debug && web3.debug.preimage)) {
+      methods.push(new web3._extend.Method({
+        name: 'preimage',
+        call: 'debug_preimage',
+        inputFormatter: [null],
+        params: 1
+      }))
+    }
+
     if (!(web3.debug && web3.debug.traceTransaction)) {
       methods.push(new web3._extend.Method({
         name: 'traceTransaction',

src/web3Provider/web3VmProvider.js

@@ -1,6 +1,7 @@
 var util = require('../helpers/util')
 var uiutil = require('../helpers/ui')
 var traceHelper = require('../helpers/traceHelper')
+var ethutil = require('ethereumjs-util')
 var Web3 = require('web3')
 
 function web3VmProvider () {
@@ -22,9 +23,11 @@ function web3VmProvider () {
   this.eth.getBlockNumber = function (cb) { return self.getBlockNumber(cb) }
   this.debug.traceTransaction = function (hash, options, cb) { return self.traceTransaction(hash, options, cb) }
   this.debug.storageRangeAt = function (blockNumber, txIndex, address, start, end, maxLength, cb) { return self.storageRangeAt(blockNumber, txIndex, address, start, end, maxLength, cb) }
+  this.debug.preimage = function (hashedKey, cb) { return self.preimage(hashedKey, cb) }
   this.providers = { 'HttpProvider': function (url) {} }
   this.currentProvider = {'host': 'vm provider'}
   this.storageCache = {}
+  this.sha3Preimages = {}
 }
 
 web3VmProvider.prototype.setVM = function (vm) {
@@ -128,6 +131,10 @@ web3VmProvider.prototype.pushTrace = function (self, data) {
       }
     }
   }
+  if (traceHelper.isSHA3Instruction(step)) {
+    var sha3Input = getSha3Input(step.stack, step.memory)
+    pushSha3Preimage(this, sha3Input)
+  }
   this.processingIndex++
   this.previousDepth = depth
 }
@@ -189,4 +196,29 @@ web3VmProvider.prototype.getTransactionFromBlock = function (blockNumber, txInde
   }
 }
 
+web3VmProvider.prototype.preimage = function (hashedKey, cb) {
+  hashedKey = hashedKey.replace('0x', '')
+  cb(null, this.sha3Preimages[hashedKey] !== undefined ? this.sha3Preimages[hashedKey].preimage : null)
+}
+
+function pushSha3Preimage (self, sha3Input) {
+  var preimage = sha3Input
+  var imageHash = ethutil.sha3('0x' + sha3Input).toString('hex')
+  self.sha3Preimages[imageHash] = {
+    'preimage': preimage
+  }
+}
+
+function getSha3Input (stack, memory) {
+  var memoryStart = stack[stack.length - 1]
+  var memoryLength = stack[stack.length - 2]
+  var memStartDec = (new ethutil.BN(memoryStart.replace('0x', ''), 16)).toString(10)
+  memoryStart = parseInt(memStartDec) * 2
+  var memLengthDec = (new ethutil.BN(memoryLength.replace('0x', ''), 16).toString(10))
+  memoryLength = parseInt(memLengthDec) * 2
+  var memoryHex = memory.join('')
+  var sha3Input = memoryHex.substr(memoryStart, memoryLength)
+  return sha3Input
+}
+
 module.exports = web3VmProvider