Merge pull request #498 from cdetrio/mapping-decoder

initial support for mapping types

src/helpers/traceHelper.js

@@ -37,6 +37,10 @@ module.exports = {
     return step.op === 'SSTORE'
   },
 
+  isSHA3Instruction: function (step) {
+    return step.op === 'SHA3'
+  },
+
   newContextStorage: function (step) {
     return step.op === 'CREATE' || step.op === 'CALL'
   },

src/solidity/decodeInfo.js

@@ -19,8 +19,19 @@ var util = require('./types/util')
   * @param {String} type - type given by the AST
   * @return {Object} returns decoded info about the current type: { storageBytes, typeName}
   */
-function mapping (type) {
-  return new MappingType()
+function mapping (type, stateDefinitions, contractName) {
+  var match = type.match(/mapping\((.*?)( =>)? (.*)\)$/)
+  var keyTypeName = match[1]
+  var valueTypeName = match[3]
+
+  var keyType = parseType(keyTypeName, stateDefinitions, contractName, 'storage')
+  var valueType = parseType(valueTypeName, stateDefinitions, contractName, 'storage')
+
+  var underlyingTypes = {
+    'keyType': keyType,
+    'valueType': valueType
+  }
+  return new MappingType(underlyingTypes, 'location', util.removeLocation(type))
 }
 
 /**
@@ -179,7 +190,7 @@ function struct (type, stateDefinitions, contractName, location) {
     if (!location) {
       location = match[2].trim()
     }
-    var memberDetails = getStructMembers(match[1], stateDefinitions, contractName, location) // type is used to extract the ast struct definition
+    var memberDetails = getStructMembers(match[1], stateDefinitions, contractName) // type is used to extract the ast struct definition
     if (!memberDetails) return null
     return new StructType(memberDetails, location, match[1])
   } else {
@@ -219,10 +230,9 @@ function getEnum (type, stateDefinitions, contractName) {
   * @param {String} typeName - name of the struct type (e.g struct <name>)
   * @param {Object} stateDefinitions  - all state definition given by the AST (including struct and enum type declaration) for all contracts
   * @param {String} contractName - contract the @args typeName belongs to
-  * @param {String} location - location of the data (storage ref| storage pointer| memory| calldata)
   * @return {Array} containing all members of the current struct type
   */
-function getStructMembers (type, stateDefinitions, contractName, location) {
+function getStructMembers (type, stateDefinitions, contractName) {
   var split = type.split('.')
   if (!split.length) {
     type = contractName + '.' + type

src/solidity/types/Mapping.js

@@ -1,20 +1,47 @@
 'use strict'
 var RefType = require('./RefType')
+var util = require('./util')
+var ethutil = require('ethereumjs-util')
 
 class Mapping extends RefType {
-  constructor () {
-    super(1, 32, 'mapping', 'storage')
+  constructor (underlyingTypes, location, fullType) {
+    super(1, 32, fullType, 'storage')
+    this.keyType = underlyingTypes.keyType
+    this.valueType = underlyingTypes.valueType
   }
 
   async decodeFromStorage (location, storageResolver) {
+    var mappingsPreimages
+    try {
+      mappingsPreimages = await storageResolver.mappingsLocation()
+    } catch (e) {
+      return {
+        value: e.message,
+        type: this.typeName
+      }
+    }
+    var mapSlot = util.normalizeHex(ethutil.bufferToHex(location.slot))
+    console.log(mapSlot, mappingsPreimages)
+    var mappingPreimages = mappingsPreimages[mapSlot]
+    var ret = {}
+    for (var i in mappingPreimages) {
+      var mapLocation = getMappingLocation(i, location.slot)
+      var globalLocation = {
+        offset: location.offset,
+        slot: mapLocation
+      }
+      ret[i] = await this.valueType.decodeFromStorage(globalLocation, storageResolver)
+    }
+
     return {
-      value: '<not implemented>',
-      length: '0x',
+      value: ret,
       type: this.typeName
     }
   }
 
   decodeFromMemoryInternal (offset, memory) {
+    // mappings can only exist in storage and not in memory
+    // so this should never be called
     return {
       value: '<not implemented>',
       length: '0x',
@@ -23,4 +50,27 @@ class Mapping extends RefType {
   }
 }
 
+function getMappingLocation (key, position) {
+  // mapping storage location decribed at http://solidity.readthedocs.io/en/develop/miscellaneous.html#layout-of-state-variables-in-storage
+  // > the value corresponding to a mapping key k is located at keccak256(k . p) where . is concatenation.
+
+  // key should be a hex string, and position an int
+  var mappingK = ethutil.toBuffer('0x' + key)
+  mappingK = ethutil.setLengthLeft(mappingK, 32)
+  var mappingP = ethutil.intToBuffer(position)
+  mappingP = ethutil.setLengthLeft(mappingP, 32)
+  var mappingKeyBuf = concatTypedArrays(mappingK, mappingP)
+  var mappingKeyPreimage = '0x' + mappingKeyBuf.toString('hex')
+  var mappingStorageLocation = ethutil.sha3(mappingKeyPreimage)
+  mappingStorageLocation = new ethutil.BN(mappingStorageLocation, 16)
+  return mappingStorageLocation
+}
+
+function concatTypedArrays (a, b) { // a, b TypedArray of same type
+  let c = new (a.constructor)(a.length + b.length)
+  c.set(a, 0)
+  c.set(b, a.length)
+  return c
+}
+
 module.exports = Mapping

src/solidity/types/util.js

@@ -10,7 +10,8 @@ module.exports = {
   toBN: toBN,
   add: add,
   extractLocation: extractLocation,
-  removeLocation: removeLocation
+  removeLocation: removeLocation,
+  normalizeHex: normalizeHex
 }
 
 function decodeIntFromHex (value, byteLength, signed) {

src/storage/mappingPreimages.js

@@ -0,0 +1,54 @@
+var global = require('../helpers/global')
+
+module.exports = {
+  decodeMappingsKeys: decodeMappingsKeys
+}
+
+/**
+  * extract the mappings location from the storage
+  * like { "<mapping_slot>" : { "<mapping-key1>": preimageOf1 }, { "<mapping-key2>": preimageOf2 }, ... }
+  *
+  * @param {Object} storage  - storage given by storage Viewer (basically a mapping hashedkey : {key, value})
+  * @param {Function} callback  - calback
+  * @return {Map} - solidity mapping location (e.g { "<mapping_slot>" : { "<mapping-key1>": preimageOf1 }, { "<mapping-key2>": preimageOf2 }, ... })
+  */
+async function decodeMappingsKeys (storage, callback) {
+  var ret = {}
+  for (var hashedLoc in storage) {
+    var preimage
+    try {
+      preimage = await getPreimage(storage[hashedLoc].key)
+    } catch (e) {
+    }
+    if (preimage) {
+      // got preimage!
+      // get mapping position (i.e. storage slot), its the last 32 bytes
+      var slotByteOffset = preimage.length - 64
+      var mappingSlot = preimage.substr(slotByteOffset)
+      var mappingKey = preimage.substr(0, slotByteOffset)
+      if (!ret[mappingSlot]) {
+        ret[mappingSlot] = {}
+      }
+      ret[mappingSlot][mappingKey] = preimage
+    }
+  }
+  callback(null, ret)
+}
+
+/**
+  * Uses web3 to return preimage of a key
+  *
+  * @param {String} key  - key to retrieve the preimage of
+  * @return {String} - preimage of the given key
+  */
+function getPreimage (key) {
+  return new Promise((resolve, reject) => {
+    global.web3.debug.preimage(key, function (error, preimage) {
+      if (error) {
+        reject(error)
+      } else {
+        resolve(preimage)
+      }
+    })
+  })
+}

src/storage/storageResolver.js

@@ -1,10 +1,16 @@
 'use strict'
 var traceHelper = require('../helpers/traceHelper')
 var util = require('../helpers/global')
+var mappingPreimages = require('./mappingPreimages')
 
+/**
+  * Basically one instance is created for one debugging session.
+  * (TODO: one instance need to be shared over all the components)
+  */
 class StorageResolver {
   constructor () {
     this.storageByAddress = {}
+    this.preimagesMappingByAddress = {}
     this.maxSize = 100
   }
 
@@ -21,6 +27,33 @@ class StorageResolver {
     storageRangeInternal(this, zeroSlot, tx, stepIndex, address, callback)
   }
 
+  /**
+    * compute the mappgings type locations for the current address (cached for a debugging session)
+    * note: that only retrieve the first 100 items.
+    *
+    * @param {String} address  - contract address
+    * @param {Object} address  - storage
+    * @return {Function} - callback
+    */
+  initialPreimagesMappings (tx, stepIndex, address, callback) {
+    if (this.preimagesMappingByAddress[address]) {
+      return callback(null, this.preimagesMappingByAddress[address])
+    }
+    this.storageRange(tx, stepIndex, address, (error, storage) => {
+      if (error) {
+        return callback(error)
+      }
+      mappingPreimages.decodeMappingsKeys(storage, (error, mappings) => {
+        if (error) {
+          callback(error)
+        } else {
+          this.preimagesMappingByAddress[address] = mappings
+          callback(null, mappings)
+        }
+      })
+    })
+  }
+
   /**
     * return a slot value for the given context (address and vm trace index)
     *
@@ -96,11 +129,11 @@ function fromCache (self, address) {
 }
 
 /**
- * store the result of `storageRangeAtInternal`
- *
- * @param {String} address  - contract address
- * @param {Object} storage  - result of `storageRangeAtInternal`, contains {key, hashedKey, value}
- */
+  * store the result of `storageRangeAtInternal`
+  *
+  * @param {String} address  - contract address
+  * @param {Object} storage  - result of `storageRangeAtInternal`, contains {key, hashedKey, value}
+  */
 function toCache (self, address, storage) {
   if (!self.storageByAddress[address]) {
     self.storageByAddress[address] = {}

src/storage/storageViewer.js

@@ -1,10 +1,17 @@
 'use strict'
 var helper = require('../helpers/util')
+var mappingPreimages = require('./mappingPreimages')
 
+ /**
+   * easier access to the storage resolver
+   * Basically one instance is created foreach execution step and foreach component that need it.
+   * (TODO: one instance need to be shared over all the components)
+   */
 class StorageViewer {
   constructor (_context, _storageResolver, _traceManager) {
     this.context = _context
     this.storageResolver = _storageResolver
+    this.completeMapingsLocationPromise = null
     _traceManager.accumulateStorageChanges(this.context.stepIndex, this.context.address, {}, (error, storageChanges) => {
       if (!error) {
         this.storageChanges = storageChanges
@@ -15,11 +22,11 @@ class StorageViewer {
   }
 
   /**
-   * return the storage for the current context (address and vm trace index)
-   * by default now returns the range 0 => 1000
-   *
-   * @param {Function} - callback - contains a map: [hashedKey] = {key, hashedKey, value}
-   */
+    * return the storage for the current context (address and vm trace index)
+    * by default now returns the range 0 => 1000
+    *
+    * @param {Function} - callback - contains a map: [hashedKey] = {key, hashedKey, value}
+    */
   storageRange (callback) {
     this.storageResolver.storageRange(this.context.tx, this.context.stepIndex, this.context.address, (error, storage) => {
       if (error) {
@@ -58,6 +65,62 @@ class StorageViewer {
   isComplete (address) {
     return this.storageResolver.isComplete(address)
   }
+
+  /**
+    * return all the possible mappings locations for the current context (cached)
+    *
+    * @param {Function} callback
+    */
+  async mappingsLocation () {
+    if (!this.completeMapingsLocationPromise) {
+      this.completeMapingsLocationPromise = new Promise((resolve, reject) => {
+        if (this.completeMappingsLocation) {
+          return this.completeMappingsLocation
+        }
+        this.storageResolver.initialPreimagesMappings(this.context.tx, this.context.stepIndex, this.context.address, (error, initialMappingsLocation) => {
+          if (error) {
+            reject(error)
+          } else {
+            this.extractMappingsLocationChanges(this.storageChanges, (error, mappingsLocationChanges) => {
+              if (error) {
+                return reject(error)
+              }
+              this.completeMappingsLocation = Object.assign({}, initialMappingsLocation)
+              for (var key in mappingsLocationChanges) {
+                if (!initialMappingsLocation[key]) {
+                  initialMappingsLocation[key] = {}
+                }
+                this.completeMappingsLocation[key] = Object.assign({}, initialMappingsLocation[key], mappingsLocationChanges[key])
+              }
+              resolve(this.completeMappingsLocation)
+            })
+          }
+        })
+      })
+    }
+    return this.completeMapingsLocationPromise
+  }
+
+  /**
+    * retrieve mapping location changes from the storage changes.
+    *
+    * @param {Function} callback
+    */
+  extractMappingsLocationChanges (storageChanges, callback) {
+    if (this.mappingsLocationChanges) {
+      return callback(null, this.mappingsLocationChanges)
+    }
+    mappingPreimages.decodeMappingsKeys(storageChanges, (error, mappings) => {
+      if (!error) {
+        this.mappingsLocationChanges = mappings
+        return callback(null, this.mappingsLocationChanges)
+      } else {
+        callback(error)
+      }
+    })
+  }
 }
 
+
+
 module.exports = StorageViewer

src/ui/SolidityTypeFormatter.js

@@ -41,6 +41,12 @@ function extractData (item, parent, key) {
     })
     ret.self = item.type
     ret.isStruct = true
+  } else if (item.type.indexOf('mapping') === 0) {
+    ret.children = Object.keys((item.value || {})).map(function (key) {
+      return {key: key, value: item.value[key]}
+    })
+    ret.isMapping = true
+    ret.self = item.type
   } else {
     ret.children = []
     ret.self = item.value
@@ -51,7 +57,7 @@ function extractData (item, parent, key) {
 
 function fontColor (data) {
   var color = '#124B46'
-  if (data.isArray || data.isStruct) {
+  if (data.isArray || data.isStruct || data.isMapping) {
     color = '#847979'
   } else if (data.type.indexOf('uint') === 0 ||
               data.type.indexOf('int') === 0 ||
@@ -63,4 +69,3 @@ function fontColor (data) {
   }
   return 'color:' + color
 }
-

src/util/web3Admin.js

@@ -6,6 +6,15 @@ module.exports = {
     }
     // DEBUG
     var methods = []
+    if (!(web3.debug && web3.debug.preimage)) {
+      methods.push(new web3._extend.Method({
+        name: 'preimage',
+        call: 'debug_preimage',
+        inputFormatter: [null],
+        params: 1
+      }))
+    }
+
     if (!(web3.debug && web3.debug.traceTransaction)) {
       methods.push(new web3._extend.Method({
         name: 'traceTransaction',

src/web3Provider/web3VmProvider.js

@@ -1,6 +1,7 @@
 var util = require('../helpers/util')
 var uiutil = require('../helpers/ui')
 var traceHelper = require('../helpers/traceHelper')
+var ethutil = require('ethereumjs-util')
 var Web3 = require('web3')
 
 function web3VmProvider () {
@@ -22,9 +23,11 @@ function web3VmProvider () {
   this.eth.getBlockNumber = function (cb) { return self.getBlockNumber(cb) }
   this.debug.traceTransaction = function (hash, options, cb) { return self.traceTransaction(hash, options, cb) }
   this.debug.storageRangeAt = function (blockNumber, txIndex, address, start, end, maxLength, cb) { return self.storageRangeAt(blockNumber, txIndex, address, start, end, maxLength, cb) }
+  this.debug.preimage = function (hashedKey, cb) { return self.preimage(hashedKey, cb) }
   this.providers = { 'HttpProvider': function (url) {} }
   this.currentProvider = {'host': 'vm provider'}
   this.storageCache = {}
+  this.sha3Preimages = {}
 }
 
 web3VmProvider.prototype.setVM = function (vm) {
@@ -128,6 +131,14 @@ web3VmProvider.prototype.pushTrace = function (self, data) {
       }
     }
   }
+  if (traceHelper.isSHA3Instruction(step)) {
+    var sha3Input = getSha3Input(step.stack, step.memory)
+    var preimage = sha3Input
+    var imageHash = ethutil.sha3('0x' + sha3Input).toString('hex')
+    self.sha3Preimages[imageHash] = {
+      'preimage': preimage
+    }
+  }
   this.processingIndex++
   this.previousDepth = depth
 }
@@ -189,4 +200,39 @@ web3VmProvider.prototype.getTransactionFromBlock = function (blockNumber, txInde
   }
 }
 
+web3VmProvider.prototype.preimage = function (hashedKey, cb) {
+  hashedKey = hashedKey.replace('0x', '')
+  cb(null, this.sha3Preimages[hashedKey] !== undefined ? this.sha3Preimages[hashedKey].preimage : null)
+}
+
+function getSha3Input (stack, memory) {
+  var memoryStart = stack[stack.length - 1]
+  var memoryLength = stack[stack.length - 2]
+  var memStartDec = (new ethutil.BN(memoryStart.replace('0x', ''), 16)).toString(10)
+  memoryStart = parseInt(memStartDec) * 2
+  var memLengthDec = (new ethutil.BN(memoryLength.replace('0x', ''), 16).toString(10))
+  memoryLength = parseInt(memLengthDec) * 2
+
+  var i = Math.floor(memoryStart / 32)
+  var maxIndex = Math.floor(memoryLength / 32)
+  if (!memory[i]) {
+    return emptyFill(memoryLength)
+  }
+  var sha3Input = memory[i].slice(memoryStart - 32 * i)
+  i++
+  while (i < maxIndex) {
+    sha3Input += memory[i] ? memory[i] : emptyFill(32)
+    i++
+  }
+  if (sha3Input.length < memoryLength) {
+    var leftSize = memoryLength - sha3Input.length
+    sha3Input += memory[i] ? memory[i].slice(0, leftSize) : emptyFill(leftSize)
+  }
+  return sha3Input
+}
+
+function emptyFill (size) {
+  return (new Array(size)).join('0')
+}
+
 module.exports = web3VmProvider