Merge pull request #677 from ethereum/compiler_wrapper

Move compiler code from browser-solidity

remix-solidity/index.js

@@ -3,11 +3,15 @@ var SolidityProxy = require('./src/decoder/solidityProxy')
 var localDecoder = require('./src/decoder/localDecoder')
 var stateDecoder = require('./src/decoder/stateDecoder')
 var CodeAnalysis = require('./src/analysis/staticAnalysisRunner')
+var Compiler = require('./src/compiler/compiler')
+var CompilerInput = require('./src/compiler/compiler-input')
 
 module.exports = {
   InternalCallTree: InternalCallTree,
   SolidityProxy: SolidityProxy,
   localDecoder: localDecoder,
   stateDecoder: stateDecoder,
-  CodeAnalysis: CodeAnalysis
+  CodeAnalysis: CodeAnalysis,
+  Compiler: Compiler,
+  CompilerInput: CompilerInput
 }

remix-solidity/package.json

@@ -23,13 +23,16 @@
     "fast-async": "^6.1.2",
     "remix-core": "latest",
     "remix-lib": "latest",
-    "solc": "^0.4.13",
+    "webworkify": "^1.2.1",
+    "solc": "https://github.com/ethereum/solc-js",
+    "npm-run-all": "^4.0.2",
     "standard": "^7.0.1",
     "tape": "^4.6.0"
   },
   "scripts": {
     "test": "standard && npm run downloadsolc && tape ./test/tests.js",
-    "downloadsolc": "test -e soljson.js || wget https://ethereum.github.io/solc-bin/soljson.js"
+    "downloadsolc": "cd node_modules/solc && (test -e soljson.js || wget https://ethereum.github.io/solc-bin/soljson.js) && cd ..",
+    "prepublish": "mkdirp build; npm-run-all -ls downloadsolc"
   },
   "standard": {
     "ignore": [

remix-solidity/src/compiler/compiler-input.js

@@ -0,0 +1,21 @@
+'use strict'
+
+module.exports = (sources, opts) => {
+  return JSON.stringify({
+    language: 'Solidity',
+    sources: sources,
+    settings: {
+      optimizer: {
+        enabled: opts.optimize === true || opts.optimize === 1,
+        runs: 200
+      },
+      libraries: opts.libraries,
+      outputSelection: {
+        '*': {
+          '': [ 'legacyAST' ],
+          '*': [ 'abi', 'metadata', 'devdoc', 'userdoc', 'evm.legacyAssembly', 'evm.bytecode', 'evm.deployedBytecode', 'evm.methodIdentifiers', 'evm.gasEstimates' ]
+        }
+      }
+    }
+  })
+}

remix-solidity/src/compiler/compiler-worker.js

@@ -0,0 +1,45 @@
+'use strict'
+
+var solc = require('solc/wrapper')
+
+var compileJSON = function () { return '' }
+var missingInputs = []
+
+module.exports = function (self) {
+  self.addEventListener('message', function (e) {
+    var data = e.data
+    switch (data.cmd) {
+      case 'loadVersion':
+        delete self.Module
+        // NOTE: workaround some browsers?
+        self.Module = undefined
+
+        compileJSON = null
+
+        self.importScripts(data.data)
+
+        var compiler = solc(self.Module)
+
+        compileJSON = function (input) {
+          try {
+            return compiler.compileStandardWrapper(input, function (path) {
+              missingInputs.push(path)
+              return { 'error': 'Deferred import' }
+            })
+          } catch (exception) {
+            return JSON.stringify({ error: 'Uncaught JavaScript exception:\n' + exception })
+          }
+        }
+
+        self.postMessage({
+          cmd: 'versionLoaded',
+          data: compiler.version()
+        })
+        break
+      case 'compile':
+        missingInputs.length = 0
+        self.postMessage({cmd: 'compiled', job: data.job, data: compileJSON(data.input), missingInputs: missingInputs})
+        break
+    }
+  }, false)
+}

remix-solidity/src/compiler/compiler.js

@@ -0,0 +1,368 @@
+'use strict'
+
+var solc = require('solc/wrapper')
+var solcABI = require('solc/abi')
+
+var webworkify = require('webworkify')
+
+var compilerInput = require('./compiler-input')
+
+var remixLib = require('remix-lib')
+var EventManager = remixLib.EventManager
+
+var txHelper = require('./txHelper')
+
+/*
+  trigger compilationFinished, compilerLoaded, compilationStarted, compilationDuration
+*/
+function Compiler (handleImportCall) {
+  var self = this
+  this.event = new EventManager()
+
+  var compileJSON
+
+  var worker = null
+
+  var currentVersion
+
+  var optimize = false
+
+  this.setOptimize = function (_optimize) {
+    optimize = _optimize
+  }
+
+  var compilationStartTime = null
+  this.event.register('compilationFinished', (success, data, source) => {
+    if (success && compilationStartTime) {
+      this.event.trigger('compilationDuration', [(new Date().getTime()) - compilationStartTime])
+    }
+    compilationStartTime = null
+  })
+
+  this.event.register('compilationStarted', () => {
+    compilationStartTime = new Date().getTime()
+  })
+
+  var internalCompile = function (files, target, missingInputs) {
+    gatherImports(files, target, missingInputs, function (error, input) {
+      if (error) {
+        self.lastCompilationResult = null
+        self.event.trigger('compilationFinished', [false, {'error': { formattedMessage: error, severity: 'error' }}, files])
+      } else {
+        compileJSON(input, optimize ? 1 : 0)
+      }
+    })
+  }
+
+  var compile = function (files, target) {
+    self.event.trigger('compilationStarted', [])
+    internalCompile(files, target)
+  }
+  this.compile = compile
+
+  function setCompileJSON (_compileJSON) {
+    compileJSON = _compileJSON
+  }
+  this.setCompileJSON = setCompileJSON // this is exposed for testing
+
+  function onCompilerLoaded (version) {
+    currentVersion = version
+    self.event.trigger('compilerLoaded', [version])
+  }
+
+  function onInternalCompilerLoaded () {
+    if (worker === null) {
+      var compiler
+      if (typeof (window) === 'undefined') {
+        compiler = require('solc')
+      } else {
+        compiler = solc(window.Module)
+      }
+
+      compileJSON = function (source, optimize, cb) {
+        var missingInputs = []
+        var missingInputsCallback = function (path) {
+          missingInputs.push(path)
+          return { error: 'Deferred import' }
+        }
+
+        var result
+        try {
+          var input = compilerInput(source.sources, {optimize: optimize, target: source.target})
+          result = compiler.compileStandardWrapper(input, missingInputsCallback)
+          result = JSON.parse(result)
+        } catch (exception) {
+          result = { error: 'Uncaught JavaScript exception:\n' + exception }
+        }
+
+        compilationFinished(result, missingInputs, source)
+      }
+      onCompilerLoaded(compiler.version())
+    }
+  }
+  // exposed for use in node
+  this.onInternalCompilerLoaded = onInternalCompilerLoaded
+
+  this.lastCompilationResult = {
+    data: null,
+    source: null
+  }
+
+  /**
+    * return the contract obj of the given @arg name. Uses last compilation result.
+    * return null if not found
+    * @param {String} name    - contract name
+    * @returns contract obj and associated file: { contract, file } or null
+    */
+  this.getContract = (name) => {
+    if (this.lastCompilationResult.data && this.lastCompilationResult.data.contracts) {
+      return txHelper.getContract(name, this.lastCompilationResult.data.contracts)
+    }
+    return null
+  }
+
+  /**
+    * call the given @arg cb (function) for all the contracts. Uses last compilation result
+    * @param {Function} cb    - callback
+    */
+  this.visitContracts = (cb) => {
+    if (this.lastCompilationResult.data && this.lastCompilationResult.data.contracts) {
+      return txHelper.visitContracts(this.lastCompilationResult.data.contracts, cb)
+    }
+    return null
+  }
+
+  /**
+    * return the compiled contracts from the last compilation result
+    * @return {Object}     - contracts
+    */
+  this.getContracts = () => {
+    if (this.lastCompilationResult.data && this.lastCompilationResult.data.contracts) {
+      return this.lastCompilationResult.data.contracts
+    }
+    return null
+  }
+
+   /**
+    * return the sources from the last compilation result
+    * @param {Object} cb    - map of sources
+    */
+  this.getSources = () => {
+    if (this.lastCompilationResult.source) {
+      return this.lastCompilationResult.source.sources
+    }
+    return null
+  }
+
+  /**
+    * return the sources @arg fileName from the last compilation result
+    * @param {Object} cb    - map of sources
+    */
+  this.getSource = (fileName) => {
+    if (this.lastCompilationResult.source) {
+      return this.lastCompilationResult.source.sources[fileName]
+    }
+    return null
+  }
+
+  /**
+    * return the source from the last compilation result that has the given index. null if source not found
+    * @param {Int} index    - index of the source
+    */
+  this.getSourceName = (index) => {
+    if (this.lastCompilationResult.data && this.lastCompilationResult.data.sources) {
+      return Object.keys(this.lastCompilationResult.data.sources)[index]
+    }
+    return null
+  }
+
+  function compilationFinished (data, missingInputs, source) {
+    var noFatalErrors = true // ie warnings are ok
+
+    function isValidError (error) {
+      // The deferred import is not a real error
+      // FIXME: maybe have a better check?
+      if (/Deferred import/.exec(error.message)) {
+        return false
+      }
+
+      return error.severity !== 'warning'
+    }
+
+    if (data['error'] !== undefined) {
+      // Ignore warnings (and the 'Deferred import' error as those are generated by us as a workaround
+      if (isValidError(data['error'])) {
+        noFatalErrors = false
+      }
+    }
+    if (data['errors'] !== undefined) {
+      data['errors'].forEach(function (err) {
+        // Ignore warnings and the 'Deferred import' error as those are generated by us as a workaround
+        if (isValidError(err)) {
+          noFatalErrors = false
+        }
+      })
+    }
+
+    if (!noFatalErrors) {
+      // There are fatal errors - abort here
+      self.lastCompilationResult = null
+      self.event.trigger('compilationFinished', [false, data, source])
+    } else if (missingInputs !== undefined && missingInputs.length > 0) {
+      // try compiling again with the new set of inputs
+      internalCompile(source.sources, source.target, missingInputs)
+    } else {
+      data = updateInterface(data)
+
+      self.lastCompilationResult = {
+        data: data,
+        source: source
+      }
+      self.event.trigger('compilationFinished', [true, data, source])
+    }
+  }
+
+  // TODO: needs to be changed to be more node friendly
+  this.loadVersion = function (usingWorker, url) {
+    console.log('Loading ' + url + ' ' + (usingWorker ? 'with worker' : 'without worker'))
+    self.event.trigger('loadingCompiler', [url, usingWorker])
+
+    if (usingWorker) {
+      loadWorker(url)
+    } else {
+      loadInternal(url)
+    }
+  }
+
+  function loadInternal (url) {
+    delete window.Module
+    // NOTE: workaround some browsers?
+    window.Module = undefined
+
+    // Set a safe fallback until the new one is loaded
+    setCompileJSON(function (source, optimize) {
+      compilationFinished({ error: { formattedMessage: 'Compiler not yet loaded.' } })
+    })
+
+    var newScript = document.createElement('script')
+    newScript.type = 'text/javascript'
+    newScript.src = url
+    document.getElementsByTagName('head')[0].appendChild(newScript)
+    var check = window.setInterval(function () {
+      if (!window.Module) {
+        return
+      }
+      window.clearInterval(check)
+      onInternalCompilerLoaded()
+    }, 200)
+  }
+
+  function loadWorker (url) {
+    if (worker !== null) {
+      worker.terminate()
+    }
+    worker = webworkify(require('./compiler-worker.js'))
+    var jobs = []
+    worker.addEventListener('message', function (msg) {
+      var data = msg.data
+      switch (data.cmd) {
+        case 'versionLoaded':
+          onCompilerLoaded(data.data)
+          break
+        case 'compiled':
+          var result
+          try {
+            result = JSON.parse(data.data)
+          } catch (exception) {
+            result = { 'error': 'Invalid JSON output from the compiler: ' + exception }
+          }
+          var sources = {}
+          if (data.job in jobs !== undefined) {
+            sources = jobs[data.job].sources
+            delete jobs[data.job]
+          }
+          compilationFinished(result, data.missingInputs, sources)
+          break
+      }
+    })
+    worker.onerror = function (msg) {
+      compilationFinished({ error: 'Worker error: ' + msg.data })
+    }
+    worker.addEventListener('error', function (msg) {
+      compilationFinished({ error: 'Worker error: ' + msg.data })
+    })
+    compileJSON = function (source, optimize) {
+      jobs.push({sources: source})
+      worker.postMessage({cmd: 'compile', job: jobs.length - 1, input: compilerInput(source.sources, {optimize: optimize, target: source.target})})
+    }
+    worker.postMessage({cmd: 'loadVersion', data: url})
+  }
+
+  function gatherImports (files, target, importHints, cb) {
+    importHints = importHints || []
+
+    // FIXME: This will only match imports if the file begins with one.
+    //        It should tokenize by lines and check each.
+    // eslint-disable-next-line no-useless-escape
+    var importRegex = /^\s*import\s*[\'\"]([^\'\"]+)[\'\"];/g
+
+    for (var fileName in files) {
+      var match
+      while ((match = importRegex.exec(files[fileName].content))) {
+        var importFilePath = match[1]
+        if (importFilePath.startsWith('./')) {
+          var path = /(.*\/).*/.exec(target)
+          if (path !== null) {
+            importFilePath = importFilePath.replace('./', path[1])
+          } else {
+            importFilePath = importFilePath.slice(2)
+          }
+        }
+
+        // FIXME: should be using includes or sets, but there's also browser compatibility..
+        if (importHints.indexOf(importFilePath) === -1) {
+          importHints.push(importFilePath)
+        }
+      }
+    }
+
+    while (importHints.length > 0) {
+      var m = importHints.pop()
+      if (m in files) {
+        continue
+      }
+
+      if (handleImportCall) {
+        handleImportCall(m, function (err, content) {
+          if (err) {
+            cb(err)
+          } else {
+            files[m] = { content }
+            gatherImports(files, target, importHints, cb)
+          }
+        })
+      }
+
+      return
+    }
+
+    cb(null, { 'sources': files, 'target': target })
+  }
+
+  function truncateVersion (version) {
+    var tmp = /^(\d+.\d+.\d+)/.exec(version)
+    if (tmp) {
+      return tmp[1]
+    }
+    return version
+  }
+
+  function updateInterface (data) {
+    txHelper.visitContracts(data.contracts, (contract) => {
+      data.contracts[contract.file][contract.name].abi = solcABI.update(truncateVersion(currentVersion), contract.object.abi)
+    })
+    return data
+  }
+}
+
+module.exports = Compiler

remix-solidity/src/compiler/txHelper.js

@@ -0,0 +1,33 @@
+'use strict'
+
+module.exports = {
+
+  /**
+    * return the contract obj of the given @arg name. Uses last compilation result.
+    * return null if not found
+    * @param {String} name    - contract name
+    * @returns contract obj and associated file: { contract, file } or null
+    */
+  getContract: (contractName, contracts) => {
+    for (var file in contracts) {
+      if (contracts[file][contractName]) {
+        return { object: contracts[file][contractName], file: file }
+      }
+    }
+    return null
+  },
+
+  /**
+    * call the given @arg cb (function) for all the contracts. Uses last compilation result
+    * stop visiting when cb return true
+    * @param {Function} cb    - callback
+    */
+  visitContracts: (contracts, cb) => {
+    for (var file in contracts) {
+      for (var name in contracts[file]) {
+        if (cb({ name: name, object: contracts[file][name], file: file })) return
+      }
+    }
+  }
+
+}

remix-solidity/test/analysis/staticAnalysisIntegration-test.js

@@ -4,8 +4,7 @@ var remixLib = require('remix-lib')
 var StatRunner = require('../../src/analysis/staticAnalysisRunner')
 var compilerInput = remixLib.helpers.compiler.compilerInput
 
-var solc = require('solc/wrapper')
-var compiler = solc(require('../../soljson'))
+var compiler = require('solc')
 
 var fs = require('fs')
 var path = require('path')

remix-solidity/test/analysis/staticAnalysisIssues-test.js

@@ -4,8 +4,7 @@ var remixLib = require('remix-lib')
 var StatRunner = require('../../src/analysis/staticAnalysisRunner')
 var compilerInput = remixLib.helpers.compiler.compilerInput
 
-var solc = require('solc/wrapper')
-var compiler = solc(require('../../soljson'))
+var compiler = require('solc')
 
 var fs = require('fs')
 var path = require('path')