Merge pull request #1047 from ethereum/tests-dir

unit testing fix for PR #1014

remix-tests/src/compiler.js

@@ -1,5 +1,5 @@
 /* eslint no-extend-native: "warn" */
-let fs = require('fs')
+let fs = require('./fs')
 var async = require('async')
 var path = require('path')
 let RemixCompiler = require('remix-solidity').Compiler
@@ -26,52 +26,60 @@ var isBrowser = !(typeof (window) === 'undefined' || userAgent.indexOf(' electro
 
 // TODO: replace this with remix's own compiler code
 function compileFileOrFiles (filename, isDirectory, opts, cb) {
-  let compiler, filepath
+  let compiler
   let accounts = opts.accounts || []
   const sources = {
     'tests.sol': { content: require('../sol/tests.sol.js') },
     'remix_tests.sol': { content: require('../sol/tests.sol.js') },
     'remix_accounts.sol': { content: writeTestAccountsContract(accounts) }
   }
+  const filepath = (isDirectory ? filename : path.dirname(filename))
   // TODO: for now assumes filepath dir contains all tests, later all this
   // should be replaced with remix's & browser solidity compiler code
-  filepath = (isDirectory ? filename : path.dirname(filename))
 
-  fs.readdirSync(filepath).forEach(file => {
+  // This logic is wrong
-    // only process .sol files
+  // We should only look into current file if a full file name with path is given
-    if (file.split('.').pop() === 'sol') {
+  // We should only walk through directory if a directory name is passed
-      let c = fs.readFileSync(path.join(filepath, file)).toString()
+  try {
-      const s = /^(import)\s['"](remix_tests.sol|tests.sol)['"];/gm
+    // walkSync only if it is a directory
-      let includeTestLibs = '\nimport \'remix_tests.sol\';\n'
+    fs.walkSync(filepath, foundpath => {
-      if (file.indexOf('_test.sol') > 0 && c.regexIndexOf(s) < 0) {
+      // only process .sol files
-        c = includeTestLibs.concat(c)
+      if (foundpath.split('.').pop() === 'sol') {
+        let c = fs.readFileSync(foundpath).toString()
+        const s = /^(import)\s['"](remix_tests.sol|tests.sol)['"];/gm
+        let includeTestLibs = '\nimport \'remix_tests.sol\';\n'
+        if (foundpath.indexOf('_test.sol') > 0 && c.regexIndexOf(s) < 0) {
+          c = includeTestLibs.concat(c)
+        }
+        sources[foundpath] = { content: c }
       }
-      sources[file] = { content: c }
+    })
-    }
+  } catch (e) {
-  })
+    throw e
-
+  } finally {
-  async.waterfall([
+    async.waterfall([
-    function loadCompiler (next) {
+      function loadCompiler (next) {
-      compiler = new RemixCompiler()
+        compiler = new RemixCompiler()
-      compiler.onInternalCompilerLoaded()
+        compiler.onInternalCompilerLoaded()
-      // compiler.event.register('compilerLoaded', this, function (version) {
+        // compiler.event.register('compilerLoaded', this, function (version) {
-      next()
+        next()
-      // });
+        // });
-    },
+      },
-    function doCompilation (next) {
+      function doCompilation (next) {
-      compiler.event.register('compilationFinished', this, function (success, data, source) {
+        compiler.event.register('compilationFinished', this, function (success, data, source) {
-        next(null, data)
+          next(null, data)
-      })
+        })
-      compiler.compile(sources, filepath)
+        compiler.compile(sources, filepath)
-    }
+      }
-  ], function (err, result) {
+    ], function (err, result) {
-    let errors = (result.errors || []).filter((e) => e.type === 'Error' || e.severity === 'error')
+      let errors = (result.errors || []).filter((e) => e.type === 'Error' || e.severity === 'error')
-    if (errors.length > 0) {
+      if (errors.length > 0) {
-      if (!isBrowser) require('signale').fatal(errors)
+        if (!isBrowser) require('signale').fatal(errors)
-      return cb(new Error('errors compiling'))
+        return cb(new Error('errors compiling'))
-    }
+      }
-    cb(err, result.contracts)
+      cb(err, result.contracts)
-  })
+    })
+  }
 }
 
 function compileContractSources (sources, importFileCb, opts, cb) {

remix-tests/src/fs.js

@@ -0,0 +1,20 @@
+// Extend fs
+var fs = require('fs')
+const path = require('path')
+
+// https://github.com/mikeal/node-utils/blob/master/file/lib/main.js
+fs.walkSync = function (start, callback) {
+  fs.readdirSync(start).forEach(name => {
+    if (name === 'node_modules') {
+      return // hack
+    }
+    var abspath = path.join(start, name)
+    if (fs.statSync(abspath).isDirectory()) {
+      fs.walkSync(abspath, callback)
+    } else {
+      callback(abspath)
+    }
+  })
+}
+
+module.exports = fs

remix-tests/src/index.js

@@ -1,234 +1,6 @@
-const async = require('async')
+const runTestFiles = require('./runTestFiles.js')
-const path = require('path')
+const runTestSources = require('./runTestSources.js')
-const fs = require('fs')
+const TestRunner = require('./testRunner.js')
-require('colors')
-
-let Compiler = require('./compiler.js')
-let Deployer = require('./deployer.js')
-let TestRunner = require('./testRunner.js')
-
-const Web3 = require('web3')
-const Provider = require('remix-simulator').Provider
-
-var createWeb3Provider = function () {
-  let web3 = new Web3()
-  web3.setProvider(new Provider())
-  return web3
-}
-
-var runTestSources = function (contractSources, testCallback, resultCallback, finalCallback, importFileCb, opts) {
-  opts = opts || {}
-  let web3 = opts.web3 || createWeb3Provider()
-  let accounts = opts.accounts || null
-  async.waterfall([
-    function getAccountList (next) {
-      if (accounts) return next()
-      web3.eth.getAccounts((_err, _accounts) => {
-        accounts = _accounts
-        next()
-      })
-    },
-    function compile (next) {
-      Compiler.compileContractSources(contractSources, importFileCb, { accounts }, next)
-    },
-    function deployAllContracts (compilationResult, next) {
-      Deployer.deployAll(compilationResult, web3, function (err, contracts) {
-        if (err) {
-          next(err)
-        }
-
-        next(null, compilationResult, contracts)
-      })
-    },
-    function determineTestContractsToRun (compilationResult, contracts, next) {
-      let contractsToTest = []
-      let contractsToTestDetails = []
-
-      for (let filename in compilationResult) {
-        if (filename.indexOf('_test.sol') < 0) {
-          continue
-        }
-        Object.keys(compilationResult[filename]).forEach(contractName => {
-          contractsToTestDetails.push(compilationResult[filename][contractName])
-          contractsToTest.push(contractName)
-        })
-      }
-
-      next(null, contractsToTest, contractsToTestDetails, contracts)
-    },
-    function runTests (contractsToTest, contractsToTestDetails, contracts, next) {
-      let totalPassing = 0
-      let totalFailing = 0
-      let totalTime = 0
-      let errors = []
-
-      var _testCallback = function (result) {
-        if (result.type === 'testFailure') {
-          errors.push(result)
-        }
-        testCallback(result)
-      }
-
-      var _resultsCallback = function (_err, result, cb) {
-        resultCallback(_err, result, () => {})
-        totalPassing += result.passingNum
-        totalFailing += result.failureNum
-        totalTime += result.timePassed
-        cb()
-      }
-
-      async.eachOfLimit(contractsToTest, 1, (contractName, index, cb) => {
-        TestRunner.runTest(contractName, contracts[contractName], contractsToTestDetails[index], { accounts }, _testCallback, (err, result) => {
-          if (err) {
-            return cb(err)
-          }
-          _resultsCallback(null, result, cb)
-        })
-      }, function (err, _results) {
-        if (err) {
-          return next(err)
-        }
-
-        let finalResults = {}
-
-        finalResults.totalPassing = totalPassing || 0
-        finalResults.totalFailing = totalFailing || 0
-        finalResults.totalTime = totalTime || 0
-        finalResults.errors = []
-
-        errors.forEach((error, _index) => {
-          finalResults.errors.push({context: error.context, value: error.value, message: error.errMsg})
-        })
-
-        next(null, finalResults)
-      })
-    }
-  ], finalCallback)
-}
-
-var runTestFiles = function (filepath, isDirectory, web3, opts) {
-  opts = opts || {}
-  const { Signale } = require('signale')
-  // signale configuration
-  const options = {
-    types: {
-      result: {
-        badge: '\t✓',
-        label: '',
-        color: 'greenBright'
-      },
-      name: {
-        badge: '\n\t◼',
-        label: '',
-        color: 'white'
-      },
-      error: {
-        badge: '\t✘',
-        label: '',
-        color: 'redBright'
-      }
-    }
-  }
-  const signale = new Signale(options)
-  let accounts = opts.accounts || null
-  async.waterfall([
-    function getAccountList (next) {
-      if (accounts) return next(null)
-      web3.eth.getAccounts((_err, _accounts) => {
-        accounts = _accounts
-        next(null)
-      })
-    },
-    function compile (next) {
-      Compiler.compileFileOrFiles(filepath, isDirectory, { accounts }, next)
-    },
-    function deployAllContracts (compilationResult, next) {
-      Deployer.deployAll(compilationResult, web3, function (err, contracts) {
-        if (err) {
-          next(err)
-        }
-        next(null, compilationResult, contracts)
-      })
-    },
-    function determineTestContractsToRun (compilationResult, contracts, next) {
-      let contractsToTest = []
-      let contractsToTestDetails = []
-      var gatherContractsFrom = (filename) => {
-        if (filename.indexOf('_test.sol') < 0) {
-          return
-        }
-        Object.keys(compilationResult[path.basename(filename)]).forEach(contractName => {
-          contractsToTest.push(contractName)
-          contractsToTestDetails.push(compilationResult[path.basename(filename)][contractName])
-        })
-      }
-
-      if (isDirectory) {
-        fs.readdirSync(filepath).forEach(filename => {
-          gatherContractsFrom(filename)
-        })
-      } else {
-        gatherContractsFrom(filepath)
-      }
-      next(null, contractsToTest, contractsToTestDetails, contracts)
-    },
-    function runTests (contractsToTest, contractsToTestDetails, contracts, next) {
-      let totalPassing = 0
-      let totalFailing = 0
-      let totalTime = 0
-      let errors = []
-
-      var testCallback = function (result) {
-        if (result.type === 'contract') {
-          signale.name(result.value.white)
-        } else if (result.type === 'testPass') {
-          signale.result(result.value)
-        } else if (result.type === 'testFailure') {
-          signale.result(result.value.red)
-          errors.push(result)
-        }
-      }
-      var resultsCallback = function (_err, result, cb) {
-        totalPassing += result.passingNum
-        totalFailing += result.failureNum
-        totalTime += result.timePassed
-        cb()
-      }
-
-      async.eachOfLimit(contractsToTest, 1, (contractName, index, cb) => {
-        TestRunner.runTest(contractName, contracts[contractName], contractsToTestDetails[index], { accounts }, testCallback, (err, result) => {
-          if (err) {
-            return cb(err)
-          }
-          resultsCallback(null, result, cb)
-        })
-      }, function (err, _results) {
-        if (err) {
-          return next(err)
-        }
-
-        console.log('\n')
-        if (totalPassing > 0) {
-          console.log(('  ' + totalPassing + ' passing ').green + ('(' + totalTime + 's)').grey)
-        }
-        if (totalFailing > 0) {
-          console.log(('  ' + totalFailing + ' failing').red)
-        }
-        console.log('')
-
-        errors.forEach((error, index) => {
-          console.log('  ' + (index + 1) + ') ' + error.context + ' ' + error.value)
-          console.log('')
-          console.log(('\t error: ' + error.errMsg).red)
-        })
-        console.log('')
-
-        next()
-      })
-    }
-  ], function () {
-  })
-}
 
 module.exports = {
   runTestFiles: runTestFiles,

remix-tests/src/runTestFiles.js

@@ -0,0 +1,133 @@
+const async = require('async')
+const path = require('path')
+const fs = require('./fs')
+const TestRunner = require('./testRunner.js')
+require('colors')
+
+let Compiler = require('./compiler.js')
+let Deployer = require('./deployer.js')
+
+const runTestFiles = function (filepath, isDirectory, web3, opts) {
+  opts = opts || {}
+  const { Signale } = require('signale')
+  // signale configuration
+  const options = {
+    types: {
+      result: {
+        badge: '\t✓',
+        label: '',
+        color: 'greenBright'
+      },
+      name: {
+        badge: '\n\t◼',
+        label: '',
+        color: 'white'
+      },
+      error: {
+        badge: '\t✘',
+        label: '',
+        color: 'redBright'
+      }
+    }
+  }
+  const signale = new Signale(options)
+  let accounts = opts.accounts || null
+  async.waterfall([
+    function getAccountList (next) {
+      if (accounts) return next(null)
+      web3.eth.getAccounts((_err, _accounts) => {
+        accounts = _accounts
+        next(null)
+      })
+    },
+    function compile (next) {
+      Compiler.compileFileOrFiles(filepath, isDirectory, { accounts }, next)
+    },
+    function deployAllContracts (compilationResult, next) {
+      Deployer.deployAll(compilationResult, web3, function (err, contracts) {
+        if (err) {
+          next(err)
+        }
+        next(null, compilationResult, contracts)
+      })
+    },
+    function determineTestContractsToRun (compilationResult, contracts, next) {
+      let contractsToTest = []
+      let contractsToTestDetails = []
+      const gatherContractsFrom = (filename) => {
+        if (filename.indexOf('_test.sol') < 0) {
+          return
+        }
+        Object.keys(compilationResult[path.basename(filename)]).forEach(contractName => {
+          contractsToTest.push(contractName)
+          contractsToTestDetails.push(compilationResult[path.basename(filename)][contractName])
+        })
+      }
+      if (isDirectory) {
+        fs.walkSync(filepath, foundpath => {
+          gatherContractsFrom(foundpath)
+        })
+      } else {
+        gatherContractsFrom(filepath)
+      }
+      next(null, contractsToTest, contractsToTestDetails, contracts)
+    },
+    function runTests (contractsToTest, contractsToTestDetails, contracts, next) {
+      let totalPassing = 0
+      let totalFailing = 0
+      let totalTime = 0
+      let errors = []
+
+      var testCallback = function (result) {
+        if (result.type === 'contract') {
+          signale.name(result.value.white)
+        } else if (result.type === 'testPass') {
+          signale.result(result.value)
+        } else if (result.type === 'testFailure') {
+          signale.result(result.value.red)
+          errors.push(result)
+        }
+      }
+      var resultsCallback = function (_err, result, cb) {
+        totalPassing += result.passingNum
+        totalFailing += result.failureNum
+        totalTime += result.timePassed
+        cb()
+      }
+
+      async.eachOfLimit(contractsToTest, 1, (contractName, index, cb) => {
+        TestRunner.runTest(contractName, contracts[contractName], contractsToTestDetails[index], { accounts }, testCallback, (err, result) => {
+          if (err) {
+            return cb(err)
+          }
+          resultsCallback(null, result, cb)
+        })
+      }, function (err, _results) {
+        if (err) {
+          return next(err)
+        }
+
+        console.log('\n')
+        if (totalPassing > 0) {
+          console.log(('  ' + totalPassing + ' passing ').green + ('(' + totalTime + 's)').grey)
+        }
+        if (totalFailing > 0) {
+          console.log(('  ' + totalFailing + ' failing').red)
+        }
+        console.log('')
+
+        errors.forEach((error, index) => {
+          console.log('  ' + (index + 1) + ') ' + error.context + ' ' + error.value)
+          console.log('')
+          console.log(('\t error: ' + error.errMsg).red)
+        })
+        console.log('')
+
+        next()
+      })
+    }
+  ], function () {
+  })
+}
+
+module.exports = runTestFiles

remix-tests/src/runTestSources.js

@@ -0,0 +1,107 @@
+const async = require('async')
+require('colors')
+
+let Compiler = require('./compiler.js')
+let Deployer = require('./deployer.js')
+let TestRunner = require('./testRunner.js')
+
+const Web3 = require('web3')
+const Provider = require('remix-simulator').Provider
+
+var createWeb3Provider = function () {
+  let web3 = new Web3()
+  web3.setProvider(new Provider())
+  return web3
+}
+
+const runTestSources = function (contractSources, testCallback, resultCallback, finalCallback, importFileCb, opts) {
+  opts = opts || {}
+  let web3 = opts.web3 || createWeb3Provider()
+  let accounts = opts.accounts || null
+  async.waterfall([
+    function getAccountList (next) {
+      if (accounts) return next()
+      web3.eth.getAccounts((_err, _accounts) => {
+        accounts = _accounts
+        next()
+      })
+    },
+    function compile (next) {
+      Compiler.compileContractSources(contractSources, importFileCb, next)
+    },
+    function deployAllContracts (compilationResult, next) {
+      Deployer.deployAll(compilationResult, web3, function (err, contracts) {
+        if (err) {
+          next(err)
+        }
+
+        next(null, compilationResult, contracts)
+      })
+    },
+    function determineTestContractsToRun (compilationResult, contracts, next) {
+      let contractsToTest = []
+      let contractsToTestDetails = []
+
+      for (let filename in compilationResult) {
+        if (filename.indexOf('_test.sol') < 0) {
+          continue
+        }
+        Object.keys(compilationResult[filename]).forEach(contractName => {
+          contractsToTestDetails.push(compilationResult[filename][contractName])
+          contractsToTest.push(contractName)
+        })
+      }
+
+      next(null, contractsToTest, contractsToTestDetails, contracts)
+    },
+    function runTests (contractsToTest, contractsToTestDetails, contracts, next) {
+      let totalPassing = 0
+      let totalFailing = 0
+      let totalTime = 0
+      let errors = []
+
+      var _testCallback = function (result) {
+        if (result.type === 'testFailure') {
+          errors.push(result)
+        }
+        testCallback(result)
+      }
+
+      var _resultsCallback = function (_err, result, cb) {
+        resultCallback(_err, result, () => {})
+        totalPassing += result.passingNum
+        totalFailing += result.failureNum
+        totalTime += result.timePassed
+        cb()
+      }
+
+      async.eachOfLimit(contractsToTest, 1, (contractName, index, cb) => {
+        TestRunner.runTest(contractName, contracts[contractName], contractsToTestDetails[index], { accounts }, _testCallback, (err, result) => {
+          if (err) {
+            return cb(err)
+          }
+          _resultsCallback(null, result, cb)
+        })
+      }, function (err, _results) {
+        if (err) {
+          return next(err)
+        }
+
+        let finalResults = {}
+
+        finalResults.totalPassing = totalPassing || 0
+        finalResults.totalFailing = totalFailing || 0
+        finalResults.totalTime = totalTime || 0
+        finalResults.errors = []
+
+        errors.forEach((error, _index) => {
+          finalResults.errors.push({context: error.context, value: error.value, message: error.errMsg})
+        })
+
+        next(null, finalResults)
+      })
+    }
+  ], finalCallback)
+}
+
+module.exports = runTestSources

remix-tests/tests/examples_3/simple_string_test.sol

@@ -12,8 +12,8 @@ contract StringTest {
     return Assert.equal(foo.get(), "Hello world!", "initial value is not correct");
   }
 
-  function valueShouldNotBeHelloWorld() public returns (bool) {
+  function valueShouldNotBeHelloWordl() public returns (bool) {
-    return Assert.notEqual(foo.get(), "Hello wordl!", "initial value is not correct");
+    return Assert.notEqual(foo.get(), "Hello wordl!", "value should not be hello world");
   }
 
   function valueShouldBeHelloWorld() public returns (bool) {

remix-tests/tests/testRunner.js

@@ -29,8 +29,8 @@ function compileAndDeploy (filename, callback) {
   ], function (_err, contracts) {
     callback(null, compilationData, contracts, accounts)
   })
-}  
+}
-  
+
 
 describe('testRunner', function () {
   describe('#runTest', function() {
@@ -47,7 +47,7 @@ describe('testRunner', function () {
             results = _results
             done()
           }
-          TestRunner.runTest('MyTest', contracts.MyTest, compilationData['simple_storage_test.sol']['MyTest'], { accounts }, testCallback, resultsCallback)
+          TestRunner.runTest('MyTest', contracts.MyTest, compilationData[filename]['MyTest'], { accounts }, testCallback, resultsCallback)
         })
       })
 
@@ -61,7 +61,7 @@ describe('testRunner', function () {
 
       it('should returns 5 messages', function () {
         assert.deepEqual(tests, [
-          { type: 'contract',    value: 'MyTest', filename: 'simple_storage_test.sol' },
+          { type: 'contract',    value: 'MyTest', filename: 'tests/examples_1/simple_storage_test.sol' },
           { type: 'testFailure', value: 'Should trigger one fail', time: 1, context: 'MyTest', errMsg: 'the test 1 fails' },
           { type: 'testPass',    value: 'Should trigger one pass', time: 1, context: 'MyTest'},
           { type: 'testPass',    value: 'Initial value should be100', time: 1, context: 'MyTest' },
@@ -83,7 +83,7 @@ describe('testRunner', function () {
             results = _results
             done()
           }
-          TestRunner.runTest('MyTest', contracts.MyTest, compilationData['simple_storage_test.sol']['MyTest'], { accounts }, testCallback, resultsCallback)
+          TestRunner.runTest('MyTest', contracts.MyTest, compilationData[filename]['MyTest'], { accounts }, testCallback, resultsCallback)
         })
       })
 
@@ -97,7 +97,7 @@ describe('testRunner', function () {
 
       it('should returns 3 messages', function () {
         assert.deepEqual(tests, [
-          { type: 'contract', value: 'MyTest', filename: 'simple_storage_test.sol' },
+          { type: 'contract', value: 'MyTest', filename: 'tests/examples_2/simple_storage_test.sol' },
           { type: 'testPass', value: 'Initial value should be100', time: 1, context: 'MyTest' },
           { type: 'testPass', value: 'Initial value should be200', time: 1, context: 'MyTest' }
         ])
@@ -118,8 +118,8 @@ describe('testRunner', function () {
             results = _results
             done()
           }
-          TestRunner.runTest('StringTest', contracts.StringTest, compilationData['simple_string_test.sol']['StringTest'], { accounts }, testCallback, resultsCallback)
+          TestRunner.runTest('StringTest', contracts.StringTest, compilationData[filename]['StringTest'], { accounts }, testCallback, resultsCallback)
-          TestRunner.runTest('StringTest2', contracts.StringTest2, compilationData['simple_string_test.sol']['StringTest2'], { accounts }, testCallback, resultsCallback)
+          TestRunner.runTest('StringTest2', contracts.StringTest2, compilationData[filename]['StringTest2'], { accounts }, testCallback, resultsCallback)
         })
       })
 
@@ -133,9 +133,9 @@ describe('testRunner', function () {
 
       it('should returns 3 messages', function () {
         assert.deepEqual(tests, [
-          { type: 'contract', value: 'StringTest', filename: 'simple_string_test.sol' },
+          { type: 'contract', value: 'StringTest', filename: 'tests/examples_3/simple_string_test.sol' },
           { type: 'testFailure', value: 'Value should be hello world', time: 1, context: 'StringTest', "errMsg": "initial value is not correct" },
-          { type: 'testPass', value: 'Value should not be hello world', time: 1, context: 'StringTest' },
+          { type: 'testPass', value: 'Value should not be hello wordl', time: 1, context: 'StringTest' },
           { type: 'testPass', value: 'Initial value should be hello', time: 1, context: 'StringTest' },
         ])
       })
@@ -155,7 +155,7 @@ describe('testRunner', function () {
             results = _results
             done()
           }
-          TestRunner.runTest('IntegerTest', contracts.IntegerTest, compilationData['number_test.sol']['IntegerTest'], { accounts }, testCallback, resultsCallback)
+          TestRunner.runTest('IntegerTest', contracts.IntegerTest, compilationData[filename]['IntegerTest'], { accounts }, testCallback, resultsCallback)
         })
       })
 
@@ -181,9 +181,9 @@ describe('testRunner', function () {
             results = _results
             done()
           }
-          
+
-          TestRunner.runTest('SenderTest', contracts.SenderTest, compilationData['sender_test.sol']['SenderTest'], { accounts }, testCallback, resultsCallback)
+          TestRunner.runTest('SenderTest', contracts.SenderTest, compilationData[filename]['SenderTest'], { accounts }, testCallback, resultsCallback)
-          
+
         })
       })