Merge pull request #741 from ethereum/remixTestsCompilerContext

custom compiler context for remix-tests CLI

libs/remix-solidity/src/compiler/compiler.ts

@@ -163,6 +163,41 @@ export class Compiler {
     }
   }
 
+  /**
+   * @dev Load compiler using given version (used by remix-tests CLI)
+   * @param version compiler version
+   */
+  
+  loadRemoteVersion (version: string): void {
+    console.log(`Loading remote solc version ${version} ...`)
+    const compiler: any = require('solc')
+    compiler.loadRemoteVersion(version, (err, remoteCompiler) => {
+      if (err) {
+        console.error('Error in loading remote solc compiler: ', err)
+      } else {
+        this.state.compileJSON = (source: SourceWithTarget) => {
+          const missingInputs: string[] = []
+          const missingInputsCallback = (path: string) => {
+            missingInputs.push(path)
+            return { error: 'Deferred import' }
+          }
+          let result: CompilationResult = {}
+          try {
+            if(source && source.sources) {
+              const {optimize, runs, evmVersion, language} = this.state
+              const input = compilerInput(source.sources, {optimize, runs, evmVersion, language})
+              result = JSON.parse(remoteCompiler.compile(input, { import: missingInputsCallback }))
+            }
+          } catch (exception) {
+            result = { error: { formattedMessage: 'Uncaught JavaScript exception:\n' + exception, severity: 'error', mode: 'panic' } }
+          }
+          this.onCompilationFinished(result, missingInputs, source)
+        }
+        this.onCompilerLoaded(version)
+      }
+    })
+  }
+
   /**
    * @dev Load compiler using given URL (used by IDE)
    * @param usingWorker if true, load compiler using worker

libs/remix-tests/package.json

@@ -37,7 +37,7 @@
   "dependencies": {
     "@remix-project/remix-lib": "^0.4.31",
     "@remix-project/remix-simulator": "^0.1.9-beta.8",
-    "@remix-project/remix-solidity": "^0.3.32",
+    "@remix-project/remix-solidity": "file:../remix-solidity",
     "ansi-gray": "^0.1.1",
     "async": "^2.6.0",
     "change-case": "^3.0.1",

libs/remix-tests/src/compiler.ts

@@ -45,15 +45,15 @@ function isRemixTestFile(path: string) {
  */
 
 function processFile(filePath: string, sources: SrcIfc, isRoot = false) {
-    const importRegEx = /import ['"](.+?)['"];/g;
-    let group: RegExpExecArray| null = null;
+    const importRegEx = /import ['"](.+?)['"];/g
+    let group: RegExpExecArray| null = null
     const isFileAlreadyInSources: boolean = Object.keys(sources).includes(filePath)
 
     // Return if file is a remix test file or already processed
     if(isRemixTestFile(filePath) || isFileAlreadyInSources)
         return
 
-    let content: string = fs.readFileSync(filePath, { encoding: 'utf-8' });
+    let content: string = fs.readFileSync(filePath, { encoding: 'utf-8' })
     const testFileImportRegEx = /^(import)\s['"](remix_tests.sol|tests.sol)['"];/gm
 
     // import 'remix_tests.sol', if file is a root test contract file and doesn't already have it 
@@ -61,13 +61,13 @@ function processFile(filePath: string, sources: SrcIfc, isRoot = false) {
         const includeTestLibs = '\nimport \'remix_tests.sol\';\n'
         content = includeTestLibs.concat(content)
     }
-    sources[filePath] = {content};
-    importRegEx.exec(''); // Resetting state of RegEx
+    sources[filePath] = {content}
+    importRegEx.exec('') // Resetting state of RegEx
 
     // Process each 'import' in file content
     while ((group = importRegEx.exec(content))) {
-        const importedFile: string = group[1];
-        const importedFilePath: string = path.join(path.dirname(filePath), importedFile);
+        const importedFile: string = group[1]
+        const importedFilePath: string = path.join(path.dirname(filePath), importedFile)
         processFile(importedFilePath, sources)
     }
 }
@@ -85,7 +85,7 @@ const isBrowser = !(typeof (window) === 'undefined' || userAgent.indexOf(' elect
  * TODO: replace this with remix's own compiler code
  */
 
-export function compileFileOrFiles(filename: string, isDirectory: boolean, opts: any, cb): void {
+export function compileFileOrFiles(filename: string, isDirectory: boolean, opts: any, compilerConfig: CompilerConfiguration, cb): void {
     let compiler: any
     const accounts: string[] = opts.accounts || []
     const sources: SrcIfc = {
@@ -103,11 +103,11 @@ export function compileFileOrFiles(filename: string, isDirectory: boolean, opts:
             }
         } else {
             // walkSync only if it is a directory
-            let testFileCount = 0;
+            let testFileCount = 0
             fs.walkSync(filepath, (foundpath: string) => {
                 // only process .sol files
                 if (foundpath.split('.').pop() === 'sol' && foundpath.endsWith('_test.sol')) {
-                    testFileCount++;
+                    testFileCount++
                     processFile(foundpath, sources, true)
                 }
             })
@@ -126,10 +126,24 @@ export function compileFileOrFiles(filename: string, isDirectory: boolean, opts:
         async.waterfall([
             function loadCompiler(next) {
                 compiler = new RemixCompiler()
-                compiler.onInternalCompilerLoaded()
-                // compiler.event.register('compilerLoaded', this, function (version) {
-                next()
-                // });
+                if(compilerConfig) {
+                    const {currentCompilerUrl, evmVersion, optimize, runs} = compilerConfig
+                    evmVersion ? compiler.set('evmVersion', evmVersion) : null
+                    optimize ? compiler.set('optimize', optimize) : null
+                    runs ? compiler.set('runs', runs) : null
+                    if(currentCompilerUrl) {
+                        compiler.loadRemoteVersion(currentCompilerUrl)
+                        compiler.event.register('compilerLoaded', this, function (version) {
+                            next()
+                        })
+                    } else {
+                        compiler.onInternalCompilerLoaded()
+                        next()
+                    }
+                } else {
+                    compiler.onInternalCompilerLoaded()
+                    next()
+                }
             },
             function doCompilation(next) {
                 // @ts-ignore

libs/remix-tests/src/run.ts

@@ -1,9 +1,11 @@
 import commander from 'commander'
-import Web3 from 'web3';
+import Web3 from 'web3'
 import path from 'path'
+import axios, { AxiosResponse } from 'axios'
 import { runTestFiles } from './runTestFiles'
 import fs from './fileSystem'
 import { Provider } from '@remix-project/remix-simulator'
+import { CompilerConfiguration } from './types'
 import Log from './logger'
 const logger = new Log()
 const log = logger.logger
@@ -21,6 +23,15 @@ function mapVerbosity (v: number) {
     }
     return levels[v]
 }
+
+function mapOptimize (v: string) {
+        const optimize = {
+                'true': true,
+                'false': false
+        }
+        return optimize[v];
+}
+
 const version = require('../package.json').version
 
 commander.version(version)
@@ -35,7 +46,11 @@ commander.command('help').description('output usage information').action(functio
 
 // get current version
 commander
-    .option('-v, --verbose <level>', 'run with verbosity', mapVerbosity)
+    .option('-c, --compiler <string>', 'set compiler version (e.g: 0.6.1, 0.7.1 etc)')
+    .option('-e, --evm <string>', 'set EVM version (e.g: petersburg, istanbul etc)')
+    .option('-o, --optimize <bool>', 'enable/disable optimization', mapOptimize)
+    .option('-r, --runs <number>', 'set runs (e.g: 150, 250 etc)')
+    .option('-v, --verbose <level>', 'set verbosity level (0 to 5)', mapVerbosity)
     .action(async (testsPath) => {
 
         // Check if path exists
@@ -62,12 +77,47 @@ commander
             log.info('verbosity level set to ' + commander.verbose.blue)
         }
 
+        let compilerConfig = {} as CompilerConfiguration
+        if (commander.compiler) {
+            const compVersion = commander.compiler
+            const baseURL = 'https://binaries.soliditylang.org/wasm/'
+            const response: AxiosResponse = await axios.get(baseURL + 'list.json')
+            const { releases, latestRelease } = response.data
+            const compString = releases[compVersion]
+            if(!compString) {
+                log.error(`No compiler found in releases with version ${compVersion}`)
+                process.exit()
+            } else {
+                compilerConfig.currentCompilerUrl = compString.replace('soljson-', '').replace('.js', '')
+                log.info(`Compiler version set to ${compVersion}. Latest version is ${latestRelease}`)
+            }
+        }
+
+        if (commander.evm) {
+            compilerConfig.evmVersion = commander.evm
+            log.info(`EVM set to ${compilerConfig.evmVersion}`)
+        }
+
+        if (commander.optimize) {
+            compilerConfig.optimize = commander.optimize
+            log.info(`Optimization is ${compilerConfig.optimize ? 'enabled' : 'disabled'}`)
+        }
+
+        if (commander.runs) {
+            if(!commander.optimize) {
+                log.error(`Optimization should be enabled for runs`)
+                process.exit()
+            }
+            compilerConfig.runs = commander.runs
+            log.info(`Runs set to ${compilerConfig.runs}`)
+        }
+
         const web3 = new Web3()
         const provider: any = new Provider()
         await provider.init()
         web3.setProvider(provider)
 
-        runTestFiles(path.resolve(testsPath), isDirectory, web3)
+        runTestFiles(path.resolve(testsPath), isDirectory, web3, compilerConfig)
     })
 
 if (!process.argv.slice(2).length) {

libs/remix-tests/src/runTestFiles.ts

@@ -1,7 +1,7 @@
 import async from 'async'
 import fs from './fileSystem'
 import { runTest } from './testRunner'
-import { TestResultInterface, ResultsInterface, compilationInterface, ASTInterface, Options, AstNode } from './types'
+import { TestResultInterface, ResultsInterface, CompilerConfiguration, compilationInterface, ASTInterface, Options, AstNode } from './types'
 import colors from 'colors'
 import Web3 from 'web3';
 
@@ -18,8 +18,9 @@ import { deployAll } from './deployer'
  */
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
-export function runTestFiles(filepath: string, isDirectory: boolean, web3: Web3, finalCallback: any = () => {}, opts?: Options) {
+export function runTestFiles(filepath: string, isDirectory: boolean, web3: Web3, compilerConfig: CompilerConfiguration, finalCallback: any = () => {}, opts?: Options) {
     opts = opts || {}
+    compilerConfig = compilerConfig || {} as CompilerConfiguration
     const sourceASTs: any = {}
     const { Signale } = require('signale')
     // signale configuration
@@ -53,7 +54,7 @@ export function runTestFiles(filepath: string, isDirectory: boolean, web3: Web3,
             })
         },
         function compile(next) {
-            compileFileOrFiles(filepath, isDirectory, { accounts }, next)
+            compileFileOrFiles(filepath, isDirectory, { accounts }, compilerConfig, next)
         },
         function deployAllContracts (compilationResult: compilationInterface, asts: ASTInterface, next) {
             // Extract AST of test contract file source

libs/remix-tests/src/types.ts

@@ -53,7 +53,7 @@ export interface CompilerConfiguration {
   currentCompilerUrl: string,
   evmVersion: string, 
   optimize: boolean, 
-  usingWorker: boolean,
+  usingWorker?: boolean,
   runs: number
 }
 

libs/remix-tests/tests/testRunner.cli.spec.ts

@@ -22,13 +22,17 @@ describe('testRunner: remix-tests CLI', () => {
         const expectedHelp = `Usage: remix-tests [options] [command]
 
 Options:
-  -V, --version          output the version number
-  -v, --verbose <level>  run with verbosity
-  -h, --help             output usage information
+  -V, --version            output the version number
+  -c, --compiler <string>  set compiler version (e.g: 0.6.1, 0.7.1 etc)
+  -e, --evm <string>       set EVM version (e.g: petersburg, istanbul etc)
+  -o, --optimize <bool>    enable/disable optimization
+  -r, --runs <number>      set runs (e.g: 150, 250 etc)
+  -v, --verbose <level>    set verbosity level (0 to 5)
+  -h, --help               output usage information
 
 Commands:
-  version                output the version number
-  help                   output usage information`
+  version                  output the version number
+  help                     output usage information`
         expect(res.stdout.toString().trim()).toBe(expectedHelp)
       })
 
@@ -41,10 +45,93 @@ Commands:
       expect(res.stdout.toString().trim()).toMatch(/AssertOkTest/)
       expect(res.stdout.toString().trim()).toMatch(/Ok pass test/)
       expect(res.stdout.toString().trim()).toMatch(/Ok fail test/)
-      // macth fail test details
+      // match fail test details
       expect(res.stdout.toString().trim()).toMatch(/error: okFailTest fails/)
       expect(res.stdout.toString().trim()).toMatch(/expected value to be ok to: true/)
       expect(res.stdout.toString().trim()).toMatch(/returned: false/)
     })
+
+    test('remix-tests running a test file with custom compiler version', () => {
+      const res = spawnSync(executablePath, ['--compiler', '0.7.4', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('Compiler version set to 0.7.4. Latest version is')).toBeTruthy()
+      expect(res.stdout.toString().trim().includes('Loading remote solc version v0.7.4+commit.3f05b770 ...')).toBeTruthy()
+      expect(res.stdout.toString().trim()).toMatch(/:: Running remix-tests - Unit testing for solidity ::/)
+      expect(res.stdout.toString().trim()).toMatch(/creation of library remix_tests.sol:Assert pending.../)
+      // match test result
+      expect(res.stdout.toString().trim()).toMatch(/Ok pass test/)
+      expect(res.stdout.toString().trim()).toMatch(/Ok fail test/)
+      // match fail test details
+      expect(res.stdout.toString().trim()).toMatch(/error: okFailTest fails/)
+    })
+
+    test('remix-tests running a test file with unavailable custom compiler version (should fail)', () => {
+      const res = spawnSync(executablePath, ['--compiler', '1.10.4', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('No compiler found in releases with version 1.10.4')).toBeTruthy()
+    })
+
+    test('remix-tests running a test file with custom EVM', () => {
+      const res = spawnSync(executablePath, ['--evm', 'petersburg', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('EVM set to petersburg')).toBeTruthy()
+      expect(res.stdout.toString().trim()).toMatch(/:: Running remix-tests - Unit testing for solidity ::/)
+      expect(res.stdout.toString().trim()).toMatch(/creation of library remix_tests.sol:Assert pending.../)
+      // match test result
+      expect(res.stdout.toString().trim()).toMatch(/Ok pass test/)
+      expect(res.stdout.toString().trim()).toMatch(/Ok fail test/)
+      // match fail test details
+      expect(res.stdout.toString().trim()).toMatch(/error: okFailTest fails/)
+    })
+
+    test('remix-tests running a test file by enabling optimization', () => {
+      const res = spawnSync(executablePath, ['--optimize', 'true', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('Optimization is enabled')).toBeTruthy()
+      expect(res.stdout.toString().trim()).toMatch(/:: Running remix-tests - Unit testing for solidity ::/)
+      expect(res.stdout.toString().trim()).toMatch(/creation of library remix_tests.sol:Assert pending.../)
+      // match test result
+      expect(res.stdout.toString().trim()).toMatch(/Ok pass test/)
+      expect(res.stdout.toString().trim()).toMatch(/Ok fail test/)
+      // match fail test details
+      expect(res.stdout.toString().trim()).toMatch(/error: okFailTest fails/)
+    })
+
+    test('remix-tests running a test file by enabling optimization and setting runs', () => {
+      const res = spawnSync(executablePath, ['--optimize', 'true', '--runs', '300', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('Optimization is enabled')).toBeTruthy()
+      expect(res.stdout.toString().trim().includes('Runs set to 300')).toBeTruthy()
+      expect(res.stdout.toString().trim()).toMatch(/:: Running remix-tests - Unit testing for solidity ::/)
+      expect(res.stdout.toString().trim()).toMatch(/creation of library remix_tests.sol:Assert pending.../)
+      // match test result
+      expect(res.stdout.toString().trim()).toMatch(/Ok pass test/)
+      expect(res.stdout.toString().trim()).toMatch(/Ok fail test/)
+      // match fail test details
+      expect(res.stdout.toString().trim()).toMatch(/error: okFailTest fails/)
+    })
+
+    test('remix-tests running a test file without enabling optimization and setting runs (should fail)', () => {
+      const res = spawnSync(executablePath, ['--runs', '300', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('Optimization should be enabled for runs')).toBeTruthy()
+    })
+
+    test('remix-tests running a test file with all options', () => {
+      const res = spawnSync(executablePath, ['--compiler', '0.7.5', '--evm', 'istanbul', '--optimize', 'true', '--runs', '250', resolve(__dirname + '/examples_0/assert_ok_test.sol')])
+      // match initial lines
+      expect(res.stdout.toString().trim().includes('Compiler version set to 0.7.5. Latest version is')).toBeTruthy()
+      expect(res.stdout.toString().trim().includes('Loading remote solc version v0.7.5+commit.eb77ed08 ...')).toBeTruthy()
+      expect(res.stdout.toString().trim().includes('EVM set to istanbul')).toBeTruthy()
+      expect(res.stdout.toString().trim().includes('Optimization is enabled')).toBeTruthy()
+      expect(res.stdout.toString().trim().includes('Runs set to 250')).toBeTruthy()
+      expect(res.stdout.toString().trim()).toMatch(/:: Running remix-tests - Unit testing for solidity ::/)
+      expect(res.stdout.toString().trim()).toMatch(/creation of library remix_tests.sol:Assert pending.../)
+      // match test result
+      expect(res.stdout.toString().trim()).toMatch(/Ok pass test/)
+      expect(res.stdout.toString().trim()).toMatch(/Ok fail test/)
+      // match fail test details
+      expect(res.stdout.toString().trim()).toMatch(/error: okFailTest fails/)
+    })
   })
 })

libs/remix-tests/tests/testRunner.spec.ts

@@ -57,7 +57,7 @@ async function compileAndDeploy(filename: string, callback: Function) {
       })
     },
     function compile(next: Function): void {
-      compileFileOrFiles(filename, false, { accounts }, next)
+      compileFileOrFiles(filename, false, { accounts }, null, next)
     },
     function deployAllContracts(compilationResult: compilationInterface, asts, next: Function): void {
       for(const filename in asts) {