From 00885a6323dd32591c9f5a1c03b8917c34db4b7f Mon Sep 17 00:00:00 2001 From: yann300 Date: Wed, 25 Nov 2020 11:20:53 +0100 Subject: [PATCH 1/3] extends contentImport with resolveAndSave --- apps/remix-ide-e2e/src/tests/terminal.test.ts | 45 ++++++++ apps/remix-ide/contracts/contract2.sol | 2 +- apps/remix-ide/src/app.js | 11 +- .../src/app/compiler/compiler-imports.js | 100 +++++++++++++++++- apps/remix-ide/src/app/files/fileManager.js | 2 - apps/remix-ide/src/app/tabs/compile-tab.js | 5 +- .../src/app/tabs/compileTab/compileTab.js | 79 +------------- apps/remix-ide/src/app/tabs/test-tab.js | 7 +- 8 files changed, 161 insertions(+), 90 deletions(-) diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 3099ed89e7..1c8e24c9f0 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -80,9 +80,32 @@ module.exports = { .journalLastChildIncludes(`[ "`) // we check if an array is present, don't need to check for the content .journalLastChildIncludes('" ]') .journalLastChildIncludes('", "') + }, + + 'Call Remix File Resolver (external URL) from a script': function (browser: NightwatchBrowser) { + browser + .click('*[data-id="terminalClearConsole"]') // clear the terminal + .addFile('resolveExternalUrlAndSave.js', { content: resolveExternalUrlAndSave }) + .openFile('browser/resolveExternalUrlAndSave.js') + .pause(1000) + .executeScript(`remix.execute('browser/resolveExternalUrlAndSave.js')`) + .pause(6000) + .journalLastChildIncludes('Implementation of the {IERC20} interface.') + }, + + 'Call Remix File Resolver (internal URL) from a script': function (browser: NightwatchBrowser) { + browser + .click('*[data-id="terminalClearConsole"]') // clear the terminal + .addFile('resolveUrl.js', { content: resolveUrl }) + .openFile('browser/resolveUrl.js') + .pause(1000) + .executeScript(`remix.execute('browser/resolveUrl.js')`) + .pause(6000) + .journalLastChildIncludes('contract Ballot {') .end() }, + tearDown: sauce } @@ -122,3 +145,25 @@ const asyncAwaitWithFileManagerAccess = ` run() ` + +const resolveExternalUrlAndSave = ` +(async () => { + try { + console.log('start') + console.log(await remix.call('contentImport', 'resolveAndSave', 'https://github.com/OpenZeppelin/openzeppelin-contracts/blob/master/contracts/token/ERC20/ERC20.sol')) + } catch (e) { + console.log(e.message) + } +})() +` + +const resolveUrl = ` +(async () => { + try { + console.log('start') + console.log(await remix.call('contentImport', 'resolveAndSave', 'browser/3_Ballot.sol')) + } catch (e) { + console.log(e.message) + } +})() +` diff --git a/apps/remix-ide/contracts/contract2.sol b/apps/remix-ide/contracts/contract2.sol index 96b59660ba..00649b1047 100644 --- a/apps/remix-ide/contracts/contract2.sol +++ b/apps/remix-ide/contracts/contract2.sol @@ -1 +1 @@ -contract test2 { function get () returns (uint) { return 9; }} \ No newline at end of file +import "openzeppelin-solidity/contracts/math/SafeMath.sol"; \ No newline at end of file diff --git a/apps/remix-ide/src/app.js b/apps/remix-ide/src/app.js index bada05e33c..b759906d21 100644 --- a/apps/remix-ide/src/app.js +++ b/apps/remix-ide/src/app.js @@ -235,8 +235,6 @@ Please make a backup of your contracts and start using http://remix.ethereum.org engine.register(appManager) // SERVICES - // ----------------- import content servive ------------------------ - const contentImport = new CompilerImport() // ----------------- theme servive --------------------------------- const themeModule = new ThemeModule(registry) registry.put({ api: themeModule, name: 'themeModule' }) @@ -255,6 +253,9 @@ Please make a backup of your contracts and start using http://remix.ethereum.org const fileManager = new FileManager(editor, appManager) registry.put({ api: fileManager, name: 'filemanager' }) + // ----------------- import content servive ------------------------ + const contentImport = new CompilerImport(fileManager) + const blockchain = new Blockchain(registry.get('config').api) const pluginUdapp = new PluginUDapp(blockchain) @@ -350,7 +351,8 @@ Please make a backup of your contracts and start using http://remix.ethereum.org registry.get('config').api, new Renderer(), registry.get('fileproviders/browser').api, - registry.get('filemanager').api + registry.get('filemanager').api, + contentImport ) const run = new RunTab( blockchain, @@ -375,7 +377,8 @@ Please make a backup of your contracts and start using http://remix.ethereum.org filePanel, compileTab, appManager, - new Renderer() + new Renderer(), + contentImport ) engine.register([ diff --git a/apps/remix-ide/src/app/compiler/compiler-imports.js b/apps/remix-ide/src/app/compiler/compiler-imports.js index 219131f67c..294e9b2be9 100644 --- a/apps/remix-ide/src/app/compiler/compiler-imports.js +++ b/apps/remix-ide/src/app/compiler/compiler-imports.js @@ -1,7 +1,10 @@ 'use strict' import { Plugin } from '@remixproject/engine' import * as packageJson from '../../../../../package.json' +const remixTests = require('@remix-project/remix-tests') const globalRegistry = require('../../global/registry') +const addTooltip = require('../ui/tooltip') +const async = require('async') var base64 = require('js-base64').Base64 var swarmgw = require('swarmgw')() var resolver = require('@resolver-engine/imports').ImportsEngine() @@ -11,12 +14,13 @@ const profile = { name: 'contentImport', displayName: 'content import', version: packageJson.version, - methods: ['resolve'] + methods: ['resolve', 'resolveAndSave'] } module.exports = class CompilerImports extends Plugin { - constructor () { + constructor (fileManager) { super(profile) + this.fileManager = fileManager this.previouslyHandled = {} // cache import so we don't make the request at each compilation. } @@ -101,6 +105,12 @@ module.exports = class CompilerImports extends Plugin { return /^([^/]+)/.exec(url) } + /** + * resolve the content of @arg url. This only resolves external URLs. + * + * @param {String} url - external URL of the content. can be basically anything like raw HTTP, ipfs URL, github address etc... + * @returns {Promise} - { content, cleanUrl, type, url } + */ resolve (url) { return new Promise((resolve, reject) => { this.import(url, null, (error, content, cleanUrl, type, url) => { @@ -171,4 +181,90 @@ module.exports = class CompilerImports extends Plugin { cb('Unable to import "' + url + '": File not found') }) } + + importExternal (url, cb) { + this.import(url, + // TODO: move to an event that is generated, the UI shouldn't be here + (loadingMsg) => { addTooltip(loadingMsg) }, + (error, content, cleanUrl, type, url) => { + if (error) return cb(error) + if (this.fileManager) { + const browser = this.fileManager.fileProviderOf('browser/') + if (browser) browser.addExternal(type + '/' + cleanUrl, content, url) + } + cb(null, content) + }) + } + + /** + * import the content of @arg url. + * first look in the browser localstorage (browser explorer) or locahost explorer. if the url start with `browser/*` or `localhost/*` + * then check if the @arg url is located in the localhost, in the node_modules or installed_contracts folder + * then check if the @arg url match any external url + * + * @param {String} url - URL of the content. can be basically anything like file located in the browser explorer, in the localhost explorer, raw HTTP, github address etc... + * @returns {Promise} - string content + */ + resolveAndSave (url) { + return new Promise((resolve, reject) => { + if (url.indexOf('remix_tests.sol') !== -1) resolve(remixTests.assertLibCode) + if (!this.fileManager) { + // fallback to just resolving the file, it won't be saved in file manager + return this.importExternal(url, (error, content) => { + if (error) return reject(error) + resolve(content) + }) + } + var provider = this.fileManager.fileProviderOf(url) + if (provider) { + if (provider.type === 'localhost' && !provider.isConnected()) { + return reject(new Error(`file provider ${provider.type} not available while trying to resolve ${url}`)) + } + provider.exists(url, (error, exist) => { + if (error) return reject(error) + if (!exist && provider.type === 'localhost') return reject(new Error(`not found ${url}`)) + + /* + if the path is absolute and the file does not exist, we can stop here + Doesn't make sense to try to resolve "localhost/node_modules/localhost/node_modules/" and we'll end in an infinite loop. + */ + if (!exist && url.startsWith('browser/')) return reject(new Error(`not found ${url}`)) + if (!exist && url.startsWith('localhost/')) return reject(new Error(`not found ${url}`)) + + if (exist) { + return provider.get(url, (error, content) => { + if (error) return reject(error) + resolve(content) + }) + } + + // try to resolve localhost modules (aka truffle imports) - e.g from the node_modules folder + const localhostProvider = this.fileManager.getProvider('localhost') + if (localhostProvider.isConnected()) { + var splitted = /([^/]+)\/(.*)$/g.exec(url) + return async.tryEach([ + (cb) => { this.resolveAndSave('localhost/installed_contracts/' + url).then((result) => cb(null, result)).catch((error) => cb(error.message)) }, + (cb) => { if (!splitted) { cb('URL not parseable: ' + url) } else { this.resolveAndSave('localhost/installed_contracts/' + splitted[1] + '/contracts/' + splitted[2]).then((result) => cb(null, result)).catch((error) => cb(error.message)) } }, + (cb) => { this.resolveAndSave('localhost/node_modules/' + url).then((result) => cb(null, result)).catch((error) => cb(error.message)) }, + (cb) => { if (!splitted) { cb('URL not parseable: ' + url) } else { this.resolveAndSave('localhost/node_modules/' + splitted[1] + '/contracts/' + splitted[2]).then((result) => cb(null, result)).catch((error) => cb(error.message)) } }], + (error, result) => { + if (error) { + return this.importExternal(url, (error, content) => { + if (error) return reject(error) + resolve(content) + }) + } + resolve(result) + }) + } else { + // try to resolve external content + this.importExternal(url, (error, content) => { + if (error) return reject(error) + resolve(content) + }) + } + }) + } + }) + } } diff --git a/apps/remix-ide/src/app/files/fileManager.js b/apps/remix-ide/src/app/files/fileManager.js index 985ff3ad81..eef16bdb51 100644 --- a/apps/remix-ide/src/app/files/fileManager.js +++ b/apps/remix-ide/src/app/files/fileManager.js @@ -6,7 +6,6 @@ import { Plugin } from '@remixproject/engine' import * as packageJson from '../../../../../package.json' const EventEmitter = require('events') const globalRegistry = require('../../global/registry') -const CompilerImport = require('../compiler/compiler-imports') const toaster = require('../ui/tooltip') const modalDialogCustom = require('../ui/modal-dialog-custom') const helper = require('../../lib/helper.js') @@ -44,7 +43,6 @@ class FileManager extends Plugin { this.events = new EventEmitter() this.editor = editor this._components = {} - this._components.compilerImport = new CompilerImport() this._components.registry = globalRegistry this.appManager = appManager this.init() diff --git a/apps/remix-ide/src/app/tabs/compile-tab.js b/apps/remix-ide/src/app/tabs/compile-tab.js index ee1e20a6f4..11de41e49e 100644 --- a/apps/remix-ide/src/app/tabs/compile-tab.js +++ b/apps/remix-ide/src/app/tabs/compile-tab.js @@ -41,7 +41,7 @@ const profile = { // - methods: ['getCompilationResult'] class CompileTab extends ViewPlugin { - constructor (editor, config, renderer, fileProvider, fileManager) { + constructor (editor, config, renderer, fileProvider, fileManager, contentImport) { super(profile) this.events = new EventEmitter() this._view = { @@ -50,6 +50,7 @@ class CompileTab extends ViewPlugin { errorContainer: null, contractEl: null } + this.contentImport = contentImport this.queryParams = new QueryParams() this.fileProvider = fileProvider // dependencies @@ -66,7 +67,7 @@ class CompileTab extends ViewPlugin { } onActivationInternal () { - this.compileTabLogic = new CompileTabLogic(this.queryParams, this.fileManager, this.editor, this.config, this.fileProvider) + this.compileTabLogic = new CompileTabLogic(this.queryParams, this.fileManager, this.editor, this.config, this.fileProvider, this.contentImport) this.compiler = this.compileTabLogic.compiler this.compileTabLogic.init() diff --git a/apps/remix-ide/src/app/tabs/compileTab/compileTab.js b/apps/remix-ide/src/app/tabs/compileTab/compileTab.js index a0c56fc01f..bc33d301c7 100644 --- a/apps/remix-ide/src/app/tabs/compileTab/compileTab.js +++ b/apps/remix-ide/src/app/tabs/compileTab/compileTab.js @@ -1,18 +1,12 @@ -const async = require('async') const EventEmitter = require('events') -var remixTests = require('@remix-project/remix-tests') var Compiler = require('@remix-project/remix-solidity').Compiler -var CompilerImport = require('../../compiler/compiler-imports') - -// TODO: move this to the UI -const addTooltip = require('../../ui/tooltip') class CompileTab { - constructor (queryParams, fileManager, editor, config, fileProvider) { + constructor (queryParams, fileManager, editor, config, fileProvider, contentImport) { this.event = new EventEmitter() this.queryParams = queryParams - this.compilerImport = new CompilerImport() - this.compiler = new Compiler((url, cb) => this.importFileCb(url, cb)) + this.compilerImport = contentImport + this.compiler = new Compiler((url, cb) => this.compilerImport.resolveAndSave(url).then((result) => cb(null, result)).catch((error) => cb(error.message))) this.fileManager = fileManager this.editor = editor this.config = config @@ -93,73 +87,6 @@ class CompileTab { console.error(err) } } - - importExternal (url, cb) { - this.compilerImport.import(url, - - // TODO: move to an event that is generated, the UI shouldn't be here - (loadingMsg) => { addTooltip(loadingMsg) }, - (error, content, cleanUrl, type, url) => { - if (error) return cb(error) - - if (this.fileProvider) { - this.fileProvider.addExternal(type + '/' + cleanUrl, content, url) - } - cb(null, content) - }) - } - - /** - * import the content of @arg url. - * first look in the browser localstorage (browser explorer) or locahost explorer. if the url start with `browser/*` or `localhost/*` - * then check if the @arg url is located in the localhost, in the node_modules or installed_contracts folder - * then check if the @arg url match any external url - * - * @param {String} url - URL of the content. can be basically anything like file located in the browser explorer, in the localhost explorer, raw HTTP, github address etc... - * @param {Function} cb - callback - */ - importFileCb (url, filecb) { - if (url.indexOf('remix_tests.sol') !== -1) return filecb(null, remixTests.assertLibCode) - - var provider = this.fileManager.fileProviderOf(url) - if (provider) { - if (provider.type === 'localhost' && !provider.isConnected()) { - return filecb(`file provider ${provider.type} not available while trying to resolve ${url}`) - } - provider.exists(url, (error, exist) => { - if (error) return filecb(error) - if (!exist && provider.type === 'localhost') return filecb(`not found ${url}`) - - /* - if the path is absolute and the file does not exist, we can stop here - Doesn't make sense to try to resolve "localhost/node_modules/localhost/node_modules/" and we'll end in an infinite loop. - */ - if (!exist && url.startsWith('browser/')) return filecb(`not found ${url}`) - if (!exist && url.startsWith('localhost/')) return filecb(`not found ${url}`) - - if (exist) return provider.get(url, filecb) - - // try to resolve localhost modules (aka truffle imports) - e.g from the node_modules folder - const localhostProvider = this.fileManager.getProvider('localhost') - if (localhostProvider.isConnected()) { - var splitted = /([^/]+)\/(.*)$/g.exec(url) - return async.tryEach([ - (cb) => { this.importFileCb('localhost/installed_contracts/' + url, cb) }, - (cb) => { if (!splitted) { cb('URL not parseable: ' + url) } else { this.importFileCb('localhost/installed_contracts/' + splitted[1] + '/contracts/' + splitted[2], cb) } }, - (cb) => { this.importFileCb('localhost/node_modules/' + url, cb) }, - (cb) => { if (!splitted) { cb('URL not parseable: ' + url) } else { this.importFileCb('localhost/node_modules/' + splitted[1] + '/contracts/' + splitted[2], cb) } }], - (error, result) => { - if (error) return this.importExternal(url, filecb) - filecb(null, result) - } - ) - } else { - // try to resolve external content - this.importExternal(url, filecb) - } - }) - } - } } module.exports = CompileTab diff --git a/apps/remix-ide/src/app/tabs/test-tab.js b/apps/remix-ide/src/app/tabs/test-tab.js index d1eb0b8e5e..807e7b5c34 100644 --- a/apps/remix-ide/src/app/tabs/test-tab.js +++ b/apps/remix-ide/src/app/tabs/test-tab.js @@ -20,9 +20,10 @@ const profile = { } module.exports = class TestTab extends ViewPlugin { - constructor (fileManager, offsetToLineColumnConverter, filePanel, compileTab, appManager, renderer) { + constructor (fileManager, offsetToLineColumnConverter, filePanel, compileTab, appManager, renderer, contentImport) { super(profile) this.compileTab = compileTab + this.contentImport = contentImport this._view = { el: null } this.fileManager = fileManager this.filePanel = filePanel @@ -373,7 +374,7 @@ module.exports = class TestTab extends ViewPlugin { if (error) return reject(error) resolve(result) }, (url, cb) => { - return this.compileTab.compileTabLogic.importFileCb(url, cb) + return this.contentImport.resolveAndSave(url).then((result) => cb(null, result)).catch((error) => cb(error.message)) }) }) } @@ -405,7 +406,7 @@ module.exports = class TestTab extends ViewPlugin { this.updateFinalResult(error, result, testFilePath) callback(error) }, (url, cb) => { - return this.compileTab.compileTabLogic.importFileCb(url, cb) + return this.contentImport.resolveAndSave(url).then((result) => cb(null, result)).catch((error) => cb(error.message)) } ) }).catch((error) => { From 65dfed3e42551a244752843d546325bb111466f9 Mon Sep 17 00:00:00 2001 From: yann300 Date: Mon, 30 Nov 2020 11:28:54 +0100 Subject: [PATCH 2/3] allow to save content to a specific path --- .../src/app/compiler/compiler-imports.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/remix-ide/src/app/compiler/compiler-imports.js b/apps/remix-ide/src/app/compiler/compiler-imports.js index 294e9b2be9..9ce5fc57b7 100644 --- a/apps/remix-ide/src/app/compiler/compiler-imports.js +++ b/apps/remix-ide/src/app/compiler/compiler-imports.js @@ -182,7 +182,7 @@ module.exports = class CompilerImports extends Plugin { }) } - importExternal (url, cb) { + importExternal (url, targetPath, cb) { this.import(url, // TODO: move to an event that is generated, the UI shouldn't be here (loadingMsg) => { addTooltip(loadingMsg) }, @@ -190,7 +190,8 @@ module.exports = class CompilerImports extends Plugin { if (error) return cb(error) if (this.fileManager) { const browser = this.fileManager.fileProviderOf('browser/') - if (browser) browser.addExternal(type + '/' + cleanUrl, content, url) + const path = targetPath || type + '/' + cleanUrl + if (browser) browser.addExternal(path, content, url) } cb(null, content) }) @@ -202,15 +203,16 @@ module.exports = class CompilerImports extends Plugin { * then check if the @arg url is located in the localhost, in the node_modules or installed_contracts folder * then check if the @arg url match any external url * - * @param {String} url - URL of the content. can be basically anything like file located in the browser explorer, in the localhost explorer, raw HTTP, github address etc... + * @param {String} url - URL of the content. can be basically anything like file located in the browser explorer, in the localhost explorer, raw HTTP, github address etc... + * @param {String} targetPath - (optional) internal path where the content should be saved to * @returns {Promise} - string content */ - resolveAndSave (url) { + resolveAndSave (url, targetPath) { return new Promise((resolve, reject) => { if (url.indexOf('remix_tests.sol') !== -1) resolve(remixTests.assertLibCode) if (!this.fileManager) { // fallback to just resolving the file, it won't be saved in file manager - return this.importExternal(url, (error, content) => { + return this.importExternal(url, targetPath, (error, content) => { if (error) return reject(error) resolve(content) }) @@ -249,7 +251,7 @@ module.exports = class CompilerImports extends Plugin { (cb) => { if (!splitted) { cb('URL not parseable: ' + url) } else { this.resolveAndSave('localhost/node_modules/' + splitted[1] + '/contracts/' + splitted[2]).then((result) => cb(null, result)).catch((error) => cb(error.message)) } }], (error, result) => { if (error) { - return this.importExternal(url, (error, content) => { + return this.importExternal(url, targetPath, (error, content) => { if (error) return reject(error) resolve(content) }) @@ -258,7 +260,7 @@ module.exports = class CompilerImports extends Plugin { }) } else { // try to resolve external content - this.importExternal(url, (error, content) => { + this.importExternal(url, targetPath, (error, content) => { if (error) return reject(error) resolve(content) }) From 90caf167817a47805404f10e41878f6efb19453b Mon Sep 17 00:00:00 2001 From: yann300 Date: Mon, 30 Nov 2020 11:37:11 +0100 Subject: [PATCH 3/3] add e2e tests --- apps/remix-ide-e2e/src/tests/terminal.test.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 1c8e24c9f0..862794d89b 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -91,6 +91,7 @@ module.exports = { .executeScript(`remix.execute('browser/resolveExternalUrlAndSave.js')`) .pause(6000) .journalLastChildIncludes('Implementation of the {IERC20} interface.') + .openFile('browser/github/OpenZeppelin/openzeppelin-contracts/contracts/token/ERC20/ERC20.sol') }, 'Call Remix File Resolver (internal URL) from a script': function (browser: NightwatchBrowser) { @@ -101,7 +102,19 @@ module.exports = { .pause(1000) .executeScript(`remix.execute('browser/resolveUrl.js')`) .pause(6000) - .journalLastChildIncludes('contract Ballot {') + .journalLastChildIncludes('contract Ballot {') + }, + + 'Call Remix File Resolver (internal URL) from a script and specify a path': function (browser: NightwatchBrowser) { + browser + .click('*[data-id="terminalClearConsole"]') // clear the terminal + .addFile('resolveExternalUrlAndSaveToaPath.js', { content: resolveExternalUrlAndSaveToaPath }) + .openFile('browser/resolveExternalUrlAndSaveToaPath.js') + .pause(1000) + .executeScript(`remix.execute('browser/resolveExternalUrlAndSaveToaPath.js')`) + .pause(6000) + .journalLastChildIncludes('abstract contract ERC20Burnable') + .openFile('browser/github/newFile.sol') .end() }, @@ -157,6 +170,17 @@ const resolveExternalUrlAndSave = ` })() ` +const resolveExternalUrlAndSaveToaPath = ` +(async () => { + try { + console.log('start') + console.log(await remix.call('contentImport', 'resolveAndSave', 'https://github.com/OpenZeppelin/openzeppelin-contracts/blob/master/contracts/token/ERC20/ERC20Burnable.sol', 'github/newFile.sol')) + } catch (e) { + console.log(e.message) + } +})() +` + const resolveUrl = ` (async () => { try {