From 049a10353f4afa1ccc7d64230a365267e2376560 Mon Sep 17 00:00:00 2001 From: filip mertens Date: Mon, 1 May 2023 12:21:16 +0200 Subject: [PATCH] grep script md5sum fix lint restore cache use file use caching fix config config fix grep fix lint nightly catch webpack url curl check curl use plus install jq use npx do not use jq rename cache filter duplicates manual versions fix manual v3 8.18 fix urls 7.6 fix regex fix latest remixd rename --- .circleci/config.yml | 15 ++- apps/remix-ide/.gitignore | 2 +- apps/remix-ide/ci/download_e2e_assets.js | 137 ++++++++++++----------- apps/remix-ide/ci/downloadsoljson.sh | 32 ++++++ apps/remix-ide/webpack.config.js | 22 +--- 5 files changed, 116 insertions(+), 92 deletions(-) create mode 100644 apps/remix-ide/ci/downloadsoljson.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d29fec3eec..78ee7b3a95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,17 +24,26 @@ jobs: key: v1-deps-{{ checksum "yarn.lock" }} paths: - node_modules - - run: name: Build command: | if [ "${CIRCLE_BRANCH}" == "master" ]; then - NX_BIN_URL=http://127.0.0.1:8080/assets/js NX_WASM_URL=http://127.0.0.1:8080/assets/js NPM_URL=http://localhost:9090/ yarn build:production + NX_BIN_URL=http://127.0.0.1:8080/assets/js/soljson NX_WASM_URL=http://127.0.0.1:8080/assets/js/soljson NPM_URL=http://localhost:9090/ yarn build:production else - NX_BIN_URL=http://127.0.0.1:8080/assets/js NX_WASM_URL=http://127.0.0.1:8080/assets/js NPM_URL=http://localhost:9090/ yarn build + NX_BIN_URL=http://127.0.0.1:8080/assets/js/soljson NX_WASM_URL=http://127.0.0.1:8080/assets/js/soljson NPM_URL=http://localhost:9090/ yarn build fi - run: yarn run build:e2e + + - run: grep -ir "[0-9]+commit" apps/* libs/* --include \*.ts > soljson-versions.txt + - restore_cache: + keys: + - soljson-v5-{{ checksum "soljson-versions.txt" }} - run: yarn run downloadsolc_assets_e2e + - save_cache: + key: soljson-v5-{{ checksum "soljson-versions.txt" }} + paths: + - dist/apps/remix-ide/assets/js/soljson + - run: mkdir persist && zip -0 -r persist/dist.zip dist - persist_to_workspace: root: . diff --git a/apps/remix-ide/.gitignore b/apps/remix-ide/.gitignore index cf8c2c0ad3..3e0c4b3136 100644 --- a/apps/remix-ide/.gitignore +++ b/apps/remix-ide/.gitignore @@ -13,4 +13,4 @@ TODO .tern-port temp_publish_docker src/assets/version.json -src/assets/js/soljson-v* \ No newline at end of file +src/assets/js/soljson/soljson-v* \ No newline at end of file diff --git a/apps/remix-ide/ci/download_e2e_assets.js b/apps/remix-ide/ci/download_e2e_assets.js index 392941536a..e8188428e1 100644 --- a/apps/remix-ide/ci/download_e2e_assets.js +++ b/apps/remix-ide/ci/download_e2e_assets.js @@ -1,80 +1,81 @@ -const testFolder = './apps/remix-ide-e2e/src/tests/'; const fs = require('fs'); +var child_process = require('child_process'); +const { exit } = require('process'); -let url = 'https://binaries.soliditylang.org/wasm/list.json' - -const axios = require('axios') - -// use axios to download the file -/* -axios({ - url: url, - method: 'GET', -}).then((response) => { - - let info = response.data; - info.builds = info.builds.filter(build => build.path.indexOf('nightly') === -1) - for (let build of info.builds) { - - const buildurl = `https://solc-bin.ethereum.org/wasm/${build.path}`; - console.log(buildurl) - - const path = `./dist/apps/remix-ide/assets/js/${build.path}`; - // use axios to get the file - try { - axios({ - method: 'get', - url: buildurl, - }).then(function (response) { - fs.writeFile(path, response.data, function (err) { - if (err) { - console.log(err); - } - }) - }) - } catch (e) { - console.log('Failed to download ' + build.path + ' from ' + buildurl) - } +var child = child_process.spawnSync('grep', ['-ir', '[0-9]+commit', 'libs/**/*', 'apps/**/*', '--include', '*.ts'], { encoding: 'utf8', cwd: process.cwd(), shell: true }); - } +if (child.error) { + console.log("ERROR: ", child); + exit(1); } -) -*/ - -fs.readdirSync(testFolder).forEach(file => { - let c = fs.readFileSync(testFolder + file, 'utf8'); - const re = /(?<=soljson).*(?=(.js))/g; - const soljson = c.match(re); - if (soljson) { - console.log(soljson) - for (let i = 0; i < soljson.length; i++) { - - const version = soljson[i]; - if (version) { - const url = `https://solc-bin.ethereum.org/bin/soljson${version}.js`; - console.log(url) - - const path = `./dist/apps/remix-ide/assets/js/soljson${version}.js`; - // use axios to get the file + +const nonNightlyRegex = /v\d*\.\d*\.\d*\+commit\.[\d\w]*/g; + +let soljson = child.stdout.match(nonNightlyRegex); +console.log('non nightly soljson versions found: ', soljson); + +const quotedVersionsRegex = /'\d*\.\d*\.\d*\+commit\.[\d\w]*/g; +let soljson2 = child.stdout.match(quotedVersionsRegex).map((item) => item.replace('\'', 'v')); +console.log('quoted soljson versions found: ', soljson2); + +const nightlyVersionsRegex = /\d*\.\d*\.\d-nightly.*\+commit\.[\d\w]*/g +let soljson3 = child.stdout.match(nightlyVersionsRegex).map((item) => 'v' + item); +console.log('nightly soljson versions found: ', soljson3); + +// merge the three arrays +soljson = soljson.concat(soljson2); +soljson = soljson.concat(soljson3); + +console.log('soljson versions found: ', soljson); + + + +if (soljson) { + // filter out duplicates + soljson = soljson.filter((item, index) => soljson.indexOf(item) === index); + + + // manually add some versions + + soljson.push('v0.7.6+commit.7338295f'); + soljson.push('v0.5.17+commit.d19bba13'); + + console.log('soljson versions found: ', soljson); + + for (let i = 0; i < soljson.length; i++) { + const version = soljson[i]; + if (version) { + let url = '' + + // if nightly + if (version.includes('nightly')) { + url = `https://binaries.soliditylang.org/bin/soljson-${version}.js`; + }else{ + url = `https://binaries.soliditylang.org/wasm/soljson-${version}.js`; + } + + const dir = './dist/apps/remix-ide/assets/js/soljson'; + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir); + } + + const path = `./dist/apps/remix-ide/assets/js/soljson/soljson-${version}.js`; + // check if the file exists + const exists = fs.existsSync(path); + if (!exists) { + console.log('URL:', url) try { - axios({ - method: 'get', - url: url, - }).then(function (response) { - fs.writeFile(path, response.data, function (err) { - if (err) { - console.log(err); - } - }) - }) + // use curl to download the file + child_process.exec(`curl -o ${path} ${url}`, { encoding: 'utf8', cwd: process.cwd(), shell: true }) } catch (e) { console.log('Failed to download soljson' + version + ' from ' + url) } - - } + } - } + + } + +} -}); diff --git a/apps/remix-ide/ci/downloadsoljson.sh b/apps/remix-ide/ci/downloadsoljson.sh new file mode 100644 index 0000000000..5fdbfddf39 --- /dev/null +++ b/apps/remix-ide/ci/downloadsoljson.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e +# check if curl is installed +if ! command -v curl &> /dev/null +then + echo "curl could not be found" + exit +fi + + +# download https://binaries.soliditylang.org/wasm/list.json as json +curl https://binaries.soliditylang.org/wasm/list.json > list.json +# get the latest version without jq +latest=$(grep 'latestRelease' list.json | cut -d '"' -f 4) +echo "latest version: $latest" +# get url +url=$(grep "\"$latest\":" list.json | cut -d '"' -f 4) +echo "url: $url" +path="https://binaries.soliditylang.org/bin/$url" +echo "path: $path" +# download the file to ./apps/remix-ide/src/assets/js/soljson.js +curl $path > ./apps/remix-ide/src/assets/js/soljson.js +# if directory ./apps/remix-ide/src/assets/js/soljson does not exist, create it +if [ ! -d "./apps/remix-ide/src/assets/js/soljson" ]; then + mkdir ./apps/remix-ide/src/assets/js/soljson +fi +cp ./apps/remix-ide/src/assets/js/soljson.js ./apps/remix-ide/src/assets/js/soljson/$url + +# remove list.json +rm list.json + diff --git a/apps/remix-ide/webpack.config.js b/apps/remix-ide/webpack.config.js index a10a6a3e47..3191b1d886 100644 --- a/apps/remix-ide/webpack.config.js +++ b/apps/remix-ide/webpack.config.js @@ -15,26 +15,8 @@ const versionData = { } const loadLocalSolJson = async () => { - let url = 'https://binaries.soliditylang.org/wasm/list.json' - axios({ - url: url, - method: 'GET', - }).then((response) => { - let info = response.data; - info.builds = info.builds.filter(build => build.path.indexOf('nightly') === -1) - info.builds = info.builds.slice(-1) - const buildurl = `https://solc-bin.ethereum.org/wasm/${info.builds[0].path}`; - console.log(`Copying... ${buildurl} to assets`) - const path = `./apps/remix-ide/src/assets/js/soljson.js`; - axios({ - method: 'get', - url: buildurl, - responseType: 'stream' - }).then(function (response) { - response.data.pipe(fs.createWriteStream(path)); - }) - } - ) + // execute apps/remix-ide/ci/downloadsoljson.sh + const child = require('child_process').execSync('bash ./apps/remix-ide/ci/downloadsoljson.sh', { encoding: 'utf8', cwd: process.cwd(), shell: true }) } fs.writeFileSync('./apps/remix-ide/src/assets/version.json', JSON.stringify(versionData))