Merge pull request #3491 from ethereum/optimizeci

optimize CI

.circleci/config.yml

@@ -24,21 +24,55 @@ jobs:
           key: v1-deps-{{ checksum "yarn.lock" }}
           paths:
             - node_modules
-      - run: NX_BIN_URL=http://127.0.0.1:8080/assets/js NX_WASM_URL=http://127.0.0.1:8080/assets/js NPM_URL=http://localhost:9090/ yarn build:production
-      - run: yarn nx build remix-ide-e2e-src-local-plugin & yarn run build:libs
-      - run: yarn nx run remixd:build
+      - run:
+          name: Build
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              NX_BIN_URL=http://127.0.0.1:8080/assets/js NX_WASM_URL=http://127.0.0.1:8080/assets/js NPM_URL=http://localhost:9090/ yarn build:production
+            else
+              NX_BIN_URL=http://127.0.0.1:8080/assets/js NX_WASM_URL=http://127.0.0.1:8080/assets/js NPM_URL=http://localhost:9090/ yarn build
+            fi
+      - run: yarn run build:e2e
       - run: mkdir persist && zip -0 -r persist/dist.zip dist
       - persist_to_workspace:
           root: .
           paths:
             - "persist"
+
+  build-plugin:
+      docker:
+      - image: cimg/node:14.17.6-browsers
+
+      resource_class:
+        xlarge
+      working_directory: ~/remix-project
+      parameters:
+        plugin:
+          type: string
+      steps:
+        - checkout
+        - restore_cache:
+            keys:
+              - v1-deps-{{ checksum "yarn.lock" }}
+        - run: yarn
+        - save_cache:
+            key: v1-deps-{{ checksum "yarn.lock" }}
+            paths:
+              - node_modules
+        - run: yarn build << parameters.plugin >>
+        - run: mkdir persist && zip -0 -r persist/plugin-<< parameters.plugin >>.zip dist
+        - persist_to_workspace:
+            root: .
+            paths:
+              - "persist"
+
   lint:
     docker:
       - image: cimg/node:14.17.6-browsers
     resource_class:
       xlarge
     working_directory: ~/remix-project
-    parallelism: 1
+
     steps:
       - checkout
       - restore_cache:
@@ -60,11 +94,11 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - run: unzip ./persist/dist.zip
       - restore_cache:
           keys:
             - v1-deps-{{ checksum "yarn.lock" }}
       - run: yarn
+      - run: yarn build:libs
       - run: cd dist/libs/remix-tests && yarn
       - run: cd dist/libs/remix-tests && yarn add @remix-project/remix-url-resolver ../../libs/remix-url-resolver
       - run: cd dist/libs/remix-tests && yarn add @remix-project/remix-lib ../../libs/remix-lib
@@ -117,12 +151,8 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - run: unzip ./persist/dist.zip 
-
-      - restore_cache:
-          keys:
-            - v1-deps-{{ checksum "yarn.lock" }}
-      - run: yarn
+      - run: unzip ./persist/dist.zip
+      - run: yarn install --cwd ./apps/remix-ide-e2e --modules-folder ../../node_modules
       - run: yarn run downloadsolc_assets_e2e && yarn run downloadsolc_assets_dist
       - run: ls -la ./dist/apps/remix-ide/assets/js
       - run: yarn run selenium-install || yarn run selenium-install
@@ -149,9 +179,12 @@ jobs:
       xlarge
     working_directory: ~/remix-project
     parameters:
-      script:
+      plugin:
         type: string
-    parallelism: 4
+      parallelism:
+        type: integer
+        default: 1
+    parallelism: << parameters.parallelism >>
     steps:
       - browser-tools/install-browser-tools:
           install-firefox: false
@@ -164,19 +197,16 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-
-      - restore_cache:
-          keys:
-            - v1-deps-{{ checksum "yarn.lock" }}
-      - run: yarn
       - run: unzip ./persist/dist.zip
+      - run: unzip ./persist/plugin-<< parameters.plugin >>.zip
+      - run: yarn install --cwd ./apps/remix-ide-e2e --modules-folder ../../node_modules
       - run: yarn run downloadsolc_assets_e2e && yarn run downloadsolc_assets_dist
       - run: yarn run selenium-install || yarn run selenium-install
       - run:
           name: Start Selenium
           command: yarn run selenium
           background: true
-      - run: ./apps/remix-ide/ci/<< parameters.script >>
+      - run: ./apps/remix-ide/ci/browser_test_plugin.sh << parameters.plugin >>
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:
@@ -273,18 +303,32 @@ workflows:
     unless: << pipeline.parameters.run_flaky_tests >>
     jobs:
       - build
+      - build-plugin:
+          matrix:
+            parameters:
+              plugin: ["etherscan", "vyper", "plugin_api"]
       - lint:
           requires:
             - build
-      - remix-libs:
-          requires:
-            - build
+      - remix-libs
       - remix-test-plugins:
+          name: test-plugin-<< matrix.plugin >>
           requires:
             - build
+            - build-plugin
           matrix:
+            alias: plugins
             parameters:
-              script: ["browser_tests_plugin_api.sh", "browser_tests_etherscan_plugin.sh", "browser_tests_vyper_plugin.sh"]
+              plugin: ["etherscan", "vyper", "plugin_api"]
+              parallelism: [1, 9]
+            exclude: 
+              - plugin: plugin_api
+                parallelism: 1
+              - plugin: etherscan
+                parallelism: 9
+              - plugin: vyper
+                parallelism: 9
+
       - remix-ide-browser:
           requires:
             - build
@@ -299,13 +343,13 @@ workflows:
             - lint
             - remix-libs
             - remix-ide-browser
-            - remix-test-plugins
+            - plugins
       - deploy-remix-live:
           requires:
             - lint
             - remix-libs
             - remix-ide-browser
-            - remix-test-plugins
+            - plugins
           filters:
             branches:
               only: remix_live
@@ -314,7 +358,7 @@ workflows:
             - lint
             - remix-libs
             - remix-ide-browser
-            - remix-test-plugins
+            - plugins
           filters:
             branches:
               only: master
@@ -323,7 +367,7 @@ workflows:
             - lint
             - remix-libs
             - remix-ide-browser
-            - remix-test-plugins
+            - plugins
           filters:
             branches:
               only: remix_beta

apps/etherscan/project.json

@@ -3,9 +3,6 @@
   "$schema": "../../node_modules/nx/schemas/project-schema.json",
   "sourceRoot": "apps/etherscan/src",
   "projectType": "application",
-  "implicitDependencies": [
-    "remix-debug"
-  ],
   "targets": {
     "build": {
       "executor": "@nrwl/webpack:webpack",

apps/remix-ide-e2e/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "remix-ide-e2e",
+  "license": "MIT",
+  "engines": {
+    "node": "^14.17.6",
+    "npm": "^6.14.15"
+  },
+  "dependencies": {
+    "@openzeppelin/contracts": "^4.7.3",
+    "@openzeppelin/contracts-upgradeable": "^4.8.1",
+    "@openzeppelin/upgrades-core": "^1.22.0",
+    "@openzeppelin/wizard": "^0.1.1",
+    "@remix-project/remixd": "../../dist/libs/remixd",
+    "deep-equal": "^1.0.1",
+    "ganache-cli": "^6.8.1",
+    "selenium-standalone": "^8.2.3",
+    "tree-kill": "^1.2.2"
+  },
+  "devDependencies": {
+    "http-server": "^14.1.1",
+    "nightwatch": "2.3"
+  }
+}

apps/remix-ide-e2e/src/local-plugin/project.json

@@ -1,11 +1,8 @@
 {
-  "name": "remix-ide-e2e-src-local-plugin",
+  "name": "plugin_api",
   "$schema": "../../node_modules/nx/schemas/project-schema.json",
   "sourceRoot": "apps/remix-ide-e2e/src/local-plugin/src/",
   "projectType": "application",
-  "implicitDependencies": [
-    "remix-debug"
-  ],
   "targets": {
     "build": {
       "executor": "@nrwl/webpack:webpack",
@@ -13,7 +10,7 @@
       "defaultConfiguration": "development",
       "options": {
         "compiler": "babel",
-        "outputPath": "dist/apps/remix-ide-e2e-src-local-plugin",
+        "outputPath": "dist/apps/plugin_api",
         "index": "apps/remix-ide-e2e/src/local-plugin/src/index.html",
         "baseHref": "/",
         "main": "apps/remix-ide-e2e/src/local-plugin/src/main.tsx",
@@ -44,16 +41,16 @@
       "executor": "@nrwl/webpack:dev-server",
       "defaultConfiguration": "development",
       "options": {
-        "buildTarget": "remix-ide-e2e-src-local-plugin:build",
+        "buildTarget": "plugin_api:build",
         "hmr": true
       },
       "configurations": {
         "development": {
-          "buildTarget": "remix-ide-e2e-src-local-plugin:build:development",
+          "buildTarget": "plugin_api:build:development",
           "port": 2020
         },
         "production": {
-          "buildTarget": "remix-ide-e2e-src-local-plugin:build:production"
+          "buildTarget": "plugin_api:build:production"
         }
       }
     }

apps/remix-ide-e2e/src/tests/etherscan_api.ts

@@ -9,7 +9,7 @@ declare global {
 module.exports = {
   '@disabled': true,
   before: function (browser: NightwatchBrowser, done: VoidFunction) {
-    init(browser, done, null, true, { name: 'etherscan', url: 'http://127.0.0.1:5003'})
+    init(browser, done, null, true, { name: 'etherscan', url: 'http://127.0.0.1:9999'})
   },
 
   'Should load etherscan plugin #group1': function (browser: NightwatchBrowser) {

apps/remix-ide-e2e/src/tests/plugin_api.ts

@@ -11,7 +11,7 @@ const localPluginData: Profile & LocationProfile & ExternalProfile = {
   name: 'localPlugin',
   displayName: 'Local Plugin',
   canActivate: ['dGitProvider', 'flattener', 'solidityUnitTesting', 'udapp', 'hardhat-provider'],
-  url: 'http://localhost:2020',
+  url: 'http://localhost:9999',
   location: 'sidePanel'
 }
 

apps/remix-ide-e2e/src/tests/remixd.test.ts

@@ -312,7 +312,7 @@ function testImportFromRemixd(browser: NightwatchBrowser, callback: VoidFunction
 }
 
 async function spawnRemixd(path: string): Promise<ChildProcess> {
-  const remixd = spawn('yarn run remixd', [`-s ${path}`], { cwd: process.cwd(), shell: true, detached: true })
+  const remixd = spawn('chmod +x dist/libs/remixd/src/bin/remixd.js && dist/libs/remixd/src/bin/remixd.js --remix-ide http://127.0.0.1:8080', [`-s ${path}`], { cwd: process.cwd(), shell: true, detached: true })
   return new Promise((resolve, reject) => {
     remixd.stdout.on('data', function (data) {
       if(

apps/remix-ide-e2e/src/tests/vyper_api.ts

@@ -9,7 +9,7 @@ declare global {
 module.exports = {
   '@disabled': true,
   before: function (browser: NightwatchBrowser, done: VoidFunction) {
-    init(browser, done, null, true, { name: 'vyper', url: 'http://127.0.0.1:5002'})
+    init(browser, done, null, true, { name: 'vyper', url: 'http://127.0.0.1:9999'})
   },
 
   'Should connect to vyper plugin #group1': function (browser: NightwatchBrowser) {

apps/remix-ide/ci/browser_test.sh

@@ -10,11 +10,8 @@ TEST_EXITCODE=0
 yarn run ganache-cli &
 npx http-server -p 9090 --cors='*' ./node_modules &
 yarn run serve:production &
-echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' &
-
 sleep 5
 
-yarn run build:e2e
 # grep -IRiL "@disabled" "dist/apps/remix-ide-e2e/src/tests" | grep "\.spec\|\.test" | xargs -I {} basename {} .test.js | grep -E "\b[${2}]"
 # TESTFILES=$(grep -IRiL "@disabled" "dist/apps/remix-ide-e2e/src/tests" | grep "\.spec\|\.test" | xargs -I {} basename {} .test.js | grep -E "\b[$2]" | circleci tests split --split-by=timings )
 node apps/remix-ide/ci/splice_tests.js $2 $3

apps/remix-ide/ci/browser_test_plugin.sh

@@ -6,16 +6,14 @@ BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
 echo "$BUILD_ID"
 TEST_EXITCODE=0
 
+npx http-server -p 9999 ./dist/apps/$1 &
 yarn run ganache-cli &
 npx http-server -p 9090 --cors='*' ./node_modules &
 yarn run serve:production &
-npx nx serve remix-ide-e2e-src-local-plugin &
 
 sleep 5
 
-yarn run build:e2e
-
-TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "plugin_api" | sort | circleci tests split )
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep $1 | sort | circleci tests split )
 for TESTFILE in $TESTFILES; do
     npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=chrome  || TEST_EXITCODE=1
 done

apps/remix-ide/ci/browser_tests_etherscan_plugin.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
-echo "$BUILD_ID"
-TEST_EXITCODE=0
-
-yarn build etherscan
-yarn run serve:production &
-npx nx serve etherscan &
-
-sleep 5
-
-yarn run build:e2e
-
-TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "etherscan_api" | sort | circleci tests split )
-for TESTFILE in $TESTFILES; do
-    npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=chrome  || TEST_EXITCODE=1
-done
-
-echo "$TEST_EXITCODE"
-if [ "$TEST_EXITCODE" -eq 1 ]
-then
-  exit 1
-fi

apps/remix-ide/ci/browser_tests_vyper_plugin.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
-echo "$BUILD_ID"
-TEST_EXITCODE=0
-
-yarn build vyper
-yarn run serve:production &
-npx nx serve vyper &
-
-sleep 5
-
-yarn run build:e2e
-
-TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "vyper_api" | sort | circleci tests split )
-for TESTFILE in $TESTFILES; do
-    npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=chrome  || TEST_EXITCODE=1
-done
-
-echo "$TEST_EXITCODE"
-if [ "$TEST_EXITCODE" -eq 1 ]
-then
-  exit 1
-fi

apps/remix-ide/ci/flaky.sh

@@ -2,7 +2,6 @@
 
 set -e
 
-yarn run build:e2e
 TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
 
 # count test files
@@ -19,10 +18,8 @@ echo "$BUILD_ID"
 TEST_EXITCODE=0
 
 yarn run ganache-cli &
+npx http-server -p 9090 --cors='*' ./node_modules &
 yarn run serve:production &
-echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' &
-npx nx serve remix-ide-e2e-src-local-plugin &
-
 sleep 5
 
 for TESTFILE in $TESTFILES; do

apps/vyper/project.json

@@ -3,9 +3,6 @@
   "$schema": "../../node_modules/nx/schemas/project-schema.json",
   "sourceRoot": "apps/vyper/src",
   "projectType": "application",
-  "implicitDependencies": [
-    "remix-debug"
-  ],
   "targets": {
     "build": {
       "executor": "@nrwl/webpack:webpack",