diff --git a/.circleci/config.yml b/.circleci/config.yml index 43991a1deb..11c6ede82b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,6 +3,10 @@ # Check https://circleci.com/docs/2.0/language-javascript/ for more details # version: 2.1 +parameters: + run_flaky_tests: + type: boolean + default: false orbs: browser-tools: circleci/browser-tools@1.2.3 jobs: @@ -91,7 +95,7 @@ jobs: - run: npm i - run: cd dist/libs/remix-tests && npm install - run: npm run test:libs - + remix-ide-chrome: docker: # specify the version you desire here @@ -135,6 +139,49 @@ jobs: - store_artifacts: path: ./reports/screenshots + flaky-chrome: + docker: + # specify the version you desire here + - image: cimg/node:14.17.6-browsers + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + resource_class: xlarge + # - image: circleci/mongo:3.4.4 + environment: + - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org" + - COMMIT_AUTHOR: "Circle CI" + working_directory: ~/remix-project + + parallelism: 80 + steps: + - browser-tools/install-chrome + - browser-tools/install-chromedriver + - run: + command: | + google-chrome --version + chromedriver --version + java -jar /usr/local/bin/selenium.jar --version + name: Check install + - checkout + - attach_workspace: + at: . + - run: unzip ./persist/dist.zip + - restore_cache: + keys: + - v1-deps-{{ checksum "package-lock.json" }} + - run: npm install + - run: + name: Start Selenium + command: java -jar /usr/local/bin/selenium.jar + background: true + - run: ./apps/remix-ide/ci/flaky.sh chrome + - store_test_results: + path: ./reports/tests + - store_artifacts: + path: ./reports/screenshots + remix-ide-firefox: docker: # specify the version you desire here @@ -177,7 +224,49 @@ jobs: path: ./reports/tests - store_artifacts: path: ./reports/screenshots + flaky-firefox: + docker: + # specify the version you desire here + - image: cimg/node:14.17.6-browsers + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + resource_class: xlarge + # - image: circleci/mongo:3.4.4 + environment: + - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org" + - COMMIT_AUTHOR: "Circle CI" + working_directory: ~/remix-project + parallelism: 80 + steps: + - browser-tools/install-firefox + - browser-tools/install-geckodriver + - run: + command: | + firefox --version + geckodriver --version + java -jar /usr/local/bin/selenium.jar --version + name: Check install + - checkout + - attach_workspace: + at: . + - run: unzip ./persist/dist.zip + - restore_cache: + keys: + - v1-deps-{{ checksum "package-lock.json" }} + - run: npm install + - run: + name: Start Selenium + command: java -jar /usr/local/bin/selenium.jar + background: true + - run: ./apps/remix-ide/ci/flaky.sh firefox + - store_test_results: + path: ./reports/tests + - store_artifacts: + path: ./reports/screenshots + remix-ide-plugin-api: docker: # specify the version you desire here @@ -306,10 +395,20 @@ jobs: if [ "${CIRCLE_BRANCH}" == "remix_beta" ]; then ./apps/remix-ide/ci/deploy_from_travis_remix-beta.sh; fi - workflows: version: 2 + run_flaky_tests: + when: << pipeline.parameters.run_flaky_tests >> + jobs: + - build + - flaky-chrome: + requires: + - build + - flaky-firefox: + requires: + - build build_all: + unless: << pipeline.parameters.run_flaky_tests >> jobs: - build - lint: diff --git a/apps/remix-ide-e2e/src/buildGroupTests.js b/apps/remix-ide-e2e/src/buildGroupTests.js index 6afd20c107..7127462125 100644 --- a/apps/remix-ide-e2e/src/buildGroupTests.js +++ b/apps/remix-ide-e2e/src/buildGroupTests.js @@ -15,19 +15,41 @@ fs.readdirSync(testFolder).forEach(file => { if (!file.includes('group')) { const content = fs.readFileSync(testFolder + file, 'utf8') const matches = content.match(/group\d+/g) - if (matches) { - const unique = matches.filter(onlyUnique) - unique.map((group) => { - const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', '')) - const extension = file.split('.') - extension.shift() - const filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}` - fs.writeFileSync(filename, rewrite) - }) - } + createFlakyTestFiles(file, content) + createFiles(file, matches) } }) -function onlyUnique (value, index, self) { +function createFiles(file, matches, flaky = false) { + if (matches) { + const unique = matches.filter(onlyUnique) + unique.map((group) => { + const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', '')) + const extension = file.split('.') + extension.shift() + let filename + if (!flaky) { + filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}` + } else { + filename = `${testFolder}${file.split('.').shift()}_${group}.flaky.ts` + } + fs.writeFileSync(filename, rewrite) + }) + } +} + +function onlyUnique(value, index, self) { return self.indexOf(value) === index } + +function createFlakyTestFiles(file, text) { + const lines = text.split('\n') + lines.forEach((line, index) => { + // if line contains #flaky + if (line.includes('#flaky')) { + const matches = line.match(/group\d+/g) + const unique = matches.filter(onlyUnique) + createFiles(file, matches, true) + } + }) +} diff --git a/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts b/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts new file mode 100644 index 0000000000..d1da53488a --- /dev/null +++ b/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts @@ -0,0 +1,19 @@ +import { NightwatchBrowser } from 'nightwatch' +import EventEmitter from 'events' + +class GetBrowserLogs extends EventEmitter { + command(this: NightwatchBrowser): NightwatchBrowser { + this.api.getLog('browser', function (logs) { + if (logs && Array.isArray(logs)) { + logs.forEach(function (log) { + console.log(log) + } + ) + } + }).perform(() => { + this.emit('complete') + }) + return this + } +} +module.exports = GetBrowserLogs \ No newline at end of file diff --git a/apps/remix-ide-e2e/src/types/index.d.ts b/apps/remix-ide-e2e/src/types/index.d.ts index 9abe5c2174..e8698d4407 100644 --- a/apps/remix-ide-e2e/src/types/index.d.ts +++ b/apps/remix-ide-e2e/src/types/index.d.ts @@ -61,6 +61,7 @@ declare module 'nightwatch' { acceptAndRemember (this: NightwatchBrowser, remember: boolean, accept: boolean): NightwatchBrowser clearConsole (this: NightwatchBrowser): NightwatchBrowser clearTransactions (this: NightwatchBrowser): NightwatchBrowser + getBrowserLogs (this: NightwatchBrowser): NightwatchBrowser currentSelectedFileIs (name: string): NightwatchBrowser } diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh new file mode 100755 index 0000000000..8a2dac3e73 --- /dev/null +++ b/apps/remix-ide/ci/flaky.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e + +npm run build:e2e +TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort ) + +# count test files +fileCount=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | wc -l ) +# if fileCount is 0 +if [ $fileCount -eq 0 ] +then + echo "No flaky tests found" + exit 0 +fi + +BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}} +echo "$BUILD_ID" +TEST_EXITCODE=0 + +npm run ganache-cli & +npm run serve:production & +echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' & +npm run remixd & +npx nx serve remix-ide-e2e-src-local-plugin & + +sleep 5 + +for TESTFILE in $TESTFILES; do + npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1 || TEST_EXITCODE=1 +done + +echo "$TEST_EXITCODE" +if [ "$TEST_EXITCODE" -eq 1 ] +then + exit 1 +fi