From c1c29bea05e450243afe52f39e8df66c5709bc47 Mon Sep 17 00:00:00 2001 From: filip mertens Date: Tue, 3 May 2022 17:44:26 +0200 Subject: [PATCH] flaky tests --- .circleci/config.yml | 51 +++++++++++++++++++++ apps/remix-ide-e2e/src/buildGroupTests.js | 45 +++++++++++++----- apps/remix-ide-e2e/src/tests/remixd.test.ts | 2 +- apps/remix-ide/ci/flaky.sh | 27 +++++++++++ 4 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 apps/remix-ide/ci/flaky.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 43991a1deb..165e6aa77c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,6 +92,49 @@ jobs: - run: cd dist/libs/remix-tests && npm install - run: npm run test:libs + flaky-chrome: + docker: + # specify the version you desire here + - image: cimg/node:14.17.6-browsers + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + resource_class: xlarge + # - image: circleci/mongo:3.4.4 + environment: + - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org" + - COMMIT_AUTHOR: "Circle CI" + working_directory: ~/remix-project + + parallelism: 80 + steps: + - browser-tools/install-chrome + - browser-tools/install-chromedriver + - run: + command: | + google-chrome --version + chromedriver --version + java -jar /usr/local/bin/selenium.jar --version + name: Check install + - checkout + - attach_workspace: + at: . + - run: unzip ./persist/dist.zip + - restore_cache: + keys: + - v1-deps-{{ checksum "package-lock.json" }} + - run: npm install + - run: + name: Start Selenium + command: java -jar /usr/local/bin/selenium.jar + background: true + - run: ./apps/remix-ide/ci/flaky.sh chrome + - store_test_results: + path: ./reports/tests + - store_artifacts: + path: ./reports/screenshots + remix-ide-chrome: docker: # specify the version you desire here @@ -312,21 +355,29 @@ workflows: build_all: jobs: - build + - flaky-chrome: + requires: + - build - lint: requires: - build + - flaky-chrome - remix-libs: requires: - build + - flaky-chrome - remix-ide-plugin-api: requires: - build + - flaky-chrome - remix-ide-chrome: requires: - build + - flaky-chrome - remix-ide-firefox: requires: - build + - flaky-chrome - deploy-remix-live: requires: - lint diff --git a/apps/remix-ide-e2e/src/buildGroupTests.js b/apps/remix-ide-e2e/src/buildGroupTests.js index 6afd20c107..0c589bd5fd 100644 --- a/apps/remix-ide-e2e/src/buildGroupTests.js +++ b/apps/remix-ide-e2e/src/buildGroupTests.js @@ -15,19 +15,42 @@ fs.readdirSync(testFolder).forEach(file => { if (!file.includes('group')) { const content = fs.readFileSync(testFolder + file, 'utf8') const matches = content.match(/group\d+/g) - if (matches) { - const unique = matches.filter(onlyUnique) - unique.map((group) => { - const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', '')) - const extension = file.split('.') - extension.shift() - const filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}` - fs.writeFileSync(filename, rewrite) - }) - } + createFlakyTestFiles(file, content) + createFiles(file, matches) } }) -function onlyUnique (value, index, self) { +function createFiles(file, matches, flaky = false) { + if (matches) { + const unique = matches.filter(onlyUnique) + unique.map((group) => { + const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', '')) + const extension = file.split('.') + extension.shift() + let filename + if (!flaky) { + filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}` + } else { + filename = `${testFolder}${file.split('.').shift()}_${group}.flaky.ts` + } + fs.writeFileSync(filename, rewrite) + }) + } +} + +function onlyUnique(value, index, self) { return self.indexOf(value) === index } + +function createFlakyTestFiles(file, text) { + const lines = text.split('\n') + lines.forEach((line, index) => { + // if line contains #flaky + if (line.includes('#flaky')) { + console.log(line) + const matches = line.match(/group\d+/g) + const unique = matches.filter(onlyUnique) + createFiles(file, matches, true) + } + }) +} diff --git a/apps/remix-ide-e2e/src/tests/remixd.test.ts b/apps/remix-ide-e2e/src/tests/remixd.test.ts index 8b7d043235..3c81e8d1ab 100644 --- a/apps/remix-ide-e2e/src/tests/remixd.test.ts +++ b/apps/remix-ide-e2e/src/tests/remixd.test.ts @@ -81,7 +81,7 @@ module.exports = { .setSolidityCompilerVersion('soljson-v0.8.0+commit.c7dfd78e.js') // open-zeppelin moved to pragma ^0.8.0 .testContracts('test_import_node_modules_with_github_import.sol', sources[4]['test_import_node_modules_with_github_import.sol'], ['ERC20', 'test11']) }, - 'Static Analysis run with remixd #group3': function (browser) { + 'Static Analysis run with remixd #group3 #flaky': function (browser) { browser.testContracts('test_static_analysis_with_remixd_and_hardhat.sol', sources[5]['test_static_analysis_with_remixd_and_hardhat.sol'], ['test5']).pause(2000) .clickLaunchIcon('solidityStaticAnalysis') .click('#staticanalysisButton button').pause(4000) diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh new file mode 100644 index 0000000000..4c9dd4e274 --- /dev/null +++ b/apps/remix-ide/ci/flaky.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}} +echo "$BUILD_ID" +TEST_EXITCODE=0 + +npm run ganache-cli & +npm run serve:production & +echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' & +npm run remixd & + +sleep 5 + +npm run build:e2e + +TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort | circleci tests split ) +for TESTFILE in $TESTFILES; do + npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1 || TEST_EXITCODE=1 +done + +echo "$TEST_EXITCODE" +if [ "$TEST_EXITCODE" -eq 1 ] +then + exit 1 +fi