flaky tests

.circleci/config.yml

@@ -92,6 +92,49 @@ jobs:
       - run: cd dist/libs/remix-tests && npm install
       - run: npm run test:libs
 
+  flaky-chrome:
+    docker:
+      # specify the version you desire here
+      - image: cimg/node:14.17.6-browsers
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+    resource_class: xlarge
+      # - image: circleci/mongo:3.4.4
+    environment:
+      - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org"
+      - COMMIT_AUTHOR: "Circle CI"
+    working_directory: ~/remix-project
+
+    parallelism: 80
+    steps:
+      - browser-tools/install-chrome
+      - browser-tools/install-chromedriver
+      - run:
+          command: |
+              google-chrome --version
+              chromedriver --version
+              java -jar /usr/local/bin/selenium.jar --version
+          name: Check install
+      - checkout
+      - attach_workspace:
+            at: .
+      - run: unzip ./persist/dist.zip
+      - restore_cache:
+          keys:
+            - v1-deps-{{ checksum "package-lock.json" }}
+      - run: npm install
+      - run:
+          name: Start Selenium
+          command: java -jar /usr/local/bin/selenium.jar
+          background: true
+      - run: ./apps/remix-ide/ci/flaky.sh chrome
+      - store_test_results:
+          path: ./reports/tests
+      - store_artifacts:
+          path: ./reports/screenshots
+          
   remix-ide-chrome:
     docker:
       # specify the version you desire here
@@ -312,21 +355,29 @@ workflows:
   build_all:
     jobs:
       - build
+      - flaky-chrome:
+          requires:
+            - build
       - lint:
           requires:
             - build
+            - flaky-chrome
       - remix-libs:
           requires:
             - build
+            - flaky-chrome
       - remix-ide-plugin-api:
           requires:
             - build
+            - flaky-chrome
       - remix-ide-chrome:
           requires:
             - build
+            - flaky-chrome
       - remix-ide-firefox:
           requires:
             - build
+            - flaky-chrome
       - deploy-remix-live:          
           requires:
             - lint

apps/remix-ide-e2e/src/buildGroupTests.js

@@ -15,19 +15,42 @@ fs.readdirSync(testFolder).forEach(file => {
   if (!file.includes('group')) {
     const content = fs.readFileSync(testFolder + file, 'utf8')
     const matches = content.match(/group\d+/g)
-    if (matches) {
+    createFlakyTestFiles(file, content)
-      const unique = matches.filter(onlyUnique)
+    createFiles(file, matches)
-      unique.map((group) => {
-        const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', ''))
-        const extension = file.split('.')
-        extension.shift()
-        const filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
-        fs.writeFileSync(filename, rewrite)
-      })
-    }
   }
 })
 
-function onlyUnique (value, index, self) {
+function createFiles(file, matches, flaky = false) {
+  if (matches) {
+    const unique = matches.filter(onlyUnique)
+    unique.map((group) => {
+      const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', ''))
+      const extension = file.split('.')
+      extension.shift()
+      let filename
+      if (!flaky) {
+        filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
+      } else {
+        filename = `${testFolder}${file.split('.').shift()}_${group}.flaky.ts`
+      }
+      fs.writeFileSync(filename, rewrite)
+    })
+  }
+}
+
+function onlyUnique(value, index, self) {
   return self.indexOf(value) === index
 }
+
+function createFlakyTestFiles(file, text) {
+  const lines = text.split('\n')
+  lines.forEach((line, index) => {
+    // if line contains #flaky
+    if (line.includes('#flaky')) {
+      console.log(line)
+      const matches = line.match(/group\d+/g)
+      const unique = matches.filter(onlyUnique)
+      createFiles(file, matches, true)
+    }
+  })
+}

apps/remix-ide-e2e/src/tests/remixd.test.ts

@@ -81,7 +81,7 @@ module.exports = {
       .setSolidityCompilerVersion('soljson-v0.8.0+commit.c7dfd78e.js') // open-zeppelin moved to pragma ^0.8.0
       .testContracts('test_import_node_modules_with_github_import.sol', sources[4]['test_import_node_modules_with_github_import.sol'], ['ERC20', 'test11'])
   },
-  'Static Analysis run with remixd #group3': function (browser) {
+  'Static Analysis run with remixd #group3 #flaky': function (browser) {
     browser.testContracts('test_static_analysis_with_remixd_and_hardhat.sol', sources[5]['test_static_analysis_with_remixd_and_hardhat.sol'], ['test5']).pause(2000)
       .clickLaunchIcon('solidityStaticAnalysis')
       .click('#staticanalysisButton button').pause(4000)

apps/remix-ide/ci/flaky.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
+echo "$BUILD_ID"
+TEST_EXITCODE=0
+
+npm run ganache-cli &
+npm run serve:production &
+echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' &
+npm run remixd &
+
+sleep 5
+
+npm run build:e2e
+
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort | circleci tests split )
+for TESTFILE in $TESTFILES; do
+    npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1  || TEST_EXITCODE=1
+done
+
+echo "$TEST_EXITCODE"
+if [ "$TEST_EXITCODE" -eq 1 ]
+then
+  exit 1
+fi