Merge pull request #2356 from ethereum/focustests

flaky test system

.circleci/config.yml

@@ -3,6 +3,10 @@
 # Check https://circleci.com/docs/2.0/language-javascript/ for more details
 #
 version: 2.1
+parameters:
+  run_flaky_tests:
+    type: boolean
+    default: false
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:
@@ -135,6 +139,49 @@ jobs:
       - store_artifacts:
           path: ./reports/screenshots
 
+  flaky-chrome:
+    docker:
+      # specify the version you desire here
+      - image: cimg/node:14.17.6-browsers
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+    resource_class: xlarge
+      # - image: circleci/mongo:3.4.4
+    environment:
+      - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org"
+      - COMMIT_AUTHOR: "Circle CI"
+    working_directory: ~/remix-project
+
+    parallelism: 80
+    steps:
+      - browser-tools/install-chrome
+      - browser-tools/install-chromedriver
+      - run:
+          command: |
+              google-chrome --version
+              chromedriver --version
+              java -jar /usr/local/bin/selenium.jar --version
+          name: Check install
+      - checkout
+      - attach_workspace:
+            at: .
+      - run: unzip ./persist/dist.zip
+      - restore_cache:
+          keys:
+            - v1-deps-{{ checksum "package-lock.json" }}
+      - run: npm install
+      - run:
+          name: Start Selenium
+          command: java -jar /usr/local/bin/selenium.jar
+          background: true
+      - run: ./apps/remix-ide/ci/flaky.sh chrome
+      - store_test_results:
+          path: ./reports/tests
+      - store_artifacts:
+          path: ./reports/screenshots
+
   remix-ide-firefox:
     docker:
       # specify the version you desire here
@@ -177,6 +224,48 @@ jobs:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
+  flaky-firefox:
+    docker:
+      # specify the version you desire here
+      - image: cimg/node:14.17.6-browsers
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+    resource_class: xlarge
+      # - image: circleci/mongo:3.4.4
+    environment:
+      - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org"
+      - COMMIT_AUTHOR: "Circle CI"
+    working_directory: ~/remix-project
+
+    parallelism: 80
+    steps:
+      - browser-tools/install-firefox
+      - browser-tools/install-geckodriver
+      - run:
+          command: |
+              firefox --version
+              geckodriver --version
+              java -jar /usr/local/bin/selenium.jar --version
+          name: Check install
+      - checkout
+      - attach_workspace:
+            at: .
+      - run: unzip ./persist/dist.zip
+      - restore_cache:
+          keys:
+            - v1-deps-{{ checksum "package-lock.json" }}
+      - run: npm install
+      - run:
+          name: Start Selenium
+          command: java -jar /usr/local/bin/selenium.jar
+          background: true
+      - run: ./apps/remix-ide/ci/flaky.sh firefox
+      - store_test_results:
+          path: ./reports/tests
+      - store_artifacts:
+          path: ./reports/screenshots
           
   remix-ide-plugin-api:
     docker:
@@ -306,10 +395,20 @@ jobs:
             if [ "${CIRCLE_BRANCH}" == "remix_beta" ]; then
               ./apps/remix-ide/ci/deploy_from_travis_remix-beta.sh;
             fi
-
 workflows:
   version: 2
+  run_flaky_tests:
+    when: << pipeline.parameters.run_flaky_tests >>
+    jobs:
+      - build
+      - flaky-chrome:
+          requires:
+            - build
+      - flaky-firefox:
+          requires:
+            - build
   build_all:
+    unless: << pipeline.parameters.run_flaky_tests >>
     jobs:
       - build
       - lint:

apps/remix-ide-e2e/src/buildGroupTests.js

@@ -15,19 +15,41 @@ fs.readdirSync(testFolder).forEach(file => {
   if (!file.includes('group')) {
     const content = fs.readFileSync(testFolder + file, 'utf8')
     const matches = content.match(/group\d+/g)
+    createFlakyTestFiles(file, content)
+    createFiles(file, matches)
+  }
+})
+
+function createFiles(file, matches, flaky = false) {
   if (matches) {
     const unique = matches.filter(onlyUnique)
     unique.map((group) => {
       const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', ''))
       const extension = file.split('.')
       extension.shift()
-        const filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
+      let filename
+      if (!flaky) {
+        filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
+      } else {
+        filename = `${testFolder}${file.split('.').shift()}_${group}.flaky.ts`
+      }
       fs.writeFileSync(filename, rewrite)
     })
   }
-  }
-})
+}
 
-function onlyUnique (value, index, self) {
+function onlyUnique(value, index, self) {
   return self.indexOf(value) === index
 }
+
+function createFlakyTestFiles(file, text) {
+  const lines = text.split('\n')
+  lines.forEach((line, index) => {
+    // if line contains #flaky
+    if (line.includes('#flaky')) {
+      const matches = line.match(/group\d+/g)
+      const unique = matches.filter(onlyUnique)
+      createFiles(file, matches, true)
+    }
+  })
+}

apps/remix-ide-e2e/src/commands/getBrowserLogs.ts

@@ -0,0 +1,19 @@
+import { NightwatchBrowser } from 'nightwatch'
+import EventEmitter from 'events'
+
+class GetBrowserLogs extends EventEmitter {
+    command(this: NightwatchBrowser): NightwatchBrowser {
+        this.api.getLog('browser', function (logs) {
+            if (logs && Array.isArray(logs)) {
+                logs.forEach(function (log) {
+                    console.log(log)
+                }
+                )
+            }
+        }).perform(() => {
+            this.emit('complete')
+        })
+        return this
+    }
+}
+module.exports = GetBrowserLogs

apps/remix-ide-e2e/src/types/index.d.ts

@@ -61,6 +61,7 @@ declare module 'nightwatch' {
         acceptAndRemember (this: NightwatchBrowser, remember: boolean, accept: boolean): NightwatchBrowser
         clearConsole (this: NightwatchBrowser): NightwatchBrowser
         clearTransactions (this: NightwatchBrowser): NightwatchBrowser
+        getBrowserLogs (this: NightwatchBrowser): NightwatchBrowser
         currentSelectedFileIs (name: string): NightwatchBrowser
     }
 

apps/remix-ide/ci/flaky.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+npm run build:e2e
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
+
+# count test files
+fileCount=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | wc -l )
+# if fileCount is 0
+if [ $fileCount -eq 0 ]
+then
+  echo "No flaky tests found"
+  exit 0
+fi
+
+BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
+echo "$BUILD_ID"
+TEST_EXITCODE=0
+
+npm run ganache-cli &
+npm run serve:production &
+echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' &
+npm run remixd &
+npx nx serve remix-ide-e2e-src-local-plugin &
+
+sleep 5
+
+for TESTFILE in $TESTFILES; do
+    npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1  || TEST_EXITCODE=1
+done
+
+echo "$TEST_EXITCODE"
+if [ "$TEST_EXITCODE" -eq 1 ]
+then
+  exit 1
+fi