From 58bcd7f20526851ecdde07f0a31ecb4d2d63faa1 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 17:44:26 +0200
Subject: [PATCH 01/26] flaky tests

---
 .circleci/config.yml                        | 51 +++++++++++++++++++++
 apps/remix-ide-e2e/src/buildGroupTests.js   | 45 +++++++++++++-----
 apps/remix-ide-e2e/src/tests/remixd.test.ts |  2 +-
 apps/remix-ide/ci/flaky.sh                  | 27 +++++++++++
 4 files changed, 113 insertions(+), 12 deletions(-)
 create mode 100644 apps/remix-ide/ci/flaky.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 43991a1deb..165e6aa77c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -92,6 +92,49 @@ jobs:
       - run: cd dist/libs/remix-tests && npm install
       - run: npm run test:libs
 
+  flaky-chrome:
+    docker:
+      # specify the version you desire here
+      - image: cimg/node:14.17.6-browsers
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+    resource_class: xlarge
+      # - image: circleci/mongo:3.4.4
+    environment:
+      - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org"
+      - COMMIT_AUTHOR: "Circle CI"
+    working_directory: ~/remix-project
+
+    parallelism: 80
+    steps:
+      - browser-tools/install-chrome
+      - browser-tools/install-chromedriver
+      - run:
+          command: |
+              google-chrome --version
+              chromedriver --version
+              java -jar /usr/local/bin/selenium.jar --version
+          name: Check install
+      - checkout
+      - attach_workspace:
+            at: .
+      - run: unzip ./persist/dist.zip
+      - restore_cache:
+          keys:
+            - v1-deps-{{ checksum "package-lock.json" }}
+      - run: npm install
+      - run:
+          name: Start Selenium
+          command: java -jar /usr/local/bin/selenium.jar
+          background: true
+      - run: ./apps/remix-ide/ci/flaky.sh chrome
+      - store_test_results:
+          path: ./reports/tests
+      - store_artifacts:
+          path: ./reports/screenshots
+          
   remix-ide-chrome:
     docker:
       # specify the version you desire here
@@ -312,21 +355,29 @@ workflows:
   build_all:
     jobs:
       - build
+      - flaky-chrome:
+          requires:
+            - build
       - lint:
           requires:
             - build
+            - flaky-chrome
       - remix-libs:
           requires:
             - build
+            - flaky-chrome
       - remix-ide-plugin-api:
           requires:
             - build
+            - flaky-chrome
       - remix-ide-chrome:
           requires:
             - build
+            - flaky-chrome
       - remix-ide-firefox:
           requires:
             - build
+            - flaky-chrome
       - deploy-remix-live:          
           requires:
             - lint
diff --git a/apps/remix-ide-e2e/src/buildGroupTests.js b/apps/remix-ide-e2e/src/buildGroupTests.js
index 6afd20c107..0c589bd5fd 100644
--- a/apps/remix-ide-e2e/src/buildGroupTests.js
+++ b/apps/remix-ide-e2e/src/buildGroupTests.js
@@ -15,19 +15,42 @@ fs.readdirSync(testFolder).forEach(file => {
   if (!file.includes('group')) {
     const content = fs.readFileSync(testFolder + file, 'utf8')
     const matches = content.match(/group\d+/g)
-    if (matches) {
-      const unique = matches.filter(onlyUnique)
-      unique.map((group) => {
-        const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', ''))
-        const extension = file.split('.')
-        extension.shift()
-        const filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
-        fs.writeFileSync(filename, rewrite)
-      })
-    }
+    createFlakyTestFiles(file, content)
+    createFiles(file, matches)
   }
 })
 
-function onlyUnique (value, index, self) {
+function createFiles(file, matches, flaky = false) {
+  if (matches) {
+    const unique = matches.filter(onlyUnique)
+    unique.map((group) => {
+      const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', ''))
+      const extension = file.split('.')
+      extension.shift()
+      let filename
+      if (!flaky) {
+        filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
+      } else {
+        filename = `${testFolder}${file.split('.').shift()}_${group}.flaky.ts`
+      }
+      fs.writeFileSync(filename, rewrite)
+    })
+  }
+}
+
+function onlyUnique(value, index, self) {
   return self.indexOf(value) === index
 }
+
+function createFlakyTestFiles(file, text) {
+  const lines = text.split('\n')
+  lines.forEach((line, index) => {
+    // if line contains #flaky
+    if (line.includes('#flaky')) {
+      console.log(line)
+      const matches = line.match(/group\d+/g)
+      const unique = matches.filter(onlyUnique)
+      createFiles(file, matches, true)
+    }
+  })
+}
diff --git a/apps/remix-ide-e2e/src/tests/remixd.test.ts b/apps/remix-ide-e2e/src/tests/remixd.test.ts
index 8b7d043235..3c81e8d1ab 100644
--- a/apps/remix-ide-e2e/src/tests/remixd.test.ts
+++ b/apps/remix-ide-e2e/src/tests/remixd.test.ts
@@ -81,7 +81,7 @@ module.exports = {
       .setSolidityCompilerVersion('soljson-v0.8.0+commit.c7dfd78e.js') // open-zeppelin moved to pragma ^0.8.0
       .testContracts('test_import_node_modules_with_github_import.sol', sources[4]['test_import_node_modules_with_github_import.sol'], ['ERC20', 'test11'])
   },
-  'Static Analysis run with remixd #group3': function (browser) {
+  'Static Analysis run with remixd #group3 #flaky': function (browser) {
     browser.testContracts('test_static_analysis_with_remixd_and_hardhat.sol', sources[5]['test_static_analysis_with_remixd_and_hardhat.sol'], ['test5']).pause(2000)
       .clickLaunchIcon('solidityStaticAnalysis')
       .click('#staticanalysisButton button').pause(4000)
diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh
new file mode 100644
index 0000000000..4c9dd4e274
--- /dev/null
+++ b/apps/remix-ide/ci/flaky.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
+echo "$BUILD_ID"
+TEST_EXITCODE=0
+
+npm run ganache-cli &
+npm run serve:production &
+echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' &
+npm run remixd &
+
+sleep 5
+
+npm run build:e2e
+
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort | circleci tests split )
+for TESTFILE in $TESTFILES; do
+    npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1  || TEST_EXITCODE=1
+done
+
+echo "$TEST_EXITCODE"
+if [ "$TEST_EXITCODE" -eq 1 ]
+then
+  exit 1
+fi

From 18748bb6131e6ba11a7522c7cf3c7a49bfc7bd53 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 17:51:09 +0200
Subject: [PATCH 02/26] permissions

---
 apps/remix-ide/ci/flaky.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 apps/remix-ide/ci/flaky.sh

diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh
old mode 100644
new mode 100755

From 596121567917478ce16753663b3e31843fd8a8d2 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:04:46 +0200
Subject: [PATCH 03/26] no splitting

---
 apps/remix-ide/ci/flaky.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh
index 4c9dd4e274..21bfa1855b 100755
--- a/apps/remix-ide/ci/flaky.sh
+++ b/apps/remix-ide/ci/flaky.sh
@@ -15,7 +15,7 @@ sleep 5
 
 npm run build:e2e
 
-TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort | circleci tests split )
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
 for TESTFILE in $TESTFILES; do
     npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1  || TEST_EXITCODE=1
 done

From d8068869e035eab900d52c0f9e19c816f78e7ac2 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:11:54 +0200
Subject: [PATCH 04/26] firefox

---
 .circleci/config.yml | 50 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 165e6aa77c..6d24922105 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -134,7 +134,48 @@ jobs:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
-          
+  flaky-ff:
+    docker:
+      # specify the version you desire here
+      - image: cimg/node:14.17.6-browsers
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+    resource_class: xlarge
+      # - image: circleci/mongo:3.4.4
+    environment:
+      - COMMIT_AUTHOR_EMAIL: "yann@ethereum.org"
+      - COMMIT_AUTHOR: "Circle CI"
+    working_directory: ~/remix-project
+
+    parallelism: 80
+    steps:
+      - browser-tools/install-chrome
+      - browser-tools/install-chromedriver
+      - run:
+          command: |
+              google-chrome --version
+              chromedriver --version
+              java -jar /usr/local/bin/selenium.jar --version
+          name: Check install
+      - checkout
+      - attach_workspace:
+            at: .
+      - run: unzip ./persist/dist.zip
+      - restore_cache:
+          keys:
+            - v1-deps-{{ checksum "package-lock.json" }}
+      - run: npm install
+      - run:
+          name: Start Selenium
+          command: java -jar /usr/local/bin/selenium.jar
+          background: true
+      - run: ./apps/remix-ide/ci/flaky.sh firefox
+      - store_test_results:
+          path: ./reports/tests
+      - store_artifacts:
+          path: ./reports/screenshots        
   remix-ide-chrome:
     docker:
       # specify the version you desire here
@@ -358,14 +399,19 @@ workflows:
       - flaky-chrome:
           requires:
             - build
+      - flaky-firefox:
+          requires:
+            - build
       - lint:
           requires:
             - build
             - flaky-chrome
+            - flaky-firefox
       - remix-libs:
           requires:
             - build
             - flaky-chrome
+            - flaky-firefox
       - remix-ide-plugin-api:
           requires:
             - build
@@ -374,10 +420,12 @@ workflows:
           requires:
             - build
             - flaky-chrome
+            - flaky-firefox
       - remix-ide-firefox:
           requires:
             - build
             - flaky-chrome
+            - flaky-firefox
       - deploy-remix-live:          
           requires:
             - lint

From ff3e634773358b84e439fa66ae61da4394c5b0d6 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:12:36 +0200
Subject: [PATCH 05/26] ff

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6d24922105..40457342b6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -134,7 +134,7 @@ jobs:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
-  flaky-ff:
+  flaky-firefox:
     docker:
       # specify the version you desire here
       - image: cimg/node:14.17.6-browsers

From 599350ccad2ab8d7e7ec5560f14af467953ba40a Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:20:27 +0200
Subject: [PATCH 06/26] ff

---
 .circleci/config.yml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 40457342b6..34e8d5003d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -134,7 +134,8 @@ jobs:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
-  flaky-firefox:
+          
+  remix-ide-chrome:
     docker:
       # specify the version you desire here
       - image: cimg/node:14.17.6-browsers
@@ -171,12 +172,13 @@ jobs:
           name: Start Selenium
           command: java -jar /usr/local/bin/selenium.jar
           background: true
-      - run: ./apps/remix-ide/ci/flaky.sh firefox
+      - run: ./apps/remix-ide/ci/browser_test.sh chrome
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:
-          path: ./reports/screenshots        
-  remix-ide-chrome:
+          path: ./reports/screenshots
+
+  remix-ide-firefox:
     docker:
       # specify the version you desire here
       - image: cimg/node:14.17.6-browsers
@@ -193,12 +195,12 @@ jobs:
 
     parallelism: 80
     steps:
-      - browser-tools/install-chrome
-      - browser-tools/install-chromedriver
+      - browser-tools/install-firefox
+      - browser-tools/install-geckodriver
       - run:
           command: |
-              google-chrome --version
-              chromedriver --version
+              firefox --version
+              geckodriver --version
               java -jar /usr/local/bin/selenium.jar --version
           name: Check install
       - checkout
@@ -213,13 +215,12 @@ jobs:
           name: Start Selenium
           command: java -jar /usr/local/bin/selenium.jar
           background: true
-      - run: ./apps/remix-ide/ci/browser_test.sh chrome
+      - run: ./apps/remix-ide/ci/browser_test.sh firefox
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
-
-  remix-ide-firefox:
+  flaky-firefox:
     docker:
       # specify the version you desire here
       - image: cimg/node:14.17.6-browsers
@@ -261,7 +262,7 @@ jobs:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
-
+          
   remix-ide-plugin-api:
     docker:
       # specify the version you desire here

From 2b105002de8daee7876f66f2c523c4d212ffb4be Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:35:34 +0200
Subject: [PATCH 07/26] count files first

---
 apps/remix-ide/ci/flaky.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh
index 21bfa1855b..f0f2ccdfa2 100755
--- a/apps/remix-ide/ci/flaky.sh
+++ b/apps/remix-ide/ci/flaky.sh
@@ -2,6 +2,17 @@
 
 set -e
 
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
+
+# count test files
+fileCount=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | wc -l )
+# if fileCount is 0
+if [ $fileCount -eq 0 ]
+then
+  echo "No flaky tests found"
+  exit 1
+fi
+
 BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
 echo "$BUILD_ID"
 TEST_EXITCODE=0
@@ -15,7 +26,6 @@ sleep 5
 
 npm run build:e2e
 
-TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
 for TESTFILE in $TESTFILES; do
     npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1  || TEST_EXITCODE=1
 done

From 8d1026f82df9201c8f71b2aeafc64de71fce1118 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:36:38 +0200
Subject: [PATCH 08/26] flaky ff

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 34e8d5003d..98d61473cc 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -257,7 +257,7 @@ jobs:
           name: Start Selenium
           command: java -jar /usr/local/bin/selenium.jar
           background: true
-      - run: ./apps/remix-ide/ci/browser_test.sh firefox
+      - run: ./apps/remix-ide/ci/flaky.sh firefox
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:

From c5c1ad2a6e39dbfebfbb23ae4a2d45830a769156 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 18:43:07 +0200
Subject: [PATCH 09/26] build first

---
 apps/remix-ide/ci/flaky.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh
index f0f2ccdfa2..111575a310 100755
--- a/apps/remix-ide/ci/flaky.sh
+++ b/apps/remix-ide/ci/flaky.sh
@@ -2,6 +2,7 @@
 
 set -e
 
+npm run build:e2e
 TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
 
 # count test files
@@ -10,7 +11,7 @@ fileCount=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/test
 if [ $fileCount -eq 0 ]
 then
   echo "No flaky tests found"
-  exit 1
+  exit 0
 fi
 
 BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
@@ -24,8 +25,6 @@ npm run remixd &
 
 sleep 5
 
-npm run build:e2e
-
 for TESTFILE in $TESTFILES; do
     npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch.js $TESTFILE --env=$1  || TEST_EXITCODE=1
 done

From 2947d1631c11bae42075f6103f7480f26c9718dd Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Tue, 3 May 2022 19:00:00 +0200
Subject: [PATCH 10/26] no flaky

---
 .circleci/config.yml                        | 10 ----------
 apps/remix-ide-e2e/src/tests/remixd.test.ts |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 98d61473cc..e544f43993 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -391,7 +391,6 @@ jobs:
             if [ "${CIRCLE_BRANCH}" == "remix_beta" ]; then
               ./apps/remix-ide/ci/deploy_from_travis_remix-beta.sh;
             fi
-
 workflows:
   version: 2
   build_all:
@@ -406,27 +405,18 @@ workflows:
       - lint:
           requires:
             - build
-            - flaky-chrome
-            - flaky-firefox
       - remix-libs:
           requires:
             - build
-            - flaky-chrome
-            - flaky-firefox
       - remix-ide-plugin-api:
           requires:
             - build
-            - flaky-chrome
       - remix-ide-chrome:
           requires:
             - build
-            - flaky-chrome
-            - flaky-firefox
       - remix-ide-firefox:
           requires:
             - build
-            - flaky-chrome
-            - flaky-firefox
       - deploy-remix-live:          
           requires:
             - lint
diff --git a/apps/remix-ide-e2e/src/tests/remixd.test.ts b/apps/remix-ide-e2e/src/tests/remixd.test.ts
index 3c81e8d1ab..8b7d043235 100644
--- a/apps/remix-ide-e2e/src/tests/remixd.test.ts
+++ b/apps/remix-ide-e2e/src/tests/remixd.test.ts
@@ -81,7 +81,7 @@ module.exports = {
       .setSolidityCompilerVersion('soljson-v0.8.0+commit.c7dfd78e.js') // open-zeppelin moved to pragma ^0.8.0
       .testContracts('test_import_node_modules_with_github_import.sol', sources[4]['test_import_node_modules_with_github_import.sol'], ['ERC20', 'test11'])
   },
-  'Static Analysis run with remixd #group3 #flaky': function (browser) {
+  'Static Analysis run with remixd #group3': function (browser) {
     browser.testContracts('test_static_analysis_with_remixd_and_hardhat.sol', sources[5]['test_static_analysis_with_remixd_and_hardhat.sol'], ['test5']).pause(2000)
       .clickLaunchIcon('solidityStaticAnalysis')
       .click('#staticanalysisButton button').pause(4000)

From 9057c0ab44736249b2357f5f0b3e588127dc5e78 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:30:26 +0200
Subject: [PATCH 11/26] test env

---
 .circleci/config.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e544f43993..0a9da6a853 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -37,6 +37,11 @@ jobs:
 
       - run: npm run build:libs
       - run: mkdir persist && zip -r persist/dist.zip dist
+      - run:
+          name: Update PATH and Define Environment Variable at Runtime
+          command: |
+            echo 'export VERY_IMPORTANT="testing"' >> $BASH_ENV
+            source $BASH_ENV
       - persist_to_workspace:
           root: .
           paths:
@@ -241,6 +246,7 @@ jobs:
       - browser-tools/install-geckodriver
       - run:
           command: |
+              echo $VERY_IMPORTANT
               firefox --version
               geckodriver --version
               java -jar /usr/local/bin/selenium.jar --version

From bedb7e4943acdb8ffd6555d6b85b36a30b3d0de1 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:42:38 +0200
Subject: [PATCH 12/26] condition pipelines

---
 .circleci/config.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a9da6a853..ebcf0d7008 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,6 +3,10 @@
 # Check https://circleci.com/docs/2.0/language-javascript/ for more details
 #
 version: 2.1
+parameters:
+  run_flaky_tests:
+    type: boolean
+    default: true
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:
@@ -399,7 +403,8 @@ jobs:
             fi
 workflows:
   version: 2
-  build_all:
+  run_flaky_tests:
+    when: << pipeline.parameters.run_flaky_tests >>
     jobs:
       - build
       - flaky-chrome:
@@ -408,6 +413,10 @@ workflows:
       - flaky-firefox:
           requires:
             - build
+  build_all:
+    when: << !pipeline.parameters.run_flaky_tests >>
+    jobs:
+      - build
       - lint:
           requires:
             - build

From 11cc87328ba535071530f556a84f623707a03f66 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:44:11 +0200
Subject: [PATCH 13/26] try

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ebcf0d7008..77317246f8 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -414,7 +414,7 @@ workflows:
           requires:
             - build
   build_all:
-    when: << !pipeline.parameters.run_flaky_tests >>
+    when: << pipeline.parameters.run_flaky_tests >>
     jobs:
       - build
       - lint:

From a98c85822a5a8f4f68615a266d199559871d76ad Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:47:14 +0200
Subject: [PATCH 14/26] unless

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 77317246f8..7ee9b751eb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -414,7 +414,7 @@ workflows:
           requires:
             - build
   build_all:
-    when: << pipeline.parameters.run_flaky_tests >>
+    unless: << pipeline.parameters.run_flaky_tests >>
     jobs:
       - build
       - lint:

From 842fd5eb7dd0debb90dc0d88e1bca5d03e6bb252 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:53:13 +0200
Subject: [PATCH 15/26] << parameters.run_flaky_parallelism >>

---
 .circleci/config.yml | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7ee9b751eb..a4399961ff 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -7,6 +7,9 @@ parameters:
   run_flaky_tests:
     type: boolean
     default: true
+  run_flaky_parallelism:
+    type: integer
+    default: 40
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:
@@ -100,8 +103,8 @@ jobs:
       - run: npm i
       - run: cd dist/libs/remix-tests && npm install
       - run: npm run test:libs
-
-  flaky-chrome:
+          
+  remix-ide-chrome:
     docker:
       # specify the version you desire here
       - image: cimg/node:14.17.6-browsers
@@ -138,13 +141,13 @@ jobs:
           name: Start Selenium
           command: java -jar /usr/local/bin/selenium.jar
           background: true
-      - run: ./apps/remix-ide/ci/flaky.sh chrome
+      - run: ./apps/remix-ide/ci/browser_test.sh chrome
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:
           path: ./reports/screenshots
-          
-  remix-ide-chrome:
+
+  flaky-chrome:
     docker:
       # specify the version you desire here
       - image: cimg/node:14.17.6-browsers
@@ -159,7 +162,7 @@ jobs:
       - COMMIT_AUTHOR: "Circle CI"
     working_directory: ~/remix-project
 
-    parallelism: 80
+    parallelism: << parameters.run_flaky_parallelism >>
     steps:
       - browser-tools/install-chrome
       - browser-tools/install-chromedriver
@@ -181,7 +184,7 @@ jobs:
           name: Start Selenium
           command: java -jar /usr/local/bin/selenium.jar
           background: true
-      - run: ./apps/remix-ide/ci/browser_test.sh chrome
+      - run: ./apps/remix-ide/ci/flaky.sh chrome
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:
@@ -244,7 +247,7 @@ jobs:
       - COMMIT_AUTHOR: "Circle CI"
     working_directory: ~/remix-project
 
-    parallelism: 80
+    parallelism: << parameters.run_flaky_parallelism >>
     steps:
       - browser-tools/install-firefox
       - browser-tools/install-geckodriver

From 7fe8a8a6cd5cb23669dbf3de8f12652b9770db45 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:55:43 +0200
Subject: [PATCH 16/26] << parameters.run_flaky_parallelism >>

---
 .circleci/config.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a4399961ff..8c9626722b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -7,9 +7,6 @@ parameters:
   run_flaky_tests:
     type: boolean
     default: true
-  run_flaky_parallelism:
-    type: integer
-    default: 40
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:
@@ -162,7 +159,7 @@ jobs:
       - COMMIT_AUTHOR: "Circle CI"
     working_directory: ~/remix-project
 
-    parallelism: << parameters.run_flaky_parallelism >>
+    parallelism: 80
     steps:
       - browser-tools/install-chrome
       - browser-tools/install-chromedriver
@@ -247,7 +244,7 @@ jobs:
       - COMMIT_AUTHOR: "Circle CI"
     working_directory: ~/remix-project
 
-    parallelism: << parameters.run_flaky_parallelism >>
+    parallelism: 80
     steps:
       - browser-tools/install-firefox
       - browser-tools/install-geckodriver

From 2b2abaeca60ffc4efa192eba68b0de8e5ab5272e Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 10:58:27 +0200
Subject: [PATCH 17/26] support api testing

---
 .circleci/config.yml                       | 6 ------
 apps/remix-ide-e2e/src/tests/plugin_api.ts | 2 +-
 apps/remix-ide/ci/flaky.sh                 | 1 +
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8c9626722b..cc732d6931 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -41,11 +41,6 @@ jobs:
 
       - run: npm run build:libs
       - run: mkdir persist && zip -r persist/dist.zip dist
-      - run:
-          name: Update PATH and Define Environment Variable at Runtime
-          command: |
-            echo 'export VERY_IMPORTANT="testing"' >> $BASH_ENV
-            source $BASH_ENV
       - persist_to_workspace:
           root: .
           paths:
@@ -250,7 +245,6 @@ jobs:
       - browser-tools/install-geckodriver
       - run:
           command: |
-              echo $VERY_IMPORTANT
               firefox --version
               geckodriver --version
               java -jar /usr/local/bin/selenium.jar --version
diff --git a/apps/remix-ide-e2e/src/tests/plugin_api.ts b/apps/remix-ide-e2e/src/tests/plugin_api.ts
index 8a7ed9a9ec..eb18137324 100644
--- a/apps/remix-ide-e2e/src/tests/plugin_api.ts
+++ b/apps/remix-ide-e2e/src/tests/plugin_api.ts
@@ -277,7 +277,7 @@ module.exports = {
     await clickAndCheckLog(browser, 'fileManager:readFile', 'test', null, 'testrename.txt')
   },
 
-  'Should create empty workspace #group2': async function (browser: NightwatchBrowser) {
+  'Should create empty workspace #flaky #group2': async function (browser: NightwatchBrowser) {
     await clickAndCheckLog(browser, 'filePanel:createWorkspace', null, null, ['emptyworkspace', true])
     await clickAndCheckLog(browser, 'filePanel:getCurrentWorkspace', { name: 'emptyworkspace', isLocalhost: false, absolutePath: '.workspaces/emptyworkspace' }, null, null)
     await clickAndCheckLog(browser, 'fileManager:readdir', {}, null, '/')
diff --git a/apps/remix-ide/ci/flaky.sh b/apps/remix-ide/ci/flaky.sh
index 111575a310..8a2dac3e73 100755
--- a/apps/remix-ide/ci/flaky.sh
+++ b/apps/remix-ide/ci/flaky.sh
@@ -22,6 +22,7 @@ npm run ganache-cli &
 npm run serve:production &
 echo 'sharing folder: ' $PWD '/apps/remix-ide/contracts' &
 npm run remixd &
+npx nx serve remix-ide-e2e-src-local-plugin &
 
 sleep 5
 

From 518e8531da5a9c9f2e346fb0734f04ca2bf5579d Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 11:10:19 +0200
Subject: [PATCH 18/26] more testing

---
 apps/remix-ide-e2e/src/tests/remixd.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/remix-ide-e2e/src/tests/remixd.test.ts b/apps/remix-ide-e2e/src/tests/remixd.test.ts
index 8b7d043235..3c81e8d1ab 100644
--- a/apps/remix-ide-e2e/src/tests/remixd.test.ts
+++ b/apps/remix-ide-e2e/src/tests/remixd.test.ts
@@ -81,7 +81,7 @@ module.exports = {
       .setSolidityCompilerVersion('soljson-v0.8.0+commit.c7dfd78e.js') // open-zeppelin moved to pragma ^0.8.0
       .testContracts('test_import_node_modules_with_github_import.sol', sources[4]['test_import_node_modules_with_github_import.sol'], ['ERC20', 'test11'])
   },
-  'Static Analysis run with remixd #group3': function (browser) {
+  'Static Analysis run with remixd #group3 #flaky': function (browser) {
     browser.testContracts('test_static_analysis_with_remixd_and_hardhat.sol', sources[5]['test_static_analysis_with_remixd_and_hardhat.sol'], ['test5']).pause(2000)
       .clickLaunchIcon('solidityStaticAnalysis')
       .click('#staticanalysisButton button').pause(4000)

From 6f17be0f6ee82c13fb13946cb607a9893432272a Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 11:10:29 +0200
Subject: [PATCH 19/26] more tests

---
 apps/remix-ide-e2e/src/tests/terminal.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts
index f262005ba1..55031eafcc 100644
--- a/apps/remix-ide-e2e/src/tests/terminal.test.ts
+++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts
@@ -194,7 +194,7 @@ module.exports = {
       .assert.visible('*[data-id="autoCompletePopUpAutoCompleteItem"]')
   },
 
-  'Should run a script right after compilation #group6': function (browser: NightwatchBrowser) {
+  'Should run a script right after compilation #group6 #flaky': function (browser: NightwatchBrowser) {
     browser
       .addFile('contracts/storage.sol', { content: scriptAutoExec.contract } )
       .addFile('scripts/deploy_storage.js', { content: scriptAutoExec.script } )

From 3ab1762d52f9c179ece1f63ccc805ea715317d28 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 11:15:43 +0200
Subject: [PATCH 20/26] no api test

---
 apps/remix-ide-e2e/src/tests/plugin_api.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/remix-ide-e2e/src/tests/plugin_api.ts b/apps/remix-ide-e2e/src/tests/plugin_api.ts
index eb18137324..8a7ed9a9ec 100644
--- a/apps/remix-ide-e2e/src/tests/plugin_api.ts
+++ b/apps/remix-ide-e2e/src/tests/plugin_api.ts
@@ -277,7 +277,7 @@ module.exports = {
     await clickAndCheckLog(browser, 'fileManager:readFile', 'test', null, 'testrename.txt')
   },
 
-  'Should create empty workspace #flaky #group2': async function (browser: NightwatchBrowser) {
+  'Should create empty workspace #group2': async function (browser: NightwatchBrowser) {
     await clickAndCheckLog(browser, 'filePanel:createWorkspace', null, null, ['emptyworkspace', true])
     await clickAndCheckLog(browser, 'filePanel:getCurrentWorkspace', { name: 'emptyworkspace', isLocalhost: false, absolutePath: '.workspaces/emptyworkspace' }, null, null)
     await clickAndCheckLog(browser, 'fileManager:readdir', {}, null, '/')

From ac3f493e14c74ee9dac85116f7ff32dae85585ed Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 11:48:25 +0200
Subject: [PATCH 21/26] rm flaky

---
 apps/remix-ide-e2e/src/tests/remixd.test.ts   | 2 +-
 apps/remix-ide-e2e/src/tests/terminal.test.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/remix-ide-e2e/src/tests/remixd.test.ts b/apps/remix-ide-e2e/src/tests/remixd.test.ts
index 3c81e8d1ab..8b7d043235 100644
--- a/apps/remix-ide-e2e/src/tests/remixd.test.ts
+++ b/apps/remix-ide-e2e/src/tests/remixd.test.ts
@@ -81,7 +81,7 @@ module.exports = {
       .setSolidityCompilerVersion('soljson-v0.8.0+commit.c7dfd78e.js') // open-zeppelin moved to pragma ^0.8.0
       .testContracts('test_import_node_modules_with_github_import.sol', sources[4]['test_import_node_modules_with_github_import.sol'], ['ERC20', 'test11'])
   },
-  'Static Analysis run with remixd #group3 #flaky': function (browser) {
+  'Static Analysis run with remixd #group3': function (browser) {
     browser.testContracts('test_static_analysis_with_remixd_and_hardhat.sol', sources[5]['test_static_analysis_with_remixd_and_hardhat.sol'], ['test5']).pause(2000)
       .clickLaunchIcon('solidityStaticAnalysis')
       .click('#staticanalysisButton button').pause(4000)
diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts
index 55031eafcc..f262005ba1 100644
--- a/apps/remix-ide-e2e/src/tests/terminal.test.ts
+++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts
@@ -194,7 +194,7 @@ module.exports = {
       .assert.visible('*[data-id="autoCompletePopUpAutoCompleteItem"]')
   },
 
-  'Should run a script right after compilation #group6 #flaky': function (browser: NightwatchBrowser) {
+  'Should run a script right after compilation #group6': function (browser: NightwatchBrowser) {
     browser
       .addFile('contracts/storage.sol', { content: scriptAutoExec.contract } )
       .addFile('scripts/deploy_storage.js', { content: scriptAutoExec.script } )

From 9ddd571818a564d6d2775ba228a4008d06806952 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Wed, 4 May 2022 11:49:20 +0200
Subject: [PATCH 22/26] set to default

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cc732d6931..11c6ede82b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ version: 2.1
 parameters:
   run_flaky_tests:
     type: boolean
-    default: true
+    default: false
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:

From a83f47c5d0492f649e91c1cdaea55ef16e247aec Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Thu, 5 May 2022 07:30:01 +0200
Subject: [PATCH 23/26] add browser logs

---
 apps/remix-ide-e2e/src/buildGroupTests.js       |  1 -
 .../src/commands/getBrowserLogs.ts              | 17 +++++++++++++++++
 apps/remix-ide-e2e/src/types/index.d.ts         |  1 +
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 apps/remix-ide-e2e/src/commands/getBrowserLogs.ts

diff --git a/apps/remix-ide-e2e/src/buildGroupTests.js b/apps/remix-ide-e2e/src/buildGroupTests.js
index 0c589bd5fd..7127462125 100644
--- a/apps/remix-ide-e2e/src/buildGroupTests.js
+++ b/apps/remix-ide-e2e/src/buildGroupTests.js
@@ -47,7 +47,6 @@ function createFlakyTestFiles(file, text) {
   lines.forEach((line, index) => {
     // if line contains #flaky
     if (line.includes('#flaky')) {
-      console.log(line)
       const matches = line.match(/group\d+/g)
       const unique = matches.filter(onlyUnique)
       createFiles(file, matches, true)
diff --git a/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts b/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts
new file mode 100644
index 0000000000..d2e07cb7a4
--- /dev/null
+++ b/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts
@@ -0,0 +1,17 @@
+import { NightwatchBrowser } from 'nightwatch'
+import EventEmitter from 'events'
+
+class GetBrowserLogs extends EventEmitter {
+    command(this: NightwatchBrowser): NightwatchBrowser {
+        this.api.getLog('browser', function (logs) {
+            logs.forEach(function (log) {
+                console.log(log)
+            }
+            )
+        }).perform(() => {
+            this.emit('complete')
+        })
+        return this
+    }
+}
+module.exports = GetBrowserLogs
\ No newline at end of file
diff --git a/apps/remix-ide-e2e/src/types/index.d.ts b/apps/remix-ide-e2e/src/types/index.d.ts
index 9abe5c2174..e8698d4407 100644
--- a/apps/remix-ide-e2e/src/types/index.d.ts
+++ b/apps/remix-ide-e2e/src/types/index.d.ts
@@ -61,6 +61,7 @@ declare module 'nightwatch' {
         acceptAndRemember (this: NightwatchBrowser, remember: boolean, accept: boolean): NightwatchBrowser
         clearConsole (this: NightwatchBrowser): NightwatchBrowser
         clearTransactions (this: NightwatchBrowser): NightwatchBrowser
+        getBrowserLogs (this: NightwatchBrowser): NightwatchBrowser
         currentSelectedFileIs (name: string): NightwatchBrowser
     }
 

From aeea09dab1e02f974aa395fab1d9261e0e8a6531 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Thu, 5 May 2022 08:33:32 +0200
Subject: [PATCH 24/26] turn on flaky

---
 .circleci/config.yml                                  | 2 +-
 apps/remix-ide-e2e/src/tests/specialFunctions.test.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 11c6ede82b..cc732d6931 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ version: 2.1
 parameters:
   run_flaky_tests:
     type: boolean
-    default: false
+    default: true
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:
diff --git a/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts b/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts
index bcdc3c4977..0271b0cdd8 100644
--- a/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts
+++ b/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts
@@ -165,7 +165,7 @@ module.exports = {
         })
       })
   },
-  'Use special functions receive/fallback - receive and fallback are declared, sending data and wei #group6': function (browser: NightwatchBrowser) {
+  'Use special functions receive/fallback - receive and fallback are declared, sending data and wei #group6 #flaky': function (browser: NightwatchBrowser) {
     browser.waitForElementVisible('#icon-panel', 10000)
       .testContracts('receiveAndFallbackBothPayable.sol', sources[4]['receiveAndFallbackBothPayable.sol'], ['CheckSpecials'])
       .clickLaunchIcon('udapp')

From b2241a3a9161f5f9f0360ae8636e654efd25fa72 Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Thu, 5 May 2022 08:35:24 +0200
Subject: [PATCH 25/26] Revert "turn on flaky"

This reverts commit aeea09dab1e02f974aa395fab1d9261e0e8a6531.
---
 .circleci/config.yml                                  | 2 +-
 apps/remix-ide-e2e/src/tests/specialFunctions.test.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cc732d6931..11c6ede82b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ version: 2.1
 parameters:
   run_flaky_tests:
     type: boolean
-    default: true
+    default: false
 orbs:
   browser-tools: circleci/browser-tools@1.2.3
 jobs:
diff --git a/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts b/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts
index 0271b0cdd8..bcdc3c4977 100644
--- a/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts
+++ b/apps/remix-ide-e2e/src/tests/specialFunctions.test.ts
@@ -165,7 +165,7 @@ module.exports = {
         })
       })
   },
-  'Use special functions receive/fallback - receive and fallback are declared, sending data and wei #group6 #flaky': function (browser: NightwatchBrowser) {
+  'Use special functions receive/fallback - receive and fallback are declared, sending data and wei #group6': function (browser: NightwatchBrowser) {
     browser.waitForElementVisible('#icon-panel', 10000)
       .testContracts('receiveAndFallbackBothPayable.sol', sources[4]['receiveAndFallbackBothPayable.sol'], ['CheckSpecials'])
       .clickLaunchIcon('udapp')

From f0ee7bcf06f580dc874c89d59fce00b477af6a2e Mon Sep 17 00:00:00 2001
From: filip mertens <filip.mertens@ethereum.org>
Date: Thu, 5 May 2022 11:22:11 +0200
Subject: [PATCH 26/26] no logs in ff

---
 apps/remix-ide-e2e/src/commands/getBrowserLogs.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts b/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts
index d2e07cb7a4..d1da53488a 100644
--- a/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts
+++ b/apps/remix-ide-e2e/src/commands/getBrowserLogs.ts
@@ -4,10 +4,12 @@ import EventEmitter from 'events'
 class GetBrowserLogs extends EventEmitter {
     command(this: NightwatchBrowser): NightwatchBrowser {
         this.api.getLog('browser', function (logs) {
-            logs.forEach(function (log) {
-                console.log(log)
+            if (logs && Array.isArray(logs)) {
+                logs.forEach(function (log) {
+                    console.log(log)
+                }
+                )
             }
-            )
         }).perform(() => {
             this.emit('complete')
         })