From 0fd0e2bf6e623ba15b72a78c684a7e32925aaa06 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Wed, 31 Aug 2022 09:20:41 +0200 Subject: [PATCH 01/10] set to flaky --- .circleci/config.yml | 2 +- apps/remix-ide-e2e/src/tests/terminal.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2095b373fe..3c9bb034fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ version: 2.1 parameters: run_flaky_tests: type: boolean - default: false + default: true orbs: browser-tools: circleci/browser-tools@1.3.0 jobs: diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index baf0639cc5..6fdacfb4e8 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -225,7 +225,7 @@ module.exports = { .waitForElementContainsText('*[data-id="terminalJournal"]', '"hex":"0x025cd8"', 120000) }, - 'Should listen on all transactions #group8': function (browser: NightwatchBrowser) { + 'Should listen on all transactions #group8 #flaky': function (browser: NightwatchBrowser) { browser .clickLaunchIcon('udapp') // connect to mainnet .switchEnvironment('External Http Provider') From c4cb4f7e5bcdeeeacbb0e39bec5ff62bdfd317d0 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Wed, 31 Aug 2022 15:01:59 +0200 Subject: [PATCH 02/10] use waituntil fix url wait add checks for connection add pause before clearing use css on start rm resolve refactor use timer add timeout selector strategy usecss abortfail slower rm end tests mv no logs url lint identifier unflaky from fail correctly increase timeout mv interval test with ganache only unused var --- .../commands/connectToExternalHttpProvider.ts | 42 ++++++++ .../src/commands/switchEnvironment.ts | 2 +- apps/remix-ide-e2e/src/tests/terminal.test.ts | 96 ++++++++++++------- apps/remix-ide-e2e/src/types/index.d.ts | 1 + 4 files changed, 106 insertions(+), 35 deletions(-) create mode 100644 apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts diff --git a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts new file mode 100644 index 0000000000..10aa2f72e4 --- /dev/null +++ b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts @@ -0,0 +1,42 @@ +import { NightwatchBrowser } from 'nightwatch' +import EventEmitter from 'events' + +class ConnectToExternalHttpProvider extends EventEmitter { + command(this: NightwatchBrowser, url: string, identifier: string): NightwatchBrowser { + this.api.element('xpath', `//*[@class='udapp_environment' and contains(.,'${identifier}')]`, + (result) => { + console.log('ConnectToExternalHttpProvider: ' + result.status, result.value) + if (result.status as any === -1 ) { + console.log("No connection to external provider found. Adding one.", url) + browser + .click({ + locateStrategy: 'css selector', + selector: '[data-id="basic-http-provider-modal-footer-ok-react"]', + abortOnFailure: false, + suppressNotFoundErrors: true, + timeout: 5000 + }) + .switchEnvironment('External Http Provider') + .waitForElementPresent('[data-id="basic-http-provider-modal-footer-ok-react"]') + .execute(() => { + (document.querySelector('*[data-id="basic-http-providerModalDialogContainer-react"] input[data-id="modalDialogCustomPromp"]') as any).focus() + }, [], () => { }) + .setValue('[data-id="modalDialogCustomPromp"]', url) + .modalFooterOKClick('basic-http-provider') + .perform((done) => { + done() + this.emit('complete') + }) + } else { + this.api.perform((done) => { + done() + this.emit('complete') + }) + } + } + ) + return this + } +} + +module.exports = ConnectToExternalHttpProvider diff --git a/apps/remix-ide-e2e/src/commands/switchEnvironment.ts b/apps/remix-ide-e2e/src/commands/switchEnvironment.ts index 03564ced41..a2fc655cac 100644 --- a/apps/remix-ide-e2e/src/commands/switchEnvironment.ts +++ b/apps/remix-ide-e2e/src/commands/switchEnvironment.ts @@ -3,7 +3,7 @@ import EventEmitter from 'events' class switchEnvironment extends EventEmitter { command (this: NightwatchBrowser, provider: string): NightwatchBrowser { - this.api.waitForElementVisible('[data-id="settingsSelectEnvOptions"]') + this.api.useCss().waitForElementVisible('[data-id="settingsSelectEnvOptions"]') .click('[data-id="settingsSelectEnvOptions"] button') .waitForElementVisible(`[data-id="dropdown-item-${provider}"]`) .click(`[data-id="dropdown-item-${provider}"]`) diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 6fdacfb4e8..0721a61f13 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -194,8 +194,8 @@ module.exports = { 'Should run a script right after compilation #group6': function (browser: NightwatchBrowser) { browser - .addFile('contracts/storage.sol', { content: scriptAutoExec.contract } ) - .addFile('scripts/deploy_storage.js', { content: scriptAutoExec.script } ) + .addFile('contracts/storage.sol', { content: scriptAutoExec.contract }) + .addFile('scripts/deploy_storage.js', { content: scriptAutoExec.script }) .openFile('contracts/storage.sol') .sendKeys('body', [browser.Keys.CONTROL, browser.Keys.SHIFT, 's']) .pause(15000) @@ -209,12 +209,12 @@ module.exports = { .waitForElementPresent('[data-id="basic-http-provider-modal-footer-ok-react"]') .execute(() => { (document.querySelector('*[data-id="basic-http-providerModalDialogContainer-react"] input[data-id="modalDialogCustomPromp"]') as any).focus() - }, [], () => {}) + }, [], () => { }) .setValue('[data-id="modalDialogCustomPromp"]', 'https://remix-goerli.ethdevops.io') .modalFooterOKClick('basic-http-provider') .clickLaunchIcon('filePanel') .openFile('README.txt') - .addFile('scripts/log_tx_block.js', { content: scriptBlockAndTransaction } ) + .addFile('scripts/log_tx_block.js', { content: scriptBlockAndTransaction }) .pause(1000) .executeScriptInTerminal('remix.execute(\'scripts/log_tx_block.js\')') // check if the input of the transaction is being logged (web3 call) @@ -223,39 +223,67 @@ module.exports = { .waitForElementContainsText('*[data-id="terminalJournal"]', '0x00000000000000000000000000100000000000000000020000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000040000000060000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000100000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000001', 120000) // check if the logsBloom is being logged (ethers.js call) .waitForElementContainsText('*[data-id="terminalJournal"]', '"hex":"0x025cd8"', 120000) - }, - - 'Should listen on all transactions #group8 #flaky': function (browser: NightwatchBrowser) { - browser - .clickLaunchIcon('udapp') // connect to mainnet - .switchEnvironment('External Http Provider') - .waitForElementPresent('[data-id="basic-http-provider-modal-footer-ok-react"]') - .execute(() => { - (document.querySelector('*[data-id="basic-http-providerModalDialogContainer-react"] input[data-id="modalDialogCustomPromp"]') as any).focus() - }, [], () => {}) - .setValue('[data-id="modalDialogCustomPromp"]', 'https://rpc.archivenode.io/e50zmkroshle2e2e50zm0044i7ao04ym') - .modalFooterOKClick('basic-http-provider') - .click('[data-id="terminalClearConsole"]') // clear the console - .click('[data-id="listenNetworkCheckInput"]') // start to listen - .waitForElementContainsText('*[data-id="terminalJournal"]', 'from:', 200000) - .waitForElementContainsText('*[data-id="terminalJournal"]', 'to:', 200000) - .click('[data-id="terminalClearConsole"]') // clear the console - .waitForElementContainsText('*[data-id="terminalJournal"]', 'from:', 200000) - .waitForElementContainsText('*[data-id="terminalJournal"]', 'to:', 200000) - .click('[data-id="listenNetworkCheckInput"]') // stop to listen - .pause(30000) - .click('[data-id="terminalClearConsole"]') // clear the console - .pause(5000) - .click('[data-id="terminalClearConsole"]') // clear the console - .pause(20000) - .execute(function () { - return (document.querySelector('[data-id="terminalJournal"]') as any).innerText - }, [], function (result) { - browser.assert.equal(result.value, '', 'terminal log should be empty') + }, + + 'Should listen on all transactions #group8 #flaky': function (browser: NightwatchBrowser) { + const url = 'http://127.0.0.1:8545' + const identifier = 'Custom' + browser + .clickLaunchIcon('udapp') // connect to mainnet + .connectToExternalHttpProvider(url, identifier) + .openFile('contracts') + .openFile('contracts/1_Storage.sol') + .pause(4000) + .clickLaunchIcon('solidity') + .click({ + selector: '*[data-id="compilerContainerCompileAndRunBtn"]', + }) + .pause(10000) + .waitForElementNotPresent({ + locateStrategy: 'xpath', + selector: "//*[@class='remix_ui_terminal_log' and contains(.,'to:') and contains(.,'from:')]", + timeout: 120000 + }) + .click({ + selector: '[data-id="listenNetworkCheckInput"]', + }) // start to listen + .pause(5000) + .findElements( + { + locateStrategy: 'xpath', + selector: "//*[@class='remix_ui_terminal_log' and contains(.,'to:') and contains(.,'from:')]", + timeout: 120000, + } + , async (result) => { + if (Array.isArray(result.value) && result.value.length == 2) { + console.log('Found ' + result.value.length + ' transactions') + browser + .click({ + selector: '[data-id="listenNetworkCheckInput"]', + }) + .click({ + selector: '*[data-id="terminalClearConsole"]', + }) + .click({ + selector: '*[data-id="compilerContainerCompileAndRunBtn"]', + }) + .pause(10000) + .waitForElementNotPresent({ + locateStrategy: 'xpath', + selector: "//*[@class='remix_ui_terminal_log' and contains(.,'to:') and contains(.,'from:')]", + timeout: 120000 + }) + .end() + } else { + browser + .assert.fail('No transaction found') + .end() + } }) - } + } } + const asyncAwait = ` var p = function () { return new Promise(function (resolve, reject) { diff --git a/apps/remix-ide-e2e/src/types/index.d.ts b/apps/remix-ide-e2e/src/types/index.d.ts index 4f5568e773..13c0c22d8b 100644 --- a/apps/remix-ide-e2e/src/types/index.d.ts +++ b/apps/remix-ide-e2e/src/types/index.d.ts @@ -65,6 +65,7 @@ declare module 'nightwatch' { currentSelectedFileIs (name: string): NightwatchBrowser, switchWorkspace: (workspaceName: string) => NightwatchBrowser switchEnvironment: (provider: string) => NightwatchBrowser + connectToExternalHttpProvider: (url: string, identifier: string) => NightwatchBrowser } export interface NightwatchBrowser { From f15ba7f24524f8a321e9155e1f17537514ee3a64 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Thu, 1 Sep 2022 09:51:41 +0200 Subject: [PATCH 03/10] gt zero --- apps/remix-ide-e2e/src/tests/terminal.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 0721a61f13..607a7c99f6 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -255,7 +255,7 @@ module.exports = { timeout: 120000, } , async (result) => { - if (Array.isArray(result.value) && result.value.length == 2) { + if (Array.isArray(result.value) && result.value.length > 0) { console.log('Found ' + result.value.length + ' transactions') browser .click({ From 3735a840793152eae5464d0a2f5f71d099f77c33 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Thu, 1 Sep 2022 10:00:21 +0200 Subject: [PATCH 04/10] click and run --- apps/remix-ide-e2e/src/tests/terminal.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 607a7c99f6..1c1d3b670c 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -247,7 +247,10 @@ module.exports = { .click({ selector: '[data-id="listenNetworkCheckInput"]', }) // start to listen - .pause(5000) + .click({ + selector: '*[data-id="compilerContainerCompileAndRunBtn"]', + }) + .pause(10000) .findElements( { locateStrategy: 'xpath', From f19eb227cf56674df72e42b1a29a03fcfe5976c7 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Thu, 1 Sep 2022 10:07:35 +0200 Subject: [PATCH 05/10] rm flaky --- .circleci/config.yml | 2 +- apps/remix-ide-e2e/src/tests/terminal.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c9bb034fc..2095b373fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ version: 2.1 parameters: run_flaky_tests: type: boolean - default: true + default: false orbs: browser-tools: circleci/browser-tools@1.3.0 jobs: diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 1c1d3b670c..f2797c8ece 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -225,7 +225,7 @@ module.exports = { .waitForElementContainsText('*[data-id="terminalJournal"]', '"hex":"0x025cd8"', 120000) }, - 'Should listen on all transactions #group8 #flaky': function (browser: NightwatchBrowser) { + 'Should listen on all transactions #group8': function (browser: NightwatchBrowser) { const url = 'http://127.0.0.1:8545' const identifier = 'Custom' browser From fb08e3a48b390f9a43d38ed345c05b239795e628 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Fri, 2 Sep 2022 10:24:57 +0200 Subject: [PATCH 06/10] Update connectToExternalHttpProvider.ts --- apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts index 10aa2f72e4..2d4e7fb35f 100644 --- a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts +++ b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts @@ -5,7 +5,6 @@ class ConnectToExternalHttpProvider extends EventEmitter { command(this: NightwatchBrowser, url: string, identifier: string): NightwatchBrowser { this.api.element('xpath', `//*[@class='udapp_environment' and contains(.,'${identifier}')]`, (result) => { - console.log('ConnectToExternalHttpProvider: ' + result.status, result.value) if (result.status as any === -1 ) { console.log("No connection to external provider found. Adding one.", url) browser From 3712321f082283d35c23cf54d2e8fac179faea12 Mon Sep 17 00:00:00 2001 From: filip mertens Date: Fri, 2 Sep 2022 10:55:09 +0200 Subject: [PATCH 07/10] use this --- .../src/commands/connectToExternalHttpProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts index 2d4e7fb35f..e7a2595d05 100644 --- a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts +++ b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts @@ -5,9 +5,9 @@ class ConnectToExternalHttpProvider extends EventEmitter { command(this: NightwatchBrowser, url: string, identifier: string): NightwatchBrowser { this.api.element('xpath', `//*[@class='udapp_environment' and contains(.,'${identifier}')]`, (result) => { - if (result.status as any === -1 ) { + if (result.status as any === -1) { console.log("No connection to external provider found. Adding one.", url) - browser + this .click({ locateStrategy: 'css selector', selector: '[data-id="basic-http-provider-modal-footer-ok-react"]', From 1329ed4007095dd289d56502d6c7f3a626097979 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Fri, 2 Sep 2022 11:00:43 +0200 Subject: [PATCH 08/10] Update terminal.test.ts --- apps/remix-ide-e2e/src/tests/terminal.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/tests/terminal.test.ts b/apps/remix-ide-e2e/src/tests/terminal.test.ts index 2f5115b5ba..6822025cca 100644 --- a/apps/remix-ide-e2e/src/tests/terminal.test.ts +++ b/apps/remix-ide-e2e/src/tests/terminal.test.ts @@ -235,7 +235,6 @@ module.exports = { .connectToExternalHttpProvider(url, identifier) .openFile('contracts') .openFile('contracts/1_Storage.sol') - .pause(4000) .clickLaunchIcon('solidity') .click({ selector: '*[data-id="compilerContainerCompileAndRunBtn"]', From cde141708d3998a7d6b3abf44be866a5b9d3324b Mon Sep 17 00:00:00 2001 From: filip mertens Date: Fri, 2 Sep 2022 11:12:50 +0200 Subject: [PATCH 09/10] use this api --- .../remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts index e7a2595d05..da457bbfc2 100644 --- a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts +++ b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts @@ -7,7 +7,7 @@ class ConnectToExternalHttpProvider extends EventEmitter { (result) => { if (result.status as any === -1) { console.log("No connection to external provider found. Adding one.", url) - this + this.api .click({ locateStrategy: 'css selector', selector: '[data-id="basic-http-provider-modal-footer-ok-react"]', From 504e49490b0f3388a4df4e054834b300ec44cd02 Mon Sep 17 00:00:00 2001 From: filip mertens Date: Fri, 2 Sep 2022 11:14:41 +0200 Subject: [PATCH 10/10] back to browser --- .../remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts index da457bbfc2..05d9256bf6 100644 --- a/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts +++ b/apps/remix-ide-e2e/src/commands/connectToExternalHttpProvider.ts @@ -7,7 +7,7 @@ class ConnectToExternalHttpProvider extends EventEmitter { (result) => { if (result.status as any === -1) { console.log("No connection to external provider found. Adding one.", url) - this.api + browser .click({ locateStrategy: 'css selector', selector: '[data-id="basic-http-provider-modal-footer-ok-react"]',