chore(test): fix flaky test cases (#3314)

* fix(test): fixed flaky reconnecting test

cc2eff2 introduced different behavior depending on whether socket reconnected or browser reconnected after reload. This lead to the reconnecting test being flaky as, depending on the timing, it will trigger second test run and result in unexpected output.

* fix(test): remove broken monitor action for Cucumber

The issue is that it resulted in the following sequence of actions:
- run karma start
- run karma run
- and then instantly kill karma start command

This resulted in flaky tests because the kill could race, resulting in
incomplete output. In fact test was unnecessary complicated as it was
enough to use runOut, which asserts output of the karma run command.

* fix(test): guard from repeated executions of the karma run

'data' event handler may be called multiple times, which will result in
multiple calls to karma run. To take it even further, its execution is
delayed by setTimeout, which means that subsequent executions will run
while next test scenario is in progress and may mess it up. To prevent
this make sure that karma run is executed only once independently of how
many time 'data' event is fired.

Author: devoto13

test/e2e/error.feature

@@ -18,7 +18,7 @@ Feature: Error Display
       """
       SyntaxError: Unexpected token }
       """
-  Scenario: Single-run Syntax Error in a test file
+  Scenario: Not single-run Syntax Error in a test file
     Given a configuration with:
       """
       files = ['error/test.js', 'error/under-test.js'];
@@ -29,10 +29,8 @@ Feature: Error Display
       ];
       singleRun = false;
       """
-    When I monitor Karma
-    And I stop when the log contains 'SyntaxError'
+    When I runOut Karma
     Then it fails with like:
       """
       SyntaxError: Unexpected token }
       """
-    And I stop a server programmatically

test/e2e/step_definitions/core_steps.js

@@ -64,7 +64,8 @@ cucumber.defineSupportCode((a) => {
         }
 
         const runOut = command === 'runOut'
-        if (command === 'run' || command === 'runOut' || command === 'monitor') {
+        if (command === 'run' || command === 'runOut') {
+          let isRun = false
           this.child = spawn('' + runtimePath, ['start', '--log-level', 'warn', configFile])
           const done = () => {
             cleansingNeeded = true
@@ -84,22 +85,24 @@ cucumber.defineSupportCode((a) => {
           this.child.stdout.on('data', (chunk) => {
             this.lastRun.stdout += chunk.toString()
             const cmd = runtimePath + ' run ' + configFile + ' ' + additionalArgs
-            setTimeout(() => {
-              exec(cmd, {
-                cwd: baseDir
-              }, (error, stdout) => {
-                if (error) {
-                  this.lastRun.error = error
-                }
-                if (runOut) {
-                  this.lastRun.stdout = stdout
-                }
-                done()
-              })
-              if (command === 'monitor') {
-                done()
-              }
-            }, 1000)
+            if (!isRun) {
+              isRun = true
+
+              setTimeout(() => {
+                exec(cmd, {
+                  cwd: baseDir
+                }, (error, stdout, stderr) => {
+                  if (error) {
+                    this.lastRun.error = error
+                  }
+                  if (runOut) {
+                    this.lastRun.stdout = stdout
+                    this.lastRun.stderr = stderr
+                  }
+                  done()
+                })
+              }, 1000)
+            }
           })
         } else {
           executor((error, stdout, stderr) => {
@@ -132,8 +135,8 @@ cucumber.defineSupportCode((a) => {
     setTimeout(function () {
       stopper.stop(_this.configFile, function (exitCode) {
         _this.stopperExitCode = exitCode
+        callback()
       })
-      callback()
     }, 1000)
   })
 
@@ -160,7 +163,7 @@ cucumber.defineSupportCode((a) => {
 
   defineParameterType({
     name: 'command',
-    regexp: /run|runOut|start|init|stop|monitor/
+    regexp: /run|runOut|start|init|stop/
   })
 
   defineParameterType({
@@ -180,15 +183,6 @@ cucumber.defineSupportCode((a) => {
     execKarma.apply(this, [command, undefined, proxyPort, proxyPath, callback])
   })
 
-  When('I stop when the log contains {string}', function (message, callback) {
-    setInterval(() => {
-      if (this.lastRun.stdout.includes(message)) {
-        this.child && this.child.kill()
-        callback()
-      }
-    }, 100)
-  })
-
   defineParameterType({
     name: 'exact',
     regexp: /no\sdebug|like/

test/e2e/stop.feature

@@ -44,7 +44,7 @@ Feature: Stop karma
     """
 
 
-  Scenario: A server can be stopped programically
+  Scenario: A server can be stopped programmatically
     Given a configuration with:
       """
       files = ['basic/plus.js', 'basic/test.js'];

test/e2e/support/reconnecting/test.js

@@ -22,8 +22,9 @@ describe('plus', function () {
 
   it('should re-connect', function (done) {
     expect(4).toBe(4)
+    // Emit reconnect, so Karma will not start new test run after reconnecting.
+    socket().emit('reconnect')
     socket().connect()
-    // window.parent.socket.socket.connect()
 
     done()
   })