Parallel Playwright

docs.openc3.com/docs/development/testing.md

@@ -39,8 +39,17 @@ sidebar_custom_props:
 
 1. Run tests (Note the --headed option visually displays tests, leave it off to run in the background)
 
+   Tests are split into a group that runs in parallel and a group that runs serially. This is done to improve overall execution time.
+
+   ```bash
+   cosmos-playwright % yarn test:parallel --headed
+   cosmos-playwright % yarn test:serial --headed
+   ```
+
+   You can run both groups together, but the --headed option will not apply to both groups:
+
    ```bash
-   cosmos-playwright % yarn playwright test --project=chromium --headed
+   cosmos-playwright % yarn test
    ```
 
 1. _[Optional]_ Fix istanbul/nyc coverage source lookups (use `fixwindows` if not on Linux).

openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/procedures/stash.rb

@@ -6,10 +6,10 @@
 puts "key2: #{stash_get('key2')}"
 check_expression("'#{stash_get('key1')}' == 'val1'")
 check_expression("'#{stash_get('key2')}' == 'val2'")
-check_expression("'#{stash_keys().to_s}' == '[\"key1\", \"key2\"]'")
 stash_set('key1', 1)
 stash_set('key2', 2)
-check_expression("'#{stash_all().to_s}' == '{\"key1\"=>1, \"key2\"=>2}'")
+check_expression("'#{stash_get('key1')}' == '1'")
+check_expression("'#{stash_get('key2')}' == '2'")
 stash_delete('key2')
 check_expression("#{stash_get('key2').nil?} == true")
 stash_delete('key1')

openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST2/procedures/stash.py

@@ -1,22 +1,22 @@
 # Stash API is useful for storing simple key/value pairs
 # to preserve state between script runs
-stash_set("key1", "val1")
-stash_set("key2", "val2")
-check_expression(f"'{stash_get('key1')}' == 'val1'")
-check_expression(f"'{stash_get('key2')}' == 'val2'")
-check_expression(f"{stash_keys()} == ['key1', 'key2']")
-stash_set("key1", 1)
-stash_set("key2", 2)
-check_expression(f"{stash_all()} == {{'key1':1, 'key2':2}}")
-stash_delete("key2")
-check_expression(f"{stash_get('key2')} == None")
-stash_delete("key1")
+stash_set("key1_py", "val1")
+stash_set("key2_py", "val2")
+check_expression(f"'{stash_get('key1_py')}' == 'val1'")
+check_expression(f"'{stash_get('key2_py')}' == 'val2'")
+stash_set("key1_py", 1)
+stash_set("key2_py", 2)
+check_expression(f"'{stash_get('key1_py')}' == '1'")
+check_expression(f"'{stash_get('key2_py')}' == '2'")
+stash_delete("key2_py")
+check_expression(f"{stash_get('key2_py')} == None")
+stash_delete("key1_py")
 data = [1, 2, [3, 4]]
-stash_set("ary", data)
-check_expression(f"{stash_get('ary')} == {data}")
-stash_delete("ary")
+stash_set("ary_py", data)
+check_expression(f"{stash_get('ary_py')} == {data}")
+stash_delete("ary_py")
 # Note: hashes with symbol keys works but get converted to string keys on stash_get
 hash = {"one": 1, "two": 2, "string": "string"}
-stash_set("hash", hash)
-check_expression(f"{stash_get('hash')} == {hash}")
-stash_delete("hash")
+stash_set("hash_py", hash)
+check_expression(f"{stash_get('hash_py')} == {hash}")
+stash_delete("hash_py")

playwright/README.md

@@ -22,23 +22,39 @@ NOTE: All commands are assumed to be executed from this (playwright) directory u
         openc3-pw-test> openc3.sh cliroot rake build VERSION=1.0.0
         openc3-pw-test> cp openc3-cosmos-pw-test-1.0.0.gem openc3-cosmos-pw-test-1.0.1.gem
 
+## Running the tests
+
+Tests are split into a parallel group and a serial group. This is done to lower overall execution time. Yarn scripts `yarn test:parallel` and `yarn test:serial` are provided to run each group individually. With these scripts, you can pass additional arguments such as `--ui` (see examples below).
+
+There is also a yarn script `yarn test` which will run the parallel group, followed by the serial group. However, due to limitations in yarn, additional arguments will only be passed to the last group (serial).
+
+These yarn scripts always pass the `--project=chromium` argument to both test groups. The script `yarn test:keycloak` is provided to run both test groups with `--project=keycloak`, but if you want to pass any other value to this argument, you will have to build the whole playwright command yourself, e.g.:
+
+```bash
+playwright> yarn playwright test ./tests/**/*.p.spec.ts --headed --project=firefox || yarn playwright test ./tests/**/*.s.spec.ts --headed --project=firefox --workers=1
+```
+
+Note the `--workers=1` passed to the serial (`*.s.spec.ts`) group. If you omit this argument, you will get very flaky test results from this group since Playwright defaults to running tests in parallel.
+
+The following examples use the parallel group, but these apply to the serial group as well.
+
 1.  Open playwright and run tests. The first example is running against Open Source, the second against Enterprise.
 
-        playwright> yarn playwright test --headed --project=chromium
-        playwright> ENTERPRISE=1 yarn playwright test --headed --project=chromium
+        playwright> yarn test:parallel --headed
+        playwright> ENTERPRISE=1 yarn test:parallel --headed
 
 1.  Playback a trace file. Note that the Github 'OpenC3 Playwright Tests' action stores the trace under the Summary link. Click Details next to the 'OpenC3 Playwright Tests' then Summary in the top left. Scroll down and download the playwright artifact. Unzip it and copy the files into the playwright/test-results directory. Then playback the trace to help debug.
 
         playwright> yarn playwright show-trace /test-results/<NAME OF TEST>/trace.zip
 
 1.  Run with the playwright UI
 
-        playwright> yarn playwright test --ui --project=chromium
+        playwright> yarn test:parallel --ui
 
 1.  Enable the playwright inspector / debugger with
 
         playwright> set PWDEBUG=1
-        playwright> yarn playwright test --headed --project=chromium
+        playwright> yarn test:parallel --headed
 
 1.  _[Optional]_ Fix istanbul/nyc coverage source lookups (use `fixlinux` if not on Windows).
     Tests will run successfully without this step and you will get coverage statistics, but line-by-line coverage won't work.

playwright/package.json

@@ -4,11 +4,17 @@
   "license": "ISC",
   "description": "OpenC3 integration testing",
   "scripts": {
+    "test": "yarn test:parallel --quiet || yarn test:serial --quiet",
+    "test:parallel": "playwright test ./tests/**/*.p.spec.ts --project=chromium",
+    "test:serial": "playwright test ./tests/**/*.s.spec.ts --project=chromium --workers=1",
+    "test:enterprise": "ENTERPRISE=1 playwright test ./tests/enterprise/*.spec.ts --project=chromium --workers=1",
+    "test:keycloak": "playwright test ./tests/**/*.p.spec.ts --project=keycloak || playwright test ./tests/**/*.s.spec.ts --project=keycloak --workers=1",
     "fixlinux": "cp ./context.js node_modules/istanbul-lib-report/lib/context.js",
     "fixwindows": "copy .\\context.js node_modules\\istanbul-lib-report\\lib\\context.js",
     "coverage": "nyc report --reporter=html",
     "cobertura": "nyc report --reporter=cobertura",
-    "clean": "rm -rf .nyc_output || rmdir /s /q .nyc_output; rm -rf coverage || rmdir /s /q coverage; rm -rf test-results || rmdir /s /q test-results"
+    "clean": "rm -rf .nyc_output || rmdir /s /q .nyc_output; rm -rf coverage || rmdir /s /q coverage; rm -rf test-results || rmdir /s /q test-results",
+    "format": "prettier -w ./"
   },
   "dependencies": {
     "@playwright/test": "1.50.1",

playwright/playwright.config.ts

@@ -14,14 +14,15 @@ export const ADMIN_STORAGE_STATE = path.join(
  */
 export default defineConfig({
   testDir: './tests',
-  /* Maximum time one test can run for. */
-  timeout: 5 * 60 * 1000, // 5 minutes
+  /* Maximum time one test can run for (default 30s). */
+  timeout: 30 * 1000, // default value
   expect: {
     /**
      * Maximum time expect() should wait for the condition to be met.
+     * (default 5s)
      * For example in `await expect(locator).toHaveText();`
      */
-    timeout: 10000,
+    timeout: 5000, // default value
   },
   /* Maximum time for the entire test run. Since we run the entire suite
      on each browser separately this should be enough. */
@@ -30,13 +31,13 @@ export default defineConfig({
   forbidOnly: !!process.env.CI,
   /* Retry once on CI */
   retries: process.env.CI ? 1 : 0,
-  workers: 1,
-  /* See if explicit WORKERS count was given, otherwise allow parallelism on CI/CD */
-  // workers: process.env.WORKERS
-  //   ? parseInt(process.env.WORKERS)
-  //   : process.env.CI
-  //   ? 3
-  //   : 1,
+  /* Allows multiple tests from each file to be run at the same time */
+  fullyParallel: true,
+  workers: process.env.WORKERS
+    ? parseInt(process.env.WORKERS) // Use explicit worker count if given
+    : process.env.CI
+      ? 3 // Otherwise use 3 on CI/CD
+      : undefined, // and a bunch locally (seems to be 7, but the default isn't documented)
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: process.env.CI ? 'github' : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */

playwright/playwright.sh

@@ -6,6 +6,7 @@ usage() {
   echo "*  build-plugin: builds the plugin to be used in the playwright tests" >&2
   echo "*  reset-storage-state: clear out cached data" >&2
   echo "*  run-chromium: runs the playwright tests against a locally running version of Cosmos using Chrome" >&2
+  echo "*  run-enterprise: runs the enterprise playwright tests against a locally running version of Cosmos using Chrome" >&2
   echo "*  run-aws: runs the playwright tests against a remotely running version of Cosmos using Chrome" >&2
   exit 1
 }
@@ -44,12 +45,15 @@ case $1 in
         ;;
 
     run-chromium )
-        yarn playwright test "${@:2}" --project=chromium
+        yarn test
         ;;
 
+    run-enterprise )
+        yarn test:enterprise
+
     run-aws )
         sed -i 's#http://localhost:2900#https://aws.openc3.com#' playwright.config.ts
-        KEYCLOAK_URL=https://aws.openc3.com/auth REDIRECT_URL=https://aws.openc3.com/* yarn playwright test --project=keycloak
-        yarn playwright test "${@:2}" --project=chromium
+        KEYCLOAK_URL=https://aws.openc3.com/auth REDIRECT_URL=https://aws.openc3.com/* yarn test:keycloak
+        yarn test
         ;;
 esac

playwright/tests/admin/plugins.spec.ts → playwright/tests/admin/plugins.p.spec.ts

@@ -201,6 +201,7 @@ test.describe(() => {
   test.use({ storageState: 'storageState.json' })
 
   test('installs, modifies, and deletes a plugin', async ({ page, utils }) => {
+    test.setTimeout(3 * 60 * 1000) // 3 minutes
     // This test goes through the gamut of plugin functionality: install, upgrade, modify, delete.
     // It is one test so that it works with Playwright's parallelization and optional order randomization.
 

playwright/tests/admin/settings.spec.ts → playwright/tests/admin/settings.p.spec.ts

@@ -43,6 +43,7 @@ test('clears default configs', async ({ page, utils }) => {
   // Visit PacketViewer and change a setting
   await page.goto('/tools/packetviewer')
   await expect(page.locator('.v-app-bar')).toContainText('Packet Viewer')
+  await page.locator('rux-icon-apps').getByRole('img').click()
   await page.locator('[data-test=packet-viewer-view]').click()
   await page.locator('text=Show Ignored').click()
   await utils.sleep(100)

playwright/tests/bucket-explorer.spec.ts → playwright/tests/bucket-explorer.p.spec.ts

@@ -161,6 +161,7 @@ test('view file', async ({ page, utils }) => {
 })
 
 test('upload and delete', async ({ page, utils }) => {
+  const randomDir = 'tmp_' + `${Math.random()}`.substring(2)
   await page.getByText('config').click()
   await expect(page).toHaveURL(/.*\/tools\/bucketexplorer\/config%2F/)
   await expect(page.locator('[data-test="file-path"]')).toHaveText('/')
@@ -178,11 +179,11 @@ test('upload and delete', async ({ page, utils }) => {
   await fileChooser1.setFiles('package.json')
   await page
     .locator('[data-test="upload-file-path"] input')
-    .fill('DEFAULT/tmp/tmp.json')
+    .fill(`DEFAULT/${randomDir}/tmp.json`)
   await page.locator('[data-test="upload-file-submit-btn"]').click()
 
   await expect(page.locator('[data-test="file-path"]')).toHaveText(
-    '/ DEFAULT / tmp /',
+    `/ DEFAULT / ${randomDir} /`,
   )
   await page.locator('tbody> tr').first().waitFor()
   let count = await page.locator('tbody > tr').count()
@@ -199,13 +200,17 @@ test('upload and delete', async ({ page, utils }) => {
   await fileChooser.setFiles('package.json')
   await page.locator('[data-test="upload-file-submit-btn"]').click()
 
-  await expect(page.locator('tbody > tr')).toHaveCount(count + 1)
+  await expect
+    .poll(() => page.locator('tbody > tr').count(), { timeout: 10000 })
+    .toBeGreaterThanOrEqual(count + 1)
   await expect(page.getByRole('cell', { name: 'package.json' })).toBeVisible()
   await page
     .locator('tr:has-text("package.json") [data-test="delete-file"]')
     .click()
   await page.locator('[data-test="confirm-dialog-delete"]').click()
-  await expect(page.locator('tbody > tr')).toHaveCount(count)
+  await expect
+    .poll(() => page.locator('tbody > tr').count(), { timeout: 10000 })
+    .toBeGreaterThanOrEqual(count)
 
   // Note that Promise.all prevents a race condition
   // between clicking and waiting for the file chooser.
@@ -219,10 +224,10 @@ test('upload and delete', async ({ page, utils }) => {
   await fileChooser2.setFiles('package.json')
   await page
     .locator('[data-test="upload-file-path"] input')
-    .fill('DEFAULT/tmp/TEST/tmp/myfile.json')
+    .fill(`DEFAULT/${randomDir}/TEST/tmp/myfile.json`)
   await page.locator('[data-test="upload-file-submit-btn"]').click()
   await expect(page.locator('[data-test="file-path"]')).toHaveText(
-    '/ DEFAULT / tmp / TEST / tmp /',
+    `/ DEFAULT / ${randomDir} / TEST / tmp /`,
   )
   await page
     .locator('tr:has-text("myfile.json") [data-test="delete-file"]')
@@ -233,14 +238,15 @@ test('upload and delete', async ({ page, utils }) => {
   await page.getByText('logs').click()
   await page.getByText('config').click()
   await page.getByRole('cell', { name: 'DEFAULT' }).click()
-  await page.getByRole('cell', { name: 'tmp' }).click()
+  await page.getByRole('cell', { name: randomDir }).click()
   await page
     .locator('tr:has-text("tmp.json") [data-test="delete-file"]')
     .click()
   await page.locator('[data-test="confirm-dialog-delete"]').click()
 })
 
 test('navigate logs and tools bucket', async ({ page, utils }) => {
+  test.setTimeout(3 * 60 * 1000) // 3 minutes
   // Keep clicking alternatively on tools and then logs to force a refresh
   // This allows the DEFAULT folder to appear in time
   await expect(async () => {
@@ -290,6 +296,7 @@ test('auto refreshes to update files', async ({
   toolPath,
   context,
 }) => {
+  const randomDir = 'tmp_' + `${Math.random()}`.substring(2)
   // Upload something from the first tab to make sure the tmp dir exists
   await page.getByText('config').click()
   await expect(page.getByLabel('prepended action')).toBeVisible()
@@ -300,15 +307,15 @@ test('auto refreshes to update files', async ({
   await fileChooser1.setFiles('package.json')
   await page
     .locator('[data-test="upload-file-path"] input')
-    .fill('DEFAULT/tmp/package1.json')
+    .fill(`DEFAULT/${randomDir}/package1.json`)
   await page.locator('[data-test="upload-file-submit-btn"]').click()
 
   // Open another tab and navigate to the tmp dir
   const pageTwo = await context.newPage()
   pageTwo.goto(toolPath)
   await pageTwo.getByText('config').click()
   await pageTwo.getByRole('cell', { name: 'DEFAULT' }).click()
-  await pageTwo.getByRole('cell', { name: 'tmp' }).click()
+  await pageTwo.getByRole('cell', { name: randomDir }).click()
 
   // Set the refresh interval on the second tab to be really slow
   await pageTwo.locator('[data-test=bucket-explorer-file]').click()
@@ -330,14 +337,14 @@ test('auto refreshes to update files', async ({
   await fileChooser2.setFiles('package.json')
   await page
     .locator('[data-test="upload-file-path"] input')
-    .fill('DEFAULT/tmp/package2.json')
+    .fill(`DEFAULT/${randomDir}/package2.json`)
   await page.locator('[data-test="upload-file-submit-btn"]').click()
 
   // The second tab shouldn't have refreshed yet, so the file shouldn't be there
   await page.locator('tbody> tr').first().waitFor()
   await expect(
     pageTwo.getByRole('cell', { name: 'package2.json' }),
-  ).not.toBeVisible()
+  ).not.toBeVisible({ timeout: 10000 })
 
   // Set the refresh interval on the second tab to 1s
   await pageTwo.locator('[data-test=bucket-explorer-file]').click()

playwright/tests/cmd-tlm-server/cmd-pkts.spec.ts → playwright/tests/cmd-tlm-server/cmd-pkts.p.spec.ts

@@ -62,7 +62,7 @@ test('displays the command count', async ({ page, utils }) => {
     .locator('[data-test=cmd-packets-table] >> tr td >> nth=2')
     .textContent()
   if (countStr === null) {
-    throw new Error("Unable to get packet count")
+    throw new Error('Unable to get packet count')
   }
   const count = parseInt(countStr)
   // Send an ABORT command
@@ -94,10 +94,11 @@ test('displays the command count', async ({ page, utils }) => {
     )
     .toEqual(2)
   await expect
-    .poll(async () => 
-      await page
-        .locator('[data-test=cmd-packets-table] >> tr td >> nth=2')
-        .textContent()
+    .poll(
+      async () =>
+        await page
+          .locator('[data-test=cmd-packets-table] >> tr td >> nth=2')
+          .textContent(),
     )
     .toEqual(`${count + 1}`)
 })

playwright/tests/cmd-tlm-server/tlm-pkts.spec.ts → playwright/tests/cmd-tlm-server/tlm-pkts.p.spec.ts

@@ -35,14 +35,18 @@ test('displays the list of telemetry', async ({ page, utils }) => {
 test('displays the packet count', async ({ page, utils }) => {
   await expect(page.locator('text=INSTHEALTH_STATUS')).toBeVisible()
   await utils.sleep(2000) // Allow the telemetry to be fetched
-  const hsCountStr = await page.locator('text=INSTHEALTH_STATUS >> td >> nth=2').textContent()
+  const hsCountStr = await page
+    .locator('text=INSTHEALTH_STATUS >> td >> nth=2')
+    .textContent()
   if (hsCountStr === null) {
-    throw new Error("Unable to get HEALTH_STATUS packet count")
+    throw new Error('Unable to get HEALTH_STATUS packet count')
   }
   expect(parseInt(hsCountStr)).toBeGreaterThan(50)
-  const adcsCountStr = await page.locator('text=INSTADCS >> td >> nth=2').textContent()
+  const adcsCountStr = await page
+    .locator('text=INSTADCS >> td >> nth=2')
+    .textContent()
   if (adcsCountStr === null) {
-    throw new Error("Unable to get HEALTH_STATUS packet count")
+    throw new Error('Unable to get HEALTH_STATUS packet count')
   }
   expect(parseInt(adcsCountStr)).toBeGreaterThan(500)
 })

playwright/tests/command-sender.spec.ts → playwright/tests/command-sender.p.spec.ts

@@ -25,6 +25,11 @@ test.use({
   toolName: 'Command Sender',
 })
 
+// Most of these tests are super flaky when run in parallel with each other.
+// Just gonna disable parallelism for now.
+// TODO: This might actually be bugginess in the app? Not sure
+test.describe.configure({ mode: 'serial' })
+
 // Helper function to select a parameter dropdown
 async function selectValue(page, param, value) {
   let row = page.locator(`tr:has-text("${param}")`)