upgrade-tests: support running against multiple apps, support sending extra jest args

.github/workflows/upgrade-tests.yml

@@ -80,30 +80,13 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       versions: ${{ steps.versions.outputs.versions }}
-      apps: ${{ steps.apps.outputs.apps }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           sparse-checkout: |
             packages/upgrade-tests/src/fixtures
 
-      - name: Get available test apps
-        id: apps
-        run: |
-          # Find all .tar.gz files in fixtures and extract app names
-          cd packages/upgrade-tests/src/fixtures
-
-          # Get list of apps and format as JSON array
-          if ls *.tar.gz >/dev/null 2>&1; then
-            APPS=$(ls *.tar.gz | sed 's/\.tar\.gz$//' | jq -R . | jq -s -c .)
-          else
-            APPS='["no-apps"]'
-          fi
-
-          echo "apps=${APPS}" >> $GITHUB_OUTPUT
-          echo "Found test apps: ${APPS}"
-
       - name: Get versions to test
         id: versions
         run: |
@@ -165,14 +148,12 @@ jobs:
 
   test-upgrade:
     needs: [build-current-version, get-test-config]
-    if: needs.get-test-config.outputs.apps != '["no-apps"]'
     strategy:
       fail-fast: false
       matrix:
         version: ${{ fromJson(needs.get-test-config.outputs.versions) }}
-        app: ${{ fromJson(needs.get-test-config.outputs.apps) }}
     runs-on: ubuntu-latest
-    name: ${{ matrix.version }} → current, ${{ matrix.app }}
+    name: ${{ matrix.version }} → current
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -212,15 +193,4 @@ jobs:
           BUDIBASE_TIMEOUT: 180000 # 3 minutes timeout for app imports
         run: |
           cd packages/upgrade-tests
-          yarn test:upgrade full --from ${{ matrix.version }} --app ${{ matrix.app }} --no-build
-
-  test-no-apps:
-    needs: get-test-config
-    if: needs.get-test-config.outputs.apps == '["no-apps"]'
-    runs-on: ubuntu-latest
-    steps:
-      - name: No test apps found
-        run: |
-          echo "⚠️ No test apps found in fixtures directory"
-          echo "Please add .tar.gz app exports to packages/upgrade-tests/src/fixtures/"
-          exit 1
+          yarn test:upgrade --from ${{ matrix.version }} --no-build

packages/upgrade-tests/package.json

@@ -34,6 +34,7 @@
     "@types/node": "22.9.0",
     "@types/semver": "^7.5.8",
     "@types/uuid": "^9.0.7",
+    "@commander-js/extra-typings": "14.0.0",
     "jest": "29.7.0",
     "ts-jest": "29.1.1",
     "typescript": "5.7.2"

packages/upgrade-tests/src/cli/index.ts

@@ -1,5 +1,5 @@
-import { Command } from "commander"
-import { bold, gray, green, red } from "chalk"
+import { program } from "@commander-js/extra-typings"
+import { bold, gray, green, red, blue } from "chalk"
 import * as fs from "fs"
 import * as path from "path"
 import {
@@ -13,19 +13,16 @@ import {
   waitForHealthy,
   buildCurrentVersion,
 } from "./docker"
-import { importApp } from "./appImport"
+import { importApp, getAvailableApps } from "./appImport"
 import { runTests } from "./testRunner"
 
-const program = new Command()
-
-// Helper to get project root
 function getProjectRoot(): string {
-  // When running from dist/src/cli, we need to go up 5 levels to reach project root
-  // dist/src/cli -> dist/src -> dist -> upgrade-tests -> packages -> project root
+  // When running from dist/src/cli, we need to go up 5 levels to reach project
+  // root dist/src/cli -> dist/src -> dist -> upgrade-tests -> packages ->
+  // project root
   return path.join(__dirname, "../../../../..")
 }
 
-// Clean up function
 async function cleanup(
   config: ReturnType<typeof generateDockerConfig>,
   showDetails = true
@@ -43,15 +40,7 @@ async function cleanup(
   }
 }
 
-// Clean up context files
 function cleanupContextFiles(silent = false) {
-  // Clean old global context file
-  const globalContext = path.join(__dirname, "../..", ".upgrade-context.json")
-  if (fs.existsSync(globalContext)) {
-    fs.unlinkSync(globalContext)
-  }
-
-  // Clean per-test context files
   const contextFiles = fs
     .readdirSync("/tmp")
     .filter(
@@ -76,7 +65,6 @@ program
   .description("Test Budibase upgrades with ease")
   .version("1.0.0")
 
-// Full upgrade test command
 program
   .command("full")
   .description("Run full upgrade test from specified version to current")
@@ -90,12 +78,18 @@ program
     "Skip building current version (assumes image already exists)"
   )
   .option("--verbose", "Show detailed output")
-  .action(async options => {
+  .allowExcessArguments()
+  .allowUnknownOption()
+  .action(async (options, { args }) => {
     const config = generateDockerConfig()
     console.log(bold("\n🚀 Starting Full Upgrade Test"))
     console.log(gray(`Upgrading from ${options.from} to ${options.to}`))
     console.log(gray(`Container: ${config.containerName}`))
 
+    if (args.length > 0) {
+      console.log(gray(`Extra Jest args: ${args.join(" ")}`))
+    }
+
     // Set up signal handlers for cleanup
     let cleanupInProgress = false
     const handleSignal = async (signal: string) => {
@@ -156,29 +150,58 @@ program
       process.env.BB_ADMIN_USER_PASSWORD = config.adminPassword
       process.env.BUDIBASE_CONTAINER_NAME = config.containerName
 
-      // Import app
-      const appToImport = options.app || "car-rental"
-      console.log(bold(`\n📱 Importing app: ${appToImport}`))
-      const { appId } = await importApp(appToImport, options.verbose)
+      // Import app(s)
+      const appMappings: Array<{ name: string; id: string }> = []
 
-      // Run pre-upgrade tests
+      if (options.app) {
+        // Single app mode
+        console.log(bold(`\n📱 Importing app: ${options.app}`))
+        const { appId, name } = await importApp(options.app, options.verbose)
+        appMappings.push({ name, id: appId })
+      } else {
+        // Multiple app mode - import all available apps
+        const availableApps = await getAvailableApps()
+        console.log(
+          bold(`\n📱 Importing ${availableApps.length} apps from fixtures`)
+        )
+
+        for (const appName of availableApps) {
+          console.log(gray(`  • Importing ${appName}...`))
+          const { appId, name } = await importApp(appName, options.verbose)
+          appMappings.push({ name, id: appId })
+        }
+      }
+
+      // Run pre-upgrade tests for each app
       console.log(bold("\n🧪 Pre-Upgrade Tests"))
-      const preSuccess = await runTests({
-        phase: "pre-upgrade",
-        verbose: options.verbose,
-        testAppId: appId,
-        testApp: options.testApp,
-        budibaseUrl,
-        internalApiKey: config.internalApiKey,
-        adminEmail: config.adminEmail,
-        adminPassword: config.adminPassword,
-        containerName: config.containerName,
-        oldVersion: options.from,
-        currentVersion: options.to,
-      })
+      let allPreTestsPassed = true
+
+      for (const app of appMappings) {
+        console.log(bold(blue(`\n  Testing app: ${app.name}`)))
+        const preSuccess = await runTests({
+          phase: "pre-upgrade",
+          verbose: options.verbose,
+          testAppId: app.id,
+          testAppName: app.name,
+          testApp: options.testApp,
+          extraArgs: args,
+          budibaseUrl,
+          internalApiKey: config.internalApiKey,
+          adminEmail: config.adminEmail,
+          adminPassword: config.adminPassword,
+          containerName: config.containerName,
+          oldVersion: options.from,
+          currentVersion: options.to,
+        })
+
+        if (!preSuccess) {
+          console.error(red(`  ✗ Pre-upgrade tests failed for ${app.name}`))
+          allPreTestsPassed = false
+        }
+      }
 
-      if (!preSuccess) {
-        throw new Error("Pre-upgrade tests failed")
+      if (!allPreTestsPassed) {
+        throw new Error("Pre-upgrade tests failed for one or more apps")
       }
 
       // Stop old version
@@ -223,24 +246,36 @@ program
       // Update environment for new URL
       process.env.BUDIBASE_URL = newBudibaseUrl
 
-      // Run post-upgrade tests
+      // Run post-upgrade tests for each app
       console.log(bold("\n🧪 Post-Upgrade Tests"))
-      const postSuccess = await runTests({
-        phase: "post-upgrade",
-        verbose: options.verbose,
-        testAppId: appId,
-        testApp: options.testApp,
-        budibaseUrl: newBudibaseUrl,
-        internalApiKey: config.internalApiKey,
-        adminEmail: config.adminEmail,
-        adminPassword: config.adminPassword,
-        containerName: config.containerName,
-        oldVersion: options.from,
-        currentVersion: options.to,
-      })
+      let allPostTestsPassed = true
+
+      for (const app of appMappings) {
+        console.log(bold(blue(`\n  Testing app: ${app.name}`)))
+        const postSuccess = await runTests({
+          phase: "post-upgrade",
+          verbose: options.verbose,
+          testAppId: app.id,
+          testAppName: app.name,
+          testApp: options.testApp,
+          extraArgs: args,
+          budibaseUrl: newBudibaseUrl,
+          internalApiKey: config.internalApiKey,
+          adminEmail: config.adminEmail,
+          adminPassword: config.adminPassword,
+          containerName: config.containerName,
+          oldVersion: options.from,
+          currentVersion: options.to,
+        })
+
+        if (!postSuccess) {
+          console.error(red(`  ✗ Post-upgrade tests failed for ${app.name}`))
+          allPostTestsPassed = false
+        }
+      }
 
-      if (!postSuccess) {
-        throw new Error("Post-upgrade tests failed")
+      if (!allPostTestsPassed) {
+        throw new Error("Post-upgrade tests failed for one or more apps")
       }
 
       // Success!
@@ -273,7 +308,9 @@ program
   .option("--from <version>", "The Budibase version")
   .option("--app <path|name>", "Path to app export or fixture name to import")
   .option("--verbose", "Show detailed output")
-  .action(async options => {
+  .allowExcessArguments()
+  .allowUnknownOption()
+  .action(async (options, { args }) => {
     console.log(bold("\n🧪 Running Pre-Upgrade Tests Only"))
 
     try {
@@ -297,6 +334,7 @@ program
       const success = await runTests({
         phase: "pre-upgrade",
         verbose: options.verbose,
+        extraArgs: args,
         budibaseUrl: process.env.BUDIBASE_URL!,
         internalApiKey: process.env.INTERNAL_API_KEY || "budibase",
         adminEmail: process.env.BB_ADMIN_USER_EMAIL || "[email protected]",
@@ -319,7 +357,9 @@ program
   .command("post")
   .description("Run only post-upgrade tests")
   .option("--verbose", "Show detailed output")
-  .action(async options => {
+  .allowExcessArguments()
+  .allowUnknownOption()
+  .action(async (options, { args }) => {
     console.log(bold("\n🧪 Running Post-Upgrade Tests Only"))
 
     if (!process.env.TEST_APP_ID) {
@@ -334,6 +374,7 @@ program
       const success = await runTests({
         phase: "post-upgrade",
         verbose: options.verbose,
+        extraArgs: args,
         testAppId: process.env.TEST_APP_ID,
         budibaseUrl: process.env.BUDIBASE_URL || "http://localhost:10000",
         internalApiKey: process.env.INTERNAL_API_KEY || "budibase",

packages/upgrade-tests/src/cli/testRunner.ts

@@ -8,7 +8,9 @@ export interface TestRunnerOptions {
   phase: TestPhase
   verbose?: boolean
   testAppId?: string
+  testAppName?: string
   testApp?: string
+  extraArgs?: string[]
   budibaseUrl: string
   internalApiKey: string
   adminEmail: string
@@ -19,7 +21,7 @@ export interface TestRunnerOptions {
 }
 
 export async function runTests(options: TestRunnerOptions): Promise<boolean> {
-  // Set up environment variables
+  // Set up environment variables - preserve NODE_OPTIONS and other env vars
   const env: NodeJS.ProcessEnv = {
     ...process.env,
     TEST_PHASE: options.phase,
@@ -30,10 +32,19 @@ export async function runTests(options: TestRunnerOptions): Promise<boolean> {
     BUDIBASE_CONTAINER_NAME: options.containerName,
   }
 
+  // Ensure NODE_OPTIONS is preserved for debugging
+  if (process.env.NODE_OPTIONS) {
+    env.NODE_OPTIONS = process.env.NODE_OPTIONS
+  }
+
   if (options.testAppId) {
     env.TEST_APP_ID = options.testAppId
   }
 
+  if (options.testAppName) {
+    env.TEST_APP_NAME = options.testAppName
+  }
+
   if (options.testApp) {
     env.TEST_APP = options.testApp
   }
@@ -48,8 +59,16 @@ export async function runTests(options: TestRunnerOptions): Promise<boolean> {
 
   console.log(bold(`\n${blue("►")} Running ${options.phase} tests...\n`))
 
+  // Build the test command with any extra arguments
+  const testCommand = ["test"]
+  const extraArgs = Array.isArray(options.extraArgs) ? options.extraArgs : []
+  if (extraArgs.length > 0) {
+    console.log(bold(`   Extra args: ${extraArgs.join(" ")}\n`))
+    testCommand.push(...extraArgs)
+  }
+
   return new Promise(resolve => {
-    const testProcess = spawn("yarn", ["test"], {
+    const testProcess = spawn("yarn", testCommand, {
       cwd: path.join(__dirname, "../.."),
       env,
       stdio: "inherit",

packages/upgrade-tests/src/utils/upgradeTest.ts

@@ -16,7 +16,13 @@ function contextFileName(): string {
   if (!state.testPath) {
     throw new Error("Test path is not set in Jest state")
   }
-  const testName = `${state.testPath}/${state.currentTestName}`
+
+  if (!process.env.TEST_APP_NAME) {
+    throw new Error("TEST_APP_NAME environment variable is not set")
+  }
+
+  const appName = process.env.TEST_APP_NAME
+  const testName = `${state.testPath}/${state.currentTestName}/${appName}`
   const name = crypto.createHash("md5").update(testName).digest("hex")
   return `/tmp/upgrade-context-${name}.json`
 }