fix(test): output parallel test results independently

.github/workflows/ci.yml

@@ -235,7 +235,7 @@ jobs:
             ~/.cargo/registry/index
             ~/.cargo/registry/cache
             ~/.cargo/git/db
-          key: 17-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
+          key: 18-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
 
       # In main branch, always creates fresh cache
       - name: Cache build output (main)
@@ -251,7 +251,7 @@ jobs:
             !./target/*/*.zip
             !./target/*/*.tar.gz
           key: |
-            17-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
+            18-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
 
       # Restore cache from the latest 'main' branch build.
       - name: Cache build output (PR)
@@ -267,7 +267,7 @@ jobs:
             !./target/*/*.tar.gz
           key: never_saved
           restore-keys: |
-            17-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
+            18-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
 
       # Don't save cache after building PRs or branches other than 'main'.
       - name: Skip save cache (PR)

cli/args/flags.rs

@@ -178,6 +178,7 @@ pub struct TestFlags {
   pub include: Vec<String>,
   pub filter: Option<String>,
   pub shuffle: Option<u64>,
+  pub parallel: bool,
   pub concurrent_jobs: NonZeroUsize,
   pub trace_ops: bool,
 }
@@ -2674,28 +2675,30 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
     }
   }
 
-  let concurrent_jobs = if matches.is_present("parallel") {
-    if let Ok(value) = env::var("DENO_JOBS") {
+  let (parallel, concurrent_jobs) = if matches.is_present("parallel") {
+    let concurrent_jobs = if let Ok(value) = env::var("DENO_JOBS") {
       value
         .parse::<NonZeroUsize>()
         .unwrap_or(NonZeroUsize::new(1).unwrap())
     } else {
       std::thread::available_parallelism()
         .unwrap_or(NonZeroUsize::new(1).unwrap())
-    }
+    };
+    (true, concurrent_jobs)
   } else if matches.is_present("jobs") {
     println!(
       "{}",
       crate::colors::yellow("Warning: --jobs flag is deprecated. Use the --parallel flag with possibly the 'DENO_JOBS' environment variable."),
     );
-    if let Some(value) = matches.value_of("jobs") {
+    let concurrent_jobs = if let Some(value) = matches.value_of("jobs") {
       value.parse().unwrap()
     } else {
       std::thread::available_parallelism()
         .unwrap_or(NonZeroUsize::new(1).unwrap())
-    }
+    };
+    (true, concurrent_jobs)
   } else {
-    NonZeroUsize::new(1).unwrap()
+    (false, NonZeroUsize::new(1).unwrap())
   };
 
   let include: Vec<String> = if matches.is_present("files") {
@@ -2719,6 +2722,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
     filter,
     shuffle,
     allow_none,
+    parallel,
     concurrent_jobs,
     trace_ops,
   });
@@ -5013,6 +5017,7 @@ mod tests {
           include: svec!["dir1/", "dir2/"],
           ignore: vec![],
           shuffle: None,
+          parallel: false,
           concurrent_jobs: NonZeroUsize::new(1).unwrap(),
           trace_ops: true,
         }),
@@ -5084,6 +5089,7 @@ mod tests {
           shuffle: None,
           include: vec![],
           ignore: vec![],
+          parallel: true,
           concurrent_jobs: NonZeroUsize::new(4).unwrap(),
           trace_ops: false,
         }),
@@ -5112,6 +5118,7 @@ mod tests {
           shuffle: None,
           include: vec![],
           ignore: vec![],
+          parallel: false,
           concurrent_jobs: NonZeroUsize::new(1).unwrap(),
           trace_ops: false,
         }),
@@ -5144,6 +5151,7 @@ mod tests {
           shuffle: None,
           include: vec![],
           ignore: vec![],
+          parallel: false,
           concurrent_jobs: NonZeroUsize::new(1).unwrap(),
           trace_ops: false,
         }),
@@ -5170,6 +5178,7 @@ mod tests {
           shuffle: Some(1),
           include: vec![],
           ignore: vec![],
+          parallel: false,
           concurrent_jobs: NonZeroUsize::new(1).unwrap(),
           trace_ops: false,
         }),
@@ -5196,6 +5205,7 @@ mod tests {
           shuffle: None,
           include: vec![],
           ignore: vec![],
+          parallel: false,
           concurrent_jobs: NonZeroUsize::new(1).unwrap(),
           trace_ops: false,
         }),
@@ -5223,6 +5233,7 @@ mod tests {
           shuffle: None,
           include: vec![],
           ignore: vec![],
+          parallel: false,
           concurrent_jobs: NonZeroUsize::new(1).unwrap(),
           trace_ops: false,
         }),

cli/lsp/testing/execution.rs

@@ -374,13 +374,12 @@ impl TestRun {
       .buffer_unordered(concurrent_jobs)
       .collect::<Vec<Result<Result<(), AnyError>, tokio::task::JoinError>>>();
 
-    let mut reporter: Box<dyn test::TestReporter + Send> =
-      Box::new(LspTestReporter::new(
-        self,
-        client.clone(),
-        maybe_root_uri,
-        self.tests.clone(),
-      ));
+    let mut reporter = Box::new(LspTestReporter::new(
+      self,
+      client.clone(),
+      maybe_root_uri,
+      self.tests.clone(),
+    ));
 
     let handler = {
       tokio::task::spawn(async move {
@@ -653,9 +652,7 @@ impl LspTestReporter {
         },
       ));
   }
-}
 
-impl test::TestReporter for LspTestReporter {
   fn report_plan(&mut self, _plan: &test::TestPlan) {}
 
   fn report_register(&mut self, desc: &test::TestDescription) {

cli/tests/integration/test_tests.rs

@@ -304,12 +304,6 @@ itest!(steps_passing_steps {
   output: "test/steps/passing_steps.out",
 });
 
-itest!(steps_passing_steps_concurrent {
-  args: "test --jobs=2 test/steps/passing_steps.ts",
-  exit_code: 0,
-  output: "test/steps/passing_steps.out",
-});
-
 itest!(steps_failing_steps {
   args: "test test/steps/failing_steps.ts",
   exit_code: 1,
@@ -447,3 +441,9 @@ itest!(non_error_thrown {
   output: "test/non_error_thrown.out",
   exit_code: 1,
 });
+
+itest!(parallel_output {
+  args: "test --parallel --reload test/parallel_output.ts",
+  output: "test/parallel_output.out",
+  exit_code: 1,
+});

cli/tests/testdata/test/parallel_output.out

@@ -0,0 +1,57 @@
+Check [WILDCARD]/test/parallel_output.ts
+./test/parallel_output.ts => step output ... step 1 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step output ... step 2 ... ok ([WILDCARD]ms)
+------- output -------
+Hello, world! (from step 3)
+----- output end -----
+./test/parallel_output.ts => step output ... step 3 ... ok ([WILDCARD]ms)
+------- output -------
+Hello, world! (from step 4)
+----- output end -----
+./test/parallel_output.ts => step output ... step 4 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step output ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step failures ... step 1 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step failures ... step 2 ... FAILED ([WILDCARD]ms)
+    error: Error: Fail.
+        throw new Error("Fail.");
+              ^
+        at file:///[WILDCARD]/test/parallel_output.ts:15:11
+        at [WILDCARD]
+        at file:///[WILDCARD]/test/parallel_output.ts:14:11
+./test/parallel_output.ts => step failures ... step 3 ... FAILED ([WILDCARD]ms)
+    error: Error: Fail.
+      await t.step("step 3", () => Promise.reject(new Error("Fail.")));
+                                                  ^
+        at file:///[WILDCARD]/test/parallel_output.ts:17:47
+        at [WILDCARD]
+        at file:///[WILDCARD]/test/parallel_output.ts:17:11
+./test/parallel_output.ts => step failures ... FAILED ([WILDCARD]ms)
+./test/parallel_output.ts => step nested failure ... step 1 ... inner 1 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step nested failure ... step 1 ... inner 2 ... FAILED ([WILDCARD]ms)
+      error: Error: Failed.
+            throw new Error("Failed.");
+                  ^
+          at file:///[WILDCARD]/test/parallel_output.ts:24:13
+          at [WILDCARD]
+          at file:///[WILDCARD]/test/parallel_output.ts:23:13
+./test/parallel_output.ts => step nested failure ... step 1 ... FAILED ([WILDCARD]ms)
+./test/parallel_output.ts => step nested failure ... FAILED ([WILDCARD]ms)
+
+ ERRORS 
+
+step failures => ./test/parallel_output.ts:12:6
+error: Error: 2 test steps failed.
+    at [WILDCARD]
+
+step nested failure => ./test/parallel_output.ts:20:6
+error: Error: 1 test step failed.
+    at [WILDCARD]
+
+ FAILURES 
+
+step failures => ./test/parallel_output.ts:12:6
+step nested failure => ./test/parallel_output.ts:20:6
+
+FAILED | 1 passed (6 steps) | 2 failed (4 steps) ([WILDCARD]ms)
+
+error: Test failed

cli/tests/testdata/test/parallel_output.ts

@@ -0,0 +1,27 @@
+Deno.test("step output", async (t) => {
+  await t.step("step 1", () => {});
+  await t.step("step 2", () => {});
+  await t.step("step 3", () => {
+    console.log("Hello, world! (from step 3)");
+  });
+  await t.step("step 4", () => {
+    console.log("Hello, world! (from step 4)");
+  });
+});
+
+Deno.test("step failures", async (t) => {
+  await t.step("step 1", () => {});
+  await t.step("step 2", () => {
+    throw new Error("Fail.");
+  });
+  await t.step("step 3", () => Promise.reject(new Error("Fail.")));
+});
+
+Deno.test("step nested failure", async (t) => {
+  await t.step("step 1", async (t) => {
+    await t.step("inner 1", () => {});
+    await t.step("inner 2", () => {
+      throw new Error("Failed.");
+    });
+  });
+});

cli/tests/testdata/test/short-pass-jobs-flag-warning.out

@@ -1,7 +1,6 @@
 Warning: --jobs flag is deprecated. Use the --parallel flag with possibly the 'DENO_JOBS' environment variable.
 Check [WILDCARD]/test/short-pass.ts
-running 1 test from ./test/short-pass.ts
-test ... ok ([WILDCARD])
+./test/short-pass.ts => test ... ok ([WILDCARD])
 
 ok | 1 passed | 0 failed ([WILDCARD])
 

cli/tests/testdata/test/short-pass.out

@@ -1,6 +1,5 @@
 Check [WILDCARD]/test/short-pass.ts
-running 1 test from ./test/short-pass.ts
-test ... ok ([WILDCARD])
+./test/short-pass.ts => test ... ok ([WILDCARD])
 
 ok | 1 passed | 0 failed ([WILDCARD])
 

cli/tests/testdata/test/steps/output_within.out

@@ -18,12 +18,10 @@ description ...
 4
 ----- output end -----
     inner 2 ... ok ([WILDCARD]ms)
-
 ------- output -------
 5
 ----- output end -----
   step 1 ... ok ([WILDCARD]ms)
-
 ------- output -------
 6
 ----- output end -----