fix(benchmark): rewrite reporter without log-update (#7019)

Author: AriPerkkio

packages/vitest/LICENSE.md

@@ -189,15 +189,13 @@
     "birpc": "0.2.19",
     "cac": "^6.7.14",
     "chai-subset": "^1.6.0",
-    "cli-truncate": "^4.0.0",
     "fast-glob": "3.3.2",
     "find-up": "^6.3.0",
     "flatted": "^3.3.2",
     "get-tsconfig": "^4.8.1",
     "happy-dom": "^15.11.7",
     "jsdom": "^25.0.1",
     "local-pkg": "^0.5.1",
-    "log-update": "^5.0.1",
     "micromatch": "^4.0.8",
     "pretty-format": "^29.7.0",
     "prompts": "^2.4.2",

packages/vitest/package.json

@@ -1113,7 +1113,6 @@ export class Vitest {
               this.logger.error('error during close', r.reason)
             }
           })
-          this.logger.logUpdate.done() // restore terminal cursor
         })
       })()
     }

packages/vitest/src/node/core.ts

@@ -7,7 +7,6 @@ import { existsSync, readFileSync } from 'node:fs'
 import { Writable } from 'node:stream'
 import { stripVTControlCharacters } from 'node:util'
 import { inspect, isPrimitive } from '@vitest/utils'
-import cliTruncate from 'cli-truncate'
 import { normalize, relative } from 'pathe'
 import c from 'tinyrainbow'
 import { TypeCheckError } from '../typecheck/typechecker'
@@ -17,7 +16,7 @@ import {
 } from '../utils/source-map'
 import { Logger } from './logger'
 import { F_POINTER } from './reporters/renderers/figures'
-import { divider } from './reporters/renderers/utils'
+import { divider, truncateString } from './reporters/renderers/utils'
 
 interface PrintErrorOptions {
   type?: string
@@ -413,7 +412,7 @@ export function generateCodeFrame(
 
         res.push(
           lineNo(j + 1)
-          + cliTruncate(lines[j].replace(/\t/g, ' '), columns - 5 - indent),
+          + truncateString(lines[j].replace(/\t/g, ' '), columns - 5 - indent),
         )
 
         if (j === i) {

packages/vitest/src/node/error.ts

@@ -8,7 +8,6 @@ import type { TestProject } from './project'
 import { Console } from 'node:console'
 import { toArray } from '@vitest/utils'
 import { parseErrorStacktrace } from '@vitest/utils/source-map'
-import { createLogUpdate } from 'log-update'
 import c from 'tinyrainbow'
 import { highlightCode } from '../utils/colors'
 import { printError } from './error'
@@ -25,19 +24,22 @@ export interface ErrorOptions {
   showCodeFrame?: boolean
 }
 
+type Listener = () => void
+
 const PAD = '      '
 
 const ESC = '\x1B['
 const ERASE_DOWN = `${ESC}J`
 const ERASE_SCROLLBACK = `${ESC}3J`
 const CURSOR_TO_START = `${ESC}1;1H`
+const HIDE_CURSOR = `${ESC}?25l`
+const SHOW_CURSOR = `${ESC}?25h`
 const CLEAR_SCREEN = '\x1Bc'
 
 export class Logger {
-  logUpdate: ReturnType<typeof createLogUpdate>
-
   private _clearScreenPending: string | undefined
   private _highlights = new Map<string, string>()
+  private cleanupListeners: Listener[] = []
   public console: Console
 
   constructor(
@@ -46,9 +48,11 @@ export class Logger {
     public errorStream: NodeJS.WriteStream | Writable = process.stderr,
   ) {
     this.console = new Console({ stdout: outputStream, stderr: errorStream })
-    this.logUpdate = createLogUpdate(this.outputStream)
     this._highlights.clear()
+    this.addCleanupListeners()
     this.registerUnhandledRejection()
+
+    ;(this.outputStream as Writable).write(HIDE_CURSOR)
   }
 
   log(...args: any[]) {
@@ -303,6 +307,44 @@ export class Logger {
     this.log(c.red(divider()))
   }
 
+  getColumns() {
+    return 'columns' in this.outputStream ? this.outputStream.columns : 80
+  }
+
+  onTerminalCleanup(listener: Listener) {
+    this.cleanupListeners.push(listener)
+  }
+
+  private addCleanupListeners() {
+    const cleanup = () => {
+      this.cleanupListeners.forEach(fn => fn())
+      ;(this.outputStream as Writable).write(SHOW_CURSOR)
+    }
+
+    const onExit = (signal?: string | number, exitCode?: number) => {
+      cleanup()
+
+      // Interrupted signals don't set exit code automatically.
+      // Use same exit code as node: https://nodejs.org/api/process.html#signal-events
+      if (process.exitCode === undefined) {
+        process.exitCode = exitCode !== undefined ? (128 + exitCode) : Number(signal)
+      }
+
+      process.exit()
+    }
+
+    process.once('SIGINT', onExit)
+    process.once('SIGTERM', onExit)
+    process.once('exit', onExit)
+
+    this.ctx.onClose(() => {
+      process.off('SIGINT', onExit)
+      process.off('SIGTERM', onExit)
+      process.off('exit', onExit)
+      cleanup()
+    })
+  }
+
   private registerUnhandledRejection() {
     const onUnhandledRejection = (err: unknown) => {
       process.exitCode = 1

packages/vitest/src/node/logger.ts

@@ -71,6 +71,9 @@ export abstract class BaseReporter implements Reporter {
     }
   }
 
+  /**
+   * Callback invoked with a single `Task` from `onTaskUpdate`
+   */
   protected printTask(task: Task) {
     if (
       !('filepath' in task)
@@ -438,7 +441,7 @@ export abstract class BaseReporter implements Reporter {
     const benches = getTests(files)
     const topBenches = benches.filter(i => i.result?.benchmark?.rank === 1)
 
-    this.log(withLabel('cyan', 'BENCH', 'Summary\n'))
+    this.log(`\n${withLabel('cyan', 'BENCH', 'Summary\n')}`)
 
     for (const bench of topBenches) {
       const group = bench.suite || bench.file
@@ -448,7 +451,7 @@ export abstract class BaseReporter implements Reporter {
       }
 
       const groupName = getFullName(group, c.dim(' > '))
-      this.log(`  ${bench.name}${c.dim(` - ${groupName}`)}`)
+      this.log(`  ${formatProjectName(bench.file.projectName)}${bench.name}${c.dim(` - ${groupName}`)}`)
 
       const siblings = group.tasks
         .filter(i => i.meta.benchmark && i.result?.benchmark && i !== bench)

packages/vitest/src/node/reporters/base.ts

@@ -1,8 +1,9 @@
-import { VerboseReporter } from '../verbose'
-import { TableReporter } from './table'
+import { BenchmarkReporter } from './reporter'
+import { VerboseBenchmarkReporter } from './verbose'
 
 export const BenchmarkReportsMap = {
-  default: TableReporter,
-  verbose: VerboseReporter,
+  default: BenchmarkReporter,
+  verbose: VerboseBenchmarkReporter,
 }
+
 export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap

packages/vitest/src/node/reporters/benchmark/index.ts

@@ -0,0 +1,69 @@
+import type { File } from '@vitest/runner'
+import type { BenchmarkResult } from '../../../runtime/types/benchmark'
+import { getFullName, getTasks } from '@vitest/runner/utils'
+
+interface Report {
+  files: {
+    filepath: string
+    groups: Group[]
+  }[]
+}
+
+interface Group {
+  fullName: string
+  benchmarks: FormattedBenchmarkResult[]
+}
+
+export type FormattedBenchmarkResult = BenchmarkResult & {
+  id: string
+}
+
+export function createBenchmarkJsonReport(files: File[]) {
+  const report: Report = { files: [] }
+
+  for (const file of files) {
+    const groups: Group[] = []
+
+    for (const task of getTasks(file)) {
+      if (task?.type === 'suite') {
+        const benchmarks: FormattedBenchmarkResult[] = []
+
+        for (const t of task.tasks) {
+          const benchmark = t.meta.benchmark && t.result?.benchmark
+
+          if (benchmark) {
+            benchmarks.push({ id: t.id, ...benchmark, samples: [] })
+          }
+        }
+
+        if (benchmarks.length) {
+          groups.push({
+            fullName: getFullName(task, ' > '),
+            benchmarks,
+          })
+        }
+      }
+    }
+
+    report.files.push({
+      filepath: file.filepath,
+      groups,
+    })
+  }
+
+  return report
+}
+
+export function flattenFormattedBenchmarkReport(report: Report) {
+  const flat: Record<FormattedBenchmarkResult['id'], FormattedBenchmarkResult> = {}
+
+  for (const file of report.files) {
+    for (const group of file.groups) {
+      for (const t of group.benchmarks) {
+        flat[t.id] = t
+      }
+    }
+  }
+
+  return flat
+}

packages/vitest/src/node/reporters/benchmark/json-formatter.ts

@@ -0,0 +1,97 @@
+import type { Task, TaskResultPack } from '@vitest/runner'
+import type { Vitest } from '../../core'
+import fs from 'node:fs'
+import { getFullName } from '@vitest/runner/utils'
+import * as pathe from 'pathe'
+import c from 'tinyrainbow'
+import { DefaultReporter } from '../default'
+import { formatProjectName, getStateSymbol } from '../renderers/utils'
+import { createBenchmarkJsonReport, flattenFormattedBenchmarkReport } from './json-formatter'
+import { renderTable } from './tableRender'
+
+export class BenchmarkReporter extends DefaultReporter {
+  compare?: Parameters<typeof renderTable>[0]['compare']
+
+  async onInit(ctx: Vitest) {
+    super.onInit(ctx)
+
+    if (this.ctx.config.benchmark?.compare) {
+      const compareFile = pathe.resolve(
+        this.ctx.config.root,
+        this.ctx.config.benchmark?.compare,
+      )
+      try {
+        this.compare = flattenFormattedBenchmarkReport(
+          JSON.parse(await fs.promises.readFile(compareFile, 'utf-8')),
+        )
+      }
+      catch (e) {
+        this.error(`Failed to read '${compareFile}'`, e)
+      }
+    }
+  }
+
+  onTaskUpdate(packs: TaskResultPack[]): void {
+    for (const pack of packs) {
+      const task = this.ctx.state.idMap.get(pack[0])
+
+      if (task?.type === 'suite' && task.result?.state !== 'run') {
+        task.tasks.filter(task => task.result?.benchmark)
+          .sort((benchA, benchB) => benchA.result!.benchmark!.mean - benchB.result!.benchmark!.mean)
+          .forEach((bench, idx) => {
+            bench.result!.benchmark!.rank = Number(idx) + 1
+          })
+      }
+    }
+
+    super.onTaskUpdate(packs)
+  }
+
+  printTask(task: Task) {
+    if (task?.type !== 'suite' || !task.result?.state || task.result?.state === 'run' || task.result?.state === 'queued') {
+      return
+    }
+
+    const benches = task.tasks.filter(t => t.meta.benchmark)
+    const duration = task.result.duration
+
+    if (benches.length > 0 && benches.every(t => t.result?.state !== 'run' && t.result?.state !== 'queued')) {
+      let title = `\n ${getStateSymbol(task)} ${formatProjectName(task.file.projectName)}${getFullName(task, c.dim(' > '))}`
+
+      if (duration != null && duration > this.ctx.config.slowTestThreshold) {
+        title += c.yellow(` ${Math.round(duration)}${c.dim('ms')}`)
+      }
+
+      this.log(title)
+      this.log(renderTable({
+        tasks: benches,
+        level: 1,
+        shallow: true,
+        columns: this.ctx.logger.getColumns(),
+        compare: this.compare,
+        showHeap: this.ctx.config.logHeapUsage,
+        slowTestThreshold: this.ctx.config.slowTestThreshold,
+      }))
+    }
+  }
+
+  async onFinished(files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors()) {
+    super.onFinished(files, errors)
+
+    // write output for future comparison
+    let outputFile = this.ctx.config.benchmark?.outputJson
+
+    if (outputFile) {
+      outputFile = pathe.resolve(this.ctx.config.root, outputFile)
+      const outputDirectory = pathe.dirname(outputFile)
+
+      if (!fs.existsSync(outputDirectory)) {
+        await fs.promises.mkdir(outputDirectory, { recursive: true })
+      }
+
+      const output = createBenchmarkJsonReport(files)
+      await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2))
+      this.log(`Benchmark report written to ${outputFile}`)
+    }
+  }
+}