|
| 1 | +import type { TaskResult } from "tinybench"; |
| 2 | +import { Bench } from "tinybench"; |
| 3 | + |
| 4 | +import type { SeedResult } from "./seed"; |
| 5 | +import { logInfo, logStep, logSuccess } from "./log"; |
| 6 | +import { formatMs, formatNumber } from "./utils"; |
| 7 | + |
| 8 | +// Type guard for completed task results |
| 9 | +type CompletedTaskResult = Extract<TaskResult, { state: "completed" }>; |
| 10 | + |
| 11 | +export interface BenchmarkRow { |
| 12 | + name: string; |
| 13 | + ops: number; |
| 14 | + mean: number; |
| 15 | + p75: number; |
| 16 | + p99: number; |
| 17 | + samples: number; |
| 18 | +} |
| 19 | + |
| 20 | +export interface BenchmarkOptions { |
| 21 | + timeMs?: number; |
| 22 | + warmupMs?: number; |
| 23 | +} |
| 24 | + |
| 25 | +export async function runBenchmarks( |
| 26 | + seed: SeedResult, |
| 27 | + options?: BenchmarkOptions, |
| 28 | +): Promise<BenchmarkRow[]> { |
| 29 | + const bench = new Bench({ |
| 30 | + time: options?.timeMs ?? 1000, |
| 31 | + warmupTime: options?.warmupMs ?? 300, |
| 32 | + }); |
| 33 | + |
| 34 | + const sampleTag = seed.tags[0]; |
| 35 | + const sampleList = seed.lists[0]; |
| 36 | + const sampleIds = seed.bookmarks.slice(0, 50).map((b) => b.id); |
| 37 | + |
| 38 | + bench.add("bookmarks.getBookmarks (page)", async () => { |
| 39 | + await seed.trpc.bookmarks.getBookmarks.query({ |
| 40 | + limit: 50, |
| 41 | + }); |
| 42 | + }); |
| 43 | + |
| 44 | + if (sampleTag) { |
| 45 | + bench.add("bookmarks.getBookmarks (tag filter)", async () => { |
| 46 | + await seed.trpc.bookmarks.getBookmarks.query({ |
| 47 | + limit: 50, |
| 48 | + tagId: sampleTag.id, |
| 49 | + }); |
| 50 | + }); |
| 51 | + } |
| 52 | + |
| 53 | + if (sampleList) { |
| 54 | + bench.add("bookmarks.getBookmarks (list filter)", async () => { |
| 55 | + await seed.trpc.bookmarks.getBookmarks.query({ |
| 56 | + limit: 50, |
| 57 | + listId: sampleList.id, |
| 58 | + }); |
| 59 | + }); |
| 60 | + } |
| 61 | + |
| 62 | + if (sampleList && sampleIds.length > 0) { |
| 63 | + bench.add("lists.getListsOfBookmark", async () => { |
| 64 | + await seed.trpc.lists.getListsOfBookmark.query({ |
| 65 | + bookmarkId: sampleIds[0], |
| 66 | + }); |
| 67 | + }); |
| 68 | + } |
| 69 | + |
| 70 | + bench.add("bookmarks.searchBookmarks", async () => { |
| 71 | + await seed.trpc.bookmarks.searchBookmarks.query({ |
| 72 | + text: seed.searchTerm, |
| 73 | + limit: 20, |
| 74 | + }); |
| 75 | + }); |
| 76 | + |
| 77 | + bench.add("bookmarks.getBookmarks (by ids)", async () => { |
| 78 | + await seed.trpc.bookmarks.getBookmarks.query({ |
| 79 | + ids: sampleIds.slice(0, 20), |
| 80 | + includeContent: false, |
| 81 | + }); |
| 82 | + }); |
| 83 | + |
| 84 | + logStep("Running benchmarks"); |
| 85 | + await bench.run(); |
| 86 | + logSuccess("Benchmarks complete"); |
| 87 | + |
| 88 | + const rows = bench.tasks |
| 89 | + .map((task) => { |
| 90 | + const result = task.result; |
| 91 | + |
| 92 | + // Check for errored state |
| 93 | + if ("error" in result) { |
| 94 | + console.error(`\n⚠️ Benchmark "${task.name}" failed with error:`); |
| 95 | + console.error(result.error); |
| 96 | + return null; |
| 97 | + } |
| 98 | + |
| 99 | + // Check if task completed successfully |
| 100 | + if (result.state !== "completed") { |
| 101 | + console.warn( |
| 102 | + `\n⚠️ Benchmark "${task.name}" did not complete. State: ${result.state}`, |
| 103 | + ); |
| 104 | + return null; |
| 105 | + } |
| 106 | + |
| 107 | + return toRow(task.name, result); |
| 108 | + }) |
| 109 | + .filter(Boolean) as BenchmarkRow[]; |
| 110 | + |
| 111 | + renderTable(rows); |
| 112 | + logInfo( |
| 113 | + "ops/s uses tinybench's hz metric; durations are recorded in milliseconds.", |
| 114 | + ); |
| 115 | + |
| 116 | + return rows; |
| 117 | +} |
| 118 | + |
| 119 | +function toRow(name: string, result: CompletedTaskResult): BenchmarkRow { |
| 120 | + // The statistics are now in result.latency and result.throughput |
| 121 | + const latency = result.latency; |
| 122 | + const throughput = result.throughput; |
| 123 | + |
| 124 | + return { |
| 125 | + name, |
| 126 | + ops: throughput.mean, // ops/s is the mean throughput |
| 127 | + mean: latency.mean, |
| 128 | + p75: latency.p75, |
| 129 | + p99: latency.p99, |
| 130 | + samples: latency.samplesCount, |
| 131 | + }; |
| 132 | +} |
| 133 | + |
| 134 | +function renderTable(rows: BenchmarkRow[]): void { |
| 135 | + const headers = ["Benchmark", "ops/s", "avg", "p75", "p99", "samples"]; |
| 136 | + |
| 137 | + const data = rows.map((row) => [ |
| 138 | + row.name, |
| 139 | + formatNumber(row.ops, 1), |
| 140 | + formatMs(row.mean), |
| 141 | + formatMs(row.p75), |
| 142 | + formatMs(row.p99), |
| 143 | + String(row.samples), |
| 144 | + ]); |
| 145 | + |
| 146 | + const columnWidths = headers.map((header, index) => |
| 147 | + Math.max(header.length, ...data.map((row) => row[index].length)), |
| 148 | + ); |
| 149 | + |
| 150 | + const formatRow = (cells: string[]): string => |
| 151 | + cells.map((cell, index) => cell.padEnd(columnWidths[index])).join(" "); |
| 152 | + |
| 153 | + console.log(""); |
| 154 | + console.log(formatRow(headers)); |
| 155 | + console.log(columnWidths.map((width) => "-".repeat(width)).join(" ")); |
| 156 | + data.forEach((row) => console.log(formatRow(row))); |
| 157 | + console.log(""); |
| 158 | +} |
0 commit comments