chore: step error recovery framework (#36996)

Author: pavelfeldman

packages/playwright-core/src/client/channelOwner.ts

@@ -20,7 +20,7 @@ import { methodMetainfo } from '../utils/isomorphic/protocolMetainfo';
 import { captureLibraryStackTrace } from './clientStackTrace';
 import { stringifyStackFrames } from '../utils/isomorphic/stackTrace';
 
-import type { ClientInstrumentation } from './clientInstrumentation';
+import type { ClientInstrumentation, RecoverFromApiErrorHandler } from './clientInstrumentation';
 import type { Connection } from './connection';
 import type { Logger } from './types';
 import type { ValidatorContext } from '../protocol/validator';
@@ -199,7 +199,14 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
       else
         e.stack = '';
       if (!options?.internal) {
+        const recoveryHandlers: RecoverFromApiErrorHandler[] = [];
         apiZone.error = e;
+        this._instrumentation.onApiCallRecovery(apiZone, e, recoveryHandlers);
+        for (const handler of recoveryHandlers) {
+          const recoverResult = await handler();
+          if (recoverResult.status === 'recovered')
+            return recoverResult.value as R;
+        }
         logApiCall(this._platform, logger, `<= ${apiZone.apiName} failed`);
         this._instrumentation.onApiCallEnd(apiZone);
       }

packages/playwright-core/src/client/clientInstrumentation.ts

@@ -28,12 +28,20 @@ export interface ApiCallData {
   error?: Error;
 }
 
+export type RecoverFromApiErrorResult = {
+  status: 'recovered' | 'failed';
+  value?: string | number | boolean | undefined;
+};
+
+export type RecoverFromApiErrorHandler = () => Promise<RecoverFromApiErrorResult>;
+
 export interface ClientInstrumentation {
   addListener(listener: ClientInstrumentationListener): void;
   removeListener(listener: ClientInstrumentationListener): void;
   removeAllListeners(): void;
   onApiCallBegin(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any> }): void;
-  onApiCallEnd(apiCal: ApiCallData): void;
+  onApiCallRecovery(apiCall: ApiCallData, error: Error, recoveryHandlers: RecoverFromApiErrorHandler[]): void;
+  onApiCallEnd(apiCall: ApiCallData): void;
   onWillPause(options: { keepTestTimeout: boolean }): void;
 
   runAfterCreateBrowserContext(context: BrowserContext): Promise<void>;
@@ -44,6 +52,7 @@ export interface ClientInstrumentation {
 
 export interface ClientInstrumentationListener {
   onApiCallBegin?(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any>  }): void;
+  onApiCallRecovery?(apiCall: ApiCallData, error: Error, recoveryHandlers: RecoverFromApiErrorHandler[]): void;
   onApiCallEnd?(apiCall: ApiCallData): void;
   onWillPause?(options: { keepTestTimeout: boolean }): void;
 

packages/playwright/src/common/ipc.ts

@@ -21,6 +21,7 @@ import { serializeCompilationCache } from '../transform/compilationCache';
 import type { ConfigLocation, FullConfigInternal } from './config';
 import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
 import type { SerializedCompilationCache  } from '../transform/compilationCache';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
 
 export type ConfigCLIOverrides = {
   debug?: boolean;
@@ -66,6 +67,7 @@ export type WorkerInitParams = {
   projectId: string;
   config: SerializedConfig;
   artifactsDir: string;
+  recoverFromStepErrors: boolean;
 };
 
 export type TestBeginPayload = {
@@ -105,6 +107,14 @@ export type StepBeginPayload = {
   location?: { file: string, line: number, column: number };
 };
 
+export type StepRecoverFromErrorPayload = {
+  testId: string;
+  stepId: string;
+  error: TestInfoErrorImpl;
+};
+
+export type ResumeAfterStepErrorPayload = RecoverFromStepErrorResult;
+
 export type StepEndPayload = {
   testId: string;
   stepId: string;

packages/playwright/src/index.ts

@@ -286,6 +286,11 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
         if (data.apiName === 'tracing.group')
           tracingGroupSteps.push(step);
       },
+      onApiCallRecovery: (data, error, recoveryHandlers) => {
+        const step = data.userData as TestStepInternal;
+        if (step)
+          recoveryHandlers.push(() => step.recoverFromStepError(error));
+      },
       onApiCallEnd: data => {
         // "tracing.group" step will end later, when "tracing.groupEnd" finishes.
         if (data.apiName === 'tracing.group')

packages/playwright/src/isomorphic/testServerConnection.ts

@@ -17,6 +17,7 @@
 import * as events from './events';
 
 import type { TestServerInterface, TestServerInterfaceEvents } from '@testIsomorphic/testServerInterface';
+import type * as reporterTypes from '../../types/testReporter';
 
 // -- Reuse boundary -- Everything below this line is reused in the vscode extension.
 
@@ -68,12 +69,14 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
   readonly onStdio: events.Event<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>;
   readonly onTestFilesChanged: events.Event<{ testFiles: string[] }>;
   readonly onLoadTraceRequested: events.Event<{ traceUrl: string }>;
+  readonly onRecoverFromStepError: events.Event<{ stepId: string, message: string, location: reporterTypes.Location }>;
 
   private _onCloseEmitter = new events.EventEmitter<void>();
   private _onReportEmitter = new events.EventEmitter<any>();
   private _onStdioEmitter = new events.EventEmitter<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>();
   private _onTestFilesChangedEmitter = new events.EventEmitter<{ testFiles: string[] }>();
   private _onLoadTraceRequestedEmitter = new events.EventEmitter<{ traceUrl: string }>();
+  private _onRecoverFromStepErrorEmitter = new events.EventEmitter<{ stepId: string, message: string, location: reporterTypes.Location }>();
 
   private _lastId = 0;
   private _transport: TestServerTransport;
@@ -87,6 +90,7 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
     this.onStdio = this._onStdioEmitter.event;
     this.onTestFilesChanged = this._onTestFilesChangedEmitter.event;
     this.onLoadTraceRequested = this._onLoadTraceRequestedEmitter.event;
+    this.onRecoverFromStepError = this._onRecoverFromStepErrorEmitter.event;
 
     this._transport = transport;
     this._transport.onmessage(data => {
@@ -147,6 +151,8 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
       this._onTestFilesChangedEmitter.fire(params);
     else if (method === 'loadTraceRequested')
       this._onLoadTraceRequestedEmitter.fire(params);
+    else if (method === 'recoverFromStepError')
+      this._onRecoverFromStepErrorEmitter.fire(params);
   }
 
   async initialize(params: Parameters<TestServerInterface['initialize']>[0]): ReturnType<TestServerInterface['initialize']> {
@@ -241,6 +247,10 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
     await this._sendMessage('closeGracefully', params);
   }
 
+  async resumeAfterStepError(params: Parameters<TestServerInterface['resumeAfterStepError']>[0]): Promise<void> {
+    await this._sendMessage('resumeAfterStepError', params);
+  }
+
   close() {
     try {
       this._transport.close();

packages/playwright/src/isomorphic/testServerInterface.ts

@@ -22,13 +22,20 @@ import type * as reporterTypes from '../../types/testReporter';
 
 export type ReportEntry = JsonEvent;
 
+export type RecoverFromStepErrorResult = {
+  stepId: string;
+  status: 'recovered' | 'failed';
+  value?: string | number | boolean | undefined;
+};
+
 export interface TestServerInterface {
   initialize(params: {
     serializer?: string,
     closeOnDisconnect?: boolean,
     interceptStdio?: boolean,
     watchTestDirs?: boolean,
     populateDependenciesOnList?: boolean,
+    recoverFromStepErrors?: boolean,
   }): Promise<void>;
 
   ping(params: {}): Promise<void>;
@@ -113,6 +120,8 @@ export interface TestServerInterface {
   stopTests(params: {}): Promise<void>;
 
   closeGracefully(params: {}): Promise<void>;
+
+  resumeAfterStepError(params: RecoverFromStepErrorResult): Promise<void>;
 }
 
 export interface TestServerInterfaceEvents {
@@ -127,4 +136,5 @@ export interface TestServerInterfaceEventEmitters {
   dispatchEvent(event: 'stdio', params: { type: 'stdout' | 'stderr', text?: string, buffer?: string }): void;
   dispatchEvent(event: 'testFilesChanged', params: { testFiles: string[] }): void;
   dispatchEvent(event: 'loadTraceRequested', params: { traceUrl: string }): void;
+  dispatchEvent(event: 'recoverFromStepError', params: { stepId: string, message: string, location: reporterTypes.Location }): void;
 }

packages/playwright/src/matchers/expect.ts

@@ -365,20 +365,37 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
 
       const step = testInfo._addStep(stepInfo);
 
-      const reportStepError = (e: Error | unknown) => {
+      const reportStepError = (isAsync: boolean, e: Error | unknown) => {
         const jestError = isJestError(e) ? e : null;
-        const error = jestError ? new ExpectError(jestError, customMessage, stackFrames) : e;
+        const expectError = jestError ? new ExpectError(jestError, customMessage, stackFrames) : undefined;
         if (jestError?.matcherResult.suggestedRebaseline) {
           // NOTE: this is a workaround for the fact that we can't pass the suggested rebaseline
           // for passing matchers. See toMatchAriaSnapshot for a counterpart.
           step.complete({ suggestedRebaseline: jestError?.matcherResult.suggestedRebaseline });
           return;
         }
+
+        const error = expectError ?? e;
         step.complete({ error });
-        if (this._info.isSoft)
-          testInfo._failWithError(error);
-        else
-          throw error;
+
+        if (!isAsync || !expectError) {
+          if (this._info.isSoft)
+            testInfo._failWithError(error);
+          else
+            throw error;
+          return;
+        }
+
+        // Recoverable async failure.
+        return (async () => {
+          const recoveryResult = await step.recoverFromStepError(expectError);
+          if (recoveryResult.status === 'recovered')
+            return recoveryResult.value as any;
+          if (this._info.isSoft)
+            testInfo._failWithError(expectError);
+          else
+            throw expectError;
+        })();
       };
 
       const finalizer = () => {
@@ -390,11 +407,11 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
         const callback = () => matcher.call(target, ...args);
         const result = currentZone().with('stepZone', step).run(callback);
         if (result instanceof Promise)
-          return result.then(finalizer).catch(reportStepError);
+          return result.then(finalizer).catch(reportStepError.bind(null, true));
         finalizer();
         return result;
       } catch (e) {
-        reportStepError(e);
+        void reportStepError(false, e);
       }
     };
   }

packages/playwright/src/runner/dispatcher.ts

@@ -26,11 +26,12 @@ import type { ProcessExitData } from './processHost';
 import type { TestGroup } from './testGroups';
 import type { TestError, TestResult, TestStep } from '../../types/testReporter';
 import type { FullConfigInternal } from '../common/config';
-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload } from '../common/ipc';
+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, StepRecoverFromErrorPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload } from '../common/ipc';
 import type { Suite } from '../common/test';
 import type { TestCase } from '../common/test';
 import type { ReporterV2 } from '../reporters/reporterV2';
 import type { RegisteredListener } from 'playwright-core/lib/utils';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
 
 
 export type EnvByProjectId = Map<string, Record<string, string | undefined>>;
@@ -218,7 +219,8 @@ export class Dispatcher {
   _createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: SerializedConfig) {
     const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;
     const outputDir = projectConfig.project.outputDir;
-    const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);
+    const recoverFromStepErrors = this._failureTracker.canRecoverFromStepError();
+    const worker = new WorkerHost(testGroup, parallelIndex, loaderData, recoverFromStepErrors, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);
     const handleOutput = (params: TestOutputPayload) => {
       const chunk = chunkFromParams(params);
       if (worker.didFail()) {
@@ -396,6 +398,26 @@ class JobDispatcher {
     this._reporter.onStepEnd?.(test, result, step);
   }
 
+  private _onStepRecoverFromError(resumeAfterStepError: (result: RecoverFromStepErrorResult) => void, params: StepRecoverFromErrorPayload) {
+    const data = this._dataByTestId.get(params.testId);
+    if (!data) {
+      resumeAfterStepError({ stepId: params.stepId, status: 'failed' });
+      return;
+    }
+    const { steps } = data;
+    const step = steps.get(params.stepId);
+    if (!step) {
+      resumeAfterStepError({ stepId: params.stepId, status: 'failed' });
+      return;
+    }
+
+    const testError: TestError = {
+      ...params.error,
+      location: step.location,
+    };
+    this._failureTracker.recoverFromStepError(params.stepId, testError, resumeAfterStepError);
+  }
+
   private _onAttach(params: AttachmentPayload) {
     const data = this._dataByTestId.get(params.testId)!;
     if (!data) {
@@ -560,12 +582,14 @@ class JobDispatcher {
       }),
     };
     worker.runTestGroup(runPayload);
+    const resumeAfterStepError = worker.resumeAfterStepError.bind(worker);
 
     this._listeners = [
       eventsHelper.addEventListener(worker, 'testBegin', this._onTestBegin.bind(this)),
       eventsHelper.addEventListener(worker, 'testEnd', this._onTestEnd.bind(this)),
       eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
       eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),
+      eventsHelper.addEventListener(worker, 'stepRecoverFromError', this._onStepRecoverFromError.bind(this, resumeAfterStepError)),
       eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),
       eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),
       eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),

packages/playwright/src/runner/failureTracker.ts

@@ -14,18 +14,30 @@
  * limitations under the License.
  */
 
-import type { TestResult } from '../../types/testReporter';
+import type { RecoverFromStepErrorResult } from '@testIsomorphic/testServerInterface';
+import type { TestResult, TestError } from '../../types/testReporter';
 import type { FullConfigInternal } from '../common/config';
 import type { Suite, TestCase } from '../common/test';
 
+export type RecoverFromStepErrorHandler = (stepId: string, error: TestError) => Promise<RecoverFromStepErrorResult>;
+
 export class FailureTracker {
   private _failureCount = 0;
   private _hasWorkerErrors = false;
   private _rootSuite: Suite | undefined;
+  private _recoverFromStepErrorHandler: RecoverFromStepErrorHandler | undefined;
 
   constructor(private _config: FullConfigInternal) {
   }
 
+  canRecoverFromStepError(): boolean {
+    return !!this._recoverFromStepErrorHandler;
+  }
+
+  setRecoverFromStepErrorHandler(recoverFromStepErrorHandler: RecoverFromStepErrorHandler) {
+    this._recoverFromStepErrorHandler = recoverFromStepErrorHandler;
+  }
+
   onRootSuite(rootSuite: Suite) {
     this._rootSuite = rootSuite;
   }
@@ -36,6 +48,14 @@ export class FailureTracker {
       ++this._failureCount;
   }
 
+  recoverFromStepError(stepId: string, error: TestError, resumeAfterStepError: (result: RecoverFromStepErrorResult) => void) {
+    if (!this._recoverFromStepErrorHandler) {
+      resumeAfterStepError({ stepId, status: 'failed' });
+      return;
+    }
+    void this._recoverFromStepErrorHandler(stepId, error).then(resumeAfterStepError).catch(() => {});
+  }
+
   onWorkerError() {
     this._hasWorkerErrors = true;
   }