feat(trace): add action screenshot and aria snapshot events

Author: pavelfeldman

docs/src/api/class-tracing.md

@@ -148,6 +148,17 @@ a timeline preview.
 
 ### option: Tracing.start.snapshots
 * since: v1.12
+* langs: js
+- `snapshots` <[boolean]|[Object]>
+  - `dom` ?<[boolean]> Capture DOM snapshot on every action and record network activity. Optional.
+  - `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional.
+  - `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional.
+
+Which snapshots to capture on every action. Passing `true` is a shortcut for `{ dom: true }`.
+
+### option: Tracing.start.snapshots
+* since: v1.12
+* langs: java, python, csharp
 - `snapshots` <[boolean]>
 
 If this option is true tracing will

packages/playwright-client/types/types.d.ts

@@ -23692,11 +23692,24 @@ export interface Tracing {
     screenshots?: boolean;
 
     /**
-     * If this option is true tracing will
-     * - capture DOM snapshot on every action
-     * - record network activity
+     * Which snapshots to capture on every action. Passing `true` is a shortcut for `{ dom: true }`.
      */
-    snapshots?: boolean;
+    snapshots?: boolean|{
+      /**
+       * Capture DOM snapshot on every action and record network activity. Optional.
+       */
+      dom?: boolean;
+
+      /**
+       * Capture aria snapshot of the page on every action. Optional.
+       */
+      aria?: boolean;
+
+      /**
+       * Capture a screenshot of the page on every action. Optional.
+       */
+      screen?: boolean;
+    };
 
     /**
      * Whether to include source files for trace actions.

packages/playwright-core/src/client/channels.d.ts

@@ -5047,14 +5047,18 @@ export interface TracingChannel extends TracingEventTarget, Channel {
 }
 export type TracingTracingStartParams = {
   name?: string,
-  snapshots?: boolean,
-  screenshots?: boolean,
+  snapshotDom?: boolean,
+  snapshotAria?: boolean,
+  snapshotScreen?: boolean,
+  screencast?: boolean,
   live?: boolean,
 };
 export type TracingTracingStartOptions = {
   name?: string,
-  snapshots?: boolean,
-  screenshots?: boolean,
+  snapshotDom?: boolean,
+  snapshotAria?: boolean,
+  snapshotScreen?: boolean,
+  screencast?: boolean,
   live?: boolean,
 };
 export type TracingTracingStartResult = void;

packages/playwright-core/src/client/tracing.ts

@@ -42,14 +42,17 @@ export class Tracing extends ChannelOwner<channels.TracingChannel> implements ap
     super(parent, type, guid, initializer);
   }
 
-  async start(options: { name?: string, title?: string, snapshots?: boolean, screenshots?: boolean, sources?: boolean, live?: boolean } = {}) {
+  async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, live?: boolean } = {}) {
     await this._wrapApiCall(async () => {
       this._includeSources = !!options.sources;
       this._isLive = !!options.live;
+      const snapshots = typeof options.snapshots === 'object' ? options.snapshots : { dom: options.snapshots };
       await this._channel.tracingStart({
         name: options.name,
-        snapshots: options.snapshots,
-        screenshots: options.screenshots,
+        snapshotDom: snapshots.dom,
+        snapshotAria: snapshots.aria,
+        snapshotScreen: snapshots.screen,
+        screencast: options.screenshots,
         live: options.live,
       }, kNoTimeout);
       const { traceName } = await this._channel.tracingStartChunk({ name: options.name, title: options.title }, kNoTimeout);

packages/playwright-core/src/server/bidi/bidiInput.ts

@@ -60,7 +60,7 @@ export class RawKeyboardImpl implements input.RawKeyboard {
         return;
       }
       try {
-        frame = await progress.race(element.contentFrame(progress));
+        frame = await element.contentFrame(progress);
       } finally {
         element.dispose();
       }

packages/playwright-core/src/server/channels.d.ts

@@ -5048,14 +5048,18 @@ export interface TracingChannel extends TracingEventTarget, Channel {
 }
 export type TracingTracingStartParams = {
   name?: string,
-  snapshots?: boolean,
-  screenshots?: boolean,
+  snapshotDom?: boolean,
+  snapshotAria?: boolean,
+  snapshotScreen?: boolean,
+  screencast?: boolean,
   live?: boolean,
 };
 export type TracingTracingStartOptions = {
   name?: string,
-  snapshots?: boolean,
-  screenshots?: boolean,
+  snapshotDom?: boolean,
+  snapshotAria?: boolean,
+  snapshotScreen?: boolean,
+  screencast?: boolean,
   live?: boolean,
 };
 export type TracingTracingStartResult = void;

packages/playwright-core/src/server/debugger.ts

@@ -117,7 +117,8 @@ export class Debugger extends SdkObject<DebuggerEventMap> implements Instrumenta
     this._muted = muted;
   }
 
-  async onBeforeCall(sdkObject: SdkObject, metadata: CallMetadata): Promise<void> {
+  async onBeforeCall(progress: Progress, sdkObject: SdkObject): Promise<void> {
+    const { metadata } = progress;
     if (!metadata.internal && metadata.method)
       this._ongoingCalls.set(metadata.id, { metadata, sentLogCount: 0, status: 'running' });
     if (this._apiCallsEnabled) {
@@ -131,10 +132,11 @@ export class Debugger extends SdkObject<DebuggerEventMap> implements Instrumenta
     const pauseBeforeAction = !!this._pauseAt.next && !!metainfo?.pause && (this._pauseBeforeWaitingActions || !metainfo?.isAutoWaiting);
     const pauseOnLocation = !!this._pauseAt.location && matchesLocation(metadata, this._pauseAt.location);
     if (pauseOnPauseCall || pauseBeforeAction || pauseOnLocation)
-      await this._pause(sdkObject, metadata);
+      await this._pause(progress, sdkObject);
   }
 
-  async onBeforeInputAction(sdkObject: SdkObject, metadata: CallMetadata, point?: Point): Promise<void> {
+  async onBeforeInputAction(progress: Progress, sdkObject: SdkObject, point?: Point): Promise<void> {
+    const { metadata } = progress;
     const call = this._ongoingCalls.get(metadata.id);
     if (call) {
       call.actionPoint = point;
@@ -148,10 +150,11 @@ export class Debugger extends SdkObject<DebuggerEventMap> implements Instrumenta
     const metainfo = getMetainfo(metadata);
     const pauseBeforeInput = !!this._pauseAt.next && !!metainfo?.pause && !!metainfo?.isAutoWaiting && !this._pauseBeforeWaitingActions;
     if (pauseBeforeInput)
-      await this._pause(sdkObject, metadata);
+      await this._pause(progress, sdkObject);
   }
 
-  async onAfterCall(sdkObject: SdkObject, metadata: CallMetadata): Promise<void> {
+  async onAfterCall(progress: Progress, sdkObject: SdkObject): Promise<void> {
+    const { metadata } = progress;
     const call = this._ongoingCalls.get(metadata.id);
     if (!call)
       return;
@@ -212,7 +215,8 @@ export class Debugger extends SdkObject<DebuggerEventMap> implements Instrumenta
       this.emit(Debugger.Events.ApiCallsUpdated, updates);
   }
 
-  private async _pause(sdkObject: SdkObject, metadata: CallMetadata) {
+  private async _pause(progress: Progress, sdkObject: SdkObject) {
+    const { metadata } = progress;
     if (this._muted || metadata.internal)
       return;
     if (this._pausedCall)

packages/playwright-core/src/server/dispatchers/dispatcher.ts

@@ -352,16 +352,21 @@ export class DispatcherConnection {
       log: [],
     };
 
-    const controller = dispatcher.createProgressController(callMetadata);
-    this._activeProgressControllers.set(callMetadata.id, controller);
+    const beforeController = dispatcher.createProgressController(callMetadata);
+    this._activeProgressControllers.set(callMetadata.id, beforeController);
+    // Be generous with the tracing timeout in case it wants to capture a screenshot, fail silently.
+    await beforeController.run(progress => sdkObject.instrumentation.onBeforeCall(progress, sdkObject), 3000).catch(() => {});
+    this._activeProgressControllers.delete(callMetadata.id);
 
-    await sdkObject.instrumentation.onBeforeCall(sdkObject, callMetadata);
     const response: any = { id };
     try {
       // If the dispatcher has been disposed while running the instrumentation call, error out.
       if (this._dispatcherByGuid.get(guid) !== dispatcher)
         throw new TargetClosedError(sdkObject.closeReason());
+      const controller = dispatcher.createProgressController(callMetadata);
+      this._activeProgressControllers.set(callMetadata.id, controller);
       const result = await controller.run(progress => (dispatcher as any)[method](validParams, progress), validMetadata.timeout);
+      this._activeProgressControllers.delete(callMetadata.id);
       const validator = findValidator(dispatcher._type, method, 'Result');
       response.result = validator(result, '', this._validatorToWireContext());
       callMetadata.result = result;
@@ -384,7 +389,10 @@ export class DispatcherConnection {
       callMetadata.error = response.error;
     } finally {
       callMetadata.endTime = monotonicTime();
-      await sdkObject.instrumentation.onAfterCall(sdkObject, callMetadata);
+      const afterController = dispatcher.createProgressController(callMetadata);
+      this._activeProgressControllers.set(callMetadata.id, afterController);
+      // Be generous with the tracing timeout in case it wants to capture a screenshot, fail silently.
+      await afterController.run(progress => sdkObject.instrumentation.onAfterCall(progress, sdkObject), 3000).catch(() => {});
       if (metainfo?.slowMo)
         await this._doSlowMo(sdkObject);
       this._activeProgressControllers.delete(callMetadata.id);
@@ -432,7 +440,8 @@ export class DispatcherConnection {
         log: [],
       };
       this._waitOperations.set(info.waitId, callMetadata);
-      await sdkObject.instrumentation.onBeforeCall(sdkObject, callMetadata).catch(() => {});
+      const controller = ProgressController.createForSdkObject(sdkObject, callMetadata);
+      await controller.run(progress => sdkObject.instrumentation.onBeforeCall(progress, sdkObject).catch(() => {}));
       return;
     }
 
@@ -448,7 +457,8 @@ export class DispatcherConnection {
       originalMetadata.endTime = monotonicTime();
       originalMetadata.error = info.error ? { error: { name: 'Error', message: info.error } } : undefined;
       this._waitOperations.delete(info.waitId);
-      await sdkObject.instrumentation.onAfterCall(sdkObject, originalMetadata).catch(() => {});
+      const controller = ProgressController.createForSdkObject(sdkObject, originalMetadata);
+      await controller.run(progress => sdkObject.instrumentation.onAfterCall(progress, sdkObject).catch(() => {}));
     }
   }
 }

packages/playwright-core/src/server/dom.ts

@@ -454,7 +454,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
     if (typeof maybeResult === 'string')
       return maybeResult;
     const point = roundPoint(maybeResult.point);
-    await progress.race(this.instrumentation.onBeforeInputAction(this, progress.metadata, point, maybeResult.box));
+    await this.instrumentation.onBeforeInputAction(progress, this, point, maybeResult.box);
 
     let hitTargetInterceptionHandle: js.JSHandle<HitTargetInterceptionResult> | undefined;
     if (force) {
@@ -577,7 +577,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
   async _selectOption(progress: Progress, elements: ElementHandle[], values: types.SelectOption[], options: types.CommonActionOptions): Promise<string[] | 'error:notconnected'> {
     let resultingOptions: string[] = [];
     const result = await this._retryAction(progress, 'select option', async progress => {
-      await progress.race(this.instrumentation.onBeforeInputAction(this, progress.metadata));
+      await this.instrumentation.onBeforeInputAction(progress, this);
       if (!options.force)
         progress.log(`  waiting for element to be visible and enabled`);
       const optionsToSelect = [...elements, ...values];
@@ -610,7 +610,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
   async _fill(progress: Progress, value: string, options: types.CommonActionOptions): Promise<'error:notconnected' | 'done'> {
     progress.log(`  fill("${value}")`);
     return await this._retryAction(progress, 'fill', async progress => {
-      await progress.race(this.instrumentation.onBeforeInputAction(this, progress.metadata));
+      await this.instrumentation.onBeforeInputAction(progress, this);
       if (!options.force)
         progress.log('  waiting for element to be visible, enabled and editable');
       const result = await progress.race(this.evaluateInUtility(async ([injected, node, { value, force }]) => {
@@ -744,7 +744,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
     if (result === 'error:notconnected' || !result.asElement())
       return 'error:notconnected';
     const retargeted = result.asElement() as ElementHandle<HTMLInputElement>;
-    await progress.race(this.instrumentation.onBeforeInputAction(this, progress.metadata));
+    await this.instrumentation.onBeforeInputAction(progress, this);
     if (localPaths || localDirectory) {
       const localPathsOrDirectory = localDirectory ? [localDirectory] : localPaths!;
       await progress.race(Promise.all((localPathsOrDirectory).map(localPath => (
@@ -785,7 +785,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
 
   async _type(progress: Progress, text: string, options: { delay?: number } & types.StrictOptions): Promise<'error:notconnected' | 'done'> {
     progress.log(`elementHandle.type("${text}")`);
-    await progress.race(this.instrumentation.onBeforeInputAction(this, progress.metadata));
+    await this.instrumentation.onBeforeInputAction(progress, this);
     const result = await this._focus(progress, true /* resetSelectionIfNotFocused */);
     if (result !== 'done')
       return result;
@@ -801,7 +801,7 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
 
   async _press(progress: Progress, key: string, options: { delay?: number, noWaitAfter?: boolean } & types.StrictOptions): Promise<'error:notconnected' | 'done'> {
     progress.log(`elementHandle.press("${key}")`);
-    await progress.race(this.instrumentation.onBeforeInputAction(this, progress.metadata));
+    await this.instrumentation.onBeforeInputAction(progress, this);
     return this._page.frameManager.waitForSignalsCreatedBy(progress, !options.noWaitAfter, async progress => {
       const result = await this._focus(progress, true /* resetSelectionIfNotFocused */);
       if (result !== 'done')

packages/playwright-core/src/server/input.ts

@@ -54,7 +54,7 @@ export class Keyboard {
   }
 
   async apiDown(progress: Progress, key: string) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page);
     await this.down(progress, key);
   }
 
@@ -82,7 +82,7 @@ export class Keyboard {
   }
 
   async apiUp(progress: Progress, key: string) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page);
     await this.up(progress, key);
   }
 
@@ -95,7 +95,7 @@ export class Keyboard {
   }
 
   async apiInsertText(progress: Progress, text: string) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page);
     await this.insertText(progress, text);
   }
 
@@ -104,7 +104,7 @@ export class Keyboard {
   }
 
   async apiType(progress: Progress, text: string, options?: { delay?: number }) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page);
     await this.type(progress, text, options);
   }
 
@@ -122,7 +122,7 @@ export class Keyboard {
   }
 
   async apiPress(progress: Progress, key: string, options: { delay?: number } = {}) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page);
     await this.press(progress, key, options);
   }
 
@@ -214,7 +214,7 @@ export class Mouse {
   }
 
   async apiMove(progress: Progress, x: number, y: number, options: { steps?: number, forClick?: boolean } = {}) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata, { x, y }));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page, { x, y });
     await this.move(progress, x, y, options);
   }
 
@@ -232,7 +232,7 @@ export class Mouse {
   }
 
   async apiDown(progress: Progress, options: { button?: types.MouseButton, clickCount?: number } = {}) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata, this._currentPoint()));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page, this._currentPoint());
     await this.down(progress, options);
   }
 
@@ -244,7 +244,7 @@ export class Mouse {
   }
 
   async apiUp(progress: Progress, options: { button?: types.MouseButton, clickCount?: number } = {}) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata, this._currentPoint()));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page, this._currentPoint());
     await this.up(progress, options);
   }
 
@@ -256,7 +256,7 @@ export class Mouse {
   }
 
   async apiClick(progress: Progress, x: number, y: number, options: { delay?: number, button?: types.MouseButton, clickCount?: number, steps?: number } = {}) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata, { x, y }));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page, { x, y });
     await this.click(progress, x, y, options);
   }
 
@@ -289,7 +289,7 @@ export class Mouse {
   }
 
   async apiWheel(progress: Progress, deltaX: number, deltaY: number) {
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page);
     await this._raw.wheel(progress, this._x, this._y, this._buttons, this._keyboard._modifiers(), deltaX, deltaY);
   }
 }
@@ -370,7 +370,7 @@ export class Touchscreen {
   async apiTap(progress: Progress, x: number, y: number) {
     if (!this._page.browserContext._options.hasTouch)
       throw new Error('hasTouch must be enabled on the browser context before using the touchscreen.');
-    await progress.race(this._page.instrumentation.onBeforeInputAction(this._page, progress.metadata, { x, y }));
+    await this._page.instrumentation.onBeforeInputAction(progress, this._page, { x, y });
     await this.tap(progress, x, y);
   }