feat(mcp): emit structured snapshot in --json responses (#42098)

Author: pavelfeldman

docs/src/api/class-locator.md

@@ -239,6 +239,61 @@ When `true`, appends each element's bounding box as `[box=x,y,width,height]` to
 relative to the viewport, in CSS pixels, as returned by [`Element.getBoundingClientRect()`](https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect).
 Defaults to `false`.
 
+## async method: Locator.ariaSnapshotJSON
+* since: v1.63
+* langs: js
+- returns: <[Serializable]>
+
+Captures the aria snapshot of the given element as a free form JSON object.
+
+**Usage**
+
+```js
+await page.getByRole('list').ariaSnapshotJSON();
+```
+
+**Details**
+
+This method returns the same tree as [`method: Locator.ariaSnapshot`], serialized as a JSON value instead of YAML markup.
+The result is a list of nodes, each node being either a plain string with static text, or an object with the following properties:
+* `role` <[string]> Aria role of the element.
+* `name` <[string]> Accessible name of the element, if any.
+* `text` <[string]> Text content of the element, when it is the only child.
+* `children` <[Array]> Child nodes and text fragments.
+* Boolean and value properties for element state flags: `checked`, `disabled`, `expanded`, `active`, `invalid`, `level`, `pressed` and `selected`.
+* Additional element properties, for example `url` for links and `placeholder` for text boxes.
+* `ref` <[string]> Element reference for AI-optimized snapshots.
+* `cursor` <[string]> Set to `"pointer"` for clickable elements in AI-optimized snapshots.
+* `box` <[Object]> Bounding box of the element when [`option: Locator.ariaSnapshotJSON.boxes`] is set.
+
+### option: Locator.ariaSnapshotJSON.mode
+* since: v1.63
+- `mode` <[AriaSnapshotMode]<"ai"|"default">>
+
+When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults to `"default"`. See details in [`method: Locator.ariaSnapshot`].
+
+### option: Locator.ariaSnapshotJSON.timeout = %%-input-timeout-%%
+* since: v1.63
+
+### option: Locator.ariaSnapshotJSON.timeout = %%-input-timeout-js-%%
+* since: v1.63
+
+### option: Locator.ariaSnapshotJSON.signal = %%-input-signal-%%
+
+### option: Locator.ariaSnapshotJSON.depth
+* since: v1.63
+- `depth` <[int]>
+
+When specified, limits the depth of the snapshot.
+
+### option: Locator.ariaSnapshotJSON.boxes
+* since: v1.63
+- `boxes` <[boolean]>
+
+When `true`, includes each element's bounding box as a `box` property with `x`, `y`, `width` and `height`. Coordinates are
+relative to the viewport, in CSS pixels, as returned by [`Element.getBoundingClientRect()`](https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect).
+Defaults to `false`.
+
 ## async method: Locator.blur
 * since: v1.28
 

docs/src/api/class-page.md

@@ -4427,6 +4427,43 @@ When `true`, appends each element's bounding box as `[box=x,y,width,height]` to
 relative to the viewport, in CSS pixels, as returned by [`Element.getBoundingClientRect()`](https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect).
 Defaults to `false`.
 
+## async method: Page.ariaSnapshotJSON
+* since: v1.63
+* langs: js
+- returns: <[Serializable]>
+
+Captures the aria snapshot of the page as a free form JSON object.
+Returns the same tree as [`method: Page.ariaSnapshot`], serialized as a JSON value instead of YAML markup.
+See [`method: Locator.ariaSnapshotJSON`] for the details of the format.
+
+### option: Page.ariaSnapshotJSON.mode
+* since: v1.63
+- `mode` <[AriaSnapshotMode]<"ai"|"default">>
+
+When set to `"ai"`, returns a snapshot optimized for AI consumption: including element references like `[ref=e2]` and snapshots of `<iframe>`s. Defaults to `"default"`.
+
+### option: Page.ariaSnapshotJSON.timeout = %%-input-timeout-%%
+* since: v1.63
+
+### option: Page.ariaSnapshotJSON.timeout = %%-input-timeout-js-%%
+* since: v1.63
+
+### option: Page.ariaSnapshotJSON.signal = %%-input-signal-%%
+
+### option: Page.ariaSnapshotJSON.depth
+* since: v1.63
+- `depth` <[int]>
+
+When specified, limits the depth of the snapshot.
+
+### option: Page.ariaSnapshotJSON.boxes
+* since: v1.63
+- `boxes` <[boolean]>
+
+When `true`, includes each element's bounding box as a `box` property with `x`, `y`, `width` and `height`. Coordinates are
+relative to the viewport, in CSS pixels, as returned by [`Element.getBoundingClientRect()`](https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect).
+Defaults to `false`.
+
 ## async method: Page.tap
 * since: v1.8
 * discouraged: Use locator-based [`method: Locator.tap`] instead. Read more about [locators](../locators.md).

packages/injected/src/ariaSnapshot.ts

@@ -583,6 +583,74 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
   return { text: lines.join('\n'), iframeDepths };
 }
 
+export function renderAriaTreeAsJSON(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions): { json: aria.AriaSnapshotJSON, iframeDepths: Record<string, number> } {
+  const options = toInternalOptions(publicOptions);
+  const iframeDepths: Record<string, number> = {};
+
+  const visit = (ariaNode: aria.AriaNode, depth: number, renderCursorPointer: boolean): aria.AriaNodeJSON => {
+    if (ariaNode.role === 'iframe' && ariaNode.ref)
+      iframeDepths[ariaNode.ref] = depth;
+
+    const node: aria.AriaNodeJSON = { role: ariaNode.role };
+    if (ariaNode.name)
+      node.name = ariaNode.name;
+    if (ariaNode.checked === 'mixed' || ariaNode.checked === true)
+      node.checked = ariaNode.checked;
+    if (ariaNode.disabled)
+      node.disabled = true;
+    if (ariaNode.expanded)
+      node.expanded = true;
+    if (ariaNode.active && options.renderActive)
+      node.active = true;
+    if (ariaNode.invalid)
+      node.invalid = ariaNode.invalid;
+    if (ariaNode.level)
+      node.level = ariaNode.level;
+    if (ariaNode.pressed === 'mixed' || ariaNode.pressed === true)
+      node.pressed = ariaNode.pressed;
+    if (ariaNode.selected === true)
+      node.selected = true;
+    if (ariaNode.ref) {
+      node.ref = ariaNode.ref;
+      if (renderCursorPointer && aria.hasPointerCursor(ariaNode))
+        node.cursor = 'pointer';
+    }
+    if (options.renderBoxes) {
+      const element = ariaNodeElement(ariaNode);
+      if (element) {
+        const r = element.getBoundingClientRect();
+        node.box = { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) };
+      }
+    }
+    for (const [name, value] of Object.entries(ariaNode.props))
+      node[name] = value;
+
+    const singleTextChild = ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' ? ariaNode.children[0] : undefined;
+    const isAtDepthLimit = !!publicOptions.depth && depth === publicOptions.depth;
+    if (singleTextChild !== undefined) {
+      node.text = singleTextChild;
+    } else if (!isAtDepthLimit && ariaNode.children.length) {
+      const inCursorPointer = !!ariaNode.ref && renderCursorPointer && aria.hasPointerCursor(ariaNode);
+      node.children = ariaNode.children.map(child => {
+        if (typeof child === 'string')
+          return child;
+        return visit(child, depth + 1, renderCursorPointer && !inCursorPointer);
+      });
+    }
+    return node;
+  };
+
+  const json: aria.AriaSnapshotJSON = [];
+  const nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];
+  for (const nodeToRender of nodesToRender) {
+    if (typeof nodeToRender === 'string')
+      json.push(nodeToRender);
+    else
+      json.push(visit(nodeToRender, 0, !!options.renderCursorPointer));
+  }
+  return { json, iframeDepths };
+}
+
 function convertToBestGuessRegex(text: string): string {
   const dynamicContent = [
     // 550e8400-e29b-41d4-a716-446655440000

packages/injected/src/injectedScript.ts

@@ -20,7 +20,7 @@ import { splitTestIdAttributeNames } from '@isomorphic/locatorUtils';
 import { parseAttributeSelector, parseSelector, stringifySelector, visitAllSelectorParts } from '@isomorphic/selectorParser';
 import { cacheNormalizedWhitespaces, normalizeWhiteSpace, trimStringWithEllipsis } from '@isomorphic/stringUtils';
 
-import { generateAriaTree, getAllElementsMatchingExpectAriaTemplate, matchesExpectAriaTemplate, renderAriaTree, findNewElement } from './ariaSnapshot';
+import { generateAriaTree, getAllElementsMatchingExpectAriaTemplate, matchesExpectAriaTemplate, renderAriaTree, renderAriaTreeAsJSON, findNewElement } from './ariaSnapshot';
 import { beginDOMCaches, enclosingShadowRootOrDocument, endDOMCaches, isElementVisible, isInsideScope, parentElementOrShadowHost, setGlobalOptions } from './domUtils';
 import { Highlight } from './highlight';
 import { kLayoutSelectorNames, layoutSelectorScore } from './layoutSelectorUtils';
@@ -33,7 +33,7 @@ import { XPathEngine } from './xpathSelectorEngine';
 import { ConsoleAPI } from './consoleApi';
 import { UtilityScript } from './utilityScript';
 
-import type { AriaTemplateNode } from '@isomorphic/ariaSnapshot';
+import type { AriaSnapshotJSON, AriaTemplateNode } from '@isomorphic/ariaSnapshot';
 import type { CSSComplexSelectorList } from '@isomorphic/cssParser';
 import type { Language } from '@isomorphic/locatorGenerators';
 import type { AttributeSelectorPart, NestedSelectorBody, ParsedSelector, ParsedSelectorPart } from '@isomorphic/selectorParser';
@@ -328,6 +328,16 @@ export class InjectedScript {
     return { text: rendered.text, iframeRefs: ariaSnapshot.iframeRefs, iframeDepths: rendered.iframeDepths };
   }
 
+  ariaSnapshotJSON(node: Node, options: AriaTreeOptions & { depth?: number }): { json: AriaSnapshotJSON, iframeRefs: string[], iframeDepths: Record<string, number> } {
+    if (node.nodeType !== Node.ELEMENT_NODE)
+      throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
+    options = { ...options, refPrefix: this._frameSeq && options.mode === 'ai' ? 'f' + this._frameSeq : '' };
+    const ariaSnapshot = generateAriaTree(node as Element, options);
+    const rendered = renderAriaTreeAsJSON(ariaSnapshot, options);
+    this._lastAriaSnapshotForQuery = ariaSnapshot;
+    return { json: rendered.json, iframeRefs: ariaSnapshot.iframeRefs, iframeDepths: rendered.iframeDepths };
+  }
+
   ariaSnapshotForRecorder(): { ariaSnapshot: string, refs: Map<Element, string> } {
     const tree = generateAriaTree(this.document.body, { mode: 'ai' });
     const { text: ariaSnapshot } = renderAriaTree(tree, { mode: 'ai' });

packages/isomorphic/ariaSnapshot.ts

@@ -55,6 +55,15 @@ export function hasPointerCursor(ariaNode: AriaNode): boolean {
   return ariaNode.box.cursor === 'pointer';
 }
 
+// Free form JSON serialization of the aria tree. Nodes are either static text
+// fragments or plain objects with the role, name, state flags and children.
+export type AriaNodeJSON = {
+  [key: string]: string | number | boolean | object | undefined;
+  children?: (AriaNodeJSON | string)[];
+};
+
+export type AriaSnapshotJSON = (AriaNodeJSON | string)[];
+
 // We pass parsed template between worlds using JSON, make it easy.
 export type AriaRegex = { pattern: string };
 

packages/isomorphic/protocolMetainfo.ts

@@ -126,6 +126,7 @@ export const methodMetainfo = new Map<string, MethodMetainfo>([
   ['Frame.addScriptTag', { title: 'Add script tag', snapshot: true, pause: true, }],
   ['Frame.addStyleTag', { title: 'Add style tag', snapshot: true, pause: true, }],
   ['Frame.ariaSnapshot', { title: 'Aria snapshot', group: 'getter', }],
+  ['Frame.ariaSnapshotJSON', { title: 'Aria snapshot JSON', group: 'getter', }],
   ['Frame.blur', { title: 'Blur', slowMo: true, snapshot: true, pause: true, }],
   ['Frame.check', { title: 'Check', slowMo: true, snapshot: true, pause: true, input: true, isAutoWaiting: true, }],
   ['Frame.click', { title: 'Click', slowMo: true, snapshot: true, pause: true, input: true, isAutoWaiting: true, }],

packages/playwright-client/types/types.d.ts

@@ -2138,6 +2138,54 @@ export interface Page {
     timeout?: number;
   }): Promise<string>;
 
+  /**
+   * Captures the aria snapshot of the page as a free form JSON object. Returns the same tree as
+   * [page.ariaSnapshot([options])](https://playwright.dev/docs/api/class-page#page-aria-snapshot), serialized as a JSON
+   * value instead of YAML markup. See
+   * [locator.ariaSnapshotJSON([options])](https://playwright.dev/docs/api/class-locator#locator-aria-snapshot-json) for
+   * the details of the format.
+   * @param options
+   */
+  ariaSnapshotJSON(options?: {
+    /**
+     * When `true`, includes each element's bounding box as a `box` property with `x`, `y`, `width` and `height`.
+     * Coordinates are relative to the viewport, in CSS pixels, as returned by
+     * [`Element.getBoundingClientRect()`](https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect).
+     * Defaults to `false`.
+     */
+    boxes?: boolean;
+
+    /**
+     * When specified, limits the depth of the snapshot.
+     */
+    depth?: number;
+
+    /**
+     * When set to `"ai"`, returns a snapshot optimized for AI consumption: including element references like `[ref=e2]`
+     * and snapshots of `<iframe>`s. Defaults to `"default"`.
+     */
+    mode?: "ai"|"default";
+
+    /**
+     * Allows to cancel the operation using an
+     * [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal). If the signal is aborted, the
+     * operation will be aborted and throw an error. Note that providing a signal does not disable the default timeout,
+     * which can be changed using
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout); pass
+     * `timeout: 0` to disable the timeout entirely.
+     */
+    signal?: AbortSignal;
+
+    /**
+     * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout`
+     * option in the config, or by using the
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+     */
+    timeout?: number;
+  }): Promise<Serializable>;
+
   /**
    * Brings page to front (activates tab).
    */
@@ -14350,6 +14398,74 @@ export interface Locator {
     timeout?: number;
   }): Promise<string>;
 
+  /**
+   * Captures the aria snapshot of the given element as a free form JSON object.
+   *
+   * **Usage**
+   *
+   * ```js
+   * await page.getByRole('list').ariaSnapshotJSON();
+   * ```
+   *
+   * **Details**
+   *
+   * This method returns the same tree as
+   * [locator.ariaSnapshot([options])](https://playwright.dev/docs/api/class-locator#locator-aria-snapshot), serialized
+   * as a JSON value instead of YAML markup. The result is a list of nodes, each node being either a plain string with
+   * static text, or an object with the following properties:
+   * - `role` <[string]> Aria role of the element.
+   * - `name` <[string]> Accessible name of the element, if any.
+   * - `text` <[string]> Text content of the element, when it is the only child.
+   * - `children` <[Array]> Child nodes and text fragments.
+   * - Boolean and value properties for element state flags: `checked`, `disabled`, `expanded`, `active`, `invalid`,
+   *   `level`, `pressed` and `selected`.
+   * - Additional element properties, for example `url` for links and `placeholder` for text boxes.
+   * - `ref` <[string]> Element reference for AI-optimized snapshots.
+   * - `cursor` <[string]> Set to `"pointer"` for clickable elements in AI-optimized snapshots.
+   * - `box` <[Object]> Bounding box of the element when
+   *   [`boxes`](https://playwright.dev/docs/api/class-locator#locator-aria-snapshot-json-option-boxes) is set.
+   * @param options
+   */
+  ariaSnapshotJSON(options?: {
+    /**
+     * When `true`, includes each element's bounding box as a `box` property with `x`, `y`, `width` and `height`.
+     * Coordinates are relative to the viewport, in CSS pixels, as returned by
+     * [`Element.getBoundingClientRect()`](https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect).
+     * Defaults to `false`.
+     */
+    boxes?: boolean;
+
+    /**
+     * When specified, limits the depth of the snapshot.
+     */
+    depth?: number;
+
+    /**
+     * When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults to `"default"`. See details in
+     * [locator.ariaSnapshot([options])](https://playwright.dev/docs/api/class-locator#locator-aria-snapshot).
+     */
+    mode?: "ai"|"default";
+
+    /**
+     * Allows to cancel the operation using an
+     * [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal). If the signal is aborted, the
+     * operation will be aborted and throw an error. Note that providing a signal does not disable the default timeout,
+     * which can be changed using
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout); pass
+     * `timeout: 0` to disable the timeout entirely.
+     */
+    signal?: AbortSignal;
+
+    /**
+     * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout`
+     * option in the config, or by using the
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+     */
+    timeout?: number;
+  }): Promise<Serializable>;
+
   /**
    * Calls [blur](https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement/blur) on the element.
    * @param options

packages/playwright-core/src/client/channels.d.ts

@@ -2179,6 +2179,7 @@ export interface FrameChannel extends FrameEventTarget, Channel {
   addScriptTag(params: FrameAddScriptTagParams, options: TimeoutOptions): Promise<FrameAddScriptTagResult>;
   addStyleTag(params: FrameAddStyleTagParams, options: TimeoutOptions): Promise<FrameAddStyleTagResult>;
   ariaSnapshot(params: FrameAriaSnapshotParams, options: TimeoutOptions): Promise<FrameAriaSnapshotResult>;
+  ariaSnapshotJSON(params: FrameAriaSnapshotJSONParams, options: TimeoutOptions): Promise<FrameAriaSnapshotJSONResult>;
   blur(params: FrameBlurParams, options: TimeoutOptions): Promise<FrameBlurResult>;
   check(params: FrameCheckParams, options: TimeoutOptions): Promise<FrameCheckResult>;
   click(params: FrameClickParams, options: TimeoutOptions): Promise<FrameClickResult>;
@@ -2301,6 +2302,21 @@ export type FrameAriaSnapshotOptions = {
 export type FrameAriaSnapshotResult = {
   snapshot: string,
 };
+export type FrameAriaSnapshotJSONParams = {
+  mode?: 'ai' | 'default',
+  selector?: string,
+  depth?: number,
+  boxes?: boolean,
+};
+export type FrameAriaSnapshotJSONOptions = {
+  mode?: 'ai' | 'default',
+  selector?: string,
+  depth?: number,
+  boxes?: boolean,
+};
+export type FrameAriaSnapshotJSONResult = {
+  snapshot: any,
+};
 export type FrameBlurParams = {
   selector: string,
   strict?: boolean,