feat(routeFromHar): add interceptAPIRequests option (#41294)

Adds opt-in `interceptAPIRequests` option to `BrowserContext.routeFromHAR` so `page.request.*` / `context.request.*` calls are also served from the HAR file.

API-request lookup is restricted to entries marked `_apiRequest: true` (already written by the HAR recorder), so browser-side recordings are never served to API requests for the same URL.

Fixes <#22869>.

Author: stkevintan

docs/src/api/class-browsercontext.md

@@ -1307,6 +1307,12 @@ When set to `minimal`, only record information necessary for routing from HAR. T
 
 Optional setting to control resource content management. If `attach` is specified, resources are persisted as separate files or entries in the ZIP archive. If `embed` is specified, content is stored inline the HAR file.
 
+### option: BrowserContext.routeFromHAR.interceptAPIRequests
+* since: v1.62
+- `interceptAPIRequests` <[boolean]>
+
+If set to `true`, requests made via [APIRequestContext] (such as [`property: BrowserContext.request`] or [`property: Page.request`]) are also served from the HAR file. By default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for backward compatibility.
+
 
 ## async method: BrowserContext.routeWebSocket
 * since: v1.48

packages/isomorphic/protocolMetainfo.ts

@@ -89,6 +89,8 @@ export const methodMetainfo = new Map<string, MethodMetainfo>([
   ['BrowserContext.setGeolocation', { title: 'Set geolocation', group: 'configuration', }],
   ['BrowserContext.setHTTPCredentials', { title: 'Set HTTP credentials', group: 'configuration', }],
   ['BrowserContext.setNetworkInterceptionPatterns', { title: 'Route requests', group: 'route', }],
+  ['BrowserContext.routeAPIRequestsFromHar', { internal: true, }],
+  ['BrowserContext.unrouteAPIRequestsFromHar', { internal: true, }],
   ['BrowserContext.setWebSocketInterceptionPatterns', { title: 'Route WebSockets', group: 'route', }],
   ['BrowserContext.setOffline', { title: 'Set offline mode', }],
   ['BrowserContext.storageState', { title: 'Get storage state', group: 'configuration', }],

packages/playwright-client/types/types.d.ts

@@ -10240,6 +10240,15 @@ export interface BrowserContext {
    * @param options
    */
   routeFromHAR(har: string, options?: {
+    /**
+     * If set to `true`, requests made via [APIRequestContext](https://playwright.dev/docs/api/class-apirequestcontext)
+     * (such as [browserContext.request](https://playwright.dev/docs/api/class-browsercontext#browser-context-request) or
+     * [page.request](https://playwright.dev/docs/api/class-page#page-request)) are also served from the HAR file. By
+     * default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for
+     * backward compatibility.
+     */
+    interceptAPIRequests?: boolean;
+
     /**
      * - If set to 'abort' any request not found in the HAR file will be aborted.
      * - If set to 'fallback' falls through to the next route handler in the handler chain.

packages/playwright-core/src/client/browserContext.ts

@@ -387,7 +387,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     await this._updateWebSocketInterceptionPatterns({ title: 'Route WebSockets' });
   }
 
-  async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean, updateContent?: 'attach' | 'embed', updateMode?: 'minimal' | 'full' } = {}): Promise<void> {
+  async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean, updateContent?: 'attach' | 'embed', updateMode?: 'minimal' | 'full', interceptAPIRequests?: boolean } = {}): Promise<void> {
     const localUtils = this._connection.localUtils();
     if (!localUtils)
       throw new Error('Route from har is not supported in thin clients');
@@ -398,6 +398,8 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     const harRouter = await HarRouter.create(localUtils, har, options.notFound || 'abort', { urlMatch: options.url });
     this._harRouters.push(harRouter);
     await harRouter.addContextRoute(this);
+    if (options.interceptAPIRequests)
+      await harRouter.addAPIRequestRoute(this);
   }
 
   private _disposeHarRouters() {

packages/playwright-core/src/client/channels.d.ts

@@ -1282,6 +1282,8 @@ export interface BrowserContextChannel extends BrowserContextEventTarget, Channe
   setGeolocation(params: BrowserContextSetGeolocationParams, signal: AbortSignal | undefined): Promise<BrowserContextSetGeolocationResult>;
   setHTTPCredentials(params: BrowserContextSetHTTPCredentialsParams, signal: AbortSignal | undefined): Promise<BrowserContextSetHTTPCredentialsResult>;
   setNetworkInterceptionPatterns(params: BrowserContextSetNetworkInterceptionPatternsParams, signal: AbortSignal | undefined): Promise<BrowserContextSetNetworkInterceptionPatternsResult>;
+  routeAPIRequestsFromHar(params: BrowserContextRouteAPIRequestsFromHarParams, signal: AbortSignal | undefined): Promise<BrowserContextRouteAPIRequestsFromHarResult>;
+  unrouteAPIRequestsFromHar(params: BrowserContextUnrouteAPIRequestsFromHarParams, signal: AbortSignal | undefined): Promise<BrowserContextUnrouteAPIRequestsFromHarResult>;
   setWebSocketInterceptionPatterns(params: BrowserContextSetWebSocketInterceptionPatternsParams, signal: AbortSignal | undefined): Promise<BrowserContextSetWebSocketInterceptionPatternsResult>;
   setOffline(params: BrowserContextSetOfflineParams, signal: AbortSignal | undefined): Promise<BrowserContextSetOfflineResult>;
   storageState(params: BrowserContextStorageStateParams, signal: AbortSignal | undefined): Promise<BrowserContextStorageStateResult>;
@@ -1515,6 +1517,28 @@ export type BrowserContextSetNetworkInterceptionPatternsOptions = {
 
 };
 export type BrowserContextSetNetworkInterceptionPatternsResult = void;
+export type BrowserContextRouteAPIRequestsFromHarParams = {
+  harId: string,
+  urlGlob?: string,
+  urlRegexSource?: string,
+  urlRegexFlags?: string,
+  notFound: 'abort' | 'fallback',
+};
+export type BrowserContextRouteAPIRequestsFromHarOptions = {
+  urlGlob?: string,
+  urlRegexSource?: string,
+  urlRegexFlags?: string,
+};
+export type BrowserContextRouteAPIRequestsFromHarResult = {
+  registrationId: string,
+};
+export type BrowserContextUnrouteAPIRequestsFromHarParams = {
+  registrationId: string,
+};
+export type BrowserContextUnrouteAPIRequestsFromHarOptions = {
+
+};
+export type BrowserContextUnrouteAPIRequestsFromHarResult = void;
 export type BrowserContextSetWebSocketInterceptionPatternsParams = {
   patterns: {
     glob?: string,

packages/playwright-core/src/client/harRouter.ts

@@ -15,6 +15,7 @@
  */
 
 import { debugLogger } from '@utils/debugLogger';
+import { isRegExp, isString } from '@isomorphic/rtti';
 
 import type { BrowserContext } from './browserContext';
 import type { LocalUtils } from './localUtils';
@@ -29,6 +30,7 @@ export class HarRouter {
   private _harId: string;
   private _notFoundAction: HarNotFoundAction;
   private _options: { urlMatch?: URLMatch; baseURL?: string; };
+  private _apiRequestRegistrations: { context: BrowserContext, registrationId: string }[] = [];
 
   static async create(localUtils: LocalUtils, file: string, notFoundAction: HarNotFoundAction, options: { urlMatch?: URLMatch }): Promise<HarRouter> {
     const { harId, error } = await localUtils.harOpen({ file });
@@ -117,11 +119,26 @@ export class HarRouter {
     await page.route(this._options.urlMatch || '**/*', route => this._handle(route));
   }
 
+  async addAPIRequestRoute(context: BrowserContext) {
+    const urlMatch = this._options.urlMatch;
+    const { registrationId } = await context._channel.routeAPIRequestsFromHar({
+      harId: this._harId,
+      urlGlob: isString(urlMatch) ? urlMatch : undefined,
+      urlRegexSource: isRegExp(urlMatch) ? urlMatch.source : undefined,
+      urlRegexFlags: isRegExp(urlMatch) ? urlMatch.flags : undefined,
+      notFound: this._notFoundAction,
+    }, undefined);
+    this._apiRequestRegistrations.push({ context, registrationId });
+  }
+
   async [Symbol.asyncDispose]() {
     await this.dispose();
   }
 
   dispose() {
+    for (const { context, registrationId } of this._apiRequestRegistrations)
+      context._channel.unrouteAPIRequestsFromHar({ registrationId }, undefined).catch(() => {});
+    this._apiRequestRegistrations = [];
     this._localUtils.harClose({ harId: this._harId }).catch(() => {});
   }
 }

packages/playwright-core/src/protocol/validator.ts

@@ -832,6 +832,20 @@ scheme.BrowserContextSetNetworkInterceptionPatternsParams = tObject({
   })),
 });
 scheme.BrowserContextSetNetworkInterceptionPatternsResult = tOptional(tObject({}));
+scheme.BrowserContextRouteAPIRequestsFromHarParams = tObject({
+  harId: tString,
+  urlGlob: tOptional(tString),
+  urlRegexSource: tOptional(tString),
+  urlRegexFlags: tOptional(tString),
+  notFound: tEnum(['abort', 'fallback']),
+});
+scheme.BrowserContextRouteAPIRequestsFromHarResult = tObject({
+  registrationId: tString,
+});
+scheme.BrowserContextUnrouteAPIRequestsFromHarParams = tObject({
+  registrationId: tString,
+});
+scheme.BrowserContextUnrouteAPIRequestsFromHarResult = tOptional(tObject({}));
 scheme.BrowserContextSetWebSocketInterceptionPatternsParams = tObject({
   patterns: tArray(tObject({
     glob: tOptional(tString),

packages/playwright-core/src/server/browserContext.ts

@@ -39,11 +39,13 @@ import type { Browser, BrowserOptions } from './browser';
 import type { ConsoleMessage } from './console';
 import type { Download } from './download';
 import type * as frames from './frames';
+import type { HarBackend } from './harBackend';
 import type { PageError } from './page';
 import type { Progress } from './progress';
 import type { ClientCertificatesProxy } from './socksClientCertificatesInterceptor';
 import type { SerializedStorage } from '@injected/storageScript';
 import type * as types from './types';
+import type { URLMatch } from '@isomorphic/urlMatch';
 import type * as channels from './channels';
 
 const BrowserContextEvent = {
@@ -120,6 +122,7 @@ export abstract class BrowserContext<EM extends EventMap = EventMap> extends Sdk
   private _playwrightBindingExposed?: Promise<void>;
   readonly dialogManager: DialogManager;
   private _consoleApiExposed = false;
+  private _harForAPIRequests: HarForAPIRequestsRegistration[] = [];
 
   constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) {
     super(browser, 'browser-context');
@@ -749,8 +752,37 @@ export abstract class BrowserContext<EM extends EventMap = EventMap> extends Sdk
   async notifyRoutesInFlightAboutRemovedHandler(handler: network.RouteHandler): Promise<void> {
     await Promise.all([...this._routesInFlight].map(route => route.removeHandler(handler)));
   }
+
+  routeAPIRequestsFromHar(options: { harBackend: HarBackend, urlMatch: URLMatch | undefined, notFound: 'abort' | 'fallback', baseURL: string | undefined }): { dispose: () => void } {
+    const registration: HarForAPIRequestsRegistration = {
+      harBackend: options.harBackend,
+      urlMatch: options.urlMatch,
+      notFound: options.notFound,
+      baseURL: options.baseURL,
+    };
+    // Give priority to the newest registration, mirroring BrowserContext.route/Page.route.
+    this._harForAPIRequests.unshift(registration);
+    return {
+      dispose: () => {
+        const index = this._harForAPIRequests.indexOf(registration);
+        if (index !== -1)
+          this._harForAPIRequests.splice(index, 1);
+      },
+    };
+  }
+
+  harForAPIRequests(): readonly HarForAPIRequestsRegistration[] {
+    return this._harForAPIRequests;
+  }
 }
 
+export type HarForAPIRequestsRegistration = {
+  harBackend: HarBackend;
+  urlMatch: URLMatch | undefined;
+  notFound: 'abort' | 'fallback';
+  baseURL: string | undefined;
+};
+
 export function validateBrowserContextOptions(options: types.BrowserContextOptions, browserOptions: BrowserOptions) {
   if (options.noDefaultViewport && options.deviceScaleFactor !== undefined)
     throw new Error(`"deviceScaleFactor" option is not supported with null "viewport"`);

packages/playwright-core/src/server/channels.d.ts

@@ -1285,6 +1285,8 @@ export interface BrowserContextChannel extends BrowserContextEventTarget, Channe
   setGeolocation(params: BrowserContextSetGeolocationParams, progress: Progress): Promise<BrowserContextSetGeolocationResult>;
   setHTTPCredentials(params: BrowserContextSetHTTPCredentialsParams, progress: Progress): Promise<BrowserContextSetHTTPCredentialsResult>;
   setNetworkInterceptionPatterns(params: BrowserContextSetNetworkInterceptionPatternsParams, progress: Progress): Promise<BrowserContextSetNetworkInterceptionPatternsResult>;
+  routeAPIRequestsFromHar(params: BrowserContextRouteAPIRequestsFromHarParams, progress: Progress): Promise<BrowserContextRouteAPIRequestsFromHarResult>;
+  unrouteAPIRequestsFromHar(params: BrowserContextUnrouteAPIRequestsFromHarParams, progress: Progress): Promise<BrowserContextUnrouteAPIRequestsFromHarResult>;
   setWebSocketInterceptionPatterns(params: BrowserContextSetWebSocketInterceptionPatternsParams, progress: Progress): Promise<BrowserContextSetWebSocketInterceptionPatternsResult>;
   setOffline(params: BrowserContextSetOfflineParams, progress: Progress): Promise<BrowserContextSetOfflineResult>;
   storageState(params: BrowserContextStorageStateParams, progress: Progress): Promise<BrowserContextStorageStateResult>;
@@ -1518,6 +1520,28 @@ export type BrowserContextSetNetworkInterceptionPatternsOptions = {
 
 };
 export type BrowserContextSetNetworkInterceptionPatternsResult = void;
+export type BrowserContextRouteAPIRequestsFromHarParams = {
+  harId: string,
+  urlGlob?: string,
+  urlRegexSource?: string,
+  urlRegexFlags?: string,
+  notFound: 'abort' | 'fallback',
+};
+export type BrowserContextRouteAPIRequestsFromHarOptions = {
+  urlGlob?: string,
+  urlRegexSource?: string,
+  urlRegexFlags?: string,
+};
+export type BrowserContextRouteAPIRequestsFromHarResult = {
+  registrationId: string,
+};
+export type BrowserContextUnrouteAPIRequestsFromHarParams = {
+  registrationId: string,
+};
+export type BrowserContextUnrouteAPIRequestsFromHarOptions = {
+
+};
+export type BrowserContextUnrouteAPIRequestsFromHarResult = void;
 export type BrowserContextSetWebSocketInterceptionPatternsParams = {
   patterns: {
     glob?: string,

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -46,10 +46,13 @@ import type { Request, Response, RouteHandler } from '../network';
 import type { InitScript, Page, PageError } from '../page';
 import type { Disposable } from '../disposable';
 import type { DispatcherScope } from './dispatcher';
+import type { LocalUtilsDispatcher } from './localUtilsDispatcher';
 import type * as channels from '../channels';
 import type { Progress } from '../progress';
 import type { URLMatch } from '@isomorphic/urlMatch';
 
+type HarForAPIRequestsDisposable = Disposable & { registrationId: string };
+
 export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channels.BrowserContextChannel, DispatcherScope> implements channels.BrowserContextChannel {
   _type_BrowserContext = true;
   private _context: BrowserContext;
@@ -335,6 +338,38 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
       this._routeWebSocketInitScript = await WebSocketRouteDispatcher.install(progress, this.connection, this._context);
   }
 
+  async routeAPIRequestsFromHar(params: channels.BrowserContextRouteAPIRequestsFromHarParams, progress: Progress): Promise<channels.BrowserContextRouteAPIRequestsFromHarResult> {
+    // Reuse the HarBackend that was already opened via localUtils.harOpen for the page-side
+    // route, rather than opening a second backend for the same HAR file. The backend is owned
+    // by LocalUtils and closed via harClose, so this registration must not dispose it.
+    const harBackend = this.connection.getDispatcher<LocalUtilsDispatcher>('LocalUtils')?.harBackendForId(params.harId);
+    if (!harBackend)
+      throw new Error('Internal error: har was not opened');
+    const urlMatch: URLMatch | undefined =
+      params.urlRegexSource !== undefined && params.urlRegexFlags !== undefined ? new RegExp(params.urlRegexSource, params.urlRegexFlags) :
+        params.urlGlob !== undefined ? params.urlGlob : undefined;
+    const registrationId = createGuid();
+    const registration = this._context.routeAPIRequestsFromHar({
+      harBackend,
+      urlMatch,
+      notFound: params.notFound,
+      baseURL: this._context._options.baseURL,
+    });
+    this._disposables.push({
+      registrationId,
+      dispose: async () => registration.dispose(),
+    } as HarForAPIRequestsDisposable);
+    return { registrationId };
+  }
+
+  async unrouteAPIRequestsFromHar(params: channels.BrowserContextUnrouteAPIRequestsFromHarParams, progress: Progress): Promise<void> {
+    const index = this._disposables.findIndex(d => (d as HarForAPIRequestsDisposable).registrationId === params.registrationId);
+    if (index === -1)
+      return;
+    const [disposable] = this._disposables.splice(index, 1);
+    await progress.race(disposable.dispose());
+  }
+
   async storageState(params: channels.BrowserContextStorageStateParams, progress: Progress): Promise<channels.BrowserContextStorageStateResult> {
     return await this._context.storageState(progress, params.indexedDB, params.credentials);
   }