diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.yml index 1beebe5249d9c..753bb7412eba5 100644 --- a/.github/workflows/tests_mcp.yml +++ b/.github/workflows/tests_mcp.yml @@ -23,7 +23,6 @@ env: # Force terminal colors. @see https://www.npmjs.com/package/colors FORCE_COLOR: 1 ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - DEBUG_GIT_COMMIT_INFO: 1 jobs: test_mcp: diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 2cb96d9ad6148..304870c65d179 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -25,7 +25,6 @@ env: # Force terminal colors. @see https://www.npmjs.com/package/colors FORCE_COLOR: 1 ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - DEBUG_GIT_COMMIT_INFO: 1 jobs: test_linux: @@ -177,7 +176,6 @@ jobs: runs-on: ubuntu-latest env: PWTEST_BOT_NAME: "vscode-extension" - DEBUG_GIT_COMMIT_INFO: "" steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 diff --git a/packages/playwright/src/mcp/test/streams.ts b/packages/playwright/src/mcp/test/streams.ts index f78eb82e3fe07..4513411016650 100644 --- a/packages/playwright/src/mcp/test/streams.ts +++ b/packages/playwright/src/mcp/test/streams.ts @@ -16,20 +16,24 @@ import { Writable } from 'stream'; +import { stripAnsiEscapes } from '../../util'; + import type { ProgressCallback } from '../sdk/server'; export class StringWriteStream extends Writable { private _progress: ProgressCallback; + private _prefix: string; - constructor(progress: ProgressCallback) { + constructor(progress: ProgressCallback, stdio: 'stdout' | 'stderr') { super(); this._progress = progress; + this._prefix = stdio === 'stdout' ? '' : '[err] '; } override _write(chunk: any, encoding: any, callback: any) { - const text = chunk.toString(); + const text = stripAnsiEscapes(chunk.toString()); // Progress wraps these as individual messages. - this._progress({ message: text.endsWith('\n') ? text.slice(0, -1) : text }); + this._progress({ message: `${this._prefix}${text.endsWith('\n') ? text.slice(0, -1) : text}` }); callback(); } } diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.ts index 60a2d16b8f250..d71a41ff48bc1 100644 --- a/packages/playwright/src/mcp/test/testContext.ts +++ b/packages/playwright/src/mcp/test/testContext.ts @@ -138,45 +138,93 @@ export class TestContext { } async runSeedTest(seedFile: string, projectName: string, progress: ProgressCallback) { - const { screen } = this.createScreen(progress); + await this.runWithGlobalSetup(async (testRunner, reporter) => { + const result = await testRunner.runTests(reporter, { + headed: !this.options?.headless, + locations: ['/' + escapeRegExp(seedFile) + '/'], + projects: [projectName], + timeout: 0, + workers: 1, + pauseAtEnd: true, + disableConfigReporters: true, + failOnLoadErrors: true, + }); + // Ideally, we should check that page was indeed created and browser mcp has kicked in. + // However, that is handled in the upper layer, so hard to check here. + if (result.status === 'passed' && !reporter.suite?.allTests().length) + throw new Error('seed test not found.'); + + if (result.status !== 'passed') + throw new Error('Errors while running the seed test.'); + }, progress); + } + + async runWithGlobalSetup( + callback: (testRunner: TestRunner, reporter: ListReporter) => Promise, + progress: ProgressCallback): Promise { + const { screen, claimStdio, releaseStdio } = createScreen(progress); const configDir = this.configLocation.configDir; - const reporter = new ListReporter({ configDir, screen }); const testRunner = await this.createTestRunner(); - const result = await testRunner.runTests(reporter, { - headed: !this.options?.headless, - locations: ['/' + escapeRegExp(seedFile) + '/'], - projects: [projectName], - timeout: 0, - workers: 1, - pauseAtEnd: true, - disableConfigReporters: true, - failOnLoadErrors: true, - }); + claimStdio(); + try { + const setupReporter = new ListReporter({ configDir, screen, includeTestId: true }); + const { status } = await testRunner.runGlobalSetup([setupReporter]); + if (status !== 'passed') + throw new Error('Failed to run global setup'); + } finally { + releaseStdio(); + } - // Ideally, we should check that page was indeed created and browser mcp has kicked in. - // However, that is handled in the upper layer, so hard to check here. - if (result.status === 'passed' && !reporter.suite?.allTests().length) - throw new Error('seed test not found.'); + try { + const reporter = new ListReporter({ configDir, screen, includeTestId: true }); + return await callback(testRunner, reporter); + } finally { + claimStdio(); + await testRunner.runGlobalTeardown().finally(() => { + releaseStdio(); + }); + } + } - if (result.status !== 'passed') - throw new Error('Errors while running the seed test.'); + async close() { } +} - createScreen(progress: ProgressCallback) { - const stream = new StringWriteStream(progress); - const screen = { - ...terminalScreen, - isTTY: false, - colors: noColors, - stdout: stream as unknown as NodeJS.WriteStream, - stderr: stream as unknown as NodeJS.WriteStream, +export function createScreen(progress: ProgressCallback) { + const stdout = new StringWriteStream(progress, 'stdout'); + const stderr = new StringWriteStream(progress, 'stderr'); + + const screen = { + ...terminalScreen, + isTTY: false, + colors: noColors, + stdout: stdout as unknown as NodeJS.WriteStream, + stderr: stderr as unknown as NodeJS.WriteStream, + }; + + /* eslint-disable no-restricted-properties */ + const originalStdoutWrite = process.stdout.write; + const originalStderrWrite = process.stderr.write; + + const claimStdio = () => { + process.stdout.write = (chunk: string | Buffer) => { + stdout.write(chunk); + return true; }; - return { screen, stream }; - } + process.stderr.write = (chunk: string | Buffer) => { + stderr.write(chunk); + return true; + }; + }; - async close() { - } + const releaseStdio = () => { + process.stdout.write = originalStdoutWrite; + process.stderr.write = originalStderrWrite; + }; + /* eslint-enable no-restricted-properties */ + + return { screen, claimStdio, releaseStdio }; } const bestPracticesMarkdown = ` diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.ts index a3c86ed080f98..7a4f0bda8dceb 100644 --- a/packages/playwright/src/mcp/test/testTools.ts +++ b/packages/playwright/src/mcp/test/testTools.ts @@ -15,9 +15,9 @@ */ import { z } from '../sdk/bundle'; -import ListReporter from '../../reporters/list'; import ListModeReporter from '../../reporters/listModeReporter'; import { defineTestTool } from './testTool'; +import { createScreen } from './testContext'; export const listTests = defineTestTool({ schema: { @@ -29,7 +29,7 @@ export const listTests = defineTestTool({ }, handle: async (context, _, progress) => { - const { screen } = context.createScreen(progress); + const { screen } = createScreen(progress); const reporter = new ListModeReporter({ screen, includeTestId: true }); const testRunner = await context.createTestRunner(); await testRunner.listTests(reporter, {}); @@ -51,15 +51,13 @@ export const runTests = defineTestTool({ }, handle: async (context, params, progress) => { - const { screen } = context.createScreen(progress); - const configDir = context.configLocation.configDir; - const reporter = new ListReporter({ configDir, screen, includeTestId: true, prefixStdio: 'out' }); - const testRunner = await context.createTestRunner(); - await testRunner.runTests(reporter, { - locations: params.locations, - projects: params.projects, - disableConfigReporters: true, - }); + await context.runWithGlobalSetup(async (testRunner, reporter) => { + await testRunner.runTests(reporter, { + locations: params.locations, + projects: params.projects, + disableConfigReporters: true, + }); + }, progress); return { content: [] }; }, @@ -80,19 +78,17 @@ export const debugTest = defineTestTool({ }, handle: async (context, params, progress) => { - const { screen } = context.createScreen(progress); - const configDir = context.configLocation.configDir; - const reporter = new ListReporter({ configDir, screen, includeTestId: true, prefixStdio: 'out' }); - const testRunner = await context.createTestRunner(); - await testRunner.runTests(reporter, { - headed: !context.options?.headless, - testIds: [params.test.id], - // For automatic recovery - timeout: 0, - workers: 1, - pauseOnError: true, - disableConfigReporters: true, - }); + await context.runWithGlobalSetup(async (testRunner, reporter) => { + await testRunner.runTests(reporter, { + headed: !context.options?.headless, + testIds: [params.test.id], + // For automatic recovery + timeout: 0, + workers: 1, + pauseOnError: true, + disableConfigReporters: true, + }); + }, progress); return { content: [] }; }, diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts index 591883e3bf33b..b2b399e6eb0d6 100644 --- a/packages/playwright/src/reporters/list.ts +++ b/packages/playwright/src/reporters/list.ts @@ -38,12 +38,10 @@ class ListReporter extends TerminalReporter { private _stepIndex = new Map(); private _needNewLine = false; private _printSteps: boolean; - private _prefixStdio?: string; - constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions & { prefixStdio?: string }) { + constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) { super(options); this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps); - this._prefixStdio = options?.prefixStdio; } override onBegin(suite: Suite) { @@ -164,10 +162,7 @@ class ListReporter extends TerminalReporter { return; const text = chunk.toString('utf-8'); this._updateLineCountAndNewLineFlagForOutput(text); - if (this._prefixStdio) - stream.write(`[${stdio}] ${chunk}`); - else - stream.write(chunk); + stream.write(chunk); } override onTestEnd(test: TestCase, result: TestResult) { diff --git a/tests/mcp/planner.spec.ts b/tests/mcp/planner.spec.ts index 64684a56c2742..1917660b52bfd 100644 --- a/tests/mcp/planner.spec.ts +++ b/tests/mcp/planner.spec.ts @@ -183,6 +183,7 @@ test('planner_setup_page (loading error)', async ({ startClient }) => { }); test('planner_setup_page (wrong test location)', async ({ startClient }) => { + await writeFiles({}); const { client } = await startClient(); expect(await client.callTool({ name: 'planner_setup_page', @@ -196,6 +197,7 @@ test('planner_setup_page (wrong test location)', async ({ startClient }) => { }); test('planner_setup_page (no test location)', async ({ startClient }) => { + await writeFiles({}); const { client } = await startClient(); expect(await client.callTool({ name: 'planner_setup_page', diff --git a/tests/mcp/seed-default-project.spec.ts b/tests/mcp/seed-default-project.spec.ts index c0acdef706d2e..beac69d682ec1 100644 --- a/tests/mcp/seed-default-project.spec.ts +++ b/tests/mcp/seed-default-project.spec.ts @@ -47,6 +47,23 @@ test('seed test runs in first top-level project by default', async ({ startClien expect(fs.existsSync(path.join(baseDir, 'third', 'seed.spec.ts'))).toBe(false); }); +test('respects provided seed test', async ({ startClient }) => { + await writeFiles({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => {}); + `, + }); + + const { client } = await startClient(); + expect(await client.callTool({ + name: 'planner_setup_page', + arguments: { + seedFile: 'a.test.ts', + }, + })).toHaveTextResponse(expect.stringContaining(`### Paused at end of test. ready for interaction`)); +}); + test('seed test runs in first top-level project with dependencies', async ({ startClient }) => { const baseDir = await writeFiles({ 'playwright.config.ts': ` diff --git a/tests/mcp/test-debug.spec.ts b/tests/mcp/test-debug.spec.ts index 0c3b031ef239a..b0f1a875816e0 100644 --- a/tests/mcp/test-debug.spec.ts +++ b/tests/mcp/test-debug.spec.ts @@ -323,7 +323,7 @@ test('test_debug w/ console.log in test', async ({ startClient }) => { }, })).toHaveTextResponse(expect.stringContaining(` Running 1 test using 1 worker -[out] console.log +console.log [err] console.error ### Paused on error: Error: expect(locator).toBeVisible() failed`)); diff --git a/tests/mcp/test-setup.spec.ts b/tests/mcp/test-setup.spec.ts new file mode 100644 index 0000000000000..455b1fcb38b82 --- /dev/null +++ b/tests/mcp/test-setup.spec.ts @@ -0,0 +1,131 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, writeFiles } from './fixtures'; + +test.use({ mcpServerType: 'test-mcp' }); + +const workspace = { + 'playwright.config.ts': ` + module.exports = { + projects: [{ name: 'chromium' }], + webServer: { + command: 'node web-server.js', + stdout: 'pipe', + stderr: 'pipe', + wait: { stderr: /started/ } + }, + globalSetup: 'global-setup.ts', + globalTeardown: 'global-teardown.ts', + }; + `, + 'web-server.js': ` + console.log('web server started'); + console.error('web server started'); + `, + 'global-setup.ts': ` + module.exports = async () => { + console.log('global setup'); + console.error('global setup'); + }; + `, + 'global-teardown.ts': ` + module.exports = async () => { + console.log('global teardown'); + console.error('global teardown'); + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + console.log('test'); + console.error('test'); + }); + ` +}; + +test('setup should run global setup and teardown', async ({ startClient }, { workerIndex }) => { + await writeFiles(workspace); + const { client } = await startClient(); + + // Call planner_setup_page without specifying a project - should use first top-level project + expect(await client.callTool({ + name: 'planner_setup_page', + arguments: { + seedFile: 'a.test.ts', + }, + })).toHaveTextResponse(expect.stringContaining(`[WebServer] web server started +[err] [WebServer] web server started +global setup +[err] global setup + +Running 1 test using 1 worker +test +[err] test +### Paused at end of test. ready for interaction`)); +}); + +test('test_run should run global setup and teardown', async ({ startClient }) => { + await writeFiles(workspace); + + const { client } = await startClient(); + expect(await client.callTool({ + name: 'test_run', + arguments: { + locations: ['a.test.ts'], + projects: ['chromium'], + }, + })).toHaveTextResponse(`[WebServer] web server started +[err] [WebServer] web server started +global setup +[err] global setup + +Running 1 test using 1 worker +test +[err] test + ok 1 [id=] [project=chromium] › a.test.ts:3:9 › test (XXms) + 1 passed (XXms) +global teardown +[err] global teardown`); +}); + +test('test_debug should run global setup and teardown', async ({ startClient }) => { + await writeFiles(workspace); + + const { client } = await startClient(); + const listResult = await client.callTool({ + name: 'test_list', + }); + const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/); + + expect(await client.callTool({ + name: 'test_debug', + arguments: { + test: { id, title: 'pass' }, + }, + })).toHaveTextResponse(`[WebServer] web server started +[err] [WebServer] web server started +global setup +[err] global setup + +Running 1 test using 1 worker +test +[err] test + ok 1 [id=] [project=chromium] › a.test.ts:3:9 › test (XXms) + 1 passed (XXms) +global teardown +[err] global teardown`); +});