From f2b99222170e70d3db370414f13c576ff19b7424 Mon Sep 17 00:00:00 2001 From: Khai Date: Thu, 10 Apr 2025 00:37:31 -0500 Subject: [PATCH 1/9] local homepage tests --- .../{production => local}/homepage.test.ts | 179 +++++++++--------- 1 file changed, 88 insertions(+), 91 deletions(-) rename apps/webapp/tests-playwright/{production => local}/homepage.test.ts (59%) diff --git a/apps/webapp/tests-playwright/production/homepage.test.ts b/apps/webapp/tests-playwright/local/homepage.test.ts similarity index 59% rename from apps/webapp/tests-playwright/production/homepage.test.ts rename to apps/webapp/tests-playwright/local/homepage.test.ts index 419de6321..79c5f9da2 100644 --- a/apps/webapp/tests-playwright/production/homepage.test.ts +++ b/apps/webapp/tests-playwright/local/homepage.test.ts @@ -1,12 +1,15 @@ import { expect, test } from '@playwright/test'; +const baseUrl = 'http://localhost:3000'; +const mainUrl = 'https://neuronpedia.org'; + test('homepage has expected title', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); await expect(page).toHaveTitle(/Neuronpedia/); }); test('API navigation link', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); // Find and click the API link const apiLink = page.getByRole('link', { name: 'API', exact: true }); @@ -15,11 +18,11 @@ test('API navigation link', async ({ page }) => { // Wait for navigation and check exact URL await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL('https://www.neuronpedia.org/api-doc'); + await expect(page).toHaveURL(baseUrl + '/api-doc'); }); test('SAE Evals link', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); // Find and click the SAE Evals link const saeEvalsLink = page.getByRole('link', { name: 'SAE Evals', exact: true }); @@ -28,18 +31,18 @@ test('SAE Evals link', async ({ page }) => { // Wait for navigation and check exact URL await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL('https://www.neuronpedia.org/sae-bench'); + await expect(page).toHaveURL(baseUrl + '/sae-bench'); }); test('Steer', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); // Find and click the Steer link const steerLink = page.getByRole('link', { name: 'Steer', exact: true }); await steerLink.waitFor({ state: 'visible' }); await steerLink.click(); - await expect(page).toHaveURL(/https:\/\/www\.neuronpedia\.org\/.*\/steer/); + await expect(page).toHaveURL(/\/steer$/); }); // Must be logged in to test this @@ -60,19 +63,20 @@ test('Steer', async ({ page }) => { // await expect(page).toHaveURL(/.*neuronpedia\.org\/user\/.*\/vectors/); // }) */ -test('Getting Started', async ({ page }) => { - await page.goto('https://neuronpedia.org'); +// no local docs currently +// test('Getting Started', async ({ page }) => { +// await page.goto(baseUrl); - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Getting Started', exact: true }).click(), - ]); +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Getting Started', exact: true }).click(), +// ]); - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/'); -}); +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/'); +// }); test('MIT Technology Review link', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -83,7 +87,7 @@ test('MIT Technology Review link', async ({ page }) => { }); test('Google Deepmind', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -94,7 +98,7 @@ test('Google Deepmind', async ({ page }) => { }); test('Fudan University', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -105,7 +109,7 @@ test('Fudan University', async ({ page }) => { }); test('Apollo Research', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -128,7 +132,7 @@ test('Apollo Research', async ({ page }) => { // }); test('EleutherAI', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -138,121 +142,124 @@ test('EleutherAI', async ({ page }) => { await expect(newPage).toHaveURL(/.*llama3\.1-8b-eleuther_gp/); }); -test('latents/features', async ({ page }) => { - await page.goto('https://neuronpedia.org'); +// no local docs +// test('latents/features', async ({ page }) => { +// await page.goto(baseUrl); - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'latents/features', exact: true }).click(), - ]); +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'latents/features', exact: true }).click(), +// ]); - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/features'); -}); +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/features'); +// }); test('concepts', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), page.getByRole('link', { name: 'concepts', exact: true }).click(), ]); - await expect(newPage).toHaveURL('https://www.neuronpedia.org/axbench'); + await expect(newPage).toHaveURL(baseUrl + '/axbench'); }); test('releases display', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); await expect(page.getByText('Llama Scope R1: SAEs for DeepSeek-R1-Distill-Llama-8B')).toBeVisible(); }); test('models display', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); await expect(page.getByText('DeepSeek-R1-Llama-8B')).toBeVisible(); }); test('jump to display', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); await expect(page.getByText('jump to source/sae')).toBeVisible(); }); test('cat steering', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), page.getByRole('link', { name: 'Try It: Gemma 2 - Cat Steering', exact: true }).click(), ]); - await expect(newPage).toHaveURL('https://www.neuronpedia.org/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5'); + await expect(newPage).toHaveURL(baseUrl + '/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5'); }); test('search by explanation', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), page.getByRole('link', { name: 'Try It: Search by Explanation', exact: true }).click(), ]); - await expect(newPage).toHaveURL('https://www.neuronpedia.org/search-explanations'); + await expect(newPage).toHaveURL( baseUrl + '/search-explanations'); }); -test('search via interference', async ({ page }) => { - await page.goto('https://neuronpedia.org'); +// no local docs +// test('search via interference', async ({ page }) => { +// await page.goto(baseUrl); - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Docs: Search via Inference', exact: true }).click(), - ]); +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Docs: Search via Inference', exact: true }).click(), +// ]); - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/search'); -}); +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/search'); +// }); test('api + libraries', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), page.getByRole('link', { name: 'API Playground', exact: true }).click(), ]); - await expect(newPage).toHaveURL('https://www.neuronpedia.org/api-doc'); + await expect(newPage).toHaveURL( baseUrl + '/api-doc'); }); test('searcher is embedded in the page', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); await page.locator('textarea[name="searchQuery"]').isVisible(); }); test('searcher example', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); await page.locator('textarea[name="Test activation with custom text./"]').isVisible(); }); -test('Docs: Lists', async ({ page }) => { - await page.goto('https://neuronpedia.org'); +// no local docs +// test('Docs: Lists', async ({ page }) => { +// await page.goto(baseUrl); - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Docs: Lists', exact: true }).click(), - ]); +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Docs: Lists', exact: true }).click(), +// ]); - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/lists'); -}); +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/lists'); +// }); -test('Docs: Embed', async ({ page }) => { - await page.goto('https://neuronpedia.org'); +// test('Docs: Embed', async ({ page }) => { +// await page.goto(baseUrl); - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Docs: Embed', exact: true }).click(), - ]); +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Docs: Embed', exact: true }).click(), +// ]); - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/embed-iframe'); -}); +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/embed-iframe'); +// }); test('Slack', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -263,7 +270,7 @@ test('Slack', async ({ page }) => { }); test('Donate', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -273,19 +280,8 @@ test('Donate', async ({ page }) => { await expect(newPage).toHaveURL(/.*every\.org*/); }); -// test('Feedback', async({ page }) => { -// await page.goto('https://neuronpedia.org'); - -// const [newPage] = await Promise.all([ -// page.waitForEvent('popup'), -// page.getByRole('button', { name: 'Feedback' }).click() -// ]); - -// await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); -// }); - test('Upskill', async ({ page }) => { - await page.goto('https://neuronpedia.org'); + await page.goto(baseUrl); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -295,17 +291,18 @@ test('Upskill', async ({ page }) => { await expect(newPage).toHaveURL('https://www.arena.education/'); }); -test('all services are online', async ({ page }) => { - await page.goto('https://neuronpedia.org'); +// online service +// test('all services are online', async ({ page }) => { +// await page.goto(baseUrl); - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page - .locator('iframe[title="Neuronpedia Status"]') - .contentFrame() - .getByRole('link', { name: 'All services are online' }) - .click(), - ]); +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page +// .locator('iframe[title="Neuronpedia Status"]') +// .contentFrame() +// .getByRole('link', { name: 'All services are online' }) +// .click(), +// ]); - await expect(newPage).toHaveURL('https://status.neuronpedia.org/'); -}); +// await expect(newPage).toHaveURL('https://status.neuronpedia.org/'); +// }); From e9b2b14d9d33dad43591c1331a4cc38c40f20b86 Mon Sep 17 00:00:00 2001 From: Khai Date: Thu, 10 Apr 2025 22:01:08 -0500 Subject: [PATCH 2/9] added fixtures --- apps/webapp/.eslintrc | 2 +- apps/webapp/package-lock.json | 102 ++-------- apps/webapp/package.json | 6 +- apps/webapp/tests-playwright/fixtures.ts | 13 ++ .../tests-playwright/local/homepage.test.ts | 60 +++--- .../production/gemmascope.test.ts | 185 +++++++++--------- .../production/search.test.ts | 4 +- .../tests-playwright/production/steer.test.ts | 111 ++++++----- apps/webapp/tsconfig.json | 3 +- package-lock.json | 73 ++++++- package.json | 5 + 11 files changed, 297 insertions(+), 267 deletions(-) create mode 100644 apps/webapp/tests-playwright/fixtures.ts create mode 100644 package.json diff --git a/apps/webapp/.eslintrc b/apps/webapp/.eslintrc index 9731fbdd6..96ee8ee86 100644 --- a/apps/webapp/.eslintrc +++ b/apps/webapp/.eslintrc @@ -18,7 +18,7 @@ "no-await-in-loop": "warn", "import/no-extraneous-dependencies": [ "error", - { "devDependencies": ["**/*.test.tsx", "**/*.spec.tsx", "**/*.test.ts", "**/*.spec.ts", "**/*.config.ts"] } + { "devDependencies": ["**/*.test.tsx", "**/*.spec.tsx", "**/*.test.ts", "**/*.spec.ts", "**/*.config.ts","**/fixtures.ts"] } ], "jsx-a11y/label-has-associated-control": [ "error", diff --git a/apps/webapp/package-lock.json b/apps/webapp/package-lock.json index 6414a0b4f..ed70bc5ef 100644 --- a/apps/webapp/package-lock.json +++ b/apps/webapp/package-lock.json @@ -120,16 +120,14 @@ "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", - "@playwright/test": "^1.50.0", - "@tailwindcss/forms": "^0.5.3", - "@tailwindcss/typography": "^0.5.9", + "@playwright/test": "^1.42.1", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/diff": "^5.2.1", "@types/lodash": "^4.6.9", "@types/mjml": "^4.7.4", "@types/ms": "^0.7.31", - "@types/node": "^22.13.1", + "@types/node": "^22.14.0", "@types/node-gzip": "^1.1.3", "@types/nodemailer": "^6.4.9", "@types/pg": "^8.11.11", @@ -5614,13 +5612,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.50.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.50.0.tgz", - "integrity": "sha512-ZGNXbt+d65EGjBORQHuYKj+XhCewlwpnSd/EDuLPZGSiEWmgOJB5RmMCCYGy5aMfTs9wx61RivfDKi8H/hcMvw==", + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.1.tgz", + "integrity": "sha512-nM+kEaTSAoVlXmMPH10017vn3FSiFqr/bh4fKg9vmAdMfd9SDqRZNvPSiAHADc/itWak+qPvMPZQOPwCBW7k7Q==", "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.50.0" + "playwright": "1.51.1" }, "bin": { "playwright": "cli.js" @@ -8578,35 +8576,6 @@ "tailwindcss": ">=3.2.0" } }, - "node_modules/@tailwindcss/forms": { - "version": "0.5.10", - "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.10.tgz", - "integrity": "sha512-utI1ONF6uf/pPNO68kmN1b8rEwNXv3czukalo8VtJH8ksIkZXr3Q3VYudZLkCsDd4Wku120uF02hYK25XGPorw==", - "dev": true, - "license": "MIT", - "dependencies": { - "mini-svg-data-uri": "^1.2.3" - }, - "peerDependencies": { - "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1 || >= 4.0.0-alpha.20 || >= 4.0.0-beta.1" - } - }, - "node_modules/@tailwindcss/typography": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.16.tgz", - "integrity": "sha512-0wDLwCVF5V3x3b1SGXPCDcdsbDHMBe+lkFzBRaHeLvNi+nrrnZ1lA18u+OTWO8iSWU2GxUOCvlXtDuqftc1oiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash.castarray": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.merge": "^4.6.2", - "postcss-selector-parser": "6.0.10" - }, - "peerDependencies": { - "tailwindcss": ">=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1" - } - }, "node_modules/@tanstack/query-core": { "version": "5.64.2", "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.64.2.tgz", @@ -9107,12 +9076,12 @@ } }, "node_modules/@types/node": { - "version": "22.13.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz", - "integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==", + "version": "22.14.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.0.tgz", + "integrity": "sha512-Kmpl+z84ILoG+3T/zQFyAJsU6EPTmOCj8/2+83fSN6djd6I4o7uOuGIH6vq3PrjY5BGitSbFuMN18j3iknubbA==", "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~6.21.0" } }, "node_modules/@types/node-fetch": { @@ -17197,13 +17166,6 @@ "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", "license": "MIT" }, - "node_modules/lodash.castarray": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz", - "integrity": "sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -18725,16 +18687,6 @@ "node": ">= 0.6" } }, - "node_modules/mini-svg-data-uri": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", - "integrity": "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==", - "dev": true, - "license": "MIT", - "bin": { - "mini-svg-data-uri": "cli.js" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -20596,13 +20548,13 @@ } }, "node_modules/playwright": { - "version": "1.50.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.50.0.tgz", - "integrity": "sha512-+GinGfGTrd2IfX1TA4N2gNmeIksSb+IAe589ZH+FlmpV3MYTx6+buChGIuDLQwrGNCw2lWibqV50fU510N7S+w==", + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz", + "integrity": "sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw==", "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.50.0" + "playwright-core": "1.51.1" }, "bin": { "playwright": "cli.js" @@ -20615,9 +20567,9 @@ } }, "node_modules/playwright-core": { - "version": "1.50.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.0.tgz", - "integrity": "sha512-CXkSSlr4JaZs2tZHI40DsZUN/NIwgaUPsyLuOAaIZp2CyF2sN5MM5NJsyB188lFSSozFxQ5fPT4qM+f0tH/6wQ==", + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz", + "integrity": "sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==", "devOptional": true, "license": "Apache-2.0", "bin": { @@ -21403,20 +21355,6 @@ "postcss": "^8.4.31" } }, - "node_modules/postcss-selector-parser": { - "version": "6.0.10", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", - "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", - "dev": true, - "license": "MIT", - "dependencies": { - "cssesc": "^3.0.0", - "util-deprecate": "^1.0.2" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/postcss-svgo": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-7.0.1.tgz", @@ -25082,9 +25020,9 @@ } }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "license": "MIT" }, "node_modules/unified": { diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 5722480ff..be61a432b 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -146,19 +146,17 @@ "zod-prisma-types": "^2.7.4" }, "devDependencies": { + "@playwright/test": "^1.42.1", "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", - "@playwright/test": "^1.50.0", - "@tailwindcss/forms": "^0.5.3", - "@tailwindcss/typography": "^0.5.9", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/diff": "^5.2.1", "@types/lodash": "^4.6.9", "@types/mjml": "^4.7.4", "@types/ms": "^0.7.31", - "@types/node": "^22.13.1", + "@types/node": "^22.14.0", "@types/node-gzip": "^1.1.3", "@types/nodemailer": "^6.4.9", "@types/pg": "^8.11.11", diff --git a/apps/webapp/tests-playwright/fixtures.ts b/apps/webapp/tests-playwright/fixtures.ts new file mode 100644 index 000000000..d9f3a538a --- /dev/null +++ b/apps/webapp/tests-playwright/fixtures.ts @@ -0,0 +1,13 @@ +import { test as base } from '@playwright/test'; + +type MyFixtures = { + baseUrl: string; + gemmaUrl: string; +}; + +export const test = base.extend({ + baseUrl: ['http://localhost:3000', { option: true }], + gemmaUrl: async ({ baseUrl }, use) => { + await use(`${baseUrl}/gemma-scope`); + }, +}); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/local/homepage.test.ts b/apps/webapp/tests-playwright/local/homepage.test.ts index 79c5f9da2..572f31d80 100644 --- a/apps/webapp/tests-playwright/local/homepage.test.ts +++ b/apps/webapp/tests-playwright/local/homepage.test.ts @@ -1,14 +1,12 @@ -import { expect, test } from '@playwright/test'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; -const baseUrl = 'http://localhost:3000'; -const mainUrl = 'https://neuronpedia.org'; - -test('homepage has expected title', async ({ page }) => { +test('homepage has expected title', async ({ page, baseUrl }) => { await page.goto(baseUrl); await expect(page).toHaveTitle(/Neuronpedia/); }); -test('API navigation link', async ({ page }) => { +test('API navigation link', async ({ page, baseUrl }) => { await page.goto(baseUrl); // Find and click the API link @@ -18,10 +16,10 @@ test('API navigation link', async ({ page }) => { // Wait for navigation and check exact URL await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL(baseUrl + '/api-doc'); + await expect(page).toHaveURL(`${baseUrl}/api-doc`); }); -test('SAE Evals link', async ({ page }) => { +test('SAE Evals link', async ({ page, baseUrl }) => { await page.goto(baseUrl); // Find and click the SAE Evals link @@ -31,10 +29,10 @@ test('SAE Evals link', async ({ page }) => { // Wait for navigation and check exact URL await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL(baseUrl + '/sae-bench'); + await expect(page).toHaveURL(`${baseUrl}/sae-bench`); }); -test('Steer', async ({ page }) => { +test('Steer', async ({ page, baseUrl }) => { await page.goto(baseUrl); // Find and click the Steer link @@ -75,7 +73,7 @@ test('Steer', async ({ page }) => { // await expect(newPage).toHaveURL('https://docs.neuronpedia.org/'); // }); -test('MIT Technology Review link', async ({ page }) => { +test('MIT Technology Review link', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -86,7 +84,7 @@ test('MIT Technology Review link', async ({ page }) => { await expect(newPage).toHaveURL(/.*technologyreview\.com/); }); -test('Google Deepmind', async ({ page }) => { +test('Google Deepmind', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -97,7 +95,7 @@ test('Google Deepmind', async ({ page }) => { await expect(newPage).toHaveURL(/.*gemma-scope/); }); -test('Fudan University', async ({ page }) => { +test('Fudan University', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -108,7 +106,7 @@ test('Fudan University', async ({ page }) => { await expect(newPage).toHaveURL(/.*llama-scope/); }); -test('Apollo Research', async ({ page }) => { +test('Apollo Research', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -131,7 +129,7 @@ test('Apollo Research', async ({ page }) => { // await expect(newPage).toHaveURL('#mats'); // }); -test('EleutherAI', async ({ page }) => { +test('EleutherAI', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -154,7 +152,7 @@ test('EleutherAI', async ({ page }) => { // await expect(newPage).toHaveURL('https://docs.neuronpedia.org/features'); // }); -test('concepts', async ({ page }) => { +test('concepts', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -162,25 +160,25 @@ test('concepts', async ({ page }) => { page.getByRole('link', { name: 'concepts', exact: true }).click(), ]); - await expect(newPage).toHaveURL(baseUrl + '/axbench'); + await expect(newPage).toHaveURL(`${baseUrl}/axbench`); }); -test('releases display', async ({ page }) => { +test('releases display', async ({ page, baseUrl }) => { await page.goto(baseUrl); await expect(page.getByText('Llama Scope R1: SAEs for DeepSeek-R1-Distill-Llama-8B')).toBeVisible(); }); -test('models display', async ({ page }) => { +test('models display', async ({ page, baseUrl }) => { await page.goto(baseUrl); await expect(page.getByText('DeepSeek-R1-Llama-8B')).toBeVisible(); }); -test('jump to display', async ({ page }) => { +test('jump to display', async ({ page, baseUrl }) => { await page.goto(baseUrl); await expect(page.getByText('jump to source/sae')).toBeVisible(); }); -test('cat steering', async ({ page }) => { +test('cat steering', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -188,10 +186,10 @@ test('cat steering', async ({ page }) => { page.getByRole('link', { name: 'Try It: Gemma 2 - Cat Steering', exact: true }).click(), ]); - await expect(newPage).toHaveURL(baseUrl + '/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5'); + await expect(newPage).toHaveURL(`${baseUrl}/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5`); }); -test('search by explanation', async ({ page }) => { +test('search by explanation', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -199,7 +197,7 @@ test('search by explanation', async ({ page }) => { page.getByRole('link', { name: 'Try It: Search by Explanation', exact: true }).click(), ]); - await expect(newPage).toHaveURL( baseUrl + '/search-explanations'); + await expect(newPage).toHaveURL( `${baseUrl}/search-explanations`); }); // no local docs @@ -214,7 +212,7 @@ test('search by explanation', async ({ page }) => { // await expect(newPage).toHaveURL('https://docs.neuronpedia.org/search'); // }); -test('api + libraries', async ({ page }) => { +test('api + libraries', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -222,15 +220,15 @@ test('api + libraries', async ({ page }) => { page.getByRole('link', { name: 'API Playground', exact: true }).click(), ]); - await expect(newPage).toHaveURL( baseUrl + '/api-doc'); + await expect(newPage).toHaveURL(`${baseUrl}/api-doc`); }); -test('searcher is embedded in the page', async ({ page }) => { +test('searcher is embedded in the page', async ({ page, baseUrl }) => { await page.goto(baseUrl); await page.locator('textarea[name="searchQuery"]').isVisible(); }); -test('searcher example', async ({ page }) => { +test('searcher example', async ({ page, baseUrl }) => { await page.goto(baseUrl); await page.locator('textarea[name="Test activation with custom text./"]').isVisible(); }); @@ -258,7 +256,7 @@ test('searcher example', async ({ page }) => { // await expect(newPage).toHaveURL('https://docs.neuronpedia.org/embed-iframe'); // }); -test('Slack', async ({ page }) => { +test('Slack', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -269,7 +267,7 @@ test('Slack', async ({ page }) => { await expect(newPage).toHaveURL(/.*slack\.com*/); }); -test('Donate', async ({ page }) => { +test('Donate', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ @@ -280,7 +278,7 @@ test('Donate', async ({ page }) => { await expect(newPage).toHaveURL(/.*every\.org*/); }); -test('Upskill', async ({ page }) => { +test('Upskill', async ({ page, baseUrl }) => { await page.goto(baseUrl); const [newPage] = await Promise.all([ diff --git a/apps/webapp/tests-playwright/production/gemmascope.test.ts b/apps/webapp/tests-playwright/production/gemmascope.test.ts index 175223563..234aeda38 100644 --- a/apps/webapp/tests-playwright/production/gemmascope.test.ts +++ b/apps/webapp/tests-playwright/production/gemmascope.test.ts @@ -1,95 +1,98 @@ -import { expect, test } from '@playwright/test'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; -test('main page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); +// http://localhost:3000/gemma-scope#main + +test('main page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); await page.locator('textarea[name="The inner workings"]').isVisible(); }); -test('microscope button', async ({ page }) => { +test('microscope button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#main'); await page.getByText("scan Gemma 2's brain to see what it's thinking").click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#microscope'); }); -test('analyze features button', async ({ page }) => { +test('analyze features button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#main'); await page.getByText('Make features fire and figure out what they do').click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#analyze'); }); -test('steer gemma button', async ({ page }) => { +test('steer gemma button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#main'); await page.getByText("Change Gemma's behavior by manipulating features").click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#steer'); }); -test('do more button', async ({ page }) => { +test('do more button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#main'); await page.getByText('Dive deeper into the exciting world of AI interpretability').click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); }); -test('browse & search saes button', async ({ page }) => { +test('browse & search saes button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#main'); await page.getByText('Directly explore the SAEs in Gemma Scope').click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#browse'); }); -test('microscope page', async ({ page }) => { +test('microscope page', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.locator('textarea[name="To understand what AI is thinking"]').isVisible(); }); -test('demo button feeling lucky', async ({ page }) => { +test('demo button feeling lucky', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.getByText("I'm Feeling Lucky").click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button food', async ({ page }) => { +test('demo button food', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.getByText('Food').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button News', async ({ page }) => { +test('demo button News', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.getByText('News').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button Literary', async ({ page }) => { +test('demo button Literary', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.getByText('Literary').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button Personal', async ({ page }) => { +test('demo button Personal', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.getByText('Personal').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button Programming', async ({ page }) => { +test('demo button Programming', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); await page.getByText('Programming').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('challenge gemma', async ({ page }) => { +test('challenge gemma', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); // challenge is unlocked after doing the demo, or pressing skip demo @@ -101,7 +104,7 @@ test('challenge gemma', async ({ page }) => { await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('next analyze features button', async ({ page }) => { +test('next analyze features button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#microscope'); // next is unlocked after doing the demo, or pressing skip demo @@ -110,37 +113,37 @@ test('next analyze features button', async ({ page }) => { await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#analyze'); }); -test('analyze features page', async ({ page }) => { +test('analyze features page', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); await page.locator('textarea[name="The features in Gemma Scope have labels"]').isVisible(); }); -test('olympic sports button', async ({ page }) => { +test('olympic sports button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); await page.getByText('Olympic sports').click(); await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); }); -test('famous cities button', async ({ page }) => { +test('famous cities button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); await page.getByText('famous cities').click(); await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); }); -test('reference to animals button', async ({ page }) => { +test('reference to animals button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); await page.getByText('references to animals').click(); await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); }); -test('none of these button', async ({ page }) => { +test('none of these button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); await page.getByText('none of these').click(); await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); }); -test('analyze steer', async ({ page }) => { +test('analyze steer', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); // unlocked after doing the demo, or pressing skip demo @@ -152,7 +155,7 @@ test('analyze steer', async ({ page }) => { await expect(page.getByText('0.00')).toBeVisible(); }); -test('puzzles labels', async ({ page }) => { +test('puzzles labels', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); // unlocked after doing analyze steer, or pressing skip analyze @@ -171,7 +174,7 @@ test('puzzles labels', async ({ page }) => { await expect(page.getByText('Bad/cringe stories')).toBeVisible(); }); -test('next steer gemma button', async ({ page }) => { +test('next steer gemma button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#analyze'); // unlocked after puzzles, or pressing skip analyze @@ -180,33 +183,33 @@ test('next steer gemma button', async ({ page }) => { await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#steer'); }); -test('steer page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#steer'); +// test('steer page', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-scope#steer'); - await page.locator('textarea[name="Let\'s put these features to use"]').isVisible(); -}); +// await page.locator('textarea[name="Let\'s put these features to use"]').isVisible(); +// }); -test('steer demo', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#steer'); +// test('steer demo', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-scope#steer'); - // check for demo question - await expect(page.getByText('What are you?').first()).toBeVisible(); -}); +// // check for demo question +// await expect(page.getByText('What are you?').first()).toBeVisible(); +// }); -test('next do more button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#steer'); +// test('next do more button', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-scope#steer'); - await page.getByText('Next - Do More').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); -}); +// await page.getByText('Next - Do More').click(); +// await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); +// }); -test('do more page', async ({ page }) => { +test('do more page', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); await page.locator('textarea[name="This demo of Gemma Scope"]').isVisible(); }); -test('advanced steer button', async ({ page }) => { +test('advanced steer button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([ @@ -216,7 +219,7 @@ test('advanced steer button', async ({ page }) => { await expect(newPage).toHaveURL(/https:\/\/www\.neuronpedia\.org\/.*\/steer/); }); -test('browse saes button', async ({ page }) => { +test('browse saes button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); await page.getByText('Browse SAEs').click(); @@ -234,7 +237,7 @@ test('browse saes button', async ({ page }) => { // await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); // }); -test('deepmind blog post button', async ({ page }) => { +test('deepmind blog post button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('DeepMind Blog Post').click()]); @@ -244,7 +247,7 @@ test('deepmind blog post button', async ({ page }) => { ); }); -test('coding tutorial button', async ({ page }) => { +test('coding tutorial button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Coding Tutorial').click()]); @@ -256,7 +259,7 @@ test('coding tutorial button', async ({ page }) => { // testing it the default method with expect().toHaveURL doesnt work, url ends up being blank // only tests for the correct link being on the page -test('technical report button', async ({ page }) => { +test('technical report button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const href = await page.getByText('Technical Report').getAttribute('href'); @@ -264,7 +267,7 @@ test('technical report button', async ({ page }) => { expect(href).toContain('storage.googleapis.com/gemma-scope/gemma-scope-report.pdf'); }); -test('huggingface button', async ({ page }) => { +test('huggingface button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('HuggingFace').click()]); @@ -272,7 +275,7 @@ test('huggingface button', async ({ page }) => { await expect(newPage).toHaveURL('https://huggingface.co/google/gemma-scope'); }); -test('get started with mech interp button', async ({ page }) => { +test('get started with mech interp button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([ @@ -283,7 +286,7 @@ test('get started with mech interp button', async ({ page }) => { await expect(newPage).toHaveURL('https://www.neelnanda.io/mechanistic-interpretability/getting-started'); }); -test('favourite mech interp papers button', async ({ page }) => { +test('favourite mech interp papers button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([ @@ -296,7 +299,7 @@ test('favourite mech interp papers button', async ({ page }) => { ); }); -test('toward monosemanticity button', async ({ page }) => { +test('toward monosemanticity button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([ @@ -307,7 +310,7 @@ test('toward monosemanticity button', async ({ page }) => { await expect(newPage).toHaveURL('https://transformer-circuits.pub/2023/monosemantic-features'); }); -test('saelens button', async ({ page }) => { +test('saelens button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('SAELens').click()]); @@ -315,7 +318,7 @@ test('saelens button', async ({ page }) => { await expect(newPage).toHaveURL('https://github.com/jbloomAus/SAELens'); }); -test('transformerlens button', async ({ page }) => { +test('transformerlens button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('TransformerLens').click()]); @@ -323,7 +326,7 @@ test('transformerlens button', async ({ page }) => { await expect(newPage).toHaveURL('https://github.com/TransformerLensOrg/TransformerLens'); }); -test('nnsight button', async ({ page }) => { +test('nnsight button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('NNsight').click()]); @@ -331,7 +334,7 @@ test('nnsight button', async ({ page }) => { await expect(newPage).toHaveURL('https://github.com/ndif-team/nnsight'); }); -test('mats program button', async ({ page }) => { +test('mats program button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('MATS Program').click()]); @@ -339,7 +342,7 @@ test('mats program button', async ({ page }) => { await expect(newPage).toHaveURL('https://www.matsprogram.org/'); }); -test('arena education button', async ({ page }) => { +test('arena education button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('ARENA Education').click()]); @@ -347,7 +350,7 @@ test('arena education button', async ({ page }) => { await expect(newPage).toHaveURL('https://www.arena.education/'); }); -test('twitter button', async ({ page }) => { +test('twitter button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Twitter').first().click()]); @@ -367,27 +370,27 @@ test('twitter button', async ({ page }) => { // await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); // }); -test('open problems button', async ({ page }) => { +test('open problems button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); await page.getByRole('button', { name: 'Open Problems' }).click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#openproblems'); }); -test('playground button', async ({ page }) => { +test('playground button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#learn'); await page.getByRole('button', { name: 'Playground' }).click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#playground'); }); -test('open problems page', async ({ page }) => { +test('open problems page', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); await page.locator('textarea[name="Interpretability has many unsolved problems"]').isVisible(); }); -test('slack link', async ({ page }) => { +test('slack link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Slack').click()]); @@ -408,7 +411,7 @@ test('slack link', async ({ page }) => { // await expect(newPage).toHaveURL('https://www.lesswrong.com'); // }); -test('wattenberg link', async ({ page }) => { +test('wattenberg link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Wattenberg et al').click()]); @@ -416,7 +419,7 @@ test('wattenberg link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.14662'); }); -test('Ziegler link', async ({ page }) => { +test('Ziegler link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Ziegler et al').click()]); @@ -424,7 +427,7 @@ test('Ziegler link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2205.01663'); }); -test('Steering Vectors link', async ({ page }) => { +test('Steering Vectors link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('steering vectors').click()]); @@ -432,7 +435,7 @@ test('Steering Vectors link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2308.10248'); }); -test('SAE feature steering link', async ({ page }) => { +test('SAE feature steering link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('SAE feature steering').click()]); @@ -442,7 +445,7 @@ test('SAE feature steering link', async ({ page }) => { ); }); -test('clamping link', async ({ page }) => { +test('clamping link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('clamping').click()]); @@ -450,7 +453,7 @@ test('clamping link', async ({ page }) => { await expect(newPage).toHaveURL('https://transformer-circuits.pub/2024/scaling-monosemanticity/index.html'); }); -test('huang et al link', async ({ page }) => { +test('huang et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Huang et al').click()]); @@ -458,7 +461,7 @@ test('huang et al link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2309.10312'); }); -test('bills et al link', async ({ page }) => { +test('bills et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Bills et al').click()]); @@ -467,7 +470,7 @@ test('bills et al link', async ({ page }) => { }); // same link appears twice -test('engels et al link', async ({ page }) => { +test('engels et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [firstPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Engels et al').first().click()]); @@ -482,7 +485,7 @@ test('engels et al link', async ({ page }) => { await expect(secondNewPage).toHaveURL('https://arxiv.org/abs/2405.14860'); }); -test('toy models link', async ({ page }) => { +test('toy models link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([ @@ -495,7 +498,7 @@ test('toy models link', async ({ page }) => { ); }); -test('stolfo et al link', async ({ page }) => { +test('stolfo et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Stolfo et al').click()]); @@ -503,7 +506,7 @@ test('stolfo et al link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2305.15054'); }); -test('marks et al link', async ({ page }) => { +test('marks et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Marks et al').click()]); @@ -511,7 +514,7 @@ test('marks et al link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2403.19647'); }); -test('mlp transcoders link', async ({ page }) => { +test('mlp transcoders link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('MLP transcoders').click()]); @@ -519,7 +522,7 @@ test('mlp transcoders link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2406.11944v1'); }); -test('jain et al link', async ({ page }) => { +test('jain et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Jain et al').click()]); @@ -527,7 +530,7 @@ test('jain et al link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.10264'); }); -test('hendel et al link', async ({ page }) => { +test('hendel et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Hendel et al').click()]); @@ -535,7 +538,7 @@ test('hendel et al link', async ({ page }) => { await expect(newPage).toHaveURL('https://arxiv.org/abs/2310.15916'); }); -test('Todd et al link', async ({ page }) => { +test('Todd et al link', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Todd et al').click()]); @@ -543,27 +546,27 @@ test('Todd et al link', async ({ page }) => { await expect(newPage).toHaveURL('https://functions.baulab.info/'); }); -test('do more button open problems', async ({ page }) => { +test('do more button open problems', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); await page.getByRole('button', { name: 'Do More' }).click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); }); -test('home button open problems', async ({ page }) => { +test('home button open problems', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); await page.getByRole('button', { name: 'Home' }).click(); await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#main'); }); -test('playground page', async ({ page }) => { +test('playground page', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#playground'); await page.locator('textarea[name="Enter something below to see"]').isVisible(); }); -test('im feeling lucky button', async ({ page }) => { +test('im feeling lucky button', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#playground'); await page.getByRole('button', { name: "I'm Feeling Lucky" }).click(); @@ -572,7 +575,7 @@ test('im feeling lucky button', async ({ page }) => { ).not.toBeVisible(); }); -test('model selector', async ({ page }) => { +test('model selector', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#playground'); await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); @@ -583,7 +586,7 @@ test('model selector', async ({ page }) => { } }); -test('source set selector', async ({ page }) => { +test('source set selector', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#playground'); await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); @@ -594,7 +597,7 @@ test('source set selector', async ({ page }) => { } }); -test('playground steering', async ({ page }) => { +test('playground steering', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#playground'); const gemmaSearch = page.getByPlaceholder('Make gemma-2-2b think about'); @@ -603,13 +606,13 @@ test('playground steering', async ({ page }) => { await expect(page.getByRole('button', { name: 'testing' })).toBeVisible(); }); -test('browse & search page', async ({ page }) => { +test('browse & search page', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); await page.locator('textarea[name="Search 50,000,000+ features"]').isVisible(); }); -test('jump to appears', async ({ page }) => { +test('jump to appears', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // Look for the h3 element with the specific classes and text @@ -621,7 +624,7 @@ test('jump to appears', async ({ page }) => { await expect(jumpToHeading).toHaveText('Jump To'); }); -test('jump to sae/source models', async ({ page }) => { +test('jump to sae/source models', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // 3 occurences of this combobox on this page @@ -633,7 +636,7 @@ test('jump to sae/source models', async ({ page }) => { } }); -test('jump to sae/source source/sae', async ({ page }) => { +test('jump to sae/source source/sae', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // 2 occurences of this combobox on this page @@ -656,7 +659,7 @@ test('jump to sae/source source/sae', async ({ page }) => { } }); -test('jump to feature models', async ({ page }) => { +test('jump to feature models', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // 3 occurences of this combobox on this page @@ -668,7 +671,7 @@ test('jump to feature models', async ({ page }) => { } }); -test('jump to feature source/sae', async ({ page }) => { +test('jump to feature source/sae', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // 2 occurences of this combobox on this page @@ -691,7 +694,7 @@ test('jump to feature source/sae', async ({ page }) => { } }); -test('search explanations appears', async ({ page }) => { +test('search explanations appears', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // Look for the h3 element with the specific classes and text @@ -703,7 +706,7 @@ test('search explanations appears', async ({ page }) => { await expect(jumpToHeading).toHaveText('Search Explanations'); }); -test('search via interference appears', async ({ page }) => { +test('search via interference appears', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); const jumpToHeading = page.locator( @@ -714,7 +717,7 @@ test('search via interference appears', async ({ page }) => { await expect(jumpToHeading).toHaveText('Search via Inference'); }); -test('search via inference models', async ({ page }) => { +test('search via inference models', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); // 3 occurences of this combobox on this page @@ -726,7 +729,7 @@ test('search via inference models', async ({ page }) => { } }); -test('search via inference source/sae', async ({ page }) => { +test('search via inference source/sae', async ({ page, gemmaUrl }) => { await page.goto('https://neuronpedia.org/gemma-scope#browse'); await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').first().click(); diff --git a/apps/webapp/tests-playwright/production/search.test.ts b/apps/webapp/tests-playwright/production/search.test.ts index a5d9fa5de..10868f572 100644 --- a/apps/webapp/tests-playwright/production/search.test.ts +++ b/apps/webapp/tests-playwright/production/search.test.ts @@ -1,5 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; +// http:localhost:3000/search test('search models', async ({ page }) => { await page.goto('https://neuronpedia.org/search'); diff --git a/apps/webapp/tests-playwright/production/steer.test.ts b/apps/webapp/tests-playwright/production/steer.test.ts index 9d43e43b2..a351300bd 100644 --- a/apps/webapp/tests-playwright/production/steer.test.ts +++ b/apps/webapp/tests-playwright/production/steer.test.ts @@ -1,4 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +// http://localhost:3000/steer defaults to localhost:3000/gemma-2-2b-it/steer test('steer page loads', async ({ page }) => { await page.goto('https://neuronpedia.org/steer'); @@ -13,10 +16,10 @@ test('model selector', async ({ page }) => { 'DEEPSEEK-R1-LLAMA-8B', 'GEMMA-2-2B', 'GEMMA-2-2B-IT', - 'GEMMA-2-9B', - 'GEMMA-2-9B-IT', + //'GEMMA-2-9B', + //'GEMMA-2-9B-IT', 'GPT2-SM', - 'LLAMA3.1-8B', + //'LLAMA3.1-8B', ]; for (const model of modelNames) { @@ -24,73 +27,73 @@ test('model selector', async ({ page }) => { } }); -test('deepseek-r1-llama-8b steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/deepseek-r1-distill-llama-8b/steer'); +// test('deepseek-r1-llama-8b steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/deepseek-r1-distill-llama-8b/steer'); - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered deepseek-r1-distill-llama-8b")).not.toBeVisible({ timeout: 30000 }); -}); +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered deepseek-r1-distill-llama-8b")).not.toBeVisible({ timeout: 30000 }); +// }); -test('gemma-2-2b steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-2-2b/steer'); +// test('gemma-2-2b steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-2-2b/steer'); - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gemma-2-2b!")).not.toBeVisible({ timeout: 30000 }); -}); +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gemma-2-2b!")).not.toBeVisible({ timeout: 30000 }); +// }); -test('gemma-2-2b-it steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-2-2b-it/steer'); +// test('gemma-2-2b-it steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-2-2b-it/steer'); - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gemma-2-2b-it!")).not.toBeVisible({ timeout: 30000 }); -}); +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gemma-2-2b-it!")).not.toBeVisible({ timeout: 30000 }); +// }); -// No presets -// test('gemma-2-9b steer', async({ page}) => { -// await page.goto('https://neuronpedia.org/gemma-2-9b/steer'); +// // No presets +// // test('gemma-2-9b steer', async({ page}) => { +// // await page.goto('https://neuronpedia.org/gemma-2-9b/steer'); -// //user first occurring demo to search -// await page.getByRole('button', { name: 'Demo' }).first().click(); +// // //user first occurring demo to search +// // await page.getByRole('button', { name: 'Demo' }).first().click(); -// //check to see if text changed due to a new response -// await expect(page.getByText("Hey, I'm steered gemma-2-9b!")).not.toBeVisible({ timeout: 30000 }); -// }); +// // //check to see if text changed due to a new response +// // await expect(page.getByText("Hey, I'm steered gemma-2-9b!")).not.toBeVisible({ timeout: 30000 }); +// // }); -test('gemma-2-9b-it steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-2-9b-it/steer'); +// test('gemma-2-9b-it steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-2-9b-it/steer'); - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gemma-2-9b-it!")).not.toBeVisible({ timeout: 30000 }); -}); +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gemma-2-9b-it!")).not.toBeVisible({ timeout: 30000 }); +// }); -test('gpt2-small steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gpt2-small/steer'); +// test('gpt2-small steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gpt2-small/steer'); - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gpt2-small!")).not.toBeVisible({ timeout: 30000 }); -}); +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gpt2-small!")).not.toBeVisible({ timeout: 30000 }); +// }); -test('llama3.1-8b steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/llama3.1-8b/steer'); +// test('llama3.1-8b steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/llama3.1-8b/steer'); - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered llama3.1-8b!")).not.toBeVisible({ timeout: 30000 }); -}); +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered llama3.1-8b!")).not.toBeVisible({ timeout: 30000 }); +// }); diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index 728abf399..fcc4016ef 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -30,7 +30,8 @@ } ], "downlevelIteration": true, - "typeRoots": ["./types", "./node_modules/@types"] + "typeRoots": ["./types", "./node_modules/@types"], + "types": ["node", "@playwright/test"] }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "middleware.ts"], "exclude": ["node_modules"] diff --git a/package-lock.json b/package-lock.json index 45583f92f..8ae54a451 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,75 @@ { - "name": "neuronpedia-monorepo", + "name": "neuronpedia", "lockfileVersion": 3, "requires": true, - "packages": {} + "packages": { + "": { + "devDependencies": { + "@playwright/test": "^1.51.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.1.tgz", + "integrity": "sha512-nM+kEaTSAoVlXmMPH10017vn3FSiFqr/bh4fKg9vmAdMfd9SDqRZNvPSiAHADc/itWak+qPvMPZQOPwCBW7k7Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.51.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz", + "integrity": "sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.51.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz", + "integrity": "sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } } diff --git a/package.json b/package.json new file mode 100644 index 000000000..f1050643c --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "@playwright/test": "^1.51.1" + } +} From 9f7e721235322c74bfcedf98440da354547cd40c Mon Sep 17 00:00:00 2001 From: Khai Date: Mon, 14 Apr 2025 12:05:16 -0500 Subject: [PATCH 3/9] updated gemmascope tests --- .../production/gemmascope.test.ts | 344 +++++++++--------- 1 file changed, 180 insertions(+), 164 deletions(-) diff --git a/apps/webapp/tests-playwright/production/gemmascope.test.ts b/apps/webapp/tests-playwright/production/gemmascope.test.ts index 234aeda38..799ad22a8 100644 --- a/apps/webapp/tests-playwright/production/gemmascope.test.ts +++ b/apps/webapp/tests-playwright/production/gemmascope.test.ts @@ -9,91 +9,91 @@ test('main page', async ({ page, gemmaUrl }) => { await page.locator('textarea[name="The inner workings"]').isVisible(); }); -test('microscope button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); +test('microscope button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); await page.getByText("scan Gemma 2's brain to see what it's thinking").click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#microscope'); + await expect(page).toHaveURL(`${gemmaUrl}#micropscope`); }); -test('analyze features button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); +test('analyze features button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); await page.getByText('Make features fire and figure out what they do').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#analyze'); + await expect(page).toHaveURL(`${gemmaUrl}#analyze`); }); -test('steer gemma button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); +test('steer gemma button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); await page.getByText("Change Gemma's behavior by manipulating features").click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#steer'); + await expect(page).toHaveURL(`${gemmaUrl}#steer`); }); -test('do more button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); +test('do more button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); await page.getByText('Dive deeper into the exciting world of AI interpretability').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); + await expect(page).toHaveURL(`${gemmaUrl}#learn`); }); -test('browse & search saes button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); +test('browse & search saes button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); await page.getByText('Directly explore the SAEs in Gemma Scope').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#browse'); + await expect(page).toHaveURL(`${gemmaUrl}#browse`); }); -test('microscope page', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('microscope page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.locator('textarea[name="To understand what AI is thinking"]').isVisible(); }); -test('demo button feeling lucky', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('demo button feeling lucky', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.getByText("I'm Feeling Lucky").click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button food', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('demo button food', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.getByText('Food').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button News', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('demo button News', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.getByText('News').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button Literary', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('demo button Literary', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.getByText('Literary').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button Personal', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('demo button Personal', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.getByText('Personal').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('demo button Programming', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('demo button Programming', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); await page.getByText('Programming').click(); await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('challenge gemma', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('challenge gemma', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); // challenge is unlocked after doing the demo, or pressing skip demo await page.getByText('skip demo').first().click(); @@ -104,47 +104,47 @@ test('challenge gemma', async ({ page, gemmaUrl }) => { await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); }); -test('next analyze features button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); +test('next analyze features button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); // next is unlocked after doing the demo, or pressing skip demo await page.getByText('skip demo').nth(1).click(); await page.getByText(' Next - Analyze Features').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#analyze'); + await expect(page).toHaveURL(`${gemmaUrl}#analyze`); }); -test('analyze features page', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('analyze features page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); await page.locator('textarea[name="The features in Gemma Scope have labels"]').isVisible(); }); -test('olympic sports button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('olympic sports button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); await page.getByText('Olympic sports').click(); await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); }); -test('famous cities button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('famous cities button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); await page.getByText('famous cities').click(); await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); }); -test('reference to animals button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('reference to animals button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); await page.getByText('references to animals').click(); await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); }); -test('none of these button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('none of these button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); await page.getByText('none of these').click(); await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); }); -test('analyze steer', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('analyze steer', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); // unlocked after doing the demo, or pressing skip demo await page.getByText('skip example').first().click(); @@ -155,8 +155,8 @@ test('analyze steer', async ({ page, gemmaUrl }) => { await expect(page.getByText('0.00')).toBeVisible(); }); -test('puzzles labels', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('puzzles labels', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); // unlocked after doing analyze steer, or pressing skip analyze await page.getByText('skip analyze').first().click(); @@ -164,9 +164,9 @@ test('puzzles labels', async ({ page, gemmaUrl }) => { // test all 3 buttons const labelButtons = await page.getByRole('button', { name: 'Reveal Our Label' }).all(); // click each button - for (const button of labelButtons) { - await button.click(); - } + await Promise.all( + labelButtons.map(button => button.click()) + ); // check labels await expect(page.getByText('Lies and falsehoods')).toBeVisible(); @@ -174,13 +174,13 @@ test('puzzles labels', async ({ page, gemmaUrl }) => { await expect(page.getByText('Bad/cringe stories')).toBeVisible(); }); -test('next steer gemma button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); +test('next steer gemma button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); // unlocked after puzzles, or pressing skip analyze await page.getByText('skip analyze').nth(1).click(); await page.getByText(' Next - Steer Gemma').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#steer'); + await expect(page).toHaveURL(`${gemmaUrl}#steer`); }); // test('steer page', async ({ page }) => { @@ -203,31 +203,31 @@ test('next steer gemma button', async ({ page, gemmaUrl }) => { // await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); // }); -test('do more page', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('do more page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); await page.locator('textarea[name="This demo of Gemma Scope"]').isVisible(); }); -test('advanced steer button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('advanced steer button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([ page.waitForEvent('popup'), - await page.locator('a[target="_blank"][href*="https://www.neuronpedia.org/steer"]').click(), + await page.locator('a[target="_blank"][href*="http://localhost:3000/steer"]').click(), ]); - await expect(newPage).toHaveURL(/https:\/\/www\.neuronpedia\.org\/.*\/steer/); + await expect(newPage).toHaveURL(/\/steer$/); }); -test('browse saes button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('browse saes button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); await page.getByText('Browse SAEs').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#browse'); + await expect(page).toHaveURL(`${gemmaUrl}#browse`); }); // test('Contact Us', async({ page }) => { -// await page.goto('https://neuronpedia.org/gemma-scope#learn'); +// await page.goto(`${gemmaUrl}#learn`); // const [newPage] = await Promise.all([ // page.waitForEvent('popup'), @@ -237,8 +237,8 @@ test('browse saes button', async ({ page, gemmaUrl }) => { // await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); // }); -test('deepmind blog post button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('deepmind blog post button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('DeepMind Blog Post').click()]); @@ -247,8 +247,8 @@ test('deepmind blog post button', async ({ page, gemmaUrl }) => { ); }); -test('coding tutorial button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('coding tutorial button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Coding Tutorial').click()]); @@ -259,24 +259,24 @@ test('coding tutorial button', async ({ page, gemmaUrl }) => { // testing it the default method with expect().toHaveURL doesnt work, url ends up being blank // only tests for the correct link being on the page -test('technical report button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('technical report button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const href = await page.getByText('Technical Report').getAttribute('href'); expect(href).toContain('storage.googleapis.com/gemma-scope/gemma-scope-report.pdf'); }); -test('huggingface button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('huggingface button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('HuggingFace').click()]); await expect(newPage).toHaveURL('https://huggingface.co/google/gemma-scope'); }); -test('get started with mech interp button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('get started with mech interp button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -286,8 +286,8 @@ test('get started with mech interp button', async ({ page, gemmaUrl }) => { await expect(newPage).toHaveURL('https://www.neelnanda.io/mechanistic-interpretability/getting-started'); }); -test('favourite mech interp papers button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('favourite mech interp papers button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -299,8 +299,8 @@ test('favourite mech interp papers button', async ({ page, gemmaUrl }) => { ); }); -test('toward monosemanticity button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('toward monosemanticity button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -310,48 +310,48 @@ test('toward monosemanticity button', async ({ page, gemmaUrl }) => { await expect(newPage).toHaveURL('https://transformer-circuits.pub/2023/monosemantic-features'); }); -test('saelens button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('saelens button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('SAELens').click()]); await expect(newPage).toHaveURL('https://github.com/jbloomAus/SAELens'); }); -test('transformerlens button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('transformerlens button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('TransformerLens').click()]); await expect(newPage).toHaveURL('https://github.com/TransformerLensOrg/TransformerLens'); }); -test('nnsight button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('nnsight button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('NNsight').click()]); await expect(newPage).toHaveURL('https://github.com/ndif-team/nnsight'); }); -test('mats program button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('mats program button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('MATS Program').click()]); await expect(newPage).toHaveURL('https://www.matsprogram.org/'); }); -test('arena education button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('arena education button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('ARENA Education').click()]); await expect(newPage).toHaveURL('https://www.arena.education/'); }); -test('twitter button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('twitter button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Twitter').first().click()]); @@ -360,7 +360,7 @@ test('twitter button', async ({ page, gemmaUrl }) => { // second contact us button on same page // test('Contact Us', async({ page }) => { -// await page.goto('https://neuronpedia.org/gemma-scope#learn'); +// await page.goto(`${gemmaUrl}#learn`); // const [newPage] = await Promise.all([ // page.waitForEvent('popup'), @@ -370,28 +370,28 @@ test('twitter button', async ({ page, gemmaUrl }) => { // await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); // }); -test('open problems button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('open problems button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); await page.getByRole('button', { name: 'Open Problems' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#openproblems'); + await expect(page).toHaveURL(`${gemmaUrl}#openproblems`); }); -test('playground button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); +test('playground button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); await page.getByRole('button', { name: 'Playground' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#playground'); + await expect(page).toHaveURL(`${gemmaUrl}#playground`); }); -test('open problems page', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +test('open problems page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); await page.locator('textarea[name="Interpretability has many unsolved problems"]').isVisible(); }); -test('slack link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +test('slack link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Slack').click()]); @@ -401,7 +401,7 @@ test('slack link', async ({ page, gemmaUrl }) => { }); // test('lesswrong link', async ({ page }) => { -// await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +// await page.goto(`${gemmaUrl}#openproblems`); // const [newPage] = await Promise.all([ // page.waitForEvent('popup'), @@ -411,32 +411,32 @@ test('slack link', async ({ page, gemmaUrl }) => { // await expect(newPage).toHaveURL('https://www.lesswrong.com'); // }); -test('wattenberg link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +test('wattenberg link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Wattenberg et al').click()]); await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.14662'); }); -test('Ziegler link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +test('Ziegler link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Ziegler et al').click()]); await expect(newPage).toHaveURL('https://arxiv.org/abs/2205.01663'); }); -test('Steering Vectors link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +test('Steering Vectors link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('steering vectors').click()]); await expect(newPage).toHaveURL('https://arxiv.org/abs/2308.10248'); }); -test('SAE feature steering link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); +test('SAE feature steering link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('SAE feature steering').click()]); @@ -446,7 +446,7 @@ test('SAE feature steering link', async ({ page, gemmaUrl }) => { }); test('clamping link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('clamping').click()]); @@ -454,7 +454,7 @@ test('clamping link', async ({ page, gemmaUrl }) => { }); test('huang et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Huang et al').click()]); @@ -462,7 +462,7 @@ test('huang et al link', async ({ page, gemmaUrl }) => { }); test('bills et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Bills et al').click()]); @@ -471,7 +471,7 @@ test('bills et al link', async ({ page, gemmaUrl }) => { // same link appears twice test('engels et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [firstPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Engels et al').first().click()]); @@ -486,7 +486,7 @@ test('engels et al link', async ({ page, gemmaUrl }) => { }); test('toy models link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([ page.waitForEvent('popup'), @@ -499,7 +499,7 @@ test('toy models link', async ({ page, gemmaUrl }) => { }); test('stolfo et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Stolfo et al').click()]); @@ -507,7 +507,7 @@ test('stolfo et al link', async ({ page, gemmaUrl }) => { }); test('marks et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Marks et al').click()]); @@ -515,7 +515,7 @@ test('marks et al link', async ({ page, gemmaUrl }) => { }); test('mlp transcoders link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('MLP transcoders').click()]); @@ -523,7 +523,7 @@ test('mlp transcoders link', async ({ page, gemmaUrl }) => { }); test('jain et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Jain et al').click()]); @@ -531,7 +531,7 @@ test('jain et al link', async ({ page, gemmaUrl }) => { }); test('hendel et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Hendel et al').click()]); @@ -539,7 +539,7 @@ test('hendel et al link', async ({ page, gemmaUrl }) => { }); test('Todd et al link', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Todd et al').click()]); @@ -547,27 +547,27 @@ test('Todd et al link', async ({ page, gemmaUrl }) => { }); test('do more button open problems', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); await page.getByRole('button', { name: 'Do More' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); + await expect(page).toHaveURL(`${gemmaUrl}#learn`); }); test('home button open problems', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); + await page.goto(`${gemmaUrl}#openproblems`); await page.getByRole('button', { name: 'Home' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#main'); + await expect(page).toHaveURL(`${gemmaUrl}#main`); }); test('playground page', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); + await page.goto(`${gemmaUrl}#playground`); await page.locator('textarea[name="Enter something below to see"]').isVisible(); }); test('im feeling lucky button', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); + await page.goto(`${gemmaUrl}#playground`); await page.getByRole('button', { name: "I'm Feeling Lucky" }).click(); await expect( @@ -576,29 +576,33 @@ test('im feeling lucky button', async ({ page, gemmaUrl }) => { }); test('model selector', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); + await page.goto(`${gemmaUrl}#playground`); await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('source set selector', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); + await page.goto(`${gemmaUrl}#playground`); await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); const modelNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('playground steering', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); + await page.goto(`${gemmaUrl}#playground`); const gemmaSearch = page.getByPlaceholder('Make gemma-2-2b think about'); await gemmaSearch.fill('testing inputs'); @@ -607,13 +611,13 @@ test('playground steering', async ({ page, gemmaUrl }) => { }); test('browse & search page', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); await page.locator('textarea[name="Search 50,000,000+ features"]').isVisible(); }); test('jump to appears', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // Look for the h3 element with the specific classes and text const jumpToHeading = page.locator( @@ -625,19 +629,21 @@ test('jump to appears', async ({ page, gemmaUrl }) => { }); test('jump to sae/source models', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // 3 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').first().click(); const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('jump to sae/source source/sae', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // 2 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').first().click(); @@ -654,25 +660,29 @@ test('jump to sae/source source/sae', async ({ page, gemmaUrl }) => { 'gemmascope-res-65k', ]; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('jump to feature models', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // 3 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(1).click(); const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('jump to feature source/sae', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // 2 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').nth(1).click(); @@ -689,13 +699,15 @@ test('jump to feature source/sae', async ({ page, gemmaUrl }) => { 'gemmascope-res-65k', ]; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('search explanations appears', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // Look for the h3 element with the specific classes and text const jumpToHeading = page.locator( @@ -707,7 +719,7 @@ test('search explanations appears', async ({ page, gemmaUrl }) => { }); test('search via interference appears', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); const jumpToHeading = page.locator( 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="search-inference-release-pane.tsx"]', @@ -718,19 +730,21 @@ test('search via interference appears', async ({ page, gemmaUrl }) => { }); test('search via inference models', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); // 3 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(2).click(); const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); test('search via inference source/sae', async ({ page, gemmaUrl }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); + await page.goto(`${gemmaUrl}#browse`); await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').first().click(); const modelNames = [ @@ -738,7 +752,9 @@ test('search via inference source/sae', async ({ page, gemmaUrl }) => { 'gemmascope-res-16k', ]; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); From 4f87975623c7db34dcabe1755956ee8dd9f646bc Mon Sep 17 00:00:00 2001 From: Khai Date: Tue, 15 Apr 2025 09:59:13 -0500 Subject: [PATCH 4/9] update search, steer, gemmascope tests --- apps/webapp/tests-playwright/fixtures.ts | 8 + .../{production => local}/gemmascope.test.ts | 0 .../tests-playwright/local/search.test.ts | 220 ++++++++++++++++++ .../{production => local}/steer.test.ts | 25 +- .../production/search.test.ts | 208 ----------------- 5 files changed, 242 insertions(+), 219 deletions(-) rename apps/webapp/tests-playwright/{production => local}/gemmascope.test.ts (100%) create mode 100644 apps/webapp/tests-playwright/local/search.test.ts rename apps/webapp/tests-playwright/{production => local}/steer.test.ts (88%) delete mode 100644 apps/webapp/tests-playwright/production/search.test.ts diff --git a/apps/webapp/tests-playwright/fixtures.ts b/apps/webapp/tests-playwright/fixtures.ts index d9f3a538a..69be593dd 100644 --- a/apps/webapp/tests-playwright/fixtures.ts +++ b/apps/webapp/tests-playwright/fixtures.ts @@ -3,6 +3,8 @@ import { test as base } from '@playwright/test'; type MyFixtures = { baseUrl: string; gemmaUrl: string; + steerUrl: string; + searchUrl: string; }; export const test = base.extend({ @@ -10,4 +12,10 @@ export const test = base.extend({ gemmaUrl: async ({ baseUrl }, use) => { await use(`${baseUrl}/gemma-scope`); }, + steerUrl: async ({ baseUrl }, use) => { + await use(`${baseUrl}/steer`) + }, + searchUrl: async ({ baseUrl }, use) => { + await use(`${baseUrl}/search`); + } }); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/production/gemmascope.test.ts b/apps/webapp/tests-playwright/local/gemmascope.test.ts similarity index 100% rename from apps/webapp/tests-playwright/production/gemmascope.test.ts rename to apps/webapp/tests-playwright/local/gemmascope.test.ts diff --git a/apps/webapp/tests-playwright/local/search.test.ts b/apps/webapp/tests-playwright/local/search.test.ts new file mode 100644 index 000000000..d1d6fd6e5 --- /dev/null +++ b/apps/webapp/tests-playwright/local/search.test.ts @@ -0,0 +1,220 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +// http:localhost:3000/search +test('search models', async ({ page, searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + const modelNames = [ + 'DEEPSEEK-R1-LLAMA-8B', + 'GEMMA-2-2B', + 'GEMMA-2-2B-IT', + 'GEMMA-2-9B', + 'GEMMA-2-9B-IT', + 'GPT2-SM', + 'LLAMA3.1-8B', + ]; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('deepseek sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('DEEPSEEK-R1-LLAMA-8B').click(); + + await expect(page.getByText('llamascope-openr1-res-32k')).toBeVisible(); +}); + +test('gemma-2-2b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + // currently default search model + // await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + // await page.getByText('GEMMA-2-2B').click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('gemma-2-9b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GEMMA-2-9B').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('gemma-2-9b-it sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GEMMA-2-9B-IT').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-131k', 'gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('gpt2-sm sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GPT2-SM').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = [ + 'att_32k-oai', + 'att-kk', + 'mlp_32k-oai', + 'res_fs12288-jb', + 'res_fs1536-jb', + 'res_fs24576-jb', + 'res_fs3072-jb', + 'res_fs49152-jb', + 'res_fs6144-jb', + 'res_fs768-jb', + 'res_fs98304-jb', + 'res_mid_32k-oai', + 'res_post_32k-oai', + 'res_sce-ajt', + 'res_scefr-ajt', + 'res_scl-ajt', + 'res_sle-ajt', + 'res_slefr-ajt', + 'res_sll-ajt', + 'res-jb', + ]; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('llama3.1-8b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('LLAMA3.1-8B').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['llamascope-res-32k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +// checks for new url in the case that the default model is changed +test('random button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Random').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('food button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Food').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('news button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('News').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('literary button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Literary').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('personal button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Personal').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('Programming button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Programming').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('Techinical button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Technical').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('academic button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Academic').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('business button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Business').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('legal button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Legal').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('educational button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Educational').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('Cultural button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Cultural').click(); + await expect(page).not.toHaveURL(searchUrl); +}); diff --git a/apps/webapp/tests-playwright/production/steer.test.ts b/apps/webapp/tests-playwright/local/steer.test.ts similarity index 88% rename from apps/webapp/tests-playwright/production/steer.test.ts rename to apps/webapp/tests-playwright/local/steer.test.ts index a351300bd..766a76bf9 100644 --- a/apps/webapp/tests-playwright/production/steer.test.ts +++ b/apps/webapp/tests-playwright/local/steer.test.ts @@ -3,30 +3,33 @@ import { test } from '../fixtures'; // http://localhost:3000/steer defaults to localhost:3000/gemma-2-2b-it/steer -test('steer page loads', async ({ page }) => { - await page.goto('https://neuronpedia.org/steer'); +test('steer page loads', async ({ page, steerUrl }) => { + await page.goto(steerUrl); await expect(page.locator('text="Steer Models"')).toBeVisible(); }); -test('model selector', async ({ page }) => { - await page.goto('https://neuronpedia.org/steer'); +test('model selector', async ({ page, steerUrl }) => { + await page.goto(steerUrl); await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); const modelNames = [ 'DEEPSEEK-R1-LLAMA-8B', 'GEMMA-2-2B', 'GEMMA-2-2B-IT', - //'GEMMA-2-9B', - //'GEMMA-2-9B-IT', - 'GPT2-SM', - //'LLAMA3.1-8B', + // 'GEMMA-2-9B', + // 'GEMMA-2-9B-IT', + 'GPT2-SMALL', + // 'LLAMA3.1-8B', ]; - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); }); +// tests designed for production // test('deepseek-r1-llama-8b steer', async ({ page }) => { // await page.goto('https://neuronpedia.org/deepseek-r1-distill-llama-8b/steer'); diff --git a/apps/webapp/tests-playwright/production/search.test.ts b/apps/webapp/tests-playwright/production/search.test.ts deleted file mode 100644 index 10868f572..000000000 --- a/apps/webapp/tests-playwright/production/search.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { expect } from '@playwright/test'; -import { test } from '../fixtures'; - -// http:localhost:3000/search -test('search models', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - const modelNames = [ - 'DEEPSEEK-R1-LLAMA-8B', - 'GEMMA-2-2B', - 'GEMMA-2-2B-IT', - 'GEMMA-2-9B', - 'GEMMA-2-9B-IT', - 'GPT2-SM', - 'LLAMA3.1-8B', - ]; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('deepseek sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('DEEPSEEK-R1-LLAMA-8B').click(); - - await expect(page.getByText('llamascope-openr1-res-32k')).toBeVisible(); -}); - -test('gemma-2-2b sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - // currently default search model - // await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - // await page.getByText('GEMMA-2-2B').click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('gemma-2-9b sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GEMMA-2-9B').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-res-16k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('gemma-2-9b-it sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GEMMA-2-9B-IT').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-res-131k', 'gemmascope-res-16k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('gpt2-sm sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GPT2-SM').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = [ - 'att_32k-oai', - 'att-kk', - 'mlp_32k-oai', - 'res_fs12288-jb', - 'res_fs1536-jb', - 'res_fs24576-jb', - 'res_fs3072-jb', - 'res_fs49152-jb', - 'res_fs6144-jb', - 'res_fs768-jb', - 'res_fs98304-jb', - 'res_mid_32k-oai', - 'res_post_32k-oai', - 'res_sce-ajt', - 'res_scefr-ajt', - 'res_scl-ajt', - 'res_sle-ajt', - 'res_slefr-ajt', - 'res_sll-ajt', - 'res-jb', - ]; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('llama3.1-8b sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('LLAMA3.1-8B').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['llamascope-res-32k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -// checks for new url in the case that the default model is changed -test('random button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Random').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('food button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Food').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('news button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('News').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('literary button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Literary').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('personal button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Personal').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('Programming button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Programming').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('Techinical button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Technical').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('academic button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Academic').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('business button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Business').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('legal button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Legal').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('educational button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Educational').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('Cultural button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Cultural').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); From d46eac369a83e4d9a2d140a3ed1bb74a1a43fed8 Mon Sep 17 00:00:00 2001 From: jonathan Date: Mon, 21 Apr 2025 20:37:10 -0500 Subject: [PATCH 5/9] localized majority of playwright tests --- .../tests-playwright/local/gemmascope.test.ts | 54 ++---------- .../tests-playwright/local/home.spec.ts | 2 +- .../tests-playwright/local/homepage.test.ts | 17 +--- .../tests-playwright/local/sae-bench.spec.ts | 2 +- .../tests-playwright/local/search.test.ts | 77 +--------------- .../tests-playwright/local/steer.test.ts | 4 +- .../production/gemmascope.test.ts | 30 +++++++ .../production/homepage.test.ts | 15 ++++ .../production/search.test.ts | 87 +++++++++++++++++++ 9 files changed, 150 insertions(+), 138 deletions(-) create mode 100644 apps/webapp/tests-playwright/production/gemmascope.test.ts create mode 100644 apps/webapp/tests-playwright/production/homepage.test.ts create mode 100644 apps/webapp/tests-playwright/production/search.test.ts diff --git a/apps/webapp/tests-playwright/local/gemmascope.test.ts b/apps/webapp/tests-playwright/local/gemmascope.test.ts index 799ad22a8..6148f2a85 100644 --- a/apps/webapp/tests-playwright/local/gemmascope.test.ts +++ b/apps/webapp/tests-playwright/local/gemmascope.test.ts @@ -13,7 +13,7 @@ test('microscope button', async ({ page, gemmaUrl }) => { await page.goto(`${gemmaUrl}#main`); await page.getByText("scan Gemma 2's brain to see what it's thinking").click(); - await expect(page).toHaveURL(`${gemmaUrl}#micropscope`); + await expect(page).toHaveURL(`${gemmaUrl}#microscope`); }); test('analyze features button', async ({ page, gemmaUrl }) => { @@ -155,24 +155,6 @@ test('analyze steer', async ({ page, gemmaUrl }) => { await expect(page.getByText('0.00')).toBeVisible(); }); -test('puzzles labels', async ({ page, gemmaUrl }) => { - await page.goto(`${gemmaUrl}#analyze`); - - // unlocked after doing analyze steer, or pressing skip analyze - await page.getByText('skip analyze').first().click(); - - // test all 3 buttons - const labelButtons = await page.getByRole('button', { name: 'Reveal Our Label' }).all(); - // click each button - await Promise.all( - labelButtons.map(button => button.click()) - ); - - // check labels - await expect(page.getByText('Lies and falsehoods')).toBeVisible(); - await expect(page.getByText('Misspellings or typos')).toBeVisible(); - await expect(page.getByText('Bad/cringe stories')).toBeVisible(); -}); test('next steer gemma button', async ({ page, gemmaUrl }) => { await page.goto(`${gemmaUrl}#analyze`); @@ -214,7 +196,7 @@ test('advanced steer button', async ({ page, gemmaUrl }) => { const [newPage] = await Promise.all([ page.waitForEvent('popup'), - await page.locator('a[target="_blank"][href*="http://localhost:3000/steer"]').click(), + await page.locator('a[target="_blank"][href*="https://www.neuronpedia.org/steer"]').click(), ]); await expect(newPage).toHaveURL(/\/steer$/); }); @@ -393,7 +375,7 @@ test('open problems page', async ({ page, gemmaUrl }) => { test('slack link', async ({ page, gemmaUrl }) => { await page.goto(`${gemmaUrl}#openproblems`); - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Slack').click()]); + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Slack').nth(1).click()]); await expect(newPage).toHaveURL( 'https://opensourcemechanistic.slack.com/join/shared_invite/zt-2o756ku1c-_yKBeUQMVfS_p_qcK6QLeA#/shared-invite/email', @@ -579,7 +561,7 @@ test('model selector', async ({ page, gemmaUrl }) => { await page.goto(`${gemmaUrl}#playground`); await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; + const modelNames = ['GEMMA-2-2B']; await Promise.all( modelNames.map(model => @@ -592,7 +574,7 @@ test('source set selector', async ({ page, gemmaUrl }) => { await page.goto(`${gemmaUrl}#playground`); await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const modelNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; + const modelNames = ['gemmascope-res-16k']; await Promise.all( modelNames.map(model => @@ -633,7 +615,7 @@ test('jump to sae/source models', async ({ page, gemmaUrl }) => { // 3 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').first().click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; + const modelNames = ['GEMMA-2-2B']; await Promise.all( modelNames.map(model => @@ -648,16 +630,7 @@ test('jump to sae/source source/sae', async ({ page, gemmaUrl }) => { // 2 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').first().click(); const modelNames = [ - 'gemmascope-att-16k', - 'gemmascope-att-65k', - 'gemmascope-mlp-16k', - 'gemmascope-mlp-65k', - 'gemmascope-res-16k', - 'gemmascope-res-1m', - 'gemmascope-res-262k', - 'gemmascope-res-32k', - 'gemmascope-res-524k', - 'gemmascope-res-65k', + 'gemmascope-res-16k' ]; await Promise.all( @@ -672,7 +645,7 @@ test('jump to feature models', async ({ page, gemmaUrl }) => { // 3 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(1).click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; + const modelNames = ['GEMMA-2-2B']; await Promise.all( modelNames.map(model => @@ -687,16 +660,7 @@ test('jump to feature source/sae', async ({ page, gemmaUrl }) => { // 2 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').nth(1).click(); const modelNames = [ - 'gemmascope-att-16k', - 'gemmascope-att-65k', - 'gemmascope-mlp-16k', - 'gemmascope-mlp-65k', 'gemmascope-res-16k', - 'gemmascope-res-1m', - 'gemmascope-res-262k', - 'gemmascope-res-32k', - 'gemmascope-res-524k', - 'gemmascope-res-65k', ]; await Promise.all( @@ -734,7 +698,7 @@ test('search via inference models', async ({ page, gemmaUrl }) => { // 3 occurences of this combobox on this page await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(2).click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; + const modelNames = ['GEMMA-2-2B']; await Promise.all( modelNames.map(model => diff --git a/apps/webapp/tests-playwright/local/home.spec.ts b/apps/webapp/tests-playwright/local/home.spec.ts index 890870908..80e9cbf5d 100644 --- a/apps/webapp/tests-playwright/local/home.spec.ts +++ b/apps/webapp/tests-playwright/local/home.spec.ts @@ -1,6 +1,6 @@ import { expect, test } from '@playwright/test'; -test('should navigate to an SAE source page', async ({ page }) => { +test.skip('should navigate to an SAE source page', async ({ page }) => { await page.goto('/'); await page.click('text=Get Started'); await expect(page.getByText('Introduction + Docs')).toBeTruthy(); diff --git a/apps/webapp/tests-playwright/local/homepage.test.ts b/apps/webapp/tests-playwright/local/homepage.test.ts index 572f31d80..aecc38369 100644 --- a/apps/webapp/tests-playwright/local/homepage.test.ts +++ b/apps/webapp/tests-playwright/local/homepage.test.ts @@ -19,19 +19,6 @@ test('API navigation link', async ({ page, baseUrl }) => { await expect(page).toHaveURL(`${baseUrl}/api-doc`); }); -test('SAE Evals link', async ({ page, baseUrl }) => { - await page.goto(baseUrl); - - // Find and click the SAE Evals link - const saeEvalsLink = page.getByRole('link', { name: 'SAE Evals', exact: true }); - await saeEvalsLink.waitFor({ state: 'visible' }); - await saeEvalsLink.click(); - - // Wait for navigation and check exact URL - await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL(`${baseUrl}/sae-bench`); -}); - test('Steer', async ({ page, baseUrl }) => { await page.goto(baseUrl); @@ -170,7 +157,7 @@ test('releases display', async ({ page, baseUrl }) => { test('models display', async ({ page, baseUrl }) => { await page.goto(baseUrl); - await expect(page.getByText('DeepSeek-R1-Llama-8B')).toBeVisible(); + await expect(page.getByText('DeepSeek-R1-Llama-8B').nth(1)).toBeVisible(); }); test('jump to display', async ({ page, baseUrl }) => { @@ -186,7 +173,7 @@ test('cat steering', async ({ page, baseUrl }) => { page.getByRole('link', { name: 'Try It: Gemma 2 - Cat Steering', exact: true }).click(), ]); - await expect(newPage).toHaveURL(`${baseUrl}/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5`); + await expect(newPage).toHaveURL(`https://www.neuronpedia.org/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5`); }); test('search by explanation', async ({ page, baseUrl }) => { diff --git a/apps/webapp/tests-playwright/local/sae-bench.spec.ts b/apps/webapp/tests-playwright/local/sae-bench.spec.ts index 645880131..464e96330 100644 --- a/apps/webapp/tests-playwright/local/sae-bench.spec.ts +++ b/apps/webapp/tests-playwright/local/sae-bench.spec.ts @@ -1,6 +1,6 @@ import { expect, test } from '@playwright/test'; -test('should navigate to the SAE Evals page', async ({ page }) => { +test.skip('should navigate to the SAE Evals page', async ({ page }) => { await page.goto('/'); await page.click('text=SAE Evals'); await expect(page).toHaveURL('/sae-bench'); diff --git a/apps/webapp/tests-playwright/local/search.test.ts b/apps/webapp/tests-playwright/local/search.test.ts index d1d6fd6e5..ec88844ef 100644 --- a/apps/webapp/tests-playwright/local/search.test.ts +++ b/apps/webapp/tests-playwright/local/search.test.ts @@ -7,13 +7,9 @@ test('search models', async ({ page, searchUrl }) => { await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); const modelNames = [ - 'DEEPSEEK-R1-LLAMA-8B', + 'DeepSeek-R1-Llama-8B', 'GEMMA-2-2B', - 'GEMMA-2-2B-IT', - 'GEMMA-2-9B', - 'GEMMA-2-9B-IT', 'GPT2-SM', - 'LLAMA3.1-8B', ]; await Promise.all( @@ -27,9 +23,9 @@ test('deepseek sae', async ({ page,searchUrl }) => { await page.goto(searchUrl); await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('DEEPSEEK-R1-LLAMA-8B').click(); + await page.getByText('DeepSeek-R1-Llama-8B').first().click(); - await expect(page.getByText('llamascope-openr1-res-32k')).toBeVisible(); + await expect(page.getByText('llamascope-slimpj-res-32k')).toBeVisible(); }); test('gemma-2-2b sae', async ({ page,searchUrl }) => { @@ -39,22 +35,6 @@ test('gemma-2-2b sae', async ({ page,searchUrl }) => { // await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); // await page.getByText('GEMMA-2-2B').click(); - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; - - await Promise.all( - saeNames.map(saeset => - expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() - ) - ); -}); - -test('gemma-2-9b sae', async ({ page,searchUrl }) => { - await page.goto(searchUrl); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GEMMA-2-9B').first().click(); - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); const saeNames = ['gemmascope-res-16k']; @@ -65,22 +45,6 @@ test('gemma-2-9b sae', async ({ page,searchUrl }) => { ); }); -test('gemma-2-9b-it sae', async ({ page,searchUrl }) => { - await page.goto(searchUrl); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GEMMA-2-9B-IT').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-res-131k', 'gemmascope-res-16k']; - - await Promise.all( - saeNames.map(saeset => - expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() - ) - ); -}); - test('gpt2-sm sae', async ({ page,searchUrl }) => { await page.goto(searchUrl); @@ -89,25 +53,6 @@ test('gpt2-sm sae', async ({ page,searchUrl }) => { await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); const saeNames = [ - 'att_32k-oai', - 'att-kk', - 'mlp_32k-oai', - 'res_fs12288-jb', - 'res_fs1536-jb', - 'res_fs24576-jb', - 'res_fs3072-jb', - 'res_fs49152-jb', - 'res_fs6144-jb', - 'res_fs768-jb', - 'res_fs98304-jb', - 'res_mid_32k-oai', - 'res_post_32k-oai', - 'res_sce-ajt', - 'res_scefr-ajt', - 'res_scl-ajt', - 'res_sle-ajt', - 'res_slefr-ajt', - 'res_sll-ajt', 'res-jb', ]; @@ -118,22 +63,6 @@ test('gpt2-sm sae', async ({ page,searchUrl }) => { ); }); -test('llama3.1-8b sae', async ({ page,searchUrl }) => { - await page.goto(searchUrl); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('LLAMA3.1-8B').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['llamascope-res-32k']; - - await Promise.all( - saeNames.map(saeset => - expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() - ) - ); -}); - // checks for new url in the case that the default model is changed test('random button', async ({ page,searchUrl }) => { await page.goto(searchUrl); diff --git a/apps/webapp/tests-playwright/local/steer.test.ts b/apps/webapp/tests-playwright/local/steer.test.ts index 766a76bf9..c24222373 100644 --- a/apps/webapp/tests-playwright/local/steer.test.ts +++ b/apps/webapp/tests-playwright/local/steer.test.ts @@ -13,12 +13,12 @@ test('model selector', async ({ page, steerUrl }) => { await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); const modelNames = [ - 'DEEPSEEK-R1-LLAMA-8B', + 'DeepSeek-R1-Llama-8B', 'GEMMA-2-2B', 'GEMMA-2-2B-IT', // 'GEMMA-2-9B', // 'GEMMA-2-9B-IT', - 'GPT2-SMALL', + 'GPT2-SM', // 'LLAMA3.1-8B', ]; diff --git a/apps/webapp/tests-playwright/production/gemmascope.test.ts b/apps/webapp/tests-playwright/production/gemmascope.test.ts new file mode 100644 index 000000000..6c6c4edbe --- /dev/null +++ b/apps/webapp/tests-playwright/production/gemmascope.test.ts @@ -0,0 +1,30 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('gemma steer demo', async ({ page }) => { + await page.goto('https://neuronpedia.org/gemma-scope#steer'); + + await page.click('text='); + await page.getByRole('button', { name: 'Tell me about yourself.' }).first().click(); + await page.waitForTimeout(3000); + await expect(page.getByText("Sorry, your message could not be sent at this time. Please try again later.")).not.toBeVisible({ timeout: 30000 }); + }); + + test('puzzles labels', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + + // unlocked after doing analyze steer, or pressing skip analyze + await page.getByText('skip analyze').first().click(); + + // test all 3 buttons + const labelButtons = await page.getByRole('button', { name: 'Reveal Our Label' }).all(); + // click each button + await Promise.all( + labelButtons.map(button => button.click()) + ); + + // check labels + await expect(page.getByText('Lies and falsehoods')).toBeVisible(); + await expect(page.getByText('Misspellings or typos')).toBeVisible(); + await expect(page.getByText('Bad/cringe stories')).toBeVisible(); + }); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/production/homepage.test.ts b/apps/webapp/tests-playwright/production/homepage.test.ts new file mode 100644 index 000000000..1d0f446cc --- /dev/null +++ b/apps/webapp/tests-playwright/production/homepage.test.ts @@ -0,0 +1,15 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('SAE Evals link', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + // Find and click the SAE Evals link + const saeEvalsLink = page.getByRole('link', { name: 'SAE Evals', exact: true }); + await saeEvalsLink.waitFor({ state: 'visible' }); + await saeEvalsLink.click(); + + // Wait for navigation and check exact URL + await page.waitForLoadState('networkidle'); + await expect(page).toHaveURL(`${baseUrl}/sae-bench`); + }); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/production/search.test.ts b/apps/webapp/tests-playwright/production/search.test.ts new file mode 100644 index 000000000..2aa5fdb26 --- /dev/null +++ b/apps/webapp/tests-playwright/production/search.test.ts @@ -0,0 +1,87 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('gemma-2-9b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GEMMA-2-9B').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); + + test('gpt2-sm sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GPT2-SM').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = [ + 'att_32k-oai', + 'att-kk', + 'mlp_32k-oai', + 'res_fs12288-jb', + 'res_fs1536-jb', + 'res_fs24576-jb', + 'res_fs3072-jb', + 'res_fs49152-jb', + 'res_fs6144-jb', + 'res_fs768-jb', + 'res_fs98304-jb', + 'res_mid_32k-oai', + 'res_post_32k-oai', + 'res_sce-ajt', + 'res_scefr-ajt', + 'res_scl-ajt', + 'res_sle-ajt', + 'res_slefr-ajt', + 'res_sll-ajt', + 'res-jb', + ]; + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); + + +test('gemma-2-9b-it sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GEMMA-2-9B-IT').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-131k', 'gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); + + test('llama3.1-8b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('LLAMA3.1-8B').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['llamascope-res-32k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); From ac9fef9ff3bbd91719f494ccd4d517f11f223de8 Mon Sep 17 00:00:00 2001 From: Khai Date: Wed, 30 Apr 2025 21:26:09 -0500 Subject: [PATCH 6/9] Add missing @tailwindcss/forms dependency --- package-lock.json | 34 +++++++++++++++++++++++++++++++++- package.json | 3 ++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8ae54a451..c18c37af8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,7 +5,8 @@ "packages": { "": { "devDependencies": { - "@playwright/test": "^1.51.1" + "@playwright/test": "^1.51.1", + "@tailwindcss/forms": "^0.5.10" } }, "node_modules/@playwright/test": { @@ -24,6 +25,19 @@ "node": ">=18" } }, + "node_modules/@tailwindcss/forms": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.10.tgz", + "integrity": "sha512-utI1ONF6uf/pPNO68kmN1b8rEwNXv3czukalo8VtJH8ksIkZXr3Q3VYudZLkCsDd4Wku120uF02hYK25XGPorw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mini-svg-data-uri": "^1.2.3" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1 || >= 4.0.0-alpha.20 || >= 4.0.0-beta.1" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -39,6 +53,16 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/mini-svg-data-uri": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", + "integrity": "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==", + "dev": true, + "license": "MIT", + "bin": { + "mini-svg-data-uri": "cli.js" + } + }, "node_modules/playwright": { "version": "1.51.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz", @@ -70,6 +94,14 @@ "engines": { "node": ">=18" } + }, + "node_modules/tailwindcss": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.5.tgz", + "integrity": "sha512-nYtSPfWGDiWgCkwQG/m+aX83XCwf62sBgg3bIlNiiOcggnS1x3uVRDAuyelBFL+vJdOPPCGElxv9DjHJjRHiVA==", + "dev": true, + "license": "MIT", + "peer": true } } } diff --git a/package.json b/package.json index f1050643c..daf1520ab 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "devDependencies": { - "@playwright/test": "^1.51.1" + "@playwright/test": "^1.51.1", + "@tailwindcss/forms": "^0.5.10" } } From 105b6d40915a87ef34db36ce3d3f8e27cceb6810 Mon Sep 17 00:00:00 2001 From: Khai Date: Wed, 30 Apr 2025 21:40:19 -0500 Subject: [PATCH 7/9] fix missing @tailwindcss/forms dependency --- apps/webapp/package-lock.json | 24 ++++++++++++++++++++++++ apps/webapp/package.json | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/apps/webapp/package-lock.json b/apps/webapp/package-lock.json index 3f57828b2..99e427332 100644 --- a/apps/webapp/package-lock.json +++ b/apps/webapp/package-lock.json @@ -123,6 +123,7 @@ "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", "@playwright/test": "^1.42.1", + "@tailwindcss/forms": "^0.5.10", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/d3": "^7.4.3", @@ -8598,6 +8599,19 @@ "tailwindcss": ">=3.2.0" } }, + "node_modules/@tailwindcss/forms": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.10.tgz", + "integrity": "sha512-utI1ONF6uf/pPNO68kmN1b8rEwNXv3czukalo8VtJH8ksIkZXr3Q3VYudZLkCsDd4Wku120uF02hYK25XGPorw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mini-svg-data-uri": "^1.2.3" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1 || >= 4.0.0-alpha.20 || >= 4.0.0-beta.1" + } + }, "node_modules/@tanstack/query-core": { "version": "5.64.2", "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.64.2.tgz", @@ -19520,6 +19534,16 @@ "node": ">= 0.6" } }, + "node_modules/mini-svg-data-uri": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", + "integrity": "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==", + "dev": true, + "license": "MIT", + "bin": { + "mini-svg-data-uri": "cli.js" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", diff --git a/apps/webapp/package.json b/apps/webapp/package.json index fb8049bda..efc52c1f9 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -149,10 +149,11 @@ "zod-prisma-types": "^2.7.4" }, "devDependencies": { - "@playwright/test": "^1.42.1", "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", + "@playwright/test": "^1.42.1", + "@tailwindcss/forms": "^0.5.10", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/d3": "^7.4.3", From 36143901dcd5bd485b42256d2a643b41a047564e Mon Sep 17 00:00:00 2001 From: Khai Date: Wed, 30 Apr 2025 21:49:42 -0500 Subject: [PATCH 8/9] Add missing @tailwindcss/typography dependency --- apps/webapp/package-lock.json | 38 +++++++++++++++++++++++++++++++++++ apps/webapp/package.json | 1 + 2 files changed, 39 insertions(+) diff --git a/apps/webapp/package-lock.json b/apps/webapp/package-lock.json index 99e427332..d5c49636b 100644 --- a/apps/webapp/package-lock.json +++ b/apps/webapp/package-lock.json @@ -124,6 +124,7 @@ "@next/env": "^15.1.6", "@playwright/test": "^1.42.1", "@tailwindcss/forms": "^0.5.10", + "@tailwindcss/typography": "^0.5.16", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/d3": "^7.4.3", @@ -8612,6 +8613,22 @@ "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1 || >= 4.0.0-alpha.20 || >= 4.0.0-beta.1" } }, + "node_modules/@tailwindcss/typography": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.16.tgz", + "integrity": "sha512-0wDLwCVF5V3x3b1SGXPCDcdsbDHMBe+lkFzBRaHeLvNi+nrrnZ1lA18u+OTWO8iSWU2GxUOCvlXtDuqftc1oiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash.castarray": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.merge": "^4.6.2", + "postcss-selector-parser": "6.0.10" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1" + } + }, "node_modules/@tanstack/query-core": { "version": "5.64.2", "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.64.2.tgz", @@ -17988,6 +18005,13 @@ "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", "license": "MIT" }, + "node_modules/lodash.castarray": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz", + "integrity": "sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -22210,6 +22234,20 @@ "postcss": "^8.4.31" } }, + "node_modules/postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/postcss-svgo": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-7.0.1.tgz", diff --git a/apps/webapp/package.json b/apps/webapp/package.json index efc52c1f9..228c75d2c 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -154,6 +154,7 @@ "@next/env": "^15.1.6", "@playwright/test": "^1.42.1", "@tailwindcss/forms": "^0.5.10", + "@tailwindcss/typography": "^0.5.16", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/d3": "^7.4.3", From 589ad4d8d7b3a67e42d5c792268f9c30dadf5be6 Mon Sep 17 00:00:00 2001 From: Khai Date: Wed, 30 Apr 2025 21:59:15 -0500 Subject: [PATCH 9/9] Add @playwright/test dependency --- apps/webapp/package-lock.json | 24 ++++++++++++------------ apps/webapp/package.json | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/webapp/package-lock.json b/apps/webapp/package-lock.json index d5c49636b..707eed8c7 100644 --- a/apps/webapp/package-lock.json +++ b/apps/webapp/package-lock.json @@ -122,7 +122,7 @@ "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", - "@playwright/test": "^1.42.1", + "@playwright/test": "^1.52.0", "@tailwindcss/forms": "^0.5.10", "@tailwindcss/typography": "^0.5.16", "@testing-library/dom": "^10.4.0", @@ -5636,13 +5636,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.51.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.1.tgz", - "integrity": "sha512-nM+kEaTSAoVlXmMPH10017vn3FSiFqr/bh4fKg9vmAdMfd9SDqRZNvPSiAHADc/itWak+qPvMPZQOPwCBW7k7Q==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz", + "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==", "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.51.1" + "playwright": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -21427,13 +21427,13 @@ } }, "node_modules/playwright": { - "version": "1.51.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz", - "integrity": "sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz", + "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==", "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.51.1" + "playwright-core": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -21446,9 +21446,9 @@ } }, "node_modules/playwright-core": { - "version": "1.51.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz", - "integrity": "sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz", + "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==", "devOptional": true, "license": "Apache-2.0", "bin": { diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 228c75d2c..8accbdeff 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -152,7 +152,7 @@ "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", - "@playwright/test": "^1.42.1", + "@playwright/test": "^1.52.0", "@tailwindcss/forms": "^0.5.10", "@tailwindcss/typography": "^0.5.16", "@testing-library/dom": "^10.4.0",