Apply v3.17.5 changes to v3.19.3 Branch

.github/workflows/playwright.yml

@@ -0,0 +1,70 @@
+name: Playwright Tests
+
+on:
+  push:
+    branches: [ main, release/* ]
+  pull_request:
+    branches: [ main, release/* ]
+env:
+  PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+  PLAYWRIGHT_BROWSERS_PATH: 0
+
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: 16
+
+    - name: Install frontend dependencies
+      run: |
+        npx playwright install-deps
+        npm ci --legacy-peer-deps
+      working-directory: src
+
+    - name: Install wait-on
+      run: npm install -g wait-on
+
+    - name: Install Playwright browsers
+      run: npx playwright install
+      working-directory: src
+
+    - name: Build and run backend
+      run: |
+        docker run --rm -p 5000:5000 -e FML_DEV_MODE=true gresearch/fasttrackml:main &
+        npx wait-on http://localhost:5000
+
+    - name: Install k6
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gnupg software-properties-common ca-certificates curl
+        curl -s https://dl.k6.io/key.gpg | sudo apt-key add -
+        echo "deb https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+        sudo apt-get update
+        sudo apt-get install -y k6
+
+    - name: Seed database with k6
+      run: |
+        wget https://raw.githubusercontent.com/G-Research/fasttrackml/main/docs/example/k6_load.js
+        k6 run k6_load.js
+
+    - name: Start frontend dev server and run E2E tests
+      run: |
+        npm start &
+        npx wait-on http://localhost:3000
+        npx playwright test src/e2e
+      working-directory: src
+
+    - name: Upload Playwright report
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30

.gitignore

@@ -1 +1,3 @@
 embed/
+.idea/
+.vscode/

README.md

@@ -26,6 +26,18 @@ An embed directory will be created with the built UI, ready to be embedded in th
 
 Once you're happy with them, create a pull request **against the `release/vX.Y.Z` branch*** that you started from (***not `main`!***). Once merged, the CI will run and build the UI. It will then push it to a new tag that is compatible with the Go module rules. For example, the first customization to `v3.16.2` of Aim will end up in a tag named `v0.31602.1`.
 
+### How to run E2E tests?
+
+The E2E tests can be run locally by following these steps:
+1. Start the FastTrackML server
+2. Run the Aim UI in development mode (on localhost:3000)
+3. In another terminal, run `cd src/e2e`
+4. Run `npx playwright test` to run the test suite
+
+New tests can be added directly to the `src/e2e` directory. You may also run `npx playwright show-report` to see the test results.
+
+For a guide on how to write a test, see [Playwright's example tests](https://github.com/microsoft/playwright/blob/main/examples/todomvc/tests/integration.spec.ts).
+
 ### How is this all enforced?
 
 A GitHub app has been created with the `contents:write` permissions on this repo. Its App ID and private key are stored as secrets under the `restricted` environment. This environment is limited to the `main` and `release/v*` branches
@@ -244,4 +256,5 @@ for rule in $(gh api /repos/G-Research/fasttrackml-ui-aim/rulesets -q '.[].id')
 do
   gh api /repos/G-Research/fasttrackml-ui-aim/rulesets/$rule | jq '[{name: .name, target: .target, conditions: .conditions, rules: .rules, bypass_actors: .bypass_actors}]'
 done | jq -s add
-```
+```
+

src/.gitignore

@@ -25,3 +25,7 @@ yarn-debug.log*
 yarn-error.log*
 
 Desktop
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

src/e2e/Dashboard/DashboardContent.spec.ts

@@ -0,0 +1,190 @@
+import { test, expect } from '@playwright/test';
+
+const BASE_URL = 'http://localhost:3000';
+
+test.describe('Dashboard', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto(BASE_URL);
+    await page.waitForLoadState('networkidle');
+  });
+
+  // Test that the total number of runs is two. This assumes that the
+  // test database has been seeded with two runs by running `k6 run k6_load.js`
+  test('has two runs', async ({ page }) => {
+    const textContent = await page.textContent(
+      'p.ProjectStatistics__totalRuns',
+    );
+    expect(textContent).toBe('Total runs: 2');
+  });
+
+  test('has two experiments', async ({ page }) => {
+    const textContent = await page.textContent('h3.ExperimentsCard__title');
+    expect(textContent).toBe('Experiments (2)');
+  });
+
+  test('Default experiment exists and has no runs', async ({ page }) => {
+    const table = await page.$('.BaseTable__body');
+    const rows = await table?.$$('.BaseTable__row');
+    expect(rows).not.toBeNull();
+
+    const firstRow = rows?.[0];
+    const experimentName = await firstRow?.$eval(
+      '.ExperimentNameBox__experimentName a',
+      (el) => el.textContent,
+    );
+    expect(experimentName).toBe('Default');
+
+    const runCount = await firstRow?.$eval('p', (el) => el.textContent);
+    expect(runCount).toBe('0');
+  });
+
+  test('generated experiment exists and has two runs', async ({ page }) => {
+    const table = await page.$('.BaseTable__body');
+    const rows = await table?.$$('.BaseTable__row');
+    expect(rows).not.toBeNull();
+
+    const secondRow = rows?.[1];
+    const experimentName = await secondRow?.$eval(
+      '.ExperimentNameBox__experimentName a',
+      (el) => el.textContent,
+    );
+    expect(experimentName).toContain('experiment-');
+
+    const runCount = await secondRow?.$eval('p', (el) => el.textContent);
+    expect(runCount).toBe('2');
+  });
+
+  test('experiment search bar filters table experiments', async ({ page }) => {
+    const searchInput = await page.$('.SearchBar input#search');
+    await searchInput?.fill('experiment-');
+    const value = await searchInput?.inputValue();
+    expect(value).toBe('experiment-');
+
+    const table = await page.$('.BaseTable__body');
+    const rows = await table?.$$('.BaseTable__row');
+    expect(rows?.length).toBe(1);
+
+    const firstRow = rows?.[0];
+    const experimentName = await firstRow?.$eval(
+      '.ExperimentNameBox__experimentName a',
+      (el) => el.textContent,
+    );
+    expect(experimentName).toContain('experiment-');
+  });
+
+  test("contributions heatmap has runs on the generated experiment's day", async ({
+    page,
+  }) => {
+    // Get the current date in format MMM DD, YYYY based on the generated experiment's timestamp
+    const rows = await page.$$('.BaseTable__row');
+    const secondRow = rows?.[1];
+    const experimentName = await secondRow?.$eval(
+      '.ExperimentNameBox__experimentName a',
+      (el) => el.textContent,
+    );
+
+    const timestamp = experimentName?.split('-')[1] ?? '';
+    const currentDate = new Date(parseInt(timestamp)).toLocaleDateString(
+      'en-US',
+      {
+        month: 'short',
+        day: 'numeric',
+        year: 'numeric',
+      },
+    );
+
+    // Cells are colored on a scale from 0 to 4, and we only have one active day in the test data
+    const heatmap = await page.$('.CalendarHeatmap');
+    const currentDayCell = await heatmap?.$('.CalendarHeatmap__cell--scale-4');
+    expect(currentDayCell).not.toBeNull();
+
+    const tooltip = await currentDayCell?.getAttribute('title');
+    expect(tooltip).toContain(currentDate);
+  });
+
+  test("clicking on the heatmap cell queries that day's runs", async ({
+    page,
+  }) => {
+    // Get the current timestamp from the generated experiment to construct the expected query
+    const rows = await page.$$('.BaseTable__row');
+    const secondRow = rows?.[1];
+
+    const experimentName = await secondRow?.$eval(
+      '.ExperimentNameBox__experimentName a',
+      (el) => el.textContent,
+    );
+    const timestamp = experimentName?.split('-')[1].trim() ?? '';
+
+    const date = new Date(parseInt(timestamp));
+    const year = date.getUTCFullYear();
+    const month = date.getUTCMonth() + 1; // getUTCMonth() is zero-based
+    const day = date.getUTCDate();
+
+    const nextDayDate = new Date(date);
+    nextDayDate.setUTCDate(day + 1);
+
+    const nextYear = nextDayDate.getUTCFullYear();
+    const nextMonth = nextDayDate.getUTCMonth() + 1; // getUTCMonth() is zero-based
+    const nextDay = nextDayDate.getUTCDate();
+
+    const expectedText = `datetime(${year},${month},${day})<=run.created_at<datetime(${nextYear},${nextMonth},${nextDay})`;
+
+    // Locate and click the current day cell in the heatmap
+    const heatmap = await page.$('.CalendarHeatmap');
+    const currentDayCell = await heatmap?.$('.CalendarHeatmap__cell--scale-4');
+    await currentDayCell?.click();
+
+    await page.waitForLoadState('networkidle');
+
+    const textBox = page.locator('.view-lines.monaco-mouse-cursor-text');
+
+    const textContent = await textBox.evaluate((el) => el.innerText);
+
+    // Remove tabs and newlines that might be present in the text content
+    const trimmedTextContent = textContent.replace(/\s/g, '');
+
+    expect(trimmedTextContent).toContain(expectedText);
+  });
+
+  test("clicking on the Default experiment link redirects to the experiment's page", async ({
+    page,
+  }) => {
+    const table = await page.$('.BaseTable__body');
+    const rows = await table?.$$('.BaseTable__row');
+    expect(rows).not.toBeNull();
+
+    const firstRow = rows?.[0];
+    const experimentLink = await firstRow?.$('a');
+    await experimentLink?.click();
+
+    await page.waitForLoadState('networkidle');
+
+    const experimentNameElement = page.locator(
+      '.ExperimentHeader__headerContainer__appBarTitleBox__container',
+    );
+    const experimentName = await experimentNameElement.innerText();
+
+    expect(experimentName).toContain('Default');
+  });
+
+  test("clicking on the generated experiment link redirects to the experiment's page", async ({
+    page,
+  }) => {
+    const table = await page.$('.BaseTable__body');
+    const rows = await table?.$$('.BaseTable__row');
+    expect(rows).not.toBeNull();
+
+    const secondRow = rows?.[1];
+    const experimentLink = await secondRow?.$('a');
+    await experimentLink?.click();
+
+    await page.waitForLoadState('networkidle');
+
+    const experimentNameElement = page.locator(
+      '.ExperimentHeader__headerContainer__appBarTitleBox__container',
+    );
+    const experimentName = await experimentNameElement.innerText();
+
+    expect(experimentName).toContain('experiment-');
+  });
+});

src/e2e/Dashboard/DashboardLogic.spec.ts

@@ -0,0 +1,52 @@
+import { test, expect } from '@playwright/test';
+
+const BASE_URL = 'http://localhost:3000';
+
+test.describe('Dashboard', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto(BASE_URL);
+  });
+
+  test('has title', async ({ page }) => {
+    await expect(page).toHaveTitle('FastTrackML (modern)');
+  });
+
+  test('active runs link redirects correctly', async ({ page }) => {
+    await page.getByText('Active Runs').click({ force: true });
+    await page.waitForLoadState('networkidle');
+
+    const expectedText = 'run.active==True';
+    const textBox = page.locator('.view-lines.monaco-mouse-cursor-text');
+    const textContent = await textBox.evaluate((el) => el.innerText);
+    const trimmedTextContent = textContent.replace(/\s/g, '');
+
+    expect(trimmedTextContent).toContain(expectedText);
+  });
+
+  test('archived runs link redirects correctly', async ({ page }) => {
+    await page.getByText('Archived Runs').click({ force: true });
+    await page.waitForLoadState('networkidle');
+
+    const expectedText = 'run.archived==True';
+    const textBox = page.locator('.view-lines.monaco-mouse-cursor-text');
+    const textContent = await textBox.evaluate((el) => el.innerText);
+    const trimmedTextContent = textContent.replace(/\s/g, '');
+
+    expect(trimmedTextContent).toContain(expectedText);
+  });
+
+  test("last week's runs link redirects correctly", async ({ page }) => {
+    await page.getByText("Last Week's Runs").click({ force: true });
+    await page.waitForLoadState('networkidle');
+
+    // The text varies depending on the current date:
+    // Example: datetime(2024, 6, 3) <= run.created_at < datetime(2024, 6, 10)
+    const queryRegex =
+      /datetime\(\d+,\d+,\d+\)<=run\.created_at<datetime\(\d+,\d+,\d+\)/;
+    const textBox = page.locator('.view-lines.monaco-mouse-cursor-text');
+    const textContent = await textBox.evaluate((el) => el.innerText);
+    const trimmedTextContent = textContent.replace(/\s/g, '');
+
+    expect(queryRegex.test(trimmedTextContent)).toBeTruthy();
+  });
+});