Skip to content

feat(eval-view): optimize dashboard load time with GCS summary manifest #1179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paulirish merged 6 commits into from
Aug 5, 2026
Merged

feat(eval-view): optimize dashboard load time with GCS summary manifest #1179

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1d719b4
feat(eval-view): optimize dashboard loading with GCS summary manifest…
micahjo7 Jul 31, 2026
eb0b6e9
refactor(eval-view): mark static manifest suites as local source and …
micahjo7 Jul 31, 2026
3588193
refactor(eval-view): fix typecheck and linting errors for preflight
micahjo7 Jul 31, 2026
2162702
perf(harness): parallelize missing suite summary backfill in upload_s…
micahjo7 Jul 31, 2026
b7be3a8
fix(eval-view): fix guide.js data loading and stats fields for browse…
micahjo7 Jul 31, 2026
f2f10c7
Merge branch 'main' into pr-1179
paulirish Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 39 additions & 19 deletions eval-view/generate-manifests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,28 +49,36 @@ export function generateMapping(outputDir = '.') {
* Generates suites.gen.json based on folders in harness/results that contain evals.json.
*/
export async function generateSuitesManifest(outputDir = '.', resultsSourceDir = resultsDir, skipFetch = false) {
const suites = new Set();
const { extractSuiteSummary } = await import('./summary-extractor.js');
/** @type {Map<string, Record<string, any>>} */
const suitesMap = new Map();
const outputPath = path.join(outputDir, 'suites.gen.json');

// 1. Fetch live manifest from GitHub Pages to preserve historical data
if (!skipFetch) {
const liveManifestUrl = 'https://googlechrome.github.io/guidance-dash/suites.gen.json';
console.log(`Fetching live manifest from ${liveManifestUrl}...`);
try {
const response = await fetch(liveManifestUrl);
if (response.ok) {
const liveSuites = await response.json();
if (Array.isArray(liveSuites)) {
console.log(`Fetched ${liveSuites.length} suites from live manifest.`);
liveSuites.forEach(s => suites.add(s));
const liveManifestUrl = 'https://googlechrome.github.io/guidance-dash/suites.gen.json';
console.log(`Fetching live manifest from ${liveManifestUrl}...`);
try {
const response = await fetch(liveManifestUrl);
if (response.ok) {
const liveSuites = await response.json();
if (Array.isArray(liveSuites)) {
console.log(`Fetched ${liveSuites.length} suites from live manifest.`);
liveSuites.forEach(s => {
if (typeof s === 'string') {
suitesMap.set(s, { testId: s, source: 'remote' });
} else if (s && s.testId) {
suitesMap.set(s.testId, s);
}
});
}
} else {
console.warn(`Failed to fetch live manifest (Status ${response.status}). Starting fresh.`);
}
} else {
console.warn(`Failed to fetch live manifest (Status ${response.status}). Starting fresh.`);
} catch (e) {
const message = e instanceof Error ? e.message : String(e);
console.warn(`Failed to fetch live manifest:`, message);
}
} catch (e) {
const message = e instanceof Error ? e.message : String(e);
console.warn(`Failed to fetch live manifest:`, message);
}
}

// 2. Add new suites from results dir
Expand All @@ -79,14 +87,26 @@ export async function generateSuitesManifest(outputDir = '.', resultsSourceDir =
for (const item of items) {
if (item.isDirectory()) {
const suiteDir = path.join(resultsSourceDir, item.name);
if (fs.existsSync(path.join(suiteDir, 'evals.json'))) {
suites.add(item.name);
const evalsPath = path.join(suiteDir, 'evals.json');
if (fs.existsSync(evalsPath)) {
try {
const rawData = JSON.parse(fs.readFileSync(evalsPath, 'utf8'));
const summary = extractSuiteSummary(item.name, rawData);
if (summary) {
suitesMap.set(item.name, summary);
}
} catch (e) {
console.warn(`Failed to parse evals.json for ${item.name}:`, e);
suitesMap.set(item.name, { testId: item.name, source: 'local' });
}
}
}
}
}

const sortedSuites = Array.from(suites).sort();
const sortedSuites = Array.from(suitesMap.values()).sort((a, b) => {
return (a.testId || '').localeCompare(b.testId || '');
});
fs.writeFileSync(outputPath, JSON.stringify(sortedSuites, null, 2));
return sortedSuites;
}
Expand Down
164 changes: 60 additions & 104 deletions eval-view/guide.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { getRunStats, initGoogleAuth, authenticatedFetch, getAccessToken, escapeHtml, parseResultKey, $ } from './utils.js';
import { initGoogleAuth, authenticatedFetch, getAccessToken, escapeHtml, $ } from './utils.js';
import { extractSuiteSummary } from './summary-extractor.js';

let allTestData = {}; // Cache all test data by testId

Expand Down Expand Up @@ -33,71 +34,27 @@ document.addEventListener('DOMContentLoaded', async () => {
}
});

function registerTestData(testId, source, parsed, forcedTimestamp) {
let serving = 'unknown';
if (parsed.serving !== undefined) {
serving = parsed.serving;
} else if (parsed.enableSkills !== undefined) {
serving = parsed.enableSkills ? 'skills' : 'mcp';
}

const compoundKey = `${testId}|||${source}`;

const guides = {};
if (parsed.results) {
Object.keys(parsed.results).forEach(key => {
const parsedKey = parseResultKey(key);
if (parsedKey) {
const { guide, runType } = parsedKey;
if (!guides[guide]) {
guides[guide] = {
guidedPassed: 0, guidedTotal: 0,
unguidedPassed: 0, unguidedTotal: 0
};
}
parsed.results[key].forEach(run => {
const s = getRunStats(run.results);
if (runType === 'guided') {
guides[guide].guidedPassed += s.passed;
guides[guide].guidedTotal += s.total;
} else if (runType === 'unguided') {
guides[guide].unguidedPassed += s.passed;
guides[guide].unguidedTotal += s.total;
}
});
}
});
}

// Convert guide raw totals into rates
const guidesWithRates = {};
Object.keys(guides).forEach(guide => {
const g = guides[guide];
const guidedRate = g.guidedTotal > 0 ? Math.round((g.guidedPassed / g.guidedTotal) * 100) : 0;
const unguidedRate = g.unguidedTotal > 0 ? Math.round((g.unguidedPassed / g.unguidedTotal) * 100) : 0;
guidesWithRates[guide] = {
guidedPassed: g.guidedPassed,
guidedTotal: g.guidedTotal,
guidedRate,
unguidedPassed: g.unguidedPassed,
unguidedTotal: g.unguidedTotal,
unguidedRate,
uplift: guidedRate - unguidedRate
};
});
function registerSuiteSummary(summary, source) {
const compoundKey = `${summary.testId}|||${source}`;

allTestData[compoundKey] = {
testId: testId,
timestamp: parsed.timestamp || forcedTimestamp || new Date().toISOString(),
data: parsed,
testId: summary.testId,
timestamp: summary.timestamp,
source: source,
agent: parsed.agent || 'unknown',
serving: serving,
model: parsed.model || 'unknown',
guides: guidesWithRates
agent: summary.agent || 'unknown',
serving: summary.serving || 'unknown',
model: summary.model || 'unknown',
guides: summary.guides || {}
};
}

function registerTestData(testId, source, parsed, forcedTimestamp) {
const summary = extractSuiteSummary(testId, parsed, forcedTimestamp);
if (summary) {
registerSuiteSummary(summary, source);
}
}

async function loadLocalTests() {
if (window.location.hostname !== 'localhost' && window.location.hostname !== '127.0.0.1') {
return;
Expand All @@ -111,27 +68,35 @@ async function loadLocalTests() {
if (!response.ok) {
const staticRes = await fetch(`/suites.gen.json?t=${Date.now()}`);
if (!staticRes.ok) return;
const suites = await staticRes.json();
manifest = { suites: suites.map(id => ({ id, source: 'local', timestamp: new Date().toISOString() })) };
const suitesData = await staticRes.json();
if (Array.isArray(suitesData)) {
manifest = { suites: suitesData };
} else {
manifest = suitesData;
}
useResultsPrefix = true;
} else {
manifest = await response.json();
const data = await response.json();
manifest = Array.isArray(data) ? { suites: data } : data;
}

for (const suite of manifest.suites) {
if (suite.source !== 'local') continue;

const testId = suite.id;
const suiteTimestamp = suite.timestamp;
try {
const fetchPath = useResultsPrefix ? `results/${testId}/evals.json` : `${testId}/evals.json`;
const response = await fetch(`${fetchPath}?source=local&t=${Date.now()}`);
if (response.ok) {
const parsed = await response.json();
registerTestData(testId, useResultsPrefix ? 'static' : 'local', parsed, suiteTimestamp);
if (typeof suite === 'object' && suite.testId && suite.guides) {
registerSuiteSummary(suite, useResultsPrefix ? 'static' : 'local');
} else {
const testId = typeof suite === 'string' ? suite : suite.id || suite.testId;
const suiteTimestamp = typeof suite === 'object' ? suite.timestamp : undefined;
if (!testId) continue;
try {
const fetchPath = useResultsPrefix ? `results/${testId}/evals.json` : `${testId}/evals.json`;
const response = await fetch(`${fetchPath}?source=local&t=${Date.now()}`);
if (response.ok) {
const parsed = await response.json();
registerTestData(testId, useResultsPrefix ? 'static' : 'local', parsed, suiteTimestamp);
}
} catch (e) {
console.warn(`Failed to load local test ${testId}:`, e);
}
} catch (e) {
console.warn(`Failed to load local test ${testId}:`, e);
}
}
} catch {
Expand All @@ -141,34 +106,18 @@ async function loadLocalTests() {

async function loadRemoteTests() {
try {
const prefixes = [];
let pageToken = '';

do {
const url = `https://storage.googleapis.com/storage/v1/b/guidance-evals/o?delimiter=/&t=${Date.now()}${pageToken ? `&pageToken=${pageToken}` : ''}`;
const response = await authenticatedFetch(url);
if (!response.ok) throw new Error('Failed to fetch remote suites');

const data = await response.json();
if (data.prefixes) {
prefixes.push(...data.prefixes);
}
pageToken = data.nextPageToken || '';
} while (pageToken);

await Promise.all(prefixes.map(async (prefix) => {
const testId = prefix.slice(0, -1);
try {
const fileUrl = `https://storage.googleapis.com/storage/v1/b/guidance-evals/o/${encodeURIComponent(prefix + 'evals.json')}?alt=media`;
const response = await authenticatedFetch(fileUrl);
if (response.ok) {
const parsed = await response.json();
registerTestData(testId, 'remote', parsed, null);
const fileUrl = `https://storage.googleapis.com/storage/v1/b/guidance-evals/o/${encodeURIComponent('suites.gen.json')}?alt=media&t=${Date.now()}`;
const response = await authenticatedFetch(fileUrl);
if (!response.ok) throw new Error('Failed to fetch remote suites manifest');

const manifest = await response.json();
if (Array.isArray(manifest) && manifest.length > 0) {
for (const item of manifest) {
if (item && item.testId) {
registerSuiteSummary(item, 'remote');
}
} catch (e) {
console.warn(`Failed to load remote test ${testId}:`, e);
}
}));
}
} catch (error) {
console.error('Error loading remote suites:', error);
}
Expand Down Expand Up @@ -324,7 +273,9 @@ function renderGraphs(guideName) {
const run = allTestData[key];
if (!run.guides || !run.guides[guideName]) return false;
const g = run.guides[guideName];
return g.guidedTotal > 0 || g.unguidedTotal > 0;
const gTotal = g.guidedTotal !== undefined ? g.guidedTotal : (g.guided?.total || 0);
const uTotal = g.unguidedTotal !== undefined ? g.unguidedTotal : (g.unguided?.total || 0);
return gTotal > 0 || uTotal > 0;
});

if (filteredKeys.length === 0) {
Expand Down Expand Up @@ -566,16 +517,21 @@ function renderGraphs(guideName) {
</div>
`;

const gPassed = stats.guidedPassed !== undefined ? stats.guidedPassed : (stats.guided?.passed || 0);
const gTotal = stats.guidedTotal !== undefined ? stats.guidedTotal : (stats.guided?.total || 0);
const uPassed = stats.unguidedPassed !== undefined ? stats.unguidedPassed : (stats.unguided?.passed || 0);
const uTotal = stats.unguidedTotal !== undefined ? stats.unguidedTotal : (stats.unguided?.total || 0);

content.innerHTML = `
<div style="color: var(--text-secondary); margin-bottom: 8px; font-size: 0.75rem;">${formattedDate}</div>
<div style="display: flex; flex-direction: column; gap: 4px;">
<div style="display: flex; justify-content: space-between;">
<span>Guided Pass Rate:</span>
<span style="font-weight: 600; color: var(--color-primary);">${stats.guidedRate}% (${stats.guidedPassed}/${stats.guidedTotal})</span>
<span style="font-weight: 600; color: var(--color-primary);">${stats.guidedRate}% (${gPassed}/${gTotal})</span>
</div>
<div style="display: flex; justify-content: space-between;">
<span>Unguided Pass Rate:</span>
<span style="font-weight: 600; color: var(--text-secondary);">${stats.unguidedRate}% (${stats.unguidedPassed}/${stats.unguidedTotal})</span>
<span style="font-weight: 600; color: var(--text-secondary);">${stats.unguidedRate}% (${uPassed}/${uTotal})</span>
</div>
<div style="display: flex; justify-content: space-between; border-top: 1px solid var(--border-color); padding-top: 4px; margin-top: 4px;">
<span style="font-weight: bold;">Uplift:</span>
Expand Down
Loading
Loading