WIP POC: run next.js e2es

.gitignore

@@ -44,3 +44,5 @@ result-*
 .corepack/
 
 .vinext
+
+tests/fixtures-repos/*/clone/

packages/vinext/package.json

@@ -33,6 +33,10 @@
       "types": "./dist/server/prod-server.d.ts",
       "import": "./dist/server/prod-server.js"
     },
+    "./build/run-prerender": {
+      "types": "./dist/build/run-prerender.d.ts",
+      "import": "./dist/build/run-prerender.js"
+    },
     "./cloudflare": {
       "types": "./dist/cloudflare/index.d.ts",
       "import": "./dist/cloudflare/index.js"

packages/vinext/src/build/prerender.ts

@@ -25,10 +25,20 @@ import type { AppRoute } from "../routing/app-router.js";
 import type { ResolvedNextConfig } from "../config/next-config.js";
 import { classifyPagesRoute, classifyAppRoute } from "./report.js";
 import { createValidFileMatcher, type ValidFileMatcher } from "../routing/file-matcher.js";
-import { NoOpCacheHandler, setCacheHandler, getCacheHandler } from "../shims/cache.js";
+import { MemoryCacheHandler, setCacheHandler, getCacheHandler } from "../shims/cache.js";
 import { runWithHeadersContext, headersContextFromRequest } from "../shims/headers.js";
 import { startProdServer } from "../server/prod-server.js";
 import { readPrerenderSecret } from "./server-manifest.js";
+import {
+  consumePrerenderPageTags,
+  enablePrerenderTagCollection,
+} from "../server/app-page-cache.js";
+import {
+  consumePrerenderFetchMetrics,
+  disablePrerenderMetricsCollection,
+  enablePrerenderMetricsCollection,
+  type PrerenderFetchMetric,
+} from "../shims/fetch-cache.js";
 export { readPrerenderSecret } from "./server-manifest.js";
 
 // ─── Public Types ─────────────────────────────────────────────────────────────
@@ -53,6 +63,23 @@ export type PrerenderRouteResult =
       path?: string;
       /** Which router produced this route. Used by cache seeding. */
       router: "app" | "pages";
+      /**
+       * ISR cache tags for this route (path tags + unstable_cache tags).
+       * Written to vinext-prerender.json and used by seedMemoryCacheFromPrerender
+       * to ensure updateTag/revalidateTag also invalidates prerendered ISR entries.
+       */
+      tags?: string[];
+      /**
+       * HTTP status code of the prerendered response.
+       * Omitted when 200. Present for routes that rendered a not-found (404) page.
+       */
+      httpStatus?: number;
+      /**
+       * Fetch metrics collected during prerender for diagnostics.
+       * Written to vinext-prerender.json and used to generate
+       * .next/diagnostics/fetch-metrics.json for test compatibility.
+       */
+      fetchMetrics?: PrerenderFetchMetric[];
     }
   | {
       route: string;
@@ -372,7 +399,7 @@ export async function prerenderPages({
   }
 
   const previousHandler = getCacheHandler();
-  setCacheHandler(new NoOpCacheHandler());
+  setCacheHandler(new MemoryCacheHandler());
   process.env.VINEXT_PRERENDER = "1";
   // ownedProdServerHandle: a prod server we started ourselves and must close in finally.
   // When the caller passes options._prodServer we use that and do NOT close it.
@@ -686,11 +713,19 @@ export async function prerenderApp({
   fs.mkdirSync(outDir, { recursive: true });
 
   const previousHandler = getCacheHandler();
-  setCacheHandler(new NoOpCacheHandler());
+  // Use a fresh MemoryCacheHandler (not NoOpCacheHandler) for the prerender phase.
+  // This ensures ISR entries start empty (no stale carry-over from previous
+  // builds), while still allowing fetch deduplication within individual renders.
+  // NoOpCacheHandler disables dedup too, causing identical fetches on the same
+  // page to return different values (breaking W3C trace context dedup tests).
+  setCacheHandler(new MemoryCacheHandler());
   // VINEXT_PRERENDER=1 tells the prod server to skip instrumentation.register()
   // and enable prerender-only endpoints (/__vinext/prerender/*).
-  // The set/delete is wrapped in try/finally so it is always restored.
+  // NEXT_PHASE=phase-production-build matches Next.js behavior during static generation:
+  // pages can check process.env.NEXT_PHASE to call notFound() at build time.
+  // Both are set/deleted here and restored in the finally block.
   process.env.VINEXT_PRERENDER = "1";
+  process.env.NEXT_PHASE = "phase-production-build";
 
   const serverDir = path.dirname(rscBundlePath);
 
@@ -969,6 +1004,14 @@ export async function prerenderApp({
 
     // ── Render each URL via direct RSC handler invocation ─────────────────────
 
+    // Enable tag collection so that renderAppPageLifecycle() records per-page
+    // unstable_cache tags into the global sidecar. consumePrerenderPageTags()
+    // reads and clears these after each render so they end up in the manifest.
+    enablePrerenderTagCollection();
+    // Enable fetch metrics collection so that every cacheable fetch during
+    // prerender records a diagnostic entry keyed by navPathname.
+    enablePrerenderMetricsCollection();
+
     /**
      * Render a single URL and return its result.
      * `onProgress` is intentionally not called here; the outer loop calls it
@@ -995,40 +1038,77 @@ export async function prerenderApp({
         const htmlRes = await runWithHeadersContext(headersContextFromRequest(htmlRequest), () =>
           rscHandler(htmlRequest),
         );
-        if (!htmlRes.ok) {
+        const httpStatus = htmlRes.status;
+
+        // Non-ok responses that are not 404 are errors (or skips for speculative routes).
+        if (!htmlRes.ok && httpStatus !== 404) {
+          consumePrerenderPageTags(urlPath);
           if (isSpeculative) {
             return { route: routePattern, status: "skipped", reason: "dynamic" };
           }
           return {
             route: routePattern,
             status: "error",
-            error: `RSC handler returned ${htmlRes.status}`,
+            error: `RSC handler returned ${httpStatus}`,
           };
         }
 
-        // Detect dynamic usage for speculative routes via Cache-Control header.
-        // When headers(), cookies(), connection(), or noStore() are called during
-        // render, the server sets Cache-Control: no-store. We treat this as a
-        // signal that the route is dynamic and should be skipped.
-        if (isSpeculative) {
+        // Speculative route that returned 404: not a static page.
+        if (isSpeculative && !htmlRes.ok) {
+          consumePrerenderPageTags(urlPath);
+          return { route: routePattern, status: "skipped", reason: "dynamic" };
+        }
+
+        // Detect dynamic usage via Cache-Control: no-store in the render response.
+        // When headers(), cookies(), noStore(), or a no-store/revalidate:0 fetch
+        // is called during render, the server sets Cache-Control: no-store.
+        // We treat this as a signal that the route is dynamic and should not be
+        // seeded into the ISR cache — even for non-speculative routes (those with
+        // an explicit generateStaticParams) whose fetches indicate dynamic data.
+        {
           const cacheControl = htmlRes.headers.get("cache-control") ?? "";
           if (cacheControl.includes("no-store")) {
+            const dynamicReason = htmlRes.headers.get("x-vinext-dynamic-reason");
             await htmlRes.body?.cancel();
+            consumePrerenderPageTags(urlPath);
+            if (dynamicReason) {
+              console.log(
+                `Static generation failed due to dynamic usage on ${urlPath}, reason: ${dynamicReason}`,
+              );
+            }
             return { route: routePattern, status: "skipped", reason: "dynamic" };
           }
         }
 
         const html = await htmlRes.text();
 
+        // Capture ISR cache tags registered during this render.
+        // Rewrite the hierarchy tags (_N_T_/.../layout and .../page) to use the
+        // route pattern instead of the concrete URL. This matches Next.js which
+        // uses revalidatePath('/blog/[slug]', 'page') to target the pattern tag.
+        const rawPageTags = consumePrerenderPageTags(urlPath);
+        const pageTags = rewriteTagsToRoutePattern(rawPageTags, urlPath, routePattern);
+        const fetchMetrics = consumePrerenderFetchMetrics(urlPath);
+
         // Fetch RSC payload via a second invocation with RSC headers
         // TODO: Extract RSC payload from the first response instead of invoking the handler twice.
-        const rscRequest = new Request(`http://localhost${urlPath}`, {
-          headers: { Accept: "text/x-component", RSC: "1" },
-        });
-        const rscRes = await runWithHeadersContext(headersContextFromRequest(rscRequest), () =>
-          rscHandler(rscRequest),
-        );
-        const rscData = rscRes.ok ? await rscRes.text() : null;
+        // Skip RSC fetch for 404 pages - they don't have a meaningful RSC payload.
+        let rscData: string | null = null;
+        if (httpStatus === 200) {
+          const rscRequest = new Request(`http://localhost${urlPath}`, {
+            headers: { Accept: "text/x-component", RSC: "1" },
+          });
+          const rscRes = await runWithHeadersContext(headersContextFromRequest(rscRequest), () =>
+            rscHandler(rscRequest),
+          );
+          rscData = rscRes.ok ? await rscRes.text() : null;
+        }
+        // Clear any sidecar entry left by the RSC-only request.
+        consumePrerenderPageTags(urlPath);
+        // Discard any fetch metrics recorded during the RSC-only request — we
+        // only want metrics from the HTML render (first pass) which mirrors what
+        // Next.js logs in its diagnostics output (one entry per unique fetch).
+        consumePrerenderFetchMetrics(urlPath);
 
         const outputFiles: string[] = [];
 
@@ -1055,6 +1135,9 @@ export async function prerenderApp({
           revalidate,
           router: "app",
           ...(urlPath !== routePattern ? { path: urlPath } : {}),
+          ...(pageTags.length > 0 ? { tags: pageTags } : {}),
+          ...(httpStatus !== 200 ? { httpStatus } : {}),
+          ...(fetchMetrics.length > 0 ? { fetchMetrics } : {}),
         };
       } catch (e) {
         if (isSpeculative) {
@@ -1117,6 +1200,8 @@ export async function prerenderApp({
   } finally {
     setCacheHandler(previousHandler);
     delete process.env.VINEXT_PRERENDER;
+    delete process.env.NEXT_PHASE;
+    disablePrerenderMetricsCollection();
     if (ownedProdServerHandle) {
       await new Promise<void>((resolve) => ownedProdServerHandle!.server.close(() => resolve()));
     }
@@ -1133,6 +1218,90 @@ export function getRscOutputPath(urlPath: string): string {
   return urlPath.replace(/^\//, "") + ".rsc";
 }
 
+// ─── Tag helpers ─────────────────────────────────────────────────────────────
+
+/**
+ * Convert an Express-style route pattern segment to Next.js bracket style.
+ *
+ * - `:slug`    → `[slug]`
+ * - `:slug+`   → `[...slug]`
+ * - `:slug*`   → `[[...slug]]`
+ */
+function expressSegToNextJs(seg: string): string {
+  const catchAll = seg.match(/^:(.+)\+$/);
+  if (catchAll) return `[...${catchAll[1]}]`;
+  const optionalCatchAll = seg.match(/^:(.+)\*$/);
+  if (optionalCatchAll) return `[[...${optionalCatchAll[1]}]]`;
+  const dynamic = seg.match(/^:(.+)$/);
+  if (dynamic) return `[${dynamic[1]}]`;
+  return seg;
+}
+
+/**
+ * Convert an Express-style route pattern (e.g. `/blog/:slug`) to Next.js
+ * file-system bracket notation (e.g. `/blog/[slug]`).
+ *
+ * vinext stores route patterns in Express style internally; Next.js tag names
+ * (`_N_T_`) use the bracket notation so revalidatePath('/blog/[slug]', 'page')
+ * targets the right cache entries.
+ */
+function expressPatternToNextJs(pattern: string): string {
+  return "/" + pattern.split("/").filter(Boolean).map(expressSegToNextJs).join("/");
+}
+
+/**
+ * Rewrite the _N_T_ hierarchy tags in `tags` to use the route pattern instead
+ * of the concrete URL path for the layout/page segment identifiers.
+ *
+ * Next.js uses the route pattern (e.g. `/blog/[slug]`) for revalidatePath()
+ * target matching, so `_N_T_/blog/[slug]/page` is the canonical tag rather
+ * than `_N_T_/blog/hello-world/page`. We post-process the tags emitted by
+ * `__pageCacheTags()` (which uses the concrete path) to match this behaviour.
+ *
+ * Result order:
+ *   [bare_pathname, ...hierarchy_tags_using_pattern, instance_tag, ...fetch_tags]
+ *
+ * @param tags         Tags from getPageTags() / consumePrerenderPageTags()
+ * @param urlPath      Concrete prerendered URL, e.g. `/blog/hello-world`
+ * @param routePattern Route file-system pattern in Express style, e.g. `/blog/:slug`
+ */
+export function rewriteTagsToRoutePattern(
+  tags: string[],
+  urlPath: string,
+  routePattern: string,
+): string[] {
+  if (urlPath === routePattern) return tags;
+
+  // Convert Express-style pattern to Next.js bracket style for tag names
+  const nextJsPattern = expressPatternToNextJs(routePattern);
+
+  // Identify concrete-path hierarchy tags generated by __pageCacheTags
+  const concreteBareSegs = urlPath.split("/").filter(Boolean);
+  const oldHierarchy = new Set<string>(["_N_T_/layout"]);
+  let cBuilt = "";
+  for (const seg of concreteBareSegs) {
+    cBuilt += "/" + seg;
+    oldHierarchy.add(`_N_T_${cBuilt}/layout`);
+  }
+  oldHierarchy.add(`_N_T_${cBuilt}/page`);
+
+  // Build hierarchy tags from the Next.js-style route pattern
+  const patternSegs = nextJsPattern.split("/").filter(Boolean);
+  const newHierarchy: string[] = ["_N_T_/layout"];
+  let pBuilt = "";
+  for (const seg of patternSegs) {
+    pBuilt += "/" + seg;
+    newHierarchy.push(`_N_T_${pBuilt}/layout`);
+  }
+  newHierarchy.push(`_N_T_${pBuilt}/page`);
+
+  // Separate remaining tags: bare pathname, instance tag, explicit fetch tags
+  const instanceTag = `_N_T_${urlPath}`;
+  const fetchTags = tags.filter((t) => !oldHierarchy.has(t) && t !== urlPath && t !== instanceTag);
+
+  return [urlPath, ...newHierarchy, instanceTag, ...fetchTags];
+}
+
 // ─── Build index ──────────────────────────────────────────────────────────────
 
 /**
@@ -1157,6 +1326,9 @@ export function writePrerenderIndex(
         revalidate: r.revalidate,
         router: r.router,
         ...(r.path ? { path: r.path } : {}),
+        ...(r.tags && r.tags.length > 0 ? { tags: r.tags } : {}),
+        ...(r.httpStatus && r.httpStatus !== 200 ? { httpStatus: r.httpStatus } : {}),
+        ...(r.fetchMetrics && r.fetchMetrics.length > 0 ? { fetchMetrics: r.fetchMetrics } : {}),
       };
     }
     if (r.status === "skipped") {

packages/vinext/src/build/run-prerender.ts

@@ -29,6 +29,7 @@ import {
   readPrerenderSecret,
 } from "./prerender.js";
 import { loadNextConfig, resolveNextConfig } from "../config/next-config.js";
+import { readBuildId } from "./server-manifest.js";
 import { pagesRouter, apiRouter } from "../routing/pages-router.js";
 import { appRouter } from "../routing/app-router.js";
 import { findDir } from "./report.js";
@@ -93,6 +94,19 @@ export type RunPrerenderOptions = {
    * Intended for tests that build to a custom outDir.
    */
   rscBundlePath?: string;
+  /**
+   * Override the output directory where prerendered HTML/RSC files are written.
+   * Defaults to `<root>/dist/server/prerendered-routes` (non-export) or
+   * `<root>/dist/client` (export).
+   * Intended for tests that build to a custom outDir.
+   */
+  outDir?: string;
+  /**
+   * Override the directory where `vinext-prerender.json` manifest is written.
+   * Defaults to `<root>/dist/server`.
+   * Intended for tests that build to a custom outDir.
+   */
+  manifestDir?: string;
 };
 
 /**
@@ -126,16 +140,28 @@ export async function runPrerender(options: RunPrerenderOptions): Promise<Preren
 
   // The manifest lands in dist/server/ alongside the server bundle so it's
   // cleaned by Vite's emptyOutDir on rebuild and co-located with server artifacts.
-  const manifestDir = path.join(root, "dist", "server");
+  const manifestDir = options.manifestDir ?? path.join(root, "dist", "server");
 
   const loadedConfig = await resolveNextConfig(await loadNextConfig(root), root);
-  const config = options.nextConfigOverride
-    ? { ...loadedConfig, ...options.nextConfigOverride }
-    : // Note: shallow merge — nested keys like `images` or `i18n` in
-      // nextConfigOverride replace the entire nested object from loadedConfig.
-      // This is intentional for test usage (top-level overrides only); a deep
-      // merge would be needed to support partial nested overrides in the future.
-      loadedConfig;
+
+  // When `rscBundlePath` is provided, read the buildId that was baked into
+  // the compiled bundle from `vinext-server.json`. This ensures that ISR cache
+  // keys written to the prerender manifest match the keys the prod server uses
+  // at request time (both derived from the same buildId). Without this, a fresh
+  // call to resolveNextConfig would generate a new random UUID that wouldn't
+  // match the compile-time define injected into the RSC bundle.
+  const compiledBuildId = options.rscBundlePath
+    ? readBuildId(path.dirname(options.rscBundlePath))
+    : undefined;
+
+  const config =
+    options.nextConfigOverride || compiledBuildId
+      ? {
+          ...loadedConfig,
+          ...(compiledBuildId ? { buildId: compiledBuildId } : {}),
+          ...options.nextConfigOverride,
+        }
+      : loadedConfig;
   // Activate export mode when next.config.js sets `output: 'export'`.
   // In export mode, SSR routes and any dynamic routes without static params are
   // build errors rather than silently skipped.
@@ -160,9 +186,10 @@ export async function runPrerender(options: RunPrerenderOptions): Promise<Preren
   // output: 'export' builds use dist/client/ (handled by static-export.ts which
   // passes its own outDir — this path is only reached for non-export builds).
   const outDir =
-    mode === "export"
+    options.outDir ??
+    (mode === "export"
       ? path.join(root, "dist", "client")
-      : path.join(root, "dist", "server", "prerendered-routes");
+      : path.join(root, "dist", "server", "prerendered-routes"));
 
   const rscBundlePath = options.rscBundlePath ?? path.join(root, "dist", "server", "index.js");
   const serverDir = path.dirname(rscBundlePath);