fix(actions): retry 504 and time out fetches in dispatch-run

.github/actions/test-system-io-dispatch-run/dist/index.js

@@ -24540,6 +24540,15 @@ function getOctokit(token, options, ...additionalPlugins) {
 
 // src/auth.ts
 var TOKEN_REFRESH_AGE_MS = 5 * 60 * 1e3;
+var TRANSIENT_HTTP_STATUS = /* @__PURE__ */ new Set([408, 429, 502, 503, 504]);
+var JSON_REQUEST_TIMEOUT_MS = 3e4;
+var UPLOAD_REQUEST_TIMEOUT_MS = 12e4;
+function isTransientHTTPStatus(status) {
+  return TRANSIENT_HTTP_STATUS.has(status);
+}
+function timeoutSignal(ms) {
+  return AbortSignal.timeout(ms);
+}
 var cachedToken = null;
 var cachedTokenMintedAt = 0;
 function invalidateToken() {
@@ -24560,22 +24569,48 @@ async function getBearer(audience) {
 function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
+function parseRetryAfterMs(res) {
+  const raw = res.headers.get("retry-after");
+  if (!raw) {
+    return 0;
+  }
+  const asSeconds = Number(raw);
+  if (Number.isFinite(asSeconds) && asSeconds >= 0) {
+    return asSeconds * 1e3;
+  }
+  const asDate = Date.parse(raw);
+  if (!Number.isNaN(asDate)) {
+    return Math.max(0, asDate - Date.now());
+  }
+  return 0;
+}
 async function fetchWithRetry(makeRequest, attempts = 4) {
   let lastErr;
+  let lastRes;
   for (let i = 0; i < attempts; i++) {
     try {
-      return await makeRequest();
+      const res = await makeRequest();
+      if (!TRANSIENT_HTTP_STATUS.has(res.status) || i === attempts - 1) {
+        return res;
+      }
+      lastRes = res;
+      const backoffMs = 500 * 2 ** i;
+      const delayMs = Math.max(backoffMs, parseRetryAfterMs(res));
+      await res.text().catch(() => void 0);
+      info(`HTTP ${res.status} from upstream; retrying in ${delayMs}ms`);
+      await sleep(delayMs);
     } catch (err) {
       lastErr = err;
       const e = err;
       const code = e?.cause?.code ?? e?.code;
-      const retryable = e?.name === "TypeError" || code === "UND_ERR_SOCKET" || code === "ECONNRESET" || code === "ETIMEDOUT" || code === "EAI_AGAIN" || code === "ENOTFOUND";
+      const retryable = e?.name === "TypeError" || e?.name === "TimeoutError" || e?.name === "AbortError" || code === "UND_ERR_SOCKET" || code === "UND_ERR_HEADERS_TIMEOUT" || code === "UND_ERR_BODY_TIMEOUT" || code === "ECONNRESET" || code === "ETIMEDOUT" || code === "EAI_AGAIN" || code === "ENOTFOUND";
       if (!retryable || i === attempts - 1) throw err;
       const delayMs = 500 * 2 ** i;
       info(`fetch transient failure (${code ?? e.name}); retrying in ${delayMs}ms`);
       await sleep(delayMs);
     }
   }
+  if (lastRes) return lastRes;
   throw lastErr;
 }
 async function fetchWithAuthRetry(makeRequest) {
@@ -24953,7 +24988,8 @@ async function uploadMultipart(cfg, urlPath, parts, defaultType) {
     return fetch(`${cfg.baseURL}${urlPath}`, {
       method: "POST",
       headers: { Authorization: `Bearer ${bearer}` },
-      body: form
+      body: form,
+      signal: timeoutSignal(UPLOAD_REQUEST_TIMEOUT_MS)
     });
   });
   if (res.status !== 200) {
@@ -25010,7 +25046,8 @@ async function postJSON(cfg, urlPath, body) {
     return fetch(`${cfg.baseURL}${urlPath}`, {
       method: "POST",
       headers: { "Content-Type": "application/json", Authorization: `Bearer ${bearer}` },
-      body: JSON.stringify(body)
+      body: JSON.stringify(body),
+      signal: timeoutSignal(JSON_REQUEST_TIMEOUT_MS)
     });
   });
   const text = await res.text();
@@ -25244,11 +25281,12 @@ async function postJSON2(cfg, urlPath, body) {
       return fetch(`${cfg.baseURL}${urlPath}`, {
         method: "POST",
         headers: { "Content-Type": "application/json", Authorization: `Bearer ${bearer}` },
-        body: JSON.stringify(body)
+        body: JSON.stringify(body),
+        signal: timeoutSignal(JSON_REQUEST_TIMEOUT_MS)
       });
     });
     const text = await res.text();
-    if (res.status >= 500 && res.status < 600 && attempt < delays.length) {
+    if (res.status >= 500 && res.status < 600 && !isTransientHTTPStatus(res.status) && attempt < delays.length) {
       const ms = delays[attempt] + Math.floor(Math.random() * 250);
       warning(
         `${urlPath}: HTTP ${res.status} (attempt ${attempt + 1}/${delays.length + 1}); retrying in ${ms}ms. body=${text.slice(0, 200)}`

.github/actions/test-system-io-dispatch-run/src/auth.ts

@@ -5,14 +5,38 @@
  * drains a queue for an hour cannot mint once and reuse — the bearer
  * expires mid-loop and the next /complete returns 401. We mint on
  * demand, cache for a few minutes, and on a 401 we invalidate and
- * retry once. Pure transient-network failures get an exponential
- * backoff on top.
+ * retry once. Pure transient-network failures, request timeouts, and
+ * transient HTTP statuses (notably 504 gateway timeouts on the
+ * end-of-worker shard upload) get exponential backoff on top.
  */
 
 import * as core from "@actions/core";
 
 const TOKEN_REFRESH_AGE_MS = 5 * 60 * 1000;
 
+/** HTTP statuses that are safe to retry (gateway blips / rate limits). */
+const TRANSIENT_HTTP_STATUS = new Set([408, 429, 502, 503, 504]);
+
+/** Request timeout for JSON control calls (checkout / complete / register). */
+export const JSON_REQUEST_TIMEOUT_MS = 30_000;
+
+/** Request timeout for multipart shard uploads — generous since large
+ *  screenshot batches stream over a slow runner uplink. */
+export const UPLOAD_REQUEST_TIMEOUT_MS = 120_000;
+
+/** Reports whether a status is one the fetch wrappers retry automatically, so
+ *  call-site retry loops can avoid double-retrying the same statuses. */
+export function isTransientHTTPStatus(status: number): boolean {
+  return TRANSIENT_HTTP_STATUS.has(status);
+}
+
+/** AbortSignal that fires after ms — attach to fetch so a hung request fails
+ *  fast (and is retried) instead of waiting on the load balancer's idle
+ *  timeout and surfacing as an opaque 504. */
+export function timeoutSignal(ms: number): AbortSignal {
+  return AbortSignal.timeout(ms);
+}
+
 let cachedToken: string | null = null;
 let cachedTokenMintedAt = 0;
 
@@ -39,28 +63,65 @@ function sleep(ms: number): Promise<void> {
   return new Promise((r) => setTimeout(r, ms));
 }
 
+/**
+ * Parse Retry-After (delay-seconds or HTTP-date) into milliseconds.
+ * Returns 0 when absent or unparseable.
+ */
+function parseRetryAfterMs(res: Response): number {
+  const raw = res.headers.get("retry-after");
+  if (!raw) {
+    return 0;
+  }
+  const asSeconds = Number(raw);
+  if (Number.isFinite(asSeconds) && asSeconds >= 0) {
+    return asSeconds * 1000;
+  }
+  const asDate = Date.parse(raw);
+  if (!Number.isNaN(asDate)) {
+    return Math.max(0, asDate - Date.now());
+  }
+  return 0;
+}
+
 /**
  * Retry transient network failures (idle keep-alive sockets closed by a
- * load balancer during a long playwright run, brief DNS hiccups, etc.).
- * Only retries on connection-level errors thrown by fetch — HTTP
- * non-2xx responses are returned to the caller verbatim so business
- * errors (e.g. RUN_NOT_IN_PROGRESS) aren't silently masked.
+ * load balancer during a long playwright run, brief DNS hiccups, request
+ * timeouts) and transient HTTP statuses (notably 504 gateway timeouts on
+ * the end-of-worker multipart shard upload). Other HTTP non-2xx responses
+ * are returned to the caller verbatim so business errors (e.g.
+ * RUN_NOT_IN_PROGRESS, WORKER_HAS_ACTIVE_LEASE) aren't silently masked.
  */
 async function fetchWithRetry(
   makeRequest: () => Promise<Response>,
   attempts = 4,
 ): Promise<Response> {
   let lastErr: unknown;
+  let lastRes: Response | undefined;
+
   for (let i = 0; i < attempts; i++) {
     try {
-      return await makeRequest();
+      const res = await makeRequest();
+      if (!TRANSIENT_HTTP_STATUS.has(res.status) || i === attempts - 1) {
+        return res;
+      }
+      lastRes = res;
+      const backoffMs = 500 * 2 ** i;
+      const delayMs = Math.max(backoffMs, parseRetryAfterMs(res));
+      // Drain so the connection can be reused on the next attempt.
+      await res.text().catch(() => undefined);
+      core.info(`HTTP ${res.status} from upstream; retrying in ${delayMs}ms`);
+      await sleep(delayMs);
     } catch (err) {
       lastErr = err;
       const e = err as { name?: string; code?: string; cause?: { code?: string } };
       const code = e?.cause?.code ?? e?.code;
       const retryable =
         e?.name === "TypeError" /* node fetch wraps net errors here */ ||
+        e?.name === "TimeoutError" /* AbortSignal.timeout fired */ ||
+        e?.name === "AbortError" ||
         code === "UND_ERR_SOCKET" ||
+        code === "UND_ERR_HEADERS_TIMEOUT" ||
+        code === "UND_ERR_BODY_TIMEOUT" ||
         code === "ECONNRESET" ||
         code === "ETIMEDOUT" ||
         code === "EAI_AGAIN" ||
@@ -71,6 +132,8 @@ async function fetchWithRetry(
       await sleep(delayMs);
     }
   }
+
+  if (lastRes) return lastRes;
   throw lastErr;
 }
 

.github/actions/test-system-io-dispatch-run/src/main.ts

@@ -13,7 +13,13 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 import * as core from "@actions/core";
 import * as github from "@actions/github";
-import { fetchWithAuthRetry, getBearer } from "./auth";
+import {
+  JSON_REQUEST_TIMEOUT_MS,
+  fetchWithAuthRetry,
+  getBearer,
+  isTransientHTTPStatus,
+  timeoutSignal,
+} from "./auth";
 import { runUnit as runPlaywrightUnit } from "./playwright";
 import { runUnit as runCypressUnit } from "./cypress";
 import { uploadShard, type UploadConfig } from "./upload";
@@ -353,10 +359,20 @@ async function postJSON<T>(
         method: "POST",
         headers: { "Content-Type": "application/json", Authorization: `Bearer ${bearer}` },
         body: JSON.stringify(body),
+        signal: timeoutSignal(JSON_REQUEST_TIMEOUT_MS),
       });
     });
     const text = await res.text();
-    if (res.status >= 500 && res.status < 600 && attempt < delays.length) {
+    // fetchWithAuthRetry already backs off on transient gateway statuses
+    // (408/429/502/503/504); only re-loop here for the other 5xx (500/501/…)
+    // so an idempotent /checkout or /complete still survives a backend blip
+    // without double-retrying the statuses the auth layer just exhausted.
+    if (
+      res.status >= 500 &&
+      res.status < 600 &&
+      !isTransientHTTPStatus(res.status) &&
+      attempt < delays.length
+    ) {
       const ms = delays[attempt]! + Math.floor(Math.random() * 250);
       core.warning(
         `${urlPath}: HTTP ${res.status} (attempt ${attempt + 1}/${delays.length + 1}); ` +

.github/actions/test-system-io-dispatch-run/src/upload.ts

@@ -11,7 +11,13 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import * as core from "@actions/core";
-import { fetchWithAuthRetry, getBearer } from "./auth";
+import {
+  JSON_REQUEST_TIMEOUT_MS,
+  UPLOAD_REQUEST_TIMEOUT_MS,
+  fetchWithAuthRetry,
+  getBearer,
+  timeoutSignal,
+} from "./auth";
 import type {
   CompositeIdentity,
   InvocationRecord,
@@ -121,6 +127,7 @@ async function uploadMultipart(
       method: "POST",
       headers: { Authorization: `Bearer ${bearer}` },
       body: form,
+      signal: timeoutSignal(UPLOAD_REQUEST_TIMEOUT_MS),
     });
   });
   if (res.status !== 200) {
@@ -191,6 +198,7 @@ async function postJSON<T>(
       method: "POST",
       headers: { "Content-Type": "application/json", Authorization: `Bearer ${bearer}` },
       body: JSON.stringify(body),
+      signal: timeoutSignal(JSON_REQUEST_TIMEOUT_MS),
     });
   });
   const text = await res.text();