Add instance/job diagnostics to SERVER_BUSY responses

Author: richiemcilroy

apps/media-server/src/routes/video.ts

@@ -63,6 +63,10 @@ const processSchema = z.object({
 	remuxOnly: z.boolean().optional(),
 });
 
+function getInstanceId(): string {
+	return process.env.HOSTNAME || `pid-${process.pid}`;
+}
+
 function isBusyError(err: unknown): boolean {
 	return err instanceof Error && err.message.includes("Server is busy");
 }
@@ -74,6 +78,8 @@ function isTimeoutError(err: unknown): boolean {
 video.get("/status", (c) => {
 	const jobs = getAllJobs();
 	return c.json({
+		instanceId: getInstanceId(),
+		pid: process.pid,
 		activeVideoProcesses: getActiveVideoProcessCount(),
 		maxConcurrentVideoProcesses: getMaxConcurrentVideoProcesses(),
 		activeProbeProcesses: getActiveProbeProcessCount(),
@@ -234,13 +240,24 @@ video.post("/process", async (c) => {
 	if (!canAcceptNewVideoProcess()) {
 		const activeVideoProcesses = getActiveVideoProcessCount();
 		const maxConcurrentVideoProcesses = getMaxConcurrentVideoProcesses();
+		const jobs = getAllJobs();
 		return c.json(
 			{
 				error: "Server is busy",
 				code: "SERVER_BUSY",
 				details: `Too many concurrent video processing jobs (${activeVideoProcesses}/${maxConcurrentVideoProcesses}), please retry later`,
+				instanceId: getInstanceId(),
+				pid: process.pid,
 				activeVideoProcesses,
 				maxConcurrentVideoProcesses,
+				jobCount: jobs.length,
+				jobs: jobs.map((job) => ({
+					jobId: job.jobId,
+					videoId: job.videoId,
+					phase: job.phase,
+					progress: job.progress,
+					updatedAt: job.updatedAt,
+				})),
 			},
 			503,
 		);

apps/web/workflows/process-video.ts

@@ -155,11 +155,35 @@ async function startMediaServerProcessJob(
 			error?: string;
 			code?: string;
 			details?: string;
+			instanceId?: string;
+			pid?: number;
+			activeVideoProcesses?: number;
+			maxConcurrentVideoProcesses?: number;
+			jobCount?: number;
 		};
-		const errorMessage =
+		const baseErrorMessage =
 			errorData.error ||
 			errorData.details ||
 			"Video processing failed to start";
+		const busyDiagnostics =
+			errorData.code === "SERVER_BUSY"
+				? [
+						errorData.instanceId ? `instance=${errorData.instanceId}` : null,
+						typeof errorData.pid === "number" ? `pid=${errorData.pid}` : null,
+						typeof errorData.activeVideoProcesses === "number" &&
+						typeof errorData.maxConcurrentVideoProcesses === "number"
+							? `active=${errorData.activeVideoProcesses}/${errorData.maxConcurrentVideoProcesses}`
+							: null,
+						typeof errorData.jobCount === "number"
+							? `jobCount=${errorData.jobCount}`
+							: null,
+					]
+						.filter(Boolean)
+						.join(", ")
+				: "";
+		const errorMessage = busyDiagnostics
+			? `${baseErrorMessage} (${busyDiagnostics})`
+			: baseErrorMessage;
 		const shouldRetry =
 			response.status === 503 &&
 			(errorData.code === "SERVER_BUSY" ||