From aec0baf1f4b9834ce7e86c0bebde2a15918c0f10 Mon Sep 17 00:00:00 2001 From: Ben Davis Date: Fri, 27 Feb 2026 21:06:48 -0800 Subject: [PATCH] fix(web): improve instance wake/provision diagnostics and retries --- apps/web/src/convex/instances/actions.ts | 83 ++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/apps/web/src/convex/instances/actions.ts b/apps/web/src/convex/instances/actions.ts index 84b7c773..367b61c8 100644 --- a/apps/web/src/convex/instances/actions.ts +++ b/apps/web/src/convex/instances/actions.ts @@ -204,6 +204,7 @@ const formatUserMessage = (operation: string, step: string | undefined, detail?: const stepLabel = step ? { load_resources: 'loading resources', + configure_git_auth: 'configuring git authentication', create_sandbox: 'creating the sandbox', get_sandbox: 'locating the sandbox', start_sandbox: 'starting the sandbox', @@ -220,6 +221,17 @@ const formatUserMessage = (operation: string, step: string | undefined, detail?: return `${base}${trimmed ? ` ${trimmed}` : ''} Please retry.`; }; +const getGitToken = () => process.env.BTCA_GIT_TOKEN?.trim(); + +async function configureSandboxGitAuth(sandbox: Sandbox): Promise { + const token = getGitToken(); + if (!token) return; + const basicToken = Buffer.from(`x-access-token:${token}`).toString('base64'); + await sandbox.process.executeCommand( + `git config --global http.https://github.com/.extraheader "AUTHORIZATION: basic ${basicToken}"` + ); +} + async function getResourceConfigs( ctx: ActionCtx, instanceId: Id<'instances'>, @@ -339,6 +351,21 @@ async function startBtcaServer(sandbox: Sandbox): Promise { } } +async function startBtcaServerWithRetry(sandbox: Sandbox, attempts = 2): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const result = await Result.tryPromise(() => startBtcaServer(sandbox)); + if (Result.isOk(result)) return result.value; + lastError = result.error; + if (attempt < attempts) { + await Result.tryPromise(() => + sandbox.process.executeCommand('pkill -f "btca serve" || true') + ); + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + async function stopSandboxIfRunning(sandbox: Sandbox): Promise { if (sandbox.state === 'started') { await sandbox.stop(60); @@ -421,11 +448,14 @@ export const provision = action({ ); sandbox = createdSandbox; + step = 'configure_git_auth'; + unwrapInstance(await withStep(step, () => configureSandboxGitAuth(createdSandbox))); + step = 'upload_config'; unwrapInstance(await withStep(step, () => uploadBtcaConfig(createdSandbox, resources))); step = 'start_btca'; - unwrapInstance(await withStep(step, () => startBtcaServer(createdSandbox))); + unwrapInstance(await withStep(step, () => startBtcaServerWithRetry(createdSandbox))); step = 'get_versions'; const versions = unwrapInstance( @@ -473,7 +503,24 @@ export const provision = action({ const errorDetails = getErrorDetails(error); const context = getErrorContext(error); const contextStep = typeof context?.step === 'string' ? context.step : step; - const message = formatUserMessage('provision', contextStep, errorDetails.message); + const health = context?.healthCheck as + | { attempts?: unknown; lastStatus?: unknown; lastError?: unknown } + | undefined; + const healthDetail = + contextStep === 'health_check' + ? `attempts=${String(health?.attempts ?? 'unknown')}, status=${String( + health?.lastStatus ?? 'none' + )}, error=${String(health?.lastError ?? 'none')}` + : undefined; + const logTail = typeof context?.btcaLogTail === 'string' ? context.btcaLogTail : undefined; + const detail = [ + errorDetails.message, + healthDetail, + logTail ? `log tail: ${logTail}` : undefined + ] + .filter(Boolean) + .join(' | '); + const message = formatUserMessage('provision', contextStep, detail); const durationMs = Date.now() - provisionStartedAt; console.error('Provisioning failed', { @@ -651,10 +698,13 @@ async function createSandboxFromScratch( ) ); + step = 'configure_git_auth'; + unwrapInstance(await withStep(step, () => configureSandboxGitAuth(sandbox))); + step = 'upload_config'; unwrapInstance(await withStep(step, () => uploadBtcaConfig(sandbox, resources))); step = 'start_btca'; - const serverUrl = unwrapInstance(await withStep(step, () => startBtcaServer(sandbox))); + const serverUrl = unwrapInstance(await withStep(step, () => startBtcaServerWithRetry(sandbox))); step = 'get_versions'; const versions = unwrapInstance(await withStep(step, () => getInstalledVersions(sandbox))); @@ -722,10 +772,12 @@ async function wakeInstanceInternal( step = 'start_sandbox'; unwrapInstance(await withStep(step, () => ensureSandboxStarted(sandbox))); + step = 'configure_git_auth'; + unwrapInstance(await withStep(step, () => configureSandboxGitAuth(sandbox))); step = 'upload_config'; unwrapInstance(await withStep(step, () => uploadBtcaConfig(sandbox, resources))); step = 'start_btca'; - serverUrl = unwrapInstance(await withStep(step, () => startBtcaServer(sandbox))); + serverUrl = unwrapInstance(await withStep(step, () => startBtcaServerWithRetry(sandbox))); sandboxId = instance.sandboxId; } @@ -750,7 +802,24 @@ async function wakeInstanceInternal( const errorDetails = getErrorDetails(error); const context = getErrorContext(error); const contextStep = typeof context?.step === 'string' ? context.step : step; - const message = formatUserMessage('wake', contextStep, errorDetails.message); + const health = context?.healthCheck as + | { attempts?: unknown; lastStatus?: unknown; lastError?: unknown } + | undefined; + const healthDetail = + contextStep === 'health_check' + ? `attempts=${String(health?.attempts ?? 'unknown')}, status=${String( + health?.lastStatus ?? 'none' + )}, error=${String(health?.lastError ?? 'none')}` + : undefined; + const logTail = typeof context?.btcaLogTail === 'string' ? context.btcaLogTail : undefined; + const detail = [ + errorDetails.message, + healthDetail, + logTail ? `log tail: ${logTail}` : undefined + ] + .filter(Boolean) + .join(' | '); + const message = formatUserMessage('wake', contextStep, detail); console.error('Wake failed', { instanceId, @@ -813,6 +882,8 @@ async function updateInstanceInternal( const wasRunning = unwrapInstance(await withStep(step, () => ensureSandboxStarted(sandbox))); await updatePackages(sandbox); + step = 'configure_git_auth'; + unwrapInstance(await withStep(step, () => configureSandboxGitAuth(sandbox))); step = 'upload_config'; unwrapInstance(await withStep(step, () => uploadBtcaConfig(sandbox, resources))); step = 'get_versions'; @@ -839,7 +910,7 @@ async function updateInstanceInternal( if (wasRunning) { await sandbox.process.executeCommand('pkill -f "btca serve" || true'); const serverUrl = unwrapInstance( - await withStep('start_btca', () => startBtcaServer(sandbox)) + await withStep('start_btca', () => startBtcaServerWithRetry(sandbox)) ); await ctx.runMutation(instanceMutations.setServerUrl, { instanceId, serverUrl }); await ctx.runMutation(instanceMutations.updateState, { instanceId, state: 'running' });