diff --git a/.changeset/capability-aware-connection-diagnostics.md b/.changeset/capability-aware-connection-diagnostics.md new file mode 100644 index 000000000..27bdf84dd --- /dev/null +++ b/.changeset/capability-aware-connection-diagnostics.md @@ -0,0 +1,10 @@ +--- +"@open-codesign/desktop": minor +"@open-codesign/shared": minor +"@open-codesign/providers": patch +"@open-codesign/i18n": patch +--- + +Upgrade connection diagnostics from pass/fail connectivity to layered capability classification (authentication, endpoint shape, wire support, model discovery, role compatibility, reasoning compatibility). Missing `/models` with working inference is now `degraded-compatible`, and responses vs chat/completions mismatches are reported as structured reasons the renderer can show. + +Fixes #213 diff --git a/apps/desktop/src/main/connection-ipc.test.ts b/apps/desktop/src/main/connection-ipc.test.ts index 5a101f81b..621153fac 100644 --- a/apps/desktop/src/main/connection-ipc.test.ts +++ b/apps/desktop/src/main/connection-ipc.test.ts @@ -923,7 +923,7 @@ describe('runProviderTest degrade-probe (issue #179)', () => { expect(res.ok).toBe(true); if (res.ok) { expect(res.probeMethod).toBe('chat_completion_degraded'); - expect(res.compatibility).toBe('degraded'); + expect(res.compatibility).toBe('degraded-compatible'); expect(res.reasonCategory).toBe('model-discovery-degraded'); } expect(calls).toHaveLength(2); @@ -1021,7 +1021,7 @@ describe('runProviderTest degrade-probe (issue #179)', () => { } }); - it('openai-chat: /models 200 → no degrade probe, probeMethod=models', async () => { + it('openai-chat: /models 200 still reports probeMethod=models after selected-wire verification', async () => { const { calls, restore } = installFakeFetch(() => ({ status: 200, body: { data: [] } })); try { const res = await runProviderTest({ @@ -1034,9 +1034,17 @@ describe('runProviderTest degrade-probe (issue #179)', () => { if (res.ok) { expect(res.probeMethod).toBe('models'); expect(res.compatibility).toBe('compatible'); + expect(res.reasons?.some((r) => r.layer === 'authentication' && r.status === 'pass')).toBe( + true, + ); + expect(res.reasons?.some((r) => r.layer === 'wire-support' && r.status === 'pass')).toBe( + true, + ); } - expect(calls).toHaveLength(1); + expect(calls).toHaveLength(2); expect(calls[0]?.method).toBe('GET'); + expect(calls[1]?.method).toBe('POST'); + expect(calls[1]?.url).toMatch(/\/chat\/completions$/); } finally { restore(); } @@ -1084,7 +1092,7 @@ describe('runProviderTest degrade-probe (issue #179)', () => { expect(res.ok).toBe(true); if (res.ok) { expect(res.probeMethod).toBe('anthropic_messages_degraded'); - expect(res.compatibility).toBe('degraded'); + expect(res.compatibility).toBe('degraded-compatible'); } expect(calls).toHaveLength(2); expect(calls[0]?.url).toMatch(/\/v1\/models$/); @@ -1138,7 +1146,7 @@ describe('runProviderTest degrade-probe (issue #179)', () => { expect(res.ok).toBe(true); if (res.ok) { expect(res.probeMethod).toBe('responses_degraded'); - expect(res.compatibility).toBe('degraded'); + expect(res.compatibility).toBe('degraded-compatible'); expect(res.reasonCategory).toBe('model-discovery-degraded'); } expect(calls).toHaveLength(2); @@ -1155,16 +1163,13 @@ describe('runProviderTest degrade-probe (issue #179)', () => { } }); - it('openai-responses: /models 404 + /responses 404 → preserves original 404 (no /chat/completions false-positive)', async () => { - // Regression: the previous implementation probed /chat/completions for - // every OpenAI-compat wire. A gateway that only implements /chat/completions - // would then report the connection healthy even though real inference (on - // /responses) would 404 at generate-time. We want the opposite: if the - // wire's real endpoint is dead, the test must fail. + it('openai-responses: /models 404 + /responses 404 + /chat/completions 200 → incompatible wrong-wire, not a false healthy', async () => { + // A gateway that only implements /chat/completions must not report the + // openai-responses connection as healthy. It SHOULD probe the alternate + // wire so the renderer can show an explicit responses vs chat mismatch. const { calls, restore } = installFakeFetch((url) => { if (url.endsWith('/models')) return { status: 404 }; if (url.endsWith('/responses')) return { status: 404 }; - // A gateway that only has /chat/completions — must not be consulted. if (url.endsWith('/chat/completions')) return { status: 200, body: { id: 'wrong-probe' } }; return { status: 500 }; }); @@ -1178,10 +1183,106 @@ describe('runProviderTest degrade-probe (issue #179)', () => { expect(res.ok).toBe(false); if (!res.ok) { expect(res.code).toBe('404'); - expect(res.message).toBe('HTTP 404'); + expect(res.compatibility).toBe('incompatible'); + expect(res.reasonCategory).toBe('wrong-wire'); + expect(res.reasons?.some((r) => r.layer === 'wire-support' && r.status === 'fail')).toBe( + true, + ); + expect(res.reasons?.find((r) => r.layer === 'wire-support')?.cause).toBe( + 'diagnostics.cause.wireMismatchResponses', + ); + expect(res.reasons?.find((r) => r.layer === 'wire-support')?.suggestedWire).toBe( + 'openai-chat', + ); + expect(res.reasons?.find((r) => r.layer === 'wire-support')?.source).toBe('probe-only'); + } + expect(calls.some((c) => c.url.endsWith('/chat/completions'))).toBe(true); + expect(calls.some((c) => c.url.endsWith('/responses'))).toBe(true); + } finally { + restore(); + } + }); + + it('openai-chat: GET /models 401 is an authentication failure, not a wire failure', async () => { + const { restore } = installFakeFetch(() => ({ status: 401, body: { error: 'invalid' } })); + try { + const res = await runProviderTest({ + provider: 'openai', + wire: 'openai-chat', + apiKey: 'sk-bad', + baseUrl: 'https://api.openai.com/v1', + }); + expect(res.ok).toBe(false); + if (!res.ok) { + expect(res.code).toBe('401'); + expect(res.reasonCategory).toBe('auth'); + expect(res.reasons?.find((r) => r.layer === 'authentication')?.status).toBe('fail'); + expect(res.reasons?.find((r) => r.layer === 'authentication')?.source).toBe( + 'shared-contract', + ); + expect( + res.reasons?.every((r) => r.layer !== 'wire-support' || r.status === 'skipped'), + ).toBe(true); + } + } finally { + restore(); + } + }); + + it('openai-chat: developer-role 400 then user-role 200 is degraded-compatible', async () => { + const { calls, restore } = installFakeFetch((url, init) => { + if (url.endsWith('/models')) return { status: 200, body: { data: [] } }; + const body = typeof init.body === 'string' ? init.body : ''; + if (body.includes('"developer"')) { + return { + status: 400, + body: { + error: { + message: + 'Invalid input: messages.0.role Input should be system, user, assistant or tool; input "developer"', + }, + }, + }; + } + return { status: 200, body: { id: 'ok' } }; + }); + try { + const res = await runProviderTest({ + provider: 'glm', + wire: 'openai-chat', + apiKey: 'sk-glm-test', + baseUrl: 'https://open.bigmodel.cn/api/paas/v4', + }); + expect(res.ok).toBe(true); + if (res.ok) { + expect(res.compatibility).toBe('degraded-compatible'); + expect(res.reasons?.find((r) => r.layer === 'role-compatibility')?.status).toBe('fail'); + expect(res.reasons?.find((r) => r.layer === 'wire-support')?.status).toBe('pass'); + expect(res.reasons?.find((r) => r.layer === 'role-compatibility')?.source).toBe( + 'probe-only', + ); + } + expect(calls.filter((c) => c.url.endsWith('/chat/completions')).length).toBeGreaterThan(1); + } finally { + restore(); + } + }); + + it('openai-responses: /models 404 + /responses 404 + /chat/completions 404 stays endpoint-not-found', async () => { + const { restore } = installFakeFetch(() => ({ status: 404 })); + try { + const res = await runProviderTest({ + provider: 'dead-gateway', + wire: 'openai-responses', + apiKey: 'sk-test', + baseUrl: 'https://gateway.example.com/v1', + }); + expect(res.ok).toBe(false); + if (!res.ok) { + expect(res.code).toBe('404'); + expect(res.reasonCategory).toBe('endpoint-not-found'); + expect(res.reasons?.find((r) => r.layer === 'endpoint-shape')?.status).toBe('fail'); } - // /chat/completions must NOT have been probed for an openai-responses wire. - expect(calls.some((c) => c.url.endsWith('/chat/completions'))).toBe(false); } finally { restore(); } @@ -1193,6 +1294,31 @@ describe('config:v1:test-endpoint response parsing', () => { vi.useRealTimers(); }); + it('treats /models 404 with a live inference route as degraded-compatible', async () => { + const { restore } = installFakeFetch((url) => { + if (url.endsWith('/models')) return { status: 404 }; + if (url.endsWith('/chat/completions')) return { status: 200, body: { id: 'ok' } }; + return { status: 500 }; + }); + try { + const res = await handleConfigV1TestEndpoint({ + wire: 'openai-chat', + baseUrl: 'https://provider.example/v1', + apiKey: 'sk-test', + }); + expect(res.ok).toBe(true); + if (res.ok) { + expect(res.modelCount).toBe(0); + expect(res.compatibility).toBe('degraded-compatible'); + expect( + res.reasons?.some((r) => r.layer === 'model-discovery' && r.status === 'degraded'), + ).toBe(true); + } + } finally { + restore(); + } + }); + it('returns a parse error when the provider response shape has no model ids', async () => { const { restore } = installFakeFetch(() => ({ status: 200, body: { unexpected: [] } })); try { diff --git a/apps/desktop/src/main/connection-ipc.ts b/apps/desktop/src/main/connection-ipc.ts index 1041c9d0d..e5dd92c94 100644 --- a/apps/desktop/src/main/connection-ipc.ts +++ b/apps/desktop/src/main/connection-ipc.ts @@ -1,16 +1,23 @@ import { createHash } from 'node:crypto'; import { isIP } from 'node:net'; +import { openaiChatShouldProbeDeveloperRole } from '@open-codesign/providers'; import { BUILTIN_PROVIDERS, CodesignError, + type ConnectionCapabilityReason, canonicalBaseUrl, + capabilityReason, + classifyInferenceProbe, type DiagnosticCategory, ERROR_CODES, ensureVersionedBase, isSupportedOnboardingProvider, + modelDiscoveryReason, type ProviderEntry, type SupportedOnboardingProvider, + skippedReason, stripInferenceEndpointSuffix, + summarizeConnectionCapabilities, type WireApi, } from '@open-codesign/shared'; import { buildAuthHeaders, buildAuthHeadersForWire } from './auth-headers'; @@ -83,8 +90,10 @@ export interface ConnectionTestResult { | 'chat_completion_degraded' | 'responses_degraded' | 'anthropic_messages_degraded'; - compatibility?: 'compatible' | 'degraded'; + compatibility?: 'compatible' | 'degraded-compatible'; reasonCategory?: DiagnosticCategory; + /** Layered capability reasons for the renderer. Always set by runProviderTest. */ + reasons?: ConnectionCapabilityReason[]; } export interface ConnectionTestError { @@ -94,6 +103,7 @@ export interface ConnectionTestError { hint: string; compatibility?: 'incompatible'; reasonCategory?: DiagnosticCategory; + reasons?: ConnectionCapabilityReason[]; } export type ConnectionTestResponse = ConnectionTestResult | ConnectionTestError; @@ -559,6 +569,82 @@ function resolveActiveCredentials(): ActiveProviderCredentials | ConnectionTestE return resolveCredentialsForProvider(active); } +type InferenceWire = 'openai-chat' | 'openai-responses' | 'anthropic'; + +function isInferenceWire(wire: WireApi): wire is InferenceWire { + return wire === 'openai-chat' || wire === 'openai-responses' || wire === 'anthropic'; +} + +function passReason( + layer: ConnectionCapabilityReason['layer'], + source: ConnectionCapabilityReason['source'], + detail: string, +): ConnectionCapabilityReason { + return capabilityReason({ + layer, + status: 'pass', + category: 'unknown', + cause: 'diagnostics.cause.capabilityPass', + source, + detail, + }); +} + +function authFailReason(detail: string): ConnectionCapabilityReason { + return capabilityReason({ + layer: 'authentication', + status: 'fail', + category: 'auth', + cause: 'diagnostics.cause.keyInvalid', + source: 'shared-contract', + detail, + }); +} + +function degradedProbeMethod( + wire: InferenceWire, +): NonNullable { + if (wire === 'openai-responses') return 'responses_degraded'; + if (wire === 'anthropic') return 'anthropic_messages_degraded'; + return 'chat_completion_degraded'; +} + +function successFromReport( + report: ReturnType, + probeMethod: ConnectionTestResult['probeMethod'], +): ConnectionTestResult { + const compatibility = + report.compatibility === 'incompatible' ? 'degraded-compatible' : report.compatibility; + return { + ok: true, + ...(probeMethod !== undefined ? { probeMethod } : {}), + compatibility, + ...(report.primary !== undefined ? { reasonCategory: report.primary.category } : {}), + reasons: report.reasons, + }; +} + +function failureFromReport( + report: ReturnType, + fields: Pick, +): ConnectionTestError { + return { + ok: false, + ...fields, + compatibility: 'incompatible', + ...(report.primary !== undefined ? { reasonCategory: report.primary.category } : {}), + reasons: report.reasons, + }; +} + +function parseJsonBody(bodyText: string): unknown { + try { + return JSON.parse(bodyText) as unknown; + } catch { + return null; + } +} + async function testChatGPTCodexOAuth(): Promise { let stored: Awaited['read']>>; try { @@ -572,26 +658,35 @@ async function testChatGPTCodexOAuth(): Promise { }; } if (stored === null) { - return { - ok: false, - code: '401', - message: 'No ChatGPT OAuth token stored', - hint: 'ChatGPT 订阅未登录，请到 Settings 登录', - compatibility: 'incompatible', - reasonCategory: 'auth', - }; + return failureFromReport( + summarizeConnectionCapabilities([authFailReason('No ChatGPT OAuth token stored')]), + { + code: '401', + message: 'No ChatGPT OAuth token stored', + hint: 'ChatGPT 订阅未登录，请到 Settings 登录', + }, + ); } if (stored.expiresAt < Date.now()) { - return { - ok: false, - code: '401', - message: 'ChatGPT OAuth token expired', - hint: 'ChatGPT 订阅登录已过期，请重新登录', - compatibility: 'incompatible', - reasonCategory: 'auth', - }; + return failureFromReport( + summarizeConnectionCapabilities([authFailReason('ChatGPT OAuth token expired')]), + { + code: '401', + message: 'ChatGPT OAuth token expired', + hint: 'ChatGPT 订阅登录已过期，请重新登录', + }, + ); } - return { ok: true, compatibility: 'compatible' }; + return successFromReport( + summarizeConnectionCapabilities([ + passReason( + 'authentication', + 'shared-contract', + 'ChatGPT OAuth token is present and unexpired', + ), + ]), + undefined, + ); } export async function runProviderTest( @@ -624,100 +719,369 @@ export async function runProviderTest( res = await fetchWithTimeout(url, { method: 'GET', headers }); } catch (err) { const { code, hint } = classifyNetworkError(err); - return { - ok: false, - code, - message: err instanceof Error ? err.message : 'Network request failed', - hint, - compatibility: 'incompatible', - reasonCategory: code === 'ECONNREFUSED' ? 'network-unreachable' : 'unknown', - }; + return failureFromReport( + summarizeConnectionCapabilities([ + skippedReason('authentication', 'No HTTP response to classify auth'), + capabilityReason({ + layer: 'endpoint-shape', + status: 'fail', + category: code === 'ECONNREFUSED' ? 'network-unreachable' : 'unknown', + cause: 'diagnostics.cause.hostUnreachable', + source: 'probe-only', + detail: err instanceof Error ? err.message : String(err), + }), + ]), + { + code, + message: err instanceof Error ? err.message : 'Network request failed', + hint, + }, + ); + } + + if (res.status === 401 || res.status === 403) { + const { code, hint } = classifyHttpError(res.status); + return failureFromReport( + summarizeConnectionCapabilities([ + authFailReason(`GET /models returned HTTP ${res.status}`), + ]), + { code, message: `HTTP ${res.status}`, hint }, + ); + } + + const modelsOk = res.ok; + const models404 = res.status === 404; + if (!modelsOk && !models404) { + const { code, hint } = classifyHttpError(res.status); + return failureFromReport( + summarizeConnectionCapabilities([ + passReason( + 'authentication', + 'shared-contract', + 'Non-auth HTTP status from GET /models using runtime auth headers', + ), + capabilityReason({ + layer: 'endpoint-shape', + status: 'fail', + category: connectionCategoryForStatus(res.status, normalizedBaseUrl), + cause: 'diagnostics.cause.serverError', + source: 'probe-only', + detail: `GET /models returned HTTP ${res.status}`, + }), + ]), + { code, message: `HTTP ${res.status}`, hint }, + ); } - if (!res.ok) { - // Some OpenAI-compatible gateways (Zhipu GLM, a handful of self-hosted - // proxies) don't expose /models but their /chat/completions works fine. - // If the primary probe 404s on those wires, degrade-probe with a tiny - // chat request before declaring the endpoint dead. We intentionally do - // not degrade anthropic — its /v1/models is standard, and skipping it - // would mask real path-shape mistakes. - if ( - res.status === 404 && - (creds.wire === 'openai-chat' || - creds.wire === 'openai-responses' || - creds.wire === 'anthropic') - ) { - const degraded = await tryDegradeProbe(creds.wire, normalizedBaseUrl, headers); - if (degraded !== null) return degraded; - // Inference endpoint also 404'd (or the network dropped) — fall through - // and report the original /models 404. + + if (!isInferenceWire(creds.wire)) { + if (modelsOk) { + return successFromReport( + summarizeConnectionCapabilities([ + passReason('authentication', 'shared-contract', 'GET /models accepted runtime auth'), + passReason('endpoint-shape', 'shared-contract', 'GET /models succeeded'), + modelDiscoveryReason({ modelsAvailable: true, inferenceAlive: false }), + ]), + 'models', + ); } const { code, hint } = classifyHttpError(res.status); - return { - ok: false, - code, - message: `HTTP ${res.status}`, - hint, - compatibility: 'incompatible', - reasonCategory: connectionCategoryForStatus(res.status, normalizedBaseUrl), - }; + return failureFromReport( + summarizeConnectionCapabilities([ + passReason('authentication', 'shared-contract', 'Non-auth HTTP from GET /models'), + capabilityReason({ + layer: 'endpoint-shape', + status: 'fail', + category: connectionCategoryForStatus(res.status, normalizedBaseUrl), + cause: 'diagnostics.cause.endpointNotFound', + source: 'probe-only', + }), + ]), + { code, message: `HTTP ${res.status}`, hint }, + ); } - return { ok: true, probeMethod: 'models', compatibility: 'compatible' }; + + return assessInferenceCapabilities({ + wire: creds.wire, + baseUrl: creds.baseUrl, + normalizedBaseUrl, + headers, + modelsOk, + }); }); } -async function tryDegradeProbe( - wire: 'openai-chat' | 'openai-responses' | 'anthropic', - normalizedBaseUrl: string, - headers: Record, -): Promise { - const probe = await probeInferenceEndpoint(wire, normalizedBaseUrl, headers); - if (probe.kind === 'pass') { - return { - ok: true, - probeMethod: - wire === 'openai-responses' - ? 'responses_degraded' - : wire === 'anthropic' - ? 'anthropic_messages_degraded' - : 'chat_completion_degraded', - compatibility: 'degraded', - reasonCategory: 'model-discovery-degraded', - }; +async function assessInferenceCapabilities(input: { + wire: InferenceWire; + baseUrl: string; + normalizedBaseUrl: string; + headers: Record; + modelsOk: boolean; +}): Promise { + const { wire, normalizedBaseUrl, headers, modelsOk } = input; + const developerRoleProbed = openaiChatShouldProbeDeveloperRole(wire, input.baseUrl); + const probe = await probeInferenceEndpoint(wire, normalizedBaseUrl, headers, { + includeDeveloperRole: developerRoleProbed, + }); + + if (probe.kind === 'network') { + if (modelsOk) { + return successFromReport( + summarizeConnectionCapabilities([ + passReason('authentication', 'shared-contract', 'GET /models accepted runtime auth'), + passReason('endpoint-shape', 'shared-contract', 'GET /models succeeded'), + skippedReason('wire-support', `Inference probe network error: ${probe.message}`), + modelDiscoveryReason({ modelsAvailable: true, inferenceAlive: false }), + ]), + 'models', + ); + } + return modelsCatalogFailure(normalizedBaseUrl, 404); + } + + let classified = classifyInferenceProbe({ + wire, + status: probe.status, + bodyText: probe.bodyText, + developerRoleProbed, + anthropicErrorShape: hasAnthropicApiErrorShape(parseJsonBody(probe.bodyText)), + }); + + if (classified.roleCompatibility === 'fail' && developerRoleProbed) { + const retry = await probeInferenceEndpoint(wire, normalizedBaseUrl, headers, { + includeDeveloperRole: false, + }); + if (retry.kind === 'response') { + const retryClass = classifyInferenceProbe({ + wire, + status: retry.status, + bodyText: retry.bodyText, + developerRoleProbed: false, + anthropicErrorShape: hasAnthropicApiErrorShape(parseJsonBody(retry.bodyText)), + }); + classified = { ...retryClass, roleCompatibility: 'fail' }; + } } - if (probe.kind === 'http' && probe.status !== 404) { + + if (classified.authentication === 'fail') { const { code, hint } = classifyHttpError(probe.status); - return { - ok: false, - code, - message: `HTTP ${probe.status}`, - hint, - compatibility: 'incompatible', - reasonCategory: connectionCategoryForStatus(probe.status, normalizedBaseUrl), - }; + return failureFromReport( + summarizeConnectionCapabilities([ + authFailReason(`Inference probe returned HTTP ${probe.status}`), + ]), + { code, message: `HTTP ${probe.status}`, hint }, + ); } - return null; + + if (classified.tryAlternateWire) { + const alt = + wire === 'openai-responses' + ? 'openai-chat' + : wire === 'openai-chat' + ? 'openai-responses' + : null; + if (alt !== null) { + const altProbe = await probeInferenceEndpoint(alt, normalizedBaseUrl, headers, { + includeDeveloperRole: false, + }); + if (altProbe.kind === 'response') { + const altClass = classifyInferenceProbe({ + wire: alt, + status: altProbe.status, + bodyText: altProbe.bodyText, + developerRoleProbed: false, + }); + if (altClass.wireSupport === 'pass') { + const suggestedWire = classified.suggestedWire ?? alt; + return failureFromReport( + summarizeConnectionCapabilities([ + passReason( + 'authentication', + 'shared-contract', + 'Non-auth HTTP using runtime auth headers', + ), + passReason( + 'endpoint-shape', + 'probe-only', + 'Alternate OpenAI wire is reachable; selected wire is not', + ), + capabilityReason({ + layer: 'wire-support', + status: 'fail', + category: 'wrong-wire', + cause: + wire === 'openai-responses' + ? 'diagnostics.cause.wireMismatchResponses' + : 'diagnostics.cause.wireMismatchChat', + source: 'probe-only', + detail: `Selected ${wire} failed; ${alt} responded as a live inference route`, + suggestedWire, + suggestedFixKind: 'switchWire', + }), + modelDiscoveryReason({ modelsAvailable: modelsOk, inferenceAlive: false }), + ]), + { + code: '404', + message: `HTTP ${probe.status}`, + hint: + wire === 'openai-responses' + ? 'This gateway speaks /chat/completions, not /responses. Switch the provider wire to openai-chat.' + : 'This gateway speaks /responses, not /chat/completions. Switch the provider wire to openai-responses.', + }, + ); + } + } + } + } + + if (classified.wireSupport === 'fail') { + if (!modelsOk) { + if (probe.status !== 404) { + const { code, hint } = classifyHttpError(probe.status); + return failureFromReport( + summarizeConnectionCapabilities([ + passReason( + 'authentication', + 'shared-contract', + 'Non-auth HTTP using runtime auth headers', + ), + capabilityReason({ + layer: 'wire-support', + status: 'fail', + category: classified.category, + cause: classified.cause, + source: 'probe-only', + }), + modelDiscoveryReason({ modelsAvailable: false, inferenceAlive: false }), + ]), + { code, message: `HTTP ${probe.status}`, hint }, + ); + } + return modelsCatalogFailure(normalizedBaseUrl, 404); + } + if (probe.status >= 500) { + return successFromReport( + summarizeConnectionCapabilities([ + passReason('authentication', 'shared-contract', 'GET /models accepted runtime auth'), + passReason('endpoint-shape', 'shared-contract', 'GET /models succeeded'), + skippedReason( + 'wire-support', + `Inference probe returned HTTP ${probe.status}; treating as unknown because /models succeeded`, + ), + modelDiscoveryReason({ modelsAvailable: true, inferenceAlive: false }), + ]), + 'models', + ); + } + const { code, hint } = classifyHttpError(probe.status); + return failureFromReport( + summarizeConnectionCapabilities([ + passReason('authentication', 'shared-contract', 'Non-auth HTTP using runtime auth headers'), + capabilityReason({ + layer: 'wire-support', + status: 'fail', + category: classified.category, + cause: classified.cause, + source: 'probe-only', + ...(classified.suggestedWire !== undefined + ? { suggestedWire: classified.suggestedWire, suggestedFixKind: 'switchWire' as const } + : {}), + }), + modelDiscoveryReason({ modelsAvailable: modelsOk, inferenceAlive: false }), + ]), + { code, message: `HTTP ${probe.status}`, hint }, + ); + } + + const observations: ConnectionCapabilityReason[] = [ + passReason('authentication', 'shared-contract', 'Non-auth HTTP using runtime auth headers'), + passReason('endpoint-shape', 'shared-contract', 'Selected-wire inference route is reachable'), + passReason('wire-support', 'probe-only', 'Selected-wire inference probe reached the route'), + modelDiscoveryReason({ modelsAvailable: modelsOk, inferenceAlive: true }), + ]; + if (classified.roleCompatibility === 'fail') { + observations.push( + capabilityReason({ + layer: 'role-compatibility', + status: 'fail', + category: 'unsupported-role', + cause: 'diagnostics.cause.unsupportedRole', + source: 'probe-only', + detail: 'Probe-only: request included a developer role message', + suggestedFixKind: 'switchWire', + suggestedWire: 'openai-chat', + }), + ); + } else if (classified.roleCompatibility === 'pass') { + observations.push( + passReason('role-compatibility', 'probe-only', 'Developer role was not rejected'), + ); + } + if (classified.reasoningCompatibility === 'fail') { + observations.push( + capabilityReason({ + layer: 'reasoning-compatibility', + status: 'fail', + category: 'reasoning-policy', + cause: 'diagnostics.cause.reasoningPolicy', + source: 'probe-only', + suggestedFixKind: 'setReasoning', + }), + ); + } else { + observations.push( + skippedReason( + 'reasoning-compatibility', + 'Connection probe does not send reasoning knobs; generate-time diagnostics still classify reasoning failures', + ), + ); + } + + const report = summarizeConnectionCapabilities(observations); + return successFromReport(report, modelsOk ? 'models' : degradedProbeMethod(wire)); +} + +function modelsCatalogFailure( + normalizedBaseUrl: string, + modelsStatus: number, +): ConnectionTestError { + const { code, hint } = classifyHttpError(modelsStatus); + const category = connectionCategoryForStatus(modelsStatus, normalizedBaseUrl); + return failureFromReport( + summarizeConnectionCapabilities([ + passReason('authentication', 'shared-contract', 'Non-auth HTTP from GET /models'), + capabilityReason({ + layer: 'endpoint-shape', + status: 'fail', + category, + cause: + category === 'missing-base-v1' + ? 'diagnostics.cause.missingV1' + : 'diagnostics.cause.endpointNotFound', + source: 'probe-only', + ...(category === 'missing-base-v1' + ? { suggestedFixKind: 'baseUrlTransform' as const } + : {}), + }), + modelDiscoveryReason({ modelsAvailable: false, inferenceAlive: false }), + ]), + { code, message: `HTTP ${modelsStatus}`, hint }, + ); } type ProbeResult = - | { kind: 'pass' } - | { kind: 'http'; status: number } + | { kind: 'response'; status: number; bodyText: string } | { kind: 'network'; message: string }; /** - * POST a minimal inference request to verify the endpoint is alive when GET - * /models returned 404. We dispatch by wire so that providers on the - * Responses API (which may not implement /chat/completions at all) can't - * false-positive via a gateway that only speaks the other shape. A 2xx - * response or any API-originated 4xx (400 model_unknown, 402 insufficient - * credits, 422, 429 — and 401/403 too, which we surface as auth) counts as - * "endpoint reachable". Only 404 and 5xx count as a real failure. The - * request body is intentionally minimal; if the gateway rejects the payload - * shape with a 4xx we still know the route exists. + * POST a minimal inference request. Dispatched by wire so a chat-only gateway + * cannot false-positive for openai-responses (and vice versa). `developer` + * role is included only for third-party openai-chat probes. */ async function probeInferenceEndpoint( - wire: 'openai-chat' | 'openai-responses' | 'anthropic', + wire: InferenceWire, normalizedBaseUrl: string, headers: Record, + options: { includeDeveloperRole?: boolean } = {}, ): Promise { const url = wire === 'anthropic' @@ -725,11 +1089,15 @@ async function probeInferenceEndpoint( : wire === 'openai-responses' ? `${normalizedBaseUrl}/responses` : `${normalizedBaseUrl}/chat/completions`; + const userMessage = { role: 'user', content: 'ping' }; + const chatMessages = options.includeDeveloperRole + ? [{ role: 'developer', content: 'ping' }, userMessage] + : [userMessage]; const body = wire === 'anthropic' ? JSON.stringify({ model: 'probe', - messages: [{ role: 'user', content: 'ping' }], + messages: [userMessage], max_tokens: 1, stream: false, }) @@ -742,7 +1110,7 @@ async function probeInferenceEndpoint( }) : JSON.stringify({ model: 'probe', - messages: [{ role: 'user', content: 'ping' }], + messages: chatMessages, max_tokens: 1, stream: false, }); @@ -756,27 +1124,8 @@ async function probeInferenceEndpoint( } catch (err) { return { kind: 'network', message: err instanceof Error ? err.message : String(err) }; } - if (res.ok) return { kind: 'pass' }; - if (res.status === 404 || res.status >= 500) return { kind: 'http', status: res.status }; - // 401/403 — endpoint alive but auth rejected; surface as auth error so the - // diagnostics panel shows the key-invalid hint instead of the 404 one. - if (res.status === 401 || res.status === 403) return { kind: 'http', status: res.status }; - if (wire === 'anthropic') { - const body = await responseJson(res); - return hasAnthropicApiErrorShape(body) - ? { kind: 'pass' } - : { kind: 'http', status: res.status }; - } - // 400/402/422/429 etc. — endpoint alive, request-level rejection. - return { kind: 'pass' }; -} - -async function responseJson(res: Response): Promise { - try { - return await res.json(); - } catch { - return null; - } + const bodyText = await res.text().catch(() => ''); + return { kind: 'response', status: res.status, bodyText }; } function isJsonRecord(value: unknown): value is Record { @@ -1082,7 +1431,7 @@ export async function handleConfigV1TestEndpoint(raw: unknown): Promise + assessInferenceCapabilities({ + wire: inferenceWire, + baseUrl: payload.baseUrl, + normalizedBaseUrl, + headers, + modelsOk: false, + }), + ); + if (assessed.ok) { + return { + ok: true, + modelCount: 0, + models: [], + compatibility: assessed.compatibility ?? 'degraded-compatible', + ...(assessed.reasons !== undefined ? { reasons: assessed.reasons } : {}), + }; + } + return { + ok: false, + error: assessed.reasonCategory === 'wrong-wire' ? 'wrong-wire' : 'not-a-model-endpoint', + message: assessed.hint, + compatibility: 'incompatible', + ...(assessed.reasons !== undefined ? { reasons: assessed.reasons } : {}), + }; + } + const statusError = classifyTestEndpointStatus(res.status); if (statusError !== null) return statusError; @@ -1257,8 +1635,20 @@ interface TestEndpointPayload { } export type TestEndpointResponse = - | { ok: true; modelCount: number; models: string[] } - | { ok: false; error: string; message: string }; + | { + ok: true; + modelCount: number; + models: string[]; + compatibility?: 'compatible' | 'degraded-compatible'; + reasons?: ConnectionCapabilityReason[]; + } + | { + ok: false; + error: string; + message: string; + compatibility?: 'incompatible'; + reasons?: ConnectionCapabilityReason[]; + }; function parseTestEndpointPayload(raw: unknown): TestEndpointPayload { if (typeof raw !== 'object' || raw === null) { diff --git a/apps/desktop/src/renderer/src/components/AddCustomProviderModal.tsx b/apps/desktop/src/renderer/src/components/AddCustomProviderModal.tsx index c0b1f6597..8e74f6883 100644 --- a/apps/desktop/src/renderer/src/components/AddCustomProviderModal.tsx +++ b/apps/desktop/src/renderer/src/components/AddCustomProviderModal.tsx @@ -1,4 +1,5 @@ import { useT } from '@open-codesign/i18n'; +import type { ConnectionCapabilityReason } from '@open-codesign/shared'; import { canonicalBaseUrl, detectWireFromBaseUrl, type WireApi } from '@open-codesign/shared'; import { Button } from '@open-codesign/ui'; import { AlertCircle, Check, CheckCircle, Loader2, X } from 'lucide-react'; @@ -48,8 +49,8 @@ interface Props { type TestState = | { kind: 'idle' } | { kind: 'testing' } - | { kind: 'ok'; modelCount: number } - | { kind: 'error'; message: string }; + | { kind: 'ok'; modelCount: number; degraded?: boolean; reasons?: ConnectionCapabilityReason[] } + | { kind: 'error'; message: string; reasons?: ConnectionCapabilityReason[] }; type DiscoveryState = | { kind: 'idle' } @@ -264,8 +265,25 @@ export function AddCustomProviderModal({ allowPrivateNetwork, ...(tlsRejectUnauthorized ? { tlsRejectUnauthorized: true } : {}), }); - if (res.ok) setTest({ kind: 'ok', modelCount: res.modelCount }); - else setTest({ kind: 'error', message: res.message }); + if (res.ok) { + setTest({ + kind: 'ok', + modelCount: res.modelCount, + ...(res.compatibility === 'degraded-compatible' ? { degraded: true } : {}), + ...(res.reasons !== undefined ? { reasons: res.reasons } : {}), + }); + } else { + const visible = (res.reasons ?? []).filter( + (reason) => reason.status === 'fail' || reason.status === 'degraded', + ); + const message = + visible.length > 0 ? visible.map((reason) => t(reason.cause)).join(' ') : res.message; + setTest({ + kind: 'error', + message, + ...(res.reasons !== undefined ? { reasons: res.reasons } : {}), + }); + } } catch (err) { setTest({ kind: 'error', message: err instanceof Error ? err.message : String(err) }); } @@ -561,8 +579,12 @@ export function AddCustomProviderModal({ {t('settings.providers.custom.test')} {test.kind === 'ok' && ( - - {t('settings.providers.custom.testOk', { count: test.modelCount })} + + {test.degraded === true + ? t('settings.providers.custom.testDegraded') + : t('settings.providers.custom.testOk', { count: test.modelCount })} )} {test.kind === 'error' && ( diff --git a/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.test.ts b/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.test.ts deleted file mode 100644 index b9a15681c..000000000 --- a/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; - -vi.mock('@open-codesign/i18n', () => ({ - useT: () => (key: string) => key, -})); - -vi.mock('../store', () => ({ - useCodesignStore: () => vi.fn(), -})); - -import { isAbsoluteHttpUrl, shouldShowGatewayAllowlistHint } from './ConnectionDiagnosticPanel'; - -describe('isAbsoluteHttpUrl', () => { - it('rejects an empty string so /v1 quick-fix cannot produce a bare "/v1"', () => { - expect(isAbsoluteHttpUrl('')).toBe(false); - expect(isAbsoluteHttpUrl(' ')).toBe(false); - }); - - it('rejects relative or scheme-less values', () => { - expect(isAbsoluteHttpUrl('api.example.com')).toBe(false); - expect(isAbsoluteHttpUrl('/v1')).toBe(false); - expect(isAbsoluteHttpUrl('ftp://api.example.com')).toBe(false); - }); - - it('accepts http and https absolute URLs', () => { - expect(isAbsoluteHttpUrl('https://api.example.com')).toBe(true); - expect(isAbsoluteHttpUrl('http://localhost:8080')).toBe(true); - expect(isAbsoluteHttpUrl(' https://api.example.com ')).toBe(true); - }); -}); - -describe('shouldShowGatewayAllowlistHint', () => { - it('shows the hint for 400-class compatibility failures on third-party gateways', () => { - expect(shouldShowGatewayAllowlistHint('400', 'https://relay.example.com/v1', undefined)).toBe( - true, - ); - expect( - shouldShowGatewayAllowlistHint( - '403', - 'https://relay.example.com/v1', - 'https://relay.example.com/v1/chat/completions', - ), - ).toBe(true); - expect(shouldShowGatewayAllowlistHint('PARSE', 'https://relay.example.com/v1')).toBe(true); - }); - - it('suppresses the hint for official providers and localhost proxies', () => { - expect(shouldShowGatewayAllowlistHint('400', 'https://api.openai.com/v1')).toBe(false); - expect(shouldShowGatewayAllowlistHint('403', 'https://api.anthropic.com')).toBe(false); - expect(shouldShowGatewayAllowlistHint('400', 'http://127.0.0.1:8317')).toBe(false); - }); - - it('suppresses the hint for unrelated error classes', () => { - expect(shouldShowGatewayAllowlistHint('404', 'https://relay.example.com/v1')).toBe(false); - expect(shouldShowGatewayAllowlistHint('429', 'https://relay.example.com/v1')).toBe(false); - expect(shouldShowGatewayAllowlistHint('ECONNREFUSED', 'https://relay.example.com/v1')).toBe( - false, - ); - }); -}); diff --git a/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.test.tsx b/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.test.tsx new file mode 100644 index 000000000..786826722 --- /dev/null +++ b/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.test.tsx @@ -0,0 +1,130 @@ +import { describe, expect, it, vi } from 'vitest'; + +vi.mock('@open-codesign/i18n', () => ({ + useT: () => (key: string) => key, +})); + +vi.mock('../store', () => ({ + useCodesignStore: () => vi.fn(), +})); + +import type { ConnectionCapabilityReason } from '@open-codesign/shared'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { + ConnectionDiagnosticPanel, + isAbsoluteHttpUrl, + selectConnectionHypotheses, + shouldShowGatewayAllowlistHint, +} from './ConnectionDiagnosticPanel'; + +describe('isAbsoluteHttpUrl', () => { + it('rejects an empty string so /v1 quick-fix cannot produce a bare "/v1"', () => { + expect(isAbsoluteHttpUrl('')).toBe(false); + expect(isAbsoluteHttpUrl(' ')).toBe(false); + }); + + it('rejects relative or scheme-less values', () => { + expect(isAbsoluteHttpUrl('api.example.com')).toBe(false); + expect(isAbsoluteHttpUrl('/v1')).toBe(false); + expect(isAbsoluteHttpUrl('ftp://api.example.com')).toBe(false); + }); + + it('accepts http and https absolute URLs', () => { + expect(isAbsoluteHttpUrl('https://api.example.com')).toBe(true); + expect(isAbsoluteHttpUrl('http://localhost:8080')).toBe(true); + expect(isAbsoluteHttpUrl(' https://api.example.com ')).toBe(true); + }); +}); + +describe('shouldShowGatewayAllowlistHint', () => { + it('shows the hint for 400-class compatibility failures on third-party gateways', () => { + expect(shouldShowGatewayAllowlistHint('400', 'https://relay.example.com/v1', undefined)).toBe( + true, + ); + expect( + shouldShowGatewayAllowlistHint( + '403', + 'https://relay.example.com/v1', + 'https://relay.example.com/v1/chat/completions', + ), + ).toBe(true); + expect(shouldShowGatewayAllowlistHint('PARSE', 'https://relay.example.com/v1')).toBe(true); + }); + + it('suppresses the hint for official providers and localhost proxies', () => { + expect(shouldShowGatewayAllowlistHint('400', 'https://api.openai.com/v1')).toBe(false); + expect(shouldShowGatewayAllowlistHint('403', 'https://api.anthropic.com')).toBe(false); + expect(shouldShowGatewayAllowlistHint('400', 'http://127.0.0.1:8317')).toBe(false); + }); + + it('suppresses the hint for unrelated error classes', () => { + expect(shouldShowGatewayAllowlistHint('404', 'https://relay.example.com/v1')).toBe(false); + expect(shouldShowGatewayAllowlistHint('429', 'https://relay.example.com/v1')).toBe(false); + expect(shouldShowGatewayAllowlistHint('ECONNREFUSED', 'https://relay.example.com/v1')).toBe( + false, + ); + }); +}); + +describe('selectConnectionHypotheses', () => { + const ctx = { provider: 'custom', baseUrl: 'https://gateway.example.com/v1' }; + + it('prefers structured capability reasons over local error-code inference', () => { + const reasons: ConnectionCapabilityReason[] = [ + { + layer: 'wire-support', + status: 'fail', + category: 'wrong-wire', + cause: 'diagnostics.cause.wireMismatchResponses', + source: 'probe-only', + suggestedWire: 'openai-chat', + }, + ]; + const result = selectConnectionHypotheses('404', ctx, reasons); + expect(result[0]?.cause).toBe('diagnostics.cause.wireMismatchResponses'); + expect(result[0]?.suggestedFix?.kind).toBe('switchWire'); + expect(result[0]?.suggestedFix?.wire).toBe('openai-chat'); + }); + + it('falls back to diagnose(errorCode) when no reasons are provided', () => { + const result = selectConnectionHypotheses('401', ctx); + expect(result[0]?.cause).toBe('diagnostics.cause.keyInvalid'); + }); +}); + +describe('ConnectionDiagnosticPanel structured reasons', () => { + it('renders capability layers and probe-only note from IPC reasons', () => { + const reasons: ConnectionCapabilityReason[] = [ + { + layer: 'authentication', + status: 'pass', + category: 'auth', + cause: 'diagnostics.cause.capabilityPass', + source: 'shared-contract', + }, + { + layer: 'wire-support', + status: 'fail', + category: 'wrong-wire', + cause: 'diagnostics.cause.wireMismatchResponses', + source: 'probe-only', + suggestedWire: 'openai-chat', + }, + ]; + const html = renderToStaticMarkup( + undefined} + onTestAgain={() => undefined} + />, + ); + expect(html).toContain('diagnostics.layersHeading'); + expect(html).toContain('diagnostics.layer.wire-support'); + expect(html).toContain('diagnostics.cause.wireMismatchResponses'); + expect(html).toContain('diagnostics.probeOnlyNote'); + }); +}); diff --git a/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.tsx b/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.tsx index 2703aefeb..e79703902 100644 --- a/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.tsx +++ b/apps/desktop/src/renderer/src/components/ConnectionDiagnosticPanel.tsx @@ -1,6 +1,13 @@ import { useT } from '@open-codesign/i18n'; -import type { ErrorCode } from '@open-codesign/shared'; -import { type DiagnoseContext, type DiagnosticHypothesis, diagnose } from '@open-codesign/shared'; +import type { + ConnectionCapabilityReason, + ConnectionCompatibility, + DiagnoseContext, + DiagnosticFix, + DiagnosticHypothesis, + ErrorCode, +} from '@open-codesign/shared'; +import { diagnose, hypothesesFromCapabilityReasons } from '@open-codesign/shared'; import { AlertCircle, ExternalLink, FileText, RefreshCw, X } from 'lucide-react'; import { useState } from 'react'; import { useCodesignStore } from '../store'; @@ -54,9 +61,22 @@ export function shouldShowGatewayAllowlistHint( return !isOfficialProviderHost(hostname); } +export function selectConnectionHypotheses( + errorCode: ErrorCode | undefined, + ctx: DiagnoseContext, + reasons?: ConnectionCapabilityReason[], +): DiagnosticHypothesis[] { + if (reasons !== undefined && reasons.length > 0) { + const fromReasons = hypothesesFromCapabilityReasons(reasons); + if (fromReasons.length > 0) return fromReasons; + } + if (errorCode === undefined) return []; + return diagnose(errorCode, ctx); +} + export interface ConnectionDiagnosticPanelProps { /** The error code returned by connection.test or generate */ - errorCode: ErrorCode; + errorCode?: ErrorCode; /** HTTP status string such as "HTTP 404", if available */ httpStatus?: string; /** The URL that was attempted */ @@ -65,8 +85,13 @@ export interface ConnectionDiagnosticPanelProps { baseUrl: string; /** Provider ID for context */ provider: string; + /** Structured capability reasons from the main-process connection test. */ + reasons?: ConnectionCapabilityReason[]; + compatibility?: ConnectionCompatibility; /** Called when the user clicks "Apply this fix" with a baseUrl transform */ onApplyFix: (newBaseUrl: string) => void; + /** Called for wire / reasoning fixes that cannot be expressed as a baseUrl change. */ + onApplySuggestedFix?: (fix: DiagnosticFix) => void; /** Called when the user clicks "Test again" */ onTestAgain: () => void; /** Called when the user dismisses the panel */ @@ -81,7 +106,10 @@ export function ConnectionDiagnosticPanel({ attemptedUrl, baseUrl, provider, + reasons, + compatibility, onApplyFix, + onApplySuggestedFix, onTestAgain, onDismiss, logsPath, @@ -91,22 +119,35 @@ export function ConnectionDiagnosticPanel({ const [fixApplied, setFixApplied] = useState(false); const ctx: DiagnoseContext = { provider, baseUrl }; - const hypotheses: DiagnosticHypothesis[] = diagnose(errorCode, ctx); + const hypotheses: DiagnosticHypothesis[] = selectConnectionHypotheses(errorCode, ctx, reasons); const primary = hypotheses[0]; const fix = primary?.suggestedFix; + const degraded = compatibility === 'degraded-compatible'; + const probeOnly = reasons?.some( + (reason) => + (reason.status === 'fail' || reason.status === 'degraded') && reason.source === 'probe-only', + ); const canTransformBaseUrl = isAbsoluteHttpUrl(baseUrl); const suggestedUrl = fix?.baseUrlTransform !== undefined && canTransformBaseUrl ? fix.baseUrlTransform(baseUrl) : undefined; - const canApplyFix = suggestedUrl !== undefined || fix?.externalUrl !== undefined; - const showGatewayAllowlistHint = shouldShowGatewayAllowlistHint(errorCode, baseUrl, attemptedUrl); + const canApplyNonUrlFix = + onApplySuggestedFix !== undefined && + (fix?.kind === 'switchWire' || fix?.kind === 'setReasoning'); + const canApplyFix = + suggestedUrl !== undefined || fix?.externalUrl !== undefined || canApplyNonUrlFix; + const showGatewayAllowlistHint = + errorCode !== undefined && shouldShowGatewayAllowlistHint(errorCode, baseUrl, attemptedUrl); function handleApplyFix() { if (suggestedUrl !== undefined) { onApplyFix(suggestedUrl); setFixApplied(true); + } else if (canApplyNonUrlFix && fix !== undefined && onApplySuggestedFix !== undefined) { + onApplySuggestedFix(fix); + setFixApplied(true); } else if (fix?.externalUrl !== undefined) { window.open(fix.externalUrl, '_blank', 'noopener,noreferrer'); } @@ -128,18 +169,22 @@ export function ConnectionDiagnosticPanel({ } } - const displayStatus = httpStatus ?? errorCode; + const displayStatus = httpStatus ?? errorCode ?? compatibility ?? ''; + const toneClass = degraded + ? 'border-[var(--color-warning)] bg-[var(--color-warning-soft,var(--color-surface))]' + : 'border-[var(--color-error)] bg-[var(--color-error-soft,var(--color-surface))]'; + const titleClass = degraded ? 'text-[var(--color-warning)]' : 'text-[var(--color-error)]'; return (
{/* Header */}
-
+
- {t('diagnostics.title')} + {degraded ? t('diagnostics.titleDegraded') : t('diagnostics.title')}
{onDismiss !== undefined && (