Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
386 changes: 162 additions & 224 deletions src/cli/__snapshots__/model-fallback.test.ts.snap

Large diffs are not rendered by default.

10 changes: 9 additions & 1 deletion src/hooks/auto-update-checker/checker/plugin-entry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,18 @@ function runFindPluginEntry(
directory: string,
envOverrides: Record<string, string | undefined> = {},
): { status: number | null; stdout: string; stderr: string } {
const isolatedConfigDir = path.join(directory, ".opencode-user")
const command = [
`import { findPluginEntry } from ${JSON.stringify("./src/hooks/auto-update-checker/checker/plugin-entry")};`,
"(async () => {",
`process.env.OPENCODE_CONFIG_DIR = ${JSON.stringify(isolatedConfigDir)};`,
`Object.assign(process.env, ${JSON.stringify(envOverrides)});`,
`const { findPluginEntry } = await import(${JSON.stringify("./src/hooks/auto-update-checker/checker/plugin-entry")});`,
`const result = findPluginEntry(${JSON.stringify(directory)});`,
"console.log(JSON.stringify(result));",
"})().catch((error) => {",
"console.error(error);",
"process.exit(1);",
"});",
].join("")

const execution = spawnSync(process.execPath, ["-e", command], {
Expand Down
6 changes: 3 additions & 3 deletions src/hooks/model-fallback/hook.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,10 @@ describe("model fallback hook", () => {

//#then - chain should progress to entry[1], not repeat entry[0]
expect(secondOutput.message["model"]).toEqual({
providerID: "opencode-go",
modelID: "kimi-k2.5",
providerID: "github-copilot",
modelID: "claude-opus-4.6",
})
expect(secondOutput.message["variant"]).toBeUndefined()
expect(secondOutput.message["variant"]).toBe("high")
})

test("does not re-arm fallback when one is already pending", () => {
Expand Down
20 changes: 10 additions & 10 deletions src/plugin/event.model-fallback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,10 @@ describe("createEventHandler - model fallback", () => {
expect(abortCalls).toEqual([sessionID])
expect(promptCalls).toEqual([sessionID])
expect(output.message["model"]).toMatchObject({
providerID: "opencode-go",
modelID: "kimi-k2.5",
providerID: "github-copilot",
modelID: "claude-opus-4.6",
})
expect(output.message["variant"]).toBeUndefined()
expect(output.message["variant"]).toBe("high")
})

test("does not spam abort/prompt when session.status retry countdown updates", async () => {
Expand Down Expand Up @@ -549,20 +549,20 @@ describe("createEventHandler - model fallback", () => {
//#when - first retry cycle
const first = await triggerRetryCycle()

//#then - first fallback entry applied (no-op skip: claude-opus-4-6 matches current model after normalization)
//#then - first reachable fallback is the adjacent Copilot Opus split entry
expect(first.message["model"]).toMatchObject({
providerID: "opencode-go",
modelID: "kimi-k2.5",
providerID: "github-copilot",
modelID: "claude-opus-4.6",
})
expect(first.message["variant"]).toBeUndefined()
expect(first.message["variant"]).toBe("high")

//#when - second retry cycle
const second = await triggerRetryCycle()

//#then - second fallback entry applied (chain advanced past opencode-go/kimi-k2.5)
//#then - chain advances past the Copilot split entry
expect(second.message["model"]).toMatchObject({
providerID: "kimi-for-coding",
modelID: "k2p5",
providerID: "opencode-go",
modelID: "kimi-k2.5",
})
expect(second.message["variant"]).toBeUndefined()
expect(abortCalls).toEqual([sessionID, sessionID])
Expand Down
47 changes: 47 additions & 0 deletions src/shared/model-capabilities.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,53 @@ describe("getModelCapabilities", () => {
})
})

describe("Copilot runtime metadata edge cases", () => {
test("resolves claude-opus-4-6 object-form variants to exactly ['low','medium','high'] via Copilot provider", () => {
findProviderModelMetadataSpy = spyOn(connectedProvidersCache, "findProviderModelMetadata").mockReturnValue(undefined)
const result = getModelCapabilities({
providerID: "github-copilot",
modelID: "claude-opus-4-6",
runtimeModel: {
variants: {
low: {},
medium: {},
high: {},
},
},
bundledSnapshot,
})

expect(result.variants).toEqual(["low", "medium", "high"])
expect(result.diagnostics).toMatchObject({
variants: { source: "runtime" },
})
})

test("does not treat empty variants object as runtime variants for gemini-3.1-pro via Copilot provider", () => {
findProviderModelMetadataSpy = spyOn(connectedProvidersCache, "findProviderModelMetadata").mockReturnValue(undefined)
const result = getModelCapabilities({
providerID: "github-copilot",
modelID: "gemini-3.1-pro",
runtimeModel: {
variants: {},
},
bundledSnapshot,
})

// An empty variant object must not produce numeric keys or phantom runtime variants.
// The runtime reader returns undefined for {}, so no runtime-sourced variants exist.
expect(result.diagnostics.variants.source).not.toBe("runtime")
// Variants may still be populated by heuristic family rules, but never from the empty object.
if (result.variants !== undefined) {
expect(result.variants).toEqual(expect.arrayContaining([expect.any(String)]))
for (const v of result.variants) {
expect(typeof v).toBe("string")
expect(Number.isInteger(Number(v))).toBe(false)
}
}
})
})

test("keeps every built-in OmO requirement model snapshot-backed", () => {
const bundledSnapshot = getBundledModelCapabilitiesSnapshot()
const requirementModels = new Set<string>()
Expand Down
105 changes: 72 additions & 33 deletions src/shared/model-requirements.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,43 +23,56 @@ describe("AGENT_MODEL_REQUIREMENTS", () => {
expect(primary.variant).toBe("high")
})

test("sisyphus has claude-opus-4-6 as primary with k2p5, kimi-k2.5, gpt-5.4 medium fallbacks", () => {
test("sisyphus keeps split claude-opus-4-6 entries and non-Copilot siblings", () => {
// #given - sisyphus agent requirement
const sisyphus = AGENT_MODEL_REQUIREMENTS["sisyphus"]

// #when - accessing Sisyphus requirement
// #then - fallbackChain has 7 entries with correct ordering
// #then - fallbackChain has 8 entries with correct ordering
expect(sisyphus).toBeDefined()
expect(sisyphus.fallbackChain).toBeArray()
expect(sisyphus.fallbackChain).toHaveLength(7)
expect(sisyphus.fallbackChain).toHaveLength(8)
expect(sisyphus.requiresAnyModel).toBe(true)

const primary = sisyphus.fallbackChain[0]
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.providers).toEqual(["anthropic", "opencode"])
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.variant).toBe("max")

const second = sisyphus.fallbackChain[1]
expect(second.providers).toEqual(["opencode-go"])
expect(second.model).toBe("kimi-k2.5")
expect(second.providers).toEqual(["github-copilot"])
expect(second.model).toBe("claude-opus-4-6")
expect(second.variant).toBe("high")

const third = sisyphus.fallbackChain[2]
expect(third.providers).toEqual(["kimi-for-coding"])
expect(third.model).toBe("k2p5")
expect(third.providers).toEqual(["opencode-go"])
expect(third.model).toBe("kimi-k2.5")

const fourth = sisyphus.fallbackChain[3]
expect(fourth.model).toBe("kimi-k2.5")
expect(fourth.providers).toEqual(["kimi-for-coding"])
expect(fourth.model).toBe("k2p5")

const fifth = sisyphus.fallbackChain[4]
expect(fifth.providers).toContain("openai")
expect(fifth.model).toBe("gpt-5.4")
expect(fifth.variant).toBe("medium")
expect(fifth.model).toBe("kimi-k2.5")
expect(fifth.providers).toEqual([
"opencode",
"moonshotai",
"moonshotai-cn",
"firmware",
"ollama-cloud",
"aihubmix",
])

const sixth = sisyphus.fallbackChain[5]
expect(sixth.providers[0]).toBe("zai-coding-plan")
expect(sixth.model).toBe("glm-5")
expect(sixth.providers).toEqual(["openai", "github-copilot", "opencode"])
expect(sixth.model).toBe("gpt-5.4")
expect(sixth.variant).toBe("medium")

const seventh = sisyphus.fallbackChain[6]
expect(seventh.providers[0]).toBe("zai-coding-plan")
expect(seventh.model).toBe("glm-5")

const last = sisyphus.fallbackChain[6]
const last = sisyphus.fallbackChain[7]
expect(last.providers[0]).toBe("opencode")
expect(last.model).toBe("big-pickle")
})
Expand Down Expand Up @@ -148,7 +161,7 @@ describe("AGENT_MODEL_REQUIREMENTS", () => {
expect(last.model).toBe("gpt-5-nano")
})

test("prometheus has claude-opus-4-6 as primary", () => {
test("prometheus keeps split claude-opus-4-6 entries with Copilot high variant", () => {
// #given - prometheus agent requirement
const prometheus = AGENT_MODEL_REQUIREMENTS["prometheus"]

Expand All @@ -160,11 +173,16 @@ describe("AGENT_MODEL_REQUIREMENTS", () => {

const primary = prometheus.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.providers).toEqual(["anthropic", "opencode"])
expect(primary.variant).toBe("max")

const sibling = prometheus.fallbackChain[1]
expect(sibling.providers).toEqual(["github-copilot"])
expect(sibling.model).toBe("claude-opus-4-6")
expect(sibling.variant).toBe("high")
})

test("metis has claude-opus-4-6 as primary", () => {
test("metis keeps split claude-opus-4-6 entries with Copilot high variant", () => {
// #given - metis agent requirement
const metis = AGENT_MODEL_REQUIREMENTS["metis"]

Expand All @@ -176,9 +194,14 @@ describe("AGENT_MODEL_REQUIREMENTS", () => {

const primary = metis.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.providers).toEqual(["anthropic", "opencode"])
expect(primary.variant).toBe("max")

const sibling = metis.fallbackChain[1]
expect(sibling.providers).toEqual(["github-copilot"])
expect(sibling.model).toBe("claude-opus-4-6")
expect(sibling.variant).toBe("high")

const openAiFallback = metis.fallbackChain.find((entry) => entry.providers.includes("openai"))
expect(openAiFallback).toEqual({
providers: ["openai", "github-copilot", "opencode"],
Expand Down Expand Up @@ -336,36 +359,47 @@ describe("CATEGORY_MODEL_REQUIREMENTS", () => {
expect(primary.providers).toContain("github-copilot")
})

test("visual-engineering has valid fallbackChain with gemini-3.1-pro high as primary", () => {
test("visual-engineering keeps split gemini and opus entries", () => {
// given - visual-engineering category requirement
const visualEngineering = CATEGORY_MODEL_REQUIREMENTS["visual-engineering"]

// when - accessing visual-engineering requirement
// then - fallbackChain: gemini-3.1-pro(high) → glm-5 → opus-4-6(max) → opencode-go/glm-5 → k2p5
// then - fallbackChain: gemini-3.1-pro(high) → github-copilot gemini → glm-5 → opus-4-6(max) → github-copilot opus → glm-5 → k2p5
expect(visualEngineering).toBeDefined()
expect(visualEngineering.fallbackChain).toBeArray()
expect(visualEngineering.fallbackChain).toHaveLength(5)
expect(visualEngineering.fallbackChain).toHaveLength(7)

const primary = visualEngineering.fallbackChain[0]
expect(primary.providers[0]).toBe("google")
expect(primary.model).toBe("gemini-3.1-pro")
expect(primary.variant).toBe("high")

const second = visualEngineering.fallbackChain[1]
expect(second.providers[0]).toBe("zai-coding-plan")
expect(second.model).toBe("glm-5")
expect(second.providers).toEqual(["github-copilot"])
expect(second.model).toBe("gemini-3.1-pro")
expect(second.variant).toBeUndefined()

const third = visualEngineering.fallbackChain[2]
expect(third.model).toBe("claude-opus-4-6")
expect(third.variant).toBe("max")
expect(third.providers[0]).toBe("zai-coding-plan")
expect(third.model).toBe("glm-5")

const fourth = visualEngineering.fallbackChain[3]
expect(fourth.providers[0]).toBe("opencode-go")
expect(fourth.model).toBe("glm-5")
expect(fourth.providers).toEqual(["anthropic", "opencode"])
expect(fourth.model).toBe("claude-opus-4-6")
expect(fourth.variant).toBe("max")

const fifth = visualEngineering.fallbackChain[4]
expect(fifth.providers[0]).toBe("kimi-for-coding")
expect(fifth.model).toBe("k2p5")
expect(fifth.providers).toEqual(["github-copilot"])
expect(fifth.model).toBe("claude-opus-4-6")
expect(fifth.variant).toBe("high")

const sixth = visualEngineering.fallbackChain[5]
expect(sixth.providers[0]).toBe("opencode-go")
expect(sixth.model).toBe("glm-5")

const seventh = visualEngineering.fallbackChain[6]
expect(seventh.providers[0]).toBe("kimi-for-coding")
expect(seventh.model).toBe("k2p5")
})

test("quick has valid fallbackChain with gpt-5.4-mini as primary and claude-haiku-4-5 as secondary", () => {
Expand Down Expand Up @@ -402,7 +436,7 @@ describe("CATEGORY_MODEL_REQUIREMENTS", () => {
expect(primary.providers[0]).toBe("anthropic")
})

test("unspecified-high has claude-opus-4-6 as primary and gpt-5.4 as secondary", () => {
test("unspecified-high keeps split claude-opus-4-6 entries and preserves gpt-5.4", () => {
// #given - unspecified-high category requirement
const unspecifiedHigh = CATEGORY_MODEL_REQUIREMENTS["unspecified-high"]

Expand All @@ -415,9 +449,14 @@ describe("CATEGORY_MODEL_REQUIREMENTS", () => {
const primary = unspecifiedHigh.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.variant).toBe("max")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.providers).toEqual(["anthropic", "opencode"])

const sibling = unspecifiedHigh.fallbackChain[1]
expect(sibling.providers).toEqual(["github-copilot"])
expect(sibling.model).toBe("claude-opus-4-6")
expect(sibling.variant).toBe("high")

const secondary = unspecifiedHigh.fallbackChain[1]
const secondary = unspecifiedHigh.fallbackChain[2]
expect(secondary.model).toBe("gpt-5.4")
expect(secondary.variant).toBe("high")
expect(secondary.providers).toEqual(["openai", "github-copilot", "opencode"])
Expand Down
Loading
Loading