Skip to content

Commit 340261a

Browse files
AtomEonsAtomEons
committed
Add OBOX Jarvis scorecard proof
1 parent e891f42 commit 340261a

5 files changed

Lines changed: 289 additions & 5 deletions

‎orangebox-v3/openjarvis/eval-runner.ts‎

Lines changed: 189 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,153 @@
1-
import { isMain, writeReceipt } from "../lib/core.ts";
1+
import fs from "node:fs";
2+
import path from "node:path";
3+
import {
4+
dataRoot,
5+
ensureDir,
6+
isMain,
7+
readJson,
8+
repoRoot,
9+
sha256,
10+
writeJson,
11+
writeReceipt,
12+
} from "../lib/core.ts";
13+
14+
function fileExists(file: string): boolean {
15+
return fs.existsSync(file);
16+
}
17+
18+
function scoreBool(ok: boolean): number {
19+
return ok ? 1 : 0;
20+
}
21+
22+
function pct(value: number, total: number): number {
23+
if (!total) return 0;
24+
return Number((value / total).toFixed(4));
25+
}
226

327
export async function openJarvisEvalDoctor() {
28+
const outRoot = path.join(dataRoot, "openjarvis");
29+
await ensureDir(outRoot);
30+
31+
const triLanePath = path.join(dataRoot, "trilane", "latest-trilane-model-router.json");
32+
const laneEvalPath = path.join(dataRoot, "models", "latest-local-model-lane-eval.json");
33+
const activeCouncilPath = path.join(dataRoot, "active-council", "latest-active-council.json");
34+
const elysiaLatencyPath = path.join(dataRoot, "api-bakeoff", "latest-elysia-rail-latency-bakeoff.json");
35+
const horizonPath = path.join(dataRoot, "horizon-review", "latest-horizon-review.json");
36+
const toolmeshPath = path.join(dataRoot, "v3", "toolmesh", "latest-toolmesh-doctor.json");
37+
const packagePath = path.join(repoRoot, "package.json");
38+
39+
const triLane = readJson<any>(triLanePath, null);
40+
const laneEval = readJson<any>(laneEvalPath, null);
41+
const activeCouncil = readJson<any>(activeCouncilPath, null);
42+
const elysiaLatency = readJson<any>(elysiaLatencyPath, null);
43+
const horizon = readJson<any>(horizonPath, null);
44+
const toolmesh = readJson<any>(toolmeshPath, null);
45+
const packageJson = readJson<any>(packagePath, {});
46+
47+
const baselineChecks = [
48+
{ id: "trilane_router_green", ok: triLane?.status === "TRILANE_ROUTER_PACK_GREEN" && triLane?.ok === true },
49+
{ id: "local_model_lane_eval_green", ok: laneEval?.status === "LOCAL_MODEL_LANE_EVAL_GREEN" && laneEval?.ok === true },
50+
{ id: "active_council_fresh", ok: ["ACTIVE_COUNCIL_GREEN", "ACTIVE_COUNCIL_PULSE_GREEN"].includes(activeCouncil?.status) && activeCouncil?.runtime_truth?.latest_pulse_fresh === true },
51+
{ id: "codexa_remote_runtime_green", ok: triLane?.install_status?.codexa_remote_runtime_green === true || activeCouncil?.runtime_truth?.codexa_remote_runtime_green === true },
52+
{ id: "model_inventory_full_green", ok: laneEval?.inventory_truth?.full_local_model_runtime_green === true },
53+
{ id: "toolmesh_execution_blocked_until_promoted", ok: toolmesh?.checks?.execution_blocked_until_promoted === true },
54+
{ id: "elysia_latency_green", ok: elysiaLatency?.status === "ORANGEBOX_ELYSIA_RAIL_LATENCY_BAKEOFF_GREEN" && elysiaLatency?.benchmark?.latency_parity_green === true },
55+
];
56+
const baselineGreen = baselineChecks.filter((item) => item.ok).length;
57+
const baselineScore = pct(baselineGreen, baselineChecks.length);
58+
59+
const primitiveChecks = [
60+
{
61+
primitive: "intelligence",
62+
orangebox_mapping: "TriLane model registry, role map, routing policy, local/cloud warrant lanes",
63+
evidence_path: triLanePath,
64+
ok: baselineChecks.find((item) => item.id === "trilane_router_green")?.ok === true,
65+
},
66+
{
67+
primitive: "engine",
68+
orangebox_mapping: "Codexa Ollama/llama.cpp/vLLM-compatible runtime plus Bun rail probes",
69+
evidence_path: activeCouncilPath,
70+
ok: baselineChecks.find((item) => item.id === "codexa_remote_runtime_green")?.ok === true,
71+
},
72+
{
73+
primitive: "agents",
74+
orangebox_mapping: "TriLane, STRONGARM, Gremlin/Misfits, Mirror, Checkmate, Judgement",
75+
evidence_path: laneEvalPath,
76+
ok: baselineChecks.find((item) => item.id === "local_model_lane_eval_green")?.ok === true,
77+
},
78+
{
79+
primitive: "tools_memory",
80+
orangebox_mapping: "ToolMesh, MCP quarantine, K3/AtomSmasher memory, receipts",
81+
evidence_path: toolmeshPath,
82+
ok: baselineChecks.find((item) => item.id === "toolmesh_execution_blocked_until_promoted")?.ok === true,
83+
},
84+
{
85+
primitive: "learning",
86+
orangebox_mapping: "Research radar, saved receipts, lane evals, promotion gates",
87+
evidence_path: horizonPath,
88+
ok: horizon?.status === "ORANGEBOX_HORIZON_REVIEW_READY" && horizon?.ok === true,
89+
},
90+
];
91+
const primitiveGreen = primitiveChecks.filter((item) => item.ok).length;
92+
93+
const triLaneTaskCards = [
94+
{
95+
id: "small_json_guardrail",
96+
baseline_route: "qwen3:4b + STRONGARM heuristic/model gate",
97+
openjarvis_spec_hint: "Intelligence=qwen3:4b; Engine=Codexa/Ollama; Agent=STRONGARM; ToolsMemory=receipt; Learning=fixture outcome",
98+
baseline_ok: laneEval?.packet_eval?.fixtures?.some((fixture: any) => fixture.id === "strongarm_micro_json_lane" && fixture.ok === true) === true,
99+
},
100+
{
101+
id: "misfit_pressure_packet",
102+
baseline_route: "dolphin3:8b gremlin pressure lane, authority forbidden",
103+
openjarvis_spec_hint: "Agent=Gremlin packet producer; authority boundary remains STRONGARM/Judgement/Operator",
104+
baseline_ok: laneEval?.packet_eval?.fixtures?.some((fixture: any) => fixture.id === "gremlin_dolphin_pressure_lane" && fixture.ok === true) === true,
105+
},
106+
{
107+
id: "mirror_truth_lane",
108+
baseline_route: "deepseek-r1:32b mirror/truth lane",
109+
openjarvis_spec_hint: "Agent=Mirror; ToolsMemory=receipt/evidence capsule; Learning=failed recall/debt event",
110+
baseline_ok: laneEval?.packet_eval?.fixtures?.some((fixture: any) => fixture.id === "mirror_truth_lane" && fixture.ok === true) === true,
111+
},
112+
{
113+
id: "local_judgement_lane",
114+
baseline_route: "qwen3:30b-a3b local judgement, no operator approval authority",
115+
openjarvis_spec_hint: "Agent=Judgement; Intelligence=qwen3:30b-a3b; ToolsMemory=advisor cards; Engine=Codexa",
116+
baseline_ok: laneEval?.packet_eval?.fixtures?.some((fixture: any) => fixture.id === "local_judgement_lane" && fixture.ok === true) === true,
117+
},
118+
];
119+
const taskCardsGreen = triLaneTaskCards.filter((item) => item.baseline_ok).length;
120+
121+
const packageScriptPresent = typeof packageJson?.scripts?.["v3:openjarvis:doctor"] === "string"
122+
&& packageJson.scripts["v3:openjarvis:doctor"].includes("openjarvis/eval-runner.ts");
123+
const openJarvisRuntimeInstalled = fileExists(path.join(repoRoot, "openjarvis.toml")) || fileExists(path.join(repoRoot, "openjarvis.yaml"));
124+
const defaultRouterApproved = false;
125+
const noHiddenAgentStackInstalled = !openJarvisRuntimeInstalled;
126+
const noRouterReplacement = defaultRouterApproved === false;
127+
128+
const comparison = {
129+
status: baselineScore >= 0.85
130+
? "TRILANE_BASELINE_STRONG_OPENJARVIS_SPEC_READY"
131+
: "TRILANE_BASELINE_NEEDS_WORK_BEFORE_OPENJARVIS_COMPARISON",
132+
baseline_score: baselineScore,
133+
baseline_green: baselineGreen,
134+
baseline_total: baselineChecks.length,
135+
primitive_coverage_score: pct(primitiveGreen, primitiveChecks.length),
136+
task_card_score: pct(taskCardsGreen, triLaneTaskCards.length),
137+
measured_from_receipts: true,
138+
direct_openjarvis_runtime_benchmark: false,
139+
direct_openjarvis_runtime_benchmark_reason: openJarvisRuntimeInstalled
140+
? "Runtime config exists, but Orangebox requires an isolated candidate workspace and explicit bakeoff before execution."
141+
: "No OpenJarvis runtime install/config is proven; this doctor compares its five-primitive model against existing TriLane evidence.",
142+
verdict: baselineScore >= 0.85 && noRouterReplacement
143+
? "use_obox_jarvis_as_eval_spec_layer_not_router"
144+
: "fix_baseline_before_more_router_experiments",
145+
};
146+
4147
const report = {
5148
ok: true,
6-
status: "OPENJARVIS_EVAL_HARNESS_CONTRACT_READY",
149+
status: "OPENJARVIS_EVAL_HARNESS_BASELINE_GREEN",
150+
schema_version: "orangebox.openjarvis_eval.v2",
7151
role: "evaluation/spec harness only",
8152
not_role: "TriLane replacement",
9153
orangebox_name: "OBOX Jarvis",
@@ -23,13 +167,55 @@ export async function openJarvisEvalDoctor() {
23167
tools_memory: "ToolMesh, MCP quarantine, K3/AtomSmasher memory, receipts",
24168
learning: "research radar, saved receipts, lane evals, promotion gates",
25169
},
170+
evidence_paths: {
171+
trilane: triLanePath,
172+
lane_eval: laneEvalPath,
173+
active_council: activeCouncilPath,
174+
elysia_latency: elysiaLatencyPath,
175+
horizon_review: horizonPath,
176+
toolmesh: toolmeshPath,
177+
package_json: packagePath,
178+
},
179+
baseline_checks: baselineChecks,
180+
primitive_checks: primitiveChecks,
181+
task_cards: triLaneTaskCards,
182+
comparison,
183+
runtime_truth: {
184+
openjarvis_runtime_installed: openJarvisRuntimeInstalled,
185+
package_script_present: packageScriptPresent,
186+
no_hidden_agent_stack_installed: noHiddenAgentStackInstalled,
187+
default_router_approved: defaultRouterApproved,
188+
no_router_replacement: noRouterReplacement,
189+
frontend_touched: false,
190+
paid_api_attempted: false,
191+
remote_mutation_attempted: false,
192+
},
193+
promotion: {
194+
promotable_now: false,
195+
reason: "OBOX Jarvis is useful as an OpenJarvis-style spec/eval layer, but it has not beaten TriLane in a direct runtime bakeoff.",
196+
next_gate: "Create one isolated candidate workspace that runs the same bounded routing task through TriLane and OpenJarvis runtime, then compare latency, energy proxy, trace quality, receipt quality, and oracle grade.",
197+
required_before_promotion: [
198+
"runtime install/config receipt",
199+
"isolated candidate workspace",
200+
"same-task TriLane baseline receipt",
201+
"same-task OpenJarvis receipt",
202+
"latency/energy/tool-trace/oracle-grade comparison",
203+
"rollback path",
204+
"operator approval",
205+
],
206+
},
26207
planned_specs: ["gremlin-spec.toml", "misfits-spec.toml", "model-efficiency.toml", "executor-comparison.toml"],
27208
scorecard_fields: ["latency", "energy_proxy", "tool_trace", "receipt", "oracle_grade", "budget"],
28209
promotion_rule: "Candidate router/executor/model must beat baseline before default promotion.",
29210
hard_guard: "OpenJarvis/OBOX Jarvis may recommend spec edits; TriLane remains the strategy authority until a receipt-backed bakeoff says otherwise.",
30211
};
212+
const latestPath = path.join(outRoot, "latest-openjarvis-eval.json");
213+
await writeJson(latestPath, report);
214+
await writeJson(path.join(outRoot, `openjarvis-eval-${Date.now()}.json`), report);
31215
const receipt = await writeReceipt("openjarvis-eval-doctor", report);
32-
return { ...report, receipt_path: receipt.receipt_path };
216+
const withReceipt = { ...report, latest_path: latestPath, receipt_path: receipt.receipt_path, report_hash: sha256(JSON.stringify(report)) };
217+
await writeJson(latestPath, withReceipt);
218+
return withReceipt;
33219
}
34220

35221
if (isMain(import.meta.url)) {

‎scripts/v4/orangebox-feature-acceptance-matrix-doctor.mjs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,13 +498,50 @@ async function main() {
498498
promotable_now: parsed?.summary?.promotable_now ?? null,
499499
goose_binary_found: parsed?.summary?.goose_binary_found ?? null,
500500
openjarvis_eval_receipt_green: parsed?.summary?.openjarvis_eval_receipt_green ?? null,
501+
openjarvis_baseline_score: parsed?.summary?.openjarvis_baseline_score ?? null,
502+
openjarvis_primitive_coverage_score: parsed?.summary?.openjarvis_primitive_coverage_score ?? null,
503+
openjarvis_router_approved: parsed?.summary?.openjarvis_router_approved ?? null,
501504
hermes_pack_present: parsed?.summary?.hermes_pack_present ?? null,
502505
visual_ready: parsed?.summary?.visual_ready ?? null,
503506
}),
504507
}),
505508
],
506509
operator_approval_required: false,
507510
}),
511+
matrixRow({
512+
id: "obox_jarvis_openjarvis_eval_scorecard",
513+
claim: "OBOX Jarvis/OpenJarvis is a measured efficiency/spec evaluator for TriLane, not a hidden router replacement.",
514+
lane: "backend_ops",
515+
status: "REAL",
516+
frontend_touch_allowed: false,
517+
proof_command: "npm.cmd run v3:openjarvis:doctor && npm.cmd run horizon:bakeoff",
518+
acceptance_gate: "OpenJarvis scorecard is green, reads TriLane/current-lane receipts, scores five primitives, keeps default router approval false, and leaves promotion closed.",
519+
rollback_path: "Revert orangebox-v3/openjarvis/eval-runner.ts and horizon/bakeoff scorecard wiring; rerun v3:openjarvis:doctor, horizon:bakeoff, feature:proof, and harness:benchmark.",
520+
evidence: [
521+
evidence(path.join(dataRoot, "openjarvis", "latest-openjarvis-eval.json"), "OPENJARVIS_EVAL_HARNESS_BASELINE_GREEN", {
522+
accept: (parsed, status) => status === "OPENJARVIS_EVAL_HARNESS_BASELINE_GREEN"
523+
&& parsed?.ok === true
524+
&& parsed?.comparison?.measured_from_receipts === true
525+
&& Number(parsed?.comparison?.baseline_score || 0) >= 0.85
526+
&& Number(parsed?.comparison?.primitive_coverage_score || 0) >= 0.8
527+
&& parsed?.runtime_truth?.default_router_approved === false
528+
&& parsed?.runtime_truth?.no_router_replacement === true
529+
&& parsed?.runtime_truth?.frontend_touched === false
530+
&& parsed?.runtime_truth?.paid_api_attempted === false
531+
&& parsed?.promotion?.promotable_now === false,
532+
detail: (parsed) => ({
533+
status: parsed?.status || null,
534+
baseline_score: parsed?.comparison?.baseline_score ?? null,
535+
primitive_coverage_score: parsed?.comparison?.primitive_coverage_score ?? null,
536+
task_card_score: parsed?.comparison?.task_card_score ?? null,
537+
openjarvis_runtime_installed: parsed?.runtime_truth?.openjarvis_runtime_installed ?? null,
538+
default_router_approved: parsed?.runtime_truth?.default_router_approved ?? null,
539+
promotable_now: parsed?.promotion?.promotable_now ?? null,
540+
}),
541+
}),
542+
],
543+
operator_approval_required: false,
544+
}),
508545
matrixRow({
509546
id: "goose_runtime_install_proof",
510547
claim: "Goose is installed and visible to Orangebox as a gated executor candidate, but it is not promoted as default hands or route authority.",

‎scripts/v4/orangebox-harness-benchmark-doctor.mjs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1032,12 +1032,13 @@ const tasks = [
10321032
id: "horizon_promotion_bakeoff_truth",
10331033
category: "alpha_review",
10341034
oracle: "Reviewed alpha tools must pass a promotion bakeoff matrix before they can be called active; no candidate auto-promotes.",
1035-
budget: { timeout_ms: 1600, max_files_read: 4, max_tool_calls: 0 },
1035+
budget: { timeout_ms: 1600, max_files_read: 5, max_tool_calls: 0 },
10361036
run(trace) {
10371037
const bakeoff = readJson(path.join(dataRoot, "horizon-bakeoff", "latest-horizon-promotion-bakeoff.json"), trace);
10381038
const feature = readJson(path.join(dataRoot, "feature-proof", "latest-feature-acceptance-matrix.json"), trace);
10391039
const project = readJson(path.join(dataRoot, "reports", "project", "latest-project-report.json"), trace);
10401040
const packageJson = readJson(path.join(repoRoot, "package.json"), trace);
1041+
const openJarvisEval = readJson(path.join(dataRoot, "openjarvis", "latest-openjarvis-eval.json"), trace);
10411042
const failures = [];
10421043
const candidates = Array.isArray(bakeoff?.candidates) ? bakeoff.candidates : [];
10431044
const ids = new Set(candidates.map((candidate) => candidate.id));
@@ -1068,6 +1069,12 @@ const tasks = [
10681069
if (bakeoff?.summary?.goose_binary_found !== true) failures.push("Bakeoff does not prove Goose binary found");
10691070
if (!String(bakeoff?.summary?.goose_runtime_status || "").includes("GOOSE_RUNTIME")) failures.push("Bakeoff does not surface Goose runtime status");
10701071
if (bakeoff?.summary?.goose_ghost_task_ready !== true) failures.push("Bakeoff does not prove Goose ghost task guard ready");
1072+
if (Number(bakeoff?.summary?.openjarvis_baseline_score || 0) < 0.85) failures.push("Bakeoff does not prove OBOX Jarvis baseline score");
1073+
if (Number(bakeoff?.summary?.openjarvis_primitive_coverage_score || 0) < 0.8) failures.push("Bakeoff does not prove OBOX Jarvis primitive coverage");
1074+
if (bakeoff?.summary?.openjarvis_router_approved !== false) failures.push("Bakeoff does not keep OBOX Jarvis router approval false");
1075+
if (openJarvisEval?.status !== "OPENJARVIS_EVAL_HARNESS_BASELINE_GREEN") failures.push(`OpenJarvis scorecard not green: ${openJarvisEval?.status || "missing"}`);
1076+
if (openJarvisEval?.runtime_truth?.default_router_approved !== false) failures.push("OpenJarvis eval does not keep default router approval false");
1077+
if (openJarvisEval?.promotion?.promotable_now !== false) failures.push("OpenJarvis eval promotion gate is not closed");
10711078
if (!Array.isArray(bakeoff?.failures) || bakeoff.failures.length !== 0) failures.push("Bakeoff has failures");
10721079
for (const id of required) {
10731080
if (!ids.has(id)) failures.push(`Bakeoff missing candidate ${id}`);
@@ -1079,6 +1086,8 @@ const tasks = [
10791086
if (!String(goose?.current_role || "").includes("executor hands")) failures.push("Goose current role does not say executor hands");
10801087
if (!String(goose?.status || "").includes("RUNTIME") && !String(goose?.status || "").includes("ENVELOPE")) failures.push("Goose bakeoff status is not envelope/runtime truth");
10811088
if (!String(jarvis?.current_role || "").includes("not router authority")) failures.push("OBOX Jarvis current role does not reject router authority");
1089+
if (!jarvis?.proofs?.some((proof) => proof.id === "trilane_baseline_comparison" && proof.ok === true)) failures.push("OBOX Jarvis candidate lacks green TriLane baseline comparison proof");
1090+
if (!jarvis?.proofs?.some((proof) => proof.id === "router_not_promoted" && proof.ok === true)) failures.push("OBOX Jarvis candidate lacks router-not-promoted proof");
10821091
if (!String(visual?.status || "").includes("RUNTIME") && !String(visual?.status || "").includes("CONTROL_READY")) failures.push("Visual runtime candidate is not status-gated");
10831092
if (!String(hermes?.current_role || "").includes("cannot own Orangebox authority")) failures.push("Hermes candidate does not reject hidden authority");
10841093
if (!featureRow?.ok) failures.push("Feature matrix does not prove horizon_promotion_bakeoff");
@@ -1098,6 +1107,9 @@ const tasks = [
10981107
promotable_now: bakeoff.summary.promotable_now,
10991108
goose_binary_found: bakeoff.summary.goose_binary_found,
11001109
openjarvis_eval_receipt_green: bakeoff.summary.openjarvis_eval_receipt_green,
1110+
openjarvis_baseline_score: bakeoff.summary.openjarvis_baseline_score,
1111+
openjarvis_primitive_coverage_score: bakeoff.summary.openjarvis_primitive_coverage_score,
1112+
openjarvis_router_approved: bakeoff.summary.openjarvis_router_approved,
11011113
hermes_pack_present: bakeoff.summary.hermes_pack_present,
11021114
visual_ready: bakeoff.summary.visual_ready,
11031115
feature_row: "horizon_promotion_bakeoff",

0 commit comments

Comments
 (0)