Merge pull request #12 from rossoctl/feat/bob-gateway

feat(proxy): Bob (BobShell) gateway — reduce model calls, pass control-plane through

Author: OsherElhadad

cmd/context-guru-proxy/main.go

@@ -33,6 +33,7 @@ func main() {
 		preset    = flag.String("preset", envOr("PRESET", "balanced"), "preset to use when --config is absent")
 		openai    = flag.String("openai-upstream", envOr("OPENAI_UPSTREAM", "https://api.openai.com"), "OpenAI upstream base URL")
 		anthropic = flag.String("anthropic-upstream", envOr("ANTHROPIC_UPSTREAM", "https://api.anthropic.com"), "Anthropic upstream base URL")
+		bob       = flag.String("bob-upstream", envOr("BOB_UPSTREAM", ""), "Bob (BobShell) backend base URL; enables the Bob gateway routes when set (e.g. https://api.us-east.bob.ibm.com)")
 		storeFlag = flag.String("store", envOr("STORE", ""), "override state store: true|false (default: config store.enabled, else on)")
 	)
 	flag.Parse()
@@ -55,6 +56,7 @@ func main() {
 	h := proxy.New(pipe, cfg.NewStore(), agg, proxy.Options{
 		OpenAIUpstream:    *openai,
 		AnthropicUpstream: *anthropic,
+		BobUpstream:       *bob, // enables the Bob gateway routes when set (BOB_UPSTREAM)
 		// Gateway mode: real provider keys live here (eval-containers passes them
 		// via env); the agent holds only a placeholder. Empty => pass client auth.
 		OpenAIKey:    os.Getenv("OPENAI_API_KEY"),

docs/integrations.md

@@ -89,3 +89,45 @@ p := bifrost.New(pipe, store)   // pipe from config.Build, store from config.New
   header / Anthropic `metadata.user_id`), else the content-hash fallback.
 - `PreLLMHook`/`PostLLMHook` are pass-throughs — the expand loop belongs in a transport wrapper
   (it must re-invoke upstream, which a hook cannot).
+
+## Use it with an agent — Bob (BobShell)
+
+IBM Bob is OpenAI-compatible via its `CUSTOM_BASE_URL`, but it calls **Bob-specific
+paths**: its model call is `POST /inference/v1/chat/completions`, and it makes
+control-plane calls (`GET /admin/v1/profile`, `/inference/v1/model/info`, …) that must
+reach the backend unmodified or the CLI won't boot. The proxy has an opt-in **Bob
+gateway** for exactly this shape — enable it with `BOB_UPSTREAM` (or `--bob-upstream`):
+
+```sh
+BOB_UPSTREAM=https://api.us-east.bob.ibm.com \
+  context-guru-proxy --preset balanced        # any deterministic preset/config
+```
+
+Then point Bob at the proxy and let it use its own key:
+
+```sh
+CUSTOM_BASE_URL=http://localhost:4000/v1 \
+BOBSHELL_DEFAULT_AUTH_TYPE=custom \
+BOBSHELL_API_KEY=<your bob key> \
+  bob --yolo "your task"
+```
+
+How the gateway routes Bob's traffic:
+
+- **Model calls** (`/inference/v1/chat/completions`) run through the pipeline like any
+  OpenAI chat and are forwarded to the same path on `BOB_UPSTREAM`. Bob's own auth is
+  passed straight through (no key injection).
+- **Control-plane calls** (everything else Bob hits) are proxied **verbatim** to
+  `BOB_UPSTREAM`, so Bob authenticates and starts normally.
+
+!!! tip "Start with deterministic components"
+    A lossless, LLM-free pipeline (`format`, `toon`, `dedup`, `failed_run`, `cmdfilter`)
+    needs no cheap-model config and leaves the transcript reversible. Verified end-to-end:
+    with `[format, toon]` Bob authenticates and answers correctly through the proxy, with
+    its model call reduced. For long Bob sessions, add `mask` (see [Choose a preset](../how-to/choose-a-preset.md)).
+
+!!! note "Bob speaks its own backend protocol"
+    Unlike Claude Code (`ANTHROPIC_BASE_URL`) or OpenAI-surface agents (`OPENAI_BASE_URL`),
+    Bob's `CUSTOM_BASE_URL` points at the proxy **host**; Bob supplies the `/inference` and
+    `/admin` paths itself. The gateway only activates when `BOB_UPSTREAM` is set, so it
+    never changes behavior for the other integrations above.

proxy/proxy.go

@@ -39,8 +39,14 @@ import (
 type Options struct {
 	OpenAIUpstream    string // e.g. https://api.openai.com
 	AnthropicUpstream string
-	OpenAIKey         string // injected as Authorization: Bearer <key>
-	AnthropicKey      string // injected as x-api-key: <key>
+	// BobUpstream, when set, enables the Bob (BobShell) gateway: Bob's
+	// OpenAI-dialect model calls (POST /inference/v1/chat/completions) are reduced
+	// and forwarded here, and every other path Bob calls (control-plane:
+	// /admin/v1/profile, /inference/v1/model/info, …) is proxied through verbatim
+	// so the CLI boots and authenticates. Point Bob's CUSTOM_BASE_URL at this proxy.
+	BobUpstream  string // e.g. https://api.us-east.bob.ibm.com
+	OpenAIKey    string // injected as Authorization: Bearer <key>
+	AnthropicKey string // injected as x-api-key: <key>
 	// ForceModel, when set, overwrites the request's "model" field. eval-containers
 	// uses this to pin every call to EVAL_MODEL regardless of what the agent asked for.
 	ForceModel string
@@ -100,9 +106,51 @@ func (h *Handler) Mux() *http.ServeMux {
 	m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok")) })
 	m.HandleFunc("GET /stats", h.stats)
 	m.HandleFunc("GET /expand", h.expand)
+	// Bob (BobShell) gateway. Bob is OpenAI-compatible but calls Bob-specific
+	// paths: its model call is POST /inference/v1/chat/completions (reduced like
+	// any OpenAI chat), and its control-plane calls (/admin/v1/profile,
+	// /inference/v1/model/info, …) must pass through verbatim so the CLI boots and
+	// authenticates. The "/" catch-all is less specific than every route above, so
+	// it only receives what nothing else matched. Enabled only when BobUpstream is
+	// set, so default proxy behavior (unknown path => 404) is unchanged.
+	if h.opts.BobUpstream != "" {
+		m.HandleFunc("POST /inference/v1/chat/completions", h.chat(bschemas.OpenAI, upstream{
+			base: h.opts.BobUpstream,
+			path: "/inference/v1/chat/completions",
+			// setKey nil: pass Bob's own auth (BOBSHELL key) straight through.
+		}))
+		m.HandleFunc("/", h.passthrough(h.opts.BobUpstream))
+	}
 	return m
 }
 
+// passthrough transparently forwards a request to the Bob upstream unchanged —
+// for Bob's control-plane calls that must not be rewritten. Bob's own auth
+// header passes straight through (no key injection); the response is streamed
+// back as-is. Only the model route is reduced; everything else Bob calls lands
+// here and is proxied verbatim.
+func (h *Handler) passthrough(base string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		target := base + r.URL.Path
+		if r.URL.RawQuery != "" {
+			target += "?" + r.URL.RawQuery
+		}
+		req, err := http.NewRequestWithContext(r.Context(), r.Method, target, strings.NewReader(string(body)))
+		if err != nil {
+			http.Error(w, "proxy: "+err.Error(), http.StatusBadGateway)
+			return
+		}
+		copyHeaders(req.Header, r.Header)
+		resp, err := h.client.Do(req)
+		if err != nil {
+			http.Error(w, "upstream: "+err.Error(), http.StatusBadGateway)
+			return
+		}
+		h.stream(w, resp)
+	}
+}
+
 // compact runs the pipeline over the request body's messages and returns the
 // rewritten body — without forwarding upstream. This is the "compact a context,
 // hand it back" endpoint: a caller (e.g. the llm-d-router request-inline-

proxy/proxy_test.go

@@ -129,6 +129,78 @@ func TestAnthropicRouteReducesToolResult(t *testing.T) {
 	}
 }
 
+// TestBobGatewayReducesModelAndPassesControlPlane drives the Bob (BobShell)
+// gateway: the OpenAI-dialect model call on /inference/v1/chat/completions is
+// reduced like any chat and forwarded to the same path, while a control-plane
+// call (GET /admin/v1/profile) passes through to the upstream verbatim.
+func TestBobGatewayReducesModelAndPassesControlPlane(t *testing.T) {
+	type hit struct {
+		method, path string
+		body         []byte
+	}
+	var hits []hit
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := io.ReadAll(r.Body)
+		hits = append(hits, hit{r.Method, r.URL.Path, b})
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"ok":true}`))
+	}))
+	defer upstream.Close()
+
+	cfg, err := config.LoadBytes([]byte("pipeline: [dedup]\ncomponents:\n  dedup: {min_tokens: 20}\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	agg := metrics.NewAggregator()
+	pipe, err := cfg.Build(agg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := proxy.New(pipe, store.NewMemory(store.Options{}), agg, proxy.Options{BobUpstream: upstream.URL})
+	srv := httptest.NewServer(h.Mux())
+	defer srv.Close()
+
+	// 1) Model call on Bob's path is reduced (dedup) and forwarded to the same path.
+	dump := strings.Repeat("verbose repeated bob tool output line\n", 40)
+	body := openAIBody(
+		map[string]any{"role": "user", "content": "do it"},
+		map[string]any{"role": "tool", "tool_call_id": "a", "content": dump},
+		map[string]any{"role": "tool", "tool_call_id": "b", "content": dump},
+	)
+	resp, err := http.Post(srv.URL+"/inference/v1/chat/completions", "application/json", strings.NewReader(string(body)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+
+	// 2) Control-plane call is proxied through verbatim.
+	cp, err := http.Get(srv.URL + "/admin/v1/profile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cp.Body.Close()
+
+	if len(hits) != 2 {
+		t.Fatalf("want 2 upstream hits, got %d: %+v", len(hits), hits)
+	}
+	model, control := hits[0], hits[1]
+	if model.path != "/inference/v1/chat/completions" {
+		t.Fatalf("model call forwarded to wrong path: %q", model.path)
+	}
+	if !strings.Contains(gjson.GetBytes(model.body, "messages.2.content").String(), "identical to an earlier") {
+		t.Fatalf("dedup did not run on the bob model call: %s", model.body)
+	}
+	if len(model.body) >= len(body) {
+		t.Fatalf("bob model call not shrunk (before=%d after=%d)", len(body), len(model.body))
+	}
+	if control.method != "GET" || control.path != "/admin/v1/profile" {
+		t.Fatalf("control-plane not passed through verbatim: %s %s", control.method, control.path)
+	}
+	if len(control.body) != 0 {
+		t.Fatalf("control-plane GET should have empty body, got %d bytes", len(control.body))
+	}
+}
+
 func TestBypassHeaderForwardsUnchanged(t *testing.T) {
 	var got []byte
 	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {