feat: local LLM support via Ollama (OpenAI-compatible, behind ENABLE_OLLAMA)

backend/.env.example

@@ -16,6 +16,18 @@ R2_BUCKET_NAME=mike
 GEMINI_API_KEY=your-gemini-key
 ANTHROPIC_API_KEY=your-anthropic-key
 OPENAI_API_KEY=your-openai-key
+# Optional: OpenAI-compatible gateway endpoint, e.g. OpenRouter/Azure proxy/Ollama-compatible bridge.
+# Must be https in production unless OPENAI_ALLOW_LOCAL_BASE_URL=true.
+OPENAI_BASE_URL=https://api.openai.com/v1
+OPENAI_ALLOW_LOCAL_BASE_URL=false
+# Local models via Ollama (opt-in, off by default). To enable: set ENABLE_OLLAMA=true
+# and point the OpenAI-compatible client at your local Ollama server:
+#   ENABLE_OLLAMA=true
+#   OPENAI_BASE_URL=http://localhost:11434/v1
+#   OPENAI_ALLOW_LOCAL_BASE_URL=true
+#   OPENAI_API_KEY=ollama          # Ollama accepts any non-empty string
+#   OLLAMA_MODELS=my-custom-model  # optional; extra models beyond the defaults
+ENABLE_OLLAMA=false
 RESEND_API_KEY=your-resend-key
 USER_API_KEYS_ENCRYPTION_SECRET=your-long-random-secret
 

backend/src/lib/__tests__/privateIp.test.ts

@@ -0,0 +1,107 @@
+import { describe, expect, it } from "vitest";
+import { isBlockedIp, isPrivateIpv4, isPrivateIpv6 } from "../privateIp";
+
+describe("isPrivateIpv4", () => {
+    it("blocks RFC1918 / loopback / link-local / CGNAT / reserved ranges", () => {
+        for (const ip of [
+            "0.0.0.0",
+            "10.0.0.1",
+            "10.255.255.255",
+            "127.0.0.1",
+            "100.64.0.1",
+            "100.127.255.255",
+            "169.254.169.254", // cloud metadata
+            "172.16.0.1",
+            "172.31.255.255",
+            "192.168.1.1",
+            "192.0.2.1",
+            "198.18.0.1",
+            "198.19.255.255",
+            "224.0.0.1", // multicast
+            "255.255.255.255",
+        ]) {
+            expect(isPrivateIpv4(ip), ip).toBe(true);
+        }
+    });
+
+    it("allows public addresses", () => {
+        for (const ip of ["8.8.8.8", "1.1.1.1", "93.184.216.34", "203.0.100.5"]) {
+            expect(isPrivateIpv4(ip), ip).toBe(false);
+        }
+    });
+
+    it("fails closed on malformed input", () => {
+        for (const ip of ["", "1.2.3", "1.2.3.4.5", "abc"]) {
+            expect(isPrivateIpv4(ip), JSON.stringify(ip)).toBe(true);
+        }
+    });
+});
+
+describe("isPrivateIpv6", () => {
+    it("blocks loopback / unspecified / ULA / link-local", () => {
+        for (const ip of [
+            "::1",
+            "::",
+            "fc00::1",
+            "fd12:3456::1",
+            "fe80::1",
+            "feaf::1",
+        ]) {
+            expect(isPrivateIpv6(ip), ip).toBe(true);
+        }
+    });
+
+    it("blocks dotted IPv4-mapped addresses that embed a private IPv4", () => {
+        expect(isPrivateIpv6("::ffff:192.168.0.1")).toBe(true);
+        expect(isPrivateIpv6("::ffff:127.0.0.1")).toBe(true);
+        expect(isPrivateIpv6("::ffff:169.254.169.254")).toBe(true);
+        expect(isPrivateIpv6("::ffff:8.8.8.8")).toBe(false);
+    });
+
+    it("blocks hex-form IPv4-mapped addresses (::ffff:c0a8:0001)", () => {
+        expect(isPrivateIpv6("::ffff:c0a8:0001")).toBe(true); // 192.168.0.1
+        expect(isPrivateIpv6("::ffff:7f00:0001")).toBe(true); // 127.0.0.1
+        expect(isPrivateIpv6("::ffff:a9fe:a9fe")).toBe(true); // 169.254.169.254
+        expect(isPrivateIpv6("::ffff:0808:0808")).toBe(false); // 8.8.8.8
+    });
+
+    it("blocks NAT64 (64:ff9b::/96) addresses that embed a private IPv4", () => {
+        expect(isPrivateIpv6("64:ff9b::c0a8:1")).toBe(true); // 192.168.0.1
+        expect(isPrivateIpv6("64:ff9b::10.0.0.1")).toBe(true); // dotted tail
+        expect(isPrivateIpv6("64:ff9b::a9fe:a9fe")).toBe(true); // 169.254.169.254
+        expect(isPrivateIpv6("64:ff9b::808:808")).toBe(false); // 8.8.8.8
+    });
+
+    it("blocks 6to4 (2002::/16) addresses that embed a private IPv4", () => {
+        expect(isPrivateIpv6("2002:c0a8:0001::")).toBe(true); // 192.168.0.1
+        expect(isPrivateIpv6("2002:7f00:0001::")).toBe(true); // 127.0.0.1
+        expect(isPrivateIpv6("2002:a9fe:a9fe::")).toBe(true); // 169.254.169.254
+        expect(isPrivateIpv6("2002:0808:0808::")).toBe(false); // 8.8.8.8
+    });
+
+    it("allows global unicast addresses", () => {
+        for (const ip of [
+            "2606:4700:4700::1111",
+            "2001:4860:4860::8888",
+            "2620:fe::fe",
+        ]) {
+            expect(isPrivateIpv6(ip), ip).toBe(false);
+        }
+    });
+});
+
+describe("isBlockedIp", () => {
+    it("routes IPv4 / IPv6 to the right classifier", () => {
+        expect(isBlockedIp("8.8.8.8")).toBe(false);
+        expect(isBlockedIp("10.0.0.1")).toBe(true);
+        expect(isBlockedIp("::1")).toBe(true);
+        expect(isBlockedIp("2002:c0a8:0001::")).toBe(true);
+        expect(isBlockedIp("2606:4700:4700::1111")).toBe(false);
+    });
+
+    it("fails closed on non-IP input", () => {
+        expect(isBlockedIp("example.com")).toBe(true);
+        expect(isBlockedIp("")).toBe(true);
+        expect(isBlockedIp("not-an-ip")).toBe(true);
+    });
+});

backend/src/lib/llm/__tests__/baseUrl.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it } from "vitest";
+
+import { openAIResponsesUrl, resolveOpenAIBaseUrl } from "../baseUrl";
+
+describe("resolveOpenAIBaseUrl", () => {
+    it("defaults to the official OpenAI v1 endpoint", () => {
+        expect(resolveOpenAIBaseUrl()).toBe("https://api.openai.com/v1");
+        expect(openAIResponsesUrl()).toBe("https://api.openai.com/v1/responses");
+    });
+
+    it("normalizes trailing slashes", () => {
+        expect(resolveOpenAIBaseUrl("https://gateway.example.com/v1///")).toBe(
+            "https://gateway.example.com/v1",
+        );
+    });
+
+    it("allows local http endpoints outside production", () => {
+        expect(resolveOpenAIBaseUrl("http://localhost:11434/v1", "development")).toBe(
+            "http://localhost:11434/v1",
+        );
+    });
+
+    it("rejects http endpoints in production", () => {
+        expect(() =>
+            resolveOpenAIBaseUrl("http://gateway.example.com/v1", "production"),
+        ).toThrow(/https in production/);
+    });
+
+    it("rejects unsupported protocols", () => {
+        expect(() => resolveOpenAIBaseUrl("file:///tmp/openai")).toThrow(
+            /http or https/,
+        );
+    });
+
+    it("rejects private/reserved IP literals in production (SSRF)", () => {
+        for (const host of [
+            "https://10.0.1.50/v1",
+            "https://172.16.5.4/v1",
+            "https://192.168.1.10/v1",
+            "https://169.254.169.254/v1", // cloud metadata
+            "https://127.0.0.1/v1",
+            "https://[::1]/v1",
+        ]) {
+            expect(() => resolveOpenAIBaseUrl(host, "production")).toThrow(
+                /private or reserved IP|localhost/,
+            );
+        }
+    });
+
+    it("allows a public IP / hostname in production", () => {
+        expect(resolveOpenAIBaseUrl("https://8.8.8.8/v1", "production")).toBe(
+            "https://8.8.8.8/v1",
+        );
+        expect(
+            resolveOpenAIBaseUrl("https://gateway.example.com/v1", "production"),
+        ).toBe("https://gateway.example.com/v1");
+    });
+});

backend/src/lib/llm/baseUrl.ts

@@ -0,0 +1,50 @@
+import net from "net";
+import { isBlockedIp } from "../privateIp";
+
+const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
+
+function isLocalHostname(hostname: string): boolean {
+    return (
+        hostname === "localhost" ||
+        hostname === "127.0.0.1" ||
+        hostname === "::1" ||
+        hostname.endsWith(".local")
+    );
+}
+
+export function resolveOpenAIBaseUrl(
+    raw = process.env.OPENAI_BASE_URL ?? DEFAULT_OPENAI_BASE_URL,
+    nodeEnv = process.env.NODE_ENV,
+): string {
+    const parsed = new URL(raw);
+    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
+        throw new Error("OPENAI_BASE_URL must use http or https");
+    }
+    if (nodeEnv === "production" && parsed.protocol !== "https:") {
+        throw new Error("OPENAI_BASE_URL must use https in production");
+    }
+    if (nodeEnv === "production" && process.env.OPENAI_ALLOW_LOCAL_BASE_URL !== "true") {
+        // URL hostnames wrap IPv6 in brackets ([::1]); strip them for net.isIP.
+        const host = parsed.hostname.replace(/^\[|\]$/g, "");
+        if (isLocalHostname(parsed.hostname)) {
+            throw new Error(
+                "OPENAI_BASE_URL cannot point at localhost in production unless OPENAI_ALLOW_LOCAL_BASE_URL=true",
+            );
+        }
+        // Reject IP literals in private/reserved ranges (SSRF) — parity with the
+        // MCP egress guard. DNS hostnames are operator config, not resolved here.
+        if (net.isIP(host) !== 0 && isBlockedIp(host)) {
+            throw new Error(
+                "OPENAI_BASE_URL cannot point at a private or reserved IP in production unless OPENAI_ALLOW_LOCAL_BASE_URL=true",
+            );
+        }
+    }
+    parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+    parsed.search = "";
+    parsed.hash = "";
+    return parsed.toString().replace(/\/$/, "");
+}
+
+export function openAIResponsesUrl(baseUrl = resolveOpenAIBaseUrl()): string {
+    return `${baseUrl}/responses`;
+}

backend/src/lib/llm/index.ts

@@ -23,10 +23,12 @@ export * from "./models";
  * Register a third-party LLM provider so it is available via
  * streamChatWithTools() and completeText().
  *
- * OpenAI-compatible providers can be added the same way — call
- * registerProvider()/registerApiKeyProvider(), no core edits.
+ * Local models via Ollama are built in but opt-in: set ENABLE_OLLAMA=true (see
+ * setupOllamaFromEnv below). Other OpenAI-compatible providers can be added the
+ * same way — call registerProvider()/registerApiKeyProvider(), no core edits.
  */
 export { registerProvider } from "./registry";
+import { setupOllamaFromEnv } from "./providers/ollama";
 
 // ---------------------------------------------------------------------------
 // Register built-in providers
@@ -35,8 +37,16 @@ export { registerProvider } from "./registry";
 // test files mock e.g. "../claude" before this module loads, so the mocked
 // function is captured here and ends up in the registry.
 
-/** Register the built-in LLM providers (claude/gemini/openai). */
-export function registerBuiltinProviders(): void {
+/**
+ * Register the built-in LLM providers (claude/gemini/openai), plus local
+ * (Ollama) models when opted in via ENABLE_OLLAMA.
+ *
+ * Reads process.env by default and is exported so it can be exercised against
+ * a controlled env.
+ */
+export function registerBuiltinProviders(
+    env: NodeJS.ProcessEnv = process.env,
+): void {
     registerProvider({
         id: "claude",
         matchesModel: (m) => m.startsWith("claude"),
@@ -58,6 +68,11 @@ export function registerBuiltinProviders(): void {
         complete: completeOpenAIText,
         models: { main: OPENAI_MAIN_MODELS, mid: OPENAI_MID_MODELS, low: OPENAI_LOW_MODELS },
     });
+
+    // Local models — opt-in via ENABLE_OLLAMA.
+    if (setupOllamaFromEnv(env)) {
+        console.log("[llm] Ollama provider enabled (ENABLE_OLLAMA=true)");
+    }
 }
 
 registerBuiltinProviders();

backend/src/lib/llm/openai.ts

@@ -6,9 +6,9 @@ import type {
   StreamChatParams,
   StreamChatResult,
 } from "./types";
+import { openAIResponsesUrl } from "./baseUrl";
 import { createRawLlmStreamRecorder, logRawLlmStream } from "./rawStreamLog";
 
-const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses";
 const MAX_OUTPUT_TOKENS = 16384;
 const COURTLISTENER_CITATION_REMINDER_TOOL_NAMES = new Set([
   "courtlistener_find_in_case",
@@ -173,7 +173,7 @@ async function createResponse(params: {
   apiKey: string;
   signal?: AbortSignal;
 }): Promise<Response> {
-  const response = await fetch(OPENAI_RESPONSES_URL, {
+  const response = await fetch(openAIResponsesUrl(), {
     method: "POST",
     headers: {
       Authorization: `Bearer ${params.apiKey}`,

backend/src/lib/llm/providers/__tests__/ollama.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+
+import { setupOllamaFromEnv } from "../ollama";
+import { getRegisteredProvider } from "../../registry";
+
+describe("setupOllamaFromEnv (ENABLE_OLLAMA gate)", () => {
+    it("does nothing when the flag is unset or not 'true'", () => {
+        expect(setupOllamaFromEnv({})).toBe(false);
+        expect(setupOllamaFromEnv({ ENABLE_OLLAMA: "false" })).toBe(false);
+        expect(setupOllamaFromEnv({ ENABLE_OLLAMA: "1" })).toBe(false);
+    });
+
+    it("registers the Ollama provider when ENABLE_OLLAMA=true", () => {
+        expect(setupOllamaFromEnv({ ENABLE_OLLAMA: "true" })).toBe(true);
+        const provider = getRegisteredProvider("ollama");
+        expect(provider).toBeDefined();
+        // A default local model routes to this provider.
+        expect(provider?.matchesModel("llama3.3")).toBe(true);
+        // A cloud model does not.
+        expect(provider?.matchesModel("gpt-4o")).toBe(false);
+    });
+
+    it("registers extra models from OLLAMA_MODELS", () => {
+        setupOllamaFromEnv({
+            ENABLE_OLLAMA: "true",
+            OLLAMA_MODELS: "my-custom-model, another-model",
+        });
+        const provider = getRegisteredProvider("ollama");
+        expect(provider?.matchesModel("my-custom-model")).toBe(true);
+        expect(provider?.matchesModel("another-model")).toBe(true);
+    });
+});