feat: add /compact endpoint and llm-d compaction service example

cmd/context-guru-proxy/main.go

@@ -11,10 +11,12 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"log"
 	"log/slog"
 	"net/http"
 	"os"
+	"strings"
 
 	"github.com/kagenti/context-guru/components"
 	_ "github.com/kagenti/context-guru/components/all"
@@ -31,13 +33,17 @@ func main() {
 		preset    = flag.String("preset", envOr("PRESET", "balanced"), "preset to use when --config is absent")
 		openai    = flag.String("openai-upstream", envOr("OPENAI_UPSTREAM", "https://api.openai.com"), "OpenAI upstream base URL")
 		anthropic = flag.String("anthropic-upstream", envOr("ANTHROPIC_UPSTREAM", "https://api.anthropic.com"), "Anthropic upstream base URL")
+		storeFlag = flag.String("store", envOr("STORE", ""), "override state store: true|false (default: config store.enabled, else on)")
 	)
 	flag.Parse()
 
 	cfg, err := loadConfig(*cfgPath, *preset)
 	if err != nil {
 		log.Fatalf("config: %v", err)
 	}
+	if v, ok := parseBool(*storeFlag); ok {
+		cfg.Store.Enabled = &v // flag/env wins over the config file when set
+	}
 
 	agg := metrics.NewAggregator()
 	emitter := metrics.Tee{agg, metrics.Slog{L: slog.Default()}}
@@ -55,6 +61,22 @@ func main() {
 		AnthropicKey: os.Getenv("ANTHROPIC_API_KEY"),
 		ForceModel:   os.Getenv("FORCE_MODEL"), // eval-containers pins EVAL_MODEL's model here
 		CheapModel:   cheapModelFromEnv(),      // static "config"-source LLM for NeedsModel components
+		// Per-request /compact override: swap the pipeline (?preset / header) while
+		// keeping this config's component blocks. nil-safe in the handler.
+		PipelineFor: func(preset string, names []string) (*components.Pipeline, error) {
+			oc := *cfg // override Pipeline only; component blocks + store carry over
+			switch {
+			case len(names) > 0:
+				oc.Pipeline = names
+			case preset != "":
+				p, ok := config.PresetPipeline(preset) // map lookup, not YAML from request input
+				if !ok {
+					return nil, fmt.Errorf("unknown preset %q", preset)
+				}
+				oc.Pipeline = p
+			}
+			return oc.Build(emitter)
+		},
 	})
 
 	slog.Info("context-guru-proxy listening", "addr", addr, "pipeline", cfg.Pipeline)
@@ -77,6 +99,18 @@ func envOr(key, def string) string {
 	return def
 }
 
+// parseBool reads a permissive bool override; ok=false for an empty/unknown
+// value so the config file's setting is left untouched.
+func parseBool(s string) (v, ok bool) {
+	switch strings.ToLower(strings.TrimSpace(s)) {
+	case "true", "1", "on", "yes":
+		return true, true
+	case "false", "0", "off", "no":
+		return false, true
+	}
+	return false, false
+}
+
 // cheapModelFromEnv builds the static "config"-source LLM client for NeedsModel
 // components (extract code/rlm, summarize with model.source=config). Returns nil
 // when CHEAP_MODEL is unset, so those components fall back / no-op.

components/offload/cmdfilter.go

@@ -87,8 +87,10 @@ func (f *Cmdfilter) Offload(req *schemas.BifrostChatRequest, rep *components.Rep
 		// original — the marker costs tokens too, so filtering that barely wins can
 		// still make the message larger (rtk never_worse, at the message level).
 		key := hashKey(content)
+		// degrade full→off when the store can't persist (no unresolvable marker).
+		mode := effectiveMode(c, f.mode)
 		var token string
-		switch f.mode {
+		switch mode {
 		case markerFull:
 			token = expand.Marker(key) + recoveryHint(loss)
 		case markerSummary:
@@ -101,7 +103,7 @@ func (f *Cmdfilter) Offload(req *schemas.BifrostChatRequest, rep *components.Rep
 		if schema.TextTokens(newText) >= schema.TextTokens(content) {
 			continue
 		}
-		if f.mode == markerFull {
+		if mode == markerFull {
 			c.Store.Put(key, []byte(content))
 			keys = append(keys, key)
 		} else {

components/offload/extract.go

@@ -39,19 +39,18 @@ type Extract struct {
 	tail        int
 	strategy    string
 	modelSource string
+	modelClient components.Model // config-pinned client (model: block), or nil
 	trigger     components.Trigger
 	mode        markerMode
 	rewrite     bool
 }
 
 type extractConfig struct {
-	MinTokens int    `yaml:"min_tokens"`
-	Head      int    `yaml:"head_lines"`
-	Tail      int    `yaml:"tail_lines"`
-	Strategy  string `yaml:"strategy"` // deterministic | code | rlm
-	Model     struct {
-		Source string `yaml:"source"` // incoming (default) | config
-	} `yaml:"model"`
+	MinTokens  int                `yaml:"min_tokens"`
+	Head       int                `yaml:"head_lines"`
+	Tail       int                `yaml:"tail_lines"`
+	Strategy   string             `yaml:"strategy"` // deterministic | code | rlm
+	Model      modelConfig        `yaml:"model"`
 	Trigger    components.Trigger `yaml:"trigger"`
 	MarkerMode string             `yaml:"marker_mode"` // full (default) | summary | off
 	// Rewrite (code strategy only): drop the deletion-only containment proof so the
@@ -72,7 +71,7 @@ func newExtract(raw []byte) (components.Component, error) {
 	if cfg.Trigger.MinOutputTokens == 0 {
 		cfg.Trigger.MinOutputTokens = cfg.MinTokens
 	}
-	return &Extract{minTokens: cfg.MinTokens, head: cfg.Head, tail: cfg.Tail, strategy: cfg.Strategy, modelSource: cfg.Model.Source, trigger: cfg.Trigger, mode: parseMarkerMode(cfg.MarkerMode), rewrite: cfg.Rewrite}, nil
+	return &Extract{minTokens: cfg.MinTokens, head: cfg.Head, tail: cfg.Tail, strategy: cfg.Strategy, modelSource: cfg.Model.Source, modelClient: cfg.Model.Client(), trigger: cfg.Trigger, mode: parseMarkerMode(cfg.MarkerMode), rewrite: cfg.Rewrite}, nil
 }
 
 // outputFloor is the minimum tokens a single tool output must have to be worth
@@ -106,7 +105,9 @@ func (e *Extract) Offload(req *bschemas.BifrostChatRequest, rep *components.Repo
 	}
 	var model components.Model
 	if e.NeedsModel() {
-		model = c.Model.For(e.modelSource)
+		if model = e.modelClient; model == nil { // config-pinned client wins
+			model = c.Model.For(e.modelSource)
+		}
 	}
 	floor := e.outputFloor()
 	keepIDs := extract.HarvestIdentifiers(goal, 40)

components/offload/marker.go

@@ -40,6 +40,17 @@ func parseMarkerMode(s string) markerMode {
 	}
 }
 
+// effectiveMode degrades a full (reversible) marker to off when the store cannot
+// persist the stash (store disabled). Without this, a full marker would leave an
+// unresolvable <<cg:HASH>> in the request and silently lose the dropped content.
+// Every Offload that honors marker_mode routes its mode through this first.
+func effectiveMode(c *components.Ctx, mode markerMode) markerMode {
+	if mode == markerFull && !c.Store.Persists() {
+		return markerOff
+	}
+	return mode
+}
+
 // mark centralizes the three marker_modes for the spot where an Offload component
 // would write its restoration marker. It returns the token to splice there and
 // the store key to append to the component's cacheKeys (empty in summary/off).
@@ -51,7 +62,7 @@ func parseMarkerMode(s string) markerMode {
 // hint is the component-specific recovery hint (e.g. " [full output: call
 // context_guru_expand]"); it is only emitted in full mode, where expand works.
 func mark(c *components.Ctx, rep *components.Report, mode markerMode, original, hint string) (token, key string) {
-	if mode == markerFull {
+	if effectiveMode(c, mode) == markerFull {
 		key = hashKey(original)
 		c.Store.Put(key, []byte(original))
 		return expand.Marker(key) + hint, key

components/offload/marker_test.go

@@ -0,0 +1,34 @@
+package offload
+
+import (
+	"testing"
+
+	"github.com/kagenti/context-guru/components"
+	"github.com/kagenti/context-guru/store"
+)
+
+// A full (reversible) marker must degrade to an irreversible off-style drop when
+// the store cannot persist the stash — otherwise it leaves an unresolvable
+// <<cg:HASH>> marker in the request and silently loses the dropped content.
+func TestMarkDegradesWhenStoreCannotPersist(t *testing.T) {
+	rep := &components.Report{}
+	c := &components.Ctx{Store: store.Nop{}}
+	tok, key := mark(c, rep, markerFull, "original content", " [hint]")
+	if tok != "" || key != "" {
+		t.Fatalf("non-persisting store: want no marker/key, got tok=%q key=%q", tok, key)
+	}
+	if !rep.Irreversible {
+		t.Fatal("degraded drop must set Irreversible so the pipeline keeps it (not reverted)")
+	}
+
+	// With a persisting store, full mode still stashes and emits a marker+key.
+	rep2 := &components.Report{}
+	c2 := &components.Ctx{Store: store.NewMemory(store.Options{})}
+	tok2, key2 := mark(c2, rep2, markerFull, "original content", "")
+	if tok2 == "" || key2 == "" {
+		t.Fatalf("persisting store: want marker+key, got tok=%q key=%q", tok2, key2)
+	}
+	if got, ok := c2.Store.Get(key2); !ok || string(got) != "original content" {
+		t.Fatalf("persisting store must retain the original, got %q ok=%v", got, ok)
+	}
+}

components/offload/modelcfg.go

@@ -0,0 +1,47 @@
+package offload
+
+import (
+	"os"
+
+	"github.com/kagenti/context-guru/components"
+	"github.com/kagenti/context-guru/internal/cheapmodel"
+)
+
+// modelConfig is the shared `model:` block for the NeedsModel components
+// (extract code/rlm, summarize). `source` picks the host-resolved client at run
+// time (incoming vs config, via Ctx.Model.For). Setting base_url/model/api_key
+// instead pins a dedicated endpoint+credentials right in the config: Client()
+// then returns that client and the component uses it directly, no CHEAP_MODEL_*
+// env required.
+type modelConfig struct {
+	Source   string `yaml:"source"`   // incoming (default) | config
+	Provider string `yaml:"provider"` // anthropic (default) | openai
+	BaseURL  string `yaml:"base_url"` // e.g. http://llm-d-gateway:8000 (default: provider public API)
+	APIKey   string `yaml:"api_key"`  // empty => provider env key (see Client)
+	Model    string `yaml:"model"`    // e.g. gpt-4o-mini; empty => not a config-pinned client
+	Auth     string `yaml:"auth"`     // anthropic only: "" | x-api-key (default) | bearer (LiteLLM/gateway)
+}
+
+// Client builds the LLM client this block pins, or nil when no model is named
+// (the component then falls back to the host-resolved Ctx.Model.For(source)).
+// An empty api_key falls back to the provider's env key: OPENAI_API_KEY for
+// OpenAI; ANTHROPIC_API_KEY then ANTHROPIC_AUTH_TOKEN (bearer gateways) for
+// Anthropic — so secrets can stay in the environment, out of the config file.
+func (m modelConfig) Client() components.Model {
+	if m.Model == "" {
+		return nil
+	}
+	key := m.APIKey
+	if m.Provider == "openai" {
+		if key == "" {
+			key = os.Getenv("OPENAI_API_KEY")
+		}
+		return cheapmodel.OpenAI{BaseURL: m.BaseURL, Model: m.Model, APIKey: key}
+	}
+	if key == "" {
+		if key = os.Getenv("ANTHROPIC_API_KEY"); key == "" {
+			key = os.Getenv("ANTHROPIC_AUTH_TOKEN")
+		}
+	}
+	return cheapmodel.Anthropic{BaseURL: m.BaseURL, Model: m.Model, APIKey: key, AuthScheme: m.Auth}
+}

components/offload/summarize.go

@@ -43,6 +43,7 @@ type Summarize struct {
 	resummarizeTokens int
 	includeToolCalls  bool
 	modelSource       string
+	modelClient       components.Model // config-pinned client (model: block), or nil
 	trigger           components.Trigger
 	mode              markerMode
 }
@@ -56,13 +57,11 @@ type summarizeConfig struct {
 	// un-summarized tail since the last checkpoint grows past this many tokens,
 	// then roll the checkpoint forward with a fresh summary. 0 = re-summarize
 	// every eligible turn (old behavior).
-	ResummarizeTokens int  `yaml:"resummarize_tokens"`
-	IncludeToolCalls  bool `yaml:"include_tool_calls"`
-	Model             struct {
-		Source string `yaml:"source"` // incoming (default) | config
-	} `yaml:"model"`
-	Trigger    components.Trigger `yaml:"trigger"`
-	MarkerMode string             `yaml:"marker_mode"` // full (default) | summary | off
+	ResummarizeTokens int                `yaml:"resummarize_tokens"`
+	IncludeToolCalls  bool               `yaml:"include_tool_calls"`
+	Model             modelConfig        `yaml:"model"`
+	Trigger           components.Trigger `yaml:"trigger"`
+	MarkerMode        string             `yaml:"marker_mode"` // full (default) | summary | off
 }
 
 func newSummarize(raw []byte) (components.Component, error) {
@@ -80,7 +79,7 @@ func newSummarize(raw []byte) (components.Component, error) {
 	return &Summarize{
 		level: cfg.SummaryLevel, keepLast: cfg.KeepLast,
 		minTokens: cfg.MinTokens, resummarizeTokens: cfg.ResummarizeTokens,
-		includeToolCalls: cfg.IncludeToolCalls, modelSource: cfg.Model.Source, trigger: cfg.Trigger,
+		includeToolCalls: cfg.IncludeToolCalls, modelSource: cfg.Model.Source, modelClient: cfg.Model.Client(), trigger: cfg.Trigger,
 		mode: parseMarkerMode(cfg.MarkerMode),
 	}, nil
 }
@@ -99,7 +98,10 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re
 		rep.Skipped = true
 		return nil, nil
 	}
-	model := c.Model.For(s.modelSource)
+	model := s.modelClient // config-pinned client wins
+	if model == nil {
+		model = c.Model.For(s.modelSource)
+	}
 	if model == nil {
 		rep.Skipped = true // NeedsModel but none available → degrade gracefully
 		return nil, nil
@@ -137,8 +139,11 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re
 	// Stash the replaced span so expand can restore it — full mode only. In
 	// summary/off there is no restoration; flag the deliberate lossy drop so the
 	// pipeline's dropped-without-stash guard permits it.
+	// full is reversible only if the store persists the stash; otherwise degrade
+	// to an irreversible off-style drop (no unresolvable marker).
+	mode := effectiveMode(c, s.mode)
 	var key string
-	if s.mode == markerFull {
+	if mode == markerFull {
 		spanJSON, err := json.Marshal(span)
 		if err != nil {
 			return nil, err
@@ -149,7 +154,7 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re
 		rep.Irreversible = true
 	}
 
-	summaryText := summaryWrapper(summary, key, s.mode)
+	summaryText := summaryWrapper(summary, key, mode)
 	summaryMsg := bschemas.ChatMessage{Role: bschemas.ChatMessageRoleSystem}
 	schema.SetMessageText(&summaryMsg, summaryText)
 

components/reformat/format.go

@@ -14,8 +14,8 @@ func init() { components.Register("format", newFormat) }
 
 // Format re-encodes JSON tool outputs denser without losing data (a Reformat):
 // pretty-printed JSON is re-marshaled compact. It's strictly lossless — same
-// value, fewer whitespace tokens — so no stash is needed. (A TOON encoder is a
-// planned future option; v1 ships json-compact only.)
+// value, fewer whitespace tokens — so no stash is needed. (For a denser tabular
+// re-encoding of uniform object arrays, see the `toon` component.)
 type Format struct{ minTokens int }
 
 type formatConfig struct {