feat: deep token reduction — 4 phases of output compression improvements

.claude/hooks/rtk-suggest.sh

@@ -1,152 +1,39 @@
 #!/usr/bin/env bash
 # RTK suggest hook for Claude Code PreToolUse:Bash
-# Emits system reminders when rtk-compatible commands are detected.
+# Delegates to `rtk suggest` to check if a command has an RTK equivalent.
 # Outputs JSON with systemMessage to inform Claude Code without modifying execution.
+#
+# This hook is intentionally thin — all rewrite logic lives in
+# src/discover/registry.rs (single source of truth).
 
 set -euo pipefail
 
 INPUT=$(cat)
 CMD=$(echo "$INPUT" | jq -r '.tool_input.command // empty')
 
-if [ -z "$CMD" ]; then
-  exit 0
-fi
+# No command → pass through
+[ -z "$CMD" ] && exit 0
 
-# Extract the first meaningful command (before pipes, &&, etc.)
-FIRST_CMD="$CMD"
-
-# Skip if already using rtk
-case "$FIRST_CMD" in
+# Already using rtk → skip
+case "$CMD" in
   rtk\ *|*/rtk\ *) exit 0 ;;
 esac
 
-# Skip commands with heredocs, variable assignments, etc.
-case "$FIRST_CMD" in
+# Heredocs → skip (not safe to rewrite)
+case "$CMD" in
   *'<<'*) exit 0 ;;
 esac
 
-SUGGESTION=""
-
-# --- Git commands ---
-if echo "$FIRST_CMD" | grep -qE '^git\s+status(\s|$)'; then
-  SUGGESTION="rtk git status"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+diff(\s|$)'; then
-  SUGGESTION="rtk git diff"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+log(\s|$)'; then
-  SUGGESTION="rtk git log"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+add(\s|$)'; then
-  SUGGESTION="rtk git add"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+commit(\s|$)'; then
-  SUGGESTION="rtk git commit"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+push(\s|$)'; then
-  SUGGESTION="rtk git push"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+pull(\s|$)'; then
-  SUGGESTION="rtk git pull"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+branch(\s|$)'; then
-  SUGGESTION="rtk git branch"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+fetch(\s|$)'; then
-  SUGGESTION="rtk git fetch"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+stash(\s|$)'; then
-  SUGGESTION="rtk git stash"
-elif echo "$FIRST_CMD" | grep -qE '^git\s+show(\s|$)'; then
-  SUGGESTION="rtk git show"
-
-# --- GitHub CLI ---
-elif echo "$FIRST_CMD" | grep -qE '^gh\s+(pr|issue|run)(\s|$)'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^gh /rtk gh /')
-
-# --- Cargo ---
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+test(\s|$)'; then
-  SUGGESTION="rtk cargo test"
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+build(\s|$)'; then
-  SUGGESTION="rtk cargo build"
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+clippy(\s|$)'; then
-  SUGGESTION="rtk cargo clippy"
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+check(\s|$)'; then
-  SUGGESTION="rtk cargo check"
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+install(\s|$)'; then
-  SUGGESTION="rtk cargo install"
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+nextest(\s|$)'; then
-  SUGGESTION="rtk cargo nextest"
-elif echo "$FIRST_CMD" | grep -qE '^cargo\s+fmt(\s|$)'; then
-  SUGGESTION="rtk cargo fmt"
-
-# --- File operations ---
-elif echo "$FIRST_CMD" | grep -qE '^cat\s+'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^cat /rtk read /')
-elif echo "$FIRST_CMD" | grep -qE '^(rg|grep)\s+'; then
-  SUGGESTION=$(echo "$CMD" | sed -E 's/^(rg|grep) /rtk grep /')
-elif echo "$FIRST_CMD" | grep -qE '^ls(\s|$)'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^ls/rtk ls/')
-elif echo "$FIRST_CMD" | grep -qE '^tree(\s|$)'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^tree/rtk tree/')
-elif echo "$FIRST_CMD" | grep -qE '^find\s+'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^find /rtk find /')
-elif echo "$FIRST_CMD" | grep -qE '^diff\s+'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^diff /rtk diff /')
-elif echo "$FIRST_CMD" | grep -qE '^head\s+'; then
-  # Suggest rtk read with --max-lines transformation
-  if echo "$FIRST_CMD" | grep -qE '^head\s+-[0-9]+\s+'; then
-    LINES=$(echo "$FIRST_CMD" | sed -E 's/^head +-([0-9]+) +.+$/\1/')
-    FILE=$(echo "$FIRST_CMD" | sed -E 's/^head +-[0-9]+ +(.+)$/\1/')
-    SUGGESTION="rtk read $FILE --max-lines $LINES"
-  elif echo "$FIRST_CMD" | grep -qE '^head\s+--lines=[0-9]+\s+'; then
-    LINES=$(echo "$FIRST_CMD" | sed -E 's/^head +--lines=([0-9]+) +.+$/\1/')
-    FILE=$(echo "$FIRST_CMD" | sed -E 's/^head +--lines=[0-9]+ +(.+)$/\1/')
-    SUGGESTION="rtk read $FILE --max-lines $LINES"
-  fi
-
-# --- JS/TS tooling ---
-elif echo "$FIRST_CMD" | grep -qE '^(pnpm\s+)?vitest(\s|$)'; then
-  SUGGESTION="rtk vitest run"
-elif echo "$FIRST_CMD" | grep -qE '^pnpm\s+test(\s|$)'; then
-  SUGGESTION="rtk vitest run"
-elif echo "$FIRST_CMD" | grep -qE '^pnpm\s+tsc(\s|$)'; then
-  SUGGESTION="rtk tsc"
-elif echo "$FIRST_CMD" | grep -qE '^(npx\s+)?tsc(\s|$)'; then
-  SUGGESTION="rtk tsc"
-elif echo "$FIRST_CMD" | grep -qE '^pnpm\s+lint(\s|$)'; then
-  SUGGESTION="rtk lint"
-elif echo "$FIRST_CMD" | grep -qE '^(npx\s+)?eslint(\s|$)'; then
-  SUGGESTION="rtk lint"
-elif echo "$FIRST_CMD" | grep -qE '^(npx\s+)?prettier(\s|$)'; then
-  SUGGESTION="rtk prettier"
-elif echo "$FIRST_CMD" | grep -qE '^(npx\s+)?playwright(\s|$)'; then
-  SUGGESTION="rtk playwright"
-elif echo "$FIRST_CMD" | grep -qE '^pnpm\s+playwright(\s|$)'; then
-  SUGGESTION="rtk playwright"
-elif echo "$FIRST_CMD" | grep -qE '^(npx\s+)?prisma(\s|$)'; then
-  SUGGESTION="rtk prisma"
-
-# --- Containers ---
-elif echo "$FIRST_CMD" | grep -qE '^docker\s+(ps|images|logs)(\s|$)'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^docker /rtk docker /')
-elif echo "$FIRST_CMD" | grep -qE '^kubectl\s+(get|logs)(\s|$)'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^kubectl /rtk kubectl /')
-
-# --- Network ---
-elif echo "$FIRST_CMD" | grep -qE '^curl\s+'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^curl /rtk curl /')
-elif echo "$FIRST_CMD" | grep -qE '^wget\s+'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^wget /rtk wget /')
-
-# --- pnpm package management ---
-elif echo "$FIRST_CMD" | grep -qE '^pnpm\s+(list|ls|outdated)(\s|$)'; then
-  SUGGESTION=$(echo "$CMD" | sed 's/^pnpm /rtk pnpm /')
-fi
-
-# If no suggestion, allow command as-is
-if [ -z "$SUGGESTION" ]; then
-  exit 0
-fi
+# Ask rtk for a suggestion (exits 1 if no equivalent)
+SUGGESTION=$(rtk suggest "$CMD" 2>/dev/null) || exit 0
 
-# Output suggestion as system message
+# Emit suggestion as system message
 jq -n \
   --arg suggestion "$SUGGESTION" \
   '{
     "hookSpecificOutput": {
       "hookEventName": "PreToolUse",
       "permissionDecision": "allow",
-      "systemMessage": ("⚡ RTK available: `" + $suggestion + "` (60-90% token savings)")
+      "systemMessage": ("RTK available: `" + $suggestion + "` (60-90% token savings)")
     }
   }'

.github/workflows/ci.yml

@@ -64,6 +64,14 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - run: cargo test --all
 
+  supply-chain:
+    name: supply-chain (cargo-deny)
+    needs: clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: EmbarkStudios/cargo-deny-action@v2
+
   security:
     name: Security Scan
     needs: clippy

README.md

@@ -485,15 +485,24 @@ brew uninstall rtk           # If installed via Homebrew
 
 ## Privacy & Telemetry
 
-RTK collects **anonymous, aggregate usage metrics** once per day, **enabled by default**. This helps prioritize development. See opt-out options below.
-
-**What is collected:**
-- Device hash (salted SHA-256 — per-user random salt stored locally, not reversible)
-- RTK version, OS, architecture
-- Command count (last 24h) and top command names (e.g. "git", "cargo" — no arguments, no file paths)
-- Token savings percentage
-
-**What is NOT collected:** source code, file paths, command arguments, secrets, environment variables, or any personally identifiable information.
+RTK collects **anonymous, aggregate usage metrics** at most once every 23 hours, **enabled by default**. This helps prioritize which commands to optimize next. See opt-out options below.
+
+**What is collected** (exact payload — see `src/core/telemetry.rs`):
+- `device_hash` — salted SHA-256 of hostname+username (per-user random salt stored locally in `~/.local/share/rtk/.device_salt`, not reversible)
+- `version` — RTK version (e.g. "0.34.3")
+- `os`, `arch` — operating system and CPU architecture
+- `install_method` — how RTK was installed (homebrew, cargo, script, nix, or other)
+- `commands_24h` — number of RTK commands run in the last 24 hours
+- `top_commands` — top 5 command names by frequency (e.g. "git status", "cargo test" — no file paths, no arguments beyond the subcommand)
+- `savings_pct` — overall token savings percentage
+- `tokens_saved_24h`, `tokens_saved_total` — token counts saved (last 24h and all-time)
+
+**What is NOT collected:** source code, file paths, full command arguments, secrets, environment variables, or any personally identifiable information.
+
+**Technical details:**
+- Telemetry URL is compiled into the binary at build time via `RTK_TELEMETRY_URL`. Open-source builds from source have no URL compiled in, so **no data is ever sent** unless you explicitly set this env var during compilation.
+- Pings use a 2-second timeout and run in a background thread — they never block CLI execution.
+- The salt file is created with `0600` permissions (owner-only read/write).
 
 **Opt-out** (any of these):
 ```bash

deny.toml

@@ -0,0 +1,41 @@
+# cargo-deny configuration for RTK
+# https://embarkstudios.github.io/cargo-deny/
+
+[advisories]
+# Deny crates with known security vulnerabilities
+vulnerability = "deny"
+# Warn about unmaintained crates
+unmaintained = "warn"
+# Warn about yanked crates
+yanked = "warn"
+
+[licenses]
+# Allow only these open-source licenses
+allow = [
+    "MIT",
+    "Apache-2.0",
+    "Apache-2.0 WITH LLVM-exception",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "ISC",
+    "Unicode-3.0",
+    "Unicode-DFS-2016",
+    "Zlib",
+    "CC0-1.0",
+]
+confidence-threshold = 0.8
+
+[bans]
+# Warn when multiple versions of the same crate are pulled in
+multiple-versions = "warn"
+# Deny wildcard dependencies (e.g. version = "*")
+wildcards = "deny"
+
+[sources]
+# Deny crates from unknown registries
+unknown-registry = "deny"
+# Deny crates fetched from unknown git repos
+unknown-git = "deny"
+# Only allow crates from crates.io
+allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+allow-git = []

src/analytics/cc_economics.rs

@@ -384,12 +384,14 @@ fn compute_totals(periods: &[PeriodEconomics]) -> Totals {
 
     // Compute global dual metrics (legacy)
     if totals.cc_total_tokens > 0 {
-        totals.blended_cpt = Some(totals.cc_cost / totals.cc_total_tokens as f64);
-        totals.savings_blended = Some(totals.rtk_saved_tokens as f64 * totals.blended_cpt.unwrap());
+        let blended = totals.cc_cost / totals.cc_total_tokens as f64;
+        totals.blended_cpt = Some(blended);
+        totals.savings_blended = Some(totals.rtk_saved_tokens as f64 * blended);
     }
     if totals.cc_active_tokens > 0 {
-        totals.active_cpt = Some(totals.cc_cost / totals.cc_active_tokens as f64);
-        totals.savings_active = Some(totals.rtk_saved_tokens as f64 * totals.active_cpt.unwrap());
+        let active = totals.cc_cost / totals.cc_active_tokens as f64;
+        totals.active_cpt = Some(active);
+        totals.savings_active = Some(totals.rtk_saved_tokens as f64 * active);
     }
 
     totals

src/cmds/cloud/curl_cmd.rs

@@ -4,6 +4,17 @@ use crate::core::tracking;
 use crate::core::utils::{exit_code_from_output, resolved_command, truncate};
 use crate::json_cmd;
 use anyhow::{Context, Result};
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    static ref HTML_TAG_RE: Regex = Regex::new(r"<[^>]+>").unwrap();
+    static ref HTML_ENTITY_RE: Regex = Regex::new(r"&(amp|lt|gt|quot|nbsp|#\d+);").unwrap();
+    static ref SCRIPT_RE: Regex =
+        Regex::new(r"(?si)<script[^>]*>.*?</script>").unwrap();
+    static ref STYLE_RE: Regex =
+        Regex::new(r"(?si)<style[^>]*>.*?</style>").unwrap();
+}
 
 /// Not using run_filtered: on failure, curl can return HTML error pages (404, 500)
 /// that the JSON schema filter would mangle. The early exit skips filtering entirely.
@@ -66,15 +77,22 @@ fn filter_curl_output(output: &str) -> String {
         }
     }
 
-    // Not JSON: truncate long output
-    let lines: Vec<&str> = trimmed.lines().collect();
+    // Detect HTML and strip tags for cleaner output
+    let text = if trimmed.contains("<!DOCTYPE") || trimmed.contains("<html") || trimmed.contains("<HTML") {
+        strip_html_tags(trimmed)
+    } else {
+        trimmed.to_string()
+    };
+
+    // Truncate long output
+    let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
     if lines.len() > 30 {
         let mut result: Vec<&str> = lines[..30].to_vec();
         result.push("");
         let msg = format!(
             "... ({} more lines, {} bytes total)",
             lines.len() - 30,
-            trimmed.len()
+            text.len()
         );
         return format!("{}\n{}", result.join("\n"), msg);
     }
@@ -87,6 +105,34 @@ fn filter_curl_output(output: &str) -> String {
         .join("\n")
 }
 
+/// Strip HTML tags and entities, returning only visible text content.
+fn strip_html_tags(html: &str) -> String {
+    // Remove <script> and <style> blocks entirely
+    let no_script = SCRIPT_RE.replace_all(html, "");
+    let no_style = STYLE_RE.replace_all(&no_script, "");
+    // Strip remaining tags
+    let no_tags = HTML_TAG_RE.replace_all(&no_style, " ");
+    // Decode common entities
+    let decoded = HTML_ENTITY_RE.replace_all(&no_tags, |caps: &regex::Captures| {
+        match &caps[1] {
+            "amp" => "&",
+            "lt" => "<",
+            "gt" => ">",
+            "quot" => "\"",
+            "nbsp" => " ",
+            _ => "",
+        }
+        .to_string()
+    });
+    // Collapse whitespace within lines, remove blank lines
+    decoded
+        .lines()
+        .map(|l| l.split_whitespace().collect::<Vec<_>>().join(" "))
+        .filter(|l| !l.is_empty())
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -135,4 +181,37 @@ mod tests {
         assert!(result.contains("Line 29"));
         assert!(result.contains("more lines"));
     }
+
+    #[test]
+    fn test_strip_html_tags_basic() {
+        let html = "<!DOCTYPE html><html><head><title>Error</title></head><body><h1>404 Not Found</h1><p>The page was not found.</p></body></html>";
+        let result = strip_html_tags(html);
+        assert!(result.contains("404 Not Found"));
+        assert!(result.contains("page was not found"));
+        assert!(!result.contains("<h1>"));
+        assert!(!result.contains("<p>"));
+    }
+
+    #[test]
+    fn test_strip_html_removes_script() {
+        let html = "<html><body><script>var x = 1;</script><p>visible</p></body></html>";
+        let result = strip_html_tags(html);
+        assert!(result.contains("visible"));
+        assert!(!result.contains("var x"));
+    }
+
+    #[test]
+    fn test_filter_curl_html_response() {
+        let html = "<!DOCTYPE html><html><body><h1>Server Error</h1><p>Something went wrong</p></body></html>";
+        let result = filter_curl_output(html);
+        assert!(result.contains("Server Error"));
+        assert!(!result.contains("<h1>"));
+    }
+
+    #[test]
+    fn test_strip_html_entities() {
+        let html = "<html><body><p>A &amp; B &lt; C</p></body></html>";
+        let result = strip_html_tags(html);
+        assert!(result.contains("A & B < C"));
+    }
 }