diff --git a/Cargo.lock b/Cargo.lock index 964aacaa2..68ca25af9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4623,16 +4623,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "forwarded-header-value" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" -dependencies = [ - "nonempty", - "thiserror 1.0.69", -] - [[package]] name = "fragile" version = "2.0.1" @@ -4748,12 +4738,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" - [[package]] name = "futures-util" version = "0.3.32" @@ -5095,29 +5079,6 @@ dependencies = [ "url", ] -[[package]] -name = "governor" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" -dependencies = [ - "cfg-if 1.0.4", - "dashmap", - "futures-sink", - "futures-timer", - "futures-util", - "getrandom 0.3.4", - "hashbrown 0.16.1", - "nonzero_ext", - "parking_lot 0.12.5", - "portable-atomic", - "quanta", - "rand 0.9.2", - "smallvec 1.15.1", - "spinning_top 0.3.0", - "web-time", -] - [[package]] name = "group" version = "0.12.1" @@ -5695,7 +5656,7 @@ dependencies = [ "sha2", "sha3", "sqlx", - "strum 0.27.2", + "strum 0.28.0", "strum_macros 0.28.0", "utoipa", ] @@ -5968,7 +5929,6 @@ dependencies = [ "sha2", "tokio", "tower-http", - "tower_governor", "tracing", "tracing-subscriber 0.3.22", "ureq", @@ -6917,7 +6877,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afa463f5405ee81cdb9cc2baf37e08ec7e4c8209442b5d72c04cfb2cd6e6286" dependencies = [ - "spinning_top 0.2.5", + "spinning_top", ] [[package]] @@ -7409,18 +7369,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nonempty" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" - -[[package]] -name = "nonzero_ext" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" - [[package]] name = "notify" version = "8.2.0" @@ -7816,9 +7764,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.28.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" +checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" dependencies = [ "futures-core", "futures-sink", @@ -7830,9 +7778,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253" +checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" dependencies = [ "async-trait", "bytes", @@ -7844,11 +7792,10 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91" +checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" dependencies = [ - "async-trait", "futures-core", "http 1.4.0", "opentelemetry", @@ -7865,9 +7812,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d" +checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -7883,18 +7830,17 @@ checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" [[package]] name = "opentelemetry_sdk" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" +checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" dependencies = [ - "async-trait", "futures-channel", "futures-executor", "futures-util", "glob", "opentelemetry", "percent-encoding", - "rand 0.8.5", + "rand 0.9.2", "serde_json", "thiserror 2.0.18", "tokio", @@ -8750,21 +8696,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" -[[package]] -name = "quanta" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" -dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -9069,15 +9000,6 @@ dependencies = [ "unicode-width 0.2.2", ] -[[package]] -name = "raw-cpuid" -version = "11.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" -dependencies = [ - "bitflags 2.10.0", -] - [[package]] name = "rayon" version = "1.11.0" @@ -11951,15 +11873,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spinning_top" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" -dependencies = [ - "lock_api", -] - [[package]] name = "spki" version = "0.7.3" @@ -12983,11 +12896,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ - "async-trait", - "axum 0.8.8", "base64 0.22.1", "bytes", - "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -12997,7 +12907,6 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", - "socket2 0.6.2", "sync_wrapper", "tokio", "tokio-rustls 0.26.4", @@ -13094,23 +13003,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" -[[package]] -name = "tower_governor" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44de9b94d849d3c46e06a883d72d408c2de6403367b39df2b1c9d9e7b6736fe6" -dependencies = [ - "axum 0.8.8", - "forwarded-header-value", - "governor", - "http 1.4.0", - "pin-project", - "thiserror 2.0.18", - "tonic 0.14.3", - "tower 0.5.3", - "tracing", -] - [[package]] name = "tracer" version = "0.2.0" @@ -13227,9 +13119,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721f2d2569dce9f3dfbbddee5906941e953bfcdf736a62da3377f5751650cc36" +checksum = "fd8e764bd6f5813fd8bebc3117875190c5b0415be8f7f8059bffb6ecd979c444" dependencies = [ "js-sys", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 129fff406..d79d0cd36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,15 +135,15 @@ k256 = { version = "0.13", default-features = false, features = [ ] } mockall = { version = "0.14.0", default-features = false } once_cell = { version = "1.19.0", default-features = false } -opentelemetry = { version = "0.28.0" } -opentelemetry-prometheus = { version = "0.28.0" } -opentelemetry_sdk = { version = "0.28.0" } -opentelemetry-otlp = { version = "0.28.0", features = [ +opentelemetry = { version = "0.29.0" } +opentelemetry-prometheus = { version = "0.29.0" } +opentelemetry_sdk = { version = "0.29.0" } +opentelemetry-otlp = { version = "0.29.0", features = [ "grpc-tonic", "trace", "metrics", ] } -opentelemetry-http = { version = "0.28" } +opentelemetry-http = { version = "0.29" } paste = { version = "1.0.15", default-features = false } prometheus = { version = "0.13.4" } quote = { version = "1.0.42", default-features = false } @@ -166,12 +166,12 @@ sp1-sdk = { version = "5.2.2", default-features = false } sp1-zkvm = { version = "5.2.2", default-features = false } sparse-merkle-tree = { version = "0.6.1", default-features = false } sqlx = { version = "0.8.6" } -strum = { version = "0.27.1" } +strum = { version = "0.28.0" } strum_macros = { version = "0.28.0", default-features = false } syn = { version = "2.0.104" } tempfile = { version = "3.20.0", default-features = false } test-log = { version = "0.2.18", default-features = false } -tracing-opentelemetry = "0.29.0" +tracing-opentelemetry = "0.30.0" tokio = { version = "1.50.0" } tokio-util = { version = "0.7.16" } toml = { version = "0.9.11", default-features = false } @@ -179,7 +179,6 @@ tower-http = { version = "0.6.5", default-features = false, features = [ "decompression-gzip", ] } tower-service = { version = "0.3.3", default-features = false } -tower_governor = { version = "0.8" } tracing = { version = "0.1", default-features = false } tracing-subscriber = { version = "0.3.19", default-features = false } turmoil = { version = "0.7" } diff --git a/crates/hyli-bus/src/utils/logger.rs b/crates/hyli-bus/src/utils/logger.rs index 310ce1134..0f4c5b3b1 100644 --- a/crates/hyli-bus/src/utils/logger.rs +++ b/crates/hyli-bus/src/utils/logger.rs @@ -13,14 +13,14 @@ mod otlp_metrics { use hyli_turmoil_shims::init_global_meter_provider; use opentelemetry_otlp::WithExportConfig; use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader; - use opentelemetry_sdk::metrics::{MetricResult, SdkMeterProvider}; + use opentelemetry_sdk::metrics::SdkMeterProvider; use opentelemetry_sdk::{runtime, Resource}; pub fn init( endpoint: String, service_name: String, push_interval: Option, - ) -> MetricResult<()> { + ) -> Result<(), opentelemetry_otlp::ExporterBuildError> { let exporter = opentelemetry_otlp::MetricExporter::builder() .with_tonic() .with_endpoint(endpoint) diff --git a/crates/hyli-tools/Cargo.toml b/crates/hyli-tools/Cargo.toml index 4a8fd8685..47b8b64a8 100644 --- a/crates/hyli-tools/Cargo.toml +++ b/crates/hyli-tools/Cargo.toml @@ -59,7 +59,6 @@ ureq = { workspace = true, features = ["json"] } # Rate limiting proxy (optional feature) axum = { workspace = true, optional = true } -tower_governor = { workspace = true, optional = true } tower-http = { workspace = true, features = [ "trace", "cors", @@ -76,7 +75,6 @@ notify = { version = "8.2.0", optional = true } risc0 = ["client-sdk/risc0"] rate-proxy = [ "dep:axum", - "dep:tower_governor", "dep:tower-http", "dep:hyper", "dep:hyper-util", @@ -85,7 +83,7 @@ rate-proxy = [ "dep:notify", ] -default = ["ui", "rate-proxy"] +default = ["rate-proxy"] ui = ["dep:ratatui", "dep:crossterm"] turmoil = [] diff --git a/crates/hyli-tools/src/bin/rate_limiter_proxy.rs b/crates/hyli-tools/src/bin/rate_limiter_proxy.rs index 18c470a76..dcaeb86fa 100644 --- a/crates/hyli-tools/src/bin/rate_limiter_proxy.rs +++ b/crates/hyli-tools/src/bin/rate_limiter_proxy.rs @@ -26,9 +26,32 @@ use std::path::PathBuf; use std::sync::RwLock; use std::{collections::HashMap, time::Duration}; use std::{collections::HashSet, sync::Arc}; -use tower_governor::key_extractor::{KeyExtractor, SmartIpKeyExtractor}; use tower_http::catch_panic::CatchPanicLayer; +/// Extract client IP from common proxy headers, falling back to "unknown". +fn extract_client_ip(req: &Request) -> String { + // Check X-Forwarded-For first (may contain comma-separated list) + if let Some(val) = req.headers().get("x-forwarded-for") + && let Ok(s) = val.to_str() + && let Some(first) = s.split(',').next() + { + let ip = first.trim(); + if !ip.is_empty() { + return ip.to_string(); + } + } + // Then X-Real-Ip + if let Some(val) = req.headers().get("x-real-ip") + && let Ok(s) = val.to_str() + { + let ip = s.trim(); + if !ip.is_empty() { + return ip.to_string(); + } + } + "unknown".to_string() +} + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { @@ -150,10 +173,7 @@ async fn contract_register_handler( State(config): State, req: Request, ) -> Result, StatusCode> { - let ip = SmartIpKeyExtractor - .extract(&req) - .map(|ip| ip.to_string()) - .unwrap_or_else(|_| "unknown".to_string()); + let ip = extract_client_ip(&req); let (parts, body) = req.into_parts(); let body_bytes = match axum::body::to_bytes(body, 1024 * 1024).await { @@ -195,10 +215,7 @@ async fn blob_proxy_handler( req: Request, ) -> Result, StatusCode> { // Extract IP for logging - let ip = SmartIpKeyExtractor - .extract(&req) - .map(|ip| ip.to_string()) - .unwrap_or_else(|_| "unknown".to_string()); + let ip = extract_client_ip(&req); // Extract and parse the request body let (parts, body) = req.into_parts(); diff --git a/tools/analyze-deps.py b/tools/analyze-deps.py new file mode 100755 index 000000000..1b8327980 --- /dev/null +++ b/tools/analyze-deps.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Analyze cargo dependency tree to find the heaviest dependencies. + +Computes the *exclusive* dependency count for each direct dependency: +how many crates would be removed from the build if you dropped that dep. +This is the actionable metric — not the total transitive count. + +Usage: python3 tools/analyze-deps.py [--top N] [--no-workspace] [--json] + +Run from the project root (where Cargo.toml lives). +Requires: cargo +""" + +import argparse +import json +import re +import subprocess +import sys +from collections import defaultdict + + +def run(cmd): + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode != 0: + print(f"Error running: {' '.join(cmd)}", file=sys.stderr) + print(r.stderr, file=sys.stderr) + sys.exit(1) + return r.stdout + + +def get_workspace_members(): + meta = json.loads(run(["cargo", "metadata", "--no-deps", "--format-version=1"])) + return {p["name"] for p in meta["packages"]} + + +def parse_tree_output(output): + """Extract unique crate names from `cargo tree --prefix none` output.""" + crates = set() + for line in output.strip().split("\n")[1:]: + line = line.strip() + if line.startswith("["): + continue + m = re.match(r"^(\S+) v\S+", line) + if m: + crates.add(m.group(1)) + return crates + + +def get_all_deps_from_tree(): + """Parse the full cargo tree once and return (direct_deps, dep_sets). + + direct_deps: list of (name, full_spec, is_first_occurrence) + dep_sets: dict name -> set of transitive dep names + + This parses the single `cargo tree` output by tracking indentation depth, + which avoids running cargo tree N times (one per dep). + """ + output = run(["cargo", "tree", "--prefix", "depth", "-e", "normal"]) + # --prefix depth outputs lines like "0hyli v0.15.0", "1anyhow v1.0.102", "2foo v0.1.0" + + direct_deps = [] # (name, full_spec) + seen_direct = set() + + # For each direct dep, collect all transitive deps by tracking the subtree + dep_sets = {} + current_direct = None + # Stack to track which direct dep "owns" deeper entries + # We only care about depth >= 1 entries that fall under a depth-1 entry + + for line in output.strip().split("\n"): + m = re.match(r"^(\d+)(\S+) (v\S+)", line) + if not m: + continue + depth = int(m.group(1)) + name = m.group(2) + ver = m.group(3) + + if depth == 0: + continue # root crate + + if depth == 1: + current_direct = name + if name not in seen_direct: + seen_direct.add(name) + direct_deps.append((name, f"{name} {ver}")) + dep_sets[name] = set() + elif current_direct and current_direct in dep_sets: + dep_sets[current_direct].add(name) + + return direct_deps, dep_sets + + +def count_duplicated_versions(): + """Find crates that appear with multiple versions.""" + output = run(["cargo", "tree", "--depth", "0", "--prefix", "none", "-e", "normal", "--duplicates"]) + versions = defaultdict(set) + for line in output.strip().split("\n"): + m = re.match(r"^(\S+) (v\S+)", line.strip()) + if m: + versions[m.group(1)].add(m.group(2)) + return {k: sorted(v) for k, v in versions.items() if len(v) > 1} + + +def main(): + parser = argparse.ArgumentParser(description="Analyze cargo dependency weight") + parser.add_argument("--top", type=int, default=20, help="Number of deps to show (default: 20)") + parser.add_argument("--no-workspace", action="store_true", help="Hide workspace crates") + parser.add_argument("--json", action="store_true", help="Output as JSON") + args = parser.parse_args() + + ws_names = get_workspace_members() + + print("Parsing dependency tree...", file=sys.stderr) + direct_deps, dep_sets = get_all_deps_from_tree() + print(f"Found {len(direct_deps)} direct dependencies.", file=sys.stderr) + + # Count how many direct deps pull in each transitive dep + # A dep is "exclusive" to crate X if no other direct dep also needs it + dep_users = defaultdict(set) # transitive_dep -> set of direct deps that need it + for direct_name, trans_deps in dep_sets.items(): + for t in trans_deps: + dep_users[t].add(direct_name) + + # For each direct dep, count exclusive deps (only pulled by this one) + results = [] + all_transitive = set() + for name, full in direct_deps: + trans = dep_sets.get(name, set()) + all_transitive |= trans + exclusive = {t for t in trans if len(dep_users[t]) == 1} + ext_trans = {t for t in trans if t not in ws_names} + ext_exclusive = {t for t in exclusive if t not in ws_names} + results.append({ + "name": name, + "full": full, + "is_workspace": name in ws_names, + "total_deps": len(trans), + "external_deps": len(ext_trans), + "exclusive_deps": len(exclusive), + "exclusive_external": len(ext_exclusive), + "exclusive_list": sorted(ext_exclusive), + }) + + external = [r for r in results if not r["is_workspace"]] + workspace = [r for r in results if r["is_workspace"]] + + external.sort(key=lambda x: -x["exclusive_external"]) + workspace.sort(key=lambda x: -x["exclusive_external"]) + + duplicates = count_duplicated_versions() + + if args.json: + print(json.dumps({ + "external": external[:args.top], + "workspace": [] if args.no_workspace else workspace[:args.top], + "duplicated_versions": duplicates, + "summary": { + "direct_deps": len(direct_deps), + "total_unique_transitive": len(all_transitive), + "external_count": len(external), + "workspace_count": len(workspace), + "duplicated_crates": len(duplicates), + }, + }, indent=2)) + return + + # --- Pretty print --- + + print() + print("=" * 72) + print(" CARGO DEPENDENCY ANALYSIS") + print(f" Direct deps: {len(direct_deps)} | " + f"Unique transitive: {len(all_transitive)} | " + f"Duplicated crates: {len(duplicates)}") + print("=" * 72) + + print() + print(f" EXTERNAL DEPENDENCIES (top {args.top})") + print(f" {'Crate':<30} {'Total':>6} {'Exclusive':>10} Exclusive deps") + print(" " + "-" * 68) + for e in external[:args.top]: + bar = "█" * e["exclusive_external"] + extras = "" + if e["exclusive_list"]: + shown = e["exclusive_list"][:5] + extras = ", ".join(shown) + if len(e["exclusive_list"]) > 5: + extras += f", +{len(e['exclusive_list']) - 5} more" + print(f" {e['full']:<30} {e['external_deps']:>6} {e['exclusive_external']:>10} {bar}") + if extras: + print(f" {'':>48} └ {extras}") + + if not args.no_workspace: + print() + print(f" WORKSPACE CRATES (top {args.top})") + print(f" {'Crate':<30} {'Total':>6} {'Exclusive':>10}") + print(" " + "-" * 48) + for e in workspace[:args.top]: + bar = "█" * e["exclusive_external"] + print(f" {e['full']:<30} {e['total_deps']:>6} {e['exclusive_external']:>10} {bar}") + + if duplicates: + print() + print(f" DUPLICATED CRATES ({len(duplicates)} crates with multiple versions)") + print(" " + "-" * 52) + for name, versions in sorted(duplicates.items()): + print(f" {name:<30} {', '.join(versions)}") + + print() + + +if __name__ == "__main__": + main()