diff --git a/crates/agentgateway/src/cel/types.rs b/crates/agentgateway/src/cel/types.rs index 3a979941a8..fc5fd10dbc 100644 --- a/crates/agentgateway/src/cel/types.rs +++ b/crates/agentgateway/src/cel/types.rs @@ -1518,6 +1518,14 @@ pub struct LLMContext { #[dynamic(rename = "cacheCreationInputTokens")] #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_tokens: Option, + /// Per-TTL cache-write split when the provider reported it (None = no + /// split — consumers fail closed rather than approximate). + #[dynamic(rename = "cacheCreation5mInputTokens")] + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_5m_input_tokens: Option, + #[dynamic(rename = "cacheCreation1hInputTokens")] + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_1h_input_tokens: Option, /// The number of tokens in the output/completion. #[dynamic(rename = "outputTokens")] #[serde(skip_serializing_if = "Option::is_none")] @@ -1634,6 +1642,8 @@ impl LLMContext { input_audio_tokens: resp.input_audio_tokens, cached_input_tokens: resp.cached_input_tokens, cache_creation_input_tokens: resp.cache_creation_input_tokens, + cache_creation_5m_input_tokens: resp.cache_creation_5m_input_tokens, + cache_creation_1h_input_tokens: resp.cache_creation_1h_input_tokens, service_tier: resp.service_tier, usage_complete: resp.usage_complete, response_model: resp.provider_model, @@ -1738,6 +1748,8 @@ impl From for LLMContext { output_audio_tokens: None, total_tokens: None, provider_total_tokens: None, + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, completion: None, tool_calls: None, reasoning_tokens: None, @@ -2453,6 +2465,8 @@ pub fn full_example_executor() -> ExecutorSerde { input_audio_tokens: Some(5), cached_input_tokens: Some(20), cache_creation_input_tokens: Some(10), + cache_creation_5m_input_tokens: Some(4), + cache_creation_1h_input_tokens: Some(6), output_tokens: Some(50), output_image_tokens: Some(30), output_text_tokens: Some(20), diff --git a/crates/agentgateway/src/cel/types_test.rs b/crates/agentgateway/src/cel/types_test.rs index b20331ffe2..0e3e0fa2db 100644 --- a/crates/agentgateway/src/cel/types_test.rs +++ b/crates/agentgateway/src/cel/types_test.rs @@ -140,6 +140,8 @@ fn build_test_request() -> crate::http::Request { count_tokens: None, reasoning_tokens: None, cache_creation_input_tokens: None, + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, cached_input_tokens: None, prompt: None, completion: Some(vec!["Hello world".to_string()]), diff --git a/crates/agentgateway/src/llm/mod.rs b/crates/agentgateway/src/llm/mod.rs index 5d87bbe397..7c91f76a1c 100644 --- a/crates/agentgateway/src/llm/mod.rs +++ b/crates/agentgateway/src/llm/mod.rs @@ -2943,6 +2943,8 @@ impl AIProvider { llm_resp.total_tokens = native.total_tokens; llm_resp.cached_input_tokens = native.cached_input_tokens; llm_resp.cache_creation_input_tokens = native.cache_creation_input_tokens; + llm_resp.cache_creation_5m_input_tokens = native.cache_creation_5m_input_tokens; + llm_resp.cache_creation_1h_input_tokens = native.cache_creation_1h_input_tokens; llm_resp.reasoning_tokens = native.reasoning_tokens; llm_resp.input_audio_tokens = native.input_audio_tokens; llm_resp.output_audio_tokens = native.output_audio_tokens; diff --git a/crates/agentgateway/src/llm/usage_extraction_tests.rs b/crates/agentgateway/src/llm/usage_extraction_tests.rs index 2b5a1fdc82..dc1424a7c8 100644 --- a/crates/agentgateway/src/llm/usage_extraction_tests.rs +++ b/crates/agentgateway/src/llm/usage_extraction_tests.rs @@ -188,3 +188,31 @@ fn native_upstream_extraction_is_usage_only() { "usage-only conversion must not extract tool-call output" ); } + +/// Anthropic-native responses carry the per-TTL cache-write split in +/// `usage.cache_creation.ephemeral_5m/1h_input_tokens`. The extraction must +/// surface it for accounting (exact 5m/1h metering) while the client-facing +/// wire stays untouched. +#[test] +fn native_upstream_anthropic_cache_ttl_split_is_extracted() { + let bytes = Bytes::from( + r#"{"id":"msg_2","type":"message","role":"assistant","model":"claude-sonnet-5-5","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":5,"cache_creation_input_tokens":300,"cache_creation":{"ephemeral_5m_input_tokens":200,"ephemeral_1h_input_tokens":100},"cache_read_input_tokens":64}}"#, + ); + let native = AIProvider::native_upstream_llm_response(ChatFormat::AnthropicMessages, &bytes) + .expect("native parse"); + assert_eq!(native.cache_creation_input_tokens, Some(300)); + assert_eq!(native.cache_creation_5m_input_tokens, Some(200)); + assert_eq!(native.cache_creation_1h_input_tokens, Some(100)); + + // Aggregate-only (older provider response shape): split stays unknown — + // never inferred from the aggregate, never zero. + let aggregate_only = Bytes::from( + r#"{"id":"msg_3","type":"message","role":"assistant","model":"claude-sonnet-5-5","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":5,"cache_creation_input_tokens":300}}"#, + ); + let native = + AIProvider::native_upstream_llm_response(ChatFormat::AnthropicMessages, &aggregate_only) + .expect("native parse"); + assert_eq!(native.cache_creation_input_tokens, Some(300)); + assert_eq!(native.cache_creation_5m_input_tokens, None); + assert_eq!(native.cache_creation_1h_input_tokens, None); +} diff --git a/crates/agentgateway/src/parse/websocket.rs b/crates/agentgateway/src/parse/websocket.rs index 6664e7150c..cd9e0fd24d 100644 --- a/crates/agentgateway/src/parse/websocket.rs +++ b/crates/agentgateway/src/parse/websocket.rs @@ -84,6 +84,8 @@ impl Parser { pages: None, reasoning_tokens: None, cache_creation_input_tokens: None, + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, cached_input_tokens: usage .input_token_details .as_ref() @@ -572,6 +574,8 @@ pub async fn guarded_realtime_proxy( pages: None, reasoning_tokens: None, cache_creation_input_tokens: None, + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, cached_input_tokens: usage_clone .input_token_details .as_ref() diff --git a/crates/agentgateway/src/telemetry/log.rs b/crates/agentgateway/src/telemetry/log.rs index cf798b8dac..bb351705ab 100644 --- a/crates/agentgateway/src/telemetry/log.rs +++ b/crates/agentgateway/src/telemetry/log.rs @@ -1864,6 +1864,25 @@ impl Drop for DropOnLog { .and_then(|l| l.cache_creation_input_tokens) .map(Into::into), ), + // Per-TTL cache-write split (Accounting consumer contract — + // synapse-runtime accounting-tap `cache_write_5m/1h_tokens`). + // Absent (None) = the provider reported no split; the tap + // leaves the classes unknown and downstream settlement fails + // closed on ambiguous aggregates rather than approximating. + ( + "gen_ai.usage.cache_creation.input_tokens.5m", + llm_response + .as_ref() + .and_then(|l| l.cache_creation_5m_input_tokens) + .map(Into::into), + ), + ( + "gen_ai.usage.cache_creation.input_tokens.1h", + llm_response + .as_ref() + .and_then(|l| l.cache_creation_1h_input_tokens) + .map(Into::into), + ), ( "gen_ai.usage.cache_read.input_tokens", llm_response @@ -3609,6 +3628,121 @@ mod tests { } #[test] + /// The per-TTL cache-write split reaches the span attributes exactly as + /// reported (Accounting tap contract `cache_write_5m/1h_tokens`); an + /// aggregate-only response leaves the split attributes absent — never + /// zero, never inferred. + #[test] + fn llm_span_emits_cache_write_ttl_split_attributes() { + let request = llm::LLMRequest { + input_tokens: None, + input_format: InputFormat::Messages, + cache_convention: llm::CacheTokenConvention::InputExcludesCache, + request_model: strng::literal!("claude"), + provider: strng::literal!("bedrock"), + streaming: false, + params: llm::LLMRequestParams::default(), + prompt: None, + provider_state: None, + }; + let split = llm::LLMResponse { + input_tokens: Some(50), + cache_creation_input_tokens: Some(300), + cache_creation_5m_input_tokens: Some(200), + cache_creation_1h_input_tokens: Some(100), + output_tokens: Some(20), + total_tokens: Some(370), + ..Default::default() + }; + let aggregate_only = llm::LLMResponse { + input_tokens: Some(50), + cache_creation_input_tokens: Some(300), + output_tokens: Some(20), + total_tokens: Some(370), + ..Default::default() + }; + + let (tracer, exporter) = test_tracer(); + let mut log = test_request_log(); + log.tracer = Some(tracer.clone()); + let mut outgoing = trc::TraceParent::new(); + outgoing.flags = 1; + log.outgoing_span = Some(outgoing); + log.llm_request = Some(request.clone()); + log + .llm_response + .store(Some(llm::LLMInfo::new(request.clone(), split))); + drop(DropOnLog::from(log)); + let _ = tracer.provider.force_flush(); + let spans = exporter.finished_spans(); + let span = spans + .iter() + .find(|span| span.name.as_ref() == "unknown") + .expect("request span should be exported"); + let value = |key: &str| { + span + .attributes + .iter() + .find(|attr| attr.key.as_str() == key) + .map(|attr| &attr.value) + }; + assert_eq!( + value("gen_ai.usage.cache_creation.input_tokens.5m"), + Some(&opentelemetry::Value::I64(200)) + ); + assert_eq!( + value("gen_ai.usage.cache_creation.input_tokens.1h"), + Some(&opentelemetry::Value::I64(100)) + ); + assert_eq!( + value("gen_ai.usage.cache_creation.input_tokens"), + Some(&opentelemetry::Value::I64(300)) + ); + + // Aggregate-only producer: split attributes ABSENT, aggregate intact. + let (tracer, exporter) = test_tracer(); + let mut log = test_request_log(); + log.tracer = Some(tracer.clone()); + let mut outgoing = trc::TraceParent::new(); + outgoing.flags = 1; + log.outgoing_span = Some(outgoing); + log.llm_request = Some(request); + log.llm_response.store(Some(llm::LLMInfo::new( + llm::LLMRequest { + input_tokens: None, + input_format: InputFormat::Messages, + cache_convention: llm::CacheTokenConvention::InputExcludesCache, + request_model: strng::literal!("claude"), + provider: strng::literal!("bedrock"), + streaming: false, + params: llm::LLMRequestParams::default(), + prompt: None, + provider_state: None, + }, + aggregate_only, + ))); + drop(DropOnLog::from(log)); + let _ = tracer.provider.force_flush(); + let spans = exporter.finished_spans(); + let span = spans + .iter() + .find(|span| span.name.as_ref() == "unknown") + .expect("request span should be exported"); + let value = |key: &str| { + span + .attributes + .iter() + .find(|attr| attr.key.as_str() == key) + .map(|attr| &attr.value) + }; + assert_eq!(value("gen_ai.usage.cache_creation.input_tokens.5m"), None); + assert_eq!(value("gen_ai.usage.cache_creation.input_tokens.1h"), None); + assert_eq!( + value("gen_ai.usage.cache_creation.input_tokens"), + Some(&opentelemetry::Value::I64(300)) + ); + } + fn llm_span_uses_cache_inclusive_input_tokens() { let request = llm::LLMRequest { input_tokens: None, diff --git a/crates/llm/src/conversion/bedrock.rs b/crates/llm/src/conversion/bedrock.rs index 2895b0a7d8..3f7637f40c 100644 --- a/crates/llm/src/conversion/bedrock.rs +++ b/crates/llm/src/conversion/bedrock.rs @@ -307,14 +307,38 @@ fn invalid_request_error(bytes: &[u8]) -> Result { impl From for super::ProviderUsage { fn from(u: bedrock::TokenUsage) -> Self { + let (cache_creation_5m_input_tokens, cache_creation_1h_input_tokens) = + cache_ttl_split(u.cache_details.as_deref()); Self { input_tokens: u.input_tokens as u64, total_tokens: u.total_tokens as u64, + cache_creation_5m_input_tokens, + cache_creation_1h_input_tokens, ..Default::default() } } } +/// Per-TTL cache-write split from AWS `TokenUsage.cacheDetails` (sorted 1h +/// before 5m; empty/absent = aggregate only). Unknown TTL classes are +/// counted under NEITHER canonical class — the aggregate attribute still +/// carries them, and the accounting consumer fails closed on ambiguous +/// meters rather than approximating. +pub(crate) fn cache_ttl_split( + details: Option<&[bedrock::CacheDetail]>, +) -> (Option, Option) { + let mut five_m = None; + let mut one_h = None; + for detail in details.unwrap_or(&[]) { + match detail.ttl { + bedrock::CacheTtl::FiveMinutes => five_m = Some(detail.input_tokens as u64), + bedrock::CacheTtl::OneHour => one_h = Some(detail.input_tokens as u64), + bedrock::CacheTtl::Unknown => {}, + } + } + (five_m, one_h) +} + pub mod from_rerank { use crate::bedrock::Provider; use crate::types::ResponseType; @@ -1191,7 +1215,7 @@ pub mod from_completions { ) -> Result, AIError> { let resp = serde_json::from_slice::(bytes) .map_err(logged_response_parsing(bytes))?; - let provider_usage = resp.usage.map(super::super::ProviderUsage::from); + let provider_usage = resp.usage.clone().map(super::super::ProviderUsage::from); let openai = translate_response_internal(resp, model, tool_name_map)?; let passthrough = json::convert::<_, types::completions::Response>(&openai) .map_err(AIError::ResponseParsing)?; @@ -1428,6 +1452,8 @@ pub mod from_completions { }, bedrock::ConverseStreamOutput::Metadata(metadata) => { if let Some(usage) = metadata.usage { + let (cache_creation_5m_input_tokens, cache_creation_1h_input_tokens) = + super::cache_ttl_split(usage.cache_details.as_deref()); log.update(|r| { r.response.output_tokens = Some(usage.output_tokens as u64); r.response.input_tokens = Some(usage.input_tokens as u64); @@ -1435,6 +1461,8 @@ pub mod from_completions { r.response.cached_input_tokens = usage.cache_read_input_tokens.map(|i| i as u64); r.response.cache_creation_input_tokens = usage.cache_write_input_tokens.map(|i| i as u64); + r.response.cache_creation_5m_input_tokens = cache_creation_5m_input_tokens; + r.response.cache_creation_1h_input_tokens = cache_creation_1h_input_tokens; if let Some(completion) = completion.take() { r.response.completion = Some(vec![completion]); } @@ -2010,7 +2038,7 @@ pub mod from_messages { ) -> Result, AIError> { let resp = serde_json::from_slice::(bytes) .map_err(logged_response_parsing(bytes))?; - let provider_usage = resp.usage.map(super::super::ProviderUsage::from); + let provider_usage = resp.usage.clone().map(super::super::ProviderUsage::from); let openai = translate_response_internal(resp, model, tool_name_map)?; let passthrough = json::convert::<_, types::messages::Response>(&openai).map_err(AIError::ResponseParsing)?; @@ -2250,7 +2278,9 @@ pub mod from_messages { }, bedrock::ConverseStreamOutput::Metadata(meta) => { if let Some(usage) = meta.usage { - pending_usage = Some(usage); + pending_usage = Some(usage.clone()); + let (cache_creation_5m_input_tokens, cache_creation_1h_input_tokens) = + super::cache_ttl_split(usage.cache_details.as_deref()); log.update(|r| { r.response.output_tokens = Some(usage.output_tokens as u64); r.response.input_tokens = Some(usage.input_tokens as u64); @@ -2258,6 +2288,8 @@ pub mod from_messages { r.response.cached_input_tokens = usage.cache_read_input_tokens.map(|i| i as u64); r.response.cache_creation_input_tokens = usage.cache_write_input_tokens.map(|i| i as u64); + r.response.cache_creation_5m_input_tokens = cache_creation_5m_input_tokens; + r.response.cache_creation_1h_input_tokens = cache_creation_1h_input_tokens; if let Some(c) = completion.take() { r.response.completion = Some(vec![c]); } @@ -3143,7 +3175,7 @@ pub mod from_responses { ) -> Result, AIError> { let resp = serde_json::from_slice::(bytes) .map_err(logged_response_parsing(bytes))?; - let provider_usage = resp.usage.map(super::super::ProviderUsage::from); + let provider_usage = resp.usage.clone().map(super::super::ProviderUsage::from); let adapter = super::ConverseResponseAdapter::from_response(resp, model)?; let mut typed = adapter.to_responses_typed(tool_name_map); if let Some(namespaces) = namespaces { @@ -3471,7 +3503,9 @@ pub mod from_responses { }, bedrock::ConverseStreamOutput::Metadata(meta) => { if let Some(usage) = meta.usage { - pending_usage = Some(usage); + pending_usage = Some(usage.clone()); + let (cache_creation_5m_input_tokens, cache_creation_1h_input_tokens) = + super::cache_ttl_split(usage.cache_details.as_deref()); log.update(|r| { r.response.output_tokens = Some(usage.output_tokens as u64); r.response.input_tokens = Some(usage.input_tokens as u64); @@ -3479,6 +3513,8 @@ pub mod from_responses { r.response.cached_input_tokens = usage.cache_read_input_tokens.map(|i| i as u64); r.response.cache_creation_input_tokens = usage.cache_write_input_tokens.map(|i| i as u64); + r.response.cache_creation_5m_input_tokens = cache_creation_5m_input_tokens; + r.response.cache_creation_1h_input_tokens = cache_creation_1h_input_tokens; }); } @@ -4072,6 +4108,7 @@ impl ConverseResponseAdapter { let usage = self .usage + .clone() .map(|token_usage| { let input_tokens = token_usage.input_tokens + token_usage.cache_read_input_tokens.unwrap_or_default() @@ -4237,7 +4274,7 @@ impl ConverseResponseAdapter { }; // Build usage - let usage = self.usage.map(|u| { + let usage = self.usage.clone().map(|u| { let input_tokens = u.input_tokens + u.cache_read_input_tokens.unwrap_or_default() + u.cache_write_input_tokens.unwrap_or_default(); @@ -4328,6 +4365,7 @@ impl ConverseResponseAdapter { let usage = self .usage + .clone() .map(|u| messagest::Usage { input_tokens: u.input_tokens, output_tokens: u.output_tokens, diff --git a/crates/llm/src/conversion/bedrock_tests.rs b/crates/llm/src/conversion/bedrock_tests.rs index 26d17b7395..2310f95524 100644 --- a/crates/llm/src/conversion/bedrock_tests.rs +++ b/crates/llm/src/conversion/bedrock_tests.rs @@ -2058,3 +2058,135 @@ fn test_responses_input_file_unknown_format_is_rejected() { "unexpected error: {err}" ); } + +// --- Cache-write TTL split (AWS TokenUsage.cacheDetails) — P0-A ------------ + +#[test] +fn test_token_usage_parses_cache_details_per_ttl_class() { + use crate::types::bedrock::{CacheTtl, TokenUsage}; + + // Mixed 5m + 1h (the API returns them sorted 1h before 5m). + let mixed: TokenUsage = serde_json::from_value(json!({ + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cacheReadInputTokens": 7, + "cacheWriteInputTokens": 300, + "cacheDetails": [ + { "ttl": "1h", "inputTokens": 100 }, + { "ttl": "5m", "inputTokens": 200 } + ] + })) + .unwrap(); + let details = mixed.cache_details.unwrap(); + assert_eq!(details.len(), 2); + assert_eq!(details[0].ttl, CacheTtl::OneHour); + assert_eq!(details[0].input_tokens, 100); + assert_eq!(details[1].ttl, CacheTtl::FiveMinutes); + assert_eq!(details[1].input_tokens, 200); + + // Aggregate-only (older models / no TTL breakdown reported). + let aggregate_only: TokenUsage = serde_json::from_value(json!({ + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cacheWriteInputTokens": 300 + })) + .unwrap(); + assert!(aggregate_only.cache_details.is_none()); +} + +#[test] +fn test_cache_ttl_split_maps_each_class_and_ignores_unknown() { + use crate::types::bedrock::{CacheDetail, CacheTtl, TokenUsage}; + + let usage = TokenUsage { + input_tokens: 1, + output_tokens: 1, + total_tokens: 2, + cache_read_input_tokens: None, + cache_write_input_tokens: Some(300), + cache_details: Some(vec![ + CacheDetail { + ttl: CacheTtl::FiveMinutes, + input_tokens: 200, + }, + CacheDetail { + ttl: CacheTtl::OneHour, + input_tokens: 100, + }, + CacheDetail { + ttl: CacheTtl::Unknown, + input_tokens: 5, + }, + ]), + }; + let provider_usage: crate::conversion::ProviderUsage = usage.into(); + assert_eq!(provider_usage.cache_creation_5m_input_tokens, Some(200)); + assert_eq!(provider_usage.cache_creation_1h_input_tokens, Some(100)); + + // Aggregate-only → split stays None (never guessed, never zero). + let aggregate_only = TokenUsage { + input_tokens: 1, + output_tokens: 1, + total_tokens: 2, + cache_read_input_tokens: None, + cache_write_input_tokens: Some(300), + cache_details: None, + }; + let provider_usage: crate::conversion::ProviderUsage = aggregate_only.into(); + assert_eq!(provider_usage.cache_creation_5m_input_tokens, None); + assert_eq!(provider_usage.cache_creation_1h_input_tokens, None); +} + +#[test] +fn test_converse_response_carries_ttl_split_into_llm_response() { + // Buffered messages-format translation (the Anthropic-messages → Bedrock + // Converse path): the TTL split must survive into the telemetry response. + let model = "eu.anthropic.claude-sonnet-5-5"; + let bedrock_resp = json!({ + "output": { + "message": { + "role": "assistant", + "content": [{ "text": "hello" }] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1000, + "outputTokens": 100, + "totalTokens": 1400, + "cacheReadInputTokens": 300, + "cacheWriteInputTokens": 300, + "cacheDetails": [ + { "ttl": "1h", "inputTokens": 100 }, + { "ttl": "5m", "inputTokens": 200 } + ] + } + }); + let bytes = serde_json::to_vec(&bedrock_resp).unwrap(); + + let translated = from_messages::translate_response(&Bytes::from(bytes), model, None).unwrap(); + let llm = translated.to_llm_response(crate::LogContentFields::default()); + assert_eq!(llm.cache_creation_input_tokens, Some(300)); + assert_eq!(llm.cache_creation_5m_input_tokens, Some(200)); + assert_eq!(llm.cache_creation_1h_input_tokens, Some(100)); + + // Aggregate-only response → split stays None on the telemetry response. + let aggregate_only = json!({ + "output": { "message": { "role": "assistant", "content": [{ "text": "hi" }] } }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1000, + "outputTokens": 100, + "totalTokens": 1300, + "cacheWriteInputTokens": 300 + } + }); + let bytes = serde_json::to_vec(&aggregate_only).unwrap(); + let translated = from_messages::translate_response(&Bytes::from(bytes), model, None).unwrap(); + let llm = translated.to_llm_response(crate::LogContentFields::default()); + assert_eq!(llm.cache_creation_input_tokens, Some(300)); + assert_eq!(llm.cache_creation_5m_input_tokens, None); + assert_eq!(llm.cache_creation_1h_input_tokens, None); +} diff --git a/crates/llm/src/conversion/completions.rs b/crates/llm/src/conversion/completions.rs index c9ce0633cb..39b08c917a 100644 --- a/crates/llm/src/conversion/completions.rs +++ b/crates/llm/src/conversion/completions.rs @@ -122,7 +122,9 @@ pub mod from_messages { .completion_tokens_details .as_ref() .and_then(|d| d.reasoning_tokens), + ..Default::default() }); + let anthropic = translate_response_internal(resp)?; Ok(Box::new(super::super::ResponseWithProviderUsage { response: anthropic, diff --git a/crates/llm/src/conversion/mod.rs b/crates/llm/src/conversion/mod.rs index 613fb71192..b95de8ce6a 100644 --- a/crates/llm/src/conversion/mod.rs +++ b/crates/llm/src/conversion/mod.rs @@ -13,6 +13,10 @@ struct ProviderUsage { input_tokens: u64, total_tokens: u64, reasoning_tokens: Option, + /// Provider-reported cache-write split per TTL class. `None` = the + /// provider reported no split (aggregate only) — never guessed. + cache_creation_5m_input_tokens: Option, + cache_creation_1h_input_tokens: Option, } /// Keep provider token counts for accounting when translating client usage conventions. @@ -28,6 +32,12 @@ impl crate::types::ResponseType for ResponseWithP response.input_tokens = Some(usage.input_tokens); response.total_tokens = Some(usage.total_tokens); response.reasoning_tokens = usage.reasoning_tokens.or(response.reasoning_tokens); + response.cache_creation_5m_input_tokens = usage + .cache_creation_5m_input_tokens + .or(response.cache_creation_5m_input_tokens); + response.cache_creation_1h_input_tokens = usage + .cache_creation_1h_input_tokens + .or(response.cache_creation_1h_input_tokens); } response } diff --git a/crates/llm/src/conversion/responses.rs b/crates/llm/src/conversion/responses.rs index 24540d8c19..495a3343d4 100644 --- a/crates/llm/src/conversion/responses.rs +++ b/crates/llm/src/conversion/responses.rs @@ -735,7 +735,9 @@ pub mod from_messages { input_tokens: u.input_tokens as u64, total_tokens: u.total_tokens as u64, reasoning_tokens: Some(u.output_tokens_details.reasoning_tokens as u64), + ..Default::default() }); + let anthropic = translate_response_internal(resp)?; Ok(Box::new(super::super::ResponseWithProviderUsage { response: anthropic, diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 45cc106db6..dec959fed1 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -394,6 +394,17 @@ pub struct LLMResponse { pub reasoning_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_tokens: Option, + /// Cache-write tokens with a 5-minute TTL class, when the provider + /// reported the per-class split (Bedrock `cacheDetails[ttl=5m]`, Anthropic + /// `usage.cache_creation.ephemeral_5m_input_tokens`). `None` = no split + /// reported — consumers must fail closed on ambiguous aggregates rather + /// than approximate the class. + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_5m_input_tokens: Option, + /// Cache-write tokens with a 1-hour TTL class, when the provider reported + /// the per-class split. + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_1h_input_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cached_input_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/llm/src/tests/response/anthropic/thinking.messages-messages.snap b/crates/llm/src/tests/response/anthropic/thinking.messages-messages.snap index b7f493b0f0..80cb4bf981 100644 --- a/crates/llm/src/tests/response/anthropic/thinking.messages-messages.snap +++ b/crates/llm/src/tests/response/anthropic/thinking.messages-messages.snap @@ -60,6 +60,8 @@ info: "output_tokens": 251, "total_tokens": 298, "cache_creation_input_tokens": 1, + "cache_creation_5m_input_tokens": 0, + "cache_creation_1h_input_tokens": 0, "cached_input_tokens": 2, "service_tier": "standard", "provider_model": "claude-opus-4-6" diff --git a/crates/llm/src/types/bedrock.rs b/crates/llm/src/types/bedrock.rs index 324e349437..6792a9ec7c 100644 --- a/crates/llm/src/types/bedrock.rs +++ b/crates/llm/src/types/bedrock.rs @@ -361,8 +361,34 @@ pub enum ConverseOutput { Unknown, } -/// Token usage information +/// TTL class of a cache-creation breakdown entry (AWS `CacheDetail.ttl`). +#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum CacheTtl { + /// 5-minute TTL class. + #[serde(rename = "5m")] + FiveMinutes, + /// 1-hour TTL class. + #[serde(rename = "1h")] + OneHour, + /// Unrecognized class — reported by the provider, never mapped. + #[serde(other)] + Unknown, +} + +/// Per-TTL cache-creation metrics (AWS `CacheDetail`): how many tokens were +/// written to cache under each TTL class. #[derive(Copy, Clone, Debug, Serialize, Deserialize)] +pub struct CacheDetail { + /// TTL duration for these cached tokens. + #[serde(rename = "ttl")] + pub ttl: CacheTtl, + /// Number of tokens written to cache with this TTL. + #[serde(rename = "inputTokens")] + pub input_tokens: usize, +} + +/// Token usage information +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct TokenUsage { /// The number of input tokens which were used #[serde(rename = "inputTokens")] @@ -385,6 +411,12 @@ pub struct TokenUsage { skip_serializing_if = "Option::is_none" )] pub cache_write_input_tokens: Option, + /// Per-TTL cache-write breakdown (AWS `cacheDetails`; the API returns it + /// sorted 1h before 5m). Absent/empty = only the aggregate + /// `cacheWriteInputTokens` was reported — consumers must not guess the + /// split. + #[serde(rename = "cacheDetails", skip_serializing_if = "Option::is_none")] + pub cache_details: Option>, } /// Metrics for the Converse call diff --git a/crates/llm/src/types/completions.rs b/crates/llm/src/types/completions.rs index cf7221890f..2274b605e0 100644 --- a/crates/llm/src/types/completions.rs +++ b/crates/llm/src/types/completions.rs @@ -210,6 +210,9 @@ impl ResponseType for Response { .and_then(|d| d.cache_write_tokens) .or(u.cache_creation_input_tokens) }), + // OpenAI-style usage carries no TTL classes — no split reported. + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, service_tier: self.service_tier.as_deref().map(Into::into), provider_model: Some(strng::new(&self.model)), completion: if log_content.completion { diff --git a/crates/llm/src/types/detect.rs b/crates/llm/src/types/detect.rs index 5938761850..0d510d2e81 100644 --- a/crates/llm/src/types/detect.rs +++ b/crates/llm/src/types/detect.rs @@ -619,6 +619,9 @@ impl ResponseType for Response { reasoning_tokens: self.lookup(lookups::REASONING, |v| v.as_u64()), cache_creation_input_tokens: self .lookup(lookups::CACHE_CREATION_INPUT_TOKENS, |v| v.as_u64()), + // Raw-JSON detection has no TTL-class lookups — no split reported. + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, cached_input_tokens: self.lookup(lookups::CACHED_INPUT_TOKENS, |v| v.as_u64()), service_tier: self .lookup(lookups::SERVICE_TIER, |v| v.as_str()) diff --git a/crates/llm/src/types/messages.rs b/crates/llm/src/types/messages.rs index f7765b4bb7..7bc1f84aa2 100644 --- a/crates/llm/src/types/messages.rs +++ b/crates/llm/src/types/messages.rs @@ -162,6 +162,25 @@ pub struct Usage { pub rest: serde_json::Value, } +impl Usage { + /// Per-TTL cache-write split from the pass-through `usage.cache_creation` + /// object (Anthropic-native `ephemeral_5m_input_tokens` / + /// `ephemeral_1h_input_tokens`). Read from `rest` so the client-facing + /// wire format stays byte-identical — no re-parsed typed field, no key + /// reordering. `None` = the provider reported no split; consumers must + /// not guess the class from the aggregate. + pub fn cache_ttl_split(&self) -> (Option, Option) { + let cache_creation = self.rest.get("cache_creation"); + let five_m = cache_creation + .and_then(|c| c.get("ephemeral_5m_input_tokens")) + .and_then(|v| v.as_u64()); + let one_h = cache_creation + .and_then(|c| c.get("ephemeral_1h_input_tokens")) + .and_then(|v| v.as_u64()); + (five_m, one_h) + } +} + pub fn get_messages_helper( messages: &[RequestMessage], system: &Option, @@ -589,6 +608,8 @@ impl ResponseType for Response { count_tokens: None, reasoning_tokens: None, cache_creation_input_tokens: self.usage.cache_creation_input_tokens, + cache_creation_5m_input_tokens: self.usage.cache_ttl_split().0, + cache_creation_1h_input_tokens: self.usage.cache_ttl_split().1, cached_input_tokens: self.usage.cache_read_input_tokens, service_tier: self.usage.service_tier.as_deref().map(Into::into), completion: if log_content.completion { @@ -1365,6 +1386,9 @@ pub mod typed { pages: None, reasoning_tokens: None, cache_creation_input_tokens: self.usage.cache_creation_input_tokens.map(|i| i as u64), + // Aggregate-only in this accumulated shape — no split reported. + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, cached_input_tokens: self.usage.cache_read_input_tokens.map(|i| i as u64), service_tier: self.usage.service_tier.as_deref().map(Into::into), provider_model: Some(agent_core::strng::new(&self.model)), diff --git a/crates/llm/src/types/responses.rs b/crates/llm/src/types/responses.rs index b84dbe57a4..524aabc72e 100644 --- a/crates/llm/src/types/responses.rs +++ b/crates/llm/src/types/responses.rs @@ -844,6 +844,9 @@ impl ResponseType for Response { .as_ref() .and_then(|d| d.cache_write_tokens) }), + // OpenAI-style usage carries no TTL classes — no split reported. + cache_creation_5m_input_tokens: None, + cache_creation_1h_input_tokens: None, service_tier: self.service_tier.as_deref().map(Into::into), provider_model: Some(strng::new(&self.model)), completion: if log_content.completion {