Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions crates/agentgateway/src/cel/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1518,6 +1518,14 @@ pub struct LLMContext {
#[dynamic(rename = "cacheCreationInputTokens")]
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_tokens: Option<u64>,
/// Per-TTL cache-write split when the provider reported it (None = no
/// split — consumers fail closed rather than approximate).
#[dynamic(rename = "cacheCreation5mInputTokens")]
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_5m_input_tokens: Option<u64>,
#[dynamic(rename = "cacheCreation1hInputTokens")]
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_1h_input_tokens: Option<u64>,
/// The number of tokens in the output/completion.
#[dynamic(rename = "outputTokens")]
#[serde(skip_serializing_if = "Option::is_none")]
Expand Down Expand Up @@ -1634,6 +1642,8 @@ impl LLMContext {
input_audio_tokens: resp.input_audio_tokens,
cached_input_tokens: resp.cached_input_tokens,
cache_creation_input_tokens: resp.cache_creation_input_tokens,
cache_creation_5m_input_tokens: resp.cache_creation_5m_input_tokens,
cache_creation_1h_input_tokens: resp.cache_creation_1h_input_tokens,
service_tier: resp.service_tier,
usage_complete: resp.usage_complete,
response_model: resp.provider_model,
Expand Down Expand Up @@ -1738,6 +1748,8 @@ impl From<llm::LLMRequest> for LLMContext {
output_audio_tokens: None,
total_tokens: None,
provider_total_tokens: None,
cache_creation_5m_input_tokens: None,
cache_creation_1h_input_tokens: None,
completion: None,
tool_calls: None,
reasoning_tokens: None,
Expand Down Expand Up @@ -2453,6 +2465,8 @@ pub fn full_example_executor() -> ExecutorSerde {
input_audio_tokens: Some(5),
cached_input_tokens: Some(20),
cache_creation_input_tokens: Some(10),
cache_creation_5m_input_tokens: Some(4),
cache_creation_1h_input_tokens: Some(6),
output_tokens: Some(50),
output_image_tokens: Some(30),
output_text_tokens: Some(20),
Expand Down
2 changes: 2 additions & 0 deletions crates/agentgateway/src/cel/types_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,8 @@ fn build_test_request() -> crate::http::Request {
count_tokens: None,
reasoning_tokens: None,
cache_creation_input_tokens: None,
cache_creation_5m_input_tokens: None,
cache_creation_1h_input_tokens: None,
cached_input_tokens: None,
prompt: None,
completion: Some(vec!["Hello world".to_string()]),
Expand Down
2 changes: 2 additions & 0 deletions crates/agentgateway/src/llm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2943,6 +2943,8 @@ impl AIProvider {
llm_resp.total_tokens = native.total_tokens;
llm_resp.cached_input_tokens = native.cached_input_tokens;
llm_resp.cache_creation_input_tokens = native.cache_creation_input_tokens;
llm_resp.cache_creation_5m_input_tokens = native.cache_creation_5m_input_tokens;
llm_resp.cache_creation_1h_input_tokens = native.cache_creation_1h_input_tokens;
llm_resp.reasoning_tokens = native.reasoning_tokens;
llm_resp.input_audio_tokens = native.input_audio_tokens;
llm_resp.output_audio_tokens = native.output_audio_tokens;
Expand Down
28 changes: 28 additions & 0 deletions crates/agentgateway/src/llm/usage_extraction_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,3 +188,31 @@ fn native_upstream_extraction_is_usage_only() {
"usage-only conversion must not extract tool-call output"
);
}

/// Anthropic-native responses carry the per-TTL cache-write split in
/// `usage.cache_creation.ephemeral_5m/1h_input_tokens`. The extraction must
/// surface it for accounting (exact 5m/1h metering) while the client-facing
/// wire stays untouched.
#[test]
fn native_upstream_anthropic_cache_ttl_split_is_extracted() {
let bytes = Bytes::from(
r#"{"id":"msg_2","type":"message","role":"assistant","model":"claude-sonnet-5-5","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":5,"cache_creation_input_tokens":300,"cache_creation":{"ephemeral_5m_input_tokens":200,"ephemeral_1h_input_tokens":100},"cache_read_input_tokens":64}}"#,
);
let native = AIProvider::native_upstream_llm_response(ChatFormat::AnthropicMessages, &bytes)
.expect("native parse");
assert_eq!(native.cache_creation_input_tokens, Some(300));
assert_eq!(native.cache_creation_5m_input_tokens, Some(200));
assert_eq!(native.cache_creation_1h_input_tokens, Some(100));

// Aggregate-only (older provider response shape): split stays unknown —
// never inferred from the aggregate, never zero.
let aggregate_only = Bytes::from(
r#"{"id":"msg_3","type":"message","role":"assistant","model":"claude-sonnet-5-5","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":5,"cache_creation_input_tokens":300}}"#,
);
let native =
AIProvider::native_upstream_llm_response(ChatFormat::AnthropicMessages, &aggregate_only)
.expect("native parse");
assert_eq!(native.cache_creation_input_tokens, Some(300));
assert_eq!(native.cache_creation_5m_input_tokens, None);
assert_eq!(native.cache_creation_1h_input_tokens, None);
}
4 changes: 4 additions & 0 deletions crates/agentgateway/src/parse/websocket.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ impl<IO> Parser<IO> {
pages: None,
reasoning_tokens: None,
cache_creation_input_tokens: None,
cache_creation_5m_input_tokens: None,
cache_creation_1h_input_tokens: None,
cached_input_tokens: usage
.input_token_details
.as_ref()
Expand Down Expand Up @@ -572,6 +574,8 @@ pub async fn guarded_realtime_proxy<C, S>(
pages: None,
reasoning_tokens: None,
cache_creation_input_tokens: None,
cache_creation_5m_input_tokens: None,
cache_creation_1h_input_tokens: None,
cached_input_tokens: usage_clone
.input_token_details
.as_ref()
Expand Down
134 changes: 134 additions & 0 deletions crates/agentgateway/src/telemetry/log.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1864,6 +1864,25 @@ impl Drop for DropOnLog {
.and_then(|l| l.cache_creation_input_tokens)
.map(Into::into),
),
// Per-TTL cache-write split (Accounting consumer contract —
// synapse-runtime accounting-tap `cache_write_5m/1h_tokens`).
// Absent (None) = the provider reported no split; the tap
// leaves the classes unknown and downstream settlement fails
// closed on ambiguous aggregates rather than approximating.
(
"gen_ai.usage.cache_creation.input_tokens.5m",
llm_response
.as_ref()
.and_then(|l| l.cache_creation_5m_input_tokens)
.map(Into::into),
),
(
"gen_ai.usage.cache_creation.input_tokens.1h",
llm_response
.as_ref()
.and_then(|l| l.cache_creation_1h_input_tokens)
.map(Into::into),
),
(
"gen_ai.usage.cache_read.input_tokens",
llm_response
Expand Down Expand Up @@ -3609,6 +3628,121 @@ mod tests {
}

#[test]
/// The per-TTL cache-write split reaches the span attributes exactly as
/// reported (Accounting tap contract `cache_write_5m/1h_tokens`); an
/// aggregate-only response leaves the split attributes absent — never
/// zero, never inferred.
#[test]
fn llm_span_emits_cache_write_ttl_split_attributes() {
let request = llm::LLMRequest {
input_tokens: None,
input_format: InputFormat::Messages,
cache_convention: llm::CacheTokenConvention::InputExcludesCache,
request_model: strng::literal!("claude"),
provider: strng::literal!("bedrock"),
streaming: false,
params: llm::LLMRequestParams::default(),
prompt: None,
provider_state: None,
};
let split = llm::LLMResponse {
input_tokens: Some(50),
cache_creation_input_tokens: Some(300),
cache_creation_5m_input_tokens: Some(200),
cache_creation_1h_input_tokens: Some(100),
output_tokens: Some(20),
total_tokens: Some(370),
..Default::default()
};
let aggregate_only = llm::LLMResponse {
input_tokens: Some(50),
cache_creation_input_tokens: Some(300),
output_tokens: Some(20),
total_tokens: Some(370),
..Default::default()
};

let (tracer, exporter) = test_tracer();
let mut log = test_request_log();
log.tracer = Some(tracer.clone());
let mut outgoing = trc::TraceParent::new();
outgoing.flags = 1;
log.outgoing_span = Some(outgoing);
log.llm_request = Some(request.clone());
log
.llm_response
.store(Some(llm::LLMInfo::new(request.clone(), split)));
drop(DropOnLog::from(log));
let _ = tracer.provider.force_flush();
let spans = exporter.finished_spans();
let span = spans
.iter()
.find(|span| span.name.as_ref() == "unknown")
.expect("request span should be exported");
let value = |key: &str| {
span
.attributes
.iter()
.find(|attr| attr.key.as_str() == key)
.map(|attr| &attr.value)
};
assert_eq!(
value("gen_ai.usage.cache_creation.input_tokens.5m"),
Some(&opentelemetry::Value::I64(200))
);
assert_eq!(
value("gen_ai.usage.cache_creation.input_tokens.1h"),
Some(&opentelemetry::Value::I64(100))
);
assert_eq!(
value("gen_ai.usage.cache_creation.input_tokens"),
Some(&opentelemetry::Value::I64(300))
);

// Aggregate-only producer: split attributes ABSENT, aggregate intact.
let (tracer, exporter) = test_tracer();
let mut log = test_request_log();
log.tracer = Some(tracer.clone());
let mut outgoing = trc::TraceParent::new();
outgoing.flags = 1;
log.outgoing_span = Some(outgoing);
log.llm_request = Some(request);
log.llm_response.store(Some(llm::LLMInfo::new(
llm::LLMRequest {
input_tokens: None,
input_format: InputFormat::Messages,
cache_convention: llm::CacheTokenConvention::InputExcludesCache,
request_model: strng::literal!("claude"),
provider: strng::literal!("bedrock"),
streaming: false,
params: llm::LLMRequestParams::default(),
prompt: None,
provider_state: None,
},
aggregate_only,
)));
drop(DropOnLog::from(log));
let _ = tracer.provider.force_flush();
let spans = exporter.finished_spans();
let span = spans
.iter()
.find(|span| span.name.as_ref() == "unknown")
.expect("request span should be exported");
let value = |key: &str| {
span
.attributes
.iter()
.find(|attr| attr.key.as_str() == key)
.map(|attr| &attr.value)
};
assert_eq!(value("gen_ai.usage.cache_creation.input_tokens.5m"), None);
assert_eq!(value("gen_ai.usage.cache_creation.input_tokens.1h"), None);
assert_eq!(
value("gen_ai.usage.cache_creation.input_tokens"),
Some(&opentelemetry::Value::I64(300))
);
}

fn llm_span_uses_cache_inclusive_input_tokens() {
let request = llm::LLMRequest {
input_tokens: None,
Expand Down
Loading
Loading