Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 62 additions & 21 deletions crates/llm/src/conversion/messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -801,10 +801,22 @@ pub mod from_completions {
message.usage.cache_read_input_tokens.map(|i| i as u64);
r.response.cache_creation_input_tokens =
message.usage.cache_creation_input_tokens.map(|i| i as u64);
// TTL split stays unread here — message_start usage
// is provisional pre-turn state; only the terminal
// message_delta is cumulative-final (see the
// passthrough MessageStart arm).
// Input-side usage (cache write split included) is final at
// `message_start`; a reported all-zero split with a
// small aggregate is the provider's authoritative
// sub-minimum no-op write evidence.
r.response.cache_creation_5m_input_tokens = message
.usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_5m_input_tokens)
.map(|i| i as u64);
r.response.cache_creation_1h_input_tokens = message
.usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_1h_input_tokens)
.map(|i| i as u64);
// Provisional evidence: the cumulative totals only
// arrive on the terminal `message_delta`.
r.response.usage_complete = Some(false);
Expand Down Expand Up @@ -946,11 +958,19 @@ pub mod from_completions {
if let Some(cwt) = usage.cache_creation_input_tokens {
r.response.cache_creation_input_tokens = Some(cwt as u64);
}
if let Some(split) = &usage.cache_creation {
r.response.cache_creation_5m_input_tokens =
split.ephemeral_5m_input_tokens.map(|i| i as u64);
r.response.cache_creation_1h_input_tokens =
split.ephemeral_1h_input_tokens.map(|i| i as u64);
// The cumulative delta wins when it carries the split;
// absent leaves earlier evidence untouched.
if usage.cache_creation.is_some() {
r.response.cache_creation_5m_input_tokens = usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_5m_input_tokens)
.map(|i| i as u64);
r.response.cache_creation_1h_input_tokens = usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_1h_input_tokens)
.map(|i| i as u64);
}
if let Some(o) = usage.output_tokens {
r.response.output_tokens = Some(o as u64);
Expand Down Expand Up @@ -1195,13 +1215,26 @@ pub fn passthrough_stream(
message.usage.cache_read_input_tokens.map(|i| i as u64);
r.response.cache_creation_input_tokens =
message.usage.cache_creation_input_tokens.map(|i| i as u64);
// NOTE: the TTL split is deliberately NOT read here.
// `message_start` usage is provisional pre-turn state —
// captured production evidence shows it reporting
// `cache_creation: {5m: 0, 1h: 0}` alongside a positive
// aggregate — while the terminal `message_delta` usage
// is cumulative-final. Only the delta may carry the
// split into evidence.
// Input-side usage (cache write split included) is
// final at `message_start`; the terminal
// `message_delta` is cumulative-final. Only
// self-consistent splits become evidence.
// Input-side usage (cache write split included) is final at
// `message_start`; a reported all-zero split with a small
// aggregate is the provider's authoritative sub-minimum
// no-op write evidence.
r.response.cache_creation_5m_input_tokens = message
.usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_5m_input_tokens)
.map(|i| i as u64);
r.response.cache_creation_1h_input_tokens = message
.usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_1h_input_tokens)
.map(|i| i as u64);
// Provisional evidence: the cumulative totals only
// arrive on the terminal `message_delta`.
r.response.usage_complete = Some(false);
Expand Down Expand Up @@ -1265,11 +1298,19 @@ pub fn passthrough_stream(
if let Some(cwt) = usage.cache_creation_input_tokens {
r.response.cache_creation_input_tokens = Some(cwt as u64);
}
if let Some(split) = &usage.cache_creation {
r.response.cache_creation_5m_input_tokens =
split.ephemeral_5m_input_tokens.map(|i| i as u64);
r.response.cache_creation_1h_input_tokens =
split.ephemeral_1h_input_tokens.map(|i| i as u64);
// The cumulative delta wins when it carries the split;
// absent leaves earlier evidence untouched.
if usage.cache_creation.is_some() {
r.response.cache_creation_5m_input_tokens = usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_5m_input_tokens)
.map(|i| i as u64);
r.response.cache_creation_1h_input_tokens = usage
.cache_creation
.as_ref()
.and_then(|c| c.ephemeral_1h_input_tokens)
.map(|i| i as u64);
}
if let Some(o) = usage.output_tokens {
r.response.output_tokens = Some(o as u64);
Expand Down
40 changes: 40 additions & 0 deletions crates/llm/src/conversion/messages_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -492,3 +492,43 @@ async fn passthrough_stream_cache_usage_is_preserved() {
"cache_creation evidence must survive extraction"
);
}

/// Captured production `message_start` shape (golden stream_basic): a
/// small aggregate alongside an all-zero split — the provider's
/// authoritative sub-minimum no-op write evidence. The split buckets are
/// forwarded EXACTLY as reported (Some(0)/Some(0) — real evidence, not
/// absence) and the terminal delta, which carries no split here, leaves
/// that evidence untouched instead of clobbering it.
#[tokio::test]
async fn passthrough_stream_sub_minimum_zero_split_is_authoritative_evidence() {
let input = format!(
"{}\n\n{}\n\n",
message_start_with_usage(
r#"{"input_tokens":15,"output_tokens":1,"cache_creation_input_tokens":13,"cache_read_input_tokens":12,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}}"#
),
r#"data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":21}}"#,
);
let info = run_passthrough(input).await;
assert_eq!(info.response.cache_creation_input_tokens, Some(13));
assert_eq!(info.response.cache_creation_5m_input_tokens, Some(0));
assert_eq!(info.response.cache_creation_1h_input_tokens, Some(0));
}

/// Real Anthropic streaming shape (production canary): the input-side
/// usage is final at `message_start` and SELF-CONSISTENT
/// (aggregate == 5m + 1h), while the terminal delta carries no split. The
/// consistent split becomes evidence at start and survives the delta.
#[tokio::test]
async fn passthrough_stream_consistent_message_start_split_becomes_evidence() {
let input = format!(
"{}\n\n{}\n\n",
message_start_with_usage(
r#"{"input_tokens":10,"output_tokens":1,"cache_creation_input_tokens":67759,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":67759,"ephemeral_1h_input_tokens":0}}"#
),
r#"data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":44}}"#,
);
let info = run_passthrough(input).await;
assert_eq!(info.response.cache_creation_input_tokens, Some(67759));
assert_eq!(info.response.cache_creation_5m_input_tokens, Some(67759));
assert_eq!(info.response.cache_creation_1h_input_tokens, Some(0));
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
---
source: crates/llm/src/golden_tests.rs
assertion_line: 846
description: src/tests/response/anthropic/stream_basic.json
---
data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":"Hi there! How are","role":"assistant"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null}
Expand All @@ -23,6 +24,8 @@ data: [DONE]
"output_tokens": 21,
"total_tokens": 36,
"cache_creation_input_tokens": 13,
"cache_creation_5m_input_tokens": 0,
"cache_creation_1h_input_tokens": 0,
"cached_input_tokens": 12,
"service_tier": "standard",
"provider_model": "claude-haiku-4-5-20251001",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
---
source: crates/llm/src/golden_tests.rs
assertion_line: 846
description: src/tests/response/anthropic/stream_basic.json
---
event: message_start
Expand Down Expand Up @@ -42,6 +43,8 @@ data: {"type":"message_stop" }
"output_tokens": 21,
"total_tokens": 36,
"cache_creation_input_tokens": 13,
"cache_creation_5m_input_tokens": 0,
"cache_creation_1h_input_tokens": 0,
"cached_input_tokens": 12,
"service_tier": "standard",
"provider_model": "claude-haiku-4-5-20251001",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
---
source: crates/llm/src/golden_tests.rs
assertion_line: 846
description: src/tests/response/anthropic/stream_thinking.json
---
data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_content":"The user is"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null}
Expand Down Expand Up @@ -31,6 +32,8 @@ data: [DONE]
"output_tokens": 264,
"total_tokens": 311,
"cache_creation_input_tokens": 0,
"cache_creation_5m_input_tokens": 0,
"cache_creation_1h_input_tokens": 0,
"cached_input_tokens": 0,
"service_tier": "standard",
"provider_model": "claude-opus-4-6",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
---
source: crates/llm/src/golden_tests.rs
assertion_line: 846
description: src/tests/response/anthropic/stream_thinking.json
---
event: message_start
Expand Down Expand Up @@ -60,6 +61,8 @@ data: {"type":"message_stop" }
"output_tokens": 264,
"total_tokens": 311,
"cache_creation_input_tokens": 0,
"cache_creation_5m_input_tokens": 0,
"cache_creation_1h_input_tokens": 0,
"cached_input_tokens": 0,
"service_tier": "standard",
"provider_model": "claude-opus-4-6",
Expand Down
5 changes: 5 additions & 0 deletions crates/llm/src/types/messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,11 @@ impl Usage {
/// wire format stays byte-identical — no re-parsed typed field, no key
/// reordering. `None` = the provider reported no split; consumers must
/// not guess the class from the aggregate.
///
/// A reported all-zero split next to a small aggregate is the
/// provider's authoritative sub-minimum no-op write evidence (observed
/// on captured production usage): the classes are authoritative and the
/// residual aggregate meters at zero — not an inconsistency to hide.
pub fn cache_ttl_split(&self) -> (Option<u64>, Option<u64>) {
let cache_creation = self.rest.get("cache_creation");
let five_m = cache_creation
Expand Down
Loading