From b0a4efb1500e0b7d5ac01512aea19d4e366d5302 Mon Sep 17 00:00:00 2001 From: James St-Pierre Date: Tue, 23 Jun 2026 19:01:58 -0400 Subject: [PATCH] open_router: Add prompt caching for Anthropic models (#57498) Closes https://github.com/zed-industries/zed/issues/52576 ## Overview Adds prompt caching support for Anthropic Claude models when accessed via OpenRouter. This mirrors Zed's native Anthropic provider by using explicit per-block `cache_control` breakpoints: - A long-lived 1-hour breakpoint on the system message's last text block, covering the tools and system prefix. - A default 5-minute breakpoint on the last `cache: true` conversation message. The implementation intentionally avoids OpenRouter's top-level automatic `cache_control` field so routing remains available across Anthropic-compatible providers including Anthropic, Bedrock, and Vertex AI. It also adds `session_id` sticky routing from the request thread ID and maps OpenRouter cache usage fields into Zed's token usage accounting. Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content is consistent with the [UI/UX checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) - [x] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable Release Notes: - Fixed Anthropic prompt caching when using OpenRouter. --------- Co-authored-by: Anant Goel --- .../src/provider/open_router.rs | 457 +++++++++++++++++- crates/open_router/src/open_router.rs | 56 ++- 2 files changed, 495 insertions(+), 18 deletions(-) diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs index 9fcb29b1fd..979022a887 100644 --- a/crates/language_models/src/provider/open_router.rs +++ b/crates/language_models/src/provider/open_router.rs @@ -30,6 +30,7 @@ const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new( const API_KEY_ENV_VAR_NAME: &str = "OPENROUTER_API_KEY"; static API_KEY_ENV_VAR: LazyLock = env_var!(API_KEY_ENV_VAR_NAME); pub(crate) const RESERVED_HEADER_NAMES: &[&str] = &["HTTP-Referer", "X-Title"]; +const MAX_OPEN_ROUTER_SESSION_ID_LENGTH: usize = 256; #[derive(Default, Clone, Debug, PartialEq)] pub struct OpenRouterSettings { @@ -449,23 +450,39 @@ pub fn into_open_router( // If we ever have a more formal distionction between the models in the future, // we should revise this to use that instead. let is_anthropic_model = model.id().starts_with("anthropic/"); + let session_id = open_router_session_id(request.thread_id); let mut messages = Vec::new(); + let mut any_message_wants_cache = false; + let mut last_cache_message_index: Option = None; + for message in request.messages { + let mut message_added_content = false; let reasoning_details_for_message = if is_anthropic_model { None } else { message.reasoning_details.clone() }; + let message_wants_cache = message.cache; + if message_wants_cache { + any_message_wants_cache = true; + } + for content in message.content { match content { - MessageContent::Text(text) => add_message_content_part( - open_router::MessagePart::Text { text }, - message.role, - &mut messages, - reasoning_details_for_message.clone(), - ), + MessageContent::Text(text) => { + add_message_content_part( + open_router::MessagePart::Text { + text, + cache_control: None, + }, + message.role, + &mut messages, + reasoning_details_for_message.clone(), + ); + message_added_content = true; + } MessageContent::Thinking { .. } => {} MessageContent::RedactedThinking(_) => {} MessageContent::Compaction(_) => {} @@ -478,6 +495,7 @@ pub fn into_open_router( &mut messages, reasoning_details_for_message.clone(), ); + message_added_content = true; } MessageContent::ToolUse(tool_use) => { let tool_call = open_router::ToolCall { @@ -503,6 +521,7 @@ pub fn into_open_router( reasoning_details: reasoning_details_for_message.clone(), }); } + message_added_content = true; } MessageContent::ToolResult(tool_result) => { let content: Vec = tool_result @@ -512,6 +531,7 @@ pub fn into_open_router( LanguageModelToolResultContent::Text(text) => { open_router::MessagePart::Text { text: text.to_string(), + cache_control: None, } } LanguageModelToolResultContent::Image(image) => { @@ -526,15 +546,42 @@ pub fn into_open_router( content: content.into(), tool_call_id: tool_result.tool_use_id.to_string(), }); + message_added_content = true; } } } + + if message_wants_cache && message_added_content { + last_cache_message_index = messages.len().checked_sub(1); + } + } + + if is_anthropic_model && any_message_wants_cache { + // OpenRouter's top-level automatic cache_control restricts routing to + // Anthropic direct; explicit block breakpoints also work on Bedrock and Vertex. + if let Some(content) = last_cache_message_index + .and_then(|index| messages.get_mut(index)) + .and_then(request_message_content_mut) + { + set_last_text_cache_control(content, cache_control(None)); + } + + if let Some(content) = messages.iter_mut().find_map(|message| match message { + open_router::RequestMessage::System { content } => Some(content), + _ => None, + }) { + set_last_text_cache_control( + content, + cache_control(Some(open_router::CacheTtl::OneHour)), + ); + } } open_router::Request { model: model.id().into(), messages, stream: true, + session_id, stop: request.stop, temperature: request.temperature.unwrap_or(0.4), max_tokens: max_output_tokens, @@ -576,6 +623,65 @@ pub fn into_open_router( } } +fn open_router_session_id(thread_id: Option) -> Option { + thread_id.map(|thread_id| { + thread_id + .chars() + .take(MAX_OPEN_ROUTER_SESSION_ID_LENGTH) + .collect() + }) +} + +fn cache_control(ttl: Option) -> open_router::CacheControl { + open_router::CacheControl { + cache_type: open_router::CacheControlType::Ephemeral, + ttl, + } +} + +fn request_message_content_mut( + message: &mut open_router::RequestMessage, +) -> Option<&mut open_router::MessageContent> { + match message { + open_router::RequestMessage::User { content } + | open_router::RequestMessage::System { content } + | open_router::RequestMessage::Tool { content, .. } => Some(content), + open_router::RequestMessage::Assistant { + content: Some(content), + .. + } => Some(content), + open_router::RequestMessage::Assistant { content: None, .. } => None, + } +} + +fn set_last_text_cache_control( + content: &mut open_router::MessageContent, + cache_control: open_router::CacheControl, +) { + match content { + open_router::MessageContent::Plain(text) => { + let text = std::mem::take(text); + *content = + open_router::MessageContent::Multipart(vec![open_router::MessagePart::Text { + text, + cache_control: Some(cache_control), + }]); + } + open_router::MessageContent::Multipart(parts) => { + for part in parts.iter_mut().rev() { + if let open_router::MessagePart::Text { + cache_control: target, + .. + } = part + { + *target = Some(cache_control); + break; + } + } + } + } +} + fn add_message_content_part( new_part: open_router::MessagePart, role: Role, @@ -651,11 +757,23 @@ impl OpenRouterEventMapper { let mut events = Vec::new(); if let Some(usage) = event.usage { + let cache_creation_input_tokens = usage + .prompt_tokens_details + .as_ref() + .map_or(0, |details| details.cache_write_tokens); + let cache_read_input_tokens = usage + .prompt_tokens_details + .as_ref() + .map_or(0, |details| details.cached_tokens); + let input_tokens = usage.prompt_tokens.saturating_sub( + cache_creation_input_tokens.saturating_add(cache_read_input_tokens), + ); + events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { - input_tokens: usage.prompt_tokens, + input_tokens, output_tokens: usage.completion_tokens, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, + cache_creation_input_tokens, + cache_read_input_tokens, }))); } @@ -1114,19 +1232,57 @@ mod tests { prompt_tokens: 12, completion_tokens: 7, total_tokens: 19, + prompt_tokens_details: Some(open_router::PromptTokensDetails { + cached_tokens: 5, + cache_write_tokens: 3, + }), }), }); assert_eq!(events.len(), 1); - match events.into_iter().next().unwrap() { - Ok(LanguageModelCompletionEvent::UsageUpdate(usage)) => { - assert_eq!(usage.input_tokens, 12); + match events.into_iter().next() { + Some(Ok(LanguageModelCompletionEvent::UsageUpdate(usage))) => { + assert_eq!(usage.input_tokens, 4); assert_eq!(usage.output_tokens, 7); + assert_eq!(usage.cache_creation_input_tokens, 3); + assert_eq!(usage.cache_read_input_tokens, 5); + assert_eq!(usage.total_tokens(), 19); } other => panic!("Expected usage update event, got: {other:?}"), } } + #[gpui::test] + async fn test_session_id_uses_thread_id() { + let model = open_router::Model::new( + "openai/gpt-4o", + Some("GPT-4o"), + Some(128000), + Some(true), + Some(false), + None, + None, + ); + let expected_session_id = "a".repeat(MAX_OPEN_ROUTER_SESSION_ID_LENGTH); + let request = LanguageModelRequest { + thread_id: Some(format!("{expected_session_id}extra")), + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::User, + content: vec![MessageContent::Text("Hello".to_string())], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }; + + let result = into_open_router(request, &model, None); + + assert_eq!( + result.session_id.as_deref(), + Some(expected_session_id.as_str()) + ); + } + #[gpui::test] async fn test_agent_prevents_empty_reasoning_details_overwrite() { // This test verifies that the agent layer prevents empty reasoning_details @@ -1168,4 +1324,281 @@ mod tests { panic!("Expected array"); } } + + #[gpui::test] + async fn test_anthropic_model_caching_two_tier() { + let model = open_router::Model::new( + "anthropic/claude-sonnet-4-5", + Some("Claude Sonnet"), + Some(200000), + Some(true), + Some(false), + None, + None, + ); + + let request = LanguageModelRequest { + messages: vec![ + language_model::LanguageModelRequestMessage { + role: Role::System, + content: vec![MessageContent::Text("You are helpful.".to_string())], + cache: false, + reasoning_details: None, + }, + language_model::LanguageModelRequestMessage { + role: Role::User, + content: vec![MessageContent::Text("Hello".to_string())], + cache: false, + reasoning_details: None, + }, + language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![MessageContent::Text("Hi there!".to_string())], + cache: false, + reasoning_details: None, + }, + language_model::LanguageModelRequestMessage { + role: Role::User, + content: vec![MessageContent::Text("What is 2+2?".to_string())], + cache: true, + reasoning_details: None, + }, + ], + stop: vec![], + temperature: None, + tools: vec![], + tool_choice: None, + thinking_allowed: false, + thinking_effort: None, + speed: None, + thread_id: None, + prompt_id: None, + intent: None, + compact_at_tokens: None, + }; + + let result = into_open_router(request, &model, None); + + let system_cache = result.messages.iter().find_map(|m| { + if let open_router::RequestMessage::System { content } = m { + if let open_router::MessageContent::Multipart(parts) = content { + parts.iter().last().and_then(|p| { + if let open_router::MessagePart::Text { cache_control, .. } = p { + *cache_control + } else { + None + } + }) + } else { + None + } + } else { + None + } + }); + assert!( + matches!( + system_cache, + Some(open_router::CacheControl { + cache_type: open_router::CacheControlType::Ephemeral, + ttl: Some(open_router::CacheTtl::OneHour), + }) + ), + "System message should have 1h cache_control, got: {system_cache:?}" + ); + + let tail_cache = result.messages.last().and_then(|last_message| { + if let open_router::RequestMessage::User { content } = last_message { + if let open_router::MessageContent::Multipart(parts) = content { + parts.iter().last().and_then(|part| { + if let open_router::MessagePart::Text { cache_control, .. } = part { + *cache_control + } else { + None + } + }) + } else { + None + } + } else { + None + } + }); + assert!( + matches!( + tail_cache, + Some(open_router::CacheControl { + cache_type: open_router::CacheControlType::Ephemeral, + ttl: None, + }) + ), + "Last cache:true message should have 5min cache_control, got: {tail_cache:?}" + ); + + for (i, message) in result.messages.iter().enumerate() { + let is_system = matches!(message, open_router::RequestMessage::System { .. }); + let is_last = i == result.messages.len() - 1; + if is_system || is_last { + continue; + } + let parts: Option<&Vec> = match message { + open_router::RequestMessage::User { content } + | open_router::RequestMessage::System { content } + | open_router::RequestMessage::Tool { content, .. } => { + if let open_router::MessageContent::Multipart(parts) = content { + Some(parts) + } else { + None + } + } + open_router::RequestMessage::Assistant { + content: Some(content), + .. + } => { + if let open_router::MessageContent::Multipart(parts) = content { + Some(parts) + } else { + None + } + } + _ => None, + }; + if let Some(parts) = parts { + for part in parts { + if let open_router::MessagePart::Text { cache_control, .. } = part { + assert!( + cache_control.is_none(), + "Message {i} should not have cache_control" + ); + } + } + } + } + } + + #[gpui::test] + async fn test_anthropic_model_no_cache_when_no_cache_flag() { + let model = open_router::Model::new( + "anthropic/claude-sonnet-4-5", + Some("Claude Sonnet"), + Some(200000), + Some(true), + Some(false), + None, + None, + ); + + let request = LanguageModelRequest { + messages: vec![ + language_model::LanguageModelRequestMessage { + role: Role::System, + content: vec![MessageContent::Text("You are helpful.".to_string())], + cache: false, + reasoning_details: None, + }, + language_model::LanguageModelRequestMessage { + role: Role::User, + content: vec![MessageContent::Text("Hello".to_string())], + cache: false, + reasoning_details: None, + }, + ], + stop: vec![], + temperature: None, + tools: vec![], + tool_choice: None, + thinking_allowed: false, + thinking_effort: None, + speed: None, + thread_id: None, + prompt_id: None, + intent: None, + compact_at_tokens: None, + }; + + let result = into_open_router(request, &model, None); + + for message in &result.messages { + let content = match message { + open_router::RequestMessage::User { content } + | open_router::RequestMessage::System { content } => Some(content), + _ => None, + }; + if let Some(content) = content { + if let open_router::MessageContent::Multipart(parts) = content { + for part in parts { + if let open_router::MessagePart::Text { cache_control, .. } = part { + assert!( + cache_control.is_none(), + "No message should have cache_control when no cache:true flags" + ); + } + } + } + } + } + } + + #[gpui::test] + async fn test_non_anthropic_model_no_cache_control() { + let model = open_router::Model::new( + "openai/gpt-4o", + Some("GPT-4o"), + Some(128000), + Some(true), + Some(false), + None, + None, + ); + + let request = LanguageModelRequest { + messages: vec![ + language_model::LanguageModelRequestMessage { + role: Role::System, + content: vec![MessageContent::Text("You are helpful.".to_string())], + cache: false, + reasoning_details: None, + }, + language_model::LanguageModelRequestMessage { + role: Role::User, + content: vec![MessageContent::Text("Hello".to_string())], + cache: true, + reasoning_details: None, + }, + ], + stop: vec![], + temperature: None, + tools: vec![], + tool_choice: None, + thinking_allowed: false, + thinking_effort: None, + speed: None, + thread_id: None, + prompt_id: None, + intent: None, + compact_at_tokens: None, + }; + + let result = into_open_router(request, &model, None); + + for message in &result.messages { + let content = match message { + open_router::RequestMessage::User { content } + | open_router::RequestMessage::System { content } => Some(content), + _ => None, + }; + if let Some(content) = content { + if let open_router::MessageContent::Multipart(parts) = content { + for part in parts { + if let open_router::MessagePart::Text { cache_control, .. } = part { + assert!( + cache_control.is_none(), + "Non-Anthropic model should never have cache_control" + ); + } + } + } + } + } + } } diff --git a/crates/open_router/src/open_router.rs b/crates/open_router/src/open_router.rs index 885e42fd9b..006b1ffc97 100644 --- a/crates/open_router/src/open_router.rs +++ b/crates/open_router/src/open_router.rs @@ -147,6 +147,8 @@ pub struct Request { pub messages: Vec, pub stream: bool, #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub stop: Vec, @@ -248,6 +250,7 @@ impl MessageContent { Self::Plain(text) => { let text_part = MessagePart::Text { text: std::mem::take(text), + cache_control: None, }; *self = Self::Multipart(vec![text_part, part]); } @@ -259,7 +262,11 @@ impl MessageContent { impl From> for MessageContent { fn from(parts: Vec) -> Self { if parts.len() == 1 - && let MessagePart::Text { text } = &parts[0] + && let MessagePart::Text { + text, + cache_control, + } = &parts[0] + && cache_control.is_none() { return Self::Plain(text.clone()); } @@ -284,7 +291,7 @@ impl MessageContent { match self { Self::Plain(text) => Some(text), Self::Multipart(parts) if parts.len() == 1 => { - if let MessagePart::Text { text } = &parts[0] { + if let MessagePart::Text { text, .. } = &parts[0] { Some(text) } else { None @@ -300,7 +307,7 @@ impl MessageContent { Self::Multipart(parts) => parts .iter() .filter_map(|part| { - if let MessagePart::Text { text } = part { + if let MessagePart::Text { text, .. } = part { Some(text.as_str()) } else { None @@ -312,16 +319,43 @@ impl MessageContent { } } +#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum CacheControlType { + Ephemeral, +} + +#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)] +pub enum CacheTtl { + /// Anthropic's default ephemeral TTL (currently 5 minutes). Refreshes for + /// free on every cache hit. + #[serde(rename = "5m")] + FiveMinutes, + /// Anthropic's extended ephemeral TTL (currently 1 hour). Costs 2x base + /// input tokens to write, but persists across longer idle gaps. + #[serde(rename = "1h")] + OneHour, +} + +#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)] +pub struct CacheControl { + #[serde(rename = "type")] + pub cache_type: CacheControlType, + /// Omitting this field uses the API's default 5-minute TTL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl: Option, +} + #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MessagePart { Text { text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, }, #[serde(rename = "image_url")] - Image { - image_url: String, - }, + Image { image_url: String }, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] @@ -371,11 +405,21 @@ pub struct FunctionChunk { pub thought_signature: Option, } +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct PromptTokensDetails { + #[serde(default)] + pub cached_tokens: u64, + #[serde(default)] + pub cache_write_tokens: u64, +} + #[derive(Serialize, Deserialize, Debug)] pub struct Usage { pub prompt_tokens: u64, pub completion_tokens: u64, pub total_tokens: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_tokens_details: Option, } #[derive(Serialize, Deserialize, Debug)]