From 7fc0b5ac6627adcfa735d868eed67986286d3cd5 Mon Sep 17 00:00:00 2001 From: Bennet Bo Fenner Date: Wed, 13 May 2026 15:34:47 +0200 Subject: [PATCH] copilot: Fix cache control error (#56632) #56472 broke Copilot chat > Failed to connect to API: 400 Bad Request {"message":"cache_control: Extra inputs are not permitted"} This PR makes it so that we still use the legacy caching approach for Copilot Release Notes: - N/A --- crates/anthropic/src/completion.rs | 154 +++++++++++++++++- .../language_models/src/provider/anthropic.rs | 3 +- .../src/provider/copilot_chat.rs | 3 +- .../language_models/src/provider/opencode.rs | 1 + .../src/language_models_cloud.rs | 3 +- 5 files changed, 152 insertions(+), 12 deletions(-) diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs index 889acfca84..5cb864a088 100644 --- a/crates/anthropic/src/completion.rs +++ b/crates/anthropic/src/completion.rs @@ -16,6 +16,51 @@ use crate::{ StringOrContents, Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage, }; +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AnthropicPromptCacheMode { + Disabled, + Legacy, + #[default] + Automatic, +} + +fn set_cache_control(content: &mut RequestContent, cache_control: Option) -> bool { + match content { + RequestContent::RedactedThinking { .. } => false, + RequestContent::Text { + cache_control: target, + .. + } + | RequestContent::Thinking { + cache_control: target, + .. + } + | RequestContent::Image { + cache_control: target, + .. + } + | RequestContent::ToolUse { + cache_control: target, + .. + } + | RequestContent::ToolResult { + cache_control: target, + .. + } => { + *target = cache_control; + true + } + } +} + +fn mark_last_cacheable_content(content: &mut [RequestContent], cache_control: CacheControl) { + for content in content.iter_mut().rev() { + if set_cache_control(content, Some(cache_control)) { + break; + } + } +} + fn to_anthropic_content(content: MessageContent) -> Option { match content { MessageContent::Text(text) => { @@ -111,6 +156,7 @@ pub fn into_anthropic( default_temperature: f32, max_output_tokens: u64, mode: AnthropicModelMode, + cache_mode: AnthropicPromptCacheMode, ) -> crate::Request { let mut new_messages: Vec = Vec::new(); let mut system_message = String::new(); @@ -125,7 +171,7 @@ pub fn into_anthropic( match message.role { Role::User | Role::Assistant => { - let anthropic_message_content: Vec = message + let mut anthropic_message_content: Vec = message .content .into_iter() .filter_map(to_anthropic_content) @@ -139,6 +185,16 @@ pub fn into_anthropic( continue; } + if cache_mode == AnthropicPromptCacheMode::Legacy && message.cache { + mark_last_cacheable_content( + &mut anthropic_message_content, + CacheControl { + cache_type: CacheControlType::Ephemeral, + ttl: None, + }, + ); + } + if let Some(last_message) = new_messages.last_mut() && last_message.role == anthropic_role { @@ -166,10 +222,12 @@ pub fn into_anthropic( // requires that longer TTLs appear earlier in the prefix, and the prefix // order is tools → system → messages, so long-TTL tools/system before a // short-TTL conversation breakpoint is a valid mix. - let long_lived_cache = any_message_wants_cache.then_some(CacheControl { - cache_type: CacheControlType::Ephemeral, - ttl: Some(CacheTtl::OneHour), - }); + let long_lived_cache = (cache_mode == AnthropicPromptCacheMode::Automatic + && any_message_wants_cache) + .then_some(CacheControl { + cache_type: CacheControlType::Ephemeral, + ttl: Some(CacheTtl::OneHour), + }); let system = if system_message.is_empty() { None @@ -208,10 +266,12 @@ pub fn into_anthropic( // tail. Omitting `ttl` uses the default (short) TTL, which refreshes // for free on every cache hit — ideal for the rapidly-changing // conversation suffix. - cache_control: any_message_wants_cache.then_some(CacheControl { - cache_type: CacheControlType::Ephemeral, - ttl: None, - }), + cache_control: (cache_mode == AnthropicPromptCacheMode::Automatic + && any_message_wants_cache) + .then_some(CacheControl { + cache_type: CacheControlType::Ephemeral, + ttl: None, + }), thinking: if request.thinking_allowed { match mode { AnthropicModelMode::Thinking { budget_tokens } => { @@ -511,6 +571,7 @@ mod tests { 0.7, 4096, AnthropicModelMode::Default, + AnthropicPromptCacheMode::Automatic, ); // No message content block should carry cache_control anymore; the @@ -571,6 +632,79 @@ mod tests { )); } + #[test] + fn test_legacy_caching_marks_last_message_content_block() { + let request = LanguageModelRequest { + messages: vec![ + LanguageModelRequestMessage { + role: Role::System, + content: vec![MessageContent::Text("You are helpful.".to_string())], + cache: false, + reasoning_details: None, + }, + LanguageModelRequestMessage { + role: Role::User, + content: vec![ + MessageContent::Text("Some prompt".to_string()), + MessageContent::Image(LanguageModelImage::empty()), + ], + cache: true, + reasoning_details: None, + }, + ], + thread_id: None, + prompt_id: None, + intent: None, + stop: vec![], + temperature: None, + tools: vec![language_model_core::LanguageModelRequestTool { + name: "do_thing".into(), + description: "Does a thing.".into(), + input_schema: serde_json::json!({"type": "object"}), + use_input_streaming: false, + }], + tool_choice: None, + thinking_allowed: true, + thinking_effort: None, + speed: None, + }; + + let anthropic_request = into_anthropic( + request, + "claude-3-5-sonnet".to_string(), + 0.7, + 4096, + AnthropicModelMode::Default, + AnthropicPromptCacheMode::Legacy, + ); + + assert!(anthropic_request.cache_control.is_none()); + assert!(matches!( + anthropic_request.system, + Some(StringOrContents::String(_)) + )); + assert_eq!(anthropic_request.tools.len(), 1); + assert!(anthropic_request.tools[0].cache_control.is_none()); + assert_eq!(anthropic_request.messages.len(), 1); + assert!(matches!( + anthropic_request.messages[0].content[0], + RequestContent::Text { + cache_control: None, + .. + } + )); + assert!(matches!( + anthropic_request.messages[0].content[1], + RequestContent::Image { + cache_control: Some(CacheControl { + cache_type: CacheControlType::Ephemeral, + ttl: None, + }), + .. + } + )); + } + #[test] fn test_no_cache_control_when_caching_disabled() { let request = LanguageModelRequest { @@ -611,6 +745,7 @@ mod tests { 0.7, 4096, AnthropicModelMode::Default, + AnthropicPromptCacheMode::Automatic, ); assert!(anthropic_request.cache_control.is_none()); @@ -654,6 +789,7 @@ mod tests { AnthropicModelMode::Thinking { budget_tokens: Some(10000), }, + AnthropicPromptCacheMode::Automatic, ) } diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs index c4ea3dcfcd..5e5fa2ad85 100644 --- a/crates/language_models/src/provider/anthropic.rs +++ b/crates/language_models/src/provider/anthropic.rs @@ -21,7 +21,7 @@ use ui::{ButtonLink, ConfiguredApiCard, List, ListBulletItem, prelude::*}; use ui_input::InputField; use util::ResultExt; -pub use anthropic::completion::{AnthropicEventMapper, into_anthropic}; +pub use anthropic::completion::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic}; pub use settings::AnthropicAvailableModel as AvailableModel; const PROVIDER_ID: LanguageModelProviderId = ANTHROPIC_PROVIDER_ID; @@ -495,6 +495,7 @@ impl LanguageModel for AnthropicModel { self.model.default_temperature, self.model.max_output_tokens, self.model.mode.clone(), + AnthropicPromptCacheMode::Automatic, ); if !self.model.supports_speed { request.speed = None; diff --git a/crates/language_models/src/provider/copilot_chat.rs b/crates/language_models/src/provider/copilot_chat.rs index 1fc1dc3ce4..cf8f11a5bb 100644 --- a/crates/language_models/src/provider/copilot_chat.rs +++ b/crates/language_models/src/provider/copilot_chat.rs @@ -31,7 +31,7 @@ use settings::SettingsStore; use ui::prelude::*; use util::debug_panic; -use crate::provider::anthropic::{AnthropicEventMapper, into_anthropic}; +use crate::provider::anthropic::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic}; use language_model::util::{fix_streamed_json, parse_tool_arguments}; const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("copilot_chat"); @@ -354,6 +354,7 @@ impl LanguageModel for CopilotChatLanguageModel { } else { AnthropicModelMode::Default }, + AnthropicPromptCacheMode::Legacy, ); anthropic_request.temperature = None; diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rs index 1611694905..64b3a6fcdb 100644 --- a/crates/language_models/src/provider/opencode.rs +++ b/crates/language_models/src/provider/opencode.rs @@ -647,6 +647,7 @@ impl LanguageModel for OpenCodeLanguageModel { 1.0, self.model.max_output_tokens().unwrap_or(8192), mode, + anthropic::completion::AnthropicPromptCacheMode::Automatic, ); let stream = self.stream_anthropic(anthropic_request, http_client, cx); async move { diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs index 928ec33648..17051ac874 100644 --- a/crates/language_models_cloud/src/language_models_cloud.rs +++ b/crates/language_models_cloud/src/language_models_cloud.rs @@ -39,7 +39,7 @@ use std::task::Poll; use std::time::Duration; use thiserror::Error; -use anthropic::completion::{AnthropicEventMapper, into_anthropic}; +use anthropic::completion::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic}; use google_ai::completion::{GoogleEventMapper, into_google}; use open_ai::completion::{ OpenAiEventMapper, OpenAiResponseEventMapper, into_open_ai, into_open_ai_response, @@ -419,6 +419,7 @@ impl LanguageModel for CloudLanguageModel