diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs index 607aeb85b4..c14739b249 100644 --- a/crates/agent/src/thread.rs +++ b/crates/agent/src/thread.rs @@ -4148,6 +4148,7 @@ impl Thread { let request = LanguageModelRequest { thread_id: Some(self.id.to_string()), + prompt_cache_key: None, prompt_id: Some(self.prompt_id.to_string()), intent: Some(completion_intent), messages, diff --git a/crates/agent_ui/src/buffer_codegen.rs b/crates/agent_ui/src/buffer_codegen.rs index e0d3b95277..38798cadd9 100644 --- a/crates/agent_ui/src/buffer_codegen.rs +++ b/crates/agent_ui/src/buffer_codegen.rs @@ -546,6 +546,7 @@ impl CodegenAlternative { LanguageModelRequest { thread_id: Some(session_id), + prompt_cache_key: None, prompt_id: None, intent: Some(CompletionIntent::InlineAssist), tools, @@ -629,6 +630,7 @@ impl CodegenAlternative { LanguageModelRequest { thread_id: Some(session_id), + prompt_cache_key: None, prompt_id: None, intent: Some(CompletionIntent::InlineAssist), tools: Vec::new(), diff --git a/crates/agent_ui/src/terminal_inline_assistant.rs b/crates/agent_ui/src/terminal_inline_assistant.rs index efbd1d2494..c7c9e7fb8e 100644 --- a/crates/agent_ui/src/terminal_inline_assistant.rs +++ b/crates/agent_ui/src/terminal_inline_assistant.rs @@ -265,6 +265,7 @@ impl TerminalInlineAssistant { LanguageModelRequest { thread_id: Some(session_id), + prompt_cache_key: None, prompt_id: None, intent: Some(CompletionIntent::TerminalInlineAssist), messages: vec![request_message], diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs index ed909521ae..7fdfeb319e 100644 --- a/crates/anthropic/src/completion.rs +++ b/crates/anthropic/src/completion.rs @@ -895,6 +895,7 @@ mod tests { }, ], thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -1008,6 +1009,7 @@ mod tests { }, ], thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -1078,6 +1080,7 @@ mod tests { reasoning_details: None, }], thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -1130,6 +1133,7 @@ mod tests { reasoning_details: None, }], thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -1200,6 +1204,7 @@ mod tests { reasoning_details: None, }], thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -1262,6 +1267,7 @@ mod tests { }, ], thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -1313,6 +1319,7 @@ mod tests { }], thinking_effort: None, thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs index 5d2ea914a5..e778a18258 100644 --- a/crates/copilot_chat/src/model.rs +++ b/crates/copilot_chat/src/model.rs @@ -1091,6 +1091,7 @@ fn into_copilot_responses( .and_then(|_| request.effective_max_output_tokens(model.max_output_tokens())); let LanguageModelRequest { thread_id: _, + prompt_cache_key: _, prompt_id: _, intent: _, messages, diff --git a/crates/git_ui/src/git_panel.rs b/crates/git_ui/src/git_panel.rs index 26126aeaa9..54582fcb0e 100644 --- a/crates/git_ui/src/git_panel.rs +++ b/crates/git_ui/src/git_panel.rs @@ -4150,6 +4150,7 @@ impl GitPanel { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: Some(CompletionIntent::GenerateGitCommitMessage), messages: vec![LanguageModelRequestMessage { diff --git a/crates/language_model_core/src/request.rs b/crates/language_model_core/src/request.rs index 6dedf9e93c..0d9752e3e8 100644 --- a/crates/language_model_core/src/request.rs +++ b/crates/language_model_core/src/request.rs @@ -469,6 +469,11 @@ pub enum CompletionIntent { #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] pub struct LanguageModelRequest { pub thread_id: Option, + /// Provider cache affinity, independent of conversation and transport session identity. + /// + /// Supporting providers fall back to `thread_id` when absent for legacy callers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_cache_key: Option, pub prompt_id: Option, pub intent: Option, pub messages: Vec, @@ -545,6 +550,33 @@ pub struct LanguageModelResponseMessage { mod tests { use super::*; + #[test] + fn request_prompt_cache_key_serialization() -> serde_json::Result<()> { + let request = LanguageModelRequest { + thread_id: Some("thread".into()), + ..Default::default() + }; + let mut serialized = serde_json::to_value(&request)?; + assert!(serialized.get("prompt_cache_key").is_none()); + assert_eq!( + serde_json::from_value::(serialized.clone())?, + request + ); + + serialized["prompt_cache_key"] = serde_json::json!("cache-affinity"); + let explicit: LanguageModelRequest = serde_json::from_value(serialized.clone())?; + assert_eq!(explicit.prompt_cache_key.as_deref(), Some("cache-affinity")); + assert_eq!(explicit.thread_id, request.thread_id); + assert_eq!(serde_json::to_value(explicit)?, serialized); + + serialized["prompt_cache_key"] = serde_json::Value::Null; + assert_eq!( + serde_json::from_value::(serialized)?, + request + ); + Ok(()) + } + #[test] fn request_output_limit_serialization() -> serde_json::Result<()> { let request = LanguageModelRequest::default(); diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rs index 0ebb4a1b69..78841f097c 100644 --- a/crates/language_models/src/provider/mistral.rs +++ b/crates/language_models/src/provider/mistral.rs @@ -877,6 +877,7 @@ mod tests { tools: vec![], tool_choice: None, thread_id: Some("abcdef".into()), + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -925,6 +926,7 @@ mod tests { tools: vec![], tool_choice: None, thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], @@ -969,6 +971,7 @@ mod tests { tool_choice: None, temperature: None, thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, stop: vec![], diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs index 368ff34c3d..9a11aedce5 100644 --- a/crates/language_models/src/provider/open_router.rs +++ b/crates/language_models/src/provider/open_router.rs @@ -876,6 +876,7 @@ mod tests { thinking_effort: None, speed: None, thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, compact_at_tokens: None, @@ -1019,6 +1020,7 @@ mod tests { thinking_effort: None, speed: None, thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, compact_at_tokens: None, @@ -1086,6 +1088,7 @@ mod tests { thinking_effort: None, speed: None, thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, compact_at_tokens: None, diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs index 7f0f1fc172..359fe281bd 100644 --- a/crates/open_ai/src/completion.rs +++ b/crates/open_ai/src/completion.rs @@ -216,7 +216,7 @@ pub fn into_open_ai( None }, prompt_cache_key: if supports_prompt_cache_key { - request.thread_id + request.prompt_cache_key.or(request.thread_id) } else { None }, @@ -268,6 +268,7 @@ pub fn into_open_ai_response( let LanguageModelRequest { thread_id, + prompt_cache_key, prompt_id: _, intent: _, messages, @@ -404,7 +405,7 @@ pub fn into_open_ai_response( }), tools, prompt_cache_key: if supports_prompt_cache_key { - thread_id + prompt_cache_key.or(thread_id) } else { None }, @@ -1524,6 +1525,51 @@ mod tests { use super::*; + #[test] + fn prompt_cache_key_respects_override_fallback_and_capability() -> Result<()> { + for (explicit, thread, supported, expected) in [ + (Some("cache"), Some("thread"), true, Some("cache")), + (Some("cache"), None, true, Some("cache")), + (None, Some("thread"), true, Some("thread")), + (None, None, true, None), + (Some("cache"), Some("thread"), false, None), + (None, Some("thread"), false, None), + ] { + let request = LanguageModelRequest { + thread_id: thread.map(str::to_owned), + prompt_cache_key: explicit.map(str::to_owned), + ..Default::default() + }; + let chat = into_open_ai( + request.clone(), + "gpt-5", + true, + supported, + None, + ChatCompletionMaxTokensParameter::MaxCompletionTokens, + None, + false, + )?; + let response = into_open_ai_response( + request, + "gpt-5", + true, + supported, + None, + None, + false, + &OPEN_AI_PROVIDER_ID, + )?; + assert_eq!(chat.prompt_cache_key.as_deref(), expected); + assert_eq!(response.prompt_cache_key.as_deref(), expected); + assert_eq!( + response.into_compact_request().prompt_cache_key.as_deref(), + expected + ); + } + Ok(()) + } + fn map_response_events(events: Vec) -> Vec { block_on(async { OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID) @@ -1795,6 +1841,7 @@ mod tests { let request = LanguageModelRequest { thread_id: Some("thread-123".into()), + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![ @@ -2001,6 +2048,7 @@ mod tests { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2095,6 +2143,7 @@ mod tests { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2187,6 +2236,7 @@ mod tests { fn into_open_ai_response_replays_reasoning_without_encrypted_content() { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2268,6 +2318,7 @@ mod tests { fn into_open_ai_response_omits_reasoning_when_thinking_is_disabled_and_none_is_unsupported() { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2362,6 +2413,7 @@ mod tests { ] { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2415,6 +2467,7 @@ mod tests { ] { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2461,6 +2514,7 @@ mod tests { fn into_open_ai_can_send_max_tokens_parameter() -> Result<()> { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2538,6 +2592,7 @@ mod tests { fn into_open_ai_response_sends_none_reasoning_when_thinking_is_disabled() -> Result<()> { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2580,6 +2635,7 @@ mod tests { fn into_open_ai_response_uses_default_effort_when_selected_effort_is_none() -> Result<()> { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2624,6 +2680,7 @@ mod tests { fn into_open_ai_response_replays_assistant_phase() { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -2719,6 +2776,7 @@ mod tests { }); let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![ @@ -2798,6 +2856,7 @@ mod tests { fn into_open_ai_response_replays_reasoning_details_but_not_thinking_text() { let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![LanguageModelRequestMessage { @@ -4098,6 +4157,7 @@ mod tests { }; let request = LanguageModelRequest { thread_id: None, + prompt_cache_key: None, prompt_id: None, intent: None, messages: vec![