From 739fdbef762f7e514d62e1fa650c1b7cac5bbff4 Mon Sep 17 00:00:00 2001 From: MartinYe1234 <52641447+MartinYe1234@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:25:47 +0000 Subject: [PATCH] Correct Copilot token-limit metadata (#64195) ## Stack Part 3 of the split. #64047 and #64194 are merged. This PR targets `main`, which has been merged into the branch without rewriting history; the already-merged output-limit and counting changes are preserved. ## Summary Expose Copilot's actual prompt, combined-context, and output limits through the shared model interface. Preserve `max_token_count()` as the existing context capacity; add `max_input_tokens()` for independent prompt limits, defaulting to context capacity for other providers. Existing total-display and telemetry callers keep using `max_token_count()` without fallback expressions. The values come from the provider's model feed, not hard-coded model tables. Missing prompt metadata falls back to context capacity; missing output metadata remains unknown. Use the independent prompt limit when deciding when to auto-compact, while retaining the combined window for total usage display and telemetry. For example, a 90k prompt limit within a 200k context window is not treated as 200k input and does not lose a second output reservation. The existing split-display ring and tooltip use the same usable input capacity. This PR does not enable split display for Copilot, so it does not introduce a visible Copilot input-limit display. No counting endpoints, consent changes, or Delta changes are included. ## Validation - After the review follow-up, all 47 selected Copilot and agent compaction tests passed. Local testing used the existing `gpui_platform/runtime_shaders` feature because the Apple Metal compiler is unavailable. - Scoped script/clippy passed for copilot_chat, agent, and agent_ui. - Formatting and whitespace checks passed. - Fixtures cover unequal prompt/context/output limits, absent/zero metadata, and existing compaction workflows. - Reviewed the complete diff against current main: only the input-limit API, Copilot metadata, its compaction/display consumers, and tests remain. The live tooltip workflow was not exercised. - No live provider calls were made. Downstream Delta integration must use `max_input_tokens()` for input admission when updating its Zed dependency; that pin update remains separate. ## Review follow-up - Added a real Thread regression with 200k context, 90k input, and 16,384 output: compaction changes from inactive at 80,999 tokens to active at 81,000; total displayed capacity remains 200k. - Changed the split input ring to use the same capacity as its tooltip. This does not enable split display for Copilot. - All 47 selected tests passed locally, along with scoped Clippy, formatting, and whitespace checks. Temporarily restoring the incorrect context-limit call made the new regression fail; that mutation was removed. Release Notes: - Fixed auto-compaction thresholds for GitHub Copilot models with a prompt limit below their context window. --- crates/agent/src/thread.rs | 93 ++++++++++++++++++- .../src/conversation_view/thread_view.rs | 21 +++-- crates/copilot_chat/src/copilot_chat.rs | 5 + crates/copilot_chat/src/model.rs | 89 ++++++++++++++++++ crates/language_model/src/fake_provider.rs | 11 +++ crates/language_model/src/language_model.rs | 11 ++- 6 files changed, 216 insertions(+), 14 deletions(-) diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs index 39f99aac0c..884a87b123 100644 --- a/crates/agent/src/thread.rs +++ b/crates/agent/src/thread.rs @@ -2434,6 +2434,16 @@ impl Thread { self.cumulative_token_usage } + /// Maximum input after reserving the selected model's output allowance. + pub fn input_token_capacity(&self) -> Option { + let model = self.model()?; + Some(compaction_input_capacity( + model.max_input_tokens(), + model.max_total_tokens(), + model.max_output_tokens(), + )) + } + pub fn latest_token_usage(&self) -> Option { let usage = self.latest_request_token_usage()?; let model = self.model()?; @@ -4392,7 +4402,7 @@ impl Thread { let model = self.model()?; let auto_compact = AgentSettings::get_global(cx).auto_compact; let max_tokens = model.max_token_count(); - let max_input_tokens = max_tokens.saturating_sub(model.max_output_tokens().unwrap_or(0)); + let max_input_tokens = self.input_token_capacity()?; let tokens_before = self .latest_request_token_usage() .map(|usage| total_input_tokens(usage).saturating_add(usage.output_tokens)); @@ -4431,10 +4441,7 @@ impl Thread { return None; } - let model = self.model()?; - let max_token_count = model.max_token_count(); - let max_input_tokens = - max_token_count.saturating_sub(model.max_output_tokens().unwrap_or(0)); + let max_input_tokens = self.input_token_capacity()?; // Models with a small context window don't leave enough headroom for a // compaction pass; the UI warns the user about the token limit instead. if max_input_tokens < MIN_COMPACTION_CONTEXT_WINDOW { @@ -4600,6 +4607,17 @@ fn total_input_tokens(usage: language_model::TokenUsage) -> u64 { .saturating_add(usage.cache_read_input_tokens) } +/// Reserves output without subtracting it from an independent input ceiling. +fn compaction_input_capacity( + input_limit: u64, + combined_limit: Option, + output_limit: Option, +) -> u64 { + combined_limit.map_or(input_limit, |combined| { + input_limit.min(combined.saturating_sub(output_limit.unwrap_or(0))) + }) +} + fn auto_compact_threshold_token_count( threshold: AutoCompactThreshold, max_token_count: u64, @@ -6883,6 +6901,30 @@ mod tests { use settings::LanguageModelProviderSetting; use std::sync::Arc; + #[test] + fn compaction_capacity_respects_prompt_and_combined_limits() { + assert_eq!( + compaction_input_capacity(90_000, Some(200_000), Some(16_384)), + 90_000 + ); + assert_eq!( + compaction_input_capacity(128_000, Some(128_000), Some(64_000)), + 64_000 + ); + assert_eq!( + compaction_input_capacity(90_000, None, Some(64_000)), + 90_000 + ); + assert_eq!( + compaction_input_capacity(2_000, Some(2_000), Some(3_000)), + 0 + ); + assert_eq!( + compaction_input_capacity(128_000, Some(128_000), None), + 128_000 + ); + } + async fn setup_thread_for_test(cx: &mut TestAppContext) -> (Entity, ThreadEventStream) { cx.update(|cx| { let settings_store = settings::SettingsStore::test(cx); @@ -7191,6 +7233,47 @@ mod tests { }); } + #[gpui::test] + async fn test_compaction_threshold_respects_independent_input_limit(cx: &mut TestAppContext) { + let (thread, _event_stream) = setup_thread_for_test(cx).await; + let mut model = FakeLanguageModel::default(); + model.set_max_token_count(200_000); + model.set_max_input_tokens(90_000); + model.set_max_output_tokens(Some(16_384)); + let model = Arc::new(model); + let user_message_id = ClientUserMessageId::new(); + + cx.update(|cx| { + set_auto_compact_settings( + cx, + agent_settings::AutoCompactSettings { + enabled: true, + threshold: AutoCompactThreshold::Percentage(0.9), + }, + ); + thread.update(cx, |thread, cx| { + thread.set_model(model, cx); + thread.messages.push(user_text_message( + user_message_id.clone(), + "near the independent input limit", + )); + for (input_tokens, expected_target) in [(80_999, None), (81_000, Some(1))] { + thread.request_token_usage.insert( + user_message_id.clone(), + language_model::TokenUsage { + input_tokens, + ..Default::default() + }, + ); + + assert_eq!(thread.compaction_message_target_ix(cx), expected_target); + assert_eq!(thread.input_token_capacity(), Some(90_000)); + assert_eq!(thread.latest_token_usage().unwrap().max_tokens, 200_000); + } + }); + }); + } + #[gpui::test] async fn test_compaction_threshold_respects_enabled_setting(cx: &mut TestAppContext) { let (thread, _event_stream) = setup_thread_for_test(cx).await; diff --git a/crates/agent_ui/src/conversation_view/thread_view.rs b/crates/agent_ui/src/conversation_view/thread_view.rs index 95b4b4fdb1..e5a862e752 100644 --- a/crates/agent_ui/src/conversation_view/thread_view.rs +++ b/crates/agent_ui/src/conversation_view/thread_view.rs @@ -4762,13 +4762,20 @@ impl ThreadView { let workspace = self.workspace.clone(); - let max_output_tokens = self + let (max_input_tokens, max_output_tokens) = self .as_native_thread(cx) - .and_then(|thread| thread.read(cx).model()) - .and_then(|model| model.max_output_tokens()) - .unwrap_or(0); - let input_max_label = - crate::humanize_token_count(usage.max_tokens.saturating_sub(max_output_tokens)); + .map(|thread| { + let thread = thread.read(cx); + ( + thread.input_token_capacity().unwrap_or(usage.max_tokens), + thread + .model() + .and_then(|model| model.max_output_tokens()) + .unwrap_or(0), + ) + }) + .unwrap_or((usage.max_tokens, 0)); + let input_max_label = crate::humanize_token_count(max_input_tokens); let output_max_label = crate::humanize_token_count(max_output_tokens); let build_tooltip = { @@ -4804,7 +4811,7 @@ impl ThreadView { }; if show_split { - let input_max_raw = usage.max_tokens.saturating_sub(max_output_tokens); + let input_max_raw = max_input_tokens; let output_max_raw = max_output_tokens; let input_ratio = if input_max_raw > 0 { diff --git a/crates/copilot_chat/src/copilot_chat.rs b/crates/copilot_chat/src/copilot_chat.rs index 71ef0895d9..9b9b208a1b 100644 --- a/crates/copilot_chat/src/copilot_chat.rs +++ b/crates/copilot_chat/src/copilot_chat.rs @@ -279,6 +279,11 @@ impl Model { self.capabilities.limits.max_context_window_tokens as u64 } + pub fn max_prompt_tokens(&self) -> Option { + let limit = self.capabilities.limits.max_prompt_tokens; + (limit > 0).then_some(limit) + } + pub fn max_output_tokens(&self) -> Option { let limit = self.capabilities.limits.max_output_tokens as u64; (limit > 0).then_some(limit) diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs index 12d524c318..5d2ea914a5 100644 --- a/crates/copilot_chat/src/model.rs +++ b/crates/copilot_chat/src/model.rs @@ -130,6 +130,16 @@ impl LanguageModel for CopilotChatLanguageModel { self.model.max_token_count() } + fn max_input_tokens(&self) -> u64 { + self.model + .max_prompt_tokens() + .unwrap_or_else(|| self.model.max_token_count()) + } + + fn max_output_tokens(&self) -> Option { + self.model.max_output_tokens() + } + fn stream_completion( &self, request: LanguageModelRequest, @@ -1301,6 +1311,85 @@ mod tests { use language_model::ProviderErrorCategory; use serde_json::json; + #[gpui::test] + fn language_model_exposes_token_limits(cx: &mut gpui::TestAppContext) { + use gpui::AppContext as _; + + let copilot_chat = cx.new(|cx| { + CopilotChat::new( + Arc::new(http_client::BlockedHttpClient::new()), + Arc::new(EmptyCredentialsProvider), + crate::CopilotChatConfiguration::default(), + cx, + ) + }); + for (limits, input, context, output) in [ + ( + json!({"max_context_window_tokens": 200_000, "max_prompt_tokens": 90_000, "max_output_tokens": 16_384}), + 90_000, + 200_000, + Some(16_384), + ), + ( + json!({"max_context_window_tokens": 128_000, "max_output_tokens": 4096}), + 128_000, + 128_000, + Some(4096), + ), + ( + json!({"max_context_window_tokens": 128_000, "max_prompt_tokens": 100_000}), + 100_000, + 128_000, + None, + ), + ( + json!({"max_context_window_tokens": 128_000, "max_prompt_tokens": 0, "max_output_tokens": 0}), + 128_000, + 128_000, + None, + ), + ] { + let mut value = serde_json::to_value(test_responses_model()).unwrap(); + value["capabilities"]["limits"] = limits; + let descriptor = serde_json::from_value(value).unwrap(); + let model = create_language_model(descriptor, copilot_chat.clone()); + assert_eq!(model.max_token_count(), context); + assert_eq!(model.max_input_tokens(), input); + assert_eq!(model.max_total_tokens(), Some(context)); + assert_eq!(model.max_output_tokens(), output); + } + } + + struct EmptyCredentialsProvider; + + impl credentials_provider::CredentialsProvider for EmptyCredentialsProvider { + fn read_credentials<'a>( + &'a self, + _url: &'a str, + _cx: &'a AsyncApp, + ) -> Pin)>>> + 'a>> { + Box::pin(async { Ok(None) }) + } + + fn write_credentials<'a>( + &'a self, + _url: &'a str, + _username: &'a str, + _password: &'a [u8], + _cx: &'a AsyncApp, + ) -> Pin> + 'a>> { + Box::pin(async { Ok(()) }) + } + + fn delete_credentials<'a>( + &'a self, + _url: &'a str, + _cx: &'a AsyncApp, + ) -> Pin> + 'a>> { + Box::pin(async { Ok(()) }) + } + } + fn map_events(events: Vec) -> Vec { futures::executor::block_on(async { CopilotResponsesEventMapper::new() diff --git a/crates/language_model/src/fake_provider.rs b/crates/language_model/src/fake_provider.rs index 94f4487f65..f9e7732707 100644 --- a/crates/language_model/src/fake_provider.rs +++ b/crates/language_model/src/fake_provider.rs @@ -121,6 +121,7 @@ pub struct FakeLanguageModel { supports_images: AtomicBool, supports_server_side_compaction: AtomicBool, max_token_count: AtomicU64, + max_input_tokens: Option, max_output_tokens: AtomicU64, input_token_counts: Mutex>, input_token_count_requests: Mutex>, @@ -141,6 +142,7 @@ impl Default for FakeLanguageModel { supports_images: AtomicBool::new(false), supports_server_side_compaction: AtomicBool::new(false), max_token_count: AtomicU64::new(1_000_000), + max_input_tokens: None, max_output_tokens: AtomicU64::new(0), input_token_counts: Mutex::new(VecDeque::new()), input_token_count_requests: Mutex::new(Vec::new()), @@ -196,6 +198,10 @@ impl FakeLanguageModel { self.max_token_count.store(count, SeqCst); } + pub fn set_max_input_tokens(&mut self, count: u64) { + self.max_input_tokens = Some(count); + } + pub fn set_max_output_tokens(&self, count: Option) { self.max_output_tokens .store(count.unwrap_or_default(), SeqCst); @@ -342,6 +348,11 @@ impl LanguageModel for FakeLanguageModel { self.max_token_count.load(SeqCst) } + fn max_input_tokens(&self) -> u64 { + self.max_input_tokens + .unwrap_or_else(|| self.max_token_count()) + } + fn max_output_tokens(&self) -> Option { let max_output_tokens = self.max_output_tokens.load(SeqCst); if max_output_tokens == 0 { diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs index 90353facb8..dc12a4559b 100644 --- a/crates/language_model/src/language_model.rs +++ b/crates/language_model/src/language_model.rs @@ -215,9 +215,16 @@ pub trait LanguageModel: Send + Sync { false } - /// Returns the input token ceiling. + /// Returns the model's context-window capacity. fn max_token_count(&self) -> u64; + /// Returns the input ceiling before reserving output from any shared window. + /// + /// Models with a separate prompt limit override the context-window default. + fn max_input_tokens(&self) -> u64 { + self.max_token_count() + } + /// Counts request input without generating output, when supported by the provider. /// /// Counts may be estimates and differ from subsequent measured usage. Callers @@ -233,7 +240,7 @@ pub trait LanguageModel: Send + Sync { /// Returns the combined input and output ceiling, if one applies. /// - /// The conservative default shares the input ceiling with output. `None` + /// The conservative default shares the context window with output. `None` /// means generation does not consume that window, not merely that the API /// validates input separately or stops generation at the window boundary. fn max_total_tokens(&self) -> Option {