Correct Copilot token-limit metadata (#64195)
## Stack Part 3 of the split. #64047 and #64194 are merged. This PR targets `main`, which has been merged into the branch without rewriting history; the already-merged output-limit and counting changes are preserved. ## Summary Expose Copilot's actual prompt, combined-context, and output limits through the shared model interface. Preserve `max_token_count()` as the existing context capacity; add `max_input_tokens()` for independent prompt limits, defaulting to context capacity for other providers. Existing total-display and telemetry callers keep using `max_token_count()` without fallback expressions. The values come from the provider's model feed, not hard-coded model tables. Missing prompt metadata falls back to context capacity; missing output metadata remains unknown. Use the independent prompt limit when deciding when to auto-compact, while retaining the combined window for total usage display and telemetry. For example, a 90k prompt limit within a 200k context window is not treated as 200k input and does not lose a second output reservation. The existing split-display ring and tooltip use the same usable input capacity. This PR does not enable split display for Copilot, so it does not introduce a visible Copilot input-limit display. No counting endpoints, consent changes, or Delta changes are included. ## Validation - After the review follow-up, all 47 selected Copilot and agent compaction tests passed. Local testing used the existing `gpui_platform/runtime_shaders` feature because the Apple Metal compiler is unavailable. - Scoped script/clippy passed for copilot_chat, agent, and agent_ui. - Formatting and whitespace checks passed. - Fixtures cover unequal prompt/context/output limits, absent/zero metadata, and existing compaction workflows. - Reviewed the complete diff against current main: only the input-limit API, Copilot metadata, its compaction/display consumers, and tests remain. The live tooltip workflow was not exercised. - No live provider calls were made. Downstream Delta integration must use `max_input_tokens()` for input admission when updating its Zed dependency; that pin update remains separate. ## Review follow-up - Added a real Thread regression with 200k context, 90k input, and 16,384 output: compaction changes from inactive at 80,999 tokens to active at 81,000; total displayed capacity remains 200k. - Changed the split input ring to use the same capacity as its tooltip. This does not enable split display for Copilot. - All 47 selected tests passed locally, along with scoped Clippy, formatting, and whitespace checks. Temporarily restoring the incorrect context-limit call made the new regression fail; that mutation was removed. Release Notes: - Fixed auto-compaction thresholds for GitHub Copilot models with a prompt limit below their context window.
This commit is contained in:
1 parent
c2451489b7
commit
739fdbef76
6 files changed
+216
-14
No files matched your search
@@ -2434,6 +2434,16 @@ impl Thread {
|
||||
self.cumulative_token_usage
|
||||
}
|
||||
|
||||
/// Maximum input after reserving the selected model's output allowance.
|
||||
pub fn input_token_capacity(&self) -> Option<u64> {
|
||||
let model = self.model()?;
|
||||
Some(compaction_input_capacity(
|
||||
model.max_input_tokens(),
|
||||
model.max_total_tokens(),
|
||||
model.max_output_tokens(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn latest_token_usage(&self) -> Option<acp_thread::TokenUsage> {
|
||||
let usage = self.latest_request_token_usage()?;
|
||||
let model = self.model()?;
|
||||
@@ -4392,7 +4402,7 @@ impl Thread {
|
||||
let model = self.model()?;
|
||||
let auto_compact = AgentSettings::get_global(cx).auto_compact;
|
||||
let max_tokens = model.max_token_count();
|
||||
let max_input_tokens = max_tokens.saturating_sub(model.max_output_tokens().unwrap_or(0));
|
||||
let max_input_tokens = self.input_token_capacity()?;
|
||||
let tokens_before = self
|
||||
.latest_request_token_usage()
|
||||
.map(|usage| total_input_tokens(usage).saturating_add(usage.output_tokens));
|
||||
@@ -4431,10 +4441,7 @@ impl Thread {
|
||||
return None;
|
||||
}
|
||||
|
||||
let model = self.model()?;
|
||||
let max_token_count = model.max_token_count();
|
||||
let max_input_tokens =
|
||||
max_token_count.saturating_sub(model.max_output_tokens().unwrap_or(0));
|
||||
let max_input_tokens = self.input_token_capacity()?;
|
||||
// Models with a small context window don't leave enough headroom for a
|
||||
// compaction pass; the UI warns the user about the token limit instead.
|
||||
if max_input_tokens < MIN_COMPACTION_CONTEXT_WINDOW {
|
||||
@@ -4600,6 +4607,17 @@ fn total_input_tokens(usage: language_model::TokenUsage) -> u64 {
|
||||
.saturating_add(usage.cache_read_input_tokens)
|
||||
}
|
||||
|
||||
/// Reserves output without subtracting it from an independent input ceiling.
|
||||
fn compaction_input_capacity(
|
||||
input_limit: u64,
|
||||
combined_limit: Option<u64>,
|
||||
output_limit: Option<u64>,
|
||||
) -> u64 {
|
||||
combined_limit.map_or(input_limit, |combined| {
|
||||
input_limit.min(combined.saturating_sub(output_limit.unwrap_or(0)))
|
||||
})
|
||||
}
|
||||
|
||||
fn auto_compact_threshold_token_count(
|
||||
threshold: AutoCompactThreshold,
|
||||
max_token_count: u64,
|
||||
@@ -6883,6 +6901,30 @@ mod tests {
|
||||
use settings::LanguageModelProviderSetting;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
fn compaction_capacity_respects_prompt_and_combined_limits() {
|
||||
assert_eq!(
|
||||
compaction_input_capacity(90_000, Some(200_000), Some(16_384)),
|
||||
90_000
|
||||
);
|
||||
assert_eq!(
|
||||
compaction_input_capacity(128_000, Some(128_000), Some(64_000)),
|
||||
64_000
|
||||
);
|
||||
assert_eq!(
|
||||
compaction_input_capacity(90_000, None, Some(64_000)),
|
||||
90_000
|
||||
);
|
||||
assert_eq!(
|
||||
compaction_input_capacity(2_000, Some(2_000), Some(3_000)),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
compaction_input_capacity(128_000, Some(128_000), None),
|
||||
128_000
|
||||
);
|
||||
}
|
||||
|
||||
async fn setup_thread_for_test(cx: &mut TestAppContext) -> (Entity<Thread>, ThreadEventStream) {
|
||||
cx.update(|cx| {
|
||||
let settings_store = settings::SettingsStore::test(cx);
|
||||
@@ -7191,6 +7233,47 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_compaction_threshold_respects_independent_input_limit(cx: &mut TestAppContext) {
|
||||
let (thread, _event_stream) = setup_thread_for_test(cx).await;
|
||||
let mut model = FakeLanguageModel::default();
|
||||
model.set_max_token_count(200_000);
|
||||
model.set_max_input_tokens(90_000);
|
||||
model.set_max_output_tokens(Some(16_384));
|
||||
let model = Arc::new(model);
|
||||
let user_message_id = ClientUserMessageId::new();
|
||||
|
||||
cx.update(|cx| {
|
||||
set_auto_compact_settings(
|
||||
cx,
|
||||
agent_settings::AutoCompactSettings {
|
||||
enabled: true,
|
||||
threshold: AutoCompactThreshold::Percentage(0.9),
|
||||
},
|
||||
);
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.set_model(model, cx);
|
||||
thread.messages.push(user_text_message(
|
||||
user_message_id.clone(),
|
||||
"near the independent input limit",
|
||||
));
|
||||
for (input_tokens, expected_target) in [(80_999, None), (81_000, Some(1))] {
|
||||
thread.request_token_usage.insert(
|
||||
user_message_id.clone(),
|
||||
language_model::TokenUsage {
|
||||
input_tokens,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(thread.compaction_message_target_ix(cx), expected_target);
|
||||
assert_eq!(thread.input_token_capacity(), Some(90_000));
|
||||
assert_eq!(thread.latest_token_usage().unwrap().max_tokens, 200_000);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_compaction_threshold_respects_enabled_setting(cx: &mut TestAppContext) {
|
||||
let (thread, _event_stream) = setup_thread_for_test(cx).await;
|
||||
|
||||
@@ -4762,13 +4762,20 @@ impl ThreadView {
|
||||
|
||||
let workspace = self.workspace.clone();
|
||||
|
||||
let max_output_tokens = self
|
||||
let (max_input_tokens, max_output_tokens) = self
|
||||
.as_native_thread(cx)
|
||||
.and_then(|thread| thread.read(cx).model())
|
||||
.and_then(|model| model.max_output_tokens())
|
||||
.unwrap_or(0);
|
||||
let input_max_label =
|
||||
crate::humanize_token_count(usage.max_tokens.saturating_sub(max_output_tokens));
|
||||
.map(|thread| {
|
||||
let thread = thread.read(cx);
|
||||
(
|
||||
thread.input_token_capacity().unwrap_or(usage.max_tokens),
|
||||
thread
|
||||
.model()
|
||||
.and_then(|model| model.max_output_tokens())
|
||||
.unwrap_or(0),
|
||||
)
|
||||
})
|
||||
.unwrap_or((usage.max_tokens, 0));
|
||||
let input_max_label = crate::humanize_token_count(max_input_tokens);
|
||||
let output_max_label = crate::humanize_token_count(max_output_tokens);
|
||||
|
||||
let build_tooltip = {
|
||||
@@ -4804,7 +4811,7 @@ impl ThreadView {
|
||||
};
|
||||
|
||||
if show_split {
|
||||
let input_max_raw = usage.max_tokens.saturating_sub(max_output_tokens);
|
||||
let input_max_raw = max_input_tokens;
|
||||
let output_max_raw = max_output_tokens;
|
||||
|
||||
let input_ratio = if input_max_raw > 0 {
|
||||
|
||||
@@ -279,6 +279,11 @@ impl Model {
|
||||
self.capabilities.limits.max_context_window_tokens as u64
|
||||
}
|
||||
|
||||
pub fn max_prompt_tokens(&self) -> Option<u64> {
|
||||
let limit = self.capabilities.limits.max_prompt_tokens;
|
||||
(limit > 0).then_some(limit)
|
||||
}
|
||||
|
||||
pub fn max_output_tokens(&self) -> Option<u64> {
|
||||
let limit = self.capabilities.limits.max_output_tokens as u64;
|
||||
(limit > 0).then_some(limit)
|
||||
|
||||
@@ -130,6 +130,16 @@ impl LanguageModel for CopilotChatLanguageModel {
|
||||
self.model.max_token_count()
|
||||
}
|
||||
|
||||
fn max_input_tokens(&self) -> u64 {
|
||||
self.model
|
||||
.max_prompt_tokens()
|
||||
.unwrap_or_else(|| self.model.max_token_count())
|
||||
}
|
||||
|
||||
fn max_output_tokens(&self) -> Option<u64> {
|
||||
self.model.max_output_tokens()
|
||||
}
|
||||
|
||||
fn stream_completion(
|
||||
&self,
|
||||
request: LanguageModelRequest,
|
||||
@@ -1301,6 +1311,85 @@ mod tests {
|
||||
use language_model::ProviderErrorCategory;
|
||||
use serde_json::json;
|
||||
|
||||
#[gpui::test]
|
||||
fn language_model_exposes_token_limits(cx: &mut gpui::TestAppContext) {
|
||||
use gpui::AppContext as _;
|
||||
|
||||
let copilot_chat = cx.new(|cx| {
|
||||
CopilotChat::new(
|
||||
Arc::new(http_client::BlockedHttpClient::new()),
|
||||
Arc::new(EmptyCredentialsProvider),
|
||||
crate::CopilotChatConfiguration::default(),
|
||||
cx,
|
||||
)
|
||||
});
|
||||
for (limits, input, context, output) in [
|
||||
(
|
||||
json!({"max_context_window_tokens": 200_000, "max_prompt_tokens": 90_000, "max_output_tokens": 16_384}),
|
||||
90_000,
|
||||
200_000,
|
||||
Some(16_384),
|
||||
),
|
||||
(
|
||||
json!({"max_context_window_tokens": 128_000, "max_output_tokens": 4096}),
|
||||
128_000,
|
||||
128_000,
|
||||
Some(4096),
|
||||
),
|
||||
(
|
||||
json!({"max_context_window_tokens": 128_000, "max_prompt_tokens": 100_000}),
|
||||
100_000,
|
||||
128_000,
|
||||
None,
|
||||
),
|
||||
(
|
||||
json!({"max_context_window_tokens": 128_000, "max_prompt_tokens": 0, "max_output_tokens": 0}),
|
||||
128_000,
|
||||
128_000,
|
||||
None,
|
||||
),
|
||||
] {
|
||||
let mut value = serde_json::to_value(test_responses_model()).unwrap();
|
||||
value["capabilities"]["limits"] = limits;
|
||||
let descriptor = serde_json::from_value(value).unwrap();
|
||||
let model = create_language_model(descriptor, copilot_chat.clone());
|
||||
assert_eq!(model.max_token_count(), context);
|
||||
assert_eq!(model.max_input_tokens(), input);
|
||||
assert_eq!(model.max_total_tokens(), Some(context));
|
||||
assert_eq!(model.max_output_tokens(), output);
|
||||
}
|
||||
}
|
||||
|
||||
struct EmptyCredentialsProvider;
|
||||
|
||||
impl credentials_provider::CredentialsProvider for EmptyCredentialsProvider {
|
||||
fn read_credentials<'a>(
|
||||
&'a self,
|
||||
_url: &'a str,
|
||||
_cx: &'a AsyncApp,
|
||||
) -> Pin<Box<dyn Future<Output = Result<Option<(String, Vec<u8>)>>> + 'a>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn write_credentials<'a>(
|
||||
&'a self,
|
||||
_url: &'a str,
|
||||
_username: &'a str,
|
||||
_password: &'a [u8],
|
||||
_cx: &'a AsyncApp,
|
||||
) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn delete_credentials<'a>(
|
||||
&'a self,
|
||||
_url: &'a str,
|
||||
_cx: &'a AsyncApp,
|
||||
) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
fn map_events(events: Vec<responses::StreamEvent>) -> Vec<LanguageModelCompletionEvent> {
|
||||
futures::executor::block_on(async {
|
||||
CopilotResponsesEventMapper::new()
|
||||
|
||||
@@ -121,6 +121,7 @@ pub struct FakeLanguageModel {
|
||||
supports_images: AtomicBool,
|
||||
supports_server_side_compaction: AtomicBool,
|
||||
max_token_count: AtomicU64,
|
||||
max_input_tokens: Option<u64>,
|
||||
max_output_tokens: AtomicU64,
|
||||
input_token_counts: Mutex<VecDeque<u64>>,
|
||||
input_token_count_requests: Mutex<Vec<LanguageModelRequest>>,
|
||||
@@ -141,6 +142,7 @@ impl Default for FakeLanguageModel {
|
||||
supports_images: AtomicBool::new(false),
|
||||
supports_server_side_compaction: AtomicBool::new(false),
|
||||
max_token_count: AtomicU64::new(1_000_000),
|
||||
max_input_tokens: None,
|
||||
max_output_tokens: AtomicU64::new(0),
|
||||
input_token_counts: Mutex::new(VecDeque::new()),
|
||||
input_token_count_requests: Mutex::new(Vec::new()),
|
||||
@@ -196,6 +198,10 @@ impl FakeLanguageModel {
|
||||
self.max_token_count.store(count, SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_max_input_tokens(&mut self, count: u64) {
|
||||
self.max_input_tokens = Some(count);
|
||||
}
|
||||
|
||||
pub fn set_max_output_tokens(&self, count: Option<u64>) {
|
||||
self.max_output_tokens
|
||||
.store(count.unwrap_or_default(), SeqCst);
|
||||
@@ -342,6 +348,11 @@ impl LanguageModel for FakeLanguageModel {
|
||||
self.max_token_count.load(SeqCst)
|
||||
}
|
||||
|
||||
fn max_input_tokens(&self) -> u64 {
|
||||
self.max_input_tokens
|
||||
.unwrap_or_else(|| self.max_token_count())
|
||||
}
|
||||
|
||||
fn max_output_tokens(&self) -> Option<u64> {
|
||||
let max_output_tokens = self.max_output_tokens.load(SeqCst);
|
||||
if max_output_tokens == 0 {
|
||||
|
||||
@@ -215,9 +215,16 @@ pub trait LanguageModel: Send + Sync {
|
||||
false
|
||||
}
|
||||
|
||||
/// Returns the input token ceiling.
|
||||
/// Returns the model's context-window capacity.
|
||||
fn max_token_count(&self) -> u64;
|
||||
|
||||
/// Returns the input ceiling before reserving output from any shared window.
|
||||
///
|
||||
/// Models with a separate prompt limit override the context-window default.
|
||||
fn max_input_tokens(&self) -> u64 {
|
||||
self.max_token_count()
|
||||
}
|
||||
|
||||
/// Counts request input without generating output, when supported by the provider.
|
||||
///
|
||||
/// Counts may be estimates and differ from subsequent measured usage. Callers
|
||||
@@ -233,7 +240,7 @@ pub trait LanguageModel: Send + Sync {
|
||||
|
||||
/// Returns the combined input and output ceiling, if one applies.
|
||||
///
|
||||
/// The conservative default shares the input ceiling with output. `None`
|
||||
/// The conservative default shares the context window with output. `None`
|
||||
/// means generation does not consume that window, not merely that the API
|
||||
/// validates input separately or stops generation at the window boundary.
|
||||
fn max_total_tokens(&self) -> Option<u64> {
|
||||
|
||||
Reference in new issue
Block a user