From f3c23ec56ba47486f3586a8a1ee63ed804b63daf Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 25 Aug 2026 17:33:00 +0000 Subject: [PATCH] open_ai: Classify billing and content policy errors (#63208) OpenAI can return structured provider failures either as Responses stream events or as non-success HTTP responses. The streaming path classified the provider's error code, while the HTTP path treated the body as opaque and classified only its status. As a result, exhausted credit could look like a retryable rate limit, and prompt policy rejections looked like generic provider failures. This change gives billing failures their existing payment-required category and adds a content-policy category for `invalid_prompt` and `cyber_policy`. It also parses OpenAI's standard HTTP error envelope and sends both HTTP and stream failures through the same classifier while preserving the original status, code, and message. Responses with another body format retain the previous status-based fallback. Release Notes: - N/A --- crates/agent_ui/src/conversation_view.rs | 1 + .../src/language_model_core.rs | 6 +- crates/open_ai/src/completion.rs | 87 ++++++++++++++---- crates/open_ai/src/open_ai.rs | 91 ++++++++++++++++++- 4 files changed, 161 insertions(+), 24 deletions(-) diff --git a/crates/agent_ui/src/conversation_view.rs b/crates/agent_ui/src/conversation_view.rs index d31efedb76..cc33d45f23 100644 --- a/crates/agent_ui/src/conversation_view.rs +++ b/crates/agent_ui/src/conversation_view.rs @@ -198,6 +198,7 @@ impl From for ThreadError { provider: provider.to_string().into(), }, ProviderErrorCategory::InvalidEncryptedContent + | ProviderErrorCategory::ContentPolicy | ProviderErrorCategory::InvalidRequest | ProviderErrorCategory::Conflict | ProviderErrorCategory::Timeout diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rs index ca74ce0314..d2975e18eb 100644 --- a/crates/language_model_core/src/language_model_core.rs +++ b/crates/language_model_core/src/language_model_core.rs @@ -128,6 +128,7 @@ pub enum ProviderErrorCategory { InvalidEncryptedContent, Authentication, Permission, + ContentPolicy, EndpointNotFound, PaymentRequired, RateLimit, @@ -426,6 +427,7 @@ fn category_from_cloud_failure(code: &str, message: &str) -> ProviderErrorCatego "authentication_error" => ProviderErrorCategory::Authentication, "billing_error" | "payment_required_error" => ProviderErrorCategory::PaymentRequired, "permission_error" => ProviderErrorCategory::Permission, + "cyber_policy" | "invalid_prompt" => ProviderErrorCategory::ContentPolicy, "not_found_error" => ProviderErrorCategory::EndpointNotFound, "conflict_error" => ProviderErrorCategory::Conflict, "rate_limit_error" | "rate_limit_exceeded" => ProviderErrorCategory::RateLimit, @@ -934,7 +936,7 @@ mod tests { } #[test] - fn test_from_cloud_failure_preserves_unknown_provider_rejection() { + fn test_from_cloud_failure_maps_content_policy_rejection() { let error = LanguageModelCompletionError::from_cloud_failure( OPEN_AI_PROVIDER_NAME, "cyber_policy".to_string(), @@ -950,7 +952,7 @@ mod tests { code: Some(code), message, retry_after: None, - category: ProviderErrorCategory::Other, + category: ProviderErrorCategory::ContentPolicy, } if provider == OPEN_AI_PROVIDER_NAME && code == "cyber_policy" && message == "This content was flagged as potentially violating our terms of use." diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs index 94028b976e..4e2d69f3e7 100644 --- a/crates/open_ai/src/completion.rs +++ b/crates/open_ai/src/completion.rs @@ -1,6 +1,7 @@ use anyhow::{Result, anyhow}; use collections::HashMap; use futures::{Stream, StreamExt}; +use http_client::StatusCode; use language_model_core::{ CompactedContext, CompactionUpdate, LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelCustomToolFormat, LanguageModelCustomToolGrammarSyntax, LanguageModelImage, @@ -1630,25 +1631,7 @@ fn completion_error_from_response_error( error: &ResponseError, provider: language_model_core::LanguageModelProviderName, ) -> LanguageModelCompletionError { - let category = match error.code.as_deref() { - Some("context_length_exceeded" | "request_too_large") => { - ProviderErrorCategory::PromptTooLarge { tokens: None } - } - Some("invalid_encrypted_content") => ProviderErrorCategory::InvalidEncryptedContent, - Some("invalid_request_error") => ProviderErrorCategory::InvalidRequest, - Some("authentication_error") => ProviderErrorCategory::Authentication, - Some("billing_error" | "payment_required_error") => ProviderErrorCategory::PaymentRequired, - Some("permission_error") => ProviderErrorCategory::Permission, - Some("not_found_error") => ProviderErrorCategory::EndpointNotFound, - Some("conflict_error") => ProviderErrorCategory::Conflict, - Some("rate_limit_error" | "rate_limit_exceeded") => ProviderErrorCategory::RateLimit, - Some("timeout_error" | "request_timed_out") => ProviderErrorCategory::Timeout, - Some("api_error" | "internal_server_error" | "server_error") => { - ProviderErrorCategory::InternalServer - } - Some("overloaded_error") => ProviderErrorCategory::Overloaded, - Some(_) | None => ProviderErrorCategory::Other, - }; + let category = response_error_category(error.code.as_deref(), None, &error.message); LanguageModelCompletionError::from_provider_response( provider, None, @@ -1659,6 +1642,43 @@ fn completion_error_from_response_error( ) } +pub(crate) fn response_error_category( + code: Option<&str>, + status: Option, + message: &str, +) -> ProviderErrorCategory { + match code { + Some("context_length_exceeded" | "request_too_large") => { + ProviderErrorCategory::PromptTooLarge { tokens: None } + } + Some("invalid_encrypted_content") => ProviderErrorCategory::InvalidEncryptedContent, + Some("invalid_request_error") => ProviderErrorCategory::InvalidRequest, + Some("authentication_error") => ProviderErrorCategory::Authentication, + Some( + "billing_error" + | "payment_required_error" + | "credit_balance_exhausted" + | "insufficient_quota" + | "organization_spend_limit_exceeded" + | "project_spend_limit_exceeded" + | "organization_usage_limit_exceeded", + ) => ProviderErrorCategory::PaymentRequired, + Some("permission_error") => ProviderErrorCategory::Permission, + Some("cyber_policy" | "invalid_prompt") => ProviderErrorCategory::ContentPolicy, + Some("not_found_error") => ProviderErrorCategory::EndpointNotFound, + Some("conflict_error") => ProviderErrorCategory::Conflict, + Some("rate_limit_error" | "rate_limit_exceeded") => ProviderErrorCategory::RateLimit, + Some("timeout_error" | "request_timed_out") => ProviderErrorCategory::Timeout, + Some("api_error" | "internal_server_error" | "server_error") => { + ProviderErrorCategory::InternalServer + } + Some("overloaded_error") => ProviderErrorCategory::Overloaded, + Some(_) | None => status + .map(|status| ProviderErrorCategory::from_http_status(status, message)) + .unwrap_or(ProviderErrorCategory::Other), + } +} + fn response_error_message(error: &ResponseError) -> String { let code = error.code.as_deref().filter(|code| !code.trim().is_empty()); let message = error.message.trim(); @@ -3207,13 +3227,40 @@ mod tests { code: Some(code), message, retry_after: None, - category: ProviderErrorCategory::Other, + category: ProviderErrorCategory::ContentPolicy, } if provider == OPEN_AI_PROVIDER_NAME && code == "cyber_policy" && message == "This content was flagged as potentially violating our terms of use." )); } + #[test] + fn responses_stream_maps_billing_codes_to_payment_required() { + for code in [ + "billing_error", + "payment_required_error", + "credit_balance_exhausted", + "insufficient_quota", + "organization_spend_limit_exceeded", + "project_spend_limit_exceeded", + "organization_usage_limit_exceeded", + ] { + assert_eq!( + response_error_category(Some(code), None, ""), + ProviderErrorCategory::PaymentRequired, + "{code}" + ); + } + } + + #[test] + fn responses_stream_maps_invalid_prompt_to_content_policy() { + assert_eq!( + response_error_category(Some("invalid_prompt"), None, ""), + ProviderErrorCategory::ContentPolicy + ); + } + #[test] fn responses_stream_maps_context_length_exceeded_to_prompt_too_large() { let event = serde_json::from_value::(json!({ diff --git a/crates/open_ai/src/open_ai.rs b/crates/open_ai/src/open_ai.rs index 2b79a5c823..f25c324966 100644 --- a/crates/open_ai/src/open_ai.rs +++ b/crates/open_ai/src/open_ai.rs @@ -444,7 +444,11 @@ impl Model { #[cfg(test)] mod tests { - use super::{Model, ReasoningEffort}; + use language_model_core::{ + LanguageModelCompletionError, OPEN_AI_PROVIDER_NAME, ProviderErrorCategory, + }; + + use super::{Model, ReasoningEffort, RequestError, StatusCode}; #[test] fn gpt_5_1_uses_none_reasoning_by_default() { @@ -516,6 +520,67 @@ mod tests { ); } + #[test] + fn http_credit_balance_exhaustion_is_payment_required() { + let error = LanguageModelCompletionError::from(RequestError::HttpResponseError { + provider: OPEN_AI_PROVIDER_NAME.to_string(), + status_code: StatusCode::TOO_MANY_REQUESTS, + body: serde_json::json!({ + "error": { + "message": "You have no credits remaining.", + "type": "insufficient_quota", + "param": null, + "code": "credit_balance_exhausted" + } + }) + .to_string(), + headers: Box::default(), + }); + + assert!(matches!( + error, + LanguageModelCompletionError::ProviderRejection { + status: Some(StatusCode::TOO_MANY_REQUESTS), + code: Some(code), + message, + category: ProviderErrorCategory::PaymentRequired, + .. + } if code == "credit_balance_exhausted" + && message == "You have no credits remaining." + )); + } + + #[test] + fn http_invalid_prompt_is_content_policy_rejection() { + let error = LanguageModelCompletionError::from(RequestError::HttpResponseError { + provider: OPEN_AI_PROVIDER_NAME.to_string(), + status_code: StatusCode::BAD_REQUEST, + body: serde_json::json!({ + "error": { + "message": "Invalid prompt: your prompt was flagged as potentially violating our usage policy.", + "type": "invalid_request_error", + "param": null, + "code": "invalid_prompt" + } + }) + .to_string(), + headers: Box::default(), + }); + + assert!(matches!( + error, + LanguageModelCompletionError::ProviderRejection { + status: Some(StatusCode::BAD_REQUEST), + code: Some(code), + message, + category: ProviderErrorCategory::ContentPolicy, + .. + } if code == "invalid_prompt" + && message + == "Invalid prompt: your prompt was flagged as potentially violating our usage policy." + )); + } + #[test] fn stream_usage_with_null_prompt_cache_tokens_is_not_an_error() { let chunk = r#"{"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":null}}}"#; @@ -855,6 +920,11 @@ pub enum ResponseStreamResult { Err { error: ResponseStreamError }, } +#[derive(Deserialize)] +struct ResponseErrorEnvelope { + error: responses::ResponseError, +} + #[derive(Serialize, Deserialize, Debug)] pub struct ResponseStreamEvent { pub choices: Vec, @@ -1197,7 +1267,24 @@ impl From for language_model_core::LanguageModelCompletionError { .and_then(|val| val.to_str().ok()?.parse::().ok()) .map(std::time::Duration::from_secs); - Self::from_http_status(provider.into(), status_code, body, retry_after) + if let Ok(error_response) = serde_json::from_str::(&body) { + let error = error_response.error; + let category = completion::response_error_category( + error.code.as_deref(), + Some(status_code), + &error.message, + ); + Self::from_provider_response( + provider.into(), + Some(status_code), + error.code, + error.message, + retry_after, + category, + ) + } else { + Self::from_http_status(provider.into(), status_code, body, retry_after) + } } RequestError::SerializeRequest { provider, error } => Self::SerializeRequest { provider: provider.into(),