diff --git a/Cargo.lock b/Cargo.lock index f43363e23c..f5803b5baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9823,6 +9823,7 @@ dependencies = [ "language", "language_model", "language_models_cloud", + "llama_cpp", "lmstudio", "log", "menu", @@ -10354,6 +10355,19 @@ dependencies = [ "zed-scap", ] +[[package]] +name = "llama_cpp" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures 0.3.32", + "http_client", + "schemars 1.0.4", + "serde", + "serde_json", + "url", +] + [[package]] name = "lmdb-master-sys" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 4f36916d2d..4d36e276a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ members = [ "crates/line_ending_selector", "crates/livekit_api", "crates/livekit_client", + "crates/llama_cpp", "crates/lmstudio", "crates/lsp", "crates/markdown", @@ -384,6 +385,7 @@ languages = { path = "crates/languages" } line_ending_selector = { path = "crates/line_ending_selector" } livekit_api = { path = "crates/livekit_api" } livekit_client = { path = "crates/livekit_client" } +llama_cpp = { path = "crates/llama_cpp" } lmstudio = { path = "crates/lmstudio" } lsp = { path = "crates/lsp" } markdown = { path = "crates/markdown" } @@ -977,6 +979,7 @@ edit_prediction_ui = { codegen-units = 1 } install_cli = { codegen-units = 1 } journal = { codegen-units = 1 } json_schema_store = { codegen-units = 1 } +llama_cpp = { codegen-units = 1 } lmstudio = { codegen-units = 1 } menu = { codegen-units = 1 } notifications = { codegen-units = 1 } diff --git a/assets/icons/ai_llama_cpp.svg b/assets/icons/ai_llama_cpp.svg new file mode 100644 index 0000000000..310ec3793a --- /dev/null +++ b/assets/icons/ai_llama_cpp.svg @@ -0,0 +1,4 @@ + + + + diff --git a/assets/settings/default.json b/assets/settings/default.json index e4051ee8f0..059608ae04 100644 --- a/assets/settings/default.json +++ b/assets/settings/default.json @@ -2427,6 +2427,9 @@ "ollama": { "api_url": "http://localhost:11434", }, + "llama_cpp": { + "api_url": "http://localhost:8080", + }, "openai": { "api_url": "https://api.openai.com/v1", }, diff --git a/crates/icons/src/icons.rs b/crates/icons/src/icons.rs index a94b43e801..56182da213 100644 --- a/crates/icons/src/icons.rs +++ b/crates/icons/src/icons.rs @@ -17,6 +17,7 @@ pub enum IconName { AiEdit, AiGemini, AiGoogle, + AiLlamaCpp, AiLmStudio, AiMistral, AiOllama, diff --git a/crates/language_models/Cargo.toml b/crates/language_models/Cargo.toml index b96a377f02..57a0d530e4 100644 --- a/crates/language_models/Cargo.toml +++ b/crates/language_models/Cargo.toml @@ -43,6 +43,7 @@ http_client.workspace = true language.workspace = true language_model.workspace = true language_models_cloud.workspace = true +llama_cpp = { workspace = true, features = ["schemars"] } lmstudio = { workspace = true, features = ["schemars"] } log.workspace = true menu.workspace = true diff --git a/crates/language_models/src/language_models.rs b/crates/language_models/src/language_models.rs index dc13cde137..1133ee2d47 100644 --- a/crates/language_models/src/language_models.rs +++ b/crates/language_models/src/language_models.rs @@ -24,6 +24,7 @@ use crate::provider::bedrock::BedrockLanguageModelProvider; use crate::provider::cloud::CloudLanguageModelProvider; use crate::provider::copilot_chat::CopilotChatLanguageModelProvider; use crate::provider::google::GoogleLanguageModelProvider; +use crate::provider::llama_cpp::LlamaCppLanguageModelProvider; use crate::provider::lmstudio::LmStudioLanguageModelProvider; pub use crate::provider::mistral::MistralLanguageModelProvider; use crate::provider::ollama::OllamaLanguageModelProvider; @@ -307,6 +308,14 @@ fn register_language_model_providers( )), cx, ); + registry.register_provider( + Arc::new(LlamaCppLanguageModelProvider::new( + client.http_client(), + credentials_provider.clone(), + cx, + )), + cx, + ); registry.register_provider( Arc::new(DeepSeekLanguageModelProvider::new( client.http_client(), diff --git a/crates/language_models/src/provider.rs b/crates/language_models/src/provider.rs index 3a1369a2d1..65fc039b07 100644 --- a/crates/language_models/src/provider.rs +++ b/crates/language_models/src/provider.rs @@ -10,6 +10,7 @@ pub mod cloud; pub mod copilot_chat; pub mod deepseek; pub mod google; +pub mod llama_cpp; pub mod lmstudio; pub mod mistral; pub mod ollama; diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rs new file mode 100644 index 0000000000..935ec74f9a --- /dev/null +++ b/crates/language_models/src/provider/llama_cpp.rs @@ -0,0 +1,2073 @@ +use anyhow::Result; +use collections::{HashMap, HashSet}; +use credentials_provider::CredentialsProvider; +use fs::Fs; +use futures::Stream; +use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream}; +use gpui::{AnyView, App, AsyncApp, Context, CursorStyle, Entity, Task, TaskExt}; +use http_client::{CustomHeaders, HttpClient}; +use language_model::util::parse_tool_arguments; +use language_model::{ + ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel, LanguageModelCompletionError, + LanguageModelCompletionEvent, LanguageModelId, LanguageModelName, LanguageModelProvider, + LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState, + LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolResultContent, + LanguageModelToolUse, MessageContent, RateLimiter, Role, StopReason, TokenUsage, env_var, +}; +use llama_cpp::{ + LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion, + stream_model_events, +}; +pub use settings::LlamaCppAvailableModel as AvailableModel; +use settings::{Settings, SettingsStore, update_settings_file}; +use std::pin::Pin; +use std::sync::LazyLock; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::time::Duration; +use ui::{ + ButtonLike, ButtonLink, ConfiguredApiCard, ElevationIndex, List, ListBulletItem, Tooltip, + prelude::*, +}; +use ui_input::InputField; +use util::ResultExt; + +use crate::AllLanguageModelSettings; + +const LLAMA_CPP_DOWNLOAD_URL: &str = "https://llama.app"; +const LLAMA_CPP_MODELS_URL: &str = "https://huggingface.co/models?library=gguf&sort=trending"; + +const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("llama_cpp"); +const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new("llama.cpp"); + +const API_KEY_ENV_VAR_NAME: &str = "LLAMACPP_API_KEY"; +static API_KEY_ENV_VAR: LazyLock = env_var!(API_KEY_ENV_VAR_NAME); + +/// How long to wait before reconnecting to `/models/sse` after the stream ends. +const MODEL_EVENT_RECONNECT_INTERVAL: Duration = Duration::from_secs(5); + +/// Context length assumed for an unloaded router model (it can't be probed +/// without loading it). Generous so early messages work; re-discovery refines +/// it once the model loads. +const ASSUMED_UNLOADED_CONTEXT: u64 = 131_072; + +#[derive(Default, Debug, Clone, PartialEq)] +pub struct LlamaCppSettings { + pub api_url: String, + pub auto_discover: bool, + pub available_models: Vec, + pub context_window: Option, + pub custom_headers: CustomHeaders, +} + +pub struct LlamaCppLanguageModelProvider { + http_client: Arc, + state: Entity, + /// Live capabilities shared with the agent's models (see [`LiveCapabilities`]). + capability_cells: CapabilityCells, + /// Live model-load progress shared with the models (see [`LoadingProgress`]). + loading_progress: LoadingProgress, +} + +pub struct State { + api_key_state: ApiKeyState, + credentials_provider: Arc, + http_client: Arc, + fetched_models: Vec, + fetch_model_task: Option>>, + /// Router-mode task on `/models/sse`; re-runs discovery as models load/unload. + model_event_task: Option>, + /// Same `Arc` as the provider's; re-discovery keeps these cells in sync. + capability_cells: CapabilityCells, + /// Same `Arc` as the provider's; the event stream updates it as a model loads. + loading_progress: LoadingProgress, +} + +impl State { + fn is_authenticated(&self) -> bool { + !self.fetched_models.is_empty() + } + + fn set_api_key(&mut self, api_key: Option, cx: &mut Context) -> Task> { + let credentials_provider = self.credentials_provider.clone(); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let task = self.api_key_state.store( + api_url, + api_key, + |this| &mut this.api_key_state, + credentials_provider, + cx, + ); + + self.fetched_models.clear(); + // Drop the event stream so it reconnects with the new key (re-fetch + // below restarts it). + self.model_event_task = None; + write_recover(&self.loading_progress).clear(); + cx.spawn(async move |this, cx| { + let result = task.await; + this.update(cx, |this, cx| this.restart_fetch_models_task(cx)) + .ok(); + result + }) + } + + fn authenticate(&mut self, cx: &mut Context) -> Task> { + let credentials_provider = self.credentials_provider.clone(); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let load_key_task = self.api_key_state.load_if_needed( + api_url, + |this| &mut this.api_key_state, + credentials_provider, + cx, + ); + + if self.is_authenticated() { + return Task::ready(Ok(())); + } + + cx.spawn(async move |this, cx| { + match load_key_task.await { + Ok(()) | Err(AuthenticateError::CredentialsNotFound) => {} + Err(error) => { + log::warn!("failed to load llama.cpp API key: {error}"); + } + } + let fetch_models_task = this.update(cx, |this, cx| this.fetch_models(cx))?; + match fetch_models_task.await { + Ok(()) => Ok(()), + Err(err) => { + // A refused connection means the server isn't running yet, not an error. + let connection_refused = err.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io_err| { + io_err.kind() == std::io::ErrorKind::ConnectionRefused + }) + }); + if connection_refused { + Err(AuthenticateError::ConnectionRefused) + } else { + Err(AuthenticateError::Other(err)) + } + } + } + }) + } + + fn fetch_models(&mut self, cx: &mut Context) -> Task> { + let http_client = Arc::clone(&self.http_client); + let settings = LlamaCppLanguageModelProvider::settings(cx); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let api_key = self.api_key_state.key(&api_url); + let extra_headers = settings.custom_headers.clone(); + + cx.spawn(async move |this, cx| { + let entries = get_models( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + &extra_headers, + ) + .await?; + + let is_router = entries.iter().any(ModelEntry::is_router_entry); + + // Models the server reports as loading, used below to prune stale + // progress labels by reconciling against the live listing (a preempted + // load or missed SSE event can skip the terminal event). + let loading_ids: HashSet = entries + .iter() + .filter(|entry| entry.is_loading()) + .map(|entry| entry.id.clone()) + .collect(); + + let models: Vec = if is_router { + // Router mode: metadata comes from `/v1/models`. We probe + // `/props` only for loaded models so listing never triggers a + // load; unloaded models use the listing's hints and overrides. + let tasks = entries.into_iter().map(|entry| { + let http_client = Arc::clone(&http_client); + let api_url = api_url.clone(); + let api_key = api_key.clone(); + let extra_headers = extra_headers.clone(); + async move { + let props = if entry.is_loaded() { + get_props( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + Some(&entry.id), + &extra_headers, + ) + .await + .log_err() + } else { + None + }; + model_from_entry(&entry, props.as_ref()) + } + }); + futures::stream::iter(tasks) + .buffer_unordered(5) + .collect() + .await + } else { + // Single-model mode: one `/props` call describes the loaded model. + let props = get_props( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + None, + &extra_headers, + ) + .await + .log_err(); + entries + .iter() + .map(|entry| model_from_entry(entry, props.as_ref())) + .collect() + }; + + this.update(cx, |this, cx| { + this.fetched_models = models; + let effective = compute_effective_models( + &this.fetched_models, + LlamaCppLanguageModelProvider::settings(cx), + ); + sync_capability_cells(&this.capability_cells, &effective); + // Drop progress labels for models no longer loading, so a stale + // "Loading …" can't stick after a preempted load or missed event. + write_recover(&this.loading_progress).retain(|id, _| loading_ids.contains(id)); + // Router mode loads models on demand: subscribe so capabilities + // self-correct as they load/unload. Start it once (events trigger + // re-discovery, not a re-spawn); single-model mode needs no stream. + if is_router { + if this.model_event_task.is_none() { + this.start_model_event_stream(cx); + } + } else { + this.model_event_task = None; + } + cx.notify(); + }) + }) + } + + /// Subscribes to `/models/sse` and re-runs discovery as models load, unload, + /// or the list changes, so capabilities stay current. Reconnects if the stream + /// drops; on builds without `/models/sse` the refresh is simply skipped. + fn start_model_event_stream(&mut self, cx: &mut Context) { + let http_client = Arc::clone(&self.http_client); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let api_key = self.api_key_state.key(&api_url); + let extra_headers = LlamaCppLanguageModelProvider::settings(cx) + .custom_headers + .clone(); + + self.model_event_task = Some(cx.spawn(async move |this, cx| { + loop { + match stream_model_events( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + &extra_headers, + ) + .await + { + Ok(mut events) => { + while let Some(event) = events.next().await { + let Some(event) = event.log_err() else { + continue; + }; + if let Some(exit_code) = event.load_failure() { + log::error!( + "llama.cpp model {} failed to load (exit code {exit_code})", + event.model + ); + } + // Loading-progress tick: record it for the selector (no + // re-discovery). `cx.notify()` drives `ProviderStateChanged`. + if let Some(progress) = event.load_progress() { + let label = SharedString::from(progress.progress_label()); + if this + .update(cx, |this, cx| { + write_recover(&this.loading_progress) + .insert(event.model.clone(), label); + cx.notify(); + }) + .is_err() + { + return; + } + continue; + } + if !event.changes_model_state() { + continue; + } + // Terminal load/unload (or list change): drop the + // progress label and re-discover to refresh capabilities. + if this + .update(cx, |this, cx| { + write_recover(&this.loading_progress).remove(&event.model); + this.restart_fetch_models_task(cx); + }) + .is_err() + { + return; + } + } + } + // Endpoint missing (older build) or connection failed; retry after a backoff. + Err(error) => { + log::warn!("llama.cpp model event stream unavailable: {error:#}"); + } + } + + cx.background_executor() + .timer(MODEL_EVENT_RECONNECT_INTERVAL) + .await; + if this.update(cx, |_, _| ()).is_err() { + return; + } + } + })); + } + + fn restart_fetch_models_task(&mut self, cx: &mut Context) { + let task = self.fetch_models(cx); + self.fetch_model_task.replace(task); + } +} + +/// Capabilities that only become known once a router model loads. `LanguageModel`'s +/// methods take no `cx`, yet the agent reads them live each turn, so we share +/// them through a map that re-discovery updates — an already-selected model +/// picks up real values without re-selection. Image support is available from +/// `/v1/models` hints, so it stays a field. +#[derive(Clone, Copy, Debug, PartialEq)] +struct LiveCapabilities { + max_tokens: u64, + supports_tools: bool, + supports_thinking: bool, +} + +impl LiveCapabilities { + fn of(model: &llama_cpp::Model) -> Self { + Self { + max_tokens: model.max_tokens, + supports_tools: model.supports_tools, + supports_thinking: model.supports_thinking, + } + } +} + +/// Live capabilities keyed by model name, shared by the provider and its models. +type CapabilityCells = Arc>>; + +/// Model name → load-status label (e.g. `"Loading weights 42%"`) while a router +/// model loads, shared so the model selector can show progress. Absent once loaded. +type LoadingProgress = Arc>>; + +/// Locks for reading, recovering instead of panicking on a poisoned lock. The +/// critical sections are infallible map ops, so poisoning is unreachable anyway. +fn read_recover(lock: &RwLock) -> RwLockReadGuard<'_, T> { + lock.read().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Locks for writing; see [`read_recover`]. +fn write_recover(lock: &RwLock) -> RwLockWriteGuard<'_, T> { + lock.write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The exact set of models `provided_models` exposes: discovery merged with the +/// `available_models` and `context_window` overrides. Shared with re-discovery. +fn compute_effective_models( + fetched_models: &[llama_cpp::Model], + settings: &LlamaCppSettings, +) -> HashMap { + let mut models: HashMap = HashMap::default(); + if settings.auto_discover { + for model in fetched_models { + let mut model = model.clone(); + if let Some(context_window) = settings.context_window { + model.max_tokens = context_window; + } + models.insert(model.name.clone(), model); + } + } + merge_settings_into_models( + &mut models, + &settings.available_models, + settings.context_window, + ); + models +} + +/// Updates the shared capability map from the effective models, so a model held +/// by an open conversation observes the new values (it reads the map by name). +fn sync_capability_cells(cells: &CapabilityCells, effective: &HashMap) { + let mut cells = write_recover(cells); + for model in effective.values() { + cells.insert(model.name.clone(), LiveCapabilities::of(model)); + } +} + +/// Builds a model from a `/v1/models` entry, refined by `/props` when the model +/// is loaded. An unloaded router model can't be probed, so we assume optimistic +/// capabilities and let re-discovery refine them on load. +fn model_from_entry(entry: &ModelEntry, props: Option<&Props>) -> llama_cpp::Model { + let max_tokens = props + .and_then(Props::context_length) + .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx)) + .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx_train)) + .unwrap_or(ASSUMED_UNLOADED_CONTEXT); + // Trust `/props` when present. Without it, assume tools for an unloaded model + // (re-discovery corrects on load) but not for a loaded model whose probe failed. + let supports_tools = match props { + Some(props) => props.supports_tools(), + None => !entry.is_loaded(), + }; + let supports_images = props.is_some_and(Props::supports_images) || entry.supports_images_hint(); + let supports_thinking = props.is_some_and(Props::supports_thinking); + + llama_cpp::Model::new( + &entry.id, + Some(&display_name_for(&entry.id)), + Some(max_tokens), + supports_tools, + supports_images, + supports_thinking, + ) +} + +/// Friendly display name from a model id, which is often a `.gguf` file path. +fn display_name_for(id: &str) -> String { + let base = id.rsplit(['/', '\\']).next().unwrap_or(id); + base.strip_suffix(".gguf").unwrap_or(base).to_string() +} + +fn telemetry_id_for(id: &str) -> String { + format!("llama_cpp/{}", display_name_for(id)) +} + +impl LlamaCppLanguageModelProvider { + pub fn new( + http_client: Arc, + credentials_provider: Arc, + cx: &mut App, + ) -> Self { + let capability_cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default())); + let loading_progress: LoadingProgress = Arc::new(RwLock::new(HashMap::default())); + let this = Self { + http_client: http_client.clone(), + capability_cells: capability_cells.clone(), + loading_progress: loading_progress.clone(), + state: cx.new(|cx| { + cx.observe_global::({ + let mut last_settings = LlamaCppLanguageModelProvider::settings(cx).clone(); + move |this: &mut State, cx| { + let current_settings = LlamaCppLanguageModelProvider::settings(cx); + let settings_changed = current_settings != &last_settings; + if settings_changed { + let url_changed = last_settings.api_url != current_settings.api_url; + last_settings = current_settings.clone(); + if url_changed { + let credentials_provider = this.credentials_provider.clone(); + let api_url = Self::api_url(cx); + this.api_key_state.handle_url_change( + api_url, + |this| &mut this.api_key_state, + credentials_provider, + cx, + ); + this.fetched_models.clear(); + // Drop the event stream so it reconnects against + // the new URL (re-auth below restarts it). + this.model_event_task = None; + write_recover(&this.loading_progress).clear(); + this.authenticate(cx).detach(); + } + cx.notify(); + } + } + }) + .detach(); + + State { + http_client, + fetched_models: Default::default(), + fetch_model_task: None, + model_event_task: None, + capability_cells, + loading_progress, + api_key_state: ApiKeyState::new(Self::api_url(cx), (*API_KEY_ENV_VAR).clone()), + credentials_provider, + } + }), + }; + // Discover eagerly so a running server is picked up without opening settings. + this.state + .update(cx, |state, cx| state.restart_fetch_models_task(cx)); + this + } + + fn settings(cx: &App) -> &LlamaCppSettings { + &AllLanguageModelSettings::get_global(cx).llama_cpp + } + + fn api_url(cx: &App) -> SharedString { + let api_url = &Self::settings(cx).api_url; + if api_url.is_empty() { + LLAMA_CPP_API_URL.into() + } else { + SharedString::new(api_url.as_str()) + } + } + + fn has_custom_url(cx: &App) -> bool { + Self::settings(cx).api_url != LLAMA_CPP_API_URL + } +} + +impl LanguageModelProviderState for LlamaCppLanguageModelProvider { + type ObservableEntity = State; + + fn observable_entity(&self) -> Option> { + Some(self.state.clone()) + } +} + +impl LanguageModelProvider for LlamaCppLanguageModelProvider { + fn id(&self) -> LanguageModelProviderId { + PROVIDER_ID + } + + fn name(&self) -> LanguageModelProviderName { + PROVIDER_NAME + } + + fn icon(&self) -> IconOrSvg { + IconOrSvg::Icon(IconName::AiLlamaCpp) + } + + fn default_model(&self, _: &App) -> Option> { + // No default model: in router mode it could trigger an expensive load of + // an unloaded model on a constrained machine. + None + } + + fn default_fast_model(&self, _: &App) -> Option> { + // See explanation for default_model. + None + } + + fn provided_models(&self, cx: &App) -> Vec> { + let settings = LlamaCppLanguageModelProvider::settings(cx); + let effective = compute_effective_models(&self.state.read(cx).fetched_models, settings); + + // Refresh the shared capability map so open conversations pick up settings changes. + sync_capability_cells(&self.capability_cells, &effective); + let mut models = effective + .into_values() + .map(|model| { + Arc::new(LlamaCppLanguageModel { + id: LanguageModelId::from(model.name.clone()), + name: model.name.clone(), + display_name: model.display_name().to_string(), + fallback_capabilities: LiveCapabilities::of(&model), + supports_images: model.supports_images, + capability_cells: self.capability_cells.clone(), + loading_progress: self.loading_progress.clone(), + http_client: self.http_client.clone(), + request_limiter: RateLimiter::new(4), + state: self.state.clone(), + }) as Arc + }) + .collect::>(); + models.sort_by_key(|model| model.name()); + models + } + + fn is_authenticated(&self, cx: &App) -> bool { + self.state.read(cx).is_authenticated() + } + + fn authenticate(&self, cx: &mut App) -> Task> { + self.state.update(cx, |state, cx| state.authenticate(cx)) + } + + fn configuration_view( + &self, + _target_agent: language_model::ConfigurationViewTargetAgent, + window: &mut Window, + cx: &mut App, + ) -> AnyView { + let state = self.state.clone(); + cx.new(|cx| ConfigurationView::new(state, window, cx)) + .into() + } + + fn reset_credentials(&self, cx: &mut App) -> Task> { + self.state + .update(cx, |state, cx| state.set_api_key(None, cx)) + } +} + +pub struct LlamaCppLanguageModel { + id: LanguageModelId, + /// The model id sent to the server. + name: String, + display_name: String, + /// Live capabilities shared with the provider, read fresh on each access so an + /// open conversation reflects the model's real values once it has loaded. + capability_cells: CapabilityCells, + /// Used when `capability_cells` has no entry (e.g. model removed mid-conversation). + fallback_capabilities: LiveCapabilities, + /// Available from `/v1/models` hints, so captured at build time. + supports_images: bool, + /// Shared with the provider; this model's load progress, read by `name` so the + /// selector can show a loading indicator. + loading_progress: LoadingProgress, + http_client: Arc, + request_limiter: RateLimiter, + state: Entity, +} + +impl LlamaCppLanguageModel { + /// The model's live capabilities, or the build-time fallback if the map lacks it. + fn capabilities(&self) -> LiveCapabilities { + read_recover(&self.capability_cells) + .get(&self.name) + .copied() + .unwrap_or(self.fallback_capabilities) + } + + /// This model's load-status label while loading, read live from the shared map. + fn loading_label(&self) -> Option { + read_recover(&self.loading_progress) + .get(&self.name) + .cloned() + } + + fn to_llama_cpp_request( + &self, + request: LanguageModelRequest, + ) -> llama_cpp::ChatCompletionRequest { + build_llama_cpp_request( + &self.name, + self.supports_images, + self.capabilities(), + request, + ) + } + + fn stream_completion( + &self, + request: llama_cpp::ChatCompletionRequest, + cx: &AsyncApp, + ) -> BoxFuture< + 'static, + Result>>, + > { + let http_client = self.http_client.clone(); + let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| { + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let extra_headers = LlamaCppLanguageModelProvider::settings(cx) + .custom_headers + .clone(); + (state.api_key_state.key(&api_url), api_url, extra_headers) + }); + + let future = self.request_limiter.stream(async move { + let stream = stream_chat_completion( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + request, + &extra_headers, + ) + .await?; + Ok(stream) + }); + + async move { Ok(future.await?.boxed()) }.boxed() + } +} + +fn build_llama_cpp_request( + model_name: &str, + supports_images: bool, + capabilities: LiveCapabilities, + request: LanguageModelRequest, +) -> llama_cpp::ChatCompletionRequest { + let supports_tools = capabilities.supports_tools; + let supports_thinking = capabilities.supports_thinking; + let mut messages = Vec::new(); + + for message in request.messages { + let mut reasoning_content: Option = None; + for content in message.content { + match content { + MessageContent::Text(text) => add_message_content_part( + llama_cpp::MessagePart::Text { text }, + message.role, + &mut messages, + if supports_thinking && message.role == Role::Assistant { + reasoning_content.take() + } else { + None + }, + ), + MessageContent::Thinking { text, .. } => { + if supports_thinking && message.role == Role::Assistant && !text.is_empty() { + reasoning_content.get_or_insert_default().push_str(&text); + } + } + MessageContent::RedactedThinking(_) => {} + MessageContent::Compaction(_) => {} + MessageContent::Image(image) => { + if supports_images { + add_message_content_part( + llama_cpp::MessagePart::Image { + image_url: llama_cpp::ImageUrl { + url: image.to_base64_url(), + detail: None, + }, + }, + message.role, + &mut messages, + if supports_thinking && message.role == Role::Assistant { + reasoning_content.take() + } else { + None + }, + ); + } + } + MessageContent::ToolUse(tool_use) => { + let tool_call = llama_cpp::ToolCall { + id: tool_use.id.to_string(), + content: llama_cpp::ToolCallContent::Function { + function: llama_cpp::FunctionContent { + name: tool_use.name.to_string(), + arguments: serde_json::to_string(&tool_use.input) + .unwrap_or_default(), + }, + }, + }; + + if let Some(llama_cpp::ChatMessage::Assistant { + tool_calls, + reasoning_content: message_reasoning_content, + .. + }) = messages.last_mut() + { + append_reasoning_content( + message_reasoning_content, + reasoning_content.take(), + ); + tool_calls.push(tool_call); + } else { + messages.push(llama_cpp::ChatMessage::Assistant { + content: None, + reasoning_content: reasoning_content.take(), + tool_calls: vec![tool_call], + }); + } + } + MessageContent::ToolResult(tool_result) => { + let content: Vec = tool_result + .content + .iter() + .filter_map(|part| match part { + LanguageModelToolResultContent::Text(text) => { + Some(llama_cpp::MessagePart::Text { + text: text.to_string(), + }) + } + LanguageModelToolResultContent::Image(image) => { + if supports_images { + Some(llama_cpp::MessagePart::Image { + image_url: llama_cpp::ImageUrl { + url: image.to_base64_url(), + detail: None, + }, + }) + } else { + None + } + } + }) + .collect(); + + messages.push(llama_cpp::ChatMessage::Tool { + content: content.into(), + tool_call_id: tool_result.tool_use_id.to_string(), + }); + } + } + } + } + + let tools: Vec = if supports_tools { + request + .tools + .into_iter() + .map(|tool| llama_cpp::ToolDefinition::Function { + function: llama_cpp::FunctionDefinition { + name: tool.name, + description: Some(tool.description), + parameters: Some(tool.input_schema), + }, + }) + .collect() + } else { + Vec::new() + }; + // Only send `tool_choice` with actual tools; some OpenAI-compatible servers + // reject it otherwise. + let tool_choice = if tools.is_empty() { + None + } else { + request.tool_choice.map(|choice| match choice { + LanguageModelToolChoice::Auto => llama_cpp::ToolChoice::Auto, + LanguageModelToolChoice::Any => llama_cpp::ToolChoice::Required, + LanguageModelToolChoice::None => llama_cpp::ToolChoice::None, + }) + }; + + llama_cpp::ChatCompletionRequest { + model: model_name.to_string(), + messages, + stream: true, + // Let the server decide the output length (its `n_predict` default). + max_tokens: None, + stop: if request.stop.is_empty() { + None + } else { + Some(request.stop) + }, + // llama.cpp models often ship recommended sampler settings, so override + // temperature only when the request sets one. + temperature: request.temperature, + tools, + tool_choice, + stream_options: Some(llama_cpp::StreamOptions { + include_usage: true, + }), + } +} + +impl LanguageModel for LlamaCppLanguageModel { + fn id(&self) -> LanguageModelId { + self.id.clone() + } + + fn name(&self) -> LanguageModelName { + match self.loading_label() { + // Surface load progress in the display name so it shows wherever the + // model is named, without provider-agnostic UI changes. The agent + // rebuilds the name on `ProviderStateChanged`, which our ticks emit. + Some(label) => LanguageModelName::from(format!("{} · {}", self.display_name, label)), + None => LanguageModelName::from(self.display_name.clone()), + } + } + + fn provider_id(&self) -> LanguageModelProviderId { + PROVIDER_ID + } + + fn provider_name(&self) -> LanguageModelProviderName { + PROVIDER_NAME + } + + fn supports_tools(&self) -> bool { + self.capabilities().supports_tools + } + + fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool { + self.supports_tools() + && match choice { + LanguageModelToolChoice::Auto => true, + LanguageModelToolChoice::Any => true, + LanguageModelToolChoice::None => true, + } + } + + fn supports_images(&self) -> bool { + self.supports_images + } + + fn supports_thinking(&self) -> bool { + self.capabilities().supports_thinking + } + + fn telemetry_id(&self) -> String { + telemetry_id_for(&self.name) + } + + fn max_token_count(&self) -> u64 { + self.capabilities().max_tokens + } + + fn stream_completion( + &self, + request: LanguageModelRequest, + cx: &AsyncApp, + ) -> BoxFuture< + 'static, + Result< + BoxStream<'static, Result>, + LanguageModelCompletionError, + >, + > { + let request = self.to_llama_cpp_request(request); + let completions = self.stream_completion(request, cx); + async move { + let mapper = LlamaCppEventMapper::new(); + Ok(mapper.map_stream(completions.await?).boxed()) + } + .boxed() + } +} + +struct LlamaCppEventMapper { + tool_calls_by_index: HashMap, +} + +impl LlamaCppEventMapper { + fn new() -> Self { + Self { + tool_calls_by_index: HashMap::default(), + } + } + + pub fn map_stream( + mut self, + events: Pin>>>, + ) -> impl Stream> + { + events.flat_map(move |event| { + futures::stream::iter(match event { + Ok(event) => self.map_event(event), + Err(error) => vec![Err(LanguageModelCompletionError::from(error))], + }) + }) + } + + pub fn map_event( + &mut self, + event: llama_cpp::ResponseStreamEvent, + ) -> Vec> { + let mut events = Vec::new(); + + if let Some(usage) = event.usage { + events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { + input_tokens: usage.prompt_tokens, + output_tokens: usage.completion_tokens, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }))); + } + + if let Some(choice) = event.choices.into_iter().next() { + if let Some(reasoning_content) = choice.delta.reasoning_content { + events.push(Ok(LanguageModelCompletionEvent::Thinking { + text: reasoning_content, + signature: None, + })); + } + + if let Some(content) = choice.delta.content { + if !content.is_empty() { + events.push(Ok(LanguageModelCompletionEvent::Text(content))); + } + } + + if let Some(tool_calls) = choice.delta.tool_calls { + for tool_call in tool_calls { + let entry = self.tool_calls_by_index.entry(tool_call.index).or_default(); + + if let Some(tool_id) = tool_call.id { + entry.id = tool_id; + } + + if let Some(function) = tool_call.function { + if let Some(name) = function.name { + // Only the first chunk carries the function name; + // later chunks send an empty name with arguments. + if !name.is_empty() { + entry.name = name; + } + } + + if let Some(arguments) = function.arguments { + entry.arguments.push_str(&arguments); + } + } + } + } + + if let Some(finish_reason) = choice.finish_reason.as_deref() { + match finish_reason { + "stop" => { + events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn))); + } + "tool_calls" => { + events.extend(self.tool_calls_by_index.drain().map(|(_, tool_call)| { + match parse_tool_arguments(&tool_call.arguments) { + Ok(input) => Ok(LanguageModelCompletionEvent::ToolUse( + LanguageModelToolUse { + id: tool_call.id.into(), + name: tool_call.name.into(), + is_input_complete: true, + input, + raw_input: tool_call.arguments, + thought_signature: None, + }, + )), + Err(error) => { + Ok(LanguageModelCompletionEvent::ToolUseJsonParseError { + id: tool_call.id.into(), + tool_name: tool_call.name.into(), + raw_input: tool_call.arguments.into(), + json_parse_error: error.to_string(), + }) + } + } + })); + + events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse))); + } + "length" => { + events.push(Ok(LanguageModelCompletionEvent::Stop( + StopReason::MaxTokens, + ))); + } + unexpected => { + log::warn!("Unexpected llama.cpp finish_reason: {unexpected:?}"); + events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn))); + } + } + } + } + + events + } +} + +#[derive(Default)] +struct RawToolCall { + id: String, + name: String, + arguments: String, +} + +fn add_message_content_part( + new_part: llama_cpp::MessagePart, + role: Role, + messages: &mut Vec, + reasoning_content: Option, +) { + match (role, messages.last_mut()) { + (Role::User, Some(llama_cpp::ChatMessage::User { content })) + | (Role::System, Some(llama_cpp::ChatMessage::System { content })) => { + content.push_part(new_part); + } + ( + Role::Assistant, + Some(llama_cpp::ChatMessage::Assistant { + content: Some(content), + reasoning_content: message_reasoning_content, + .. + }), + ) => { + append_reasoning_content(message_reasoning_content, reasoning_content); + content.push_part(new_part); + } + _ => { + messages.push(match role { + Role::User => llama_cpp::ChatMessage::User { + content: llama_cpp::MessageContent::from(vec![new_part]), + }, + Role::Assistant => llama_cpp::ChatMessage::Assistant { + content: Some(llama_cpp::MessageContent::from(vec![new_part])), + reasoning_content, + tool_calls: Vec::new(), + }, + Role::System => llama_cpp::ChatMessage::System { + content: llama_cpp::MessageContent::from(vec![new_part]), + }, + }); + } + } +} + +fn append_reasoning_content(target: &mut Option, content: Option) { + let Some(content) = content else { + return; + }; + if content.is_empty() { + return; + } + target.get_or_insert_default().push_str(&content); +} + +fn merge_settings_into_models( + models: &mut HashMap, + available_models: &[AvailableModel], + context_window: Option, +) { + for setting_model in available_models { + if let Some(model) = models.get_mut(&setting_model.name) { + if context_window.is_none() { + model.max_tokens = setting_model.max_tokens; + } + if setting_model.display_name.is_some() { + model.display_name = setting_model.display_name.clone(); + } + if let Some(supports_tools) = setting_model.supports_tools { + model.supports_tools = supports_tools; + } + if let Some(supports_images) = setting_model.supports_images { + model.supports_images = supports_images; + } + if let Some(supports_thinking) = setting_model.supports_thinking { + model.supports_thinking = supports_thinking; + } + } else { + models.insert( + setting_model.name.clone(), + llama_cpp::Model { + name: setting_model.name.clone(), + display_name: setting_model.display_name.clone(), + max_tokens: context_window.unwrap_or(setting_model.max_tokens), + supports_tools: setting_model.supports_tools.unwrap_or(false), + supports_images: setting_model.supports_images.unwrap_or(false), + supports_thinking: setting_model.supports_thinking.unwrap_or(false), + }, + ); + } + } +} + +struct ConfigurationView { + api_key_editor: Entity, + api_url_editor: Entity, + context_window_editor: Entity, + state: Entity, +} + +impl ConfigurationView { + pub fn new(state: Entity, window: &mut Window, cx: &mut Context) -> Self { + let api_key_editor = cx.new(|cx| InputField::new(window, cx, "sk-...").label("API key")); + + let api_url_editor = cx.new(|cx| { + let input = InputField::new(window, cx, LLAMA_CPP_API_URL).label("API URL"); + input.set_text(&LlamaCppLanguageModelProvider::api_url(cx), window, cx); + input + }); + + let context_window_editor = cx.new(|cx| { + let input = InputField::new(window, cx, "8192").label("Context Window"); + if let Some(context_window) = LlamaCppLanguageModelProvider::settings(cx).context_window + { + input.set_text(&context_window.to_string(), window, cx); + } + input + }); + + cx.observe(&state, |_, _, cx| { + cx.notify(); + }) + .detach(); + + Self { + api_key_editor, + api_url_editor, + context_window_editor, + state, + } + } + + fn retry_connection(&mut self, window: &mut Window, cx: &mut Context) { + let has_api_url = LlamaCppLanguageModelProvider::has_custom_url(cx); + let has_api_key = self + .state + .read_with(cx, |state, _| state.api_key_state.has_key()); + if !has_api_url { + self.save_api_url(cx); + } + if !has_api_key { + self.save_api_key(&Default::default(), window, cx); + } + + self.state.update(cx, |state, cx| { + state.restart_fetch_models_task(cx); + }); + } + + fn save_api_key(&mut self, _: &menu::Confirm, window: &mut Window, cx: &mut Context) { + let api_key = self.api_key_editor.read(cx).text(cx).trim().to_string(); + if api_key.is_empty() { + return; + } + + // A URL change can cause the editor to be shown again. + self.api_key_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + + let state = self.state.clone(); + cx.spawn_in(window, async move |_, cx| { + state + .update(cx, |state, cx| state.set_api_key(Some(api_key), cx)) + .await + }) + .detach_and_log_err(cx); + } + + fn reset_api_key(&mut self, window: &mut Window, cx: &mut Context) { + self.api_key_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + + let state = self.state.clone(); + cx.spawn_in(window, async move |_, cx| { + state + .update(cx, |state, cx| state.set_api_key(None, cx)) + .await + }) + .detach_and_log_err(cx); + + cx.notify(); + } + + fn save_api_url(&self, cx: &mut Context) { + let api_url = self.api_url_editor.read(cx).text(cx).trim().to_string(); + let current_url = LlamaCppLanguageModelProvider::api_url(cx); + if !api_url.is_empty() && &api_url != ¤t_url { + let fs = ::global(cx); + update_settings_file(fs, cx, move |settings, _| { + settings + .language_models + .get_or_insert_default() + .llama_cpp + .get_or_insert_default() + .api_url = Some(api_url); + }); + } + } + + fn reset_api_url(&mut self, window: &mut Window, cx: &mut Context) { + self.api_url_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + let fs = ::global(cx); + update_settings_file(fs, cx, |settings, _cx| { + if let Some(settings) = settings + .language_models + .as_mut() + .and_then(|models| models.llama_cpp.as_mut()) + { + settings.api_url = Some(LLAMA_CPP_API_URL.into()); + } + }); + cx.notify(); + } + + fn save_context_window(&mut self, cx: &mut Context) { + let context_window_str = self + .context_window_editor + .read(cx) + .text(cx) + .trim() + .to_string(); + let current_context_window = LlamaCppLanguageModelProvider::settings(cx).context_window; + + if let Ok(context_window) = context_window_str.parse::() { + if Some(context_window) != current_context_window { + let fs = ::global(cx); + update_settings_file(fs, cx, move |settings, _| { + settings + .language_models + .get_or_insert_default() + .llama_cpp + .get_or_insert_default() + .context_window = Some(context_window); + }); + } + } else if context_window_str.is_empty() && current_context_window.is_some() { + let fs = ::global(cx); + update_settings_file(fs, cx, move |settings, _| { + settings + .language_models + .get_or_insert_default() + .llama_cpp + .get_or_insert_default() + .context_window = None; + }); + } + } + + fn reset_context_window(&mut self, window: &mut Window, cx: &mut Context) { + self.context_window_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + let fs = ::global(cx); + update_settings_file(fs, cx, |settings, _cx| { + if let Some(settings) = settings + .language_models + .as_mut() + .and_then(|models| models.llama_cpp.as_mut()) + { + settings.context_window = None; + } + }); + cx.notify(); + } + + fn render_instructions(cx: &App) -> Div { + v_flex() + .gap_2() + .child(Label::new( + "Run open models locally with llama.cpp's built-in server, or connect to a \ + remote llama.cpp server.", + )) + .child(Label::new("To use a local llama.cpp server:")) + .child( + List::new() + .child( + ListBulletItem::new("") + .child(Label::new("Install llama.cpp from")) + .child(ButtonLink::new("llama.app", LLAMA_CPP_DOWNLOAD_URL)), + ) + .child( + ListBulletItem::new("") + .child(Label::new("Start the server in router mode:")) + .child(Label::new("llama serve").inline_code(cx)), + ) + .child(ListBulletItem::new( + "Click 'Connect' below to start using llama.cpp in Zed", + )), + ) + .child(Label::new( + "Alternatively, you can connect to a remote llama.cpp server by specifying its \ + URL and API key (set with --api-key, may not be required):", + )) + } + + fn render_api_key_editor(&self, cx: &Context) -> impl IntoElement { + let state = self.state.read(cx); + let env_var_set = state.api_key_state.is_from_env_var(); + let configured_card_label = if env_var_set { + format!("API key set in {API_KEY_ENV_VAR_NAME} environment variable.") + } else { + "API key configured".to_string() + }; + + if !state.api_key_state.has_key() { + v_flex() + .on_action(cx.listener(Self::save_api_key)) + .child(self.api_key_editor.clone()) + .child( + Label::new(format!( + "You can also set the {API_KEY_ENV_VAR_NAME} environment variable and restart Zed." + )) + .size(LabelSize::Small) + .color(Color::Muted), + ) + .into_any_element() + } else { + ConfiguredApiCard::new(configured_card_label) + .disabled(env_var_set) + .on_click(cx.listener(|this, _, window, cx| this.reset_api_key(window, cx))) + .when(env_var_set, |this| { + this.tooltip_label(format!( + "To reset your API key, unset the {API_KEY_ENV_VAR_NAME} environment variable." + )) + }) + .into_any_element() + } + } + + fn render_context_window_editor(&self, cx: &Context) -> Div { + let settings = LlamaCppLanguageModelProvider::settings(cx); + let custom_context_window_set = settings.context_window.is_some(); + + if custom_context_window_set { + h_flex() + .p_3() + .justify_between() + .rounded_md() + .border_1() + .border_color(cx.theme().colors().border) + .bg(cx.theme().colors().elevated_surface_background) + .child( + h_flex() + .gap_2() + .child(Icon::new(IconName::Check).color(Color::Success)) + .child(v_flex().gap_1().child(Label::new(format!( + "Context Window: {}", + settings.context_window.unwrap_or_default() + )))), + ) + .child( + Button::new("reset-context-window", "Reset") + .label_size(LabelSize::Small) + .start_icon(Icon::new(IconName::Undo).size(IconSize::Small)) + .layer(ElevationIndex::ModalSurface) + .on_click( + cx.listener(|this, _, window, cx| { + this.reset_context_window(window, cx) + }), + ), + ) + } else { + v_flex() + .on_action( + cx.listener(|this, _: &menu::Confirm, _window, cx| { + this.save_context_window(cx) + }), + ) + .child(self.context_window_editor.clone()) + .child( + Label::new("Default: discovered from the server") + .size(LabelSize::Small) + .color(Color::Muted), + ) + } + } + + fn render_api_url_editor(&self, cx: &Context) -> Div { + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let custom_api_url_set = api_url != LLAMA_CPP_API_URL; + + if custom_api_url_set { + h_flex() + .p_3() + .justify_between() + .rounded_md() + .border_1() + .border_color(cx.theme().colors().border) + .bg(cx.theme().colors().elevated_surface_background) + .child( + h_flex() + .gap_2() + .child(Icon::new(IconName::Check).color(Color::Success)) + .child(v_flex().gap_1().child(Label::new(api_url))), + ) + .child( + Button::new("reset-api-url", "Reset API URL") + .label_size(LabelSize::Small) + .start_icon(Icon::new(IconName::Undo).size(IconSize::Small)) + .layer(ElevationIndex::ModalSurface) + .on_click( + cx.listener(|this, _, window, cx| this.reset_api_url(window, cx)), + ), + ) + } else { + v_flex() + .on_action(cx.listener(|this, _: &menu::Confirm, _window, cx| { + this.save_api_url(cx); + cx.notify(); + })) + .gap_2() + .child(self.api_url_editor.clone()) + } + } +} + +impl Render for ConfigurationView { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let is_authenticated = self.state.read(cx).is_authenticated(); + + v_flex() + .gap_2() + .child(Self::render_instructions(cx)) + .child(self.render_api_url_editor(cx)) + .child(self.render_context_window_editor(cx)) + .child(self.render_api_key_editor(cx)) + .child( + h_flex() + .w_full() + .justify_between() + .gap_2() + .child( + h_flex() + .w_full() + .gap_2() + .map(|this| { + if is_authenticated { + this.child( + Button::new("llama-cpp-webui", "Open WebUI") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| { + let url = + LlamaCppLanguageModelProvider::api_url(cx); + cx.open_url(&url); + }) + .into_any_element(), + ) + .child( + Button::new("llama-cpp-site", "llama.cpp") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| { + cx.open_url(LLAMA_CPP_DOWNLOAD_URL) + }) + .into_any_element(), + ) + } else { + this.child( + Button::new("download_llama_cpp_button", "Get llama.cpp") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| { + cx.open_url(LLAMA_CPP_DOWNLOAD_URL) + }) + .into_any_element(), + ) + } + }) + .child( + Button::new("view-models", "Browse GGUF Models") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| cx.open_url(LLAMA_CPP_MODELS_URL)), + ), + ) + .map(|this| { + if is_authenticated { + this.child( + ButtonLike::new("connected") + .disabled(true) + .cursor_style(CursorStyle::Arrow) + .child( + h_flex() + .gap_2() + .child(Icon::new(IconName::Check).color(Color::Success)) + .child(Label::new("Connected")) + .into_any_element(), + ) + .child( + IconButton::new("refresh-models", IconName::RotateCcw) + .tooltip(Tooltip::text("Refresh Models")) + .on_click(cx.listener(|this, _, window, cx| { + this.state.update(cx, |state, _| { + state.fetched_models.clear(); + }); + this.retry_connection(window, cx); + })), + ), + ) + } else { + this.child( + Button::new("retry_llama_cpp_models", "Connect") + .start_icon( + Icon::new(IconName::PlayOutlined).size(IconSize::XSmall), + ) + .on_click(cx.listener(move |this, _, window, cx| { + this.retry_connection(window, cx) + })), + ) + } + }), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use gpui::TestAppContext; + use http_client::FakeHttpClient; + use parking_lot::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FakeCredentialsProvider { + api_key: Vec, + } + + impl CredentialsProvider for FakeCredentialsProvider { + fn read_credentials<'a>( + &'a self, + _url: &'a str, + _cx: &'a AsyncApp, + ) -> std::pin::Pin< + Box)>>> + 'a>, + > { + let api_key = self.api_key.clone(); + Box::pin(async move { Ok(Some(("Bearer".to_string(), api_key))) }) + } + + fn write_credentials<'a>( + &'a self, + _url: &'a str, + _username: &'a str, + _password: &'a [u8], + _cx: &'a AsyncApp, + ) -> std::pin::Pin> + 'a>> { + Box::pin(async { Ok(()) }) + } + + fn delete_credentials<'a>( + &'a self, + _url: &'a str, + _cx: &'a AsyncApp, + ) -> std::pin::Pin> + 'a>> { + Box::pin(async { Ok(()) }) + } + } + + fn entry(id: &str, n_ctx: Option, n_ctx_train: Option) -> ModelEntry { + ModelEntry { + id: id.to_string(), + meta: Some(llama_cpp::ModelMeta { n_ctx, n_ctx_train }), + architecture: None, + status: None, + } + } + + #[test] + fn display_name_strips_path_and_extension() { + assert_eq!( + display_name_for("../models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"), + "Qwen2.5-Coder-7B-Instruct-Q4_K_M" + ); + assert_eq!(display_name_for("my-alias"), "my-alias"); + } + + #[test] + fn telemetry_id_strips_local_model_paths() { + assert_eq!( + telemetry_id_for("/Users/alice/models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"), + "llama_cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M" + ); + assert_eq!( + telemetry_id_for(r"C:\Users\alice\models\Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"), + "llama_cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M" + ); + assert_eq!(telemetry_id_for("my-alias"), "llama_cpp/my-alias"); + } + + #[test] + fn model_uses_props_then_meta_for_context() { + let props = Props { + default_generation_settings: Some(llama_cpp::GenerationSettings { n_ctx: Some(8192) }), + modalities: Some(llama_cpp::Modalities { vision: true }), + chat_template_caps: Some(llama_cpp::ChatTemplateCaps { + supports_tool_calls: true, + supports_preserve_reasoning: true, + ..Default::default() + }), + }; + // /props wins when present. + let model = model_from_entry(&entry("m", Some(4096), Some(131072)), Some(&props)); + assert_eq!(model.max_tokens, 8192); + assert!(model.supports_tools); + assert!(model.supports_images); + assert!(model.supports_thinking); + + // Unprobed: falls back to the listing's runtime context, then trained + // context. Tools are assumed supported until the model loads. + let model = model_from_entry(&entry("m", Some(4096), Some(131072)), None); + assert_eq!(model.max_tokens, 4096); + assert!(model.supports_tools); + assert!(!model.supports_thinking); + + let model = model_from_entry(&entry("m", None, Some(131072)), None); + assert_eq!(model.max_tokens, 131072); + + // Nothing reported -> the optimistic unloaded-context assumption. + let model = model_from_entry(&entry("m", None, None), None); + assert_eq!(model.max_tokens, ASSUMED_UNLOADED_CONTEXT); + assert!(model.supports_tools); + } + + #[test] + fn router_entry_detects_vision_from_modalities() { + let router_entry = ModelEntry { + id: "vlm".to_string(), + meta: None, + architecture: Some(llama_cpp::Architecture { + input_modalities: vec!["text".to_string(), "image".to_string()], + }), + status: Some(llama_cpp::ModelStatus { + value: "unloaded".to_string(), + }), + }; + let model = model_from_entry(&router_entry, None); + assert!(model.supports_images); + // Unprobed router models optimistically advertise tools until loaded. + assert!(model.supports_tools); + } + + #[test] + fn settings_override_capabilities_and_context() { + let mut models: HashMap = HashMap::default(); + models.insert( + "qwen".to_string(), + llama_cpp::Model::new("qwen", Some("qwen"), Some(8192), false, false, false), + ); + + let available = vec![AvailableModel { + name: "qwen".to_string(), + display_name: Some("Qwen Coder".to_string()), + max_tokens: 16384, + supports_tools: Some(true), + supports_images: None, + supports_thinking: Some(true), + }]; + + merge_settings_into_models(&mut models, &available, None); + + let model = models.get("qwen").unwrap(); + assert_eq!(model.display_name.as_deref(), Some("Qwen Coder")); + assert_eq!(model.max_tokens, 16384); + assert!(model.supports_tools); + assert!(model.supports_thinking); + // Unspecified capability keeps the discovered value. + assert!(!model.supports_images); + } + + #[test] + fn capability_cells_update_when_a_model_loads() { + let cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default())); + let settings = LlamaCppSettings { + auto_discover: true, + ..Default::default() + }; + + // Cold: the optimistic unloaded-context assumption. + let cold = vec![llama_cpp::Model::new( + "m", + Some("m"), + Some(ASSUMED_UNLOADED_CONTEXT), + true, + false, + false, + )]; + sync_capability_cells(&cells, &compute_effective_models(&cold, &settings)); + assert_eq!( + cells.read().unwrap().get("m").unwrap().max_tokens, + ASSUMED_UNLOADED_CONTEXT + ); + + // The model loads and reports its real context. The shared map must + // reflect the new value so a model reading it by name (an open + // conversation) is no longer stuck on the cold-start assumption. + let loaded = vec![llama_cpp::Model::new( + "m", + Some("m"), + Some(262_144), + true, + false, + true, + )]; + sync_capability_cells(&cells, &compute_effective_models(&loaded, &settings)); + assert_eq!(cells.read().unwrap().get("m").unwrap().max_tokens, 262_144); + assert!(cells.read().unwrap().get("m").unwrap().supports_thinking); + } + + #[test] + fn request_preserves_assistant_thinking_when_supported() { + let request = build_llama_cpp_request( + "test-model", + false, + LiveCapabilities { + max_tokens: 8192, + supports_tools: false, + supports_thinking: true, + }, + LanguageModelRequest { + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![ + MessageContent::Thinking { + text: "reasoning".to_string(), + signature: None, + }, + MessageContent::Text("answer".to_string()), + ], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }, + ); + + assert_eq!(request.messages.len(), 1); + match &request.messages[0] { + llama_cpp::ChatMessage::Assistant { + content: Some(llama_cpp::MessageContent::Plain(content)), + reasoning_content: Some(reasoning_content), + tool_calls, + } => { + assert_eq!(content, "answer"); + assert_eq!(reasoning_content, "reasoning"); + assert!(tool_calls.is_empty()); + } + message => panic!("unexpected message: {message:?}"), + } + } + + #[test] + fn request_skips_assistant_thinking_when_unsupported() { + let request = build_llama_cpp_request( + "test-model", + false, + LiveCapabilities { + max_tokens: 8192, + supports_tools: false, + supports_thinking: false, + }, + LanguageModelRequest { + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![ + MessageContent::Thinking { + text: "reasoning".to_string(), + signature: None, + }, + MessageContent::RedactedThinking("encrypted".to_string()), + MessageContent::Text("answer".to_string()), + ], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }, + ); + + assert_eq!(request.messages.len(), 1); + match &request.messages[0] { + llama_cpp::ChatMessage::Assistant { + content: Some(llama_cpp::MessageContent::Plain(content)), + reasoning_content, + tool_calls, + } => { + assert_eq!(content, "answer"); + assert_eq!(reasoning_content, &None); + assert!(tool_calls.is_empty()); + } + message => panic!("unexpected message: {message:?}"), + } + } + + #[test] + fn request_preserves_thinking_for_assistant_tool_calls_when_supported() { + let request = build_llama_cpp_request( + "test-model", + false, + LiveCapabilities { + max_tokens: 8192, + supports_tools: true, + supports_thinking: true, + }, + LanguageModelRequest { + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![ + MessageContent::Thinking { + text: "reasoning".to_string(), + signature: None, + }, + MessageContent::ToolUse(LanguageModelToolUse { + id: "call_1".into(), + name: "weather".into(), + raw_input: r#"{"city":"Oslo"}"#.to_string(), + input: serde_json::json!({ "city": "Oslo" }), + is_input_complete: true, + thought_signature: None, + }), + ], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }, + ); + + assert_eq!(request.messages.len(), 1); + match &request.messages[0] { + llama_cpp::ChatMessage::Assistant { + content: None, + reasoning_content: Some(reasoning_content), + tool_calls, + } => { + assert_eq!(reasoning_content, "reasoning"); + assert_eq!(tool_calls.len(), 1); + } + message => panic!("unexpected message: {message:?}"), + } + } + + #[test] + fn usage_event_precedes_stop_event() { + let mut mapper = LlamaCppEventMapper::new(); + let events = mapper.map_event(llama_cpp::ResponseStreamEvent { + model: "test-model".to_string(), + object: "chat.completion.chunk".to_string(), + choices: vec![llama_cpp::ChoiceDelta { + index: 0, + delta: llama_cpp::ResponseMessageDelta { + content: None, + reasoning_content: None, + tool_calls: None, + }, + finish_reason: Some("stop".to_string()), + }], + usage: Some(llama_cpp::Usage { + prompt_tokens: 11, + completion_tokens: 7, + total_tokens: 18, + }), + }); + + assert!(matches!( + events.as_slice(), + [ + Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { + input_tokens: 11, + output_tokens: 7, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + })), + Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)), + ] + )); + } + + #[test] + fn usage_event_precedes_tool_use_stop_event() { + let mut mapper = LlamaCppEventMapper::new(); + let events = mapper.map_event(llama_cpp::ResponseStreamEvent { + model: "test-model".to_string(), + object: "chat.completion.chunk".to_string(), + choices: vec![llama_cpp::ChoiceDelta { + index: 0, + delta: llama_cpp::ResponseMessageDelta { + content: None, + reasoning_content: None, + tool_calls: Some(vec![llama_cpp::ToolCallChunk { + index: 0, + id: Some("tool-call-id".to_string()), + function: Some(llama_cpp::FunctionChunk { + name: Some("test_tool".to_string()), + arguments: Some(r#"{"value":1}"#.to_string()), + }), + }]), + }, + finish_reason: Some("tool_calls".to_string()), + }], + usage: Some(llama_cpp::Usage { + prompt_tokens: 13, + completion_tokens: 5, + total_tokens: 18, + }), + }); + + assert!(matches!( + events.as_slice(), + [ + Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { + input_tokens: 13, + output_tokens: 5, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + })), + Ok(LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse { + id, + name, + .. + })), + Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)), + ] if id.to_string() == "tool-call-id" && name.as_ref() == "test_tool" + )); + } + + #[gpui::test] + async fn authenticate_fetches_models_after_loading_api_key(cx: &mut TestAppContext) { + cx.update(|cx| { + let settings_store = SettingsStore::test(cx); + cx.set_global(settings_store); + }); + + let model_request_authorizations = Arc::new(Mutex::new(Vec::new())); + let model_request_count = Arc::new(AtomicUsize::new(0)); + let http_client = FakeHttpClient::create({ + let model_request_authorizations = model_request_authorizations.clone(); + let model_request_count = model_request_count.clone(); + move |request| { + let model_request_authorizations = model_request_authorizations.clone(); + let model_request_count = model_request_count.clone(); + async move { + let path = request.uri().path(); + let authorization = request + .headers() + .get("Authorization") + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + + if path == "/v1/models" { + model_request_authorizations.lock().push(authorization); + let request_index = model_request_count.fetch_add(1, Ordering::SeqCst); + if request_index == 0 { + return Ok(http_client::Response::builder() + .status(503) + .body(http_client::AsyncBody::from("not ready"))?); + } + + return Ok(http_client::Response::builder().status(200).body( + http_client::AsyncBody::from( + r#"{"data":[{"id":"test-model","meta":{"n_ctx":4096}}]}"#, + ), + )?); + } + + if path == "/props" { + return Ok(http_client::Response::builder() + .status(200) + .body(http_client::AsyncBody::from("{}"))?); + } + + Ok(http_client::Response::builder() + .status(404) + .body(http_client::AsyncBody::default())?) + } + } + }); + let credentials_provider = Arc::new(FakeCredentialsProvider { + api_key: b"loaded-key".to_vec(), + }); + let provider = cx + .update(|cx| LlamaCppLanguageModelProvider::new(http_client, credentials_provider, cx)); + + cx.run_until_parked(); + + let result = cx.update(|cx| provider.authenticate(cx)).await; + assert!( + result.is_ok(), + "authenticate should discover models after loading credentials" + ); + assert_eq!( + &*model_request_authorizations.lock(), + &[None, Some("Bearer loaded-key".to_string())] + ); + } +} diff --git a/crates/language_models/src/settings.rs b/crates/language_models/src/settings.rs index d2fc9f59a2..1ce7caa3a6 100644 --- a/crates/language_models/src/settings.rs +++ b/crates/language_models/src/settings.rs @@ -6,10 +6,11 @@ use settings::RegisterSetting; use crate::provider::{ anthropic, anthropic::AnthropicSettings, anthropic_compatible::AnthropicCompatibleSettings, bedrock, bedrock::AmazonBedrockSettings, cloud::ZedDotDevSettings, deepseek::DeepSeekSettings, - google::GoogleSettings, lmstudio::LmStudioSettings, mistral, mistral::MistralSettings, - ollama::OllamaSettings, open_ai::OpenAiSettings, open_ai_compatible::OpenAiCompatibleSettings, - open_router, open_router::OpenRouterSettings, opencode, opencode::OpenCodeSettings, - resolve_custom_headers, vercel_ai_gateway::VercelAiGatewaySettings, x_ai::XAiSettings, + google::GoogleSettings, llama_cpp::LlamaCppSettings, lmstudio::LmStudioSettings, mistral, + mistral::MistralSettings, ollama::OllamaSettings, open_ai::OpenAiSettings, + open_ai_compatible::OpenAiCompatibleSettings, open_router, open_router::OpenRouterSettings, + opencode, opencode::OpenCodeSettings, resolve_custom_headers, + vercel_ai_gateway::VercelAiGatewaySettings, x_ai::XAiSettings, }; #[derive(Debug, RegisterSetting)] @@ -19,6 +20,7 @@ pub struct AllLanguageModelSettings { pub bedrock: AmazonBedrockSettings, pub deepseek: DeepSeekSettings, pub google: GoogleSettings, + pub llama_cpp: LlamaCppSettings, pub lmstudio: LmStudioSettings, pub mistral: MistralSettings, pub ollama: OllamaSettings, @@ -52,6 +54,7 @@ impl settings::Settings for AllLanguageModelSettings { let bedrock = language_models.bedrock.unwrap(); let deepseek = language_models.deepseek.unwrap(); let google = language_models.google.unwrap(); + let llama_cpp = language_models.llama_cpp.unwrap(); let lmstudio = language_models.lmstudio.unwrap(); let mistral = language_models.mistral.unwrap(); let ollama = language_models.ollama.unwrap(); @@ -116,6 +119,13 @@ impl settings::Settings for AllLanguageModelSettings { available_models: google.available_models.unwrap_or_default(), custom_headers: custom_headers_from("Google AI", google.custom_headers, &[]), }, + llama_cpp: LlamaCppSettings { + api_url: llama_cpp.api_url.unwrap(), + auto_discover: llama_cpp.auto_discover.unwrap_or(true), + available_models: llama_cpp.available_models.unwrap_or_default(), + context_window: llama_cpp.context_window, + custom_headers: custom_headers_from("llama.cpp", llama_cpp.custom_headers, &[]), + }, lmstudio: LmStudioSettings { api_url: lmstudio.api_url.unwrap(), available_models: lmstudio.available_models.unwrap_or_default(), diff --git a/crates/llama_cpp/Cargo.toml b/crates/llama_cpp/Cargo.toml new file mode 100644 index 0000000000..c77c2dd817 --- /dev/null +++ b/crates/llama_cpp/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "llama_cpp" +version = "0.1.0" +edition.workspace = true +publish.workspace = true +license = "GPL-3.0-or-later" + +[lints] +workspace = true + +[lib] +path = "src/llama_cpp.rs" + +[features] +default = [] +schemars = ["dep:schemars"] + +[dependencies] +anyhow.workspace = true +futures.workspace = true +http_client.workspace = true +schemars = { workspace = true, optional = true } +serde.workspace = true +serde_json.workspace = true +url.workspace = true diff --git a/crates/llama_cpp/LICENSE-GPL b/crates/llama_cpp/LICENSE-GPL new file mode 120000 index 0000000000..89e542f750 --- /dev/null +++ b/crates/llama_cpp/LICENSE-GPL @@ -0,0 +1 @@ +../../LICENSE-GPL \ No newline at end of file diff --git a/crates/llama_cpp/src/llama_cpp.rs b/crates/llama_cpp/src/llama_cpp.rs new file mode 100644 index 0000000000..5d5238c4b1 --- /dev/null +++ b/crates/llama_cpp/src/llama_cpp.rs @@ -0,0 +1,911 @@ +use anyhow::{Context as _, Result, anyhow}; +use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream}; +use http_client::{ + AsyncBody, CustomHeaders, HttpClient, HttpRequestExt, Method, Request as HttpRequest, + RequestBuilderExt, http, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use url::Url; + +pub const LLAMA_CPP_API_URL: &str = "http://localhost:8080"; + +const DEFAULT_CONTEXT_LENGTH: u64 = 4096; + +/// A model exposed to the rest of Zed, after merging API discovery with +/// user-configured overrides. +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct Model { + pub name: String, + pub display_name: Option, + pub max_tokens: u64, + pub supports_tools: bool, + pub supports_images: bool, + pub supports_thinking: bool, +} + +impl Model { + pub fn new( + name: &str, + display_name: Option<&str>, + max_tokens: Option, + supports_tools: bool, + supports_images: bool, + supports_thinking: bool, + ) -> Self { + Self { + name: name.to_owned(), + display_name: display_name.map(ToString::to_string), + max_tokens: max_tokens.unwrap_or(DEFAULT_CONTEXT_LENGTH), + supports_tools, + supports_images, + supports_thinking, + } + } + + pub fn display_name(&self) -> &str { + self.display_name.as_deref().unwrap_or(&self.name) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ToolChoice { + Auto, + Required, + None, +} + +#[derive(Clone, Deserialize, Serialize, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolDefinition { + Function { function: FunctionDefinition }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FunctionDefinition { + pub name: String, + pub description: Option, + pub parameters: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "role", rename_all = "lowercase")] +pub enum ChatMessage { + Assistant { + #[serde(default)] + content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tool_calls: Vec, + }, + User { + content: MessageContent, + }, + System { + content: MessageContent, + }, + Tool { + content: MessageContent, + tool_call_id: String, + }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +#[serde(untagged)] +pub enum MessageContent { + Plain(String), + Multipart(Vec), +} + +impl MessageContent { + pub fn push_part(&mut self, part: MessagePart) { + match self { + MessageContent::Plain(text) => { + *self = + MessageContent::Multipart(vec![MessagePart::Text { text: text.clone() }, part]); + } + MessageContent::Multipart(parts) if parts.is_empty() => match part { + MessagePart::Text { text } => *self = MessageContent::Plain(text), + MessagePart::Image { .. } => *self = MessageContent::Multipart(vec![part]), + }, + MessageContent::Multipart(parts) => parts.push(part), + } + } +} + +impl From> for MessageContent { + fn from(mut parts: Vec) -> Self { + if let [MessagePart::Text { text }] = parts.as_mut_slice() { + MessageContent::Plain(std::mem::take(text)) + } else { + MessageContent::Multipart(parts) + } + } +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MessagePart { + Text { + text: String, + }, + #[serde(rename = "image_url")] + Image { + image_url: ImageUrl, + }, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] +pub struct ImageUrl { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct ToolCall { + pub id: String, + #[serde(flatten)] + pub content: ToolCallContent, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum ToolCallContent { + Function { function: FunctionContent }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct FunctionContent { + pub name: String, + pub arguments: String, +} + +#[derive(Serialize, Debug)] +pub struct ChatCompletionRequest { + pub model: String, + pub messages: Vec, + pub stream: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tools: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, +} + +/// Asks the server to include a final `usage` chunk in the stream. +#[derive(Serialize, Debug)] +pub struct StreamOptions { + pub include_usage: bool, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct Usage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct LlamaCppError { + pub message: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +pub enum ResponseStreamResult { + Ok(ResponseStreamEvent), + Err { error: LlamaCppError }, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ResponseStreamEvent { + pub model: String, + pub object: String, + pub choices: Vec, + pub usage: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ChoiceDelta { + pub index: u32, + pub delta: ResponseMessageDelta, + pub finish_reason: Option, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct ResponseMessageDelta { + pub content: Option, + /// `llama-server` emits reasoning as a dedicated `reasoning_content` field + /// when started with a reasoning format (e.g. `--reasoning-format deepseek`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct ToolCallChunk { + pub index: usize, + pub id: Option, + pub function: Option, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct FunctionChunk { + pub name: Option, + pub arguments: Option, +} + +/// Response of `GET /v1/models`. +/// +/// In single-model mode `data` has exactly one entry describing the loaded +/// model; in router mode it lists every model the server knows about. +#[derive(Deserialize, Debug)] +pub struct ListModelsResponse { + #[serde(default)] + pub data: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModelEntry { + pub id: String, + /// Present in single-model mode; carries context information. + #[serde(default)] + pub meta: Option, + /// Present in router mode; reports the modalities the model accepts. + #[serde(default)] + pub architecture: Option, + /// Present in router mode; reports whether the model is currently loaded. + #[serde(default)] + pub status: Option, +} + +impl ModelEntry { + /// Whether this entry came from a server running in router mode. + pub fn is_router_entry(&self) -> bool { + self.status.is_some() + } + + /// Whether the model is loaded and can be probed for capabilities without + /// triggering a (potentially expensive) load. + pub fn is_loaded(&self) -> bool { + self.status + .as_ref() + .is_some_and(|status| status.value == "loaded") + } + + /// Whether the model is currently loading, so a progress label applies. + pub fn is_loading(&self) -> bool { + self.status + .as_ref() + .is_some_and(|status| status.value == "loading") + } + + pub fn supports_images_hint(&self) -> bool { + self.architecture + .as_ref() + .is_some_and(|architecture| architecture.input_modalities.iter().any(|m| m == "image")) + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ModelMeta { + /// The runtime per-slot context size. + #[serde(default)] + pub n_ctx: Option, + /// The context size the model was trained with. + #[serde(default)] + pub n_ctx_train: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct Architecture { + #[serde(default)] + pub input_modalities: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModelStatus { + /// One of `loaded`, `loading`, `unloaded`, `downloading`, `downloaded`, `sleeping`. + pub value: String, +} + +/// Response of `GET /props`, describing the loaded model. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct Props { + #[serde(default)] + pub default_generation_settings: Option, + #[serde(default)] + pub modalities: Option, + #[serde(default)] + pub chat_template_caps: Option, +} + +impl Props { + /// The runtime context length the loaded model was configured with. + pub fn context_length(&self) -> Option { + self.default_generation_settings + .as_ref() + .and_then(|settings| settings.n_ctx) + .filter(|n_ctx| *n_ctx > 0) + } + + pub fn supports_images(&self) -> bool { + self.modalities + .as_ref() + .is_some_and(|modalities| modalities.vision) + } + + pub fn supports_tools(&self) -> bool { + self.chat_template_caps + .as_ref() + .is_some_and(|caps| caps.supports_tool_calls || caps.supports_tools) + } + + pub fn supports_thinking(&self) -> bool { + self.chat_template_caps + .as_ref() + .is_some_and(|caps| caps.supports_preserve_reasoning) + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct GenerationSettings { + #[serde(default)] + pub n_ctx: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct Modalities { + #[serde(default)] + pub vision: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ChatTemplateCaps { + /// llama.cpp's `/props` reports both of these keys, so we read them into + /// separate fields (aliasing one to the other would be a serde duplicate-field + /// error) and treat either being true as tool support. `supports_tools` is the + /// older name; some builds report only one of the two. + #[serde(default)] + pub supports_tool_calls: bool, + #[serde(default)] + pub supports_tools: bool, + #[serde(default)] + pub supports_preserve_reasoning: bool, +} + +/// An event from the router's `/models/sse` feed, which the provider subscribes +/// to so model capabilities stay current as models load and unload. `model` is +/// `*` for events that aren't about a single model (e.g. the list reloading). +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModelEvent { + #[serde(default)] + pub model: String, + pub event: String, + #[serde(default)] + pub data: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ModelEventData { + #[serde(default)] + pub status: Option, + /// Present on an `unloaded` status; non-zero means the model failed to load. + #[serde(default)] + pub exit_code: Option, + /// Present on a `loading` status, reporting per-stage load progress. + #[serde(default)] + pub progress: Option, +} + +/// Per-stage load progress carried by a `loading` event. A model loads its +/// stages in order (the text model, plus an optional draft and/or multimodal +/// projector), each reporting a `0.0..=1.0` fraction. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct LoadProgress { + #[serde(default)] + pub stages: Vec, + #[serde(default)] + pub current: String, + #[serde(default)] + pub value: f32, +} + +impl LoadProgress { + /// A human label for the stage currently loading, matching the llama.cpp + /// WebUI's labels. We report the current stage's own `value` rather than a + /// blended overall percentage (as the WebUI does), so each stage runs + /// `0→100%` and the label says which stage it is. + pub fn stage_label(&self) -> &'static str { + match self.current.as_str() { + "text_model" => "Loading weights", + "spec_model" => "Loading draft", + "mmproj_model" => "Loading projector", + _ => "Loading", + } + } + + /// The full load-status label shown in the model selector, e.g. + /// `"Loading weights 42%"`: the current stage plus its own progress rounded + /// to a percentage. + pub fn progress_label(&self) -> String { + format!( + "{} {}%", + self.stage_label(), + (self.value * 100.0).round() as u32 + ) + } +} + +impl ModelEvent { + /// Whether this event means the set of models or their loaded state changed, + /// so the provider should re-run discovery. Intermediate `loading` progress + /// ticks return `false` — only terminal load/unload and list changes matter. + pub fn changes_model_state(&self) -> bool { + match self.event.as_str() { + "models_reload" | "model_remove" => true, + _ => matches!( + self.data.as_ref().and_then(|data| data.status.as_deref()), + Some("loaded" | "unloaded") + ), + } + } + + /// The non-zero exit code of a failed load, if this event reports one. + pub fn load_failure(&self) -> Option { + let data = self.data.as_ref()?; + if data.status.as_deref() == Some("unloaded") { + data.exit_code.filter(|code| *code != 0) + } else { + None + } + } + + /// This event's load progress if it is a loading event carrying usable stage + /// data, else `None`. + pub fn load_progress(&self) -> Option<&LoadProgress> { + let data = self.data.as_ref()?; + if data.status.as_deref() != Some("loading") { + return None; + } + let progress = data.progress.as_ref()?; + // The server also emits bare stage-transition markers (e.g. + // `{"stage": "mmproj_model"}`) with no `stages`/`current`/`value`. Skip + // them so the indicator holds its last value rather than dropping to 0%. + if progress.stages.is_empty() || progress.current.is_empty() { + return None; + } + Some(progress) + } +} + +pub async fn stream_chat_completion( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + request: ChatCompletionRequest, + extra_headers: &CustomHeaders, +) -> Result>> { + let uri = format!("{api_url}/v1/chat/completions"); + let request_builder = http::Request::builder() + .method(Method::POST) + .uri(uri) + .header("Content-Type", "application/json") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }); + + let request = request_builder + .extra_headers(extra_headers) + .body(AsyncBody::from(serde_json::to_string(&request)?))?; + let mut response = client.send(request).await?; + if response.status().is_success() { + let reader = BufReader::new(response.into_body()); + Ok(reader + .lines() + .filter_map(|line| async move { + match line { + Ok(line) => { + let line = line.strip_prefix("data: ")?; + if line == "[DONE]" { + None + } else { + match serde_json::from_str(line) { + Ok(ResponseStreamResult::Ok(response)) => Some(Ok(response)), + Ok(ResponseStreamResult::Err { error }) => { + Some(Err(anyhow!(error.message))) + } + Err(error) => Some(Err(anyhow!(error))), + } + } + } + Err(error) => Some(Err(anyhow!(error))), + } + }) + .boxed()) + } else { + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + anyhow::bail!( + "Failed to connect to llama.cpp API: {} {}", + response.status(), + body, + ); + } +} + +/// Lists the models the server is serving via `GET /v1/models`. +pub async fn get_models( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + extra_headers: &CustomHeaders, +) -> Result> { + let uri = format!("{api_url}/v1/models"); + let request = HttpRequest::builder() + .method(Method::GET) + .uri(uri) + .header("Accept", "application/json") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }) + .extra_headers(extra_headers) + .body(AsyncBody::default())?; + + let mut response = client.send(request).await?; + + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + + anyhow::ensure!( + response.status().is_success(), + "Failed to connect to llama.cpp API: {} {}", + response.status(), + body, + ); + let response: ListModelsResponse = + serde_json::from_str(&body).context("Unable to parse llama.cpp models response")?; + Ok(response.data) +} + +/// Fetches `GET /props` to discover the loaded model's capabilities. +/// +/// In router mode `model` selects which model instance to query; passing +/// `None` queries the single loaded model in single-model mode. +pub async fn get_props( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + model: Option<&str>, + extra_headers: &CustomHeaders, +) -> Result { + // Router-mode `/props` takes a `model` query parameter selecting which + // instance to describe. Model ids contain `/` and `:` (e.g. + // `unsloth/Qwen3.5-2B-GGUF:Q4_1`), so the value is URL-encoded. + let uri = match model { + Some(model) => Url::parse_with_params(&format!("{api_url}/props"), [("model", model)]) + .context("invalid llama.cpp API URL")? + .to_string(), + None => format!("{api_url}/props"), + }; + let request = HttpRequest::builder() + .method(Method::GET) + .uri(uri) + .header("Accept", "application/json") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }) + .extra_headers(extra_headers) + .body(AsyncBody::default())?; + + let mut response = client.send(request).await?; + + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + + anyhow::ensure!( + response.status().is_success(), + "Failed to connect to llama.cpp API: {} {}", + response.status(), + body, + ); + let props: Props = + serde_json::from_str(&body).context("Unable to parse llama.cpp props response")?; + Ok(props) +} + +/// Opens the router's `GET /models/sse` event stream. Each item is one parsed +/// event; the stream ends when the connection closes. Only available on builds +/// that expose `/models/sse` (router mode). +pub async fn stream_model_events( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + extra_headers: &CustomHeaders, +) -> Result>> { + let uri = format!("{api_url}/models/sse"); + let request = HttpRequest::builder() + .method(Method::GET) + .uri(uri) + .header("Accept", "text/event-stream") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }) + .extra_headers(extra_headers) + .body(AsyncBody::default())?; + + let mut response = client.send(request).await?; + if !response.status().is_success() { + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + anyhow::bail!( + "Failed to open llama.cpp model event stream: {} {}", + response.status(), + body, + ); + } + + let reader = BufReader::new(response.into_body()); + Ok(reader + .lines() + .filter_map(|line| async move { + // Each event is a single `data:` line carrying the JSON envelope; + // other SSE lines (comments, blank separators) are ignored. + let line = line.ok()?; + let payload = line.strip_prefix("data:")?.trim_start(); + Some(serde_json::from_str::(payload).map_err(|error| anyhow!(error))) + }) + .boxed()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_single_model_listing() { + let response = serde_json::json!({ + "object": "list", + "data": [ + { + "id": "../models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", + "object": "model", + "created": 1735142223u64, + "owned_by": "llamacpp", + "meta": { + "n_ctx": 8192, + "n_ctx_train": 131072, + "n_embd": 4096, + } + } + ] + }); + let response: ListModelsResponse = serde_json::from_value(response).unwrap(); + assert_eq!(response.data.len(), 1); + let entry = &response.data[0]; + assert!(!entry.is_router_entry()); + assert_eq!(entry.meta.as_ref().unwrap().n_ctx, Some(8192)); + assert_eq!(entry.meta.as_ref().unwrap().n_ctx_train, Some(131072)); + } + + #[test] + fn parse_router_model_listing() { + let response = serde_json::json!({ + "object": "list", + "data": [ + { + "id": "qwen2.5-coder", + "object": "model", + "owned_by": "llamacpp", + "status": { "value": "loaded", "args": [] }, + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"] + } + }, + { + "id": "gemma-3", + "object": "model", + "owned_by": "llamacpp", + "status": { "value": "unloaded" }, + "architecture": { "input_modalities": ["text"] } + } + ] + }); + let response: ListModelsResponse = serde_json::from_value(response).unwrap(); + assert_eq!(response.data.len(), 2); + + let loaded = &response.data[0]; + assert!(loaded.is_router_entry()); + assert!(loaded.is_loaded()); + assert!(loaded.supports_images_hint()); + + let unloaded = &response.data[1]; + assert!(unloaded.is_router_entry()); + assert!(!unloaded.is_loaded()); + assert!(!unloaded.supports_images_hint()); + } + + #[test] + fn parse_props() { + let response = serde_json::json!({ + "default_generation_settings": { + "id": 0, + "n_ctx": 8192, + "params": {} + }, + "total_slots": 1, + "model_path": "../models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", + "modalities": { "vision": true, "audio": false }, + // The real `/props` reports both keys; parsing must not treat that as + // a duplicate field. + "chat_template_caps": { + "supports_tools": true, + "supports_tool_calls": true, + "supports_preserve_reasoning": true, + "supports_system_role": true + } + }); + let props: Props = serde_json::from_value(response).unwrap(); + assert_eq!(props.context_length(), Some(8192)); + assert!(props.supports_images()); + assert!(props.supports_tools()); + assert!(props.supports_thinking()); + } + + fn model_event(value: serde_json::Value) -> ModelEvent { + serde_json::from_value(value).unwrap() + } + + #[test] + fn model_event_state_changes() { + // Terminal load/unload and list changes warrant re-discovery. + assert!( + model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "loaded" } + })) + .changes_model_state() + ); + assert!( + model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "unloaded" } + })) + .changes_model_state() + ); + assert!( + model_event(serde_json::json!({ "model": "*", "event": "models_reload" })) + .changes_model_state() + ); + assert!( + model_event(serde_json::json!({ "model": "m", "event": "model_remove" })) + .changes_model_state() + ); + + // Intermediate loading-progress ticks do not. + assert!( + !model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", "progress": { "value": 0.4 } } + })) + .changes_model_state() + ); + assert!( + !model_event(serde_json::json!({ "model": "m", "event": "download_progress" })) + .changes_model_state() + ); + } + + #[test] + fn model_event_load_failure() { + let failed = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "unloaded", "exit_code": 1 } + })); + assert_eq!(failed.load_failure(), Some(1)); + + // A clean unload (exit code 0 or absent) is not a failure. + let clean = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "unloaded", "exit_code": 0 } + })); + assert_eq!(clean.load_failure(), None); + let loaded = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "loaded" } + })); + assert_eq!(loaded.load_failure(), None); + } + + #[test] + fn model_event_load_progress() { + // We report the current stage's own value (0→1), not a blended overall. + let weights = model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", + "progress": { "stages": ["text_model"], "current": "text_model", "value": 0.4 } } + })); + let progress = weights.load_progress().unwrap(); + assert!((progress.value - 0.4).abs() < 1e-4); + assert_eq!(progress.stage_label(), "Loading weights"); + assert_eq!(progress.progress_label(), "Loading weights 40%"); + + // The projector stage runs 0→1 on its own (not 90→100%). + let projector = model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", + "progress": { "stages": ["text_model", "mmproj_model"], + "current": "mmproj_model", "value": 0.5 } } + })); + let progress = projector.load_progress().unwrap(); + assert!((progress.value - 0.5).abs() < 1e-4); + assert_eq!(progress.stage_label(), "Loading projector"); + + // Non-loading events carry no progress. + let loaded = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "loaded" } + })); + assert!(loaded.load_progress().is_none()); + + // Bare stage-transition markers (no stages/current/value) are skipped so + // the indicator doesn't flicker to 0% between stages. + let transition = model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", "progress": { "stage": "mmproj_model" } } + })); + assert!(transition.load_progress().is_none()); + } + + #[test] + fn parse_streaming_reasoning_and_tool_calls() { + let event = serde_json::json!({ + "model": "llama", + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": null, + "reasoning_content": "thinking...", + "tool_calls": [ + { + "index": 0, + "id": "call_1", + "function": { "name": "weather", "arguments": "{\"city\":" } + } + ] + }, + "finish_reason": null + } + ] + }); + let event: ResponseStreamEvent = serde_json::from_value(event).unwrap(); + let delta = &event.choices[0].delta; + assert_eq!(delta.reasoning_content.as_deref(), Some("thinking...")); + assert_eq!(delta.tool_calls.as_ref().unwrap().len(), 1); + } + + #[test] + fn encodes_router_model_id_for_props_query() { + // Router ids contain `/` and `:`, which must be URL-encoded in the + // `?model=` query, while unreserved characters (`.`, `-`) stay literal. + let url = Url::parse_with_params( + "http://localhost:8080/props", + [("model", "unsloth/Qwen3.5-2B-GGUF:Q4_1")], + ) + .unwrap(); + assert_eq!( + url.as_str(), + "http://localhost:8080/props?model=unsloth%2FQwen3.5-2B-GGUF%3AQ4_1" + ); + } +} diff --git a/crates/settings_content/src/language_model.rs b/crates/settings_content/src/language_model.rs index a673ecc7a8..ab11097728 100644 --- a/crates/settings_content/src/language_model.rs +++ b/crates/settings_content/src/language_model.rs @@ -15,6 +15,7 @@ pub struct AllLanguageModelSettingsContent { pub bedrock: Option, pub deepseek: Option, pub google: Option, + pub llama_cpp: Option, pub lmstudio: Option, pub mistral: Option, pub ollama: Option, @@ -262,6 +263,36 @@ pub struct LmStudioAvailableModel { pub supports_images: bool, } +#[with_fallible_options] +#[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema, MergeFrom)] +pub struct LlamaCppSettingsContent { + pub api_url: Option, + /// Whether to automatically discover models served by the llama.cpp server. + /// Defaults to true. + pub auto_discover: Option, + pub available_models: Option>, + /// Overrides the context length reported for every llama.cpp model. + pub context_window: Option, + pub custom_headers: Option>, +} + +#[with_fallible_options] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema, MergeFrom)] +pub struct LlamaCppAvailableModel { + /// The model id reported by the llama.cpp server (its `--alias` or the model file path). + pub name: String, + /// The model's name in Zed's UI, such as in the model selector dropdown menu in the agent panel. + pub display_name: Option, + /// The Context Length parameter to the model (aka n_ctx). + pub max_tokens: u64, + /// Whether the model supports tools. + pub supports_tools: Option, + /// Whether the model supports vision. + pub supports_images: Option, + /// Whether the model emits reasoning/thinking content. + pub supports_thinking: Option, +} + #[with_fallible_options] #[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema, MergeFrom)] pub struct DeepseekSettingsContent { diff --git a/docs/src/ai/by-company.md b/docs/src/ai/by-company.md index 5eb3d27a7c..e477bdf6b2 100644 --- a/docs/src/ai/by-company.md +++ b/docs/src/ai/by-company.md @@ -97,8 +97,9 @@ Paid DeepSeek usage is API access in Zed, not subscription sign-in. | Tool | Support level | What you get | Account / billing | Setup | | --------------------------------- | ----------------- | -------------------------------------- | ----------------- | ------------------------------------------------------------- | -| Ollama | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#ollama) | +| llama.cpp | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#llama-cpp) | | LM Studio | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#lm-studio) | +| Ollama | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#ollama) | | Local OpenAI-compatible server | Configured in Zed | Local or self-hosted model endpoint | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#openai-compatible) | | Local/self-hosted edit prediction | Configured in Zed | Edit predictions from a local provider | Local/self-hosted | [Edit Prediction](./edit-prediction.md) | diff --git a/docs/src/ai/use-a-local-model.md b/docs/src/ai/use-a-local-model.md index 7cd7836b69..89984d10f3 100644 --- a/docs/src/ai/use-a-local-model.md +++ b/docs/src/ai/use-a-local-model.md @@ -1,6 +1,6 @@ --- title: Use a Local Model - Zed -description: Configure Ollama, LM Studio, local OpenAI-compatible servers, and local edit prediction in Zed. +description: Configure llama.cpp, Ollama, LM Studio, local OpenAI-compatible servers, and local edit prediction in Zed. --- # Use a Local Model @@ -9,11 +9,69 @@ Use local models when you run the model on your machine or on infrastructure you | Local path | Zed AI features | External Agents | Terminal Threads | Notes | | --------------------------------- | -------------------- | --------------- | ---------------- | -------------------------------------------------- | -| Ollama | Yes | Separate config | Separate config | Configure Ollama for Zed AI features | +| llama.cpp | Yes | Separate config | Separate config | Configure a llama.cpp server for Zed AI features | | LM Studio | Yes | Separate config | Separate config | Configure LM Studio for Zed AI features | +| Ollama | Yes | Separate config | Separate config | Configure Ollama for Zed AI features | | Local OpenAI-compatible server | Yes | Separate config | Separate config | Configure base URL, model, and key if needed | | Local/self-hosted edit prediction | Edit Prediction only | No | No | Uses [Edit Prediction](./edit-prediction.md) setup | +## llama.cpp {#llama-cpp} + +Use [llama.cpp](https://llama.app) and its built-in server for local models with Zed Agent, Inline Assistant, and similar model-backed Zed AI features. + +1. Install llama.cpp from [llama.app](https://llama.app). +2. Start the server in [router mode](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md): + + ```sh + llama serve + ``` + + It loads models from the llama.cpp cache on demand. To download and run a specific model in one step, pass `-hf`: + + ```sh + llama serve -hf unsloth/gemma-4-26B-A4B-it-GGUF:BF16 + ``` + +3. In Zed, select a llama.cpp model from the model dropdown. + +Zed automatically discovers the served models with their context length and tool/vision capabilities. In router mode these are refined once a model loads, via the server's `/models/sse` stream (which requires a recent llama.cpp build). To list models yourself instead, set `auto_discover` to `false`: + +```json [settings] +{ + "language_models": { + "llama_cpp": { + "api_url": "http://localhost:8080", + "auto_discover": false, + "available_models": [ + { + "name": "gemma-4-12b-it-GGUF:BF16", + "display_name": "gemma-4-12b-it-GGUF:BF16", + "max_tokens": 32768, + "supports_tools": true, + "supports_images": false + } + ] + } + } +} +``` + +### llama.cpp Context Length {#llama-cpp-context} + +Zed uses the context length the server reports (`/props`). Override it for all models with `context_window`, or per model with `max_tokens` in `available_models`: + +```json [settings] +{ + "language_models": { + "llama_cpp": { + "context_window": 8192 + } + } +} +``` + +If your llama.cpp server requires a key, enter it in the provider UI or set `LLAMACPP_API_KEY`. For a remote server, set the API URL to its endpoint and provide the key (set on the server with `--api-key`). + ## Ollama {#ollama} Use Ollama for local models with Zed Agent, Inline Assistant, and similar model-backed Zed AI features.