From c93d478488513dba2f6124147050f2d9d5c4ff8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?c=C3=A9lina?= Date: Mon, 29 Jun 2026 15:24:33 +0200 Subject: [PATCH] Add llama.cpp as a language model provider (#59964) Hi there, I'm Celina from Hugging Face! Opening this PR to add [llama.cpp](https://llama.app) as a model provider # Objective Today Zed users running llama.cpp have to fall back to the generic OpenAI-compatible provider, which means no auto-discovery (the router mode (`llama serve`) discovers models from the cache and loads them on demand) and manual configuration of every model and its capabilities. This PR makes `llama.cpp` a first-class provider with the same auto-discovery experience. ## Solution - Add a `llama_cpp` client crate with the OpenAI-compatible chat types (`/v1/chat/completions`, including `reasoning_content`) and the discovery types (`/v1/models`, `/props`), mirroring the existing `ollama` crate. - Add the provider in `crates/language_models/src/provider/llama_cpp.rs`, modeled on the Ollama provider (settings, configuration view, event mapping). - Auto discover served models and their context length and tool/vision support from `/props`. Set `auto_discover: false` to list models manually instead. - An unloaded model can't be inspected without loading it, so it is listed with optimistic defaults (large context, tools enabled) and is usable from the first message; its real context length and tool support are filled in once it loads. These live behind a shared map, so a model already selected in an open conversation picks them up without being re-selected. - Show load progress. The provider subscribes to `/models/sse` and surfaces each model's load progress (e.g. "Loading weights 42%") in its display name, reconciling stale labels against `/v1/models`. Builds without `/models/sse` degrade gracefully - no progress, and no capability refresh after the initial discovery. - Add settings (`api_url`, `auto_discover`, `available_models` with per-model `max_tokens` / `supports_tools` / `supports_images`, `context_window`, `custom_headers`), the provider icon, a `default.json` entry, and documentation under "Use a Local Model". No new dependencies: the crate reuses existing workspace dependencies, and shared state uses `std::sync::RwLock`. ## Testing - Unit tests in both new crates cover wire/response parsing, model discovery for single-model and router shapes, the cold-start optimistic defaults, the in-place capability refresh once a model loads, and the `/models/sse` event handling (state changes, load failure, load progress). - Built Zed locally and ran it against a local `llama serve` router: confirmed models are discovered without manual configuration, that the first message works before a model has finished loading, that load progress is shown in the model's display name while it loads, and that the reported context length and tool support refine to the model's real values once it finishes loading. - Platforms: tested on macOS (Apple Silicon). ## Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content adheres to Zed's UI standards ([UX/UI](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) and [icon](https://github.com/zed-industries/zed/blob/main/crates/icons/README.md) guidelines) - [x] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable ## Showcase The generation speed (tokens/sec) depends on the machine you're running the model, here it's a Apple M3 Max 64GB running a 4-bit quant of https://huggingface.co/Qwen/Qwen3.5-35B-A3B. For the load progress status, make sure to upgrade your llama.cpp version to the latest build. https://github.com/user-attachments/assets/0254f6ef-abe9-42ed-810b-ef1a5b8fa3bd --- Release Notes: - Added llama.cpp as a language model provider --------- Co-authored-by: Ben Brandt --- Cargo.lock | 14 + Cargo.toml | 3 + assets/icons/ai_llama_cpp.svg | 4 + assets/settings/default.json | 3 + crates/icons/src/icons.rs | 1 + crates/language_models/Cargo.toml | 1 + crates/language_models/src/language_models.rs | 9 + crates/language_models/src/provider.rs | 1 + .../language_models/src/provider/llama_cpp.rs | 2073 +++++++++++++++++ crates/language_models/src/settings.rs | 18 +- crates/llama_cpp/Cargo.toml | 25 + crates/llama_cpp/LICENSE-GPL | 1 + crates/llama_cpp/src/llama_cpp.rs | 911 ++++++++ crates/settings_content/src/language_model.rs | 31 + docs/src/ai/by-company.md | 3 +- docs/src/ai/use-a-local-model.md | 62 +- 16 files changed, 3153 insertions(+), 7 deletions(-) create mode 100644 assets/icons/ai_llama_cpp.svg create mode 100644 crates/language_models/src/provider/llama_cpp.rs create mode 100644 crates/llama_cpp/Cargo.toml create mode 120000 crates/llama_cpp/LICENSE-GPL create mode 100644 crates/llama_cpp/src/llama_cpp.rs diff --git a/Cargo.lock b/Cargo.lock index f43363e23c..f5803b5baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9823,6 +9823,7 @@ dependencies = [ "language", "language_model", "language_models_cloud", + "llama_cpp", "lmstudio", "log", "menu", @@ -10354,6 +10355,19 @@ dependencies = [ "zed-scap", ] +[[package]] +name = "llama_cpp" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures 0.3.32", + "http_client", + "schemars 1.0.4", + "serde", + "serde_json", + "url", +] + [[package]] name = "lmdb-master-sys" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 4f36916d2d..4d36e276a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ members = [ "crates/line_ending_selector", "crates/livekit_api", "crates/livekit_client", + "crates/llama_cpp", "crates/lmstudio", "crates/lsp", "crates/markdown", @@ -384,6 +385,7 @@ languages = { path = "crates/languages" } line_ending_selector = { path = "crates/line_ending_selector" } livekit_api = { path = "crates/livekit_api" } livekit_client = { path = "crates/livekit_client" } +llama_cpp = { path = "crates/llama_cpp" } lmstudio = { path = "crates/lmstudio" } lsp = { path = "crates/lsp" } markdown = { path = "crates/markdown" } @@ -977,6 +979,7 @@ edit_prediction_ui = { codegen-units = 1 } install_cli = { codegen-units = 1 } journal = { codegen-units = 1 } json_schema_store = { codegen-units = 1 } +llama_cpp = { codegen-units = 1 } lmstudio = { codegen-units = 1 } menu = { codegen-units = 1 } notifications = { codegen-units = 1 } diff --git a/assets/icons/ai_llama_cpp.svg b/assets/icons/ai_llama_cpp.svg new file mode 100644 index 0000000000..310ec3793a --- /dev/null +++ b/assets/icons/ai_llama_cpp.svg @@ -0,0 +1,4 @@ + + + + diff --git a/assets/settings/default.json b/assets/settings/default.json index e4051ee8f0..059608ae04 100644 --- a/assets/settings/default.json +++ b/assets/settings/default.json @@ -2427,6 +2427,9 @@ "ollama": { "api_url": "http://localhost:11434", }, + "llama_cpp": { + "api_url": "http://localhost:8080", + }, "openai": { "api_url": "https://api.openai.com/v1", }, diff --git a/crates/icons/src/icons.rs b/crates/icons/src/icons.rs index a94b43e801..56182da213 100644 --- a/crates/icons/src/icons.rs +++ b/crates/icons/src/icons.rs @@ -17,6 +17,7 @@ pub enum IconName { AiEdit, AiGemini, AiGoogle, + AiLlamaCpp, AiLmStudio, AiMistral, AiOllama, diff --git a/crates/language_models/Cargo.toml b/crates/language_models/Cargo.toml index b96a377f02..57a0d530e4 100644 --- a/crates/language_models/Cargo.toml +++ b/crates/language_models/Cargo.toml @@ -43,6 +43,7 @@ http_client.workspace = true language.workspace = true language_model.workspace = true language_models_cloud.workspace = true +llama_cpp = { workspace = true, features = ["schemars"] } lmstudio = { workspace = true, features = ["schemars"] } log.workspace = true menu.workspace = true diff --git a/crates/language_models/src/language_models.rs b/crates/language_models/src/language_models.rs index dc13cde137..1133ee2d47 100644 --- a/crates/language_models/src/language_models.rs +++ b/crates/language_models/src/language_models.rs @@ -24,6 +24,7 @@ use crate::provider::bedrock::BedrockLanguageModelProvider; use crate::provider::cloud::CloudLanguageModelProvider; use crate::provider::copilot_chat::CopilotChatLanguageModelProvider; use crate::provider::google::GoogleLanguageModelProvider; +use crate::provider::llama_cpp::LlamaCppLanguageModelProvider; use crate::provider::lmstudio::LmStudioLanguageModelProvider; pub use crate::provider::mistral::MistralLanguageModelProvider; use crate::provider::ollama::OllamaLanguageModelProvider; @@ -307,6 +308,14 @@ fn register_language_model_providers( )), cx, ); + registry.register_provider( + Arc::new(LlamaCppLanguageModelProvider::new( + client.http_client(), + credentials_provider.clone(), + cx, + )), + cx, + ); registry.register_provider( Arc::new(DeepSeekLanguageModelProvider::new( client.http_client(), diff --git a/crates/language_models/src/provider.rs b/crates/language_models/src/provider.rs index 3a1369a2d1..65fc039b07 100644 --- a/crates/language_models/src/provider.rs +++ b/crates/language_models/src/provider.rs @@ -10,6 +10,7 @@ pub mod cloud; pub mod copilot_chat; pub mod deepseek; pub mod google; +pub mod llama_cpp; pub mod lmstudio; pub mod mistral; pub mod ollama; diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rs new file mode 100644 index 0000000000..935ec74f9a --- /dev/null +++ b/crates/language_models/src/provider/llama_cpp.rs @@ -0,0 +1,2073 @@ +use anyhow::Result; +use collections::{HashMap, HashSet}; +use credentials_provider::CredentialsProvider; +use fs::Fs; +use futures::Stream; +use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream}; +use gpui::{AnyView, App, AsyncApp, Context, CursorStyle, Entity, Task, TaskExt}; +use http_client::{CustomHeaders, HttpClient}; +use language_model::util::parse_tool_arguments; +use language_model::{ + ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel, LanguageModelCompletionError, + LanguageModelCompletionEvent, LanguageModelId, LanguageModelName, LanguageModelProvider, + LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState, + LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolResultContent, + LanguageModelToolUse, MessageContent, RateLimiter, Role, StopReason, TokenUsage, env_var, +}; +use llama_cpp::{ + LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion, + stream_model_events, +}; +pub use settings::LlamaCppAvailableModel as AvailableModel; +use settings::{Settings, SettingsStore, update_settings_file}; +use std::pin::Pin; +use std::sync::LazyLock; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::time::Duration; +use ui::{ + ButtonLike, ButtonLink, ConfiguredApiCard, ElevationIndex, List, ListBulletItem, Tooltip, + prelude::*, +}; +use ui_input::InputField; +use util::ResultExt; + +use crate::AllLanguageModelSettings; + +const LLAMA_CPP_DOWNLOAD_URL: &str = "https://llama.app"; +const LLAMA_CPP_MODELS_URL: &str = "https://huggingface.co/models?library=gguf&sort=trending"; + +const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("llama_cpp"); +const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new("llama.cpp"); + +const API_KEY_ENV_VAR_NAME: &str = "LLAMACPP_API_KEY"; +static API_KEY_ENV_VAR: LazyLock = env_var!(API_KEY_ENV_VAR_NAME); + +/// How long to wait before reconnecting to `/models/sse` after the stream ends. +const MODEL_EVENT_RECONNECT_INTERVAL: Duration = Duration::from_secs(5); + +/// Context length assumed for an unloaded router model (it can't be probed +/// without loading it). Generous so early messages work; re-discovery refines +/// it once the model loads. +const ASSUMED_UNLOADED_CONTEXT: u64 = 131_072; + +#[derive(Default, Debug, Clone, PartialEq)] +pub struct LlamaCppSettings { + pub api_url: String, + pub auto_discover: bool, + pub available_models: Vec, + pub context_window: Option, + pub custom_headers: CustomHeaders, +} + +pub struct LlamaCppLanguageModelProvider { + http_client: Arc, + state: Entity, + /// Live capabilities shared with the agent's models (see [`LiveCapabilities`]). + capability_cells: CapabilityCells, + /// Live model-load progress shared with the models (see [`LoadingProgress`]). + loading_progress: LoadingProgress, +} + +pub struct State { + api_key_state: ApiKeyState, + credentials_provider: Arc, + http_client: Arc, + fetched_models: Vec, + fetch_model_task: Option>>, + /// Router-mode task on `/models/sse`; re-runs discovery as models load/unload. + model_event_task: Option>, + /// Same `Arc` as the provider's; re-discovery keeps these cells in sync. + capability_cells: CapabilityCells, + /// Same `Arc` as the provider's; the event stream updates it as a model loads. + loading_progress: LoadingProgress, +} + +impl State { + fn is_authenticated(&self) -> bool { + !self.fetched_models.is_empty() + } + + fn set_api_key(&mut self, api_key: Option, cx: &mut Context) -> Task> { + let credentials_provider = self.credentials_provider.clone(); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let task = self.api_key_state.store( + api_url, + api_key, + |this| &mut this.api_key_state, + credentials_provider, + cx, + ); + + self.fetched_models.clear(); + // Drop the event stream so it reconnects with the new key (re-fetch + // below restarts it). + self.model_event_task = None; + write_recover(&self.loading_progress).clear(); + cx.spawn(async move |this, cx| { + let result = task.await; + this.update(cx, |this, cx| this.restart_fetch_models_task(cx)) + .ok(); + result + }) + } + + fn authenticate(&mut self, cx: &mut Context) -> Task> { + let credentials_provider = self.credentials_provider.clone(); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let load_key_task = self.api_key_state.load_if_needed( + api_url, + |this| &mut this.api_key_state, + credentials_provider, + cx, + ); + + if self.is_authenticated() { + return Task::ready(Ok(())); + } + + cx.spawn(async move |this, cx| { + match load_key_task.await { + Ok(()) | Err(AuthenticateError::CredentialsNotFound) => {} + Err(error) => { + log::warn!("failed to load llama.cpp API key: {error}"); + } + } + let fetch_models_task = this.update(cx, |this, cx| this.fetch_models(cx))?; + match fetch_models_task.await { + Ok(()) => Ok(()), + Err(err) => { + // A refused connection means the server isn't running yet, not an error. + let connection_refused = err.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io_err| { + io_err.kind() == std::io::ErrorKind::ConnectionRefused + }) + }); + if connection_refused { + Err(AuthenticateError::ConnectionRefused) + } else { + Err(AuthenticateError::Other(err)) + } + } + } + }) + } + + fn fetch_models(&mut self, cx: &mut Context) -> Task> { + let http_client = Arc::clone(&self.http_client); + let settings = LlamaCppLanguageModelProvider::settings(cx); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let api_key = self.api_key_state.key(&api_url); + let extra_headers = settings.custom_headers.clone(); + + cx.spawn(async move |this, cx| { + let entries = get_models( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + &extra_headers, + ) + .await?; + + let is_router = entries.iter().any(ModelEntry::is_router_entry); + + // Models the server reports as loading, used below to prune stale + // progress labels by reconciling against the live listing (a preempted + // load or missed SSE event can skip the terminal event). + let loading_ids: HashSet = entries + .iter() + .filter(|entry| entry.is_loading()) + .map(|entry| entry.id.clone()) + .collect(); + + let models: Vec = if is_router { + // Router mode: metadata comes from `/v1/models`. We probe + // `/props` only for loaded models so listing never triggers a + // load; unloaded models use the listing's hints and overrides. + let tasks = entries.into_iter().map(|entry| { + let http_client = Arc::clone(&http_client); + let api_url = api_url.clone(); + let api_key = api_key.clone(); + let extra_headers = extra_headers.clone(); + async move { + let props = if entry.is_loaded() { + get_props( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + Some(&entry.id), + &extra_headers, + ) + .await + .log_err() + } else { + None + }; + model_from_entry(&entry, props.as_ref()) + } + }); + futures::stream::iter(tasks) + .buffer_unordered(5) + .collect() + .await + } else { + // Single-model mode: one `/props` call describes the loaded model. + let props = get_props( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + None, + &extra_headers, + ) + .await + .log_err(); + entries + .iter() + .map(|entry| model_from_entry(entry, props.as_ref())) + .collect() + }; + + this.update(cx, |this, cx| { + this.fetched_models = models; + let effective = compute_effective_models( + &this.fetched_models, + LlamaCppLanguageModelProvider::settings(cx), + ); + sync_capability_cells(&this.capability_cells, &effective); + // Drop progress labels for models no longer loading, so a stale + // "Loading …" can't stick after a preempted load or missed event. + write_recover(&this.loading_progress).retain(|id, _| loading_ids.contains(id)); + // Router mode loads models on demand: subscribe so capabilities + // self-correct as they load/unload. Start it once (events trigger + // re-discovery, not a re-spawn); single-model mode needs no stream. + if is_router { + if this.model_event_task.is_none() { + this.start_model_event_stream(cx); + } + } else { + this.model_event_task = None; + } + cx.notify(); + }) + }) + } + + /// Subscribes to `/models/sse` and re-runs discovery as models load, unload, + /// or the list changes, so capabilities stay current. Reconnects if the stream + /// drops; on builds without `/models/sse` the refresh is simply skipped. + fn start_model_event_stream(&mut self, cx: &mut Context) { + let http_client = Arc::clone(&self.http_client); + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let api_key = self.api_key_state.key(&api_url); + let extra_headers = LlamaCppLanguageModelProvider::settings(cx) + .custom_headers + .clone(); + + self.model_event_task = Some(cx.spawn(async move |this, cx| { + loop { + match stream_model_events( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + &extra_headers, + ) + .await + { + Ok(mut events) => { + while let Some(event) = events.next().await { + let Some(event) = event.log_err() else { + continue; + }; + if let Some(exit_code) = event.load_failure() { + log::error!( + "llama.cpp model {} failed to load (exit code {exit_code})", + event.model + ); + } + // Loading-progress tick: record it for the selector (no + // re-discovery). `cx.notify()` drives `ProviderStateChanged`. + if let Some(progress) = event.load_progress() { + let label = SharedString::from(progress.progress_label()); + if this + .update(cx, |this, cx| { + write_recover(&this.loading_progress) + .insert(event.model.clone(), label); + cx.notify(); + }) + .is_err() + { + return; + } + continue; + } + if !event.changes_model_state() { + continue; + } + // Terminal load/unload (or list change): drop the + // progress label and re-discover to refresh capabilities. + if this + .update(cx, |this, cx| { + write_recover(&this.loading_progress).remove(&event.model); + this.restart_fetch_models_task(cx); + }) + .is_err() + { + return; + } + } + } + // Endpoint missing (older build) or connection failed; retry after a backoff. + Err(error) => { + log::warn!("llama.cpp model event stream unavailable: {error:#}"); + } + } + + cx.background_executor() + .timer(MODEL_EVENT_RECONNECT_INTERVAL) + .await; + if this.update(cx, |_, _| ()).is_err() { + return; + } + } + })); + } + + fn restart_fetch_models_task(&mut self, cx: &mut Context) { + let task = self.fetch_models(cx); + self.fetch_model_task.replace(task); + } +} + +/// Capabilities that only become known once a router model loads. `LanguageModel`'s +/// methods take no `cx`, yet the agent reads them live each turn, so we share +/// them through a map that re-discovery updates — an already-selected model +/// picks up real values without re-selection. Image support is available from +/// `/v1/models` hints, so it stays a field. +#[derive(Clone, Copy, Debug, PartialEq)] +struct LiveCapabilities { + max_tokens: u64, + supports_tools: bool, + supports_thinking: bool, +} + +impl LiveCapabilities { + fn of(model: &llama_cpp::Model) -> Self { + Self { + max_tokens: model.max_tokens, + supports_tools: model.supports_tools, + supports_thinking: model.supports_thinking, + } + } +} + +/// Live capabilities keyed by model name, shared by the provider and its models. +type CapabilityCells = Arc>>; + +/// Model name → load-status label (e.g. `"Loading weights 42%"`) while a router +/// model loads, shared so the model selector can show progress. Absent once loaded. +type LoadingProgress = Arc>>; + +/// Locks for reading, recovering instead of panicking on a poisoned lock. The +/// critical sections are infallible map ops, so poisoning is unreachable anyway. +fn read_recover(lock: &RwLock) -> RwLockReadGuard<'_, T> { + lock.read().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Locks for writing; see [`read_recover`]. +fn write_recover(lock: &RwLock) -> RwLockWriteGuard<'_, T> { + lock.write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The exact set of models `provided_models` exposes: discovery merged with the +/// `available_models` and `context_window` overrides. Shared with re-discovery. +fn compute_effective_models( + fetched_models: &[llama_cpp::Model], + settings: &LlamaCppSettings, +) -> HashMap { + let mut models: HashMap = HashMap::default(); + if settings.auto_discover { + for model in fetched_models { + let mut model = model.clone(); + if let Some(context_window) = settings.context_window { + model.max_tokens = context_window; + } + models.insert(model.name.clone(), model); + } + } + merge_settings_into_models( + &mut models, + &settings.available_models, + settings.context_window, + ); + models +} + +/// Updates the shared capability map from the effective models, so a model held +/// by an open conversation observes the new values (it reads the map by name). +fn sync_capability_cells(cells: &CapabilityCells, effective: &HashMap) { + let mut cells = write_recover(cells); + for model in effective.values() { + cells.insert(model.name.clone(), LiveCapabilities::of(model)); + } +} + +/// Builds a model from a `/v1/models` entry, refined by `/props` when the model +/// is loaded. An unloaded router model can't be probed, so we assume optimistic +/// capabilities and let re-discovery refine them on load. +fn model_from_entry(entry: &ModelEntry, props: Option<&Props>) -> llama_cpp::Model { + let max_tokens = props + .and_then(Props::context_length) + .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx)) + .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx_train)) + .unwrap_or(ASSUMED_UNLOADED_CONTEXT); + // Trust `/props` when present. Without it, assume tools for an unloaded model + // (re-discovery corrects on load) but not for a loaded model whose probe failed. + let supports_tools = match props { + Some(props) => props.supports_tools(), + None => !entry.is_loaded(), + }; + let supports_images = props.is_some_and(Props::supports_images) || entry.supports_images_hint(); + let supports_thinking = props.is_some_and(Props::supports_thinking); + + llama_cpp::Model::new( + &entry.id, + Some(&display_name_for(&entry.id)), + Some(max_tokens), + supports_tools, + supports_images, + supports_thinking, + ) +} + +/// Friendly display name from a model id, which is often a `.gguf` file path. +fn display_name_for(id: &str) -> String { + let base = id.rsplit(['/', '\\']).next().unwrap_or(id); + base.strip_suffix(".gguf").unwrap_or(base).to_string() +} + +fn telemetry_id_for(id: &str) -> String { + format!("llama_cpp/{}", display_name_for(id)) +} + +impl LlamaCppLanguageModelProvider { + pub fn new( + http_client: Arc, + credentials_provider: Arc, + cx: &mut App, + ) -> Self { + let capability_cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default())); + let loading_progress: LoadingProgress = Arc::new(RwLock::new(HashMap::default())); + let this = Self { + http_client: http_client.clone(), + capability_cells: capability_cells.clone(), + loading_progress: loading_progress.clone(), + state: cx.new(|cx| { + cx.observe_global::({ + let mut last_settings = LlamaCppLanguageModelProvider::settings(cx).clone(); + move |this: &mut State, cx| { + let current_settings = LlamaCppLanguageModelProvider::settings(cx); + let settings_changed = current_settings != &last_settings; + if settings_changed { + let url_changed = last_settings.api_url != current_settings.api_url; + last_settings = current_settings.clone(); + if url_changed { + let credentials_provider = this.credentials_provider.clone(); + let api_url = Self::api_url(cx); + this.api_key_state.handle_url_change( + api_url, + |this| &mut this.api_key_state, + credentials_provider, + cx, + ); + this.fetched_models.clear(); + // Drop the event stream so it reconnects against + // the new URL (re-auth below restarts it). + this.model_event_task = None; + write_recover(&this.loading_progress).clear(); + this.authenticate(cx).detach(); + } + cx.notify(); + } + } + }) + .detach(); + + State { + http_client, + fetched_models: Default::default(), + fetch_model_task: None, + model_event_task: None, + capability_cells, + loading_progress, + api_key_state: ApiKeyState::new(Self::api_url(cx), (*API_KEY_ENV_VAR).clone()), + credentials_provider, + } + }), + }; + // Discover eagerly so a running server is picked up without opening settings. + this.state + .update(cx, |state, cx| state.restart_fetch_models_task(cx)); + this + } + + fn settings(cx: &App) -> &LlamaCppSettings { + &AllLanguageModelSettings::get_global(cx).llama_cpp + } + + fn api_url(cx: &App) -> SharedString { + let api_url = &Self::settings(cx).api_url; + if api_url.is_empty() { + LLAMA_CPP_API_URL.into() + } else { + SharedString::new(api_url.as_str()) + } + } + + fn has_custom_url(cx: &App) -> bool { + Self::settings(cx).api_url != LLAMA_CPP_API_URL + } +} + +impl LanguageModelProviderState for LlamaCppLanguageModelProvider { + type ObservableEntity = State; + + fn observable_entity(&self) -> Option> { + Some(self.state.clone()) + } +} + +impl LanguageModelProvider for LlamaCppLanguageModelProvider { + fn id(&self) -> LanguageModelProviderId { + PROVIDER_ID + } + + fn name(&self) -> LanguageModelProviderName { + PROVIDER_NAME + } + + fn icon(&self) -> IconOrSvg { + IconOrSvg::Icon(IconName::AiLlamaCpp) + } + + fn default_model(&self, _: &App) -> Option> { + // No default model: in router mode it could trigger an expensive load of + // an unloaded model on a constrained machine. + None + } + + fn default_fast_model(&self, _: &App) -> Option> { + // See explanation for default_model. + None + } + + fn provided_models(&self, cx: &App) -> Vec> { + let settings = LlamaCppLanguageModelProvider::settings(cx); + let effective = compute_effective_models(&self.state.read(cx).fetched_models, settings); + + // Refresh the shared capability map so open conversations pick up settings changes. + sync_capability_cells(&self.capability_cells, &effective); + let mut models = effective + .into_values() + .map(|model| { + Arc::new(LlamaCppLanguageModel { + id: LanguageModelId::from(model.name.clone()), + name: model.name.clone(), + display_name: model.display_name().to_string(), + fallback_capabilities: LiveCapabilities::of(&model), + supports_images: model.supports_images, + capability_cells: self.capability_cells.clone(), + loading_progress: self.loading_progress.clone(), + http_client: self.http_client.clone(), + request_limiter: RateLimiter::new(4), + state: self.state.clone(), + }) as Arc + }) + .collect::>(); + models.sort_by_key(|model| model.name()); + models + } + + fn is_authenticated(&self, cx: &App) -> bool { + self.state.read(cx).is_authenticated() + } + + fn authenticate(&self, cx: &mut App) -> Task> { + self.state.update(cx, |state, cx| state.authenticate(cx)) + } + + fn configuration_view( + &self, + _target_agent: language_model::ConfigurationViewTargetAgent, + window: &mut Window, + cx: &mut App, + ) -> AnyView { + let state = self.state.clone(); + cx.new(|cx| ConfigurationView::new(state, window, cx)) + .into() + } + + fn reset_credentials(&self, cx: &mut App) -> Task> { + self.state + .update(cx, |state, cx| state.set_api_key(None, cx)) + } +} + +pub struct LlamaCppLanguageModel { + id: LanguageModelId, + /// The model id sent to the server. + name: String, + display_name: String, + /// Live capabilities shared with the provider, read fresh on each access so an + /// open conversation reflects the model's real values once it has loaded. + capability_cells: CapabilityCells, + /// Used when `capability_cells` has no entry (e.g. model removed mid-conversation). + fallback_capabilities: LiveCapabilities, + /// Available from `/v1/models` hints, so captured at build time. + supports_images: bool, + /// Shared with the provider; this model's load progress, read by `name` so the + /// selector can show a loading indicator. + loading_progress: LoadingProgress, + http_client: Arc, + request_limiter: RateLimiter, + state: Entity, +} + +impl LlamaCppLanguageModel { + /// The model's live capabilities, or the build-time fallback if the map lacks it. + fn capabilities(&self) -> LiveCapabilities { + read_recover(&self.capability_cells) + .get(&self.name) + .copied() + .unwrap_or(self.fallback_capabilities) + } + + /// This model's load-status label while loading, read live from the shared map. + fn loading_label(&self) -> Option { + read_recover(&self.loading_progress) + .get(&self.name) + .cloned() + } + + fn to_llama_cpp_request( + &self, + request: LanguageModelRequest, + ) -> llama_cpp::ChatCompletionRequest { + build_llama_cpp_request( + &self.name, + self.supports_images, + self.capabilities(), + request, + ) + } + + fn stream_completion( + &self, + request: llama_cpp::ChatCompletionRequest, + cx: &AsyncApp, + ) -> BoxFuture< + 'static, + Result>>, + > { + let http_client = self.http_client.clone(); + let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| { + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let extra_headers = LlamaCppLanguageModelProvider::settings(cx) + .custom_headers + .clone(); + (state.api_key_state.key(&api_url), api_url, extra_headers) + }); + + let future = self.request_limiter.stream(async move { + let stream = stream_chat_completion( + http_client.as_ref(), + &api_url, + api_key.as_deref(), + request, + &extra_headers, + ) + .await?; + Ok(stream) + }); + + async move { Ok(future.await?.boxed()) }.boxed() + } +} + +fn build_llama_cpp_request( + model_name: &str, + supports_images: bool, + capabilities: LiveCapabilities, + request: LanguageModelRequest, +) -> llama_cpp::ChatCompletionRequest { + let supports_tools = capabilities.supports_tools; + let supports_thinking = capabilities.supports_thinking; + let mut messages = Vec::new(); + + for message in request.messages { + let mut reasoning_content: Option = None; + for content in message.content { + match content { + MessageContent::Text(text) => add_message_content_part( + llama_cpp::MessagePart::Text { text }, + message.role, + &mut messages, + if supports_thinking && message.role == Role::Assistant { + reasoning_content.take() + } else { + None + }, + ), + MessageContent::Thinking { text, .. } => { + if supports_thinking && message.role == Role::Assistant && !text.is_empty() { + reasoning_content.get_or_insert_default().push_str(&text); + } + } + MessageContent::RedactedThinking(_) => {} + MessageContent::Compaction(_) => {} + MessageContent::Image(image) => { + if supports_images { + add_message_content_part( + llama_cpp::MessagePart::Image { + image_url: llama_cpp::ImageUrl { + url: image.to_base64_url(), + detail: None, + }, + }, + message.role, + &mut messages, + if supports_thinking && message.role == Role::Assistant { + reasoning_content.take() + } else { + None + }, + ); + } + } + MessageContent::ToolUse(tool_use) => { + let tool_call = llama_cpp::ToolCall { + id: tool_use.id.to_string(), + content: llama_cpp::ToolCallContent::Function { + function: llama_cpp::FunctionContent { + name: tool_use.name.to_string(), + arguments: serde_json::to_string(&tool_use.input) + .unwrap_or_default(), + }, + }, + }; + + if let Some(llama_cpp::ChatMessage::Assistant { + tool_calls, + reasoning_content: message_reasoning_content, + .. + }) = messages.last_mut() + { + append_reasoning_content( + message_reasoning_content, + reasoning_content.take(), + ); + tool_calls.push(tool_call); + } else { + messages.push(llama_cpp::ChatMessage::Assistant { + content: None, + reasoning_content: reasoning_content.take(), + tool_calls: vec![tool_call], + }); + } + } + MessageContent::ToolResult(tool_result) => { + let content: Vec = tool_result + .content + .iter() + .filter_map(|part| match part { + LanguageModelToolResultContent::Text(text) => { + Some(llama_cpp::MessagePart::Text { + text: text.to_string(), + }) + } + LanguageModelToolResultContent::Image(image) => { + if supports_images { + Some(llama_cpp::MessagePart::Image { + image_url: llama_cpp::ImageUrl { + url: image.to_base64_url(), + detail: None, + }, + }) + } else { + None + } + } + }) + .collect(); + + messages.push(llama_cpp::ChatMessage::Tool { + content: content.into(), + tool_call_id: tool_result.tool_use_id.to_string(), + }); + } + } + } + } + + let tools: Vec = if supports_tools { + request + .tools + .into_iter() + .map(|tool| llama_cpp::ToolDefinition::Function { + function: llama_cpp::FunctionDefinition { + name: tool.name, + description: Some(tool.description), + parameters: Some(tool.input_schema), + }, + }) + .collect() + } else { + Vec::new() + }; + // Only send `tool_choice` with actual tools; some OpenAI-compatible servers + // reject it otherwise. + let tool_choice = if tools.is_empty() { + None + } else { + request.tool_choice.map(|choice| match choice { + LanguageModelToolChoice::Auto => llama_cpp::ToolChoice::Auto, + LanguageModelToolChoice::Any => llama_cpp::ToolChoice::Required, + LanguageModelToolChoice::None => llama_cpp::ToolChoice::None, + }) + }; + + llama_cpp::ChatCompletionRequest { + model: model_name.to_string(), + messages, + stream: true, + // Let the server decide the output length (its `n_predict` default). + max_tokens: None, + stop: if request.stop.is_empty() { + None + } else { + Some(request.stop) + }, + // llama.cpp models often ship recommended sampler settings, so override + // temperature only when the request sets one. + temperature: request.temperature, + tools, + tool_choice, + stream_options: Some(llama_cpp::StreamOptions { + include_usage: true, + }), + } +} + +impl LanguageModel for LlamaCppLanguageModel { + fn id(&self) -> LanguageModelId { + self.id.clone() + } + + fn name(&self) -> LanguageModelName { + match self.loading_label() { + // Surface load progress in the display name so it shows wherever the + // model is named, without provider-agnostic UI changes. The agent + // rebuilds the name on `ProviderStateChanged`, which our ticks emit. + Some(label) => LanguageModelName::from(format!("{} · {}", self.display_name, label)), + None => LanguageModelName::from(self.display_name.clone()), + } + } + + fn provider_id(&self) -> LanguageModelProviderId { + PROVIDER_ID + } + + fn provider_name(&self) -> LanguageModelProviderName { + PROVIDER_NAME + } + + fn supports_tools(&self) -> bool { + self.capabilities().supports_tools + } + + fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool { + self.supports_tools() + && match choice { + LanguageModelToolChoice::Auto => true, + LanguageModelToolChoice::Any => true, + LanguageModelToolChoice::None => true, + } + } + + fn supports_images(&self) -> bool { + self.supports_images + } + + fn supports_thinking(&self) -> bool { + self.capabilities().supports_thinking + } + + fn telemetry_id(&self) -> String { + telemetry_id_for(&self.name) + } + + fn max_token_count(&self) -> u64 { + self.capabilities().max_tokens + } + + fn stream_completion( + &self, + request: LanguageModelRequest, + cx: &AsyncApp, + ) -> BoxFuture< + 'static, + Result< + BoxStream<'static, Result>, + LanguageModelCompletionError, + >, + > { + let request = self.to_llama_cpp_request(request); + let completions = self.stream_completion(request, cx); + async move { + let mapper = LlamaCppEventMapper::new(); + Ok(mapper.map_stream(completions.await?).boxed()) + } + .boxed() + } +} + +struct LlamaCppEventMapper { + tool_calls_by_index: HashMap, +} + +impl LlamaCppEventMapper { + fn new() -> Self { + Self { + tool_calls_by_index: HashMap::default(), + } + } + + pub fn map_stream( + mut self, + events: Pin>>>, + ) -> impl Stream> + { + events.flat_map(move |event| { + futures::stream::iter(match event { + Ok(event) => self.map_event(event), + Err(error) => vec![Err(LanguageModelCompletionError::from(error))], + }) + }) + } + + pub fn map_event( + &mut self, + event: llama_cpp::ResponseStreamEvent, + ) -> Vec> { + let mut events = Vec::new(); + + if let Some(usage) = event.usage { + events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { + input_tokens: usage.prompt_tokens, + output_tokens: usage.completion_tokens, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }))); + } + + if let Some(choice) = event.choices.into_iter().next() { + if let Some(reasoning_content) = choice.delta.reasoning_content { + events.push(Ok(LanguageModelCompletionEvent::Thinking { + text: reasoning_content, + signature: None, + })); + } + + if let Some(content) = choice.delta.content { + if !content.is_empty() { + events.push(Ok(LanguageModelCompletionEvent::Text(content))); + } + } + + if let Some(tool_calls) = choice.delta.tool_calls { + for tool_call in tool_calls { + let entry = self.tool_calls_by_index.entry(tool_call.index).or_default(); + + if let Some(tool_id) = tool_call.id { + entry.id = tool_id; + } + + if let Some(function) = tool_call.function { + if let Some(name) = function.name { + // Only the first chunk carries the function name; + // later chunks send an empty name with arguments. + if !name.is_empty() { + entry.name = name; + } + } + + if let Some(arguments) = function.arguments { + entry.arguments.push_str(&arguments); + } + } + } + } + + if let Some(finish_reason) = choice.finish_reason.as_deref() { + match finish_reason { + "stop" => { + events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn))); + } + "tool_calls" => { + events.extend(self.tool_calls_by_index.drain().map(|(_, tool_call)| { + match parse_tool_arguments(&tool_call.arguments) { + Ok(input) => Ok(LanguageModelCompletionEvent::ToolUse( + LanguageModelToolUse { + id: tool_call.id.into(), + name: tool_call.name.into(), + is_input_complete: true, + input, + raw_input: tool_call.arguments, + thought_signature: None, + }, + )), + Err(error) => { + Ok(LanguageModelCompletionEvent::ToolUseJsonParseError { + id: tool_call.id.into(), + tool_name: tool_call.name.into(), + raw_input: tool_call.arguments.into(), + json_parse_error: error.to_string(), + }) + } + } + })); + + events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse))); + } + "length" => { + events.push(Ok(LanguageModelCompletionEvent::Stop( + StopReason::MaxTokens, + ))); + } + unexpected => { + log::warn!("Unexpected llama.cpp finish_reason: {unexpected:?}"); + events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn))); + } + } + } + } + + events + } +} + +#[derive(Default)] +struct RawToolCall { + id: String, + name: String, + arguments: String, +} + +fn add_message_content_part( + new_part: llama_cpp::MessagePart, + role: Role, + messages: &mut Vec, + reasoning_content: Option, +) { + match (role, messages.last_mut()) { + (Role::User, Some(llama_cpp::ChatMessage::User { content })) + | (Role::System, Some(llama_cpp::ChatMessage::System { content })) => { + content.push_part(new_part); + } + ( + Role::Assistant, + Some(llama_cpp::ChatMessage::Assistant { + content: Some(content), + reasoning_content: message_reasoning_content, + .. + }), + ) => { + append_reasoning_content(message_reasoning_content, reasoning_content); + content.push_part(new_part); + } + _ => { + messages.push(match role { + Role::User => llama_cpp::ChatMessage::User { + content: llama_cpp::MessageContent::from(vec![new_part]), + }, + Role::Assistant => llama_cpp::ChatMessage::Assistant { + content: Some(llama_cpp::MessageContent::from(vec![new_part])), + reasoning_content, + tool_calls: Vec::new(), + }, + Role::System => llama_cpp::ChatMessage::System { + content: llama_cpp::MessageContent::from(vec![new_part]), + }, + }); + } + } +} + +fn append_reasoning_content(target: &mut Option, content: Option) { + let Some(content) = content else { + return; + }; + if content.is_empty() { + return; + } + target.get_or_insert_default().push_str(&content); +} + +fn merge_settings_into_models( + models: &mut HashMap, + available_models: &[AvailableModel], + context_window: Option, +) { + for setting_model in available_models { + if let Some(model) = models.get_mut(&setting_model.name) { + if context_window.is_none() { + model.max_tokens = setting_model.max_tokens; + } + if setting_model.display_name.is_some() { + model.display_name = setting_model.display_name.clone(); + } + if let Some(supports_tools) = setting_model.supports_tools { + model.supports_tools = supports_tools; + } + if let Some(supports_images) = setting_model.supports_images { + model.supports_images = supports_images; + } + if let Some(supports_thinking) = setting_model.supports_thinking { + model.supports_thinking = supports_thinking; + } + } else { + models.insert( + setting_model.name.clone(), + llama_cpp::Model { + name: setting_model.name.clone(), + display_name: setting_model.display_name.clone(), + max_tokens: context_window.unwrap_or(setting_model.max_tokens), + supports_tools: setting_model.supports_tools.unwrap_or(false), + supports_images: setting_model.supports_images.unwrap_or(false), + supports_thinking: setting_model.supports_thinking.unwrap_or(false), + }, + ); + } + } +} + +struct ConfigurationView { + api_key_editor: Entity, + api_url_editor: Entity, + context_window_editor: Entity, + state: Entity, +} + +impl ConfigurationView { + pub fn new(state: Entity, window: &mut Window, cx: &mut Context) -> Self { + let api_key_editor = cx.new(|cx| InputField::new(window, cx, "sk-...").label("API key")); + + let api_url_editor = cx.new(|cx| { + let input = InputField::new(window, cx, LLAMA_CPP_API_URL).label("API URL"); + input.set_text(&LlamaCppLanguageModelProvider::api_url(cx), window, cx); + input + }); + + let context_window_editor = cx.new(|cx| { + let input = InputField::new(window, cx, "8192").label("Context Window"); + if let Some(context_window) = LlamaCppLanguageModelProvider::settings(cx).context_window + { + input.set_text(&context_window.to_string(), window, cx); + } + input + }); + + cx.observe(&state, |_, _, cx| { + cx.notify(); + }) + .detach(); + + Self { + api_key_editor, + api_url_editor, + context_window_editor, + state, + } + } + + fn retry_connection(&mut self, window: &mut Window, cx: &mut Context) { + let has_api_url = LlamaCppLanguageModelProvider::has_custom_url(cx); + let has_api_key = self + .state + .read_with(cx, |state, _| state.api_key_state.has_key()); + if !has_api_url { + self.save_api_url(cx); + } + if !has_api_key { + self.save_api_key(&Default::default(), window, cx); + } + + self.state.update(cx, |state, cx| { + state.restart_fetch_models_task(cx); + }); + } + + fn save_api_key(&mut self, _: &menu::Confirm, window: &mut Window, cx: &mut Context) { + let api_key = self.api_key_editor.read(cx).text(cx).trim().to_string(); + if api_key.is_empty() { + return; + } + + // A URL change can cause the editor to be shown again. + self.api_key_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + + let state = self.state.clone(); + cx.spawn_in(window, async move |_, cx| { + state + .update(cx, |state, cx| state.set_api_key(Some(api_key), cx)) + .await + }) + .detach_and_log_err(cx); + } + + fn reset_api_key(&mut self, window: &mut Window, cx: &mut Context) { + self.api_key_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + + let state = self.state.clone(); + cx.spawn_in(window, async move |_, cx| { + state + .update(cx, |state, cx| state.set_api_key(None, cx)) + .await + }) + .detach_and_log_err(cx); + + cx.notify(); + } + + fn save_api_url(&self, cx: &mut Context) { + let api_url = self.api_url_editor.read(cx).text(cx).trim().to_string(); + let current_url = LlamaCppLanguageModelProvider::api_url(cx); + if !api_url.is_empty() && &api_url != ¤t_url { + let fs = ::global(cx); + update_settings_file(fs, cx, move |settings, _| { + settings + .language_models + .get_or_insert_default() + .llama_cpp + .get_or_insert_default() + .api_url = Some(api_url); + }); + } + } + + fn reset_api_url(&mut self, window: &mut Window, cx: &mut Context) { + self.api_url_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + let fs = ::global(cx); + update_settings_file(fs, cx, |settings, _cx| { + if let Some(settings) = settings + .language_models + .as_mut() + .and_then(|models| models.llama_cpp.as_mut()) + { + settings.api_url = Some(LLAMA_CPP_API_URL.into()); + } + }); + cx.notify(); + } + + fn save_context_window(&mut self, cx: &mut Context) { + let context_window_str = self + .context_window_editor + .read(cx) + .text(cx) + .trim() + .to_string(); + let current_context_window = LlamaCppLanguageModelProvider::settings(cx).context_window; + + if let Ok(context_window) = context_window_str.parse::() { + if Some(context_window) != current_context_window { + let fs = ::global(cx); + update_settings_file(fs, cx, move |settings, _| { + settings + .language_models + .get_or_insert_default() + .llama_cpp + .get_or_insert_default() + .context_window = Some(context_window); + }); + } + } else if context_window_str.is_empty() && current_context_window.is_some() { + let fs = ::global(cx); + update_settings_file(fs, cx, move |settings, _| { + settings + .language_models + .get_or_insert_default() + .llama_cpp + .get_or_insert_default() + .context_window = None; + }); + } + } + + fn reset_context_window(&mut self, window: &mut Window, cx: &mut Context) { + self.context_window_editor + .update(cx, |input, cx| input.set_text("", window, cx)); + let fs = ::global(cx); + update_settings_file(fs, cx, |settings, _cx| { + if let Some(settings) = settings + .language_models + .as_mut() + .and_then(|models| models.llama_cpp.as_mut()) + { + settings.context_window = None; + } + }); + cx.notify(); + } + + fn render_instructions(cx: &App) -> Div { + v_flex() + .gap_2() + .child(Label::new( + "Run open models locally with llama.cpp's built-in server, or connect to a \ + remote llama.cpp server.", + )) + .child(Label::new("To use a local llama.cpp server:")) + .child( + List::new() + .child( + ListBulletItem::new("") + .child(Label::new("Install llama.cpp from")) + .child(ButtonLink::new("llama.app", LLAMA_CPP_DOWNLOAD_URL)), + ) + .child( + ListBulletItem::new("") + .child(Label::new("Start the server in router mode:")) + .child(Label::new("llama serve").inline_code(cx)), + ) + .child(ListBulletItem::new( + "Click 'Connect' below to start using llama.cpp in Zed", + )), + ) + .child(Label::new( + "Alternatively, you can connect to a remote llama.cpp server by specifying its \ + URL and API key (set with --api-key, may not be required):", + )) + } + + fn render_api_key_editor(&self, cx: &Context) -> impl IntoElement { + let state = self.state.read(cx); + let env_var_set = state.api_key_state.is_from_env_var(); + let configured_card_label = if env_var_set { + format!("API key set in {API_KEY_ENV_VAR_NAME} environment variable.") + } else { + "API key configured".to_string() + }; + + if !state.api_key_state.has_key() { + v_flex() + .on_action(cx.listener(Self::save_api_key)) + .child(self.api_key_editor.clone()) + .child( + Label::new(format!( + "You can also set the {API_KEY_ENV_VAR_NAME} environment variable and restart Zed." + )) + .size(LabelSize::Small) + .color(Color::Muted), + ) + .into_any_element() + } else { + ConfiguredApiCard::new(configured_card_label) + .disabled(env_var_set) + .on_click(cx.listener(|this, _, window, cx| this.reset_api_key(window, cx))) + .when(env_var_set, |this| { + this.tooltip_label(format!( + "To reset your API key, unset the {API_KEY_ENV_VAR_NAME} environment variable." + )) + }) + .into_any_element() + } + } + + fn render_context_window_editor(&self, cx: &Context) -> Div { + let settings = LlamaCppLanguageModelProvider::settings(cx); + let custom_context_window_set = settings.context_window.is_some(); + + if custom_context_window_set { + h_flex() + .p_3() + .justify_between() + .rounded_md() + .border_1() + .border_color(cx.theme().colors().border) + .bg(cx.theme().colors().elevated_surface_background) + .child( + h_flex() + .gap_2() + .child(Icon::new(IconName::Check).color(Color::Success)) + .child(v_flex().gap_1().child(Label::new(format!( + "Context Window: {}", + settings.context_window.unwrap_or_default() + )))), + ) + .child( + Button::new("reset-context-window", "Reset") + .label_size(LabelSize::Small) + .start_icon(Icon::new(IconName::Undo).size(IconSize::Small)) + .layer(ElevationIndex::ModalSurface) + .on_click( + cx.listener(|this, _, window, cx| { + this.reset_context_window(window, cx) + }), + ), + ) + } else { + v_flex() + .on_action( + cx.listener(|this, _: &menu::Confirm, _window, cx| { + this.save_context_window(cx) + }), + ) + .child(self.context_window_editor.clone()) + .child( + Label::new("Default: discovered from the server") + .size(LabelSize::Small) + .color(Color::Muted), + ) + } + } + + fn render_api_url_editor(&self, cx: &Context) -> Div { + let api_url = LlamaCppLanguageModelProvider::api_url(cx); + let custom_api_url_set = api_url != LLAMA_CPP_API_URL; + + if custom_api_url_set { + h_flex() + .p_3() + .justify_between() + .rounded_md() + .border_1() + .border_color(cx.theme().colors().border) + .bg(cx.theme().colors().elevated_surface_background) + .child( + h_flex() + .gap_2() + .child(Icon::new(IconName::Check).color(Color::Success)) + .child(v_flex().gap_1().child(Label::new(api_url))), + ) + .child( + Button::new("reset-api-url", "Reset API URL") + .label_size(LabelSize::Small) + .start_icon(Icon::new(IconName::Undo).size(IconSize::Small)) + .layer(ElevationIndex::ModalSurface) + .on_click( + cx.listener(|this, _, window, cx| this.reset_api_url(window, cx)), + ), + ) + } else { + v_flex() + .on_action(cx.listener(|this, _: &menu::Confirm, _window, cx| { + this.save_api_url(cx); + cx.notify(); + })) + .gap_2() + .child(self.api_url_editor.clone()) + } + } +} + +impl Render for ConfigurationView { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let is_authenticated = self.state.read(cx).is_authenticated(); + + v_flex() + .gap_2() + .child(Self::render_instructions(cx)) + .child(self.render_api_url_editor(cx)) + .child(self.render_context_window_editor(cx)) + .child(self.render_api_key_editor(cx)) + .child( + h_flex() + .w_full() + .justify_between() + .gap_2() + .child( + h_flex() + .w_full() + .gap_2() + .map(|this| { + if is_authenticated { + this.child( + Button::new("llama-cpp-webui", "Open WebUI") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| { + let url = + LlamaCppLanguageModelProvider::api_url(cx); + cx.open_url(&url); + }) + .into_any_element(), + ) + .child( + Button::new("llama-cpp-site", "llama.cpp") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| { + cx.open_url(LLAMA_CPP_DOWNLOAD_URL) + }) + .into_any_element(), + ) + } else { + this.child( + Button::new("download_llama_cpp_button", "Get llama.cpp") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| { + cx.open_url(LLAMA_CPP_DOWNLOAD_URL) + }) + .into_any_element(), + ) + } + }) + .child( + Button::new("view-models", "Browse GGUF Models") + .style(ButtonStyle::Subtle) + .end_icon( + Icon::new(IconName::ArrowUpRight) + .size(IconSize::XSmall) + .color(Color::Muted), + ) + .on_click(move |_, _, cx| cx.open_url(LLAMA_CPP_MODELS_URL)), + ), + ) + .map(|this| { + if is_authenticated { + this.child( + ButtonLike::new("connected") + .disabled(true) + .cursor_style(CursorStyle::Arrow) + .child( + h_flex() + .gap_2() + .child(Icon::new(IconName::Check).color(Color::Success)) + .child(Label::new("Connected")) + .into_any_element(), + ) + .child( + IconButton::new("refresh-models", IconName::RotateCcw) + .tooltip(Tooltip::text("Refresh Models")) + .on_click(cx.listener(|this, _, window, cx| { + this.state.update(cx, |state, _| { + state.fetched_models.clear(); + }); + this.retry_connection(window, cx); + })), + ), + ) + } else { + this.child( + Button::new("retry_llama_cpp_models", "Connect") + .start_icon( + Icon::new(IconName::PlayOutlined).size(IconSize::XSmall), + ) + .on_click(cx.listener(move |this, _, window, cx| { + this.retry_connection(window, cx) + })), + ) + } + }), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use gpui::TestAppContext; + use http_client::FakeHttpClient; + use parking_lot::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FakeCredentialsProvider { + api_key: Vec, + } + + impl CredentialsProvider for FakeCredentialsProvider { + fn read_credentials<'a>( + &'a self, + _url: &'a str, + _cx: &'a AsyncApp, + ) -> std::pin::Pin< + Box)>>> + 'a>, + > { + let api_key = self.api_key.clone(); + Box::pin(async move { Ok(Some(("Bearer".to_string(), api_key))) }) + } + + fn write_credentials<'a>( + &'a self, + _url: &'a str, + _username: &'a str, + _password: &'a [u8], + _cx: &'a AsyncApp, + ) -> std::pin::Pin> + 'a>> { + Box::pin(async { Ok(()) }) + } + + fn delete_credentials<'a>( + &'a self, + _url: &'a str, + _cx: &'a AsyncApp, + ) -> std::pin::Pin> + 'a>> { + Box::pin(async { Ok(()) }) + } + } + + fn entry(id: &str, n_ctx: Option, n_ctx_train: Option) -> ModelEntry { + ModelEntry { + id: id.to_string(), + meta: Some(llama_cpp::ModelMeta { n_ctx, n_ctx_train }), + architecture: None, + status: None, + } + } + + #[test] + fn display_name_strips_path_and_extension() { + assert_eq!( + display_name_for("../models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"), + "Qwen2.5-Coder-7B-Instruct-Q4_K_M" + ); + assert_eq!(display_name_for("my-alias"), "my-alias"); + } + + #[test] + fn telemetry_id_strips_local_model_paths() { + assert_eq!( + telemetry_id_for("/Users/alice/models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"), + "llama_cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M" + ); + assert_eq!( + telemetry_id_for(r"C:\Users\alice\models\Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"), + "llama_cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M" + ); + assert_eq!(telemetry_id_for("my-alias"), "llama_cpp/my-alias"); + } + + #[test] + fn model_uses_props_then_meta_for_context() { + let props = Props { + default_generation_settings: Some(llama_cpp::GenerationSettings { n_ctx: Some(8192) }), + modalities: Some(llama_cpp::Modalities { vision: true }), + chat_template_caps: Some(llama_cpp::ChatTemplateCaps { + supports_tool_calls: true, + supports_preserve_reasoning: true, + ..Default::default() + }), + }; + // /props wins when present. + let model = model_from_entry(&entry("m", Some(4096), Some(131072)), Some(&props)); + assert_eq!(model.max_tokens, 8192); + assert!(model.supports_tools); + assert!(model.supports_images); + assert!(model.supports_thinking); + + // Unprobed: falls back to the listing's runtime context, then trained + // context. Tools are assumed supported until the model loads. + let model = model_from_entry(&entry("m", Some(4096), Some(131072)), None); + assert_eq!(model.max_tokens, 4096); + assert!(model.supports_tools); + assert!(!model.supports_thinking); + + let model = model_from_entry(&entry("m", None, Some(131072)), None); + assert_eq!(model.max_tokens, 131072); + + // Nothing reported -> the optimistic unloaded-context assumption. + let model = model_from_entry(&entry("m", None, None), None); + assert_eq!(model.max_tokens, ASSUMED_UNLOADED_CONTEXT); + assert!(model.supports_tools); + } + + #[test] + fn router_entry_detects_vision_from_modalities() { + let router_entry = ModelEntry { + id: "vlm".to_string(), + meta: None, + architecture: Some(llama_cpp::Architecture { + input_modalities: vec!["text".to_string(), "image".to_string()], + }), + status: Some(llama_cpp::ModelStatus { + value: "unloaded".to_string(), + }), + }; + let model = model_from_entry(&router_entry, None); + assert!(model.supports_images); + // Unprobed router models optimistically advertise tools until loaded. + assert!(model.supports_tools); + } + + #[test] + fn settings_override_capabilities_and_context() { + let mut models: HashMap = HashMap::default(); + models.insert( + "qwen".to_string(), + llama_cpp::Model::new("qwen", Some("qwen"), Some(8192), false, false, false), + ); + + let available = vec![AvailableModel { + name: "qwen".to_string(), + display_name: Some("Qwen Coder".to_string()), + max_tokens: 16384, + supports_tools: Some(true), + supports_images: None, + supports_thinking: Some(true), + }]; + + merge_settings_into_models(&mut models, &available, None); + + let model = models.get("qwen").unwrap(); + assert_eq!(model.display_name.as_deref(), Some("Qwen Coder")); + assert_eq!(model.max_tokens, 16384); + assert!(model.supports_tools); + assert!(model.supports_thinking); + // Unspecified capability keeps the discovered value. + assert!(!model.supports_images); + } + + #[test] + fn capability_cells_update_when_a_model_loads() { + let cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default())); + let settings = LlamaCppSettings { + auto_discover: true, + ..Default::default() + }; + + // Cold: the optimistic unloaded-context assumption. + let cold = vec![llama_cpp::Model::new( + "m", + Some("m"), + Some(ASSUMED_UNLOADED_CONTEXT), + true, + false, + false, + )]; + sync_capability_cells(&cells, &compute_effective_models(&cold, &settings)); + assert_eq!( + cells.read().unwrap().get("m").unwrap().max_tokens, + ASSUMED_UNLOADED_CONTEXT + ); + + // The model loads and reports its real context. The shared map must + // reflect the new value so a model reading it by name (an open + // conversation) is no longer stuck on the cold-start assumption. + let loaded = vec![llama_cpp::Model::new( + "m", + Some("m"), + Some(262_144), + true, + false, + true, + )]; + sync_capability_cells(&cells, &compute_effective_models(&loaded, &settings)); + assert_eq!(cells.read().unwrap().get("m").unwrap().max_tokens, 262_144); + assert!(cells.read().unwrap().get("m").unwrap().supports_thinking); + } + + #[test] + fn request_preserves_assistant_thinking_when_supported() { + let request = build_llama_cpp_request( + "test-model", + false, + LiveCapabilities { + max_tokens: 8192, + supports_tools: false, + supports_thinking: true, + }, + LanguageModelRequest { + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![ + MessageContent::Thinking { + text: "reasoning".to_string(), + signature: None, + }, + MessageContent::Text("answer".to_string()), + ], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }, + ); + + assert_eq!(request.messages.len(), 1); + match &request.messages[0] { + llama_cpp::ChatMessage::Assistant { + content: Some(llama_cpp::MessageContent::Plain(content)), + reasoning_content: Some(reasoning_content), + tool_calls, + } => { + assert_eq!(content, "answer"); + assert_eq!(reasoning_content, "reasoning"); + assert!(tool_calls.is_empty()); + } + message => panic!("unexpected message: {message:?}"), + } + } + + #[test] + fn request_skips_assistant_thinking_when_unsupported() { + let request = build_llama_cpp_request( + "test-model", + false, + LiveCapabilities { + max_tokens: 8192, + supports_tools: false, + supports_thinking: false, + }, + LanguageModelRequest { + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![ + MessageContent::Thinking { + text: "reasoning".to_string(), + signature: None, + }, + MessageContent::RedactedThinking("encrypted".to_string()), + MessageContent::Text("answer".to_string()), + ], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }, + ); + + assert_eq!(request.messages.len(), 1); + match &request.messages[0] { + llama_cpp::ChatMessage::Assistant { + content: Some(llama_cpp::MessageContent::Plain(content)), + reasoning_content, + tool_calls, + } => { + assert_eq!(content, "answer"); + assert_eq!(reasoning_content, &None); + assert!(tool_calls.is_empty()); + } + message => panic!("unexpected message: {message:?}"), + } + } + + #[test] + fn request_preserves_thinking_for_assistant_tool_calls_when_supported() { + let request = build_llama_cpp_request( + "test-model", + false, + LiveCapabilities { + max_tokens: 8192, + supports_tools: true, + supports_thinking: true, + }, + LanguageModelRequest { + messages: vec![language_model::LanguageModelRequestMessage { + role: Role::Assistant, + content: vec![ + MessageContent::Thinking { + text: "reasoning".to_string(), + signature: None, + }, + MessageContent::ToolUse(LanguageModelToolUse { + id: "call_1".into(), + name: "weather".into(), + raw_input: r#"{"city":"Oslo"}"#.to_string(), + input: serde_json::json!({ "city": "Oslo" }), + is_input_complete: true, + thought_signature: None, + }), + ], + cache: false, + reasoning_details: None, + }], + ..Default::default() + }, + ); + + assert_eq!(request.messages.len(), 1); + match &request.messages[0] { + llama_cpp::ChatMessage::Assistant { + content: None, + reasoning_content: Some(reasoning_content), + tool_calls, + } => { + assert_eq!(reasoning_content, "reasoning"); + assert_eq!(tool_calls.len(), 1); + } + message => panic!("unexpected message: {message:?}"), + } + } + + #[test] + fn usage_event_precedes_stop_event() { + let mut mapper = LlamaCppEventMapper::new(); + let events = mapper.map_event(llama_cpp::ResponseStreamEvent { + model: "test-model".to_string(), + object: "chat.completion.chunk".to_string(), + choices: vec![llama_cpp::ChoiceDelta { + index: 0, + delta: llama_cpp::ResponseMessageDelta { + content: None, + reasoning_content: None, + tool_calls: None, + }, + finish_reason: Some("stop".to_string()), + }], + usage: Some(llama_cpp::Usage { + prompt_tokens: 11, + completion_tokens: 7, + total_tokens: 18, + }), + }); + + assert!(matches!( + events.as_slice(), + [ + Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { + input_tokens: 11, + output_tokens: 7, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + })), + Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)), + ] + )); + } + + #[test] + fn usage_event_precedes_tool_use_stop_event() { + let mut mapper = LlamaCppEventMapper::new(); + let events = mapper.map_event(llama_cpp::ResponseStreamEvent { + model: "test-model".to_string(), + object: "chat.completion.chunk".to_string(), + choices: vec![llama_cpp::ChoiceDelta { + index: 0, + delta: llama_cpp::ResponseMessageDelta { + content: None, + reasoning_content: None, + tool_calls: Some(vec![llama_cpp::ToolCallChunk { + index: 0, + id: Some("tool-call-id".to_string()), + function: Some(llama_cpp::FunctionChunk { + name: Some("test_tool".to_string()), + arguments: Some(r#"{"value":1}"#.to_string()), + }), + }]), + }, + finish_reason: Some("tool_calls".to_string()), + }], + usage: Some(llama_cpp::Usage { + prompt_tokens: 13, + completion_tokens: 5, + total_tokens: 18, + }), + }); + + assert!(matches!( + events.as_slice(), + [ + Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage { + input_tokens: 13, + output_tokens: 5, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + })), + Ok(LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse { + id, + name, + .. + })), + Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)), + ] if id.to_string() == "tool-call-id" && name.as_ref() == "test_tool" + )); + } + + #[gpui::test] + async fn authenticate_fetches_models_after_loading_api_key(cx: &mut TestAppContext) { + cx.update(|cx| { + let settings_store = SettingsStore::test(cx); + cx.set_global(settings_store); + }); + + let model_request_authorizations = Arc::new(Mutex::new(Vec::new())); + let model_request_count = Arc::new(AtomicUsize::new(0)); + let http_client = FakeHttpClient::create({ + let model_request_authorizations = model_request_authorizations.clone(); + let model_request_count = model_request_count.clone(); + move |request| { + let model_request_authorizations = model_request_authorizations.clone(); + let model_request_count = model_request_count.clone(); + async move { + let path = request.uri().path(); + let authorization = request + .headers() + .get("Authorization") + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + + if path == "/v1/models" { + model_request_authorizations.lock().push(authorization); + let request_index = model_request_count.fetch_add(1, Ordering::SeqCst); + if request_index == 0 { + return Ok(http_client::Response::builder() + .status(503) + .body(http_client::AsyncBody::from("not ready"))?); + } + + return Ok(http_client::Response::builder().status(200).body( + http_client::AsyncBody::from( + r#"{"data":[{"id":"test-model","meta":{"n_ctx":4096}}]}"#, + ), + )?); + } + + if path == "/props" { + return Ok(http_client::Response::builder() + .status(200) + .body(http_client::AsyncBody::from("{}"))?); + } + + Ok(http_client::Response::builder() + .status(404) + .body(http_client::AsyncBody::default())?) + } + } + }); + let credentials_provider = Arc::new(FakeCredentialsProvider { + api_key: b"loaded-key".to_vec(), + }); + let provider = cx + .update(|cx| LlamaCppLanguageModelProvider::new(http_client, credentials_provider, cx)); + + cx.run_until_parked(); + + let result = cx.update(|cx| provider.authenticate(cx)).await; + assert!( + result.is_ok(), + "authenticate should discover models after loading credentials" + ); + assert_eq!( + &*model_request_authorizations.lock(), + &[None, Some("Bearer loaded-key".to_string())] + ); + } +} diff --git a/crates/language_models/src/settings.rs b/crates/language_models/src/settings.rs index d2fc9f59a2..1ce7caa3a6 100644 --- a/crates/language_models/src/settings.rs +++ b/crates/language_models/src/settings.rs @@ -6,10 +6,11 @@ use settings::RegisterSetting; use crate::provider::{ anthropic, anthropic::AnthropicSettings, anthropic_compatible::AnthropicCompatibleSettings, bedrock, bedrock::AmazonBedrockSettings, cloud::ZedDotDevSettings, deepseek::DeepSeekSettings, - google::GoogleSettings, lmstudio::LmStudioSettings, mistral, mistral::MistralSettings, - ollama::OllamaSettings, open_ai::OpenAiSettings, open_ai_compatible::OpenAiCompatibleSettings, - open_router, open_router::OpenRouterSettings, opencode, opencode::OpenCodeSettings, - resolve_custom_headers, vercel_ai_gateway::VercelAiGatewaySettings, x_ai::XAiSettings, + google::GoogleSettings, llama_cpp::LlamaCppSettings, lmstudio::LmStudioSettings, mistral, + mistral::MistralSettings, ollama::OllamaSettings, open_ai::OpenAiSettings, + open_ai_compatible::OpenAiCompatibleSettings, open_router, open_router::OpenRouterSettings, + opencode, opencode::OpenCodeSettings, resolve_custom_headers, + vercel_ai_gateway::VercelAiGatewaySettings, x_ai::XAiSettings, }; #[derive(Debug, RegisterSetting)] @@ -19,6 +20,7 @@ pub struct AllLanguageModelSettings { pub bedrock: AmazonBedrockSettings, pub deepseek: DeepSeekSettings, pub google: GoogleSettings, + pub llama_cpp: LlamaCppSettings, pub lmstudio: LmStudioSettings, pub mistral: MistralSettings, pub ollama: OllamaSettings, @@ -52,6 +54,7 @@ impl settings::Settings for AllLanguageModelSettings { let bedrock = language_models.bedrock.unwrap(); let deepseek = language_models.deepseek.unwrap(); let google = language_models.google.unwrap(); + let llama_cpp = language_models.llama_cpp.unwrap(); let lmstudio = language_models.lmstudio.unwrap(); let mistral = language_models.mistral.unwrap(); let ollama = language_models.ollama.unwrap(); @@ -116,6 +119,13 @@ impl settings::Settings for AllLanguageModelSettings { available_models: google.available_models.unwrap_or_default(), custom_headers: custom_headers_from("Google AI", google.custom_headers, &[]), }, + llama_cpp: LlamaCppSettings { + api_url: llama_cpp.api_url.unwrap(), + auto_discover: llama_cpp.auto_discover.unwrap_or(true), + available_models: llama_cpp.available_models.unwrap_or_default(), + context_window: llama_cpp.context_window, + custom_headers: custom_headers_from("llama.cpp", llama_cpp.custom_headers, &[]), + }, lmstudio: LmStudioSettings { api_url: lmstudio.api_url.unwrap(), available_models: lmstudio.available_models.unwrap_or_default(), diff --git a/crates/llama_cpp/Cargo.toml b/crates/llama_cpp/Cargo.toml new file mode 100644 index 0000000000..c77c2dd817 --- /dev/null +++ b/crates/llama_cpp/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "llama_cpp" +version = "0.1.0" +edition.workspace = true +publish.workspace = true +license = "GPL-3.0-or-later" + +[lints] +workspace = true + +[lib] +path = "src/llama_cpp.rs" + +[features] +default = [] +schemars = ["dep:schemars"] + +[dependencies] +anyhow.workspace = true +futures.workspace = true +http_client.workspace = true +schemars = { workspace = true, optional = true } +serde.workspace = true +serde_json.workspace = true +url.workspace = true diff --git a/crates/llama_cpp/LICENSE-GPL b/crates/llama_cpp/LICENSE-GPL new file mode 120000 index 0000000000..89e542f750 --- /dev/null +++ b/crates/llama_cpp/LICENSE-GPL @@ -0,0 +1 @@ +../../LICENSE-GPL \ No newline at end of file diff --git a/crates/llama_cpp/src/llama_cpp.rs b/crates/llama_cpp/src/llama_cpp.rs new file mode 100644 index 0000000000..5d5238c4b1 --- /dev/null +++ b/crates/llama_cpp/src/llama_cpp.rs @@ -0,0 +1,911 @@ +use anyhow::{Context as _, Result, anyhow}; +use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream}; +use http_client::{ + AsyncBody, CustomHeaders, HttpClient, HttpRequestExt, Method, Request as HttpRequest, + RequestBuilderExt, http, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use url::Url; + +pub const LLAMA_CPP_API_URL: &str = "http://localhost:8080"; + +const DEFAULT_CONTEXT_LENGTH: u64 = 4096; + +/// A model exposed to the rest of Zed, after merging API discovery with +/// user-configured overrides. +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct Model { + pub name: String, + pub display_name: Option, + pub max_tokens: u64, + pub supports_tools: bool, + pub supports_images: bool, + pub supports_thinking: bool, +} + +impl Model { + pub fn new( + name: &str, + display_name: Option<&str>, + max_tokens: Option, + supports_tools: bool, + supports_images: bool, + supports_thinking: bool, + ) -> Self { + Self { + name: name.to_owned(), + display_name: display_name.map(ToString::to_string), + max_tokens: max_tokens.unwrap_or(DEFAULT_CONTEXT_LENGTH), + supports_tools, + supports_images, + supports_thinking, + } + } + + pub fn display_name(&self) -> &str { + self.display_name.as_deref().unwrap_or(&self.name) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ToolChoice { + Auto, + Required, + None, +} + +#[derive(Clone, Deserialize, Serialize, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolDefinition { + Function { function: FunctionDefinition }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FunctionDefinition { + pub name: String, + pub description: Option, + pub parameters: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "role", rename_all = "lowercase")] +pub enum ChatMessage { + Assistant { + #[serde(default)] + content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tool_calls: Vec, + }, + User { + content: MessageContent, + }, + System { + content: MessageContent, + }, + Tool { + content: MessageContent, + tool_call_id: String, + }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +#[serde(untagged)] +pub enum MessageContent { + Plain(String), + Multipart(Vec), +} + +impl MessageContent { + pub fn push_part(&mut self, part: MessagePart) { + match self { + MessageContent::Plain(text) => { + *self = + MessageContent::Multipart(vec![MessagePart::Text { text: text.clone() }, part]); + } + MessageContent::Multipart(parts) if parts.is_empty() => match part { + MessagePart::Text { text } => *self = MessageContent::Plain(text), + MessagePart::Image { .. } => *self = MessageContent::Multipart(vec![part]), + }, + MessageContent::Multipart(parts) => parts.push(part), + } + } +} + +impl From> for MessageContent { + fn from(mut parts: Vec) -> Self { + if let [MessagePart::Text { text }] = parts.as_mut_slice() { + MessageContent::Plain(std::mem::take(text)) + } else { + MessageContent::Multipart(parts) + } + } +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MessagePart { + Text { + text: String, + }, + #[serde(rename = "image_url")] + Image { + image_url: ImageUrl, + }, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] +pub struct ImageUrl { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct ToolCall { + pub id: String, + #[serde(flatten)] + pub content: ToolCallContent, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum ToolCallContent { + Function { function: FunctionContent }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct FunctionContent { + pub name: String, + pub arguments: String, +} + +#[derive(Serialize, Debug)] +pub struct ChatCompletionRequest { + pub model: String, + pub messages: Vec, + pub stream: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tools: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, +} + +/// Asks the server to include a final `usage` chunk in the stream. +#[derive(Serialize, Debug)] +pub struct StreamOptions { + pub include_usage: bool, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct Usage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct LlamaCppError { + pub message: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +pub enum ResponseStreamResult { + Ok(ResponseStreamEvent), + Err { error: LlamaCppError }, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ResponseStreamEvent { + pub model: String, + pub object: String, + pub choices: Vec, + pub usage: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ChoiceDelta { + pub index: u32, + pub delta: ResponseMessageDelta, + pub finish_reason: Option, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct ResponseMessageDelta { + pub content: Option, + /// `llama-server` emits reasoning as a dedicated `reasoning_content` field + /// when started with a reasoning format (e.g. `--reasoning-format deepseek`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct ToolCallChunk { + pub index: usize, + pub id: Option, + pub function: Option, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct FunctionChunk { + pub name: Option, + pub arguments: Option, +} + +/// Response of `GET /v1/models`. +/// +/// In single-model mode `data` has exactly one entry describing the loaded +/// model; in router mode it lists every model the server knows about. +#[derive(Deserialize, Debug)] +pub struct ListModelsResponse { + #[serde(default)] + pub data: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModelEntry { + pub id: String, + /// Present in single-model mode; carries context information. + #[serde(default)] + pub meta: Option, + /// Present in router mode; reports the modalities the model accepts. + #[serde(default)] + pub architecture: Option, + /// Present in router mode; reports whether the model is currently loaded. + #[serde(default)] + pub status: Option, +} + +impl ModelEntry { + /// Whether this entry came from a server running in router mode. + pub fn is_router_entry(&self) -> bool { + self.status.is_some() + } + + /// Whether the model is loaded and can be probed for capabilities without + /// triggering a (potentially expensive) load. + pub fn is_loaded(&self) -> bool { + self.status + .as_ref() + .is_some_and(|status| status.value == "loaded") + } + + /// Whether the model is currently loading, so a progress label applies. + pub fn is_loading(&self) -> bool { + self.status + .as_ref() + .is_some_and(|status| status.value == "loading") + } + + pub fn supports_images_hint(&self) -> bool { + self.architecture + .as_ref() + .is_some_and(|architecture| architecture.input_modalities.iter().any(|m| m == "image")) + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ModelMeta { + /// The runtime per-slot context size. + #[serde(default)] + pub n_ctx: Option, + /// The context size the model was trained with. + #[serde(default)] + pub n_ctx_train: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct Architecture { + #[serde(default)] + pub input_modalities: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModelStatus { + /// One of `loaded`, `loading`, `unloaded`, `downloading`, `downloaded`, `sleeping`. + pub value: String, +} + +/// Response of `GET /props`, describing the loaded model. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct Props { + #[serde(default)] + pub default_generation_settings: Option, + #[serde(default)] + pub modalities: Option, + #[serde(default)] + pub chat_template_caps: Option, +} + +impl Props { + /// The runtime context length the loaded model was configured with. + pub fn context_length(&self) -> Option { + self.default_generation_settings + .as_ref() + .and_then(|settings| settings.n_ctx) + .filter(|n_ctx| *n_ctx > 0) + } + + pub fn supports_images(&self) -> bool { + self.modalities + .as_ref() + .is_some_and(|modalities| modalities.vision) + } + + pub fn supports_tools(&self) -> bool { + self.chat_template_caps + .as_ref() + .is_some_and(|caps| caps.supports_tool_calls || caps.supports_tools) + } + + pub fn supports_thinking(&self) -> bool { + self.chat_template_caps + .as_ref() + .is_some_and(|caps| caps.supports_preserve_reasoning) + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct GenerationSettings { + #[serde(default)] + pub n_ctx: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct Modalities { + #[serde(default)] + pub vision: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ChatTemplateCaps { + /// llama.cpp's `/props` reports both of these keys, so we read them into + /// separate fields (aliasing one to the other would be a serde duplicate-field + /// error) and treat either being true as tool support. `supports_tools` is the + /// older name; some builds report only one of the two. + #[serde(default)] + pub supports_tool_calls: bool, + #[serde(default)] + pub supports_tools: bool, + #[serde(default)] + pub supports_preserve_reasoning: bool, +} + +/// An event from the router's `/models/sse` feed, which the provider subscribes +/// to so model capabilities stay current as models load and unload. `model` is +/// `*` for events that aren't about a single model (e.g. the list reloading). +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModelEvent { + #[serde(default)] + pub model: String, + pub event: String, + #[serde(default)] + pub data: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ModelEventData { + #[serde(default)] + pub status: Option, + /// Present on an `unloaded` status; non-zero means the model failed to load. + #[serde(default)] + pub exit_code: Option, + /// Present on a `loading` status, reporting per-stage load progress. + #[serde(default)] + pub progress: Option, +} + +/// Per-stage load progress carried by a `loading` event. A model loads its +/// stages in order (the text model, plus an optional draft and/or multimodal +/// projector), each reporting a `0.0..=1.0` fraction. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct LoadProgress { + #[serde(default)] + pub stages: Vec, + #[serde(default)] + pub current: String, + #[serde(default)] + pub value: f32, +} + +impl LoadProgress { + /// A human label for the stage currently loading, matching the llama.cpp + /// WebUI's labels. We report the current stage's own `value` rather than a + /// blended overall percentage (as the WebUI does), so each stage runs + /// `0→100%` and the label says which stage it is. + pub fn stage_label(&self) -> &'static str { + match self.current.as_str() { + "text_model" => "Loading weights", + "spec_model" => "Loading draft", + "mmproj_model" => "Loading projector", + _ => "Loading", + } + } + + /// The full load-status label shown in the model selector, e.g. + /// `"Loading weights 42%"`: the current stage plus its own progress rounded + /// to a percentage. + pub fn progress_label(&self) -> String { + format!( + "{} {}%", + self.stage_label(), + (self.value * 100.0).round() as u32 + ) + } +} + +impl ModelEvent { + /// Whether this event means the set of models or their loaded state changed, + /// so the provider should re-run discovery. Intermediate `loading` progress + /// ticks return `false` — only terminal load/unload and list changes matter. + pub fn changes_model_state(&self) -> bool { + match self.event.as_str() { + "models_reload" | "model_remove" => true, + _ => matches!( + self.data.as_ref().and_then(|data| data.status.as_deref()), + Some("loaded" | "unloaded") + ), + } + } + + /// The non-zero exit code of a failed load, if this event reports one. + pub fn load_failure(&self) -> Option { + let data = self.data.as_ref()?; + if data.status.as_deref() == Some("unloaded") { + data.exit_code.filter(|code| *code != 0) + } else { + None + } + } + + /// This event's load progress if it is a loading event carrying usable stage + /// data, else `None`. + pub fn load_progress(&self) -> Option<&LoadProgress> { + let data = self.data.as_ref()?; + if data.status.as_deref() != Some("loading") { + return None; + } + let progress = data.progress.as_ref()?; + // The server also emits bare stage-transition markers (e.g. + // `{"stage": "mmproj_model"}`) with no `stages`/`current`/`value`. Skip + // them so the indicator holds its last value rather than dropping to 0%. + if progress.stages.is_empty() || progress.current.is_empty() { + return None; + } + Some(progress) + } +} + +pub async fn stream_chat_completion( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + request: ChatCompletionRequest, + extra_headers: &CustomHeaders, +) -> Result>> { + let uri = format!("{api_url}/v1/chat/completions"); + let request_builder = http::Request::builder() + .method(Method::POST) + .uri(uri) + .header("Content-Type", "application/json") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }); + + let request = request_builder + .extra_headers(extra_headers) + .body(AsyncBody::from(serde_json::to_string(&request)?))?; + let mut response = client.send(request).await?; + if response.status().is_success() { + let reader = BufReader::new(response.into_body()); + Ok(reader + .lines() + .filter_map(|line| async move { + match line { + Ok(line) => { + let line = line.strip_prefix("data: ")?; + if line == "[DONE]" { + None + } else { + match serde_json::from_str(line) { + Ok(ResponseStreamResult::Ok(response)) => Some(Ok(response)), + Ok(ResponseStreamResult::Err { error }) => { + Some(Err(anyhow!(error.message))) + } + Err(error) => Some(Err(anyhow!(error))), + } + } + } + Err(error) => Some(Err(anyhow!(error))), + } + }) + .boxed()) + } else { + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + anyhow::bail!( + "Failed to connect to llama.cpp API: {} {}", + response.status(), + body, + ); + } +} + +/// Lists the models the server is serving via `GET /v1/models`. +pub async fn get_models( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + extra_headers: &CustomHeaders, +) -> Result> { + let uri = format!("{api_url}/v1/models"); + let request = HttpRequest::builder() + .method(Method::GET) + .uri(uri) + .header("Accept", "application/json") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }) + .extra_headers(extra_headers) + .body(AsyncBody::default())?; + + let mut response = client.send(request).await?; + + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + + anyhow::ensure!( + response.status().is_success(), + "Failed to connect to llama.cpp API: {} {}", + response.status(), + body, + ); + let response: ListModelsResponse = + serde_json::from_str(&body).context("Unable to parse llama.cpp models response")?; + Ok(response.data) +} + +/// Fetches `GET /props` to discover the loaded model's capabilities. +/// +/// In router mode `model` selects which model instance to query; passing +/// `None` queries the single loaded model in single-model mode. +pub async fn get_props( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + model: Option<&str>, + extra_headers: &CustomHeaders, +) -> Result { + // Router-mode `/props` takes a `model` query parameter selecting which + // instance to describe. Model ids contain `/` and `:` (e.g. + // `unsloth/Qwen3.5-2B-GGUF:Q4_1`), so the value is URL-encoded. + let uri = match model { + Some(model) => Url::parse_with_params(&format!("{api_url}/props"), [("model", model)]) + .context("invalid llama.cpp API URL")? + .to_string(), + None => format!("{api_url}/props"), + }; + let request = HttpRequest::builder() + .method(Method::GET) + .uri(uri) + .header("Accept", "application/json") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }) + .extra_headers(extra_headers) + .body(AsyncBody::default())?; + + let mut response = client.send(request).await?; + + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + + anyhow::ensure!( + response.status().is_success(), + "Failed to connect to llama.cpp API: {} {}", + response.status(), + body, + ); + let props: Props = + serde_json::from_str(&body).context("Unable to parse llama.cpp props response")?; + Ok(props) +} + +/// Opens the router's `GET /models/sse` event stream. Each item is one parsed +/// event; the stream ends when the connection closes. Only available on builds +/// that expose `/models/sse` (router mode). +pub async fn stream_model_events( + client: &dyn HttpClient, + api_url: &str, + api_key: Option<&str>, + extra_headers: &CustomHeaders, +) -> Result>> { + let uri = format!("{api_url}/models/sse"); + let request = HttpRequest::builder() + .method(Method::GET) + .uri(uri) + .header("Accept", "text/event-stream") + .when_some(api_key, |builder, api_key| { + builder.header("Authorization", format!("Bearer {api_key}")) + }) + .extra_headers(extra_headers) + .body(AsyncBody::default())?; + + let mut response = client.send(request).await?; + if !response.status().is_success() { + let mut body = String::new(); + response.body_mut().read_to_string(&mut body).await?; + anyhow::bail!( + "Failed to open llama.cpp model event stream: {} {}", + response.status(), + body, + ); + } + + let reader = BufReader::new(response.into_body()); + Ok(reader + .lines() + .filter_map(|line| async move { + // Each event is a single `data:` line carrying the JSON envelope; + // other SSE lines (comments, blank separators) are ignored. + let line = line.ok()?; + let payload = line.strip_prefix("data:")?.trim_start(); + Some(serde_json::from_str::(payload).map_err(|error| anyhow!(error))) + }) + .boxed()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_single_model_listing() { + let response = serde_json::json!({ + "object": "list", + "data": [ + { + "id": "../models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", + "object": "model", + "created": 1735142223u64, + "owned_by": "llamacpp", + "meta": { + "n_ctx": 8192, + "n_ctx_train": 131072, + "n_embd": 4096, + } + } + ] + }); + let response: ListModelsResponse = serde_json::from_value(response).unwrap(); + assert_eq!(response.data.len(), 1); + let entry = &response.data[0]; + assert!(!entry.is_router_entry()); + assert_eq!(entry.meta.as_ref().unwrap().n_ctx, Some(8192)); + assert_eq!(entry.meta.as_ref().unwrap().n_ctx_train, Some(131072)); + } + + #[test] + fn parse_router_model_listing() { + let response = serde_json::json!({ + "object": "list", + "data": [ + { + "id": "qwen2.5-coder", + "object": "model", + "owned_by": "llamacpp", + "status": { "value": "loaded", "args": [] }, + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"] + } + }, + { + "id": "gemma-3", + "object": "model", + "owned_by": "llamacpp", + "status": { "value": "unloaded" }, + "architecture": { "input_modalities": ["text"] } + } + ] + }); + let response: ListModelsResponse = serde_json::from_value(response).unwrap(); + assert_eq!(response.data.len(), 2); + + let loaded = &response.data[0]; + assert!(loaded.is_router_entry()); + assert!(loaded.is_loaded()); + assert!(loaded.supports_images_hint()); + + let unloaded = &response.data[1]; + assert!(unloaded.is_router_entry()); + assert!(!unloaded.is_loaded()); + assert!(!unloaded.supports_images_hint()); + } + + #[test] + fn parse_props() { + let response = serde_json::json!({ + "default_generation_settings": { + "id": 0, + "n_ctx": 8192, + "params": {} + }, + "total_slots": 1, + "model_path": "../models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", + "modalities": { "vision": true, "audio": false }, + // The real `/props` reports both keys; parsing must not treat that as + // a duplicate field. + "chat_template_caps": { + "supports_tools": true, + "supports_tool_calls": true, + "supports_preserve_reasoning": true, + "supports_system_role": true + } + }); + let props: Props = serde_json::from_value(response).unwrap(); + assert_eq!(props.context_length(), Some(8192)); + assert!(props.supports_images()); + assert!(props.supports_tools()); + assert!(props.supports_thinking()); + } + + fn model_event(value: serde_json::Value) -> ModelEvent { + serde_json::from_value(value).unwrap() + } + + #[test] + fn model_event_state_changes() { + // Terminal load/unload and list changes warrant re-discovery. + assert!( + model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "loaded" } + })) + .changes_model_state() + ); + assert!( + model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "unloaded" } + })) + .changes_model_state() + ); + assert!( + model_event(serde_json::json!({ "model": "*", "event": "models_reload" })) + .changes_model_state() + ); + assert!( + model_event(serde_json::json!({ "model": "m", "event": "model_remove" })) + .changes_model_state() + ); + + // Intermediate loading-progress ticks do not. + assert!( + !model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", "progress": { "value": 0.4 } } + })) + .changes_model_state() + ); + assert!( + !model_event(serde_json::json!({ "model": "m", "event": "download_progress" })) + .changes_model_state() + ); + } + + #[test] + fn model_event_load_failure() { + let failed = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "unloaded", "exit_code": 1 } + })); + assert_eq!(failed.load_failure(), Some(1)); + + // A clean unload (exit code 0 or absent) is not a failure. + let clean = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "unloaded", "exit_code": 0 } + })); + assert_eq!(clean.load_failure(), None); + let loaded = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "loaded" } + })); + assert_eq!(loaded.load_failure(), None); + } + + #[test] + fn model_event_load_progress() { + // We report the current stage's own value (0→1), not a blended overall. + let weights = model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", + "progress": { "stages": ["text_model"], "current": "text_model", "value": 0.4 } } + })); + let progress = weights.load_progress().unwrap(); + assert!((progress.value - 0.4).abs() < 1e-4); + assert_eq!(progress.stage_label(), "Loading weights"); + assert_eq!(progress.progress_label(), "Loading weights 40%"); + + // The projector stage runs 0→1 on its own (not 90→100%). + let projector = model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", + "progress": { "stages": ["text_model", "mmproj_model"], + "current": "mmproj_model", "value": 0.5 } } + })); + let progress = projector.load_progress().unwrap(); + assert!((progress.value - 0.5).abs() < 1e-4); + assert_eq!(progress.stage_label(), "Loading projector"); + + // Non-loading events carry no progress. + let loaded = model_event(serde_json::json!({ + "model": "m", "event": "status_change", "data": { "status": "loaded" } + })); + assert!(loaded.load_progress().is_none()); + + // Bare stage-transition markers (no stages/current/value) are skipped so + // the indicator doesn't flicker to 0% between stages. + let transition = model_event(serde_json::json!({ + "model": "m", "event": "status_change", + "data": { "status": "loading", "progress": { "stage": "mmproj_model" } } + })); + assert!(transition.load_progress().is_none()); + } + + #[test] + fn parse_streaming_reasoning_and_tool_calls() { + let event = serde_json::json!({ + "model": "llama", + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": null, + "reasoning_content": "thinking...", + "tool_calls": [ + { + "index": 0, + "id": "call_1", + "function": { "name": "weather", "arguments": "{\"city\":" } + } + ] + }, + "finish_reason": null + } + ] + }); + let event: ResponseStreamEvent = serde_json::from_value(event).unwrap(); + let delta = &event.choices[0].delta; + assert_eq!(delta.reasoning_content.as_deref(), Some("thinking...")); + assert_eq!(delta.tool_calls.as_ref().unwrap().len(), 1); + } + + #[test] + fn encodes_router_model_id_for_props_query() { + // Router ids contain `/` and `:`, which must be URL-encoded in the + // `?model=` query, while unreserved characters (`.`, `-`) stay literal. + let url = Url::parse_with_params( + "http://localhost:8080/props", + [("model", "unsloth/Qwen3.5-2B-GGUF:Q4_1")], + ) + .unwrap(); + assert_eq!( + url.as_str(), + "http://localhost:8080/props?model=unsloth%2FQwen3.5-2B-GGUF%3AQ4_1" + ); + } +} diff --git a/crates/settings_content/src/language_model.rs b/crates/settings_content/src/language_model.rs index a673ecc7a8..ab11097728 100644 --- a/crates/settings_content/src/language_model.rs +++ b/crates/settings_content/src/language_model.rs @@ -15,6 +15,7 @@ pub struct AllLanguageModelSettingsContent { pub bedrock: Option, pub deepseek: Option, pub google: Option, + pub llama_cpp: Option, pub lmstudio: Option, pub mistral: Option, pub ollama: Option, @@ -262,6 +263,36 @@ pub struct LmStudioAvailableModel { pub supports_images: bool, } +#[with_fallible_options] +#[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema, MergeFrom)] +pub struct LlamaCppSettingsContent { + pub api_url: Option, + /// Whether to automatically discover models served by the llama.cpp server. + /// Defaults to true. + pub auto_discover: Option, + pub available_models: Option>, + /// Overrides the context length reported for every llama.cpp model. + pub context_window: Option, + pub custom_headers: Option>, +} + +#[with_fallible_options] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema, MergeFrom)] +pub struct LlamaCppAvailableModel { + /// The model id reported by the llama.cpp server (its `--alias` or the model file path). + pub name: String, + /// The model's name in Zed's UI, such as in the model selector dropdown menu in the agent panel. + pub display_name: Option, + /// The Context Length parameter to the model (aka n_ctx). + pub max_tokens: u64, + /// Whether the model supports tools. + pub supports_tools: Option, + /// Whether the model supports vision. + pub supports_images: Option, + /// Whether the model emits reasoning/thinking content. + pub supports_thinking: Option, +} + #[with_fallible_options] #[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema, MergeFrom)] pub struct DeepseekSettingsContent { diff --git a/docs/src/ai/by-company.md b/docs/src/ai/by-company.md index 5eb3d27a7c..e477bdf6b2 100644 --- a/docs/src/ai/by-company.md +++ b/docs/src/ai/by-company.md @@ -97,8 +97,9 @@ Paid DeepSeek usage is API access in Zed, not subscription sign-in. | Tool | Support level | What you get | Account / billing | Setup | | --------------------------------- | ----------------- | -------------------------------------- | ----------------- | ------------------------------------------------------------- | -| Ollama | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#ollama) | +| llama.cpp | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#llama-cpp) | | LM Studio | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#lm-studio) | +| Ollama | Configured in Zed | Local models for Zed AI features | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#ollama) | | Local OpenAI-compatible server | Configured in Zed | Local or self-hosted model endpoint | Local/self-hosted | [Use a Local Model](./use-a-local-model.md#openai-compatible) | | Local/self-hosted edit prediction | Configured in Zed | Edit predictions from a local provider | Local/self-hosted | [Edit Prediction](./edit-prediction.md) | diff --git a/docs/src/ai/use-a-local-model.md b/docs/src/ai/use-a-local-model.md index 7cd7836b69..89984d10f3 100644 --- a/docs/src/ai/use-a-local-model.md +++ b/docs/src/ai/use-a-local-model.md @@ -1,6 +1,6 @@ --- title: Use a Local Model - Zed -description: Configure Ollama, LM Studio, local OpenAI-compatible servers, and local edit prediction in Zed. +description: Configure llama.cpp, Ollama, LM Studio, local OpenAI-compatible servers, and local edit prediction in Zed. --- # Use a Local Model @@ -9,11 +9,69 @@ Use local models when you run the model on your machine or on infrastructure you | Local path | Zed AI features | External Agents | Terminal Threads | Notes | | --------------------------------- | -------------------- | --------------- | ---------------- | -------------------------------------------------- | -| Ollama | Yes | Separate config | Separate config | Configure Ollama for Zed AI features | +| llama.cpp | Yes | Separate config | Separate config | Configure a llama.cpp server for Zed AI features | | LM Studio | Yes | Separate config | Separate config | Configure LM Studio for Zed AI features | +| Ollama | Yes | Separate config | Separate config | Configure Ollama for Zed AI features | | Local OpenAI-compatible server | Yes | Separate config | Separate config | Configure base URL, model, and key if needed | | Local/self-hosted edit prediction | Edit Prediction only | No | No | Uses [Edit Prediction](./edit-prediction.md) setup | +## llama.cpp {#llama-cpp} + +Use [llama.cpp](https://llama.app) and its built-in server for local models with Zed Agent, Inline Assistant, and similar model-backed Zed AI features. + +1. Install llama.cpp from [llama.app](https://llama.app). +2. Start the server in [router mode](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md): + + ```sh + llama serve + ``` + + It loads models from the llama.cpp cache on demand. To download and run a specific model in one step, pass `-hf`: + + ```sh + llama serve -hf unsloth/gemma-4-26B-A4B-it-GGUF:BF16 + ``` + +3. In Zed, select a llama.cpp model from the model dropdown. + +Zed automatically discovers the served models with their context length and tool/vision capabilities. In router mode these are refined once a model loads, via the server's `/models/sse` stream (which requires a recent llama.cpp build). To list models yourself instead, set `auto_discover` to `false`: + +```json [settings] +{ + "language_models": { + "llama_cpp": { + "api_url": "http://localhost:8080", + "auto_discover": false, + "available_models": [ + { + "name": "gemma-4-12b-it-GGUF:BF16", + "display_name": "gemma-4-12b-it-GGUF:BF16", + "max_tokens": 32768, + "supports_tools": true, + "supports_images": false + } + ] + } + } +} +``` + +### llama.cpp Context Length {#llama-cpp-context} + +Zed uses the context length the server reports (`/props`). Override it for all models with `context_window`, or per model with `max_tokens` in `available_models`: + +```json [settings] +{ + "language_models": { + "llama_cpp": { + "context_window": 8192 + } + } +} +``` + +If your llama.cpp server requires a key, enter it in the provider UI or set `LLAMACPP_API_KEY`. For a remote server, set the API URL to its endpoint and provide the key (set on the server with `--api-key`). + ## Ollama {#ollama} Use Ollama for local models with Zed Agent, Inline Assistant, and similar model-backed Zed AI features.