From e7b4a54c531ebfc144c4a558b0eca52b31234f21 Mon Sep 17 00:00:00 2001 From: Finn Evers Date: Wed, 19 Aug 2026 16:26:14 +0000 Subject: [PATCH] language_core: Represent supported queries using an enum (#62707) Refactors our query loading to instead have an enum of supported queries and load the query files themselves in parallel, where possible. We will use this to warn extension authors when unsupported queries are present, as we no longer want to ship those as part of the extension. It also now enforces one file per supported query - only one extension currently utilizes this feature and was undocumented previously, so I decided in favor of removing it here, since this will help with enforcing extensions to just ship query files we actually support in the future. The extension has also been migrated so that it does not break with this effort Side-effect of this change is that we now load both embedded and external languages up to at least 10%/5% faster, since the files are now read in parallel as opposed to sequentially one by one. Release Notes: - N/A --- Cargo.lock | 2 + crates/extension/src/extension_host_proxy.rs | 6 +- crates/extension_host/Cargo.toml | 1 - crates/extension_host/src/extension_host.rs | 157 ++++++++----- .../src/extension_store_test.rs | 38 +++- crates/extension_host/src/headless_host.rs | 20 +- crates/grammars/Cargo.toml | 2 +- crates/grammars/src/grammars.rs | 37 +-- crates/language/src/available_languages.rs | 7 +- crates/language/src/language.rs | 5 +- crates/language/src/language_registry.rs | 32 ++- crates/language_core/Cargo.toml | 2 + crates/language_core/src/language_core.rs | 2 +- crates/language_core/src/queries.rs | 215 ++++++++++++++++-- .../src/language_extension.rs | 4 +- crates/languages/src/lib.rs | 22 +- 16 files changed, 418 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84a6a2d41a..fcbb281d43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9811,6 +9811,7 @@ name = "language_core" version = "0.1.0" dependencies = [ "anyhow", + "bitflags 2.13.1", "collections", "gpui_shared_string", "gpui_util", @@ -9821,6 +9822,7 @@ dependencies = [ "schemars 1.0.4", "serde", "serde_json", + "strum 0.27.2", "toml 0.8.23", "tree-sitter", ] diff --git a/crates/extension/src/extension_host_proxy.rs b/crates/extension/src/extension_host_proxy.rs index 1329dd4fe4..77fd57696d 100644 --- a/crates/extension/src/extension_host_proxy.rs +++ b/crates/extension/src/extension_host_proxy.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use anyhow::Result; use fs::Fs; use gpui::{App, Global, ReadGlobal, SharedString, Task}; -use language::{BinaryStatus, LanguageMatcher, LanguageName, LoadedLanguage}; +use language::{BinaryStatus, LanguageLoader, LanguageMatcher, LanguageName}; use lsp::LanguageServerName; use parking_lot::RwLock; @@ -234,7 +234,7 @@ pub trait ExtensionLanguageProxy: Send + Sync + 'static { grammar: Option>, matcher: Arc, hidden: bool, - load: Arc Result + Send + Sync + 'static>, + load: LanguageLoader, ); fn remove_languages( @@ -252,7 +252,7 @@ impl ExtensionLanguageProxy for ExtensionHostProxy { grammar: Option>, matcher: Arc, hidden: bool, - load: Arc Result + Send + Sync + 'static>, + load: LanguageLoader, ) { let Some(proxy) = self.language_proxy.read().clone() else { return; diff --git a/crates/extension_host/Cargo.toml b/crates/extension_host/Cargo.toml index 8dd949844f..1895b8b6ec 100644 --- a/crates/extension_host/Cargo.toml +++ b/crates/extension_host/Cargo.toml @@ -65,7 +65,6 @@ language = { workspace = true, features = ["test-support"] } language_extension.workspace = true parking_lot.workspace = true project = { workspace = true, features = ["test-support"] } - reqwest_client.workspace = true theme = { workspace = true, features = ["test-support"] } theme_settings.workspace = true diff --git a/crates/extension_host/src/extension_host.rs b/crates/extension_host/src/extension_host.rs index 21f5b3d369..48547167de 100644 --- a/crates/extension_host/src/extension_host.rs +++ b/crates/extension_host/src/extension_host.rs @@ -36,8 +36,8 @@ use gpui::{ }; use http_client::{AsyncBody, HttpClient, HttpClientWithUrl}; use language::{ - LanguageConfig, LanguageMatcher, LanguageName, LanguageQueries, LoadedLanguage, - QUERY_FILENAME_PREFIXES, Rope, + LanguageConfig, LanguageMatcher, LanguageName, LanguageQueries, LoadedLanguage, QueryFile, + QueryFileContents, QueryFiles, Rope, }; use node_runtime::NodeRuntime; use project::ContextProviderWithTasks; @@ -50,6 +50,7 @@ use std::ops::RangeInclusive; use std::str::FromStr; use std::sync::LazyLock; use std::{ + borrow::Cow, cmp::Ordering, path::{self, Path, PathBuf}, sync::Arc, @@ -252,6 +253,8 @@ pub struct ExtensionIndexLanguageEntry { pub matcher: Arc, pub hidden: bool, pub grammar: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query_files: Option, } actions!( @@ -1411,26 +1414,15 @@ impl ExtensionStore { language.grammar.clone(), language.matcher.clone(), language.hidden, - Arc::new(move || { - let config = - LanguageConfig::load(language_path.join(LanguageConfig::FILE_NAME))?; - let queries = load_plugin_queries(&language_path); - let context_provider = - std::fs::read_to_string(language_path.join(TaskTemplates::FILE_NAME)) - .ok() - .and_then(|contents| { - let definitions = - serde_json_lenient::from_str(&contents).log_err()?; - Some(Arc::new(ContextProviderWithTasks::new(definitions)) as Arc<_>) - }); - - Ok(LoadedLanguage { - config, - queries, - context_provider, - toolchain_provider: None, - manifest_name: None, - }) + Arc::new({ + let fs = self.fs.clone(); + let query_files = language.query_files; + move || { + let fs = fs.clone(); + let language_path = language_path.clone(); + async move { load_plugin_language(fs, &language_path, query_files).await } + .boxed() + } }), ); } @@ -1668,11 +1660,22 @@ impl ExtensionStore { if !fs_metadata.is_dir { continue; } - let language_config_path = language_path.join(LanguageConfig::FILE_NAME); - let config = fs.load(&language_config_path).await.with_context(|| { - format!("loading language config from {language_config_path:?}") - })?; - let config = ::toml::from_str::(&config)?; + let config = { + let fs = fs.clone(); + let language_config_path = language_path.join(LanguageConfig::FILE_NAME); + async move { + let config = fs.load(&language_config_path).await.with_context(|| { + format!("loading language config from {language_config_path:?}") + })?; + ::toml::from_str::(&config).map_err(anyhow::Error::from) + } + }; + let query_files = async { + Ok(discover_query_files(fs.clone(), &language_path) + .await + .log_err()) + }; + let (config, query_files) = futures::try_join!(config, query_files)?; let relative_path = relative_path.to_rel_path_buf()?; if !extension_manifest.languages.contains(&relative_path) { @@ -1687,6 +1690,7 @@ impl ExtensionStore { matcher: config.matcher, hidden: config.hidden, grammar: config.grammar, + query_files, }, ); } @@ -1965,31 +1969,80 @@ impl ExtensionStore { } } -fn load_plugin_queries(root_path: &Path) -> LanguageQueries { - let mut result = LanguageQueries::default(); - if let Some(entries) = std::fs::read_dir(root_path).log_err() { - for entry in entries { - let Some(entry) = entry.log_err() else { - continue; - }; - let path = entry.path(); - if let Some(remainder) = path.strip_prefix(root_path).ok().and_then(|p| p.to_str()) { - if !remainder.ends_with(".scm") { - continue; - } - for (name, query) in QUERY_FILENAME_PREFIXES { - if remainder.starts_with(name) { - if let Some(contents) = std::fs::read_to_string(&path).log_err() { - match query(&mut result) { - None => *query(&mut result) = Some(contents.into()), - Some(r) => r.to_mut().push_str(contents.as_ref()), - } - } - break; - } - } - } +async fn load_plugin_language( + fs: Arc, + language_path: &Path, + query_files: Option, +) -> Result { + let config = { + let fs = fs.clone(); + let config_path = language_path.join(LanguageConfig::FILE_NAME); + async move { + let contents = fs.load(&config_path).await?; + toml::from_str::(&contents).map_err(anyhow::Error::from) } + }; + let context_provider = { + let fs = fs.clone(); + let tasks_path = language_path.join(TaskTemplates::FILE_NAME); + async move { + fs.load(&tasks_path).await.ok().and_then(|contents| { + serde_json_lenient::from_str(&contents) + .log_err() + .map(|definitions| { + Arc::new(ContextProviderWithTasks::new(definitions)) as Arc<_> + }) + }) + } + }; + let (config, queries, context_provider) = futures::try_join!( + config, + async move { Ok(load_plugin_queries(fs, &language_path, query_files).await) }, + async move { Ok(context_provider.await) } + )?; + + Ok(LoadedLanguage { + config, + queries, + context_provider, + toolchain_provider: None, + manifest_name: None, + }) +} + +async fn discover_query_files(fs: Arc, root_path: &Path) -> Result { + let mut paths = fs.read_dir(root_path).await?; + let mut query_files = QueryFiles::empty(); + while let Some(path) = paths.next().await { + let path = path?; + let Some(query_file) = path + .file_name() + .and_then(|file_name| file_name.to_str()) + .and_then(|file_name| file_name.parse::().ok()) + else { + continue; + }; + query_files.insert(query_file.into()); } - result + Ok(query_files) +} + +async fn load_plugin_queries( + fs: Arc, + root_path: &Path, + query_files: Option, +) -> LanguageQueries { + let query_files = query_files.unwrap_or_else(QueryFiles::all); + let files = join_all(query_files.query_files().map(|query_file| { + let fs = fs.clone(); + let path = root_path.join(query_file.file_name()); + async move { + fs.load(&path) + .await + .ok() + .map(|contents| QueryFileContents::new(query_file, Cow::Owned(contents))) + } + })) + .await; + LanguageQueries::from_files(files.into_iter().flatten()) } diff --git a/crates/extension_host/src/extension_store_test.rs b/crates/extension_host/src/extension_store_test.rs index 68b799c4fc..dd6470234c 100644 --- a/crates/extension_host/src/extension_store_test.rs +++ b/crates/extension_host/src/extension_store_test.rs @@ -1,7 +1,7 @@ use crate::{ Event, ExtensionIndex, ExtensionIndexEntry, ExtensionIndexLanguageEntry, ExtensionIndexThemeEntry, ExtensionManifest, ExtensionStore, GrammarManifestEntry, - RELOAD_DEBOUNCE_DURATION, SchemaVersion, + RELOAD_DEBOUNCE_DURATION, SchemaVersion, load_plugin_queries, }; use async_compression::futures::bufread::GzipEncoder; use collections::{BTreeMap, HashSet}; @@ -10,7 +10,7 @@ use fs::{FakeFs, Fs, RealFs}; use futures::{AsyncReadExt, FutureExt, StreamExt, io::BufReader}; use gpui::{AppContext as _, BackgroundExecutor, TaskExt, TestAppContext}; use http_client::{FakeHttpClient, Response}; -use language::{BinaryStatus, LanguageMatcher, LanguageName, LanguageRegistry}; +use language::{BinaryStatus, LanguageMatcher, LanguageName, LanguageRegistry, QueryFiles}; use language_extension::LspAccess; use lsp::LanguageServerName; use node_runtime::NodeRuntime; @@ -34,6 +34,36 @@ fn init_logger() { zlog::init_test(); } +#[gpui::test] +async fn test_load_plugin_queries(executor: BackgroundExecutor) { + let fs = FakeFs::new(executor); + fs.insert_tree( + "/queries", + json!({ + "highlights.scm": "highlight query", + "outline.scm": "outline query", + "highlights_extra.scm": "ignored query", + "locals.scm": "unrelated query", + }), + ) + .await; + + let queries = load_plugin_queries( + fs.clone(), + Path::new("/queries"), + Some(QueryFiles::HIGHLIGHTS), + ) + .await; + assert_eq!(queries.highlights.as_deref(), Some("highlight query")); + assert!(queries.outline.is_none()); + assert!(queries.brackets.is_none()); + + let queries = load_plugin_queries(fs, Path::new("/queries"), None).await; + assert_eq!(queries.highlights.as_deref(), Some("highlight query")); + assert_eq!(queries.outline.as_deref(), Some("outline query")); + assert!(queries.brackets.is_none()); +} + fn remote_sync_entry(id: &str, manifest_body: &str) -> ExtensionIndexEntry { let manifest = format!( r#" @@ -59,6 +89,7 @@ fn remote_sync_language_entry(extension: &str, path: &str) -> ExtensionIndexLang matcher: LanguageMatcher::default().into(), hidden: false, grammar: None, + query_files: None, } } @@ -297,6 +328,7 @@ async fn test_extension_store(cx: &mut TestAppContext) { path_suffixes = ["rb"] "#, "highlights.scm": "", + "outline.scm": "", }, "erb": { "config.toml": r#" @@ -399,6 +431,7 @@ async fn test_extension_store(cx: &mut TestAppContext) { ..LanguageMatcher::default() }) .into(), + query_files: Some(QueryFiles::HIGHLIGHTS), }, ), ( @@ -414,6 +447,7 @@ async fn test_extension_store(cx: &mut TestAppContext) { ..LanguageMatcher::default() }) .into(), + query_files: Some(QueryFiles::HIGHLIGHTS | QueryFiles::OUTLINE), }, ), ] diff --git a/crates/extension_host/src/headless_host.rs b/crates/extension_host/src/headless_host.rs index 725e8e571d..53ef6db2d6 100644 --- a/crates/extension_host/src/headless_host.rs +++ b/crates/extension_host/src/headless_host.rs @@ -8,7 +8,7 @@ use extension::{ ExtensionLanguageServerProxy, ExtensionManifest, }; use fs::{Fs, RemoveOptions, RenameOptions}; -use futures::future::join_all; +use futures::future::{FutureExt as _, join_all}; use gpui::{App, AppContext as _, AsyncApp, Context, Entity, Task, WeakEntity}; use http_client::HttpClient; use language::{LanguageConfig, LanguageName, LanguageQueries, LoadedLanguage}; @@ -157,13 +157,17 @@ impl HeadlessExtensionStore { config.matcher.clone(), config.hidden, Arc::new(move || { - Ok(LoadedLanguage { - config: config.clone(), - queries: LanguageQueries::default(), - context_provider: None, - toolchain_provider: None, - manifest_name: None, - }) + let config = config.clone(); + async move { + Ok(LoadedLanguage { + config, + queries: LanguageQueries::default(), + context_provider: None, + toolchain_provider: None, + manifest_name: None, + }) + } + .boxed() }), ); })?; diff --git a/crates/grammars/Cargo.toml b/crates/grammars/Cargo.toml index 9d37f6cf26..3cd75c8c2f 100644 --- a/crates/grammars/Cargo.toml +++ b/crates/grammars/Cargo.toml @@ -11,9 +11,9 @@ workspace = true path = "src/grammars.rs" [dependencies] +anyhow.workspace = true language_core.workspace = true rust-embed.workspace = true -anyhow.workspace = true toml.workspace = true tree-sitter = { workspace = true, optional = true } diff --git a/crates/grammars/src/grammars.rs b/crates/grammars/src/grammars.rs index e3c70c9dc3..b0bdc50b48 100644 --- a/crates/grammars/src/grammars.rs +++ b/crates/grammars/src/grammars.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use anyhow::Context as _; -use language_core::{LanguageConfig, LanguageQueries, QUERY_FILENAME_PREFIXES}; +use language_core::{LanguageConfig, LanguageQueries, QueryFile, QueryFileContents}; use rust_embed::RustEmbed; #[derive(RustEmbed)] @@ -83,30 +83,15 @@ pub fn get_file(path: &str) -> Option { GrammarDir::get(path) } -/// Load all `.scm` query files for a given language name into a `LanguageQueries`. -/// -/// Multiple `.scm` files with the same prefix (e.g. `highlights.scm` and -/// `highlights_extra.scm`) are concatenated together with their contents appended. +/// Load all Tree-sitter query files for a given language name. pub fn load_queries(name: &str) -> LanguageQueries { - let mut result = LanguageQueries::default(); - for path in GrammarDir::iter() { - if let Some(remainder) = path.strip_prefix(name).and_then(|p| p.strip_prefix('/')) { - if !remainder.ends_with(".scm") { - continue; - } - for (prefix, query) in QUERY_FILENAME_PREFIXES { - if remainder.starts_with(prefix) { - let contents = match GrammarDir::get(path.as_ref()).unwrap().data { - Cow::Borrowed(bytes) => Cow::Borrowed(std::str::from_utf8(bytes).unwrap()), - Cow::Owned(bytes) => Cow::Owned(String::from_utf8(bytes).unwrap()), - }; - match query(&mut result) { - None => *query(&mut result) = Some(contents), - Some(existing) => existing.to_mut().push_str(contents.as_ref()), - } - } - } - } - } - result + LanguageQueries::from_files(GrammarDir::iter().filter_map(|path| { + let file_name = path.strip_prefix(name)?.strip_prefix('/')?; + let query_file = file_name.parse::().ok()?; + let contents = match GrammarDir::get(path.as_ref())?.data { + Cow::Borrowed(bytes) => Cow::Borrowed(std::str::from_utf8(bytes).ok()?), + Cow::Owned(bytes) => Cow::Owned(String::from_utf8(bytes).ok()?), + }; + Some(QueryFileContents::new(query_file, contents)) + })) } diff --git a/crates/language/src/available_languages.rs b/crates/language/src/available_languages.rs index 236726b318..c68aa9dc0d 100644 --- a/crates/language/src/available_languages.rs +++ b/crates/language/src/available_languages.rs @@ -1,5 +1,4 @@ -use crate::{LanguageId, LanguageMatcher, LanguageName, LoadedLanguage, ManifestName}; -use anyhow::Result; +use crate::{LanguageId, LanguageLoader, LanguageMatcher, LanguageName, ManifestName}; use collections::FxHashMap; use globset::GlobSet; use smallvec::SmallVec; @@ -15,7 +14,7 @@ pub struct AvailableLanguage { pub(super) grammar: Option>, pub(super) matcher: Arc, pub(super) hidden: bool, - pub(super) load: Arc Result + 'static + Send + Sync>, + pub(super) load: LanguageLoader, pub(super) loaded: bool, pub(super) manifest_name: Option, } @@ -57,7 +56,7 @@ impl AvailableLanguages { matcher: Arc, hidden: bool, manifest_name: Option, - load: Arc Result + 'static + Send + Sync>, + load: LanguageLoader, ) -> bool { if let Some(existing_language) = self .0 diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index ea41bd6f0c..fadca37b97 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -58,7 +58,7 @@ pub use language_core::{ serialize_regex, }; pub use language_registry::{ - LanguageName, LanguageServerStatusUpdate, LoadedLanguage, ServerHealth, + LanguageLoader, LanguageName, LanguageServerStatusUpdate, LoadedLanguage, ServerHealth, }; use lsp::{ CodeActionKind, InitializeParams, LanguageServerBinary, LanguageServerBinaryOptions, Uri, @@ -106,7 +106,8 @@ pub use file_content::{ ByteContent, DecodedText, FILE_ANALYSIS_BYTES, analyze_byte_content, decode_text, encode_text, }; pub use language_registry::{ - BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QUERY_FILENAME_PREFIXES, + BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QueryFile, + QueryFileContents, QueryFiles, }; pub use lsp::{LanguageServerId, LanguageServerName}; pub use outline::*; diff --git a/crates/language/src/language_registry.rs b/crates/language/src/language_registry.rs index 2e4f7200a3..c58aca7b9c 100644 --- a/crates/language/src/language_registry.rs +++ b/crates/language/src/language_registry.rs @@ -8,14 +8,15 @@ use crate::{ use anyhow::{Context as _, Result, anyhow}; use collections::{FxHashMap, HashMap, HashSet, hash_map}; pub use language_core::{ - BinaryStatus, LanguageName, LanguageQueries, LanguageServerStatusUpdate, - QUERY_FILENAME_PREFIXES, ServerHealth, + BinaryStatus, LanguageName, LanguageQueries, LanguageServerStatusUpdate, QueryFile, + QueryFileContents, QueryFiles, ServerHealth, }; use settings::{AllLanguageSettingsContent, LanguageSettingsContent}; use futures::{ Future, channel::{mpsc, oneshot}, + future::{BoxFuture, FutureExt as _}, }; use globset::GlobSet; use gpui::{App, BackgroundExecutor}; @@ -101,6 +102,9 @@ pub struct LoadedLanguage { pub manifest_name: Option, } +pub type LanguageLoader = + Arc BoxFuture<'static, Result> + Send + Sync + 'static>; + impl LanguageRegistry { pub fn new(executor: BackgroundExecutor) -> Self { let this = Self { @@ -191,13 +195,17 @@ impl LanguageRegistry { config.hidden, None, Arc::new(move || { - Ok(LoadedLanguage { - config: config.clone(), - queries: Default::default(), - toolchain_provider: None, - context_provider: None, - manifest_name: None, - }) + let config = config.clone(); + async move { + Ok(LoadedLanguage { + config, + queries: Default::default(), + toolchain_provider: None, + context_provider: None, + manifest_name: None, + }) + } + .boxed() }), ) } @@ -374,7 +382,7 @@ impl LanguageRegistry { matcher: Arc, hidden: bool, manifest_name: Option, - load: Arc Result + 'static + Send + Sync>, + load: LanguageLoader, ) { let state = &mut *self.state.write(); @@ -459,7 +467,7 @@ impl LanguageRegistry { matcher: language.config.matcher.clone(), hidden: language.config.hidden, manifest_name: None, - load: Arc::new(|| Err(anyhow!("already loaded"))), + load: Arc::new(|| async { Err(anyhow!("already loaded")) }.boxed()), loaded: true, }); state.add(language); @@ -633,7 +641,7 @@ impl LanguageRegistry { self.executor .spawn(async move { let language = async { - let loaded_language = (language_load)()?; + let loaded_language = (language_load)().await?; if let Some(grammar) = loaded_language.config.grammar.clone() { let grammar = Some(this.get_or_load_grammar(grammar).await?); diff --git a/crates/language_core/Cargo.toml b/crates/language_core/Cargo.toml index 7d77847a9c..54ec3ce12d 100644 --- a/crates/language_core/Cargo.toml +++ b/crates/language_core/Cargo.toml @@ -9,6 +9,7 @@ path = "src/language_core.rs" [dependencies] anyhow.workspace = true +bitflags.workspace = true collections.workspace = true gpui_shared_string.workspace = true gpui_util.workspace = true @@ -19,6 +20,7 @@ regex.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +strum.workspace = true toml.workspace = true tree-sitter.workspace = true diff --git a/crates/language_core/src/language_core.rs b/crates/language_core/src/language_core.rs index 88d24867c9..f3cb3ab51c 100644 --- a/crates/language_core/src/language_core.rs +++ b/crates/language_core/src/language_core.rs @@ -31,5 +31,5 @@ pub use code_label::{CodeLabel, CodeLabelBuilder, Symbol, SymbolKind}; pub use language_name::{LanguageId, LanguageName}; pub use lsp_adapter::{BinaryStatus, LanguageServerStatusUpdate, ServerHealth}; pub use manifest::ManifestName; -pub use queries::{LanguageQueries, QUERY_FILENAME_PREFIXES}; +pub use queries::{LanguageQueries, QueryFile, QueryFileContents, QueryFiles}; pub use toolchain::{Toolchain, ToolchainList, ToolchainMetadata, ToolchainScope}; diff --git a/crates/language_core/src/queries.rs b/crates/language_core/src/queries.rs index 510fb2e03c..56d9ad81d5 100644 --- a/crates/language_core/src/queries.rs +++ b/crates/language_core/src/queries.rs @@ -1,19 +1,119 @@ use std::borrow::Cow; -pub type QueryFieldAccessor = fn(&mut LanguageQueries) -> &mut Option>; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use strum::IntoEnumIterator; -pub const QUERY_FILENAME_PREFIXES: &[(&str, QueryFieldAccessor)] = &[ - ("highlights", |q| &mut q.highlights), - ("brackets", |q| &mut q.brackets), - ("outline", |q| &mut q.outline), - ("indents", |q| &mut q.indents), - ("injections", |q| &mut q.injections), - ("overrides", |q| &mut q.overrides), - ("redactions", |q| &mut q.redactions), - ("runnables", |q| &mut q.runnables), - ("debugger", |q| &mut q.debugger), - ("textobjects", |q| &mut q.text_objects), -]; +#[derive( + Clone, Copy, Debug, Eq, PartialEq, strum::EnumIter, strum::EnumString, strum::IntoStaticStr, +)] +pub enum QueryFile { + #[strum(serialize = "highlights.scm")] + Highlights, + #[strum(serialize = "brackets.scm")] + Brackets, + #[strum(serialize = "outline.scm")] + Outline, + #[strum(serialize = "indents.scm")] + Indents, + #[strum(serialize = "injections.scm")] + Injections, + #[strum(serialize = "overrides.scm")] + Overrides, + #[strum(serialize = "redactions.scm")] + Redactions, + #[strum(serialize = "runnables.scm")] + Runnables, + #[strum(serialize = "debugger.scm")] + Debugger, + #[strum(serialize = "textobjects.scm")] + TextObjects, +} + +impl QueryFile { + pub fn file_name(self) -> &'static str { + self.into() + } +} + +bitflags::bitflags! { + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd, Hash)] + pub struct QueryFiles: u16 { + const HIGHLIGHTS = 1 << 0; + const BRACKETS = 1 << 1; + const OUTLINE = 1 << 2; + const INDENTS = 1 << 3; + const INJECTIONS = 1 << 4; + const OVERRIDES = 1 << 5; + const REDACTIONS = 1 << 6; + const RUNNABLES = 1 << 7; + const DEBUGGER = 1 << 8; + const TEXT_OBJECTS = 1 << 9; + } +} + +impl QueryFiles { + pub fn query_files(self) -> impl Iterator { + QueryFile::iter().filter(move |query_file| self.contains((*query_file).into())) + } +} + +impl From for QueryFiles { + fn from(query_file: QueryFile) -> Self { + match query_file { + QueryFile::Highlights => Self::HIGHLIGHTS, + QueryFile::Brackets => Self::BRACKETS, + QueryFile::Outline => Self::OUTLINE, + QueryFile::Indents => Self::INDENTS, + QueryFile::Injections => Self::INJECTIONS, + QueryFile::Overrides => Self::OVERRIDES, + QueryFile::Redactions => Self::REDACTIONS, + QueryFile::Runnables => Self::RUNNABLES, + QueryFile::Debugger => Self::DEBUGGER, + QueryFile::TextObjects => Self::TEXT_OBJECTS, + } + } +} + +impl FromIterator for QueryFiles { + fn from_iter>(query_files: T) -> Self { + query_files + .into_iter() + .fold(Self::empty(), |files, query_file| files | query_file.into()) + } +} + +impl Serialize for QueryFiles { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u16(self.bits()) + } +} + +impl<'de> Deserialize<'de> for QueryFiles { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Self::from_bits_retain(u16::deserialize(deserializer)?)) + } +} + +#[derive(Debug)] +pub struct QueryFileContents { + pub query_file: QueryFile, + pub contents: Cow<'static, str>, +} + +impl QueryFileContents { + pub fn new(query_file: QueryFile, contents: Cow<'static, str>) -> Self { + Self { + query_file, + contents, + } + } +} /// Tree-sitter language queries for a given language. #[derive(Debug, Default)] @@ -29,3 +129,92 @@ pub struct LanguageQueries { pub text_objects: Option>, pub debugger: Option>, } + +impl LanguageQueries { + pub fn from_files(files: impl IntoIterator) -> Self { + let mut queries = Self::default(); + for QueryFileContents { + query_file, + contents, + } in files + { + let field = match query_file { + QueryFile::Highlights => &mut queries.highlights, + QueryFile::Brackets => &mut queries.brackets, + QueryFile::Outline => &mut queries.outline, + QueryFile::Indents => &mut queries.indents, + QueryFile::Injections => &mut queries.injections, + QueryFile::Overrides => &mut queries.overrides, + QueryFile::Redactions => &mut queries.redactions, + QueryFile::Runnables => &mut queries.runnables, + QueryFile::Debugger => &mut queries.debugger, + QueryFile::TextObjects => &mut queries.text_objects, + }; + *field = Some(contents); + } + queries + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn query_files_have_unique_names_and_fields() { + let queries = LanguageQueries::from_files(QueryFile::iter().map(|query_file| { + QueryFileContents::new(query_file, Cow::Borrowed(query_file.into())) + })); + + assert_eq!(queries.highlights.as_deref(), Some("highlights.scm")); + assert_eq!(queries.brackets.as_deref(), Some("brackets.scm")); + assert_eq!(queries.outline.as_deref(), Some("outline.scm")); + assert_eq!(queries.indents.as_deref(), Some("indents.scm")); + assert_eq!(queries.injections.as_deref(), Some("injections.scm")); + assert_eq!(queries.overrides.as_deref(), Some("overrides.scm")); + assert_eq!(queries.redactions.as_deref(), Some("redactions.scm")); + assert_eq!(queries.runnables.as_deref(), Some("runnables.scm")); + assert_eq!(queries.debugger.as_deref(), Some("debugger.scm")); + assert_eq!(queries.text_objects.as_deref(), Some("textobjects.scm")); + + for query_file in QueryFile::iter() { + let file_name: &'static str = query_file.into(); + assert_eq!(file_name.parse(), Ok(query_file)); + } + assert_eq!( + "highlights_extra.scm".parse::(), + Err(strum::ParseError::VariantNotFound) + ); + } + + #[test] + fn query_files_bitflags_map_to_query_files() { + let all_query_files = QueryFile::iter().collect::(); + assert_eq!(all_query_files, QueryFiles::all()); + assert_eq!( + all_query_files.query_files().collect::>(), + QueryFile::iter().collect::>() + ); + + let selected = QueryFiles::from(QueryFile::Highlights) | QueryFile::Outline.into(); + assert_eq!( + selected.query_files().collect::>(), + vec![QueryFile::Highlights, QueryFile::Outline] + ); + } + + #[test] + fn query_files_serialize_as_bits() { + let query_files = QueryFiles::HIGHLIGHTS | QueryFiles::TEXT_OBJECTS; + let serialized = serde_json::to_string(&query_files).unwrap(); + assert_eq!(serialized, "513"); + assert_eq!( + serde_json::from_str::(&serialized).unwrap(), + query_files + ); + + let unknown_bit = 1 << 15; + let query_files = serde_json::from_str::(&unknown_bit.to_string()).unwrap(); + assert_eq!(query_files.bits(), unknown_bit); + } +} diff --git a/crates/language_extension/src/language_extension.rs b/crates/language_extension/src/language_extension.rs index 89246d538b..e7843138ea 100644 --- a/crates/language_extension/src/language_extension.rs +++ b/crates/language_extension/src/language_extension.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use anyhow::Result; use extension::{ExtensionGrammarProxy, ExtensionHostProxy, ExtensionLanguageProxy}; use gpui::{App, Entity, WeakEntity}; -use language::{LanguageMatcher, LanguageName, LanguageRegistry, LoadedLanguage}; +use language::{LanguageLoader, LanguageMatcher, LanguageName, LanguageRegistry}; use project::LspStore; #[derive(Clone)] @@ -50,7 +50,7 @@ impl ExtensionLanguageProxy for LanguageServerRegistryProxy { grammar: Option>, matcher: Arc, hidden: bool, - load: Arc Result + Send + Sync + 'static>, + load: LanguageLoader, ) { self.language_registry .register_language(language, grammar, matcher, hidden, None, load); diff --git a/crates/languages/src/lib.rs b/crates/languages/src/lib.rs index 2f0893fda4..b3993ee481 100644 --- a/crates/languages/src/lib.rs +++ b/crates/languages/src/lib.rs @@ -1,3 +1,4 @@ +use futures::FutureExt as _; use gpui::{App, SharedString, UpdateGlobal}; use node_runtime::NodeRuntime; use project::Fs; @@ -363,13 +364,20 @@ fn register_language( config.hidden, manifest_name.clone(), Arc::new(move || { - Ok(LoadedLanguage { - config: config.clone(), - queries: grammars::load_queries(name), - context_provider: context.clone(), - toolchain_provider: toolchain.clone(), - manifest_name: manifest_name.clone(), - }) + let config = config.clone(); + let context = context.clone(); + let toolchain = toolchain.clone(); + let manifest_name = manifest_name.clone(); + async move { + Ok(LoadedLanguage { + config, + queries: grammars::load_queries(name), + context_provider: context, + toolchain_provider: toolchain, + manifest_name, + }) + } + .boxed() }), ); }