editor: Add language detection for untitled buffers (#61201)
# Objective Closes #4868. Untitled buffers start as Plain Text and require users to select a language manually before receiving syntax highlighting. Add lightweight automatic language detection for code entered or pasted into untitled buffers. ## Solution This builds on [Max Stevens's earlier language-detection work](https://github.com/zed-industries/zed/pull/43057), replacing Magika with [Betlang](https://github.com/DioxusLabs/betlang). While researching smaller and faster alternatives to Magika, I came across Betlang, a recently introduced language detection library developed by DioxusLabs for dioxus-code. The fact that it comes from DioxusLabs gave me more confidence in evaluating this relatively new dependency for Zed. Betlang embeds an approximately 50 KB model, is MIT-licensed, and depends only on `fearless_simd`, making it well suited to Zed's cross-platform embedding requirements. Detection runs on the background executor with bounded input sampling. It is restricted to untitled buffers, skips content shorter than 20 bytes, and requires at least 50% confidence. These limits have worked well in local testing. In release builds on Linux with an Intel Core i5-13600KF, the `betlang::detect` call alone typically completes within 3 ms when processing the maximum sampled input. ## Testing I added a test for language detection in untitled buffers that covers both manually entered and pasted content. The test passes successfully. I also manually tested the feature to verify that the overall experience works well. ## Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content adheres to Zed's UI standards ([UX/UI](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) and [icon](https://github.com/zed-industries/zed/blob/main/crates/icons/README.md) guidelines) - [x] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable ## Showcase https://github.com/user-attachments/assets/96e28ad7-2968-4325-9aff-37fe813a2da7 --- Release Notes: - Added automatic language detection for untitled buffers. --------- Co-authored-by: Max Stevens <maxstevens2708@gmail.com>
This commit is contained in:
ᴀᴍᴛᴏᴀᴇʀandMax Stevens authored and GitHub committed
1 parent
4a1ed927ff
commit
a98c7e2fa8
8 files changed
+227
-3
No files matched your search
Generated
+25
@@ -2176,6 +2176,15 @@ dependencies = [
|
||||
"zed_actions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "betlang"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9a5811a1a59386e19785588e5bb2384b7d543b4870acedd1c7a1cb177b13b7d"
|
||||
dependencies = [
|
||||
"fearless_simd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bigdecimal"
|
||||
version = "0.4.8"
|
||||
@@ -5766,6 +5775,7 @@ dependencies = [
|
||||
"indoc",
|
||||
"itertools 0.14.0",
|
||||
"language",
|
||||
"language_detection",
|
||||
"languages",
|
||||
"linkify",
|
||||
"log",
|
||||
@@ -6555,6 +6565,12 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fearless_simd"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693"
|
||||
|
||||
[[package]]
|
||||
name = "feature_flags"
|
||||
version = "0.1.0"
|
||||
@@ -9771,6 +9787,15 @@ dependencies = [
|
||||
"tree-sitter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "language_detection"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"betlang",
|
||||
"gpui",
|
||||
"language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "language_extension"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -114,6 +114,7 @@ members = [
|
||||
"crates/keymap_editor",
|
||||
"crates/language",
|
||||
"crates/language_core",
|
||||
"crates/language_detection",
|
||||
"crates/language_extension",
|
||||
"crates/language_model",
|
||||
"crates/language_model_core",
|
||||
@@ -376,6 +377,7 @@ json_schema_store = { path = "crates/json_schema_store" }
|
||||
keymap_editor = { path = "crates/keymap_editor" }
|
||||
language = { path = "crates/language" }
|
||||
language_core = { path = "crates/language_core" }
|
||||
language_detection = { path = "crates/language_detection" }
|
||||
language_extension = { path = "crates/language_extension" }
|
||||
language_model = { path = "crates/language_model" }
|
||||
language_model_core = { path = "crates/language_model_core" }
|
||||
@@ -554,6 +556,7 @@ aws-smithy-runtime-api = { version = "1.9.2", features = ["http-1x", "client"] }
|
||||
aws-smithy-types = { version = "1.3.4", features = ["http-body-1-x"] }
|
||||
backtrace = "0.3"
|
||||
base64 = "0.22"
|
||||
betlang = "0.1.0"
|
||||
bitflags = "2.6.0"
|
||||
block2 = "0.6"
|
||||
brotli = "8.0.2"
|
||||
|
||||
@@ -56,6 +56,7 @@ indoc.workspace = true
|
||||
edit_prediction_types.workspace = true
|
||||
itertools.workspace = true
|
||||
language.workspace = true
|
||||
language_detection.workspace = true
|
||||
linkify.workspace = true
|
||||
log.workspace = true
|
||||
lsp.workspace = true
|
||||
|
||||
@@ -190,6 +190,7 @@ use language::{
|
||||
},
|
||||
point_from_lsp, point_to_lsp, text_diff_with_options,
|
||||
};
|
||||
use language_detection::detect_language;
|
||||
use linked_editing_ranges::refresh_linked_ranges;
|
||||
use lsp::{
|
||||
CodeActionKind, CompletionItemKind, CompletionTriggerKind, InsertTextFormat, InsertTextMode,
|
||||
@@ -296,6 +297,8 @@ const CURSOR_BLINK_INTERVAL: Duration = Duration::from_millis(500);
|
||||
const MAX_LINE_LEN: usize = 1024;
|
||||
const MIN_NAVIGATION_HISTORY_ROW_DELTA: i64 = 10;
|
||||
const MAX_SELECTION_HISTORY_LEN: usize = 1024;
|
||||
const MIN_LANGUAGE_DETECTION_LEN: usize = 20;
|
||||
const MIN_LANGUAGE_DETECTION_CONFIDENCE: f32 = 0.5;
|
||||
pub(crate) const CURSORS_VISIBLE_FOR: Duration = Duration::from_millis(2000);
|
||||
#[doc(hidden)]
|
||||
pub const CODE_ACTIONS_DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(250);
|
||||
@@ -9554,8 +9557,9 @@ impl Editor {
|
||||
cx.emit(EditorEvent::TitleChanged);
|
||||
}
|
||||
|
||||
let buffer_id = buffer.read(cx).remote_id();
|
||||
|
||||
if self.project.is_some() {
|
||||
let buffer_id = buffer.read(cx).remote_id();
|
||||
self.register_buffer(buffer_id, cx);
|
||||
self.update_lsp_data(Some(buffer_id), window, cx);
|
||||
self.refresh_inlay_hints(
|
||||
@@ -9563,6 +9567,8 @@ impl Editor {
|
||||
cx,
|
||||
);
|
||||
}
|
||||
|
||||
self.detect_buffer_language(buffer_id, cx);
|
||||
}
|
||||
|
||||
cx.emit(EditorEvent::BufferEdited);
|
||||
@@ -10816,6 +10822,46 @@ impl Editor {
|
||||
self.refresh_document_symbols(for_buffer, cx);
|
||||
}
|
||||
|
||||
fn detect_buffer_language(&self, buffer_id: BufferId, cx: &mut Context<Self>) {
|
||||
if DisableAiSettings::get_global(cx).disable_ai {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(buffer_entity) = self.buffer().read(cx).buffer(buffer_id) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let buffer = buffer_entity.read(cx);
|
||||
if buffer.file().is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
let buffer_snapshot = buffer.snapshot();
|
||||
if buffer_snapshot.len() < MIN_LANGUAGE_DETECTION_LEN {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(language_registry) = buffer.language_registry() else {
|
||||
return;
|
||||
};
|
||||
let buffer_version = buffer_snapshot.version().clone();
|
||||
let detected_language = detect_language(buffer_snapshot, language_registry, cx);
|
||||
|
||||
cx.spawn(async move |_, cx| {
|
||||
if let Some((detected_language, confidence)) = detected_language.await {
|
||||
if confidence < MIN_LANGUAGE_DETECTION_CONFIDENCE {
|
||||
return;
|
||||
}
|
||||
buffer_entity.update(cx, |buffer, cx| {
|
||||
if buffer.file().is_none() && !buffer.version().changed_since(&buffer_version) {
|
||||
buffer.set_language(Some(detected_language), cx);
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
}
|
||||
|
||||
fn register_visible_buffers(&mut self, cx: &mut Context<Self>) {
|
||||
if !self.lsp_data_enabled() {
|
||||
return;
|
||||
|
||||
@@ -35,8 +35,7 @@ use language::{
|
||||
tree_sitter_python,
|
||||
};
|
||||
use language_settings::Formatter;
|
||||
use languages::markdown_lang;
|
||||
use languages::rust_lang;
|
||||
use languages::{language, markdown_lang, rust_lang};
|
||||
use lsp::{CompletionParams, DEFAULT_LSP_REQUEST_TIMEOUT};
|
||||
use multi_buffer::{IndentGuide, MultiBuffer, MultiBufferOffset, MultiBufferOffsetUtf16, PathKey};
|
||||
use parking_lot::Mutex;
|
||||
@@ -9603,6 +9602,80 @@ async fn test_kill_ring_yank_pastes_accumulated_kill_at_each_cursor(cx: &mut Tes
|
||||
cx.assert_editor_state("aone\nˇ bone\nˇ");
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_editing_untitled_buffer_redetects_language(cx: &mut TestAppContext) {
|
||||
init_test(cx, |_| {});
|
||||
|
||||
let fs = FakeFs::new(cx.executor());
|
||||
let project = Project::test(fs, [], cx).await;
|
||||
let go_language = language("go", tree_sitter_go::LANGUAGE.into());
|
||||
project.read_with(cx, |project, _| {
|
||||
project.languages().add(rust_lang());
|
||||
project.languages().add(go_language.clone());
|
||||
});
|
||||
let buffer = project
|
||||
.update(cx, |project, cx| project.create_buffer(None, true, cx))
|
||||
.await
|
||||
.unwrap();
|
||||
let window = cx.add_window(|window, cx| {
|
||||
let editor = build_editor_with_project(
|
||||
project,
|
||||
MultiBuffer::build_from_buffer(buffer.clone(), cx),
|
||||
window,
|
||||
cx,
|
||||
);
|
||||
window.focus(&editor.focus_handle(cx), cx);
|
||||
editor
|
||||
});
|
||||
let editor = window.root(cx).unwrap();
|
||||
let cx = &mut VisualTestContext::from_window(*window, cx);
|
||||
|
||||
assert_eq!(
|
||||
buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()),
|
||||
PLAIN_TEXT.name()
|
||||
);
|
||||
|
||||
editor.update_in(cx, |editor, window, cx| {
|
||||
editor.insert("fn main() {}", window, cx);
|
||||
});
|
||||
cx.run_until_parked();
|
||||
|
||||
assert_eq!(
|
||||
buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()),
|
||||
PLAIN_TEXT.name()
|
||||
);
|
||||
|
||||
editor.update_in(cx, |editor, window, cx| {
|
||||
editor.select_all(&SelectAll, window, cx);
|
||||
editor.insert("fn main() { println!(\"hello\"); }", window, cx);
|
||||
});
|
||||
cx.run_until_parked();
|
||||
|
||||
assert_eq!(
|
||||
buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()),
|
||||
rust_lang().name()
|
||||
);
|
||||
|
||||
editor.update_in(cx, |editor, window, cx| {
|
||||
editor.select_all(&SelectAll, window, cx);
|
||||
editor.backspace(&Backspace, window, cx);
|
||||
});
|
||||
cx.run_until_parked();
|
||||
|
||||
editor.update_in(cx, |editor, window, cx| {
|
||||
cx.write_to_clipboard(ClipboardItem::new_string(
|
||||
"package main\n\nimport \"fmt\"\n\nfunc main() { fmt.Println(\"hello\") }".to_string(),
|
||||
));
|
||||
editor.paste(&Paste, window, cx);
|
||||
});
|
||||
cx.run_until_parked();
|
||||
|
||||
assert_eq!(
|
||||
buffer.read_with(cx, |buffer, _| buffer.language().unwrap().name()),
|
||||
go_language.name()
|
||||
);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_clipboard(cx: &mut TestAppContext) {
|
||||
init_test(cx, |_| {});
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "language_detection"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
publish.workspace = true
|
||||
license = "GPL-3.0-or-later"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/language_detection.rs"
|
||||
|
||||
[dependencies]
|
||||
betlang.workspace = true
|
||||
gpui.workspace = true
|
||||
language.workspace = true
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../LICENSE-GPL
|
||||
@@ -0,0 +1,58 @@
|
||||
use gpui::{App, AppContext, Task};
|
||||
use language::{BufferSnapshot, Language, LanguageRegistry};
|
||||
use std::sync::Arc;
|
||||
|
||||
const SAMPLE_BLOCK_SIZE: usize = 4096;
|
||||
|
||||
fn language_registry_key(language: betlang::Language) -> &'static str {
|
||||
match language {
|
||||
betlang::Language::ObjectiveC => "Objective-C",
|
||||
betlang::Language::Shell => "Shell Script",
|
||||
_ => language.slug(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_language(
|
||||
buffer: BufferSnapshot,
|
||||
language_registry: Arc<LanguageRegistry>,
|
||||
cx: &mut App,
|
||||
) -> Task<Option<(Arc<Language>, f32)>> {
|
||||
let source = extract_sample(&buffer);
|
||||
cx.background_spawn(async move {
|
||||
let detection = betlang::detect(source);
|
||||
let (confidence, language) = detection.top_languages().next()?;
|
||||
let language = language_registry
|
||||
.language_for_name_or_extension(language_registry_key(language))
|
||||
.await
|
||||
.ok()?;
|
||||
|
||||
Some((language, confidence))
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_sample(buffer: &BufferSnapshot) -> Vec<u8> {
|
||||
let source_length = buffer.len();
|
||||
let ranges = if source_length <= SAMPLE_BLOCK_SIZE * 2 {
|
||||
vec![0..source_length]
|
||||
} else {
|
||||
vec![
|
||||
0..SAMPLE_BLOCK_SIZE,
|
||||
source_length - SAMPLE_BLOCK_SIZE..source_length,
|
||||
]
|
||||
};
|
||||
|
||||
ranges
|
||||
.into_iter()
|
||||
.flat_map(|range| buffer.bytes_in_range(range))
|
||||
.flat_map(|chunk| chunk.iter().copied())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn detects_rust_source() {
|
||||
let detection = betlang::detect("fn main() { println!(\"hello\"); }");
|
||||
assert_eq!(detection.language(), Some(betlang::Language::Rust));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user