diff --git a/Cargo.lock b/Cargo.lock index 1e74bce459..38da980bb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9555,12 +9555,14 @@ dependencies = [ "gpui_shared_string", "gpui_util", "log", + "lru", "parking_lot", "path", "regex", "schemars 1.0.4", "serde", "serde_json", + "smallvec", "strum 0.28.0", "toml 0.8.23", "tree-sitter", diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 5bf8e87112..246b786ee3 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -19,8 +19,8 @@ use crate::{ unified_diff_with_offsets, }; pub use crate::{ - Grammar, HighlightId, HighlightMap, Language, LanguageRegistry, diagnostic_set::DiagnosticSet, - proto, + CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry, + diagnostic_set::DiagnosticSet, proto, }; use anyhow::{Context as _, Result}; @@ -5553,8 +5553,8 @@ impl<'a> BufferChunks<'a> { { let next_capture_end = capture.node.end_byte(); if range.start < next_capture_end - && let Some(capture_id) = - highlights.highlight_maps[capture.grammar_index].get(capture.index) + && let Some(capture_id) = highlights.highlight_maps[capture.grammar_index] + .get(CaptureId(capture.index)) { highlights.stack.push((next_capture_end, capture_id)); } @@ -5689,8 +5689,8 @@ impl<'a> Iterator for BufferChunks<'a> { next_capture_start = capture.node.start_byte(); break; } else { - let highlight_id = - highlights.highlight_maps[capture.grammar_index].get(capture.index); + let highlight_id = highlights.highlight_maps[capture.grammar_index] + .get(CaptureId(capture.index)); if let Some(highlight_id) = highlight_id { highlights .stack diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index a7791c51e7..6dcb998500 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -44,10 +44,12 @@ use http_client::HttpClient; pub use language_core::{ SymbolKind, - highlight_map::{HighlightId, HighlightMap}, + highlight_cache::ResolvedHighlights, + highlight_map::{CaptureId, HighlightId, HighlightMap}, }; use futures::future::FutureExt as _; +use language_core::highlight_cache::{MAX_TEXT_HIGHLIGHT_ENTRY_BYTES, TextHighlightKey}; pub use language_core::{ BlockCommentConfig, BracketPair, BracketPairConfig, BracketPairContent, BracketsConfig, BracketsPatternConfig, CodeLabel, CodeLabelBuilder, DebugVariablesConfig, DebuggerTextObject, @@ -83,7 +85,7 @@ use std::{ str, sync::{Arc, LazyLock}, }; -use syntax_map::{QueryCursorHandle, SyntaxSnapshot}; +use syntax_map::{QueryCursorHandle, SyntaxSnapshot, flattened_highlight_regions}; use task::RunnableTag; pub use task_context::{ContextLocation, ContextProvider}; pub use text_diff::{ @@ -1120,29 +1122,94 @@ impl Language { text: &'a Rope, range: Range, ) -> Vec<(Range, HighlightId)> { - let mut result = Vec::new(); - if let Some(grammar) = &self.grammar { - let tree = parse_text(grammar, text, None); - let captures = - SyntaxSnapshot::single_tree_captures(range.clone(), text, &tree, self, |grammar| { - grammar - .highlights_config - .as_ref() - .map(|config| &config.query) - }); - let highlight_maps = vec![grammar.highlight_map()]; - let mut offset = 0; - for chunk in - BufferChunks::new(text, range, Some((captures, highlight_maps)), false, None) + self.highlight_text_resolved(text, range).runs.to_vec() + } + + pub fn highlight_text_resolved( + self: &Arc, + text: &Rope, + range: Range, + ) -> ResolvedHighlights { + let Some(grammar) = &self.grammar else { + return ResolvedHighlights::default(); + }; + let Some(highlights_config) = &grammar.highlights_config else { + return ResolvedHighlights::default(); + }; + let highlights = if text.len() > MAX_TEXT_HIGHLIGHT_ENTRY_BYTES { + self.compute_resolved_highlights(grammar, text) + } else { + let key = TextHighlightKey::new(text.chunks(), text.len()); + match highlights_config + .text_highlight_cache + .get(&key, text.chunks()) { - let end_offset = offset + chunk.text.len(); - if let Some(highlight_id) = chunk.syntax_highlight_id { - result.push((offset..end_offset, highlight_id)); + Some(highlights) => highlights, + None => highlights_config.text_highlight_cache.insert( + key, + Arc::from(text.chunks().collect::()), + self.compute_resolved_highlights(grammar, text), + ), + } + }; + if range.start == 0 && range.end >= text.len() { + return highlights; + } + ResolvedHighlights { + sources: highlights.sources.clone(), + runs: highlights + .runs + .iter() + .filter(|(run_range, _)| run_range.start < range.end && run_range.end > range.start) + .map(|(run_range, highlight_id)| { + ( + run_range.start.max(range.start) - range.start + ..run_range.end.min(range.end) - range.start, + *highlight_id, + ) + }) + .collect(), + } + } + + fn compute_resolved_highlights( + self: &Arc, + grammar: &Arc, + text: &Rope, + ) -> ResolvedHighlights { + let highlight_map = grammar.highlight_map(); + let tree = parse_text(grammar, text, None); + let captures = + SyntaxSnapshot::single_tree_captures(0..text.len(), text, &tree, self, |grammar| { + grammar + .highlights_config + .as_ref() + .map(|config| &config.query) + }); + let mut runs = Vec::<(Range, HighlightId)>::new(); + for region in flattened_highlight_regions(captures, 0..text.len()) { + let highlight_id = region + .stack + .iter() + .rev() + .find_map(|capture| highlight_map.get(capture.capture_id)); + let Some(highlight_id) = highlight_id else { + continue; + }; + match runs.last_mut() { + Some((last_range, last_highlight_id)) + if *last_highlight_id == highlight_id + && last_range.end == region.range.start => + { + last_range.end = region.range.end; } - offset = end_offset; + _ => runs.push((region.range, highlight_id)), } } - result + ResolvedHighlights { + sources: [(Arc::clone(grammar), highlight_map)].into_iter().collect(), + runs: runs.into(), + } } pub fn path_suffixes(&self) -> &[String] { @@ -1686,19 +1753,163 @@ mod tests { let map = build_highlight_map(capture_names, &theme); assert_eq!( - theme.get_capture_name(map.get(0).unwrap()), + theme.get_capture_name(map.get(CaptureId(0)).unwrap()), Some("function") ); assert_eq!( - theme.get_capture_name(map.get(1).unwrap()), + theme.get_capture_name(map.get(CaptureId(1)).unwrap()), Some("function.async") ); assert_eq!( - theme.get_capture_name(map.get(2).unwrap()), + theme.get_capture_name(map.get(CaptureId(2)).unwrap()), Some("variable.builtin") ); } + #[test] + fn test_highlight_text_resolves_nested_captures_with_theme_fallback() { + let language = Arc::new( + Language::new( + LanguageConfig { + name: "Rust".into(), + ..LanguageConfig::default() + }, + Some(tree_sitter_rust::LANGUAGE.into()), + ) + .with_highlights_query( + r#" + (function_item) @function.definition + (identifier) @variable + "fn" @keyword + "#, + ) + .unwrap(), + ); + + let theme = SyntaxTheme::new( + [ + ("function", rgba(0x100000ff)), + ("keyword", rgba(0x200000ff)), + ] + .iter() + .map(|(name, color)| (name.to_string(), (*color).into())), + ); + language.set_theme(&theme); + + let code = "fn main() {}"; + let highlights = language.highlight_text_resolved(&Rope::from(code), 0..code.len()); + assert!(highlights.is_current()); + let named = highlights + .runs + .iter() + .map(|(range, highlight_id)| { + ( + range.clone(), + theme.get_capture_name(*highlight_id).unwrap(), + ) + }) + .collect::>(); + assert_eq!( + named, + vec![(0..2, "keyword"), (2..12, "function")], + "an inner capture missing from the theme must fall back to its outer capture" + ); + + let memoized = language.highlight_text_resolved(&Rope::from(code), 0..code.len()); + assert!( + Arc::ptr_eq(&highlights.runs, &memoized.runs), + "repeated highlighting of the same text must be memoized" + ); + let partial = language.highlight_text_resolved(&Rope::from(code), 0..2); + assert!( + !Arc::ptr_eq(&highlights.runs, &partial.runs), + "a different range must not reuse the memoized highlights" + ); + assert_eq!(partial.runs.as_ref(), &[(0..2, highlights.runs[0].1)]); + + let richer_theme = SyntaxTheme::new( + [ + ("function", rgba(0x100000ff)), + ("keyword", rgba(0x200000ff)), + ("variable", rgba(0x300000ff)), + ] + .iter() + .map(|(name, color)| (name.to_string(), (*color).into())), + ); + language.set_theme(&richer_theme); + assert!( + !highlights.is_current(), + "a theme change must invalidate previously resolved highlights" + ); + let rethemed = language.highlight_text_resolved(&Rope::from(code), 0..code.len()); + assert!(rethemed.is_current()); + let renamed = rethemed + .runs + .iter() + .map(|(range, highlight_id)| { + ( + range.clone(), + richer_theme.get_capture_name(*highlight_id).unwrap(), + ) + }) + .collect::>(); + assert_eq!( + renamed, + vec![ + (0..2, "keyword"), + (2..3, "function"), + (3..7, "variable"), + (7..12, "function"), + ], + "a theme change must invalidate memoized highlights and resolve against the new theme" + ); + } + + #[test] + fn test_oversized_text_is_highlighted_without_caching() { + let language = rust_lang(); + let theme = SyntaxTheme::new( + [ + ("function", rgba(0x100000ff)), + ("keyword", rgba(0x200000ff)), + ] + .iter() + .map(|(name, color)| (name.to_string(), (*color).into())), + ); + language.set_theme(&theme); + + let small_code = "fn main() {}"; + let small_highlights = + language.highlight_text_resolved(&Rope::from(small_code), 0..small_code.len()); + assert!( + !small_highlights.runs.is_empty(), + "texts within the size cap must be highlighted" + ); + + let oversized_code = format!( + "fn main() {{}}{}", + " ".repeat(MAX_TEXT_HIGHLIGHT_ENTRY_BYTES) + ); + let oversized_rope = Rope::from(oversized_code.as_str()); + let first_highlights = + language.highlight_text_resolved(&oversized_rope, 0..oversized_code.len()); + assert_eq!( + first_highlights.runs.as_ref(), + small_highlights.runs.as_ref(), + "texts over the cache entry cap must still be highlighted" + ); + let second_highlights = + language.highlight_text_resolved(&oversized_rope, 0..oversized_code.len()); + assert_eq!( + second_highlights.runs.as_ref(), + first_highlights.runs.as_ref() + ); + assert!( + !Arc::ptr_eq(&first_highlights.runs, &second_highlights.runs), + "texts over the cache entry cap must not be memoized" + ); + } + #[test] fn test_with_parser_resets_after_cancellation() { use std::ops::ControlFlow; diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index 9ff17ce2de..26a53bfd5b 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -2,11 +2,14 @@ mod syntax_map_tests; use crate::{ - Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS, with_parser, + CaptureId, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS, + with_parser, }; use collections::HashMap; use futures::FutureExt; use gpui::SharedString; +use language_core::highlight_map::HighlightCaptureRef; +use smallvec::SmallVec; use std::{ borrow::Cow, cmp::{self, Ordering, Reverse}, @@ -73,7 +76,7 @@ impl Drop for SyntaxSnapshot { pub struct SyntaxMapCaptures<'a> { layers: Vec>, active_layer_count: usize, - grammars: Vec<&'a Grammar>, + grammars: Vec<&'a Arc>, } #[derive(Default)] @@ -90,6 +93,73 @@ pub struct SyntaxMapCapture<'a> { pub grammar_index: usize, } +pub(crate) struct HighlightCaptureRegion { + pub range: Range, + pub stack: SmallVec<[HighlightCaptureRef; 4]>, +} + +pub(crate) fn flattened_highlight_regions( + mut captures: SyntaxMapCaptures<'_>, + range: Range, +) -> Vec { + let capture_refs = iter::from_fn(move || { + let capture = captures.next()?; + Some(( + capture.node.byte_range(), + HighlightCaptureRef { + grammar_index: capture.grammar_index, + capture_id: CaptureId(capture.index), + }, + )) + }); + flatten_capture_regions(range, capture_refs) +} + +fn flatten_capture_regions( + range: Range, + mut captures: impl Iterator, HighlightCaptureRef)>, +) -> Vec { + let mut result = Vec::new(); + let mut stack = Vec::<(Range, HighlightCaptureRef)>::new(); + let mut offset = range.start; + let mut next_capture = captures.next(); + loop { + stack.retain(|(capture_range, _)| capture_range.end > offset); + while let Some((capture_range, capture_ref)) = next_capture.take() { + if capture_range.start > offset { + next_capture = Some((capture_range, capture_ref)); + break; + } + if capture_range.end > offset { + stack.push((capture_range, capture_ref)); + } + next_capture = captures.next(); + } + let mut next_boundary = range.end; + if let Some(min_end) = stack + .iter() + .map(|(capture_range, _)| capture_range.end) + .min() + { + next_boundary = next_boundary.min(min_end); + } + if let Some((capture_range, _)) = &next_capture { + next_boundary = next_boundary.min(capture_range.start); + } + if !stack.is_empty() && next_boundary > offset { + result.push(HighlightCaptureRegion { + range: offset..next_boundary, + stack: stack.iter().map(|(_, capture_ref)| *capture_ref).collect(), + }); + } + if next_boundary >= range.end { + break; + } + offset = next_boundary; + } + result +} + #[derive(Debug)] pub struct SyntaxMapMatch<'a> { pub language: Arc, @@ -1201,7 +1271,7 @@ impl<'a> SyntaxMapCaptures<'a> { result } - pub fn grammars(&self) -> &[&'a Grammar] { + pub fn grammars(&self) -> &[&'a Arc] { &self.grammars } diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs index d4eecbe04c..4c57b38e6f 100644 --- a/crates/language/src/syntax_map/syntax_map_tests.rs +++ b/crates/language/src/syntax_map/syntax_map_tests.rs @@ -1341,6 +1341,48 @@ fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) { test_random_edits(text, registry, language, rng); } +#[test] +fn test_flatten_capture_regions_with_nested_captures() { + let outer = capture_ref(1); + let inner = capture_ref(2); + assert_eq!( + flattened(0..12, &[(0..10, outer), (2..5, inner)]), + vec![ + (0..2, vec![outer]), + (2..5, vec![outer, inner]), + (5..10, vec![outer]), + ], + ); +} + +#[test] +fn test_flatten_capture_regions_with_overlapping_captures() { + let first = capture_ref(1); + let second = capture_ref(2); + assert_eq!( + flattened(0..25, &[(0..10, first), (2..20, second)]), + vec![ + (0..2, vec![first]), + (2..10, vec![first, second]), + (10..20, vec![second]), + ], + "a capture must not extend past its own end when overlapping another capture" + ); +} + +#[test] +fn test_flatten_capture_regions_clips_to_the_requested_range() { + let capture = capture_ref(1); + assert_eq!( + flattened(5..8, &[(0..10, capture)]), + vec![(5..8, vec![capture])], + ); + assert_eq!( + flattened(0..6, &[(4..10, capture)]), + vec![(4..6, vec![capture])], + ); +} + fn test_random_edits( text: String, registry: Arc, @@ -1762,6 +1804,23 @@ fn sql_lang() -> Language { ) } +fn capture_ref(capture_id: u32) -> HighlightCaptureRef { + HighlightCaptureRef { + grammar_index: 0, + capture_id: CaptureId(capture_id), + } +} + +fn flattened( + range: Range, + captures: &[(Range, HighlightCaptureRef)], +) -> Vec<(Range, Vec)> { + flatten_capture_regions(range, captures.iter().cloned()) + .into_iter() + .map(|region| (region.range, region.stack.to_vec())) + .collect() +} + fn range_for_text(buffer: &Buffer, text: &str) -> Range { let start = buffer.as_rope().to_string().find(text).unwrap(); start..start + text.len() diff --git a/crates/language_core/Cargo.toml b/crates/language_core/Cargo.toml index 672d0dabd5..9941b1f8bf 100644 --- a/crates/language_core/Cargo.toml +++ b/crates/language_core/Cargo.toml @@ -14,12 +14,14 @@ collections.workspace = true gpui_shared_string.workspace = true gpui_util.workspace = true log.workspace = true +lru.workspace = true parking_lot.workspace = true path.workspace = true regex.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +smallvec.workspace = true strum.workspace = true toml.workspace = true tree-sitter.workspace = true diff --git a/crates/language_core/src/grammar.rs b/crates/language_core/src/grammar.rs index ff4dc0c885..fc45de7706 100644 --- a/crates/language_core/src/grammar.rs +++ b/crates/language_core/src/grammar.rs @@ -1,6 +1,6 @@ use crate::{ - HighlightId, HighlightMap, LanguageConfig, LanguageConfigOverride, LanguageName, - LanguageQueries, language_config::BracketPairConfig, + CaptureId, HighlightId, HighlightMap, LanguageConfig, LanguageConfigOverride, LanguageName, + LanguageQueries, highlight_cache::TextHighlightCache, language_config::BracketPairConfig, }; use anyhow::{Context as _, Result}; use collections::HashMap; @@ -103,6 +103,7 @@ pub struct Grammar { pub struct HighlightsConfig { pub query: Query, pub identifier_capture_indices: Vec, + pub text_highlight_cache: TextHighlightCache, } pub struct IndentConfig { @@ -367,12 +368,16 @@ impl Grammar { self.highlight_map.lock().clone() } + pub fn highlight_map_matches(&self, highlight_map: &HighlightMap) -> bool { + self.highlight_map.lock().same(highlight_map) + } + pub fn highlight_id_for_name(&self, name: &str) -> Option { self.highlights_config .as_ref()? .query .capture_index_for_name(name) - .and_then(|capture_id| self.highlight_map.lock().get(capture_id)) + .and_then(|capture_id| self.highlight_map.lock().get(CaptureId(capture_id))) } pub fn debug_variables_config(&self) -> Option<&DebugVariablesConfig> { @@ -469,6 +474,7 @@ impl Grammar { self.highlights_config = Some(HighlightsConfig { query, identifier_capture_indices, + text_highlight_cache: TextHighlightCache::default(), }); Ok(self) diff --git a/crates/language_core/src/highlight_cache.rs b/crates/language_core/src/highlight_cache.rs new file mode 100644 index 0000000000..b820787f71 --- /dev/null +++ b/crates/language_core/src/highlight_cache.rs @@ -0,0 +1,284 @@ +use crate::grammar::Grammar; +use crate::highlight_map::{HighlightId, HighlightMap}; +use collections::FxHasher; +use lru::LruCache; +use parking_lot::Mutex; +use smallvec::SmallVec; +use std::{ + fmt, + hash::{Hash, Hasher as _}, + ops::Range, + sync::Arc, +}; + +pub const MAX_TEXT_HIGHLIGHT_ENTRY_BYTES: usize = MAX_TEXT_HIGHLIGHT_CACHE_BYTES / 8; + +const MAX_TEXT_HIGHLIGHT_CACHE_BYTES: usize = 4 * 1024 * 1024; +const APPROXIMATE_LRU_NODE_BYTES: usize = 4 * size_of::(); + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TextHighlightKey { + text_hash: u64, + text_len: usize, +} + +impl TextHighlightKey { + pub fn new<'a>(text_chunks: impl Iterator, text_len: usize) -> Self { + let mut hasher = FxHasher::default(); + for chunk in text_chunks { + hasher.write(chunk.as_bytes()); + } + Self { + text_hash: hasher.finish(), + text_len, + } + } +} + +#[derive(Clone, Default)] +pub struct ResolvedHighlights { + pub sources: SmallVec<[(Arc, HighlightMap); 2]>, + pub runs: Arc<[(Range, HighlightId)]>, +} + +impl ResolvedHighlights { + pub fn is_current(&self) -> bool { + self.sources + .iter() + .all(|(grammar, highlight_map)| grammar.highlight_map_matches(highlight_map)) + } + + pub fn cost_bytes(&self) -> usize { + self.sources.len() * size_of::<(Arc, HighlightMap)>() + + self.runs.len() * size_of::<(Range, HighlightId)>() + } +} + +impl fmt::Debug for ResolvedHighlights { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ResolvedHighlights") + .field("runs", &self.runs) + .finish_non_exhaustive() + } +} + +pub struct TextHighlightCache(Mutex>); + +impl Default for TextHighlightCache { + fn default() -> Self { + Self(Mutex::new(CostBudgetedLru::new( + MAX_TEXT_HIGHLIGHT_CACHE_BYTES, + MAX_TEXT_HIGHLIGHT_ENTRY_BYTES, + ))) + } +} + +impl TextHighlightCache { + pub fn get<'a>( + &self, + key: &TextHighlightKey, + text_chunks: impl Iterator, + ) -> Option { + let mut cache = self.0.lock(); + let entry = cache.get(key)?; + if !entry.highlights.is_current() { + return None; + } + if !chunks_match_text(&entry.text, text_chunks) { + return None; + } + Some(entry.highlights.clone()) + } + + pub fn insert( + &self, + key: TextHighlightKey, + text: Arc, + highlights: ResolvedHighlights, + ) -> ResolvedHighlights { + let cost = text.len() + highlights.cost_bytes(); + self.0.lock().insert( + key, + TextHighlightEntry { + text, + highlights: highlights.clone(), + }, + cost, + ); + highlights + } +} + +struct CostBudgetedLru { + entries: LruCache, + total_cost: usize, + max_total_cost: usize, + max_entry_cost: usize, +} + +impl CostBudgetedLru { + const ENTRY_OVERHEAD_BYTES: usize = + size_of::() + size_of::<(V, usize)>() + APPROXIMATE_LRU_NODE_BYTES; + + fn new(max_total_cost: usize, max_entry_cost: usize) -> Self { + Self { + entries: LruCache::unbounded(), + total_cost: 0, + max_total_cost, + max_entry_cost, + } + } + + fn get(&mut self, key: &K) -> Option<&V> { + self.entries.get(key).map(|(value, _)| value) + } + + fn insert(&mut self, key: K, value: V, cost: usize) { + if cost > self.max_entry_cost { + return; + } + let cost = cost + Self::ENTRY_OVERHEAD_BYTES; + if let Some((_, old_cost)) = self.entries.put(key, (value, cost)) { + self.total_cost -= old_cost; + } + self.total_cost += cost; + while self.total_cost > self.max_total_cost { + let Some((_, (_, evicted_cost))) = self.entries.pop_lru() else { + break; + }; + self.total_cost -= evicted_cost; + } + } +} + +struct TextHighlightEntry { + text: Arc, + highlights: ResolvedHighlights, +} + +fn chunks_match_text<'a>(text: &str, text_chunks: impl Iterator) -> bool { + let mut remaining = text; + for chunk in text_chunks { + let Some(rest) = remaining.strip_prefix(chunk) else { + return false; + }; + remaining = rest; + } + remaining.is_empty() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_budget_evicts_least_recently_used() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(2 * (40 + overhead), 100); + cache.insert("a", 1, 40); + cache.insert("b", 2, 40); + cache.get(&"a"); + cache.insert("c", 3, 40); + assert_eq!(cache.get(&"a"), Some(&1)); + assert_eq!(cache.get(&"b"), None); + assert_eq!(cache.get(&"c"), Some(&3)); + assert_eq!(cache.total_cost, 2 * (40 + overhead)); + } + + #[test] + fn test_replacing_an_entry_updates_the_budget() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(100 + 2 * overhead, 100); + cache.insert("a", 1, 60); + cache.insert("a", 2, 30); + assert_eq!(cache.total_cost, 30 + overhead); + assert_eq!(cache.get(&"a"), Some(&2)); + cache.insert("b", 3, 70); + assert_eq!(cache.get(&"a"), Some(&2)); + assert_eq!(cache.get(&"b"), Some(&3)); + } + + #[test] + fn test_oversized_entries_are_rejected_without_flushing() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(100 + overhead, 50); + cache.insert("a", 1, 40); + cache.insert("b", 2, 60); + assert_eq!(cache.get(&"a"), Some(&1)); + assert_eq!(cache.get(&"b"), None); + assert_eq!(cache.total_cost, 40 + overhead); + } + + #[test] + fn test_zero_cost_entries_still_consume_budget() { + let overhead = CostBudgetedLru::::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::::new(3 * overhead, 100); + for key in 0..5_usize { + cache.insert(key, key as u32, 0); + } + assert_eq!(cache.get(&0), None); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.get(&2), Some(&2)); + assert_eq!(cache.get(&3), Some(&3)); + assert_eq!(cache.get(&4), Some(&4)); + assert_eq!( + cache.total_cost, + 3 * overhead, + "Entries with zero caller-supplied cost must still be bounded by the budget" + ); + } + + #[test] + fn test_oversized_highlights_are_returned_but_not_cached() { + let text = Arc::::from("fn main() {}"); + let run_bytes = size_of::<(Range, HighlightId)>(); + let max_entry_cost = text.len() + 2 * run_bytes; + let cache = TextHighlightCache(Mutex::new(CostBudgetedLru::new( + max_entry_cost * 10, + max_entry_cost, + ))); + let resolved = |ranges: &[Range]| ResolvedHighlights { + sources: SmallVec::new(), + runs: ranges + .iter() + .map(|range| (range.clone(), HighlightId::new(0))) + .collect(), + }; + + let small_text = Arc::::from("fn f() {}"); + let small_key = TextHighlightKey::new([small_text.as_ref()].into_iter(), small_text.len()); + let small_range = 0..2; + let small_highlights = cache.insert( + small_key.clone(), + Arc::clone(&small_text), + resolved(std::slice::from_ref(&small_range)), + ); + assert_eq!( + small_highlights.runs.as_ref(), + resolved(std::slice::from_ref(&small_range)).runs.as_ref() + ); + assert_eq!( + cache + .get(&small_key, [small_text.as_ref()].into_iter()) + .map(|highlights| highlights.runs), + Some(small_highlights.runs.clone()), + "highlights within the entry budget must be cached as-is" + ); + + let big_key = TextHighlightKey::new([text.as_ref()].into_iter(), text.len()); + let big_highlights_source = resolved(&[0..2, 3..7, 8..9]); + let big_highlights = cache.insert( + big_key.clone(), + Arc::clone(&text), + big_highlights_source.clone(), + ); + assert!( + Arc::ptr_eq(&big_highlights.runs, &big_highlights_source.runs), + "highlights over the entry budget must be returned unchanged" + ); + assert!( + cache.get(&big_key, [text.as_ref()].into_iter()).is_none(), + "highlights over the entry budget must not be cached" + ); + } +} diff --git a/crates/language_core/src/highlight_map.rs b/crates/language_core/src/highlight_map.rs index cba5cda6f7..b5ddcf670e 100644 --- a/crates/language_core/src/highlight_map.rs +++ b/crates/language_core/src/highlight_map.rs @@ -3,6 +3,15 @@ use std::{num::NonZeroU32, sync::Arc}; #[derive(Clone, Debug)] pub struct HighlightMap(Arc<[Option]>); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct CaptureId(pub u32); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HighlightCaptureRef { + pub grammar_index: usize, + pub capture_id: CaptureId, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct HighlightId(NonZeroU32); @@ -28,8 +37,13 @@ impl HighlightMap { } #[inline] - pub fn get(&self, capture_id: u32) -> Option { - self.0.get(capture_id as usize).copied().flatten() + pub fn get(&self, capture_id: CaptureId) -> Option { + self.0.get(capture_id.0 as usize).copied().flatten() + } + + #[inline] + pub fn same(&self, other: &HighlightMap) -> bool { + Arc::ptr_eq(&self.0, &other.0) } } diff --git a/crates/language_core/src/language_core.rs b/crates/language_core/src/language_core.rs index f3cb3ab51c..e2050f325e 100644 --- a/crates/language_core/src/language_core.rs +++ b/crates/language_core/src/language_core.rs @@ -2,6 +2,7 @@ // language configuration, and highlight mapping. pub mod grammar; +pub mod highlight_cache; pub mod highlight_map; pub mod language_config; @@ -11,7 +12,7 @@ pub use grammar::{ NEXT_GRAMMAR_ID, OutlineConfig, OverrideConfig, OverrideEntry, ParseableLanguage, RedactionConfig, RunnableCapture, RunnableConfig, TextObject, TextObjectConfig, }; -pub use highlight_map::{HighlightId, HighlightMap}; +pub use highlight_map::{CaptureId, HighlightId, HighlightMap}; pub use language_config::{ BlockCommentConfig, BracketPair, BracketPairConfig, BracketPairContent, DecreaseIndentConfig, JsxTagAutoCloseConfig, LanguageConfig, LanguageConfigOverride, LanguageMatcher, diff --git a/crates/language_tools/src/highlights_tree_view.rs b/crates/language_tools/src/highlights_tree_view.rs index e63f6e3051..b3ac451d54 100644 --- a/crates/language_tools/src/highlights_tree_view.rs +++ b/crates/language_tools/src/highlights_tree_view.rs @@ -1190,7 +1190,8 @@ fn build_highlight_entries( .collect::>(); for capture in captures { - let Some(highlight_id) = highlight_maps[capture.grammar_index].get(capture.index) + let Some(highlight_id) = + highlight_maps[capture.grammar_index].get(language::CaptureId(capture.index)) else { continue; }; diff --git a/crates/languages/src/rust.rs b/crates/languages/src/rust.rs index 5bb3089c79..d0f468a6b2 100644 --- a/crates/languages/src/rust.rs +++ b/crates/languages/src/rust.rs @@ -1842,8 +1842,7 @@ mod tests { vec![ (10..13, HighlightId::TABSTOP_INSERT_ID), (16..19, HighlightId::TABSTOP_INSERT_ID), - (0..7, HighlightId::new(2)), - (7..8, HighlightId::new(2)), + (0..8, HighlightId::new(2)), ], )) ); @@ -1870,8 +1869,7 @@ mod tests { 0..4, vec![ (5..9, HighlightId::TABSTOP_REPLACE_ID), - (0..3, HighlightId::new(2)), - (3..4, HighlightId::new(2)), + (0..4, HighlightId::new(2)), ], )) ); @@ -1956,8 +1954,7 @@ mod tests { vec![ (15..20, HighlightId::TABSTOP_REPLACE_ID), (16..19, HighlightId::TABSTOP_INSERT_ID), - (0..13, HighlightId::new(2)), - (13..14, HighlightId::new(2)), + (0..14, HighlightId::new(2)), ], )) ); diff --git a/crates/markdown/src/markdown.rs b/crates/markdown/src/markdown.rs index c8f10d3117..9dc936e3c7 100644 --- a/crates/markdown/src/markdown.rs +++ b/crates/markdown/src/markdown.rs @@ -5,7 +5,6 @@ mod path_range; mod selection; use base64::Engine as _; - use gpui::EdgesRefinement; use gpui::HitboxBehavior; use gpui::UnderlineStyle; @@ -41,7 +40,7 @@ use gpui::{ TextStyle, TextStyleRefinement, WrappedLineLayout, actions, canvas, img, point, quad, relative, size, }; -use language::{CharClassifier, Language, LanguageRegistry, Rope}; +use language::{CharClassifier, Language, LanguageRegistry, ResolvedHighlights, Rope}; use parser::CodeBlockMetadata; use parser::{ MarkdownEvent, MarkdownTag, MarkdownTagEnd, ParsedMetadataBlock, parse_links_only, @@ -977,20 +976,7 @@ impl Markdown { return None; }; - match kind { - CodeBlockKind::FencedLang(language) => self - .parsed_markdown - .languages_by_name - .get(language) - .cloned(), - CodeBlockKind::FencedSrc(path_range) => self - .parsed_markdown - .languages_by_path - .get(&path_range.path) - .cloned(), - CodeBlockKind::Fenced => self.parsed_markdown.fallback_code_block_language.clone(), - CodeBlockKind::Indented => None, - } + self.parsed_markdown.code_block_language(kind) }) } @@ -1258,6 +1244,7 @@ impl Markdown { footnote_definitions: HashMap::default(), link_definition_spans: Arc::default(), fallback_code_block_language: None, + code_block_highlights: Arc::default(), }, Default::default(), ); @@ -1335,23 +1322,23 @@ impl Markdown { } } - ( - ParsedMarkdown { - source, - events: Arc::from(events), - languages_by_name, - languages_by_path, - root_block_starts: Arc::from(root_block_starts), - html_blocks, - metadata_blocks, - mermaid_diagrams, - heading_slugs, - footnote_definitions, - link_definition_spans: Arc::from(link_definition_spans), - fallback_code_block_language, - }, - images_by_source_offset, - ) + let mut parsed = ParsedMarkdown { + source, + events: Arc::from(events), + languages_by_name, + languages_by_path, + root_block_starts: Arc::from(root_block_starts), + html_blocks, + metadata_blocks, + mermaid_diagrams, + heading_slugs, + footnote_definitions, + link_definition_spans: Arc::from(link_definition_spans), + fallback_code_block_language, + code_block_highlights: Arc::default(), + }; + parsed.code_block_highlights = Arc::new(compute_code_block_highlights(&parsed)); + (parsed, images_by_source_offset) }); cx.spawn(async move |this, cx| { @@ -1497,8 +1484,11 @@ pub struct ParsedMarkdown { pub footnote_definitions: HashMap, pub(crate) link_definition_spans: Arc<[Range]>, pub(crate) fallback_code_block_language: Option>, + pub(crate) code_block_highlights: Arc, } +pub(crate) type CodeBlockHighlights = HashMap; + impl ParsedMarkdown { pub fn source(&self) -> &SharedString { &self.source @@ -1606,6 +1596,109 @@ impl ParsedMarkdown { selection, ) } + + pub(crate) fn code_block_language(&self, kind: &CodeBlockKind) -> Option> { + match kind { + CodeBlockKind::FencedLang(name) => self.languages_by_name.get(name).cloned(), + CodeBlockKind::FencedSrc(path_range) => { + self.languages_by_path.get(&path_range.path).cloned() + } + CodeBlockKind::Fenced => self.fallback_code_block_language.clone(), + CodeBlockKind::Indented => None, + } + } +} + +struct PendingCodeBlock<'a> { + language: Arc, + texts: Vec<(Range, &'a str)>, +} + +fn compute_code_block_highlights(parsed: &ParsedMarkdown) -> CodeBlockHighlights { + let mut code_block_highlights = CodeBlockHighlights::default(); + let mut pending_block: Option = None; + for (range, event) in parsed.events.iter() { + match event { + MarkdownEvent::Start(MarkdownTag::CodeBlock { kind, .. }) => { + if parsed.mermaid_diagrams.contains_key(&range.start) { + pending_block = None; + continue; + } + pending_block = parsed + .code_block_language(kind) + .map(|language| PendingCodeBlock { + language, + texts: Vec::new(), + }); + } + MarkdownEvent::End(MarkdownTagEnd::CodeBlock) => { + if let Some(block) = pending_block.take() { + highlight_code_block(block, &mut code_block_highlights); + } + } + MarkdownEvent::Text => { + if let Some(block) = &mut pending_block { + block + .texts + .push((range.clone(), &parsed.source[range.clone()])); + } + } + MarkdownEvent::SubstitutedText(text) => { + if let Some(block) = &mut pending_block { + block.texts.push((range.clone(), text.as_str())); + } + } + _ => {} + } + } + code_block_highlights +} + +fn highlight_code_block(block: PendingCodeBlock, code_block_highlights: &mut CodeBlockHighlights) { + let mut combined = String::new(); + let mut text_offsets = Vec::with_capacity(block.texts.len()); + for (_, text) in &block.texts { + text_offsets.push(combined.len()); + combined.push_str(text); + } + let resolved = block + .language + .highlight_text_resolved(&Rope::from(combined.as_str()), 0..combined.len()); + if resolved.runs.is_empty() { + return; + } + if let [(source_range, _)] = block.texts.as_slice() { + code_block_highlights.insert(source_range.start, resolved); + return; + } + let mut runs = resolved.runs.iter().peekable(); + for ((source_range, text), text_offset) in block.texts.iter().zip(text_offsets) { + let text_end = text_offset + text.len(); + let mut text_runs = Vec::new(); + while let Some((run_range, highlight_id)) = runs.peek() { + if run_range.start >= text_end { + break; + } + let start = run_range.start.max(text_offset); + let end = run_range.end.min(text_end); + if end > start { + text_runs.push((start - text_offset..end - text_offset, *highlight_id)); + } + if run_range.end > text_end { + break; + } + runs.next(); + } + if !text_runs.is_empty() { + code_block_highlights.insert( + source_range.start, + ResolvedHighlights { + sources: resolved.sources.clone(), + runs: text_runs.into(), + }, + ); + } + } } pub enum AutoscrollBehavior { @@ -2491,12 +2584,7 @@ impl Element for MarkdownElement { next_search_highlight_ix: 0, } }; - let mut builder = MarkdownElementBuilder::new( - &self.style.container_style, - self.style.base_text_style.clone(), - self.style.syntax.clone(), - highlights, - ); + let (parsed_markdown, images, active_root_block, render_mermaid_diagrams, mermaid_state) = { let markdown = self.markdown.read(cx); ( @@ -2507,6 +2595,13 @@ impl Element for MarkdownElement { markdown.mermaid_state.clone(), ) }; + let mut builder = MarkdownElementBuilder::new( + &self.style.container_style, + self.style.base_text_style.clone(), + self.style.syntax.clone(), + highlights, + parsed_markdown.code_block_highlights.clone(), + ); let markdown_end = if let Some(last) = parsed_markdown.events.last() { last.0.end } else { @@ -2670,19 +2765,7 @@ impl Element for MarkdownElement { continue; } - let language = match kind { - CodeBlockKind::Fenced => { - parsed_markdown.fallback_code_block_language.clone() - } - CodeBlockKind::FencedLang(language) => { - parsed_markdown.languages_by_name.get(language).cloned() - } - CodeBlockKind::FencedSrc(path_range) => parsed_markdown - .languages_by_path - .get(&path_range.path) - .cloned(), - _ => None, - }; + let language = parsed_markdown.code_block_language(kind); let is_indented = matches!(kind, CodeBlockKind::Indented); let scroll_handle = if self.style.code_block_overflow_x_scroll { @@ -3587,6 +3670,7 @@ struct MarkdownElementBuilder { base_text_style: TextStyle, text_style_stack: Vec, code_block_stack: Vec>>, + code_block_highlights: Arc, link_depth: usize, list_stack: Vec, table: TableState, @@ -3688,6 +3772,7 @@ impl MarkdownElementBuilder { base_text_style: TextStyle, syntax_theme: Arc, highlights: MarkdownHighlights, + code_block_highlights: Arc, ) -> Self { Self { div_stack: vec![{ @@ -3706,6 +3791,7 @@ impl MarkdownElementBuilder { base_text_style, text_style_stack: Vec::new(), code_block_stack: Vec::new(), + code_block_highlights, link_depth: 0, list_stack: Vec::new(), table: TableState::default(), @@ -3937,24 +4023,34 @@ impl MarkdownElementBuilder { // Compute the base text style once let text_style = self.text_style(); - if let Some(Some(language)) = self.code_block_stack.last() { + if let Some(language) = self.code_block_stack.last().and_then(Option::as_ref) + && let Some(resolved) = self.code_block_highlights.get(&source_range.start) + { + let runs = if resolved.is_current() { + resolved.runs.clone() + } else { + language + .highlight_text_resolved(&Rope::from(text), 0..text.len()) + .runs + }; let mut offset = 0; - for (range, highlight_id) in language.highlight_text(&Rope::from(text), 0..text.len()) { - if range.start > offset { + for (run_range, highlight_id) in runs.iter() { + if run_range.start > offset { self.pending_line .runs - .push(text_style.to_run(range.start - offset)); + .push(text_style.to_run(run_range.start - offset)); } - let run_len = range.len(); - if let Some(highlight) = self.syntax_theme.get(highlight_id).cloned() { + let run_len = run_range.len(); + let highlight = self.syntax_theme.get(*highlight_id).cloned(); + if let Some(highlight) = highlight { self.pending_line .runs .push(text_style.clone().highlight(highlight).to_run(run_len)); } else { self.pending_line.runs.push(text_style.to_run(run_len)); } - offset = range.end; + offset = run_range.end; } if offset < text.len() { @@ -4080,7 +4176,11 @@ impl MarkdownElementBuilder { layout: text.layout().clone(), source_mappings: line.source_mappings, source_end: self.current_source_index, - language: self.code_block_stack.last().cloned().flatten(), + language: self + .code_block_stack + .last() + .and_then(|entry| entry.as_ref()) + .cloned(), text_align, highlights, code_chips: line.code_chips.into_iter().collect(), @@ -5164,9 +5264,118 @@ mod tests { }); } + #[gpui::test] + fn test_code_block_highlights_cached_at_parse_time(cx: &mut TestAppContext) { + let source = "```rust\nfn main() {}\n```"; + let (language, markdown) = markdown_with_rust_language(source, cx); + + let code_text = "fn main() {}\n"; + let code_start = source.find(code_text).unwrap(); + let cached = cached_code_block_highlights(&markdown, code_start, cx); + let expected = language.highlight_text(&Rope::from(code_text), 0..code_text.len()); + assert_eq!( + cached.runs.to_vec(), + expected, + "highlights cached at parse time must match direct highlighting" + ); + assert!(!expected.is_empty(), "rust code must produce highlights"); + assert!(cached.is_current()); + + language.set_theme(&rust_test_theme()); + let stale = cached_code_block_highlights(&markdown, code_start, cx); + assert!( + !stale.is_current(), + "a theme change must make parse-time highlights stale so rendering re-resolves them" + ); + } + + #[gpui::test] + fn test_code_block_highlights_reused_when_streaming(cx: &mut TestAppContext) { + let source = "```rust\nfn main() {}\n```\n\nSome trailing text"; + let (language, markdown) = markdown_with_rust_language(source, cx); + + let code_start = source.find("fn main").unwrap(); + let first_parse_highlights = cached_code_block_highlights(&markdown, code_start, cx); + + markdown.update(cx, |markdown, cx| { + markdown.append(" that keeps streaming in\n\n```rust\nlet x = 1;\n```", cx) + }); + cx.run_until_parked(); + + let second_parse_highlights = cached_code_block_highlights(&markdown, code_start, cx); + assert!( + Arc::ptr_eq(&first_parse_highlights.runs, &second_parse_highlights.runs), + "highlights of an unchanged code block must be reused across parses" + ); + + markdown.update(cx, |markdown, cx| { + markdown.replace("```rust\nfn main() { panic!() }\n```", cx) + }); + cx.run_until_parked(); + + let changed_highlights = cached_code_block_highlights(&markdown, code_start, cx); + let changed_code = "fn main() { panic!() }\n"; + assert_eq!( + changed_highlights.runs.as_ref(), + language + .highlight_text_resolved(&Rope::from(changed_code), 0..changed_code.len()) + .runs + .as_ref(), + "highlights of a changed code block must be recomputed" + ); + } + + #[gpui::test] + fn test_mermaid_blocks_are_not_highlighted(cx: &mut TestAppContext) { + ensure_theme_initialized(cx); + let mermaid_language = Arc::new( + Language::new( + LanguageConfig { + name: "mermaid".into(), + ..LanguageConfig::default() + }, + Some(language::rust_lang().grammar().unwrap().ts_language.clone()), + ) + .with_highlights_query("(identifier) @variable") + .unwrap(), + ); + let language_registry = Arc::new(LanguageRegistry::test(cx.executor())); + language_registry.add(mermaid_language); + + let source = "```mermaid\ngraph TD;\n```"; + let markdown = cx.new(|cx| { + Markdown::new_with_options( + source.into(), + Some(language_registry), + None, + MarkdownOptions { + render_mermaid_diagrams: true, + ..MarkdownOptions::default() + }, + cx, + ) + }); + cx.run_until_parked(); + + markdown.read_with(cx, |markdown, _| { + let parsed = markdown.parsed_markdown(); + assert_eq!( + parsed.mermaid_diagrams.len(), + 1, + "the block must be recognized as a mermaid diagram" + ); + assert_eq!( + parsed.code_block_highlights.len(), + 0, + "mermaid blocks must not be highlighted" + ); + }); + } + #[gpui::test] fn test_fallback_language_highlights_untagged_code_blocks(cx: &mut TestAppContext) { let language = language::rust_lang(); + language.set_theme(&rust_test_theme()); let language_registry = Arc::new(LanguageRegistry::test(cx.executor())); language_registry.add(language.clone()); @@ -5186,19 +5395,19 @@ mod tests { cx.run_until_parked(); markdown.read_with(cx, |markdown, _| { - let fallback = markdown - .parsed_markdown() + let parsed = markdown.parsed_markdown(); + let fallback = parsed .fallback_code_block_language .as_ref() .expect("the fallback language must be resolved for untagged code blocks"); assert_eq!(fallback.name(), language.name()); - assert_eq!( - markdown - .first_code_block_language() - .map(|language| language.name()), - Some(language.name()), - "untagged code blocks must resolve to the fallback language" - ); + + let code_start = source.find("fn main").unwrap(); + let cached = parsed + .code_block_highlights + .get(&code_start) + .expect("untagged code blocks must be highlighted with the fallback language"); + assert!(!cached.runs.is_empty()); }); } @@ -7167,4 +7376,53 @@ mod tests { }; (prose.layout.line_height(), code.layout.line_height()) } + + fn rust_test_theme() -> SyntaxTheme { + SyntaxTheme::new( + [ + ("keyword", gpui::red()), + ("function", gpui::blue()), + ("type", gpui::green()), + ] + .into_iter() + .map(|(name, color)| { + ( + name.to_owned(), + gpui::HighlightStyle { + color: Some(color), + ..gpui::HighlightStyle::default() + }, + ) + }), + ) + } + + fn markdown_with_rust_language( + source: &str, + cx: &mut TestAppContext, + ) -> (Arc, Entity) { + let language = language::rust_lang(); + language.set_theme(&rust_test_theme()); + let language_registry = Arc::new(LanguageRegistry::test(cx.executor())); + language_registry.add(language.clone()); + let source = SharedString::from(source.to_owned()); + let markdown = cx.new(|cx| Markdown::new(source, Some(language_registry), None, cx)); + cx.run_until_parked(); + (language, markdown) + } + + fn cached_code_block_highlights( + markdown: &Entity, + code_start: usize, + cx: &mut TestAppContext, + ) -> ResolvedHighlights { + markdown.read_with(cx, |markdown, _| { + markdown + .parsed_markdown() + .code_block_highlights + .get(&code_start) + .expect("code block highlights must be computed during parse") + .clone() + }) + } }