From ed192bf4f1edcaba15dacc138a5b9487874f2f1c Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 9 Sep 2026 14:31:14 +0000 Subject: [PATCH] markdown: Cache code block syntax highlighting at parse time instead of re-highlighting every frame (#63138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of https://github.com/zed-industries/zed/issues/57349 `MarkdownElement` re-highlights every code block with tree-sitter on every frame. Each visible code block pays a full parse per frame, which makes agent panel streaming and hover popovers janky once the content grows. This PR computes code block highlights once, in the background parse task, instead of at render time: - Highlights are resolved into `(Range, HighlightId)` runs at parse time; each entry stores the grammar and `HighlightMap` it was resolved against and is served only while the grammar still holds the pointer-identical map (`ResolvedHighlights::is_current`), so theme changes need no invalidation hooks, and a block whose entry went stale falls back to direct fragment highlighting at render until the next parse. - `Language::highlight_text_resolved` memoizes results in a content-keyed LRU (4MB budget, 512KB per-entry cap), so the re-parses triggered by streaming appends reuse the highlights of unchanged blocks instead of re-running tree-sitter on them. - Blocks larger than the per-entry cap bypass the cache and are highlighted directly, as before. `CostBudgetedLru` is generic rather than specialized because a follow-up PR reuses it for per-buffer highlight caching in `BufferSnapshot::chunks`; the capture flattening keyed by grammar index (`HighlightCaptureRef`) is shared with that follow-up as well. ## Benchmarks `cargo bench -p benchmarks --bench markdown_renderer` renders generated documents that mix paragraphs, headings, tables, and Rust code blocks. **Run 1 — baseline, `main` (`ee2366a395`):** ``` Markdown render/min_bytes/5000 time: [1.2956 ms 1.2974 ms 1.3016 ms] Markdown render/min_bytes/10000 time: [1.7919 ms 1.7955 ms 1.8026 ms] Markdown render/min_bytes/50000 time: [6.5510 ms 6.6241 ms 6.6671 ms] Markdown render/min_bytes/250000 time: [37.644 µs 2.4819 ms 4.3791 ms] <- bimodal already on main ``` **Run 2 — this branch:** ``` Markdown render/min_bytes/5000 time: [940.11 µs 945.28 µs 948.79 µs] change: [-29.317% -28.385% -27.607%] (p = 0.00 < 0.05) Markdown render/min_bytes/10000 time: [1.1162 ms 1.1331 ms 1.1407 ms] change: [-40.859% -39.151% -37.719%] (p = 0.00 < 0.05) Markdown render/min_bytes/50000 time: [2.4943 ms 2.9225 ms 3.1051 ms] change: [-80.088% -67.885% -58.058%] (p = 0.00 < 0.05) Markdown render/min_bytes/250000 time: [43.442 µs 44.661 µs 45.514 µs] change: [-98.210% -95.806% +5.5167%] (p = 0.17 > 0.05) ``` Over the same runs, GPUI reports window draw mean 2.22 ms → 1.26 ms, p99 31.2 ms → 16.0 ms, and the share of frames that blow the frame budget drops from 3.9% to 2.0%. Note: at the 250,000-byte case root blocks render lazily, which makes the measurement bimodal and inconclusive in both configurations. Release Notes: - Improved rendering performance of markdown code blocks (agent panel, hover popovers, markdown preview) --- Cargo.lock | 2 + crates/language/src/buffer.rs | 12 +- crates/language/src/language.rs | 259 ++++++++++-- crates/language/src/syntax_map.rs | 76 +++- .../src/syntax_map/syntax_map_tests.rs | 59 +++ crates/language_core/Cargo.toml | 2 + crates/language_core/src/grammar.rs | 12 +- crates/language_core/src/highlight_cache.rs | 284 +++++++++++++ crates/language_core/src/highlight_map.rs | 18 +- crates/language_core/src/language_core.rs | 3 +- .../src/highlights_tree_view.rs | 3 +- crates/languages/src/rust.rs | 9 +- crates/markdown/src/markdown.rs | 396 +++++++++++++++--- 13 files changed, 1020 insertions(+), 115 deletions(-) create mode 100644 crates/language_core/src/highlight_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 1e74bce459..38da980bb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9555,12 +9555,14 @@ dependencies = [ "gpui_shared_string", "gpui_util", "log", + "lru", "parking_lot", "path", "regex", "schemars 1.0.4", "serde", "serde_json", + "smallvec", "strum 0.28.0", "toml 0.8.23", "tree-sitter", diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 5bf8e87112..246b786ee3 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -19,8 +19,8 @@ use crate::{ unified_diff_with_offsets, }; pub use crate::{ - Grammar, HighlightId, HighlightMap, Language, LanguageRegistry, diagnostic_set::DiagnosticSet, - proto, + CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry, + diagnostic_set::DiagnosticSet, proto, }; use anyhow::{Context as _, Result}; @@ -5553,8 +5553,8 @@ impl<'a> BufferChunks<'a> { { let next_capture_end = capture.node.end_byte(); if range.start < next_capture_end - && let Some(capture_id) = - highlights.highlight_maps[capture.grammar_index].get(capture.index) + && let Some(capture_id) = highlights.highlight_maps[capture.grammar_index] + .get(CaptureId(capture.index)) { highlights.stack.push((next_capture_end, capture_id)); } @@ -5689,8 +5689,8 @@ impl<'a> Iterator for BufferChunks<'a> { next_capture_start = capture.node.start_byte(); break; } else { - let highlight_id = - highlights.highlight_maps[capture.grammar_index].get(capture.index); + let highlight_id = highlights.highlight_maps[capture.grammar_index] + .get(CaptureId(capture.index)); if let Some(highlight_id) = highlight_id { highlights .stack diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index a7791c51e7..6dcb998500 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -44,10 +44,12 @@ use http_client::HttpClient; pub use language_core::{ SymbolKind, - highlight_map::{HighlightId, HighlightMap}, + highlight_cache::ResolvedHighlights, + highlight_map::{CaptureId, HighlightId, HighlightMap}, }; use futures::future::FutureExt as _; +use language_core::highlight_cache::{MAX_TEXT_HIGHLIGHT_ENTRY_BYTES, TextHighlightKey}; pub use language_core::{ BlockCommentConfig, BracketPair, BracketPairConfig, BracketPairContent, BracketsConfig, BracketsPatternConfig, CodeLabel, CodeLabelBuilder, DebugVariablesConfig, DebuggerTextObject, @@ -83,7 +85,7 @@ use std::{ str, sync::{Arc, LazyLock}, }; -use syntax_map::{QueryCursorHandle, SyntaxSnapshot}; +use syntax_map::{QueryCursorHandle, SyntaxSnapshot, flattened_highlight_regions}; use task::RunnableTag; pub use task_context::{ContextLocation, ContextProvider}; pub use text_diff::{ @@ -1120,29 +1122,94 @@ impl Language { text: &'a Rope, range: Range, ) -> Vec<(Range, HighlightId)> { - let mut result = Vec::new(); - if let Some(grammar) = &self.grammar { - let tree = parse_text(grammar, text, None); - let captures = - SyntaxSnapshot::single_tree_captures(range.clone(), text, &tree, self, |grammar| { - grammar - .highlights_config - .as_ref() - .map(|config| &config.query) - }); - let highlight_maps = vec![grammar.highlight_map()]; - let mut offset = 0; - for chunk in - BufferChunks::new(text, range, Some((captures, highlight_maps)), false, None) + self.highlight_text_resolved(text, range).runs.to_vec() + } + + pub fn highlight_text_resolved( + self: &Arc, + text: &Rope, + range: Range, + ) -> ResolvedHighlights { + let Some(grammar) = &self.grammar else { + return ResolvedHighlights::default(); + }; + let Some(highlights_config) = &grammar.highlights_config else { + return ResolvedHighlights::default(); + }; + let highlights = if text.len() > MAX_TEXT_HIGHLIGHT_ENTRY_BYTES { + self.compute_resolved_highlights(grammar, text) + } else { + let key = TextHighlightKey::new(text.chunks(), text.len()); + match highlights_config + .text_highlight_cache + .get(&key, text.chunks()) { - let end_offset = offset + chunk.text.len(); - if let Some(highlight_id) = chunk.syntax_highlight_id { - result.push((offset..end_offset, highlight_id)); + Some(highlights) => highlights, + None => highlights_config.text_highlight_cache.insert( + key, + Arc::from(text.chunks().collect::()), + self.compute_resolved_highlights(grammar, text), + ), + } + }; + if range.start == 0 && range.end >= text.len() { + return highlights; + } + ResolvedHighlights { + sources: highlights.sources.clone(), + runs: highlights + .runs + .iter() + .filter(|(run_range, _)| run_range.start < range.end && run_range.end > range.start) + .map(|(run_range, highlight_id)| { + ( + run_range.start.max(range.start) - range.start + ..run_range.end.min(range.end) - range.start, + *highlight_id, + ) + }) + .collect(), + } + } + + fn compute_resolved_highlights( + self: &Arc, + grammar: &Arc, + text: &Rope, + ) -> ResolvedHighlights { + let highlight_map = grammar.highlight_map(); + let tree = parse_text(grammar, text, None); + let captures = + SyntaxSnapshot::single_tree_captures(0..text.len(), text, &tree, self, |grammar| { + grammar + .highlights_config + .as_ref() + .map(|config| &config.query) + }); + let mut runs = Vec::<(Range, HighlightId)>::new(); + for region in flattened_highlight_regions(captures, 0..text.len()) { + let highlight_id = region + .stack + .iter() + .rev() + .find_map(|capture| highlight_map.get(capture.capture_id)); + let Some(highlight_id) = highlight_id else { + continue; + }; + match runs.last_mut() { + Some((last_range, last_highlight_id)) + if *last_highlight_id == highlight_id + && last_range.end == region.range.start => + { + last_range.end = region.range.end; } - offset = end_offset; + _ => runs.push((region.range, highlight_id)), } } - result + ResolvedHighlights { + sources: [(Arc::clone(grammar), highlight_map)].into_iter().collect(), + runs: runs.into(), + } } pub fn path_suffixes(&self) -> &[String] { @@ -1686,19 +1753,163 @@ mod tests { let map = build_highlight_map(capture_names, &theme); assert_eq!( - theme.get_capture_name(map.get(0).unwrap()), + theme.get_capture_name(map.get(CaptureId(0)).unwrap()), Some("function") ); assert_eq!( - theme.get_capture_name(map.get(1).unwrap()), + theme.get_capture_name(map.get(CaptureId(1)).unwrap()), Some("function.async") ); assert_eq!( - theme.get_capture_name(map.get(2).unwrap()), + theme.get_capture_name(map.get(CaptureId(2)).unwrap()), Some("variable.builtin") ); } + #[test] + fn test_highlight_text_resolves_nested_captures_with_theme_fallback() { + let language = Arc::new( + Language::new( + LanguageConfig { + name: "Rust".into(), + ..LanguageConfig::default() + }, + Some(tree_sitter_rust::LANGUAGE.into()), + ) + .with_highlights_query( + r#" + (function_item) @function.definition + (identifier) @variable + "fn" @keyword + "#, + ) + .unwrap(), + ); + + let theme = SyntaxTheme::new( + [ + ("function", rgba(0x100000ff)), + ("keyword", rgba(0x200000ff)), + ] + .iter() + .map(|(name, color)| (name.to_string(), (*color).into())), + ); + language.set_theme(&theme); + + let code = "fn main() {}"; + let highlights = language.highlight_text_resolved(&Rope::from(code), 0..code.len()); + assert!(highlights.is_current()); + let named = highlights + .runs + .iter() + .map(|(range, highlight_id)| { + ( + range.clone(), + theme.get_capture_name(*highlight_id).unwrap(), + ) + }) + .collect::>(); + assert_eq!( + named, + vec![(0..2, "keyword"), (2..12, "function")], + "an inner capture missing from the theme must fall back to its outer capture" + ); + + let memoized = language.highlight_text_resolved(&Rope::from(code), 0..code.len()); + assert!( + Arc::ptr_eq(&highlights.runs, &memoized.runs), + "repeated highlighting of the same text must be memoized" + ); + let partial = language.highlight_text_resolved(&Rope::from(code), 0..2); + assert!( + !Arc::ptr_eq(&highlights.runs, &partial.runs), + "a different range must not reuse the memoized highlights" + ); + assert_eq!(partial.runs.as_ref(), &[(0..2, highlights.runs[0].1)]); + + let richer_theme = SyntaxTheme::new( + [ + ("function", rgba(0x100000ff)), + ("keyword", rgba(0x200000ff)), + ("variable", rgba(0x300000ff)), + ] + .iter() + .map(|(name, color)| (name.to_string(), (*color).into())), + ); + language.set_theme(&richer_theme); + assert!( + !highlights.is_current(), + "a theme change must invalidate previously resolved highlights" + ); + let rethemed = language.highlight_text_resolved(&Rope::from(code), 0..code.len()); + assert!(rethemed.is_current()); + let renamed = rethemed + .runs + .iter() + .map(|(range, highlight_id)| { + ( + range.clone(), + richer_theme.get_capture_name(*highlight_id).unwrap(), + ) + }) + .collect::>(); + assert_eq!( + renamed, + vec![ + (0..2, "keyword"), + (2..3, "function"), + (3..7, "variable"), + (7..12, "function"), + ], + "a theme change must invalidate memoized highlights and resolve against the new theme" + ); + } + + #[test] + fn test_oversized_text_is_highlighted_without_caching() { + let language = rust_lang(); + let theme = SyntaxTheme::new( + [ + ("function", rgba(0x100000ff)), + ("keyword", rgba(0x200000ff)), + ] + .iter() + .map(|(name, color)| (name.to_string(), (*color).into())), + ); + language.set_theme(&theme); + + let small_code = "fn main() {}"; + let small_highlights = + language.highlight_text_resolved(&Rope::from(small_code), 0..small_code.len()); + assert!( + !small_highlights.runs.is_empty(), + "texts within the size cap must be highlighted" + ); + + let oversized_code = format!( + "fn main() {{}}{}", + " ".repeat(MAX_TEXT_HIGHLIGHT_ENTRY_BYTES) + ); + let oversized_rope = Rope::from(oversized_code.as_str()); + let first_highlights = + language.highlight_text_resolved(&oversized_rope, 0..oversized_code.len()); + assert_eq!( + first_highlights.runs.as_ref(), + small_highlights.runs.as_ref(), + "texts over the cache entry cap must still be highlighted" + ); + let second_highlights = + language.highlight_text_resolved(&oversized_rope, 0..oversized_code.len()); + assert_eq!( + second_highlights.runs.as_ref(), + first_highlights.runs.as_ref() + ); + assert!( + !Arc::ptr_eq(&first_highlights.runs, &second_highlights.runs), + "texts over the cache entry cap must not be memoized" + ); + } + #[test] fn test_with_parser_resets_after_cancellation() { use std::ops::ControlFlow; diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index 9ff17ce2de..26a53bfd5b 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -2,11 +2,14 @@ mod syntax_map_tests; use crate::{ - Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS, with_parser, + CaptureId, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS, + with_parser, }; use collections::HashMap; use futures::FutureExt; use gpui::SharedString; +use language_core::highlight_map::HighlightCaptureRef; +use smallvec::SmallVec; use std::{ borrow::Cow, cmp::{self, Ordering, Reverse}, @@ -73,7 +76,7 @@ impl Drop for SyntaxSnapshot { pub struct SyntaxMapCaptures<'a> { layers: Vec>, active_layer_count: usize, - grammars: Vec<&'a Grammar>, + grammars: Vec<&'a Arc>, } #[derive(Default)] @@ -90,6 +93,73 @@ pub struct SyntaxMapCapture<'a> { pub grammar_index: usize, } +pub(crate) struct HighlightCaptureRegion { + pub range: Range, + pub stack: SmallVec<[HighlightCaptureRef; 4]>, +} + +pub(crate) fn flattened_highlight_regions( + mut captures: SyntaxMapCaptures<'_>, + range: Range, +) -> Vec { + let capture_refs = iter::from_fn(move || { + let capture = captures.next()?; + Some(( + capture.node.byte_range(), + HighlightCaptureRef { + grammar_index: capture.grammar_index, + capture_id: CaptureId(capture.index), + }, + )) + }); + flatten_capture_regions(range, capture_refs) +} + +fn flatten_capture_regions( + range: Range, + mut captures: impl Iterator, HighlightCaptureRef)>, +) -> Vec { + let mut result = Vec::new(); + let mut stack = Vec::<(Range, HighlightCaptureRef)>::new(); + let mut offset = range.start; + let mut next_capture = captures.next(); + loop { + stack.retain(|(capture_range, _)| capture_range.end > offset); + while let Some((capture_range, capture_ref)) = next_capture.take() { + if capture_range.start > offset { + next_capture = Some((capture_range, capture_ref)); + break; + } + if capture_range.end > offset { + stack.push((capture_range, capture_ref)); + } + next_capture = captures.next(); + } + let mut next_boundary = range.end; + if let Some(min_end) = stack + .iter() + .map(|(capture_range, _)| capture_range.end) + .min() + { + next_boundary = next_boundary.min(min_end); + } + if let Some((capture_range, _)) = &next_capture { + next_boundary = next_boundary.min(capture_range.start); + } + if !stack.is_empty() && next_boundary > offset { + result.push(HighlightCaptureRegion { + range: offset..next_boundary, + stack: stack.iter().map(|(_, capture_ref)| *capture_ref).collect(), + }); + } + if next_boundary >= range.end { + break; + } + offset = next_boundary; + } + result +} + #[derive(Debug)] pub struct SyntaxMapMatch<'a> { pub language: Arc, @@ -1201,7 +1271,7 @@ impl<'a> SyntaxMapCaptures<'a> { result } - pub fn grammars(&self) -> &[&'a Grammar] { + pub fn grammars(&self) -> &[&'a Arc] { &self.grammars } diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs index d4eecbe04c..4c57b38e6f 100644 --- a/crates/language/src/syntax_map/syntax_map_tests.rs +++ b/crates/language/src/syntax_map/syntax_map_tests.rs @@ -1341,6 +1341,48 @@ fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) { test_random_edits(text, registry, language, rng); } +#[test] +fn test_flatten_capture_regions_with_nested_captures() { + let outer = capture_ref(1); + let inner = capture_ref(2); + assert_eq!( + flattened(0..12, &[(0..10, outer), (2..5, inner)]), + vec![ + (0..2, vec![outer]), + (2..5, vec![outer, inner]), + (5..10, vec![outer]), + ], + ); +} + +#[test] +fn test_flatten_capture_regions_with_overlapping_captures() { + let first = capture_ref(1); + let second = capture_ref(2); + assert_eq!( + flattened(0..25, &[(0..10, first), (2..20, second)]), + vec![ + (0..2, vec![first]), + (2..10, vec![first, second]), + (10..20, vec![second]), + ], + "a capture must not extend past its own end when overlapping another capture" + ); +} + +#[test] +fn test_flatten_capture_regions_clips_to_the_requested_range() { + let capture = capture_ref(1); + assert_eq!( + flattened(5..8, &[(0..10, capture)]), + vec![(5..8, vec![capture])], + ); + assert_eq!( + flattened(0..6, &[(4..10, capture)]), + vec![(4..6, vec![capture])], + ); +} + fn test_random_edits( text: String, registry: Arc, @@ -1762,6 +1804,23 @@ fn sql_lang() -> Language { ) } +fn capture_ref(capture_id: u32) -> HighlightCaptureRef { + HighlightCaptureRef { + grammar_index: 0, + capture_id: CaptureId(capture_id), + } +} + +fn flattened( + range: Range, + captures: &[(Range, HighlightCaptureRef)], +) -> Vec<(Range, Vec)> { + flatten_capture_regions(range, captures.iter().cloned()) + .into_iter() + .map(|region| (region.range, region.stack.to_vec())) + .collect() +} + fn range_for_text(buffer: &Buffer, text: &str) -> Range { let start = buffer.as_rope().to_string().find(text).unwrap(); start..start + text.len() diff --git a/crates/language_core/Cargo.toml b/crates/language_core/Cargo.toml index 672d0dabd5..9941b1f8bf 100644 --- a/crates/language_core/Cargo.toml +++ b/crates/language_core/Cargo.toml @@ -14,12 +14,14 @@ collections.workspace = true gpui_shared_string.workspace = true gpui_util.workspace = true log.workspace = true +lru.workspace = true parking_lot.workspace = true path.workspace = true regex.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +smallvec.workspace = true strum.workspace = true toml.workspace = true tree-sitter.workspace = true diff --git a/crates/language_core/src/grammar.rs b/crates/language_core/src/grammar.rs index ff4dc0c885..fc45de7706 100644 --- a/crates/language_core/src/grammar.rs +++ b/crates/language_core/src/grammar.rs @@ -1,6 +1,6 @@ use crate::{ - HighlightId, HighlightMap, LanguageConfig, LanguageConfigOverride, LanguageName, - LanguageQueries, language_config::BracketPairConfig, + CaptureId, HighlightId, HighlightMap, LanguageConfig, LanguageConfigOverride, LanguageName, + LanguageQueries, highlight_cache::TextHighlightCache, language_config::BracketPairConfig, }; use anyhow::{Context as _, Result}; use collections::HashMap; @@ -103,6 +103,7 @@ pub struct Grammar { pub struct HighlightsConfig { pub query: Query, pub identifier_capture_indices: Vec, + pub text_highlight_cache: TextHighlightCache, } pub struct IndentConfig { @@ -367,12 +368,16 @@ impl Grammar { self.highlight_map.lock().clone() } + pub fn highlight_map_matches(&self, highlight_map: &HighlightMap) -> bool { + self.highlight_map.lock().same(highlight_map) + } + pub fn highlight_id_for_name(&self, name: &str) -> Option { self.highlights_config .as_ref()? .query .capture_index_for_name(name) - .and_then(|capture_id| self.highlight_map.lock().get(capture_id)) + .and_then(|capture_id| self.highlight_map.lock().get(CaptureId(capture_id))) } pub fn debug_variables_config(&self) -> Option<&DebugVariablesConfig> { @@ -469,6 +474,7 @@ impl Grammar { self.highlights_config = Some(HighlightsConfig { query, identifier_capture_indices, + text_highlight_cache: TextHighlightCache::default(), }); Ok(self) diff --git a/crates/language_core/src/highlight_cache.rs b/crates/language_core/src/highlight_cache.rs new file mode 100644 index 0000000000..b820787f71 --- /dev/null +++ b/crates/language_core/src/highlight_cache.rs @@ -0,0 +1,284 @@ +use crate::grammar::Grammar; +use crate::highlight_map::{HighlightId, HighlightMap}; +use collections::FxHasher; +use lru::LruCache; +use parking_lot::Mutex; +use smallvec::SmallVec; +use std::{ + fmt, + hash::{Hash, Hasher as _}, + ops::Range, + sync::Arc, +}; + +pub const MAX_TEXT_HIGHLIGHT_ENTRY_BYTES: usize = MAX_TEXT_HIGHLIGHT_CACHE_BYTES / 8; + +const MAX_TEXT_HIGHLIGHT_CACHE_BYTES: usize = 4 * 1024 * 1024; +const APPROXIMATE_LRU_NODE_BYTES: usize = 4 * size_of::(); + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TextHighlightKey { + text_hash: u64, + text_len: usize, +} + +impl TextHighlightKey { + pub fn new<'a>(text_chunks: impl Iterator, text_len: usize) -> Self { + let mut hasher = FxHasher::default(); + for chunk in text_chunks { + hasher.write(chunk.as_bytes()); + } + Self { + text_hash: hasher.finish(), + text_len, + } + } +} + +#[derive(Clone, Default)] +pub struct ResolvedHighlights { + pub sources: SmallVec<[(Arc, HighlightMap); 2]>, + pub runs: Arc<[(Range, HighlightId)]>, +} + +impl ResolvedHighlights { + pub fn is_current(&self) -> bool { + self.sources + .iter() + .all(|(grammar, highlight_map)| grammar.highlight_map_matches(highlight_map)) + } + + pub fn cost_bytes(&self) -> usize { + self.sources.len() * size_of::<(Arc, HighlightMap)>() + + self.runs.len() * size_of::<(Range, HighlightId)>() + } +} + +impl fmt::Debug for ResolvedHighlights { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ResolvedHighlights") + .field("runs", &self.runs) + .finish_non_exhaustive() + } +} + +pub struct TextHighlightCache(Mutex>); + +impl Default for TextHighlightCache { + fn default() -> Self { + Self(Mutex::new(CostBudgetedLru::new( + MAX_TEXT_HIGHLIGHT_CACHE_BYTES, + MAX_TEXT_HIGHLIGHT_ENTRY_BYTES, + ))) + } +} + +impl TextHighlightCache { + pub fn get<'a>( + &self, + key: &TextHighlightKey, + text_chunks: impl Iterator, + ) -> Option { + let mut cache = self.0.lock(); + let entry = cache.get(key)?; + if !entry.highlights.is_current() { + return None; + } + if !chunks_match_text(&entry.text, text_chunks) { + return None; + } + Some(entry.highlights.clone()) + } + + pub fn insert( + &self, + key: TextHighlightKey, + text: Arc, + highlights: ResolvedHighlights, + ) -> ResolvedHighlights { + let cost = text.len() + highlights.cost_bytes(); + self.0.lock().insert( + key, + TextHighlightEntry { + text, + highlights: highlights.clone(), + }, + cost, + ); + highlights + } +} + +struct CostBudgetedLru { + entries: LruCache, + total_cost: usize, + max_total_cost: usize, + max_entry_cost: usize, +} + +impl CostBudgetedLru { + const ENTRY_OVERHEAD_BYTES: usize = + size_of::() + size_of::<(V, usize)>() + APPROXIMATE_LRU_NODE_BYTES; + + fn new(max_total_cost: usize, max_entry_cost: usize) -> Self { + Self { + entries: LruCache::unbounded(), + total_cost: 0, + max_total_cost, + max_entry_cost, + } + } + + fn get(&mut self, key: &K) -> Option<&V> { + self.entries.get(key).map(|(value, _)| value) + } + + fn insert(&mut self, key: K, value: V, cost: usize) { + if cost > self.max_entry_cost { + return; + } + let cost = cost + Self::ENTRY_OVERHEAD_BYTES; + if let Some((_, old_cost)) = self.entries.put(key, (value, cost)) { + self.total_cost -= old_cost; + } + self.total_cost += cost; + while self.total_cost > self.max_total_cost { + let Some((_, (_, evicted_cost))) = self.entries.pop_lru() else { + break; + }; + self.total_cost -= evicted_cost; + } + } +} + +struct TextHighlightEntry { + text: Arc, + highlights: ResolvedHighlights, +} + +fn chunks_match_text<'a>(text: &str, text_chunks: impl Iterator) -> bool { + let mut remaining = text; + for chunk in text_chunks { + let Some(rest) = remaining.strip_prefix(chunk) else { + return false; + }; + remaining = rest; + } + remaining.is_empty() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_budget_evicts_least_recently_used() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(2 * (40 + overhead), 100); + cache.insert("a", 1, 40); + cache.insert("b", 2, 40); + cache.get(&"a"); + cache.insert("c", 3, 40); + assert_eq!(cache.get(&"a"), Some(&1)); + assert_eq!(cache.get(&"b"), None); + assert_eq!(cache.get(&"c"), Some(&3)); + assert_eq!(cache.total_cost, 2 * (40 + overhead)); + } + + #[test] + fn test_replacing_an_entry_updates_the_budget() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(100 + 2 * overhead, 100); + cache.insert("a", 1, 60); + cache.insert("a", 2, 30); + assert_eq!(cache.total_cost, 30 + overhead); + assert_eq!(cache.get(&"a"), Some(&2)); + cache.insert("b", 3, 70); + assert_eq!(cache.get(&"a"), Some(&2)); + assert_eq!(cache.get(&"b"), Some(&3)); + } + + #[test] + fn test_oversized_entries_are_rejected_without_flushing() { + let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::<&str, u32>::new(100 + overhead, 50); + cache.insert("a", 1, 40); + cache.insert("b", 2, 60); + assert_eq!(cache.get(&"a"), Some(&1)); + assert_eq!(cache.get(&"b"), None); + assert_eq!(cache.total_cost, 40 + overhead); + } + + #[test] + fn test_zero_cost_entries_still_consume_budget() { + let overhead = CostBudgetedLru::::ENTRY_OVERHEAD_BYTES; + let mut cache = CostBudgetedLru::::new(3 * overhead, 100); + for key in 0..5_usize { + cache.insert(key, key as u32, 0); + } + assert_eq!(cache.get(&0), None); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.get(&2), Some(&2)); + assert_eq!(cache.get(&3), Some(&3)); + assert_eq!(cache.get(&4), Some(&4)); + assert_eq!( + cache.total_cost, + 3 * overhead, + "Entries with zero caller-supplied cost must still be bounded by the budget" + ); + } + + #[test] + fn test_oversized_highlights_are_returned_but_not_cached() { + let text = Arc::::from("fn main() {}"); + let run_bytes = size_of::<(Range, HighlightId)>(); + let max_entry_cost = text.len() + 2 * run_bytes; + let cache = TextHighlightCache(Mutex::new(CostBudgetedLru::new( + max_entry_cost * 10, + max_entry_cost, + ))); + let resolved = |ranges: &[Range]| ResolvedHighlights { + sources: SmallVec::new(), + runs: ranges + .iter() + .map(|range| (range.clone(), HighlightId::new(0))) + .collect(), + }; + + let small_text = Arc::::from("fn f() {}"); + let small_key = TextHighlightKey::new([small_text.as_ref()].into_iter(), small_text.len()); + let small_range = 0..2; + let small_highlights = cache.insert( + small_key.clone(), + Arc::clone(&small_text), + resolved(std::slice::from_ref(&small_range)), + ); + assert_eq!( + small_highlights.runs.as_ref(), + resolved(std::slice::from_ref(&small_range)).runs.as_ref() + ); + assert_eq!( + cache + .get(&small_key, [small_text.as_ref()].into_iter()) + .map(|highlights| highlights.runs), + Some(small_highlights.runs.clone()), + "highlights within the entry budget must be cached as-is" + ); + + let big_key = TextHighlightKey::new([text.as_ref()].into_iter(), text.len()); + let big_highlights_source = resolved(&[0..2, 3..7, 8..9]); + let big_highlights = cache.insert( + big_key.clone(), + Arc::clone(&text), + big_highlights_source.clone(), + ); + assert!( + Arc::ptr_eq(&big_highlights.runs, &big_highlights_source.runs), + "highlights over the entry budget must be returned unchanged" + ); + assert!( + cache.get(&big_key, [text.as_ref()].into_iter()).is_none(), + "highlights over the entry budget must not be cached" + ); + } +} diff --git a/crates/language_core/src/highlight_map.rs b/crates/language_core/src/highlight_map.rs index cba5cda6f7..b5ddcf670e 100644 --- a/crates/language_core/src/highlight_map.rs +++ b/crates/language_core/src/highlight_map.rs @@ -3,6 +3,15 @@ use std::{num::NonZeroU32, sync::Arc}; #[derive(Clone, Debug)] pub struct HighlightMap(Arc<[Option]>); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct CaptureId(pub u32); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HighlightCaptureRef { + pub grammar_index: usize, + pub capture_id: CaptureId, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct HighlightId(NonZeroU32); @@ -28,8 +37,13 @@ impl HighlightMap { } #[inline] - pub fn get(&self, capture_id: u32) -> Option { - self.0.get(capture_id as usize).copied().flatten() + pub fn get(&self, capture_id: CaptureId) -> Option { + self.0.get(capture_id.0 as usize).copied().flatten() + } + + #[inline] + pub fn same(&self, other: &HighlightMap) -> bool { + Arc::ptr_eq(&self.0, &other.0) } } diff --git a/crates/language_core/src/language_core.rs b/crates/language_core/src/language_core.rs index f3cb3ab51c..e2050f325e 100644 --- a/crates/language_core/src/language_core.rs +++ b/crates/language_core/src/language_core.rs @@ -2,6 +2,7 @@ // language configuration, and highlight mapping. pub mod grammar; +pub mod highlight_cache; pub mod highlight_map; pub mod language_config; @@ -11,7 +12,7 @@ pub use grammar::{ NEXT_GRAMMAR_ID, OutlineConfig, OverrideConfig, OverrideEntry, ParseableLanguage, RedactionConfig, RunnableCapture, RunnableConfig, TextObject, TextObjectConfig, }; -pub use highlight_map::{HighlightId, HighlightMap}; +pub use highlight_map::{CaptureId, HighlightId, HighlightMap}; pub use language_config::{ BlockCommentConfig, BracketPair, BracketPairConfig, BracketPairContent, DecreaseIndentConfig, JsxTagAutoCloseConfig, LanguageConfig, LanguageConfigOverride, LanguageMatcher, diff --git a/crates/language_tools/src/highlights_tree_view.rs b/crates/language_tools/src/highlights_tree_view.rs index e63f6e3051..b3ac451d54 100644 --- a/crates/language_tools/src/highlights_tree_view.rs +++ b/crates/language_tools/src/highlights_tree_view.rs @@ -1190,7 +1190,8 @@ fn build_highlight_entries( .collect::>(); for capture in captures { - let Some(highlight_id) = highlight_maps[capture.grammar_index].get(capture.index) + let Some(highlight_id) = + highlight_maps[capture.grammar_index].get(language::CaptureId(capture.index)) else { continue; }; diff --git a/crates/languages/src/rust.rs b/crates/languages/src/rust.rs index 5bb3089c79..d0f468a6b2 100644 --- a/crates/languages/src/rust.rs +++ b/crates/languages/src/rust.rs @@ -1842,8 +1842,7 @@ mod tests { vec![ (10..13, HighlightId::TABSTOP_INSERT_ID), (16..19, HighlightId::TABSTOP_INSERT_ID), - (0..7, HighlightId::new(2)), - (7..8, HighlightId::new(2)), + (0..8, HighlightId::new(2)), ], )) ); @@ -1870,8 +1869,7 @@ mod tests { 0..4, vec![ (5..9, HighlightId::TABSTOP_REPLACE_ID), - (0..3, HighlightId::new(2)), - (3..4, HighlightId::new(2)), + (0..4, HighlightId::new(2)), ], )) ); @@ -1956,8 +1954,7 @@ mod tests { vec![ (15..20, HighlightId::TABSTOP_REPLACE_ID), (16..19, HighlightId::TABSTOP_INSERT_ID), - (0..13, HighlightId::new(2)), - (13..14, HighlightId::new(2)), + (0..14, HighlightId::new(2)), ], )) ); diff --git a/crates/markdown/src/markdown.rs b/crates/markdown/src/markdown.rs index c8f10d3117..9dc936e3c7 100644 --- a/crates/markdown/src/markdown.rs +++ b/crates/markdown/src/markdown.rs @@ -5,7 +5,6 @@ mod path_range; mod selection; use base64::Engine as _; - use gpui::EdgesRefinement; use gpui::HitboxBehavior; use gpui::UnderlineStyle; @@ -41,7 +40,7 @@ use gpui::{ TextStyle, TextStyleRefinement, WrappedLineLayout, actions, canvas, img, point, quad, relative, size, }; -use language::{CharClassifier, Language, LanguageRegistry, Rope}; +use language::{CharClassifier, Language, LanguageRegistry, ResolvedHighlights, Rope}; use parser::CodeBlockMetadata; use parser::{ MarkdownEvent, MarkdownTag, MarkdownTagEnd, ParsedMetadataBlock, parse_links_only, @@ -977,20 +976,7 @@ impl Markdown { return None; }; - match kind { - CodeBlockKind::FencedLang(language) => self - .parsed_markdown - .languages_by_name - .get(language) - .cloned(), - CodeBlockKind::FencedSrc(path_range) => self - .parsed_markdown - .languages_by_path - .get(&path_range.path) - .cloned(), - CodeBlockKind::Fenced => self.parsed_markdown.fallback_code_block_language.clone(), - CodeBlockKind::Indented => None, - } + self.parsed_markdown.code_block_language(kind) }) } @@ -1258,6 +1244,7 @@ impl Markdown { footnote_definitions: HashMap::default(), link_definition_spans: Arc::default(), fallback_code_block_language: None, + code_block_highlights: Arc::default(), }, Default::default(), ); @@ -1335,23 +1322,23 @@ impl Markdown { } } - ( - ParsedMarkdown { - source, - events: Arc::from(events), - languages_by_name, - languages_by_path, - root_block_starts: Arc::from(root_block_starts), - html_blocks, - metadata_blocks, - mermaid_diagrams, - heading_slugs, - footnote_definitions, - link_definition_spans: Arc::from(link_definition_spans), - fallback_code_block_language, - }, - images_by_source_offset, - ) + let mut parsed = ParsedMarkdown { + source, + events: Arc::from(events), + languages_by_name, + languages_by_path, + root_block_starts: Arc::from(root_block_starts), + html_blocks, + metadata_blocks, + mermaid_diagrams, + heading_slugs, + footnote_definitions, + link_definition_spans: Arc::from(link_definition_spans), + fallback_code_block_language, + code_block_highlights: Arc::default(), + }; + parsed.code_block_highlights = Arc::new(compute_code_block_highlights(&parsed)); + (parsed, images_by_source_offset) }); cx.spawn(async move |this, cx| { @@ -1497,8 +1484,11 @@ pub struct ParsedMarkdown { pub footnote_definitions: HashMap, pub(crate) link_definition_spans: Arc<[Range]>, pub(crate) fallback_code_block_language: Option>, + pub(crate) code_block_highlights: Arc, } +pub(crate) type CodeBlockHighlights = HashMap; + impl ParsedMarkdown { pub fn source(&self) -> &SharedString { &self.source @@ -1606,6 +1596,109 @@ impl ParsedMarkdown { selection, ) } + + pub(crate) fn code_block_language(&self, kind: &CodeBlockKind) -> Option> { + match kind { + CodeBlockKind::FencedLang(name) => self.languages_by_name.get(name).cloned(), + CodeBlockKind::FencedSrc(path_range) => { + self.languages_by_path.get(&path_range.path).cloned() + } + CodeBlockKind::Fenced => self.fallback_code_block_language.clone(), + CodeBlockKind::Indented => None, + } + } +} + +struct PendingCodeBlock<'a> { + language: Arc, + texts: Vec<(Range, &'a str)>, +} + +fn compute_code_block_highlights(parsed: &ParsedMarkdown) -> CodeBlockHighlights { + let mut code_block_highlights = CodeBlockHighlights::default(); + let mut pending_block: Option = None; + for (range, event) in parsed.events.iter() { + match event { + MarkdownEvent::Start(MarkdownTag::CodeBlock { kind, .. }) => { + if parsed.mermaid_diagrams.contains_key(&range.start) { + pending_block = None; + continue; + } + pending_block = parsed + .code_block_language(kind) + .map(|language| PendingCodeBlock { + language, + texts: Vec::new(), + }); + } + MarkdownEvent::End(MarkdownTagEnd::CodeBlock) => { + if let Some(block) = pending_block.take() { + highlight_code_block(block, &mut code_block_highlights); + } + } + MarkdownEvent::Text => { + if let Some(block) = &mut pending_block { + block + .texts + .push((range.clone(), &parsed.source[range.clone()])); + } + } + MarkdownEvent::SubstitutedText(text) => { + if let Some(block) = &mut pending_block { + block.texts.push((range.clone(), text.as_str())); + } + } + _ => {} + } + } + code_block_highlights +} + +fn highlight_code_block(block: PendingCodeBlock, code_block_highlights: &mut CodeBlockHighlights) { + let mut combined = String::new(); + let mut text_offsets = Vec::with_capacity(block.texts.len()); + for (_, text) in &block.texts { + text_offsets.push(combined.len()); + combined.push_str(text); + } + let resolved = block + .language + .highlight_text_resolved(&Rope::from(combined.as_str()), 0..combined.len()); + if resolved.runs.is_empty() { + return; + } + if let [(source_range, _)] = block.texts.as_slice() { + code_block_highlights.insert(source_range.start, resolved); + return; + } + let mut runs = resolved.runs.iter().peekable(); + for ((source_range, text), text_offset) in block.texts.iter().zip(text_offsets) { + let text_end = text_offset + text.len(); + let mut text_runs = Vec::new(); + while let Some((run_range, highlight_id)) = runs.peek() { + if run_range.start >= text_end { + break; + } + let start = run_range.start.max(text_offset); + let end = run_range.end.min(text_end); + if end > start { + text_runs.push((start - text_offset..end - text_offset, *highlight_id)); + } + if run_range.end > text_end { + break; + } + runs.next(); + } + if !text_runs.is_empty() { + code_block_highlights.insert( + source_range.start, + ResolvedHighlights { + sources: resolved.sources.clone(), + runs: text_runs.into(), + }, + ); + } + } } pub enum AutoscrollBehavior { @@ -2491,12 +2584,7 @@ impl Element for MarkdownElement { next_search_highlight_ix: 0, } }; - let mut builder = MarkdownElementBuilder::new( - &self.style.container_style, - self.style.base_text_style.clone(), - self.style.syntax.clone(), - highlights, - ); + let (parsed_markdown, images, active_root_block, render_mermaid_diagrams, mermaid_state) = { let markdown = self.markdown.read(cx); ( @@ -2507,6 +2595,13 @@ impl Element for MarkdownElement { markdown.mermaid_state.clone(), ) }; + let mut builder = MarkdownElementBuilder::new( + &self.style.container_style, + self.style.base_text_style.clone(), + self.style.syntax.clone(), + highlights, + parsed_markdown.code_block_highlights.clone(), + ); let markdown_end = if let Some(last) = parsed_markdown.events.last() { last.0.end } else { @@ -2670,19 +2765,7 @@ impl Element for MarkdownElement { continue; } - let language = match kind { - CodeBlockKind::Fenced => { - parsed_markdown.fallback_code_block_language.clone() - } - CodeBlockKind::FencedLang(language) => { - parsed_markdown.languages_by_name.get(language).cloned() - } - CodeBlockKind::FencedSrc(path_range) => parsed_markdown - .languages_by_path - .get(&path_range.path) - .cloned(), - _ => None, - }; + let language = parsed_markdown.code_block_language(kind); let is_indented = matches!(kind, CodeBlockKind::Indented); let scroll_handle = if self.style.code_block_overflow_x_scroll { @@ -3587,6 +3670,7 @@ struct MarkdownElementBuilder { base_text_style: TextStyle, text_style_stack: Vec, code_block_stack: Vec>>, + code_block_highlights: Arc, link_depth: usize, list_stack: Vec, table: TableState, @@ -3688,6 +3772,7 @@ impl MarkdownElementBuilder { base_text_style: TextStyle, syntax_theme: Arc, highlights: MarkdownHighlights, + code_block_highlights: Arc, ) -> Self { Self { div_stack: vec![{ @@ -3706,6 +3791,7 @@ impl MarkdownElementBuilder { base_text_style, text_style_stack: Vec::new(), code_block_stack: Vec::new(), + code_block_highlights, link_depth: 0, list_stack: Vec::new(), table: TableState::default(), @@ -3937,24 +4023,34 @@ impl MarkdownElementBuilder { // Compute the base text style once let text_style = self.text_style(); - if let Some(Some(language)) = self.code_block_stack.last() { + if let Some(language) = self.code_block_stack.last().and_then(Option::as_ref) + && let Some(resolved) = self.code_block_highlights.get(&source_range.start) + { + let runs = if resolved.is_current() { + resolved.runs.clone() + } else { + language + .highlight_text_resolved(&Rope::from(text), 0..text.len()) + .runs + }; let mut offset = 0; - for (range, highlight_id) in language.highlight_text(&Rope::from(text), 0..text.len()) { - if range.start > offset { + for (run_range, highlight_id) in runs.iter() { + if run_range.start > offset { self.pending_line .runs - .push(text_style.to_run(range.start - offset)); + .push(text_style.to_run(run_range.start - offset)); } - let run_len = range.len(); - if let Some(highlight) = self.syntax_theme.get(highlight_id).cloned() { + let run_len = run_range.len(); + let highlight = self.syntax_theme.get(*highlight_id).cloned(); + if let Some(highlight) = highlight { self.pending_line .runs .push(text_style.clone().highlight(highlight).to_run(run_len)); } else { self.pending_line.runs.push(text_style.to_run(run_len)); } - offset = range.end; + offset = run_range.end; } if offset < text.len() { @@ -4080,7 +4176,11 @@ impl MarkdownElementBuilder { layout: text.layout().clone(), source_mappings: line.source_mappings, source_end: self.current_source_index, - language: self.code_block_stack.last().cloned().flatten(), + language: self + .code_block_stack + .last() + .and_then(|entry| entry.as_ref()) + .cloned(), text_align, highlights, code_chips: line.code_chips.into_iter().collect(), @@ -5164,9 +5264,118 @@ mod tests { }); } + #[gpui::test] + fn test_code_block_highlights_cached_at_parse_time(cx: &mut TestAppContext) { + let source = "```rust\nfn main() {}\n```"; + let (language, markdown) = markdown_with_rust_language(source, cx); + + let code_text = "fn main() {}\n"; + let code_start = source.find(code_text).unwrap(); + let cached = cached_code_block_highlights(&markdown, code_start, cx); + let expected = language.highlight_text(&Rope::from(code_text), 0..code_text.len()); + assert_eq!( + cached.runs.to_vec(), + expected, + "highlights cached at parse time must match direct highlighting" + ); + assert!(!expected.is_empty(), "rust code must produce highlights"); + assert!(cached.is_current()); + + language.set_theme(&rust_test_theme()); + let stale = cached_code_block_highlights(&markdown, code_start, cx); + assert!( + !stale.is_current(), + "a theme change must make parse-time highlights stale so rendering re-resolves them" + ); + } + + #[gpui::test] + fn test_code_block_highlights_reused_when_streaming(cx: &mut TestAppContext) { + let source = "```rust\nfn main() {}\n```\n\nSome trailing text"; + let (language, markdown) = markdown_with_rust_language(source, cx); + + let code_start = source.find("fn main").unwrap(); + let first_parse_highlights = cached_code_block_highlights(&markdown, code_start, cx); + + markdown.update(cx, |markdown, cx| { + markdown.append(" that keeps streaming in\n\n```rust\nlet x = 1;\n```", cx) + }); + cx.run_until_parked(); + + let second_parse_highlights = cached_code_block_highlights(&markdown, code_start, cx); + assert!( + Arc::ptr_eq(&first_parse_highlights.runs, &second_parse_highlights.runs), + "highlights of an unchanged code block must be reused across parses" + ); + + markdown.update(cx, |markdown, cx| { + markdown.replace("```rust\nfn main() { panic!() }\n```", cx) + }); + cx.run_until_parked(); + + let changed_highlights = cached_code_block_highlights(&markdown, code_start, cx); + let changed_code = "fn main() { panic!() }\n"; + assert_eq!( + changed_highlights.runs.as_ref(), + language + .highlight_text_resolved(&Rope::from(changed_code), 0..changed_code.len()) + .runs + .as_ref(), + "highlights of a changed code block must be recomputed" + ); + } + + #[gpui::test] + fn test_mermaid_blocks_are_not_highlighted(cx: &mut TestAppContext) { + ensure_theme_initialized(cx); + let mermaid_language = Arc::new( + Language::new( + LanguageConfig { + name: "mermaid".into(), + ..LanguageConfig::default() + }, + Some(language::rust_lang().grammar().unwrap().ts_language.clone()), + ) + .with_highlights_query("(identifier) @variable") + .unwrap(), + ); + let language_registry = Arc::new(LanguageRegistry::test(cx.executor())); + language_registry.add(mermaid_language); + + let source = "```mermaid\ngraph TD;\n```"; + let markdown = cx.new(|cx| { + Markdown::new_with_options( + source.into(), + Some(language_registry), + None, + MarkdownOptions { + render_mermaid_diagrams: true, + ..MarkdownOptions::default() + }, + cx, + ) + }); + cx.run_until_parked(); + + markdown.read_with(cx, |markdown, _| { + let parsed = markdown.parsed_markdown(); + assert_eq!( + parsed.mermaid_diagrams.len(), + 1, + "the block must be recognized as a mermaid diagram" + ); + assert_eq!( + parsed.code_block_highlights.len(), + 0, + "mermaid blocks must not be highlighted" + ); + }); + } + #[gpui::test] fn test_fallback_language_highlights_untagged_code_blocks(cx: &mut TestAppContext) { let language = language::rust_lang(); + language.set_theme(&rust_test_theme()); let language_registry = Arc::new(LanguageRegistry::test(cx.executor())); language_registry.add(language.clone()); @@ -5186,19 +5395,19 @@ mod tests { cx.run_until_parked(); markdown.read_with(cx, |markdown, _| { - let fallback = markdown - .parsed_markdown() + let parsed = markdown.parsed_markdown(); + let fallback = parsed .fallback_code_block_language .as_ref() .expect("the fallback language must be resolved for untagged code blocks"); assert_eq!(fallback.name(), language.name()); - assert_eq!( - markdown - .first_code_block_language() - .map(|language| language.name()), - Some(language.name()), - "untagged code blocks must resolve to the fallback language" - ); + + let code_start = source.find("fn main").unwrap(); + let cached = parsed + .code_block_highlights + .get(&code_start) + .expect("untagged code blocks must be highlighted with the fallback language"); + assert!(!cached.runs.is_empty()); }); } @@ -7167,4 +7376,53 @@ mod tests { }; (prose.layout.line_height(), code.layout.line_height()) } + + fn rust_test_theme() -> SyntaxTheme { + SyntaxTheme::new( + [ + ("keyword", gpui::red()), + ("function", gpui::blue()), + ("type", gpui::green()), + ] + .into_iter() + .map(|(name, color)| { + ( + name.to_owned(), + gpui::HighlightStyle { + color: Some(color), + ..gpui::HighlightStyle::default() + }, + ) + }), + ) + } + + fn markdown_with_rust_language( + source: &str, + cx: &mut TestAppContext, + ) -> (Arc, Entity) { + let language = language::rust_lang(); + language.set_theme(&rust_test_theme()); + let language_registry = Arc::new(LanguageRegistry::test(cx.executor())); + language_registry.add(language.clone()); + let source = SharedString::from(source.to_owned()); + let markdown = cx.new(|cx| Markdown::new(source, Some(language_registry), None, cx)); + cx.run_until_parked(); + (language, markdown) + } + + fn cached_code_block_highlights( + markdown: &Entity, + code_start: usize, + cx: &mut TestAppContext, + ) -> ResolvedHighlights { + markdown.read_with(cx, |markdown, _| { + markdown + .parsed_markdown() + .code_block_highlights + .get(&code_start) + .expect("code block highlights must be computed during parse") + .clone() + }) + } }