From e950abb32b69935f21f0ef28d274ed0b0505f6a3 Mon Sep 17 00:00:00 2001 From: Xin Zhao Date: Tue, 7 Apr 2026 22:15:33 +0800 Subject: [PATCH] editor: Fix diagnostic rendering when semantic tokens set to full (#53008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content is consistent with the [UI/UX checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) - [ ] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable Closes #50212 There are two unreasonable coupling account for this issue, the coupling of `use_tree_sitter` with `languge_aware` in https://github.com/zed-industries/zed/blob/ca8bd8681c123b995b495a79d760bc0d88220fd0/crates/editor/src/element.rs#L3820-L3822 and the coupling of `language_aware` with `diagnostics` in https://github.com/zed-industries/zed/blob/ca8bd8681c123b995b495a79d760bc0d88220fd0/crates/language/src/buffer.rs#L3736-L3746 Because of these couplings, when the editor stops using Tree-sitter highlighting when `"semantic_tokens"` set to `"full"`, it also accidentally stops fetching diagnostic information. This is why error and warning underlines disappear. I’ve fixed this by adding a separate `use_tree_sitter` parameter to `highlighted_chunks`. This way, we can keep `language_aware` true to get the diagnostic data we need, but still decide whether or not to apply Tree-sitter highlights. I chose to fix this at the `highlighted_chunks` level because I’m worried that changing the logic in the deeper layers of the DisplayMap or Buffer might have too many side effects that are hard to predict. This approach feels like a safer way to solve the problem. Release Notes: - Fixed a bug where diagnostic underlines disappeared when "semantic_tokens" set to "full" --------- Co-authored-by: Kirill Bulatov --- crates/editor/src/display_map.rs | 48 +++++-- crates/editor/src/display_map/block_map.rs | 14 +- .../src/display_map/custom_highlights.rs | 9 +- crates/editor/src/display_map/fold_map.rs | 33 ++++- crates/editor/src/display_map/inlay_map.rs | 37 +++-- crates/editor/src/display_map/tab_map.rs | 71 ++++++++-- crates/editor/src/display_map/wrap_map.rs | 19 ++- crates/editor/src/editor.rs | 22 ++- crates/editor/src/element.rs | 17 ++- crates/editor/src/semantic_tokens.rs | 132 +++++++++++++++++- crates/language/src/buffer.rs | 43 +++++- crates/language/src/buffer_tests.rs | 8 +- crates/multi_buffer/src/multi_buffer.rs | 33 +++-- crates/multi_buffer/src/multi_buffer_tests.rs | 24 +++- crates/outline_panel/src/outline_panel.rs | 13 +- crates/project/src/lsp_store.rs | 15 +- .../tests/integration/project_tests.rs | 15 +- crates/vim/src/state.rs | 12 +- 18 files changed, 468 insertions(+), 97 deletions(-) diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs index f95f103027..7cb8040e28 100644 --- a/crates/editor/src/display_map.rs +++ b/crates/editor/src/display_map.rs @@ -98,7 +98,7 @@ use gpui::{ WeakEntity, }; use language::{ - Point, Subscription as BufferSubscription, + LanguageAwareStyling, Point, Subscription as BufferSubscription, language_settings::{AllLanguageSettings, LanguageSettings}, }; @@ -1769,7 +1769,10 @@ impl DisplaySnapshot { self.block_snapshot .chunks( BlockRow(display_row.0)..BlockRow(self.max_point().row().next_row().0), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, self.masked, Highlights::default(), ) @@ -1783,7 +1786,10 @@ impl DisplaySnapshot { self.block_snapshot .chunks( BlockRow(row)..BlockRow(row + 1), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, self.masked, Highlights::default(), ) @@ -1798,7 +1804,7 @@ impl DisplaySnapshot { pub fn chunks( &self, display_rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlight_styles: HighlightStyles, ) -> DisplayChunks<'_> { self.block_snapshot.chunks( @@ -1818,7 +1824,7 @@ impl DisplaySnapshot { pub fn highlighted_chunks<'a>( &'a self, display_rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, editor_style: &'a EditorStyle, ) -> impl Iterator> { self.chunks( @@ -1910,7 +1916,10 @@ impl DisplaySnapshot { let chunks = custom_highlights::CustomHighlightsChunks::new( multibuffer_range, - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, None, Some(&self.semantic_token_highlights), multibuffer, @@ -1961,7 +1970,14 @@ impl DisplaySnapshot { let mut line = String::new(); let range = display_row..display_row.next_row(); - for chunk in self.highlighted_chunks(range, false, editor_style) { + for chunk in self.highlighted_chunks( + range, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + editor_style, + ) { line.push_str(chunk.text); let text_style = if let Some(style) = chunk.style { @@ -3388,7 +3404,14 @@ pub mod tests { let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); let mut chunks = Vec::<(String, Option, Rgba)>::new(); - for chunk in snapshot.chunks(DisplayRow(0)..DisplayRow(5), true, Default::default()) { + for chunk in snapshot.chunks( + DisplayRow(0)..DisplayRow(5), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + Default::default(), + ) { let color = chunk .highlight_style .and_then(|style| style.color) @@ -3940,7 +3963,14 @@ pub mod tests { ) -> Vec<(String, Option, Option)> { let snapshot = map.update(cx, |map, cx| map.snapshot(cx)); let mut chunks: Vec<(String, Option, Option)> = Vec::new(); - for chunk in snapshot.chunks(rows, true, HighlightStyles::default()) { + for chunk in snapshot.chunks( + rows, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + HighlightStyles::default(), + ) { let syntax_color = chunk .syntax_highlight_id .and_then(|id| theme.get(id)?.color); diff --git a/crates/editor/src/display_map/block_map.rs b/crates/editor/src/display_map/block_map.rs index 67318e3300..17fa7e3de4 100644 --- a/crates/editor/src/display_map/block_map.rs +++ b/crates/editor/src/display_map/block_map.rs @@ -9,7 +9,7 @@ use crate::{ }; use collections::{Bound, HashMap, HashSet}; use gpui::{AnyElement, App, EntityId, Pixels, Window}; -use language::{Patch, Point}; +use language::{LanguageAwareStyling, Patch, Point}; use multi_buffer::{ Anchor, ExcerptBoundaryInfo, MultiBuffer, MultiBufferOffset, MultiBufferPoint, MultiBufferRow, MultiBufferSnapshot, RowInfo, ToOffset, ToPoint as _, @@ -2140,7 +2140,10 @@ impl BlockSnapshot { pub fn text(&self) -> String { self.chunks( BlockRow(0)..self.transforms.summary().output_rows, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, false, Highlights::default(), ) @@ -2152,7 +2155,7 @@ impl BlockSnapshot { pub(crate) fn chunks<'a>( &'a self, rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, masked: bool, highlights: Highlights<'a>, ) -> BlockChunks<'a> { @@ -4300,7 +4303,10 @@ mod tests { let actual_text = blocks_snapshot .chunks( BlockRow(start_row as u32)..BlockRow(end_row as u32), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, false, Highlights::default(), ) diff --git a/crates/editor/src/display_map/custom_highlights.rs b/crates/editor/src/display_map/custom_highlights.rs index 39eabef2f9..6e93e56217 100644 --- a/crates/editor/src/display_map/custom_highlights.rs +++ b/crates/editor/src/display_map/custom_highlights.rs @@ -1,6 +1,6 @@ use collections::BTreeMap; use gpui::HighlightStyle; -use language::Chunk; +use language::{Chunk, LanguageAwareStyling}; use multi_buffer::{MultiBufferChunks, MultiBufferOffset, MultiBufferSnapshot, ToOffset as _}; use std::{ cmp, @@ -34,7 +34,7 @@ impl<'a> CustomHighlightsChunks<'a> { #[ztracing::instrument(skip_all)] pub fn new( range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, text_highlights: Option<&'a TextHighlights>, semantic_token_highlights: Option<&'a SemanticTokensHighlights>, multibuffer_snapshot: &'a MultiBufferSnapshot, @@ -308,7 +308,10 @@ mod tests { // Get all chunks and verify their bitmaps let chunks = CustomHighlightsChunks::new( MultiBufferOffset(0)..buffer_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, None, None, &buffer_snapshot, diff --git a/crates/editor/src/display_map/fold_map.rs b/crates/editor/src/display_map/fold_map.rs index 1554bb96da..4c6c04b86c 100644 --- a/crates/editor/src/display_map/fold_map.rs +++ b/crates/editor/src/display_map/fold_map.rs @@ -5,7 +5,7 @@ use super::{ inlay_map::{InlayBufferRows, InlayChunks, InlayEdit, InlayOffset, InlayPoint, InlaySnapshot}, }; use gpui::{AnyElement, App, ElementId, HighlightStyle, Pixels, SharedString, Stateful, Window}; -use language::{Edit, HighlightId, Point}; +use language::{Edit, HighlightId, LanguageAwareStyling, Point}; use multi_buffer::{ Anchor, AnchorRangeExt, MBTextSummary, MultiBufferOffset, MultiBufferRow, MultiBufferSnapshot, RowInfo, ToOffset, @@ -707,7 +707,10 @@ impl FoldSnapshot { pub fn text(&self) -> String { self.chunks( FoldOffset(MultiBufferOffset(0))..self.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|c| c.text) @@ -909,7 +912,7 @@ impl FoldSnapshot { pub(crate) fn chunks<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> FoldChunks<'a> { let mut transform_cursor = self @@ -954,7 +957,10 @@ impl FoldSnapshot { pub fn chars_at(&self, start: FoldPoint) -> impl '_ + Iterator { self.chunks( start.to_offset(self)..self.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .flat_map(|chunk| chunk.text.chars()) @@ -964,7 +970,10 @@ impl FoldSnapshot { pub fn chunks_at(&self, start: FoldPoint) -> FoldChunks<'_> { self.chunks( start.to_offset(self)..self.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) } @@ -2131,7 +2140,14 @@ mod tests { let text = &expected_text[start.0.0..end.0.0]; assert_eq!( snapshot - .chunks(start..end, false, Highlights::default()) + .chunks( + start..end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default() + ) .map(|c| c.text) .collect::(), text, @@ -2303,7 +2319,10 @@ mod tests { // Get all chunks and verify their bitmaps let chunks = snapshot.chunks( FoldOffset(MultiBufferOffset(0))..FoldOffset(snapshot.len().0), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ); diff --git a/crates/editor/src/display_map/inlay_map.rs b/crates/editor/src/display_map/inlay_map.rs index 47ca295ccb..698b58682d 100644 --- a/crates/editor/src/display_map/inlay_map.rs +++ b/crates/editor/src/display_map/inlay_map.rs @@ -10,7 +10,7 @@ use crate::{ inlays::{Inlay, InlayContent}, }; use collections::BTreeSet; -use language::{Chunk, Edit, Point, TextSummary}; +use language::{Chunk, Edit, LanguageAwareStyling, Point, TextSummary}; use multi_buffer::{ MBTextSummary, MultiBufferOffset, MultiBufferRow, MultiBufferRows, MultiBufferSnapshot, RowInfo, ToOffset, @@ -1200,7 +1200,7 @@ impl InlaySnapshot { pub(crate) fn chunks<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> InlayChunks<'a> { let mut cursor = self @@ -1234,9 +1234,16 @@ impl InlaySnapshot { #[cfg(test)] #[ztracing::instrument(skip_all)] pub fn text(&self) -> String { - self.chunks(Default::default()..self.len(), false, Highlights::default()) - .map(|chunk| chunk.chunk.text) - .collect() + self.chunks( + Default::default()..self.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) + .map(|chunk| chunk.chunk.text) + .collect() } #[ztracing::instrument(skip_all)] @@ -1979,7 +1986,10 @@ mod tests { let actual_text = inlay_snapshot .chunks( range, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights { text_highlights: Some(&text_highlights), inlay_highlights: Some(&inlay_highlights), @@ -2158,7 +2168,10 @@ mod tests { // Get all chunks and verify their bitmaps let chunks = snapshot.chunks( InlayOffset(MultiBufferOffset(0))..snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ); @@ -2293,7 +2306,10 @@ mod tests { let chunks: Vec<_> = inlay_snapshot .chunks( InlayOffset(MultiBufferOffset(0))..inlay_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, highlights, ) .collect(); @@ -2408,7 +2424,10 @@ mod tests { let chunks: Vec<_> = inlay_snapshot .chunks( InlayOffset(MultiBufferOffset(0))..inlay_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, highlights, ) .collect(); diff --git a/crates/editor/src/display_map/tab_map.rs b/crates/editor/src/display_map/tab_map.rs index 187ed8614e..bb0e642df3 100644 --- a/crates/editor/src/display_map/tab_map.rs +++ b/crates/editor/src/display_map/tab_map.rs @@ -3,7 +3,7 @@ use super::{ fold_map::{self, Chunk, FoldChunks, FoldEdit, FoldPoint, FoldSnapshot}, }; -use language::Point; +use language::{LanguageAwareStyling, Point}; use multi_buffer::MultiBufferSnapshot; use std::{cmp, num::NonZeroU32, ops::Range}; use sum_tree::Bias; @@ -101,7 +101,10 @@ impl TabMap { let mut last_tab_with_changed_expansion_offset = None; 'outer: for chunk in old_snapshot.fold_snapshot.chunks( fold_edit.old.end..old_end_row_successor_offset, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) { let mut remaining_tabs = chunk.tabs; @@ -244,7 +247,14 @@ impl TabSnapshot { self.max_point() }; let first_line_chars = self - .chunks(range.start..line_end, false, Highlights::default()) + .chunks( + range.start..line_end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) .flat_map(|chunk| chunk.text.chars()) .take_while(|&c| c != '\n') .count() as u32; @@ -254,7 +264,10 @@ impl TabSnapshot { } else { self.chunks( TabPoint::new(range.end.row(), 0)..range.end, - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .flat_map(|chunk| chunk.text.chars()) @@ -274,7 +287,7 @@ impl TabSnapshot { pub(crate) fn chunks<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> TabChunks<'a> { let (input_start, expanded_char_column, to_next_stop) = @@ -324,7 +337,10 @@ impl TabSnapshot { pub fn text(&self) -> String { self.chunks( TabPoint::zero()..self.max_point(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|chunk| chunk.text) @@ -1170,7 +1186,10 @@ mod tests { tab_snapshot .chunks( TabPoint::new(0, ix as u32)..tab_snapshot.max_point(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|c| c.text) @@ -1246,8 +1265,14 @@ mod tests { let mut chunks = Vec::new(); let mut was_tab = false; let mut text = String::new(); - for chunk in snapshot.chunks(start..snapshot.max_point(), false, Highlights::default()) - { + for chunk in snapshot.chunks( + start..snapshot.max_point(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) { if chunk.is_tab != was_tab { if !text.is_empty() { chunks.push((mem::take(&mut text), was_tab)); @@ -1296,7 +1321,14 @@ mod tests { // This should not panic. let result: String = tab_snapshot - .chunks(start..end, false, Highlights::default()) + .chunks( + start..end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default(), + ) .map(|c| c.text) .collect(); assert!(!result.is_empty()); @@ -1354,7 +1386,14 @@ mod tests { let expected_summary = TextSummary::from(expected_text.as_str()); assert_eq!( tabs_snapshot - .chunks(start..end, false, Highlights::default()) + .chunks( + start..end, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + Highlights::default() + ) .map(|c| c.text) .collect::(), expected_text, @@ -1436,7 +1475,10 @@ mod tests { let (_, fold_snapshot) = FoldMap::new(inlay_snapshot); let chunks = fold_snapshot.chunks( FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Default::default(), ); let mut cursor = TabStopCursor::new(chunks); @@ -1598,7 +1640,10 @@ mod tests { let (_, fold_snapshot) = FoldMap::new(inlay_snapshot); let chunks = fold_snapshot.chunks( FoldOffset(MultiBufferOffset(0))..fold_snapshot.len(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Default::default(), ); let mut cursor = TabStopCursor::new(chunks); diff --git a/crates/editor/src/display_map/wrap_map.rs b/crates/editor/src/display_map/wrap_map.rs index d21642977e..4ff11b1ef6 100644 --- a/crates/editor/src/display_map/wrap_map.rs +++ b/crates/editor/src/display_map/wrap_map.rs @@ -5,7 +5,7 @@ use super::{ tab_map::{self, TabEdit, TabPoint, TabSnapshot}, }; use gpui::{App, AppContext as _, Context, Entity, Font, LineWrapper, Pixels, Task}; -use language::Point; +use language::{LanguageAwareStyling, Point}; use multi_buffer::{MultiBufferSnapshot, RowInfo}; use smol::future::yield_now; use std::{cmp, collections::VecDeque, mem, ops::Range, sync::LazyLock, time::Duration}; @@ -513,7 +513,10 @@ impl WrapSnapshot { let mut remaining = None; let mut chunks = new_tab_snapshot.chunks( TabPoint::new(edit.new_rows.start, 0)..new_tab_snapshot.max_point(), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ); let mut edit_transforms = Vec::::new(); @@ -656,7 +659,7 @@ impl WrapSnapshot { pub(crate) fn chunks<'a>( &'a self, rows: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, highlights: Highlights<'a>, ) -> WrapChunks<'a> { let output_start = WrapPoint::new(rows.start, 0); @@ -960,7 +963,10 @@ impl WrapSnapshot { pub fn text_chunks(&self, wrap_row: WrapRow) -> impl Iterator { self.chunks( wrap_row..self.max_point().row() + WrapRow(1), - false, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, Highlights::default(), ) .map(|h| h.text) @@ -1719,7 +1725,10 @@ mod tests { let actual_text = self .chunks( WrapRow(start_row)..WrapRow(end_row), - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, Highlights::default(), ) .map(|c| c.text) diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 6550d79c9f..ae852b1055 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -132,9 +132,9 @@ use language::{ AutoindentMode, BlockCommentConfig, BracketMatch, BracketPair, Buffer, BufferRow, BufferSnapshot, Capability, CharClassifier, CharKind, CharScopeContext, CodeLabel, CursorShape, DiagnosticEntryRef, DiffOptions, EditPredictionsMode, EditPreview, HighlightedText, IndentKind, - IndentSize, Language, LanguageName, LanguageRegistry, LanguageScope, LocalFile, OffsetRangeExt, - OutlineItem, Point, Selection, SelectionGoal, TextObject, TransactionId, TreeSitterOptions, - WordsQuery, + IndentSize, Language, LanguageAwareStyling, LanguageName, LanguageRegistry, LanguageScope, + LocalFile, OffsetRangeExt, OutlineItem, Point, Selection, SelectionGoal, TextObject, + TransactionId, TreeSitterOptions, WordsQuery, language_settings::{ self, AllLanguageSettings, LanguageSettings, LspInsertMode, RewrapBehavior, WordsCompletionMode, all_language_settings, @@ -19147,7 +19147,13 @@ impl Editor { let range = buffer.anchor_before(rename_start)..buffer.anchor_after(rename_end); let mut old_highlight_id = None; let old_name: Arc = buffer - .chunks(rename_start..rename_end, true) + .chunks( + rename_start..rename_end, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) .map(|chunk| { if old_highlight_id.is_none() { old_highlight_id = chunk.syntax_highlight_id; @@ -25005,7 +25011,13 @@ impl Editor { selection.range() }; - let chunks = snapshot.chunks(range, true); + let chunks = snapshot.chunks( + range, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ); let mut lines = Vec::new(); let mut line: VecDeque = VecDeque::new(); diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index 7a532dc7a7..512fbb8855 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -51,7 +51,10 @@ use gpui::{ pattern_slash, point, px, quad, relative, size, solid_background, transparent_black, }; use itertools::Itertools; -use language::{HighlightedText, IndentGuideSettings, language_settings::ShowWhitespaceSetting}; +use language::{ + HighlightedText, IndentGuideSettings, LanguageAwareStyling, + language_settings::ShowWhitespaceSetting, +}; use markdown::Markdown; use multi_buffer::{ Anchor, ExcerptBoundaryInfo, ExpandExcerptDirection, ExpandInfo, MultiBufferPoint, @@ -3819,7 +3822,11 @@ impl EditorElement { } else { let use_tree_sitter = !snapshot.semantic_tokens_enabled || snapshot.use_tree_sitter_for_syntax(rows.start, cx); - let chunks = snapshot.highlighted_chunks(rows.clone(), use_tree_sitter, style); + let language_aware = LanguageAwareStyling { + tree_sitter: use_tree_sitter, + diagnostics: true, + }; + let chunks = snapshot.highlighted_chunks(rows.clone(), language_aware, style); LineWithInvisibles::from_chunks( chunks, style, @@ -11999,7 +12006,11 @@ pub fn layout_line( ) -> LineWithInvisibles { let use_tree_sitter = !snapshot.semantic_tokens_enabled || snapshot.use_tree_sitter_for_syntax(row, cx); - let chunks = snapshot.highlighted_chunks(row..row + DisplayRow(1), use_tree_sitter, style); + let language_aware = LanguageAwareStyling { + tree_sitter: use_tree_sitter, + diagnostics: true, + }; + let chunks = snapshot.highlighted_chunks(row..row + DisplayRow(1), language_aware, style); LineWithInvisibles::from_chunks( chunks, style, diff --git a/crates/editor/src/semantic_tokens.rs b/crates/editor/src/semantic_tokens.rs index 5e78be70d5..d485cfa702 100644 --- a/crates/editor/src/semantic_tokens.rs +++ b/crates/editor/src/semantic_tokens.rs @@ -475,13 +475,17 @@ mod tests { use gpui::{ AppContext as _, Entity, Focusable as _, HighlightStyle, TestAppContext, UpdateGlobal as _, }; - use language::{Language, LanguageConfig, LanguageMatcher}; + use language::{ + Diagnostic, DiagnosticEntry, DiagnosticSet, Language, LanguageAwareStyling, LanguageConfig, + LanguageMatcher, + }; use languages::FakeLspAdapter; + use lsp::LanguageServerId; use multi_buffer::{ AnchorRangeExt, ExpandExcerptDirection, MultiBuffer, MultiBufferOffset, PathKey, }; use project::Project; - use rope::Point; + use rope::{Point, PointUtf16}; use serde_json::json; use settings::{ GlobalLspSettingsContent, LanguageSettingsContent, SemanticTokenRule, SemanticTokenRules, @@ -2088,6 +2092,130 @@ mod tests { ); } + #[gpui::test] + async fn test_diagnostics_visible_when_semantic_token_set_to_full(cx: &mut TestAppContext) { + init_test(cx, |_| {}); + + update_test_language_settings(cx, &|language_settings| { + language_settings.languages.0.insert( + "Rust".into(), + LanguageSettingsContent { + semantic_tokens: Some(SemanticTokens::Full), + ..LanguageSettingsContent::default() + }, + ); + }); + + let mut cx = EditorLspTestContext::new_rust( + lsp::ServerCapabilities { + semantic_tokens_provider: Some( + lsp::SemanticTokensServerCapabilities::SemanticTokensOptions( + lsp::SemanticTokensOptions { + legend: lsp::SemanticTokensLegend { + token_types: vec!["function".into()], + token_modifiers: Vec::new(), + }, + full: Some(lsp::SemanticTokensFullOptions::Delta { delta: None }), + ..lsp::SemanticTokensOptions::default() + }, + ), + ), + ..lsp::ServerCapabilities::default() + }, + cx, + ) + .await; + + let mut full_request = cx + .set_request_handler::( + move |_, _, _| { + async move { + Ok(Some(lsp::SemanticTokensResult::Tokens( + lsp::SemanticTokens { + data: vec![ + 0, // delta_line + 3, // delta_start + 4, // length + 0, // token_type + 0, // token_modifiers_bitset + ], + result_id: Some("a".into()), + }, + ))) + } + }, + ); + + cx.set_state("ˇfn main() {}"); + assert!(full_request.next().await.is_some()); + + let task = cx.update_editor(|e, _, _| e.semantic_token_state.take_update_task()); + task.await; + + cx.update_buffer(|buffer, cx| { + buffer.update_diagnostics( + LanguageServerId(0), + DiagnosticSet::new( + [DiagnosticEntry { + range: PointUtf16::new(0, 3)..PointUtf16::new(0, 7), + diagnostic: Diagnostic { + severity: lsp::DiagnosticSeverity::ERROR, + group_id: 1, + message: "unused function".into(), + ..Default::default() + }, + }], + buffer, + ), + cx, + ) + }); + + cx.run_until_parked(); + let chunks = cx.update_editor(|editor, window, cx| { + editor + .snapshot(window, cx) + .display_snapshot + .chunks( + crate::display_map::DisplayRow(0)..crate::display_map::DisplayRow(1), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: true, + }, + crate::HighlightStyles::default(), + ) + .map(|chunk| { + ( + chunk.text.to_string(), + chunk.diagnostic_severity, + chunk.highlight_style, + ) + }) + .collect::>() + }); + + assert_eq!( + extract_semantic_highlights(&cx.editor, &cx), + vec![MultiBufferOffset(3)..MultiBufferOffset(7)] + ); + + assert!( + chunks.iter().any( + |(text, severity, style): &( + String, + Option, + Option + )| { + text == "main" + && *severity == Some(lsp::DiagnosticSeverity::ERROR) + && style.is_some() + } + ), + "expected 'main' chunk to have both diagnostic and semantic styling: {:?}", + chunks + ); + } + fn extract_semantic_highlight_styles( editor: &Entity, cx: &TestAppContext, diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index a467cd7895..1e54134efc 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -3733,16 +3733,24 @@ impl BufferSnapshot { /// returned in chunks where each chunk has a single syntax highlighting style and /// diagnostic status. #[ztracing::instrument(skip_all)] - pub fn chunks(&self, range: Range, language_aware: bool) -> BufferChunks<'_> { + pub fn chunks( + &self, + range: Range, + language_aware: LanguageAwareStyling, + ) -> BufferChunks<'_> { let range = range.start.to_offset(self)..range.end.to_offset(self); let mut syntax = None; - if language_aware { + if language_aware.tree_sitter { syntax = Some(self.get_highlights(range.clone())); } - // We want to look at diagnostic spans only when iterating over language-annotated chunks. - let diagnostics = language_aware; - BufferChunks::new(self.text.as_rope(), range, syntax, diagnostics, Some(self)) + BufferChunks::new( + self.text.as_rope(), + range, + syntax, + language_aware.diagnostics, + Some(self), + ) } pub fn highlighted_text_for_range( @@ -4477,7 +4485,13 @@ impl BufferSnapshot { let mut text = String::new(); let mut highlight_ranges = Vec::new(); let mut name_ranges = Vec::new(); - let mut chunks = self.chunks(source_range_for_text.clone(), true); + let mut chunks = self.chunks( + source_range_for_text.clone(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ); let mut last_buffer_range_end = 0; for (buffer_range, is_name) in buffer_ranges { let space_added = !text.is_empty() && buffer_range.start > last_buffer_range_end; @@ -5402,7 +5416,13 @@ impl BufferSnapshot { let mut words = BTreeMap::default(); let mut current_word_start_ix = None; let mut chunk_ix = query.range.start; - for chunk in self.chunks(query.range, false) { + for chunk in self.chunks( + query.range, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ) { for (i, c) in chunk.text.char_indices() { let ix = chunk_ix + i; if classifier.is_word(c) { @@ -5441,6 +5461,15 @@ impl BufferSnapshot { } } +/// A configuration to use when producing styled text chunks. +#[derive(Clone, Copy)] +pub struct LanguageAwareStyling { + /// Whether to highlight text chunks using tree-sitter. + pub tree_sitter: bool, + /// Whether to highlight text chunks based on the diagnostics data. + pub diagnostics: bool, +} + pub struct WordsQuery<'a> { /// Only returns words with all chars from the fuzzy string in them. pub fuzzy_contents: Option<&'a str>, diff --git a/crates/language/src/buffer_tests.rs b/crates/language/src/buffer_tests.rs index 9308ee6f0a..9f4562bf54 100644 --- a/crates/language/src/buffer_tests.rs +++ b/crates/language/src/buffer_tests.rs @@ -4102,7 +4102,13 @@ fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) { let snapshot = buffer.read(cx).snapshot(); // Get all chunks and verify their bitmaps - let chunks = snapshot.chunks(0..snapshot.len(), false); + let chunks = snapshot.chunks( + 0..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); for chunk in chunks { let chunk_text = chunk.text; diff --git a/crates/multi_buffer/src/multi_buffer.rs b/crates/multi_buffer/src/multi_buffer.rs index a54ff64af0..47c1288c8f 100644 --- a/crates/multi_buffer/src/multi_buffer.rs +++ b/crates/multi_buffer/src/multi_buffer.rs @@ -21,9 +21,9 @@ use itertools::Itertools; use language::{ AutoindentMode, Buffer, BufferChunks, BufferRow, BufferSnapshot, Capability, CharClassifier, CharKind, CharScopeContext, Chunk, CursorShape, DiagnosticEntryRef, File, IndentGuideSettings, - IndentSize, Language, LanguageScope, OffsetRangeExt, OffsetUtf16, Outline, OutlineItem, Point, - PointUtf16, Selection, TextDimension, TextObject, ToOffset as _, ToPoint as _, TransactionId, - TreeSitterOptions, Unclipped, + IndentSize, Language, LanguageAwareStyling, LanguageScope, OffsetRangeExt, OffsetUtf16, + Outline, OutlineItem, Point, PointUtf16, Selection, TextDimension, TextObject, ToOffset as _, + ToPoint as _, TransactionId, TreeSitterOptions, Unclipped, language_settings::{AllLanguageSettings, LanguageSettings}, }; @@ -1072,7 +1072,7 @@ pub struct MultiBufferChunks<'a> { range: Range, excerpt_offset_range: Range, excerpt_chunks: Option>, - language_aware: bool, + language_aware: LanguageAwareStyling, snapshot: &'a MultiBufferSnapshot, } @@ -3340,9 +3340,15 @@ impl EventEmitter for MultiBuffer {} impl MultiBufferSnapshot { pub fn text(&self) -> String { - self.chunks(MultiBufferOffset::ZERO..self.len(), false) - .map(|chunk| chunk.text) - .collect() + self.chunks( + MultiBufferOffset::ZERO..self.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ) + .map(|chunk| chunk.text) + .collect() } pub fn reversed_chars_at(&self, position: T) -> impl Iterator + '_ { @@ -3378,7 +3384,14 @@ impl MultiBufferSnapshot { } pub fn text_for_range(&self, range: Range) -> impl Iterator + '_ { - self.chunks(range, false).map(|chunk| chunk.text) + self.chunks( + range, + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ) + .map(|chunk| chunk.text) } pub fn is_line_blank(&self, row: MultiBufferRow) -> bool { @@ -4178,7 +4191,7 @@ impl MultiBufferSnapshot { pub fn chunks( &self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, ) -> MultiBufferChunks<'_> { let mut chunks = MultiBufferChunks { excerpt_offset_range: ExcerptDimension(MultiBufferOffset::ZERO) @@ -7227,7 +7240,7 @@ impl Excerpt { fn chunks_in_range<'a>( &'a self, range: Range, - language_aware: bool, + language_aware: LanguageAwareStyling, snapshot: &'a MultiBufferSnapshot, ) -> ExcerptChunks<'a> { let buffer = self.buffer_snapshot(snapshot); diff --git a/crates/multi_buffer/src/multi_buffer_tests.rs b/crates/multi_buffer/src/multi_buffer_tests.rs index bc904d1a05..cebc9073e9 100644 --- a/crates/multi_buffer/src/multi_buffer_tests.rs +++ b/crates/multi_buffer/src/multi_buffer_tests.rs @@ -5039,7 +5039,13 @@ fn check_edits( fn assert_chunks_in_ranges(snapshot: &MultiBufferSnapshot) { let full_text = snapshot.text(); for ix in 0..full_text.len() { - let mut chunks = snapshot.chunks(MultiBufferOffset(0)..snapshot.len(), false); + let mut chunks = snapshot.chunks( + MultiBufferOffset(0)..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); chunks.seek(MultiBufferOffset(ix)..snapshot.len()); let tail = chunks.map(|chunk| chunk.text).collect::(); assert_eq!(tail, &full_text[ix..], "seek to range: {:?}", ix..); @@ -5300,7 +5306,13 @@ fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) { let snapshot = multibuffer.read(cx).snapshot(cx); - let chunks = snapshot.chunks(MultiBufferOffset(0)..snapshot.len(), false); + let chunks = snapshot.chunks( + MultiBufferOffset(0)..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); for chunk in chunks { let chunk_text = chunk.text; @@ -5466,7 +5478,13 @@ fn test_random_chunk_bitmaps_with_diffs(cx: &mut App, mut rng: StdRng) { let snapshot = multibuffer.read(cx).snapshot(cx); - let chunks = snapshot.chunks(MultiBufferOffset(0)..snapshot.len(), false); + let chunks = snapshot.chunks( + MultiBufferOffset(0)..snapshot.len(), + LanguageAwareStyling { + tree_sitter: false, + diagnostics: false, + }, + ); for chunk in chunks { let chunk_text = chunk.text; diff --git a/crates/outline_panel/src/outline_panel.rs b/crates/outline_panel/src/outline_panel.rs index b7d5afcb68..fa23b805cd 100644 --- a/crates/outline_panel/src/outline_panel.rs +++ b/crates/outline_panel/src/outline_panel.rs @@ -23,8 +23,8 @@ use gpui::{ uniform_list, }; use itertools::Itertools; -use language::language_settings::LanguageSettings; use language::{Anchor, BufferId, BufferSnapshot, OffsetRangeExt, OutlineItem}; +use language::{LanguageAwareStyling, language_settings::LanguageSettings}; use menu::{Cancel, SelectFirst, SelectLast, SelectNext, SelectPrevious}; use std::{ @@ -217,10 +217,13 @@ impl SearchState { let mut offset = context_offset_range.start; let mut context_text = String::new(); let mut highlight_ranges = Vec::new(); - for mut chunk in highlight_arguments - .multi_buffer_snapshot - .chunks(context_offset_range.start..context_offset_range.end, true) - { + for mut chunk in highlight_arguments.multi_buffer_snapshot.chunks( + context_offset_range.start..context_offset_range.end, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) { if !non_whitespace_symbol_occurred { for c in chunk.text.chars() { if c.is_whitespace() { diff --git a/crates/project/src/lsp_store.rs b/crates/project/src/lsp_store.rs index 2f579f5a72..9ea50fdc8f 100644 --- a/crates/project/src/lsp_store.rs +++ b/crates/project/src/lsp_store.rs @@ -72,9 +72,10 @@ use itertools::Itertools as _; use language::{ Bias, BinaryStatus, Buffer, BufferRow, BufferSnapshot, CachedLspAdapter, Capability, CodeLabel, CodeLabelExt, Diagnostic, DiagnosticEntry, DiagnosticSet, DiagnosticSourceKind, Diff, - File as _, Language, LanguageName, LanguageRegistry, LocalFile, LspAdapter, LspAdapterDelegate, - LspInstaller, ManifestDelegate, ManifestName, ModelineSettings, OffsetUtf16, Patch, PointUtf16, - TextBufferSnapshot, ToOffset, ToOffsetUtf16, ToPointUtf16, Toolchain, Transaction, Unclipped, + File as _, Language, LanguageAwareStyling, LanguageName, LanguageRegistry, LocalFile, + LspAdapter, LspAdapterDelegate, LspInstaller, ManifestDelegate, ManifestName, ModelineSettings, + OffsetUtf16, Patch, PointUtf16, TextBufferSnapshot, ToOffset, ToOffsetUtf16, ToPointUtf16, + Toolchain, Transaction, Unclipped, language_settings::{ AllLanguageSettings, FormatOnSave, Formatter, LanguageSettings, all_language_settings, }, @@ -13527,7 +13528,13 @@ fn resolve_word_completion(snapshot: &BufferSnapshot, completion: &mut Completio } let mut offset = 0; - for chunk in snapshot.chunks(word_range.clone(), true) { + for chunk in snapshot.chunks( + word_range.clone(), + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) { let end_offset = offset + chunk.text.len(); if let Some(highlight_id) = chunk.syntax_highlight_id { completion diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs index d6c2ce37c9..f680ccee78 100644 --- a/crates/project/tests/integration/project_tests.rs +++ b/crates/project/tests/integration/project_tests.rs @@ -41,9 +41,10 @@ use gpui::{ use itertools::Itertools; use language::{ Buffer, BufferEvent, Diagnostic, DiagnosticEntry, DiagnosticEntryRef, DiagnosticSet, - DiagnosticSourceKind, DiskState, FakeLspAdapter, Language, LanguageConfig, LanguageMatcher, - LanguageName, LineEnding, ManifestName, ManifestProvider, ManifestQuery, OffsetRangeExt, Point, - ToPoint, Toolchain, ToolchainList, ToolchainLister, ToolchainMetadata, + DiagnosticSourceKind, DiskState, FakeLspAdapter, Language, LanguageAwareStyling, + LanguageConfig, LanguageMatcher, LanguageName, LineEnding, ManifestName, ManifestProvider, + ManifestQuery, OffsetRangeExt, Point, ToPoint, Toolchain, ToolchainList, ToolchainLister, + ToolchainMetadata, language_settings::{LanguageSettings, LanguageSettingsContent}, markdown_lang, rust_lang, tree_sitter_typescript, }; @@ -4382,7 +4383,13 @@ fn chunks_with_diagnostics( range: Range, ) -> Vec<(String, Option)> { let mut chunks: Vec<(String, Option)> = Vec::new(); - for chunk in buffer.snapshot().chunks(range, true) { + for chunk in buffer.snapshot().chunks( + range, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, + ) { if chunks .last() .is_some_and(|prev_chunk| prev_chunk.1 == chunk.diagnostic_severity) diff --git a/crates/vim/src/state.rs b/crates/vim/src/state.rs index 4dd557199a..9e9b42d319 100644 --- a/crates/vim/src/state.rs +++ b/crates/vim/src/state.rs @@ -17,7 +17,7 @@ use gpui::{ Action, App, AppContext, BorrowAppContext, ClipboardEntry, ClipboardItem, DismissEvent, Entity, EntityId, Global, HighlightStyle, StyledText, Subscription, Task, TextStyle, WeakEntity, }; -use language::{Buffer, BufferEvent, BufferId, Chunk, Point}; +use language::{Buffer, BufferEvent, BufferId, Chunk, LanguageAwareStyling, Point}; use multi_buffer::MultiBufferRow; use picker::{Picker, PickerDelegate}; @@ -1504,7 +1504,10 @@ impl PickerDelegate for MarksViewDelegate { position.row, snapshot.line_len(MultiBufferRow(position.row)), ), - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, ); matches.push(MarksMatch { name: name.clone(), @@ -1530,7 +1533,10 @@ impl PickerDelegate for MarksViewDelegate { let chunks = snapshot.chunks( Point::new(position.row, 0) ..Point::new(position.row, snapshot.line_len(position.row)), - true, + LanguageAwareStyling { + tree_sitter: true, + diagnostics: true, + }, ); matches.push(MarksMatch {