Reuse char-scan invisibles detection in highlight_invisibles (#62715)

Follow-up to
https://github.com/zed-industries/zed/pull/62478#discussion_r3769810809

New bench results:

| corpus | old | new | speedup |
|---|---|---|---|
| ascii, no invisibles | 83 MB/s | 580 MB/s | **7.0x** |
| unicode, no invisibles | 88 MB/s | 442 MB/s | **5.0x** |
| sparse invisibles | 63 MB/s | 431 MB/s | **6.9x** |
| dense invisibles | 82 MB/s | 109 MB/s | 1.3x |


Release Notes:

- N/A

crates/benchmarks/benches/display_map.rs

@@ -1,5 +1,5 @@
 use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
-use editor::{MultiBuffer, display_map::*};
+use editor::{EditorStyle, MultiBuffer, display_map::*};
 use gpui::{AppContext as _, HighlightStyle, Hsla, TestDispatcher, font, px};
 use itertools::Itertools;
 use multi_buffer::MultiBufferOffset;
@@ -205,10 +205,87 @@ fn create_highlight_endpoints_benchmark(c: &mut Criterion) {
     group.finish();
 }
 
+fn highlighted_chunks_benchmark(c: &mut Criterion) {
+    const LINE_COUNT: usize = 500;
+
+    let dispatcher = TestDispatcher::new(1);
+    let mut cx = gpui::TestAppContext::build(dispatcher, None);
+    cx.update(|cx| {
+        let store = SettingsStore::test(cx);
+        cx.set_global(store);
+        editor::init(cx);
+    });
+
+    let corpora = [
+        (
+            "ascii",
+            "    let chunks = snapshot.highlighted_chunks(rows.clone(), language_aware, style);",
+        ),
+        (
+            "unicode",
+            "の設定を変更する — émojis 🧑\u{200d}✈\u{fe0f} und Ümläute überall, здесь тоже текст",
+        ),
+        (
+            "sparse_invisibles",
+            "normal text here\u{200b}and some more text that goes on for a while without issues",
+        ),
+        (
+            "dense_invisibles",
+            "a\u{200b}b\u{ad}c\u{2060}d\u{feff}e\u{200b}f\u{ad}g\u{2060}h",
+        ),
+    ];
+
+    let mut group = c.benchmark_group("Highlighted chunks");
+    for (name, line) in corpora {
+        let text = std::iter::repeat_n(line, LINE_COUNT)
+            .collect::<Vec<_>>()
+            .join("\n");
+        let buffer = cx.update(|cx| MultiBuffer::build_simple(&text, cx));
+        let map = cx.new(|cx| {
+            DisplayMap::new(
+                buffer,
+                font("Courier"),
+                px(16.0),
+                None,
+                1,
+                1,
+                FoldPlaceholder::default(),
+                DiagnosticSeverity::Warning,
+                cx,
+            )
+        });
+        let snapshot = cx.update(|cx| map.update(cx, |map, cx| map.snapshot(cx)));
+        let editor_style = EditorStyle::default();
+        group.bench_with_input(
+            BenchmarkId::new("highlighted_chunks", name),
+            &snapshot,
+            |bench, snapshot| {
+                bench.iter(|| {
+                    let mut total_len = 0usize;
+                    let chunks = snapshot.highlighted_chunks(
+                        DisplayRow(0)..DisplayRow(LINE_COUNT as u32),
+                        language::LanguageAwareStyling {
+                            tree_sitter: false,
+                            diagnostics: false,
+                        },
+                        &editor_style,
+                    );
+                    for chunk in chunks {
+                        total_len += black_box(chunk.text).len();
+                    }
+                    black_box(total_len);
+                });
+            },
+        );
+    }
+    group.finish();
+}
+
 criterion_group!(
     benches,
     to_tab_point_benchmark,
     to_fold_point_benchmark,
-    create_highlight_endpoints_benchmark
+    create_highlight_endpoints_benchmark,
+    highlighted_chunks_benchmark
 );
 criterion_main!(benches);

crates/editor/src/display_map.rs

@@ -89,6 +89,7 @@ pub use fold_map::{
     ChunkRenderer, ChunkRendererContext, ChunkRendererId, Fold, FoldId, FoldPlaceholder, FoldPoint,
 };
 pub use inlay_map::{InlayOffset, InlayPoint};
+use invisibles::is_standalone_grapheme;
 pub use invisibles::{is_invisible, replacement};
 pub use wrap_map::{WrapPoint, WrapRow, WrapSnapshot};
 
@@ -1420,24 +1421,25 @@ impl<'a> HighlightedChunk<'a> {
         self,
         editor_style: &'a EditorStyle,
     ) -> impl Iterator<Item = Self> + 'a {
-        let mut chunks = self.text.graphemes(true).peekable();
         let mut text = self.text;
         let style = self.style;
         let is_tab = self.is_tab;
         let renderer = self.replacement;
         let is_inlay = self.is_inlay;
         iter::from_fn(move || {
-            let mut prefix_len = 0;
-            while let Some(&chunk) = chunks.peek() {
-                let mut chars = chunk.chars();
-                let Some(ch) = chars.next() else { break };
-                if chunk.len() != ch.len_utf8() || !is_invisible(ch) {
-                    prefix_len += chunk.len();
-                    chunks.next();
+            if text.is_empty() {
+                return None;
+            }
+            for (offset, ch) in text.char_indices() {
+                if !is_invisible(ch) {
                     continue;
                 }
-                if prefix_len > 0 {
-                    let (prefix, suffix) = text.split_at(prefix_len);
+                let ch_end = offset + ch.len_utf8();
+                if !is_standalone_grapheme(text, offset, ch_end) {
+                    continue;
+                }
+                if offset > 0 {
+                    let (prefix, suffix) = text.split_at(offset);
                     text = suffix;
                     return Some(HighlightedChunk {
                         text: prefix,
@@ -1447,70 +1449,44 @@ impl<'a> HighlightedChunk<'a> {
                         replacement: renderer.clone(),
                     });
                 }
-                chunks.next();
-                let (prefix, suffix) = text.split_at(chunk.len());
+                let (invisible_text, suffix) = text.split_at(ch_end);
                 text = suffix;
-                if let Some(replacement) = replacement(ch) {
-                    let invisible_highlight = HighlightStyle {
-                        background_color: Some(editor_style.status.hint_background),
-                        underline: Some(UnderlineStyle {
-                            color: Some(editor_style.status.hint),
-                            thickness: px(1.),
-                            wavy: false,
-                        }),
-                        ..Default::default()
-                    };
-                    let invisible_style = if let Some(style) = style {
-                        style.highlight(invisible_highlight)
-                    } else {
-                        invisible_highlight
-                    };
-                    return Some(HighlightedChunk {
-                        text: prefix,
-                        style: Some(invisible_style),
-                        is_tab: false,
-                        is_inlay,
-                        replacement: Some(ChunkReplacement::Str(replacement.into())),
-                    });
+                let invisible_highlight = HighlightStyle {
+                    background_color: Some(editor_style.status.hint_background),
+                    underline: Some(UnderlineStyle {
+                        color: Some(editor_style.status.hint),
+                        thickness: px(1.),
+                        wavy: false,
+                    }),
+                    ..Default::default()
+                };
+                let invisible_style = if let Some(style) = style {
+                    style.highlight(invisible_highlight)
                 } else {
-                    let invisible_highlight = HighlightStyle {
-                        background_color: Some(editor_style.status.hint_background),
-                        underline: Some(UnderlineStyle {
-                            color: Some(editor_style.status.hint),
-                            thickness: px(1.),
-                            wavy: false,
-                        }),
-                        ..Default::default()
-                    };
-                    let invisible_style = if let Some(style) = style {
-                        style.highlight(invisible_highlight)
-                    } else {
-                        invisible_highlight
-                    };
-
-                    return Some(HighlightedChunk {
-                        text: prefix,
-                        style: Some(invisible_style),
-                        is_tab: false,
-                        is_inlay,
-                        replacement: renderer.clone(),
-                    });
-                }
-            }
-
-            if !text.is_empty() {
-                let remainder = text;
-                text = "";
-                Some(HighlightedChunk {
-                    text: remainder,
-                    style,
-                    is_tab,
+                    invisible_highlight
+                };
+                return Some(HighlightedChunk {
+                    text: invisible_text,
+                    style: Some(invisible_style),
+                    is_tab: false,
                     is_inlay,
-                    replacement: renderer.clone(),
-                })
-            } else {
-                None
+                    replacement: match replacement(ch) {
+                        Some(replacement) => {
+                            Some(ChunkReplacement::Str(SharedString::from(replacement)))
+                        }
+                        None => renderer.clone(),
+                    },
+                });
             }
+            let remainder = text;
+            text = "";
+            Some(HighlightedChunk {
+                text: remainder,
+                style,
+                is_tab,
+                is_inlay,
+                replacement: renderer.clone(),
+            })
         })
     }
 }

crates/editor/src/display_map/invisibles.rs

@@ -30,6 +30,8 @@
 // ref: https://gist.github.com/ConradIrwin/f759e1fc29267143c4c7895aa495dca5?h=1
 // ref: https://unicode.org/Public/emoji/13.0/emoji-test.txt
 // https://github.com/bits/UTF-8-Unicode-Test-Documents/blob/master/UTF-8_sequence_separated/utf8_sequence_0-0x10ffff_assigned_including-unprintable-asis.txt
+use unicode_segmentation::GraphemeCursor;
+
 #[ztracing::instrument(skip_all)]
 pub fn is_invisible(c: char) -> bool {
     if c <= '\u{1f}' {
@@ -111,6 +113,15 @@ fn should_preserve_invisible_character(c: char) -> bool {
     }
 }
 
+pub fn is_standalone_grapheme(text: &str, start: usize, end: usize) -> bool {
+    let mut cursor = GraphemeCursor::new(start, text.len(), true);
+    if cursor.is_boundary(text, 0) != Ok(true) {
+        return false;
+    }
+    cursor.set_cursor(end);
+    cursor.is_boundary(text, 0) == Ok(true)
+}
+
 const FIXED_WIDTH_SPACE: char = '\u{2007}';
 
 // IDEOGRAPHIC SPACE is common alongside Chinese and other wide character sets.

crates/editor/src/display_map/wrap_map.rs

@@ -2,7 +2,7 @@ use super::{
     Highlights,
     dimensions::RowDelta,
     fold_map::{Chunk, FoldRows},
-    invisibles::{is_invisible, replacement},
+    invisibles::{is_invisible, is_standalone_grapheme, replacement},
     tab_map::{self, TabEdit, TabPoint, TabSnapshot},
 };
 
@@ -23,7 +23,6 @@ use std::{
 };
 use sum_tree::{Bias, Cursor, Dimensions, SumTree};
 use text::Patch;
-use unicode_segmentation::GraphemeCursor;
 
 pub use super::tab_map::TextSummary;
 pub type WrapEdit = text::Edit<WrapRow>;
@@ -142,15 +141,6 @@ impl LineFragmentBuilder {
     }
 }
 
-fn is_standalone_grapheme(text: &str, start: usize, end: usize) -> bool {
-    let mut cursor = GraphemeCursor::new(start, text.len(), true);
-    if cursor.is_boundary(text, 0) != Ok(true) {
-        return false;
-    }
-    cursor.set_cursor(end);
-    cursor.is_boundary(text, 0) == Ok(true)
-}
-
 pub struct WrapChunks<'a> {
     input_chunks: tab_map::TabChunks<'a>,
     input_chunk: Chunk<'a>,