diff --git a/crates/grammars/src/python/injections.scm b/crates/grammars/src/python/injections.scm index bc47469dc8..8981caffac 100644 --- a/crates/grammars/src/python/injections.scm +++ b/crates/grammars/src/python/injections.scm @@ -13,14 +13,14 @@ arguments: (argument_list (comment) @_comment (string - (string_content) @injection.content))) + (string_content) @injection.content) @injection.host)) ; string variables ((comment) @_comment . (expression_statement (assignment right: (string - (string_content) @injection.content)))) + (string_content) @injection.content) @injection.host))) ] (#match? @_comment "^(#|#\\s+)(?i:sql)\\s*$") (#set! injection.language "sql")) diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index 7f9d377e6d..9ff17ce2de 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -253,6 +253,21 @@ enum ParseMode { }, } +/// Identifies a set of injection matches that should share a single syntax layer. +#[derive(PartialEq, Eq, Hash)] +enum InjectionGroupKey { + /// Every match of an `injection.combined` pattern for a given language shares one + /// layer spanning the whole parent layer. + Combined(LanguageId), + /// Every match whose `@injection.host` capture resolves to the same node shares one + /// layer spanning that node. This keeps interpolated strings such as Python + /// f-strings, which produce one match per string fragment, in a single layer. + Host { + language: LanguageId, + host_range: Range, + }, +} + #[derive(Debug, PartialEq, Eq)] struct ChangedRegion { depth: usize, @@ -561,7 +576,7 @@ impl SyntaxSnapshot { let mut changed_regions = ChangeRegionSet::default(); let mut queue = BinaryHeap::new(); - let mut combined_injection_ranges = HashMap::default(); + let mut injection_groups = HashMap::default(); queue.push(ParseStep { depth: 0, language: ParseStepLanguage::Loaded { @@ -860,25 +875,29 @@ impl SyntaxSnapshot { registry, step.depth + 1, &expanded_ranges, - &mut combined_injection_ranges, + &mut injection_groups, &mut queue, ); } - let included_sub_ranges: Option>> = if is_combined { - Some( - included_ranges - .into_iter() - .filter(|r| r.start_byte < r.end_byte) - .map(|r| { - text.anchor_before(r.start_byte + step_start_byte) - ..text.anchor_after(r.end_byte + step_start_byte) - }) - .collect(), - ) - } else { - None - }; + // Layers built from more than one included range don't cover their + // whole span, so record the sub-ranges to let callers tell which + // offsets actually belong to this layer. + let included_sub_ranges: Option>> = + if is_combined || included_ranges.len() > 1 { + Some( + included_ranges + .into_iter() + .filter(|r| r.start_byte < r.end_byte) + .map(|r| { + text.anchor_before(r.start_byte + step_start_byte) + ..text.anchor_after(r.end_byte + step_start_byte) + }) + .collect(), + ) + } else { + None + }; SyntaxLayerContent::Parsed { tree, language, @@ -1583,7 +1602,7 @@ fn get_injections( language_registry: &Arc, depth: usize, changed_ranges: &[Range], - combined_injection_ranges: &mut HashMap, Vec)>, + injection_groups: &mut HashMap, Vec)>, queue: &mut BinaryHeap, ) { let mut query_cursor = QueryCursorHandle::new(); @@ -1591,7 +1610,7 @@ fn get_injections( // Ensure that a `ParseStep` is created for every combined injection language, even // if there currently no matches for that injection. - combined_injection_ranges.clear(); + injection_groups.clear(); for pattern in &config.patterns { if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) && let Some(language) = language_registry @@ -1599,7 +1618,10 @@ fn get_injections( .now_or_never() .and_then(|language| language.ok()) { - combined_injection_ranges.insert(language.id, (language, Vec::new())); + injection_groups.insert( + InjectionGroupKey::Combined(language.id), + (language, Vec::new()), + ); } } @@ -1629,6 +1651,11 @@ fn get_injections( prev_match = Some((mat.pattern_index, content_range.clone())); let combined = config.patterns[mat.pattern_index].combined; + let host_range = config + .host_capture_ix + .and_then(|ix| mat.nodes_for_capture_index(ix).next()) + .map(|host_node| host_node.byte_range()); + let mut step_range = content_range.clone(); let language_name = if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() { @@ -1658,11 +1685,23 @@ fn get_injections( .now_or_never() .and_then(|language| language.ok()); let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end); + if let Some(language) = language { - if combined { - combined_injection_ranges - .entry(language.id) - .or_insert_with(|| (language.clone(), vec![])) + let group_key = if let Some(host_range) = host_range { + Some(InjectionGroupKey::Host { + language: language.id, + host_range, + }) + } else if combined { + Some(InjectionGroupKey::Combined(language.id)) + } else { + None + }; + + if let Some(group_key) = group_key { + injection_groups + .entry(group_key) + .or_insert_with(|| (language.clone(), Vec::new())) .1 .extend(content_ranges); } else { @@ -1689,19 +1728,34 @@ fn get_injections( } } - for (_, (language, mut included_ranges)) in combined_injection_ranges.drain() { + for (group_key, (language, mut included_ranges)) in injection_groups.drain() { included_ranges.sort_unstable_by(|a, b| { Ord::cmp(&a.start_byte, &b.start_byte).then_with(|| Ord::cmp(&a.end_byte, &b.end_byte)) }); + // Overlapping changed ranges can yield the same match more than once, and + // `set_included_ranges` rejects overlapping ranges. + included_ranges.dedup(); + + let (range, mode) = match group_key { + InjectionGroupKey::Combined(_) => ( + outer_range.clone(), + ParseMode::Combined { + parent_layer_range: outer_range.to_offset(text), + parent_layer_changed_ranges: changed_ranges.to_vec(), + }, + ), + InjectionGroupKey::Host { host_range, .. } => ( + text.anchor_before(host_range.start)..text.anchor_after(host_range.end), + ParseMode::Single, + ), + }; + queue.push(ParseStep { depth, language: ParseStepLanguage::Loaded { language }, - range: outer_range.clone(), + range, included_ranges, - mode: ParseMode::Combined { - parent_layer_range: outer_range.to_offset(text), - parent_layer_changed_ranges: changed_ranges.to_vec(), - }, + mode, }) } } diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs index 442e6890fe..d4eecbe04c 100644 --- a/crates/language/src/syntax_map/syntax_map_tests.rs +++ b/crates/language/src/syntax_map/syntax_map_tests.rs @@ -4,6 +4,7 @@ use crate::{ buffer_tests::markdown_inline_lang, markdown_lang, rust_lang, }; use gpui::App; +use indoc::indoc; use pretty_assertions::assert_eq; use rand::rngs::StdRng; use std::borrow::Cow; @@ -1049,6 +1050,99 @@ fn test_comment_triggered_injection_toggle(cx: &mut App) { ); } +#[gpui::test] +fn test_injection_grouped_by_host(cx: &mut App) { + // Each SQL layer is described by the buffer text of the fragments it is parsed from, + // so that interpolated strings are asserted to produce a single layer covering every + // fragment of the string. + let cases: &[(&str, &[&[&str]])] = &[ + ( + indoc! {r#" + # sql + cmd = "SELECT col1, col2 FROM tbl" + "#}, + &[&["SELECT col1, col2 FROM tbl"]], + ), + ( + indoc! {r#" + # sql + cmd = f"SELECT col1 FROM tbl WHERE col2 = '{my_var}'" + "#}, + &[&["SELECT col1 FROM tbl WHERE col2 = '", "'"]], + ), + ( + indoc! {r#" + # sql + cmd = f"SELECT {col1}, {col2} FROM {tbl}" + "#}, + &[&["SELECT ", ", ", " FROM "]], + ), + ( + indoc! {r#" + cursor.execute( + # sql + f"SELECT col1 FROM tbl WHERE col2 = '{my_var}'" + ) + "#}, + &[&["SELECT col1 FROM tbl WHERE col2 = '", "'"]], + ), + ( + indoc! {r#" + cursor.execute( + # sql + f"SELECT col1 FROM tbl WHERE col2 = '{my_var}'" + ) + + cursor.execute( + # sql + f"INSERT INTO tbl VALUES ('{val1}')" + ) + "#}, + &[ + &["SELECT col1 FROM tbl WHERE col2 = '", "'"], + &["INSERT INTO tbl VALUES ('", "')"], + ], + ), + ]; + + let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone())); + let python = Arc::new(python_lang()); + registry.add(python.clone()); + registry.add(Arc::new(comment_lang())); + registry.add(Arc::new(sql_lang())); + + for (buffer_index, (text, expected_layers)) in cases.iter().enumerate() { + let buffer = Buffer::new( + ReplicaId::LOCAL, + BufferId::new(buffer_index as u64 + 1).unwrap(), + text.to_string(), + ); + let mut syntax_map = SyntaxMap::new(&buffer); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(python.clone(), &buffer); + + let sql_layers = syntax_map + .layers_for_range(0..buffer.len(), &buffer, true) + .filter(|layer| layer.language.name() == "SQL") + .map(|layer| match layer.included_sub_ranges { + Some(sub_ranges) => sub_ranges + .iter() + .map(|range| buffer.text_for_range(range.clone()).collect::()) + .collect::>(), + None => { + vec![ + buffer + .text_for_range(layer.node().byte_range()) + .collect::(), + ] + } + }) + .collect::>(); + + assert_eq!(sql_layers, *expected_layers, "buffer text:\n{text}"); + } +} + #[gpui::test] fn test_syntax_map_languages_loading_with_erb(cx: &mut App) { let text = r#" @@ -1188,6 +1282,32 @@ fn test_random_syntax_map_edits_with_erb(rng: StdRng, cx: &mut App) { test_random_edits(text, registry, language, rng); } +#[gpui::test(iterations = 50)] +fn test_random_syntax_map_edits_with_python_sql(rng: StdRng, cx: &mut App) { + let text = r#" + # sql + query = f"SELECT {col} FROM tbl WHERE name = '{name}' AND id = {id}" + + cursor.execute( + # sql + f""" + INSERT INTO tbl (col1, col2) + VALUES ('{one}', '{two}') + """ + ) + "# + .unindent() + .repeat(5); + + let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone())); + let language = Arc::new(python_lang()); + registry.add(language.clone()); + registry.add(Arc::new(comment_lang())); + registry.add(Arc::new(sql_lang())); + + test_random_edits(text, registry, language, rng); +} + #[gpui::test(iterations = 50)] fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) { let text = r#" @@ -1630,6 +1750,18 @@ fn comment_lang() -> Language { ) } +/// Stands in for the real SQL language, which this crate doesn't depend on. The grammar +/// is irrelevant here; only the layers Python's injection query produces are asserted on. +fn sql_lang() -> Language { + Language::new( + LanguageConfig { + name: "SQL".into(), + ..Default::default() + }, + Some(tree_sitter_json::LANGUAGE.into()), + ) +} + fn range_for_text(buffer: &Buffer, text: &str) -> Range { let start = buffer.as_rope().to_string().find(text).unwrap(); start..start + text.len() diff --git a/crates/language_core/src/grammar.rs b/crates/language_core/src/grammar.rs index 42c398d3cd..ff4dc0c885 100644 --- a/crates/language_core/src/grammar.rs +++ b/crates/language_core/src/grammar.rs @@ -183,6 +183,7 @@ pub struct InjectionConfig { pub query: Query, pub content_capture_ix: u32, pub language_capture_ix: Option, + pub host_capture_ix: Option, pub patterns: Vec, } @@ -687,6 +688,7 @@ impl Grammar { let mut injection_language_capture_ix = None; let mut content_capture_ix = None; let mut injection_content_capture_ix = None; + let mut host_capture_ix = None; if populate_capture_indices( &query, language_name, @@ -697,6 +699,7 @@ impl Grammar { Capture::Optional("injection.language", &mut injection_language_capture_ix), Capture::Optional("content", &mut content_capture_ix), Capture::Optional("injection.content", &mut injection_content_capture_ix), + Capture::Optional("injection.host", &mut host_capture_ix), ], ) { language_capture_ix = match (language_capture_ix, injection_language_capture_ix) { @@ -735,6 +738,7 @@ impl Grammar { query, language_capture_ix, content_capture_ix, + host_capture_ix, patterns, }); } else {