use collections::{BTreeMap, HashMap, HashSet}; use gpui::SharedString; use linkify::LinkFinder; pub use pulldown_cmark::TagEnd as MarkdownTagEnd; use pulldown_cmark::{ Alignment, CowStr, HeadingLevel, LinkType, MetadataBlockKind, Options, Parser, }; use std::{ops::Range, sync::Arc}; use util::markdown::generate_heading_slug; use crate::{html, path_range::PathWithRange}; pub const PARSE_OPTIONS: Options = Options::ENABLE_TABLES .union(Options::ENABLE_FOOTNOTES) .union(Options::ENABLE_STRIKETHROUGH) .union(Options::ENABLE_TASKLISTS) .union(Options::ENABLE_SMART_PUNCTUATION) .union(Options::ENABLE_HEADING_ATTRIBUTES) .union(Options::ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS) .union(Options::ENABLE_OLD_FOOTNOTES) .union(Options::ENABLE_GFM) .union(Options::ENABLE_SUPERSCRIPT) .union(Options::ENABLE_SUBSCRIPT); #[derive(Default)] struct ParseState { events: Vec<(Range, MarkdownEvent)>, root_block_starts: Vec, depth: usize, } #[derive(Debug, Default)] #[cfg_attr(test, derive(PartialEq))] pub(crate) struct ParsedMarkdownData { pub events: Vec<(Range, MarkdownEvent)>, pub language_names: HashSet, pub language_paths: HashSet>, pub root_block_starts: Vec, pub html_blocks: BTreeMap, pub heading_slugs: HashMap, pub footnote_definitions: HashMap, } impl ParseState { fn push_event(&mut self, range: Range, event: MarkdownEvent) { match &event { MarkdownEvent::Start(_) => { if self.depth == 0 { self.root_block_starts.push(range.start); self.events.push((range.clone(), MarkdownEvent::RootStart)); } self.depth += 1; self.events.push((range, event)); } MarkdownEvent::End(_) => { self.events.push((range.clone(), event)); if self.depth > 0 { self.depth -= 1; if self.depth == 0 { let root_block_index = self.root_block_starts.len() - 1; self.events .push((range, MarkdownEvent::RootEnd(root_block_index))); } } } MarkdownEvent::Rule => { if self.depth == 0 && !range.is_empty() { self.root_block_starts.push(range.start); let root_block_index = self.root_block_starts.len() - 1; self.events.push((range.clone(), MarkdownEvent::RootStart)); self.events.push((range.clone(), event)); self.events .push((range, MarkdownEvent::RootEnd(root_block_index))); } else { self.events.push((range, event)); } } _ => { self.events.push((range, event)); } } } } const MAX_DUPLICATE_HEADING_SLUGS: usize = 128; fn build_heading_slugs( source: &str, events: &[(Range, MarkdownEvent)], ) -> HashMap { let mut slugs = HashMap::default(); let mut slug_counts: HashMap = HashMap::default(); let mut inside_heading = false; let mut heading_text = String::new(); let mut heading_source_start: Option = None; for (range, event) in events { match event { MarkdownEvent::Start(MarkdownTag::Heading { .. }) => { inside_heading = true; heading_text.clear(); heading_source_start = None; } MarkdownEvent::End(MarkdownTagEnd::Heading(_)) => { if inside_heading { let source_offset = heading_source_start.unwrap_or(range.start); let base_slug = generate_heading_slug(&heading_text); let count = slug_counts.entry(base_slug.clone()).or_insert(0); let mut slug = if *count == 0 { base_slug.clone() } else { format!("{base_slug}-{count}") }; *count += 1; while slugs.contains_key(slug.as_str()) { let Some(count) = slug_counts.get_mut(&base_slug) else { slug.clear(); break; }; if *count >= MAX_DUPLICATE_HEADING_SLUGS { slug.clear(); break; } slug = format!("{base_slug}-{count}"); *count += 1; } if !slug.is_empty() { slugs.insert(SharedString::from(slug), source_offset); } inside_heading = false; } } MarkdownEvent::Text | MarkdownEvent::Code if inside_heading => { if heading_source_start.is_none() { heading_source_start = Some(range.start); } heading_text.push_str(&source[range.clone()]); } MarkdownEvent::SubstitutedText(substituted) if inside_heading => { if heading_source_start.is_none() { heading_source_start = Some(range.start); } heading_text.push_str(substituted); } _ => {} } } slugs } pub(crate) fn parse_markdown_with_options( text: &str, parse_html: bool, parse_heading_slugs: bool, ) -> ParsedMarkdownData { let mut state = ParseState::default(); let mut language_names = HashSet::default(); let mut language_paths = HashSet::default(); let mut html_blocks = BTreeMap::default(); let mut within_link = false; let mut within_code_block = false; let mut within_metadata = false; let mut parser = Parser::new_ext(text, PARSE_OPTIONS) .into_offset_iter() .peekable(); while let Some((pulldown_event, range)) = parser.next() { if within_metadata { if let pulldown_cmark::Event::End(pulldown_cmark::TagEnd::MetadataBlock { .. }) = pulldown_event { within_metadata = false; } continue; } match pulldown_event { pulldown_cmark::Event::Start(tag) => { if let pulldown_cmark::Tag::HtmlBlock = &tag { state.push_event(range.clone(), MarkdownEvent::Start(MarkdownTag::HtmlBlock)); if parse_html { if let Some(block) = html::html_parser::parse_html_block(&text[range.clone()], range.clone()) { html_blocks.insert(range.start, block); while let Some((event, end_range)) = parser.next() { if let pulldown_cmark::Event::End( pulldown_cmark::TagEnd::HtmlBlock, ) = event { state.push_event( end_range, MarkdownEvent::End(MarkdownTagEnd::HtmlBlock), ); break; } } } } continue; } let tag = match tag { pulldown_cmark::Tag::Link { link_type, dest_url, title, id, } => { within_link = true; MarkdownTag::Link { link_type, dest_url: SharedString::from(dest_url.into_string()), title: SharedString::from(title.into_string()), id: SharedString::from(id.into_string()), } } pulldown_cmark::Tag::MetadataBlock(_kind) => { within_metadata = true; continue; } pulldown_cmark::Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Indented) => { within_code_block = true; MarkdownTag::CodeBlock { kind: CodeBlockKind::Indented, metadata: CodeBlockMetadata { content_range: range.clone(), line_count: 1, }, } } pulldown_cmark::Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Fenced( ref info, )) => { within_code_block = true; let content_range = extract_code_block_content_range(&text[range.clone()]); let content_range = content_range.start + range.start..content_range.end + range.start; // Valid to use bytes since multi-byte UTF-8 doesn't use ASCII chars. let line_count = text[content_range.clone()] .bytes() .filter(|c| *c == b'\n') .count(); let metadata = CodeBlockMetadata { content_range, line_count, }; let info = info.trim(); let kind = if info.is_empty() { CodeBlockKind::Fenced // Languages should never contain a slash, and PathRanges always should. // (Models are told to specify them relative to a workspace root.) } else if info.contains('/') { let path_range = PathWithRange::new(info); language_paths.insert(path_range.path.clone()); CodeBlockKind::FencedSrc(path_range) } else { let language = SharedString::from(info.to_string()); language_names.insert(language.clone()); CodeBlockKind::FencedLang(language) }; MarkdownTag::CodeBlock { kind, metadata } } pulldown_cmark::Tag::Paragraph => MarkdownTag::Paragraph, pulldown_cmark::Tag::Heading { level, id, classes, attrs, } => { let id = id.map(|id| SharedString::from(id.into_string())); let classes = classes .into_iter() .map(|c| SharedString::from(c.into_string())) .collect(); let attrs = attrs .into_iter() .map(|(key, value)| { ( SharedString::from(key.into_string()), value.map(|v| SharedString::from(v.into_string())), ) }) .collect(); MarkdownTag::Heading { level, id, classes, attrs, } } pulldown_cmark::Tag::BlockQuote(kind) => MarkdownTag::BlockQuote(kind), pulldown_cmark::Tag::List(start_number) => MarkdownTag::List(start_number), pulldown_cmark::Tag::Item => MarkdownTag::Item, pulldown_cmark::Tag::FootnoteDefinition(label) => { MarkdownTag::FootnoteDefinition(SharedString::from(label.to_string())) } pulldown_cmark::Tag::Table(alignments) => MarkdownTag::Table(alignments), pulldown_cmark::Tag::TableHead => MarkdownTag::TableHead, pulldown_cmark::Tag::TableRow => MarkdownTag::TableRow, pulldown_cmark::Tag::TableCell => MarkdownTag::TableCell, pulldown_cmark::Tag::Emphasis => MarkdownTag::Emphasis, pulldown_cmark::Tag::Strong => MarkdownTag::Strong, pulldown_cmark::Tag::Strikethrough => MarkdownTag::Strikethrough, pulldown_cmark::Tag::Superscript => MarkdownTag::Superscript, pulldown_cmark::Tag::Subscript => MarkdownTag::Subscript, pulldown_cmark::Tag::Image { link_type, dest_url, title, id, } => MarkdownTag::Image { link_type, dest_url: SharedString::from(dest_url.into_string()), title: SharedString::from(title.into_string()), id: SharedString::from(id.into_string()), }, pulldown_cmark::Tag::HtmlBlock => MarkdownTag::HtmlBlock, // this is handled above separately pulldown_cmark::Tag::DefinitionList => MarkdownTag::DefinitionList, pulldown_cmark::Tag::DefinitionListTitle => MarkdownTag::DefinitionListTitle, pulldown_cmark::Tag::DefinitionListDefinition => { MarkdownTag::DefinitionListDefinition } }; state.push_event(range, MarkdownEvent::Start(tag)) } pulldown_cmark::Event::End(tag) => { if let pulldown_cmark::TagEnd::Link = tag { within_link = false; } else if let pulldown_cmark::TagEnd::CodeBlock = tag { within_code_block = false; } state.push_event(range, MarkdownEvent::End(tag)); } pulldown_cmark::Event::Text(parsed) => { fn event_for( text: &str, range: Range, str: &str, ) -> (Range, MarkdownEvent) { if str == &text[range.clone()] { (range, MarkdownEvent::Text) } else { (range, MarkdownEvent::SubstitutedText(str.to_owned())) } } if within_code_block { let (range, event) = event_for(text, range, &parsed); state.push_event(range, event); continue; } #[derive(Debug)] struct TextRange<'a> { source_range: Range, merged_range: Range, parsed: CowStr<'a>, } let mut last_len = parsed.len(); let mut ranges = vec![TextRange { source_range: range.clone(), merged_range: 0..last_len, parsed, }]; while matches!(parser.peek(), Some((pulldown_cmark::Event::Text(_), _))) || (parse_html && matches!( parser.peek(), Some((pulldown_cmark::Event::InlineHtml(_), _)) )) { let Some((next_event, next_range)) = parser.next() else { unreachable!() }; let next_text = match next_event { pulldown_cmark::Event::Text(next_event) => next_event, pulldown_cmark::Event::InlineHtml(_) => CowStr::Borrowed(""), _ => unreachable!(), }; let next_len = last_len + next_text.len(); ranges.push(TextRange { source_range: next_range.clone(), merged_range: last_len..next_len, parsed: next_text, }); last_len = next_len; } let mut merged_text = String::with_capacity(ranges.last().unwrap().merged_range.end); for range in &ranges { merged_text.push_str(&range.parsed); } let mut ranges = ranges.into_iter().peekable(); if !within_link && !within_code_block { let mut finder = LinkFinder::new(); finder.kinds(&[linkify::LinkKind::Url]); // Find links in the merged text for link in finder.links(&merged_text) { let link_start_in_merged = link.start(); let link_end_in_merged = link.end(); while ranges .peek() .is_some_and(|range| range.merged_range.end <= link_start_in_merged) { let range = ranges.next().unwrap(); let (range, event) = event_for(text, range.source_range, &range.parsed); state.push_event(range, event); } let Some(range) = ranges.peek_mut() else { continue; }; let prefix_len = link_start_in_merged - range.merged_range.start; if prefix_len > 0 { let (head, tail) = range.parsed.split_at(prefix_len); let (event_range, event) = event_for( text, range.source_range.start..range.source_range.start + prefix_len, head, ); state.push_event(event_range, event); range.parsed = CowStr::Boxed(tail.into()); range.merged_range.start += prefix_len; range.source_range.start += prefix_len; } let link_start_in_source = range.source_range.start; let mut link_end_in_source = range.source_range.end; let mut link_events = Vec::new(); while ranges .peek() .is_some_and(|range| range.merged_range.end <= link_end_in_merged) { let range = ranges.next().unwrap(); link_end_in_source = range.source_range.end; link_events.push(event_for(text, range.source_range, &range.parsed)); } if let Some(range) = ranges.peek_mut() { let prefix_len = link_end_in_merged - range.merged_range.start; if prefix_len > 0 { let (head, tail) = range.parsed.split_at(prefix_len); link_events.push(event_for( text, range.source_range.start..range.source_range.start + prefix_len, head, )); range.parsed = CowStr::Boxed(tail.into()); range.merged_range.start += prefix_len; range.source_range.start += prefix_len; link_end_in_source = range.source_range.start; } } let link_range = link_start_in_source..link_end_in_source; state.push_event( link_range.clone(), MarkdownEvent::Start(MarkdownTag::Link { link_type: LinkType::Autolink, dest_url: SharedString::from(link.as_str().to_string()), title: SharedString::default(), id: SharedString::default(), }), ); for (range, event) in link_events { state.push_event(range, event); } state.push_event( link_range.clone(), MarkdownEvent::End(MarkdownTagEnd::Link), ); } } for range in ranges { let (range, event) = event_for(text, range.source_range, &range.parsed); state.push_event(range, event); } } pulldown_cmark::Event::Code(_) => { let content_range = extract_code_content_range(&text[range.clone()]); let content_range = content_range.start + range.start..content_range.end + range.start; state.push_event(content_range, MarkdownEvent::Code) } pulldown_cmark::Event::Html(_) => state.push_event(range, MarkdownEvent::Html), pulldown_cmark::Event::InlineHtml(_) => { state.push_event(range, MarkdownEvent::InlineHtml) } pulldown_cmark::Event::FootnoteReference(label) => state.push_event( range, MarkdownEvent::FootnoteReference(SharedString::from(label.to_string())), ), pulldown_cmark::Event::SoftBreak => state.push_event(range, MarkdownEvent::SoftBreak), pulldown_cmark::Event::HardBreak => state.push_event(range, MarkdownEvent::HardBreak), pulldown_cmark::Event::Rule => state.push_event(range, MarkdownEvent::Rule), pulldown_cmark::Event::TaskListMarker(checked) => { state.push_event(range, MarkdownEvent::TaskListMarker(checked)) } pulldown_cmark::Event::InlineMath(_) | pulldown_cmark::Event::DisplayMath(_) => {} } } let heading_slugs = if parse_heading_slugs { build_heading_slugs(text, &state.events) } else { HashMap::default() }; let footnote_definitions = build_footnote_definitions(&state.events); ParsedMarkdownData { events: state.events, language_names, language_paths, root_block_starts: state.root_block_starts, html_blocks, heading_slugs, footnote_definitions, } } fn build_footnote_definitions( events: &[(Range, MarkdownEvent)], ) -> HashMap { let mut definitions = HashMap::default(); let mut current_label: Option = None; for (range, event) in events { match event { MarkdownEvent::Start(MarkdownTag::FootnoteDefinition(label)) => { current_label = Some(label.clone()); } MarkdownEvent::End(MarkdownTagEnd::FootnoteDefinition) => { current_label = None; } MarkdownEvent::Text if current_label.is_some() => { if let Some(label) = current_label.take() { definitions.entry(label).or_insert(range.start); } } _ => {} } } definitions } pub fn parse_links_only(text: &str) -> Vec<(Range, MarkdownEvent)> { let mut events = Vec::new(); let mut finder = LinkFinder::new(); finder.kinds(&[linkify::LinkKind::Url]); let mut text_range = Range { start: 0, end: text.len(), }; for link in finder.links(text) { let link_range = link.start()..link.end(); if link_range.start > text_range.start { events.push((text_range.start..link_range.start, MarkdownEvent::Text)); } events.push(( link_range.clone(), MarkdownEvent::Start(MarkdownTag::Link { link_type: LinkType::Autolink, dest_url: SharedString::from(link.as_str().to_string()), title: SharedString::default(), id: SharedString::default(), }), )); events.push((link_range.clone(), MarkdownEvent::Text)); events.push((link_range.clone(), MarkdownEvent::End(MarkdownTagEnd::Link))); text_range.start = link_range.end; } if text_range.end > text_range.start { events.push((text_range, MarkdownEvent::Text)); } events } /// A static-lifetime equivalent of pulldown_cmark::Event so we can cache the /// parse result for rendering without resorting to unsafe lifetime coercion. #[derive(Clone, Debug, PartialEq)] pub enum MarkdownEvent { /// Start of a tagged element. Events that are yielded after this event /// and before its corresponding `End` event are inside this element. /// Start and end events are guaranteed to be balanced. Start(MarkdownTag), /// End of a tagged element. End(MarkdownTagEnd), /// Text that uses the associated range from the markdown source. Text, /// Text that differs from the markdown source - typically due to substitution of HTML entities /// and smart punctuation. SubstitutedText(String), /// An inline code node. Code, /// An HTML node. Html, /// An inline HTML node. InlineHtml, /// A reference to a footnote with given label, which may or may not be defined /// by an event with a `Tag::FootnoteDefinition` tag. Definitions and references to them may /// occur in any order. FootnoteReference(SharedString), /// A soft line break. SoftBreak, /// A hard line break. HardBreak, /// A horizontal ruler. Rule, /// A task list marker, rendered as a checkbox in HTML. Contains a true when it is checked. TaskListMarker(bool), /// Start of a root-level block (a top-level structural element like a paragraph, heading, list, etc.). RootStart, /// End of a root-level block. Contains the root block index. RootEnd(usize), } /// Tags for elements that can contain other elements. #[derive(Clone, Debug, PartialEq)] pub enum MarkdownTag { /// A paragraph of text and other inline elements. Paragraph, /// A heading, with optional identifier, classes and custom attributes. /// The identifier is prefixed with `#` and the last one in the attributes /// list is chosen, classes are prefixed with `.` and custom attributes /// have no prefix and can optionally have a value (`myattr` o `myattr=myvalue`). Heading { level: HeadingLevel, id: Option, classes: Vec, /// The first item of the tuple is the attr and second one the value. attrs: Vec<(SharedString, Option)>, }, BlockQuote(Option), /// A code block. CodeBlock { kind: CodeBlockKind, metadata: CodeBlockMetadata, }, /// A HTML block. HtmlBlock, /// A list. If the list is ordered the field indicates the number of the first item. /// Contains only list items. List(Option), // TODO: add delim and tight for ast (not needed for html) /// A list item. Item, /// A footnote definition. The value contained is the footnote's label by which it can /// be referred to. FootnoteDefinition(SharedString), /// A table. Contains a vector describing the text-alignment for each of its columns. Table(Vec), /// A table header. Contains only `TableCell`s. Note that the table body starts immediately /// after the closure of the `TableHead` tag. There is no `TableBody` tag. TableHead, /// A table row. Is used both for header rows as body rows. Contains only `TableCell`s. TableRow, TableCell, // span-level tags Emphasis, Strong, Strikethrough, Superscript, Subscript, /// A link. Link { link_type: LinkType, dest_url: SharedString, title: SharedString, /// Identifier of reference links, e.g. `world` in the link `[hello][world]`. id: SharedString, }, /// An image. The first field is the link type, the second the destination URL and the third is a title, /// the fourth is the link identifier. Image { link_type: LinkType, dest_url: SharedString, title: SharedString, /// Identifier of reference links, e.g. `world` in the link `[hello][world]`. id: SharedString, }, /// A metadata block. MetadataBlock(MetadataBlockKind), DefinitionList, DefinitionListTitle, DefinitionListDefinition, } #[derive(Clone, Debug, PartialEq)] pub enum CodeBlockKind { Indented, /// "Fenced" means "surrounded by triple backticks." /// There can optionally be either a language after the backticks (like in traditional Markdown) /// or, if an agent is specifying a path for a source location in the project, it can be a PathRange, /// e.g. ```path/to/foo.rs#L123-456 instead of ```rust Fenced, FencedLang(SharedString), FencedSrc(PathWithRange), } #[derive(Default, Clone, Debug, PartialEq)] pub struct CodeBlockMetadata { pub content_range: Range, pub line_count: usize, } fn extract_code_content_range(text: &str) -> Range { let text_len = text.len(); if text_len == 0 { return 0..0; } let start_ticks = text.chars().take_while(|&c| c == '`').count(); if start_ticks == 0 || start_ticks > text_len { return 0..text_len; } let end_ticks = text.chars().rev().take_while(|&c| c == '`').count(); if end_ticks != start_ticks || text_len < start_ticks + end_ticks { return 0..text_len; } start_ticks..text_len - end_ticks } pub(crate) fn extract_code_block_content_range(text: &str) -> Range { let mut range = 0..text.len(); if text.starts_with("```") { range.start += 3; if let Some(newline_ix) = text[range.clone()].find('\n') { range.start += newline_ix + 1; } } if !range.is_empty() && text.ends_with("```") { range.end -= 3; } if range.start > range.end { range.end = range.start; } range } #[cfg(test)] mod tests { use super::MarkdownEvent::*; use super::MarkdownTag::*; use super::*; const UNWANTED_OPTIONS: Options = Options::ENABLE_YAML_STYLE_METADATA_BLOCKS .union(Options::ENABLE_MATH) .union(Options::ENABLE_DEFINITION_LIST) .union(Options::ENABLE_WIKILINKS); #[test] fn all_options_considered() { // The purpose of this is to fail when new options are added to pulldown_cmark, so that they // can be evaluated for inclusion. assert_eq!(PARSE_OPTIONS.union(UNWANTED_OPTIONS), Options::all()); } #[test] fn wanted_and_unwanted_options_disjoint() { assert_eq!( PARSE_OPTIONS.intersection(UNWANTED_OPTIONS), Options::empty() ); } #[test] fn test_html_comments() { assert_eq!( parse_markdown_with_options(" \nReturns", false, false), ParsedMarkdownData { events: vec![ (2..30, RootStart), (2..30, Start(HtmlBlock)), (2..2, SubstitutedText(" ".into())), (2..7, Html), (7..26, Html), (26..30, Html), (2..30, End(MarkdownTagEnd::HtmlBlock)), (2..30, RootEnd(0)), (30..37, RootStart), (30..37, Start(Paragraph)), (30..37, Text), (30..37, End(MarkdownTagEnd::Paragraph)), (30..37, RootEnd(1)), ], root_block_starts: vec![2, 30], ..Default::default() } ) } #[test] fn test_plain_urls_and_escaped_text() { assert_eq!( parse_markdown_with_options( "   https://some.url some \\`►\\` text", false, false, ), ParsedMarkdownData { events: vec![ (0..51, RootStart), (0..51, Start(Paragraph)), (0..6, SubstitutedText("\u{a0}".into())), (6..12, SubstitutedText("\u{a0}".into())), (12..13, Text), ( 13..29, Start(Link { link_type: LinkType::Autolink, dest_url: "https://some.url".into(), title: "".into(), id: "".into(), }) ), (13..29, Text), (13..29, End(MarkdownTagEnd::Link)), (29..35, Text), (36..37, Text), // Escaped backtick (37..44, SubstitutedText("►".into())), (45..46, Text), // Escaped backtick (46..51, Text), (0..51, End(MarkdownTagEnd::Paragraph)), (0..51, RootEnd(0)), ], root_block_starts: vec![0], ..Default::default() } ); } #[test] fn test_incomplete_link() { assert_eq!( parse_markdown_with_options( "You can use the [GitHub Search API](https://docs.github.com/en", false, false, ) .events, vec![ (0..62, RootStart), (0..62, Start(Paragraph)), (0..16, Text), (16..17, Text), (17..34, Text), (34..35, Text), (35..36, Text), ( 36..62, Start(Link { link_type: LinkType::Autolink, dest_url: "https://docs.github.com/en".into(), title: "".into(), id: "".into() }) ), (36..62, Text), (36..62, End(MarkdownTagEnd::Link)), (0..62, End(MarkdownTagEnd::Paragraph)), (0..62, RootEnd(0)), ], ); } #[test] fn test_smart_punctuation() { assert_eq!( parse_markdown_with_options( "-- --- ... \"double quoted\" 'single quoted' ----------", false, false, ), ParsedMarkdownData { events: vec![ (0..53, RootStart), (0..53, Start(Paragraph)), (0..2, SubstitutedText("–".into())), (2..3, Text), (3..6, SubstitutedText("—".into())), (6..7, Text), (7..10, SubstitutedText("…".into())), (10..11, Text), (11..12, SubstitutedText("\u{201c}".into())), (12..25, Text), (25..26, SubstitutedText("\u{201d}".into())), (26..27, Text), (27..28, SubstitutedText("\u{2018}".into())), (28..41, Text), (41..42, SubstitutedText("\u{2019}".into())), (42..43, Text), (43..53, SubstitutedText("–––––".into())), (0..53, End(MarkdownTagEnd::Paragraph)), (0..53, RootEnd(0)), ], root_block_starts: vec![0], ..Default::default() } ) } #[test] fn test_code_block_metadata() { assert_eq!( parse_markdown_with_options("```rust\nfn main() {\n let a = 1;\n}\n```", false, false), ParsedMarkdownData { events: vec![ (0..37, RootStart), ( 0..37, Start(CodeBlock { kind: CodeBlockKind::FencedLang("rust".into()), metadata: CodeBlockMetadata { content_range: 8..34, line_count: 3 } }) ), (8..34, Text), (0..37, End(MarkdownTagEnd::CodeBlock)), (0..37, RootEnd(0)), ], language_names: { let mut h = HashSet::default(); h.insert("rust".into()); h }, root_block_starts: vec![0], ..Default::default() } ); assert_eq!( parse_markdown_with_options(" fn main() {}", false, false), ParsedMarkdownData { events: vec![ (4..16, RootStart), ( 4..16, Start(CodeBlock { kind: CodeBlockKind::Indented, metadata: CodeBlockMetadata { content_range: 4..16, line_count: 1 } }) ), (4..16, Text), (4..16, End(MarkdownTagEnd::CodeBlock)), (4..16, RootEnd(0)), ], root_block_starts: vec![4], ..Default::default() } ); } fn assert_code_block_does_not_emit_links(markdown: &str) { let parsed = parse_markdown_with_options(markdown, false, false); let mut code_block_depth = 0; let mut code_block_count = 0; let mut saw_text_inside_code_block = false; for (_, event) in &parsed.events { match event { Start(CodeBlock { .. }) => { code_block_depth += 1; code_block_count += 1; } End(MarkdownTagEnd::CodeBlock) => { assert!( code_block_depth > 0, "encountered a code block end without a matching start" ); code_block_depth -= 1; } Start(Link { .. }) | End(MarkdownTagEnd::Link) => { assert_eq!( code_block_depth, 0, "code blocks should not emit link events" ); } Text | SubstitutedText(_) if code_block_depth > 0 => { saw_text_inside_code_block = true; } _ => {} } } assert_eq!(code_block_count, 1, "expected exactly one code block"); assert_eq!(code_block_depth, 0, "unterminated code block"); assert!( saw_text_inside_code_block, "expected text inside the code block" ); } #[test] fn test_code_blocks_do_not_autolink_urls() { assert_code_block_does_not_emit_links("```txt\nhttps://example.com\n```"); assert_code_block_does_not_emit_links(" https://example.com"); assert_code_block_does_not_emit_links( "```txt\r\nhttps:/\\/example.com\r\nhttps://example.com\r\n```", ); assert_code_block_does_not_emit_links( " https:/\\/example.com\r\n https://example.com", ); } #[test] fn test_metadata_blocks_do_not_affect_root_blocks() { assert_eq!( parse_markdown_with_options("+++\ntitle = \"Example\"\n+++\n\nParagraph", false, false), ParsedMarkdownData { events: vec![ (27..36, RootStart), (27..36, Start(Paragraph)), (27..36, Text), (27..36, End(MarkdownTagEnd::Paragraph)), (27..36, RootEnd(0)), ], root_block_starts: vec![27], ..Default::default() } ); } #[test] fn test_table_checkboxes_remain_text_in_cells() { let markdown = "\ | Done | Task | |------|---------| | [x] | Fix bug | | [ ] | Add feature |"; let parsed = parse_markdown_with_options(markdown, false, false); let mut in_table = false; let mut saw_task_list_marker = false; let mut cell_texts = Vec::new(); let mut current_cell = String::new(); for (range, event) in &parsed.events { match event { Start(Table(_)) => in_table = true, End(MarkdownTagEnd::Table) => in_table = false, Start(TableCell) => current_cell.clear(), End(MarkdownTagEnd::TableCell) => { if in_table { cell_texts.push(current_cell.clone()); } } Text if in_table => current_cell.push_str(&markdown[range.clone()]), TaskListMarker(_) if in_table => saw_task_list_marker = true, _ => {} } } let checkbox_cells: Vec<&str> = cell_texts .iter() .map(|cell| cell.trim()) .filter(|cell| *cell == "[x]" || *cell == "[X]" || *cell == "[ ]") .collect(); assert!( !saw_task_list_marker, "Table checkboxes should remain text, not task-list markers" ); assert_eq!(checkbox_cells, vec!["[x]", "[ ]"]); } #[test] fn test_extract_code_content_range() { let input = "```let x = 5;```"; assert_eq!(extract_code_content_range(input), 3..13); let input = "``let x = 5;``"; assert_eq!(extract_code_content_range(input), 2..12); let input = "`let x = 5;`"; assert_eq!(extract_code_content_range(input), 1..11); let input = "plain text"; assert_eq!(extract_code_content_range(input), 0..10); let input = "``let x = 5;`"; assert_eq!(extract_code_content_range(input), 0..13); } #[test] fn test_extract_code_block_content_range() { let input = "```rust\nlet x = 5;\n```"; assert_eq!(extract_code_block_content_range(input), 8..19); let input = "plain text"; assert_eq!(extract_code_block_content_range(input), 0..10); let input = "```python\nprint('hello')\nprint('world')\n```"; assert_eq!(extract_code_block_content_range(input), 10..40); // Malformed input let input = "`````"; assert_eq!(extract_code_block_content_range(input), 3..3); } #[test] fn test_footnotes() { let parsed = parse_markdown_with_options( "Text with a footnote[^1] and some more text.\n\n[^1]: This is the footnote content.", false, false, ); assert_eq!( parsed.events, vec![ (0..45, RootStart), (0..45, Start(Paragraph)), (0..20, Text), (20..24, FootnoteReference("1".into())), (24..44, Text), (0..45, End(MarkdownTagEnd::Paragraph)), (0..45, RootEnd(0)), (46..81, RootStart), (46..81, Start(FootnoteDefinition("1".into()))), (52..81, Start(Paragraph)), (52..81, Text), (52..81, End(MarkdownTagEnd::Paragraph)), (46..81, End(MarkdownTagEnd::FootnoteDefinition)), (46..81, RootEnd(1)), ] ); assert_eq!(parsed.footnote_definitions.len(), 1); assert_eq!(parsed.footnote_definitions.get("1").copied(), Some(52)); } #[test] fn test_footnote_definitions_multiple() { let parsed = parse_markdown_with_options( "Text[^a] and[^b].\n\n[^a]: First.\n\n[^b]: Second.", false, false, ); assert_eq!(parsed.footnote_definitions.len(), 2); assert!(parsed.footnote_definitions.contains_key("a")); assert!(parsed.footnote_definitions.contains_key("b")); } #[test] fn test_links_split_across_fragments() { // This test verifies that links split across multiple text fragments due to escaping or other issues // are correctly detected and processed // Note: In real usage, pulldown_cmark creates separate text events for the escaped character // We're verifying our parser can handle this correctly assert_eq!( parse_markdown_with_options( "https:/\\/example.com is equivalent to https://example.com!", false, false, ) .events, vec![ (0..62, RootStart), (0..62, Start(Paragraph)), ( 0..20, Start(Link { link_type: LinkType::Autolink, dest_url: "https://example.com".into(), title: "".into(), id: "".into() }) ), (0..7, Text), (8..20, Text), (0..20, End(MarkdownTagEnd::Link)), (20..38, Text), ( 38..61, Start(Link { link_type: LinkType::Autolink, dest_url: "https://example.com".into(), title: "".into(), id: "".into() }) ), (38..53, Text), (53..58, SubstitutedText(".".into())), (58..61, Text), (38..61, End(MarkdownTagEnd::Link)), (61..62, Text), (0..62, End(MarkdownTagEnd::Paragraph)), (0..62, RootEnd(0)), ], ); assert_eq!( parse_markdown_with_options( "Visit https://example.com/cat\\/é‍☕ for coffee!", false, false, ) .events, [ (0..55, RootStart), (0..55, Start(Paragraph)), (0..6, Text), ( 6..43, Start(Link { link_type: LinkType::Autolink, dest_url: "https://example.com/cat/é\u{200d}☕".into(), title: "".into(), id: "".into() }) ), (6..29, Text), (30..33, Text), (33..40, SubstitutedText("\u{200d}".into())), (40..43, Text), (6..43, End(MarkdownTagEnd::Link)), (43..55, Text), (0..55, End(MarkdownTagEnd::Paragraph)), (0..55, RootEnd(0)), ] ); } #[test] fn test_heading_slugs() { let parsed = parse_markdown_with_options( "# Hello World\n\n## Code `block`\n\n### Third Level\n\n#### Fourth Level\n\n## Hello World", false, true, ); assert_eq!(parsed.heading_slugs.len(), 5); assert!(parsed.heading_slugs.contains_key("hello-world")); assert!(parsed.heading_slugs.contains_key("code-block")); assert!(parsed.heading_slugs.contains_key("third-level")); assert!(parsed.heading_slugs.contains_key("fourth-level")); assert!(parsed.heading_slugs.contains_key("hello-world-1")); } #[test] fn test_heading_source_index_for_slug() { let parsed = parse_markdown_with_options( "# Duplicate\n\nText\n\n## Duplicate\n\nMore text", false, true, ); let first = parsed.heading_slugs.get("duplicate").copied(); let second = parsed.heading_slugs.get("duplicate-1").copied(); assert!(first.is_some()); assert!(second.is_some()); assert!(first.expect("first slug missing") < second.expect("second slug missing")); } #[test] fn test_heading_slug_collision_with_dedup_suffix() { let parsed = parse_markdown_with_options("# Foo\n\n## Foo\n\n## Foo 1", false, true); assert_eq!(parsed.heading_slugs.len(), 3); assert!(parsed.heading_slugs.contains_key("foo")); assert!(parsed.heading_slugs.contains_key("foo-1")); assert!(parsed.heading_slugs.contains_key("foo-1-1")); } #[test] fn test_gfm_alert_block_quote_kinds() { use pulldown_cmark::BlockQuoteKind; let markdown = "\n> [!NOTE]\n> A note.\n\n> [!TIP]\n> A tip.\n\n> [!IMPORTANT]\n> Important.\n\n> [!WARNING]\n> A warning.\n\n> [!CAUTION]\n> A caution.\n\n> Plain quote.\n"; let parsed = parse_markdown_with_options(markdown, false, false); let block_quote_kinds: Vec<_> = parsed .events .iter() .filter_map(|(_, event)| match event { Start(BlockQuote(kind)) => Some(*kind), _ => None, }) .collect(); assert_eq!( block_quote_kinds, vec![ Some(BlockQuoteKind::Note), Some(BlockQuoteKind::Tip), Some(BlockQuoteKind::Important), Some(BlockQuoteKind::Warning), Some(BlockQuoteKind::Caution), None, ] ); } }