From a5ce6f8dfbdf0ba65ae82b3cbc4bea47a7205dad Mon Sep 17 00:00:00 2001 From: deadcode-walker Date: Thu, 23 Apr 2026 12:42:10 +0000 Subject: [PATCH] Fix freeze when binary content written to open empty file (#53074) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening an empty file then writing binary content to it on disk causes a permanent freeze (not a crash — requires force kill). `reload_impl` loads raw bytes via `load_bytes` and decodes with `encoding_rs` but never checks if the content is binary. The existing binary check in `decode_file_text` only runs on fresh opens, not reloads. So binary content enters the buffer as lossy UTF-8. Binary data has almost no newlines, producing a single enormous row. The wrap map's sync fast path in `flush_edits` checks row count (< 100 rows) but not line length, so it runs `wrap_line` on a multi-MB single line synchronously on the main thread via `smol::block_on`. Font shaping millions of non-ASCII replacement characters blocks the UI thread indefinitely. Two fixes, both one-condition guards: - Null-byte check in `reload_impl` before decoding, same heuristic (first 8000 bytes) used by `decode_file_text` on fresh opens. Binary content never enters the buffer. - Column-length guard (`MAX_SYNC_WRAP_COLUMNS = 10_000`) on the wrap map sync fast path so absurdly long lines fall through to the async background path. Defense in depth for any single-line content that's too long to shape synchronously. ## Test plan - [ ] Open an empty file in Zed, write binary content to it externally (e.g. `cp /bin/ls /path/to/open-file`) — should show "Binary files are not supported" instead of freezing - [ ] Open a compressed file (zip, gz) that starts empty and gets filled — same behavior - [ ] Normal text file reload still works (no regression) - [ ] Very long single-line text files (>10k columns) don't freeze the editor --------- Co-authored-by: deadcode-walker <268043493+deadcode-walker@users.noreply.github.com> Co-authored-by: Smit Barmase --- crates/language/src/buffer.rs | 13 +- crates/language/src/file_content.rs | 158 +++++++++++++++++ crates/language/src/language.rs | 2 + .../tests/integration/project_tests.rs | 33 ++++ crates/worktree/src/worktree.rs | 161 +----------------- 5 files changed, 203 insertions(+), 164 deletions(-) create mode 100644 crates/language/src/file_content.rs diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index 3770e4ccf1..e15568affc 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -1,8 +1,8 @@ pub mod row_chunk; use crate::{ - DebuggerTextObject, LanguageScope, ModelineSettings, Outline, OutlineConfig, PLAIN_TEXT, - RunnableCapture, RunnableTag, TextObject, TreeSitterOptions, + ByteContent, DebuggerTextObject, LanguageScope, ModelineSettings, Outline, OutlineConfig, + PLAIN_TEXT, RunnableCapture, RunnableTag, TextObject, TreeSitterOptions, analyze_byte_content, diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup}, language_settings::{AutoIndentMode, LanguageSettings}, outline::OutlineItem, @@ -1579,16 +1579,21 @@ impl Buffer { let target_encoding = force_encoding.unwrap_or(current_encoding); + let bytes = load_bytes_task.await?; + + anyhow::ensure!( + analyze_byte_content(&bytes) != ByteContent::Binary, + "Binary files are not supported" + ); + let is_unicode = target_encoding == encoding_rs::UTF_8 || target_encoding == encoding_rs::UTF_16LE || target_encoding == encoding_rs::UTF_16BE; let (new_text, has_bom, encoding_used) = if force_encoding.is_some() && !is_unicode { - let bytes = load_bytes_task.await?; let (cow, _had_errors) = target_encoding.decode_without_bom_handling(&bytes); (cow.into_owned(), false, target_encoding) } else { - let bytes = load_bytes_task.await?; let (cow, used_enc, _had_errors) = target_encoding.decode(&bytes); let actual_has_bom = if used_enc == encoding_rs::UTF_8 { diff --git a/crates/language/src/file_content.rs b/crates/language/src/file_content.rs new file mode 100644 index 0000000000..bb543478e5 --- /dev/null +++ b/crates/language/src/file_content.rs @@ -0,0 +1,158 @@ +pub const FILE_ANALYSIS_BYTES: usize = 1024; + +#[derive(Debug, PartialEq)] +pub enum ByteContent { + Utf16Le, + Utf16Be, + Binary, + Unknown, +} + +// Heuristic check using null byte distribution plus a generic text-likeness +// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise +// distinguishes between text-like and binary-like content. +pub fn analyze_byte_content(bytes: &[u8]) -> ByteContent { + if bytes.len() < 2 { + return ByteContent::Unknown; + } + + if is_known_binary_header(bytes) { + return ByteContent::Binary; + } + + let limit = bytes.len().min(FILE_ANALYSIS_BYTES); + let mut even_null_count = 0usize; + let mut odd_null_count = 0usize; + let mut non_text_like_count = 0usize; + + for (i, &byte) in bytes[..limit].iter().enumerate() { + if byte == 0 { + if i % 2 == 0 { + even_null_count += 1; + } else { + odd_null_count += 1; + } + non_text_like_count += 1; + continue; + } + + let is_text_like = match byte { + b'\t' | b'\n' | b'\r' | 0x0C => true, + 0x20..=0x7E => true, + // Treat bytes that are likely part of UTF-8 or single-byte encodings as text-like. + 0x80..=0xBF | 0xC2..=0xF4 => true, + _ => false, + }; + + if !is_text_like { + non_text_like_count += 1; + } + } + + let total_null_count = even_null_count + odd_null_count; + + // If there are no NUL bytes at all, this is overwhelmingly likely to be text. + if total_null_count == 0 { + return ByteContent::Unknown; + } + + let has_significant_nulls = total_null_count >= limit / 16; + let nulls_skew_to_even = even_null_count > odd_null_count * 4; + let nulls_skew_to_odd = odd_null_count > even_null_count * 4; + + if has_significant_nulls { + let sample = &bytes[..limit]; + + // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first) + // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first) + + if nulls_skew_to_even && is_plausible_utf16_text(sample, false) { + return ByteContent::Utf16Be; + } + + if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) { + return ByteContent::Utf16Le; + } + + return ByteContent::Binary; + } + + if non_text_like_count * 100 < limit * 8 { + ByteContent::Unknown + } else { + ByteContent::Binary + } +} + +fn is_known_binary_header(bytes: &[u8]) -> bool { + bytes.starts_with(b"%PDF-") // PDF + || bytes.starts_with(b"PK\x03\x04") // ZIP local header + || bytes.starts_with(b"PK\x05\x06") // ZIP end of central directory + || bytes.starts_with(b"PK\x07\x08") // ZIP spanning/splitting + || bytes.starts_with(b"\x89PNG\r\n\x1a\n") // PNG + || bytes.starts_with(b"\xFF\xD8\xFF") // JPEG + || bytes.starts_with(b"GIF87a") // GIF87a + || bytes.starts_with(b"GIF89a") // GIF89a + || bytes.starts_with(b"IWAD") // Doom IWAD archive + || bytes.starts_with(b"PWAD") // Doom PWAD archive + || bytes.starts_with(b"RIFF") // WAV, AVI, WebP + || bytes.starts_with(b"OggS") // OGG (Vorbis, Opus, FLAC) + || bytes.starts_with(b"fLaC") // FLAC + || bytes.starts_with(b"ID3") // MP3 with ID3v2 tag + || bytes.starts_with(b"\xFF\xFB") // MP3 frame sync (MPEG1 Layer3) + || bytes.starts_with(b"\xFF\xFA") // MP3 frame sync (MPEG1 Layer3) + || bytes.starts_with(b"\xFF\xF3") // MP3 frame sync (MPEG2 Layer3) + || bytes.starts_with(b"\xFF\xF2") // MP3 frame sync (MPEG2 Layer3) +} + +// Null byte skew alone is not enough to identify UTF-16 -- binary formats with +// small 16-bit values (like PCM audio) produce the same pattern. Decode the +// bytes as UTF-16 and reject if too many code units land in control character +// ranges or form unpaired surrogates, which real text almost never contains. +fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool { + let mut suspicious_count = 0usize; + let mut total = 0usize; + + let mut i = 0; + while let Some(code_unit) = read_u16(bytes, i, little_endian) { + total += 1; + + match code_unit { + 0x0009 | 0x000A | 0x000C | 0x000D => {} + // C0/C1 control characters and non-characters + 0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1, + 0xD800..=0xDBFF => { + let next_offset = i + 2; + let has_low_surrogate = read_u16(bytes, next_offset, little_endian) + .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next)); + if has_low_surrogate { + total += 1; + i += 2; + } else { + suspicious_count += 1; + } + } + // Lone low surrogate without a preceding high surrogate + 0xDC00..=0xDFFF => suspicious_count += 1, + _ => {} + } + + i += 2; + } + + if total == 0 { + return false; + } + + // Real UTF-16 text has near-zero control characters; binary data with + // small 16-bit values typically exceeds 5%. 2% provides a safe margin. + suspicious_count * 100 < total * 2 +} + +fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option { + let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?]; + if little_endian { + return Some(u16::from_le_bytes(pair)); + } + Some(u16::from_be_bytes(pair)) +} diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index 505fbb4d7d..ea8a0c69a1 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -9,6 +9,7 @@ mod buffer; mod diagnostic; mod diagnostic_set; +mod file_content; mod language_registry; pub mod language_settings; @@ -91,6 +92,7 @@ pub use buffer::Operation; pub use buffer::*; pub use diagnostic::{Diagnostic, DiagnosticSourceKind}; pub use diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup}; +pub use file_content::{ByteContent, FILE_ANALYSIS_BYTES, analyze_byte_content}; pub use language_registry::{ AvailableLanguage, BinaryStatus, LanguageNotFound, LanguageQueries, LanguageRegistry, QUERY_FILENAME_PREFIXES, diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs index 05e29d0017..12f139daca 100644 --- a/crates/project/tests/integration/project_tests.rs +++ b/crates/project/tests/integration/project_tests.rs @@ -6278,6 +6278,39 @@ async fn test_dirty_buffer_reloads_after_undo(cx: &mut gpui::TestAppContext) { }); } +#[gpui::test] +async fn test_buffer_file_change_to_binary_fails(cx: &mut gpui::TestAppContext) { + init_test(cx); + + let fs = FakeFs::new(cx.executor()); + fs.insert_tree( + path!("/dir"), + json!({ + "file.txt": "", + }), + ) + .await; + + let project = Project::test(fs.clone(), [path!("/dir").as_ref()], cx).await; + let buffer = project + .update(cx, |p, cx| p.open_local_buffer(path!("/dir/file.txt"), cx)) + .await + .unwrap(); + + fs.write( + path!("/dir/file.txt").as_ref(), + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01", + ) + .await + .unwrap(); + cx.executor().run_until_parked(); + + // Test that existing buffer is left untouched + buffer.read_with(cx, |buffer, _| { + assert_eq!(buffer.text(), ""); + }); +} + #[gpui::test] async fn test_buffer_file_changes_on_disk(cx: &mut gpui::TestAppContext) { init_test(cx); diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs index 5e658659e2..4a85e91c5a 100644 --- a/crates/worktree/src/worktree.rs +++ b/crates/worktree/src/worktree.rs @@ -29,7 +29,7 @@ use gpui::{ Task, }; use ignore::IgnoreStack; -use language::DiskState; +use language::{ByteContent, DiskState, FILE_ANALYSIS_BYTES, analyze_byte_content}; use parking_lot::Mutex; use paths::{local_settings_folder_name, local_vscode_folder_name}; @@ -6337,8 +6337,6 @@ impl fs::Watcher for NullWatcher { } } -const FILE_ANALYSIS_BYTES: usize = 1024; - async fn decode_file_text( fs: &dyn Fs, abs_path: &Path, @@ -6453,163 +6451,6 @@ fn decode_byte_full( } } -#[derive(Debug, PartialEq)] -enum ByteContent { - Utf16Le, - Utf16Be, - Binary, - Unknown, -} - -// Heuristic check using null byte distribution plus a generic text-likeness -// heuristic. This prefers UTF-16 when many bytes are NUL and otherwise -// distinguishes between text-like and binary-like content. -fn analyze_byte_content(bytes: &[u8]) -> ByteContent { - if bytes.len() < 2 { - return ByteContent::Unknown; - } - - if is_known_binary_header(bytes) { - return ByteContent::Binary; - } - - let limit = bytes.len().min(FILE_ANALYSIS_BYTES); - let mut even_null_count = 0usize; - let mut odd_null_count = 0usize; - let mut non_text_like_count = 0usize; - - for (i, &byte) in bytes[..limit].iter().enumerate() { - if byte == 0 { - if i % 2 == 0 { - even_null_count += 1; - } else { - odd_null_count += 1; - } - non_text_like_count += 1; - continue; - } - - let is_text_like = match byte { - b'\t' | b'\n' | b'\r' | 0x0C => true, - 0x20..=0x7E => true, - // Treat bytes that are likely part of UTF-8 or single-byte encodings as text-like. - 0x80..=0xBF | 0xC2..=0xF4 => true, - _ => false, - }; - - if !is_text_like { - non_text_like_count += 1; - } - } - - let total_null_count = even_null_count + odd_null_count; - - // If there are no NUL bytes at all, this is overwhelmingly likely to be text. - if total_null_count == 0 { - return ByteContent::Unknown; - } - - let has_significant_nulls = total_null_count >= limit / 16; - let nulls_skew_to_even = even_null_count > odd_null_count * 4; - let nulls_skew_to_odd = odd_null_count > even_null_count * 4; - - if has_significant_nulls { - let sample = &bytes[..limit]; - - // UTF-16BE ASCII: [0x00, char] — nulls at even positions (high byte first) - // UTF-16LE ASCII: [char, 0x00] — nulls at odd positions (low byte first) - - if nulls_skew_to_even && is_plausible_utf16_text(sample, false) { - return ByteContent::Utf16Be; - } - - if nulls_skew_to_odd && is_plausible_utf16_text(sample, true) { - return ByteContent::Utf16Le; - } - - return ByteContent::Binary; - } - - if non_text_like_count * 100 < limit * 8 { - ByteContent::Unknown - } else { - ByteContent::Binary - } -} - -fn is_known_binary_header(bytes: &[u8]) -> bool { - bytes.starts_with(b"%PDF-") // PDF - || bytes.starts_with(b"PK\x03\x04") // ZIP local header - || bytes.starts_with(b"PK\x05\x06") // ZIP end of central directory - || bytes.starts_with(b"PK\x07\x08") // ZIP spanning/splitting - || bytes.starts_with(b"\x89PNG\r\n\x1a\n") // PNG - || bytes.starts_with(b"\xFF\xD8\xFF") // JPEG - || bytes.starts_with(b"GIF87a") // GIF87a - || bytes.starts_with(b"GIF89a") // GIF89a - || bytes.starts_with(b"IWAD") // Doom IWAD archive - || bytes.starts_with(b"PWAD") // Doom PWAD archive - || bytes.starts_with(b"RIFF") // WAV, AVI, WebP - || bytes.starts_with(b"OggS") // OGG (Vorbis, Opus, FLAC) - || bytes.starts_with(b"fLaC") // FLAC - || bytes.starts_with(b"ID3") // MP3 with ID3v2 tag - || bytes.starts_with(b"\xFF\xFB") // MP3 frame sync (MPEG1 Layer3) - || bytes.starts_with(b"\xFF\xFA") // MP3 frame sync (MPEG1 Layer3) - || bytes.starts_with(b"\xFF\xF3") // MP3 frame sync (MPEG2 Layer3) - || bytes.starts_with(b"\xFF\xF2") // MP3 frame sync (MPEG2 Layer3) -} - -// Null byte skew alone is not enough to identify UTF-16 -- binary formats with -// small 16-bit values (like PCM audio) produce the same pattern. Decode the -// bytes as UTF-16 and reject if too many code units land in control character -// ranges or form unpaired surrogates, which real text almost never contains. -fn is_plausible_utf16_text(bytes: &[u8], little_endian: bool) -> bool { - let mut suspicious_count = 0usize; - let mut total = 0usize; - - let mut i = 0; - while let Some(code_unit) = read_u16(bytes, i, little_endian) { - total += 1; - - match code_unit { - 0x0009 | 0x000A | 0x000C | 0x000D => {} - // C0/C1 control characters and non-characters - 0x0000..=0x001F | 0x007F..=0x009F | 0xFFFE | 0xFFFF => suspicious_count += 1, - 0xD800..=0xDBFF => { - let next_offset = i + 2; - let has_low_surrogate = read_u16(bytes, next_offset, little_endian) - .is_some_and(|next| (0xDC00..=0xDFFF).contains(&next)); - if has_low_surrogate { - total += 1; - i += 2; - } else { - suspicious_count += 1; - } - } - // Lone low surrogate without a preceding high surrogate - 0xDC00..=0xDFFF => suspicious_count += 1, - _ => {} - } - - i += 2; - } - - if total == 0 { - return false; - } - - // Real UTF-16 text has near-zero control characters; binary data with - // small 16-bit values typically exceeds 5%. 2% provides a safe margin. - suspicious_count * 100 < total * 2 -} - -fn read_u16(bytes: &[u8], offset: usize, little_endian: bool) -> Option { - let pair = [*bytes.get(offset)?, *bytes.get(offset + 1)?]; - if little_endian { - return Some(u16::from_le_bytes(pair)); - } - Some(u16::from_be_bytes(pair)) -} - #[cfg(test)] mod tests { use super::*;