diff --git a/Cargo.lock b/Cargo.lock index 28b906a59f..d84cdc4cb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -826,9 +826,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -1358,7 +1355,6 @@ dependencies = [ "collections", "cpal", "crossbeam", - "denoise", "gpui", "libwebrtc", "log", @@ -2689,53 +2685,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "candle-core" -version = "0.9.1" -source = "git+https://github.com/zed-industries/candle?branch=9.1-patched#724d75eb3deebefe83f2a7381a45d4fac6eda383" -dependencies = [ - "byteorder", - "float8", - "gemm 0.17.1", - "half", - "memmap2", - "num-traits", - "num_cpus", - "rand 0.9.4", - "rand_distr", - "rayon", - "safetensors", - "thiserror 1.0.69", - "ug", - "yoke 0.7.5", - "zip 1.1.4", -] - -[[package]] -name = "candle-nn" -version = "0.9.1" -source = "git+https://github.com/zed-industries/candle?branch=9.1-patched#724d75eb3deebefe83f2a7381a45d4fac6eda383" -dependencies = [ - "candle-core", - "half", - "libc", - "num-traits", - "rayon", - "safetensors", - "serde", - "thiserror 1.0.69", -] - -[[package]] -name = "candle-onnx" -version = "0.9.1" -source = "git+https://github.com/zed-industries/candle?branch=9.1-patched#724d75eb3deebefe83f2a7381a45d4fac6eda383" -dependencies = [ - "candle-core", - "candle-nn", - "prost 0.12.6", -] - [[package]] name = "cap-fs-ext" version = "3.4.4" @@ -5047,19 +4996,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" -[[package]] -name = "denoise" -version = "0.1.0" -dependencies = [ - "candle-core", - "candle-onnx", - "log", - "realfft", - "rodio", - "rustfft", - "thiserror 2.0.17", -] - [[package]] name = "der" version = "0.6.1" @@ -5091,17 +5027,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "derive_builder" version = "0.12.0" @@ -5497,32 +5422,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "dyn-stack" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" -dependencies = [ - "bytemuck", - "reborrow", -] - -[[package]] -name = "dyn-stack" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" -dependencies = [ - "bytemuck", - "dyn-stack-macros", -] - -[[package]] -name = "dyn-stack-macros" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" - [[package]] name = "ec4rs" version = "1.2.0" @@ -6005,18 +5904,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3d8a32ae18130a3c84dd492d4215c3d913c3b07c6b63c2eb3eb7ff1101ab7bf" -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "enumflags2" version = "0.7.12" @@ -6736,18 +6623,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" -[[package]] -name = "float8" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4203231de188ebbdfb85c11f3c20ca2b063945710de04e7b59268731e728b462" -dependencies = [ - "half", - "num-traits", - "rand 0.9.4", - "rand_distr", -] - [[package]] name = "float_next_after" version = "1.0.0" @@ -7213,243 +7088,6 @@ dependencies = [ "triomphe", ] -[[package]] -name = "gemm" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-c32 0.17.1", - "gemm-c64 0.17.1", - "gemm-common 0.17.1", - "gemm-f16 0.17.1", - "gemm-f32 0.17.1", - "gemm-f64 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" -dependencies = [ - "dyn-stack 0.13.2", - "gemm-c32 0.18.2", - "gemm-c64 0.18.2", - "gemm-common 0.18.2", - "gemm-f16 0.18.2", - "gemm-f32 0.18.2", - "gemm-f64 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.6.0", - "seq-macro", -] - -[[package]] -name = "gemm-c32" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-c32" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" -dependencies = [ - "dyn-stack 0.13.2", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.6.0", - "seq-macro", -] - -[[package]] -name = "gemm-c64" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-c64" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" -dependencies = [ - "dyn-stack 0.13.2", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.6.0", - "seq-macro", -] - -[[package]] -name = "gemm-common" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" -dependencies = [ - "bytemuck", - "dyn-stack 0.10.0", - "half", - "num-complex", - "num-traits", - "once_cell", - "paste", - "pulp 0.18.22", - "raw-cpuid 10.7.0", - "rayon", - "seq-macro", - "sysctl 0.5.5", -] - -[[package]] -name = "gemm-common" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" -dependencies = [ - "bytemuck", - "dyn-stack 0.13.2", - "half", - "libm", - "num-complex", - "num-traits", - "once_cell", - "paste", - "pulp 0.21.5", - "raw-cpuid 11.6.0", - "rayon", - "seq-macro", - "sysctl 0.6.0", -] - -[[package]] -name = "gemm-f16" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "gemm-f32 0.17.1", - "half", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "rayon", - "seq-macro", -] - -[[package]] -name = "gemm-f16" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" -dependencies = [ - "dyn-stack 0.13.2", - "gemm-common 0.18.2", - "gemm-f32 0.18.2", - "half", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.6.0", - "rayon", - "seq-macro", -] - -[[package]] -name = "gemm-f32" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-f32" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" -dependencies = [ - "dyn-stack 0.13.2", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.6.0", - "seq-macro", -] - -[[package]] -name = "gemm-f64" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-f64" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" -dependencies = [ - "dyn-stack 0.13.2", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.6.0", - "seq-macro", -] - [[package]] name = "generator" version = "0.8.7" @@ -8479,12 +8117,9 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "bytemuck", "cfg-if", "crunchy", "num-traits", - "rand 0.9.4", - "rand_distr", "zerocopy", ] @@ -9096,7 +8731,7 @@ checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", - "yoke 0.8.0", + "yoke", "zerofrom", "zerovec", ] @@ -9168,7 +8803,7 @@ dependencies = [ "stable_deref_trait", "tinystr", "writeable", - "yoke 0.8.0", + "yoke", "zerofrom", "zerotrie", "zerovec", @@ -11188,7 +10823,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" dependencies = [ "libc", - "stable_deref_trait", ] [[package]] @@ -12001,7 +11635,6 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ - "bytemuck", "num-traits", ] @@ -14120,15 +13753,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "primal-check" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" -dependencies = [ - "num-integer", -] - [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -14727,32 +14351,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "pulp" -version = "0.18.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" -dependencies = [ - "bytemuck", - "libm", - "num-complex", - "reborrow", -] - -[[package]] -name = "pulp" -version = "0.21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907" -dependencies = [ - "bytemuck", - "cfg-if", - "libm", - "num-complex", - "reborrow", - "version_check", -] - [[package]] name = "pxfm" version = "0.1.25" @@ -15101,24 +14699,6 @@ dependencies = [ "rgb", ] -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "raw-cpuid" -version = "11.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" -dependencies = [ - "bitflags 2.10.0", -] - [[package]] name = "raw-window-handle" version = "0.6.2" @@ -15193,21 +14773,6 @@ dependencies = [ "font-types 0.11.0", ] -[[package]] -name = "realfft" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" -dependencies = [ - "rustfft", -] - -[[package]] -name = "reborrow" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" - [[package]] name = "recent_projects" version = "0.1.0" @@ -16024,20 +15589,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustfft" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" -dependencies = [ - "num-complex", - "num-integer", - "num-traits", - "primal-check", - "strength_reduce", - "transpose", -] - [[package]] name = "rustix" version = "0.38.44" @@ -16277,16 +15828,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "safetensors" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "same-file" version = "1.0.6" @@ -16690,12 +16231,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - [[package]] name = "serde" version = "1.0.228" @@ -17900,12 +17435,6 @@ dependencies = [ "util", ] -[[package]] -name = "strength_reduce" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" - [[package]] name = "strict-num" version = "0.1.1" @@ -18392,34 +17921,6 @@ dependencies = [ "libc", ] -[[package]] -name = "sysctl" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" -dependencies = [ - "bitflags 2.10.0", - "byteorder", - "enum-as-inner", - "libc", - "thiserror 1.0.69", - "walkdir", -] - -[[package]] -name = "sysctl" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" -dependencies = [ - "bitflags 2.10.0", - "byteorder", - "enum-as-inner", - "libc", - "thiserror 1.0.69", - "walkdir", -] - [[package]] name = "sysinfo" version = "0.31.4" @@ -19612,16 +19113,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "transpose" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" -dependencies = [ - "num-integer", - "strength_reduce", -] - [[package]] name = "trash" version = "5.2.5" @@ -20012,27 +19503,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "ug" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90b70b37e9074642bc5f60bb23247fd072a84314ca9e71cdf8527593406a0dd3" -dependencies = [ - "gemm 0.18.2", - "half", - "libloading", - "memmap2", - "num", - "num-traits", - "num_cpus", - "rayon", - "safetensors", - "serde", - "thiserror 1.0.69", - "tracing", - "yoke 0.7.5", -] - [[package]] name = "ui" version = "0.1.0" @@ -21423,7 +20893,7 @@ dependencies = [ "reqwest 0.12.24", "scratch", "semver", - "zip 0.6.6", + "zip", ] [[package]] @@ -23234,18 +22704,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive 0.7.5", - "zerofrom", -] - [[package]] name = "yoke" version = "0.8.0" @@ -23254,22 +22712,10 @@ checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive 0.8.0", + "yoke-derive", "zerofrom", ] -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "synstructure", -] - [[package]] name = "yoke-derive" version = "0.8.0" @@ -23849,7 +23295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" dependencies = [ "displaydoc", - "yoke 0.8.0", + "yoke", "zerofrom", ] @@ -23859,7 +23305,7 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ - "yoke 0.8.0", + "yoke", "zerofrom", "zerovec-derive", ] @@ -23906,21 +23352,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "zip" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "indexmap 2.11.4", - "num_enum", - "thiserror 1.0.69", -] - [[package]] name = "zlog" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 809a68a5e3..a293adfb82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,6 @@ members = [ "crates/debugger_tools", "crates/debugger_ui", "crates/deepseek", - "crates/denoise", "crates/dev_container", "crates/diagnostics", "crates/docs_preprocessor", diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index 59df698af1..156915b4ba 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -18,7 +18,6 @@ collections.workspace = true cpal.workspace = true crossbeam.workspace = true gpui.workspace = true -denoise = { path = "../denoise" } log.workspace = true parking_lot.workspace = true rodio.workspace = true diff --git a/crates/audio/src/audio_pipeline/rodio_ext.rs b/crates/audio/src/audio_pipeline/rodio_ext.rs index ab74c59fe6..8fc716e033 100644 --- a/crates/audio/src/audio_pipeline/rodio_ext.rs +++ b/crates/audio/src/audio_pipeline/rodio_ext.rs @@ -8,7 +8,6 @@ use std::{ }; use crossbeam::queue::ArrayQueue; -use denoise::{Denoiser, DenoiserError}; use log::warn; use rodio::{ ChannelCount, Sample, SampleRate, Source, conversions::SampleRateConverter, nz, @@ -36,7 +35,6 @@ pub trait RodioExt: Source + Sized { duration: Duration, ) -> Result<(Replay, Replayable), ReplayDurationTooShort>; fn take_samples(self, n: usize) -> TakeSamples; - fn denoise(self) -> Result, DenoiserError>; fn constant_params( self, channel_count: ChannelCount, @@ -120,10 +118,6 @@ impl RodioExt for S { left_to_take: n, } } - fn denoise(self) -> Result, DenoiserError> { - let res = Denoiser::try_new(self); - res - } fn constant_params( self, channel_count: ChannelCount, diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.toml deleted file mode 100644 index 7d4644a610..0000000000 --- a/crates/denoise/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "denoise" -version = "0.1.0" -edition.workspace = true -publish.workspace = true -license = "GPL-3.0-or-later" - -[lints] -workspace = true - -[dependencies] -candle-core = { version = "0.9.1", git ="https://github.com/zed-industries/candle", branch = "9.1-patched" } -candle-onnx = { version = "0.9.1", git ="https://github.com/zed-industries/candle", branch = "9.1-patched" } -log.workspace = true - -rodio = { workspace = true, features = ["wav_output"] } - -rustfft = { version = "6.2.0", features = ["avx"] } -realfft = "3.4.0" -thiserror.workspace = true diff --git a/crates/denoise/LICENSE-GPL b/crates/denoise/LICENSE-GPL deleted file mode 120000 index 89e542f750..0000000000 --- a/crates/denoise/LICENSE-GPL +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE-GPL \ No newline at end of file diff --git a/crates/denoise/README.md b/crates/denoise/README.md deleted file mode 100644 index d7486da36e..0000000000 --- a/crates/denoise/README.md +++ /dev/null @@ -1,20 +0,0 @@ -Real time streaming audio denoising using a [Dual-Signal Transformation LSTM Network for Real-Time Noise Suppression](https://arxiv.org/abs/2005.07551). - -Trivial to build as it uses the native rust Candle crate for inference. Easy to integrate into any Rodio pipeline. - -```rust - # use rodio::{nz, source::UniformSourceIterator, wav_to_file}; - let file = std::fs::File::open("clips_airconditioning.wav")?; - let decoder = rodio::Decoder::try_from(file)?; - let resampled = UniformSourceIterator::new(decoder, nz!(1), nz!(16_000)); - - let mut denoised = denoise::Denoiser::try_new(resampled)?; - wav_to_file(&mut denoised, "denoised.wav")?; - Result::Ok<(), Box> -``` - -## Acknowledgements & License - -The trained models in this repo are optimized versions of the models in the [breizhn/DTLN](https://github.com/breizhn/DTLN?tab=readme-ov-file#model-conversion-and-real-time-processing-with-onnx). These are licensed under MIT. - -The FFT code was adapted from Datadog's [dtln-rs Repo](https://github.com/DataDog/dtln-rs/tree/main) also licensed under MIT. diff --git a/crates/denoise/examples/denoise.rs b/crates/denoise/examples/denoise.rs deleted file mode 100644 index a4d89d7e51..0000000000 --- a/crates/denoise/examples/denoise.rs +++ /dev/null @@ -1,11 +0,0 @@ -use rodio::{nz, source::UniformSourceIterator, wav_to_file}; - -fn main() -> Result<(), Box> { - let file = std::fs::File::open("airconditioning.wav")?; - let decoder = rodio::Decoder::try_from(file)?; - let resampled = UniformSourceIterator::new(decoder, nz!(1), nz!(16_000)); - - let mut denoised = denoise::Denoiser::try_new(resampled)?; - wav_to_file(&mut denoised, "denoised.wav")?; - Ok(()) -} diff --git a/crates/denoise/examples/enable_disable.rs b/crates/denoise/examples/enable_disable.rs deleted file mode 100644 index 1cffadbce2..0000000000 --- a/crates/denoise/examples/enable_disable.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::time::Duration; - -use rodio::Source; -use rodio::wav_to_file; -use rodio::{nz, source::UniformSourceIterator}; - -fn main() -> Result<(), Box> { - let file = std::fs::File::open("clips_airconditioning.wav")?; - let decoder = rodio::Decoder::try_from(file)?; - let resampled = UniformSourceIterator::new(decoder, nz!(1), nz!(16_000)); - - let mut enabled = true; - let denoised = denoise::Denoiser::try_new(resampled)?.periodic_access( - Duration::from_secs(2), - |denoised| { - enabled = !enabled; - denoised.set_enabled(enabled); - }, - ); - - wav_to_file(denoised, "processed.wav")?; - Ok(()) -} diff --git a/crates/denoise/models/model_1_converted_simplified.onnx b/crates/denoise/models/model_1_converted_simplified.onnx deleted file mode 100644 index 821cb73bd7..0000000000 Binary files a/crates/denoise/models/model_1_converted_simplified.onnx and /dev/null differ diff --git a/crates/denoise/models/model_2_converted_simplified.onnx b/crates/denoise/models/model_2_converted_simplified.onnx deleted file mode 100644 index a83023ab22..0000000000 Binary files a/crates/denoise/models/model_2_converted_simplified.onnx and /dev/null differ diff --git a/crates/denoise/src/engine.rs b/crates/denoise/src/engine.rs deleted file mode 100644 index be0548c689..0000000000 --- a/crates/denoise/src/engine.rs +++ /dev/null @@ -1,204 +0,0 @@ -/// use something like https://netron.app/ to inspect the models and understand -/// the flow -use std::collections::HashMap; - -use candle_core::{Device, IndexOp, Tensor}; -use candle_onnx::onnx::ModelProto; -use candle_onnx::prost::Message; -use realfft::RealFftPlanner; -use rustfft::num_complex::Complex; - -pub struct Engine { - spectral_model: ModelProto, - signal_model: ModelProto, - - fft_planner: RealFftPlanner, - fft_scratch: Vec>, - spectrum: [Complex; FFT_OUT_SIZE], - signal: [f32; BLOCK_LEN], - - in_magnitude: [f32; FFT_OUT_SIZE], - in_phase: [f32; FFT_OUT_SIZE], - - spectral_memory: Tensor, - signal_memory: Tensor, - - in_buffer: [f32; BLOCK_LEN], - out_buffer: [f32; BLOCK_LEN], -} - -// 32 ms @ 16khz per DTLN docs: https://github.com/breizhn/DTLN -pub const BLOCK_LEN: usize = 512; -// 8 ms @ 16khz per DTLN docs. -pub const BLOCK_SHIFT: usize = 128; -pub const FFT_OUT_SIZE: usize = BLOCK_LEN / 2 + 1; - -impl Engine { - pub fn new() -> Self { - let mut fft_planner = RealFftPlanner::new(); - let fft_planned = fft_planner.plan_fft_forward(BLOCK_LEN); - let scratch_len = fft_planned.get_scratch_len(); - Self { - // Models are 1.5MB and 2.5MB respectively. Its worth the binary - // size increase not to have to distribute the models separately. - spectral_model: ModelProto::decode( - include_bytes!("../models/model_1_converted_simplified.onnx").as_slice(), - ) - .expect("The model should decode"), - signal_model: ModelProto::decode( - include_bytes!("../models/model_2_converted_simplified.onnx").as_slice(), - ) - .expect("The model should decode"), - fft_planner, - fft_scratch: vec![Complex::ZERO; scratch_len], - spectrum: [Complex::ZERO; FFT_OUT_SIZE], - signal: [0f32; BLOCK_LEN], - - in_magnitude: [0f32; FFT_OUT_SIZE], - in_phase: [0f32; FFT_OUT_SIZE], - - spectral_memory: Tensor::from_slice::<_, f32>( - &[0f32; 512], - (1, 2, BLOCK_SHIFT, 2), - &Device::Cpu, - ) - .expect("Tensor has the correct dimensions"), - signal_memory: Tensor::from_slice::<_, f32>( - &[0f32; 512], - (1, 2, BLOCK_SHIFT, 2), - &Device::Cpu, - ) - .expect("Tensor has the correct dimensions"), - out_buffer: [0f32; BLOCK_LEN], - in_buffer: [0f32; BLOCK_LEN], - } - } - - /// Add a clunk of samples and get the denoised chunk 4 feeds later - pub fn feed(&mut self, samples: &[f32]) -> [f32; BLOCK_SHIFT] { - /// The name of the output node of the onnx network - /// [Dual-Signal Transformation LSTM Network for Real-Time Noise Suppression](https://arxiv.org/abs/2005.07551). - const MEMORY_OUTPUT: &'static str = "Identity_1"; - - debug_assert_eq!(samples.len(), BLOCK_SHIFT); - - // place new samples at the end of the `in_buffer` - self.in_buffer.copy_within(BLOCK_SHIFT.., 0); - self.in_buffer[(BLOCK_LEN - BLOCK_SHIFT)..].copy_from_slice(&samples); - - // run inference - let inputs = self.spectral_inputs(); - let mut spectral_outputs = candle_onnx::simple_eval(&self.spectral_model, inputs) - .expect("The embedded file must be valid"); - self.spectral_memory = spectral_outputs - .remove(MEMORY_OUTPUT) - .expect("The model has an output named Identity_1"); - let inputs = self.signal_inputs(spectral_outputs); - let mut signal_outputs = candle_onnx::simple_eval(&self.signal_model, inputs) - .expect("The embedded file must be valid"); - self.signal_memory = signal_outputs - .remove(MEMORY_OUTPUT) - .expect("The model has an output named Identity_1"); - let model_output = model_outputs(signal_outputs); - - // place processed samples at the start of the `out_buffer` - // shift the rest left, fill the end with zeros. Zeros are needed as - // the out buffer is part of the input of the network - self.out_buffer.copy_within(BLOCK_SHIFT.., 0); - self.out_buffer[BLOCK_LEN - BLOCK_SHIFT..].fill(0f32); - for (a, b) in self.out_buffer.iter_mut().zip(model_output) { - *a += b; - } - - // samples at the front of the `out_buffer` are now denoised - self.out_buffer[..BLOCK_SHIFT] - .try_into() - .expect("len is correct") - } - - fn spectral_inputs(&mut self) -> HashMap { - // Prepare FFT input - let fft = self.fft_planner.plan_fft_forward(BLOCK_LEN); - - // Perform real-to-complex FFT - let mut fft_in = self.in_buffer; - fft.process_with_scratch(&mut fft_in, &mut self.spectrum, &mut self.fft_scratch) - .expect("The fft should run, there is enough scratch space"); - - // Generate magnitude and phase - for ((magnitude, phase), complex) in self - .in_magnitude - .iter_mut() - .zip(self.in_phase.iter_mut()) - .zip(self.spectrum) - { - *magnitude = complex.norm(); - *phase = complex.arg(); - } - - const SPECTRUM_INPUT: &str = "input_2"; - const MEMORY_INPUT: &str = "input_3"; - let spectrum = - Tensor::from_slice::<_, f32>(&self.in_magnitude, (1, 1, FFT_OUT_SIZE), &Device::Cpu) - .expect("the in magnitude has enough elements to fill the Tensor"); - - let inputs = HashMap::from([ - (SPECTRUM_INPUT.to_string(), spectrum), - (MEMORY_INPUT.to_string(), self.spectral_memory.clone()), - ]); - inputs - } - - fn signal_inputs(&mut self, outputs: HashMap) -> HashMap { - let magnitude_weight = model_outputs(outputs); - - // Apply mask and reconstruct complex spectrum - let mut spectrum = [Complex::I; FFT_OUT_SIZE]; - for i in 0..FFT_OUT_SIZE { - let magnitude = self.in_magnitude[i] * magnitude_weight[i]; - let phase = self.in_phase[i]; - let real = magnitude * phase.cos(); - let imag = magnitude * phase.sin(); - spectrum[i] = Complex::new(real, imag); - } - - // Handle DC component (i = 0) - let magnitude = self.in_magnitude[0] * magnitude_weight[0]; - spectrum[0] = Complex::new(magnitude, 0.0); - - // Handle Nyquist component (i = N/2) - let magnitude = self.in_magnitude[FFT_OUT_SIZE - 1] * magnitude_weight[FFT_OUT_SIZE - 1]; - spectrum[FFT_OUT_SIZE - 1] = Complex::new(magnitude, 0.0); - - // Perform complex-to-real IFFT - let ifft = self.fft_planner.plan_fft_inverse(BLOCK_LEN); - ifft.process_with_scratch(&mut spectrum, &mut self.signal, &mut self.fft_scratch) - .expect("The fft should run, there is enough scratch space"); - - // Normalize the IFFT output - for real in &mut self.signal { - *real /= BLOCK_LEN as f32; - } - - const SIGNAL_INPUT: &str = "input_4"; - const SIGNAL_MEMORY: &str = "input_5"; - let signal_input = - Tensor::from_slice::<_, f32>(&self.signal, (1, 1, BLOCK_LEN), &Device::Cpu).unwrap(); - - HashMap::from([ - (SIGNAL_INPUT.to_string(), signal_input), - (SIGNAL_MEMORY.to_string(), self.signal_memory.clone()), - ]) - } -} - -// Both models put their outputs in the same location -fn model_outputs(mut outputs: HashMap) -> Vec { - const NON_MEMORY_OUTPUT: &str = "Identity"; - outputs - .remove(NON_MEMORY_OUTPUT) - .expect("The model has this output") - .i((0, 0)) - .and_then(|tensor| tensor.to_vec1()) - .expect("The tensor has the correct dimensions") -} diff --git a/crates/denoise/src/lib.rs b/crates/denoise/src/lib.rs deleted file mode 100644 index e92831b965..0000000000 --- a/crates/denoise/src/lib.rs +++ /dev/null @@ -1,269 +0,0 @@ -mod engine; - -use core::fmt; -use std::{collections::VecDeque, sync::mpsc, thread}; - -pub use engine::Engine; -use rodio::{ChannelCount, Sample, SampleRate, Source, nz}; - -use crate::engine::BLOCK_SHIFT; - -const SUPPORTED_SAMPLE_RATE: SampleRate = nz!(16_000); -const SUPPORTED_CHANNEL_COUNT: ChannelCount = nz!(1); - -pub struct Denoiser { - inner: S, - input_tx: mpsc::Sender<[Sample; BLOCK_SHIFT]>, - denoised_rx: mpsc::Receiver<[Sample; BLOCK_SHIFT]>, - ready: [Sample; BLOCK_SHIFT], - next: usize, - state: IterState, - // When disabled instead of reading denoised sub-blocks from the engine through - // `denoised_rx` we read unprocessed from this queue. This maintains the same - // latency so we can 'trivially' re-enable - queued: Queue, -} - -impl fmt::Debug for Denoiser { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Denoiser") - .field("state", &self.state) - .finish_non_exhaustive() - } -} - -struct Queue(VecDeque<[Sample; BLOCK_SHIFT]>); - -impl Queue { - fn new() -> Self { - Self(VecDeque::new()) - } - fn push(&mut self, block: [Sample; BLOCK_SHIFT]) { - self.0.push_back(block); - self.0.resize(4, [0f32; BLOCK_SHIFT]); - } - fn pop(&mut self) -> [Sample; BLOCK_SHIFT] { - debug_assert!(self.0.len() == 4); - self.0.pop_front().expect( - "There is no State where the queue is popped while there are less then 4 entries", - ) - } -} - -#[derive(Debug, Clone, Copy)] -pub enum IterState { - Enabled, - StartingMidAudio { fed_to_denoiser: usize }, - Disabled, - Startup { enabled: bool }, -} - -#[derive(Debug, thiserror::Error)] -pub enum DenoiserError { - #[error("This denoiser only works on sources with samplerate 16000")] - UnsupportedSampleRate, - #[error("This denoiser only works on mono sources (1 channel)")] - UnsupportedChannelCount, -} - -// todo dvdsk needs constant source upstream in rodio -impl Denoiser { - pub fn try_new(source: S) -> Result { - if source.sample_rate() != SUPPORTED_SAMPLE_RATE { - return Err(DenoiserError::UnsupportedSampleRate); - } - if source.channels() != SUPPORTED_CHANNEL_COUNT { - return Err(DenoiserError::UnsupportedChannelCount); - } - - let (input_tx, input_rx) = mpsc::channel(); - let (denoised_tx, denoised_rx) = mpsc::channel(); - - thread::Builder::new() - .name("NeuralDenoiser".to_owned()) - .spawn(move || { - run_neural_denoiser(denoised_tx, input_rx); - }) - .expect("Should be ablet to spawn threads"); - - Ok(Self { - inner: source, - input_tx, - denoised_rx, - ready: [0.0; BLOCK_SHIFT], - state: IterState::Startup { enabled: true }, - next: BLOCK_SHIFT, - queued: Queue::new(), - }) - } - - pub fn set_enabled(&mut self, enabled: bool) { - self.state = match (enabled, self.state) { - (false, IterState::StartingMidAudio { .. }) | (false, IterState::Enabled) => { - IterState::Disabled - } - (false, IterState::Startup { enabled: true }) => IterState::Startup { enabled: false }, - (true, IterState::Disabled) => IterState::StartingMidAudio { fed_to_denoiser: 0 }, - (_, state) => state, - }; - } - - fn feed(&self, sub_block: [f32; BLOCK_SHIFT]) { - self.input_tx.send(sub_block).unwrap(); - } -} - -fn run_neural_denoiser( - denoised_tx: mpsc::Sender<[f32; BLOCK_SHIFT]>, - input_rx: mpsc::Receiver<[f32; BLOCK_SHIFT]>, -) { - let mut engine = Engine::new(); - // until tx is dropped - while let Ok(sub_block) = input_rx.recv() { - let denoised_sub_block = engine.feed(&sub_block); - if denoised_tx.send(denoised_sub_block).is_err() { - break; - } - } -} - -impl Source for Denoiser { - fn current_span_len(&self) -> Option { - self.inner.current_span_len() - } - - fn channels(&self) -> rodio::ChannelCount { - self.inner.channels() - } - - fn sample_rate(&self) -> rodio::SampleRate { - self.inner.sample_rate() - } - - fn total_duration(&self) -> Option { - self.inner.total_duration() - } -} - -impl Iterator for Denoiser { - type Item = Sample; - - #[inline] - fn next(&mut self) -> Option { - self.next += 1; - if self.next < self.ready.len() { - let sample = self.ready[self.next]; - return Some(sample); - } - - // This is a separate function to prevent it from being inlined - // as this code only runs once every 128 samples - self.prepare_next_ready() - .inspect_err(|_| { - log::error!("Denoise engine crashed"); - }) - .ok() - .flatten() - } -} - -#[derive(Debug, thiserror::Error)] -#[error("Could not send or receive from denoise thread. It must have crashed")] -struct DenoiseEngineCrashed; - -impl Denoiser { - #[cold] - fn prepare_next_ready(&mut self) -> Result, DenoiseEngineCrashed> { - self.state = match self.state { - IterState::Startup { enabled } => { - // guaranteed to be coming from silence - for _ in 0..3 { - let Some(sub_block) = read_sub_block(&mut self.inner) else { - return Ok(None); - }; - self.queued.push(sub_block); - self.input_tx - .send(sub_block) - .map_err(|_| DenoiseEngineCrashed)?; - } - let Some(sub_block) = read_sub_block(&mut self.inner) else { - return Ok(None); - }; - self.queued.push(sub_block); - self.input_tx - .send(sub_block) - .map_err(|_| DenoiseEngineCrashed)?; - // throw out old blocks that are denoised silence - let _ = self.denoised_rx.iter().take(3).count(); - self.ready = self.denoised_rx.recv().map_err(|_| DenoiseEngineCrashed)?; - - let Some(sub_block) = read_sub_block(&mut self.inner) else { - return Ok(None); - }; - self.queued.push(sub_block); - self.feed(sub_block); - - if enabled { - IterState::Enabled - } else { - IterState::Disabled - } - } - IterState::Enabled => { - self.ready = self.denoised_rx.recv().map_err(|_| DenoiseEngineCrashed)?; - let Some(sub_block) = read_sub_block(&mut self.inner) else { - return Ok(None); - }; - self.queued.push(sub_block); - self.input_tx - .send(sub_block) - .map_err(|_| DenoiseEngineCrashed)?; - IterState::Enabled - } - IterState::Disabled => { - // Need to maintain the same 512 samples delay such that - // we can re-enable at any point. - self.ready = self.queued.pop(); - let Some(sub_block) = read_sub_block(&mut self.inner) else { - return Ok(None); - }; - self.queued.push(sub_block); - IterState::Disabled - } - IterState::StartingMidAudio { - fed_to_denoiser: mut sub_blocks_fed, - } => { - self.ready = self.queued.pop(); - let Some(sub_block) = read_sub_block(&mut self.inner) else { - return Ok(None); - }; - self.queued.push(sub_block); - self.input_tx - .send(sub_block) - .map_err(|_| DenoiseEngineCrashed)?; - sub_blocks_fed += 1; - if sub_blocks_fed > 4 { - // throw out partially denoised blocks, - // next will be correctly denoised - let _ = self.denoised_rx.iter().take(3).count(); - IterState::Enabled - } else { - IterState::StartingMidAudio { - fed_to_denoiser: sub_blocks_fed, - } - } - } - }; - - self.next = 0; - Ok(Some(self.ready[0])) - } -} - -fn read_sub_block(s: &mut impl Source) -> Option<[f32; BLOCK_SHIFT]> { - let mut res = [0f32; BLOCK_SHIFT]; - for sample in &mut res { - *sample = s.next()?; - } - Some(res) -}