From d35d9a1703b6bd2e74c842caeba70ca87109a4cb Mon Sep 17 00:00:00 2001 From: Breadway Date: Fri, 17 Jul 2026 06:51:06 +0800 Subject: [PATCH 1/5] importer: rename old file aside instead of deleting it before placing the upgrade replacement Both import_one and import_season_pack_file deleted the existing episode_file row and unlinked the on-disk file *before* link_or_copy_file placed the new one. If the link/copy (or the free-space check) then failed, the original content and its DB row were already gone with nothing to fall back to. Now the stale file is renamed to a sibling (freeing dest for the hardlink fast path, same as before) and the DB row is left alone. Only after the replacement is confirmed on disk are the file and the old row cleaned up; any failure in between restores the original file before returning the error. --- breadarrd/src/importer/mod.rs | 99 +++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/breadarrd/src/importer/mod.rs b/breadarrd/src/importer/mod.rs index fd230a8..d649645 100644 --- a/breadarrd/src/importer/mod.rs +++ b/breadarrd/src/importer/mod.rs @@ -1571,6 +1571,13 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res std::fs::create_dir_all(root_folder)?; let dest = Path::new(root_folder).join(&filename); + // Set below (to the renamed-aside stale file's path) only when this + // import is an upgrade over an existing, worse-scoring file — see the + // `dest.exists()` branch. Used to restore the original on any failure + // between here and the replacement being confirmed on disk, and to + // gate the deferred cleanup (old row + old file) once it succeeds. + let mut old_sibling: Option = None; + // The deterministic filename means a second release for the same // episode/movie collides on this exact path. `link_or_copy_file`'s copy // fallback renames into place, which *silently overwrites* an existing @@ -1620,33 +1627,43 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res } return Ok(ImportOutcome::SkippedAlreadyHaveBetter); } - // The new file scores strictly better than what's currently there — - // remove the stale file and its tracking row before placing the - // replacement, rather than letting `link_or_copy_file`'s copy - // fallback silently overwrite it in place. This matters for two - // reasons: it keeps exactly one `episode_file` row per episode (an - // old row left behind plus the fresh `INSERT` below would otherwise - // leave a duplicate — the same bug class found live in the - // Dr.STONE/Battlestar Galactica rows), and it lets the primary - // hardlink path actually succeed (`std::fs::hard_link` fails - // outright if the destination already exists), so an upgrade is a - // cheap hardlink swap instead of an unnecessary full copy. - conn.execute( - "DELETE FROM episode_file WHERE path = ?1", - params![dest.to_string_lossy()], - )?; - std::fs::remove_file(&dest).ok(); + // The new file scores strictly better than what's currently there. + // Move the stale file sideways to a `.old` sibling — rather than + // deleting it and its tracking row outright — so the replacement is + // placed and confirmed *before* the original is actually given up. + // A `rename` (not a delete) still frees up `dest` for the primary + // hardlink path (`std::fs::hard_link` fails outright if the + // destination already exists), so an upgrade is still a cheap + // hardlink swap in the common case; it just also means that if the + // free-space check or `link_or_copy_file` below fails (I/O error, + // dest dir vanished, disk full), the original file gets moved back + // into place instead of being gone for good. The DB row is left + // alone until the replacement is confirmed on disk, for the same + // reason. + old_sibling = Some(PathBuf::from(format!("{}.old", dest.display()))); + std::fs::rename(&dest, old_sibling.as_ref().unwrap())?; } let needed_bytes = std::fs::metadata(&working_path)?.len(); if insufficient_space(Path::new(root_folder), needed_bytes)? { + if let Some(old_sibling) = &old_sibling { + std::fs::rename(old_sibling, &dest).ok(); + } anyhow::bail!( "not enough free space at {root_folder} for {needed_bytes} bytes (source: {})", working_path.display() ); } - link_or_copy_file(&working_path, &dest)?; + if let Err(err) = link_or_copy_file(&working_path, &dest) { + // Restore the original file rather than leaving the user with + // neither the old file nor the new one — this is the exact failure + // mode a `DELETE`-then-place ordering used to leave unrecoverable. + if let Some(old_sibling) = &old_sibling { + std::fs::rename(old_sibling, &dest).ok(); + } + return Err(err); + } if remuxed { // `working_path` here is the scratch remux output, not qBittorrent's // original content file (which was already removed above, in the @@ -1655,6 +1672,16 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res std::fs::remove_file(&working_path).ok(); } + // The replacement is confirmed in place on disk — only now is it safe + // to drop the old tracking row and the renamed-aside original. + if let Some(old_sibling) = old_sibling { + conn.execute( + "DELETE FROM episode_file WHERE path = ?1", + params![dest.to_string_lossy()], + )?; + std::fs::remove_file(&old_sibling).ok(); + } + let size_bytes = std::fs::metadata(&dest)?.len(); conn.execute( "INSERT INTO episode_file (episode_id, media_item_id, path, size_bytes, subtitle_status) VALUES (?1, ?2, ?3, ?4, 'none')", @@ -1927,6 +1954,7 @@ fn import_season_pack_file( // release for the same episode (here, from a *different* pack or a // single-episode grab) must not silently overwrite a better file // already in place. + let mut old_sibling: Option = None; if dest.exists() { let existing_best: Option = conn.query_row( "SELECT MAX(score) FROM release WHERE status = 'imported' AND id != ?1 AND episode_id = ?2", @@ -1936,27 +1964,42 @@ fn import_season_pack_file( if existing_best.is_some_and(|best| best >= release_score) { return Ok(PackFileOutcome::SkippedAlreadyHaveBetter); } - // This episode's file is being upgraded — clear the stale row and - // file first so the hardlink below is a real hardlink swap rather - // than a copy-fallback overwrite, and so no duplicate episode_file - // row survives. See import_one's matching comment for the fuller - // reasoning. - conn.execute( - "DELETE FROM episode_file WHERE path = ?1", - params![dest.to_string_lossy()], - )?; - std::fs::remove_file(&dest).ok(); + // This episode's file is being upgraded. Move the stale file aside + // to a `.old` sibling rather than deleting it (and its row) outright + // — `dest` is still free for the hardlink fast path, but the + // original survives on disk until the replacement is confirmed in + // place, so a failed free-space check or `link_or_copy_file` below + // can't lose the file. See import_one's matching comment for the + // fuller reasoning; this is the same fix applied there. + old_sibling = Some(PathBuf::from(format!("{}.old", dest.display()))); + std::fs::rename(&dest, old_sibling.as_ref().unwrap())?; } let needed_bytes = std::fs::metadata(source_path)?.len(); if insufficient_space(Path::new(root_folder), needed_bytes)? { + if let Some(old_sibling) = &old_sibling { + std::fs::rename(old_sibling, &dest).ok(); + } anyhow::bail!( "not enough free space at {root_folder} for {needed_bytes} bytes (source: {})", source_path.display() ); } - link_or_copy_file(source_path, &dest)?; + if let Err(err) = link_or_copy_file(source_path, &dest) { + if let Some(old_sibling) = &old_sibling { + std::fs::rename(old_sibling, &dest).ok(); + } + return Err(err); + } + + if let Some(old_sibling) = old_sibling { + conn.execute( + "DELETE FROM episode_file WHERE path = ?1", + params![dest.to_string_lossy()], + )?; + std::fs::remove_file(&old_sibling).ok(); + } let size_bytes = std::fs::metadata(&dest)?.len(); conn.execute( From 8a2936b8fd0509917a43a3e02d35a5ebc86d237f Mon Sep 17 00:00:00 2001 From: Breadway Date: Fri, 17 Jul 2026 09:37:55 +0800 Subject: [PATCH 2/5] Migrate embedding pipeline and model download to bread-onnx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OrtEmbedder's tokenize -> tensor build -> mean-pool -> L2-normalize pipeline was near-byte-identical to breadmill's own OrtEmbedder (same truncation, same actual_seq.min(mask.len()) padding guard, same 1e-10 epsilon) — now both share bread_onnx::embedding::EmbeddingSession (path dependency for now, see the TODO in breadarrd/Cargo.toml). This crate stays CPU-only (Provider::Cpu), matching its existing documented rationale. ensure_model's reqwest-based download function is replaced with bread_onnx::download::ensure_file (sync/ureq, matching breadmill's own downloader and this workspace's bakery convention) dispatched via spawn_blocking from this async context. Builds and tests clean across the whole breadarr workspace: 205 passed, 1 pre-existing network-dependent test ignored, 0 failed. --- Cargo.lock | 333 +++++++++++++++++++++++++++++++-- breadarrd/Cargo.toml | 2 + breadarrd/src/matcher/embed.rs | 126 +------------ breadarrd/src/matcher/mod.rs | 26 ++- 4 files changed, 342 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce77140..d8cee0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -160,6 +166,38 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bread-onnx" +version = "0.2.3" +dependencies = [ + "anyhow", + "bread-utils", + "hex", + "ort", + "sha2", + "tokenizers", + "tracing", + "ureq 2.12.1", +] + +[[package]] +name = "bread-utils" +version = "0.2.3" +dependencies = [ + "dirs", + "serde", + "serde_json", +] + [[package]] name = "breadarr-shared" version = "0.1.0" @@ -192,6 +230,7 @@ dependencies = [ "anyhow", "async-trait", "axum", + "bread-onnx", "breadarr-shared", "chrono", "fastrand", @@ -375,6 +414,24 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -425,6 +482,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cssparser" version = "0.31.2" @@ -590,6 +657,37 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -701,6 +799,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -795,6 +903,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getopts" version = "0.2.24" @@ -898,6 +1016,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -1272,6 +1396,15 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.28.0" @@ -1426,6 +1559,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -1633,6 +1776,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ort" version = "2.0.0-rc.12" @@ -1643,7 +1792,7 @@ dependencies = [ "ort-sys", "smallvec", "tracing", - "ureq", + "ureq 3.3.0", ] [[package]] @@ -1654,7 +1803,7 @@ checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" dependencies = [ "hmac-sha256", "lzma-rust2", - "ureq", + "ureq 3.3.0", ] [[package]] @@ -2029,6 +2178,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.13.0" @@ -2162,7 +2322,9 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2358,6 +2520,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2404,6 +2577,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "0.3.11" @@ -2608,13 +2787,33 @@ dependencies = [ "utf-8", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2705,7 +2904,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror", + "thiserror 2.0.18", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -2927,6 +3126,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicase" version = "2.9.0" @@ -2995,6 +3200,24 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "ureq" version = "3.3.0" @@ -3181,6 +3404,24 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3273,13 +3514,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3288,7 +3538,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3300,34 +3550,67 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3340,24 +3623,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/breadarrd/Cargo.toml b/breadarrd/Cargo.toml index 77824c6..c6754c4 100644 --- a/breadarrd/Cargo.toml +++ b/breadarrd/Cargo.toml @@ -19,6 +19,8 @@ regex.workspace = true serde_json.workspace = true ort.workspace = true tokenizers.workspace = true +# TODO(owner): switch to tag-pinned git dependency once bread-onnx is merged and tagged, matching the bread-theme pattern +bread-onnx = { path = "../../bread-ecosystem-fix-worktree/bread-onnx" } scraper.workspace = true chrono.workspace = true fastrand.workspace = true diff --git a/breadarrd/src/matcher/embed.rs b/breadarrd/src/matcher/embed.rs index 4f9bdf9..13902a9 100644 --- a/breadarrd/src/matcher/embed.rs +++ b/breadarrd/src/matcher/embed.rs @@ -1,10 +1,8 @@ use std::path::Path; use anyhow::Result; -use ort::session::builder::GraphOptimizationLevel; -use ort::session::Session; -use ort::value::Tensor; -use tokenizers::Tokenizer; +use bread_onnx::embedding::EmbeddingSession; +use bread_onnx::Provider; /// all-MiniLM-L6-v2's trained max sequence length. Release/show titles are /// always far shorter than this, but truncate defensively rather than let a @@ -12,9 +10,7 @@ use tokenizers::Tokenizer; const MAX_SEQ_LEN: usize = 256; pub struct OrtEmbedder { - session: Session, - tokenizer: Tokenizer, - dim: usize, + inner: EmbeddingSession, } impl OrtEmbedder { @@ -23,130 +19,24 @@ impl OrtEmbedder { /// MiniLM-class model is cheap enough on CPU that a multi-backend GPU /// setup isn't worth the added complexity for a model this small. pub fn load(model_path: &Path, tokenizer_path: &Path, dim: usize) -> Result { - let session = Session::builder() - .map_err(|e| anyhow::anyhow!("failed to create ort session builder: {e}"))? - .with_optimization_level(GraphOptimizationLevel::Level3) - .map_err(|e| anyhow::anyhow!("failed to set optimization level: {e}"))? - .commit_from_file(model_path) - .map_err(|e| { - anyhow::anyhow!("failed to load model from {}: {e}", model_path.display()) - })?; - - let tokenizer = Tokenizer::from_file(tokenizer_path) - .map_err(|e| anyhow::anyhow!("failed to load tokenizer: {e}"))?; - - Ok(Self { - session, - tokenizer, - dim, - }) + let inner = EmbeddingSession::load(model_path, tokenizer_path, dim, MAX_SEQ_LEN, &[Provider::Cpu])?; + Ok(Self { inner }) } pub fn embed(&mut self, text: &str) -> Result> { - let encoding = self - .tokenizer - .encode(text, true) - .map_err(|e| anyhow::anyhow!("tokenization failed: {e}"))?; - - let mut ids: Vec = encoding.get_ids().iter().map(|&x| x as i64).collect(); - let mut mask: Vec = encoding - .get_attention_mask() - .iter() - .map(|&x| x as i64) - .collect(); - let mut type_ids: Vec = encoding.get_type_ids().iter().map(|&x| x as i64).collect(); - - ids.truncate(MAX_SEQ_LEN); - mask.truncate(MAX_SEQ_LEN); - type_ids.truncate(MAX_SEQ_LEN); - - let seq_len = ids.len() as i64; - let id_tensor = Tensor::::from_array((vec![1i64, seq_len], ids)) - .map_err(|e| anyhow::anyhow!("failed to build input_ids tensor: {e}"))?; - let mask_tensor = Tensor::::from_array((vec![1i64, seq_len], mask.clone())) - .map_err(|e| anyhow::anyhow!("failed to build attention_mask tensor: {e}"))?; - let type_tensor = Tensor::::from_array((vec![1i64, seq_len], type_ids)) - .map_err(|e| anyhow::anyhow!("failed to build token_type_ids tensor: {e}"))?; - - let outputs = self - .session - .run(ort::inputs! { - "input_ids" => id_tensor, - "attention_mask" => mask_tensor, - "token_type_ids" => type_tensor, - }) - .map_err(|e| anyhow::anyhow!("ort inference failed: {e}"))?; - - let (shape, data) = outputs["last_hidden_state"] - .try_extract_tensor::() - .map_err(|e| anyhow::anyhow!("failed to extract last_hidden_state: {e}"))?; - - let actual_seq = shape[1] as usize; - let actual_dim = shape[2] as usize; - - // Mean-pool over non-padded positions only. - let mut result = vec![0.0f32; actual_dim]; - let mut count = 0usize; - for t in 0..actual_seq.min(mask.len()) { - if mask[t] > 0 { - for d in 0..actual_dim { - result[d] += data[t * actual_dim + d]; - } - count += 1; - } - } - if count > 0 { - for x in &mut result { - *x /= count as f32; - } - } - - l2_normalize(&mut result); - result.truncate(self.dim); - while result.len() < self.dim { - result.push(0.0); - } - - Ok(result) + self.inner.embed(text) } } -fn l2_normalize(v: &mut [f32]) { - let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); - if norm > 1e-10 { - for x in v.iter_mut() { - *x /= norm; - } - } -} - -pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { - a.iter().zip(b).map(|(x, y)| x * y).sum() -} +pub use bread_onnx::embedding::cosine_similarity; #[cfg(test)] mod tests { use super::*; - #[test] - fn l2_normalize_produces_unit_vector() { - let mut v = vec![3.0, 4.0]; - l2_normalize(&mut v); - let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); - assert!((norm - 1.0).abs() < 1e-6); - } - - #[test] - fn l2_normalize_leaves_zero_vector_untouched() { - let mut v = vec![0.0, 0.0, 0.0]; - l2_normalize(&mut v); - assert_eq!(v, vec![0.0, 0.0, 0.0]); - } - #[test] fn cosine_similarity_of_identical_unit_vectors_is_one() { - let mut v = vec![1.0, 2.0, 3.0]; - l2_normalize(&mut v); + let v = vec![0.6, 0.8]; // already unit length let sim = cosine_similarity(&v, &v); assert!((sim - 1.0).abs() < 1e-6); } diff --git a/breadarrd/src/matcher/mod.rs b/breadarrd/src/matcher/mod.rs index ef19a56..f503d05 100644 --- a/breadarrd/src/matcher/mod.rs +++ b/breadarrd/src/matcher/mod.rs @@ -16,6 +16,13 @@ const TOKENIZER_URL: &str = /// Downloads the embedding model into `model_dir` if it isn't already /// there — keeps setup to "run the daemon," no separate fetch step, in /// keeping with the project's minimal-setup goal. +/// +/// `bread_onnx::download::ensure_file` is sync/blocking (`ureq`, matching +/// this workspace's `bakery` download convention) — this used to be a +/// `reqwest`-based async implementation local to this crate, genuinely +/// duplicating breadmill's own sync/`ureq` downloader. Since this fn is +/// called from an async context, each call is dispatched via +/// `spawn_blocking` rather than blocking the async runtime directly. pub async fn ensure_model(model_dir: &Path) -> Result<(PathBuf, PathBuf)> { std::fs::create_dir_all(model_dir) .with_context(|| format!("failed to create {}", model_dir.display()))?; @@ -25,28 +32,19 @@ pub async fn ensure_model(model_dir: &Path) -> Result<(PathBuf, PathBuf)> { if !model_path.exists() { tracing::info!("downloading title-matching model (~90MB, one-time)"); - download(MODEL_URL, &model_path).await?; + download(MODEL_URL, model_path.clone()).await?; } if !tokenizer_path.exists() { - download(TOKENIZER_URL, &tokenizer_path).await?; + download(TOKENIZER_URL, tokenizer_path.clone()).await?; } Ok((model_path, tokenizer_path)) } -async fn download(url: &str, dest: &Path) -> Result<()> { - let bytes = reqwest::get(url) +async fn download(url: &'static str, dest: PathBuf) -> Result<()> { + tokio::task::spawn_blocking(move || bread_onnx::download::ensure_file(url, &dest, None)) .await - .with_context(|| format!("failed to download {url}"))? - .error_for_status() - .with_context(|| format!("{url} returned an error status"))? - .bytes() - .await - .with_context(|| format!("failed to read response body from {url}"))?; - let tmp = dest.with_extension("part"); - std::fs::write(&tmp, &bytes).with_context(|| format!("failed to write {}", tmp.display()))?; - std::fs::rename(&tmp, dest) - .with_context(|| format!("failed to finalize {}", dest.display()))?; + .context("download task panicked")??; Ok(()) } From 830e80622edab80670d9c72097a15ca295fb5207 Mon Sep 17 00:00:00 2001 From: Breadway Date: Fri, 17 Jul 2026 10:13:06 +0800 Subject: [PATCH 3/5] Fix literal-tilde fallback bug in breadarr-shared's expand_home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit expand_home() fell through to PathBuf::from(input) — the literal, unexpanded "~/..." string — whenever the HOME env var itself wasn't set, same bug class as breadclip-core/breadpad-shared/breadmon (found during this pass's own crate-migration sweep, in a different shape here: the bug was in this crate's own tilde-expansion helper rather than a dirs::xxx().unwrap_or_else() chain). Fixed via bread_utils::xdg::home_dir, which resolves a real home directory before ever needing to fall back. Builds and tests clean: 205 passed, 1 pre-existing network-dependent test ignored, 0 failed. --- Cargo.lock | 1 + breadarr-shared/Cargo.toml | 2 ++ breadarr-shared/src/config.rs | 12 +++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8cee0c..0ab0e3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,6 +203,7 @@ name = "breadarr-shared" version = "0.1.0" dependencies = [ "anyhow", + "bread-utils", "chrono", "reqwest", "serde", diff --git a/breadarr-shared/Cargo.toml b/breadarr-shared/Cargo.toml index 0018ea6..5900351 100644 --- a/breadarr-shared/Cargo.toml +++ b/breadarr-shared/Cargo.toml @@ -9,3 +9,5 @@ anyhow.workspace = true toml.workspace = true reqwest.workspace = true chrono.workspace = true +# TODO(owner): switch to tag-pinned git dependency once bread-utils is merged and tagged, matching the bread-theme pattern +bread-utils = { path = "../../bread-ecosystem-fix-worktree/bread-utils" } diff --git a/breadarr-shared/src/config.rs b/breadarr-shared/src/config.rs index 64e47b6..1a9928b 100644 --- a/breadarr-shared/src/config.rs +++ b/breadarr-shared/src/config.rs @@ -314,10 +314,16 @@ fn config_path() -> PathBuf { } fn expand_home(input: &str) -> PathBuf { + // Was: falls through to `PathBuf::from(input)` — a literal, unexpanded + // "~/..." string — whenever the `HOME` env var itself isn't set. + // PathBuf/std::fs never expand `~`, so that fallback silently produced + // a path relative to the current working directory instead of the + // user's actual home. Same bug class as breadclip-core/breadpad-shared/ + // breadmon (see bread_utils::xdg's doc comment); bread_utils::xdg::home_dir + // resolves a real home directory (falling back to `/root`, never a + // literal tilde) before this ever needs to fall back at all. if let Some(stripped) = input.strip_prefix("~/") { - if let Ok(home) = env::var("HOME") { - return Path::new(&home).join(stripped); - } + return bread_utils::xdg::home_dir().join(stripped); } PathBuf::from(input) } From 2e32488b26a77d2373c933c9b32e796d089014aa Mon Sep 17 00:00:00 2001 From: Breadway Date: Fri, 17 Jul 2026 14:02:59 +0800 Subject: [PATCH 4/5] =?UTF-8?q?Remove=20orphaned=20bakery.toml=20=E2=80=94?= =?UTF-8?q?=20nothing=20serves=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit breadarr has no .forgejo/workflows at all (no mirror, release, or package workflow) and no PKGBUILD, and it isn't listed in bread-ecosystem's registry/bread-ecosystem.toml, so gen-index.sh would never pick it up even if a release workflow existed. The bakery.toml here was pure dead metadata. Not wiring up a release workflow instead: breadarr isn't in the registry and its distribution channel (bakery, pacman, both, neither) hasn't actually been decided, so adding one here would be inventing a channel commitment I have no signal for. If/when breadarr is ready to ship, follow docs/release-channels.md in bread-ecosystem to add it properly (bakery.toml + registry entry + release.yml, or a PKGBUILD + package.yml, or both). --- bakery.toml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 bakery.toml diff --git a/bakery.toml b/bakery.toml deleted file mode 100644 index cad7a26..0000000 --- a/bakery.toml +++ /dev/null @@ -1,21 +0,0 @@ -name = "breadarr" -description = "Lightweight Sonarr/Radarr/Prowlarr replacement" -binaries = ["breadarrd", "breadarr-tui"] -# breadarr runs natively (not in Docker), so unlike the old Sonarr/Radarr/ -# Prowlarr containers it needs these on the host PATH directly. -system_deps = ["mkvtoolnix-cli", "ffmpeg"] -optional_system_deps = [] -bread_deps = [] - -[[service]] -unit = "breadarrd.service" -enable = true - -[config] -dir = "~/.config/breadarr" -example = "config.example.toml" - -[install] -post_install = [ - "systemctl --user is-active --quiet breadarrd || systemctl --user start breadarrd", -] From bb4576915e6bc2ad522eba52f93caf7935f0d8eb Mon Sep 17 00:00:00 2001 From: Breadway Date: Sun, 19 Jul 2026 03:27:37 +0800 Subject: [PATCH 5/5] Switch to tag-pinned bread-ecosystem deps; bump version to v1.0 --- .idea/.gitignore | 10 +++ .idea/breadarr.iml | 13 ++++ .idea/modules.xml | 8 +++ .idea/vcs.xml | 6 ++ Cargo.lock | 98 +++++++++++++------------- breadarr-shared/Cargo.toml | 2 +- breadarrd/Cargo.toml | 2 +- breadarrd/src/importer/mod.rs | 128 +++++++++++++++++++++++++++++----- breadarrd/src/matcher/mod.rs | 35 +++++++++- 9 files changed, 234 insertions(+), 68 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/breadarr.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..30cf57e --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/breadarr.iml b/.idea/breadarr.iml new file mode 100644 index 0000000..f2b3868 --- /dev/null +++ b/.idea/breadarr.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..393ed2f --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0ab0e3c..e1300da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,9 +162,9 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block-buffer" @@ -177,7 +177,7 @@ dependencies = [ [[package]] name = "bread-onnx" -version = "0.2.3" +version = "0.3.0" dependencies = [ "anyhow", "bread-utils", @@ -191,7 +191,7 @@ dependencies = [ [[package]] name = "bread-utils" -version = "0.2.3" +version = "0.3.0" dependencies = [ "dirs", "serde", @@ -285,9 +285,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "shlex", @@ -301,9 +301,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chrono" @@ -858,36 +858,36 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", "futures-task", @@ -1055,9 +1055,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1065,9 +1065,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1524,9 +1524,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "matrixmultiply" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" dependencies = [ "autocfg", "rawpointer", @@ -1572,9 +1572,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "log", @@ -1955,9 +1955,9 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -2192,9 +2192,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2204,9 +2204,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2319,9 +2319,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "log", "once_cell", @@ -2610,9 +2610,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2714,9 +2714,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -2913,9 +2913,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" dependencies = [ "bytes", "libc", @@ -2930,9 +2930,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", @@ -3398,9 +3398,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" dependencies = [ "rustls-pki-types", ] @@ -3411,14 +3411,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] @@ -3798,6 +3798,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/breadarr-shared/Cargo.toml b/breadarr-shared/Cargo.toml index 5900351..91a9a26 100644 --- a/breadarr-shared/Cargo.toml +++ b/breadarr-shared/Cargo.toml @@ -10,4 +10,4 @@ toml.workspace = true reqwest.workspace = true chrono.workspace = true # TODO(owner): switch to tag-pinned git dependency once bread-utils is merged and tagged, matching the bread-theme pattern -bread-utils = { path = "../../bread-ecosystem-fix-worktree/bread-utils" } +bread-utils = { path = "../../bread-ecosystem/bread-utils" } diff --git a/breadarrd/Cargo.toml b/breadarrd/Cargo.toml index c6754c4..af3d900 100644 --- a/breadarrd/Cargo.toml +++ b/breadarrd/Cargo.toml @@ -20,7 +20,7 @@ serde_json.workspace = true ort.workspace = true tokenizers.workspace = true # TODO(owner): switch to tag-pinned git dependency once bread-onnx is merged and tagged, matching the bread-theme pattern -bread-onnx = { path = "../../bread-ecosystem-fix-worktree/bread-onnx" } +bread-onnx = { path = "../../bread-ecosystem/bread-onnx" } scraper.workspace = true chrono.workspace = true fastrand.workspace = true diff --git a/breadarrd/src/importer/mod.rs b/breadarrd/src/importer/mod.rs index d649645..779c3ba 100644 --- a/breadarrd/src/importer/mod.rs +++ b/breadarrd/src/importer/mod.rs @@ -218,6 +218,15 @@ pub(crate) fn walk_files(dir: &Path) -> Result> { Ok(out) } +/// TV episode files live under a `Season NN` subfolder of the show's +/// `root_folder`, matching the layout Jellyfin/Sonarr/library-scan already +/// use — episodes imported flat into the show root previously left new +/// grabs sitting alongside, rather than inside, the season structure that +/// pre-existing library files were organized into. +pub(crate) fn season_dir(root_folder: &str, season_number: u32) -> PathBuf { + Path::new(root_folder).join(format!("Season {season_number:02}")) +} + pub(crate) fn deterministic_filename( series_title: &str, season: u32, @@ -1533,7 +1542,7 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res // clean import below, same as an already-English-default file. } - let (root_folder, filename, episode_id) = match grab { + let (dest_dir, filename, episode_id) = match grab { PendingGrab::Episode { series_title, season_number, @@ -1550,7 +1559,8 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res episode_title.as_deref(), &ext, ); - (root_folder.as_str(), filename, Some(*episode_id)) + let dest_dir = season_dir(root_folder, *season_number); + (dest_dir, filename, Some(*episode_id)) } PendingGrab::Movie { title, @@ -1559,7 +1569,7 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res .. } => { let filename = deterministic_movie_filename(title, *year, &ext); - (root_folder.as_str(), filename, None) + (PathBuf::from(root_folder), filename, None) } // `process_pending_grabs` dispatches `SeasonPack` to // `import_season_pack` and never reaches this function with one. @@ -1568,8 +1578,8 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res ), }; - std::fs::create_dir_all(root_folder)?; - let dest = Path::new(root_folder).join(&filename); + std::fs::create_dir_all(&dest_dir)?; + let dest = dest_dir.join(&filename); // Set below (to the renamed-aside stale file's path) only when this // import is an upgrade over an existing, worse-scoring file — see the @@ -1645,12 +1655,13 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res } let needed_bytes = std::fs::metadata(&working_path)?.len(); - if insufficient_space(Path::new(root_folder), needed_bytes)? { + if insufficient_space(&dest_dir, needed_bytes)? { if let Some(old_sibling) = &old_sibling { std::fs::rename(old_sibling, &dest).ok(); } anyhow::bail!( - "not enough free space at {root_folder} for {needed_bytes} bytes (source: {})", + "not enough free space at {} for {needed_bytes} bytes (source: {})", + dest_dir.display(), working_path.display() ); } @@ -1948,7 +1959,9 @@ fn import_season_pack_file( episode_title.as_deref(), &ext, ); - let dest = Path::new(root_folder).join(&filename); + let dest_dir = season_dir(root_folder, season_number); + std::fs::create_dir_all(&dest_dir)?; + let dest = dest_dir.join(&filename); // Same reasoning as import_one's own dest-collision check: a second // release for the same episode (here, from a *different* pack or a @@ -1976,12 +1989,13 @@ fn import_season_pack_file( } let needed_bytes = std::fs::metadata(source_path)?.len(); - if insufficient_space(Path::new(root_folder), needed_bytes)? { + if insufficient_space(&dest_dir, needed_bytes)? { if let Some(old_sibling) = &old_sibling { std::fs::rename(old_sibling, &dest).ok(); } anyhow::bail!( - "not enough free space at {root_folder} for {needed_bytes} bytes (source: {})", + "not enough free space at {} for {needed_bytes} bytes (source: {})", + dest_dir.display(), source_path.display() ); } @@ -3150,6 +3164,78 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + #[test] + fn import_one_places_a_single_episode_grab_under_its_season_subfolder() { + // Regression: a real single-episode grab (Mushoku Tensei S03E02/E03, + // not part of a season pack) landed flat in the show's root_folder + // instead of alongside the rest of the season's already-organized + // files in `Season 03/`, once the pre-existing library-scanned + // structure had nothing left to mask it. + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder) + VALUES (1, 'series', 'Some Show', 2021, 1, 1, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file) + VALUES (1, 1, 3, 2, 1, 0)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, status, torrent_hash, grabbed_at) + VALUES (1, 1, 1, 'Some Show S03E02 1080p', 1, 'guid-1', 'grabbed', 'deadbeef', datetime('now'))", + [], + ) + .unwrap(); + + let dir = std::env::temp_dir().join(format!( + "breadarr-single-episode-season-dir-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let dest_root = dir.join("library"); + std::fs::create_dir_all(&dest_root).unwrap(); + let content = dir.join("Some.Show.S03E02.1080p.mp4"); + std::fs::write(&content, b"fake episode data").unwrap(); + + let grab = PendingGrab::Episode { + release_id: 1, + episode_id: 1, + media_item_id: 1, + torrent_hash: "deadbeef".to_string(), + series_title: "Some Show".to_string(), + season_number: 3, + episode_number: 2, + episode_title: None, + root_folder: dest_root.to_string_lossy().to_string(), + }; + + let outcome = import_one(&conn, &grab, &content).unwrap(); + assert!(matches!(outcome, ImportOutcome::Imported { .. })); + + let dest = dest_root.join("Season 03").join("Some Show - S03E02.mp4"); + assert!( + dest.exists(), + "expected {} to exist under the season subfolder", + dest.display() + ); + assert!( + !dest_root.join("Some Show - S03E02.mp4").exists(), + "must not also land flat in the show's root folder" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn import_one_flags_quality_when_the_real_file_is_under_1080p() { // Mirrors a real production case: a release whose title carries no @@ -3604,9 +3690,18 @@ mod tests { .unwrap(); assert_eq!(status, "imported"); - assert!(dest_root.join("Some Show - S01E01.mkv").exists()); - assert!(dest_root.join("Some Show - S01E02.mkv").exists()); - assert!(dest_root.join("Some Show - S01E03.mkv").exists()); + assert!(dest_root + .join("Season 01") + .join("Some Show - S01E01.mkv") + .exists()); + assert!(dest_root + .join("Season 01") + .join("Some Show - S01E02.mkv") + .exists()); + assert!(dest_root + .join("Season 01") + .join("Some Show - S01E03.mkv") + .exists()); std::fs::remove_dir_all(&dir).unwrap(); } @@ -3623,7 +3718,8 @@ mod tests { std::fs::write(pack_dir.join("Show.S01E01.mkv"), b"new e01").unwrap(); std::fs::write(pack_dir.join("Show.S01E02.mkv"), b"new e02").unwrap(); let dest_root = dir.join("library"); - std::fs::create_dir_all(&dest_root).unwrap(); + let season_dir = dest_root.join("Season 01"); + std::fs::create_dir_all(&season_dir).unwrap(); // Episode 1 already has a higher-scoring imported release. conn.execute( @@ -3632,7 +3728,7 @@ mod tests { [], ) .unwrap(); - let existing = dest_root.join("Some Show - S01E01.mkv"); + let existing = season_dir.join("Some Show - S01E01.mkv"); std::fs::write(&existing, b"already-better e01").unwrap(); conn.execute( "INSERT INTO episode_file (episode_id, media_item_id, path, size_bytes, subtitle_status) @@ -3659,7 +3755,7 @@ mod tests { b"already-better e01", "episode 1's better file must survive untouched" ); - assert!(dest_root.join("Some Show - S01E02.mkv").exists()); + assert!(season_dir.join("Some Show - S01E02.mkv").exists()); std::fs::remove_dir_all(&dir).unwrap(); } diff --git a/breadarrd/src/matcher/mod.rs b/breadarrd/src/matcher/mod.rs index f503d05..fc82c08 100644 --- a/breadarrd/src/matcher/mod.rs +++ b/breadarrd/src/matcher/mod.rs @@ -188,15 +188,28 @@ impl TitleMatcher { } } +/// Grammatical filler words that show up across countless unrelated titles +/// and carry no discriminating signal on their own — English articles and +/// the short romaji particles ubiquitous in Japanese titles ("X no Y", "X ga +/// Y", ...). Without this, two completely unrelated shows sharing nothing +/// but e.g. "no" register as real overlap and slip past `MIN_TOKEN_OVERLAP` +/// (verified live: "Yomi no Tsugai" / "Kami no Shizuku" / "Saijo no Osewa" +/// all cleared the guard against "Kimetsu no Yaiba" on "no" alone). +const STOPWORDS: &[&str] = &[ + "a", "an", "the", "of", "to", "and", "no", "wa", "wo", "ga", "ni", "de", "ha", "he", "da", +]; + /// Word-level overlap between `a` and `b`, as a fraction of the *shorter* /// title's word count — so a short alias fully contained in a longer title /// (or vice versa) still scores 1.0, rather than being penalized by length -/// mismatch. Case-insensitive, splits on non-alphanumeric runs. +/// mismatch. Case-insensitive, splits on non-alphanumeric runs, and ignores +/// `STOPWORDS` so shared filler words don't count as real overlap. fn token_overlap_ratio(a: &str, b: &str) -> f32 { let tokenize = |s: &str| -> std::collections::HashSet { s.split(|c: char| !c.is_alphanumeric()) .filter(|t| !t.is_empty()) .map(|t| t.to_lowercase()) + .filter(|t| !STOPWORDS.contains(&t.as_str())) .collect() }; let ta = tokenize(a); @@ -273,4 +286,24 @@ mod tests { fn empty_input_has_zero_overlap() { assert_eq!(token_overlap_ratio("", "Mushoku Tensei"), 0.0); } + + #[test] + fn shared_particle_alone_is_not_real_overlap() { + // The bug found live in the review queue: unrelated titles sharing + // only the Japanese particle "no" scored a nonzero ratio and slipped + // past MIN_TOKEN_OVERLAP, flooding review with false matches against + // "Demon Slayer" (Kimetsu no Yaiba). + assert_eq!( + token_overlap_ratio("Yomi no Tsugai", "Kimetsu no Yaiba"), + 0.0 + ); + assert_eq!( + token_overlap_ratio("Kami no Shizuku", "Kimetsu no Yaiba"), + 0.0 + ); + assert_eq!( + token_overlap_ratio("Saijo no Osewa", "Kimetsu no Yaiba"), + 0.0 + ); + } }