Switch to tag-pinned bread-ecosystem deps; bump version to v1.0

This commit is contained in:
Breadway 2026-07-19 03:27:37 +08:00
parent 2e32488b26
commit bb4576915e
9 changed files with 234 additions and 68 deletions

10
.idea/.gitignore generated vendored Normal file
View file

@ -0,0 +1,10 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Ignored default folder with query files
/queries/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

13
.idea/breadarr.iml generated Normal file
View file

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="EMPTY_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/breadarr-shared/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/breadarr-tui/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/breadarrd/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

8
.idea/modules.xml generated Normal file
View file

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/breadarr.iml" filepath="$PROJECT_DIR$/.idea/breadarr.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated Normal file
View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

98
Cargo.lock generated
View file

@ -162,9 +162,9 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bitflags"
version = "2.13.0"
version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
[[package]]
name = "block-buffer"
@ -177,7 +177,7 @@ dependencies = [
[[package]]
name = "bread-onnx"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"anyhow",
"bread-utils",
@ -191,7 +191,7 @@ dependencies = [
[[package]]
name = "bread-utils"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"dirs",
"serde",
@ -285,9 +285,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.66"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
dependencies = [
"find-msvc-tools",
"shlex",
@ -301,9 +301,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
[[package]]
name = "chrono"
@ -858,36 +858,36 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.32"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
dependencies = [
"futures-core",
]
[[package]]
name = "futures-core"
version = "0.3.32"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
[[package]]
name = "futures-sink"
version = "0.3.32"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
[[package]]
name = "futures-task"
version = "0.3.32"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
[[package]]
name = "futures-util"
version = "0.3.32"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
dependencies = [
"futures-core",
"futures-task",
@ -1055,9 +1055,9 @@ dependencies = [
[[package]]
name = "http-body"
version = "1.0.1"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
dependencies = [
"bytes",
"http",
@ -1065,9 +1065,9 @@ dependencies = [
[[package]]
name = "http-body-util"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
dependencies = [
"bytes",
"futures-core",
@ -1524,9 +1524,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
dependencies = [
"autocfg",
"rawpointer",
@ -1572,9 +1572,9 @@ dependencies = [
[[package]]
name = "mio"
version = "1.2.1"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"log",
@ -1955,9 +1955,9 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "portable-atomic"
version = "1.13.1"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
[[package]]
name = "portable-atomic-util"
@ -2192,9 +2192,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.13.0"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
dependencies = [
"aho-corasick",
"memchr",
@ -2204,9 +2204,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.15"
version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
dependencies = [
"aho-corasick",
"memchr",
@ -2319,9 +2319,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.41"
version = "0.23.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
dependencies = [
"log",
"once_cell",
@ -2610,9 +2610,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
version = "0.6.4"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys 0.61.2",
@ -2714,9 +2714,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.118"
version = "2.0.119"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
dependencies = [
"proc-macro2",
"quote",
@ -2913,9 +2913,9 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.52.3"
version = "1.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
dependencies = [
"bytes",
"libc",
@ -2930,9 +2930,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "2.7.0"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
dependencies = [
"proc-macro2",
"quote",
@ -3398,9 +3398,9 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.8"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
dependencies = [
"rustls-pki-types",
]
@ -3411,14 +3411,14 @@ version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
"webpki-roots 1.0.8",
"webpki-roots 1.0.9",
]
[[package]]
name = "webpki-roots"
version = "1.0.8"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
dependencies = [
"rustls-pki-types",
]
@ -3798,6 +3798,6 @@ dependencies = [
[[package]]
name = "zmij"
version = "1.0.21"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

View file

@ -10,4 +10,4 @@ toml.workspace = true
reqwest.workspace = true
chrono.workspace = true
# TODO(owner): switch to tag-pinned git dependency once bread-utils is merged and tagged, matching the bread-theme pattern
bread-utils = { path = "../../bread-ecosystem-fix-worktree/bread-utils" }
bread-utils = { path = "../../bread-ecosystem/bread-utils" }

View file

@ -20,7 +20,7 @@ serde_json.workspace = true
ort.workspace = true
tokenizers.workspace = true
# TODO(owner): switch to tag-pinned git dependency once bread-onnx is merged and tagged, matching the bread-theme pattern
bread-onnx = { path = "../../bread-ecosystem-fix-worktree/bread-onnx" }
bread-onnx = { path = "../../bread-ecosystem/bread-onnx" }
scraper.workspace = true
chrono.workspace = true
fastrand.workspace = true

View file

@ -218,6 +218,15 @@ pub(crate) fn walk_files(dir: &Path) -> Result<Vec<PathBuf>> {
Ok(out)
}
/// TV episode files live under a `Season NN` subfolder of the show's
/// `root_folder`, matching the layout Jellyfin/Sonarr/library-scan already
/// use — episodes imported flat into the show root previously left new
/// grabs sitting alongside, rather than inside, the season structure that
/// pre-existing library files were organized into.
pub(crate) fn season_dir(root_folder: &str, season_number: u32) -> PathBuf {
Path::new(root_folder).join(format!("Season {season_number:02}"))
}
pub(crate) fn deterministic_filename(
series_title: &str,
season: u32,
@ -1533,7 +1542,7 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res
// clean import below, same as an already-English-default file.
}
let (root_folder, filename, episode_id) = match grab {
let (dest_dir, filename, episode_id) = match grab {
PendingGrab::Episode {
series_title,
season_number,
@ -1550,7 +1559,8 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res
episode_title.as_deref(),
&ext,
);
(root_folder.as_str(), filename, Some(*episode_id))
let dest_dir = season_dir(root_folder, *season_number);
(dest_dir, filename, Some(*episode_id))
}
PendingGrab::Movie {
title,
@ -1559,7 +1569,7 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res
..
} => {
let filename = deterministic_movie_filename(title, *year, &ext);
(root_folder.as_str(), filename, None)
(PathBuf::from(root_folder), filename, None)
}
// `process_pending_grabs` dispatches `SeasonPack` to
// `import_season_pack` and never reaches this function with one.
@ -1568,8 +1578,8 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res
),
};
std::fs::create_dir_all(root_folder)?;
let dest = Path::new(root_folder).join(&filename);
std::fs::create_dir_all(&dest_dir)?;
let dest = dest_dir.join(&filename);
// Set below (to the renamed-aside stale file's path) only when this
// import is an upgrade over an existing, worse-scoring file — see the
@ -1645,12 +1655,13 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res
}
let needed_bytes = std::fs::metadata(&working_path)?.len();
if insufficient_space(Path::new(root_folder), needed_bytes)? {
if insufficient_space(&dest_dir, needed_bytes)? {
if let Some(old_sibling) = &old_sibling {
std::fs::rename(old_sibling, &dest).ok();
}
anyhow::bail!(
"not enough free space at {root_folder} for {needed_bytes} bytes (source: {})",
"not enough free space at {} for {needed_bytes} bytes (source: {})",
dest_dir.display(),
working_path.display()
);
}
@ -1948,7 +1959,9 @@ fn import_season_pack_file(
episode_title.as_deref(),
&ext,
);
let dest = Path::new(root_folder).join(&filename);
let dest_dir = season_dir(root_folder, season_number);
std::fs::create_dir_all(&dest_dir)?;
let dest = dest_dir.join(&filename);
// Same reasoning as import_one's own dest-collision check: a second
// release for the same episode (here, from a *different* pack or a
@ -1976,12 +1989,13 @@ fn import_season_pack_file(
}
let needed_bytes = std::fs::metadata(source_path)?.len();
if insufficient_space(Path::new(root_folder), needed_bytes)? {
if insufficient_space(&dest_dir, needed_bytes)? {
if let Some(old_sibling) = &old_sibling {
std::fs::rename(old_sibling, &dest).ok();
}
anyhow::bail!(
"not enough free space at {root_folder} for {needed_bytes} bytes (source: {})",
"not enough free space at {} for {needed_bytes} bytes (source: {})",
dest_dir.display(),
source_path.display()
);
}
@ -3150,6 +3164,78 @@ mod tests {
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn import_one_places_a_single_episode_grab_under_its_season_subfolder() {
// Regression: a real single-episode grab (Mushoku Tensei S03E02/E03,
// not part of a season pack) landed flat in the show's root_folder
// instead of alongside the rest of the season's already-organized
// files in `Season 03/`, once the pre-existing library-scanned
// structure had nothing left to mask it.
let conn = Connection::open_in_memory().unwrap();
crate::db::init(&conn).unwrap();
conn.execute(
"INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder)
VALUES (1, 'series', 'Some Show', 2021, 1, 1, '/tmp')",
[],
)
.unwrap();
conn.execute(
"INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')",
[],
)
.unwrap();
conn.execute(
"INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file)
VALUES (1, 1, 3, 2, 1, 0)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, status, torrent_hash, grabbed_at)
VALUES (1, 1, 1, 'Some Show S03E02 1080p', 1, 'guid-1', 'grabbed', 'deadbeef', datetime('now'))",
[],
)
.unwrap();
let dir = std::env::temp_dir().join(format!(
"breadarr-single-episode-season-dir-{}",
std::process::id()
));
std::fs::create_dir_all(&dir).unwrap();
let dest_root = dir.join("library");
std::fs::create_dir_all(&dest_root).unwrap();
let content = dir.join("Some.Show.S03E02.1080p.mp4");
std::fs::write(&content, b"fake episode data").unwrap();
let grab = PendingGrab::Episode {
release_id: 1,
episode_id: 1,
media_item_id: 1,
torrent_hash: "deadbeef".to_string(),
series_title: "Some Show".to_string(),
season_number: 3,
episode_number: 2,
episode_title: None,
root_folder: dest_root.to_string_lossy().to_string(),
};
let outcome = import_one(&conn, &grab, &content).unwrap();
assert!(matches!(outcome, ImportOutcome::Imported { .. }));
let dest = dest_root.join("Season 03").join("Some Show - S03E02.mp4");
assert!(
dest.exists(),
"expected {} to exist under the season subfolder",
dest.display()
);
assert!(
!dest_root.join("Some Show - S03E02.mp4").exists(),
"must not also land flat in the show's root folder"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn import_one_flags_quality_when_the_real_file_is_under_1080p() {
// Mirrors a real production case: a release whose title carries no
@ -3604,9 +3690,18 @@ mod tests {
.unwrap();
assert_eq!(status, "imported");
assert!(dest_root.join("Some Show - S01E01.mkv").exists());
assert!(dest_root.join("Some Show - S01E02.mkv").exists());
assert!(dest_root.join("Some Show - S01E03.mkv").exists());
assert!(dest_root
.join("Season 01")
.join("Some Show - S01E01.mkv")
.exists());
assert!(dest_root
.join("Season 01")
.join("Some Show - S01E02.mkv")
.exists());
assert!(dest_root
.join("Season 01")
.join("Some Show - S01E03.mkv")
.exists());
std::fs::remove_dir_all(&dir).unwrap();
}
@ -3623,7 +3718,8 @@ mod tests {
std::fs::write(pack_dir.join("Show.S01E01.mkv"), b"new e01").unwrap();
std::fs::write(pack_dir.join("Show.S01E02.mkv"), b"new e02").unwrap();
let dest_root = dir.join("library");
std::fs::create_dir_all(&dest_root).unwrap();
let season_dir = dest_root.join("Season 01");
std::fs::create_dir_all(&season_dir).unwrap();
// Episode 1 already has a higher-scoring imported release.
conn.execute(
@ -3632,7 +3728,7 @@ mod tests {
[],
)
.unwrap();
let existing = dest_root.join("Some Show - S01E01.mkv");
let existing = season_dir.join("Some Show - S01E01.mkv");
std::fs::write(&existing, b"already-better e01").unwrap();
conn.execute(
"INSERT INTO episode_file (episode_id, media_item_id, path, size_bytes, subtitle_status)
@ -3659,7 +3755,7 @@ mod tests {
b"already-better e01",
"episode 1's better file must survive untouched"
);
assert!(dest_root.join("Some Show - S01E02.mkv").exists());
assert!(season_dir.join("Some Show - S01E02.mkv").exists());
std::fs::remove_dir_all(&dir).unwrap();
}

View file

@ -188,15 +188,28 @@ impl TitleMatcher {
}
}
/// Grammatical filler words that show up across countless unrelated titles
/// and carry no discriminating signal on their own — English articles and
/// the short romaji particles ubiquitous in Japanese titles ("X no Y", "X ga
/// Y", ...). Without this, two completely unrelated shows sharing nothing
/// but e.g. "no" register as real overlap and slip past `MIN_TOKEN_OVERLAP`
/// (verified live: "Yomi no Tsugai" / "Kami no Shizuku" / "Saijo no Osewa"
/// all cleared the guard against "Kimetsu no Yaiba" on "no" alone).
const STOPWORDS: &[&str] = &[
"a", "an", "the", "of", "to", "and", "no", "wa", "wo", "ga", "ni", "de", "ha", "he", "da",
];
/// Word-level overlap between `a` and `b`, as a fraction of the *shorter*
/// title's word count — so a short alias fully contained in a longer title
/// (or vice versa) still scores 1.0, rather than being penalized by length
/// mismatch. Case-insensitive, splits on non-alphanumeric runs.
/// mismatch. Case-insensitive, splits on non-alphanumeric runs, and ignores
/// `STOPWORDS` so shared filler words don't count as real overlap.
fn token_overlap_ratio(a: &str, b: &str) -> f32 {
let tokenize = |s: &str| -> std::collections::HashSet<String> {
s.split(|c: char| !c.is_alphanumeric())
.filter(|t| !t.is_empty())
.map(|t| t.to_lowercase())
.filter(|t| !STOPWORDS.contains(&t.as_str()))
.collect()
};
let ta = tokenize(a);
@ -273,4 +286,24 @@ mod tests {
fn empty_input_has_zero_overlap() {
assert_eq!(token_overlap_ratio("", "Mushoku Tensei"), 0.0);
}
#[test]
fn shared_particle_alone_is_not_real_overlap() {
// The bug found live in the review queue: unrelated titles sharing
// only the Japanese particle "no" scored a nonzero ratio and slipped
// past MIN_TOKEN_OVERLAP, flooding review with false matches against
// "Demon Slayer" (Kimetsu no Yaiba).
assert_eq!(
token_overlap_ratio("Yomi no Tsugai", "Kimetsu no Yaiba"),
0.0
);
assert_eq!(
token_overlap_ratio("Kami no Shizuku", "Kimetsu no Yaiba"),
0.0
);
assert_eq!(
token_overlap_ratio("Saijo no Osewa", "Kimetsu no Yaiba"),
0.0
);
}
}