can't be bothered writing a commit message
This commit is contained in:
commit
697b009627
55 changed files with 21320 additions and 0 deletions
25
breadarrd/Cargo.toml
Normal file
25
breadarrd/Cargo.toml
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[package]
|
||||
name = "breadarrd"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
breadarr-shared.workspace = true
|
||||
tokio.workspace = true
|
||||
anyhow.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
axum.workspace = true
|
||||
serde.workspace = true
|
||||
reqwest.workspace = true
|
||||
rusqlite.workspace = true
|
||||
quick-xml.workspace = true
|
||||
async-trait.workspace = true
|
||||
regex.workspace = true
|
||||
serde_json.workspace = true
|
||||
ort.workspace = true
|
||||
tokenizers.workspace = true
|
||||
scraper.workspace = true
|
||||
chrono.workspace = true
|
||||
fastrand.workspace = true
|
||||
nix.workspace = true
|
||||
180
breadarrd/src/api/mod.rs
Normal file
180
breadarrd/src/api/mod.rs
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
pub mod routes;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::{Request, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::middleware::{self, Next};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::Router;
|
||||
use rusqlite::Connection;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::metadata::tmdb::TmdbClient;
|
||||
use crate::metadata::tvdb::TvdbClient;
|
||||
use crate::qbit::QbitClient;
|
||||
use crate::scheduler::SearchCycleStats;
|
||||
|
||||
/// A `tokio::sync::Mutex`, not `std::sync::Mutex` — several handlers (e.g.
|
||||
/// review-queue approval) interleave synchronous DB calls with `.await`ed
|
||||
/// qBittorrent/TVDB calls, and a sync `MutexGuard` can't be held across an
|
||||
/// await point. This one can.
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub conn: Arc<Mutex<Connection>>,
|
||||
pub tvdb: Option<Arc<TvdbClient>>,
|
||||
pub tmdb: Option<Arc<TmdbClient>>,
|
||||
pub qbit: Option<Arc<QbitClient>>,
|
||||
pub qbit_category: String,
|
||||
pub cycle_status: Arc<std::sync::Mutex<CycleStatus>>,
|
||||
pub config: breadarr_shared::Config,
|
||||
/// `None` when the background loop isn't running (qbit not
|
||||
/// configured) — routes that need the search pipeline (manual
|
||||
/// "search now", the candidate picker) dispatch through this rather
|
||||
/// than running it inline, since that pipeline holds a `Connection`/
|
||||
/// `&dyn ReleaseSource` across `.await` points and is therefore
|
||||
/// `!Send`, which axum's `Handler` trait doesn't allow. Routing
|
||||
/// through the loop's own already-running instance also means these
|
||||
/// reuse its already-loaded title-matcher/sources for free.
|
||||
pub background_tx: Option<tokio::sync::mpsc::Sender<BackgroundRequest>>,
|
||||
}
|
||||
|
||||
/// Everything an API handler needs the background loop to do on its
|
||||
/// behalf, because the work involves `!Send` state (see `AppState::
|
||||
/// background_tx`'s doc comment) that can't be run directly inside an
|
||||
/// axum handler.
|
||||
pub enum BackgroundRequest {
|
||||
SearchNow {
|
||||
media_item_id: i64,
|
||||
reply: tokio::sync::oneshot::Sender<anyhow::Result<SearchCycleStats>>,
|
||||
},
|
||||
FetchCandidates {
|
||||
media_item_id: i64,
|
||||
episode_id: Option<i64>,
|
||||
reply: tokio::sync::oneshot::Sender<
|
||||
anyhow::Result<Vec<breadarr_shared::dto::ReleaseCandidate>>,
|
||||
>,
|
||||
},
|
||||
GrabCandidate {
|
||||
media_item_id: i64,
|
||||
episode_id: Option<i64>,
|
||||
source_id: i64,
|
||||
raw_title: String,
|
||||
link: String,
|
||||
guid: String,
|
||||
reply: tokio::sync::oneshot::Sender<anyhow::Result<()>>,
|
||||
},
|
||||
}
|
||||
|
||||
/// One record per completed grab/import cycle attempt — plain `std::sync::
|
||||
/// Mutex` is fine here (unlike `conn`) since updates are a single field
|
||||
/// write with no `.await` in between. Exists so a silently-stalled
|
||||
/// background loop (e.g. every cycle erroring for hours) is visible from a
|
||||
/// single `/health` call instead of only in the journal.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CycleStatus {
|
||||
pub last_grab: Option<CycleRecord>,
|
||||
pub last_import: Option<CycleRecord>,
|
||||
pub last_search: Option<CycleRecord>,
|
||||
pub last_upgrade: Option<CycleRecord>,
|
||||
/// Set once the search-driven loop's consecutive-failure backoff hits
|
||||
/// its ceiling — still ticking at max backoff underneath (self-healing
|
||||
/// if the source recovers), but worth a loud, easy-to-spot signal that
|
||||
/// something's wrong with 1337x/nyaa-search specifically.
|
||||
pub search_halted: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CycleRecord {
|
||||
pub at: chrono::DateTime<chrono::Utc>,
|
||||
pub ok: bool,
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
/// Rejects any request lacking `Authorization: Bearer <config.daemon.api_token>`
|
||||
/// once a token is actually configured — a no-op (every request passes)
|
||||
/// when it's empty, so an unconfigured install behaves exactly as before.
|
||||
/// `/health` is deliberately exempt even with a token configured: it's
|
||||
/// commonly polled by external monitoring (e.g. an uptime dashboard) that
|
||||
/// has no reason to hold the same credential as the TUI/API client, and it
|
||||
/// exposes nothing more sensitive than "is the process alive."
|
||||
async fn require_api_token(State(state): State<AppState>, req: Request, next: Next) -> Response {
|
||||
if state.config.daemon.api_token.is_empty() || req.uri().path() == "/health" {
|
||||
return next.run(req).await;
|
||||
}
|
||||
let authorized = req
|
||||
.headers()
|
||||
.get(axum::http::header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.is_some_and(|token| token == state.config.daemon.api_token);
|
||||
if !authorized {
|
||||
return (StatusCode::UNAUTHORIZED, "missing or invalid API token").into_response();
|
||||
}
|
||||
next.run(req).await
|
||||
}
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(routes::health::health))
|
||||
.route("/media", get(routes::media::list).post(routes::media::add))
|
||||
.route(
|
||||
"/media/:id",
|
||||
get(routes::media::detail).delete(routes::media::delete),
|
||||
)
|
||||
.route("/media/movie", post(routes::media::add_movie))
|
||||
.route("/media/:id/search", post(routes::media::search_now))
|
||||
.route(
|
||||
"/media/:id/candidates",
|
||||
get(routes::media::list_candidates).post(routes::media::grab_candidate),
|
||||
)
|
||||
.route(
|
||||
"/episode/:id/candidates",
|
||||
get(routes::media::list_episode_candidates).post(routes::media::grab_episode_candidate),
|
||||
)
|
||||
.route("/media/:id/monitor", post(routes::media::monitor))
|
||||
.route("/media/:id/unmonitor", post(routes::media::unmonitor))
|
||||
.route(
|
||||
"/media/:id/season/:season_number/monitor",
|
||||
post(routes::media::monitor_season),
|
||||
)
|
||||
.route(
|
||||
"/media/:id/season/:season_number/unmonitor",
|
||||
post(routes::media::unmonitor_season),
|
||||
)
|
||||
.route("/episode/:id/monitor", post(routes::media::monitor_episode))
|
||||
.route(
|
||||
"/episode/:id/unmonitor",
|
||||
post(routes::media::unmonitor_episode),
|
||||
)
|
||||
.route(
|
||||
"/episode/:id/file",
|
||||
axum::routing::delete(routes::media::delete_episode_file),
|
||||
)
|
||||
.route(
|
||||
"/media/:id/file",
|
||||
axum::routing::delete(routes::media::delete_movie_file),
|
||||
)
|
||||
.route("/releases", get(routes::releases::list))
|
||||
.route("/review", get(routes::review::list))
|
||||
.route("/review/:id/approve", post(routes::review::approve))
|
||||
.route("/review/:id/reject", post(routes::review::reject))
|
||||
.route("/search", get(routes::search::search))
|
||||
.route("/stuck", get(routes::stuck::stuck))
|
||||
.route("/calendar", get(routes::calendar::calendar))
|
||||
.route(
|
||||
"/library/health",
|
||||
get(routes::library_health::library_health),
|
||||
)
|
||||
.route("/quality-profiles", get(routes::quality_profiles::list))
|
||||
.route(
|
||||
"/quality-profiles/:id/weights",
|
||||
axum::routing::put(routes::quality_profiles::update_weights),
|
||||
)
|
||||
.layer(middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
require_api_token,
|
||||
))
|
||||
.with_state(state)
|
||||
}
|
||||
52
breadarrd/src/api/routes/calendar.rs
Normal file
52
breadarrd/src/api/routes/calendar.rs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::CalendarEntry;
|
||||
|
||||
use crate::api::AppState;
|
||||
|
||||
/// How far back/forward "upcoming" spans — matches the common Sonarr
|
||||
/// calendar default window (a week either side) rather than an arbitrary
|
||||
/// wider range, since the point is "what's relevant right now."
|
||||
const DAYS_PAST: i64 = 7;
|
||||
const DAYS_FUTURE: i64 = 21;
|
||||
|
||||
pub async fn calendar(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<CalendarEntry>>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT e.media_item_id, m.title, e.season_number, e.episode_number, e.title,
|
||||
e.air_date, e.monitored, e.has_file
|
||||
FROM episode e JOIN media_item m ON m.id = e.media_item_id
|
||||
WHERE e.air_date IS NOT NULL
|
||||
AND date(e.air_date) BETWEEN date('now', ?1) AND date('now', ?2)
|
||||
ORDER BY e.air_date ASC",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let rows = stmt
|
||||
.query_map(
|
||||
[format!("-{DAYS_PAST} days"), format!("+{DAYS_FUTURE} days")],
|
||||
|row| {
|
||||
Ok(CalendarEntry {
|
||||
media_item_id: row.get(0)?,
|
||||
media_title: row.get(1)?,
|
||||
season_number: row.get(2)?,
|
||||
episode_number: row.get(3)?,
|
||||
title: row.get(4)?,
|
||||
air_date: row.get(5)?,
|
||||
monitored: row.get::<_, i64>(6)? != 0,
|
||||
has_file: row.get::<_, i64>(7)? != 0,
|
||||
})
|
||||
},
|
||||
)
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
25
breadarrd/src/api/routes/health.rs
Normal file
25
breadarrd/src/api/routes/health.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
use axum::extract::State;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::{CycleInfo, HealthDetail};
|
||||
|
||||
use crate::api::AppState;
|
||||
|
||||
fn to_info(r: &crate::api::CycleRecord) -> CycleInfo {
|
||||
CycleInfo {
|
||||
at: r.at,
|
||||
ok: r.ok,
|
||||
detail: r.detail.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn health(State(state): State<AppState>) -> Json<HealthDetail> {
|
||||
let status = state.cycle_status.lock().expect("cycle_status poisoned");
|
||||
Json(HealthDetail {
|
||||
status: "ok".to_string(),
|
||||
last_grab_cycle: status.last_grab.as_ref().map(to_info),
|
||||
last_import_cycle: status.last_import.as_ref().map(to_info),
|
||||
last_search_cycle: status.last_search.as_ref().map(to_info),
|
||||
last_upgrade_cycle: status.last_upgrade.as_ref().map(to_info),
|
||||
search_halted: status.search_halted,
|
||||
})
|
||||
}
|
||||
321
breadarrd/src/api/routes/library_health.rs
Normal file
321
breadarrd/src/api/routes/library_health.rs
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::{
|
||||
CodecCount, DuplicateGroup, FlaggedFile, LibraryHealthReport, LibrarySummary,
|
||||
};
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::api::AppState;
|
||||
|
||||
/// Every flag category shares the same shape (id/title/episode-label/path),
|
||||
/// resolved through the same TV-or-movie join every time — `episode_id` is
|
||||
/// set for a TV file (join via `episode`→`media_item`) and NULL for a movie
|
||||
/// file (join `media_item` directly via `episode_file.media_item_id`), the
|
||||
/// same nullable-episode_id convention used throughout this schema.
|
||||
const FLAGGED_FILE_SELECT: &str = "
|
||||
SELECT ef.id,
|
||||
COALESCE(mi_direct.title, mi_ep.title) AS media_title,
|
||||
CASE WHEN ef.episode_id IS NOT NULL
|
||||
THEN 'S' || printf('%02d', e.season_number) || 'E' || printf('%02d', e.episode_number)
|
||||
ELSE NULL END AS episode_label,
|
||||
ef.path
|
||||
FROM episode_file ef
|
||||
JOIN media_file_probe p ON p.episode_file_id = ef.id
|
||||
LEFT JOIN media_item mi_direct ON mi_direct.id = ef.media_item_id
|
||||
LEFT JOIN episode e ON e.id = ef.episode_id
|
||||
LEFT JOIN media_item mi_ep ON mi_ep.id = e.media_item_id
|
||||
WHERE ";
|
||||
|
||||
fn fetch_flagged(conn: &Connection, where_clause: &str) -> rusqlite::Result<Vec<FlaggedFile>> {
|
||||
let sql = format!("{FLAGGED_FILE_SELECT}{where_clause}");
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(FlaggedFile {
|
||||
episode_file_id: row.get(0)?,
|
||||
media_title: row.get(1)?,
|
||||
episode_label: row.get(2)?,
|
||||
path: row.get(3)?,
|
||||
})
|
||||
})?;
|
||||
rows.collect()
|
||||
}
|
||||
|
||||
fn fetch_duplicate_groups(conn: &Connection) -> rusqlite::Result<Vec<DuplicateGroup>> {
|
||||
let mut groups = Vec::new();
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT m.title,
|
||||
'S' || printf('%02d', e.season_number) || 'E' || printf('%02d', e.episode_number),
|
||||
group_concat(ef.path, '||')
|
||||
FROM episode_file ef
|
||||
JOIN episode e ON e.id = ef.episode_id
|
||||
JOIN media_item m ON m.id = e.media_item_id
|
||||
GROUP BY ef.episode_id
|
||||
HAVING count(*) > 1",
|
||||
)?;
|
||||
groups.extend(
|
||||
stmt.query_map([], |row| {
|
||||
let paths: String = row.get(2)?;
|
||||
Ok(DuplicateGroup {
|
||||
media_title: row.get(0)?,
|
||||
episode_label: row.get(1)?,
|
||||
paths: paths.split("||").map(str::to_string).collect(),
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?,
|
||||
);
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT m.title, group_concat(ef.path, '||')
|
||||
FROM episode_file ef
|
||||
JOIN media_item m ON m.id = ef.media_item_id
|
||||
WHERE ef.episode_id IS NULL
|
||||
GROUP BY ef.media_item_id
|
||||
HAVING count(*) > 1",
|
||||
)?;
|
||||
groups.extend(
|
||||
stmt.query_map([], |row| {
|
||||
let paths: String = row.get(1)?;
|
||||
Ok(DuplicateGroup {
|
||||
media_title: row.get(0)?,
|
||||
episode_label: None,
|
||||
paths: paths.split("||").map(str::to_string).collect(),
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?,
|
||||
);
|
||||
|
||||
Ok(groups)
|
||||
}
|
||||
|
||||
fn fetch_summary(conn: &Connection) -> rusqlite::Result<LibrarySummary> {
|
||||
let (total_files, total_size_bytes): (i64, i64) = conn.query_row(
|
||||
"SELECT count(*), COALESCE(sum(size_bytes), 0) FROM episode_file",
|
||||
[],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
let probed_files: i64 = conn.query_row("SELECT count(*) FROM media_file_probe", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT video_codec, count(*) FROM media_file_probe
|
||||
WHERE video_codec IS NOT NULL GROUP BY video_codec ORDER BY count(*) DESC",
|
||||
)?;
|
||||
let by_video_codec = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(CodecCount {
|
||||
codec: row.get(0)?,
|
||||
count: row.get(1)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
// Resolution buckets are computed against `height` directly rather than
|
||||
// reusing `flag_under_quality` (which is only a <1080p boolean) — this
|
||||
// view wants the full breakdown, not just the pass/fail split.
|
||||
let bucket = |where_clause: &str| -> rusqlite::Result<i64> {
|
||||
conn.query_row(
|
||||
&format!(
|
||||
"SELECT count(*) FROM media_file_probe WHERE height IS NOT NULL AND {where_clause}"
|
||||
),
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
};
|
||||
let sd_count = bucket("height < 720")?;
|
||||
let hd_720p_count = bucket("height >= 720 AND height < 1080")?;
|
||||
let full_hd_1080p_count = bucket("height >= 1080 AND height < 2160")?;
|
||||
let uhd_4k_count = bucket("height >= 2160")?;
|
||||
|
||||
let with_subs: i64 = conn.query_row(
|
||||
"SELECT count(*) FROM media_file_probe WHERE flag_no_subtitles = 0",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
let pct_with_subtitles = if probed_files > 0 {
|
||||
(with_subs as f64 / probed_files as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
Ok(LibrarySummary {
|
||||
total_files,
|
||||
total_size_bytes,
|
||||
probed_files,
|
||||
by_video_codec,
|
||||
sd_count,
|
||||
hd_720p_count,
|
||||
full_hd_1080p_count,
|
||||
uhd_4k_count,
|
||||
pct_with_subtitles,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn library_health(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<LibraryHealthReport>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
|
||||
let report = LibraryHealthReport {
|
||||
corrupt_files: fetch_flagged(
|
||||
&conn,
|
||||
"p.corruption_status IN ('probe_failed','decode_failed')",
|
||||
)
|
||||
.map_err(internal)?,
|
||||
under_quality_files: fetch_flagged(&conn, "p.flag_under_quality = 1").map_err(internal)?,
|
||||
no_subtitle_files: fetch_flagged(&conn, "p.flag_no_subtitles = 1").map_err(internal)?,
|
||||
no_english_audio_files: fetch_flagged(&conn, "p.flag_no_english_audio = 1")
|
||||
.map_err(internal)?,
|
||||
non_english_default_audio_files: fetch_flagged(
|
||||
&conn,
|
||||
"p.flag_non_english_default_audio = 1",
|
||||
)
|
||||
.map_err(internal)?,
|
||||
duplicate_groups: fetch_duplicate_groups(&conn).map_err(internal)?,
|
||||
summary: fetch_summary(&conn).map_err(internal)?,
|
||||
};
|
||||
|
||||
Ok(Json(report))
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A TV episode file (id=1, under-quality) and a movie file (id=2,
|
||||
/// no English audio) with `media_file_probe` rows already populated —
|
||||
/// exercises both halves of the nullable-episode_id join every query in
|
||||
/// this file relies on.
|
||||
fn seeded_conn() -> Connection {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
crate::db::init(&conn).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder)
|
||||
VALUES (1, 'series', 'Some Show', 2020, 1, 1, '/tmp')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file)
|
||||
VALUES (1, 1, 2, 5, 1, 1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status)
|
||||
VALUES (1, 1, NULL, '/tmp/show.mkv', 1000, 'none')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO media_file_probe (episode_file_id, probed_at, probe_size_bytes, probe_mtime, height, video_codec,
|
||||
corruption_status, flag_under_quality, flag_no_subtitles, flag_no_english_audio, flag_non_english_default_audio)
|
||||
VALUES (1, datetime('now'), 1000, 0, 480, 'h264', 'probe_ok', 1, 0, 0, 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder)
|
||||
VALUES (2, 'movie', 'Some Movie', 2019, 1, 2, '/tmp')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status)
|
||||
VALUES (2, NULL, 2, '/tmp/movie.mkv', 2000, 'none')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO media_file_probe (episode_file_id, probed_at, probe_size_bytes, probe_mtime, height, video_codec,
|
||||
corruption_status, flag_under_quality, flag_no_subtitles, flag_no_english_audio, flag_non_english_default_audio)
|
||||
VALUES (2, datetime('now'), 2000, 0, 1080, 'hevc', 'probe_ok', 0, 1, 1, 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_flagged_resolves_titles_for_both_tv_and_movie_files() {
|
||||
let conn = seeded_conn();
|
||||
|
||||
let under_quality = fetch_flagged(&conn, "p.flag_under_quality = 1").unwrap();
|
||||
assert_eq!(under_quality.len(), 1);
|
||||
assert_eq!(under_quality[0].media_title, "Some Show");
|
||||
assert_eq!(under_quality[0].episode_label.as_deref(), Some("S02E05"));
|
||||
|
||||
let no_english = fetch_flagged(&conn, "p.flag_no_english_audio = 1").unwrap();
|
||||
assert_eq!(no_english.len(), 1);
|
||||
assert_eq!(no_english[0].media_title, "Some Movie");
|
||||
assert_eq!(no_english[0].episode_label, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_duplicate_groups_finds_both_tv_and_movie_duplicates() {
|
||||
let conn = seeded_conn();
|
||||
// A second file for the same TV episode, and a second for the same movie.
|
||||
conn.execute(
|
||||
"INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status)
|
||||
VALUES (3, 1, NULL, '/tmp/show-dup.mkv', 1000, 'none')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status)
|
||||
VALUES (4, NULL, 2, '/tmp/movie-dup.mkv', 2000, 'none')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let groups = fetch_duplicate_groups(&conn).unwrap();
|
||||
assert_eq!(groups.len(), 2);
|
||||
let tv_group = groups
|
||||
.iter()
|
||||
.find(|g| g.media_title == "Some Show")
|
||||
.unwrap();
|
||||
assert_eq!(tv_group.paths.len(), 2);
|
||||
let movie_group = groups
|
||||
.iter()
|
||||
.find(|g| g.media_title == "Some Movie")
|
||||
.unwrap();
|
||||
assert_eq!(movie_group.paths.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_duplicate_groups_ignores_files_without_duplicates() {
|
||||
let conn = seeded_conn();
|
||||
assert!(fetch_duplicate_groups(&conn).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_summary_buckets_resolutions_and_computes_subtitle_percentage() {
|
||||
let conn = seeded_conn();
|
||||
let summary = fetch_summary(&conn).unwrap();
|
||||
|
||||
assert_eq!(summary.total_files, 2);
|
||||
assert_eq!(summary.total_size_bytes, 3000);
|
||||
assert_eq!(summary.probed_files, 2);
|
||||
assert_eq!(summary.sd_count, 1); // the 480p show episode
|
||||
assert_eq!(summary.full_hd_1080p_count, 1); // the 1080p movie
|
||||
// One of the two probed files (the movie) has flag_no_subtitles=1.
|
||||
assert_eq!(summary.pct_with_subtitles, 50.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_summary_handles_an_empty_library_without_dividing_by_zero() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
crate::db::init(&conn).unwrap();
|
||||
let summary = fetch_summary(&conn).unwrap();
|
||||
assert_eq!(summary.total_files, 0);
|
||||
assert_eq!(summary.pct_with_subtitles, 0.0);
|
||||
}
|
||||
}
|
||||
507
breadarrd/src/api/routes/media.rs
Normal file
507
breadarrd/src/api/routes/media.rs
Normal file
|
|
@ -0,0 +1,507 @@
|
|||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::{
|
||||
AddMovieRequest, AddMovieResponse, AddSeriesRequest, AddSeriesResponse, EpisodeSummary,
|
||||
MediaItemDetail, MediaItemSummary, SearchNowResult,
|
||||
};
|
||||
use rusqlite::{params, OptionalExtension};
|
||||
|
||||
use crate::api::AppState;
|
||||
use crate::metadata;
|
||||
|
||||
pub async fn list(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<MediaItemSummary>>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT m.id, m.kind, m.title, m.year, m.monitored,
|
||||
(SELECT count(*) FROM episode e WHERE e.media_item_id = m.id) AS episode_count,
|
||||
(SELECT count(*) FROM episode e WHERE e.media_item_id = m.id AND e.monitored = 1 AND e.has_file = 0) AS missing_count
|
||||
FROM media_item m ORDER BY m.title",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(MediaItemSummary {
|
||||
id: row.get(0)?,
|
||||
kind: row.get(1)?,
|
||||
title: row.get(2)?,
|
||||
year: row.get(3)?,
|
||||
monitored: row.get::<_, i64>(4)? != 0,
|
||||
episode_count: row.get(5)?,
|
||||
missing_count: row.get(6)?,
|
||||
})
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
pub async fn detail(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<Json<MediaItemDetail>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let (kind, title, year, monitored, root_folder) = conn
|
||||
.query_row(
|
||||
"SELECT kind, title, year, monitored, root_folder FROM media_item WHERE id = ?1",
|
||||
params![id],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, Option<i64>>(2)?,
|
||||
row.get::<_, i64>(3)? != 0,
|
||||
row.get::<_, String>(4)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.map_err(|e| (StatusCode::NOT_FOUND, e.to_string()))?;
|
||||
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT id, season_number, episode_number, title, air_date, monitored, has_file
|
||||
FROM episode WHERE media_item_id = ?1 ORDER BY season_number, episode_number",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let episodes = stmt
|
||||
.query_map(params![id], |row| {
|
||||
Ok(EpisodeSummary {
|
||||
id: row.get(0)?,
|
||||
season_number: row.get(1)?,
|
||||
episode_number: row.get(2)?,
|
||||
title: row.get(3)?,
|
||||
air_date: row.get(4)?,
|
||||
monitored: row.get::<_, i64>(5)? != 0,
|
||||
has_file: row.get::<_, i64>(6)? != 0,
|
||||
})
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
|
||||
Ok(Json(MediaItemDetail {
|
||||
id,
|
||||
kind,
|
||||
title,
|
||||
year,
|
||||
monitored,
|
||||
root_folder,
|
||||
episodes,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn add(
|
||||
State(state): State<AppState>,
|
||||
Json(req): Json<AddSeriesRequest>,
|
||||
) -> Result<Json<AddSeriesResponse>, (StatusCode, String)> {
|
||||
let Some(tvdb) = &state.tvdb else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"tvdb.api_key is not configured".into(),
|
||||
));
|
||||
};
|
||||
|
||||
// Fetch before taking the lock — a sync MutexGuard can't be held across
|
||||
// an `.await` point.
|
||||
let episodes = tvdb.episodes(&req.tvdb_id).await.map_err(internal)?;
|
||||
|
||||
let media_item_id = {
|
||||
let conn = state.conn.lock().await;
|
||||
metadata::insert_series(
|
||||
&conn,
|
||||
&req.tvdb_id,
|
||||
&req.title,
|
||||
req.year.map(|y| y as u32),
|
||||
&req.aliases,
|
||||
&req.root_folder,
|
||||
1,
|
||||
&episodes,
|
||||
)
|
||||
.map_err(internal)?
|
||||
};
|
||||
|
||||
Ok(Json(AddSeriesResponse { media_item_id }))
|
||||
}
|
||||
|
||||
pub async fn add_movie(
|
||||
State(state): State<AppState>,
|
||||
Json(req): Json<AddMovieRequest>,
|
||||
) -> Result<Json<AddMovieResponse>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let media_item_id = metadata::insert_movie(
|
||||
&conn,
|
||||
&req.tmdb_id,
|
||||
&req.title,
|
||||
req.year.map(|y| y as u32),
|
||||
&req.root_folder,
|
||||
2,
|
||||
)
|
||||
.map_err(internal)?;
|
||||
Ok(Json(AddMovieResponse { media_item_id }))
|
||||
}
|
||||
|
||||
pub async fn monitor(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
set_monitored(&state, id, true).await
|
||||
}
|
||||
|
||||
pub async fn unmonitor(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
set_monitored(&state, id, false).await
|
||||
}
|
||||
|
||||
async fn set_monitored(
|
||||
state: &AppState,
|
||||
id: i64,
|
||||
monitored: bool,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let rows = conn
|
||||
.execute(
|
||||
"UPDATE media_item SET monitored = ?1 WHERE id = ?2",
|
||||
params![monitored as i64, id],
|
||||
)
|
||||
.map_err(internal)?;
|
||||
if rows == 0 {
|
||||
return Err((StatusCode::NOT_FOUND, format!("no media_item {id}")));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
pub async fn monitor_episode(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
set_episode_monitored(&state, id, true).await
|
||||
}
|
||||
|
||||
pub async fn unmonitor_episode(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
set_episode_monitored(&state, id, false).await
|
||||
}
|
||||
|
||||
async fn set_episode_monitored(
|
||||
state: &AppState,
|
||||
episode_id: i64,
|
||||
monitored: bool,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let rows = conn
|
||||
.execute(
|
||||
"UPDATE episode SET monitored = ?1 WHERE id = ?2",
|
||||
params![monitored as i64, episode_id],
|
||||
)
|
||||
.map_err(internal)?;
|
||||
if rows == 0 {
|
||||
return Err((StatusCode::NOT_FOUND, format!("no episode {episode_id}")));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
/// Toggles every episode in one season at once — a season-level "row" isn't
|
||||
/// separately tracked (the `season` table exists in the schema but was
|
||||
/// never actually populated by any insert path, so resurrecting it just to
|
||||
/// hold one redundant monitored flag would mean keeping two copies of the
|
||||
/// same state in sync for no behavioral gain); bulk-updating the episodes
|
||||
/// directly gets the identical practical effect — this season's episodes
|
||||
/// stop appearing in search enumeration — with one source of truth.
|
||||
pub async fn monitor_season(
|
||||
State(state): State<AppState>,
|
||||
Path((media_item_id, season_number)): Path<(i64, i64)>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
set_season_monitored(&state, media_item_id, season_number, true).await
|
||||
}
|
||||
|
||||
pub async fn unmonitor_season(
|
||||
State(state): State<AppState>,
|
||||
Path((media_item_id, season_number)): Path<(i64, i64)>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
set_season_monitored(&state, media_item_id, season_number, false).await
|
||||
}
|
||||
|
||||
async fn set_season_monitored(
|
||||
state: &AppState,
|
||||
media_item_id: i64,
|
||||
season_number: i64,
|
||||
monitored: bool,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let rows = conn
|
||||
.execute(
|
||||
"UPDATE episode SET monitored = ?1 WHERE media_item_id = ?2 AND season_number = ?3",
|
||||
params![monitored as i64, media_item_id, season_number],
|
||||
)
|
||||
.map_err(internal)?;
|
||||
if rows == 0 {
|
||||
return Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
format!("no episodes in media_item {media_item_id} season {season_number}"),
|
||||
));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
/// Removes a media item (and, via cascade, its episodes/aliases/releases)
|
||||
/// from the library. Deliberately does not touch anything on disk — the
|
||||
/// same "stop managing this, don't delete the user's files" default
|
||||
/// Sonarr/Radarr use.
|
||||
pub async fn delete(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let rows = conn
|
||||
.execute("DELETE FROM media_item WHERE id = ?1", params![id])
|
||||
.map_err(internal)?;
|
||||
if rows == 0 {
|
||||
return Err((StatusCode::NOT_FOUND, format!("no media_item {id}")));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
/// Deletes an episode's imported file — both the row *and* the actual file
|
||||
/// on disk — and clears `has_file`, freeing the episode to be grabbed again
|
||||
/// by a future search. Unlike `delete` above, this deliberately does touch
|
||||
/// disk: the whole point is "this specific file is wrong/broken," and
|
||||
/// leaving it in place would (a) mislead the user into thinking they still
|
||||
/// have a good copy, and (b) block a future re-grab outright — `import_one`
|
||||
/// refuses to overwrite an existing file with an equal-or-lower-scoring one
|
||||
/// (see its dest-collision check), so a bad file that happened to score
|
||||
/// high would silently reject its own replacement forever if left on disk.
|
||||
pub async fn delete_episode_file(
|
||||
State(state): State<AppState>,
|
||||
Path(episode_id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let row: Option<(i64, String)> = conn
|
||||
.query_row(
|
||||
"SELECT id, path FROM episode_file WHERE episode_id = ?1",
|
||||
params![episode_id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
.map_err(internal)?;
|
||||
let Some((file_id, path)) = row else {
|
||||
return Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
format!("no file tracked for episode {episode_id}"),
|
||||
));
|
||||
};
|
||||
delete_file_and_clear(&conn, &path, file_id)?;
|
||||
conn.execute(
|
||||
"UPDATE episode SET has_file = 0 WHERE id = ?1",
|
||||
params![episode_id],
|
||||
)
|
||||
.map_err(internal)?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
/// Movie counterpart to `delete_episode_file` — movies have no `has_file`
|
||||
/// column of their own (`movie_needs_grab` derives "has a file" purely from
|
||||
/// `episode_file` row existence), so clearing the row is the entire state
|
||||
/// reset needed.
|
||||
pub async fn delete_movie_file(
|
||||
State(state): State<AppState>,
|
||||
Path(media_item_id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let row: Option<(i64, String)> = conn
|
||||
.query_row(
|
||||
"SELECT id, path FROM episode_file WHERE media_item_id = ?1 AND episode_id IS NULL",
|
||||
params![media_item_id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
.map_err(internal)?;
|
||||
let Some((file_id, path)) = row else {
|
||||
return Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
format!("no file tracked for media_item {media_item_id}"),
|
||||
));
|
||||
};
|
||||
delete_file_and_clear(&conn, &path, file_id)?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
/// Shared cleanup: removes the file from disk (tolerating "already gone" —
|
||||
/// nothing left to do, not an error) and deletes its `episode_file` row by
|
||||
/// that row's own primary key — never by `episode_id`/`media_item_id`
|
||||
/// alone, which a TV show shares across every one of its episode rows and
|
||||
/// would otherwise risk wiping an entire show's tracked files instead of
|
||||
/// the one the caller actually looked up.
|
||||
fn delete_file_and_clear(
|
||||
conn: &rusqlite::Connection,
|
||||
path: &str,
|
||||
file_id: i64,
|
||||
) -> Result<(), (StatusCode, String)> {
|
||||
if let Err(e) = std::fs::remove_file(path) {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
return Err((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to delete {path}: {e}"),
|
||||
));
|
||||
}
|
||||
}
|
||||
conn.execute("DELETE FROM episode_file WHERE id = ?1", params![file_id])
|
||||
.map_err(internal)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Manual "search now" for a single show/movie's whole current backlog —
|
||||
/// bypasses the background loop's per-cycle budget and due-ness cadence
|
||||
/// (those exist to pace unattended, indefinite operation; a single
|
||||
/// user-triggered request doesn't need throttling against itself).
|
||||
///
|
||||
/// Dispatched to the background loop via a channel rather than run inline
|
||||
/// here: the search pipeline (`execute_search_targets` and everything it
|
||||
/// calls) holds a `Connection`/`&dyn ReleaseSource` across `.await` points,
|
||||
/// which makes it `!Send` — fine for the background loop (part of the root
|
||||
/// future, never `Send`-constrained) but not callable directly from an axum
|
||||
/// handler (whose future axum requires to be `Send`). Routing through the
|
||||
/// background loop's own already-running instance also means this reuses
|
||||
/// its already-loaded title-matcher/sources instead of paying to construct
|
||||
/// fresh ones per request.
|
||||
pub async fn search_now(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<Json<SearchNowResult>, (StatusCode, String)> {
|
||||
let Some(tx) = &state.background_tx else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"background loop is not running (qbit.base_url is not configured)".into(),
|
||||
));
|
||||
};
|
||||
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
|
||||
tx.send(crate::api::BackgroundRequest::SearchNow {
|
||||
media_item_id: id,
|
||||
reply: reply_tx,
|
||||
})
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
let stats = reply_rx.await.map_err(internal)?.map_err(internal)?;
|
||||
|
||||
Ok(Json(SearchNowResult {
|
||||
targets: stats.targets,
|
||||
searched: stats.searched,
|
||||
grabbed: stats.grabbed,
|
||||
errors: stats.errors,
|
||||
source_exhausted: stats.source_exhausted,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Movie candidate picker: `episode_id` is always `None` for a movie
|
||||
/// target, matching every other movie/episode split in this codebase.
|
||||
pub async fn list_candidates(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<Json<Vec<breadarr_shared::dto::ReleaseCandidate>>, (StatusCode, String)> {
|
||||
fetch_candidates_via_background(&state, id, None).await
|
||||
}
|
||||
|
||||
pub async fn grab_candidate(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
Json(req): Json<breadarr_shared::dto::GrabCandidateRequest>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
grab_candidate_via_background(&state, id, None, req).await
|
||||
}
|
||||
|
||||
/// Episode candidate picker — resolves the episode's own `media_item_id`
|
||||
/// first (every other candidate/search-target function needs it), same
|
||||
/// pattern as `set_episode_monitored`.
|
||||
pub async fn list_episode_candidates(
|
||||
State(state): State<AppState>,
|
||||
Path(episode_id): Path<i64>,
|
||||
) -> Result<Json<Vec<breadarr_shared::dto::ReleaseCandidate>>, (StatusCode, String)> {
|
||||
let media_item_id = episode_media_item_id(&state, episode_id).await?;
|
||||
fetch_candidates_via_background(&state, media_item_id, Some(episode_id)).await
|
||||
}
|
||||
|
||||
pub async fn grab_episode_candidate(
|
||||
State(state): State<AppState>,
|
||||
Path(episode_id): Path<i64>,
|
||||
Json(req): Json<breadarr_shared::dto::GrabCandidateRequest>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let media_item_id = episode_media_item_id(&state, episode_id).await?;
|
||||
grab_candidate_via_background(&state, media_item_id, Some(episode_id), req).await
|
||||
}
|
||||
|
||||
async fn episode_media_item_id(
|
||||
state: &AppState,
|
||||
episode_id: i64,
|
||||
) -> Result<i64, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
conn.query_row(
|
||||
"SELECT media_item_id FROM episode WHERE id = ?1",
|
||||
params![episode_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(internal)?
|
||||
.ok_or_else(|| (StatusCode::NOT_FOUND, format!("no episode {episode_id}")))
|
||||
}
|
||||
|
||||
async fn fetch_candidates_via_background(
|
||||
state: &AppState,
|
||||
media_item_id: i64,
|
||||
episode_id: Option<i64>,
|
||||
) -> Result<Json<Vec<breadarr_shared::dto::ReleaseCandidate>>, (StatusCode, String)> {
|
||||
let Some(tx) = &state.background_tx else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"background loop is not running (qbit.base_url is not configured)".into(),
|
||||
));
|
||||
};
|
||||
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
|
||||
tx.send(crate::api::BackgroundRequest::FetchCandidates {
|
||||
media_item_id,
|
||||
episode_id,
|
||||
reply: reply_tx,
|
||||
})
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
let candidates = reply_rx.await.map_err(internal)?.map_err(internal)?;
|
||||
Ok(Json(candidates))
|
||||
}
|
||||
|
||||
async fn grab_candidate_via_background(
|
||||
state: &AppState,
|
||||
media_item_id: i64,
|
||||
episode_id: Option<i64>,
|
||||
req: breadarr_shared::dto::GrabCandidateRequest,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let Some(tx) = &state.background_tx else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"background loop is not running (qbit.base_url is not configured)".into(),
|
||||
));
|
||||
};
|
||||
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
|
||||
tx.send(crate::api::BackgroundRequest::GrabCandidate {
|
||||
media_item_id,
|
||||
episode_id,
|
||||
source_id: req.source_id,
|
||||
raw_title: req.raw_title,
|
||||
link: req.link,
|
||||
guid: req.guid,
|
||||
reply: reply_tx,
|
||||
})
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
reply_rx.await.map_err(internal)?.map_err(internal)?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
9
breadarrd/src/api/routes/mod.rs
Normal file
9
breadarrd/src/api/routes/mod.rs
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
pub mod calendar;
|
||||
pub mod health;
|
||||
pub mod library_health;
|
||||
pub mod media;
|
||||
pub mod quality_profiles;
|
||||
pub mod releases;
|
||||
pub mod review;
|
||||
pub mod search;
|
||||
pub mod stuck;
|
||||
87
breadarrd/src/api/routes/quality_profiles.rs
Normal file
87
breadarrd/src/api/routes/quality_profiles.rs
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::{QualityProfileSummary, UpdateQualityProfileWeightsRequest, WeightsDto};
|
||||
|
||||
use crate::api::AppState;
|
||||
use crate::scoring::{ProfileKind, QualityProfile};
|
||||
|
||||
fn to_dto(weights: &crate::scoring::profile::Weights) -> WeightsDto {
|
||||
WeightsDto {
|
||||
seeder: weights.seeder,
|
||||
resolution_tier: weights.resolution_tier,
|
||||
source_tier: weights.source_tier,
|
||||
codec_tier: weights.codec_tier,
|
||||
bit_depth: weights.bit_depth,
|
||||
container: weights.container,
|
||||
group_allowlist: weights.group_allowlist,
|
||||
repack: weights.repack,
|
||||
hdr: weights.hdr,
|
||||
}
|
||||
}
|
||||
|
||||
/// Lists every quality profile with its currently-effective weights
|
||||
/// (built-in defaults plus any stored override already applied) — never
|
||||
/// the raw, possibly-partial stored JSON, so a client always has a
|
||||
/// complete, concrete set of numbers to show and re-submit.
|
||||
pub async fn list(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<QualityProfileSummary>>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT id, name, kind, weights FROM quality_profile ORDER BY id")
|
||||
.map_err(internal)?;
|
||||
let rows: Vec<(i64, String, String, String)> = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<_>>()
|
||||
.map_err(internal)?;
|
||||
|
||||
let profiles = rows
|
||||
.into_iter()
|
||||
.map(|(id, name, kind, weights_json)| {
|
||||
let profile_kind = if kind == "movie" {
|
||||
ProfileKind::Movie
|
||||
} else {
|
||||
ProfileKind::Tv
|
||||
};
|
||||
let resolved = QualityProfile::with_weights_override(profile_kind, &weights_json);
|
||||
QualityProfileSummary {
|
||||
id,
|
||||
name,
|
||||
kind,
|
||||
weights: to_dto(&resolved.weights),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(Json(profiles))
|
||||
}
|
||||
|
||||
/// Overwrites a profile's stored `weights` column with exactly the
|
||||
/// submitted values (a full replacement, not a partial patch) — the TUI
|
||||
/// always submits the complete resolved set it displayed, so there's no
|
||||
/// ambiguity about what "the rest stays as it was" would even mean here.
|
||||
pub async fn update_weights(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
Json(req): Json<UpdateQualityProfileWeightsRequest>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let weights_json = serde_json::to_string(&req.weights).map_err(internal)?;
|
||||
let conn = state.conn.lock().await;
|
||||
let updated = conn
|
||||
.execute(
|
||||
"UPDATE quality_profile SET weights = ?1 WHERE id = ?2",
|
||||
rusqlite::params![weights_json, id],
|
||||
)
|
||||
.map_err(internal)?;
|
||||
if updated == 0 {
|
||||
return Err((StatusCode::NOT_FOUND, "quality profile not found".into()));
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
38
breadarrd/src/api/routes/releases.rs
Normal file
38
breadarrd/src/api/routes/releases.rs
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::ReleaseSummary;
|
||||
|
||||
use crate::api::AppState;
|
||||
|
||||
pub async fn list(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<ReleaseSummary>>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT r.id, m.title, r.raw_title, r.score, r.status, r.grabbed_at
|
||||
FROM release r JOIN media_item m ON m.id = r.media_item_id
|
||||
ORDER BY r.grabbed_at DESC LIMIT 200",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(ReleaseSummary {
|
||||
id: row.get(0)?,
|
||||
media_title: row.get(1)?,
|
||||
raw_title: row.get(2)?,
|
||||
score: row.get(3)?,
|
||||
status: row.get(4)?,
|
||||
grabbed_at: row.get(5)?,
|
||||
})
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
108
breadarrd/src/api/routes/review.rs
Normal file
108
breadarrd/src/api/routes/review.rs
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::ReviewQueueEntry;
|
||||
|
||||
use crate::api::AppState;
|
||||
use crate::scheduler;
|
||||
|
||||
pub async fn list(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<ReviewQueueEntry>>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT rq.id, rq.raw_release_title, m.title, rq.confidence, rq.status, rq.created_at
|
||||
FROM review_queue rq LEFT JOIN media_item m ON m.id = rq.candidate_media_item_id
|
||||
WHERE rq.status = 'pending'
|
||||
ORDER BY rq.created_at DESC",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(ReviewQueueEntry {
|
||||
id: row.get(0)?,
|
||||
raw_release_title: row.get(1)?,
|
||||
candidate_media_title: row.get(2)?,
|
||||
confidence: row.get(3)?,
|
||||
status: row.get(4)?,
|
||||
created_at: row.get(5)?,
|
||||
})
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
pub async fn approve(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let Some(qbit) = &state.qbit else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"qbit.base_url is not configured".into(),
|
||||
));
|
||||
};
|
||||
|
||||
let prep = {
|
||||
let conn = state.conn.lock().await;
|
||||
scheduler::prepare_review_approval(&conn, id).map_err(internal)?
|
||||
};
|
||||
let prepared = match prep {
|
||||
scheduler::ApprovalPrep::Ready(p) => p,
|
||||
scheduler::ApprovalPrep::NotPending => {
|
||||
return Err((StatusCode::CONFLICT, "review item is not pending".into()))
|
||||
}
|
||||
scheduler::ApprovalPrep::MissingGrabData => {
|
||||
return Err((
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
"review item has no stored link/source to grab".into(),
|
||||
))
|
||||
}
|
||||
scheduler::ApprovalPrep::CouldNotResolveEpisode => {
|
||||
return Err((
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
"could not resolve which episode this release is".into(),
|
||||
))
|
||||
}
|
||||
scheduler::ApprovalPrep::NotMonitoredOrAlreadyHave => {
|
||||
return Err((
|
||||
StatusCode::CONFLICT,
|
||||
"episode is not monitored or is already downloaded".into(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let torrent_hash = scheduler::grab_prepared_approval(qbit, &state.qbit_category, &prepared)
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
|
||||
{
|
||||
let conn = state.conn.lock().await;
|
||||
scheduler::finalize_review_approval(
|
||||
&conn,
|
||||
id,
|
||||
&prepared,
|
||||
&state.qbit_category,
|
||||
torrent_hash.as_deref(),
|
||||
)
|
||||
.map_err(internal)?;
|
||||
}
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
pub async fn reject(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
scheduler::reject_review(&conn, id).map_err(internal)?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
63
breadarrd/src/api/routes/search.rs
Normal file
63
breadarrd/src/api/routes/search.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::SearchResult;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::api::AppState;
|
||||
|
||||
fn default_kind() -> String {
|
||||
"series".to_string()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SearchParams {
|
||||
q: String,
|
||||
#[serde(default = "default_kind")]
|
||||
kind: String,
|
||||
}
|
||||
|
||||
pub async fn search(
|
||||
State(state): State<AppState>,
|
||||
Query(params): Query<SearchParams>,
|
||||
) -> Result<Json<Vec<SearchResult>>, (StatusCode, String)> {
|
||||
if params.kind == "movie" {
|
||||
let Some(tmdb) = &state.tmdb else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"tmdb.bearer_token is not configured".into(),
|
||||
));
|
||||
};
|
||||
let results = tmdb
|
||||
.search_movie(¶ms.q)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))?
|
||||
.into_iter()
|
||||
.map(|r| SearchResult {
|
||||
external_id: r.external_id,
|
||||
title: r.title,
|
||||
year: r.year.map(|y| y as i64),
|
||||
})
|
||||
.collect();
|
||||
return Ok(Json(results));
|
||||
}
|
||||
|
||||
let Some(tvdb) = &state.tvdb else {
|
||||
return Err((
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"tvdb.api_key is not configured".into(),
|
||||
));
|
||||
};
|
||||
let results = tvdb
|
||||
.search_series(¶ms.q)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))?
|
||||
.into_iter()
|
||||
.map(|r| SearchResult {
|
||||
external_id: r.external_id,
|
||||
title: r.name,
|
||||
year: r.year.map(|y| y as i64),
|
||||
})
|
||||
.collect();
|
||||
Ok(Json(results))
|
||||
}
|
||||
84
breadarrd/src/api/routes/stuck.rs
Normal file
84
breadarrd/src/api/routes/stuck.rs
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use breadarr_shared::dto::{MaxedSearchTarget, StalledGrab, StuckReport};
|
||||
|
||||
use crate::api::AppState;
|
||||
|
||||
/// Deliberately earlier than importer::mod's own `STALL_THRESHOLD_HOURS`
|
||||
/// (72h) — this report exists to surface a stuck grab *before* the
|
||||
/// daemon's own auto-fail kicks in, not just repeat it after the fact.
|
||||
const STALLED_GRAB_HOURS: f64 = 48.0;
|
||||
|
||||
/// Matches the point where the search-driven loop's exponential backoff
|
||||
/// (`6h * 2^(count-1)`, capped at 168h) saturates — see `scheduler::
|
||||
/// DUE_CLAUSE`. A target still at this count with no successful grab has
|
||||
/// been failing its search every single time for at least a week.
|
||||
const MAXED_SEARCH_COUNT: i64 = 6;
|
||||
|
||||
pub async fn stuck(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<StuckReport>, (StatusCode, String)> {
|
||||
let conn = state.conn.lock().await;
|
||||
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT r.id, m.title, r.raw_title, r.grabbed_at
|
||||
FROM release r JOIN media_item m ON m.id = r.media_item_id
|
||||
WHERE r.status = 'grabbed'
|
||||
AND (julianday('now') - julianday(r.grabbed_at)) * 24.0 > ?1
|
||||
ORDER BY r.grabbed_at ASC",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let stalled_grabs = stmt
|
||||
.query_map([STALLED_GRAB_HOURS], |row| {
|
||||
Ok(StalledGrab {
|
||||
release_id: row.get(0)?,
|
||||
media_title: row.get(1)?,
|
||||
raw_title: row.get(2)?,
|
||||
grabbed_at: row.get(3)?,
|
||||
})
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
|
||||
let review_queue_depth: i64 = conn
|
||||
.query_row(
|
||||
"SELECT count(*) FROM review_queue WHERE status = 'pending'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map_err(internal)?;
|
||||
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT ss.media_item_id, m.title, ss.search_count, ss.last_searched_at
|
||||
FROM search_state ss JOIN media_item m ON m.id = ss.media_item_id
|
||||
WHERE ss.search_count >= ?1 AND ss.last_result != 'grabbed'
|
||||
ORDER BY ss.search_count DESC",
|
||||
)
|
||||
.map_err(internal)?;
|
||||
let maxed_out_search_targets = stmt
|
||||
.query_map([MAXED_SEARCH_COUNT], |row| {
|
||||
Ok(MaxedSearchTarget {
|
||||
media_item_id: row.get(0)?,
|
||||
media_title: row.get(1)?,
|
||||
search_count: row.get(2)?,
|
||||
last_searched_at: row.get(3)?,
|
||||
})
|
||||
})
|
||||
.map_err(internal)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(internal)?;
|
||||
|
||||
Ok(Json(StuckReport {
|
||||
stalled_grabs,
|
||||
review_queue_depth,
|
||||
maxed_out_search_targets,
|
||||
}))
|
||||
}
|
||||
|
||||
fn internal<E: std::fmt::Display>(e: E) -> (StatusCode, String) {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
|
||||
}
|
||||
615
breadarrd/src/db.rs
Normal file
615
breadarrd/src/db.rs
Normal file
|
|
@ -0,0 +1,615 @@
|
|||
use rusqlite::Connection;
|
||||
|
||||
/// How many recent backups `backup_before_open` keeps around — bounded so
|
||||
/// a long-running daemon (restarted routinely by systemd on crash/update)
|
||||
/// doesn't slowly fill the disk with an ever-growing pile of copies.
|
||||
const MAX_BACKUPS: usize = 5;
|
||||
|
||||
/// Copies the database (and its WAL/SHM sidecar files, if present — WAL
|
||||
/// mode means the real state can be split across all three) to a timestamped
|
||||
/// backup before the daemon opens it, then prunes old backups beyond
|
||||
/// `MAX_BACKUPS`. A no-op if there's no existing database yet (fresh
|
||||
/// install — nothing to back up). Sonarr/Radarr back themselves up before
|
||||
/// every upgrade; breadarr has no migration framework to trigger that same
|
||||
/// moment, so this runs on every startup instead, which is a superset of
|
||||
/// the same protection.
|
||||
pub fn backup_before_open(db_path: &std::path::Path) -> anyhow::Result<()> {
|
||||
if !db_path.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let backup_dir = db_path
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("backups");
|
||||
std::fs::create_dir_all(&backup_dir)?;
|
||||
|
||||
let stem = db_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("breadarr.db");
|
||||
// Millisecond precision (not just seconds) so two backups triggered
|
||||
// within the same second — restart-crash-loop territory — still sort
|
||||
// and dedupe correctly instead of the second one silently overwriting.
|
||||
let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S%3fZ");
|
||||
let dest = backup_dir.join(format!("{timestamp}-{stem}"));
|
||||
std::fs::copy(db_path, &dest)?;
|
||||
|
||||
for sidecar_ext in ["-wal", "-shm"] {
|
||||
let sidecar = std::path::PathBuf::from(format!("{}{sidecar_ext}", db_path.display()));
|
||||
if sidecar.exists() {
|
||||
let dest_sidecar = backup_dir.join(format!("{timestamp}-{stem}{sidecar_ext}"));
|
||||
std::fs::copy(&sidecar, &dest_sidecar)?;
|
||||
}
|
||||
}
|
||||
|
||||
prune_old_backups(&backup_dir, stem)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prune_old_backups(backup_dir: &std::path::Path, stem: &str) -> anyhow::Result<()> {
|
||||
let mut entries: Vec<_> = std::fs::read_dir(backup_dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
let name = e.file_name();
|
||||
let name = name.to_string_lossy();
|
||||
name.ends_with(stem) && !name.ends_with("-wal") && !name.ends_with("-shm")
|
||||
})
|
||||
.collect();
|
||||
// Filenames are `<timestamp>-<stem>` with a sortable timestamp format,
|
||||
// so lexical order is chronological order.
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
if entries.len() > MAX_BACKUPS {
|
||||
for e in &entries[..entries.len() - MAX_BACKUPS] {
|
||||
let base = e.file_name().to_string_lossy().to_string();
|
||||
let _ = std::fs::remove_file(e.path());
|
||||
for sidecar_ext in ["-wal", "-shm"] {
|
||||
let _ = std::fs::remove_file(backup_dir.join(format!("{base}{sidecar_ext}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ad-hoc migration for a column introduced after the initial schema. No
|
||||
/// migration framework exists (single production deployment, `CREATE TABLE
|
||||
/// IF NOT EXISTS` is the only mechanism otherwise) — idempotent by simply
|
||||
/// ignoring "duplicate column name", so it's safe to call on every startup
|
||||
/// against both a fresh database and one that already has the column.
|
||||
fn add_column_if_missing(
|
||||
conn: &Connection,
|
||||
table: &str,
|
||||
column: &str,
|
||||
ddl: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let sql = format!("ALTER TABLE {table} ADD COLUMN {column} {ddl}");
|
||||
match conn.execute(&sql, []) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(rusqlite::Error::SqliteFailure(_, Some(msg)))
|
||||
if msg.contains("duplicate column name") =>
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn init(conn: &Connection) -> anyhow::Result<()> {
|
||||
conn.execute_batch(
|
||||
"PRAGMA journal_mode=WAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
PRAGMA busy_timeout=5000;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS quality_profile (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('movie','tv')),
|
||||
cutoff REAL,
|
||||
weights TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media_item (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('movie','series')),
|
||||
title TEXT NOT NULL,
|
||||
year INTEGER,
|
||||
tvdb_id INTEGER,
|
||||
tmdb_id INTEGER,
|
||||
anidb_id INTEGER,
|
||||
monitored INTEGER NOT NULL DEFAULT 1,
|
||||
quality_profile_id INTEGER NOT NULL REFERENCES quality_profile(id),
|
||||
root_folder TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alias (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
source TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS season (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
season_number INTEGER NOT NULL,
|
||||
monitored INTEGER NOT NULL DEFAULT 1,
|
||||
UNIQUE(media_item_id, season_number)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS episode (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
season_number INTEGER NOT NULL,
|
||||
episode_number INTEGER NOT NULL,
|
||||
absolute_number INTEGER,
|
||||
title TEXT,
|
||||
air_date TEXT,
|
||||
monitored INTEGER NOT NULL DEFAULT 1,
|
||||
has_file INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE(media_item_id, season_number, episode_number)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS episode_file (
|
||||
id INTEGER PRIMARY KEY,
|
||||
episode_id INTEGER REFERENCES episode(id) ON DELETE CASCADE,
|
||||
media_item_id INTEGER REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
video_codec TEXT,
|
||||
resolution TEXT,
|
||||
audio_langs TEXT,
|
||||
quality_score REAL,
|
||||
subtitle_status TEXT NOT NULL DEFAULT 'none'
|
||||
CHECK (subtitle_status IN ('none','not_needed','queued','done'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('rss','scrape')),
|
||||
base_url TEXT NOT NULL,
|
||||
poll_interval_secs INTEGER NOT NULL DEFAULT 300,
|
||||
enabled INTEGER NOT NULL DEFAULT 1
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS release (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
episode_id INTEGER REFERENCES episode(id) ON DELETE CASCADE,
|
||||
raw_title TEXT NOT NULL,
|
||||
source_id INTEGER NOT NULL REFERENCES source(id),
|
||||
guid TEXT NOT NULL,
|
||||
score REAL,
|
||||
torrent_hash TEXT,
|
||||
status TEXT NOT NULL
|
||||
CHECK (status IN ('grabbed','downloading','imported','upgraded','failed')),
|
||||
grabbed_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS anime_mapping (
|
||||
anidb_id INTEGER PRIMARY KEY,
|
||||
tvdb_id INTEGER,
|
||||
tmdb_id INTEGER,
|
||||
season_offset INTEGER,
|
||||
episode_offset INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- Anime *movies'* TMDB ids, separate from `anime_mapping` above:
|
||||
-- one AniDB entry can map to several TMDB movie ids (rereleases,
|
||||
-- split cuts), which doesn't fit `anime_mapping`'s one-row-per-
|
||||
-- anidb_id shape (that shape exists for TV's season/episode-offset
|
||||
-- upsert semantics, meaningless for movies) — and nothing about a
|
||||
-- movie needs the anidb_id linkage anyway, only is-this-tmdb_id-
|
||||
-- an-anime-movie membership.
|
||||
CREATE TABLE IF NOT EXISTS anime_tmdb_movie (
|
||||
tmdb_id INTEGER PRIMARY KEY
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS review_queue (
|
||||
id INTEGER PRIMARY KEY,
|
||||
raw_release_title TEXT NOT NULL,
|
||||
candidate_media_item_id INTEGER REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
confidence REAL NOT NULL,
|
||||
link TEXT,
|
||||
source_id INTEGER REFERENCES source(id),
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending','approved','rejected')),
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS seen_guid (
|
||||
source_id INTEGER NOT NULL REFERENCES source(id) ON DELETE CASCADE,
|
||||
guid TEXT NOT NULL,
|
||||
seen_at TEXT NOT NULL,
|
||||
PRIMARY KEY (source_id, guid)
|
||||
);
|
||||
|
||||
-- One row per monitored episode/movie the search-driven loop has
|
||||
-- attempted at least once. `episode_id` NULL means the row is for
|
||||
-- a movie (same nullable-episode_id convention as episode_file) —
|
||||
-- a plain UNIQUE(media_item_id, episode_id) can't express that,
|
||||
-- since SQLite treats NULLs as distinct, so two partial unique
|
||||
-- indexes stand in for it instead.
|
||||
CREATE TABLE IF NOT EXISTS search_state (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
episode_id INTEGER REFERENCES episode(id) ON DELETE CASCADE,
|
||||
last_searched_at TEXT NOT NULL,
|
||||
search_count INTEGER NOT NULL DEFAULT 1,
|
||||
last_result TEXT NOT NULL
|
||||
CHECK (last_result IN ('grabbed','no_results','no_viable','error'))
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_search_state_episode
|
||||
ON search_state(episode_id) WHERE episode_id IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_search_state_movie
|
||||
ON search_state(media_item_id) WHERE episode_id IS NULL;
|
||||
|
||||
-- Same shape as `search_state`, but for the separate upgrade-search
|
||||
-- cadence: already-imported episodes/movies periodically re-checked
|
||||
-- for a better release. Kept as its own table rather than reusing
|
||||
-- `search_state` so the two cadences (missing-content vs. upgrade)
|
||||
-- never interfere with each other's due-ness clock.
|
||||
CREATE TABLE IF NOT EXISTS upgrade_state (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
|
||||
episode_id INTEGER REFERENCES episode(id) ON DELETE CASCADE,
|
||||
last_checked_at TEXT NOT NULL,
|
||||
check_count INTEGER NOT NULL DEFAULT 1,
|
||||
last_result TEXT NOT NULL
|
||||
CHECK (last_result IN ('grabbed','no_results','no_viable','error'))
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_upgrade_state_episode
|
||||
ON upgrade_state(episode_id) WHERE episode_id IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_upgrade_state_movie
|
||||
ON upgrade_state(media_item_id) WHERE episode_id IS NULL;
|
||||
|
||||
-- A queryable decision log (grabbed/review_queued/imported/failed),
|
||||
-- separate from `release`'s current-state columns — added after
|
||||
-- a night of reconstructing why-did-it-do-that purely from
|
||||
-- journal logs. `media_item_id` has no FK so a row survives even
|
||||
-- after its media_item is deleted (the point is historical
|
||||
-- record, not live referential integrity).
|
||||
CREATE TABLE IF NOT EXISTS event_history (
|
||||
id INTEGER PRIMARY KEY,
|
||||
media_item_id INTEGER,
|
||||
episode_id INTEGER,
|
||||
event_type TEXT NOT NULL
|
||||
CHECK (event_type IN ('grabbed','review_queued','imported','failed')),
|
||||
detail TEXT NOT NULL,
|
||||
occurred_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_event_history_media_item
|
||||
ON event_history(media_item_id, occurred_at);
|
||||
|
||||
-- Placeholder profiles until Phase 6 builds real quality-scoring
|
||||
-- weights; media_item.quality_profile_id needs something to
|
||||
-- reference in the meantime.
|
||||
INSERT OR IGNORE INTO quality_profile (id, name, kind, weights)
|
||||
VALUES (1, 'Default TV', 'tv', '{}');
|
||||
INSERT OR IGNORE INTO quality_profile (id, name, kind, weights)
|
||||
VALUES (2, 'Default Movie', 'movie', '{}');
|
||||
|
||||
-- One row per `episode_file`, ffprobe's ground-truth read of what's
|
||||
-- actually on disk — a deliberate companion table rather than more
|
||||
-- columns on `episode_file` (which already carries three vestigial,
|
||||
-- never-populated columns from an earlier attempt at this: don't
|
||||
-- compound that mistake). `probe_size_bytes`/`probe_mtime` are a
|
||||
-- freshness guard, not just metadata: `reconcile_missing_files`
|
||||
-- repairs a stale `episode_file.path` onto the *same row id* when
|
||||
-- a file is renamed or transcoded in place (e.g. Tdarr converting
|
||||
-- codec/container), so a probe keyed only to that row id would
|
||||
-- silently describe a file that no longer exists unless something
|
||||
-- notices the underlying bytes changed.
|
||||
CREATE TABLE IF NOT EXISTS media_file_probe (
|
||||
episode_file_id INTEGER PRIMARY KEY
|
||||
REFERENCES episode_file(id) ON DELETE CASCADE,
|
||||
probed_at TEXT NOT NULL,
|
||||
probe_size_bytes INTEGER NOT NULL,
|
||||
probe_mtime INTEGER NOT NULL,
|
||||
|
||||
duration_secs REAL,
|
||||
container_bitrate INTEGER,
|
||||
video_codec TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
video_bitrate INTEGER,
|
||||
audio_codecs TEXT,
|
||||
audio_langs TEXT,
|
||||
default_audio_lang TEXT,
|
||||
subtitle_langs TEXT,
|
||||
|
||||
corruption_status TEXT NOT NULL DEFAULT 'unknown'
|
||||
CHECK (corruption_status IN
|
||||
('unknown','probe_ok','probe_failed','decode_ok','decode_failed')),
|
||||
corruption_checked_at TEXT,
|
||||
|
||||
flag_under_quality INTEGER NOT NULL DEFAULT 0,
|
||||
flag_no_subtitles INTEGER NOT NULL DEFAULT 0,
|
||||
flag_no_english_audio INTEGER NOT NULL DEFAULT 0,
|
||||
flag_non_english_default_audio INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_probe_flags
|
||||
ON media_file_probe(flag_under_quality, flag_no_subtitles,
|
||||
flag_no_english_audio, flag_non_english_default_audio);
|
||||
CREATE INDEX IF NOT EXISTS idx_probe_corruption
|
||||
ON media_file_probe(corruption_status);
|
||||
|
||||
-- Permanent audit trail of every torrent breadarr has ever handed
|
||||
-- to qBittorrent, independent of whatever later happens to the
|
||||
-- `release` row it's associated with (which can be deleted, or
|
||||
-- flipped between grabbed/failed/imported/upgraded over time). No
|
||||
-- foreign keys — same reasoning as `event_history`: the point is a
|
||||
-- durable historical record, not live referential integrity, so a
|
||||
-- row must survive its source/media_item being deleted.
|
||||
CREATE TABLE IF NOT EXISTS torrent_fetch (
|
||||
id INTEGER PRIMARY KEY,
|
||||
torrent_hash TEXT,
|
||||
name TEXT NOT NULL,
|
||||
size_bytes INTEGER,
|
||||
category TEXT,
|
||||
source_id INTEGER,
|
||||
media_item_id INTEGER,
|
||||
episode_id INTEGER,
|
||||
status TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_torrent_fetch_hash ON torrent_fetch(torrent_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_torrent_fetch_fetched_at ON torrent_fetch(fetched_at);",
|
||||
)?;
|
||||
|
||||
// Progress watermark for stalled-download detection (added after the
|
||||
// initial `release` schema — see `add_column_if_missing`).
|
||||
add_column_if_missing(conn, "release", "last_seen_progress", "REAL")?;
|
||||
add_column_if_missing(conn, "release", "last_progress_at", "TEXT")?;
|
||||
// Counts consecutive `import_one` failures for a completed torrent —
|
||||
// without this, a persistent import error (a bad path remap, an
|
||||
// unreadable file) retried forever every cycle with no escalation,
|
||||
// permanently blocking the episode/movie from being re-searched via a
|
||||
// different release (`status` stays `grabbed`) while never actually
|
||||
// succeeding either.
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"release",
|
||||
"import_error_count",
|
||||
"INTEGER NOT NULL DEFAULT 0",
|
||||
)?;
|
||||
// Set only for season-pack grabs (`episode_id` stays NULL for these,
|
||||
// same convention as movies — but unlike a movie, one media_item can
|
||||
// have many *different* season packs, so episode_id-is-NULL alone
|
||||
// can't disambiguate "season 1 pack" from "season 2 pack" the way it
|
||||
// disambiguates "this movie" from nothing else. This column is that
|
||||
// disambiguator, used to scope existing-grab/upgrade comparisons and
|
||||
// in-flight checks to the right season.
|
||||
add_column_if_missing(conn, "release", "season_number", "INTEGER")?;
|
||||
|
||||
// Extra per-file metadata beyond the original probe columns — kept as
|
||||
// its own batch of `add_column_if_missing` calls (rather than folded
|
||||
// into the original `CREATE TABLE`) since that table already exists on
|
||||
// deployed databases. None of this feeds any decision yet; the point is
|
||||
// to have it already sitting in the DB, ready for whatever uses it
|
||||
// later, rather than needing a second full-library re-probe when that
|
||||
// day comes.
|
||||
add_column_if_missing(conn, "media_file_probe", "frame_rate", "REAL")?;
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"media_file_probe",
|
||||
"default_audio_channels",
|
||||
"INTEGER",
|
||||
)?;
|
||||
// Best-effort classification (see MediaProbe::is_hdr) computed at
|
||||
// probe time from the raw color_transfer tag, which is also kept
|
||||
// verbatim below rather than only storing the reduced boolean.
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"media_file_probe",
|
||||
"hdr",
|
||||
"INTEGER NOT NULL DEFAULT 0",
|
||||
)?;
|
||||
add_column_if_missing(conn, "media_file_probe", "color_transfer", "TEXT")?;
|
||||
add_column_if_missing(conn, "media_file_probe", "container_format", "TEXT")?;
|
||||
// The complete ffprobe JSON response, verbatim — the escape hatch for
|
||||
// anything not already worth its own column (chapters, encoder tag,
|
||||
// less-common stream metadata). See `ffprobe::MediaProbe::raw_json`.
|
||||
add_column_if_missing(conn, "media_file_probe", "raw_ffprobe_json", "TEXT")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends one row to `event_history`. `event_type` must be one of the
|
||||
/// values the table's `CHECK` constraint allows — a bad value surfaces
|
||||
/// immediately as a `rusqlite::Error`, not a silent no-op.
|
||||
pub fn record_event(
|
||||
conn: &Connection,
|
||||
media_item_id: i64,
|
||||
episode_id: Option<i64>,
|
||||
event_type: &str,
|
||||
detail: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO event_history (media_item_id, episode_id, event_type, detail, occurred_at)
|
||||
VALUES (?1, ?2, ?3, ?4, datetime('now'))",
|
||||
rusqlite::params![media_item_id, episode_id, event_type, detail],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends one row to `torrent_fetch` — the permanent audit log, distinct
|
||||
/// from `release` (whose status/hash can change or which can be deleted
|
||||
/// alongside its media_item). Called once per actual qBittorrent add,
|
||||
/// regardless of what happens to the grab afterward.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn record_torrent_fetch(
|
||||
conn: &Connection,
|
||||
torrent_hash: Option<&str>,
|
||||
name: &str,
|
||||
size_bytes: Option<u64>,
|
||||
category: &str,
|
||||
source_id: i64,
|
||||
media_item_id: i64,
|
||||
episode_id: Option<i64>,
|
||||
status: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO torrent_fetch (torrent_hash, name, size_bytes, category, source_id, media_item_id, episode_id, status, fetched_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, datetime('now'))",
|
||||
rusqlite::params![torrent_hash, name, size_bytes, category, source_id, media_item_id, episode_id, status],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn init_is_idempotent() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
init(&conn).unwrap();
|
||||
init(&conn).unwrap();
|
||||
|
||||
let table_count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT count(*) FROM sqlite_master WHERE type='table'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(table_count, 17);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_event_inserts_a_queryable_row() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
init(&conn).unwrap();
|
||||
|
||||
record_event(&conn, 42, Some(7), "grabbed", "score=8.5").unwrap();
|
||||
|
||||
let (media_item_id, episode_id, event_type, detail): (i64, Option<i64>, String, String) =
|
||||
conn.query_row(
|
||||
"SELECT media_item_id, episode_id, event_type, detail FROM event_history",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(media_item_id, 42);
|
||||
assert_eq!(episode_id, Some(7));
|
||||
assert_eq!(event_type, "grabbed");
|
||||
assert_eq!(detail, "score=8.5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_event_rejects_an_unknown_event_type() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
init(&conn).unwrap();
|
||||
assert!(record_event(&conn, 1, None, "not_a_real_type", "x").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_torrent_fetch_inserts_a_queryable_row() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
init(&conn).unwrap();
|
||||
|
||||
record_torrent_fetch(
|
||||
&conn,
|
||||
Some("deadbeef"),
|
||||
"Some.Release.Title",
|
||||
Some(1_500_000_000),
|
||||
"breadarr",
|
||||
1,
|
||||
42,
|
||||
Some(7),
|
||||
"grabbed",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (hash, name, size, status): (Option<String>, String, Option<i64>, String) = conn
|
||||
.query_row(
|
||||
"SELECT torrent_hash, name, size_bytes, status FROM torrent_fetch",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(hash.as_deref(), Some("deadbeef"));
|
||||
assert_eq!(name, "Some.Release.Title");
|
||||
assert_eq!(size, Some(1_500_000_000));
|
||||
assert_eq!(status, "grabbed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_torrent_fetch_allows_a_null_hash_and_size() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
init(&conn).unwrap();
|
||||
|
||||
record_torrent_fetch(
|
||||
&conn,
|
||||
None,
|
||||
"Uncorrelated Add",
|
||||
None,
|
||||
"breadarr",
|
||||
1,
|
||||
42,
|
||||
None,
|
||||
"failed",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let count: i64 = conn
|
||||
.query_row("SELECT count(*) FROM torrent_fetch", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_before_open_is_a_noop_when_no_database_exists_yet() {
|
||||
let dir = std::env::temp_dir().join(format!("breadarr-backup-noop-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let db_path = dir.join("breadarr.db");
|
||||
|
||||
backup_before_open(&db_path).unwrap();
|
||||
let backup_dir = dir.join("backups");
|
||||
assert!(
|
||||
!backup_dir.exists(),
|
||||
"no backup dir should be created for a fresh install"
|
||||
);
|
||||
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_before_open_copies_an_existing_database() {
|
||||
let dir = std::env::temp_dir().join(format!("breadarr-backup-copy-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let db_path = dir.join("breadarr.db");
|
||||
std::fs::write(&db_path, b"fake sqlite data").unwrap();
|
||||
|
||||
backup_before_open(&db_path).unwrap();
|
||||
|
||||
let backup_dir = dir.join("backups");
|
||||
let backups: Vec<_> = std::fs::read_dir(&backup_dir).unwrap().collect();
|
||||
assert_eq!(backups.len(), 1, "expected exactly one backup file");
|
||||
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_before_open_prunes_beyond_max_backups() {
|
||||
let dir =
|
||||
std::env::temp_dir().join(format!("breadarr-backup-prune-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let db_path = dir.join("breadarr.db");
|
||||
std::fs::write(&db_path, b"fake sqlite data").unwrap();
|
||||
|
||||
// One more than MAX_BACKUPS, sleeping a few ms between each so the
|
||||
// millisecond-resolution timestamp in the filename is guaranteed to
|
||||
// differ and sort correctly.
|
||||
for _ in 0..(MAX_BACKUPS + 2) {
|
||||
backup_before_open(&db_path).unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
}
|
||||
|
||||
let backup_dir = dir.join("backups");
|
||||
let backups: Vec<_> = std::fs::read_dir(&backup_dir).unwrap().collect();
|
||||
assert_eq!(backups.len(), MAX_BACKUPS);
|
||||
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
379
breadarrd/src/importer/ffprobe.rs
Normal file
379
breadarrd/src/importer/ffprobe.rs
Normal file
|
|
@ -0,0 +1,379 @@
|
|||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AudioStream {
|
||||
pub codec: Option<String>,
|
||||
pub language: Option<String>,
|
||||
pub is_default: bool,
|
||||
pub channels: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SubtitleStream {
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct MediaProbe {
|
||||
pub duration_secs: Option<f64>,
|
||||
pub container_bitrate: Option<i64>,
|
||||
/// ffprobe's own long-form container name (e.g. "Matroska / WebM") —
|
||||
/// distinct from the file extension, which can lie or just be absent.
|
||||
pub container_format: Option<String>,
|
||||
pub video_codec: Option<String>,
|
||||
pub width: Option<i64>,
|
||||
pub height: Option<i64>,
|
||||
pub video_bitrate: Option<i64>,
|
||||
pub frame_rate: Option<f64>,
|
||||
/// The primary video stream's `color_transfer` tag as ffprobe reports
|
||||
/// it (e.g. "smpte2084", "arib-std-b67", "bt709") — kept as the raw
|
||||
/// string rather than pre-reduced to a bool so a future caller isn't
|
||||
/// stuck with just today's idea of what counts as "HDR" (Dolby Vision
|
||||
/// profiles, for instance, don't all surface the same way here).
|
||||
pub color_transfer: Option<String>,
|
||||
pub audio: Vec<AudioStream>,
|
||||
pub subtitles: Vec<SubtitleStream>,
|
||||
/// The complete ffprobe JSON response, kept verbatim. Every field
|
||||
/// above is a deliberately-narrow, purpose-built read of this same
|
||||
/// payload; this is the escape hatch — future functionality that needs
|
||||
/// something not already surfaced as its own column (chapters, extra
|
||||
/// stream tags, encoder info, etc.) can mine it here without requiring
|
||||
/// a re-probe of the whole library.
|
||||
pub raw_json: String,
|
||||
}
|
||||
|
||||
impl MediaProbe {
|
||||
/// Best-effort HDR classification from `color_transfer` — PQ
|
||||
/// (smpte2084, the common HDR10/HDR10+/Dolby-Vision-base-layer
|
||||
/// transfer function) or HLG (arib-std-b67). Not exhaustive by design;
|
||||
/// see `color_transfer`'s own doc comment for why the raw value is kept
|
||||
/// around too.
|
||||
pub fn is_hdr(&self) -> bool {
|
||||
matches!(
|
||||
self.color_transfer.as_deref(),
|
||||
Some("smpte2084") | Some("arib-std-b67")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_english(lang: Option<&str>) -> bool {
|
||||
matches!(lang, Some("eng") | Some("en"))
|
||||
}
|
||||
|
||||
impl MediaProbe {
|
||||
pub fn has_english_audio(&self) -> bool {
|
||||
self.audio.iter().any(|a| is_english(a.language.as_deref()))
|
||||
}
|
||||
|
||||
/// True when the default-disposition audio track (if any) is not
|
||||
/// English — mirrors `mkv::default_track_is_english_or_unset`'s "unset
|
||||
/// default isn't a problem" behavior, since the codebase's post-download
|
||||
/// remux fix already treats those two cases identically.
|
||||
pub fn default_audio_is_non_english(&self) -> bool {
|
||||
match self.audio.iter().find(|a| a.is_default) {
|
||||
Some(a) => !is_english(a.language.as_deref()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One `ffprobe` call gets container + every stream's codec/resolution/
|
||||
/// bitrate/language/default-disposition in a single JSON payload — no need
|
||||
/// for separate audio/video/subtitle passes.
|
||||
pub fn probe(path: &Path) -> Result<MediaProbe> {
|
||||
let output = Command::new("ffprobe")
|
||||
.args([
|
||||
"-v",
|
||||
"quiet",
|
||||
"-print_format",
|
||||
"json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
])
|
||||
.arg(path)
|
||||
.output()
|
||||
.context("failed to run ffprobe")?;
|
||||
|
||||
if !output.status.success() {
|
||||
bail!(
|
||||
"ffprobe failed for {}: {}",
|
||||
path.display(),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
let raw_json = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&output.stdout).context("ffprobe output was not valid JSON")?;
|
||||
|
||||
let format = &json["format"];
|
||||
let duration_secs = format["duration"]
|
||||
.as_str()
|
||||
.and_then(|s| s.parse::<f64>().ok());
|
||||
let container_bitrate = format["bit_rate"]
|
||||
.as_str()
|
||||
.and_then(|s| s.parse::<i64>().ok());
|
||||
let container_format = format["format_long_name"].as_str().map(str::to_string);
|
||||
|
||||
let streams = json["streams"].as_array().cloned().unwrap_or_default();
|
||||
|
||||
let mut probe = MediaProbe {
|
||||
duration_secs,
|
||||
container_bitrate,
|
||||
container_format,
|
||||
raw_json,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for stream in &streams {
|
||||
let codec_type = stream["codec_type"].as_str().unwrap_or("");
|
||||
let language = stream["tags"]["language"]
|
||||
.as_str()
|
||||
.or_else(|| stream["tags"]["LANGUAGE"].as_str())
|
||||
.map(str::to_string);
|
||||
match codec_type {
|
||||
"video" if probe.video_codec.is_none() => {
|
||||
// First video stream only — a second "video" stream in a
|
||||
// real-world file is almost always an embedded cover-art
|
||||
// thumbnail, not a second picture track.
|
||||
probe.video_codec = stream["codec_name"].as_str().map(str::to_string);
|
||||
probe.width = stream["width"].as_i64();
|
||||
probe.height = stream["height"].as_i64();
|
||||
probe.video_bitrate = stream["bit_rate"]
|
||||
.as_str()
|
||||
.and_then(|s| s.parse::<i64>().ok());
|
||||
probe.frame_rate = stream["avg_frame_rate"]
|
||||
.as_str()
|
||||
.and_then(parse_frame_rate_fraction)
|
||||
.or_else(|| {
|
||||
stream["r_frame_rate"]
|
||||
.as_str()
|
||||
.and_then(parse_frame_rate_fraction)
|
||||
});
|
||||
probe.color_transfer = stream["color_transfer"].as_str().map(str::to_string);
|
||||
}
|
||||
"audio" => {
|
||||
probe.audio.push(AudioStream {
|
||||
codec: stream["codec_name"].as_str().map(str::to_string),
|
||||
language,
|
||||
is_default: stream["disposition"]["default"].as_i64() == Some(1),
|
||||
channels: stream["channels"].as_i64(),
|
||||
});
|
||||
}
|
||||
"subtitle" => {
|
||||
probe.subtitles.push(SubtitleStream { language });
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(probe)
|
||||
}
|
||||
|
||||
/// ffprobe reports frame rate as a "num/den" fraction string (e.g.
|
||||
/// "24000/1001" for 23.976fps) rather than a plain number. `0/0` means
|
||||
/// "unknown" (common for e.g. still-image or data streams that don't
|
||||
/// really have one), not a real rate.
|
||||
fn parse_frame_rate_fraction(s: &str) -> Option<f64> {
|
||||
let (num, den) = s.split_once('/')?;
|
||||
let num: f64 = num.parse().ok()?;
|
||||
let den: f64 = den.parse().ok()?;
|
||||
if den == 0.0 {
|
||||
return None;
|
||||
}
|
||||
Some(num / den)
|
||||
}
|
||||
|
||||
/// A cheap header-level probe succeeding only proves the container is
|
||||
/// parseable — it doesn't catch a truncated file or mid-stream bit-rot,
|
||||
/// which requires actually decoding every frame. `ffmpeg -xerror` does
|
||||
/// that: any decode error either exits non-zero or writes to stderr. This
|
||||
/// is CPU-bound and can take minutes per file (it reads and decodes the
|
||||
/// whole thing), so it's deliberately a separate, opt-in pass rather than
|
||||
/// part of every routine scan — see `verify-library` in `main.rs`.
|
||||
pub enum DecodeCheck {
|
||||
Ok,
|
||||
Corrupt(String),
|
||||
}
|
||||
|
||||
pub fn verify_decodable(path: &Path) -> Result<DecodeCheck> {
|
||||
let output = Command::new("ffmpeg")
|
||||
.args(["-v", "error", "-xerror", "-i"])
|
||||
.arg(path)
|
||||
.args(["-f", "null", "-"])
|
||||
.output()
|
||||
.context("failed to run ffmpeg for decode verification")?;
|
||||
|
||||
if output.status.success() && output.stderr.is_empty() {
|
||||
Ok(DecodeCheck::Ok)
|
||||
} else {
|
||||
Ok(DecodeCheck::Corrupt(
|
||||
String::from_utf8_lossy(&output.stderr).into_owned(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn has_english_audio_true_when_any_track_is_english() {
|
||||
let probe = MediaProbe {
|
||||
audio: vec![
|
||||
AudioStream {
|
||||
codec: None,
|
||||
language: Some("jpn".to_string()),
|
||||
is_default: true,
|
||||
channels: None,
|
||||
},
|
||||
AudioStream {
|
||||
codec: None,
|
||||
language: Some("eng".to_string()),
|
||||
is_default: false,
|
||||
channels: None,
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(probe.has_english_audio());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_english_audio_false_with_no_tracks() {
|
||||
assert!(!MediaProbe::default().has_english_audio());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_audio_is_non_english_true_when_default_track_is_not_english() {
|
||||
let probe = MediaProbe {
|
||||
audio: vec![AudioStream {
|
||||
codec: None,
|
||||
language: Some("ita".to_string()),
|
||||
is_default: true,
|
||||
channels: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(probe.default_audio_is_non_english());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_audio_is_non_english_false_when_no_default_is_set() {
|
||||
// Mirrors mkv::default_track_is_english_or_unset: an unset default
|
||||
// isn't treated as a problem.
|
||||
let probe = MediaProbe {
|
||||
audio: vec![AudioStream {
|
||||
codec: None,
|
||||
language: Some("ita".to_string()),
|
||||
is_default: false,
|
||||
channels: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!probe.default_audio_is_non_english());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_audio_is_non_english_false_when_default_is_english() {
|
||||
let probe = MediaProbe {
|
||||
audio: vec![AudioStream {
|
||||
codec: None,
|
||||
language: Some("eng".to_string()),
|
||||
is_default: true,
|
||||
channels: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!probe.default_audio_is_non_english());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_fails_gracefully_for_a_nonexistent_file() {
|
||||
let result = probe(Path::new("/nonexistent/path/to/nothing.mkv"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
/// Generates a tiny real video via ffmpeg's `lavfi` synthetic source —
|
||||
/// validates the actual JSON field extraction (width/height/codec/
|
||||
/// duration/audio language+default) against genuine ffprobe output,
|
||||
/// not just hand-built `MediaProbe` fixtures.
|
||||
fn generate_test_clip(dir: &Path, width: u32, height: u32) -> std::path::PathBuf {
|
||||
let path = dir.join("clip.mkv");
|
||||
let status = Command::new("ffmpeg")
|
||||
.args(["-y", "-f", "lavfi", "-i"])
|
||||
.arg(format!("testsrc=size={width}x{height}:duration=1:rate=1"))
|
||||
.args(["-f", "lavfi", "-i", "sine=frequency=1000:duration=1"])
|
||||
.args(["-metadata:s:a:0", "language=eng"])
|
||||
.args(["-c:v", "libx264", "-c:a", "aac"])
|
||||
.arg(&path)
|
||||
.output()
|
||||
.expect("failed to run ffmpeg to generate a test clip");
|
||||
assert!(
|
||||
status.status.success(),
|
||||
"ffmpeg failed to generate test clip: {}",
|
||||
String::from_utf8_lossy(&status.stderr)
|
||||
);
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_extracts_real_resolution_and_audio_language_from_a_generated_clip() {
|
||||
let dir =
|
||||
std::env::temp_dir().join(format!("breadarr-ffprobe-test-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let clip = generate_test_clip(&dir, 640, 360);
|
||||
|
||||
let probe = probe(&clip).unwrap();
|
||||
assert_eq!(probe.width, Some(640));
|
||||
assert_eq!(probe.height, Some(360));
|
||||
assert!(probe.video_codec.is_some());
|
||||
assert!(probe.duration_secs.unwrap_or(0.0) > 0.0);
|
||||
assert!(probe.has_english_audio());
|
||||
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_extracts_extra_metadata_from_a_generated_clip() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"breadarr-ffprobe-extra-test-{}",
|
||||
std::process::id()
|
||||
));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
// Generated at rate=1 (see generate_test_clip), so frame_rate
|
||||
// should come back at (or very near) 1.0.
|
||||
let clip = generate_test_clip(&dir, 640, 360);
|
||||
|
||||
let probe = probe(&clip).unwrap();
|
||||
assert!(
|
||||
probe.frame_rate.is_some_and(|r| (r - 1.0).abs() < 0.1),
|
||||
"frame_rate was {:?}",
|
||||
probe.frame_rate
|
||||
);
|
||||
assert!(
|
||||
probe
|
||||
.container_format
|
||||
.as_deref()
|
||||
.is_some_and(|f| !f.is_empty()),
|
||||
"container_format should be populated"
|
||||
);
|
||||
assert!(
|
||||
probe.audio.first().is_some_and(|a| a.channels.is_some()),
|
||||
"audio channel count should be populated"
|
||||
);
|
||||
// A real generated clip's default color_transfer is unlikely to be
|
||||
// an HDR transfer function — this is really asserting `is_hdr()`
|
||||
// doesn't spuriously fire on ordinary SDR content.
|
||||
assert!(!probe.is_hdr());
|
||||
// The raw payload is kept verbatim and should parse as JSON on its
|
||||
// own — a caller mining it later needs that to actually be true.
|
||||
assert!(!probe.raw_json.is_empty());
|
||||
assert!(serde_json::from_str::<serde_json::Value>(&probe.raw_json).is_ok());
|
||||
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
94
breadarrd/src/importer/mkv.rs
Normal file
94
breadarrd/src/importer/mkv.rs
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AudioTrack {
|
||||
pub id: u32,
|
||||
pub language: Option<String>,
|
||||
pub is_default: bool,
|
||||
}
|
||||
|
||||
/// Runs `mkvmerge -J` (JSON track info) — mkvtoolnix's own structured-output
|
||||
/// mode, so track inspection doesn't depend on parsing human-readable text.
|
||||
pub fn inspect_audio_tracks(path: &Path) -> Result<Vec<AudioTrack>> {
|
||||
let output = Command::new("mkvmerge")
|
||||
.arg("-J")
|
||||
.arg(path)
|
||||
.output()
|
||||
.context("failed to run mkvmerge -J")?;
|
||||
|
||||
if !output.status.success() {
|
||||
bail!(
|
||||
"mkvmerge -J failed for {}: {}",
|
||||
path.display(),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&output.stdout).context("mkvmerge -J output was not valid JSON")?;
|
||||
|
||||
let tracks = json["tracks"]
|
||||
.as_array()
|
||||
.context("mkvmerge -J output had no tracks array")?;
|
||||
|
||||
Ok(tracks
|
||||
.iter()
|
||||
.filter(|t| t["type"] == "audio")
|
||||
.map(|t| AudioTrack {
|
||||
id: t["id"].as_u64().unwrap_or(0) as u32,
|
||||
language: t["properties"]["language"].as_str().map(str::to_string),
|
||||
is_default: t["properties"]["default_track"].as_bool().unwrap_or(false),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn is_english(lang: Option<&str>) -> bool {
|
||||
matches!(lang, Some("eng") | Some("en"))
|
||||
}
|
||||
|
||||
pub fn has_english_track(tracks: &[AudioTrack]) -> bool {
|
||||
tracks.iter().any(|t| is_english(t.language.as_deref()))
|
||||
}
|
||||
|
||||
/// True if there's no fix needed: either the default track is already
|
||||
/// English, or there's no explicit default at all (mkvmerge/players
|
||||
/// typically fall back to the first audio track, and single-track files
|
||||
/// have nothing to reorder).
|
||||
pub fn default_track_is_english_or_unset(tracks: &[AudioTrack]) -> bool {
|
||||
match tracks.iter().find(|t| t.is_default) {
|
||||
Some(t) => is_english(t.language.as_deref()),
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-flags the English track as default (and all others as non-default)
|
||||
/// without re-encoding — fixes the common "Italian track 1, English track
|
||||
/// 2" pattern. Returns an error if there's no English track to promote.
|
||||
pub fn remux_english_default(input: &Path, output: &Path, tracks: &[AudioTrack]) -> Result<()> {
|
||||
let english_id = tracks
|
||||
.iter()
|
||||
.find(|t| is_english(t.language.as_deref()))
|
||||
.map(|t| t.id)
|
||||
.context("no English audio track to promote")?;
|
||||
|
||||
let mut cmd = Command::new("mkvmerge");
|
||||
cmd.arg("-o").arg(output);
|
||||
for track in tracks {
|
||||
let flag = if track.id == english_id { "yes" } else { "no" };
|
||||
cmd.arg("--default-track-flag")
|
||||
.arg(format!("{}:{flag}", track.id));
|
||||
}
|
||||
cmd.arg(input);
|
||||
|
||||
let output_result = cmd.output().context("failed to run mkvmerge remux")?;
|
||||
if !output_result.status.success() {
|
||||
bail!(
|
||||
"mkvmerge remux failed: {}",
|
||||
String::from_utf8_lossy(&output_result.stderr)
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
3651
breadarrd/src/importer/mod.rs
Normal file
3651
breadarrd/src/importer/mod.rs
Normal file
File diff suppressed because it is too large
Load diff
41
breadarrd/src/jellyfin.rs
Normal file
41
breadarrd/src/jellyfin.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use anyhow::{bail, Context, Result};
|
||||
|
||||
pub struct JellyfinClient {
|
||||
base_url: String,
|
||||
api_key: String,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl JellyfinClient {
|
||||
pub fn new(base_url: impl Into<String>, api_key: impl Into<String>) -> Self {
|
||||
Self {
|
||||
base_url: base_url.into(),
|
||||
api_key: api_key.into(),
|
||||
// No total timeout is reqwest's default — fine for a one-shot
|
||||
// debug command, but the daemon's background loop holds the DB
|
||||
// mutex across this call, so a stalled connection here would
|
||||
// hang the whole daemon indefinitely.
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn refresh_library(&self) -> Result<()> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(format!("{}/Library/Refresh", self.base_url))
|
||||
.header("X-Emby-Token", &self.api_key)
|
||||
.send()
|
||||
.await
|
||||
.context("jellyfin library refresh request failed")?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
bail!("jellyfin library refresh failed: status={status} body={body:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
861
breadarrd/src/library_scan.rs
Normal file
861
breadarrd/src/library_scan.rs
Normal file
|
|
@ -0,0 +1,861 @@
|
|||
use std::path::Path;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use regex::Regex;
|
||||
use rusqlite::{params, Connection, OptionalExtension};
|
||||
|
||||
use crate::importer;
|
||||
use crate::matcher::TitleMatcher;
|
||||
use crate::metadata::tmdb::TmdbClient;
|
||||
use crate::metadata::tvdb::TvdbClient;
|
||||
use crate::metadata::{self, MovieSearchResult, SeriesSearchResult};
|
||||
use crate::parser;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ScanReport {
|
||||
pub matched: Vec<(String, i64)>,
|
||||
pub unmatched: Vec<String>,
|
||||
pub files_linked: usize,
|
||||
pub files_renamed: usize,
|
||||
}
|
||||
|
||||
fn get_episode_title(conn: &Connection, episode_id: i64) -> Result<Option<String>> {
|
||||
conn.query_row(
|
||||
"SELECT title FROM episode WHERE id = ?1",
|
||||
params![episode_id],
|
||||
|row| row.get::<_, Option<String>>(0),
|
||||
)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Renames `file` to `target_name` within the same directory (never moves
|
||||
/// across directories/filesystems) — a plain filesystem rename, so it's
|
||||
/// fast and doesn't touch file content. Refuses to clobber an existing,
|
||||
/// differently-named file already at the target path.
|
||||
fn rename_in_place(file: &Path, target_name: &str) -> Result<std::path::PathBuf> {
|
||||
let Some(parent) = file.parent() else {
|
||||
return Ok(file.to_path_buf());
|
||||
};
|
||||
let target = parent.join(target_name);
|
||||
if target == *file {
|
||||
return Ok(file.to_path_buf());
|
||||
}
|
||||
if target.exists() {
|
||||
tracing::warn!(
|
||||
from = %file.display(),
|
||||
to = %target.display(),
|
||||
"normalization target already exists, leaving file as-is"
|
||||
);
|
||||
return Ok(file.to_path_buf());
|
||||
}
|
||||
std::fs::rename(file, &target)?;
|
||||
Ok(target)
|
||||
}
|
||||
|
||||
// A "[Group]" or "[Tag]" prefix at the very start (common anime release-
|
||||
// group convention, e.g. "[SubsPlease] Show Name") — stripped before any
|
||||
// other parsing so it doesn't pollute the metadata search query or survive
|
||||
// into the normalized title.
|
||||
static LEADING_BRACKET_TAG_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\[[^\]]+\]\s*").unwrap());
|
||||
// "<Series> Movie 01 - <Subtitle>" is a common anime-movie flat-file naming
|
||||
// convention, but TMDB's search is not fuzzy enough to see past the "Movie
|
||||
// 01" token — it returns zero results for the full string even though
|
||||
// "<Series> <Subtitle>" alone matches immediately. Only fires when a dash
|
||||
// follows (an un-subtitled "Some Movie 2" is left alone).
|
||||
static MOVIE_NUMBER_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bmovie\s*\d{1,3}\b\s*-\s*").unwrap());
|
||||
static FOLDER_BRACKET_YEAR_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[(\[]((?:19|20)\d{2})[)\]]").unwrap());
|
||||
static FOLDER_BARE_YEAR_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\b((?:19|20)\d{2})\b").unwrap());
|
||||
// Common release-tag vocabulary (quality/source/codec/audio/season markers)
|
||||
// that shows up in folder names that were never cleaned up after being
|
||||
// dropped straight out of a torrent client — everything from the first
|
||||
// match onward is discarded, since normalization keeps only Name and Year.
|
||||
static FOLDER_JUNK_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)\b(1080p|720p|2160p|480p|4k|bluray|blu-ray|web-?dl|webrip|hdtv|dvdrip|remux|x264|x265|h\.?264|h\.?265|hevc|av1|dual[- ]?audio|multi[- ]?audio|dual|multi|proper|repack|extended|unrated|directors?\.?cut|10bit|8bit|complete|s\d{1,2}(?:e\d{1,3})?|season\s*\d+)\b",
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
fn clean_title_edge(s: &str) -> String {
|
||||
s.trim()
|
||||
.trim_end_matches(['-', ':', '(', '['])
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Extracts just Name and Year out of a folder name, discarding everything
|
||||
/// else — release-tag cruft (resolution/source/codec/group/season markers)
|
||||
/// that's meaningful on a torrent's own filename has no business surviving
|
||||
/// into the library's folder layout. "Arrival (2016)" -> ("Arrival",
|
||||
/// Some(2016)); "Arrival.2016.1080p.BluRay.x264-GROUP" -> ("Arrival",
|
||||
/// Some(2016)); "Attack on Titan S01 1080p Dual Audio [Group]" -> ("Attack
|
||||
/// on Titan", None); "Black Adder" -> ("Black Adder", None).
|
||||
fn parse_folder_name(name: &str) -> (String, Option<u32>) {
|
||||
let name = LEADING_BRACKET_TAG_RE.replace(name, "");
|
||||
let name = MOVIE_NUMBER_RE.replace(&name, "");
|
||||
let name = name.as_ref();
|
||||
|
||||
// Scene-style names pack everything into dot/underscore-separated
|
||||
// tokens with no real spaces at all — normalize those to spaces before
|
||||
// hunting for a year/junk boundary. Left untouched whenever real spaces
|
||||
// are already present, so legitimately dotted titles ("Mr. Robot")
|
||||
// aren't mangled.
|
||||
let normalized = if !name.contains(' ') && (name.contains('.') || name.contains('_')) {
|
||||
name.replace(['.', '_'], " ")
|
||||
} else {
|
||||
name.to_string()
|
||||
};
|
||||
let normalized = parser::WS_RE
|
||||
.replace_all(normalized.trim(), " ")
|
||||
.to_string();
|
||||
|
||||
if let Some(c) = FOLDER_BRACKET_YEAR_RE.captures(&normalized) {
|
||||
let year = c[1].parse().ok();
|
||||
let title = clean_title_edge(&normalized[..c.get(0).unwrap().start()]);
|
||||
return (title, year);
|
||||
}
|
||||
// A bare year is only trusted as *the* year when something precedes it
|
||||
// — otherwise a movie literally titled after a year ("1917", "2012")
|
||||
// would have its own title mistaken for a year with nothing left over.
|
||||
if let Some(m) = FOLDER_BARE_YEAR_RE.find(&normalized) {
|
||||
if m.start() > 0 {
|
||||
let year = normalized[m.start()..m.end()].parse().ok();
|
||||
let title = clean_title_edge(&normalized[..m.start()]);
|
||||
return (title, year);
|
||||
}
|
||||
}
|
||||
if let Some(m) = FOLDER_JUNK_RE.find(&normalized) {
|
||||
return (clean_title_edge(&normalized[..m.start()]), None);
|
||||
}
|
||||
(normalized.trim().to_string(), None)
|
||||
}
|
||||
|
||||
fn canonical_folder_name(title: &str, year: Option<u32>) -> String {
|
||||
match year {
|
||||
Some(y) => format!("{} ({y})", importer::sanitize(title)),
|
||||
None => importer::sanitize(title),
|
||||
}
|
||||
}
|
||||
|
||||
/// Renames a media folder in place so its name is exactly the canonical
|
||||
/// "{Title} ({Year})" form, discarding whatever release-tag/resolution/
|
||||
/// group cruft the original folder name carried — normalization is meant to
|
||||
/// leave nothing but name and year behind. Same-filesystem rename, doesn't
|
||||
/// touch file content; refuses to clobber an existing, different folder
|
||||
/// already at the target name; a no-op if already canonical.
|
||||
fn normalize_folder_name(
|
||||
root: &Path,
|
||||
current_dir: &Path,
|
||||
title: &str,
|
||||
year: Option<u32>,
|
||||
) -> Result<std::path::PathBuf> {
|
||||
let target_dir = root.join(canonical_folder_name(title, year));
|
||||
if target_dir == *current_dir {
|
||||
return Ok(current_dir.to_path_buf());
|
||||
}
|
||||
if target_dir.exists() {
|
||||
tracing::warn!(
|
||||
from = %current_dir.display(),
|
||||
to = %target_dir.display(),
|
||||
"normalized folder name already exists, leaving folder as-is"
|
||||
);
|
||||
return Ok(current_dir.to_path_buf());
|
||||
}
|
||||
std::fs::rename(current_dir, &target_dir)?;
|
||||
Ok(target_dir)
|
||||
}
|
||||
|
||||
fn subdirectories(root: &Path) -> Result<Vec<std::fs::DirEntry>> {
|
||||
let mut entries: Vec<_> = std::fs::read_dir(root)?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.path().is_dir())
|
||||
.collect();
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Below this cosine similarity, the "best" candidate among *multiple*
|
||||
/// options still isn't good enough to trust over the alternatives — treated
|
||||
/// as no match at all. Only applied when there's more than one candidate to
|
||||
/// choose between: a lone candidate is accepted unconditionally, since a
|
||||
/// low score there usually just means the provider has no English name or
|
||||
/// alias for it at all (e.g. TVDB's only entry for a well-known show can be
|
||||
/// its native-script title with non-English aliases), not that it's wrong.
|
||||
const MIN_MATCH_CONFIDENCE: f32 = 0.5;
|
||||
|
||||
/// TVDB/TMDB's own search relevance ranking isn't reliable enough to trust
|
||||
/// blindly — e.g. searching "Attack on Titan" ranks the spinoff "Attack on
|
||||
/// Titan: Counter Rockets" above the real 2013 series, whose own alias
|
||||
/// list has the far-closer "Attack on Titan (2013)". Exact year match (when
|
||||
/// the folder name has one) is tried first since it's a hard, cheap
|
||||
/// signal; the embedding matcher — comparing the query against every
|
||||
/// result's name *and* aliases, not just the top hit — breaks ties and
|
||||
/// covers the (common) case where no year is available at all.
|
||||
///
|
||||
/// When a year *is* known but no exact match exists, candidates whose year
|
||||
/// is off by more than one are excluded before the embedding pass — without
|
||||
/// this, a title-only match (a misplaced TV season-pack folder colliding
|
||||
/// with an unrelated same-named movie already in the catalog) can look
|
||||
/// identical to a real hit purely on text similarity.
|
||||
fn pick_series_match(
|
||||
matcher: &mut TitleMatcher,
|
||||
query: &str,
|
||||
results: Vec<SeriesSearchResult>,
|
||||
want_year: Option<u32>,
|
||||
) -> Result<Option<SeriesSearchResult>> {
|
||||
// No "exact year match wins outright" shortcut here on purpose — a
|
||||
// year match alone doesn't disambiguate title (e.g. two same-year,
|
||||
// differently-titled shows), and bypassing the embedding-similarity
|
||||
// check below on year alone is the same incident class as the
|
||||
// Naruto-Kai merge documented further down this file, except upstream
|
||||
// of it: that guard only catches a bad match on *reuse* of an existing
|
||||
// row, not on picking the wrong search result for a brand-new add.
|
||||
// Year still narrows the candidate pool below, just never on its own.
|
||||
let pool = match want_year {
|
||||
Some(want) => year_filtered_pool(&results, want),
|
||||
None => results.iter().collect::<Vec<_>>(),
|
||||
};
|
||||
if pool.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut candidates = Vec::new();
|
||||
for (idx, r) in pool.iter().enumerate() {
|
||||
candidates.push((idx, r.name.clone()));
|
||||
for alias in &r.aliases {
|
||||
candidates.push((idx, alias.clone()));
|
||||
}
|
||||
}
|
||||
let Some((best_idx, score)) = matcher.best_match_index(query, &candidates)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Gated on the *pre-filter* candidate count, not `pool.len()`: the year
|
||||
// filter can take several wrong search results down to a single
|
||||
// survivor that merely has a nearby year, which is exactly the kind of
|
||||
// wrong-but-unopposed match this floor exists to catch. The single-
|
||||
// candidate bypass is only safe when the provider's search itself
|
||||
// returned one result (e.g. TVDB's only entry for a show has no English
|
||||
// name/alias at all — still unambiguous), not when filtering produced one.
|
||||
if results.len() > 1 && score < MIN_MATCH_CONFIDENCE {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(pool.get(best_idx).map(|r| (*r).clone()))
|
||||
}
|
||||
|
||||
fn year_filtered_pool<T>(results: &[T], want: u32) -> Vec<&T>
|
||||
where
|
||||
T: HasYear,
|
||||
{
|
||||
results
|
||||
.iter()
|
||||
.filter(|r| r.year().is_none_or(|y| y.abs_diff(want) <= 1))
|
||||
.collect()
|
||||
}
|
||||
|
||||
trait HasYear {
|
||||
fn year(&self) -> Option<u32>;
|
||||
}
|
||||
|
||||
impl HasYear for SeriesSearchResult {
|
||||
fn year(&self) -> Option<u32> {
|
||||
self.year
|
||||
}
|
||||
}
|
||||
|
||||
impl HasYear for MovieSearchResult {
|
||||
fn year(&self) -> Option<u32> {
|
||||
self.year
|
||||
}
|
||||
}
|
||||
|
||||
fn pick_movie_match(
|
||||
matcher: &mut TitleMatcher,
|
||||
query: &str,
|
||||
results: Vec<MovieSearchResult>,
|
||||
want_year: Option<u32>,
|
||||
) -> Result<Option<MovieSearchResult>> {
|
||||
// See `pick_series_match` — no exact-year-wins-outright shortcut here
|
||||
// either, for the same reason.
|
||||
let pool = match want_year {
|
||||
Some(want) => year_filtered_pool(&results, want),
|
||||
None => results.iter().collect::<Vec<_>>(),
|
||||
};
|
||||
if pool.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// TMDB movie results don't carry an alias list the way TVDB does, but
|
||||
// running the same title through the matcher still catches the same
|
||||
// class of "wrong entry ranked first" problem when titles are close
|
||||
// but not identical (sequels, re-releases, regional retitles).
|
||||
let candidates: Vec<(usize, String)> = pool
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, r)| (idx, r.title.clone()))
|
||||
.collect();
|
||||
let Some((best_idx, score)) = matcher.best_match_index(query, &candidates)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// See `pick_series_match` — gated on the pre-filter count, not `pool`.
|
||||
if results.len() > 1 && score < MIN_MATCH_CONFIDENCE {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(pool.get(best_idx).map(|r| (*r).clone()))
|
||||
}
|
||||
|
||||
fn get_media_item_by_tvdb_id(conn: &Connection, tvdb_id: i64) -> Result<Option<i64>> {
|
||||
conn.query_row(
|
||||
"SELECT id FROM media_item WHERE tvdb_id = ?1",
|
||||
params![tvdb_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn get_media_item_by_tmdb_id(conn: &Connection, tmdb_id: i64) -> Result<Option<i64>> {
|
||||
conn.query_row(
|
||||
"SELECT id FROM media_item WHERE tmdb_id = ?1 AND kind = 'movie'",
|
||||
params![tmdb_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Guards against blindly repointing an existing row's `root_folder` onto
|
||||
/// whatever folder a provider-ID lookup happened to match, without ever
|
||||
/// checking the two actually agree on *which show*. A provider ID can end
|
||||
/// up on the wrong row for reasons entirely outside this function's control
|
||||
/// (a bad selection when the show was originally added, a data-quality
|
||||
/// slip on the provider's own end) — verified live: this exact gap let
|
||||
/// "Boy Swallows Universe" (added with a TVDB ID that collided with Naruto
|
||||
/// Kai's real one) silently absorb Naruto Kai's folder path on a later
|
||||
/// scan, and Naruto Kai's own tracking row simply vanished, merged into an
|
||||
/// unrelated show. Reuses the same embedding similarity the rest of the
|
||||
/// matcher uses, so "close enough" here means the same thing it means
|
||||
/// everywhere else in the app.
|
||||
fn existing_row_title_plausibly_matches(
|
||||
conn: &Connection,
|
||||
matcher: &mut TitleMatcher,
|
||||
media_item_id: i64,
|
||||
candidate_title: &str,
|
||||
) -> Result<bool> {
|
||||
let existing_title: String = conn.query_row(
|
||||
"SELECT title FROM media_item WHERE id = ?1",
|
||||
params![media_item_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
let score = matcher
|
||||
.best_match_index(candidate_title, &[(0, existing_title.clone())])?
|
||||
.map(|(_, score)| score)
|
||||
.unwrap_or(0.0);
|
||||
let plausible = score >= crate::matcher::MIN_CONFIDENCE;
|
||||
if !plausible {
|
||||
tracing::warn!(
|
||||
media_item_id,
|
||||
existing_title = %existing_title,
|
||||
candidate_title,
|
||||
score,
|
||||
"provider-ID match found an existing row, but its title doesn't \
|
||||
plausibly match the folder being scanned — refusing to repoint \
|
||||
its root_folder onto a possibly-unrelated show"
|
||||
);
|
||||
}
|
||||
Ok(plausible)
|
||||
}
|
||||
|
||||
fn find_episode_id(
|
||||
conn: &Connection,
|
||||
media_item_id: i64,
|
||||
season: u32,
|
||||
episode: u32,
|
||||
) -> Result<Option<i64>> {
|
||||
conn.query_row(
|
||||
"SELECT id FROM episode WHERE media_item_id = ?1 AND season_number = ?2 AND episode_number = ?3",
|
||||
params![media_item_id, season, episode],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn episode_has_file_row(conn: &Connection, episode_id: i64) -> Result<bool> {
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT count(*) FROM episode_file WHERE episode_id = ?1",
|
||||
params![episode_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
fn movie_has_file_row(conn: &Connection, media_item_id: i64) -> Result<bool> {
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT count(*) FROM episode_file WHERE media_item_id = ?1 AND episode_id IS NULL",
|
||||
params![media_item_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
/// Imports every show folder under `root` as a monitored series: matches
|
||||
/// it to TVDB, populates its full episode list, then walks its actual
|
||||
/// files and marks whichever episodes are already present as `has_file`
|
||||
/// (instead of a fresh "add show" which starts with nothing on disk) —
|
||||
/// idempotent, safe to re-run against the same root later.
|
||||
pub async fn scan_tv_root(
|
||||
conn: &Connection,
|
||||
tvdb: &TvdbClient,
|
||||
matcher: &mut TitleMatcher,
|
||||
root: &Path,
|
||||
quality_profile_id: i64,
|
||||
jellyfin: Option<&crate::jellyfin::JellyfinClient>,
|
||||
) -> Result<ScanReport> {
|
||||
let mut report = ScanReport::default();
|
||||
|
||||
for entry in subdirectories(root)? {
|
||||
let folder_name = entry.file_name().to_string_lossy().to_string();
|
||||
let (title, year) = parse_folder_name(&folder_name);
|
||||
|
||||
let results = match tvdb.search_series(&title).await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(folder = %folder_name, error = %e, "tvdb search failed");
|
||||
report.unmatched.push(folder_name);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(best) = pick_series_match(matcher, &title, results, year)? else {
|
||||
report.unmatched.push(folder_name);
|
||||
continue;
|
||||
};
|
||||
let tvdb_id: i64 = best.external_id.parse().unwrap_or_default();
|
||||
let canonical_year = best.year.or(year);
|
||||
// Normalize the folder itself down to "Title (Year)" — release
|
||||
// tags/resolution/group cruft in the original folder name isn't
|
||||
// wanted in the library layout, only in the source torrent name.
|
||||
let series_dir = normalize_folder_name(root, &entry.path(), &title, canonical_year)?;
|
||||
|
||||
let existing_id = get_media_item_by_tvdb_id(conn, tvdb_id)?;
|
||||
let reuse_existing = match existing_id {
|
||||
Some(id) => existing_row_title_plausibly_matches(conn, matcher, id, &title)?,
|
||||
None => false,
|
||||
};
|
||||
|
||||
let media_item_id = if let (Some(id), true) = (existing_id, reuse_existing) {
|
||||
conn.execute(
|
||||
"UPDATE media_item SET root_folder = ?1 WHERE id = ?2",
|
||||
params![series_dir.to_string_lossy(), id],
|
||||
)?;
|
||||
id
|
||||
} else {
|
||||
let episodes = tvdb.episodes(&best.external_id).await?;
|
||||
// Use the folder's own title, not TVDB's `name` field — for
|
||||
// non-English-origin shows that's often the original-language
|
||||
// title (e.g. "進撃の巨人" for Attack on Titan), while the
|
||||
// folder name reflects how the user actually organizes their
|
||||
// library already.
|
||||
metadata::insert_series(
|
||||
conn,
|
||||
&best.external_id,
|
||||
&title,
|
||||
canonical_year,
|
||||
&best.aliases,
|
||||
&series_dir.to_string_lossy(),
|
||||
quality_profile_id,
|
||||
&episodes,
|
||||
)?
|
||||
};
|
||||
report.matched.push((folder_name, media_item_id));
|
||||
|
||||
for file in importer::walk_files(&series_dir)? {
|
||||
let ext = file
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("")
|
||||
.to_lowercase();
|
||||
if !importer::VIDEO_EXTS.contains(&ext.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let filename = file.file_name().unwrap_or_default().to_string_lossy();
|
||||
let parsed = parser::parse(&filename);
|
||||
// `parsed.season` is `None` for the dominant fansub naming
|
||||
// convention ("[SubsPlease] Show - 05.mkv" — absolute episode
|
||||
// numbering only, no season marker at all), which previously
|
||||
// skipped linking these files entirely: `has_file` never got
|
||||
// set, so the daemon kept re-downloading episodes it already
|
||||
// had on disk. `crate::scheduler::resolve_episode` already
|
||||
// handles exactly this via the anime absolute-numbering map —
|
||||
// reused here instead of duplicating it.
|
||||
let Some((season, episode)) =
|
||||
crate::scheduler::resolve_episode(conn, Some(tvdb_id), &parsed)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(episode_id) = find_episode_id(conn, media_item_id, season, episode)? else {
|
||||
continue;
|
||||
};
|
||||
if episode_has_file_row(conn, episode_id)? {
|
||||
continue;
|
||||
}
|
||||
|
||||
let episode_title = get_episode_title(conn, episode_id)?;
|
||||
let target_name = importer::deterministic_filename(
|
||||
&title,
|
||||
season,
|
||||
episode,
|
||||
episode_title.as_deref(),
|
||||
&ext,
|
||||
);
|
||||
let final_path = match rename_in_place(&file, &target_name) {
|
||||
Ok(p) => {
|
||||
if p != file {
|
||||
report.files_renamed += 1;
|
||||
}
|
||||
p
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(file = %file.display(), error = %e, "rename failed, keeping original name");
|
||||
file.clone()
|
||||
}
|
||||
};
|
||||
|
||||
let size = std::fs::metadata(&final_path)?.len();
|
||||
conn.execute(
|
||||
"INSERT INTO episode_file (episode_id, path, size_bytes, subtitle_status) VALUES (?1, ?2, ?3, 'none')",
|
||||
params![episode_id, final_path.to_string_lossy(), size],
|
||||
)?;
|
||||
let episode_file_id = conn.last_insert_rowid();
|
||||
conn.execute(
|
||||
"UPDATE episode SET has_file = 1 WHERE id = ?1",
|
||||
params![episode_id],
|
||||
)?;
|
||||
report.files_linked += 1;
|
||||
// Best-effort: a newly-linked file should get its ground-truth
|
||||
// metadata right away rather than waiting for the next
|
||||
// `probe_library` sweep, but a probing failure here must never
|
||||
// fail the scan itself.
|
||||
if let Err(e) = importer::ensure_probed(conn, episode_file_id, &final_path) {
|
||||
tracing::warn!(episode_file_id, error = %e, "post-scan probing failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Folder renames (via `normalize_folder_name`) change paths Jellyfin
|
||||
// already has indexed under their old names — without an explicit
|
||||
// refresh here, Jellyfin's own library index silently falls out of
|
||||
// sync with disk until its *own* next scheduled scan happens to run,
|
||||
// which could be hours away. Verified live: exactly this happened —
|
||||
// renamed anime folders vanished from Jellyfin's library view because
|
||||
// nothing told it to look again.
|
||||
if !report.matched.is_empty() {
|
||||
if let Some(jellyfin) = jellyfin {
|
||||
refresh_jellyfin_after_rename(jellyfin, "tv scan").await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Refreshes Jellyfin's library index after a scan that may have renamed
|
||||
/// folders — a single fire-and-forget attempt previously left Jellyfin's
|
||||
/// index silently stale for however long it took its own next scheduled
|
||||
/// scan to notice on a transient failure (Jellyfin mid-restart, a momentary
|
||||
/// network blip — the realistic failure mode, not a persistent outage).
|
||||
/// Retries a few times with backoff before giving up, and logs at `error`
|
||||
/// (not `warn`) once retries are exhausted, since a permanently stale
|
||||
/// Jellyfin index is exactly the kind of thing worth actually noticing
|
||||
/// rather than scrolling past in the journal.
|
||||
async fn refresh_jellyfin_after_rename(jellyfin: &crate::jellyfin::JellyfinClient, context: &str) {
|
||||
const ATTEMPTS: u32 = 3;
|
||||
for attempt in 1..=ATTEMPTS {
|
||||
match jellyfin.refresh_library().await {
|
||||
Ok(()) => return,
|
||||
Err(e) if attempt < ATTEMPTS => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
attempt,
|
||||
context,
|
||||
"jellyfin library refresh failed, retrying"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(5 * u64::from(attempt))).await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
context,
|
||||
"jellyfin library refresh failed after {ATTEMPTS} attempts — its index may \
|
||||
stay stale relative to disk until its own next scheduled scan runs"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One thing to match+import: either a "one folder per movie" entry (the
|
||||
/// common layout) or a bare video file sitting directly under the root
|
||||
/// (some libraries — e.g. this one's "Anime Movies" — are organized flat,
|
||||
/// one file per movie with no per-movie folder at all).
|
||||
struct MovieCandidate {
|
||||
display_name: String,
|
||||
/// Where to search for the actual video file: the folder itself for a
|
||||
/// subdirectory entry, or the file's own path for a flat file.
|
||||
video_source: std::path::PathBuf,
|
||||
}
|
||||
|
||||
fn movie_candidates(root: &Path) -> Result<Vec<MovieCandidate>> {
|
||||
let mut out: Vec<MovieCandidate> = subdirectories(root)?
|
||||
.into_iter()
|
||||
.map(|entry| MovieCandidate {
|
||||
display_name: entry.file_name().to_string_lossy().to_string(),
|
||||
video_source: entry.path(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
for entry in std::fs::read_dir(root)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let ext = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("")
|
||||
.to_lowercase();
|
||||
if !importer::VIDEO_EXTS.contains(&ext.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let display_name = path
|
||||
.file_stem()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
out.push(MovieCandidate {
|
||||
display_name,
|
||||
video_source: path,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Ensures a movie has its own folder before it's otherwise processed —
|
||||
/// restructures a bare file directly under `root` into "{Title}
|
||||
/// ({Year})/{file}", matching the "one folder per movie" convention the
|
||||
/// rest of the library already uses (some libraries — e.g. this one's
|
||||
/// "Anime Movies" — were organized flat, one file per movie with no
|
||||
/// per-movie folder at all). Same-filesystem rename, doesn't touch file
|
||||
/// content; a no-op if `video_source` is already a folder.
|
||||
fn ensure_movie_folder(
|
||||
root: &Path,
|
||||
video_source: &Path,
|
||||
title: &str,
|
||||
year: Option<u32>,
|
||||
) -> Result<std::path::PathBuf> {
|
||||
if video_source.is_dir() {
|
||||
return normalize_folder_name(root, video_source, title, year);
|
||||
}
|
||||
let target_dir = root.join(canonical_folder_name(title, year));
|
||||
std::fs::create_dir_all(&target_dir)?;
|
||||
|
||||
let file_name = video_source
|
||||
.file_name()
|
||||
.context("video file has no filename")?;
|
||||
let target_file = target_dir.join(file_name);
|
||||
if target_file != *video_source {
|
||||
std::fs::rename(video_source, &target_file)?;
|
||||
}
|
||||
Ok(target_dir)
|
||||
}
|
||||
|
||||
/// Same idea as [`scan_tv_root`] but for movies: one media_item per
|
||||
/// candidate (folder or bare file, restructured into its own folder if it
|
||||
/// wasn't already one), matched to TMDB, with its video file linked as its
|
||||
/// `episode_file` (movies have no `episode` row — `episode_id` is NULL).
|
||||
pub async fn scan_movie_root(
|
||||
conn: &Connection,
|
||||
tmdb: &TmdbClient,
|
||||
matcher: &mut TitleMatcher,
|
||||
root: &Path,
|
||||
quality_profile_id: i64,
|
||||
jellyfin: Option<&crate::jellyfin::JellyfinClient>,
|
||||
) -> Result<ScanReport> {
|
||||
let mut report = ScanReport::default();
|
||||
|
||||
for candidate in movie_candidates(root)? {
|
||||
let folder_name = candidate.display_name;
|
||||
let (title, year) = parse_folder_name(&folder_name);
|
||||
|
||||
let results = match tmdb.search_movie(&title).await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(folder = %folder_name, error = %e, "tmdb search failed");
|
||||
report.unmatched.push(folder_name);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(best) = pick_movie_match(matcher, &title, results, year)? else {
|
||||
report.unmatched.push(folder_name);
|
||||
continue;
|
||||
};
|
||||
let tmdb_id: i64 = best.external_id.parse().unwrap_or_default();
|
||||
let canonical_year = best.year.or(year);
|
||||
|
||||
// Normalizes the folder itself down to "Title (Year)" too — release
|
||||
// tags/resolution/group cruft from the original folder/file name
|
||||
// isn't wanted in the library layout, only in the source torrent
|
||||
// name (covers both a pre-existing messily-named folder and a flat
|
||||
// file getting its own folder created for the first time).
|
||||
let item_root_folder =
|
||||
ensure_movie_folder(root, &candidate.video_source, &title, canonical_year)?;
|
||||
|
||||
let existing_id = get_media_item_by_tmdb_id(conn, tmdb_id)?;
|
||||
let reuse_existing = match existing_id {
|
||||
Some(id) => existing_row_title_plausibly_matches(conn, matcher, id, &title)?,
|
||||
None => false,
|
||||
};
|
||||
|
||||
let media_item_id = if let (Some(id), true) = (existing_id, reuse_existing) {
|
||||
conn.execute(
|
||||
"UPDATE media_item SET root_folder = ?1 WHERE id = ?2",
|
||||
params![item_root_folder.to_string_lossy(), id],
|
||||
)?;
|
||||
id
|
||||
} else {
|
||||
conn.execute(
|
||||
"INSERT INTO media_item (kind, title, year, tmdb_id, monitored, quality_profile_id, root_folder)
|
||||
VALUES ('movie', ?1, ?2, ?3, 1, ?4, ?5)",
|
||||
params![
|
||||
title,
|
||||
canonical_year,
|
||||
tmdb_id,
|
||||
quality_profile_id,
|
||||
item_root_folder.to_string_lossy()
|
||||
],
|
||||
)?;
|
||||
conn.last_insert_rowid()
|
||||
};
|
||||
report.matched.push((folder_name, media_item_id));
|
||||
|
||||
if !movie_has_file_row(conn, media_item_id)? {
|
||||
if let Ok(file) = importer::largest_video_file(&item_root_folder) {
|
||||
let ext = file
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("mkv")
|
||||
.to_lowercase();
|
||||
let target_name = importer::deterministic_movie_filename(
|
||||
&title,
|
||||
canonical_year.map(i64::from),
|
||||
&ext,
|
||||
);
|
||||
let final_path = match rename_in_place(&file, &target_name) {
|
||||
Ok(p) => {
|
||||
if p != file {
|
||||
report.files_renamed += 1;
|
||||
}
|
||||
p
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(file = %file.display(), error = %e, "rename failed, keeping original name");
|
||||
file.clone()
|
||||
}
|
||||
};
|
||||
|
||||
let size = std::fs::metadata(&final_path)?.len();
|
||||
conn.execute(
|
||||
"INSERT INTO episode_file (media_item_id, path, size_bytes, subtitle_status) VALUES (?1, ?2, ?3, 'none')",
|
||||
params![media_item_id, final_path.to_string_lossy(), size],
|
||||
)?;
|
||||
let episode_file_id = conn.last_insert_rowid();
|
||||
report.files_linked += 1;
|
||||
if let Err(e) = importer::ensure_probed(conn, episode_file_id, &final_path) {
|
||||
tracing::warn!(episode_file_id, error = %e, "post-scan probing failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// See the matching comment in `scan_tv_root` — folder renames need an
|
||||
// explicit refresh or Jellyfin's index silently falls out of sync with
|
||||
// disk until its own next scheduled scan.
|
||||
if !report.matched.is_empty() {
|
||||
if let Some(jellyfin) = jellyfin {
|
||||
refresh_jellyfin_after_rename(jellyfin, "movie scan").await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_folder_name_with_year() {
|
||||
assert_eq!(
|
||||
parse_folder_name("Arrival (2016)"),
|
||||
("Arrival".to_string(), Some(2016))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_folder_name_without_year() {
|
||||
assert_eq!(
|
||||
parse_folder_name("Black Adder"),
|
||||
("Black Adder".to_string(), None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_scene_style_dot_separated_tags() {
|
||||
assert_eq!(
|
||||
parse_folder_name("Arrival.2016.1080p.BluRay.x264-GROUP"),
|
||||
("Arrival".to_string(), Some(2016))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_season_and_quality_tags_from_series_folder() {
|
||||
assert_eq!(
|
||||
parse_folder_name("Attack on Titan S01 1080p Dual Audio [Group]"),
|
||||
("Attack on Titan".to_string(), None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_bare_numeric_title_when_no_other_year_found() {
|
||||
assert_eq!(
|
||||
parse_folder_name("1917.1080p.BluRay.x264-GROUP"),
|
||||
("1917".to_string(), None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_mangle_titles_with_real_dots() {
|
||||
assert_eq!(
|
||||
parse_folder_name("Mr. Robot (2015)"),
|
||||
("Mr. Robot".to_string(), Some(2015))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_leading_release_group_bracket_tag_and_movie_number() {
|
||||
assert_eq!(
|
||||
parse_folder_name(
|
||||
"[Judas] Code Geass Movie 01 - Lelouch of the Rebellion - Initiation"
|
||||
),
|
||||
(
|
||||
"Code Geass Lelouch of the Rebellion - Initiation".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
1208
breadarrd/src/main.rs
Normal file
1208
breadarrd/src/main.rs
Normal file
File diff suppressed because it is too large
Load diff
160
breadarrd/src/matcher/embed.rs
Normal file
160
breadarrd/src/matcher/embed.rs
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
use ort::session::builder::GraphOptimizationLevel;
|
||||
use ort::session::Session;
|
||||
use ort::value::Tensor;
|
||||
use tokenizers::Tokenizer;
|
||||
|
||||
/// all-MiniLM-L6-v2's trained max sequence length. Release/show titles are
|
||||
/// always far shorter than this, but truncate defensively rather than let a
|
||||
/// pathological input blow up attention memory.
|
||||
const MAX_SEQ_LEN: usize = 256;
|
||||
|
||||
pub struct OrtEmbedder {
|
||||
session: Session,
|
||||
tokenizer: Tokenizer,
|
||||
dim: usize,
|
||||
}
|
||||
|
||||
impl OrtEmbedder {
|
||||
/// CPU-only — no execution-provider selection. A GPU on a typical media
|
||||
/// server is usually already busy with transcoding, and a 90MB
|
||||
/// MiniLM-class model is cheap enough on CPU that a multi-backend GPU
|
||||
/// setup isn't worth the added complexity for a model this small.
|
||||
pub fn load(model_path: &Path, tokenizer_path: &Path, dim: usize) -> Result<Self> {
|
||||
let session = Session::builder()
|
||||
.map_err(|e| anyhow::anyhow!("failed to create ort session builder: {e}"))?
|
||||
.with_optimization_level(GraphOptimizationLevel::Level3)
|
||||
.map_err(|e| anyhow::anyhow!("failed to set optimization level: {e}"))?
|
||||
.commit_from_file(model_path)
|
||||
.map_err(|e| {
|
||||
anyhow::anyhow!("failed to load model from {}: {e}", model_path.display())
|
||||
})?;
|
||||
|
||||
let tokenizer = Tokenizer::from_file(tokenizer_path)
|
||||
.map_err(|e| anyhow::anyhow!("failed to load tokenizer: {e}"))?;
|
||||
|
||||
Ok(Self {
|
||||
session,
|
||||
tokenizer,
|
||||
dim,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn embed(&mut self, text: &str) -> Result<Vec<f32>> {
|
||||
let encoding = self
|
||||
.tokenizer
|
||||
.encode(text, true)
|
||||
.map_err(|e| anyhow::anyhow!("tokenization failed: {e}"))?;
|
||||
|
||||
let mut ids: Vec<i64> = encoding.get_ids().iter().map(|&x| x as i64).collect();
|
||||
let mut mask: Vec<i64> = encoding
|
||||
.get_attention_mask()
|
||||
.iter()
|
||||
.map(|&x| x as i64)
|
||||
.collect();
|
||||
let mut type_ids: Vec<i64> = encoding.get_type_ids().iter().map(|&x| x as i64).collect();
|
||||
|
||||
ids.truncate(MAX_SEQ_LEN);
|
||||
mask.truncate(MAX_SEQ_LEN);
|
||||
type_ids.truncate(MAX_SEQ_LEN);
|
||||
|
||||
let seq_len = ids.len() as i64;
|
||||
let id_tensor = Tensor::<i64>::from_array((vec![1i64, seq_len], ids))
|
||||
.map_err(|e| anyhow::anyhow!("failed to build input_ids tensor: {e}"))?;
|
||||
let mask_tensor = Tensor::<i64>::from_array((vec![1i64, seq_len], mask.clone()))
|
||||
.map_err(|e| anyhow::anyhow!("failed to build attention_mask tensor: {e}"))?;
|
||||
let type_tensor = Tensor::<i64>::from_array((vec![1i64, seq_len], type_ids))
|
||||
.map_err(|e| anyhow::anyhow!("failed to build token_type_ids tensor: {e}"))?;
|
||||
|
||||
let outputs = self
|
||||
.session
|
||||
.run(ort::inputs! {
|
||||
"input_ids" => id_tensor,
|
||||
"attention_mask" => mask_tensor,
|
||||
"token_type_ids" => type_tensor,
|
||||
})
|
||||
.map_err(|e| anyhow::anyhow!("ort inference failed: {e}"))?;
|
||||
|
||||
let (shape, data) = outputs["last_hidden_state"]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(|e| anyhow::anyhow!("failed to extract last_hidden_state: {e}"))?;
|
||||
|
||||
let actual_seq = shape[1] as usize;
|
||||
let actual_dim = shape[2] as usize;
|
||||
|
||||
// Mean-pool over non-padded positions only.
|
||||
let mut result = vec![0.0f32; actual_dim];
|
||||
let mut count = 0usize;
|
||||
for t in 0..actual_seq.min(mask.len()) {
|
||||
if mask[t] > 0 {
|
||||
for d in 0..actual_dim {
|
||||
result[d] += data[t * actual_dim + d];
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
if count > 0 {
|
||||
for x in &mut result {
|
||||
*x /= count as f32;
|
||||
}
|
||||
}
|
||||
|
||||
l2_normalize(&mut result);
|
||||
result.truncate(self.dim);
|
||||
while result.len() < self.dim {
|
||||
result.push(0.0);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
fn l2_normalize(v: &mut [f32]) {
|
||||
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if norm > 1e-10 {
|
||||
for x in v.iter_mut() {
|
||||
*x /= norm;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
a.iter().zip(b).map(|(x, y)| x * y).sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn l2_normalize_produces_unit_vector() {
|
||||
let mut v = vec![3.0, 4.0];
|
||||
l2_normalize(&mut v);
|
||||
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
assert!((norm - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn l2_normalize_leaves_zero_vector_untouched() {
|
||||
let mut v = vec![0.0, 0.0, 0.0];
|
||||
l2_normalize(&mut v);
|
||||
assert_eq!(v, vec![0.0, 0.0, 0.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_similarity_of_identical_unit_vectors_is_one() {
|
||||
let mut v = vec![1.0, 2.0, 3.0];
|
||||
l2_normalize(&mut v);
|
||||
let sim = cosine_similarity(&v, &v);
|
||||
assert!((sim - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_similarity_of_orthogonal_vectors_is_zero() {
|
||||
let a = vec![1.0, 0.0];
|
||||
let b = vec![0.0, 1.0];
|
||||
assert!(cosine_similarity(&a, &b).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
278
breadarrd/src/matcher/mod.rs
Normal file
278
breadarrd/src/matcher/mod.rs
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
pub mod embed;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rusqlite::{params, Connection};
|
||||
|
||||
use embed::{cosine_similarity, OrtEmbedder};
|
||||
|
||||
const MODEL_URL: &str =
|
||||
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx";
|
||||
const TOKENIZER_URL: &str =
|
||||
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/tokenizer.json";
|
||||
|
||||
/// Downloads the embedding model into `model_dir` if it isn't already
|
||||
/// there — keeps setup to "run the daemon," no separate fetch step, in
|
||||
/// keeping with the project's minimal-setup goal.
|
||||
pub async fn ensure_model(model_dir: &Path) -> Result<(PathBuf, PathBuf)> {
|
||||
std::fs::create_dir_all(model_dir)
|
||||
.with_context(|| format!("failed to create {}", model_dir.display()))?;
|
||||
|
||||
let model_path = model_dir.join("model.onnx");
|
||||
let tokenizer_path = model_dir.join("tokenizer.json");
|
||||
|
||||
if !model_path.exists() {
|
||||
tracing::info!("downloading title-matching model (~90MB, one-time)");
|
||||
download(MODEL_URL, &model_path).await?;
|
||||
}
|
||||
if !tokenizer_path.exists() {
|
||||
download(TOKENIZER_URL, &tokenizer_path).await?;
|
||||
}
|
||||
|
||||
Ok((model_path, tokenizer_path))
|
||||
}
|
||||
|
||||
async fn download(url: &str, dest: &Path) -> Result<()> {
|
||||
let bytes = reqwest::get(url)
|
||||
.await
|
||||
.with_context(|| format!("failed to download {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("{url} returned an error status"))?
|
||||
.bytes()
|
||||
.await
|
||||
.with_context(|| format!("failed to read response body from {url}"))?;
|
||||
let tmp = dest.with_extension("part");
|
||||
std::fs::write(&tmp, &bytes).with_context(|| format!("failed to write {}", tmp.display()))?;
|
||||
std::fs::rename(&tmp, dest)
|
||||
.with_context(|| format!("failed to finalize {}", dest.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const EMBEDDING_DIM: usize = 384;
|
||||
|
||||
/// Below this cosine similarity, no match is proposed at all.
|
||||
pub(crate) const MIN_CONFIDENCE: f32 = 0.55;
|
||||
/// At or above this, auto-match without queuing for manual review — but see
|
||||
/// `token_overlap_ratio` below, which gates this further.
|
||||
const AUTO_MATCH_CONFIDENCE: f32 = 0.85;
|
||||
/// Auto-match additionally requires at least this much literal word overlap
|
||||
/// between the query and the matched title/alias. all-MiniLM-L6-v2 is
|
||||
/// trained on English semantic similarity; on romanized Japanese anime
|
||||
/// titles it doesn't actually discriminate between shows — it clusters
|
||||
/// *any* two romaji titles close together (shared particles/phonetics read
|
||||
/// as "similar foreign text"), producing confidently wrong matches between
|
||||
/// completely unrelated series (verified live: multiple unrelated
|
||||
/// currently-airing anime auto-matched to "Demon Slayer" and "Mushoku
|
||||
/// Tensei" at >0.85 confidence with zero actual relation). Requiring some
|
||||
/// real word overlap catches exactly this failure mode — a genuine match
|
||||
/// (including a registered alias) shares at least one real token; a
|
||||
/// same-cluster false positive usually shares none.
|
||||
const MIN_TOKEN_OVERLAP: f32 = 0.2;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct MatchCandidate {
|
||||
pub media_item_id: i64,
|
||||
pub matched_text: String,
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum MatchOutcome {
|
||||
Auto(MatchCandidate),
|
||||
NeedsReview(MatchCandidate),
|
||||
NoMatch,
|
||||
}
|
||||
|
||||
pub struct TitleMatcher {
|
||||
embedder: OrtEmbedder,
|
||||
cache: HashMap<String, Vec<f32>>,
|
||||
}
|
||||
|
||||
impl TitleMatcher {
|
||||
pub fn load(model_path: &Path, tokenizer_path: &Path) -> Result<Self> {
|
||||
Ok(Self {
|
||||
embedder: OrtEmbedder::load(model_path, tokenizer_path, EMBEDDING_DIM)?,
|
||||
cache: HashMap::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn embed_cached(&mut self, text: &str) -> Result<Vec<f32>> {
|
||||
if let Some(v) = self.cache.get(text) {
|
||||
return Ok(v.clone());
|
||||
}
|
||||
let v = self.embedder.embed(text)?;
|
||||
self.cache.insert(text.to_string(), v.clone());
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Given a flat list of candidate texts (e.g. every search result's
|
||||
/// name plus its aliases, flattened with an index back to which result
|
||||
/// each one belongs to), returns the index of whichever candidate text
|
||||
/// best matches `query` and its similarity score. Generic version of
|
||||
/// the logic `match_title` uses against the DB — also used to
|
||||
/// disambiguate metadata-provider search results, where the top result
|
||||
/// isn't always the right entry (TVDB's own relevance ranking can
|
||||
/// place a spinoff/short above the main series, e.g. searching "Attack
|
||||
/// on Titan" ranks "Attack on Titan: Counter Rockets" first even
|
||||
/// though the real series' own alias list has "Attack on Titan
|
||||
/// (2013)", which should score far closer to the query).
|
||||
pub fn best_match_index(
|
||||
&mut self,
|
||||
query: &str,
|
||||
candidates: &[(usize, String)],
|
||||
) -> Result<Option<(usize, f32)>> {
|
||||
let query_emb = self.embed_cached(query)?;
|
||||
let mut best: Option<(usize, f32)> = None;
|
||||
for (owner_index, text) in candidates {
|
||||
let emb = self.embed_cached(text)?;
|
||||
let score = cosine_similarity(&query_emb, &emb);
|
||||
if best.as_ref().is_none_or(|(_, s)| score > *s) {
|
||||
best = Some((*owner_index, score));
|
||||
}
|
||||
}
|
||||
Ok(best)
|
||||
}
|
||||
|
||||
/// Matches `query` against every monitored media_item's title + known
|
||||
/// aliases, returning the single best match and whether it clears the
|
||||
/// auto-match bar or needs a human to confirm it in the review queue.
|
||||
pub fn match_title(&mut self, conn: &Connection, query: &str) -> Result<MatchOutcome> {
|
||||
let query_emb = self.embed_cached(query)?;
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, title FROM media_item WHERE monitored = 1
|
||||
UNION ALL
|
||||
SELECT a.media_item_id, a.text FROM alias a
|
||||
JOIN media_item m ON m.id = a.media_item_id WHERE m.monitored = 1",
|
||||
)?;
|
||||
let candidates: Vec<(i64, String)> = stmt
|
||||
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
|
||||
let mut best: Option<MatchCandidate> = None;
|
||||
for (media_item_id, text) in candidates {
|
||||
let emb = self.embed_cached(&text)?;
|
||||
let confidence = cosine_similarity(&query_emb, &emb);
|
||||
if confidence < MIN_CONFIDENCE {
|
||||
continue;
|
||||
}
|
||||
if best.as_ref().is_none_or(|b| confidence > b.confidence) {
|
||||
best = Some(MatchCandidate {
|
||||
media_item_id,
|
||||
matched_text: text,
|
||||
confidence,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(match best {
|
||||
None => MatchOutcome::NoMatch,
|
||||
// The embedding-clustering failure mode this guards against
|
||||
// (see this struct's field docs / `MIN_TOKEN_OVERLAP`) lands
|
||||
// its false positives anywhere in [MIN_CONFIDENCE,
|
||||
// AUTO_MATCH_CONFIDENCE) just as often as above it — a
|
||||
// completely unrelated romaji title scores confidently
|
||||
// *enough* to pass MIN_CONFIDENCE, just not confidently enough
|
||||
// to auto-match. Gating only the auto-match branch left every
|
||||
// one of those false positives landing in the review queue
|
||||
// instead, silently flooding it (verified live: dozens of
|
||||
// zero-overlap titles queued against a handful of "attractor"
|
||||
// shows). Zero real word overlap means "not a candidate at
|
||||
// all," not "uncertain, ask a human."
|
||||
Some(c) if token_overlap_ratio(query, &c.matched_text) < MIN_TOKEN_OVERLAP => {
|
||||
MatchOutcome::NoMatch
|
||||
}
|
||||
Some(c) if c.confidence >= AUTO_MATCH_CONFIDENCE => MatchOutcome::Auto(c),
|
||||
Some(c) => MatchOutcome::NeedsReview(c),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Word-level overlap between `a` and `b`, as a fraction of the *shorter*
|
||||
/// title's word count — so a short alias fully contained in a longer title
|
||||
/// (or vice versa) still scores 1.0, rather than being penalized by length
|
||||
/// mismatch. Case-insensitive, splits on non-alphanumeric runs.
|
||||
fn token_overlap_ratio(a: &str, b: &str) -> f32 {
|
||||
let tokenize = |s: &str| -> std::collections::HashSet<String> {
|
||||
s.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|t| !t.is_empty())
|
||||
.map(|t| t.to_lowercase())
|
||||
.collect()
|
||||
};
|
||||
let ta = tokenize(a);
|
||||
let tb = tokenize(b);
|
||||
let shorter = ta.len().min(tb.len());
|
||||
if shorter == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let overlap = ta.intersection(&tb).count();
|
||||
overlap as f32 / shorter as f32
|
||||
}
|
||||
|
||||
/// `link`/`source_id` are stored (not just the title) so an approval can
|
||||
/// later actually complete the grab, rather than just recording a decision.
|
||||
/// Both `None` when there's no real source item behind the query (e.g. an
|
||||
/// ad hoc matcher test) — such an entry can be reviewed but not approved.
|
||||
pub fn queue_for_review(
|
||||
conn: &Connection,
|
||||
raw_release_title: &str,
|
||||
candidate: &MatchCandidate,
|
||||
link: Option<&str>,
|
||||
source_id: Option<i64>,
|
||||
) -> Result<i64> {
|
||||
conn.execute(
|
||||
"INSERT INTO review_queue (raw_release_title, candidate_media_item_id, confidence, link, source_id, status, created_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, 'pending', datetime('now'))",
|
||||
params![raw_release_title, candidate.media_item_id, candidate.confidence, link, source_id],
|
||||
)?;
|
||||
crate::db::record_event(
|
||||
conn,
|
||||
candidate.media_item_id,
|
||||
None,
|
||||
"review_queued",
|
||||
&format!(
|
||||
"confidence={:.2} title={raw_release_title:?}",
|
||||
candidate.confidence
|
||||
),
|
||||
)?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn identical_titles_fully_overlap() {
|
||||
assert_eq!(token_overlap_ratio("Mushoku Tensei", "Mushoku Tensei"), 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_alias_contained_in_longer_title_fully_overlaps() {
|
||||
// The alias is fully covered by the longer title's words, so this
|
||||
// should score 1.0 despite the length difference — an alias like
|
||||
// "Mushoku Tensei" shouldn't be penalized against the full title
|
||||
// "Mushoku Tensei: Jobless Reincarnation".
|
||||
let ratio = token_overlap_ratio("Mushoku Tensei", "Mushoku Tensei Jobless Reincarnation");
|
||||
assert!((ratio - 1.0).abs() < 1e-6, "ratio was {ratio}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_titles_have_no_overlap() {
|
||||
// The real failure case this guards against: two unrelated
|
||||
// romanized Japanese anime titles that an English-trained embedding
|
||||
// model scores as deceptively similar.
|
||||
let ratio = token_overlap_ratio(
|
||||
"Sekai Saikyou no Kouei",
|
||||
"Kimi wo Aisuru Ki wa Nai to Itta Jiki Koushaku-sama ga Nazeka Dekiai Shitekimasu",
|
||||
);
|
||||
assert_eq!(ratio, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_has_zero_overlap() {
|
||||
assert_eq!(token_overlap_ratio("", "Mushoku Tensei"), 0.0);
|
||||
}
|
||||
}
|
||||
200
breadarrd/src/metadata/anime_map.rs
Normal file
200
breadarrd/src/metadata/anime_map.rs
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
use anyhow::{Context, Result};
|
||||
use rusqlite::{params, Connection};
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Fribb/anime-lists cross-references AniDB anime entries (roughly one per
|
||||
/// TV season/cour) to TVDB/TMDB ids and season numbers — this is what
|
||||
/// Sonarr itself relies on for anime scene-numbering, and unlike
|
||||
/// manami-project/anime-offline-database it actually carries TVDB/TMDB ids
|
||||
/// (verified: the offline-database has none at all).
|
||||
const SOURCE_URL: &str =
|
||||
"https://raw.githubusercontent.com/Fribb/anime-lists/master/anime-list-full.json";
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Entry {
|
||||
#[serde(default)]
|
||||
anidb_id: Option<i64>,
|
||||
#[serde(default)]
|
||||
tvdb_id: Option<i64>,
|
||||
#[serde(default)]
|
||||
themoviedb_id: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
season: Option<SeasonField>,
|
||||
#[serde(default)]
|
||||
episode_offset: Option<OffsetField>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SeasonField {
|
||||
tvdb: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct OffsetField {
|
||||
tvdb: Option<i64>,
|
||||
}
|
||||
|
||||
/// Downloads the current Fribb/anime-lists dataset and upserts it into
|
||||
/// `anime_mapping` (TV) and `anime_tmdb_movie` (movies). Returns the number
|
||||
/// of `anime_mapping` entries processed (movie ids aren't counted the same
|
||||
/// way — one entry can contribute several).
|
||||
pub async fn refresh(conn: &mut Connection, client: &reqwest::Client) -> Result<usize> {
|
||||
let bytes = client
|
||||
.get(SOURCE_URL)
|
||||
.send()
|
||||
.await
|
||||
.context("anime-lists download failed")?
|
||||
.error_for_status()
|
||||
.context("anime-lists download returned an error status")?
|
||||
.bytes()
|
||||
.await
|
||||
.context("anime-lists download body read failed")?;
|
||||
|
||||
let entries: Vec<Entry> =
|
||||
serde_json::from_slice(&bytes).context("anime-lists response was not valid JSON")?;
|
||||
|
||||
let tx = conn.transaction()?;
|
||||
let mut written = 0;
|
||||
for e in &entries {
|
||||
// `themoviedb_id` carries either a `tv` id (a single number) or a
|
||||
// `movie` id list (rereleases/split cuts can give one AniDB entry
|
||||
// several TMDB movie ids) — never both in practice, but nothing
|
||||
// guarantees that, so both are checked independently rather than
|
||||
// assuming one implies the absence of the other.
|
||||
let tv_tmdb_id = e
|
||||
.themoviedb_id
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("tv"))
|
||||
.and_then(|v| v.as_i64());
|
||||
for movie_id in e
|
||||
.themoviedb_id
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("movie"))
|
||||
.and_then(|v| v.as_array())
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|v| v.as_i64())
|
||||
{
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO anime_tmdb_movie (tmdb_id) VALUES (?1)",
|
||||
params![movie_id],
|
||||
)?;
|
||||
}
|
||||
|
||||
// ~63% of entries have no AniDB cross-reference at all (AniList/MAL/Kitsu-only
|
||||
// listings) — irrelevant to a table keyed on anidb_id, skip them.
|
||||
let Some(anidb_id) = e.anidb_id else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let season_number = e.season.as_ref().and_then(|s| s.tvdb);
|
||||
let episode_offset = e.episode_offset.as_ref().and_then(|o| o.tvdb).unwrap_or(0);
|
||||
|
||||
tx.execute(
|
||||
"INSERT INTO anime_mapping (anidb_id, tvdb_id, tmdb_id, season_offset, episode_offset)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(anidb_id) DO UPDATE SET
|
||||
tvdb_id = excluded.tvdb_id,
|
||||
tmdb_id = excluded.tmdb_id,
|
||||
season_offset = excluded.season_offset,
|
||||
episode_offset = excluded.episode_offset",
|
||||
params![
|
||||
anidb_id,
|
||||
e.tvdb_id,
|
||||
tv_tmdb_id,
|
||||
season_number,
|
||||
episode_offset
|
||||
],
|
||||
)?;
|
||||
written += 1;
|
||||
}
|
||||
tx.commit()?;
|
||||
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Resolves an absolute anime episode number to a (season, episode) pair
|
||||
/// for a known TVDB series.
|
||||
///
|
||||
/// One TVDB season is often stitched together from several AniDB cours,
|
||||
/// each its own `anime_mapping` row with its own `episode_offset` — e.g.
|
||||
/// tvdb_id 366263 (Ascendance of a Bookworm) has cours starting at absolute
|
||||
/// episodes 1, 15, 27, 37 (offsets 0, 14, 26, 36). The matching cour is the
|
||||
/// one with the largest offset that's still below the target absolute
|
||||
/// episode (offset = episode count preceding that cour, so
|
||||
/// `local_episode = absolute - offset`).
|
||||
pub fn resolve_absolute_episode(
|
||||
conn: &Connection,
|
||||
tvdb_id: i64,
|
||||
absolute_episode: u32,
|
||||
) -> Result<Option<(u32, u32)>> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT season_offset, episode_offset FROM anime_mapping
|
||||
WHERE tvdb_id = ?1 AND season_offset IS NOT NULL",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map(params![tvdb_id], |row| {
|
||||
Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
let absolute = absolute_episode as i64;
|
||||
let best = rows
|
||||
.into_iter()
|
||||
.filter(|&(_, offset)| offset < absolute)
|
||||
.max_by_key(|&(_, offset)| offset);
|
||||
|
||||
Ok(best.map(|(season, offset)| (season as u32, (absolute - offset) as u32)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn seeded_conn() -> Connection {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE anime_mapping (
|
||||
anidb_id INTEGER PRIMARY KEY, tvdb_id INTEGER, tmdb_id INTEGER,
|
||||
season_offset INTEGER, episode_offset INTEGER NOT NULL DEFAULT 0
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
// Mirrors real data observed for tvdb_id 366263: one TVDB season
|
||||
// stitched from four AniDB cours starting at absolute eps 1/15/27/37.
|
||||
for (anidb_id, offset) in [(1, 0), (2, 14), (3, 26), (4, 36)] {
|
||||
conn.execute(
|
||||
"INSERT INTO anime_mapping (anidb_id, tvdb_id, season_offset, episode_offset)
|
||||
VALUES (?1, 366263, 1, ?2)",
|
||||
params![anidb_id, offset],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_first_cour() {
|
||||
let conn = seeded_conn();
|
||||
assert_eq!(
|
||||
resolve_absolute_episode(&conn, 366263, 5).unwrap(),
|
||||
Some((1, 5))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_later_cour_using_its_offset() {
|
||||
let conn = seeded_conn();
|
||||
// absolute ep 20 falls in the cour starting at 15 (offset 14)
|
||||
assert_eq!(
|
||||
resolve_absolute_episode(&conn, 366263, 20).unwrap(),
|
||||
Some((1, 6))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_none_for_unmapped_series() {
|
||||
let conn = seeded_conn();
|
||||
assert_eq!(resolve_absolute_episode(&conn, 999999, 1).unwrap(), None);
|
||||
}
|
||||
}
|
||||
148
breadarrd/src/metadata/mod.rs
Normal file
148
breadarrd/src/metadata/mod.rs
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
pub mod anime_map;
|
||||
pub mod tmdb;
|
||||
pub mod tvdb;
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use rusqlite::{params, Connection};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SeriesSearchResult {
|
||||
pub external_id: String,
|
||||
pub name: String,
|
||||
pub year: Option<u32>,
|
||||
pub aliases: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct MovieSearchResult {
|
||||
pub external_id: String,
|
||||
pub title: String,
|
||||
pub year: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct EpisodeInfo {
|
||||
pub season_number: u32,
|
||||
pub episode_number: u32,
|
||||
pub absolute_number: Option<u32>,
|
||||
pub title: Option<String>,
|
||||
pub air_date: Option<String>,
|
||||
}
|
||||
|
||||
/// Inserts a series and its full episode list into the library, using a
|
||||
/// TVDB series id as the source of episode data. Returns the new
|
||||
/// `media_item.id`.
|
||||
///
|
||||
/// Split into a fetch step and a write step (see [`insert_series`]) rather
|
||||
/// than one function that both awaits and holds `conn`, because a caller
|
||||
/// using `Mutex<Connection>` (e.g. an axum handler) can't hold a sync
|
||||
/// `MutexGuard` across an `.await` point — this convenience wrapper is only
|
||||
/// safe for callers with an owned, unshared `Connection` (e.g. the debug
|
||||
/// CLI commands).
|
||||
pub async fn add_series(
|
||||
conn: &Connection,
|
||||
tvdb: &tvdb::TvdbClient,
|
||||
tvdb_series_id: &str,
|
||||
title: &str,
|
||||
year: Option<u32>,
|
||||
aliases: &[String],
|
||||
root_folder: &str,
|
||||
quality_profile_id: i64,
|
||||
) -> Result<i64> {
|
||||
let episodes = tvdb.episodes(tvdb_series_id).await?;
|
||||
insert_series(
|
||||
conn,
|
||||
tvdb_series_id,
|
||||
title,
|
||||
year,
|
||||
aliases,
|
||||
root_folder,
|
||||
quality_profile_id,
|
||||
&episodes,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn insert_series(
|
||||
conn: &Connection,
|
||||
tvdb_series_id: &str,
|
||||
title: &str,
|
||||
year: Option<u32>,
|
||||
aliases: &[String],
|
||||
root_folder: &str,
|
||||
quality_profile_id: i64,
|
||||
episodes: &[EpisodeInfo],
|
||||
) -> Result<i64> {
|
||||
conn.execute(
|
||||
"INSERT INTO media_item (kind, title, year, tvdb_id, monitored, quality_profile_id, root_folder)
|
||||
VALUES ('series', ?1, ?2, ?3, 1, ?4, ?5)",
|
||||
params![
|
||||
title,
|
||||
year,
|
||||
tvdb_series_id.parse::<i64>().ok(),
|
||||
quality_profile_id,
|
||||
root_folder
|
||||
],
|
||||
)?;
|
||||
let media_item_id = conn.last_insert_rowid();
|
||||
|
||||
for alias in aliases {
|
||||
conn.execute(
|
||||
"INSERT INTO alias (media_item_id, text, source) VALUES (?1, ?2, 'tvdb')",
|
||||
params![media_item_id, alias],
|
||||
)?;
|
||||
}
|
||||
|
||||
let mut seasons_seen = HashSet::new();
|
||||
for ep in episodes {
|
||||
if seasons_seen.insert(ep.season_number) {
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO season (media_item_id, season_number, monitored) VALUES (?1, ?2, 1)",
|
||||
params![media_item_id, ep.season_number],
|
||||
)?;
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO episode
|
||||
(media_item_id, season_number, episode_number, absolute_number, title, air_date, monitored, has_file)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, 0)",
|
||||
params![
|
||||
media_item_id,
|
||||
ep.season_number,
|
||||
ep.episode_number,
|
||||
ep.absolute_number,
|
||||
ep.title,
|
||||
ep.air_date
|
||||
],
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(media_item_id)
|
||||
}
|
||||
|
||||
/// Inserts a movie into the library. No fetch step needed (unlike
|
||||
/// `add_series`/`insert_series`) — TMDB's movie search result already
|
||||
/// carries everything a movie needs (title, year); there's no separate
|
||||
/// episode-list call the way a series has.
|
||||
pub fn insert_movie(
|
||||
conn: &Connection,
|
||||
tmdb_movie_id: &str,
|
||||
title: &str,
|
||||
year: Option<u32>,
|
||||
root_folder: &str,
|
||||
quality_profile_id: i64,
|
||||
) -> Result<i64> {
|
||||
conn.execute(
|
||||
"INSERT INTO media_item (kind, title, year, tmdb_id, monitored, quality_profile_id, root_folder)
|
||||
VALUES ('movie', ?1, ?2, ?3, 1, ?4, ?5)",
|
||||
params![
|
||||
title,
|
||||
year,
|
||||
tmdb_movie_id.parse::<i64>().ok(),
|
||||
quality_profile_id,
|
||||
root_folder
|
||||
],
|
||||
)?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
145
breadarrd/src/metadata/tmdb.rs
Normal file
145
breadarrd/src/metadata/tmdb.rs
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{EpisodeInfo, MovieSearchResult, SeriesSearchResult};
|
||||
|
||||
pub struct TmdbClient {
|
||||
bearer_token: String,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl TmdbClient {
|
||||
pub fn new(bearer_token: impl Into<String>) -> Self {
|
||||
Self {
|
||||
bearer_token: bearer_token.into(),
|
||||
// No total timeout is reqwest's default — the background loop
|
||||
// holds the DB mutex across calls into this client, so a
|
||||
// stalled connection would hang the whole daemon.
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn search_tv(&self, query: &str) -> Result<Vec<SeriesSearchResult>> {
|
||||
#[derive(Deserialize)]
|
||||
struct SearchResponse {
|
||||
results: Vec<TvItem>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct TvItem {
|
||||
id: u64,
|
||||
name: String,
|
||||
first_air_date: Option<String>,
|
||||
}
|
||||
|
||||
let resp: SearchResponse = self
|
||||
.client
|
||||
.get("https://api.themoviedb.org/3/search/tv")
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.query(&[("query", query)])
|
||||
.send()
|
||||
.await
|
||||
.context("tmdb tv search request failed")?
|
||||
.error_for_status()
|
||||
.context("tmdb tv search returned an error status")?
|
||||
.json()
|
||||
.await
|
||||
.context("tmdb tv search response was not valid JSON")?;
|
||||
|
||||
Ok(resp
|
||||
.results
|
||||
.into_iter()
|
||||
.map(|item| SeriesSearchResult {
|
||||
external_id: item.id.to_string(),
|
||||
name: item.name,
|
||||
year: year_from_date(item.first_air_date.as_deref()),
|
||||
aliases: Vec::new(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn search_movie(&self, query: &str) -> Result<Vec<MovieSearchResult>> {
|
||||
#[derive(Deserialize)]
|
||||
struct SearchResponse {
|
||||
results: Vec<MovieItem>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct MovieItem {
|
||||
id: u64,
|
||||
title: String,
|
||||
release_date: Option<String>,
|
||||
}
|
||||
|
||||
let resp: SearchResponse = self
|
||||
.client
|
||||
.get("https://api.themoviedb.org/3/search/movie")
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.query(&[("query", query)])
|
||||
.send()
|
||||
.await
|
||||
.context("tmdb movie search request failed")?
|
||||
.error_for_status()
|
||||
.context("tmdb movie search returned an error status")?
|
||||
.json()
|
||||
.await
|
||||
.context("tmdb movie search response was not valid JSON")?;
|
||||
|
||||
Ok(resp
|
||||
.results
|
||||
.into_iter()
|
||||
.map(|item| MovieSearchResult {
|
||||
external_id: item.id.to_string(),
|
||||
title: item.title,
|
||||
year: year_from_date(item.release_date.as_deref()),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn tv_season_episodes(&self, tv_id: u64, season: u32) -> Result<Vec<EpisodeInfo>> {
|
||||
#[derive(Deserialize)]
|
||||
struct SeasonResponse {
|
||||
#[serde(default)]
|
||||
episodes: Vec<EpisodeItem>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct EpisodeItem {
|
||||
season_number: u32,
|
||||
episode_number: u32,
|
||||
name: Option<String>,
|
||||
air_date: Option<String>,
|
||||
}
|
||||
|
||||
let resp: SeasonResponse = self
|
||||
.client
|
||||
.get(format!(
|
||||
"https://api.themoviedb.org/3/tv/{tv_id}/season/{season}"
|
||||
))
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.send()
|
||||
.await
|
||||
.context("tmdb season request failed")?
|
||||
.error_for_status()
|
||||
.context("tmdb season returned an error status")?
|
||||
.json()
|
||||
.await
|
||||
.context("tmdb season response was not valid JSON")?;
|
||||
|
||||
Ok(resp
|
||||
.episodes
|
||||
.into_iter()
|
||||
.map(|e| EpisodeInfo {
|
||||
season_number: e.season_number,
|
||||
episode_number: e.episode_number,
|
||||
absolute_number: None,
|
||||
title: e.name,
|
||||
air_date: e.air_date,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
fn year_from_date(date: Option<&str>) -> Option<u32> {
|
||||
date.and_then(|d| d.get(0..4)).and_then(|y| y.parse().ok())
|
||||
}
|
||||
162
breadarrd/src/metadata/tvdb.rs
Normal file
162
breadarrd/src/metadata/tvdb.rs
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{EpisodeInfo, SeriesSearchResult};
|
||||
|
||||
/// TVDB v4 JWTs are valid for roughly a month; refresh well before that so
|
||||
/// clock skew or early invalidation on their end doesn't strand us.
|
||||
const TOKEN_TTL: Duration = Duration::from_secs(20 * 60 * 60);
|
||||
|
||||
pub struct TvdbClient {
|
||||
api_key: String,
|
||||
client: reqwest::Client,
|
||||
token: Mutex<Option<(String, Instant)>>,
|
||||
}
|
||||
|
||||
impl TvdbClient {
|
||||
pub fn new(api_key: impl Into<String>) -> Self {
|
||||
Self {
|
||||
api_key: api_key.into(),
|
||||
// No total timeout is reqwest's default — the background loop
|
||||
// holds the DB mutex across calls into this client, so a
|
||||
// stalled connection would hang the whole daemon.
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
token: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn token(&self) -> Result<String> {
|
||||
let cached = self.token.lock().unwrap().clone();
|
||||
if let Some((token, obtained_at)) = cached {
|
||||
if obtained_at.elapsed() < TOKEN_TTL {
|
||||
return Ok(token);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LoginResponse {
|
||||
data: LoginData,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LoginData {
|
||||
token: String,
|
||||
}
|
||||
|
||||
let resp: LoginResponse = self
|
||||
.client
|
||||
.post("https://api4.thetvdb.com/v4/login")
|
||||
.json(&serde_json::json!({ "apikey": self.api_key }))
|
||||
.send()
|
||||
.await
|
||||
.context("tvdb login request failed")?
|
||||
.error_for_status()
|
||||
.context("tvdb login returned an error status")?
|
||||
.json()
|
||||
.await
|
||||
.context("tvdb login response was not valid JSON")?;
|
||||
|
||||
*self.token.lock().unwrap() = Some((resp.data.token.clone(), Instant::now()));
|
||||
Ok(resp.data.token)
|
||||
}
|
||||
|
||||
pub async fn search_series(&self, query: &str) -> Result<Vec<SeriesSearchResult>> {
|
||||
let token = self.token().await?;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SearchResponse {
|
||||
data: Vec<SearchItem>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct SearchItem {
|
||||
tvdb_id: Option<String>,
|
||||
name: Option<String>,
|
||||
year: Option<String>,
|
||||
#[serde(default)]
|
||||
aliases: Vec<String>,
|
||||
}
|
||||
|
||||
let resp: SearchResponse = self
|
||||
.client
|
||||
.get("https://api4.thetvdb.com/v4/search")
|
||||
.bearer_auth(token)
|
||||
.query(&[("query", query), ("type", "series")])
|
||||
.send()
|
||||
.await
|
||||
.context("tvdb search request failed")?
|
||||
.error_for_status()
|
||||
.context("tvdb search returned an error status")?
|
||||
.json()
|
||||
.await
|
||||
.context("tvdb search response was not valid JSON")?;
|
||||
|
||||
Ok(resp
|
||||
.data
|
||||
.into_iter()
|
||||
.filter_map(|item| {
|
||||
Some(SeriesSearchResult {
|
||||
external_id: item.tvdb_id?,
|
||||
name: item.name?,
|
||||
year: item.year.and_then(|y| y.parse().ok()),
|
||||
aliases: item.aliases,
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn episodes(&self, series_id: &str) -> Result<Vec<EpisodeInfo>> {
|
||||
let token = self.token().await?;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EpisodesResponse {
|
||||
data: EpisodesData,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct EpisodesData {
|
||||
episodes: Vec<EpisodeItem>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct EpisodeItem {
|
||||
#[serde(rename = "seasonNumber")]
|
||||
season_number: u32,
|
||||
number: u32,
|
||||
#[serde(rename = "absoluteNumber")]
|
||||
absolute_number: Option<u32>,
|
||||
name: Option<String>,
|
||||
aired: Option<String>,
|
||||
}
|
||||
|
||||
let resp: EpisodesResponse = self
|
||||
.client
|
||||
.get(format!(
|
||||
"https://api4.thetvdb.com/v4/series/{series_id}/episodes/default"
|
||||
))
|
||||
.bearer_auth(token)
|
||||
.send()
|
||||
.await
|
||||
.context("tvdb episodes request failed")?
|
||||
.error_for_status()
|
||||
.context("tvdb episodes returned an error status")?
|
||||
.json()
|
||||
.await
|
||||
.context("tvdb episodes response was not valid JSON")?;
|
||||
|
||||
Ok(resp
|
||||
.data
|
||||
.episodes
|
||||
.into_iter()
|
||||
.map(|e| EpisodeInfo {
|
||||
season_number: e.season_number,
|
||||
episode_number: e.number,
|
||||
absolute_number: e.absolute_number.filter(|&n| n != 0),
|
||||
title: e.name,
|
||||
air_date: e.aired,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
70
breadarrd/src/notify.rs
Normal file
70
breadarrd/src/notify.rs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
use anyhow::Result;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Payload<'a> {
|
||||
title: &'a str,
|
||||
message: &'a str,
|
||||
}
|
||||
|
||||
/// Best-effort push notification — failures are logged, never propagated.
|
||||
/// The events this fires for (a review-queue item, a run of import
|
||||
/// failures, the search loop halting) already have a durable home in
|
||||
/// `event_history`/the TUI; the notification is a convenience nudge on top
|
||||
/// of that, not the record of truth, so it should never be able to fail an
|
||||
/// otherwise-successful grab/import/search cycle.
|
||||
pub struct Notifier {
|
||||
client: reqwest::Client,
|
||||
webhook_url: String,
|
||||
}
|
||||
|
||||
impl Notifier {
|
||||
/// Returns `None` when `webhook_url` is empty — callers hold an
|
||||
/// `Option<Notifier>` and simply skip notifying rather than every call
|
||||
/// site needing its own empty-string check.
|
||||
pub fn new(webhook_url: &str) -> Option<Self> {
|
||||
if webhook_url.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
webhook_url: webhook_url.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn send(&self, title: &str, message: &str) {
|
||||
let result = self
|
||||
.client
|
||||
.post(&self.webhook_url)
|
||||
.json(&Payload { title, message })
|
||||
.send()
|
||||
.await;
|
||||
match result {
|
||||
Ok(resp) if !resp.status().is_success() => {
|
||||
tracing::warn!(status = %resp.status(), "notification webhook returned an error status");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "notification webhook request failed");
|
||||
}
|
||||
Ok(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn new_returns_none_for_an_empty_url() {
|
||||
assert!(Notifier::new("").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_returns_some_for_a_configured_url() {
|
||||
assert!(Notifier::new("http://localhost:5600/message?token=x").is_some());
|
||||
}
|
||||
}
|
||||
291
breadarrd/src/parser/mod.rs
Normal file
291
breadarrd/src/parser/mod.rs
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
mod tokens;
|
||||
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Source {
|
||||
Hdtv,
|
||||
WebRip,
|
||||
WebDl,
|
||||
BluRay,
|
||||
Remux,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Codec {
|
||||
H264,
|
||||
Hevc,
|
||||
Av1,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct ParsedRelease {
|
||||
pub title_raw: String,
|
||||
pub title_normalized: String,
|
||||
pub group: Option<String>,
|
||||
pub season: Option<u32>,
|
||||
pub episode: Option<u32>,
|
||||
pub absolute_episode: Option<u32>,
|
||||
pub year: Option<u32>,
|
||||
pub resolution: Option<u32>,
|
||||
pub source: Option<Source>,
|
||||
pub codec: Option<Codec>,
|
||||
pub bit_depth: Option<u8>,
|
||||
pub container: Option<String>,
|
||||
pub is_repack: bool,
|
||||
}
|
||||
|
||||
pub fn parse(raw_title: &str) -> ParsedRelease {
|
||||
let mut work = raw_title.to_string();
|
||||
|
||||
let group = tokens::extract_group(&work);
|
||||
if group.is_some() {
|
||||
work = tokens::GROUP_PREFIX_RE
|
||||
.replace(&work, "")
|
||||
.trim()
|
||||
.to_string();
|
||||
}
|
||||
|
||||
let container = tokens::extract_container(&work);
|
||||
let resolution = tokens::extract_resolution(&work);
|
||||
let source = tokens::extract_source(&work);
|
||||
let codec = tokens::extract_codec(&work);
|
||||
let bit_depth = tokens::extract_bit_depth(&work);
|
||||
let is_repack = tokens::REPACK_RE.is_match(&work);
|
||||
let year = tokens::extract_year(&work);
|
||||
let (season, episode, absolute_episode, title_span_end) = tokens::extract_episode_info(&work);
|
||||
|
||||
let title_normalized = tokens::derive_title(&work, title_span_end);
|
||||
|
||||
ParsedRelease {
|
||||
title_raw: raw_title.to_string(),
|
||||
title_normalized,
|
||||
group,
|
||||
season,
|
||||
episode,
|
||||
absolute_episode,
|
||||
year,
|
||||
resolution,
|
||||
source,
|
||||
codec,
|
||||
bit_depth,
|
||||
container,
|
||||
is_repack,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) static WS_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\s+").unwrap());
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const CORPUS: &[&str] = &[
|
||||
"Aime ton prochain (01-06) (Chida) (2019) [Digital-1920] [Manga FR] (PapriKa+)",
|
||||
"[ANi] 小書痴的下剋上 為了成為圖書管理員不擇手段!領主的養女 - 13 [1080P][Baha][WEB-DL][AAC AVC][CHT][MP4]",
|
||||
"Ascendance of a Bookworm S04E11 The Gathering of Gutenberg 1080p CR WEB-DL DUAL AAC2.0 H 264-VARYG (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Dual-Audio, Multi-Subs)",
|
||||
"Ascendance of a Bookworm S04E12 The Winter Social Season and Debut 1080p CR WEB-DL DUAL AAC2.0 H 264-VARYG (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Dual-Audio, Multi-Subs)",
|
||||
"Ascendance of a Bookworm S04E13 VOSTFR 1080p WEB x264 AAC -Tsundere-Raws (CR) (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen - Ryoushu no Youjo,Ascendance of a Bookworm: Adopted Daughter of an Archduke)",
|
||||
"Ascendance of a Bookworm S04E13 VOSTFR 720p WEB x264 AAC -Tsundere-Raws (CR) (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen - Ryoushu no Youjo,Ascendance of a Bookworm: Adopted Daughter of an Archduke)",
|
||||
"Ascendance of a Bookworm S04E13 Winter Material Gathering 1080p CR WEB-DL AAC2.0 H 264-VARYG (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Multi-Subs)",
|
||||
"Assassin's Creed - Blade of Shao Jun (01-04) (Kurata) (2020) [Digital-1920] [Manga FR] (PapriKa+)",
|
||||
"[Erai-raws] Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen - Ryushu no Youjo - 13 [1080p CR WEB-DL AVC AAC][MultiSub][18261A02]",
|
||||
"[Erai-raws] Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen - Ryushu no Youjo - 13 [480p CR WEB-DL AVC AAC][MultiSub][A22FE68E]",
|
||||
"[Erai-raws] Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen - Ryushu no Youjo - 13 [720p CR WEB-DL AVC AAC][MultiSub][4D409859]",
|
||||
"[GM-Team][国漫][仙逆][Renegade Immortal][2023][148][AVC][GB][1080P]",
|
||||
"[GM-Team][国漫][光阴之外][Beyond Time's Gaze][2025][29][GB][4K HEVC 10Bit]",
|
||||
"[GM-Team][国漫][吞噬星空][Swallowed Star][2021][231][AVC][GB][1080P]",
|
||||
"[GM-Team][国漫][斗破苍穹 第5季][Fights Break Sphere Ⅴ][2022][206][AVC][GB][1080P]",
|
||||
"[GM-Team][国漫][斗破苍穹 第5季][Fights Break Sphere Ⅴ][2022][206][HEVC][GB][4K]",
|
||||
"[GM-Team][国漫][牧神记][Tales of Qin Mu][2024][90][AVC][GB][1080P]",
|
||||
"KILL BLUE S01E12 Life Paths 1080p AMZN WEB-DL MULTi DDP2.0 H 264-VARYG (Kill Ao, Multi-Audio, Multi-Subs)",
|
||||
"[LoliHouse] 与奔驰于透明之夜的你,谈一场看不见的恋爱。 / 透明な夜に駆ける君と、目に見えない恋をした。 / KakeKoi - 01 [WebRip 1080p HEVC-10bit AAC][简繁内封字幕]",
|
||||
"RILAKKUMA S01E13 1080p CR WEB-DL MULTi AAC2.0 H 264-VARYG (Multi-Audio, Multi-Subs)",
|
||||
"[RUBaDUB] Kaiju No. 8 (S1 Complete) (1080p) (Dual Audio)",
|
||||
"[SubsPlease] Honzuki no Gekokujou S4 - 13 (1080p) [A4FE0990].mkv",
|
||||
"[SubsPlease] Honzuki no Gekokujou S4 - 13 (480p) [17152F60].mkv",
|
||||
"[SubsPlease] Honzuki no Gekokujou S4 - 13 (720p) [97744C2D].mkv",
|
||||
"That Time I Got Reincarnated as a Slime S04E12 Tempest Evolves 1080p CR WEB-DL DUAL AAC2.0 H 264-VARYG (Tensei shitara Slime Datta Ken, Dual-Audio, Multi-Subs)",
|
||||
"The Drops of God S01E12 Take the Distant Path 1080p CR WEB-DL DUAL AAC2.0 H 264-VARYG (Kami no Shizuku, Dual-Audio, Multi-Subs)",
|
||||
"[ToonsHub] Ascendance of a Bookworm S04E11 1080p CR WEB-DL DUAL AAC2.0 H.264 (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Dual-Audio, Multi-Subs)",
|
||||
"[ToonsHub] Ascendance of a Bookworm S04E12 1080p CR WEB-DL DUAL AAC2.0 H.264 (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Dual-Audio, Multi-Subs)",
|
||||
"[ToonsHub] Ascendance of a Bookworm S04E13 1080p CR WEB-DL AAC2.0 H.264 (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Multi-Subs)",
|
||||
"[ToonsHub] Honzuki no Gekokujou S04E13 1080p AMZN WEB-DL DDP2.0 H.264 (Ascendance of a Bookworm Side Story)",
|
||||
"[Yameii] Ascendance of a Bookworm - S04E11 [English Dub] [CR WEB-DL 1080p H264 AAC] [8ACE7B72] (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan o Erande Iraremasen - Ryoushu no Youjo | Adopted Daughter of an Archduke)",
|
||||
"[Yameii] Ascendance of a Bookworm - S04E12 [English Dub] [CR WEB-DL 1080p H264 AAC] [ACE19921] (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan o Erande Iraremasen - Ryoushu no Youjo | Adopted Daughter of an Archduke)",
|
||||
"[桜都字幕组] 入间同学入魔了 第四季 / Mairimashita! Iruma-kun (2026) [13][1080P][简体内嵌]",
|
||||
"[桜都字幕组] 入间同学入魔了 第四季 / Mairimashita! Iruma-kun (2026) [13][1080P][简繁内封]",
|
||||
"[桜都字幕组] 入间同学入魔了 第四季 / Mairimashita! Iruma-kun (2026) [13][1080P][繁体内嵌]",
|
||||
"[桜都字幕组] 入间同学入魔了 第四季 / Mairimashita! Iruma-kun (2026) [14][1080P][简繁内封]",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn parses_standard_sxxexx_with_group_and_quality_chain() {
|
||||
let p = parse("Ascendance of a Bookworm S04E11 The Gathering of Gutenberg 1080p CR WEB-DL DUAL AAC2.0 H 264-VARYG (Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen, Dual-Audio, Multi-Subs)");
|
||||
assert_eq!(p.season, Some(4));
|
||||
assert_eq!(p.episode, Some(11));
|
||||
assert_eq!(p.resolution, Some(1080));
|
||||
assert_eq!(p.source, Some(Source::WebDl));
|
||||
assert_eq!(p.codec, Some(Codec::H264));
|
||||
assert_eq!(p.title_normalized, "Ascendance of a Bookworm");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_subsplease_dash_episode_with_group_and_hash() {
|
||||
let p = parse("[SubsPlease] Honzuki no Gekokujou S4 - 13 (1080p) [A4FE0990].mkv");
|
||||
assert_eq!(p.group.as_deref(), Some("SubsPlease"));
|
||||
assert_eq!(p.season, Some(4));
|
||||
assert_eq!(p.episode, Some(13));
|
||||
assert_eq!(p.resolution, Some(1080));
|
||||
assert_eq!(p.container.as_deref(), Some("mkv"));
|
||||
assert_eq!(p.title_normalized, "Honzuki no Gekokujou");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_erai_raws_bracket_quality_block() {
|
||||
let p = parse("[Erai-raws] Honzuki no Gekokujou: Shisho ni Naru Tame ni wa Shudan wo Erandeiraremasen - Ryushu no Youjo - 13 [1080p CR WEB-DL AVC AAC][MultiSub][18261A02]");
|
||||
assert_eq!(p.group.as_deref(), Some("Erai-raws"));
|
||||
assert_eq!(p.season, None);
|
||||
assert_eq!(p.episode, Some(13));
|
||||
assert_eq!(p.resolution, Some(1080));
|
||||
assert_eq!(p.source, Some(Source::WebDl));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_lolihouse_webrip_hevc_10bit() {
|
||||
let p = parse("[LoliHouse] Some Title - 01 [WebRip 1080p HEVC-10bit AAC][Subs]");
|
||||
assert_eq!(p.group.as_deref(), Some("LoliHouse"));
|
||||
assert_eq!(p.episode, Some(1));
|
||||
assert_eq!(p.source, Some(Source::WebRip));
|
||||
assert_eq!(p.codec, Some(Codec::Hevc));
|
||||
assert_eq!(p.bit_depth, Some(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_season_pack_no_episode() {
|
||||
let p = parse("[RUBaDUB] Kaiju No. 8 (S1 Complete) (1080p) (Dual Audio)");
|
||||
assert_eq!(p.group.as_deref(), Some("RUBaDUB"));
|
||||
assert_eq!(p.season, Some(1));
|
||||
assert_eq!(p.episode, None);
|
||||
assert_eq!(p.resolution, Some(1080));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_yameii_dash_sxxexx_with_english_dub_tag() {
|
||||
let p = parse("[Yameii] Ascendance of a Bookworm - S04E11 [English Dub] [CR WEB-DL 1080p H264 AAC] [8ACE7B72] (Honzuki no Gekokujou)");
|
||||
assert_eq!(p.group.as_deref(), Some("Yameii"));
|
||||
assert_eq!(p.season, Some(4));
|
||||
assert_eq!(p.episode, Some(11));
|
||||
assert_eq!(p.codec, Some(Codec::H264));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_year_and_bare_episode_number() {
|
||||
let p = parse("[GM-Team][Renegade Immortal][2023][148][AVC][GB][1080P]");
|
||||
assert_eq!(p.year, Some(2023));
|
||||
assert_eq!(p.resolution, Some(1080));
|
||||
assert_eq!(p.codec, Some(Codec::H264).or(Some(Codec::H264))); // AVC == H264 family
|
||||
}
|
||||
|
||||
/// A batch of real, unmodified nyaa.si release titles pulled live
|
||||
/// across several categories — not hand-picked for parseability. Not a
|
||||
/// correctness check (no ground truth for the messier CJK/bracket-only
|
||||
/// shapes), just a robustness net: parse() must never panic, and on
|
||||
/// the common WEB-DL/anime shapes it should extract *something*.
|
||||
#[test]
|
||||
fn does_not_panic_on_real_corpus() {
|
||||
let mut resolution_hits = 0;
|
||||
for title in CORPUS {
|
||||
let p = parse(title);
|
||||
if p.resolution.is_some() {
|
||||
resolution_hits += 1;
|
||||
}
|
||||
assert!(!p.title_normalized.is_empty(), "empty title for {title:?}");
|
||||
}
|
||||
// Most of this corpus carries an explicit resolution tag; a low hit
|
||||
// rate would mean the resolution regex regressed, not just that a
|
||||
// few CJK/manga edge cases were missed.
|
||||
assert!(
|
||||
resolution_hits * 2 > CORPUS.len(),
|
||||
"only {resolution_hits}/{} titles yielded a resolution",
|
||||
CORPUS.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_to_resolve_a_bracketed_batch_range_to_one_episode() {
|
||||
// Previously resolved to episode 12, silently attributing the
|
||||
// whole 12-episode batch torrent to one guessed episode.
|
||||
let p = parse("[Judas] Some Show (01-12) [BD 1080p]");
|
||||
assert_eq!(p.episode, None);
|
||||
assert_eq!(p.absolute_episode, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_to_resolve_a_dash_prefixed_batch_range_to_one_episode() {
|
||||
// Previously resolved to episode 1 (the first number after the
|
||||
// leading dash), same underlying problem as above.
|
||||
let p = parse("Show - 01-12 [1080p]");
|
||||
assert_eq!(p.episode, None);
|
||||
assert_eq!(p.absolute_episode, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_to_resolve_an_sxxexx_range_to_one_episode() {
|
||||
// Previously resolved to S01E01 via SXXEXX_RE matching only the
|
||||
// first half of the range.
|
||||
let p = parse("Show S01E01-E12 1080p WEB-DL");
|
||||
assert_eq!(p.season, Some(1));
|
||||
assert_eq!(p.episode, None);
|
||||
assert_eq!(p.absolute_episode, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_episode_dash_titles_are_unaffected_by_range_detection() {
|
||||
// A genuine single episode using dash notation must still resolve
|
||||
// normally — only real ranges should be refused.
|
||||
let p = parse("[SubsPlease] Some Show - 05 (1080p) [ABCD1234].mkv");
|
||||
assert_eq!(p.episode, Some(5));
|
||||
assert_eq!(p.absolute_episode, Some(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_a_bare_year_from_a_scene_style_movie_name() {
|
||||
// Previously parsed as year=None entirely, since these releases
|
||||
// never bracket the year — silently disabling movie_year_mismatch,
|
||||
// the only defense against a same-named-but-wrong film, on exactly
|
||||
// the naming convention TPB/1337x results actually use.
|
||||
let p = parse("Dune.1984.1080p.BluRay.x264-GROUP");
|
||||
assert_eq!(p.year, Some(1984));
|
||||
|
||||
let p = parse("Dune 1984 1080p BluRay");
|
||||
assert_eq!(p.year, Some(1984));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_mistake_a_year_titled_movie_for_a_bare_year() {
|
||||
// "1917" and "2012" are real movie titles — a bare year at the very
|
||||
// start of the string must not be treated as a release year, or
|
||||
// the title would be mistaken for empty.
|
||||
let p = parse("1917 1080p BluRay x264-GROUP");
|
||||
assert_eq!(p.year, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brackets_still_take_priority_over_a_coincidental_bare_year() {
|
||||
let p = parse("Some.Show.2024.S01E01.(2023).1080p.WEB-DL");
|
||||
assert_eq!(p.year, Some(2023));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_panic_on_unparsable_manga_release() {
|
||||
// Not a video release at all — should degrade gracefully, not crash.
|
||||
let p =
|
||||
parse("Aime ton prochain (01-06) (Chida) (2019) [Digital-1920] [Manga FR] (PapriKa+)");
|
||||
assert_eq!(p.year, Some(2019));
|
||||
// season/episode extraction is allowed to miss here; the important
|
||||
// thing is it returns *something* rather than panicking.
|
||||
let _ = p.season;
|
||||
}
|
||||
}
|
||||
206
breadarrd/src/parser/tokens.rs
Normal file
206
breadarrd/src/parser/tokens.rs
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use super::{Codec, Source, WS_RE};
|
||||
|
||||
pub(super) static GROUP_PREFIX_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\[[^\]]+\]\s*").unwrap());
|
||||
|
||||
static LEADING_GROUP_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\[([^\]]+)\]").unwrap());
|
||||
|
||||
static CONTAINER_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\.(mkv|mp4|avi)\b").unwrap());
|
||||
|
||||
static RESOLUTION_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(2160p|1080p|720p|480p|4k)\b").unwrap());
|
||||
|
||||
static SOURCE_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\b(BDRemux|Remux|Blu-?Ray|BDRip|WEB-?DL|WEBRip|HDTV)\b").unwrap()
|
||||
});
|
||||
|
||||
static CODEC_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\b(AV1|HEVC|H\.?\s?265|x265|H\.?\s?264|x264|AVC)\b").unwrap()
|
||||
});
|
||||
|
||||
static BIT_DEPTH_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(8|10)-?bit\b").unwrap());
|
||||
|
||||
pub(super) static REPACK_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(REPACK|PROPER)\b").unwrap());
|
||||
|
||||
static YEAR_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[(\[](19|20)\d{2}[)\]]").unwrap());
|
||||
// Scene-style releases ("Dune.1984.1080p.BluRay.x264-GROUP") carry the year
|
||||
// bare, with no surrounding brackets — `YEAR_RE` above never matches these
|
||||
// at all, which meant `movie_year_mismatch` (the only defense against a
|
||||
// same-named-but-wrong film once a title auto-matches) silently never fired
|
||||
// on exactly the naming convention TPB/1337x results actually use.
|
||||
static BARE_YEAR_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\b((?:19|20)\d{2})\b").unwrap());
|
||||
|
||||
static SXXEXX_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bS(\d{1,2})E(\d{1,3})\b").unwrap());
|
||||
static SXX_DASH_EP_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bS(\d{1,2})\s*-\s*(\d{1,3})\b").unwrap());
|
||||
static SEASON_PACK_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\(S?(\d{1,2})\s*Complete\)").unwrap());
|
||||
static DASH_EPISODE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"-\s*(\d{1,3})\b").unwrap());
|
||||
|
||||
// A batch/season-pack release covering many episodes in one torrent.
|
||||
// `SXXEXX_RE`/`DASH_EPISODE_RE` above would otherwise happily resolve one
|
||||
// of these to a single arbitrary episode number (verified live:
|
||||
// "Show (01-12) [1080p]" resolves to episode 12, "Show - 01-12 [1080p]"
|
||||
// resolves to episode 1, "Show S01E01-E12" resolves to S01E01) — the
|
||||
// importer then grabs the whole multi-episode torrent, picks whichever
|
||||
// file happens to be largest, and files it under that one guessed episode
|
||||
// while silently discarding the rest. `looks_like_episode_range` below is
|
||||
// checked first so these get refused rather than mis-resolved.
|
||||
static BATCH_WORD_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(batch|complete)\b").unwrap());
|
||||
// "S01E01-E12" / "S01E01-12": no word-boundary exists between the digits
|
||||
// and letters in a run like "S01E01-E12" (letters and digits are both
|
||||
// \w), so a boundary-anchored generic range pattern can't find this —
|
||||
// needs its own literal S..E..-..E?.. shape.
|
||||
static SXX_EPISODE_RANGE_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bS(\d{1,2})E(\d{1,3})\s*-\s*E?(\d{1,3})\b").unwrap());
|
||||
// Bare numeric ranges: "(01-12)", "01-12", "01~12". Requires the second
|
||||
// number to be strictly larger than the first (a real episode range
|
||||
// always counts up) so this doesn't fire on, say, an unrelated dash
|
||||
// elsewhere in the title with a smaller trailing number.
|
||||
static BARE_EPISODE_RANGE_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\b(\d{1,3})\s*[-~]\s*(\d{1,3})\b").unwrap());
|
||||
|
||||
pub(super) fn looks_like_episode_range(s: &str) -> bool {
|
||||
if BATCH_WORD_RE.is_match(s) || SXX_EPISODE_RANGE_RE.is_match(s) {
|
||||
return true;
|
||||
}
|
||||
BARE_EPISODE_RANGE_RE.captures(s).is_some_and(|c| {
|
||||
let a: u32 = c[1].parse().unwrap_or(0);
|
||||
let b: u32 = c[2].parse().unwrap_or(0);
|
||||
b > a
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn extract_group(s: &str) -> Option<String> {
|
||||
LEADING_GROUP_RE
|
||||
.captures(s)
|
||||
.map(|c| c[1].trim().to_string())
|
||||
}
|
||||
|
||||
pub(super) fn extract_container(s: &str) -> Option<String> {
|
||||
CONTAINER_RE.captures(s).map(|c| c[1].to_lowercase())
|
||||
}
|
||||
|
||||
pub(super) fn extract_resolution(s: &str) -> Option<u32> {
|
||||
let m = RESOLUTION_RE.captures(s)?;
|
||||
let token = m[1].to_lowercase();
|
||||
if token == "4k" {
|
||||
return Some(2160);
|
||||
}
|
||||
token.trim_end_matches('p').parse().ok()
|
||||
}
|
||||
|
||||
pub(super) fn extract_source(s: &str) -> Option<Source> {
|
||||
let token = SOURCE_RE.captures(s)?[1].to_lowercase();
|
||||
Some(match token.as_str() {
|
||||
"bdremux" | "remux" => Source::Remux,
|
||||
t if t.replace('-', "") == "bluray" => Source::BluRay,
|
||||
"bdrip" => Source::BluRay,
|
||||
t if t.replace('-', "") == "webdl" => Source::WebDl,
|
||||
"webrip" => Source::WebRip,
|
||||
"hdtv" => Source::Hdtv,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn extract_codec(s: &str) -> Option<Codec> {
|
||||
let token = CODEC_RE.captures(s)?[1]
|
||||
.to_lowercase()
|
||||
.replace([' ', '.'], "");
|
||||
Some(match token.as_str() {
|
||||
"av1" => Codec::Av1,
|
||||
"hevc" | "h265" | "x265" => Codec::Hevc,
|
||||
"h264" | "x264" | "avc" => Codec::H264,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn extract_bit_depth(s: &str) -> Option<u8> {
|
||||
BIT_DEPTH_RE.captures(s)?[1].parse().ok()
|
||||
}
|
||||
|
||||
pub(super) fn extract_year(s: &str) -> Option<u32> {
|
||||
if let Some(m) = YEAR_RE.find(s) {
|
||||
return s[m.start() + 1..m.end() - 1].parse().ok();
|
||||
}
|
||||
// Bare-year fallback, but never at the very start of the string — a
|
||||
// movie literally titled after a year ("1917", "2012") would otherwise
|
||||
// have its own title mistaken for a year with nothing left over. Same
|
||||
// guard `library_scan.rs`'s `FOLDER_BARE_YEAR_RE` uses.
|
||||
let m = BARE_YEAR_RE.find(s)?;
|
||||
if m.start() == 0 {
|
||||
return None;
|
||||
}
|
||||
s[m.start()..m.end()].parse().ok()
|
||||
}
|
||||
|
||||
/// Returns (season, episode, absolute_episode, title_span_end) — the last
|
||||
/// element is the byte offset in `s` where the episode/season token (or,
|
||||
/// failing that, the first quality marker) begins, used to slice out the
|
||||
/// title portion.
|
||||
pub(super) fn extract_episode_info(s: &str) -> (Option<u32>, Option<u32>, Option<u32>, usize) {
|
||||
if looks_like_episode_range(s) {
|
||||
// Season is still useful to surface (e.g. for display/logging)
|
||||
// when it's unambiguous, but episode/absolute_episode are
|
||||
// deliberately left unresolved — see `looks_like_episode_range`'s
|
||||
// doc comment for why guessing one is actively harmful here.
|
||||
let season = SXXEXX_RE
|
||||
.captures(s)
|
||||
.and_then(|c| c[1].parse().ok())
|
||||
.or_else(|| SEASON_PACK_RE.captures(s).and_then(|c| c[1].parse().ok()));
|
||||
let end = first_quality_marker(s).unwrap_or(s.len());
|
||||
return (season, None, None, end);
|
||||
}
|
||||
if let Some(c) = SXXEXX_RE.captures(s) {
|
||||
let season = c[1].parse().ok();
|
||||
let episode = c[2].parse().ok();
|
||||
return (season, episode, None, c.get(0).unwrap().start());
|
||||
}
|
||||
if let Some(c) = SXX_DASH_EP_RE.captures(s) {
|
||||
let season = c[1].parse().ok();
|
||||
let episode = c[2].parse().ok();
|
||||
return (season, episode, None, c.get(0).unwrap().start());
|
||||
}
|
||||
if let Some(c) = SEASON_PACK_RE.captures(s) {
|
||||
let season = c[1].parse().ok();
|
||||
return (season, None, None, c.get(0).unwrap().start());
|
||||
}
|
||||
if let Some(c) = DASH_EPISODE_RE.captures(s) {
|
||||
let episode: Option<u32> = c[1].parse().ok();
|
||||
return (None, episode, episode, c.get(0).unwrap().start());
|
||||
}
|
||||
|
||||
let end = first_quality_marker(s).unwrap_or(s.len());
|
||||
(None, None, None, end)
|
||||
}
|
||||
|
||||
fn first_quality_marker(s: &str) -> Option<usize> {
|
||||
[
|
||||
RESOLUTION_RE.find(s).map(|m| m.start()),
|
||||
SOURCE_RE.find(s).map(|m| m.start()),
|
||||
CODEC_RE.find(s).map(|m| m.start()),
|
||||
YEAR_RE.find(s).map(|m| m.start()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min()
|
||||
}
|
||||
|
||||
pub(super) fn derive_title(s: &str, span_end: usize) -> String {
|
||||
let candidate = &s[..span_end.min(s.len())];
|
||||
let trimmed = candidate
|
||||
.trim()
|
||||
.trim_end_matches(['-', ':', '('])
|
||||
.trim_end_matches(char::is_whitespace);
|
||||
WS_RE.replace_all(trimmed, " ").trim().to_string()
|
||||
}
|
||||
265
breadarrd/src/qbit/mod.rs
Normal file
265
breadarrd/src/qbit/mod.rs
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
use anyhow::{bail, Context, Result};
|
||||
use serde::Deserialize;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
static BTIH_RE: LazyLock<regex::Regex> =
|
||||
LazyLock::new(|| regex::Regex::new(r"(?i)urn:btih:([0-9a-f]{40}|[2-7a-z]{32})").unwrap());
|
||||
|
||||
/// Pulls a magnet link's own infohash out of it directly — no need to ask
|
||||
/// qBittorrent to correlate anything when the caller already handed us the
|
||||
/// hash. Returns `None` for anything that isn't a magnet URI (e.g. nyaa's
|
||||
/// `.torrent`-file download URLs), which callers fall back to polling for.
|
||||
/// Lowercased to match the casing qBittorrent's own API always returns.
|
||||
pub(crate) fn extract_btih(link: &str) -> Option<String> {
|
||||
BTIH_RE.captures(link).map(|c| c[1].to_lowercase())
|
||||
}
|
||||
|
||||
/// qBittorrent rejected a magnet outright (HTTP 200 with body "Fails.") —
|
||||
/// deterministic for a given hash (a dead/unreachable torrent, a malformed
|
||||
/// magnet), unlike a network-level failure. A distinct type so callers can
|
||||
/// `downcast_ref` and treat it differently from a transient error: retrying
|
||||
/// the exact same hash next cycle would just fail identically forever,
|
||||
/// where a real network blip is worth retrying.
|
||||
#[derive(Debug)]
|
||||
pub struct MagnetRejected {
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for MagnetRejected {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "qbit rejected the magnet: body={:?}", self.body)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for MagnetRejected {}
|
||||
|
||||
pub struct QbitClient {
|
||||
base_url: String,
|
||||
client: reqwest::Client,
|
||||
/// Credentials from the last successful `login()`, kept so a session
|
||||
/// that's expired mid-run (a qBittorrent container restarting under it
|
||||
/// is common in a Docker-based setup) can be silently re-established
|
||||
/// instead of every subsequent call failing until breadarrd itself is
|
||||
/// restarted.
|
||||
credentials: tokio::sync::RwLock<Option<(String, String)>>,
|
||||
/// Serializes the add-then-correlate-hash sequence (see
|
||||
/// `scheduler::grab_and_capture_hash`) across every caller — the
|
||||
/// background grab loop and the API's review-approve handler both add
|
||||
/// torrents against the same category and can run concurrently.
|
||||
/// Without this, two adds interleaved between one caller's before/after
|
||||
/// `torrents/info` snapshots let the *other* caller's torrent look like
|
||||
/// "the new one," recording the wrong hash against the wrong release
|
||||
/// (verified as a real risk, not just theoretical, during the Fable 5
|
||||
/// audit — see the caller for the fuller writeup).
|
||||
grab_lock: tokio::sync::Mutex<()>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TorrentInfo {
|
||||
pub hash: String,
|
||||
pub name: String,
|
||||
pub state: String,
|
||||
pub progress: f64,
|
||||
pub save_path: String,
|
||||
/// Full path to the torrent's content (file or directory root) —
|
||||
/// qBittorrent resolves this for us, so importers don't need to guess
|
||||
/// at how save_path and name combine for a given torrent.
|
||||
pub content_path: String,
|
||||
}
|
||||
|
||||
impl QbitClient {
|
||||
pub fn new(base_url: impl Into<String>) -> Result<Self> {
|
||||
// No total timeout is reqwest's default — the background loop holds
|
||||
// the DB mutex across calls into this client, so a stalled
|
||||
// connection would hang the whole daemon.
|
||||
let client = reqwest::Client::builder()
|
||||
.cookie_store(true)
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()?;
|
||||
Ok(Self {
|
||||
base_url: base_url.into(),
|
||||
client,
|
||||
credentials: tokio::sync::RwLock::new(None),
|
||||
grab_lock: tokio::sync::Mutex::new(()),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn login(&self, username: &str, password: &str) -> Result<()> {
|
||||
self.do_login(username, password).await?;
|
||||
*self.credentials.write().await = Some((username.to_string(), password.to_string()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn do_login(&self, username: &str, password: &str) -> Result<()> {
|
||||
let resp = self
|
||||
.client
|
||||
.post(format!("{}/api/v2/auth/login", self.base_url))
|
||||
.form(&[("username", username), ("password", password)])
|
||||
.send()
|
||||
.await
|
||||
.context("qbit login request failed")?;
|
||||
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
if !status.is_success() || body.trim() != "Ok." {
|
||||
bail!("qbit login failed: status={status} body={body:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Re-authenticates using the credentials from the last successful
|
||||
/// `login()`, if any. Returns `true` on success so callers know it's
|
||||
/// worth retrying the request that got a 403 in the first place.
|
||||
async fn try_reauth(&self) -> bool {
|
||||
let creds = self.credentials.read().await.clone();
|
||||
let Some((username, password)) = creds else {
|
||||
return false;
|
||||
};
|
||||
self.do_login(&username, &password).await.is_ok()
|
||||
}
|
||||
|
||||
pub async fn add_magnet(&self, magnet: &str, category: &str) -> Result<()> {
|
||||
for attempt in 0..2 {
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.text("urls", magnet.to_string())
|
||||
.text("category", category.to_string());
|
||||
|
||||
let resp = self
|
||||
.client
|
||||
.post(format!("{}/api/v2/torrents/add", self.base_url))
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await
|
||||
.context("qbit add-torrent request failed")?;
|
||||
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::FORBIDDEN && attempt == 0 && self.try_reauth().await {
|
||||
continue;
|
||||
}
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
if !status.is_success() {
|
||||
bail!("qbit add-torrent failed: status={status} body={body:?}");
|
||||
}
|
||||
// qBittorrent's add-torrent endpoint returns HTTP 200 even when
|
||||
// it rejects the magnet outright (a dead/malformed hash, one it
|
||||
// already knows is unreachable) — the *only* signal is the
|
||||
// response body text ("Ok." vs "Fails."). Without this check a
|
||||
// rejected magnet looks identical to a real success: the caller
|
||||
// records a `release` row as grabbed and nothing ever
|
||||
// downloads, silently and permanently (verified live — this
|
||||
// happened for a real release).
|
||||
if body.trim() != "Ok." {
|
||||
return Err(anyhow::Error::new(MagnetRejected { body }));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
unreachable!("loop always returns or bails on its second iteration")
|
||||
}
|
||||
|
||||
pub async fn list_torrents(&self, category: Option<&str>) -> Result<Vec<TorrentInfo>> {
|
||||
for attempt in 0..2 {
|
||||
let mut req = self
|
||||
.client
|
||||
.get(format!("{}/api/v2/torrents/info", self.base_url));
|
||||
if let Some(category) = category {
|
||||
req = req.query(&[("category", category)]);
|
||||
}
|
||||
|
||||
let resp = req.send().await.context("qbit list-torrents failed")?;
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::FORBIDDEN && attempt == 0 && self.try_reauth().await {
|
||||
continue;
|
||||
}
|
||||
if !status.is_success() {
|
||||
bail!("qbit list-torrents failed: status={status}");
|
||||
}
|
||||
return Ok(resp.json().await?);
|
||||
}
|
||||
unreachable!("loop always returns or bails on its second iteration")
|
||||
}
|
||||
|
||||
/// Sends a POST form request, retrying once after a silent re-login if
|
||||
/// the session had expired (403). Shared by the file-relocation methods
|
||||
/// below — `add_magnet`/`list_torrents` predate this and are left as
|
||||
/// they are rather than churned for the sake of it.
|
||||
async fn post_form(&self, path: &str, form: &[(&str, &str)]) -> Result<String> {
|
||||
for attempt in 0..2 {
|
||||
let resp = self
|
||||
.client
|
||||
.post(format!("{}{path}", self.base_url))
|
||||
.form(form)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("qbit {path} request failed"))?;
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::FORBIDDEN && attempt == 0 && self.try_reauth().await {
|
||||
continue;
|
||||
}
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
if !status.is_success() {
|
||||
bail!("qbit {path} failed: status={status} body={body:?}");
|
||||
}
|
||||
return Ok(body);
|
||||
}
|
||||
unreachable!("loop always returns or bails on its second iteration")
|
||||
}
|
||||
|
||||
/// Moves a torrent's save location — qBittorrent physically relocates
|
||||
/// the underlying file(s) itself and continues seeding from the new
|
||||
/// path, rather than breadarr keeping a second permanent copy purely to
|
||||
/// satisfy its own import step.
|
||||
/// Held for the duration of an add-then-correlate-hash sequence — see
|
||||
/// `grab_lock`'s doc comment on why this needs to be process-wide, not
|
||||
/// just per-call.
|
||||
pub(crate) async fn lock_for_grab(&self) -> tokio::sync::MutexGuard<'_, ()> {
|
||||
self.grab_lock.lock().await
|
||||
}
|
||||
|
||||
pub async fn set_location(&self, hash: &str, location: &str) -> Result<()> {
|
||||
self.post_form(
|
||||
"/api/v2/torrents/setLocation",
|
||||
&[("hashes", hash), ("location", location)],
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extracts_hash_from_a_real_magnet() {
|
||||
let link = "magnet:?xt=urn:btih:3F493B821C8B13CA2EE6DA0183B4803B187F9363&dn=Some+Show&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce";
|
||||
assert_eq!(
|
||||
extract_btih(link).as_deref(),
|
||||
Some("3f493b821c8b13ca2ee6da0183b4803b187f9363")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_base32_hash_from_a_magnet() {
|
||||
let link = "magnet:?xt=urn:btih:jz2eqzsm3mmirrbcaacegz3czfzwxjbc&dn=Some+Show";
|
||||
assert_eq!(
|
||||
extract_btih(link).as_deref(),
|
||||
Some("jz2eqzsm3mmirrbcaacegz3czfzwxjbc")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_none_for_a_torrent_download_url() {
|
||||
assert_eq!(
|
||||
extract_btih("https://nyaa.si/download/2131680.torrent"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_none_for_a_1337x_page_url() {
|
||||
assert_eq!(
|
||||
extract_btih("https://1337x.to/torrent/3740704/Some-Show/"),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
3065
breadarrd/src/scheduler.rs
Normal file
3065
breadarrd/src/scheduler.rs
Normal file
File diff suppressed because it is too large
Load diff
262
breadarrd/src/scoring/gate.rs
Normal file
262
breadarrd/src/scoring/gate.rs
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
use crate::parser::ParsedRelease;
|
||||
|
||||
use super::profile::QualityProfile;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RejectReason {
|
||||
MissingSeederData,
|
||||
BelowMinSeeders,
|
||||
SizeOutOfRange,
|
||||
GroupDenylisted,
|
||||
NoEnglishAudio,
|
||||
/// Below 1080p while a 1080p-or-better candidate exists for the same
|
||||
/// target in this same search batch — rejected in favor of the better
|
||||
/// one, not because low resolution is inherently disqualifying (a
|
||||
/// release with no 1080p+ option anywhere is still accepted; see
|
||||
/// `GateContext::better_resolution_available`).
|
||||
LowResolutionAlternativeExists,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum GateResult {
|
||||
Accept,
|
||||
Reject(RejectReason),
|
||||
}
|
||||
|
||||
pub struct GateContext {
|
||||
pub seeders: Option<u32>,
|
||||
pub size_bytes: Option<u64>,
|
||||
/// Episode/movie runtime, when known — used to sanity-check size against
|
||||
/// implied bitrate. Frequently unavailable (neither TVDB's nor TMDB's
|
||||
/// episode-level data reliably carries this), so size checks fall back
|
||||
/// to absolute per-resolution bounds when absent.
|
||||
pub runtime_minutes: Option<u32>,
|
||||
pub has_english_audio: bool,
|
||||
/// Anime gets a carve-out on the English-audio gate: falls back to
|
||||
/// Japanese-only audio when no English release exists at all.
|
||||
pub is_anime: bool,
|
||||
/// True when at least one other candidate in the same search batch
|
||||
/// parsed to 1080p or better — the signal that makes a sub-1080p
|
||||
/// candidate here a real downgrade rather than the only option. Always
|
||||
/// `false` on the RSS feed-watch path (no batch of alternatives to
|
||||
/// compare against there), so this gate never fires for that path.
|
||||
pub better_resolution_available: bool,
|
||||
/// A season/batch release covering many episodes in one torrent — its
|
||||
/// total size is naturally a multiple of a single episode's, so the
|
||||
/// per-episode size/bitrate sanity check would reject it as absurdly
|
||||
/// oversized every time. There's no reliable episode count to scale
|
||||
/// the bounds by before the torrent is actually inspected, so the size
|
||||
/// gate is skipped entirely rather than guessed at.
|
||||
pub is_season_pack: bool,
|
||||
}
|
||||
|
||||
pub fn evaluate(parsed: &ParsedRelease, ctx: &GateContext, profile: &QualityProfile) -> GateResult {
|
||||
let Some(seeders) = ctx.seeders else {
|
||||
return GateResult::Reject(RejectReason::MissingSeederData);
|
||||
};
|
||||
if seeders < profile.min_seeders {
|
||||
return GateResult::Reject(RejectReason::BelowMinSeeders);
|
||||
}
|
||||
|
||||
if !ctx.is_season_pack {
|
||||
if let (Some(size), Some(resolution)) = (ctx.size_bytes, parsed.resolution) {
|
||||
if !size_is_sane(size, resolution, ctx.runtime_minutes) {
|
||||
return GateResult::Reject(RejectReason::SizeOutOfRange);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ctx.better_resolution_available && parsed.resolution.is_some_and(|r| r < 1080) {
|
||||
return GateResult::Reject(RejectReason::LowResolutionAlternativeExists);
|
||||
}
|
||||
|
||||
if let Some(group) = &parsed.group {
|
||||
if profile
|
||||
.group_denylist
|
||||
.iter()
|
||||
.any(|g| g.eq_ignore_ascii_case(group))
|
||||
{
|
||||
return GateResult::Reject(RejectReason::GroupDenylisted);
|
||||
}
|
||||
}
|
||||
|
||||
if !ctx.has_english_audio && !ctx.is_anime {
|
||||
return GateResult::Reject(RejectReason::NoEnglishAudio);
|
||||
}
|
||||
|
||||
GateResult::Accept
|
||||
}
|
||||
|
||||
fn size_is_sane(size_bytes: u64, resolution: u32, runtime_minutes: Option<u32>) -> bool {
|
||||
let Some(runtime_minutes) = runtime_minutes.filter(|&r| r > 0) else {
|
||||
return absolute_size_bounds(resolution).contains(&size_bytes);
|
||||
};
|
||||
|
||||
let seconds = (runtime_minutes as u64) * 60;
|
||||
let bits_per_second = (size_bytes * 8) / seconds;
|
||||
let (min_bps, max_bps) = bitrate_bounds(resolution);
|
||||
(min_bps..=max_bps).contains(&bits_per_second)
|
||||
}
|
||||
|
||||
fn bitrate_bounds(resolution: u32) -> (u64, u64) {
|
||||
match resolution {
|
||||
r if r >= 2160 => (2_000_000, 80_000_000),
|
||||
r if r >= 1080 => (800_000, 40_000_000),
|
||||
r if r >= 720 => (400_000, 20_000_000),
|
||||
_ => (150_000, 10_000_000),
|
||||
}
|
||||
}
|
||||
|
||||
fn absolute_size_bounds(resolution: u32) -> std::ops::RangeInclusive<u64> {
|
||||
match resolution {
|
||||
r if r >= 2160 => 200_000_000..=40_000_000_000,
|
||||
r if r >= 1080 => 50_000_000..=15_000_000_000,
|
||||
r if r >= 720 => 20_000_000..=8_000_000_000,
|
||||
_ => 5_000_000..=4_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::parser;
|
||||
|
||||
fn ctx() -> GateContext {
|
||||
GateContext {
|
||||
seeders: Some(50),
|
||||
size_bytes: Some(400_000_000),
|
||||
runtime_minutes: Some(24),
|
||||
has_english_audio: true,
|
||||
is_anime: false,
|
||||
better_resolution_available: false,
|
||||
is_season_pack: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_a_healthy_english_release() {
|
||||
let parsed = parser::parse("Some Show S01E01 1080p WEB-DL H264");
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &ctx(), &QualityProfile::default_tv()),
|
||||
GateResult::Accept
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_seeder_data() {
|
||||
let parsed = parser::parse("Some Show S01E01 1080p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.seeders = None;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Reject(RejectReason::MissingSeederData)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_below_min_seeders() {
|
||||
let parsed = parser::parse("Some Show S01E01 1080p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.seeders = Some(1);
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Reject(RejectReason::BelowMinSeeders)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_absurdly_small_file_for_claimed_resolution() {
|
||||
let parsed = parser::parse("Some Movie 2024 2160p BluRay");
|
||||
let mut c = ctx();
|
||||
c.size_bytes = Some(1_000_000); // 1MB claiming to be a 4K release
|
||||
c.runtime_minutes = None;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_movie()),
|
||||
GateResult::Reject(RejectReason::SizeOutOfRange)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_denylisted_group() {
|
||||
let parsed = parser::parse("[BadGroup] Some Show - 01 [1080p]");
|
||||
let mut c = ctx();
|
||||
c.is_anime = true;
|
||||
let mut profile = QualityProfile::default_tv();
|
||||
profile.group_denylist.push("BadGroup".to_string());
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &profile),
|
||||
GateResult::Reject(RejectReason::GroupDenylisted)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_english_audio_for_non_anime() {
|
||||
let parsed = parser::parse("Some Show S01E01 1080p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.has_english_audio = false;
|
||||
c.is_anime = false;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Reject(RejectReason::NoEnglishAudio)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anime_without_english_audio_is_allowed_through() {
|
||||
let parsed = parser::parse("[SubsPlease] Some Anime - 01 (1080p)");
|
||||
let mut c = ctx();
|
||||
c.has_english_audio = false;
|
||||
c.is_anime = true;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Accept
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_below_1080p_when_a_better_alternative_exists() {
|
||||
let parsed = parser::parse("Some Show S01E01 720p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.better_resolution_available = true;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Reject(RejectReason::LowResolutionAlternativeExists)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_below_1080p_when_it_is_the_only_option() {
|
||||
let parsed = parser::parse("Some Show S01E01 720p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.better_resolution_available = false;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Accept
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn season_pack_bypasses_the_size_sanity_check() {
|
||||
let parsed = parser::parse("Some Show S01 1080p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.is_season_pack = true;
|
||||
// A whole season's worth of episodes at once — would fail
|
||||
// size_is_sane badly if evaluated as if it were one episode.
|
||||
c.size_bytes = Some(400_000_000 * 12);
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Accept
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_1080p_regardless_of_better_resolution_available() {
|
||||
let parsed = parser::parse("Some Show S01E01 1080p WEB-DL H264");
|
||||
let mut c = ctx();
|
||||
c.better_resolution_available = true;
|
||||
assert_eq!(
|
||||
evaluate(&parsed, &c, &QualityProfile::default_tv()),
|
||||
GateResult::Accept
|
||||
);
|
||||
}
|
||||
}
|
||||
7
breadarrd/src/scoring/mod.rs
Normal file
7
breadarrd/src/scoring/mod.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
pub mod gate;
|
||||
pub mod profile;
|
||||
pub mod score;
|
||||
|
||||
pub use gate::{evaluate as evaluate_gates, GateContext, GateResult, RejectReason};
|
||||
pub use profile::{ProfileKind, QualityProfile};
|
||||
pub use score::score;
|
||||
211
breadarrd/src/scoring/profile.rs
Normal file
211
breadarrd/src/scoring/profile.rs
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProfileKind {
|
||||
Movie,
|
||||
Tv,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Weights {
|
||||
pub seeder: f32,
|
||||
/// Weighted heavily on purpose: resolution is the single most visible
|
||||
/// quality axis, and without it in the score, same-episode releases at
|
||||
/// 480p/720p/1080p (routine — SubsPlease and friends publish all three
|
||||
/// of an episode within the same feed window) scored identically apart
|
||||
/// from source/codec, letting a lower-resolution duplicate arrive later
|
||||
/// and outscore (and at import time, overwrite) an already-grabbed
|
||||
/// better one on nothing but a seeder-count edge.
|
||||
pub resolution_tier: f32,
|
||||
pub source_tier: f32,
|
||||
pub codec_tier: f32,
|
||||
pub bit_depth: f32,
|
||||
pub container: f32,
|
||||
pub group_allowlist: f32,
|
||||
pub repack: f32,
|
||||
/// HDR/Dolby Vision bonus. Always 0 on TV/anime profiles — only movie
|
||||
/// profiles score this axis at all, per the product decision that HDR
|
||||
/// only matters for movies.
|
||||
pub hdr: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct QualityProfile {
|
||||
pub kind: ProfileKind,
|
||||
pub min_seeders: u32,
|
||||
pub group_denylist: Vec<String>,
|
||||
pub group_allowlist: Vec<String>,
|
||||
pub weights: Weights,
|
||||
}
|
||||
|
||||
impl QualityProfile {
|
||||
pub fn default_tv() -> Self {
|
||||
Self {
|
||||
kind: ProfileKind::Tv,
|
||||
min_seeders: 3,
|
||||
group_denylist: Vec::new(),
|
||||
group_allowlist: Vec::new(),
|
||||
weights: Weights {
|
||||
seeder: 1.0,
|
||||
resolution_tier: 3.0,
|
||||
source_tier: 3.0,
|
||||
codec_tier: 2.0,
|
||||
bit_depth: 0.5,
|
||||
container: 0.25,
|
||||
group_allowlist: 1.0,
|
||||
repack: 1.5,
|
||||
hdr: 0.0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_movie() -> Self {
|
||||
Self {
|
||||
kind: ProfileKind::Movie,
|
||||
min_seeders: 3,
|
||||
group_denylist: Vec::new(),
|
||||
group_allowlist: Vec::new(),
|
||||
weights: Weights {
|
||||
seeder: 1.0,
|
||||
resolution_tier: 3.0,
|
||||
source_tier: 3.0,
|
||||
codec_tier: 2.0,
|
||||
bit_depth: 0.5,
|
||||
container: 0.25,
|
||||
group_allowlist: 1.0,
|
||||
repack: 1.5,
|
||||
hdr: 2.0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the default profile for `kind`, then applies a user-supplied
|
||||
/// JSON weights override on top of it — the `quality_profile.weights`
|
||||
/// column's actual content, finally read instead of ignored. Every
|
||||
/// field is individually optional, so e.g. `{"hdr": 5.0}` overrides
|
||||
/// only the HDR bonus and leaves every other axis at its built-in
|
||||
/// default, rather than requiring a full weights object to change one
|
||||
/// number. Empty (`""`/`"{}"`), missing, or malformed JSON all
|
||||
/// harmlessly fall back to exactly the hardcoded defaults — a bad
|
||||
/// config value must never break scoring or gating.
|
||||
pub fn with_weights_override(kind: ProfileKind, weights_json: &str) -> Self {
|
||||
let mut profile = match kind {
|
||||
ProfileKind::Movie => Self::default_movie(),
|
||||
ProfileKind::Tv => Self::default_tv(),
|
||||
};
|
||||
if weights_json.trim().is_empty() || weights_json.trim() == "{}" {
|
||||
return profile;
|
||||
}
|
||||
match serde_json::from_str::<WeightsOverride>(weights_json) {
|
||||
Ok(o) => profile.weights = profile.weights.with_override(&o),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
weights_json,
|
||||
"quality_profile.weights is not valid JSON; using built-in defaults"
|
||||
);
|
||||
}
|
||||
}
|
||||
profile
|
||||
}
|
||||
}
|
||||
|
||||
/// Every field optional so a partial JSON object only overrides the axes it
|
||||
/// actually names — matches `serde_json::from_str`'s normal "missing field
|
||||
/// stays at its `Default`" behavior for a struct made entirely of
|
||||
/// `Option<f32>` fields.
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
struct WeightsOverride {
|
||||
seeder: Option<f32>,
|
||||
resolution_tier: Option<f32>,
|
||||
source_tier: Option<f32>,
|
||||
codec_tier: Option<f32>,
|
||||
bit_depth: Option<f32>,
|
||||
container: Option<f32>,
|
||||
group_allowlist: Option<f32>,
|
||||
repack: Option<f32>,
|
||||
hdr: Option<f32>,
|
||||
}
|
||||
|
||||
impl Weights {
|
||||
fn with_override(mut self, o: &WeightsOverride) -> Self {
|
||||
if let Some(v) = o.seeder {
|
||||
self.seeder = v;
|
||||
}
|
||||
if let Some(v) = o.resolution_tier {
|
||||
self.resolution_tier = v;
|
||||
}
|
||||
if let Some(v) = o.source_tier {
|
||||
self.source_tier = v;
|
||||
}
|
||||
if let Some(v) = o.codec_tier {
|
||||
self.codec_tier = v;
|
||||
}
|
||||
if let Some(v) = o.bit_depth {
|
||||
self.bit_depth = v;
|
||||
}
|
||||
if let Some(v) = o.container {
|
||||
self.container = v;
|
||||
}
|
||||
if let Some(v) = o.group_allowlist {
|
||||
self.group_allowlist = v;
|
||||
}
|
||||
if let Some(v) = o.repack {
|
||||
self.repack = v;
|
||||
}
|
||||
if let Some(v) = o.hdr {
|
||||
self.hdr = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn empty_or_absent_json_falls_back_to_exact_defaults() {
|
||||
for json in ["", "{}", " ", " {} "] {
|
||||
let profile = QualityProfile::with_weights_override(ProfileKind::Tv, json);
|
||||
assert_eq!(profile.weights.resolution_tier, 3.0);
|
||||
assert_eq!(profile.weights.hdr, 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_override_changes_only_the_named_axis() {
|
||||
let profile = QualityProfile::with_weights_override(ProfileKind::Movie, r#"{"hdr": 5.0}"#);
|
||||
assert_eq!(profile.weights.hdr, 5.0);
|
||||
// Every other axis stays at the movie default, untouched.
|
||||
assert_eq!(profile.weights.resolution_tier, 3.0);
|
||||
assert_eq!(profile.weights.codec_tier, 2.0);
|
||||
assert_eq!(profile.weights.seeder, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_override_replaces_every_axis() {
|
||||
let json = r#"{"seeder": 9.0, "resolution_tier": 1.0, "source_tier": 1.0,
|
||||
"codec_tier": 9.0, "bit_depth": 9.0, "container": 9.0,
|
||||
"group_allowlist": 9.0, "repack": 9.0, "hdr": 9.0}"#;
|
||||
let profile = QualityProfile::with_weights_override(ProfileKind::Tv, json);
|
||||
assert_eq!(profile.weights.seeder, 9.0);
|
||||
assert_eq!(profile.weights.resolution_tier, 1.0);
|
||||
assert_eq!(profile.weights.codec_tier, 9.0);
|
||||
assert_eq!(profile.weights.hdr, 9.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_json_falls_back_to_defaults_instead_of_panicking() {
|
||||
let profile = QualityProfile::with_weights_override(ProfileKind::Tv, "not json at all");
|
||||
assert_eq!(profile.weights.resolution_tier, 3.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn movie_and_tv_still_get_different_hdr_defaults_with_no_override() {
|
||||
let movie = QualityProfile::with_weights_override(ProfileKind::Movie, "{}");
|
||||
let tv = QualityProfile::with_weights_override(ProfileKind::Tv, "{}");
|
||||
assert_eq!(movie.weights.hdr, 2.0);
|
||||
assert_eq!(tv.weights.hdr, 0.0);
|
||||
}
|
||||
}
|
||||
165
breadarrd/src/scoring/score.rs
Normal file
165
breadarrd/src/scoring/score.rs
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
use crate::parser::ParsedRelease;
|
||||
|
||||
use super::profile::{ProfileKind, QualityProfile};
|
||||
|
||||
pub fn score(parsed: &ParsedRelease, seeders: u32, has_hdr: bool, profile: &QualityProfile) -> f32 {
|
||||
let w = &profile.weights;
|
||||
let mut total = 0.0;
|
||||
|
||||
total += w.seeder * seeder_score(seeders);
|
||||
total += w.resolution_tier * resolution_tier(parsed.resolution);
|
||||
total += w.source_tier * parsed.source.map(|s| s as u8 as f32).unwrap_or(0.0);
|
||||
total += w.codec_tier * parsed.codec.map(|c| c as u8 as f32).unwrap_or(0.0);
|
||||
|
||||
if parsed.bit_depth == Some(10) {
|
||||
total += w.bit_depth;
|
||||
}
|
||||
if parsed.container.as_deref() == Some("mkv") {
|
||||
total += w.container;
|
||||
}
|
||||
if let Some(group) = &parsed.group {
|
||||
if profile
|
||||
.group_allowlist
|
||||
.iter()
|
||||
.any(|g| g.eq_ignore_ascii_case(group))
|
||||
{
|
||||
total += w.group_allowlist;
|
||||
}
|
||||
}
|
||||
if parsed.is_repack {
|
||||
total += w.repack;
|
||||
}
|
||||
if has_hdr && profile.kind == ProfileKind::Movie {
|
||||
total += w.hdr;
|
||||
}
|
||||
|
||||
total
|
||||
}
|
||||
|
||||
/// Log-scaled so 5 vs 50 seeders matters far more than 400 vs 4000.
|
||||
fn seeder_score(seeders: u32) -> f32 {
|
||||
((seeders as f32) + 1.0).ln()
|
||||
}
|
||||
|
||||
/// An unparsed resolution scores the same as the bottom tier — no signal
|
||||
/// either way, not a reason to reject, but not a reason to prefer it either.
|
||||
fn resolution_tier(resolution: Option<u32>) -> f32 {
|
||||
match resolution {
|
||||
Some(r) if r >= 2160 => 3.0,
|
||||
Some(r) if r >= 1080 => 2.0,
|
||||
Some(r) if r >= 720 => 1.0,
|
||||
_ => 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::parser;
|
||||
|
||||
#[test]
|
||||
fn av1_outscores_hevc_outscores_h264_all_else_equal() {
|
||||
let profile = QualityProfile::default_tv();
|
||||
let av1 = parser::parse("Show S01E01 1080p WEB-DL AV1");
|
||||
let hevc = parser::parse("Show S01E01 1080p WEB-DL HEVC");
|
||||
let h264 = parser::parse("Show S01E01 1080p WEB-DL H264");
|
||||
|
||||
let s_av1 = score(&av1, 50, false, &profile);
|
||||
let s_hevc = score(&hevc, 50, false, &profile);
|
||||
let s_h264 = score(&h264, 50, false, &profile);
|
||||
|
||||
assert!(s_av1 > s_hevc);
|
||||
assert!(s_hevc > s_h264);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remux_outscores_webrip_all_else_equal() {
|
||||
let profile = QualityProfile::default_movie();
|
||||
let remux = parser::parse("Movie 2024 2160p Remux H264");
|
||||
let webrip = parser::parse("Movie 2024 2160p WEBRip H264");
|
||||
assert!(score(&remux, 50, false, &profile) > score(&webrip, 50, false, &profile));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_seeders_scores_higher_but_with_diminishing_returns() {
|
||||
let profile = QualityProfile::default_tv();
|
||||
let parsed = parser::parse("Show S01E01 1080p WEB-DL H264");
|
||||
|
||||
let low = score(&parsed, 5, false, &profile);
|
||||
let mid = score(&parsed, 50, false, &profile);
|
||||
let high = score(&parsed, 4000, false, &profile);
|
||||
|
||||
assert!(mid > low);
|
||||
assert!(high > mid);
|
||||
// Diminishing returns on a log scale: the *same absolute* jump of
|
||||
// +45 seeders matters far more starting from 5 than starting from
|
||||
// 4000 (equal-ratio jumps like 5->50 vs 400->4000 are a different,
|
||||
// roughly-equal-delta comparison — not what's being tested here).
|
||||
let low_plus_45 = score(&parsed, 50, false, &profile) - low;
|
||||
let high_plus_45 = score(&parsed, 4045, false, &profile) - high;
|
||||
assert!(low_plus_45 > high_plus_45);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hdr_bonus_applies_to_movies_not_tv() {
|
||||
let parsed = parser::parse("Title 2024 2160p BluRay H264");
|
||||
let movie_profile = QualityProfile::default_movie();
|
||||
let tv_profile = QualityProfile::default_tv();
|
||||
|
||||
let movie_with_hdr = score(&parsed, 50, true, &movie_profile);
|
||||
let movie_without_hdr = score(&parsed, 50, false, &movie_profile);
|
||||
assert!(movie_with_hdr > movie_without_hdr);
|
||||
|
||||
let tv_with_hdr = score(&parsed, 50, true, &tv_profile);
|
||||
let tv_without_hdr = score(&parsed, 50, false, &tv_profile);
|
||||
assert_eq!(
|
||||
tv_with_hdr, tv_without_hdr,
|
||||
"HDR must not affect TV/anime scoring"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ten_bit_and_mkv_and_repack_each_add_a_bonus() {
|
||||
let profile = QualityProfile::default_tv();
|
||||
let base = parser::parse("Show S01E01 1080p WEB-DL H264");
|
||||
let ten_bit = parser::parse("Show S01E01 1080p WEB-DL H264 10bit");
|
||||
let mkv = parser::parse("Show S01E01 1080p WEB-DL H264.mkv");
|
||||
let repack = parser::parse("Show S01E01 1080p WEB-DL H264 REPACK");
|
||||
|
||||
let base_score = score(&base, 50, false, &profile);
|
||||
assert!(score(&ten_bit, 50, false, &profile) > base_score);
|
||||
assert!(score(&mkv, 50, false, &profile) > base_score);
|
||||
assert!(score(&repack, 50, false, &profile) > base_score);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn higher_resolution_outscores_lower_all_else_equal() {
|
||||
// The real scenario this guards: SubsPlease et al. publish 480p,
|
||||
// 720p, and 1080p of the same episode within the same feed
|
||||
// window — without a resolution term, these scored identically
|
||||
// apart from seeders, letting a later lower-res duplicate outscore
|
||||
// (and overwrite at import) an already-grabbed 1080p file.
|
||||
let profile = QualityProfile::default_tv();
|
||||
let p1080 = parser::parse("[SubsPlease] Show - 01 (1080p) [ABCD1234].mkv");
|
||||
let p720 = parser::parse("[SubsPlease] Show - 01 (720p) [ABCD1234].mkv");
|
||||
let p480 = parser::parse("[SubsPlease] Show - 01 (480p) [ABCD1234].mkv");
|
||||
|
||||
let s1080 = score(&p1080, 50, false, &profile);
|
||||
let s720 = score(&p720, 50, false, &profile);
|
||||
let s480 = score(&p480, 50, false, &profile);
|
||||
|
||||
assert!(s1080 > s720);
|
||||
assert!(s720 > s480);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allowlisted_group_scores_higher_than_unlisted() {
|
||||
let mut profile = QualityProfile::default_tv();
|
||||
profile.group_allowlist.push("SubsPlease".to_string());
|
||||
|
||||
let allowlisted = parser::parse("[SubsPlease] Show - 01 [1080p]");
|
||||
let unlisted = parser::parse("[RandomGroup] Show - 01 [1080p]");
|
||||
|
||||
assert!(score(&allowlisted, 50, false, &profile) > score(&unlisted, 50, false, &profile));
|
||||
}
|
||||
}
|
||||
83
breadarrd/src/sources/mod.rs
Normal file
83
breadarrd/src/sources/mod.rs
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
pub mod rss;
|
||||
pub mod scrape;
|
||||
pub mod tpb;
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RawReleaseItem {
|
||||
pub title: String,
|
||||
/// Magnet URI or direct .torrent download URL — qBittorrent's add-by-URL
|
||||
/// endpoint accepts either identically.
|
||||
pub link: String,
|
||||
pub guid: String,
|
||||
pub size_bytes: Option<u64>,
|
||||
pub seeders: Option<u32>,
|
||||
pub leechers: Option<u32>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ReleaseSource {
|
||||
/// Fetch current candidate releases. `query` is used by search-driven
|
||||
/// sources (e.g. a scraped search page); feed-based sources ignore it
|
||||
/// and return everything currently in the feed.
|
||||
async fn fetch(&self, query: Option<&str>) -> Result<Vec<RawReleaseItem>>;
|
||||
}
|
||||
|
||||
pub(crate) fn urlencode(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '~') {
|
||||
c.to_string()
|
||||
} else if c == ' ' {
|
||||
"%20".to_string()
|
||||
} else {
|
||||
let mut buf = [0u8; 4];
|
||||
c.encode_utf8(&mut buf)
|
||||
.bytes()
|
||||
.map(|b| format!("%{b:02X}"))
|
||||
.collect()
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn parse_human_size(s: &str) -> Option<u64> {
|
||||
let s = s.trim();
|
||||
let (num_part, unit) = s.split_once(' ')?;
|
||||
let num: f64 = num_part.parse().ok()?;
|
||||
let mult = match unit {
|
||||
"B" => 1.0,
|
||||
// 1337x labels these "KB/MB/GB" (decimal-looking) but the numbers
|
||||
// are actually binary (1024-based), matching every other torrent
|
||||
// site's convention — treat them the same as the KiB/MiB/GiB nyaa
|
||||
// uses.
|
||||
"KiB" | "KB" => 1024.0,
|
||||
"MiB" | "MB" => 1024.0 * 1024.0,
|
||||
"GiB" | "GB" => 1024.0 * 1024.0 * 1024.0,
|
||||
"TiB" | "TB" => 1024.0_f64.powi(4),
|
||||
_ => return None,
|
||||
};
|
||||
Some((num * mult) as u64)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_gib() {
|
||||
assert_eq!(parse_human_size("38.1 GiB"), Some(40909563494));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_mib() {
|
||||
assert_eq!(parse_human_size("356.5 MiB"), Some(373817344));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_unit() {
|
||||
assert_eq!(parse_human_size("5 XiB"), None);
|
||||
}
|
||||
}
|
||||
196
breadarrd/src/sources/rss.rs
Normal file
196
breadarrd/src/sources/rss.rs
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use quick_xml::escape::unescape;
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::reader::Reader;
|
||||
|
||||
use super::{parse_human_size, urlencode, RawReleaseItem, ReleaseSource};
|
||||
|
||||
/// RSS source for nyaa.si-style feeds, whose `nyaa:seeders`/`nyaa:leechers`/
|
||||
/// `nyaa:size` custom-namespaced fields carry the seeder data this project
|
||||
/// needs — generic feed libraries (e.g. feed-rs) don't surface arbitrary
|
||||
/// vendor namespaces, so this parses the raw XML directly.
|
||||
pub struct RssSource {
|
||||
feed_url: String,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl RssSource {
|
||||
pub fn new(feed_url: impl Into<String>) -> Self {
|
||||
Self {
|
||||
feed_url: feed_url.into(),
|
||||
// The background loop holds the DB mutex across this fetch — no
|
||||
// total timeout (reqwest's default) means a stalled connection
|
||||
// hangs the whole daemon, not just this one request.
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReleaseSource for RssSource {
|
||||
async fn fetch(&self, query: Option<&str>) -> Result<Vec<RawReleaseItem>> {
|
||||
let url = build_search_url(&self.feed_url, query);
|
||||
let bytes = self.client.get(&url).send().await?.bytes().await?;
|
||||
parse_nyaa_rss(&bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// nyaa's search *is* its RSS feed with a `q` param appended — same
|
||||
/// endpoint, same custom-namespace fields, so the parser needs no changes
|
||||
/// at all for search-driven use.
|
||||
fn build_search_url(feed_url: &str, query: Option<&str>) -> String {
|
||||
match query {
|
||||
Some(q) => {
|
||||
let sep = if feed_url.contains('?') { '&' } else { '?' };
|
||||
format!("{feed_url}{sep}q={}", urlencode(q))
|
||||
}
|
||||
None => feed_url.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_nyaa_rss(bytes: &[u8]) -> Result<Vec<RawReleaseItem>> {
|
||||
let mut reader = Reader::from_reader(bytes);
|
||||
reader.config_mut().trim_text(true);
|
||||
|
||||
let mut items = Vec::new();
|
||||
let mut buf = Vec::new();
|
||||
|
||||
let mut in_item = false;
|
||||
let mut cur_tag = String::new();
|
||||
let mut title = None;
|
||||
let mut link = None;
|
||||
let mut guid = None;
|
||||
let mut seeders = None;
|
||||
let mut leechers = None;
|
||||
let mut size_bytes = None;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf)? {
|
||||
Event::Eof => break,
|
||||
Event::Start(e) => {
|
||||
let name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
|
||||
if name == "item" {
|
||||
in_item = true;
|
||||
title = None;
|
||||
link = None;
|
||||
guid = None;
|
||||
seeders = None;
|
||||
leechers = None;
|
||||
size_bytes = None;
|
||||
}
|
||||
cur_tag = name;
|
||||
}
|
||||
Event::Text(t) if in_item => {
|
||||
let raw = t.decode()?;
|
||||
let text = unescape(&raw)?.into_owned();
|
||||
match cur_tag.as_str() {
|
||||
"title" => title = Some(text),
|
||||
"link" => link = Some(text),
|
||||
"guid" => guid = Some(text),
|
||||
"nyaa:seeders" => seeders = text.parse().ok(),
|
||||
"nyaa:leechers" => leechers = text.parse().ok(),
|
||||
"nyaa:size" => size_bytes = parse_human_size(&text),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Event::End(e) => {
|
||||
let name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
|
||||
if name == "item" {
|
||||
if let (Some(title), Some(link), Some(guid)) =
|
||||
(title.take(), link.take(), guid.take())
|
||||
{
|
||||
items.push(RawReleaseItem {
|
||||
title,
|
||||
link,
|
||||
guid,
|
||||
size_bytes,
|
||||
seeders,
|
||||
leechers,
|
||||
});
|
||||
}
|
||||
in_item = false;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:nyaa="https://nyaa.si/xmlns/nyaa" version="2.0">
|
||||
<channel>
|
||||
<title>Nyaa - Home - Torrent File RSS</title>
|
||||
<item>
|
||||
<title>[Group] Some Show - 05 [1080p]</title>
|
||||
<link>https://nyaa.si/download/2130903.torrent</link>
|
||||
<guid isPermaLink="true">https://nyaa.si/view/2130903</guid>
|
||||
<pubDate>Sat, 11 Jul 2026 08:51:04 -0000</pubDate>
|
||||
<nyaa:seeders>12</nyaa:seeders>
|
||||
<nyaa:leechers>3</nyaa:leechers>
|
||||
<nyaa:size>356.5 MiB</nyaa:size>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>"#;
|
||||
|
||||
#[test]
|
||||
fn parses_nyaa_item_with_custom_fields() {
|
||||
let items = parse_nyaa_rss(SAMPLE.as_bytes()).unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
let item = &items[0];
|
||||
assert_eq!(item.title, "[Group] Some Show - 05 [1080p]");
|
||||
assert_eq!(item.link, "https://nyaa.si/download/2130903.torrent");
|
||||
assert_eq!(item.guid, "https://nyaa.si/view/2130903");
|
||||
assert_eq!(item.seeders, Some(12));
|
||||
assert_eq!(item.leechers, Some(3));
|
||||
assert_eq!(item.size_bytes, Some(373817344));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_search_url_appends_query_param() {
|
||||
assert_eq!(
|
||||
build_search_url("https://nyaa.si/?page=rss&c=1_2", Some("Some Movie 2016")),
|
||||
"https://nyaa.si/?page=rss&c=1_2&q=Some%20Movie%202016"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_search_url_handles_a_feed_url_with_no_existing_query_string() {
|
||||
assert_eq!(
|
||||
build_search_url("https://nyaa.si/rss", Some("Some Movie")),
|
||||
"https://nyaa.si/rss?q=Some%20Movie"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_search_url_is_unchanged_without_a_query() {
|
||||
assert_eq!(
|
||||
build_search_url("https://nyaa.si/?page=rss", None),
|
||||
"https://nyaa.si/?page=rss"
|
||||
);
|
||||
}
|
||||
|
||||
/// Hits the real nyaa.si feed — not run by default. `cargo test -- --ignored`
|
||||
/// to sanity-check the parser against live markup if nyaa changes their format.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn parses_live_nyaa_feed() {
|
||||
let source = RssSource::new("https://nyaa.si/?page=rss");
|
||||
let items = source.fetch(None).await.unwrap();
|
||||
assert!(
|
||||
!items.is_empty(),
|
||||
"expected at least one item from live feed"
|
||||
);
|
||||
assert!(items.iter().any(|i| i.seeders.is_some()));
|
||||
}
|
||||
}
|
||||
549
breadarrd/src/sources/scrape.rs
Normal file
549
breadarrd/src/sources/scrape.rs
Normal file
|
|
@ -0,0 +1,549 @@
|
|||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
use super::{parse_human_size, urlencode, RawReleaseItem, ReleaseSource};
|
||||
|
||||
const BASE_COOLDOWN_SECS: u64 = 5 * 60;
|
||||
const MAX_COOLDOWN: Duration = Duration::from_secs(6 * 60 * 60);
|
||||
const RATE_LIMIT_MIN_COOLDOWN: Duration = Duration::from_secs(30 * 60);
|
||||
const RATE_LIMIT_MAX_RETRY_AFTER: Duration = Duration::from_secs(60 * 60);
|
||||
/// Hard cap on the tracked failure streak — well past the point where
|
||||
/// `BASE_COOLDOWN_SECS * 4^(streak-1)` already exceeds `MAX_COOLDOWN`, it
|
||||
/// exists purely so the exponent can never grow large enough to overflow.
|
||||
const MAX_FAILURE_STREAK: u32 = 10;
|
||||
|
||||
/// 1337x's main domain (1337x.to) bans IPs at the Cloudflare WAF level
|
||||
/// after bursts of automated traffic — a network-level block that no
|
||||
/// amount of browser-fingerprint evasion gets around. Its community
|
||||
/// mirrors run on separate domains/Cloudflare zones, so a ban on one
|
||||
/// doesn't carry over. Requests are round-robined across mirrors (not just
|
||||
/// tried in a fixed fallback order) so no single domain absorbs the bulk of
|
||||
/// traffic, and a mirror that fails is demoted into a cooldown rather than
|
||||
/// re-probed on the very next search.
|
||||
pub struct ScrapeSource {
|
||||
mirrors: Vec<String>,
|
||||
client: reqwest::Client,
|
||||
ring: Mutex<MirrorRing>,
|
||||
}
|
||||
|
||||
struct MirrorRing {
|
||||
next: usize,
|
||||
cooldown_until: Vec<Option<Instant>>,
|
||||
failure_streak: Vec<u32>,
|
||||
}
|
||||
|
||||
impl MirrorRing {
|
||||
fn new(count: usize) -> Self {
|
||||
Self {
|
||||
next: 0,
|
||||
cooldown_until: vec![None; count],
|
||||
failure_streak: vec![0; count],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Distinguishes *why* a mirror attempt failed, since the right cooldown
|
||||
/// differs: a WAF-issued rate-limit/block response means "this domain is
|
||||
/// now watching you" and gets a firm minimum cooldown regardless of streak,
|
||||
/// while a timeout or a challenge page gets the milder exponential ladder.
|
||||
#[derive(Debug)]
|
||||
enum MirrorError {
|
||||
/// No results table found at all — most likely a Cloudflare challenge
|
||||
/// page served with an HTTP 200 (so `error_for_status` wouldn't catch
|
||||
/// it), or the mirror's HTML layout has drifted.
|
||||
Challenge,
|
||||
/// 403/429/503 — an explicit throttle/block signal from the WAF, not a
|
||||
/// generic network failure.
|
||||
RateLimited {
|
||||
retry_after_secs: Option<u64>,
|
||||
},
|
||||
Other(anyhow::Error),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for MirrorError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
MirrorError::Challenge => write!(f, "no results table found (challenge page?)"),
|
||||
MirrorError::RateLimited { retry_after_secs } => {
|
||||
write!(f, "rate limited (retry_after={retry_after_secs:?})")
|
||||
}
|
||||
MirrorError::Other(e) => write!(f, "{e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returned by [`ScrapeSource::fetch`] when every mirror is either on
|
||||
/// cooldown or failed this attempt — a distinct type (rather than a plain
|
||||
/// string error) so callers can `downcast_ref` to trigger cycle-level
|
||||
/// backoff without string-matching an error message.
|
||||
#[derive(Debug)]
|
||||
pub struct AllMirrorsFailed;
|
||||
|
||||
impl std::fmt::Display for AllMirrorsFailed {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "all 1337x mirrors failed or are in cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AllMirrorsFailed {}
|
||||
|
||||
impl ScrapeSource {
|
||||
pub fn new(mirrors: Vec<String>) -> Self {
|
||||
let count = mirrors.len();
|
||||
Self {
|
||||
mirrors,
|
||||
client: reqwest::Client::builder()
|
||||
.user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
|
||||
.timeout(Duration::from_secs(30))
|
||||
// Cloudflare's cf_clearance cookie is per-zone (per mirror
|
||||
// domain) — holding it means a mirror that challenged once
|
||||
// can pass on subsequent requests instead of re-challenging
|
||||
// every single time.
|
||||
.cookie_store(true)
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
ring: Mutex::new(MirrorRing::new(count)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirror indices to try, starting from the round-robin cursor and
|
||||
/// wrapping, skipping any still in cooldown. Advances the cursor
|
||||
/// unconditionally (not just on success) so consecutive searches keep
|
||||
/// moving through the ring instead of retrying the same start point.
|
||||
fn candidate_order(&self) -> Vec<usize> {
|
||||
let mut ring = self.ring.lock().expect("mirror ring poisoned");
|
||||
let start = ring.next;
|
||||
if !self.mirrors.is_empty() {
|
||||
ring.next = (ring.next + 1) % self.mirrors.len();
|
||||
}
|
||||
compute_candidate_order(
|
||||
self.mirrors.len(),
|
||||
start,
|
||||
&ring.cooldown_until,
|
||||
Instant::now(),
|
||||
)
|
||||
}
|
||||
|
||||
fn record_success(&self, idx: usize) {
|
||||
let mut ring = self.ring.lock().expect("mirror ring poisoned");
|
||||
ring.failure_streak[idx] = 0;
|
||||
}
|
||||
|
||||
fn demote(&self, idx: usize, err: &MirrorError) {
|
||||
let mut ring = self.ring.lock().expect("mirror ring poisoned");
|
||||
let cooldown = match err {
|
||||
MirrorError::RateLimited { retry_after_secs } => {
|
||||
// A rate-limit response isn't generic flakiness — don't let
|
||||
// it ratchet the exponential streak, just apply a firm
|
||||
// minimum (or the server's own Retry-After, capped).
|
||||
let retry_after = retry_after_secs
|
||||
.map(|s| Duration::from_secs(s).min(RATE_LIMIT_MAX_RETRY_AFTER));
|
||||
retry_after
|
||||
.unwrap_or(RATE_LIMIT_MIN_COOLDOWN)
|
||||
.max(RATE_LIMIT_MIN_COOLDOWN)
|
||||
}
|
||||
MirrorError::Challenge | MirrorError::Other(_) => {
|
||||
let streak = (ring.failure_streak[idx] + 1).min(MAX_FAILURE_STREAK);
|
||||
ring.failure_streak[idx] = streak;
|
||||
let secs = BASE_COOLDOWN_SECS.saturating_mul(4u64.saturating_pow(streak - 1));
|
||||
Duration::from_secs(secs).min(MAX_COOLDOWN)
|
||||
}
|
||||
};
|
||||
ring.cooldown_until[idx] = Some(Instant::now() + cooldown);
|
||||
}
|
||||
|
||||
async fn search_mirror(
|
||||
&self,
|
||||
mirror: &str,
|
||||
query: &str,
|
||||
) -> std::result::Result<Vec<RawReleaseItem>, MirrorError> {
|
||||
let url = format!(
|
||||
"{}/search/{}/1/",
|
||||
mirror.trim_end_matches('/'),
|
||||
urlencode(query)
|
||||
);
|
||||
let resp = self.client.get(&url).send().await.map_err(|e| {
|
||||
MirrorError::Other(anyhow::Error::new(e).context(format!("request to {url} failed")))
|
||||
})?;
|
||||
|
||||
let status = resp.status();
|
||||
if matches!(status.as_u16(), 403 | 429 | 503) {
|
||||
let retry_after_secs = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok());
|
||||
return Err(MirrorError::RateLimited { retry_after_secs });
|
||||
}
|
||||
let resp = resp.error_for_status().map_err(|e| {
|
||||
MirrorError::Other(
|
||||
anyhow::Error::new(e).context(format!("{url} returned an error status")),
|
||||
)
|
||||
})?;
|
||||
let body = resp.text().await.map_err(|e| {
|
||||
MirrorError::Other(anyhow::Error::new(e).context("failed to read response body"))
|
||||
})?;
|
||||
parse_search_results(&body, mirror).ok_or(MirrorError::Challenge)
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirror indices to try, in round-robin order starting at `start`,
|
||||
/// excluding any whose cooldown hasn't elapsed yet. Pure and separately
|
||||
/// testable from the ring's locking/mutation.
|
||||
fn compute_candidate_order(
|
||||
len: usize,
|
||||
start: usize,
|
||||
cooldown_until: &[Option<Instant>],
|
||||
now: Instant,
|
||||
) -> Vec<usize> {
|
||||
if len == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
(0..len)
|
||||
.map(|offset| (start + offset) % len)
|
||||
.filter(|&i| cooldown_until[i].is_none_or(|until| now >= until))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// True if `link` is already directly usable by qBittorrent's add-by-URL
|
||||
/// endpoint (a magnet URI or a direct .torrent download link) — false for
|
||||
/// a detail-page URL that needs resolving first (what search results give
|
||||
/// us, since the results table doesn't carry the magnet directly).
|
||||
pub fn needs_resolution(link: &str) -> bool {
|
||||
!(link.starts_with("magnet:") || link.ends_with(".torrent"))
|
||||
}
|
||||
|
||||
/// The search results table only links to a torrent's detail page, not its
|
||||
/// magnet URI directly — fetches that page and extracts the magnet link.
|
||||
/// A free function (not tied to a `ScrapeSource`/mirror list) because the
|
||||
/// detail URL captured at search time already has the right mirror domain
|
||||
/// baked in, and this needs to be callable from the grab path generically
|
||||
/// (including review-queue approval, which only has a stored link, not a
|
||||
/// `ScrapeSource` instance) — called only for the one candidate actually
|
||||
/// being grabbed, not for every search result, to keep request volume low.
|
||||
pub async fn resolve_magnet(client: &reqwest::Client, detail_url: &str) -> Result<String> {
|
||||
let resp = client
|
||||
.get(detail_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("request to {detail_url} failed"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("{detail_url} returned an error status"))?;
|
||||
let body = resp.text().await.context("failed to read response body")?;
|
||||
extract_magnet(&body).with_context(|| format!("no magnet link found on {detail_url}"))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReleaseSource for ScrapeSource {
|
||||
async fn fetch(&self, query: Option<&str>) -> Result<Vec<RawReleaseItem>> {
|
||||
let Some(query) = query else {
|
||||
bail!(
|
||||
"ScrapeSource requires a search query (this is a search-driven source, not a feed)"
|
||||
);
|
||||
};
|
||||
if self.mirrors.is_empty() {
|
||||
bail!("no 1337x mirrors configured");
|
||||
}
|
||||
|
||||
for idx in self.candidate_order() {
|
||||
let mirror = &self.mirrors[idx];
|
||||
match self.search_mirror(mirror, query).await {
|
||||
Ok(items) => {
|
||||
self.record_success(idx);
|
||||
return Ok(items);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(mirror, error = %e, "1337x mirror failed, trying next");
|
||||
self.demote(idx, &e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(anyhow::Error::new(AllMirrorsFailed))
|
||||
}
|
||||
}
|
||||
|
||||
/// Pulls the numeric torrent ID out of a 1337x listing/detail href of the
|
||||
/// shape `/torrent/<id>/<slug>/` (relative) or `https://<mirror>/torrent/
|
||||
/// <id>/<slug>/` (absolute) — the one part of the URL that's identical
|
||||
/// across every mirror, unlike the domain. `None` if the href doesn't
|
||||
/// contain `/torrent/` at all (an unexpected HTML shape), in which case
|
||||
/// the caller falls back to the full URL rather than losing the item.
|
||||
fn extract_torrent_id(href: &str) -> Option<&str> {
|
||||
href.split("/torrent/")
|
||||
.nth(1)?
|
||||
.split('/')
|
||||
.next()
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
/// `None` when the results table itself is missing from the document —
|
||||
/// most likely a Cloudflare challenge page served with a 200 status (so it
|
||||
/// never trips `error_for_status`), or the mirror's HTML layout drifted.
|
||||
/// `Some(vec![])` is a genuine, trustworthy "no results" — those two cases
|
||||
/// must not be conflated, since the former should demote the mirror and
|
||||
/// the latter should not.
|
||||
fn parse_search_results(html: &str, mirror: &str) -> Option<Vec<RawReleaseItem>> {
|
||||
let doc = Html::parse_document(html);
|
||||
let table_sel = Selector::parse("table.table-list").unwrap();
|
||||
doc.select(&table_sel).next()?;
|
||||
|
||||
let row_sel = Selector::parse("table.table-list tbody tr").unwrap();
|
||||
let name_link_sel = Selector::parse("td.coll-1.name a:not(.icon)").unwrap();
|
||||
let seeds_sel = Selector::parse("td.coll-2").unwrap();
|
||||
let leeches_sel = Selector::parse("td.coll-3").unwrap();
|
||||
let size_sel = Selector::parse("td.coll-4").unwrap();
|
||||
|
||||
let mirror = mirror.trim_end_matches('/');
|
||||
let mut items = Vec::new();
|
||||
|
||||
for row in doc.select(&row_sel) {
|
||||
let Some(link_el) = row.select(&name_link_sel).next() else {
|
||||
continue;
|
||||
};
|
||||
let Some(href) = link_el.value().attr("href") else {
|
||||
continue;
|
||||
};
|
||||
let title: String = link_el.text().collect();
|
||||
let title = title.trim().to_string();
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let detail_url = if href.starts_with("http") {
|
||||
href.to_string()
|
||||
} else {
|
||||
format!("{mirror}{href}")
|
||||
};
|
||||
// The dedup key must not depend on which mirror answered: the same
|
||||
// physical torrent's `href` path (`/torrent/<id>/<slug>/`) is
|
||||
// identical across every mirror, but `detail_url` bakes in
|
||||
// whichever mirror happened to serve this particular search — with
|
||||
// ~10 mirrors round-robined for ban resilience, using the full URL
|
||||
// as `guid` meant the same release looked "new" up to 10 times
|
||||
// over, defeating `is_seen`/`mark_seen` dedup entirely and flooding
|
||||
// the review queue with duplicates of the same low-confidence
|
||||
// match (verified live: one title queued 13 times). The numeric
|
||||
// torrent ID is the stable, mirror-invariant identity instead.
|
||||
let guid = extract_torrent_id(href)
|
||||
.map(|id| format!("1337x:{id}"))
|
||||
.unwrap_or_else(|| detail_url.clone());
|
||||
|
||||
let seeders = row
|
||||
.select(&seeds_sel)
|
||||
.next()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse().ok());
|
||||
let leechers = row
|
||||
.select(&leeches_sel)
|
||||
.next()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse().ok());
|
||||
let size_bytes = row
|
||||
.select(&size_sel)
|
||||
.next()
|
||||
.and_then(|e| parse_human_size(e.text().collect::<String>().trim()));
|
||||
|
||||
items.push(RawReleaseItem {
|
||||
title,
|
||||
// Not directly grabbable yet — a detail-page URL, resolved to
|
||||
// a real magnet link via `resolve_magnet` right before the
|
||||
// winning candidate is actually sent to qBittorrent.
|
||||
link: detail_url,
|
||||
guid,
|
||||
size_bytes,
|
||||
seeders,
|
||||
leechers,
|
||||
});
|
||||
}
|
||||
|
||||
Some(items)
|
||||
}
|
||||
|
||||
fn extract_magnet(html: &str) -> Option<String> {
|
||||
let doc = Html::parse_document(html);
|
||||
let sel = Selector::parse(r#"a[href^="magnet:"]"#).unwrap();
|
||||
doc.select(&sel)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE_ROW: &str = r#"
|
||||
<table class="table-list table table-responsive table-striped">
|
||||
<thead><tr><th class="coll-1 name">name</th><th class="coll-2">se</th><th class="coll-3">le</th><th class="coll-date">time</th><th class="coll-4"><span class="size">size</span></th><th class="coll-5">uploader</th></tr></thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="coll-1 name"><a href="/sub/tv/HD/1/" class="icon"><i class="flaticon-hd"></i></a><a href="/torrent/3250239/The-Big-Bang-Theory-S12E01-720p-HDTV-x264-KILLERS-eztv/">The Big Bang Theory S12E01 720p HDTV x264-KILLERS [eztv]</a></td>
|
||||
<td class="coll-2 seeds">3978</td>
|
||||
<td class="coll-3 leeches">1443</td>
|
||||
<td class="coll-date">Sep. 25th '18</td>
|
||||
<td class="coll-4 size mob-uploader">559 MB</td>
|
||||
<td class="coll-5 uploader"><a href="/user/EZTVag/">EZTVag</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>"#;
|
||||
|
||||
#[test]
|
||||
fn parses_a_real_captured_result_row() {
|
||||
let items = parse_search_results(SAMPLE_ROW, "https://13377x.info").unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
let item = &items[0];
|
||||
assert_eq!(
|
||||
item.title,
|
||||
"The Big Bang Theory S12E01 720p HDTV x264-KILLERS [eztv]"
|
||||
);
|
||||
assert_eq!(
|
||||
item.link,
|
||||
"https://13377x.info/torrent/3250239/The-Big-Bang-Theory-S12E01-720p-HDTV-x264-KILLERS-eztv/"
|
||||
);
|
||||
assert_eq!(item.guid, "1337x:3250239");
|
||||
assert_eq!(item.seeders, Some(3978));
|
||||
assert_eq!(item.leechers, Some(1443));
|
||||
assert_eq!(item.size_bytes, Some(586153984));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guid_is_identical_across_different_mirrors_for_the_same_torrent() {
|
||||
let a = parse_search_results(SAMPLE_ROW, "https://13377x.info").unwrap();
|
||||
let b = parse_search_results(SAMPLE_ROW, "https://1337x.maskbay.info").unwrap();
|
||||
// The `link` legitimately differs (it's used to actually fetch from
|
||||
// whichever mirror answered this search) but the dedup `guid` must
|
||||
// not, or mirror rotation defeats `is_seen`/`mark_seen` entirely —
|
||||
// this was a real bug that flooded the review queue with up to 13
|
||||
// duplicate entries for the same release.
|
||||
assert_ne!(a[0].link, b[0].link);
|
||||
assert_eq!(a[0].guid, b[0].guid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_torrent_id_handles_relative_and_absolute_hrefs() {
|
||||
assert_eq!(
|
||||
extract_torrent_id("/torrent/3602010/Instant-Family-2018/"),
|
||||
Some("3602010")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_torrent_id("https://13377x.info/torrent/3602010/Instant-Family-2018/"),
|
||||
Some("3602010")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_torrent_id_is_none_for_an_unexpected_shape() {
|
||||
assert_eq!(extract_torrent_id("/sub/tv/HD/1/"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_none_when_results_table_is_absent() {
|
||||
let challenge_page = "<html><body><h1>Just a moment...</h1></body></html>";
|
||||
assert_eq!(
|
||||
parse_search_results(challenge_page, "https://13377x.info"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_empty_vec_for_a_genuine_no_results_page() {
|
||||
let empty_results = r#"<table class="table-list table table-responsive table-striped">
|
||||
<thead><tr><th class="coll-1 name">name</th></tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>"#;
|
||||
assert_eq!(
|
||||
parse_search_results(empty_results, "https://13377x.info"),
|
||||
Some(vec![])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_magnet_from_detail_page() {
|
||||
let html = r#"<a href="magnet:?xt=urn:btih:F30F455DC4C64A38E18C79C853B2A80B417C2343&dn=test">Magnet Download</a>"#;
|
||||
assert_eq!(
|
||||
extract_magnet(html).as_deref(),
|
||||
Some("magnet:?xt=urn:btih:F30F455DC4C64A38E18C79C853B2A80B417C2343&dn=test")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn urlencode_handles_spaces_and_unicode() {
|
||||
assert_eq!(urlencode("Big Buck Bunny"), "Big%20Buck%20Bunny");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn candidate_order_round_robins_from_start() {
|
||||
let cooldowns = vec![None, None, None, None];
|
||||
let order = compute_candidate_order(4, 2, &cooldowns, Instant::now());
|
||||
assert_eq!(order, vec![2, 3, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn candidate_order_skips_mirrors_still_in_cooldown() {
|
||||
let now = Instant::now();
|
||||
let cooldowns = vec![
|
||||
None,
|
||||
Some(now + Duration::from_secs(600)), // still cooling down
|
||||
None,
|
||||
Some(now - Duration::from_secs(1)), // cooldown already elapsed
|
||||
];
|
||||
let order = compute_candidate_order(4, 0, &cooldowns, now);
|
||||
assert_eq!(order, vec![0, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn candidate_order_empty_when_every_mirror_cooling_down() {
|
||||
let now = Instant::now();
|
||||
let cooldowns = vec![Some(now + Duration::from_secs(60)); 3];
|
||||
let order = compute_candidate_order(3, 0, &cooldowns, now);
|
||||
assert!(order.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn demote_applies_a_firm_minimum_cooldown_for_rate_limiting() {
|
||||
let source = ScrapeSource::new(vec!["https://a".into(), "https://b".into()]);
|
||||
source.demote(
|
||||
0,
|
||||
&MirrorError::RateLimited {
|
||||
retry_after_secs: None,
|
||||
},
|
||||
);
|
||||
let ring = source.ring.lock().unwrap();
|
||||
let until = ring.cooldown_until[0].expect("should be cooling down");
|
||||
assert!(until >= Instant::now() + Duration::from_secs(29 * 60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn demote_caps_rate_limit_retry_after_at_one_hour() {
|
||||
let source = ScrapeSource::new(vec!["https://a".into()]);
|
||||
source.demote(
|
||||
0,
|
||||
&MirrorError::RateLimited {
|
||||
retry_after_secs: Some(999_999),
|
||||
},
|
||||
);
|
||||
let ring = source.ring.lock().unwrap();
|
||||
let until = ring.cooldown_until[0].expect("should be cooling down");
|
||||
assert!(until <= Instant::now() + RATE_LIMIT_MAX_RETRY_AFTER + Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn demote_escalates_generic_failures_exponentially() {
|
||||
let source = ScrapeSource::new(vec!["https://a".into()]);
|
||||
source.demote(0, &MirrorError::Challenge);
|
||||
let first = source.ring.lock().unwrap().cooldown_until[0].unwrap();
|
||||
source.demote(0, &MirrorError::Challenge);
|
||||
let second = source.ring.lock().unwrap().cooldown_until[0].unwrap();
|
||||
assert!(second > first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_success_resets_failure_streak() {
|
||||
let source = ScrapeSource::new(vec!["https://a".into()]);
|
||||
source.demote(0, &MirrorError::Challenge);
|
||||
assert_eq!(source.ring.lock().unwrap().failure_streak[0], 1);
|
||||
source.record_success(0);
|
||||
assert_eq!(source.ring.lock().unwrap().failure_streak[0], 0);
|
||||
}
|
||||
}
|
||||
132
breadarrd/src/sources/tpb.rs
Normal file
132
breadarrd/src/sources/tpb.rs
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{urlencode, RawReleaseItem, ReleaseSource};
|
||||
|
||||
/// A community-run JSON API mirror of The Pirate Bay's search — unlike
|
||||
/// 1337x, this is a genuine machine-readable API (not HTML scraping), and
|
||||
/// unlike 1337x's HTML results table, `info_hash` is enough to build a
|
||||
/// magnet directly: no second detail-page fetch needed to resolve a link
|
||||
/// before grabbing. Also proved dramatically more precise in practice —
|
||||
/// its search actually ranks by relevance, where 1337x's is a pure
|
||||
/// seeder-count sort that buries genuine matches for common-word titles
|
||||
/// under unrelated, much-more-seeded content.
|
||||
pub struct TpbSource {
|
||||
api_url: String,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct TpbResult {
|
||||
id: String,
|
||||
name: String,
|
||||
info_hash: String,
|
||||
seeders: String,
|
||||
leechers: String,
|
||||
size: String,
|
||||
}
|
||||
|
||||
const TRACKERS: &[&str] = &[
|
||||
"udp://tracker.opentrackr.org:1337/announce",
|
||||
"udp://open.stealth.si:80/announce",
|
||||
"udp://tracker.torrent.eu.org:451/announce",
|
||||
"udp://tracker.openbittorrent.com:6969/announce",
|
||||
"udp://exodus.desync.com:6969/announce",
|
||||
];
|
||||
|
||||
fn build_magnet(info_hash: &str, name: &str) -> String {
|
||||
let mut magnet = format!("magnet:?xt=urn:btih:{info_hash}&dn={}", urlencode(name));
|
||||
for t in TRACKERS {
|
||||
magnet.push_str("&tr=");
|
||||
magnet.push_str(&urlencode(t));
|
||||
}
|
||||
magnet
|
||||
}
|
||||
|
||||
impl TpbSource {
|
||||
pub fn new(api_url: impl Into<String>) -> Self {
|
||||
Self {
|
||||
api_url: api_url.into(),
|
||||
// No total timeout is reqwest's default — the background loop
|
||||
// holds the DB mutex across this fetch, so a stalled connection
|
||||
// would hang the whole daemon.
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("reqwest client build"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReleaseSource for TpbSource {
|
||||
async fn fetch(&self, query: Option<&str>) -> Result<Vec<RawReleaseItem>> {
|
||||
let Some(query) = query else {
|
||||
anyhow::bail!(
|
||||
"TpbSource requires a search query (this is a search-driven source, not a feed)"
|
||||
);
|
||||
};
|
||||
let url = format!("{}?q={}", self.api_url, urlencode(query));
|
||||
let results: Vec<TpbResult> = self
|
||||
.client
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("request to {url} failed"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("{url} returned an error status"))?
|
||||
.json()
|
||||
.await
|
||||
.context("failed to parse apibay response as JSON")?;
|
||||
|
||||
Ok(results
|
||||
.into_iter()
|
||||
// A query with no matches returns a single sentinel row
|
||||
// (id="0", an all-zero info_hash) rather than an empty array —
|
||||
// has to be filtered out explicitly or it'd be treated as one
|
||||
// real (and completely bogus) result.
|
||||
.filter(|r| r.id != "0" && !r.info_hash.chars().all(|c| c == '0'))
|
||||
.map(|r| RawReleaseItem {
|
||||
title: r.name.clone(),
|
||||
link: build_magnet(&r.info_hash, &r.name),
|
||||
guid: r.info_hash,
|
||||
size_bytes: r.size.parse().ok(),
|
||||
seeders: r.seeders.parse().ok(),
|
||||
leechers: r.leechers.parse().ok(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn build_magnet_includes_hash_name_and_trackers() {
|
||||
let magnet = build_magnet("ABC123", "Some.Show.S01E01");
|
||||
assert!(magnet.starts_with("magnet:?xt=urn:btih:ABC123&dn=Some.Show.S01E01"));
|
||||
assert!(magnet.contains("tracker.opentrackr.org"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_real_captured_response() {
|
||||
let body = r#"[{"id":"51630137","name":"Modern Family S01 (1080p BluRay)","info_hash":"8F87C7C186172F17E35F4512BB1A3E93B614ADED","leechers":"309","seeders":"379","size":"5395577424","num_files":"28","username":"rjaa","added":"1629816694","status":"vip","category":"208","imdb":""}]"#;
|
||||
let results: Vec<TpbResult> = serde_json::from_str(body).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].name, "Modern Family S01 (1080p BluRay)");
|
||||
assert_eq!(results[0].seeders, "379");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filters_out_the_no_results_sentinel() {
|
||||
let body = r#"[{"id":"0","name":"No results returned","info_hash":"0000000000000000000000000000000000000000","leechers":"0","seeders":"0","size":"0","num_files":"0","username":"","added":"0","status":"","category":"0","imdb":""}]"#;
|
||||
let results: Vec<TpbResult> = serde_json::from_str(body).unwrap();
|
||||
let filtered: Vec<_> = results
|
||||
.into_iter()
|
||||
.filter(|r| r.id != "0" && !r.info_hash.chars().all(|c| c == '0'))
|
||||
.collect();
|
||||
assert!(filtered.is_empty());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue