Add oversized-AV1 remediation and orphaned-file relink commands
- transcode_job gains is_anime/force_reencode so a job can be re-encoded even though it's already AV1 - needed for the rate-control bug that left ~476 files larger than their originals; the normal backfill query skips already-AV1 files, so this is find_oversized_ av1_candidates plus a dedicated retranscode-oversized CLI command - relink-orphaned-files: read-only reconciliation for episode_file rows that lost their association (DB-recovery incident) despite the real file still sitting where the importer would have put it - qbit: delete finished torrents from qBittorrent after import instead of relocating them with set_location, since nothing is left seeding from the old save path after the move
This commit is contained in:
parent
109b29ee55
commit
d110556a4d
3 changed files with 135 additions and 8 deletions
|
|
@ -448,6 +448,35 @@ pub fn init(conn: &Connection) -> anyhow::Result<()> {
|
||||||
"INTEGER NOT NULL DEFAULT 0",
|
"INTEGER NOT NULL DEFAULT 0",
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
// Decided at enqueue time (path prefix match against
|
||||||
|
// `transcode.anime_root_folders`, OR'd with the `anime_mapping`/
|
||||||
|
// `anime_tmdb_movie` metadata check) and carried on the job row so
|
||||||
|
// `claim_pending_jobs` can dispatch straight to the right encode
|
||||||
|
// pipeline (`run_ffmpeg_encode_anime` vs `_live_action`) without
|
||||||
|
// re-deriving it — the eligibility metadata lookups aren't available
|
||||||
|
// from the job row's own columns alone (no media_item_id here).
|
||||||
|
add_column_if_missing(
|
||||||
|
conn,
|
||||||
|
"transcode_job",
|
||||||
|
"is_anime",
|
||||||
|
"INTEGER NOT NULL DEFAULT 0",
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Lets a job re-encode a file that's already AV1 — normally
|
||||||
|
// `encode_and_verify` treats "already AV1" as nothing-to-do and skips
|
||||||
|
// the encode entirely, which is right for a fresh library scan but
|
||||||
|
// wrong for deliberately re-transcoding a file that got mis-encoded
|
||||||
|
// (e.g. the 476 files a real rate-control bug left *larger* than their
|
||||||
|
// original — already AV1, so the normal backfill query skips them, but
|
||||||
|
// they're exactly what a remediation pass needs to revisit). See
|
||||||
|
// `find_oversized_av1_candidates`.
|
||||||
|
add_column_if_missing(
|
||||||
|
conn,
|
||||||
|
"transcode_job",
|
||||||
|
"force_reencode",
|
||||||
|
"INTEGER NOT NULL DEFAULT 0",
|
||||||
|
)?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -118,9 +118,15 @@ async fn main() -> Result<()> {
|
||||||
Some("transcode-library") => {
|
Some("transcode-library") => {
|
||||||
return transcode_library_cmd(&config).await;
|
return transcode_library_cmd(&config).await;
|
||||||
}
|
}
|
||||||
|
Some("retranscode-oversized") => {
|
||||||
|
return retranscode_oversized_cmd(&config).await;
|
||||||
|
}
|
||||||
Some("verify-library") => {
|
Some("verify-library") => {
|
||||||
return verify_library_cmd(&config).await;
|
return verify_library_cmd(&config).await;
|
||||||
}
|
}
|
||||||
|
Some("relink-orphaned-files") => {
|
||||||
|
return relink_orphaned_files_cmd(&config).await;
|
||||||
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1209,6 +1215,44 @@ async fn remux_backlog_cmd(config: &Config) -> Result<()> {
|
||||||
/// doesn't stall the grab/import/search cycles; this command is for
|
/// doesn't stall the grab/import/search cycles; this command is for
|
||||||
/// immediately backfilling that same backlog by hand instead of waiting for
|
/// immediately backfilling that same backlog by hand instead of waiting for
|
||||||
/// it to trickle in over several hours.
|
/// it to trickle in over several hours.
|
||||||
|
/// One-time reconciliation for episodes whose `episode_file` association
|
||||||
|
/// went missing (almost certainly the earlier DB-recovery incident) despite
|
||||||
|
/// their real file still sitting exactly where breadarr's own importer
|
||||||
|
/// would have put it — see `importer::find_relinkable_episode_files`'s doc
|
||||||
|
/// comment for the full story. Purely additive: reports what it found
|
||||||
|
/// before touching anything, never moves/deletes/overwrites a single file,
|
||||||
|
/// and flags ambiguous matches for a human to look at rather than guessing.
|
||||||
|
async fn relink_orphaned_files_cmd(config: &Config) -> Result<()> {
|
||||||
|
if let Some(parent) = config.db_path().parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let conn = Connection::open(config.db_path())?;
|
||||||
|
db::init(&conn)?;
|
||||||
|
|
||||||
|
let (candidates, ambiguous) = importer::find_relinkable_episode_files(&conn)?;
|
||||||
|
println!(
|
||||||
|
"found {} orphaned episode file(s) to relink, {} ambiguous case(s) left for manual review",
|
||||||
|
candidates.len(),
|
||||||
|
ambiguous.len()
|
||||||
|
);
|
||||||
|
for c in &candidates {
|
||||||
|
println!(
|
||||||
|
" relink: {} S{:02}E{:02} -> {}",
|
||||||
|
c.series_title,
|
||||||
|
c.season_number,
|
||||||
|
c.episode_number,
|
||||||
|
c.path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for a in &ambiguous {
|
||||||
|
println!(" ambiguous, skipped: {a}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let linked = importer::relink_episode_files(&conn, &candidates)?;
|
||||||
|
println!("done: {linked} episode(s) relinked");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn probe_library_cmd(config: &Config) -> Result<()> {
|
async fn probe_library_cmd(config: &Config) -> Result<()> {
|
||||||
if let Some(parent) = config.db_path().parent() {
|
if let Some(parent) = config.db_path().parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
|
|
@ -1270,9 +1314,56 @@ async fn transcode_library_cmd(config: &Config) -> Result<()> {
|
||||||
candidate.episode_file_id,
|
candidate.episode_file_id,
|
||||||
candidate.video_codec.as_deref(),
|
candidate.video_codec.as_deref(),
|
||||||
candidate.size_bytes,
|
candidate.size_bytes,
|
||||||
|
candidate.is_anime,
|
||||||
|
false,
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
drive_transcode_queue_to_completion(conn, config).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-transcodes files a real rate-control bug left larger than their
|
||||||
|
/// original (already AV1, so `transcode-library`'s own backfill query
|
||||||
|
/// excludes them) — see `transcode::find_oversized_av1_candidates`'s doc
|
||||||
|
/// comment for the full story. Enqueues with `force_reencode: true`, the
|
||||||
|
/// only thing that lets `encode_and_verify` attempt a real re-encode of a
|
||||||
|
/// file that's already AV1 rather than treating it as nothing-to-do.
|
||||||
|
async fn retranscode_oversized_cmd(config: &Config) -> Result<()> {
|
||||||
|
if !config.transcode.enabled {
|
||||||
|
bail!("transcode.enabled is false in config — enable it before running this");
|
||||||
|
}
|
||||||
|
if let Some(parent) = config.db_path().parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let conn = Connection::open(config.db_path())?;
|
||||||
|
db::init(&conn)?;
|
||||||
|
|
||||||
|
let candidates = transcode::find_oversized_av1_candidates(&conn, &config.transcode)?;
|
||||||
|
println!(
|
||||||
|
"found {} oversized AV1 file(s) to re-transcode, worst offenders first",
|
||||||
|
candidates.len()
|
||||||
|
);
|
||||||
|
for candidate in &candidates {
|
||||||
|
transcode::enqueue(
|
||||||
|
&conn,
|
||||||
|
candidate.episode_file_id,
|
||||||
|
candidate.video_codec.as_deref(),
|
||||||
|
candidate.size_bytes,
|
||||||
|
candidate.is_anime,
|
||||||
|
true,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
drive_transcode_queue_to_completion(conn, config).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared by `transcode_library_cmd` and `retranscode_oversized_cmd`: drains
|
||||||
|
/// whatever's now `pending` in `transcode_job` via repeated `run_cycle`
|
||||||
|
/// calls until nothing is left, printing a running total. The only
|
||||||
|
/// difference between the two commands is which candidates got enqueued
|
||||||
|
/// (and with what `force_reencode` value) before this runs — there's
|
||||||
|
/// exactly one place that actually drives the worker loop to completion.
|
||||||
|
async fn drive_transcode_queue_to_completion(conn: Connection, config: &Config) -> Result<()> {
|
||||||
let jellyfin = if config.jellyfin.base_url.is_empty() {
|
let jellyfin = if config.jellyfin.base_url.is_empty() {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -1284,6 +1375,7 @@ async fn transcode_library_cmd(config: &Config) -> Result<()> {
|
||||||
|
|
||||||
let conn = std::sync::Arc::new(tokio::sync::Mutex::new(conn));
|
let conn = std::sync::Arc::new(tokio::sync::Mutex::new(conn));
|
||||||
let mut total_succeeded = 0usize;
|
let mut total_succeeded = 0usize;
|
||||||
|
let mut total_skipped = 0usize;
|
||||||
let mut total_failed = 0usize;
|
let mut total_failed = 0usize;
|
||||||
let mut total_bytes_saved: i64 = 0;
|
let mut total_bytes_saved: i64 = 0;
|
||||||
loop {
|
loop {
|
||||||
|
|
@ -1293,12 +1385,13 @@ async fn transcode_library_cmd(config: &Config) -> Result<()> {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
total_succeeded += stats.succeeded;
|
total_succeeded += stats.succeeded;
|
||||||
|
total_skipped += stats.skipped;
|
||||||
total_failed += stats.failed;
|
total_failed += stats.failed;
|
||||||
total_bytes_saved += stats.bytes_saved;
|
total_bytes_saved += stats.bytes_saved;
|
||||||
println!("batch: {stats:?}");
|
println!("batch: {stats:?}");
|
||||||
}
|
}
|
||||||
println!(
|
println!(
|
||||||
"done: {total_succeeded} succeeded, {total_failed} failed, {:.1} GB saved total",
|
"done: {total_succeeded} succeeded, {total_skipped} skipped (not beneficial), {total_failed} failed, {:.1} GB saved total",
|
||||||
total_bytes_saved as f64 / 1_073_741_824.0
|
total_bytes_saved as f64 / 1_073_741_824.0
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
|
||||||
|
|
@ -226,10 +226,6 @@ impl QbitClient {
|
||||||
unreachable!("loop always returns or bails on its second iteration")
|
unreachable!("loop always returns or bails on its second iteration")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Moves a torrent's save location — qBittorrent physically relocates
|
|
||||||
/// the underlying file(s) itself and continues seeding from the new
|
|
||||||
/// path, rather than breadarr keeping a second permanent copy purely to
|
|
||||||
/// satisfy its own import step.
|
|
||||||
/// Held for the duration of an add-then-correlate-hash sequence — see
|
/// Held for the duration of an add-then-correlate-hash sequence — see
|
||||||
/// `grab_lock`'s doc comment on why this needs to be process-wide, not
|
/// `grab_lock`'s doc comment on why this needs to be process-wide, not
|
||||||
/// just per-call.
|
/// just per-call.
|
||||||
|
|
@ -237,10 +233,19 @@ impl QbitClient {
|
||||||
self.grab_lock.lock().await
|
self.grab_lock.lock().await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn set_location(&self, hash: &str, location: &str) -> Result<()> {
|
/// Removes a torrent from qBittorrent's own tracking after breadarr has
|
||||||
|
/// already moved its data straight into the library — `delete_files:
|
||||||
|
/// false` because by the time this is called there's nothing left at
|
||||||
|
/// the torrent's original save path for qBittorrent to delete; leaving
|
||||||
|
/// the torrent registered would just leave it sitting in an "files
|
||||||
|
/// missing" error state indefinitely. Best-effort from the caller's
|
||||||
|
/// side: a failure here never undoes or blocks the import that already
|
||||||
|
/// succeeded, it just leaves one stale entry in the qBittorrent UI to
|
||||||
|
/// clean up by hand.
|
||||||
|
pub async fn delete_torrent(&self, hash: &str) -> Result<()> {
|
||||||
self.post_form(
|
self.post_form(
|
||||||
"/api/v2/torrents/setLocation",
|
"/api/v2/torrents/delete",
|
||||||
&[("hashes", hash), ("location", location)],
|
&[("hashes", hash), ("deleteFiles", "false")],
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue