diff --git a/.forgejo/workflows/check.yml b/.forgejo/workflows/check.yml new file mode 100644 index 0000000..8b14fac --- /dev/null +++ b/.forgejo/workflows/check.yml @@ -0,0 +1,25 @@ +name: check + +# Fast-fail lint/test on short-lived work branches, before it ever reaches +# main and triggers a dev-track release build. +on: + push: + branches: ['feature/**', 'fix/**', 'main'] + pull_request: + +jobs: + check: + runs-on: [self-hosted, hestia] + steps: + - name: checkout + run: | + set -euo pipefail + rm -rf src && mkdir src + git clone --branch "${GITHUB_REF_NAME}" --depth 1 \ + "https://git.breadway.dev/${GITHUB_REPOSITORY}.git" src + + - name: clippy + run: cd src && bash ci/build.sh cargo clippy --workspace --all-targets --locked -- -D warnings + + - name: test + run: cd src && bash ci/build.sh cargo test --workspace --locked diff --git a/.forgejo/workflows/dev-release.yml b/.forgejo/workflows/dev-release.yml new file mode 100644 index 0000000..dc91327 --- /dev/null +++ b/.forgejo/workflows/dev-release.yml @@ -0,0 +1,99 @@ +name: dev release + +# Publishes a dev-track build on every push to `main` (the trunk branch — +# there is no separate `dev` branch). See bread-ecosystem's +# docs/release-channels.md for the release-track policy this is part of. +on: + push: + branches: ['main'] + +jobs: + build: + runs-on: [self-hosted, hestia] + steps: + - name: checkout + run: | + set -euo pipefail + rm -rf src && mkdir src + git clone --branch main --depth 1 \ + "https://git.breadway.dev/${GITHUB_REPOSITORY}.git" src + + - name: build + run: cd src && bash ci/build.sh cargo build --release --locked + + - name: compute dev version + run: | + set -euo pipefail + cd src + # Base the dev version off the latest published stable tag, not + # Cargo.toml — Cargo.toml can go stale relative to the last real + # release, which would make a dev build sort as OLDER than what's + # already installed and bakery would correctly refuse it. + LATEST_TAG="$(git ls-remote --tags --refs \ + "https://git.breadway.dev/${GITHUB_REPOSITORY}.git" 'v*' \ + | awk -F/ '{print $NF}' | sed 's/^v//' | (grep -v -- '-' || true) | sort -V | tail -1)" + if [ -n "${LATEST_TAG}" ]; then + CUR="${LATEST_TAG}" + else + # breadarr's own Cargo.toml is a virtual workspace manifest with + # no [workspace.package] version — breadarrd/Cargo.toml is the + # daemon crate's own version, used as the fallback instead. + CUR="$(grep -m1 '^version' breadarrd/Cargo.toml | sed -E 's/.*"(.*)".*/\1/')" + fi + IFS='.' read -r MA MI PA <<< "${CUR}" + SHA="$(git rev-parse --short HEAD)" + TS="$(date -u +%Y%m%d%H%M%S)" + echo "VERSION=${MA}.${MI}.$((PA + 1))-dev.${TS}+${SHA}" >> "$GITHUB_ENV" + + - name: prepare artifacts + run: | + set -euo pipefail + PKG_DIR="/srv/breadway-dl/dev/breadarr/${VERSION}" + mkdir -p "${PKG_DIR}" + for bin in breadarrd breadarr-tui; do + cp "src/target/release/${bin}" "${PKG_DIR}/${bin}-x86_64" + strip "${PKG_DIR}/${bin}-x86_64" + sha256sum "${PKG_DIR}/${bin}-x86_64" | awk '{print $1}' \ + > "${PKG_DIR}/${bin}-x86_64.sha256" + done + cp src/packaging/systemd/breadarrd.service "${PKG_DIR}/" + cp src/bakery.toml "${PKG_DIR}/bakery.toml" + ln -sfn "${VERSION}" "/srv/breadway-dl/dev/breadarr/latest" + + - name: sign dev binaries + env: + MINISIGN_SEC_KEY: ${{ secrets.BAKERY_MINISIGN_SEC_KEY_PATH }} + run: | + set -euo pipefail + PKG_DIR="/srv/breadway-dl/dev/breadarr/${VERSION}" + if [ -n "${MINISIGN_SEC_KEY:-}" ]; then + for bin in breadarrd breadarr-tui; do + minisign -W -S -s "${MINISIGN_SEC_KEY}" -m "${PKG_DIR}/${bin}-x86_64" \ + -x "${PKG_DIR}/${bin}-x86_64.minisig" /dev/null || true + # mktemp: a fixed clone path races when multiple repos' dev/rc + # workflows run close together on the same self-hosted runner. + ECOSYSTEM_CI_DIR="$(mktemp -d /tmp/bread-ecosystem-ci-XXXXXX)" + git clone --branch main https://git.breadway.dev/Breadway/bread-ecosystem.git "${ECOSYSTEM_CI_DIR}" + TRACK=dev bash "${ECOSYSTEM_CI_DIR}/scripts/gen-index.sh" + rm -rf "${ECOSYSTEM_CI_DIR}" diff --git a/.forgejo/workflows/rc-release.yml b/.forgejo/workflows/rc-release.yml new file mode 100644 index 0000000..779fb9d --- /dev/null +++ b/.forgejo/workflows/rc-release.yml @@ -0,0 +1,79 @@ +name: beta (rc) release + +# Publishes a beta-track build for any `vX.Y.Z-rc.N` prerelease tag pushed +# to `main` — there is no separate `beta` branch; "freezing" is just +# pausing pushes to main while an RC gets tested. See bread-ecosystem's +# docs/release-channels.md for the release-track policy. +on: + push: + tags: ['v*'] + +jobs: + build: + if: ${{ contains(github.ref_name, '-rc.') }} + runs-on: [self-hosted, hestia] + steps: + - name: checkout + run: | + set -euo pipefail + rm -rf src && mkdir src + git clone --branch "${GITHUB_REF_NAME}" --depth 1 \ + "https://git.breadway.dev/${GITHUB_REPOSITORY}.git" src + + - name: build + run: cd src && bash ci/build.sh cargo build --release --locked + + - name: prepare artifacts + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + PKG_DIR="/srv/breadway-dl/beta/breadarr/${VERSION}" + mkdir -p "${PKG_DIR}" + for bin in breadarrd breadarr-tui; do + cp "src/target/release/${bin}" "${PKG_DIR}/${bin}-x86_64" + strip "${PKG_DIR}/${bin}-x86_64" + sha256sum "${PKG_DIR}/${bin}-x86_64" | awk '{print $1}' \ + > "${PKG_DIR}/${bin}-x86_64.sha256" + done + cp src/packaging/systemd/breadarrd.service "${PKG_DIR}/" + cp src/bakery.toml "${PKG_DIR}/bakery.toml" + ln -sfn "${VERSION}" "/srv/breadway-dl/beta/breadarr/latest" + + - name: sign beta binaries + env: + MINISIGN_SEC_KEY: ${{ secrets.BAKERY_MINISIGN_SEC_KEY_PATH }} + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + PKG_DIR="/srv/breadway-dl/beta/breadarr/${VERSION}" + if [ -n "${MINISIGN_SEC_KEY:-}" ]; then + for bin in breadarrd breadarr-tui; do + minisign -W -S -s "${MINISIGN_SEC_KEY}" -m "${PKG_DIR}/${bin}-x86_64" \ + -x "${PKG_DIR}/${bin}-x86_64.minisig" /dev/null || true + # mktemp: a fixed clone path races when multiple repos' dev/rc + # workflows run close together on the same self-hosted runner. + ECOSYSTEM_CI_DIR="$(mktemp -d /tmp/bread-ecosystem-ci-XXXXXX)" + git clone https://git.breadway.dev/Breadway/bread-ecosystem.git "${ECOSYSTEM_CI_DIR}" + TRACK=beta bash "${ECOSYSTEM_CI_DIR}/scripts/gen-index.sh" + rm -rf "${ECOSYSTEM_CI_DIR}" diff --git a/.forgejo/workflows/release.yml b/.forgejo/workflows/release.yml new file mode 100644 index 0000000..9839f3a --- /dev/null +++ b/.forgejo/workflows/release.yml @@ -0,0 +1,79 @@ +name: release + +on: + push: + tags: ['v*'] + +jobs: + build: + if: ${{ !contains(github.ref_name, '-rc.') }} + runs-on: [self-hosted, hestia] + steps: + - name: checkout + run: | + set -euo pipefail + rm -rf src && mkdir src + git clone --branch "${GITHUB_REF_NAME}" --depth 1 \ + "https://git.breadway.dev/${GITHUB_REPOSITORY}.git" src + + - name: build + run: cd src && bash ci/build.sh cargo build --release --locked + + - name: prepare artifacts + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + PKG_DIR="/srv/breadway-dl/breadarr/${VERSION}" + mkdir -p "${PKG_DIR}" + for bin in breadarrd breadarr-tui; do + cp "src/target/release/${bin}" "${PKG_DIR}/${bin}-x86_64" + strip "${PKG_DIR}/${bin}-x86_64" + sha256sum "${PKG_DIR}/${bin}-x86_64" | awk '{print $1}' \ + > "${PKG_DIR}/${bin}-x86_64.sha256" + done + cp src/packaging/systemd/breadarrd.service "${PKG_DIR}/" + cp src/bakery.toml "${PKG_DIR}/bakery.toml" + ln -sfn "${VERSION}" "/srv/breadway-dl/breadarr/latest" + + # Signs with the shared bakery ecosystem signing key (same key that + # signs index.json). BAKERY_MINISIGN_SEC_KEY_PATH is a *path on this + # runner's disk* (hestia has persistent storage), not the key + # contents. Dormant (binaries ship unsigned) until that secret is + # provisioned for this repo. + - name: sign release binaries + env: + MINISIGN_SEC_KEY: ${{ secrets.BAKERY_MINISIGN_SEC_KEY_PATH }} + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + PKG_DIR="/srv/breadway-dl/breadarr/${VERSION}" + if [ -n "${MINISIGN_SEC_KEY:-}" ]; then + for bin in breadarrd breadarr-tui; do + minisign -W -S -s "${MINISIGN_SEC_KEY}" -m "${PKG_DIR}/${bin}-x86_64" \ + -x "${PKG_DIR}/${bin}-x86_64.minisig" /dev/null || true + # mktemp: a fixed clone path races when multiple repos' release + # workflows run close together on the same self-hosted runner. + ECOSYSTEM_CI_DIR="$(mktemp -d /tmp/bread-ecosystem-ci-XXXXXX)" + git clone https://git.breadway.dev/Breadway/bread-ecosystem.git "${ECOSYSTEM_CI_DIR}" + bash "${ECOSYSTEM_CI_DIR}/scripts/gen-index.sh" + rm -rf "${ECOSYSTEM_CI_DIR}" + + # No GitHub Release upload step — breadarr has no GitHub mirror, + # unlike most sibling bread-ecosystem repos. dl.breadway.dev is the + # only distribution point for this repo. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..acc1967 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +graphify-out/graph.json merge=graphify diff --git a/.gitignore b/.gitignore index c2594e0..1b8749f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,16 @@ /target +/.ci-old-glibc config.toml +breadarrd.toml *.db *.db-wal *.db-shm + +# Local hygiene notes (not for commit) +CLAUDE.md + +# Leftover source tarballs (never commit these) +**/src.tar.xz + +# graphify knowledge-graph output (local tool cache, not for commit) +graphify-out/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..959acd8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,53 @@ +# AGENTS.md — Repo hygiene + +Scope: this file covers *repo hygiene* — branching, remotes, CI, cleanup. It is not project documentation. + +This repo follows the branch/release workflow documented in `CONTRIBUTING.md` +— read and follow it for any git, branch, or release work here (the +single-trunk model, `feature/x`/`fix/x` branch naming, how RC tags work, +etc). Don't improvise a different workflow. The short version: there is one +long-lived branch, `main` — no `dev` or `beta` branch exists. `main` +auto-publishes a bakery dev-track build on every push. "Beta" and "stable" +are both just tags, not branches: push a `vX.Y.Z-rc.N` tag to publish a +beta-track build, push a plain `vX.Y.Z` tag to cut the signed stable +release. "Freezing" for stabilization means pausing pushes to `main`, not +moving a branch. + +## Product identity + +breadarr is a **homelab** product (single-daemon Sonarr/Radarr/Prowlarr +replacement: `breadarrd` + `breadarr-tui`). It is bakery-distributed +(`bakery install breadarr`), **not** baked into the BOS ISO, **not** a +GTK/desktop-shell app, and does **not** emit or subscribe to bread events. +bread's `KNOWN_APPS` list already reserves `"arr"`; do not start emitting +on that id unless an explicit integration pass asks for it. + +Do not add a bos-settings panel, a breadway.dev website entry, or an ISO +bake. Those are out of scope for this product. + +## Remotes +- `origin` — Forgejo (`git.breadway.dev`) only. Unlike most sibling + bread-ecosystem repos, this one has no GitHub mirror — don't assume a + `github` remote exists. Push `origin` only. + +## CI + +Workflows live under `.forgejo/workflows/` (not `.github/`): + +- `check.yml` — clippy + test on push to `feature/**` and `fix/**`, + before a change reaches `main` and triggers a dev-track release. +- `dev-release.yml` — bakery dev-track build on every push to `main`. +- `rc-release.yml` — bakery beta-track build on any `vX.Y.Z-rc.N` tag. +- `release.yml` — signed bakery stable release on any other `v*` tag + (skips tags containing `-rc.`). + +All four run on the self-hosted `hestia` runner. There is no GitHub +Release upload (no GitHub mirror). Do not claim this repo has no CI. + +## Cleanup +- Delete feature/fix branches once merged. Check with `git branch --merged main`. + +## Don't +- Don't embed credentials in remote URLs — SSH or a credential helper only. +- Don't emit bread events, add a settings panel, website entry, or ISO bake. +- Don't commit leftover source tarballs (`**/src.tar.xz`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f2c30f2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,98 @@ +# Contributing + +`breadarr` — single-daemon Sonarr+Radarr+Prowlarr replacement for a +homelab host (`breadarrd` + `breadarr-tui`). + +It is bakery-distributed (`bakery install breadarr`), **not** baked into +the BOS ISO, **not** a GTK/desktop-shell app, and does **not** emit bread +events. bread's `KNOWN_APPS` list reserves `"arr"`; this repo does not +emit on that id. + +Part of the bread ecosystem; this repo follows the same single-trunk +branch/release workflow as every other bakery-channel product. + +## Branches + +There is one long-lived branch: **`main`**. All day-to-day work lands here. +Every push to `main` automatically builds and publishes a **dev-track** +build (see Tracks below) — a real install you can test before cutting +anything more formal. + +New work — features and bug fixes alike — goes on a short-lived branch: + +``` +feature/ +fix/ +``` + +Branch off `main`, open a PR/push back into `main` when ready. Short-lived +branches get deleted on merge — they never accumulate the kind of drift a +second long-lived branch does. + +## The release cycle + +There's no separate `beta` or release branch — "stable" and "beta" are both +just **tags** on `main`, not branches that need to be kept in sync: + +1. Work accumulates on `main` via `feature/x` / `fix/x` branches. Each push + auto-publishes a dev build — install it with `bakery track set dev` and + `bakery update --all`, then fix anything broken with another push. +2. When you want to stabilize before a real release, tag a release + candidate: `git tag vX.Y.Z-rc.1 && git push origin vX.Y.Z-rc.1`. That + tag alone triggers a beta-track build — "freezing" is just pausing + pushes to `main` while you test it, not a branch operation. Cut + `-rc.2`, `-rc.3`, etc. for further fixes. +3. Once an RC has gone without issues, tag the real release: + `git tag vX.Y.Z && git push origin vX.Y.Z` — that's what triggers the + signed stable release build. + +This repo has no GitHub mirror. Push tags and branches to `origin` +(Forgejo) only. + +## Tracks, from a user's perspective + +``` +bakery track show # what you're currently on (defaults to stable) +bakery track set dev # or beta, or stable +bakery update --all # pull the latest build on your current track +``` + +| Track | What it is | Published from | +|--------|-----------|-----------------| +| `stable` | The last tagged release | a `vX.Y.Z` tag | +| `beta` | Latest release candidate | a `vX.Y.Z-rc.N` tag | +| `dev` | Bleeding edge | `main`, on every push | + +Dev versions are auto-computed (`X.Y.Z-dev.+`) from the +latest published stable tag, so they always sort as newer than what you +have installed — no manual version bumping needed. Beta versions are just +the RC tag itself (already valid semver, already sorts below the real +release it's a candidate for). + +## Local development + +```sh +cargo build --release --workspace --locked +cargo test --workspace --locked +cargo clippy --workspace --all-targets --locked -- -D warnings +``` + +Host tools the daemon shells out to: `mkvtoolnix-cli` (`mkvmerge`) and +`ffmpeg`/`ffprobe`. See `bakery.toml` and the README setup section. + +## CI + +- `check.yml` — clippy + test on push to `feature/**`, `fix/**`, and + `main`, and on pull requests. +- `dev-release.yml` — triggered on push to `main`. +- `rc-release.yml` — triggered on any `vX.Y.Z-rc.N` tag push. +- `release.yml` — triggered on any other `v*` tag push, cuts the actual + stable release. + +All CI runs on a self-hosted runner. See +[bread-ecosystem's docs/release-channels.md](https://git.breadway.dev/Breadway/bread-ecosystem/src/branch/main/docs/release-channels.md) +for the full policy, including how a new product gets wired onto these tracks. + +## Questions + +Open an issue on this repo's Forgejo tracker. diff --git a/Cargo.lock b/Cargo.lock index e1300da..6663c83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,8 @@ dependencies = [ [[package]] name = "bread-onnx" -version = "0.3.0" +version = "0.7.2" +source = "git+https://git.breadway.dev/Breadway/bread-ecosystem?tag=v0.7.2#30517f161724132cdeb658c04cf5e490be07ee73" dependencies = [ "anyhow", "bread-utils", @@ -191,7 +192,8 @@ dependencies = [ [[package]] name = "bread-utils" -version = "0.3.0" +version = "0.7.2" +source = "git+https://git.breadway.dev/Breadway/bread-ecosystem?tag=v0.7.2#30517f161724132cdeb658c04cf5e490be07ee73" dependencies = [ "dirs", "serde", @@ -200,7 +202,7 @@ dependencies = [ [[package]] name = "breadarr-shared" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "bread-utils", @@ -212,7 +214,7 @@ dependencies = [ [[package]] name = "breadarr-tui" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "breadarr-shared", @@ -226,7 +228,7 @@ dependencies = [ [[package]] name = "breadarrd" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "async-trait", @@ -236,6 +238,7 @@ dependencies = [ "chrono", "fastrand", "nix", + "openssl-sys", "ort", "quick-xml", "regex", @@ -1765,6 +1768,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.117" @@ -1773,6 +1785,7 @@ checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/README.md b/README.md index efa64cc..8ef047b 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,10 @@ A single-daemon Rust replacement for the Sonarr + Radarr + Prowlarr stack — on - **`breadarrd`** — the daemon. Watches sources, matches releases to your library, scores and grabs candidates, imports completed downloads, probes them for real ground-truth quality, and refreshes Jellyfin. Runs as a `systemd --user` service. - **`breadarr-tui`** — a terminal client (ratatui) that talks to `breadarrd`'s local HTTP API. No web UI, on purpose. +## Distribution + +breadarr is a **homelab** product. It is distributed through bakery (`bakery install breadarr`) and is **not** baked into the BOS ISO. It is not a GTK/desktop-shell app, has no bos-settings panel, and does not subscribe to or emit bread events. bread's `KNOWN_APPS` list reserves the `"arr"` app id for a possible future integration; this repo does not emit on that id. + ## Why this exists Sonarr + Radarr + Prowlarr is three separate services, three databases, three web UIs, and a lot of setup surface for one workflow: watch for releases, pick the best one, download it, file it correctly, tell Jellyfin. breadarr collapses that into a single daemon, with a few specific problems solved directly rather than configured around: @@ -22,13 +26,17 @@ Sonarr + Radarr + Prowlarr is three separate services, three databases, three we - **nyaa.si** (RSS) — anime TV, polled continuously. Full-auto, no request budget concerns (it's a plain RSS feed). - **apibay.org** (a community JSON API mirror of The Pirate Bay's search) — the primary search-driven source for general TV and movies. Unlike 1337x this is a genuine machine-readable API, needs no HTML scraping, and its search actually ranks by relevance rather than pure seeder count, which matters a lot for titles made of common words. -- **1337x** (scraped HTML, via community mirrors) — secondary search-driven source, tried after TPB. 1337x's main domain is Cloudflare-protected and has a ban history, so requests are round-robined across mirrors with automatic cooldown/backoff on failures, jittered between searches, and rate-limit responses are honored explicitly. -- **nyaa.si search mode** — anime movies specifically route here instead of TPB/1337x, since nyaa is the safe, official-RSS-interface target and gives materially better results for anime content. +- **torrents-csv** (JSON DHT-dump search) — first general-content fallback when TPB fails or returns nothing. Same hash-to-magnet grab as TPB; covers movies and TV. +- **YTS** (JSON `list_movies` API via `yts.lt` — `yts.mx` no longer resolves) — movie-only fallback after TPB. Each hit expands into one candidate per quality so the scorer sees 720p/1080p/2160p separately. +- **1337x** (scraped HTML, via community mirrors) — last-resort general-content fallback after the JSON sources. 1337x's main domain is Cloudflare-protected and has a ban history, so requests are round-robined across mirrors with automatic cooldown/backoff on failures, jittered between searches, and rate-limit responses are honored explicitly. +- **nyaa.si search mode** — anime movies specifically route here instead of the general-content chain, since nyaa is the safe, official-RSS-interface target and gives materially better results for anime content. -All three search-driven sources share one per-cycle request budget (default: 5 searches per 30-minute cycle, TPB tried first, then 1337x, then nyaa search) — kept conservative since 1337x has a ban history and the goal is steady backlog clearing, not maximum throughput. A whole cycle failing outright backs off the *next cycle's* interval (1h → 2h → 4h, capped), on top of each source's own per-mirror cooldowns. +Search-driven sources share one per-cycle request budget (default: 5 searches per 30-minute cycle). General content tries TPB, then YTS (movies) / torrents-csv, then 1337x; anime movies go to nyaa search. Kept conservative since 1337x has a ban history and the goal is steady backlog clearing, not maximum throughput. A whole cycle failing outright backs off the *next cycle's* interval (1h → 2h → 4h, capped), on top of each source's own per-mirror cooldowns. ## Setup +Install via bakery (`bakery install breadarr`) on a homelab host, or build from this repo. breadarr is not on the BOS ISO. + 1. Copy `config.example.toml` to `~/.config/breadarr/breadarrd.toml` and fill in: - qBittorrent WebUI URL/credentials - Jellyfin URL + API key @@ -46,24 +54,27 @@ All three search-driven sources share one per-cycle request budget (default: 5 s ## Using the TUI -`Tab` cycles Library / History / Review Queue / Add Show / Stuck / Calendar / Health. `j`/`k` or arrow keys navigate, `Enter` opens detail or runs a search, `Esc` backs out. +`Tab` / `Shift+Tab` cycle Library / History / Review / Add / Stuck / Calendar / Health / Profiles; `1`–`8` jump straight to a tab. `j`/`k` or arrows move, `g`/`G` jump to first/last, `PgUp`/`PgDn` page, `Enter` opens detail or runs a search, `Esc` backs out. `?` is the full key list. -- **Review Queue** — `a` approves and `r` rejects a pending low-confidence title match. Check this periodically, especially early on. -- **Library** (with an item's detail open) — `s` triggers an immediate search-now pass for that item's backlog; `m`/`e`/`S` toggle monitored on the show/episode/season respectively; `x` (confirm with a second `x`) removes the item from tracking without touching files on disk; `d` (confirm with a second `d`) deletes a bad imported file from disk and clears its tracking, freeing the episode/movie to be re-grabbed on the next cycle — the redownload path for a file that turned out to be wrong or broken; `c` fetches the manual release picker for the selected episode/movie, `Enter` grabs the highlighted candidate, `Esc` cancels. -- **Stuck** — surfaces grabs that look stalled (no download progress advancing, or missing from qBittorrent) before the daemon's own auto-fail timers would catch them. -- **Calendar** — upcoming/recently-aired episodes in a roughly week-either-side window. -- **Health** — the library-health report (see below) rendered as a tab instead of curled by hand. +- **Review** — `a` approves and `r` rejects a pending low-confidence title match. The tab title badges the pending count. Check this periodically, especially early on. +- **Library** — `/` incrementally filters the list by title; `f`/`o` cycle kind-filter and sort. With an item's detail open: `s` triggers an immediate search-now pass for that item's backlog; `n` jumps to the next missing monitored episode; `m`/`e`/`S` toggle monitored on the show/episode/season respectively; `x` (confirm with a second `x`) removes the item from tracking without touching files on disk; `d` (confirm with a second `d`) deletes a bad imported file from disk and clears its tracking, freeing the episode/movie to be re-grabbed on the next cycle — the redownload path for a file that turned out to be wrong or broken; `c` fetches the manual release picker for the selected episode/movie, `Enter` grabs the highlighted candidate, `Esc` cancels. Movies (no episode list) show a summary pane with the same keys. +- **Stuck** — surfaces grabs that look stalled (no download progress advancing, or missing from qBittorrent) before the daemon's own auto-fail timers would catch them. `Enter` jumps to that show. +- **Calendar** — upcoming/recently-aired episodes in a roughly week-either-side window. `Enter` opens the matching episode. +- **Health** — daemon cycle outcomes plus the library-health report (see below), scrollable. +- **Profiles** — quality-profile weight axes; open a profile and edit a weight in place. ## Operational notes -- `curl http://127.0.0.1:7879/health` reports daemon status plus the last grab/import/search/upgrade cycle outcome — the cheapest way to confirm the automation loop is actually alive. +- `curl http://127.0.0.1:7879/health` is liveness-only (is the process up). Cycle outcomes live at `/health/detail` (requires `Authorization: Bearer ` when `daemon.api_token` is set). - `curl http://127.0.0.1:7879/library/health` (or the TUI's Health tab) reports corrupt files, under-1080p files, missing-English-audio files, non-English-default-audio files, missing-subtitle files, duplicate-file groups, and library-wide summary stats (codec breakdown, resolution distribution, subtitle coverage %) — all derived from `ffprobe` data, not release-title claims. +- The API is plaintext HTTP — there is no TLS. Binding `listen_addr` off loopback requires a non-empty `daemon.api_token`. - If `daemon.api_token` is set, every route except `/health` requires `Authorization: Bearer `. - Library normalization is a manual, explicit action (not run automatically against your files): `breadarrd debug-scan-tv ` / `breadarrd debug-scan-movies `. - `breadarrd remux-backlog` sweeps the whole library for files with a non-English default audio track and applies the same track-promotion fix used automatically on fresh imports — a one-time (or occasional) pass against files that predate the fix, or were imported before breadarr started tracking them. - `breadarrd probe-library` backfills `ffprobe` data for the whole library in one run (the running daemon does this incrementally, a bounded batch per hour, so it doesn't stall the grab/import/search cycles — this command is for getting it all done immediately instead). - `breadarrd verify-library` runs the expensive full-decode corruption check (`ffmpeg -xerror`, actually decoding every frame) against every file whose cheap header probe succeeded but hasn't been decode-verified yet. This is opt-in and can take minutes per file, so — unlike `probe-library` — it's never run automatically by any ticker; run it by hand (or on a cron) whenever you want a real, not-just-header-parseable confirmation the library is intact. - Other `debug-*` subcommands are diagnostics for exercising one piece of the pipeline directly — run `breadarrd ` with no args to see its usage. Currently: `debug-qbit-add`, `debug-qbit-list`, `debug-jellyfin-refresh`, `debug-tvdb-add`, `debug-tvdb-search`, `debug-anime-map-refresh`, `debug-match-title`, `debug-grab-cycle`, `debug-import-cycle`, `debug-1337x-search`, `debug-scan-tv`, `debug-scan-movies`, `debug-search-show` (a manually-triggered, unthrottled search pass over one already-tracked title's whole backlog), `debug-reconcile-report` (dry-run of the disk-reconciliation pass — safe to run against a freshly-restored or otherwise suspect database before trusting the hourly ticker with it unattended). +- `breadarrd transcode-library` backfills AV1 transcode over existing library files (requires `[transcode] enabled = true`). `breadarrd retranscode-oversized` re-encodes already-AV1 files that landed larger than the current ceiling. `breadarrd relink-orphaned-files` reattaches episode files on disk that tracking lost. - The embedding model (~90MB) downloads automatically on first run. - The database is backed up (with its WAL/SHM sidecars) to `/backups/` on every daemon startup, keeping the 5 most recent copies — there's no migration framework, so this stands in for the pre-upgrade backup Sonarr/Radarr do on every schema change. - An hourly background pass reconciles tracked `episode_file` paths against what's actually on disk: a file renamed or transcoded in place (e.g. Tdarr converting codec/container) gets its path repaired rather than being wrongly treated as deleted; a file genuinely gone gets cleared from tracking so it becomes searchable again. A circuit breaker refuses to touch anything if an anomalous fraction of the library looks missing at once (the classic false signal of an offline mount), rather than mass-clearing tracking for files that are actually still there. @@ -71,8 +82,8 @@ All three search-driven sources share one per-cycle request budget (default: 5 s ## Known limitations - No subtitle generation yet — a Whisper-based reimplementation of an existing external tool is planned (see [Roadmap](#roadmap)), with the schema (`episode_file.subtitle_status`) already reserved for it. -- No Sonarr/Radarr API compatibility shim, so tools expecting that API (e.g. Overseerr/Seerr) can't integrate directly yet — also planned, with an empty `compat/` module reserved so it isn't a retrofit later. -- **Quality-scoring weights are hardcoded, not configurable.** Every axis — resolution tier, source tier, codec tier, bit depth, HDR, repack/proper bonus, and so on — is a fixed constant in `QualityProfile::default_tv`/`default_movie` (`scoring/profile.rs`). The `quality_profile` table and its `weights` JSON column already exist in the schema and are already parsed on load; `scoring/profile.rs` just doesn't read them yet, so every install currently gets the same scoring behavior regardless of what tradeoffs you'd actually prefer (e.g. valuing efficient codecs over raw resolution, or not caring about HDR at all). This is the single biggest gap between "works great for the specific setup it shipped with" and "works well for a range of libraries and preferences" — see [Roadmap](#roadmap). +- No Sonarr/Radarr API compatibility shim yet, so tools expecting that API (e.g. Overseerr/Seerr) can't integrate directly — still a future idea (see [Roadmap](#roadmap)). +- Quality-profile weights are loaded and editable (Profiles tab). `min_seeders` and the group denylist are still hardcoded defaults, not user-configurable. - The title-matching embedding model (all-MiniLM-L6-v2) doesn't actually discriminate between unrelated romanized-Japanese titles — it clusters any two romaji strings as "similar foreign text" regardless of content (verified live: several unrelated anime auto-matched at >0.85 confidence against a handful of "attractor" shows with zero real relation). A token-overlap gate (`MIN_TOKEN_OVERLAP` in `matcher/mod.rs`) blocks this from both auto-matching *and* reaching the review queue, but the underlying model limitation is a workaround, not a fix. - The search loop's per-cycle budget is intentionally conservative; raising it trades faster backlog clearing for more request volume against 1337x specifically, which has a ban history. The upgrade-search loop shares the same underlying sources and the same conservatism applies. @@ -80,10 +91,6 @@ All three search-driven sources share one per-cycle request budget (default: 5 s Everything above is shipped and running. This section is the honest, disciplined version of "what would this become if taken all the way" — grounded in subsystems that already exist, not a wishlist. Nothing here contradicts the project's two foundational design choices (a small hardcoded set of sources, not a general indexer-plugin architecture; a TUI, not a web UI) — anything that would require reconsidering either is flagged as such. -### Near-term (extends existing, working subsystems) - -- **Configurable quality-profile weights.** The most user-facing gap in the project today. Different people reasonably want different tradeoffs — some care most about resolution, some would rather have a smaller, more-efficient-codec file, some don't watch anything HDR and don't want it influencing scores at all — and right now every install gets identical hardcoded behavior. The `quality_profile.weights` column already exists in the schema and is already parsed as JSON on load; the gap is purely that `scoring/profile.rs` ignores it in favor of the `default_tv`/`default_movie` constants. Wiring the column through the scoring engine and exposing it as an editable profile in the TUI turns an already-half-built feature into a real one, without changing the scoring model's shape — same axes, same gates, just user-owned numbers instead of fixed ones. - ### Medium-term (closes the loop on data already being collected) - **Tdarr hand-off.** `media_file_probe` already has `video_codec`, `container_bitrate`, `video_bitrate`, and `hdr` per file — everything needed to generate a "these files are still H.264/high-bitrate and are good AV1-transcode candidates" list without re-scanning the filesystem. Whether that's a report the user acts on manually or a direct trigger into the existing Tdarr pipeline is a judgment call for whenever this is built — but the *data* side of this is already sitting in the database, unused. @@ -94,7 +101,7 @@ Everything above is shipped and running. This section is the honest, disciplined ### Longer-term (larger, still-plausible extensions) - **Learning from review-queue decisions.** Every approve/reject in the review queue is already a labeled example of "was this match actually correct." Logging outcomes (not just acting on them) and periodically comparing auto-tuned per-library confidence thresholds against the fixed `AUTO_MATCH_CONFIDENCE`/`MIN_TOKEN_OVERLAP` constants in `matcher/mod.rs` is a plausible way to let the matcher get measurably better over time for *this* library's actual title vocabulary — anime is the library segment most exposed to the token-overlap workaround today, so it's also the segment most likely to benefit first. This is a genuine judgment call (a wrong auto-tune silently degrades match quality with no review-queue visibility into it happening), not a slam dunk — worth prototyping as an offline analysis of logged decisions before it's ever allowed to write back to live thresholds. -- **Seerr/Overseerr compatibility shim, then real request fulfillment.** The `compat/` extension point has been reserved from the start for a Sonarr/Radarr v3-API-compatible shim, since that's what Seerr's client code expects. Once that shim exists, the natural next step isn't just protocol compatibility — it's tracking *fulfillment* end-to-end (a request maps to a `media_item`, which maps to `release`/`event_history` rows that already record exactly when and how it was grabbed and imported), giving a requester real status instead of Seerr's own best-effort polling. +- **Seerr/Overseerr compatibility shim, then real request fulfillment.** A Sonarr/Radarr v3-API-compatible shim is the integration path Seerr's client code expects. Once that shim exists, the natural next step isn't just protocol compatibility — it's tracking *fulfillment* end-to-end (a request maps to a `media_item`, which maps to `release`/`event_history` rows that already record exactly when and how it was grabbed and imported), giving a requester real status instead of Seerr's own best-effort polling. - **Whisper subtitle generation.** Fully speced already: reimplement the existing external Python/Whisper tool's exact behavior — "Full Subtitles" (everything transcribed) and "Foreign Parts" (segments where a translate-pass diverges from the transcribe-pass, via text-similarity diff at a 0.80 threshold) SRT tracks, muxed with `mkvmerge` — but gated and on-demand (triggered post-import only when a file lacks subtitles and has non-English/fallback audio), unlike the external tool's full-library batch sweep. `episode_file.subtitle_status` is already wired through the schema for this. The open technical questions are a Rust equivalent of Python's `difflib.SequenceMatcher` for the similarity diff, and which Whisper-in-Rust GPU backend story actually works well across the range of consumer GPU hardware this runs on in practice (CUDA/Metal/Vulkan-centric crates like `whisper-rs` have no strong story for e.g. Intel Arc or other OpenVINO-friendly hardware). ### Deliberately not on this list diff --git a/bakery.toml b/bakery.toml new file mode 100644 index 0000000..186ac4e --- /dev/null +++ b/bakery.toml @@ -0,0 +1,45 @@ +name = "breadarr" +description = "Single-daemon Sonarr+Radarr+Prowlarr replacement — release watching, matching, grabbing, importing, and a terminal UI, no web UI" +binaries = ["breadarrd", "breadarr-tui"] +# mkvtoolnix-cli / ffmpeg: `mkvmerge` and `ffprobe`/`ffmpeg` are shelled out +# to directly via Command::new() (breadarrd/src/importer/mkv.rs for the +# audio-track remux fix; breadarrd/src/importer/ffprobe.rs and +# breadarrd/src/transcode/mod.rs for media probing/corruption verification/ +# transcode-backlog work) rather than linked, so neither shows up in +# `ldd target/release/breadarrd` -- confirmed by grepping breadarrd/src for +# Command::new("mkvmerge"|"ffprobe"|"ffmpeg") and cross-checking against the +# README's own host-requirements instructions. The ONNX embedding model (the +# `ort` crate, used for fuzzy title matching) needs no system_deps entry: +# `ldd target/release/breadarrd` lists no libonnxruntime.so, and `strings` +# on the built binary is full of onnxruntime's own C++ symbol names -- +# confirming the `ort` crate's default strategy statically bundled its own +# onnxruntime build directly into breadarrd rather than dynamically linking +# the system onnxruntime-cpu package. TLS is vendored: breadarrd builds +# OpenSSL via openssl-sys's `vendored` feature, so the shipped binary does +# not depend on the host's libssl/libcrypto. libstdc++/libgcc/glibc/zlib/ +# zstd/brotli also appear in `ldd` but are omitted here the same way +# breadcast's bakery.toml omits them: glibc/gcc-libs are unavoidable +# base-system dependencies, and zlib/zstd/brotli are themselves +# transitive deps of curl/pacman already guaranteed present on any real +# Arch install. +system_deps = [ + "mkvtoolnix-cli", + "ffmpeg", +] +optional_system_deps = [] +bread_deps = [] +license_file = "LICENSE" + +[[service]] +unit = "breadarrd.service" +enable = true + +[config] +dir = "~/.config/breadarr" +example = "config.example.toml" + +[install] +post_install = [ + "loginctl enable-linger \"$USER\" || true", + "systemctl --user is-active --quiet breadarrd || systemctl --user start breadarrd", +] diff --git a/breadarr-shared/Cargo.toml b/breadarr-shared/Cargo.toml index 91a9a26..635eb41 100644 --- a/breadarr-shared/Cargo.toml +++ b/breadarr-shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "breadarr-shared" -version = "0.1.0" +version = "0.1.1" edition = "2021" [dependencies] @@ -9,5 +9,4 @@ anyhow.workspace = true toml.workspace = true reqwest.workspace = true chrono.workspace = true -# TODO(owner): switch to tag-pinned git dependency once bread-utils is merged and tagged, matching the bread-theme pattern -bread-utils = { path = "../../bread-ecosystem/bread-utils" } +bread-utils = { git = "https://git.breadway.dev/Breadway/bread-ecosystem", tag = "v0.7.2" } diff --git a/breadarr-shared/src/client.rs b/breadarr-shared/src/client.rs index 174caad..5496747 100644 --- a/breadarr-shared/src/client.rs +++ b/breadarr-shared/src/client.rs @@ -7,6 +7,7 @@ use crate::dto::{ SearchResult, StuckReport, UpdateQualityProfileWeightsRequest, WeightsDto, }; +#[derive(Clone)] pub struct DaemonClient { base_url: String, client: reqwest::Client, @@ -16,8 +17,9 @@ impl DaemonClient { /// `api_token` mirrors `config.daemon.api_token` server-side — empty /// means "no auth configured," so this stays a no-op default header /// rather than sending a meaningless empty bearer token on every - /// request. - pub fn new(base_url: impl Into, api_token: &str) -> Self { + /// request. A non-empty token that cannot be encoded as an HTTP header + /// is an error (not a silent unauthenticated client). + pub fn new(base_url: impl Into, api_token: &str) -> Result { let mut builder = reqwest::Client::builder() // A default so no request can hang the TUI forever with zero // feedback if the daemon is unreachable or a connection stalls. @@ -26,20 +28,23 @@ impl DaemonClient { // which overrides this. .timeout(std::time::Duration::from_secs(30)); if !api_token.is_empty() { + let token = api_token.replace(['\r', '\n'], ""); + anyhow::ensure!( + !token.is_empty(), + "daemon.api_token is non-empty but contains only CR/LF" + ); + let value = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")) + .context("daemon.api_token is not a valid HTTP header value")?; let mut headers = reqwest::header::HeaderMap::new(); - if let Ok(value) = - reqwest::header::HeaderValue::from_str(&format!("Bearer {api_token}")) - { - headers.insert(reqwest::header::AUTHORIZATION, value); - } + headers.insert(reqwest::header::AUTHORIZATION, value); builder = builder.default_headers(headers); } - Self { + Ok(Self { base_url: base_url.into(), client: builder .build() .expect("reqwest client builder should not fail with only a timeout/headers set"), - } + }) } pub async fn health(&self) -> Result { @@ -59,7 +64,7 @@ impl DaemonClient { pub async fn health_detail(&self) -> Result { let resp = self .client - .get(format!("{}/health", self.base_url)) + .get(format!("{}/health/detail", self.base_url)) .timeout(std::time::Duration::from_secs(2)) .send() .await diff --git a/breadarr-shared/src/config.rs b/breadarr-shared/src/config.rs index 1a9928b..b60c84a 100644 --- a/breadarr-shared/src/config.rs +++ b/breadarr-shared/src/config.rs @@ -23,21 +23,35 @@ pub struct Config { pub sources: SourcesConfig, #[serde(default)] pub notifications: NotificationsConfig, + #[serde(default)] + pub transcode: TranscodeConfig, } -/// Where the TUI's "add show" flow places new series by default. Sonarr/ -/// Radarr let you pick a root folder per add; a single configured default -/// is a reasonable v1 simplification — per-add picking can follow later. +/// Where the TUI's "add" flow places new series/movies by default. Sonarr/ +/// Radarr let you pick a root folder per add; a single configured default per +/// kind is a reasonable v1 simplification — per-add picking can follow later. +/// Series and movies need *separate* defaults (not one shared value) because +/// they live under different category roots on disk (e.g. `TV Shows/` vs +/// `Movies/`) — a real production bug had both kinds falling back to one +/// bare library root with no per-item subfolder, landing new grabs directly +/// in the library root instead of inside their own show/movie folder, +/// invisible to Jellyfin's per-category libraries. #[derive(Debug, Clone, Default, Deserialize)] pub struct LibraryConfig { #[serde(default = "default_root_folder")] pub default_root_folder: String, + #[serde(default = "default_movies_root_folder")] + pub movies_root_folder: String, } fn default_root_folder() -> String { "~/breadarr-library".to_string() } +fn default_movies_root_folder() -> String { + "~/breadarr-library/Movies".to_string() +} + #[derive(Debug, Clone, Deserialize)] pub struct SourcesConfig { /// 1337x's main domain bans IPs at the Cloudflare WAF level after @@ -55,6 +69,12 @@ pub struct SourcesConfig { pub nyaa_rss_url: String, #[serde(default = "default_grab_poll_interval_secs")] pub grab_poll_interval_secs: u64, + /// Human kill switch for the passive RSS-feed grab loop (nyaa, anime + /// only) — same reasoning as `search_enabled`/`upgrade_enabled`, kept + /// as its own flag since this loop watches a different source and can + /// need to be paused independently of the search-driven ones. + #[serde(default = "default_grab_enabled")] + pub grab_enabled: bool, #[serde(default = "default_import_poll_interval_secs")] pub import_poll_interval_secs: u64, #[serde(default = "default_search_poll_interval_secs")] @@ -77,6 +97,16 @@ pub struct SourcesConfig { /// titles made of common words. #[serde(default = "default_tpb_api_url")] pub tpb_api_url: String, + /// torrents.csv DHT-dump search — first fallback when TPB fails or + /// returns nothing. Same hash-to-magnet grab shape as TPB, covers + /// movies and TV. + #[serde(default = "default_torrents_csv_url")] + pub torrents_csv_url: String, + /// YTS v2 list_movies JSON — movie-only fallback after TPB / torrents-csv. + /// `yts.mx` does not resolve; this default is a working host as of + /// 2026-08-16. + #[serde(default = "default_yts_api_url")] + pub yts_api_url: String, /// Human kill switch for the upgrade-search loop, same reasoning as /// `search_enabled` — off by default would mean nothing ever improves, /// but a user who's happy with their current files (or wants to save @@ -108,11 +138,14 @@ impl Default for SourcesConfig { torrent_1337x_mirrors: default_1337x_mirrors(), nyaa_rss_url: default_nyaa_rss_url(), grab_poll_interval_secs: default_grab_poll_interval_secs(), + grab_enabled: default_grab_enabled(), import_poll_interval_secs: default_import_poll_interval_secs(), search_poll_interval_secs: default_search_poll_interval_secs(), search_budget_per_cycle: default_search_budget_per_cycle(), search_enabled: default_search_enabled(), tpb_api_url: default_tpb_api_url(), + torrents_csv_url: default_torrents_csv_url(), + yts_api_url: default_yts_api_url(), upgrade_enabled: default_upgrade_enabled(), upgrade_poll_interval_secs: default_upgrade_poll_interval_secs(), upgrade_budget_per_cycle: default_upgrade_budget_per_cycle(), @@ -125,6 +158,14 @@ fn default_tpb_api_url() -> String { "https://apibay.org/q.php".to_string() } +fn default_torrents_csv_url() -> String { + "https://torrents-csv.com/service/search".to_string() +} + +fn default_yts_api_url() -> String { + "https://yts.lt/api/v2/list_movies.json".to_string() +} + fn default_upgrade_enabled() -> bool { true } @@ -161,12 +202,17 @@ fn default_grab_poll_interval_secs() -> u64 { 300 } +fn default_grab_enabled() -> bool { + true +} + fn default_import_poll_interval_secs() -> u64 { 60 } fn default_1337x_mirrors() -> Vec { [ + "https://www.1337xx.to", "https://13377x.info", "https://13377x.email", "https://1337xto.info", @@ -197,12 +243,10 @@ pub struct DaemonConfig { #[serde(default = "default_model_dir")] pub model_dir: String, /// Bearer token required on every API request when non-empty. Empty - /// (the default) means auth is off entirely — `listen_addr` defaults to - /// loopback-only, so a fresh install isn't suddenly locked out of its - /// own unconfigured daemon. This matters once `listen_addr` is changed - /// to bind non-loopback (e.g. so a TUI on a different host on the same - /// tailnet can reach it) — without a token, that's unauthenticated - /// add/delete/search access to anyone who can reach the port. + /// (the default) means auth is off entirely — allowed only when + /// `listen_addr` is loopback. A non-loopback bind with an empty token + /// is rejected at load. When set, the token must be at least 16 + /// characters so a length-oracle of short guesses is useless. #[serde(default)] pub api_token: String, } @@ -266,6 +310,283 @@ pub struct JellyfinConfig { pub api_key: String, } +/// GPU-accelerated AV1 transcode: re-encodes freshly-grabbed and existing +/// library files down to a space-reasonable size instead of keeping +/// whatever the source release happened to be (REMUX, huge season packs, +/// etc). `enabled` defaults false — this needs a manual calibration pass +/// against real content on the target GPU before it's safe to run +/// unattended against a whole library. +#[derive(Debug, Clone, Deserialize)] +pub struct TranscodeConfig { + #[serde(default)] + pub enabled: bool, + /// Tight on purpose — the actual pace is bottlenecked by encode time + /// (minutes per file), not this interval; a short poll just means a + /// freshly-completed job's slot gets refilled promptly instead of + /// sitting idle for the rest of a longer interval. + #[serde(default = "default_transcode_poll_interval_secs")] + pub poll_interval_secs: u64, + #[serde(default = "default_vaapi_device")] + pub vaapi_device: String, + /// Applies to both pipelines identically when Jellyfin reports an + /// active transcoding session — deliberately not split per-pipeline; + /// when someone's actually watching something, both the GPU (which + /// they're using) and CPU (contending for the same box) should back off. + #[serde(default = "default_parallelism_min")] + pub parallelism_min: usize, + /// The total concurrent **live-action** (`av1_vaapi`, GPU-bound) stream + /// budget — not just "the batch job's cap", but the real ceiling the + /// GPU can sustain at all, batch work and live Jellyfin viewers + /// combined. `transcode::live_action_parallelism_for` subtracts however + /// many Jellyfin transcode sessions are actually active from this + /// number to get the batch job's actual parallelism each cycle, so + /// real viewers get exactly the headroom they need rather than the + /// batch job dropping to a flat minimum regardless of how many people + /// are watching. Deliberately separate from `parallelism_max_anime` — + /// the two pipelines contend for genuinely different hardware (GPU + /// encode engine vs CPU threads), so raising one shouldn't raise the + /// other. Empirically calibrated on Hestia's Arc A380: per-stream + /// throughput stays above 1.5x realtime through 7 concurrent streams, + /// crosses below it at 8 (aggregate throughput itself plateaus around + /// ~12x realtime from 5-6 streams on, i.e. the GPU's actual saturation + /// point) — see [[breadarr-av1-transcode]] for the full scaling-test + /// numbers. + #[serde(default = "default_parallelism_max")] + pub parallelism_max: usize, + /// Ramp-up ceiling for concurrent **anime** (`libsvtav1`, CPU-bound) + /// encode streams — capped separately from `parallelism_max` (see its + /// doc comment) precisely because a single shared cap would let "raise + /// GPU parallelism" accidentally also raise anime concurrency, and + /// anime jobs are CPU-thread-hungry (`anime_svtav1_max_threads` each) + /// in a way live-action jobs aren't. Kept at the original + /// conservative shared-cap default (2) since concurrent-anime-job + /// memory/CPU behavior at higher counts hasn't been load-tested the + /// way the live-action GPU path has. + #[serde(default = "default_parallelism_max_anime")] + pub parallelism_max_anime: usize, + /// The "looks fine, no complaints" calibration reference: a real + /// bitrate (Mbps, in kbps here) from content already in the library at + /// `reference_height` that the user is happy with. New AV1 encodes are + /// targeted relative to this, scaled by resolution and AV1's encoding + /// efficiency, rather than picking a bitrate out of thin air. + #[serde(default = "default_reference_bitrate_kbps")] + pub reference_bitrate_kbps: u32, + #[serde(default = "default_reference_height")] + pub reference_height: u32, + /// AV1 reaches equivalent perceived quality to HEVC at a meaningfully + /// lower bitrate — this factor is applied on top of the resolution + /// scaling so the AV1 target isn't just a like-for-like copy of the + /// HEVC/H264 reference bitrate. Conservative (not maximally aggressive) + /// on purpose: erring toward "still clearly smaller" over "as small as + /// AV1 could theoretically go" leaves margin against visible artifacts. + #[serde(default = "default_av1_efficiency_factor")] + pub av1_efficiency_factor: f32, + /// HDR10/Dolby Vision metadata preservation through the GPU encoder + /// hasn't been verified yet — excluded from both the backfill and the + /// post-grab path until that's specifically checked on a few samples. + #[serde(default = "default_exclude_hdr")] + pub exclude_hdr: bool, + /// Excludes the 2160p tier from the first pass for the same reason as + /// `exclude_hdr` (most current 4K content in this library is HDR + /// anyway) — revisit once HDR handling is confirmed safe. + #[serde(default = "default_exclude_min_height")] + pub exclude_min_height: u32, + /// `global_quality` for the live-action `av1_vaapi` `QVBR` encode — the + /// actual quality driver now that rate control is quality-based rather + /// than a flat bitrate target (see `run_ffmpeg_encode_live_action`). + /// `reference_bitrate_kbps`/`av1_efficiency_factor` still compute a + /// `-b:v`/`-maxrate`/`-bufsize` ceiling alongside this, so a source that's + /// already unusually efficient doesn't get inflated up toward the + /// ceiling — QVBR only spends up to it on content that actually needs it. + #[serde(default = "default_quality_live_action")] + pub quality_live_action: u32, + /// Root folder path prefixes (exact string prefix match against + /// `episode_file.path`) routed to the anime encode pipeline + /// (`run_ffmpeg_encode_anime`) instead of the live-action one, regardless + /// of `anime_mapping`/`anime_tmdb_movie` metadata coverage — path is a + /// more reliable signal than TVDB/TMDB anime-list membership, which has + /// real gaps (e.g. Avatar: The Last Airbender and some Dragon Ball movies + /// were missing from those tables and slipped through as "not anime"). + /// Empty by default (a no-op) — set per-deployment to match how the + /// library is actually organized. + #[serde(default)] + pub anime_root_folders: Vec, + /// CRF for the anime pipeline's software `libsvtav1` encode (0-63, lower + /// = higher quality/larger). No hardware AV1 10-bit encode entrypoint + /// exists on Hestia's Arc A380 (`vainfo` only lists `AV1Profile0`, + /// 8-bit) — anime needs true 10-bit output to avoid banding in the flat + /// gradients the art style is full of, so this pipeline trades GPU + /// offload for CPU-based `libsvtav1` specifically to get it. + #[serde(default = "default_quality_anime")] + pub quality_anime: u32, + /// `libsvtav1` preset (0-13, lower = slower/better compression AND more + /// memory-hungry — SVT-AV1's lookahead/reference buffering scales with + /// preset, not just thread count). Raised from an initial guess of 6 to + /// 10 after a real validation run hit a genuine kernel OOM: preset 6 on + /// a single 1080p anime episode grew to 9.3GB resident memory on + /// Hestia's 6-core/12-thread box. This runs as unattended background + /// work, so trading some compression efficiency for a much smaller, + /// safer memory footprint is the right call — see + /// `anime_svtav1_max_threads` for the other half of that fix. + #[serde(default = "default_anime_svtav1_preset")] + pub anime_svtav1_preset: u32, + /// Passed to `libsvtav1` as `-svtav1-params lp=N` — caps how many + /// worker threads it uses, independent of preset. More parallel workers + /// means more concurrently-buffered frames, so this is the other lever + /// (alongside `anime_svtav1_preset`) for bounding the encoder's peak + /// memory to something predictable regardless of how many cores the + /// host actually has. Default is conservative (well under a typical + /// modern host's core count) after the same OOM incident that raised + /// the preset default. + #[serde(default = "default_anime_svtav1_max_threads")] + pub anime_svtav1_max_threads: u32, + /// Hard floor on what counts as "worth keeping": a transcode whose + /// output isn't at least this fraction smaller than the original is + /// discarded (job marked `skipped`, original left untouched) rather than + /// swapped in. Exists because quality-driven rate control can still + /// occasionally produce an output that's the same size as or larger than + /// an already-efficient source — this is the invariant that makes that + /// safe regardless of how good the rate-control tuning is, after a real + /// incident where flat-bitrate VBR targeting silently produced files + /// *larger* than the original on the majority of a backfill. + #[serde(default = "default_min_size_reduction_pct")] + pub min_size_reduction_pct: f64, + /// Skip attempting a transcode at all (no GPU/CPU time spent) when the + /// source's current bitrate is already at or below this fraction of the + /// resolution-scaled ceiling (`target_bitrate_kbps`) — a strong signal + /// there's little room left to save, so it's not worth the encode time + /// to find out (the `min_size_reduction_pct` check above would reject + /// most of these anyway, this just avoids paying for that finding). + #[serde(default = "default_skip_below_ceiling_ratio")] + pub skip_below_ceiling_ratio: f64, + /// Size (seconds) of each of the three start/middle/end windows + /// `ffprobe::verify_decodable_sampled` actually decodes, instead of the + /// whole file — a full decode verification was measured as the actual + /// CPU bottleneck of a transcode cycle (400%+ CPU per job, dwarfing the + /// GPU encode time), not the encode itself. Bounds verification cost to + /// a small constant regardless of source length. + #[serde(default = "default_verify_sample_secs")] + pub verify_sample_secs: f64, +} + +impl Default for TranscodeConfig { + fn default() -> Self { + Self { + enabled: false, + poll_interval_secs: default_transcode_poll_interval_secs(), + vaapi_device: default_vaapi_device(), + parallelism_min: default_parallelism_min(), + parallelism_max: default_parallelism_max(), + parallelism_max_anime: default_parallelism_max_anime(), + reference_bitrate_kbps: default_reference_bitrate_kbps(), + reference_height: default_reference_height(), + av1_efficiency_factor: default_av1_efficiency_factor(), + exclude_hdr: default_exclude_hdr(), + exclude_min_height: default_exclude_min_height(), + quality_live_action: default_quality_live_action(), + anime_root_folders: Vec::new(), + quality_anime: default_quality_anime(), + anime_svtav1_preset: default_anime_svtav1_preset(), + anime_svtav1_max_threads: default_anime_svtav1_max_threads(), + min_size_reduction_pct: default_min_size_reduction_pct(), + skip_below_ceiling_ratio: default_skip_below_ceiling_ratio(), + verify_sample_secs: default_verify_sample_secs(), + } + } +} + +fn default_transcode_poll_interval_secs() -> u64 { + 60 +} + +fn default_vaapi_device() -> String { + "/dev/dri/renderD128".to_string() +} + +fn default_parallelism_min() -> usize { + 1 +} + +fn default_parallelism_max() -> usize { + // The measured total GPU budget (not "batch cap plus a static + // reservation") — `live_action_parallelism_for` dynamically subtracts + // real Jellyfin transcode sessions from this each cycle. Raised from + // an initial conservative guess of 2 after an actual concurrent-stream + // scaling test on Hestia's Arc A380 (see `parallelism_max`'s doc + // comment): per-stream throughput stays above 1.5x realtime through 7 + // total concurrent streams, crossing below at 8. + 7 +} + +fn default_parallelism_max_anime() -> usize { + // Kept at the original conservative shared-cap value — unlike + // `parallelism_max`, this hasn't been load-tested at higher counts. + // A real incident already showed a *single* uncapped anime job could + // hit 9.3GB resident memory; multiple concurrent anime jobs (each its + // own `anime_svtav1_max_threads`-sized thread pool) multiply both CPU + // thread contention and memory pressure in a way the GPU path doesn't + // have to worry about. Raise deliberately, per-deployment, only after + // watching `free -h` and CPU load under real concurrent-anime load. + 2 +} + +fn default_reference_bitrate_kbps() -> u32 { + 5320 +} + +fn default_reference_height() -> u32 { + 1080 +} + +fn default_av1_efficiency_factor() -> f32 { + 0.7 +} + +fn default_exclude_hdr() -> bool { + true +} + +fn default_exclude_min_height() -> u32 { + 2000 +} + +// Starting point for `av1_vaapi`'s `-global_quality` under `QVBR`, needs the +// same real-hardware calibration pass as the bitrate reference did — this is +// a reasonable guess (roughly x264/x265 "visually near-lossless" territory +// on the encoder's internal QP-like scale), not a measured value. +fn default_quality_live_action() -> u32 { + 26 +} + +// SVT-AV1 CRF starting point for the anime pipeline — slightly lower +// (higher quality) than the live-action guess above since flat-color/ +// gradient-heavy anime content shows banding more readily than live-action +// grain/texture does at the same nominal quality level. Also unvalidated +// against real hardware/content yet. +fn default_quality_anime() -> u32 { + 24 +} + +fn default_anime_svtav1_preset() -> u32 { + 10 +} + +fn default_anime_svtav1_max_threads() -> u32 { + 4 +} + +fn default_min_size_reduction_pct() -> f64 { + 0.10 +} + +fn default_skip_below_ceiling_ratio() -> f64 { + 0.5 +} + +fn default_verify_sample_secs() -> f64 { + 20.0 +} + /// TVDB v4 API key, exchanged for a short-lived JWT at request time. #[derive(Debug, Clone, Default, Deserialize)] pub struct TvdbConfig { @@ -289,9 +610,100 @@ impl Config { let raw = fs::read_to_string(&path)?; let cfg: Config = toml::from_str(&raw)?; + cfg.validate()?; Ok(cfg) } + /// Rejects values that are individually syntactically valid TOML but + /// make the daemon unsafe or the transcode pipeline's math nonsensical + /// — a typo here would otherwise only surface much later, or bind an + /// unauthenticated API on a reachable address. + fn validate(&self) -> Result<()> { + // `target_bitrate_kbps` divides by `reference_height` (via + // `reference_pixels`); zero makes that ratio `f64::INFINITY`, which + // saturates to `u32::MAX` on the cast back to `u32` — then + // `run_ffmpeg_encode_live_action`'s `bitrate_ceiling_kbps * 3` + // overflows that `u32::MAX` (panics in a debug build, silently + // wraps to a nonsense small value in release). + anyhow::ensure!( + self.transcode.reference_height > 0, + "transcode.reference_height must be greater than 0" + ); + // `is_beneficial` computes `original_bytes * (1.0 - + // min_size_reduction_pct)` as the max allowed output size — a + // negative value here would raise that ceiling *above* the + // original, letting a transcode that actually grew the file still + // count as "beneficial." That's the exact failure mode + // `min_size_reduction_pct` exists to prevent (see its own doc + // comment: a real incident where flat-bitrate VBR silently produced + // files larger than the original). + anyhow::ensure!( + (0.0..=1.0).contains(&self.transcode.min_size_reduction_pct), + "transcode.min_size_reduction_pct must be between 0.0 and 1.0" + ); + anyhow::ensure!( + (0.0..=1.0).contains(&self.transcode.skip_below_ceiling_ratio), + "transcode.skip_below_ceiling_ratio must be between 0.0 and 1.0" + ); + anyhow::ensure!( + self.transcode.parallelism_min >= 1, + "transcode.parallelism_min must be at least 1" + ); + anyhow::ensure!( + self.transcode.parallelism_max >= self.transcode.parallelism_min, + "transcode.parallelism_max must be >= parallelism_min" + ); + anyhow::ensure!( + self.transcode.parallelism_max_anime >= 1, + "transcode.parallelism_max_anime must be at least 1" + ); + anyhow::ensure!( + (0..=63).contains(&self.transcode.quality_anime), + "transcode.quality_anime must be between 0 and 63" + ); + anyhow::ensure!( + (0..=13).contains(&self.transcode.anime_svtav1_preset), + "transcode.anime_svtav1_preset must be between 0 and 13" + ); + anyhow::ensure!( + self.transcode.verify_sample_secs.is_finite() + && self.transcode.verify_sample_secs > 0.0, + "transcode.verify_sample_secs must be greater than 0" + ); + const LOG_LEVELS: &[&str] = &["error", "warn", "info", "debug", "trace", "off"]; + anyhow::ensure!( + LOG_LEVELS + .iter() + .any(|l| self.daemon.log_level.eq_ignore_ascii_case(l)), + "daemon.log_level must be one of error, warn, info, debug, trace, off" + ); + if !self.daemon.api_token.is_empty() { + anyhow::ensure!( + self.daemon.api_token.len() >= 16, + "daemon.api_token must be at least 16 characters when set" + ); + } + if self.daemon.api_token.is_empty() && !is_loopback_listen_addr(&self.daemon.listen_addr) { + anyhow::bail!( + "daemon.api_token is required when listen_addr ({}) is not loopback", + self.daemon.listen_addr + ); + } + Ok(()) + } + + /// `true` when `daemon.listen_addr` is loopback (`127.0.0.1`, `::1`, + /// `localhost`). Used at startup to decide whether an empty token is + /// merely a local-process warning or already refused by `validate`. + pub fn listen_is_loopback(&self) -> bool { + is_loopback_listen_addr(&self.daemon.listen_addr) + } + + /// Expand a `~/...` path the same way configured library roots are. + pub fn expand_path(input: &str) -> PathBuf { + expand_home(input) + } + pub fn db_path(&self) -> PathBuf { expand_home(&self.daemon.db_path) } @@ -303,6 +715,10 @@ impl Config { pub fn default_root_folder(&self) -> PathBuf { expand_home(&self.library.default_root_folder) } + + pub fn movies_root_folder(&self) -> PathBuf { + expand_home(&self.library.movies_root_folder) + } } fn config_path() -> PathBuf { @@ -328,6 +744,29 @@ fn expand_home(input: &str) -> PathBuf { PathBuf::from(input) } +/// Loopback hosts we allow to bind without an API token: IPv4/IPv6 +/// loopback socket addresses, plus the `localhost` hostname form. +fn is_loopback_listen_addr(listen_addr: &str) -> bool { + if let Ok(addr) = listen_addr.parse::() { + return addr.ip().is_loopback(); + } + let host = if let Some(rest) = listen_addr.strip_prefix('[') { + rest.split(']').next().unwrap_or(rest) + } else if let Some((h, port)) = listen_addr.rsplit_once(':') { + if port.parse::().is_ok() && !h.contains(':') { + h + } else { + listen_addr + } + } else { + listen_addr + }; + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) +} + fn default_log_level() -> String { "info".to_string() } @@ -378,4 +817,132 @@ mod tests { assert_eq!(cfg.daemon.log_level, "debug"); assert_eq!(cfg.daemon.listen_addr, "127.0.0.1:7879"); } + + #[test] + fn default_config_passes_validation() { + Config::default().validate().unwrap(); + } + + // Regression test for a real gap found in review: `reference_height = + // 0` makes `target_bitrate_kbps`'s resolution-scaling ratio divide by + // zero, which eventually overflows a `u32` multiplication deep inside + // the live-action encoder's maxrate calculation — a config typo that + // used to only surface as a panic/garbage value in the middle of an + // encode, not at startup. + #[test] + fn rejects_a_zero_reference_height() { + let cfg: Config = toml::from_str("[transcode]\nreference_height = 0\n").unwrap(); + assert!(cfg.validate().is_err()); + } + + // Regression test for a real gap found in review: a negative + // `min_size_reduction_pct` would let `is_beneficial` accept an encode + // that actually *grew* the file — the exact invariant this field exists + // to guarantee against. + #[test] + fn rejects_a_negative_min_size_reduction_pct() { + let cfg: Config = toml::from_str("[transcode]\nmin_size_reduction_pct = -0.1\n").unwrap(); + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_a_min_size_reduction_pct_above_one() { + let cfg: Config = toml::from_str("[transcode]\nmin_size_reduction_pct = 1.5\n").unwrap(); + assert!(cfg.validate().is_err()); + } + + #[test] + fn default_loopback_with_empty_token_is_ok() { + Config::default().validate().unwrap(); + assert!(is_loopback_listen_addr("127.0.0.1:7879")); + assert!(is_loopback_listen_addr("localhost:7879")); + assert!(is_loopback_listen_addr("[::1]:7879")); + assert!(is_loopback_listen_addr("::1")); + } + + #[test] + fn non_loopback_without_token_is_rejected() { + let mut cfg = Config::default(); + cfg.daemon.listen_addr = "0.0.0.0:7879".into(); + assert!(cfg.validate().is_err()); + } + + #[test] + fn non_loopback_with_token_is_ok() { + let mut cfg = Config::default(); + cfg.daemon.listen_addr = "0.0.0.0:7879".into(); + cfg.daemon.api_token = "a-token-16-chars+".into(); + cfg.validate().unwrap(); + } + + #[test] + fn short_api_token_is_rejected() { + let mut cfg = Config::default(); + cfg.daemon.api_token = "tooshort".into(); + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_invalid_log_level() { + let mut cfg = Config::default(); + cfg.daemon.log_level = "loud".into(); + assert!(cfg.validate().is_err()); + } + + #[test] + fn accepts_off_log_level() { + let mut cfg = Config::default(); + cfg.daemon.log_level = "off".into(); + cfg.validate().unwrap(); + } + + #[test] + fn rejects_skip_below_ceiling_ratio_out_of_range() { + let mut cfg = Config::default(); + cfg.transcode.skip_below_ceiling_ratio = 1.5; + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_zero_parallelism_min() { + let mut cfg = Config::default(); + cfg.transcode.parallelism_min = 0; + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_parallelism_max_below_min() { + let mut cfg = Config::default(); + cfg.transcode.parallelism_min = 3; + cfg.transcode.parallelism_max = 2; + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_zero_parallelism_max_anime() { + let mut cfg = Config::default(); + cfg.transcode.parallelism_max_anime = 0; + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_quality_anime_out_of_range() { + let mut cfg = Config::default(); + cfg.transcode.quality_anime = 64; + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_anime_svtav1_preset_out_of_range() { + let mut cfg = Config::default(); + cfg.transcode.anime_svtav1_preset = 14; + assert!(cfg.validate().is_err()); + } + + #[test] + fn rejects_zero_verify_sample_secs() { + let mut cfg = Config::default(); + cfg.transcode.verify_sample_secs = 0.0; + assert!(cfg.validate().is_err()); + } } diff --git a/breadarr-shared/src/dto.rs b/breadarr-shared/src/dto.rs index 72e330b..7289b6c 100644 --- a/breadarr-shared/src/dto.rs +++ b/breadarr-shared/src/dto.rs @@ -56,6 +56,7 @@ pub struct ReviewQueueEntry { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StalledGrab { pub release_id: i64, + pub media_item_id: i64, pub media_title: String, pub raw_title: String, pub grabbed_at: String, @@ -135,6 +136,12 @@ pub struct CalendarEntry { pub has_file: bool, } +/// Cheap liveness payload for unauthenticated `GET /health`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthStatus { + pub status: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealthDetail { pub status: String, @@ -142,6 +149,7 @@ pub struct HealthDetail { pub last_import_cycle: Option, pub last_search_cycle: Option, pub last_upgrade_cycle: Option, + pub last_transcode_cycle: Option, pub search_halted: bool, } diff --git a/breadarr-tui/Cargo.toml b/breadarr-tui/Cargo.toml index 90bcac8..0617ee8 100644 --- a/breadarr-tui/Cargo.toml +++ b/breadarr-tui/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "breadarr-tui" -version = "0.1.0" +version = "0.1.1" edition = "2021" [dependencies] diff --git a/breadarr-tui/src/app.rs b/breadarr-tui/src/app.rs index f3b1ac5..b991885 100644 --- a/breadarr-tui/src/app.rs +++ b/breadarr-tui/src/app.rs @@ -1,11 +1,52 @@ use anyhow::Result; use breadarr_shared::dto::{ - CalendarEntry, HealthDetail, LibraryHealthReport, MediaItemDetail, MediaItemSummary, - QualityProfileSummary, ReleaseCandidate, ReleaseSummary, ReviewQueueEntry, SearchResult, - StuckReport, WeightsDto, + CalendarEntry, EpisodeSummary, FlaggedFile, HealthDetail, LibraryHealthReport, MediaItemDetail, + MediaItemSummary, QualityProfileSummary, ReleaseCandidate, ReleaseSummary, ReviewQueueEntry, + SearchNowResult, SearchResult, StuckReport, WeightsDto, }; use breadarr_shared::DaemonClient; use ratatui::widgets::ListState; +use std::path::Path; +use std::time::{Duration, Instant}; + +/// Category roots the TUI's "Add" flow can place new items under — kept as +/// two separate paths (not one shared default) since a series and a movie +/// added with the same title must not collide on disk, and Jellyfin's +/// per-category libraries only see files under their own category root. +pub struct LibraryRoots { + pub series: String, + pub movies: String, +} + +/// Builds the per-item folder a freshly added series/movie lands in — +/// `{root}/{Title} ({Year})`, matching the convention the importer already +/// assumes (`import_one` places a movie's file directly inside its +/// `media_item.root_folder`, with no further subfolder of its own, and +/// `season_dir` does the same for a show's `Season NN` folders). Passing the +/// bare category root straight through as `root_folder` — the bug this +/// replaces — landed every new grab directly in that shared root instead of +/// its own show/movie folder. +fn item_root_folder(root: &str, title: &str, year: Option) -> String { + let sanitized: String = title + .chars() + .map(|c| if "/\\:*?\"<>|".contains(c) { '_' } else { c }) + .collect(); + // `.` / `..` survive the character class above and would make + // `root.join(...)` walk out of the library root. + let sanitized = if sanitized == "." || sanitized == ".." { + "_".to_string() + } else { + sanitized + }; + let folder_name = match year { + Some(y) => format!("{sanitized} ({y})"), + None => sanitized, + }; + Path::new(root) + .join(folder_name) + .to_string_lossy() + .to_string() +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Tab { @@ -35,8 +76,8 @@ impl Tab { match self { Tab::Library => "Library", Tab::History => "History", - Tab::Review => "Review Queue", - Tab::Add => "Add Show", + Tab::Review => "Review", + Tab::Add => "Add", Tab::Stuck => "Stuck", Tab::Calendar => "Calendar", Tab::LibraryHealth => "Health", @@ -49,6 +90,10 @@ pub enum Focus { List, AddSearchInput, AddResults, + /// Incremental title filter on the Library list — `App::library_query` + /// holds the in-progress text, same priority-over-global-keys pattern + /// as `AddSearchInput`. + LibraryFilterInput, /// The manual release picker overlay — `App::candidates` holds the /// list, `App::candidates_episode_id` remembers which episode (or, if /// `None`, the open movie) it was fetched for so a grab can be @@ -74,6 +119,88 @@ impl AddKind { } } +/// Client-side filter over the already-fetched `media_items` — the server +/// has no filter/sort query params, and the library is small enough that +/// refiltering the in-memory `Vec` on every keypress is free. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LibraryFilter { + All, + Series, + Movies, + Missing, + Unmonitored, +} + +impl LibraryFilter { + pub const ALL: [LibraryFilter; 5] = [ + LibraryFilter::All, + LibraryFilter::Series, + LibraryFilter::Movies, + LibraryFilter::Missing, + LibraryFilter::Unmonitored, + ]; + + pub fn next(self) -> Self { + let idx = Self::ALL.iter().position(|f| *f == self).unwrap_or(0); + Self::ALL[(idx + 1) % Self::ALL.len()] + } + + pub fn label(&self) -> &'static str { + match self { + LibraryFilter::All => "all", + LibraryFilter::Series => "series", + LibraryFilter::Movies => "movies", + LibraryFilter::Missing => "missing", + LibraryFilter::Unmonitored => "unmonitored", + } + } + + fn matches(&self, m: &MediaItemSummary) -> bool { + match self { + LibraryFilter::All => true, + LibraryFilter::Series => m.kind == "series", + LibraryFilter::Movies => m.kind == "movie", + LibraryFilter::Missing => m.missing_count > 0, + LibraryFilter::Unmonitored => !m.monitored, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LibrarySort { + TitleAsc, + MissingDesc, + Kind, +} + +impl LibrarySort { + pub const ALL: [LibrarySort; 3] = [ + LibrarySort::TitleAsc, + LibrarySort::MissingDesc, + LibrarySort::Kind, + ]; + + pub fn next(self) -> Self { + let idx = Self::ALL.iter().position(|s| *s == self).unwrap_or(0); + Self::ALL[(idx + 1) % Self::ALL.len()] + } + + pub fn label(&self) -> &'static str { + match self { + LibrarySort::TitleAsc => "title", + LibrarySort::MissingDesc => "missing", + LibrarySort::Kind => "kind", + } + } +} + +/// Which half of the Stuck tab has selection/arrow-key focus. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StuckSection { + Stalled, + Maxed, +} + /// A weight axis's display name plus a getter/setter pair, so /// `WEIGHT_FIELDS` can enumerate `WeightsDto`'s fields by position instead /// of every caller matching on an index. @@ -115,6 +242,73 @@ pub struct AddResult { pub result: SearchResult, } +/// Result of a long `DaemonClient` call spawned off the draw loop so +/// `search_now` / candidate fetch / add-tab search cannot freeze key +/// handling. Add-search is typically a few seconds (TVDB + TMDB together) +/// rather than minutes, but it still shouldn't stall j/k or Tab. +enum BackgroundOutcome { + SearchNow(Result), + Candidates { + episode_id: Option, + result: Result>, + }, + AddSearch { + results: Vec, + errors: Vec, + }, +} + +/// Case-insensitive substring match used by the Library `/` filter. +fn title_matches_query(title: &str, query: &str) -> bool { + query.is_empty() || title.to_lowercase().contains(&query.to_lowercase()) +} + +/// Next monitored-and-missing episode, wrapping from `start` so `n` can +/// be mashed through a season without first jumping to the top. +fn next_missing_index(episodes: &[EpisodeSummary], start: usize) -> Option { + if episodes.is_empty() { + return None; + } + let start = start.min(episodes.len()); + episodes + .iter() + .enumerate() + .skip(start) + .chain(episodes.iter().enumerate().take(start)) + .find(|(_, e)| e.monitored && !e.has_file) + .map(|(i, _)| i) +} + +fn clamp_list_state(state: &mut ListState, len: usize) { + match (state.selected(), len) { + (_, 0) => state.select(None), + (None, _) => state.select(Some(0)), + (Some(i), _) if i >= len => state.select(Some(len - 1)), + _ => {} + } +} + +/// Row count of the Health tab's flagged-files list, including section +/// headers — must stay in lockstep with `draw_library_health` so j/k +/// doesn't walk off the rendered items. +pub fn health_row_count(report: &LibraryHealthReport) -> usize { + let mut n = 0; + let mut add_files = |files: &[FlaggedFile]| { + if !files.is_empty() { + n += 1 + files.len(); + } + }; + add_files(&report.corrupt_files); + add_files(&report.under_quality_files); + add_files(&report.no_english_audio_files); + add_files(&report.non_english_default_audio_files); + add_files(&report.no_subtitle_files); + if !report.duplicate_groups.is_empty() { + n += 1 + report.duplicate_groups.len(); + } + n.max(1) +} + pub struct App { pub client: DaemonClient, pub daemon_up: bool, @@ -122,10 +316,32 @@ pub struct App { pub tab: Tab, pub focus: Focus, pub status: String, + /// When `status` was last written — the draw loop clears it after a + /// few seconds so action feedback doesn't permanently hide the + /// keybinding hints in the status bar. + status_set_at: Option, pub should_quit: bool, + /// Toggled by `?`; intercepted at the top of `main.rs`'s key handler + /// like `Focus::AddSearchInput`/`Focus::WeightInput`, but kept as its + /// own bool (not folded into `Focus`) since it needs to open from any + /// tab rather than being scoped to one. + pub help_visible: bool, + /// Set by the `R` key; consumed by `main.rs`'s refresh loop to bypass + /// the normal 3s throttle for one immediate refresh. + pub force_refresh: bool, pub media_items: Vec, pub media_state: ListState, + /// Filtered+sorted indices into `media_items`, recomputed by + /// `recompute_library_view` — the top-level Library list and + /// `media_state` always operate over this, never over `media_items` + /// directly, so filter/sort never has to touch the raw fetched data. + pub library_view: Vec, + pub library_filter: LibraryFilter, + pub library_sort: LibrarySort, + /// Incremental title filter from `/` — applied on top of + /// `library_filter`/`library_sort` inside `recompute_library_view`. + pub library_query: String, pub detail: Option, /// Selection within `detail.episodes` — separate from `media_state` /// since they're two different lists sharing the same tab. @@ -136,14 +352,24 @@ pub struct App { pub review_items: Vec, pub review_state: ListState, + /// Last-seen review-queue depth, kept even when the Review tab isn't + /// the one being refreshed so the tab strip can badge it. + pub review_count: usize, pub add_query: String, pub add_results: Vec, pub add_results_state: ListState, pub stuck: Option, + pub stuck_focus: StuckSection, + pub stalled_state: ListState, + pub maxed_state: ListState, + /// Last-seen stalled+maxed count for the Stuck tab badge. + pub stuck_count: usize, pub calendar: Vec, + pub calendar_state: ListState, pub library_health: Option, + pub health_state: ListState, pub candidates: Vec, pub candidates_state: ListState, @@ -173,6 +399,10 @@ pub struct App { pub profile_weight_state: ListState, /// In-progress digits while `Focus::WeightInput` is active. pub weight_input_buffer: String, + + /// True while a long search-now / candidate-fetch task is in flight. + pub busy: bool, + background: Option>, } impl App { @@ -184,21 +414,35 @@ impl App { tab: Tab::Library, focus: Focus::List, status: String::new(), + status_set_at: None, should_quit: false, + help_visible: false, + force_refresh: false, media_items: Vec::new(), media_state: ListState::default(), + library_view: Vec::new(), + library_filter: LibraryFilter::All, + library_sort: LibrarySort::TitleAsc, + library_query: String::new(), detail: None, episode_state: ListState::default(), releases: Vec::new(), releases_state: ListState::default(), review_items: Vec::new(), review_state: ListState::default(), + review_count: 0, add_query: String::new(), add_results: Vec::new(), add_results_state: ListState::default(), stuck: None, + stuck_focus: StuckSection::Stalled, + stalled_state: ListState::default(), + maxed_state: ListState::default(), + stuck_count: 0, calendar: Vec::new(), + calendar_state: ListState::default(), library_health: None, + health_state: ListState::default(), candidates: Vec::new(), candidates_state: ListState::default(), candidates_episode_id: None, @@ -209,6 +453,137 @@ impl App { profile_detail: None, profile_weight_state: ListState::default(), weight_input_buffer: String::new(), + busy: false, + background: None, + } + } + + pub fn set_status(&mut self, msg: impl Into) { + self.status = msg.into(); + self.status_set_at = if self.status.is_empty() { + None + } else { + Some(Instant::now()) + }; + } + + /// Drops stale action feedback so the status bar can show keybinding + /// hints again. In-flight work (`busy`) keeps its "searching..." line. + pub fn expire_status(&mut self) { + if self.busy { + return; + } + if let Some(at) = self.status_set_at { + if at.elapsed() >= Duration::from_secs(5) { + self.status.clear(); + self.status_set_at = None; + } + } + } + + /// The list currently driven by j/k / g/G / PageUp/PageDown. + fn active_list(&mut self) -> (&mut ListState, usize) { + match self.tab { + Tab::Library if matches!(self.focus, Focus::Candidates) => { + (&mut self.candidates_state, self.candidates.len()) + } + Tab::Library if self.detail.is_none() => { + (&mut self.media_state, self.library_view.len()) + } + Tab::Library => ( + &mut self.episode_state, + self.detail.as_ref().map_or(0, |d| d.episodes.len()), + ), + Tab::History => (&mut self.releases_state, self.releases.len()), + Tab::Review => (&mut self.review_state, self.review_items.len()), + Tab::Add => (&mut self.add_results_state, self.add_results.len()), + Tab::Stuck => { + let report_len = self.stuck.as_ref().map_or((0, 0), |r| { + (r.stalled_grabs.len(), r.maxed_out_search_targets.len()) + }); + match self.stuck_focus { + StuckSection::Stalled => (&mut self.stalled_state, report_len.0), + StuckSection::Maxed => (&mut self.maxed_state, report_len.1), + } + } + Tab::Calendar => (&mut self.calendar_state, self.calendar.len()), + Tab::LibraryHealth => { + let n = self + .library_health + .as_ref() + .map(health_row_count) + .unwrap_or(0); + (&mut self.health_state, n) + } + Tab::Profiles if self.profile_detail.is_some() => { + (&mut self.profile_weight_state, WEIGHT_FIELDS.len()) + } + Tab::Profiles => (&mut self.profiles_state, self.quality_profiles.len()), + } + } + + pub fn select_edge(&mut self, last: bool) { + let (state, len) = self.active_list(); + if len == 0 { + return; + } + state.select(Some(if last { len - 1 } else { 0 })); + } + + /// Applies a finished background search/candidate task. Only `.await`s + /// a handle that `is_finished()`, so the draw loop stays responsive. + pub async fn poll_background(&mut self) { + let Some(handle) = &self.background else { + return; + }; + if !handle.is_finished() { + return; + } + let handle = self.background.take().expect("just checked is_finished"); + self.busy = false; + match handle.await { + Ok(BackgroundOutcome::SearchNow(Ok(stats))) => { + self.set_status(format!( + "search complete: {} target(s), {} grabbed, {} error(s)", + stats.targets, stats.grabbed, stats.errors + )); + } + Ok(BackgroundOutcome::SearchNow(Err(e))) => { + self.set_status(format!("search failed: {e}")); + } + Ok(BackgroundOutcome::Candidates { + episode_id, + result: Ok(candidates), + }) => { + self.set_status(format!("{} candidate(s) found", candidates.len())); + if matches!(self.tab, Tab::Library) && self.detail.is_some() { + self.candidates_state.select(if candidates.is_empty() { + None + } else { + Some(0) + }); + self.candidates = candidates; + self.candidates_episode_id = episode_id; + self.focus = Focus::Candidates; + } + } + Ok(BackgroundOutcome::Candidates { result: Err(e), .. }) => { + self.set_status(format!("candidate fetch failed: {e}")); + } + Ok(BackgroundOutcome::AddSearch { results, errors }) => { + self.add_results_state + .select(if results.is_empty() { None } else { Some(0) }); + self.add_results = results; + self.set_status(if errors.is_empty() { + String::new() + } else { + errors.join("; ") + }); + if matches!(self.tab, Tab::Add) { + self.focus = Focus::AddResults; + } + } + Err(e) => self.set_status(format!("background task failed: {e}")), } } @@ -224,7 +599,7 @@ impl App { } } if !self.daemon_up { - self.status = "daemon unreachable".to_string(); + self.set_status("daemon unreachable"); return; } let result: Result<()> = async { @@ -236,50 +611,70 @@ impl App { } } else { self.media_items = self.client.list_media().await?; - if self.media_state.selected().is_none() && !self.media_items.is_empty() { - self.media_state.select(Some(0)); - } + self.recompute_library_view(); } } Tab::History => { self.releases = self.client.releases().await?; - if self.releases_state.selected().is_none() && !self.releases.is_empty() { - self.releases_state.select(Some(0)); - } + clamp_list_state(&mut self.releases_state, self.releases.len()); } Tab::Review => { self.review_items = self.client.review_queue().await?; - if self.review_state.selected().is_none() && !self.review_items.is_empty() { - self.review_state.select(Some(0)); - } + self.review_count = self.review_items.len(); + clamp_list_state(&mut self.review_state, self.review_items.len()); } Tab::Add => {} Tab::Stuck => { - self.stuck = Some(self.client.stuck().await?); + let report = self.client.stuck().await?; + self.stuck_count = + report.stalled_grabs.len() + report.maxed_out_search_targets.len(); + clamp_list_state(&mut self.stalled_state, report.stalled_grabs.len()); + clamp_list_state(&mut self.maxed_state, report.maxed_out_search_targets.len()); + self.stuck = Some(report); } Tab::Calendar => { self.calendar = self.client.calendar().await?; + clamp_list_state(&mut self.calendar_state, self.calendar.len()); } Tab::LibraryHealth => { self.library_health = Some(self.client.library_health().await?); + let n = self + .library_health + .as_ref() + .map(health_row_count) + .unwrap_or(0); + clamp_list_state(&mut self.health_state, n); } Tab::Profiles => { if self.profile_detail.is_none() { self.quality_profiles = self.client.quality_profiles().await?; - if self.profiles_state.selected().is_none() - && !self.quality_profiles.is_empty() - { - self.profiles_state.select(Some(0)); - } + clamp_list_state(&mut self.profiles_state, self.quality_profiles.len()); } } } + self.refresh_badge_counts().await; Ok(()) } .await; if let Err(e) = result { - self.status = format!("error: {e}"); + self.set_status(format!("error: {e}")); + } + } + + /// Cheap counts for the tab-strip badges. Skips the resource the + /// active tab already fetched so we don't double-hit the same route. + async fn refresh_badge_counts(&mut self) { + if !matches!(self.tab, Tab::Review) { + if let Ok(items) = self.client.review_queue().await { + self.review_count = items.len(); + } + } + if !matches!(self.tab, Tab::Stuck) { + if let Ok(report) = self.client.stuck().await { + self.stuck_count = + report.stalled_grabs.len() + report.maxed_out_search_targets.len(); + } } } @@ -287,27 +682,71 @@ impl App { self.detail.as_ref().map(|d| d.id) } + /// Rebuilds `library_view` from `media_items` under the current + /// filter/sort, then re-selects whichever item was selected before (by + /// id, not raw index) if it's still in view — falls back to the first + /// item, or no selection if the view is now empty. Called after + /// `media_items` changes, and after `library_filter`/`library_sort` + /// change. + pub fn recompute_library_view(&mut self) { + let previously_selected_id = self.selected_media_item().map(|m| m.id); + + let query = self.library_query.clone(); + let mut indices: Vec = self + .media_items + .iter() + .enumerate() + .filter(|(_, m)| { + self.library_filter.matches(m) && title_matches_query(&m.title, &query) + }) + .map(|(i, _)| i) + .collect(); + + match self.library_sort { + LibrarySort::TitleAsc => indices.sort_by(|&a, &b| { + self.media_items[a] + .title + .to_lowercase() + .cmp(&self.media_items[b].title.to_lowercase()) + }), + LibrarySort::MissingDesc => indices.sort_by(|&a, &b| { + self.media_items[b] + .missing_count + .cmp(&self.media_items[a].missing_count) + }), + LibrarySort::Kind => indices.sort_by(|&a, &b| { + self.media_items[a] + .kind + .cmp(&self.media_items[b].kind) + .then_with(|| { + self.media_items[a] + .title + .to_lowercase() + .cmp(&self.media_items[b].title.to_lowercase()) + }) + }), + } + self.library_view = indices; + + match previously_selected_id.and_then(|id| { + self.library_view + .iter() + .position(|&i| self.media_items[i].id == id) + }) { + Some(pos) => self.media_state.select(Some(pos)), + None if !self.library_view.is_empty() => self.media_state.select(Some(0)), + None => self.media_state.select(None), + } + } + + pub fn selected_media_item(&self) -> Option<&MediaItemSummary> { + let idx = self.media_state.selected()?; + let real_idx = *self.library_view.get(idx)?; + self.media_items.get(real_idx) + } + pub fn move_selection(&mut self, delta: i32) { - let (state, len) = match self.tab { - Tab::Library if matches!(self.focus, Focus::Candidates) => { - (&mut self.candidates_state, self.candidates.len()) - } - Tab::Library if self.detail.is_none() => { - (&mut self.media_state, self.media_items.len()) - } - Tab::Library => ( - &mut self.episode_state, - self.detail.as_ref().map_or(0, |d| d.episodes.len()), - ), - Tab::History => (&mut self.releases_state, self.releases.len()), - Tab::Review => (&mut self.review_state, self.review_items.len()), - Tab::Add => (&mut self.add_results_state, self.add_results.len()), - Tab::Profiles if self.profile_detail.is_some() => { - (&mut self.profile_weight_state, WEIGHT_FIELDS.len()) - } - Tab::Profiles => (&mut self.profiles_state, self.quality_profiles.len()), - _ => return, - }; + let (state, len) = self.active_list(); if len == 0 { return; } @@ -320,10 +759,7 @@ impl App { if !matches!(self.tab, Tab::Library) || self.detail.is_some() { return; } - let Some(idx) = self.media_state.selected() else { - return; - }; - let Some(item) = self.media_items.get(idx) else { + let Some(item) = self.selected_media_item() else { return; }; match self.client.media_detail(item.id).await { @@ -335,13 +771,47 @@ impl App { }); self.detail = Some(detail); } - Err(e) => self.status = format!("error loading detail: {e}"), + Err(e) => self.set_status(format!("error loading detail: {e}")), } } pub fn close_detail(&mut self) { self.detail = None; self.episode_state.select(None); + if matches!(self.focus, Focus::Candidates) { + self.close_candidates(); + } + } + + pub fn start_library_filter(&mut self) { + if self.detail.is_some() { + return; + } + self.focus = Focus::LibraryFilterInput; + } + + pub fn confirm_library_filter(&mut self) { + self.focus = Focus::List; + } + + pub fn clear_library_filter(&mut self) { + self.library_query.clear(); + self.focus = Focus::List; + self.recompute_library_view(); + } + + /// Advances the episode selection to the next monitored episode that + /// still has no file, wrapping so a second `n` at the end of the list + /// starts over rather than doing nothing. + pub fn select_next_missing_episode(&mut self) { + let Some(detail) = &self.detail else { + return; + }; + let start = self.episode_state.selected().map(|i| i + 1).unwrap_or(0); + match next_missing_index(&detail.episodes, start) { + Some(i) => self.episode_state.select(Some(i)), + None => self.set_status("no missing monitored episodes"), + } } /// Toggles monitored on whichever episode is currently selected in the @@ -364,14 +834,14 @@ impl App { }; match result { Ok(()) => { - self.status = "episode monitor state updated".to_string(); + self.set_status("episode monitor state updated".to_string()); if let Some(id) = self.selected_media_id() { if let Ok(fresh) = self.client.media_detail(id).await { self.detail = Some(fresh); } } } - Err(e) => self.status = format!("episode monitor toggle failed: {e}"), + Err(e) => self.set_status(format!("episode monitor toggle failed: {e}")), } } @@ -388,7 +858,7 @@ impl App { return; }; let (media_item_id, season_number, monitored) = - (detail.id, episode.season_number as i64, episode.monitored); + (detail.id, episode.season_number, episode.monitored); let result = if monitored { self.client .unmonitor_season(media_item_id, season_number) @@ -400,12 +870,12 @@ impl App { }; match result { Ok(()) => { - self.status = format!("season {season_number} monitor state updated"); + self.set_status(format!("season {season_number} monitor state updated")); if let Ok(fresh) = self.client.media_detail(media_item_id).await { self.detail = Some(fresh); } } - Err(e) => self.status = format!("season monitor toggle failed: {e}"), + Err(e) => self.set_status(format!("season monitor toggle failed: {e}")), } } @@ -417,10 +887,12 @@ impl App { return; }; match self.client.approve_review(item.id).await { - Ok(()) => self.status = format!("approved: {}", item.raw_release_title), - Err(e) => self.status = format!("approve failed: {e}"), + Ok(()) => self.set_status(format!("approved: {}", item.raw_release_title)), + Err(e) => self.set_status(format!("approve failed: {e}")), } self.review_items = self.client.review_queue().await.unwrap_or_default(); + self.review_count = self.review_items.len(); + clamp_list_state(&mut self.review_state, self.review_items.len()); } pub async fn reject_selected_review(&mut self) { @@ -431,10 +903,12 @@ impl App { return; }; match self.client.reject_review(item.id).await { - Ok(()) => self.status = format!("rejected: {}", item.raw_release_title), - Err(e) => self.status = format!("reject failed: {e}"), + Ok(()) => self.set_status(format!("rejected: {}", item.raw_release_title)), + Err(e) => self.set_status(format!("reject failed: {e}")), } self.review_items = self.client.review_queue().await.unwrap_or_default(); + self.review_count = self.review_items.len(); + clamp_list_state(&mut self.review_state, self.review_items.len()); } /// Searches series and movies together instead of requiring the user to @@ -445,44 +919,39 @@ impl App { /// a movie search returned nothing because the mode had never actually /// switched). Querying both up front removes the failure mode entirely. pub async fn run_add_search(&mut self) { - if self.add_query.trim().is_empty() { + if self.add_query.trim().is_empty() || self.busy { return; } - let (series_result, movie_result) = tokio::join!( - self.client.search_series(&self.add_query), - self.client.search_movies(&self.add_query) - ); + self.set_status("searching movies and TV..."); + self.busy = true; + let client = self.client.clone(); + let query = self.add_query.clone(); + self.background = Some(tokio::spawn(async move { + let (series_result, movie_result) = + tokio::join!(client.search_series(&query), client.search_movies(&query)); - let mut results = Vec::new(); - let mut errors = Vec::new(); - match series_result { - Ok(hits) => results.extend(hits.into_iter().map(|result| AddResult { - kind: AddKind::Series, - result, - })), - Err(e) => errors.push(format!("series search failed: {e}")), - } - match movie_result { - Ok(hits) => results.extend(hits.into_iter().map(|result| AddResult { - kind: AddKind::Movie, - result, - })), - Err(e) => errors.push(format!("movie search failed: {e}")), - } - results.sort_by_key(|a| a.result.title.to_lowercase()); - - self.add_results_state - .select(if results.is_empty() { None } else { Some(0) }); - self.add_results = results; - self.status = if errors.is_empty() { - String::new() - } else { - errors.join("; ") - }; - self.focus = Focus::AddResults; + let mut results = Vec::new(); + let mut errors = Vec::new(); + match series_result { + Ok(hits) => results.extend(hits.into_iter().map(|result| AddResult { + kind: AddKind::Series, + result, + })), + Err(e) => errors.push(format!("series search failed: {e}")), + } + match movie_result { + Ok(hits) => results.extend(hits.into_iter().map(|result| AddResult { + kind: AddKind::Movie, + result, + })), + Err(e) => errors.push(format!("movie search failed: {e}")), + } + results.sort_by_key(|a| a.result.title.to_lowercase()); + BackgroundOutcome::AddSearch { results, errors } + })); } - pub async fn add_selected_search_result(&mut self, root_folder: &str) { + pub async fn add_selected_search_result(&mut self, roots: &LibraryRoots) { let Some(idx) = self.add_results_state.selected() else { return; }; @@ -496,7 +965,7 @@ impl App { title: result.title.clone(), year: result.year, aliases: Vec::new(), - root_folder: root_folder.to_string(), + root_folder: item_root_folder(&roots.series, &result.title, result.year), }; self.client.add_series(&req).await.map(|r| r.media_item_id) } @@ -505,40 +974,42 @@ impl App { tmdb_id: result.external_id.clone(), title: result.title.clone(), year: result.year, - root_folder: root_folder.to_string(), + root_folder: item_root_folder(&roots.movies, &result.title, result.year), }; self.client.add_movie(&req).await.map(|r| r.media_item_id) } }; match outcome { Ok(media_item_id) => { - self.status = format!("added {:?} (media_item_id={media_item_id})", result.title); + self.set_status(format!( + "added {:?} (media_item_id={media_item_id})", + result.title + )); self.add_results.clear(); self.add_query.clear(); self.focus = Focus::AddSearchInput; } - Err(e) => self.status = format!("add failed: {e}"), + Err(e) => self.set_status(format!("add failed: {e}")), } } /// Manual "search now" for the show/movie currently open in the Library /// detail view — can take a while (jittered, one request per missing - /// item), so the status line makes that explicit rather than looking - /// like the UI hung. + /// item), so this is spawned off the draw loop and the status line + /// shows that work is in flight. A second press while busy is ignored. pub async fn search_now_selected(&mut self) { let Some(id) = self.detail.as_ref().map(|d| d.id) else { return; }; - self.status = "searching now (this can take a while)...".to_string(); - match self.client.search_now(id).await { - Ok(stats) => { - self.status = format!( - "search complete: {} target(s), {} grabbed, {} error(s)", - stats.targets, stats.grabbed, stats.errors - ); - } - Err(e) => self.status = format!("search failed: {e}"), + if self.busy { + return; } + self.set_status("searching now...".to_string()); + self.busy = true; + let client = self.client.clone(); + self.background = Some(tokio::spawn(async move { + BackgroundOutcome::SearchNow(client.search_now(id).await) + })); } /// Fetches candidates for whatever's selected in the open detail view — @@ -561,23 +1032,20 @@ impl App { Some(episode.id) }; let media_item_id = detail.id; - - self.status = "fetching candidates (this can take a while)...".to_string(); - let result = match episode_id { - Some(id) => self.client.episode_candidates(id).await, - None => self.client.movie_candidates(media_item_id).await, - }; - match result { - Ok(candidates) => { - self.status = format!("{} candidate(s) found", candidates.len()); - self.candidates_state - .select(if candidates.is_empty() { None } else { Some(0) }); - self.candidates = candidates; - self.candidates_episode_id = episode_id; - self.focus = Focus::Candidates; - } - Err(e) => self.status = format!("candidate fetch failed: {e}"), + if self.busy { + return; } + + self.set_status("fetching candidates...".to_string()); + self.busy = true; + let client = self.client.clone(); + self.background = Some(tokio::spawn(async move { + let result = match episode_id { + Some(id) => client.episode_candidates(id).await, + None => client.movie_candidates(media_item_id).await, + }; + BackgroundOutcome::Candidates { episode_id, result } + })); } /// Grabs whichever candidate is currently selected in the picker @@ -606,13 +1074,13 @@ impl App { }; match result { Ok(()) => { - self.status = format!("grabbed: {}", candidate.raw_title); + self.set_status(format!("grabbed: {}", candidate.raw_title)); self.close_candidates(); if let Ok(fresh) = self.client.media_detail(media_item_id).await { self.detail = Some(fresh); } } - Err(e) => self.status = format!("grab failed: {e}"), + Err(e) => self.set_status(format!("grab failed: {e}")), } } @@ -682,7 +1150,10 @@ impl App { let value: f32 = match self.weight_input_buffer.trim().parse() { Ok(v) => v, Err(_) => { - self.status = format!("'{}' is not a valid number", self.weight_input_buffer); + self.set_status(format!( + "'{}' is not a valid number", + self.weight_input_buffer + )); return; } }; @@ -699,11 +1170,33 @@ impl App { .await { Ok(()) => { - self.status = format!("{name} updated to {value}"); + self.set_status(format!("{name} updated to {value}")); self.weight_input_buffer.clear(); self.focus = Focus::List; } - Err(e) => self.status = format!("weight update failed: {e}"), + Err(e) => self.set_status(format!("weight update failed: {e}")), + } + } + + /// Toggles monitored for whatever's selected in the top-level Library + /// list, without needing to open its detail view first. + pub async fn toggle_monitor_list_selected(&mut self) { + let Some(item) = self.selected_media_item() else { + return; + }; + let (id, monitored) = (item.id, item.monitored); + let result = if monitored { + self.client.unmonitor(id).await + } else { + self.client.monitor(id).await + }; + match result { + Ok(()) => { + self.set_status("monitor state updated".to_string()); + self.media_items = self.client.list_media().await.unwrap_or_default(); + self.recompute_library_view(); + } + Err(e) => self.set_status(format!("monitor toggle failed: {e}")), } } @@ -719,12 +1212,12 @@ impl App { }; match result { Ok(()) => { - self.status = "monitor state updated".to_string(); + self.set_status("monitor state updated".to_string()); if let Ok(fresh) = self.client.media_detail(id).await { self.detail = Some(fresh); } } - Err(e) => self.status = format!("monitor toggle failed: {e}"), + Err(e) => self.set_status(format!("monitor toggle failed: {e}")), } } @@ -736,17 +1229,18 @@ impl App { }; if !self.confirm_delete { self.confirm_delete = true; - self.status = "press x again to confirm delete".to_string(); + self.set_status("press x again to confirm delete".to_string()); return; } self.confirm_delete = false; match self.client.delete_media(id).await { Ok(()) => { - self.status = "deleted".to_string(); + self.set_status("deleted".to_string()); self.detail = None; self.media_items = self.client.list_media().await.unwrap_or_default(); + self.recompute_library_view(); } - Err(e) => self.status = format!("delete failed: {e}"), + Err(e) => self.set_status(format!("delete failed: {e}")), } } @@ -761,7 +1255,7 @@ impl App { }; if !self.confirm_delete_file { self.confirm_delete_file = true; - self.status = "press d again to confirm deleting this file".to_string(); + self.set_status("press d again to confirm deleting this file".to_string()); return; } self.confirm_delete_file = false; @@ -779,14 +1273,202 @@ impl App { }; match result { Ok(()) => { - self.status = "file deleted, will be re-searched".to_string(); + self.set_status("file deleted, will be re-searched".to_string()); if let Some(id) = self.selected_media_id() { if let Ok(fresh) = self.client.media_detail(id).await { self.detail = Some(fresh); } } } - Err(e) => self.status = format!("file delete failed: {e}"), + Err(e) => self.set_status(format!("file delete failed: {e}")), + } + } + + /// Jumps from whichever Stuck-tab row is selected straight to that + /// show's Library detail view. + pub async fn jump_to_stuck_target(&mut self) { + let Some(report) = &self.stuck else { + return; + }; + let media_item_id = match self.stuck_focus { + StuckSection::Maxed => self + .maxed_state + .selected() + .and_then(|i| report.maxed_out_search_targets.get(i)) + .map(|t| t.media_item_id), + StuckSection::Stalled => self + .stalled_state + .selected() + .and_then(|i| report.stalled_grabs.get(i)) + .map(|g| g.media_item_id), + }; + let Some(id) = media_item_id else { + return; + }; + match self.client.media_detail(id).await { + Ok(detail) => { + self.tab = Tab::Library; + self.episode_state.select(if detail.episodes.is_empty() { + None + } else { + Some(0) + }); + self.detail = Some(detail); + self.focus = Focus::List; + } + Err(e) => self.set_status(format!("error loading detail: {e}")), + } + } + + /// Jumps from the selected Calendar row to that episode's Library + /// detail — same idea as `jump_to_stuck_target`, but also lands on the + /// matching SxxExx so `c`/`d` apply to the aired episode, not S01E01. + pub async fn jump_to_calendar_entry(&mut self) { + let Some(idx) = self.calendar_state.selected() else { + return; + }; + let Some(entry) = self.calendar.get(idx).cloned() else { + return; + }; + match self.client.media_detail(entry.media_item_id).await { + Ok(detail) => { + let ep = detail.episodes.iter().position(|e| { + e.season_number == entry.season_number + && e.episode_number == entry.episode_number + }); + self.tab = Tab::Library; + self.episode_state + .select(ep.or(if detail.episodes.is_empty() { + None + } else { + Some(0) + })); + self.detail = Some(detail); + self.focus = Focus::List; + } + Err(e) => self.set_status(format!("error loading detail: {e}")), } } } + +#[cfg(test)] +mod tests { + use super::{ + clamp_list_state, health_row_count, item_root_folder, next_missing_index, + title_matches_query, + }; + use breadarr_shared::dto::{ + DuplicateGroup, EpisodeSummary, FlaggedFile, LibraryHealthReport, LibrarySummary, + }; + use ratatui::widgets::ListState; + + #[test] + fn item_root_folder_replaces_dot_and_dotdot() { + assert_eq!(item_root_folder("/lib", "..", None), "/lib/_"); + assert_eq!(item_root_folder("/lib", ".", None), "/lib/_"); + assert_eq!(item_root_folder("/lib", "..", Some(2020)), "/lib/_ (2020)"); + } + + #[test] + fn item_root_folder_sanitizes_hostile_characters() { + assert_eq!( + item_root_folder("/lib", "Foo: Bar/Baz", Some(2020)), + "/lib/Foo_ Bar_Baz (2020)" + ); + } + + #[test] + fn title_matches_query_is_case_insensitive_substring() { + assert!(title_matches_query("The Matrix", "matrix")); + assert!(title_matches_query("The Matrix", "")); + assert!(!title_matches_query("The Matrix", "inception")); + } + + fn ep(season: i64, number: i64, monitored: bool, has_file: bool) -> EpisodeSummary { + EpisodeSummary { + id: season * 100 + number, + season_number: season, + episode_number: number, + title: None, + air_date: None, + monitored, + has_file, + } + } + + #[test] + fn next_missing_index_wraps_and_skips_owned_or_unmonitored() { + let episodes = vec![ + ep(1, 1, true, true), + ep(1, 2, true, false), + ep(1, 3, false, false), + ep(1, 4, true, false), + ]; + assert_eq!(next_missing_index(&episodes, 0), Some(1)); + assert_eq!(next_missing_index(&episodes, 2), Some(3)); + assert_eq!(next_missing_index(&episodes, 4), Some(1)); + assert_eq!(next_missing_index(&[], 0), None); + assert_eq!( + next_missing_index(&[ep(1, 1, true, true), ep(1, 2, false, false)], 0), + None + ); + } + + #[test] + fn clamp_list_state_pins_past_the_end_and_clears_empty() { + let mut state = ListState::default(); + state.select(Some(4)); + clamp_list_state(&mut state, 3); + assert_eq!(state.selected(), Some(2)); + clamp_list_state(&mut state, 0); + assert_eq!(state.selected(), None); + clamp_list_state(&mut state, 2); + assert_eq!(state.selected(), Some(0)); + } + + fn empty_health() -> LibraryHealthReport { + LibraryHealthReport { + corrupt_files: vec![], + under_quality_files: vec![], + no_subtitle_files: vec![], + no_english_audio_files: vec![], + non_english_default_audio_files: vec![], + duplicate_groups: vec![], + summary: LibrarySummary { + total_files: 0, + total_size_bytes: 0, + probed_files: 0, + by_video_codec: vec![], + sd_count: 0, + hd_720p_count: 0, + full_hd_1080p_count: 0, + uhd_4k_count: 0, + pct_with_subtitles: 0.0, + }, + } + } + + fn flagged(title: &str) -> FlaggedFile { + FlaggedFile { + episode_file_id: 1, + media_title: title.to_string(), + episode_label: None, + path: "/x".to_string(), + } + } + + #[test] + fn health_row_count_includes_headers_and_empty_placeholder() { + let mut report = empty_health(); + assert_eq!(health_row_count(&report), 1); + report.corrupt_files.push(flagged("A")); + report.corrupt_files.push(flagged("B")); + report.duplicate_groups.push(DuplicateGroup { + media_title: "C".into(), + episode_label: None, + paths: vec!["/a".into(), "/b".into()], + }); + // header + 2 files + header + 1 group + assert_eq!(health_row_count(&report), 5); + } +} diff --git a/breadarr-tui/src/main.rs b/breadarr-tui/src/main.rs index bfff7bd..d8af4b0 100644 --- a/breadarr-tui/src/main.rs +++ b/breadarr-tui/src/main.rs @@ -6,7 +6,7 @@ use std::time::Duration; use anyhow::Result; use breadarr_shared::{Config, DaemonClient}; -use crossterm::event::{self, Event, KeyCode, KeyEventKind}; +use crossterm::event::{self, Event, KeyCode, KeyEvent, KeyEventKind}; use crossterm::execute; use crossterm::terminal::{ disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen, @@ -14,14 +14,17 @@ use crossterm::terminal::{ use ratatui::backend::CrosstermBackend; use ratatui::Terminal; -use app::{App, Focus, Tab}; +use app::{App, Focus, LibraryRoots, StuckSection, Tab}; #[tokio::main] async fn main() -> Result<()> { let config = Config::load()?; let base_url = format!("http://{}", config.daemon.listen_addr); - let client = DaemonClient::new(base_url, &config.daemon.api_token); - let root_folder = config.default_root_folder().to_string_lossy().to_string(); + let client = DaemonClient::new(base_url, &config.daemon.api_token)?; + let roots = LibraryRoots { + series: config.default_root_folder().to_string_lossy().to_string(), + movies: config.movies_root_folder().to_string_lossy().to_string(), + }; enable_raw_mode()?; let mut stdout = io::stdout(); @@ -30,7 +33,7 @@ async fn main() -> Result<()> { let mut terminal = Terminal::new(backend)?; let mut app = App::new(client); - let result = run(&mut terminal, &mut app, &root_folder).await; + let result = run(&mut terminal, &mut app, &roots).await; disable_raw_mode()?; execute!(terminal.backend_mut(), LeaveAlternateScreen)?; @@ -42,13 +45,17 @@ async fn main() -> Result<()> { async fn run( terminal: &mut Terminal>, app: &mut App, - root_folder: &str, + roots: &LibraryRoots, ) -> Result<()> { let mut last_refresh = tokio::time::Instant::now() - Duration::from_secs(10); loop { - if last_refresh.elapsed() >= Duration::from_secs(3) { + app.poll_background().await; + app.expire_status(); + + if last_refresh.elapsed() >= Duration::from_secs(3) || app.force_refresh { app.refresh_active_tab().await; + app.force_refresh = false; last_refresh = tokio::time::Instant::now(); } @@ -59,7 +66,7 @@ async fn run( if key.kind != KeyEventKind::Press { continue; } - handle_key(app, key.code, root_folder).await; + handle_key(app, key, roots).await; if app.should_quit { return Ok(()); } @@ -68,7 +75,9 @@ async fn run( } } -async fn handle_key(app: &mut App, code: KeyCode, root_folder: &str) { +async fn handle_key(app: &mut App, key: KeyEvent, roots: &LibraryRoots) { + let code = key.code; + // Typing into the add-show search box takes priority over global keys. if matches!(app.tab, Tab::Add) && matches!(app.focus, Focus::AddSearchInput) { match code { @@ -77,8 +86,31 @@ async fn handle_key(app: &mut App, code: KeyCode, root_folder: &str) { KeyCode::Backspace => { app.add_query.pop(); } - KeyCode::Esc => app.should_quit = true, - KeyCode::Tab => cycle_tab(app), + KeyCode::Esc => { + app.add_query.clear(); + } + KeyCode::Tab => switch_tab(app, tab_offset(app.tab, 1)), + KeyCode::BackTab => switch_tab(app, tab_offset(app.tab, -1)), + _ => {} + } + return; + } + + // Incremental Library title filter — same input-mode isolation as Add. + if matches!(app.tab, Tab::Library) && matches!(app.focus, Focus::LibraryFilterInput) { + match code { + KeyCode::Enter => app.confirm_library_filter(), + KeyCode::Char(c) => { + app.library_query.push(c); + app.recompute_library_view(); + } + KeyCode::Backspace => { + app.library_query.pop(); + app.recompute_library_view(); + } + KeyCode::Esc => app.clear_library_filter(), + KeyCode::Tab => switch_tab(app, tab_offset(app.tab, 1)), + KeyCode::BackTab => switch_tab(app, tab_offset(app.tab, -1)), _ => {} } return; @@ -101,6 +133,16 @@ async fn handle_key(app: &mut App, code: KeyCode, root_folder: &str) { return; } + // Help overlay intercepts everything while open, same + // priority-over-global-keys idiom as the two input modes above. + if app.help_visible { + match code { + KeyCode::Char('?') | KeyCode::Esc => app.help_visible = false, + _ => {} + } + return; + } + // Any key other than a second `x`/`d` clears a pending delete // confirmation — the confirmation must be the very next keypress, not // just "any keypress before the user gets distracted." @@ -113,12 +155,25 @@ async fn handle_key(app: &mut App, code: KeyCode, root_folder: &str) { match code { KeyCode::Char('q') => app.should_quit = true, - KeyCode::Tab => cycle_tab(app), + KeyCode::Tab => switch_tab(app, tab_offset(app.tab, 1)), + KeyCode::BackTab => switch_tab(app, tab_offset(app.tab, -1)), + KeyCode::Char(c) if c.is_ascii_digit() => { + if let Some(n) = c.to_digit(10) { + if (1..=Tab::ALL.len() as u32).contains(&n) { + switch_tab(app, Tab::ALL[(n as usize) - 1]); + } + } + } KeyCode::Char('j') | KeyCode::Down => app.move_selection(1), KeyCode::Char('k') | KeyCode::Up => app.move_selection(-1), + KeyCode::Char('g') => app.select_edge(false), + KeyCode::Char('G') => app.select_edge(true), + KeyCode::PageDown => app.move_selection(10), + KeyCode::PageUp => app.move_selection(-10), KeyCode::Esc => match app.tab { Tab::Library if matches!(app.focus, Focus::Candidates) => app.close_candidates(), Tab::Library if app.detail.is_some() => app.close_detail(), + Tab::Library if !app.library_query.is_empty() => app.clear_library_filter(), Tab::Add => app.focus = Focus::AddSearchInput, Tab::Profiles if app.profile_detail.is_some() => app.close_profile_detail(), _ => {} @@ -129,21 +184,52 @@ async fn handle_key(app: &mut App, code: KeyCode, root_folder: &str) { } Tab::Library => app.open_detail().await, Tab::Add => match app.focus { - Focus::AddResults => app.add_selected_search_result(root_folder).await, + Focus::AddResults => app.add_selected_search_result(roots).await, _ => app.focus = Focus::AddSearchInput, }, Tab::Profiles if app.profile_detail.is_some() => { app.start_editing_selected_weight(); } Tab::Profiles => app.open_profile_detail(), + Tab::Stuck => app.jump_to_stuck_target().await, + Tab::Calendar => app.jump_to_calendar_entry().await, _ => {} }, + KeyCode::Left | KeyCode::Right if matches!(app.tab, Tab::Stuck) => { + app.stuck_focus = match app.stuck_focus { + StuckSection::Stalled => StuckSection::Maxed, + StuckSection::Maxed => StuckSection::Stalled, + }; + } + KeyCode::Char('?') => app.help_visible = true, + KeyCode::Char('R') => app.force_refresh = true, + KeyCode::Char('f') if matches!(app.tab, Tab::Library) && app.detail.is_none() => { + app.library_filter = app.library_filter.next(); + app.recompute_library_view(); + } + KeyCode::Char('o') if matches!(app.tab, Tab::Library) && app.detail.is_none() => { + app.library_sort = app.library_sort.next(); + app.recompute_library_view(); + } + KeyCode::Char('m') if matches!(app.tab, Tab::Library) && app.detail.is_none() => { + app.toggle_monitor_list_selected().await; + } KeyCode::Char('a') if matches!(app.tab, Tab::Review) => { app.approve_selected_review().await; } KeyCode::Char('r') if matches!(app.tab, Tab::Review) => { app.reject_selected_review().await; } + KeyCode::Char('/') if matches!(app.tab, Tab::Library) && app.detail.is_none() => { + app.start_library_filter(); + } + KeyCode::Char('n') + if matches!(app.tab, Tab::Library) + && app.detail.is_some() + && !matches!(app.focus, Focus::Candidates) => + { + app.select_next_missing_episode(); + } KeyCode::Char('s') if matches!(app.tab, Tab::Library) && app.detail.is_some() => { app.search_now_selected().await; } @@ -173,9 +259,20 @@ async fn handle_key(app: &mut App, code: KeyCode, root_folder: &str) { } } -fn cycle_tab(app: &mut App) { - let idx = Tab::ALL.iter().position(|t| *t == app.tab).unwrap_or(0); - app.tab = Tab::ALL[(idx + 1) % Tab::ALL.len()]; +fn tab_offset(current: Tab, delta: i32) -> Tab { + let idx = Tab::ALL.iter().position(|t| *t == current).unwrap_or(0) as i32; + let len = Tab::ALL.len() as i32; + Tab::ALL[(idx + delta).rem_euclid(len) as usize] +} + +fn switch_tab(app: &mut App, tab: Tab) { + if app.tab == tab { + return; + } + if matches!(app.focus, Focus::Candidates) { + app.close_candidates(); + } + app.tab = tab; app.detail = None; app.profile_detail = None; app.profile_weight_state.select(None); diff --git a/breadarr-tui/src/ui.rs b/breadarr-tui/src/ui.rs index 20d6910..77ea537 100644 --- a/breadarr-tui/src/ui.rs +++ b/breadarr-tui/src/ui.rs @@ -1,10 +1,21 @@ use ratatui::layout::{Constraint, Direction, Layout, Rect}; use ratatui::style::{Color, Modifier, Style}; use ratatui::text::{Line, Span}; -use ratatui::widgets::{Block, Borders, List, ListItem, Paragraph, Tabs}; +use ratatui::widgets::{Block, Borders, Clear, List, ListItem, Paragraph, Tabs}; use ratatui::Frame; -use crate::app::{App, Focus, Tab}; +use crate::app::{App, Focus, StuckSection, Tab}; +use breadarr_shared::dto::CycleInfo; + +// Shared color palette — kept to these meanings so a color never has to be +// second-guessed at a glance: +// Green healthy / 0 missing / high confidence / not stuck +// Yellow warning / low missing / mid confidence / at backoff ceiling +// Red critical / high missing / low confidence / past ceiling +// DarkGray unmonitored / muted / not currently relevant +// Blue TV kind tag (categorical, not severity) +// Magenta Movie kind tag (categorical, not severity) +// Cyan focus/interactive accent (tab highlight, active input, focused section) — never severity pub fn draw(frame: &mut Frame, app: &App) { let chunks = Layout::default() @@ -30,10 +41,154 @@ pub fn draw(frame: &mut Frame, app: &App) { } draw_status(frame, chunks[2], app); + + if app.help_visible { + draw_help_overlay(frame, frame.area(), app); + } +} + +/// Keybindings relevant to the current tab/focus/detail state, in the order +/// they should be shown — the single source of truth shared by the status +/// bar (which shows a short prefix) and the help overlay (which shows all of +/// it plus `GLOBAL_KEYS`), so the two can't drift apart. +fn context_keybindings(app: &App) -> Vec<(&'static str, &'static str)> { + match app.tab { + Tab::Library if matches!(app.focus, Focus::Candidates) => { + vec![("Enter", "grab"), ("Esc", "cancel")] + } + Tab::Library if app.detail.is_some() => vec![ + ("Esc", "back"), + ("s", "search now"), + ("n", "next missing"), + ("m", "monitor show"), + ("e", "monitor episode"), + ("S", "monitor season"), + ("x", "delete show"), + ("d", "delete file"), + ("c", "pick release"), + ], + Tab::Library if matches!(app.focus, Focus::LibraryFilterInput) => { + vec![("Enter", "keep filter"), ("Esc", "clear filter")] + } + Tab::Library => vec![ + ("Enter", "open detail"), + ("/", "filter title"), + ("f", "cycle filter"), + ("o", "cycle sort"), + ("m", "monitor toggle"), + ], + Tab::Review => vec![("a", "approve"), ("r", "reject")], + Tab::Add => match app.focus { + Focus::AddSearchInput => vec![("Enter", "search"), ("Esc", "clear")], + _ => vec![("Enter", "add"), ("Esc", "back to search")], + }, + Tab::Profiles if app.profile_detail.is_some() => { + vec![("Enter", "edit weight"), ("Esc", "back")] + } + Tab::Profiles => vec![("Enter", "open profile")], + Tab::Stuck => vec![("Left/Right", "switch section"), ("Enter", "jump to show")], + Tab::Calendar => vec![("Enter", "jump to show")], + _ => vec![], + } +} + +const GLOBAL_KEYS: &[(&str, &str)] = &[ + ("Tab/S-Tab", "switch tab"), + ("1-8", "jump tab"), + ("j/k", "move"), + ("g/G", "first/last"), + ("PgUp/PgDn", "page"), + ("?", "help"), + ("R", "refresh now"), + ("q", "quit"), +]; + +/// Centers a `width` x `height` rect inside `area` — standard ratatui idiom +/// for a popup/overlay. +fn centered_rect(width: u16, height: u16, area: Rect) -> Rect { + let width = width.min(area.width); + let height = height.min(area.height); + Rect { + x: area.x + (area.width.saturating_sub(width)) / 2, + y: area.y + (area.height.saturating_sub(height)) / 2, + width, + height, + } +} + +fn draw_help_overlay(frame: &mut Frame, area: Rect, app: &App) { + let mut lines: Vec = context_keybindings(app) + .into_iter() + .map(|(key, desc)| { + Line::from(vec![ + Span::styled(format!("{key:12}"), Style::default().fg(Color::Cyan)), + Span::raw(desc), + ]) + }) + .collect(); + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + "Global", + Style::default().add_modifier(Modifier::BOLD), + ))); + lines.extend(GLOBAL_KEYS.iter().map(|(key, desc)| { + Line::from(vec![ + Span::styled(format!("{key:12}"), Style::default().fg(Color::Cyan)), + Span::raw(*desc), + ]) + })); + + let popup = centered_rect(52, lines.len() as u16 + 2, area); + frame.render_widget(Clear, popup); + let paragraph = Paragraph::new(lines).block( + Block::default() + .borders(Borders::ALL) + .title("Help — ? or Esc to close"), + ); + frame.render_widget(paragraph, popup); +} + +fn tab_label(tab: Tab, app: &App) -> String { + match tab { + Tab::Library => { + let missing: i64 = app.media_items.iter().map(|m| m.missing_count).sum(); + if missing > 0 { + format!("Library ({missing})") + } else { + tab.title().to_string() + } + } + Tab::Review => { + let n = if matches!(app.tab, Tab::Review) { + app.review_items.len() + } else { + app.review_count + }; + if n > 0 { + format!("Review ({n})") + } else { + tab.title().to_string() + } + } + Tab::Stuck => { + let n = app.stuck.as_ref().map_or(app.stuck_count, |r| { + r.stalled_grabs.len() + r.maxed_out_search_targets.len() + }); + if n > 0 { + format!("Stuck ({n})") + } else { + tab.title().to_string() + } + } + _ => tab.title().to_string(), + } } fn draw_tabs(frame: &mut Frame, area: Rect, app: &App) { - let titles: Vec = Tab::ALL.iter().map(|t| Line::from(t.title())).collect(); + let titles: Vec = Tab::ALL + .iter() + .map(|t| Line::from(tab_label(*t, app))) + .collect(); let selected = Tab::ALL.iter().position(|t| *t == app.tab).unwrap_or(0); let (daemon_label, daemon_color) = match (&app.daemon_up, &app.health) { @@ -84,22 +239,30 @@ fn draw_library(frame: &mut Frame, area: Rect, app: &App) { return; } + if detail.kind == "movie" || detail.episodes.is_empty() { + draw_movie_detail(frame, area, app, detail); + return; + } + let items: Vec = detail .episodes .iter() .map(|e| { - let status = if e.has_file { - "✓" + let (status, color) = if e.has_file { + ("✓", Color::Green) } else if e.monitored { - "…" + ("…", Color::Yellow) } else { - "-" + ("-", Color::DarkGray) }; let title = e.title.as_deref().unwrap_or(""); - ListItem::new(format!( - "{status} S{:02}E{:02} {title}", - e.season_number, e.episode_number - )) + ListItem::new(Line::from(vec![ + Span::styled(status, Style::default().fg(color)), + Span::raw(format!( + " S{:02}E{:02} {title}", + e.season_number, e.episode_number + )), + ])) }) .collect(); let monitor_label = if detail.monitored { @@ -116,9 +279,7 @@ fn draw_library(frame: &mut Frame, area: Rect, app: &App) { }; let list = List::new(items) .block(Block::default().borders(Borders::ALL).title(format!( - "{} ({}) [{monitor_label}] — Esc: back s: search now m: monitor show \ - e: monitor episode S: monitor season x: delete show d: delete file \ - c: pick release{confirm}", + "{} ({}) [{monitor_label}]{confirm}", detail.title, detail.year.map(|y| y.to_string()).unwrap_or_default() ))) @@ -129,33 +290,134 @@ fn draw_library(frame: &mut Frame, area: Rect, app: &App) { } let items: Vec = app - .media_items + .library_view .iter() - .map(|m| { - let missing = if m.missing_count > 0 { + .map(|&i| { + let m = &app.media_items[i]; + let kind_tag = if m.kind == "movie" { "[Movie]" } else { "[TV]" }; + let kind_color = if m.kind == "movie" { + Color::Magenta + } else { + Color::Blue + }; + let ratio = if m.episode_count > 0 { + m.missing_count as f64 / m.episode_count as f64 + } else if m.missing_count > 0 { + 1.0 + } else { + 0.0 + }; + // Mute severity color for unmonitored items — a missing count + // on something deliberately unmonitored isn't actionable. + let missing_color = if !m.monitored || m.missing_count == 0 { + Color::DarkGray + } else if ratio <= 0.25 { + Color::Yellow + } else { + Color::Red + }; + let missing_text = if m.missing_count > 0 { format!(" — {} missing", m.missing_count) } else { String::new() }; - ListItem::new(format!( - "{} ({}){}", - m.title, - m.year.map(|y| y.to_string()).unwrap_or_default(), - missing - )) + let title_style = if m.monitored { + Style::default() + } else { + Style::default().fg(Color::DarkGray) + }; + ListItem::new(Line::from(vec![ + Span::styled(format!("{kind_tag} "), Style::default().fg(kind_color)), + Span::styled( + format!( + "{} ({})", + m.title, + m.year.map(|y| y.to_string()).unwrap_or_default() + ), + title_style, + ), + Span::styled(missing_text, Style::default().fg(missing_color)), + ])) }) .collect(); + let search = if app.library_query.is_empty() { + String::new() + } else { + format!(" search: {}", app.library_query) + }; + let filter_caret = if matches!(app.focus, Focus::LibraryFilterInput) { + "▋" + } else { + "" + }; let list = List::new(items) - .block( - Block::default() - .borders(Borders::ALL) - .title("Monitored Shows — Enter for detail"), - ) + .block(Block::default().borders(Borders::ALL).title(format!( + "Library ({}/{}) — filter: {} sort: {}{search}{filter_caret}", + app.library_view.len(), + app.media_items.len(), + app.library_filter.label(), + app.library_sort.label() + ))) .highlight_style(Style::default().add_modifier(Modifier::REVERSED)); let mut state = app.media_state.clone(); frame.render_stateful_widget(list, area, &mut state); } +fn draw_movie_detail( + frame: &mut Frame, + area: Rect, + app: &App, + detail: &breadarr_shared::dto::MediaItemDetail, +) { + let confirm = if app.confirm_delete { + " — x AGAIN TO DELETE" + } else if app.confirm_delete_file { + " — d AGAIN TO DELETE FILE" + } else { + "" + }; + let monitor_label = if detail.monitored { + "monitored" + } else { + "unmonitored" + }; + let have = app + .media_items + .iter() + .find(|m| m.id == detail.id) + .map(|m| m.missing_count == 0); + let file_line = match have { + Some(true) => ("file on disk", Color::Green), + Some(false) => ("missing", Color::Yellow), + None => ("file status unknown", Color::DarkGray), + }; + let kind = if detail.kind == "movie" { + "Movie" + } else { + "Series" + }; + let year = detail + .year + .map(|y| y.to_string()) + .unwrap_or_else(|| "—".into()); + let lines = vec![ + Line::from(Span::styled( + format!("{} ({year})", detail.title), + Style::default().add_modifier(Modifier::BOLD), + )), + Line::from(""), + Line::from(format!("{kind} · {monitor_label}")), + Line::from(Span::styled(file_line.0, Style::default().fg(file_line.1))), + Line::from(format!("root: {}", detail.root_folder)), + Line::from(""), + Line::from("s search now c pick release m monitor d delete file x remove"), + ]; + let paragraph = Paragraph::new(lines).block(Block::default().borders(Borders::ALL).title( + format!("{} ({year}) [{monitor_label}]{confirm}", detail.title), + )); + frame.render_widget(paragraph, area); +} + /// Manual release picker — candidates for whatever episode/movie was /// selected when `c` was pressed, scored (or gate-rejected with a reason) /// exactly like the automatic search pipeline would see them. @@ -187,15 +449,19 @@ fn draw_candidates(frame: &mut Frame, area: Rect, app: &App, media_title: &str) if c.is_season_pack { " [PACK]" } else { "" }, if c.is_repack { " [REPACK]" } else { "" }, ); - let verdict = match (c.score, &c.rejected_reason) { - (Some(score), _) => format!("score {score:.1}"), - (None, Some(reason)) => format!("REJECTED: {reason}"), - (None, None) => "unscored".to_string(), + let (verdict, color) = match (c.score, &c.rejected_reason) { + (Some(score), _) if score >= 8.0 => (format!("score {score:.1}"), Color::Green), + (Some(score), _) => (format!("score {score:.1}"), Color::Yellow), + (None, Some(reason)) => (format!("REJECTED: {reason}"), Color::Red), + (None, None) => ("unscored".to_string(), Color::DarkGray), }; - ListItem::new(format!( - "[{}] {} — {seeders} seeders, {size}{flags} — {verdict}", - c.source_name, c.raw_title - )) + ListItem::new(Line::from(vec![ + Span::raw(format!( + "[{}] {} — {seeders} seeders, {size}{flags} — ", + c.source_name, c.raw_title + )), + Span::styled(verdict, Style::default().fg(color)), + ])) }) .collect(); let list = List::new(items) @@ -207,18 +473,32 @@ fn draw_candidates(frame: &mut Frame, area: Rect, app: &App, media_title: &str) frame.render_stateful_widget(list, area, &mut state); } +fn history_status_color(status: &str) -> Color { + match status { + "imported" => Color::Green, + "grabbed" => Color::Yellow, + "failed" => Color::Red, + _ => Color::DarkGray, + } +} + fn draw_history(frame: &mut Frame, area: Rect, app: &App) { let items: Vec = app .releases .iter() .map(|r| { - ListItem::new(format!( - "[{}] {} — {} (score {:.1})", - r.status, - r.media_title, - r.raw_title, - r.score.unwrap_or(0.0) - )) + ListItem::new(Line::from(vec![ + Span::styled( + format!("[{}]", r.status), + Style::default().fg(history_status_color(&r.status)), + ), + Span::raw(format!( + " {} — {} (score {:.1})", + r.media_title, + r.raw_title, + r.score.unwrap_or(0.0) + )), + ])) }) .collect(); let list = List::new(items) @@ -233,12 +513,24 @@ fn draw_review(frame: &mut Frame, area: Rect, app: &App) { .review_items .iter() .map(|r| { - ListItem::new(format!( - "({:.0}%) {} -> {}", - r.confidence * 100.0, - r.raw_release_title, - r.candidate_media_title.as_deref().unwrap_or("?") - )) + let color = if r.confidence < 0.70 { + Color::Red + } else if r.confidence < 0.85 { + Color::Yellow + } else { + Color::Green + }; + ListItem::new(Line::from(vec![ + Span::styled( + format!("({:.0}%)", r.confidence * 100.0), + Style::default().fg(color), + ), + Span::raw(format!( + " {} -> {}", + r.raw_release_title, + r.candidate_media_title.as_deref().unwrap_or("?") + )), + ])) }) .collect(); let list = List::new(items) @@ -256,6 +548,11 @@ fn draw_review(frame: &mut Frame, area: Rect, app: &App) { /// than expected, how deep the review queue has backed up, and search /// targets that have been failing every attempt long enough for their /// backoff to hit its ceiling. Read-only report, no selection/navigation. +// Mirrors breadarrd/src/api/routes/stuck.rs::MAXED_SEARCH_COUNT. Duplicated +// because the daemon doesn't expose it via the API; if it drifts this is +// cosmetic only (wrong shade), not a behavior bug. +const MAXED_SEARCH_COUNT: i64 = 6; + fn draw_stuck(frame: &mut Frame, area: Rect, app: &App) { let Some(report) = &app.stuck else { let placeholder = Paragraph::new("loading...").block( @@ -272,42 +569,74 @@ fn draw_stuck(frame: &mut Frame, area: Rect, app: &App) { .constraints([Constraint::Min(3), Constraint::Min(3)]) .split(area); + let stalled_border = if matches!(app.stuck_focus, StuckSection::Stalled) { + Color::Cyan + } else { + Color::Reset + }; let stalled_items: Vec = report .stalled_grabs .iter() .map(|g| { - ListItem::new(format!( - "{} — {} (grabbed {})", - g.media_title, g.raw_title, g.grabbed_at - )) + ListItem::new(Line::from(Span::styled( + format!( + "{} — {} (grabbed {})", + g.media_title, g.raw_title, g.grabbed_at + ), + Style::default().fg(Color::Yellow), + ))) }) .collect(); - let stalled_list = - List::new(stalled_items).block(Block::default().borders(Borders::ALL).title(format!( - "Stalled grabs ({}) — review queue: {} pending", - report.stalled_grabs.len(), - report.review_queue_depth - ))); - frame.render_widget(stalled_list, chunks[0]); + let stalled_list = List::new(stalled_items) + .block( + Block::default() + .borders(Borders::ALL) + .border_style(Style::default().fg(stalled_border)) + .title(format!( + "Stalled grabs ({}) — review queue: {} pending", + report.stalled_grabs.len(), + report.review_queue_depth + )), + ) + .highlight_style(Style::default().add_modifier(Modifier::REVERSED)); + let mut stalled_state = app.stalled_state.clone(); + frame.render_stateful_widget(stalled_list, chunks[0], &mut stalled_state); + let maxed_border = if matches!(app.stuck_focus, StuckSection::Maxed) { + Color::Cyan + } else { + Color::Reset + }; let maxed_items: Vec = report .maxed_out_search_targets .iter() .map(|t| { - ListItem::new(format!( - "{} — {} attempts, last searched {}", - t.media_title, - t.search_count, - t.last_searched_at.as_deref().unwrap_or("never") - )) + let color = if t.search_count > MAXED_SEARCH_COUNT { + Color::Red + } else { + Color::Yellow + }; + ListItem::new(Line::from(Span::styled( + format!( + "{} — {} attempts, last searched {}", + t.media_title, + t.search_count, + t.last_searched_at.as_deref().unwrap_or("never") + ), + Style::default().fg(color), + ))) }) .collect(); - let maxed_list = List::new(maxed_items).block( - Block::default() - .borders(Borders::ALL) - .title("Search targets at max backoff"), - ); - frame.render_widget(maxed_list, chunks[1]); + let maxed_list = List::new(maxed_items) + .block( + Block::default() + .borders(Borders::ALL) + .border_style(Style::default().fg(maxed_border)) + .title("Search targets at max backoff — Enter: jump to show"), + ) + .highlight_style(Style::default().add_modifier(Modifier::REVERSED)); + let mut maxed_state = app.maxed_state.clone(); + frame.render_stateful_widget(maxed_list, chunks[1], &mut maxed_state); } fn draw_add(frame: &mut Frame, area: Rect, app: &App) { @@ -333,12 +662,21 @@ fn draw_add(frame: &mut Frame, area: Rect, app: &App) { .add_results .iter() .map(|r| { - ListItem::new(format!( - "[{}] {} ({})", - r.kind.label(), - r.result.title, - r.result.year.map(|y| y.to_string()).unwrap_or_default() - )) + let kind_color = match r.kind { + crate::app::AddKind::Movie => Color::Magenta, + crate::app::AddKind::Series => Color::Blue, + }; + ListItem::new(Line::from(vec![ + Span::styled( + format!("[{}] ", r.kind.label()), + Style::default().fg(kind_color), + ), + Span::raw(format!( + "{} ({})", + r.result.title, + r.result.year.map(|y| y.to_string()).unwrap_or_default() + )), + ])) }) .collect(); let list = List::new(items) @@ -353,36 +691,43 @@ fn draw_add(frame: &mut Frame, area: Rect, app: &App) { } /// What's aired recently or airs soon (a week back, three weeks forward — -/// see `calendar::DAYS_PAST`/`DAYS_FUTURE` server-side). Read-only, no -/// selection — a lookahead view, not something acted on directly here. +/// see `calendar::DAYS_PAST`/`DAYS_FUTURE` server-side). Selectable — +/// Enter jumps to that episode in Library. fn draw_calendar(frame: &mut Frame, area: Rect, app: &App) { let today = chrono::Local::now().date_naive().to_string(); let items: Vec = app .calendar .iter() .map(|e| { - let status = if e.has_file { - "✓" + let (status, color) = if e.has_file { + ("✓", Color::Green) } else if !e.monitored { - "-" + ("-", Color::DarkGray) } else if e.air_date.as_str() > today.as_str() { - "…" + ("…", Color::Yellow) } else { - "!" // aired, monitored, still missing + ("!", Color::Red) }; + let today_mark = if e.air_date == today { " today" } else { "" }; let title = e.title.as_deref().unwrap_or(""); - ListItem::new(format!( - "{status} {} {} S{:02}E{:02} {title}", - e.air_date, e.media_title, e.season_number, e.episode_number - )) + ListItem::new(Line::from(vec![ + Span::styled(status, Style::default().fg(color)), + Span::raw(format!( + " {} {} S{:02}E{:02} {title}{today_mark}", + e.air_date, e.media_title, e.season_number, e.episode_number + )), + ])) }) .collect(); - let list = List::new(items).block( - Block::default() - .borders(Borders::ALL) - .title("Calendar — ✓ have it … upcoming ! aired but missing"), - ); - frame.render_widget(list, area); + let list = List::new(items) + .block( + Block::default() + .borders(Borders::ALL) + .title("Calendar — Enter: jump ✓ have it … upcoming ! aired but missing"), + ) + .highlight_style(Style::default().add_modifier(Modifier::REVERSED)); + let mut state = app.calendar_state.clone(); + frame.render_stateful_widget(list, area, &mut state); } /// Read-only report on `media_file_probe` state: corruption, under-quality, @@ -402,7 +747,7 @@ fn draw_library_health(frame: &mut Frame, area: Rect, app: &App) { let chunks = Layout::default() .direction(Direction::Vertical) - .constraints([Constraint::Length(5), Constraint::Min(3)]) + .constraints([Constraint::Length(6), Constraint::Min(3)]) .split(area); let s = &report.summary; @@ -412,8 +757,13 @@ fn draw_library_health(frame: &mut Frame, area: Rect, app: &App) { .map(|c| format!("{}={}", c.codec, c.count)) .collect::>() .join(", "); + let cycles = app + .health + .as_ref() + .map(cycle_summary_line) + .unwrap_or_default(); let summary_text = format!( - "{} files, {:.1} GB total, {} probed — resolution: SD={} 720p={} 1080p={} 4K={} \ + "{cycles}\n{} files, {:.1} GB total, {} probed — resolution: SD={} 720p={} 1080p={} 4K={} \ — subtitles: {:.0}% — codecs: {codec_summary}", s.total_files, s.total_size_bytes as f64 / 1_073_741_824.0, @@ -482,12 +832,38 @@ fn draw_library_health(frame: &mut Frame, area: Rect, app: &App) { items.push(ListItem::new("Nothing flagged — library looks clean.")); } - let list = List::new(items).block( - Block::default() - .borders(Borders::ALL) - .title("Flagged files"), - ); - frame.render_widget(list, chunks[1]); + let list = List::new(items) + .block( + Block::default() + .borders(Borders::ALL) + .title("Flagged files"), + ) + .highlight_style(Style::default().add_modifier(Modifier::REVERSED)); + let mut state = app.health_state.clone(); + frame.render_stateful_widget(list, chunks[1], &mut state); +} + +fn cycle_bit(name: &str, info: &Option) -> String { + match info { + Some(c) if c.ok => format!("{name}: ok"), + Some(_) => format!("{name}: FAIL"), + None => format!("{name}: —"), + } +} + +fn cycle_summary_line(h: &breadarr_shared::dto::HealthDetail) -> String { + format!( + "{} | {} | {} | {}{}", + cycle_bit("grab", &h.last_grab_cycle), + cycle_bit("import", &h.last_import_cycle), + cycle_bit("search", &h.last_search_cycle), + cycle_bit("upgrade", &h.last_upgrade_cycle), + if h.search_halted { + " | search HALTED" + } else { + "" + } + ) } /// Quality-profile weight editing — list of profiles, then (once one is @@ -549,11 +925,23 @@ fn draw_profiles(frame: &mut Frame, area: Rect, app: &App) { } fn draw_status(frame: &mut Frame, area: Rect, app: &App) { - let text = if app.status.is_empty() { - "Tab: switch view | j/k: move | q: quit".to_string() - } else { + let text = if !app.status.is_empty() { app.status.clone() + } else { + let hints: Vec = context_keybindings(app) + .into_iter() + .take(4) + .map(|(key, desc)| format!("{key}: {desc}")) + .collect(); + format!("{} | ?: help", hints.join(" | ")) }; - let status = Paragraph::new(text).block(Block::default().borders(Borders::ALL)); + let style = if app.busy { + Style::default().fg(Color::Yellow) + } else { + Style::default() + }; + let status = Paragraph::new(text) + .style(style) + .block(Block::default().borders(Borders::ALL)); frame.render_widget(status, area); } diff --git a/breadarrd/Cargo.toml b/breadarrd/Cargo.toml index af3d900..48cebe6 100644 --- a/breadarrd/Cargo.toml +++ b/breadarrd/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "breadarrd" -version = "0.1.0" +version = "0.1.1" edition = "2021" [dependencies] @@ -19,9 +19,15 @@ regex.workspace = true serde_json.workspace = true ort.workspace = true tokenizers.workspace = true -# TODO(owner): switch to tag-pinned git dependency once bread-onnx is merged and tagged, matching the bread-theme pattern -bread-onnx = { path = "../../bread-ecosystem/bread-onnx" } +bread-onnx = { git = "https://git.breadway.dev/Breadway/bread-ecosystem", tag = "v0.7.2" } scraper.workspace = true chrono.workspace = true fastrand.workspace = true nix.workspace = true +# Forces reqwest's native-tls backend to statically build and link its own +# OpenSSL instead of dynamically linking whatever libssl/libcrypto happens +# to be on the build host — CI (see ci/build.sh) links against an archived +# old-glibc sysroot for hestia compatibility, which has no libssl/libcrypto +# of its own to dynamically link against. Also means the shipped binary no +# longer depends on the target system's OpenSSL version at all. +openssl-sys = { version = "0.9", features = ["vendored"] } diff --git a/breadarrd/src/api/mod.rs b/breadarrd/src/api/mod.rs index 2c6c309..ec8932d 100644 --- a/breadarrd/src/api/mod.rs +++ b/breadarrd/src/api/mod.rs @@ -71,13 +71,14 @@ pub enum BackgroundRequest { /// Mutex` is fine here (unlike `conn`) since updates are a single field /// write with no `.await` in between. Exists so a silently-stalled /// background loop (e.g. every cycle erroring for hours) is visible from a -/// single `/health` call instead of only in the journal. +/// single `/health/detail` call instead of only in the journal. #[derive(Clone, Default)] pub struct CycleStatus { pub last_grab: Option, pub last_import: Option, pub last_search: Option, pub last_upgrade: Option, + pub last_transcode: Option, /// Set once the search-driven loop's consecutive-failure backoff hits /// its ceiling — still ticking at max backoff underneath (self-healing /// if the source recovers), but worth a loud, easy-to-spot signal that @@ -95,10 +96,11 @@ pub struct CycleRecord { /// Rejects any request lacking `Authorization: Bearer ` /// once a token is actually configured — a no-op (every request passes) /// when it's empty, so an unconfigured install behaves exactly as before. -/// `/health` is deliberately exempt even with a token configured: it's -/// commonly polled by external monitoring (e.g. an uptime dashboard) that -/// has no reason to hold the same credential as the TUI/API client, and it -/// exposes nothing more sensitive than "is the process alive." +/// Exact `/health` is deliberately exempt even with a token configured: +/// it's commonly polled by external monitoring (e.g. an uptime dashboard) +/// that has no reason to hold the same credential as the TUI/API client, +/// and it exposes nothing more sensitive than "is the process alive." +/// `/health/detail` is *not* exempt — it includes cycle status. async fn require_api_token(State(state): State, req: Request, next: Next) -> Response { if state.config.daemon.api_token.is_empty() || req.uri().path() == "/health" { return next.run(req).await; @@ -108,16 +110,36 @@ async fn require_api_token(State(state): State, req: Request, next: Ne .get(axum::http::header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) .and_then(|v| v.strip_prefix("Bearer ")) - .is_some_and(|token| token == state.config.daemon.api_token); + .is_some_and(|token| constant_time_eq(token, &state.config.daemon.api_token)); if !authorized { return (StatusCode::UNAUTHORIZED, "missing or invalid API token").into_response(); } next.run(req).await } +/// Byte-wise `==` short-circuits on the first mismatching byte, making +/// comparison time a (weak, but real) signal of how many leading bytes of a +/// guessed token were correct — a classic timing oracle. This always walks +/// the longer input (padding the shorter against a dummy) and folds a +/// length mismatch into the accumulator so a length difference is not a +/// first-instruction return. Pair with `api_token.len() >= 16` in +/// `Config::validate` so a length-oracle of short guesses is useless. +fn constant_time_eq(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + let max = a.len().max(b.len()); + let mut acc = u8::from(a.len() != b.len()); + for i in 0..max { + let x = *a.get(i).unwrap_or(&0); + let y = *b.get(i).unwrap_or(&0); + acc |= x ^ y; + } + acc == 0 +} + pub fn router(state: AppState) -> Router { Router::new() .route("/health", get(routes::health::health)) + .route("/health/detail", get(routes::health::health_detail)) .route("/media", get(routes::media::list).post(routes::media::add)) .route( "/media/:id", @@ -178,3 +200,28 @@ pub fn router(state: AppState) -> Router { )) .with_state(state) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn constant_time_eq_matches_identical_strings() { + assert!(constant_time_eq("secret-token", "secret-token")); + } + + #[test] + fn constant_time_eq_rejects_different_strings_of_the_same_length() { + assert!(!constant_time_eq("secret-token", "secret-toke1")); + } + + #[test] + fn constant_time_eq_rejects_different_lengths() { + assert!(!constant_time_eq("short", "a-much-longer-token")); + } + + #[test] + fn constant_time_eq_treats_empty_strings_as_equal() { + assert!(constant_time_eq("", "")); + } +} diff --git a/breadarrd/src/api/routes/health.rs b/breadarrd/src/api/routes/health.rs index 4333efd..3e43f4b 100644 --- a/breadarrd/src/api/routes/health.rs +++ b/breadarrd/src/api/routes/health.rs @@ -1,6 +1,6 @@ use axum::extract::State; use axum::Json; -use breadarr_shared::dto::{CycleInfo, HealthDetail}; +use breadarr_shared::dto::{CycleInfo, HealthDetail, HealthStatus}; use crate::api::AppState; @@ -12,7 +12,15 @@ fn to_info(r: &crate::api::CycleRecord) -> CycleInfo { } } -pub async fn health(State(state): State) -> Json { +/// Unauthenticated liveness — cheap, no cycle detail. +pub async fn health() -> Json { + Json(HealthStatus { + status: "ok".to_string(), + }) +} + +/// Authenticated cycle-status payload (same as the old `/health`). +pub async fn health_detail(State(state): State) -> Json { let status = state.cycle_status.lock().expect("cycle_status poisoned"); Json(HealthDetail { status: "ok".to_string(), @@ -20,6 +28,7 @@ pub async fn health(State(state): State) -> Json { last_import_cycle: status.last_import.as_ref().map(to_info), last_search_cycle: status.last_search.as_ref().map(to_info), last_upgrade_cycle: status.last_upgrade.as_ref().map(to_info), + last_transcode_cycle: status.last_transcode.as_ref().map(to_info), search_halted: status.search_halted, }) } diff --git a/breadarrd/src/api/routes/media.rs b/breadarrd/src/api/routes/media.rs index 0c96885..83c0de3 100644 --- a/breadarrd/src/api/routes/media.rs +++ b/breadarrd/src/api/routes/media.rs @@ -1,3 +1,5 @@ +use std::path::{Component, Path as FsPath, PathBuf}; + use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::Json; @@ -5,6 +7,7 @@ use breadarr_shared::dto::{ AddMovieRequest, AddMovieResponse, AddSeriesRequest, AddSeriesResponse, EpisodeSummary, MediaItemDetail, MediaItemSummary, SearchNowResult, }; +use breadarr_shared::Config; use rusqlite::{params, OptionalExtension}; use crate::api::AppState; @@ -45,7 +48,7 @@ pub async fn detail( Path(id): Path, ) -> Result, (StatusCode, String)> { let conn = state.conn.lock().await; - let (kind, title, year, monitored, root_folder) = conn + let row = conn .query_row( "SELECT kind, title, year, monitored, root_folder FROM media_item WHERE id = ?1", params![id], @@ -59,7 +62,11 @@ pub async fn detail( )) }, ) - .map_err(|e| (StatusCode::NOT_FOUND, e.to_string()))?; + .optional() + .map_err(internal)?; + let Some((kind, title, year, monitored, root_folder)) = row else { + return Err((StatusCode::NOT_FOUND, format!("no media_item {id}"))); + }; let mut stmt = conn .prepare( @@ -105,6 +112,8 @@ pub async fn add( )); }; + let root_folder = constrain_root_folder(&req.root_folder, &state.config.default_root_folder())?; + // Fetch before taking the lock — a sync MutexGuard can't be held across // an `.await` point. let episodes = tvdb.episodes(&req.tvdb_id).await.map_err(internal)?; @@ -117,7 +126,7 @@ pub async fn add( &req.title, req.year.map(|y| y as u32), &req.aliases, - &req.root_folder, + &root_folder, 1, &episodes, ) @@ -131,13 +140,14 @@ pub async fn add_movie( State(state): State, Json(req): Json, ) -> Result, (StatusCode, String)> { + let root_folder = constrain_root_folder(&req.root_folder, &state.config.movies_root_folder())?; let conn = state.conn.lock().await; let media_item_id = metadata::insert_movie( &conn, &req.tmdb_id, &req.title, req.year.map(|y| y as u32), - &req.root_folder, + &root_folder, 2, ) .map_err(internal)?; @@ -208,13 +218,9 @@ async fn set_episode_monitored( Ok(StatusCode::NO_CONTENT) } -/// Toggles every episode in one season at once — a season-level "row" isn't -/// separately tracked (the `season` table exists in the schema but was -/// never actually populated by any insert path, so resurrecting it just to -/// hold one redundant monitored flag would mean keeping two copies of the -/// same state in sync for no behavioral gain); bulk-updating the episodes -/// directly gets the identical practical effect — this season's episodes -/// stop appearing in search enumeration — with one source of truth. +/// Toggles every episode in one season and the `season.monitored` flag +/// when a season row exists (`insert_series` writes those). A missing +/// season row is not an error — episodes still update. pub async fn monitor_season( State(state): State, Path((media_item_id, season_number)): Path<(i64, i64)>, @@ -242,6 +248,11 @@ async fn set_season_monitored( params![monitored as i64, media_item_id, season_number], ) .map_err(internal)?; + // Season row is best-effort — older libraries (or movies) may have none. + let _ = conn.execute( + "UPDATE season SET monitored = ?1 WHERE media_item_id = ?2 AND season_number = ?3", + params![monitored as i64, media_item_id, season_number], + ); if rows == 0 { return Err(( StatusCode::NOT_FOUND, @@ -283,21 +294,20 @@ pub async fn delete_episode_file( Path(episode_id): Path, ) -> Result { let conn = state.conn.lock().await; - let row: Option<(i64, String)> = conn - .query_row( - "SELECT id, path FROM episode_file WHERE episode_id = ?1", - params![episode_id], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional() - .map_err(internal)?; - let Some((file_id, path)) = row else { + let files = tracked_files( + &conn, + "SELECT id, path FROM episode_file WHERE episode_id = ?1", + episode_id, + )?; + if files.is_empty() { return Err(( StatusCode::NOT_FOUND, format!("no file tracked for episode {episode_id}"), )); - }; - delete_file_and_clear(&conn, &path, file_id)?; + } + for (file_id, path) in files { + delete_file_and_clear(&conn, &path, file_id)?; + } conn.execute( "UPDATE episode SET has_file = 0 WHERE id = ?1", params![episode_id], @@ -315,21 +325,20 @@ pub async fn delete_movie_file( Path(media_item_id): Path, ) -> Result { let conn = state.conn.lock().await; - let row: Option<(i64, String)> = conn - .query_row( - "SELECT id, path FROM episode_file WHERE media_item_id = ?1 AND episode_id IS NULL", - params![media_item_id], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional() - .map_err(internal)?; - let Some((file_id, path)) = row else { + let files = tracked_files( + &conn, + "SELECT id, path FROM episode_file WHERE media_item_id = ?1 AND episode_id IS NULL", + media_item_id, + )?; + if files.is_empty() { return Err(( StatusCode::NOT_FOUND, format!("no file tracked for media_item {media_item_id}"), )); - }; - delete_file_and_clear(&conn, &path, file_id)?; + } + for (file_id, path) in files { + delete_file_and_clear(&conn, &path, file_id)?; + } Ok(StatusCode::NO_CONTENT) } @@ -339,6 +348,20 @@ pub async fn delete_movie_file( /// alone, which a TV show shares across every one of its episode rows and /// would otherwise risk wiping an entire show's tracked files instead of /// the one the caller actually looked up. +fn tracked_files( + conn: &rusqlite::Connection, + sql: &str, + id: i64, +) -> Result, (StatusCode, String)> { + let mut stmt = conn.prepare(sql).map_err(internal)?; + let files = stmt + .query_map(params![id], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(internal)? + .collect::>>() + .map_err(internal)?; + Ok(files) +} + fn delete_file_and_clear( conn: &rusqlite::Connection, path: &str, @@ -502,6 +525,59 @@ async fn grab_candidate_via_background( Ok(StatusCode::NO_CONTENT) } +/// After `~/` expand and lexical normalize, the path must stay under +/// `allowed_root`. `..` components and absolute paths outside that root +/// are rejected with 400. +fn constrain_root_folder( + requested: &str, + allowed_root: &FsPath, +) -> Result { + let requested = requested.trim(); + if requested.is_empty() { + return Err(( + StatusCode::BAD_REQUEST, + "root_folder must not be empty".into(), + )); + } + let expanded = Config::expand_path(requested); + if expanded + .components() + .any(|c| matches!(c, Component::ParentDir)) + { + return Err(( + StatusCode::BAD_REQUEST, + "root_folder must not contain '..'".into(), + )); + } + let candidate = if expanded.is_absolute() { + normalize_lexically(&expanded) + } else { + normalize_lexically(&allowed_root.join(expanded)) + }; + let root = normalize_lexically(allowed_root); + if !candidate.starts_with(&root) { + return Err(( + StatusCode::BAD_REQUEST, + format!("root_folder must be under {}", root.display()), + )); + } + Ok(candidate.to_string_lossy().into_owned()) +} + +fn normalize_lexically(path: &FsPath) -> PathBuf { + let mut out = PathBuf::new(); + for c in path.components() { + match c { + Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + other => out.push(other), + } + } + out +} + fn internal(e: E) -> (StatusCode, String) { (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) } diff --git a/breadarrd/src/api/routes/quality_profiles.rs b/breadarrd/src/api/routes/quality_profiles.rs index 0387a7b..50814e9 100644 --- a/breadarrd/src/api/routes/quality_profiles.rs +++ b/breadarrd/src/api/routes/quality_profiles.rs @@ -68,6 +68,12 @@ pub async fn update_weights( Path(id): Path, Json(req): Json, ) -> Result { + if !weights_are_valid(&req.weights) { + return Err(( + StatusCode::BAD_REQUEST, + "quality-profile weights must be finite and >= 0".into(), + )); + } let weights_json = serde_json::to_string(&req.weights).map_err(internal)?; let conn = state.conn.lock().await; let updated = conn @@ -82,6 +88,59 @@ pub async fn update_weights( Ok(StatusCode::NO_CONTENT) } +fn weights_are_valid(w: &WeightsDto) -> bool { + [ + w.seeder, + w.resolution_tier, + w.source_tier, + w.codec_tier, + w.bit_depth, + w.container, + w.group_allowlist, + w.repack, + w.hdr, + ] + .into_iter() + .all(|v| v.is_finite() && v >= 0.0) +} + fn internal(e: E) -> (StatusCode, String) { (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn valid_weights() -> WeightsDto { + WeightsDto { + seeder: 1.0, + resolution_tier: 1.0, + source_tier: 1.0, + codec_tier: 1.0, + bit_depth: 1.0, + container: 1.0, + group_allowlist: 1.0, + repack: 1.0, + hdr: 1.0, + } + } + + #[test] + fn weights_are_valid_accepts_finite_non_negative() { + assert!(weights_are_valid(&valid_weights())); + } + + #[test] + fn weights_are_valid_rejects_nan_inf_and_negative() { + let mut w = valid_weights(); + w.seeder = f32::NAN; + assert!(!weights_are_valid(&w)); + w = valid_weights(); + w.hdr = f32::INFINITY; + assert!(!weights_are_valid(&w)); + w = valid_weights(); + w.repack = -0.1; + assert!(!weights_are_valid(&w)); + } +} diff --git a/breadarrd/src/api/routes/review.rs b/breadarrd/src/api/routes/review.rs index f16fbeb..8a846bc 100644 --- a/breadarrd/src/api/routes/review.rs +++ b/breadarrd/src/api/routes/review.rs @@ -75,15 +75,27 @@ pub async fn approve( } }; - let torrent_hash = scheduler::grab_prepared_approval(qbit, &state.qbit_category, &prepared) - .await - .map_err(internal)?; + let torrent_hash = + match scheduler::grab_prepared_approval(qbit, &state.qbit_category, &prepared).await { + Ok(hash) => hash, + Err(e) => { + // The grab errored outright (not just "added but no hash + // captured" — `finalize_review_approval` below handles that + // case and still runs to completion). `prepare_review_approval` + // already claimed this row into `approved` before we got here; + // without releasing it back to `pending`, a transient + // qBittorrent error would strand the review permanently + // unapprovable with nothing ever recorded for it. + let conn = state.conn.lock().await; + let _ = scheduler::release_review_claim(&conn, id); + return Err(internal(e)); + } + }; { let conn = state.conn.lock().await; scheduler::finalize_review_approval( &conn, - id, &prepared, &state.qbit_category, torrent_hash.as_deref(), @@ -99,7 +111,12 @@ pub async fn reject( Path(id): Path, ) -> Result { let conn = state.conn.lock().await; - scheduler::reject_review(&conn, id).map_err(internal)?; + if !scheduler::reject_review(&conn, id).map_err(internal)? { + return Err(( + StatusCode::NOT_FOUND, + format!("no pending review item {id}"), + )); + } Ok(StatusCode::NO_CONTENT) } diff --git a/breadarrd/src/api/routes/search.rs b/breadarrd/src/api/routes/search.rs index cc032e4..6d6ea7b 100644 --- a/breadarrd/src/api/routes/search.rs +++ b/breadarrd/src/api/routes/search.rs @@ -21,43 +21,50 @@ pub async fn search( State(state): State, Query(params): Query, ) -> Result>, (StatusCode, String)> { - if params.kind == "movie" { - let Some(tmdb) = &state.tmdb else { - return Err(( - StatusCode::PRECONDITION_FAILED, - "tmdb.bearer_token is not configured".into(), - )); - }; - let results = tmdb - .search_movie(¶ms.q) - .await - .map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))? - .into_iter() - .map(|r| SearchResult { - external_id: r.external_id, - title: r.title, - year: r.year.map(|y| y as i64), - }) - .collect(); - return Ok(Json(results)); + match params.kind.as_str() { + "movie" => { + let Some(tmdb) = &state.tmdb else { + return Err(( + StatusCode::PRECONDITION_FAILED, + "tmdb.bearer_token is not configured".into(), + )); + }; + let results = tmdb + .search_movie(¶ms.q) + .await + .map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))? + .into_iter() + .map(|r| SearchResult { + external_id: r.external_id, + title: r.title, + year: r.year.map(|y| y as i64), + }) + .collect(); + Ok(Json(results)) + } + "series" => { + let Some(tvdb) = &state.tvdb else { + return Err(( + StatusCode::PRECONDITION_FAILED, + "tvdb.api_key is not configured".into(), + )); + }; + let results = tvdb + .search_series(¶ms.q) + .await + .map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))? + .into_iter() + .map(|r| SearchResult { + external_id: r.external_id, + title: r.name, + year: r.year.map(|y| y as i64), + }) + .collect(); + Ok(Json(results)) + } + other => Err(( + StatusCode::BAD_REQUEST, + format!("kind must be series or movie, got {other:?}"), + )), } - - let Some(tvdb) = &state.tvdb else { - return Err(( - StatusCode::PRECONDITION_FAILED, - "tvdb.api_key is not configured".into(), - )); - }; - let results = tvdb - .search_series(¶ms.q) - .await - .map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))? - .into_iter() - .map(|r| SearchResult { - external_id: r.external_id, - title: r.name, - year: r.year.map(|y| y as i64), - }) - .collect(); - Ok(Json(results)) } diff --git a/breadarrd/src/api/routes/stuck.rs b/breadarrd/src/api/routes/stuck.rs index c4d6888..5cae652 100644 --- a/breadarrd/src/api/routes/stuck.rs +++ b/breadarrd/src/api/routes/stuck.rs @@ -23,7 +23,7 @@ pub async fn stuck( let mut stmt = conn .prepare( - "SELECT r.id, m.title, r.raw_title, r.grabbed_at + "SELECT r.id, r.media_item_id, m.title, r.raw_title, r.grabbed_at FROM release r JOIN media_item m ON m.id = r.media_item_id WHERE r.status = 'grabbed' AND (julianday('now') - julianday(r.grabbed_at)) * 24.0 > ?1 @@ -34,9 +34,10 @@ pub async fn stuck( .query_map([STALLED_GRAB_HOURS], |row| { Ok(StalledGrab { release_id: row.get(0)?, - media_title: row.get(1)?, - raw_title: row.get(2)?, - grabbed_at: row.get(3)?, + media_item_id: row.get(1)?, + media_title: row.get(2)?, + raw_title: row.get(3)?, + grabbed_at: row.get(4)?, }) }) .map_err(internal)? diff --git a/breadarrd/src/db.rs b/breadarrd/src/db.rs index 88ff19a..6ecee1d 100644 --- a/breadarrd/src/db.rs +++ b/breadarrd/src/db.rs @@ -5,14 +5,14 @@ use rusqlite::Connection; /// doesn't slowly fill the disk with an ever-growing pile of copies. const MAX_BACKUPS: usize = 5; -/// Copies the database (and its WAL/SHM sidecar files, if present — WAL -/// mode means the real state can be split across all three) to a timestamped -/// backup before the daemon opens it, then prunes old backups beyond +/// Writes a consistent snapshot of the existing database to a timestamped +/// file under `/backups/`, then prunes old backups beyond /// `MAX_BACKUPS`. A no-op if there's no existing database yet (fresh -/// install — nothing to back up). Sonarr/Radarr back themselves up before -/// every upgrade; breadarr has no migration framework to trigger that same -/// moment, so this runs on every startup instead, which is a superset of -/// the same protection. +/// install — nothing to back up). Uses `VACUUM INTO` so WAL state is +/// folded into one standalone file; a raw `fs::copy` of a live WAL +/// database can be torn. Sonarr/Radarr back themselves up before every +/// upgrade; breadarr has no migration framework to trigger that same +/// moment, so this runs on every startup instead. pub fn backup_before_open(db_path: &std::path::Path) -> anyhow::Result<()> { if !db_path.exists() { return Ok(()); @@ -32,15 +32,13 @@ pub fn backup_before_open(db_path: &std::path::Path) -> anyhow::Result<()> { // and dedupe correctly instead of the second one silently overwriting. let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S%3fZ"); let dest = backup_dir.join(format!("{timestamp}-{stem}")); - std::fs::copy(db_path, &dest)?; - for sidecar_ext in ["-wal", "-shm"] { - let sidecar = std::path::PathBuf::from(format!("{}{sidecar_ext}", db_path.display())); - if sidecar.exists() { - let dest_sidecar = backup_dir.join(format!("{timestamp}-{stem}{sidecar_ext}")); - std::fs::copy(&sidecar, &dest_sidecar)?; - } - } + let src = Connection::open_with_flags(db_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?; + // Path is interpolated (VACUUM INTO does not bind `?` parameters); + // single quotes in the path are doubled so the SQL string stays valid. + let dest_sql = dest.to_string_lossy().replace('\'', "''"); + src.execute(&format!("VACUUM INTO '{dest_sql}'"), [])?; + drop(src); prune_old_backups(&backup_dir, stem)?; Ok(()) @@ -280,9 +278,7 @@ pub fn init(conn: &Connection) -> anyhow::Result<()> { CREATE INDEX IF NOT EXISTS idx_event_history_media_item ON event_history(media_item_id, occurred_at); - -- Placeholder profiles until Phase 6 builds real quality-scoring - -- weights; media_item.quality_profile_id needs something to - -- reference in the meantime. + -- Default profiles. Scoring reads these rows' `weights` on every grab. INSERT OR IGNORE INTO quality_profile (id, name, kind, weights) VALUES (1, 'Default TV', 'tv', '{}'); INSERT OR IGNORE INTO quality_profile (id, name, kind, weights) @@ -353,7 +349,39 @@ pub fn init(conn: &Connection) -> anyhow::Result<()> { fetched_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_torrent_fetch_hash ON torrent_fetch(torrent_hash); - CREATE INDEX IF NOT EXISTS idx_torrent_fetch_fetched_at ON torrent_fetch(fetched_at);", + CREATE INDEX IF NOT EXISTS idx_torrent_fetch_fetched_at ON torrent_fetch(fetched_at); + + -- One row per file queued for AV1 transcoding, whether from the + -- post-import async hook or the `transcode-library` backfill sweep. + -- Persisted (not an in-memory queue) so a `pending`/`running` row + -- left over from a daemon crash mid-encode just gets picked up + -- again on the next tick instead of silently vanishing. + CREATE TABLE IF NOT EXISTS transcode_job ( + id INTEGER PRIMARY KEY, + episode_file_id INTEGER NOT NULL REFERENCES episode_file(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','running','done','failed','skipped')), + original_codec TEXT, + original_bytes INTEGER, + new_bytes INTEGER, + error TEXT, + queued_at TEXT NOT NULL, + finished_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_transcode_job_status ON transcode_job(status); + -- Prevents two jobs for the same file ever being active at once — + -- closes the door on the same file getting encoded twice + -- concurrently regardless of how it happened (a stray duplicate + -- enqueue, a daemon-restart reset racing a still-alive backfill). + CREATE UNIQUE INDEX IF NOT EXISTS idx_transcode_job_active_episode_file + ON transcode_job(episode_file_id) WHERE status IN ('pending','running'); + + CREATE INDEX IF NOT EXISTS idx_release_status ON release(status); + CREATE INDEX IF NOT EXISTS idx_release_torrent_hash ON release(torrent_hash); + CREATE INDEX IF NOT EXISTS idx_release_episode_id ON release(episode_id); + CREATE INDEX IF NOT EXISTS idx_episode_file_episode_id ON episode_file(episode_id); + CREATE INDEX IF NOT EXISTS idx_review_queue_status ON review_queue(status); + CREATE INDEX IF NOT EXISTS idx_episode_air_date ON episode(air_date);", )?; // Progress watermark for stalled-download detection (added after the @@ -411,6 +439,196 @@ pub fn init(conn: &Connection) -> anyhow::Result<()> { // less-common stream metadata). See `ffprobe::MediaProbe::raw_json`. add_column_if_missing(conn, "media_file_probe", "raw_ffprobe_json", "TEXT")?; + // Set once a file has been through a successful local AV1 transcode — + // stops the upgrade cycle from treating it as still needing a bigger + // HEVC/H264 release, since `best_existing_score` otherwise only ever + // sees the stored `release.score` from original-grab time, which a + // local re-encode never touches. + add_column_if_missing( + conn, + "episode_file", + "upgrade_locked", + "INTEGER NOT NULL DEFAULT 0", + )?; + + // Decided at enqueue time (path prefix match against + // `transcode.anime_root_folders`, OR'd with the `anime_mapping`/ + // `anime_tmdb_movie` metadata check) and carried on the job row so + // `claim_pending_jobs` can dispatch straight to the right encode + // pipeline (`run_ffmpeg_encode_anime` vs `_live_action`) without + // re-deriving it — the eligibility metadata lookups aren't available + // from the job row's own columns alone (no media_item_id here). + add_column_if_missing( + conn, + "transcode_job", + "is_anime", + "INTEGER NOT NULL DEFAULT 0", + )?; + + // Lets a job re-encode a file that's already AV1 — normally + // `encode_and_verify` treats "already AV1" as nothing-to-do and skips + // the encode entirely, which is right for a fresh library scan but + // wrong for deliberately re-transcoding a file that got mis-encoded + // (e.g. the 476 files a real rate-control bug left *larger* than their + // original — already AV1, so the normal backfill query skips them, but + // they're exactly what a remediation pass needs to revisit). See + // `find_oversized_av1_candidates`. + add_column_if_missing( + conn, + "transcode_job", + "force_reencode", + "INTEGER NOT NULL DEFAULT 0", + )?; + + ensure_indexes(conn)?; + + Ok(()) +} + +fn index_exists(conn: &Connection, name: &str) -> anyhow::Result { + let n: i64 = conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name = ?1", + [name], + |row| row.get(0), + )?; + Ok(n > 0) +} + +/// `CREATE UNIQUE INDEX IF NOT EXISTS` still errors when existing rows +/// violate uniqueness (IF NOT EXISTS only checks the index name). A dirty +/// production DB must still start, so uniqueness failures warn and skip. +fn create_unique_index_best_effort(conn: &Connection, name: &str, ddl: &str) -> bool { + match conn.execute(ddl, []) { + Ok(_) => true, + Err(e) => { + tracing::warn!( + index = name, + error = %e, + "skipping unique index; existing rows would violate it" + ); + false + } + } +} + +/// Delete extra `media_item` rows that share a non-NULL `tvdb_id`/`tmdb_id`, +/// keeping the lowest `id`. Extras with any `episode` or `episode_file` +/// rows are left in place — those are not safe to drop. Returns whether +/// every duplicate group was reduced to a single row. +fn dedupe_media_item_external_id(conn: &Connection, column: &str) -> anyhow::Result { + debug_assert!(column == "tvdb_id" || column == "tmdb_id"); + let sql = format!( + "SELECT {column}, MIN(id) FROM media_item + WHERE {column} IS NOT NULL + GROUP BY {column} + HAVING COUNT(*) > 1" + ); + let dupes: Vec<(i64, i64)> = { + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect::>>()? + }; + + let mut safe = true; + let extra_sql = format!("SELECT id FROM media_item WHERE {column} = ?1 AND id != ?2"); + for (ext_id, keep_id) in dupes { + let extras: Vec = { + let mut stmt = conn.prepare(&extra_sql)?; + let rows = stmt.query_map(rusqlite::params![ext_id, keep_id], |row| row.get(0))?; + rows.collect::>>()? + }; + for extra_id in extras { + let episode_count: i64 = conn.query_row( + "SELECT count(*) FROM episode WHERE media_item_id = ?1", + [extra_id], + |row| row.get(0), + )?; + let file_count: i64 = conn.query_row( + "SELECT count(*) FROM episode_file WHERE media_item_id = ?1", + [extra_id], + |row| row.get(0), + )?; + if episode_count == 0 && file_count == 0 { + conn.execute("DELETE FROM media_item WHERE id = ?1", [extra_id])?; + } else { + tracing::warn!( + column, + ext_id, + keep_id, + extra_id, + episode_count, + file_count, + "cannot safely dedupe media_item; extra row has episodes or files" + ); + safe = false; + } + } + } + Ok(safe) +} + +fn ensure_unique_external_id_index( + conn: &Connection, + column: &str, + unique_name: &str, + unique_ddl: &str, + lookup_ddl: &str, +) -> anyhow::Result<()> { + let unique_ok = if dedupe_media_item_external_id(conn, column)? { + create_unique_index_best_effort(conn, unique_name, unique_ddl) + } else { + tracing::warn!( + index = unique_name, + column, + "skipping unique index; media_item still has unsafely-duplicated rows" + ); + false + }; + if !unique_ok && !index_exists(conn, unique_name)? { + conn.execute(lookup_ddl, [])?; + } + Ok(()) +} + +fn ensure_indexes(conn: &Connection) -> anyhow::Result<()> { + ensure_unique_external_id_index( + conn, + "tvdb_id", + "idx_media_item_tvdb", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_media_item_tvdb + ON media_item(tvdb_id) WHERE tvdb_id IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_media_item_tvdb_lookup ON media_item(tvdb_id)", + )?; + ensure_unique_external_id_index( + conn, + "tmdb_id", + "idx_media_item_tmdb", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_media_item_tmdb + ON media_item(tmdb_id) WHERE tmdb_id IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_media_item_tmdb_lookup ON media_item(tmdb_id)", + )?; + + create_unique_index_best_effort( + conn, + "idx_episode_file_path", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_episode_file_path ON episode_file(path)", + ); + create_unique_index_best_effort( + conn, + "idx_alias_item_text", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_alias_item_text ON alias(media_item_id, text)", + ); + create_unique_index_best_effort( + conn, + "idx_review_pending_title", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_review_pending_title + ON review_queue(candidate_media_item_id, raw_release_title) + WHERE status = 'pending'", + ); + // No unique on release(source_id, guid): status can cycle and a + // re-grab of the same guid is legitimate. `seen_guid` already records + // first-seen. + Ok(()) } @@ -473,7 +691,7 @@ mod tests { |row| row.get(0), ) .unwrap(); - assert_eq!(table_count, 17); + assert_eq!(table_count, 18); } #[test] @@ -574,19 +792,145 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + fn write_real_sqlite(db_path: &std::path::Path) { + let conn = Connection::open(db_path).unwrap(); + init(&conn).unwrap(); + drop(conn); + } + + fn seed_movie(conn: &Connection, title: &str, tmdb_id: i64) -> i64 { + conn.execute( + "INSERT INTO media_item (kind, title, year, tmdb_id, monitored, quality_profile_id, root_folder) + VALUES ('movie', ?1, NULL, ?2, 1, 2, '/tmp')", + rusqlite::params![title, tmdb_id], + ) + .unwrap(); + conn.last_insert_rowid() + } + + #[test] + fn init_creates_unique_and_lookup_indexes_on_a_clean_database() { + let conn = Connection::open_in_memory().unwrap(); + init(&conn).unwrap(); + + assert!(index_exists(&conn, "idx_media_item_tvdb").unwrap()); + assert!(index_exists(&conn, "idx_media_item_tmdb").unwrap()); + assert!(index_exists(&conn, "idx_episode_file_path").unwrap()); + assert!(index_exists(&conn, "idx_alias_item_text").unwrap()); + assert!(index_exists(&conn, "idx_release_status").unwrap()); + assert!(index_exists(&conn, "idx_release_torrent_hash").unwrap()); + assert!(index_exists(&conn, "idx_release_episode_id").unwrap()); + assert!(index_exists(&conn, "idx_episode_file_episode_id").unwrap()); + assert!(index_exists(&conn, "idx_review_queue_status").unwrap()); + assert!(index_exists(&conn, "idx_review_pending_title").unwrap()); + assert!(index_exists(&conn, "idx_episode_air_date").unwrap()); + assert!(!index_exists(&conn, "idx_media_item_tvdb_lookup").unwrap()); + assert!(!index_exists(&conn, "idx_media_item_tmdb_lookup").unwrap()); + } + + #[test] + fn init_dedupes_empty_duplicate_tmdb_rows_and_creates_unique_index() { + let conn = Connection::open_in_memory().unwrap(); + init(&conn).unwrap(); + conn.execute("DROP INDEX IF EXISTS idx_media_item_tmdb", []) + .unwrap(); + + let first = seed_movie(&conn, "A", 99); + seed_movie(&conn, "B", 99); + + init(&conn).unwrap(); + + let count: i64 = conn + .query_row( + "SELECT count(*) FROM media_item WHERE tmdb_id = 99", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(count, 1); + let kept: i64 = conn + .query_row("SELECT id FROM media_item WHERE tmdb_id = 99", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(kept, first); + assert!(index_exists(&conn, "idx_media_item_tmdb").unwrap()); + } + + #[test] + fn init_skips_tmdb_unique_index_when_duplicate_has_files() { + let conn = Connection::open_in_memory().unwrap(); + init(&conn).unwrap(); + conn.execute("DROP INDEX IF EXISTS idx_media_item_tmdb", []) + .unwrap(); + + seed_movie(&conn, "A", 99); + let extra = seed_movie(&conn, "B", 99); + conn.execute( + "INSERT INTO episode_file (media_item_id, path, size_bytes, subtitle_status) + VALUES (?1, '/tmp/b.mkv', 1, 'none')", + [extra], + ) + .unwrap(); + + init(&conn).unwrap(); + + let count: i64 = conn + .query_row( + "SELECT count(*) FROM media_item WHERE tmdb_id = 99", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(count, 2); + assert!(!index_exists(&conn, "idx_media_item_tmdb").unwrap()); + assert!(index_exists(&conn, "idx_media_item_tmdb_lookup").unwrap()); + } + + #[test] + fn init_skips_path_unique_index_when_duplicates_exist() { + let conn = Connection::open_in_memory().unwrap(); + init(&conn).unwrap(); + conn.execute("DROP INDEX IF EXISTS idx_episode_file_path", []) + .unwrap(); + + let id = seed_movie(&conn, "A", 1); + for _ in 0..2 { + conn.execute( + "INSERT INTO episode_file (media_item_id, path, size_bytes, subtitle_status) + VALUES (?1, '/tmp/x.mkv', 1, 'none')", + [id], + ) + .unwrap(); + } + + init(&conn).unwrap(); + assert!(!index_exists(&conn, "idx_episode_file_path").unwrap()); + } + #[test] fn backup_before_open_copies_an_existing_database() { let dir = std::env::temp_dir().join(format!("breadarr-backup-copy-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let db_path = dir.join("breadarr.db"); - std::fs::write(&db_path, b"fake sqlite data").unwrap(); + write_real_sqlite(&db_path); backup_before_open(&db_path).unwrap(); let backup_dir = dir.join("backups"); - let backups: Vec<_> = std::fs::read_dir(&backup_dir).unwrap().collect(); + let backups: Vec<_> = std::fs::read_dir(&backup_dir) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); assert_eq!(backups.len(), 1, "expected exactly one backup file"); + let verify = Connection::open(&backups[0]).unwrap(); + let n: i64 = verify + .query_row("SELECT count(*) FROM quality_profile", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 2); + std::fs::remove_dir_all(&dir).unwrap(); } @@ -594,9 +938,10 @@ mod tests { fn backup_before_open_prunes_beyond_max_backups() { let dir = std::env::temp_dir().join(format!("breadarr-backup-prune-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let db_path = dir.join("breadarr.db"); - std::fs::write(&db_path, b"fake sqlite data").unwrap(); + write_real_sqlite(&db_path); // One more than MAX_BACKUPS, sleeping a few ms between each so the // millisecond-resolution timestamp in the filename is guaranteed to diff --git a/breadarrd/src/importer/ffprobe.rs b/breadarrd/src/importer/ffprobe.rs index 5f576e5..835c6ad 100644 --- a/breadarrd/src/importer/ffprobe.rs +++ b/breadarrd/src/importer/ffprobe.rs @@ -14,6 +14,11 @@ pub struct AudioStream { #[derive(Debug, Clone, PartialEq)] pub struct SubtitleStream { pub language: Option, + /// mov_text (mp4's timed-text subtitle codec) isn't valid inside a + /// Matroska container — a transcode pipeline that always outputs `.mkv` + /// needs to know this per-stream to convert rather than blindly stream + /// copy. See `transcode::subtitle_codec_args`. + pub codec: Option, } #[derive(Debug, Clone, Default, PartialEq)] @@ -109,6 +114,16 @@ pub fn probe(path: &Path) -> Result { let json: serde_json::Value = serde_json::from_slice(&output.stdout).context("ffprobe output was not valid JSON")?; + Ok(build_media_probe(&json, raw_json)) +} + +/// The actual JSON-to-`MediaProbe` mapping, split out from `probe` so it can +/// be unit-tested directly against hand-built ffprobe-shaped JSON — real +/// muxers are inconsistent enough about stream ordering/disposition (see +/// `probe_finds_the_real_video_stream_even_when_attached_pic_comes_first`'s +/// doc comment) that constructing every case as an actual file via `ffmpeg` +/// isn't always practical. +fn build_media_probe(json: &serde_json::Value, raw_json: String) -> MediaProbe { let format = &json["format"]; let duration_secs = format["duration"] .as_str() @@ -134,11 +149,18 @@ pub fn probe(path: &Path) -> Result { .as_str() .or_else(|| stream["tags"]["LANGUAGE"].as_str()) .map(str::to_string); + let is_attached_pic = stream["disposition"]["attached_pic"].as_i64() == Some(1); match codec_type { - "video" if probe.video_codec.is_none() => { - // First video stream only — a second "video" stream in a - // real-world file is almost always an embedded cover-art - // thumbnail, not a second picture track. + // First non-attached-pic video stream — a second "video" stream + // in a real-world file is almost always an embedded cover-art + // thumbnail, not a second picture track, and ffmpeg does not + // guarantee it comes *after* the real content stream (some mp4 + // remuxes and mkvmerge outputs put it first). Explicitly + // checking `disposition.attached_pic` rather than relying on + // stream order means a cover-art-first file no longer has its + // actual video codec/height silently replaced by the + // thumbnail's. + "video" if probe.video_codec.is_none() && !is_attached_pic => { probe.video_codec = stream["codec_name"].as_str().map(str::to_string); probe.width = stream["width"].as_i64(); probe.height = stream["height"].as_i64(); @@ -164,13 +186,14 @@ pub fn probe(path: &Path) -> Result { }); } "subtitle" => { - probe.subtitles.push(SubtitleStream { language }); + let codec = stream["codec_name"].as_str().map(str::to_string); + probe.subtitles.push(SubtitleStream { language, codec }); } _ => {} } } - Ok(probe) + probe } /// ffprobe reports frame rate as a "num/den" fraction string (e.g. @@ -216,6 +239,59 @@ pub fn verify_decodable(path: &Path) -> Result { } } +/// Bounded, sampled variant of `verify_decodable` for callers where a full +/// decode's O(duration) cost is the actual bottleneck — the transcode +/// pipeline measured this in practice: two concurrent full-file decode +/// verifications pinned two CPU cores at 400%+ each for the whole +/// verification pass, dwarfing the GPU encode time itself for long files. +/// +/// Decodes only fixed-size windows (`sample_secs` each) near the start, +/// middle, and end of the file, rather than every frame — a deliberate +/// trade of "catches most real corruption cheaply" for "bounded cost +/// regardless of file length", not equivalent thoroughness to a full +/// decode. Truncation specifically doesn't need this: `encode_and_verify`'s +/// separate duration-match check against the original already catches that +/// regardless of what this function samples, since a truncated output's +/// container-reported duration comes up short either way. +/// +/// Falls back to a full `verify_decodable` when `duration_secs` is small +/// enough that sampling wouldn't save meaningful time anyway. +pub fn verify_decodable_sampled( + path: &Path, + duration_secs: f64, + sample_secs: f64, +) -> Result { + if duration_secs <= sample_secs * 3.0 { + return verify_decodable(path); + } + + let windows = [ + 0.0, + (duration_secs / 2.0 - sample_secs / 2.0).max(0.0), + (duration_secs - sample_secs).max(0.0), + ]; + + for start in windows { + let mut cmd = Command::new("ffmpeg"); + cmd.args(["-v", "error", "-xerror"]); + if start > 0.0 { + cmd.args(["-ss", &format!("{start:.2}")]); + } + cmd.arg("-i").arg(path); + cmd.args(["-t", &format!("{sample_secs:.2}"), "-f", "null", "-"]); + let output = cmd + .output() + .context("failed to run ffmpeg for sampled decode verification")?; + + if !(output.status.success() && output.stderr.is_empty()) { + return Ok(DecodeCheck::Corrupt( + String::from_utf8_lossy(&output.stderr).into_owned(), + )); + } + } + Ok(DecodeCheck::Ok) +} + #[cfg(test)] mod tests { use super::*; @@ -297,6 +373,78 @@ mod tests { assert!(result.is_err()); } + fn generate_clip(dir: &Path, name: &str, duration_secs: u32) -> std::path::PathBuf { + let path = dir.join(name); + let status = Command::new("ffmpeg") + .args([ + "-y", + "-f", + "lavfi", + "-i", + &format!("testsrc=size=320x240:duration={duration_secs}:rate=5"), + ]) + .args(["-c:v", "libx264", "-preset", "ultrafast"]) + .arg(&path) + .output() + .expect("failed to run ffmpeg to generate a test clip"); + assert!( + status.status.success(), + "ffmpeg failed to generate a test clip: {}", + String::from_utf8_lossy(&status.stderr) + ); + path + } + + #[test] + fn verify_decodable_sampled_passes_a_genuinely_intact_short_file() { + // Short enough to hit the "falls back to a full check" path. + let dir = std::env::temp_dir().join(format!( + "breadarr-ffprobe-sampled-short-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_clip(&dir, "short.mkv", 2); + + let result = verify_decodable_sampled(&clip, 2.0, 20.0).unwrap(); + assert!(matches!(result, DecodeCheck::Ok)); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn verify_decodable_sampled_passes_a_genuinely_intact_long_file() { + // Long enough (duration > sample_secs * 3) to actually exercise the + // windowed start/middle/end sampling path, not the short-file + // fallback. + let dir = std::env::temp_dir().join(format!( + "breadarr-ffprobe-sampled-long-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_clip(&dir, "long.mkv", 10); + + let result = verify_decodable_sampled(&clip, 10.0, 2.0).unwrap(); + assert!(matches!(result, DecodeCheck::Ok)); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn verify_decodable_sampled_flags_a_file_that_does_not_decode_at_all() { + let dir = std::env::temp_dir().join(format!( + "breadarr-ffprobe-sampled-corrupt-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("corrupt.mkv"); + std::fs::write(&path, b"this is not a real video file").unwrap(); + + let result = verify_decodable_sampled(&path, 10.0, 2.0).unwrap(); + assert!(matches!(result, DecodeCheck::Corrupt(_))); + + std::fs::remove_dir_all(&dir).unwrap(); + } + /// Generates a tiny real video via ffmpeg's `lavfi` synthetic source — /// validates the actual JSON field extraction (width/height/codec/ /// duration/audio language+default) against genuine ffprobe output, @@ -376,4 +524,82 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + + // Regression test for a real gap found in review: `probe`'s "first video + // stream wins" selection used to have no idea about + // `disposition.attached_pic` and just trusted stream order — a + // convention real muxers don't reliably follow (ffmpeg's own mp4 muxer + // was observed reordering an attached-pic stream to the *end* + // regardless of requested `-map` order, which makes constructing a + // genuine cover-art-*first* file via `ffmpeg` impractical — so this + // exercises `build_media_probe` directly against hand-built, + // real-shaped ffprobe JSON instead of a generated file, deterministically + // covering the ordering `ffmpeg`'s own tooling won't produce). Unlike + // `generate_clip_with_attached_pic` in `transcode::mod::tests` (cover + // art second, the already-handled case), this puts it *first* to prove + // the fix is order-independent, not just "skip the second video + // stream". + #[test] + fn probe_finds_the_real_video_stream_even_when_attached_pic_comes_first() { + let json = serde_json::json!({ + "format": { "duration": "10.0", "bit_rate": "5000000", "format_long_name": "Matroska / WebM" }, + "streams": [ + { + "index": 0, + "codec_type": "video", + "codec_name": "png", + "width": 64, + "height": 64, + "disposition": { "attached_pic": 1 } + }, + { + "index": 1, + "codec_type": "video", + "codec_name": "h264", + "width": 640, + "height": 360, + "disposition": { "attached_pic": 0 } + } + ] + }); + + let probe = build_media_probe(&json, "{}".to_string()); + assert_eq!(probe.width, Some(640), "must pick the real content stream's width, not the 64x64 cover art's"); + assert_eq!(probe.height, Some(360), "must pick the real content stream's height, not the 64x64 cover art's"); + assert_eq!(probe.video_codec.as_deref(), Some("h264"), "must pick the real content stream's codec, not the cover art's png"); + } + + // Companion case: attached-pic *second* (the ordering the code + // previously assumed was the only one) must still work exactly as + // before — this fix is additive, not a behavior change for the + // already-handled ordering. + #[test] + fn probe_finds_the_real_video_stream_when_attached_pic_comes_second() { + let json = serde_json::json!({ + "format": { "duration": "10.0", "bit_rate": "5000000", "format_long_name": "Matroska / WebM" }, + "streams": [ + { + "index": 0, + "codec_type": "video", + "codec_name": "h264", + "width": 640, + "height": 360, + "disposition": { "attached_pic": 0 } + }, + { + "index": 1, + "codec_type": "video", + "codec_name": "png", + "width": 64, + "height": 64, + "disposition": { "attached_pic": 1 } + } + ] + }); + + let probe = build_media_probe(&json, "{}".to_string()); + assert_eq!(probe.width, Some(640)); + assert_eq!(probe.height, Some(360)); + assert_eq!(probe.video_codec.as_deref(), Some("h264")); + } } diff --git a/breadarrd/src/importer/mod.rs b/breadarrd/src/importer/mod.rs index 779c3ba..1bce250 100644 --- a/breadarrd/src/importer/mod.rs +++ b/breadarrd/src/importer/mod.rs @@ -99,14 +99,6 @@ impl PendingGrab { PendingGrab::Movie { .. } | PendingGrab::SeasonPack { .. } => None, } } - - fn root_folder(&self) -> &str { - match self { - PendingGrab::Episode { root_folder, .. } - | PendingGrab::Movie { root_folder, .. } - | PendingGrab::SeasonPack { root_folder, .. } => root_folder, - } - } } /// Three separate queries (rather than one `LEFT JOIN episode`) because a @@ -209,9 +201,17 @@ pub(crate) fn walk_files(dir: &Path) -> Result> { let mut out = Vec::new(); for entry in std::fs::read_dir(dir)? { let path = entry?.path(); - if path.is_dir() { + // `path.is_dir()` follows symlinks, which would let a planted + // directory link walk the importer (and library-scan) out of the + // download/library root. Skip every symlink — file links to videos + // included — and only recurse into real directories. + let ft = std::fs::symlink_metadata(&path)?.file_type(); + if ft.is_symlink() { + continue; + } + if ft.is_dir() { out.extend(walk_files(&path)?); - } else { + } else if ft.is_file() { out.push(path); } } @@ -227,6 +227,24 @@ pub(crate) fn season_dir(root_folder: &str, season_number: u32) -> PathBuf { Path::new(root_folder).join(format!("Season {season_number:02}")) } +/// TVDB stores some shows' episode titles only in their original-airing +/// language (verified live: entire seasons of otherwise-English-titled +/// shows came back with Japanese-only episode titles, no English fallback +/// available from the API at all) — using that title verbatim in a +/// generated filename buries a CJK title inside an otherwise-Latin-script +/// library, which nothing downstream (search, external tools, a user +/// scanning a directory listing) can actually read. Better to just omit the +/// episode title than emit a filename with random plaintext. +fn has_cjk(s: &str) -> bool { + s.chars().any(|c| { + matches!(c, + '\u{3040}'..='\u{30FF}' // hiragana + katakana + | '\u{4E00}'..='\u{9FFF}' // CJK unified ideographs + | '\u{FF00}'..='\u{FFEF}' // fullwidth forms + ) + }) +} + pub(crate) fn deterministic_filename( series_title: &str, season: u32, @@ -235,7 +253,7 @@ pub(crate) fn deterministic_filename( ext: &str, ) -> String { let series = sanitize(series_title); - match episode_title.filter(|t| !t.is_empty()) { + match episode_title.filter(|t| !t.is_empty() && !has_cjk(t)) { Some(t) => format!( "{series} - S{season:02}E{episode:02} - {}.{ext}", sanitize(t) @@ -253,9 +271,25 @@ pub(crate) fn deterministic_movie_filename(title: &str, year: Option, ext: } pub(crate) fn sanitize(s: &str) -> String { - s.chars() - .map(|c| if "/\\:*?\"<>|".contains(c) { '_' } else { c }) - .collect() + let cleaned: String = s + .chars() + .map(|c| { + if c.is_ascii_control() || "/\\:*?\"<>|".contains(c) { + '_' + } else { + c + } + }) + .collect(); + // After the replacements above, a title of `.` or `..` is still a + // hostile path component (`root_folder/../file`). Slash-containing + // titles become `foo_.._bar`, which is fine — only a lone `.`/`..` + // can climb. + if cleaned == "." || cleaned == ".." { + "_".to_string() + } else { + cleaned + } } /// True when there isn't enough free space at `dir` to hold `needed_bytes`. @@ -271,31 +305,199 @@ fn insufficient_space(dir: &Path, needed_bytes: u64) -> Result { Ok(available < needed_bytes) } -/// Hardlinks into the destination (same filesystem, effectively free) and -/// falls back to a plain copy cross-filesystem — deliberately leaves `src` -/// untouched either way. The previous `rename`-then-delete behavior yanked -/// the file out of qBittorrent's payload directory on every import, leaving -/// qBittorrent holding a torrent whose data had vanished (an errored -/// "missing files" state, seeding stopped instantly). A hardlink costs -/// nothing extra on disk and lets qBittorrent keep seeding after import; -/// even the copy fallback preserves seeding, just at the cost of double -/// disk usage for that one file. +/// `true` if `a` and `b` live on the same filesystem (compares `st_dev`), +/// `false` if they don't *or* either can't be stat'd. Used to skip the +/// free-space check ahead of `move_or_copy_file`: that function tries +/// `rename` first, which needs essentially zero additional space, so +/// checking `dest`'s free space against the *full* source file size is a +/// false positive whenever downloads and the library share a volume (a +/// common setup) — a real production shape found in review: a season-pack +/// file that would `rename` instantly got rejected as "not enough free +/// space", retried and failed identically every cycle +/// (`MAX_IMPORT_ERRORS`), then got released back to the search pool where +/// it was re-grabbed and failed the same way again, forever. Erring toward +/// `false` (i.e. still running the space check) on a stat failure is the +/// safe direction — it only means checking a space guarantee that wasn't +/// strictly needed, never skipping one that was. +fn same_filesystem(a: &Path, b: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + match (std::fs::metadata(a), std::fs::metadata(b)) { + (Ok(a), Ok(b)) => a.dev() == b.dev(), + _ => false, + } +} + +/// Moves `src` into `dest`: a same-filesystem `rename` where possible +/// (instant, atomic, no extra disk usage), falling back to copy-then-delete +/// across a filesystem boundary. Nothing is left behind at `src` either way +/// — there used to be a deliberate hardlink-and-leave-`src`-alone scheme +/// here so qBittorrent could keep seeding the original download after +/// import, but with nothing seeding after download completes anymore, that +/// complexity (a same-filesystem staging relocate before every import, so +/// the hardlink was guaranteed rather than falling back to a permanent +/// second copy) bought nothing but risk. The file just lands directly at +/// its final destination. /// -/// The copy path writes to a `.part` sibling of `dest` and only renames it -/// into place once the copy is complete (a same-filesystem rename, so +/// The copy fallback writes to a `.part` sibling of `dest` and only renames +/// it into place once the copy is complete (a same-filesystem rename, so /// atomic) — a crash mid-copy leaves an orphaned `.part` file rather than a /// truncated file at `dest`, so a retried import can't ever double-count a -/// half-written file as already present. -fn link_or_copy_file(src: &Path, dest: &Path) -> Result<()> { - if std::fs::hard_link(src, dest).is_ok() { +/// half-written file as already present. `src` is only removed once that +/// copy is confirmed in place, so a crash between the copy and the removal +/// leaves both copies on disk (recoverable) rather than neither. +fn move_or_copy_file(src: &Path, dest: &Path) -> Result<()> { + if std::fs::rename(src, dest).is_ok() { return Ok(()); } - copy_via_temp_file(src, dest) + copy_via_temp_file(src, dest)?; + std::fs::remove_file(src).with_context(|| { + format!( + "copied {} to {} but failed to remove the original", + src.display(), + dest.display() + ) + }) +} + +fn is_video_path(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .map(|e| VIDEO_EXTS.contains(&e.to_lowercase().as_str())) + .unwrap_or(false) +} + +/// Lexical `..`/`.` collapse without touching the filesystem — used when +/// `canonicalize` can't (path missing) and by `remap_path` before the +/// joined result is allowed to escape `host_prefix`. +fn normalize_lexically(path: &Path) -> PathBuf { + use std::path::Component; + let mut out = PathBuf::new(); + for comp in path.components() { + match comp { + Component::Prefix(p) => out.push(p.as_os_str()), + Component::RootDir => out.push(Component::RootDir.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + Component::Normal(c) => out.push(c), + } + } + out +} + +fn canonicalize_or_normalize(path: &Path) -> PathBuf { + if let Ok(canon) = std::fs::canonicalize(path) { + return canon; + } + if let Some(parent) = path.parent() { + if let (Ok(canon_parent), Some(name)) = (std::fs::canonicalize(parent), path.file_name()) { + return canon_parent.join(name); + } + } + if path.is_absolute() { + return normalize_lexically(path); + } + std::env::current_dir() + .map(|cwd| normalize_lexically(&cwd.join(path))) + .unwrap_or_else(|_| normalize_lexically(path)) +} + +fn is_strictly_inside(path: &Path, root: &Path) -> bool { + let path = canonicalize_or_normalize(path); + let root = canonicalize_or_normalize(root); + path.starts_with(&root) && path != root +} + +fn is_common_filesystem_root(path: &Path) -> bool { + let normalized = canonicalize_or_normalize(path); + if normalized == Path::new("/") + || normalized == Path::new("/mnt") + || normalized == Path::new("/media") + { + return true; + } + if let Some(home) = std::env::var_os("HOME") { + if normalized == Path::new(&home) { + return true; + } + } + false +} + +/// `true` only for a per-torrent folder we are willing to `remove_dir_all`. +/// A configured `host_downloads_path` must strictly contain `content_path`; +/// an empty host falls back to "has a real parent that isn't `/` and isn't +/// a well-known root" so a default-empty config cannot wipe `/` or `$HOME`. +fn is_safe_cleanup_target(content_path: &Path, host_downloads_path: &str) -> bool { + let meta = match std::fs::symlink_metadata(content_path) { + Ok(m) => m, + Err(_) => return false, + }; + if !meta.is_dir() || meta.file_type().is_symlink() { + return false; + } + if !host_downloads_path.is_empty() { + return is_strictly_inside(content_path, Path::new(host_downloads_path)); + } + if is_common_filesystem_root(content_path) { + return false; + } + let Some(parent) = content_path.parent() else { + return false; + }; + if parent == Path::new("/") || parent.as_os_str().is_empty() || !parent.exists() { + return false; + } + content_path != parent +} + +fn dir_contains_video_files(dir: &Path) -> bool { + match walk_files(dir) { + Ok(files) => files.iter().any(|p| is_video_path(p)), + // A walk failure must not authorize a wipe. + Err(_) => true, + } +} + +fn remove_non_video_files(dir: &Path) -> Result<()> { + for path in walk_files(dir)? { + if !is_video_path(&path) { + let _ = std::fs::remove_file(&path); + } + } + Ok(()) +} + +/// Removes whatever's left of a torrent's download folder once every file +/// breadarr cares about has already been moved out of it — split out from +/// `run_import_cycle` so it's directly testable without a real qBittorrent +/// server. `move_or_copy_file` only ever relocates the one video file it +/// located; a multi-file release's own folder (sample clips, `.nfo`/`.srt`/ +/// `.jpg` sidecars) is otherwise left behind forever, since nothing else +/// ever points at it once the torrent itself is gone from qBittorrent. A +/// bare file (a release with no wrapping folder) needs no cleanup here — +/// `move_or_copy_file` already consumed it. +/// +/// Never wipes a directory that still contains a video we didn't import +/// (unmatched pack files), and never `remove_dir_all`s the downloads root +/// itself — `host_downloads_path` defaults to `""`, so a raw `!= host` +/// comparison is not enough. +fn cleanup_leftover_download_dir(content_path: &Path, host_downloads_path: &str) -> Result<()> { + if !is_safe_cleanup_target(content_path, host_downloads_path) { + return Ok(()); + } + if dir_contains_video_files(content_path) { + remove_non_video_files(content_path)?; + return Ok(()); + } + std::fs::remove_dir_all(content_path)?; + Ok(()) } /// The copy fallback's actual mechanics, split out so it's directly /// testable without needing a real cross-filesystem boundary to force -/// `hard_link` to fail. +/// `rename` to fail. fn copy_via_temp_file(src: &Path, dest: &Path) -> Result<()> { let tmp = PathBuf::from(format!("{}.part", dest.display())); std::fs::copy(src, &tmp) @@ -660,6 +862,156 @@ fn find_by_stem(root: &Path, stem: &std::ffi::OsStr) -> Option { None } +/// One TV episode whose file already sits on disk — matching breadarr's own +/// `"S{season:02}E{episode:02}"` naming marker, in exactly the season +/// directory this episode's own metadata says it should be in — despite +/// having no `episode_file` row at all. `has_file = 0` lies about it, so +/// the missing-content search loop treats it as genuinely absent and keeps +/// trying to (re)download something already there. +#[derive(Debug, PartialEq)] +pub struct RelinkCandidate { + pub episode_id: i64, + pub media_item_id: i64, + pub series_title: String, + pub season_number: i64, + pub episode_number: i64, + pub path: PathBuf, +} + +/// Every video file found at any depth under `root` — deliberately +/// unbounded by a fixed season-folder shape, since a real production audit +/// found layouts varying wildly per show: a plain `Season N`/`Season 0N` +/// directly under the show root in most cases, but at least one show (a +/// BluRay box-set release) nests an extra layer — the whole release's own +/// folder name — between the show root and its `Season N` directories. +/// Bounded to one show's own folder (a handful of seasons deep at most), +/// same scope as `find_by_basename`/`find_by_stem`. +fn collect_video_files(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { + return Vec::new(); + }; + let mut found = Vec::new(); + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + found.extend(collect_video_files(&path)); + } else if path + .extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| VIDEO_EXTS.contains(&e.to_lowercase().as_str())) + { + found.push(path); + } + } + found +} + +/// Finds every `has_file = 0` TV episode whose show folder contains exactly +/// one video file matching its own `"S{season:02}E{episode:02}"` naming +/// marker (breadarr's own convention, which the vast majority of +/// scene/fansub releases also happen to embed verbatim even under otherwise +/// raw filenames) — found via a real production audit: several shows had +/// `episode`/`media_item` rows (almost certainly rebuilt by an earlier +/// DB-recovery incident) with no matching `episode_file` row, even though +/// the actual video files were never touched and still sat right where they +/// always had, often under a pre-breadarr folder layout (unpadded `Season +/// N`, or an extra nested release-folder level) that a check scoped only to +/// `season_dir`'s exact zero-padded shape would never find. Walks the whole +/// show folder once and reuses that listing for every missing episode in +/// it, rather than re-walking per episode. Purely a finder — see +/// `relink_episode_files` for the (additive-only) part that actually links +/// anything in. +/// +/// Deliberately conservative in both directions: a show folder with *zero* +/// marker matches for an episode is left alone (genuinely missing, nothing +/// to relink here — `reconcile_missing_files`/the normal search loop are the +/// right tools for an actually-absent file), and *more than one* match is +/// left alone too rather than guessed at, returned separately so a human +/// can look instead of silently picking one. A show using pure absolute +/// numbering with no season/episode marker at all in its filenames (a real +/// pattern found on some anime releases) simply never matches either way — +/// the safe failure mode, not a wrong guess. +pub fn find_relinkable_episode_files( + conn: &Connection, +) -> Result<(Vec, Vec)> { + let mut show_stmt = conn.prepare( + "SELECT DISTINCT m.id, m.title, m.root_folder + FROM episode e + JOIN media_item m ON m.id = e.media_item_id + WHERE e.has_file = 0 AND m.kind = 'series'", + )?; + let shows: Vec<(i64, String, String)> = show_stmt + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))? + .collect::>()?; + + let mut candidates = Vec::new(); + let mut ambiguous = Vec::new(); + + let mut ep_stmt = conn.prepare( + "SELECT id, season_number, episode_number FROM episode + WHERE media_item_id = ?1 AND has_file = 0", + )?; + for (media_item_id, series_title, root_folder) in shows { + let all_video_files = collect_video_files(Path::new(&root_folder)); + let episodes: Vec<(i64, i64, i64)> = ep_stmt + .query_map(params![media_item_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .collect::>()?; + + for (episode_id, season_number, episode_number) in episodes { + let marker = format!("S{season_number:02}E{episode_number:02}"); + let matches: Vec<&PathBuf> = all_video_files + .iter() + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.contains(&marker)) + }) + .collect(); + match matches.len() { + 0 => {} + 1 => candidates.push(RelinkCandidate { + episode_id, + media_item_id, + series_title: series_title.clone(), + season_number, + episode_number, + path: matches[0].clone(), + }), + n => ambiguous.push(format!( + "{series_title} S{season_number:02}E{episode_number:02}: {n} candidate files under {root_folder}" + )), + } + } + } + Ok((candidates, ambiguous)) +} + +/// Links each `RelinkCandidate` into `episode_file` (probing it first, same +/// as a real import) and marks its episode `has_file = 1`. Purely additive +/// — never moves, renames, or deletes anything on disk, since the file was +/// already exactly where a real import would have put it; this only makes +/// breadarr's own bookkeeping admit what's already true. +pub fn relink_episode_files(conn: &Connection, candidates: &[RelinkCandidate]) -> Result { + let mut linked = 0; + for c in candidates { + let size_bytes = std::fs::metadata(&c.path)?.len() as i64; + conn.execute( + "INSERT INTO episode_file (episode_id, media_item_id, path, size_bytes, subtitle_status) VALUES (?1, ?2, ?3, ?4, 'none')", + params![c.episode_id, c.media_item_id, c.path.to_string_lossy(), size_bytes], + )?; + let episode_file_id = conn.last_insert_rowid(); + conn.execute( + "UPDATE episode SET has_file = 1 WHERE id = ?1", + params![c.episode_id], + )?; + ensure_probed(conn, episode_file_id, &c.path)?; + linked += 1; + } + Ok(linked) +} + /// Runs ffprobe on `path` and upserts the result into `media_file_probe` /// against `episode_file_id` — but only if the file's size or mtime has /// actually changed since the last probe, so a routine sweep over a large, @@ -1080,12 +1432,22 @@ fn remux_one_backlog_file(conn: &Connection, episode_file_id: i64, path: &Path) } let tmp = path.with_extension("fixed.mkv"); - mkv::remux_english_default(path, &tmp, &tracks)?; + if let Err(e) = mkv::remux_english_default(path, &tmp, &tracks) { + // Leaked otherwise: a stray `.fixed.mkv` sitting in the library + // directory forever, matched by `find_relinkable_episode_files` on + // the same `SxxExx` substring as the real file and reported as an + // ambiguous, unrelinkable episode. + std::fs::remove_file(&tmp).ok(); + return Err(e); + } // Same atomic-swap shape as `copy_via_temp_file`: rename the freshly // remuxed output over the original on the same filesystem, so a crash // mid-swap can never leave a half-written file at the real path. - std::fs::rename(&tmp, path)?; + if let Err(e) = std::fs::rename(&tmp, path) { + std::fs::remove_file(&tmp).ok(); + return Err(e.into()); + } let size_bytes = std::fs::metadata(path)?.len(); conn.execute( @@ -1115,198 +1477,27 @@ fn remap_path(reported: &str, container_prefix: &str, host_prefix: &str) -> Path return PathBuf::from(reported); } match reported.strip_prefix(container_prefix) { - Some(rest) => PathBuf::from(format!("{host_prefix}{rest}")), + Some(rest) => { + let rest_path = Path::new(rest); + if rest_path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return PathBuf::from(reported); + } + let joined = PathBuf::from(format!("{host_prefix}{rest}")); + let normalized = normalize_lexically(&joined); + let host = normalize_lexically(Path::new(host_prefix)); + if !normalized.starts_with(&host) { + return PathBuf::from(reported); + } + normalized + } None => PathBuf::from(reported), } } -/// Marker directory name for the seeding-preserving staging area — checked -/// as a plain substring of a torrent's reported `content_path` to tell -/// whether it's already been relocated there in a previous cycle. -const STAGING_DIR_NAME: &str = ".breadarr-staging"; - -/// How many times to poll qBittorrent for a `setLocation` move to finish -/// before giving up for this cycle (it'll simply be retried next cycle — -/// see `relocate_completed_to_staging`). -const RELOCATE_POLL_ATTEMPTS: u32 = 5; -const RELOCATE_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); - -/// Finds the nearest ancestor of `path` that actually exists — `path` -/// itself (e.g. a show's season folder) may not have been created yet. -fn nearest_existing_ancestor(path: &Path) -> Result { - let mut current = path; - loop { - if current.exists() { - return Ok(current.to_path_buf()); - } - current = current - .parent() - .with_context(|| format!("no existing ancestor found for {}", path.display()))?; - } -} - -/// Finds the filesystem mount-point directory containing `path`, by -/// walking up parents until the device id changes. Used to place the -/// seeding-staging directory on the same physical filesystem as the final -/// destination — the whole point of staging is that the later hardlink-out -/// in `import_one` is guaranteed to succeed as a true hardlink rather than -/// silently falling back to a full copy, and a hardlink can never cross a -/// filesystem boundary. -fn find_mount_root(path: &Path) -> Result { - use std::os::unix::fs::MetadataExt; - let start = nearest_existing_ancestor(path)?; - let dev = std::fs::metadata(&start)?.dev(); - let mut current = start; - loop { - let Some(parent) = current.parent() else { - return Ok(current); - }; - let Ok(parent_meta) = std::fs::metadata(parent) else { - return Ok(current); - }; - if parent_meta.dev() != dev { - return Ok(current); - } - current = parent.to_path_buf(); - } -} - -/// The staging directory a given torrent's data should be relocated to — -/// on the same filesystem as `dest_dir`, named by torrent hash so multiple -/// torrents' leftover extras (samples/nfo/screenshots) never collide. -fn staging_dir_for(dest_dir: &Path, torrent_hash: &str) -> Result { - let mount_root = find_mount_root(dest_dir)?; - Ok(mount_root.join(STAGING_DIR_NAME).join(torrent_hash)) -} - -/// For every pending grab whose torrent has finished downloading but whose -/// data isn't already staged, relocates it via qBittorrent's own -/// `setLocation` to a directory on the same filesystem as its eventual -/// destination, then waits (briefly, bounded) for the move to actually -/// finish — `setLocation` returns before the physical move completes. -/// -/// Best-effort and self-healing by design: a grab that isn't staged yet -/// this cycle is simply retried on the next one (nothing here ever fails -/// the whole import cycle), so a slow move or a transient qBit API error -/// never blocks or loses anything — it just costs one extra cycle. -/// -/// Returns whether anything was actually relocated, so the caller knows -/// whether it's worth re-fetching the torrent list before importing (a -/// relocated torrent's `content_path` only reflects its new home after a -/// fresh `list_torrents` call). -async fn relocate_completed_to_staging( - qbit: &QbitClient, - pending: &[PendingGrab], - torrents: &[crate::qbit::TorrentInfo], - category: &str, -) -> bool { - let mut relocated_any = false; - for grab in pending { - let Some(torrent) = torrents.iter().find(|t| t.hash == grab.torrent_hash()) else { - continue; - }; - if torrent.progress < 1.0 { - continue; - } - if torrent.content_path.contains(STAGING_DIR_NAME) { - continue; // already staged in a previous cycle - } - - let staging = match staging_dir_for(Path::new(grab.root_folder()), grab.torrent_hash()) { - Ok(s) => s, - Err(e) => { - tracing::warn!( - error = %e, - hash = grab.torrent_hash(), - "could not determine a staging directory this cycle" - ); - continue; - } - }; - - if let Err(e) = qbit - .set_location(grab.torrent_hash(), &staging.to_string_lossy()) - .await - { - tracing::warn!( - error = %e, - hash = grab.torrent_hash(), - "qbit relocate-to-staging failed, will retry next cycle" - ); - continue; - } - - if wait_for_relocation(qbit, grab.torrent_hash(), category, &staging).await { - relocated_any = true; - } else { - tracing::warn!( - hash = grab.torrent_hash(), - "qbit relocate-to-staging didn't finish in time, will retry next cycle" - ); - } - } - relocated_any -} - -/// Waits for qBittorrent to report the torrent's `content_path` as staged, -/// then verifies the reported path actually landed where expected — -/// qBittorrent (when it's running in its own Docker container) reports *its -/// own* filesystem view, and `staging_dir_for`/`set_location` currently -/// rely on every library mount being set up as an identity mount (container -/// path == host path) for that reported path to be directly meaningful -/// from the host's side. That's a deployment convention, not something the -/// code enforces, so it can be wrong for a given install's mount layout. If -/// it is, treating the reported path as trustworthy without checking could -/// let `import_one`'s later hardlink-out silently fall back to a full -/// cross-device copy (or fail outright) instead of the guaranteed-cheap -/// hardlink staging exists to provide — so this confirms the reported path -/// resolves, from the host, to the *same physical filesystem* as -/// `expected_staging` before trusting it. -async fn wait_for_relocation( - qbit: &QbitClient, - hash: &str, - category: &str, - expected_staging: &Path, -) -> bool { - use std::os::unix::fs::MetadataExt; - - for _ in 0..RELOCATE_POLL_ATTEMPTS { - tokio::time::sleep(RELOCATE_POLL_INTERVAL).await; - let Ok(torrents) = qbit.list_torrents(Some(category)).await else { - continue; - }; - let Some(t) = torrents.iter().find(|t| t.hash == hash) else { - continue; - }; - if !t.content_path.contains(STAGING_DIR_NAME) { - continue; - } - let reported = Path::new(&t.content_path); - match ( - std::fs::metadata(reported), - std::fs::metadata(expected_staging), - ) { - (Ok(reported_meta), Ok(expected_meta)) - if reported_meta.dev() == expected_meta.dev() => - { - return true; - } - _ => { - tracing::error!( - hash, - reported = %t.content_path, - expected = %expected_staging.display(), - "qbit reports the torrent as staged, but its content_path isn't visible \ - on the same host filesystem as expected — the container's mount for this \ - library path may not be an identity mount; refusing to trust this relocation" - ); - return false; - } - } - } - false -} - +#[allow(clippy::too_many_arguments)] pub async fn run_import_cycle( conn: &Connection, qbit: &QbitClient, @@ -1314,6 +1505,7 @@ pub async fn run_import_cycle( category: &str, container_downloads_path: &str, host_downloads_path: &str, + transcode_cfg: Option<&breadarr_shared::config::TranscodeConfig>, ) -> Result { let pending = fetch_pending_grabs(conn)?; if pending.is_empty() { @@ -1322,30 +1514,48 @@ pub async fn run_import_cycle( let torrents = qbit.list_torrents(Some(category)).await?; - // Relocate completed-but-not-yet-staged torrents onto the same - // filesystem as their destination *before* importing, so the - // hardlink-out below (`link_or_copy_file`, unchanged) is a true - // hardlink instead of a full cross-drive copy — qBittorrent keeps - // seeding indefinitely from the staged location afterward, with no - // permanent second copy of the data anywhere. - let relocated_any = relocate_completed_to_staging(qbit, &pending, &torrents, category).await; - let torrents = if relocated_any { - qbit.list_torrents(Some(category)).await? - } else { - torrents - }; - - let stats = process_pending_grabs( + let (stats, imported_hashes) = process_pending_grabs( conn, &pending, &torrents, container_downloads_path, host_downloads_path, + transcode_cfg, )?; + // The file(s) are already gone from qBittorrent's download directory by + // this point — moved into the library, or deleted outright because a + // better file already existed — no seeding to preserve either way, so + // the torrent itself is just forgotten rather than left sitting around + // in a "files missing" error state. Best-effort: a failed delete here + // never undoes or blocks the import that already succeeded. + for (hash, content_path) in &imported_hashes { + if let Err(e) = qbit.delete_torrent(hash).await { + tracing::warn!(hash, error = %e, "failed to remove completed torrent from qbittorrent"); + } + if let Err(e) = cleanup_leftover_download_dir(content_path, host_downloads_path) { + tracing::warn!( + hash, + path = %content_path.display(), + error = %e, + "failed to remove leftover download directory after import" + ); + } + } + + // Best-effort, same reasoning as the `delete_torrent` cleanup just + // above: by this point files have already been moved, DB rows written, + // and torrents removed from qBittorrent — a real import that already + // succeeded. Propagating a Jellyfin 500/timeout here used to discard + // `stats` entirely and report the whole cycle as failed, which also + // skipped `stats.failed`/`stats.quality_flagged` notifications for + // imports that had nothing to do with Jellyfin. The library picks up + // the new files on its own next scheduled scan either way. if stats.imported > 0 { if let Some(jellyfin) = jellyfin { - jellyfin.refresh_library().await?; + if let Err(e) = jellyfin.refresh_library().await { + tracing::warn!(error = %e, "failed to trigger jellyfin library refresh"); + } } } @@ -1366,14 +1576,27 @@ fn process_pending_grabs( torrents: &[crate::qbit::TorrentInfo], container_downloads_path: &str, host_downloads_path: &str, -) -> Result { + transcode_cfg: Option<&breadarr_shared::config::TranscodeConfig>, +) -> Result<(ImportStats, Vec<(String, PathBuf)>)> { let mut stats = ImportStats::default(); + // Torrents whose data has been fully dealt with this cycle — imported, + // skipped as already-better, or given up on (stall / missing past grace + // / MAX_IMPORT_ERRORS). The caller (`run_import_cycle`) removes each + // from qBittorrent afterward and runs leftover-dir cleanup, which now + // refuses to wipe leftover videos or the downloads root. `fail_grab` + // itself only writes SQLite; hashes collected here are how the torrent + // actually leaves qBit. + let mut imported_hashes: Vec<(String, PathBuf)> = Vec::new(); for grab in pending { let Some(torrent) = torrents.iter().find(|t| t.hash == grab.torrent_hash()) else { if grab_missing_past_grace(conn, grab.release_id())? { fail_grab(conn, grab, "torrent hash absent from qBittorrent")?; stats.failed += 1; + // Torrent is already gone from qBit; still collect the hash + // so delete is attempted (best-effort) and any known path + // would be cleaned. No `content_path` is available here. + imported_hashes.push((grab.torrent_hash().to_string(), PathBuf::new())); } continue; }; @@ -1387,15 +1610,21 @@ fn process_pending_grabs( "no progress for longer than the stall threshold", )?; stats.failed += 1; + let content_path = remap_path( + &torrent.content_path, + container_downloads_path, + host_downloads_path, + ); + imported_hashes.push((grab.torrent_hash().to_string(), content_path)); } continue; } if torrent.state == "moving" { - // qBittorrent's own `setLocation` relocation (or a manual move) - // is still physically in flight — `content_path` may already - // point at the new location while the actual bytes are still - // being copied there. Importing now risks hardlinking/copying - // a partially-moved (truncated) file into the library and + // A category/save-path change (e.g. a manual move in the + // qBittorrent UI) is still physically in flight — `content_path` + // may already point at the new location while the actual bytes + // are still being copied there. Importing now risks moving a + // partially-relocated (truncated) file into the library and // marking it complete. Simply wait; this is checked every // cycle, so it proceeds as soon as the move finishes. stats.skipped_incomplete += 1; @@ -1425,10 +1654,18 @@ fn process_pending_grabs( *season_number, root_folder, &content_path, + transcode_cfg, ) { Ok(outcome) => { stats.imported += outcome.episodes_imported; stats.quality_flagged += outcome.quality_flagged; + // Imported files *or* a no-op upgrade (`upgraded` with + // 0 imported): nothing breadarr still needs lives in + // qBit. Unmatched leftover videos are left on disk by + // `cleanup_leftover_download_dir`. + if outcome.episodes_imported > 0 || outcome.episodes_already_had_better > 0 { + imported_hashes.push((grab.torrent_hash().to_string(), content_path.clone())); + } } Err(e) => { let error_count = record_import_error(conn, *release_id)?; @@ -1439,6 +1676,7 @@ fn process_pending_grabs( &format!("season pack import failed {error_count} times in a row: {e}"), )?; stats.failed += 1; + imported_hashes.push((grab.torrent_hash().to_string(), content_path.clone())); } else { tracing::warn!( error = %e, @@ -1453,7 +1691,7 @@ fn process_pending_grabs( continue; } - match import_one(conn, grab, &content_path) { + match import_one(conn, grab, &content_path, transcode_cfg) { Ok(ImportOutcome::Imported { remuxed, quality_flagged, @@ -1465,8 +1703,14 @@ fn process_pending_grabs( if quality_flagged { stats.quality_flagged += 1; } + imported_hashes.push((grab.torrent_hash().to_string(), content_path.clone())); + } + Ok(ImportOutcome::SkippedAlreadyHaveBetter) => { + // The download's data is already handled — `import_one` + // deleted the losing file itself — so there's nothing left + // for qBittorrent to track either. + imported_hashes.push((grab.torrent_hash().to_string(), content_path.clone())); } - Ok(ImportOutcome::SkippedAlreadyHaveBetter) => {} Err(e) => { let error_count = record_import_error(conn, grab.release_id())?; if error_count >= MAX_IMPORT_ERRORS { @@ -1476,6 +1720,7 @@ fn process_pending_grabs( &format!("import failed {error_count} times in a row: {e}"), )?; stats.failed += 1; + imported_hashes.push((grab.torrent_hash().to_string(), content_path.clone())); } else { tracing::warn!( error = %e, @@ -1489,7 +1734,7 @@ fn process_pending_grabs( } } - Ok(stats) + Ok((stats, imported_hashes)) } /// What actually happened when `import_one` was asked to import a @@ -1504,7 +1749,75 @@ enum ImportOutcome { SkippedAlreadyHaveBetter, } -fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Result { +/// Shared by every place a freshly-imported file becomes eligible for the +/// async transcode queue (`import_one`, season-pack import) — reads the +/// probe data `ensure_probed` just wrote, delegates the codec/HDR/height +/// eligibility call to `transcode::should_enqueue`, and — if eligible — +/// decides the encode pipeline (`transcode::is_anime_content`, path-prefix +/// first then metadata fallback) before enqueueing. Never propagates a +/// failure into the caller's import result — same "never fail an +/// otherwise-successful import" treatment as probing. +fn maybe_enqueue_transcode( + conn: &Connection, + media_item_id: i64, + episode_file_id: i64, + cfg: &breadarr_shared::config::TranscodeConfig, +) -> Result<()> { + let probe: Option<(Option, i64, Option, i64)> = conn + .query_row( + "SELECT video_codec, hdr, height, probe_size_bytes FROM media_file_probe WHERE episode_file_id = ?1", + params![episode_file_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .optional()?; + let Some((video_codec, hdr, height, size_bytes)) = probe else { + // Probing itself failed (recorded as `probe_failed`) — nothing + // reliable to decide eligibility from yet; skip rather than guess. + return Ok(()); + }; + + if !crate::transcode::should_enqueue(video_codec.as_deref(), hdr != 0, height, cfg) { + return Ok(()); + } + + let path: String = conn.query_row( + "SELECT path FROM episode_file WHERE id = ?1", + params![episode_file_id], + |row| row.get(0), + )?; + let is_anime = crate::transcode::is_anime_content(conn, media_item_id, Path::new(&path), cfg)?; + crate::transcode::enqueue(conn, episode_file_id, video_codec.as_deref(), size_bytes, is_anime, false)?; + Ok(()) +} + +/// A tracked file renamed aside to make room for an incoming upgrade — +/// `parked_at` may be empty (never actually renamed anything) when the +/// tracked `episode_file` row's file was already missing from disk; `row_id` +/// is `None` when this represents a stray untracked file at `dest` rather +/// than an actual tracked row to drop. +struct StaleFile { + parked_at: PathBuf, + restore_to: PathBuf, + row_id: Option, +} + +impl StaleFile { + /// Best-effort: puts the parked file back where it came from after a + /// later step (free-space check, the actual move) fails, so a failed + /// import never leaves neither the old file nor the new one behind. + fn restore(&self) { + if !self.parked_at.as_os_str().is_empty() { + std::fs::rename(&self.parked_at, &self.restore_to).ok(); + } + } +} + +fn import_one( + conn: &Connection, + grab: &PendingGrab, + content_path: &Path, + transcode_cfg: Option<&breadarr_shared::config::TranscodeConfig>, +) -> Result { let source_path = locate_video_file(content_path)?; let ext = source_path .extension() @@ -1521,19 +1834,22 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res !mkv::default_track_is_english_or_unset(&tracks) && mkv::has_english_track(&tracks); if needs_fix { let tmp = source_path.with_extension("fixed.mkv"); - mkv::remux_english_default(&source_path, &tmp, &tracks)?; - // `source_path` is qBittorrent's actual seeding payload for - // this torrent — deleting it here, before the free-space check - // and the import below have even run, defeats the whole - // staging design (whose point is to keep seeding intact) and - // risks real data loss if either subsequent step fails: the - // original would already be gone, `working_path` might not - // have made it into the library either, and qBittorrent can't - // necessarily re-fetch a dead swarm. `link_or_copy_file` - // already leaves its source untouched for exactly this reason - // in the non-remux path (see its own doc comment) — the remux - // scratch output below gets the same treatment: only removed - // once it's been successfully imported. + if let Err(e) = mkv::remux_english_default(&source_path, &tmp, &tracks) { + // A failed remux can still leave a partially written `tmp` + // behind; left uncleaned it sits in the library/download + // directory as a stray `.fixed.mkv`, which + // `find_relinkable_episode_files` can then match on the same + // `SxxExx` substring as the real file and report the episode + // as ambiguous. + std::fs::remove_file(&tmp).ok(); + return Err(e); + } + // Deliberately not deleting `source_path` here, before the + // free-space check and the import below have even run: doing so + // risks real data loss if either subsequent step fails, leaving + // neither the original nor a successfully-placed replacement + // anywhere. `source_path` is only ever removed once the remuxed + // derivative has actually landed in the library (see below). working_path = tmp; remuxed = true; } @@ -1581,23 +1897,67 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res std::fs::create_dir_all(&dest_dir)?; let dest = dest_dir.join(&filename); - // Set below (to the renamed-aside stale file's path) only when this - // import is an upgrade over an existing, worse-scoring file — see the - // `dest.exists()` branch. Used to restore the original on any failure - // between here and the replacement being confirmed on disk, and to - // gate the deferred cleanup (old row + old file) once it succeeds. - let mut old_sibling: Option = None; + // Looked up by *identity* (episode_id for TV, media_item_id with a NULL + // episode_id for movies) rather than by whether a file happens to sit at + // `dest` right now. A real production bug found the hard way: a movie's + // `root_folder` had drifted (recategorized between library folders) + // after it was already imported, so its tracked `episode_file.path` + // no longer matched the *current* `dest` — `dest.exists()` came back + // false, the whole comparison below was skipped entirely, and a fresh + // grab imported right in alongside the untouched original, a real + // duplicate neither this function nor the score comparison ever knew + // to look for. Keying off identity means the comparison still happens + // even when the tracked file's path and the freshly computed `dest` + // disagree. + // Collects *every* matching row, not just one: nothing in the schema + // enforces a single `episode_file` per episode (the `library_health` + // duplicate-groups report exists precisely because duplicates occur + // here), and `query_row` would silently pick an arbitrary one of them, + // leaving any other duplicate's row and file untouched — orphaned + // pointing at a file this import may have just deleted (via the + // `old_sibling`/`dest` overwrite below), or, worse, read as the *only* + // existing file for the `upgrade_locked` check so a transcoded file's + // lock could be missed if it happened to live in the row `query_row` + // didn't return. + let existing_files: Vec<(i64, String, i64)> = conn + .prepare( + "SELECT id, path, upgrade_locked FROM episode_file + WHERE (episode_id = ?1 AND ?1 IS NOT NULL) OR (media_item_id = ?2 AND ?1 IS NULL)", + )? + .query_map(params![episode_id, grab.media_item_id()], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .collect::>>()?; - // The deterministic filename means a second release for the same - // episode/movie collides on this exact path. `link_or_copy_file`'s copy - // fallback renames into place, which *silently overwrites* an existing - // file with no comparison at all — a lower-scored duplicate arriving - // second (e.g. a 480p release importing after an already-imported - // 1080p one, entirely possible before resolution was scored, and still - // possible from a race between two grabs of the same episode) would - // quietly replace the better file already in the library. Checked here - // rather than left to the filesystem to decide by import order. - if dest.exists() { + // Trigger the comparison whenever *either* signal says something might + // already be here: a tracked row (regardless of where its file actually + // is), or a physical file already sitting at `dest` (a stray, + // not-yet-reconciled leftover with no tracked row at all — the case the + // original `dest.exists()`-only check covered). Comparing on either + // condition alone would miss the other's failure mode; comparing on + // both is a strict superset of what the original single check did. + let mut old_siblings: Vec = Vec::new(); + if !existing_files.is_empty() || dest.exists() { + // Belt-and-suspenders: the missing-content/upgrade search paths + // already refuse to re-grab an `upgrade_locked` file (it isn't + // "missing" and the upgrade loop skips it), so this shouldn't be + // reachable for one today — but a locally AV1-transcoded file + // should never be silently overwritten on score alone regardless + // of which path produced the incoming grab, same guard as + // `import_season_pack_file`'s matching check. Locked if *any* + // duplicate row is locked, not just whichever one a plain + // `query_row` would have happened to return. + let upgrade_locked = existing_files.iter().any(|(_, _, ul)| *ul != 0); + if upgrade_locked { + if remuxed { + std::fs::remove_file(&working_path).ok(); + } + // Nothing seeds this download anymore and it isn't going + // anywhere — the episode already has a file in place, so + // there's no reason to leave a duplicate orphaned on disk. + std::fs::remove_file(&source_path).ok(); + return Ok(ImportOutcome::SkippedAlreadyHaveBetter); + } let current_score: f32 = conn .query_row( "SELECT score FROM release WHERE id = ?1", @@ -1635,29 +1995,73 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res if remuxed { std::fs::remove_file(&working_path).ok(); } + // Same reasoning as the upgrade_locked case above — this + // download lost the comparison and nothing seeds it, so it's + // just wasted disk space if left in place. + std::fs::remove_file(&source_path).ok(); return Ok(ImportOutcome::SkippedAlreadyHaveBetter); } // The new file scores strictly better than what's currently there. - // Move the stale file sideways to a `.old` sibling — rather than - // deleting it and its tracking row outright — so the replacement is - // placed and confirmed *before* the original is actually given up. - // A `rename` (not a delete) still frees up `dest` for the primary - // hardlink path (`std::fs::hard_link` fails outright if the - // destination already exists), so an upgrade is still a cheap - // hardlink swap in the common case; it just also means that if the - // free-space check or `link_or_copy_file` below fails (I/O error, - // dest dir vanished, disk full), the original file gets moved back - // into place instead of being gone for good. The DB row is left - // alone until the replacement is confirmed on disk, for the same - // reason. - old_sibling = Some(PathBuf::from(format!("{}.old", dest.display()))); - std::fs::rename(&dest, old_sibling.as_ref().unwrap())?; + // Park aside whatever's actually here — the tracked row's real file + // (not necessarily `dest` — see above) and/or a stray file sitting + // at `dest` with no tracked row — rather than deleting anything + // outright, so the replacement is placed and confirmed *before* + // the original is actually given up. If the free-space check or + // `move_or_copy_file` below then fails (I/O error, dest dir + // vanished, disk full), everything parked gets moved back into + // place instead of being gone for good. The DB row is left alone + // until the replacement is confirmed on disk, for the same reason. + // Park *every* matching row, not just one — a second (duplicate) + // row left untouched here would otherwise keep pointing at a file + // that may be gone once the replacement lands (the incoming file + // gets moved to `dest`, and any duplicate's file sitting elsewhere + // is simply orphaned in the DB with no cleanup). + for (existing_id, existing_path, _) in &existing_files { + let existing_path = PathBuf::from(existing_path); + if existing_path.exists() { + let parked_at = PathBuf::from(format!("{}.old", existing_path.display())); + std::fs::rename(&existing_path, &parked_at)?; + old_siblings.push(StaleFile { + parked_at, + restore_to: existing_path, + row_id: Some(*existing_id), + }); + } else { + // The tracked row survived but its file didn't (e.g. + // deleted out from under breadarr) — nothing to park, but + // the stale row still needs dropping once the new file is + // confirmed in place. + old_siblings.push(StaleFile { + parked_at: PathBuf::new(), + restore_to: PathBuf::new(), + row_id: Some(*existing_id), + }); + } + } + if dest.exists() && old_siblings.iter().all(|s| s.restore_to != dest) { + let parked_at = PathBuf::from(format!("{}.old", dest.display())); + std::fs::rename(&dest, &parked_at)?; + // Doesn't duplicate an existing park (from the tracked-row loop + // above) — only added when none of those already cover `dest`. + old_siblings.push(StaleFile { + parked_at, + restore_to: dest.clone(), + row_id: None, + }); + } } let needed_bytes = std::fs::metadata(&working_path)?.len(); - if insufficient_space(&dest_dir, needed_bytes)? { - if let Some(old_sibling) = &old_sibling { - std::fs::rename(old_sibling, &dest).ok(); + if !same_filesystem(&working_path, &dest_dir) && insufficient_space(&dest_dir, needed_bytes)? { + for stale in &old_siblings { + stale.restore(); + } + // `working_path` is the remux scratch copy (`source_path` is the + // real, still-intact download) — disposable, and left uncleaned it + // leaks into whichever directory it was written in (see the same + // cleanup added above for a failed remux). + if remuxed { + std::fs::remove_file(&working_path).ok(); } anyhow::bail!( "not enough free space at {} for {needed_bytes} bytes (source: {})", @@ -1666,31 +2070,39 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res ); } - if let Err(err) = link_or_copy_file(&working_path, &dest) { + if let Err(err) = move_or_copy_file(&working_path, &dest) { // Restore the original file rather than leaving the user with // neither the old file nor the new one — this is the exact failure // mode a `DELETE`-then-place ordering used to leave unrecoverable. - if let Some(old_sibling) = &old_sibling { - std::fs::rename(old_sibling, &dest).ok(); + for stale in &old_siblings { + stale.restore(); + } + if remuxed { + std::fs::remove_file(&working_path).ok(); } return Err(err); } if remuxed { - // `working_path` here is the scratch remux output, not qBittorrent's - // original content file (which was already removed above, in the - // remux branch, to make way for it) — nothing else reads it, so - // unlike the plain-import case there's no seeding reason to keep it. - std::fs::remove_file(&working_path).ok(); + // `working_path` (the remux scratch output) was already consumed by + // the move above either way — nothing left to clean up there. What's + // left is `source_path`: qBittorrent's original pre-remux download, + // a genuinely separate file from `working_path`. Nothing seeds it + // anymore and its remuxed derivative is now safely in the library, + // so it's just wasted disk space if left behind. + std::fs::remove_file(&source_path).ok(); } // The replacement is confirmed in place on disk — only now is it safe - // to drop the old tracking row and the renamed-aside original. - if let Some(old_sibling) = old_sibling { - conn.execute( - "DELETE FROM episode_file WHERE path = ?1", - params![dest.to_string_lossy()], - )?; - std::fs::remove_file(&old_sibling).ok(); + // to drop the old tracking row(s) (by id, not by path — a tracked row's + // path may never have matched `dest` in the first place) and the + // renamed-aside original(s). + for stale in old_siblings { + if let Some(row_id) = stale.row_id { + conn.execute("DELETE FROM episode_file WHERE id = ?1", params![row_id])?; + } + if !stale.parked_at.as_os_str().is_empty() { + std::fs::remove_file(&stale.parked_at).ok(); + } } let size_bytes = std::fs::metadata(&dest)?.len(); @@ -1745,6 +2157,17 @@ fn import_one(conn: &Connection, grab: &PendingGrab, content_path: &Path) -> Res )?; } + // Queue this freshly-imported file for the async AV1 transcode swap + // (see `transcode::run_cycle`) — the file is available in the library + // immediately, exactly as today; the transcode happens later in the + // background. Never blocks or fails the import itself: enqueue errors + // are logged and swallowed, same treatment as probing failures above. + if let Some(cfg) = transcode_cfg { + if let Err(e) = maybe_enqueue_transcode(conn, grab.media_item_id(), episode_file_id, cfg) { + tracing::warn!(episode_file_id, error = %e, "failed to check/enqueue transcode job"); + } + } + Ok(ImportOutcome::Imported { remuxed, quality_flagged, @@ -1774,6 +2197,7 @@ struct SeasonPackImportOutcome { /// double a season pack's import time. Any file that needs it is still /// reachable afterward via `remux_backlog`, which sweeps the whole library /// including season-pack imports. +#[allow(clippy::too_many_arguments)] fn import_season_pack( conn: &Connection, release_id: i64, @@ -1782,6 +2206,7 @@ fn import_season_pack( season_number: u32, root_folder: &str, content_path: &Path, + transcode_cfg: Option<&breadarr_shared::config::TranscodeConfig>, ) -> Result { let release_score: f32 = conn .query_row( @@ -1811,6 +2236,12 @@ fn import_season_pack( std::fs::create_dir_all(root_folder)?; + let tvdb_id: Option = conn.query_row( + "SELECT tvdb_id FROM media_item WHERE id = ?1", + params![media_item_id], + |row| row.get(0), + )?; + let mut outcome = SeasonPackImportOutcome::default(); for source_path in &video_files { let filename_only = source_path @@ -1818,18 +2249,24 @@ fn import_season_pack( .and_then(|f| f.to_str()) .unwrap_or_default(); let parsed = crate::parser::parse(filename_only); - let Some(episode_number) = parsed.episode.or(parsed.absolute_episode) else { - outcome.episodes_unmatched += 1; - tracing::warn!( - file = filename_only, - "season pack: could not determine an episode number for this file, skipping" - ); - continue; + // Anime packs name files `[SubsPlease] Show - 15.mkv` (absolute + // only). Treating that as S01E15 skipped the AniDB map that + // `library_scan` already uses via `resolve_episode`. + let resolved = crate::scheduler::resolve_episode(conn, tvdb_id, &parsed)?; + let (file_season, episode_number) = match resolved { + Some(pair) => pair, + None => match (parsed.season, parsed.episode) { + (Some(season), Some(episode)) => (season, episode), + _ => { + outcome.episodes_unmatched += 1; + tracing::warn!( + file = filename_only, + "season pack: could not determine an episode number for this file, skipping" + ); + continue; + } + }, }; - // Prefer the individual file's own season marker when it has one - // (a pack can occasionally mix seasons); fall back to the pack's - // own season otherwise. - let file_season = parsed.season.unwrap_or(season_number); let episode_id = match crate::scheduler::find_episode_id( conn, @@ -1861,6 +2298,7 @@ fn import_season_pack( episode_number, root_folder, source_path, + transcode_cfg, ) { Ok(PackFileOutcome::Imported { quality_flagged }) => { outcome.episodes_imported += 1; @@ -1926,7 +2364,7 @@ enum PackFileOutcome { } /// One file's worth of the season-pack import: the same dest-collision -/// scoring, free-space check, hardlink-or-copy, `episode_file` bookkeeping, +/// scoring, free-space check, move-or-copy, `episode_file` bookkeeping, /// and post-import probe that `import_one` does for a single-episode grab, /// scoped to one already-identified `episode_id` within a larger pack. #[allow(clippy::too_many_arguments)] @@ -1941,6 +2379,7 @@ fn import_season_pack_file( episode_number: u32, root_folder: &str, source_path: &Path, + transcode_cfg: Option<&breadarr_shared::config::TranscodeConfig>, ) -> Result { let ext = source_path .extension() @@ -1963,35 +2402,97 @@ fn import_season_pack_file( std::fs::create_dir_all(&dest_dir)?; let dest = dest_dir.join(&filename); - // Same reasoning as import_one's own dest-collision check: a second - // release for the same episode (here, from a *different* pack or a - // single-episode grab) must not silently overwrite a better file - // already in place. - let mut old_sibling: Option = None; - if dest.exists() { + // Looked up by identity (this episode's id), same reasoning as + // import_one's matching fix: a season pack's incoming file must be + // compared against whatever this episode is *actually* tracked as + // having, not against whatever happens to physically sit at `dest` + // right now — those can disagree if `root_folder` ever drifted after + // the tracked file was imported. + // Every matching row, not just one — same duplicate-row fix as + // `import_one` (see its comment): nothing enforces a single + // `episode_file` per episode, and `query_row` would silently pick an + // arbitrary one, leaving any duplicate's row/file untouched and + // potentially missing its `upgrade_locked` flag. + let mut old_siblings: Vec = Vec::new(); + let existing_files: Vec<(i64, String, i64)> = conn + .prepare("SELECT id, path, upgrade_locked FROM episode_file WHERE episode_id = ?1")? + .query_map(params![episode_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .collect::>>()?; + + // Trigger on either signal, same generalization as import_one's matching + // fix: a tracked row (wherever its file really is) or a stray physical + // file at `dest` with no tracked row at all. + if !existing_files.is_empty() || dest.exists() { + // A locally AV1-transcoded file is a deliberate shrink, not + // something a season pack (scored against its own, much larger, + // original release) should be allowed to silently overwrite just + // because its release score happens to be higher — that score was + // never computed against a transcoded file's actual size/quality + // tradeoff. Checked before the score comparison, same reasoning as + // `movie_eligible_for_upgrade`/`enumerate_upgrade_targets`. Locked + // if *any* duplicate row is locked. + let upgrade_locked = existing_files.iter().any(|(_, _, ul)| *ul != 0); + if upgrade_locked { + // Nothing seeds this download anymore and it isn't going + // anywhere — this one file within the pack loses to what's + // already in place, so there's no reason to leave it orphaned + // on disk (other files in the same pack are handled by their + // own separate calls to this function, so only this one file + // is removed here, not the whole pack directory). + std::fs::remove_file(source_path).ok(); + return Ok(PackFileOutcome::SkippedAlreadyHaveBetter); + } let existing_best: Option = conn.query_row( "SELECT MAX(score) FROM release WHERE status = 'imported' AND id != ?1 AND episode_id = ?2", params![release_id, episode_id], |row| row.get(0), )?; if existing_best.is_some_and(|best| best >= release_score) { + std::fs::remove_file(source_path).ok(); return Ok(PackFileOutcome::SkippedAlreadyHaveBetter); } - // This episode's file is being upgraded. Move the stale file aside - // to a `.old` sibling rather than deleting it (and its row) outright - // — `dest` is still free for the hardlink fast path, but the - // original survives on disk until the replacement is confirmed in - // place, so a failed free-space check or `link_or_copy_file` below - // can't lose the file. See import_one's matching comment for the - // fuller reasoning; this is the same fix applied there. - old_sibling = Some(PathBuf::from(format!("{}.old", dest.display()))); - std::fs::rename(&dest, old_sibling.as_ref().unwrap())?; + // This episode's file is being upgraded. Park aside whatever's + // actually here — the tracked row's real file and/or a stray file + // at `dest` with no tracked row — rather than deleting anything + // outright, so the replacement is placed and confirmed *before* + // the original is actually given up. See import_one's matching + // comment for the fuller reasoning; this is the same fix applied + // there. + for (existing_id, existing_path, _) in &existing_files { + let existing_path = PathBuf::from(existing_path); + if existing_path.exists() { + let parked_at = PathBuf::from(format!("{}.old", existing_path.display())); + std::fs::rename(&existing_path, &parked_at)?; + old_siblings.push(StaleFile { + parked_at, + restore_to: existing_path, + row_id: Some(*existing_id), + }); + } else { + old_siblings.push(StaleFile { + parked_at: PathBuf::new(), + restore_to: PathBuf::new(), + row_id: Some(*existing_id), + }); + } + } + if dest.exists() && old_siblings.iter().all(|s| s.restore_to != dest) { + let parked_at = PathBuf::from(format!("{}.old", dest.display())); + std::fs::rename(&dest, &parked_at)?; + old_siblings.push(StaleFile { + parked_at, + restore_to: dest.clone(), + row_id: None, + }); + } } let needed_bytes = std::fs::metadata(source_path)?.len(); - if insufficient_space(&dest_dir, needed_bytes)? { - if let Some(old_sibling) = &old_sibling { - std::fs::rename(old_sibling, &dest).ok(); + if !same_filesystem(source_path, &dest_dir) && insufficient_space(&dest_dir, needed_bytes)? { + for stale in &old_siblings { + stale.restore(); } anyhow::bail!( "not enough free space at {} for {needed_bytes} bytes (source: {})", @@ -2000,19 +2501,20 @@ fn import_season_pack_file( ); } - if let Err(err) = link_or_copy_file(source_path, &dest) { - if let Some(old_sibling) = &old_sibling { - std::fs::rename(old_sibling, &dest).ok(); + if let Err(err) = move_or_copy_file(source_path, &dest) { + for stale in &old_siblings { + stale.restore(); } return Err(err); } - if let Some(old_sibling) = old_sibling { - conn.execute( - "DELETE FROM episode_file WHERE path = ?1", - params![dest.to_string_lossy()], - )?; - std::fs::remove_file(&old_sibling).ok(); + for stale in old_siblings { + if let Some(row_id) = stale.row_id { + conn.execute("DELETE FROM episode_file WHERE id = ?1", params![row_id])?; + } + if !stale.parked_at.as_os_str().is_empty() { + std::fs::remove_file(&stale.parked_at).ok(); + } } let size_bytes = std::fs::metadata(&dest)?.len(); @@ -2034,6 +2536,12 @@ fn import_season_pack_file( } }; + if let Some(cfg) = transcode_cfg { + if let Err(e) = maybe_enqueue_transcode(conn, media_item_id, episode_file_id, cfg) { + tracing::warn!(episode_file_id, error = %e, "failed to check/enqueue transcode job"); + } + } + Ok(PackFileOutcome::Imported { quality_flagged }) } @@ -2605,6 +3113,124 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + // Regression coverage for a real production finding: several shows had + // `episode` rows marked `has_file = 0` whose real file was sitting + // exactly where breadarr's own importer would have put it, with no + // `episode_file` row at all — almost certainly a residue of an earlier + // DB-recovery incident. This models that shape directly: an episode row + // with no matching episode_file, and a real file already at the + // expected season directory bearing breadarr's own SxxEyy marker. + #[test] + fn find_relinkable_episode_files_finds_a_file_with_no_tracked_row() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + let dir = std::env::temp_dir().join(format!("breadarr-relink-{}", std::process::id())); + media_item_with_root(&conn, 1, &dir); + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, title, has_file) + VALUES (1, 1, 2, 1, 'Seven Years Later', 0)", + [], + ) + .unwrap(); + + let season_dir = dir.join("Season 02"); + std::fs::create_dir_all(&season_dir).unwrap(); + let real_file = season_dir.join("Some Show - S02E01 - Seven Years Later.mkv"); + std::fs::write(&real_file, b"already on disk, never linked").unwrap(); + + let (candidates, ambiguous) = find_relinkable_episode_files(&conn).unwrap(); + assert!(ambiguous.is_empty()); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].episode_id, 1); + assert_eq!(candidates[0].path, real_file); + + let linked = relink_episode_files(&conn, &candidates).unwrap(); + assert_eq!(linked, 1); + + let has_file: i64 = conn + .query_row("SELECT has_file FROM episode WHERE id = 1", [], |r| r.get(0)) + .unwrap(); + assert_eq!(has_file, 1); + let tracked_path: String = conn + .query_row( + "SELECT path FROM episode_file WHERE episode_id = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(tracked_path, real_file.to_string_lossy()); + // Purely additive — the file itself was never touched. + assert_eq!(std::fs::read(&real_file).unwrap(), b"already on disk, never linked"); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + // Regression coverage for the wider pattern found across ~20 shows on + // real production data: a pre-breadarr library organized entirely under + // the unpadded "Season N" convention, with no zero-padded folder at all + // for that season. `season_dir` alone would never find anything here. + #[test] + fn find_relinkable_episode_files_falls_back_to_the_unpadded_season_folder() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + let dir = std::env::temp_dir().join(format!("breadarr-relink-unpadded-{}", std::process::id())); + media_item_with_root(&conn, 1, &dir); + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, title, has_file) + VALUES (1, 1, 1, 1, 'Pilot', 0)", + [], + ) + .unwrap(); + + // No "Season 01" anywhere — only the unpadded convention. + let season_dir = dir.join("Season 1"); + std::fs::create_dir_all(&season_dir).unwrap(); + let real_file = season_dir.join("Some.Show.S01E01.Pilot.1080p.mkv"); + std::fs::write(&real_file, b"pre-breadarr library content").unwrap(); + + let (candidates, ambiguous) = find_relinkable_episode_files(&conn).unwrap(); + assert!(ambiguous.is_empty()); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].path, real_file); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn find_relinkable_episode_files_skips_an_ambiguous_match_rather_than_guessing() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + let dir = + std::env::temp_dir().join(format!("breadarr-relink-ambiguous-{}", std::process::id())); + media_item_with_root(&conn, 1, &dir); + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, title, has_file) + VALUES (1, 1, 1, 1, 'Pilot', 0)", + [], + ) + .unwrap(); + + let season_dir = dir.join("Season 01"); + std::fs::create_dir_all(&season_dir).unwrap(); + std::fs::write(season_dir.join("Some Show - S01E01 - Pilot.mkv"), b"one").unwrap(); + std::fs::write( + season_dir.join("[Group] Some Show - S01E01 (dual audio).mkv"), + b"two", + ) + .unwrap(); + + let (candidates, ambiguous) = find_relinkable_episode_files(&conn).unwrap(); + assert!(candidates.is_empty(), "an ambiguous match must not be guessed at"); + assert_eq!(ambiguous.len(), 1); + + let tracked_count: i64 = conn + .query_row("SELECT count(*) FROM episode_file", [], |r| r.get(0)) + .unwrap(); + assert_eq!(tracked_count, 0); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn a_fresh_grab_is_not_stalled() { let (conn, release_id) = seeded_release_conn(1.0); @@ -2712,44 +3338,21 @@ mod tests { #[test] fn sanitizes_path_hostile_characters() { assert_eq!(sanitize("Kill: Ao / Blue?"), "Kill_ Ao _ Blue_"); + assert_eq!(sanitize(".."), "_"); + assert_eq!(sanitize("."), "_"); + // Slashes become `_` first, so this is not a lone `..` component. + assert_eq!(sanitize("foo/../bar"), "foo_.._bar"); + assert_eq!(sanitize("title\nwith\x00ctrl"), "title_with_ctrl"); } #[test] - fn nearest_existing_ancestor_returns_the_path_itself_when_it_exists() { - let dir = std::env::temp_dir(); - assert_eq!(nearest_existing_ancestor(&dir).unwrap(), dir); - } - - #[test] - fn nearest_existing_ancestor_walks_up_past_nonexistent_components() { - let dir = std::env::temp_dir().join(format!( - "breadarr-ancestor-test-{}/does/not/exist/yet", - std::process::id() - )); - let expected = - std::env::temp_dir().join(format!("breadarr-ancestor-test-{}", std::process::id())); - std::fs::create_dir_all(&expected).unwrap(); - assert_eq!(nearest_existing_ancestor(&dir).unwrap(), expected); - std::fs::remove_dir_all(&expected).unwrap(); - } - - #[test] - fn staging_dir_for_is_named_by_torrent_hash_under_the_marker_directory() { - let dest = std::env::temp_dir(); - let staging = staging_dir_for(&dest, "deadbeef1234").unwrap(); + fn omits_a_cjk_only_episode_title_instead_of_embedding_it() { + // TVDB sometimes has no English episode title at all for a given + // show, only the original Japanese one — verified live across + // several real shows' entire seasons. assert_eq!( - staging.file_name().unwrap().to_str().unwrap(), - "deadbeef1234" - ); - assert_eq!( - staging - .parent() - .unwrap() - .file_name() - .unwrap() - .to_str() - .unwrap(), - STAGING_DIR_NAME + deterministic_filename("Helck", 1, 3, Some("未知の敵"), "mkv"), + "Helck - S01E03.mkv" ); } @@ -2764,6 +3367,42 @@ mod tests { assert!(insufficient_space(&std::env::temp_dir(), u64::MAX / 2).unwrap()); } + // Regression test for a real gap found in review: `insufficient_space` + // checks `dest`'s free space against the *full* source file size, but + // `move_or_copy_file` tries a same-filesystem `rename` first, which + // needs essentially none. Two paths under the same temp dir are + // guaranteed to share a device, so this exercises the exact case that + // used to produce false "not enough free space" rejections (and, via + // the grab-fail-search retry loop, a permanent stuck cycle) whenever + // downloads and the library share a volume. + #[test] + fn same_filesystem_is_true_for_two_paths_under_the_same_temp_dir() { + let dir = std::env::temp_dir().join(format!("breadarr-same-fs-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let a = dir.join("a.mkv"); + let b = dir.join("subdir"); + std::fs::create_dir_all(&b).unwrap(); + std::fs::write(&a, b"x").unwrap(); + + assert!(same_filesystem(&a, &b), "two paths under the same temp dir must share a device"); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn same_filesystem_is_false_when_either_path_cannot_be_stat_d() { + let dir = std::env::temp_dir().join(format!("breadarr-same-fs-missing-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let missing = dir.join("does-not-exist.mkv"); + + assert!( + !same_filesystem(&dir, &missing), + "a stat failure must default to 'different filesystems' so the free-space check still runs" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn copy_via_temp_file_writes_through_a_part_file_and_renames_into_place() { let dir = std::env::temp_dir().join(format!("breadarr-copy-test-{}", std::process::id())); @@ -2786,22 +3425,137 @@ mod tests { } #[test] - fn link_or_copy_file_leaves_the_source_in_place() { - let dir = std::env::temp_dir().join(format!("breadarr-link-test-{}", std::process::id())); + fn move_or_copy_file_moves_the_source_out() { + let dir = std::env::temp_dir().join(format!("breadarr-move-test-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let src = dir.join("source.mkv"); std::fs::write(&src, b"fake video data").unwrap(); let dest = dir.join("dest.mkv"); - link_or_copy_file(&src, &dest).unwrap(); + move_or_copy_file(&src, &dest).unwrap(); - assert!(src.exists(), "source should survive a hardlink-or-copy"); + assert!(!src.exists(), "source should be gone after a move — nothing seeds it anymore"); assert!(dest.exists()); - assert_eq!(std::fs::read(&src).unwrap(), std::fs::read(&dest).unwrap()); + assert_eq!(std::fs::read(&dest).unwrap(), b"fake video data"); std::fs::remove_dir_all(&dir).unwrap(); } + #[test] + fn cleanup_leftover_download_dir_removes_a_folder_with_only_sidecars_left() { + let downloads = + std::env::temp_dir().join(format!("breadarr-cleanup-test-{}", std::process::id())); + let release_dir = downloads.join("Some Movie 2016 1080p"); + std::fs::create_dir_all(&release_dir).unwrap(); + // The video was already moved out by `move_or_copy_file`; only the + // sidecar junk qBittorrent downloaded alongside it remains. + std::fs::write(release_dir.join("poster.jpg"), b"jpeg bytes").unwrap(); + std::fs::write(release_dir.join("release.nfo"), b"nfo text").unwrap(); + + cleanup_leftover_download_dir(&release_dir, &downloads.to_string_lossy()).unwrap(); + + assert!(!release_dir.exists(), "the now-empty-of-video release folder should be gone"); + assert!(downloads.exists(), "the shared downloads root itself must survive"); + + std::fs::remove_dir_all(&downloads).unwrap(); + } + + #[test] + fn cleanup_leftover_download_dir_is_a_noop_for_a_bare_file() { + let downloads = + std::env::temp_dir().join(format!("breadarr-cleanup-file-test-{}", std::process::id())); + std::fs::create_dir_all(&downloads).unwrap(); + // A release with no wrapping folder — `move_or_copy_file` already + // consumed it, so `content_path` no longer exists at all. + let bare = downloads.join("Some.Movie.2016.1080p.mp4"); + + cleanup_leftover_download_dir(&bare, &downloads.to_string_lossy()).unwrap(); + + std::fs::remove_dir_all(&downloads).unwrap(); + } + + #[test] + fn cleanup_leftover_download_dir_refuses_to_remove_the_downloads_root_itself() { + let downloads = std::env::temp_dir() + .join(format!("breadarr-cleanup-guard-test-{}", std::process::id())); + std::fs::create_dir_all(&downloads).unwrap(); + std::fs::write(downloads.join("unrelated-other-download.mkv"), b"data").unwrap(); + + // A malformed `content_path` that happens to equal the downloads + // root itself must never be wiped, even though it passes the + // `is_dir()` check. + cleanup_leftover_download_dir(&downloads, &downloads.to_string_lossy()).unwrap(); + + assert!(downloads.exists()); + assert!(downloads.join("unrelated-other-download.mkv").exists()); + + std::fs::remove_dir_all(&downloads).unwrap(); + } + + #[test] + fn cleanup_leftover_download_dir_leaves_unmatched_videos() { + let downloads = std::env::temp_dir().join(format!( + "breadarr-cleanup-leftover-video-{}", + std::process::id() + )); + let release_dir = downloads.join("Some Show S01"); + std::fs::create_dir_all(&release_dir).unwrap(); + let unmatched = release_dir.join("Show - 15.mkv"); + std::fs::write(&unmatched, b"unmatched video").unwrap(); + std::fs::write(release_dir.join("release.nfo"), b"nfo").unwrap(); + std::fs::write(release_dir.join("poster.jpg"), b"jpeg").unwrap(); + + cleanup_leftover_download_dir(&release_dir, &downloads.to_string_lossy()).unwrap(); + + assert!( + unmatched.exists(), + "an unmatched video must survive leftover-dir cleanup" + ); + assert!( + !release_dir.join("release.nfo").exists(), + "sidecars next to leftover videos should still be dropped" + ); + assert!(!release_dir.join("poster.jpg").exists()); + assert!(release_dir.exists()); + + std::fs::remove_dir_all(&downloads).unwrap(); + } + + #[test] + fn cleanup_leftover_download_dir_empty_host_still_removes_a_per_torrent_folder() { + let downloads = std::env::temp_dir().join(format!( + "breadarr-cleanup-empty-host-{}", + std::process::id() + )); + let release_dir = downloads.join("Some Movie 2016 1080p"); + std::fs::create_dir_all(&release_dir).unwrap(); + std::fs::write(release_dir.join("poster.jpg"), b"jpeg").unwrap(); + + cleanup_leftover_download_dir(&release_dir, "").unwrap(); + + assert!(!release_dir.exists()); + assert!(downloads.exists()); + + std::fs::remove_dir_all(&downloads).unwrap(); + } + + #[test] + fn cleanup_leftover_download_dir_empty_host_refuses_common_roots() { + cleanup_leftover_download_dir(Path::new("/"), "").unwrap(); + assert!(Path::new("/").exists()); + + cleanup_leftover_download_dir(Path::new("/mnt"), "").unwrap(); + cleanup_leftover_download_dir(Path::new("/media"), "").unwrap(); + + if let Some(home) = std::env::var_os("HOME") { + let home_path = PathBuf::from(&home); + if home_path.is_dir() { + cleanup_leftover_download_dir(&home_path, "").unwrap(); + assert!(home_path.exists(), "must never wipe $HOME"); + } + } + } + #[test] fn locates_a_single_file_torrent() { let dir = std::env::temp_dir().join(format!("breadarr-test-{}", std::process::id())); @@ -2815,6 +3569,42 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + #[test] + fn walk_files_does_not_follow_symlinks() { + let dir = std::env::temp_dir().join(format!( + "breadarr-walk-symlink-{}", + std::process::id() + )); + let outside = std::env::temp_dir().join(format!( + "breadarr-walk-outside-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(dir.join("real.mkv"), b"inside").unwrap(); + std::fs::write(outside.join("secret.mkv"), b"escaped").unwrap(); + std::os::unix::fs::symlink(&outside, dir.join("escape")).unwrap(); + std::os::unix::fs::symlink(outside.join("secret.mkv"), dir.join("link.mkv")).unwrap(); + + let walked = walk_files(&dir).unwrap(); + let names: Vec = walked + .iter() + .filter_map(|p| p.file_name().and_then(|n| n.to_str()).map(str::to_string)) + .collect(); + assert!(names.contains(&"real.mkv".to_string())); + assert!( + !names.contains(&"secret.mkv".to_string()), + "a directory symlink must not pull files from outside the scan root" + ); + assert!( + !names.contains(&"link.mkv".to_string()), + "a file symlink to a video must be skipped" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + std::fs::remove_dir_all(&outside).unwrap(); + } + #[test] fn remap_path_translates_container_prefix_to_host_prefix() { assert_eq!( @@ -2835,6 +3625,31 @@ mod tests { ); } + #[test] + fn remap_path_rejects_parent_dir_escape() { + let remapped = remap_path( + "/downloads/../../etc", + "/downloads", + "/home/breadway/downloads", + ); + assert_eq!(remapped, PathBuf::from("/downloads/../../etc")); + assert!( + !normalize_lexically(&remapped).starts_with("/home/breadway/downloads") + || remapped.as_os_str() == "/downloads/../../etc", + "a `..` remainder must not be rewritten under the host prefix" + ); + + let remapped = remap_path( + "/downloads/show/../../../etc/passwd", + "/downloads", + "/home/breadway/downloads", + ); + assert_eq!( + remapped, + PathBuf::from("/downloads/show/../../../etc/passwd") + ); + } + #[test] fn process_pending_grabs_routes_each_grab_by_torrent_state() { use crate::qbit::TorrentInfo; @@ -2940,12 +3755,22 @@ mod tests { // absent — simulating torrents qBit no longer knows about. ]; - let stats = process_pending_grabs(&conn, &pending, &torrents, "", "").unwrap(); + let (stats, hashes) = process_pending_grabs(&conn, &pending, &torrents, "", "", None).unwrap(); assert_eq!(stats.imported, 1); assert_eq!(stats.skipped_incomplete, 1); assert_eq!(stats.failed, 1); assert_eq!(stats.errors, 0); + assert!( + hashes.iter().any(|(h, _)| h == "hash-complete"), + "a successful import must still be queued for qBit delete" + ); + assert!( + hashes.iter().any(|(h, _)| h == "hash-missing-stale"), + "a grab failed for a missing torrent must be queued for qBit delete" + ); + assert!(!hashes.iter().any(|(h, _)| h == "hash-downloading")); + assert!(!hashes.iter().any(|(h, _)| h == "hash-missing-fresh")); let statuses: Vec<(i64, String)> = { let mut stmt = conn @@ -3002,7 +3827,7 @@ mod tests { content_path: "/tmp/somewhere-mid-move".to_string(), }]; - let stats = process_pending_grabs(&conn, &pending, &torrents, "", "").unwrap(); + let (stats, _) = process_pending_grabs(&conn, &pending, &torrents, "", "", None).unwrap(); assert_eq!(stats.imported, 0); assert_eq!(stats.skipped_incomplete, 1); assert_eq!(stats.errors, 0); @@ -3059,7 +3884,7 @@ mod tests { for i in 1..MAX_IMPORT_ERRORS { let pending = fetch_pending_grabs(&conn).unwrap(); - let stats = process_pending_grabs(&conn, &pending, &torrents, "", "").unwrap(); + let (stats, _) = process_pending_grabs(&conn, &pending, &torrents, "", "", None).unwrap(); assert_eq!(stats.errors, 1, "iteration {i}"); assert_eq!(stats.failed, 0, "iteration {i}"); let status: String = conn @@ -3070,9 +3895,13 @@ mod tests { // The Nth failure crosses the threshold and gives up. let pending = fetch_pending_grabs(&conn).unwrap(); - let stats = process_pending_grabs(&conn, &pending, &torrents, "", "").unwrap(); + let (stats, hashes) = process_pending_grabs(&conn, &pending, &torrents, "", "", None).unwrap(); assert_eq!(stats.failed, 1); assert_eq!(stats.errors, 0); + assert!( + hashes.iter().any(|(h, _)| h == "hash-broken"), + "a grab failed after MAX_IMPORT_ERRORS must be queued for qBit delete" + ); let status: String = conn .query_row("SELECT status FROM release WHERE id = 1", [], |r| r.get(0)) .unwrap(); @@ -3085,6 +3914,51 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + #[test] + fn a_stalled_grab_is_queued_for_qbit_delete() { + use crate::qbit::TorrentInfo; + + let (conn, release_id) = seeded_release_conn(STALL_THRESHOLD_HOURS + 1.0); + // Same progress already recorded — otherwise `update_grab_progress` + // would stamp `last_progress_at = now` and reset the stall clock. + conn.execute( + "UPDATE release SET last_seen_progress = 0.4, last_progress_at = datetime('now', ?1) WHERE id = ?2", + params![format!("-{} hours", STALL_THRESHOLD_HOURS + 1.0), release_id], + ) + .unwrap(); + let dir = std::env::temp_dir().join(format!( + "breadarr-stall-delete-{}", + std::process::id() + )); + let content = dir.join("partial"); + std::fs::create_dir_all(&content).unwrap(); + std::fs::write(content.join("movie.mp4"), b"partial").unwrap(); + + let pending = fetch_pending_grabs(&conn).unwrap(); + let torrents = vec![TorrentInfo { + hash: "deadbeef".to_string(), + name: "stalled".to_string(), + state: "downloading".to_string(), + progress: 0.4, + save_path: dir.to_string_lossy().to_string(), + content_path: content.to_string_lossy().to_string(), + }]; + + let (stats, hashes) = process_pending_grabs(&conn, &pending, &torrents, "", "", None).unwrap(); + assert_eq!(stats.failed, 1); + assert!( + hashes.iter().any(|(h, p)| h == "deadbeef" && p == &content), + "a stalled grab must be queued for qBit delete with its content_path" + ); + cleanup_leftover_download_dir(&content, "").unwrap(); + assert!( + content.join("movie.mp4").exists(), + "cleanup of a failed grab must not wipe leftover videos" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn imports_a_movie_release_with_no_episode_id() { let conn = Connection::open_in_memory().unwrap(); @@ -3124,7 +3998,7 @@ mod tests { root_folder: dest_root.to_string_lossy().to_string(), }; - let outcome = import_one(&conn, &grab, &content).unwrap(); + let outcome = import_one(&conn, &grab, &content, None).unwrap(); let ImportOutcome::Imported { remuxed, .. } = outcome else { panic!("expected a real import, got a skip"); }; @@ -3133,8 +4007,8 @@ mod tests { let dest = dest_root.join("Some Movie (2016).mp4"); assert!(dest.exists(), "expected {} to exist", dest.display()); assert!( - content.exists(), - "source file should survive import so qBittorrent can keep seeding" + !content.exists(), + "source file should be moved into place, not left behind" ); let status: String = conn @@ -3164,6 +4038,152 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + #[test] + fn import_one_enqueues_a_transcode_job_when_transcode_is_enabled() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder) + VALUES (1, 'movie', 'Some Movie', 2016, 1, 1, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, status, torrent_hash, grabbed_at) + VALUES (1, 1, NULL, 'Some Movie 2016 1080p', 1, 'guid-1', 'grabbed', 'deadbeef', datetime('now'))", + [], + ) + .unwrap(); + + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-enqueue-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let dest_root = dir.join("library"); + std::fs::create_dir_all(&dest_root).unwrap(); + // A real, probeable h264/1080p clip, not placeholder bytes: since + // `should_enqueue` now requires a successful probe (a `probe_failed` + // NULL-codec/NULL-height row must never enqueue an unclaimable job — + // see `should_enqueue`'s doc comment), a fake file would make + // `ensure_probed` record `probe_failed` and this test would no + // longer exercise the real "should this file be transcoded" path. + let content = dir.join("Some.Movie.2016.1080p.mkv"); + let clip = generate_test_clip(&dir, 1920, 1080); + std::fs::rename(&clip, &content).unwrap(); + + let grab = PendingGrab::Movie { + release_id: 1, + media_item_id: 1, + torrent_hash: "deadbeef".to_string(), + title: "Some Movie".to_string(), + year: Some(2016), + root_folder: dest_root.to_string_lossy().to_string(), + }; + + import_one(&conn, &grab, &content, Some(&breadarr_shared::config::TranscodeConfig::default())).unwrap(); + + let episode_file_id: i64 = conn + .query_row( + "SELECT id FROM episode_file WHERE media_item_id = 1", + [], + |r| r.get(0), + ) + .unwrap(); + let job_count: i64 = conn + .query_row( + "SELECT count(*) FROM transcode_job WHERE episode_file_id = ?1 AND status = 'pending'", + params![episode_file_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(job_count, 1); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn import_one_enqueues_an_anime_tagged_transcode_job_for_anime() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, tvdb_id, monitored, quality_profile_id, root_folder) + VALUES (1, 'series', 'Some Anime', 999, 1, 1, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO anime_mapping (anidb_id, tvdb_id) VALUES (1, 999)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file) + VALUES (1, 1, 1, 1, 1, 0)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'nyaa', 'rss', 'http://x')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, status, torrent_hash, grabbed_at) + VALUES (1, 1, 1, 'Some Anime S01E01', 1, 'guid-1', 'grabbed', 'deadbeef', datetime('now'))", + [], + ) + .unwrap(); + + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-anime-skip-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let dest_root = dir.join("library"); + std::fs::create_dir_all(&dest_root).unwrap(); + // Real, probeable content — see the sibling + // `import_one_enqueues_a_transcode_job_when_transcode_is_enabled` + // test for why a fake file no longer exercises this path now that + // `should_enqueue` requires an actual successful probe. + let content = dir.join("Some.Anime.S01E01.mkv"); + let clip = generate_test_clip(&dir, 1920, 1080); + std::fs::rename(&clip, &content).unwrap(); + + let grab = PendingGrab::Episode { + release_id: 1, + media_item_id: 1, + episode_id: 1, + torrent_hash: "deadbeef".to_string(), + series_title: "Some Anime".to_string(), + season_number: 1, + episode_number: 1, + episode_title: None, + root_folder: dest_root.to_string_lossy().to_string(), + }; + + import_one(&conn, &grab, &content, Some(&breadarr_shared::config::TranscodeConfig::default())).unwrap(); + + // Anime is no longer excluded from transcoding — it's routed to its + // own pipeline (`is_anime = 1` on the job row), not skipped. + let (job_count, is_anime): (i64, i64) = conn + .query_row( + "SELECT count(*), max(is_anime) FROM transcode_job", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(job_count, 1); + assert_eq!(is_anime, 1, "job must be tagged is_anime via anime_mapping"); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn import_one_places_a_single_episode_grab_under_its_season_subfolder() { // Regression: a real single-episode grab (Mushoku Tensei S03E02/E03, @@ -3219,7 +4239,7 @@ mod tests { root_folder: dest_root.to_string_lossy().to_string(), }; - let outcome = import_one(&conn, &grab, &content).unwrap(); + let outcome = import_one(&conn, &grab, &content, None).unwrap(); assert!(matches!(outcome, ImportOutcome::Imported { .. })); let dest = dest_root.join("Season 03").join("Some Show - S03E02.mp4"); @@ -3283,7 +4303,7 @@ mod tests { root_folder: dest_root.to_string_lossy().to_string(), }; - let outcome = import_one(&conn, &grab, &content).unwrap(); + let outcome = import_one(&conn, &grab, &content, None).unwrap(); let ImportOutcome::Imported { quality_flagged, .. } = outcome @@ -3493,7 +4513,7 @@ mod tests { root_folder: dest_root.to_string_lossy().to_string(), }; - let outcome = import_one(&conn, &grab, &content).unwrap(); + let outcome = import_one(&conn, &grab, &content, None).unwrap(); assert!(matches!(outcome, ImportOutcome::SkippedAlreadyHaveBetter)); // The existing better file must be untouched, not overwritten. @@ -3501,6 +4521,9 @@ mod tests { std::fs::read(&dest).unwrap(), b"the better file, already imported" ); + // The losing download is cleaned up rather than left as a dangling + // duplicate — nothing seeds it and it lost the comparison. + assert!(!content.exists(), "the losing download should be deleted, not left behind"); let status: String = conn .query_row("SELECT status FROM release WHERE id = 2", [], |r| r.get(0)) @@ -3511,9 +4534,7 @@ mod tests { } #[test] - fn a_strictly_better_release_replaces_the_existing_file_via_a_real_hardlink_swap() { - use std::os::unix::fs::MetadataExt; - + fn a_strictly_better_release_replaces_the_existing_file_via_a_real_move() { let conn = Connection::open_in_memory().unwrap(); crate::db::init(&conn).unwrap(); conn.execute( @@ -3571,18 +4592,14 @@ mod tests { root_folder: dest_root.to_string_lossy().to_string(), }; - let outcome = import_one(&conn, &grab, &content).unwrap(); + let outcome = import_one(&conn, &grab, &content, None).unwrap(); assert!(matches!(outcome, ImportOutcome::Imported { .. })); // The new file's content landed at the shared deterministic path. assert_eq!(std::fs::read(&dest).unwrap(), b"the new, better file"); - // It's a genuine hardlink to the source (same inode), not a copy — - // confirms the old file/row was cleared first so `hard_link` - // itself succeeded instead of falling back to `copy_via_temp_file`. - assert_eq!( - std::fs::metadata(&dest).unwrap().ino(), - std::fs::metadata(&content).unwrap().ino() - ); + // The source is gone — it was moved, not copied or hardlinked + // alongside a surviving original. + assert!(!content.exists(), "source should be moved, not left behind"); // Exactly one episode_file row survives for this movie — the old // one was removed, not left behind as a duplicate alongside the new. @@ -3598,6 +4615,200 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + // Regression test for a real gap found in review: nothing in the schema + // enforces one `episode_file` per episode (the `library_health` + // duplicate-groups report exists precisely because duplicates occur), + // but the old lookup used `query_row`, which silently returns only one + // arbitrary matching row. A second duplicate row was left completely + // untouched — its `upgrade_locked` flag never even consulted, and its + // file never parked-and-cleaned-up alongside the winning replacement. + // This seeds two rows for the same movie and asserts the upgrade sweeps + // both: both old files gone from disk, both old rows gone from the DB, + // exactly one row/file survives. + #[test] + fn an_upgrade_swap_sweeps_every_duplicate_episode_file_row_not_just_one() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder) + VALUES (1, 'movie', 'Some Movie', 2016, 1, 1, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, score, status, torrent_hash, grabbed_at) + VALUES (2, 1, NULL, 'Some Movie 2016 1080p', 1, 'guid-2', 20.0, 'grabbed', 'bbbb', datetime('now'))", + [], + ) + .unwrap(); + + let dir = std::env::temp_dir().join(format!( + "breadarr-duplicate-episode-file-sweep-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let dest_root = dir.join("library"); + std::fs::create_dir_all(&dest_root).unwrap(); + + // Two duplicate rows for the same movie, each with its own real + // file on disk, neither at the deterministic `dest` path (so this + // also exercises the identity-lookup path, not the `dest.exists()` + // fallback). + let stray_a = dir.join("stray-a.mp4"); + std::fs::write(&stray_a, b"duplicate row A's file").unwrap(); + conn.execute( + "INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status) + VALUES (10, NULL, 1, ?1, 20, 'none')", + params![stray_a.to_string_lossy()], + ) + .unwrap(); + let stray_b = dir.join("stray-b.mp4"); + std::fs::write(&stray_b, b"duplicate row B's file").unwrap(); + conn.execute( + "INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status) + VALUES (11, NULL, 1, ?1, 20, 'none')", + params![stray_b.to_string_lossy()], + ) + .unwrap(); + + let content = dir.join("Some.Movie.2016.1080p.mp4"); + std::fs::write(&content, b"the new, better file").unwrap(); + + let grab = PendingGrab::Movie { + release_id: 2, + media_item_id: 1, + torrent_hash: "bbbb".to_string(), + title: "Some Movie".to_string(), + year: Some(2016), + root_folder: dest_root.to_string_lossy().to_string(), + }; + + let outcome = import_one(&conn, &grab, &content, None).unwrap(); + assert!(matches!(outcome, ImportOutcome::Imported { .. })); + + assert!(!stray_a.exists(), "duplicate row A's file must be cleaned up, not orphaned"); + assert!(!stray_b.exists(), "duplicate row B's file must be cleaned up, not orphaned"); + + let remaining: Vec = conn + .prepare("SELECT id FROM episode_file WHERE media_item_id = 1") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + remaining.len(), + 1, + "both duplicate rows must be swept, leaving exactly the new one, got {remaining:?}" + ); + assert!( + !remaining.contains(&10) && !remaining.contains(&11), + "the surviving row must be the newly inserted one, not a stale duplicate" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + // Regression test for a real production bug: a movie's `root_folder` + // drifted (recategorized between library folders — the exact shape of + // a real incident found on "Cars 3", tracked at `.../Kids Movies/...` + // while `root_folder` had since moved to `.../Movies/...`) after it was + // already imported. The old dest-collision check only fired on + // `dest.exists()`, which came back false once the destination path no + // longer matched where the tracked file actually lived — so the score + // comparison was skipped entirely and a fresh grab landed right in + // alongside the untouched original, a real duplicate. This asserts the + // comparison still happens (and the two copies get consolidated into + // one) even when the tracked path and the freshly computed `dest` + // disagree. + #[test] + fn a_drifted_root_folder_does_not_defeat_the_dest_collision_check() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, year, monitored, quality_profile_id, root_folder) + VALUES (1, 'movie', 'Cars 3', 2017, 1, 1, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, score, status, torrent_hash, grabbed_at) + VALUES (1, 1, NULL, 'Cars 3 2017 720p', 1, 'guid-1', 5.0, 'imported', 'aaaa', datetime('now'))", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, score, status, torrent_hash, grabbed_at) + VALUES (2, 1, NULL, 'Cars 3 2017 1080p', 1, 'guid-2', 20.0, 'grabbed', 'bbbb', datetime('now'))", + [], + ) + .unwrap(); + + let dir = std::env::temp_dir().join(format!("breadarr-drift-{}", std::process::id())); + // The tracked file's real home: an old category folder, no longer + // matching the movie's current `root_folder`. + let old_root = dir.join("Kids Movies").join("Cars 3 (2017)"); + std::fs::create_dir_all(&old_root).unwrap(); + let old_path = old_root.join("Cars 3 (2017).mp4"); + std::fs::write(&old_path, b"the old, smaller tracked file").unwrap(); + conn.execute( + "INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes, subtitle_status) + VALUES (1, NULL, 1, ?1, 20, 'none')", + params![old_path.to_string_lossy()], + ) + .unwrap(); + + // The movie's root_folder has since drifted to a different folder — + // `dest` will never equal `old_path`. + let new_root = dir.join("Movies").join("Cars 3 (2017)"); + std::fs::create_dir_all(&new_root).unwrap(); + + let content = dir.join("Cars.3.2017.1080p.mp4"); + std::fs::write(&content, b"the new, better file").unwrap(); + + let grab = PendingGrab::Movie { + release_id: 2, + media_item_id: 1, + torrent_hash: "bbbb".to_string(), + title: "Cars 3".to_string(), + year: Some(2017), + root_folder: new_root.to_string_lossy().to_string(), + }; + + let outcome = import_one(&conn, &grab, &content, None).unwrap(); + assert!(matches!(outcome, ImportOutcome::Imported { .. })); + + let dest = new_root.join("Cars 3 (2017).mp4"); + assert_eq!(std::fs::read(&dest).unwrap(), b"the new, better file"); + // The old tracked file, at its own (different) path, is gone — + // consolidated, not left behind as an untracked duplicate. + assert!( + !old_path.exists(), + "the old tracked file should be cleaned up even though its path never matched dest" + ); + + let file_count: i64 = conn + .query_row( + "SELECT count(*) FROM episode_file WHERE media_item_id = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(file_count, 1, "exactly one tracked file should survive, not two"); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn locates_the_largest_video_file_in_a_directory() { let dir = std::env::temp_dir().join(format!("breadarr-test-dir-{}", std::process::id())); @@ -3669,6 +4880,7 @@ mod tests { 1, &dest_root.to_string_lossy(), &pack_dir, + None, ) .unwrap(); @@ -3745,6 +4957,7 @@ mod tests { 1, &dest_root.to_string_lossy(), &pack_dir, + None, ) .unwrap(); @@ -3760,6 +4973,67 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + #[test] + fn import_season_pack_skips_an_upgrade_locked_episode_even_with_a_lower_scoring_existing_release() { + let conn = seeded_season_pack_conn(2); + let dir = std::env::temp_dir().join(format!( + "breadarr-season-pack-locked-{}", + std::process::id() + )); + let pack_dir = dir.join("pack"); + std::fs::create_dir_all(&pack_dir).unwrap(); + std::fs::write(pack_dir.join("Show.S01E01.mkv"), b"new e01").unwrap(); + std::fs::write(pack_dir.join("Show.S01E02.mkv"), b"new e02").unwrap(); + let dest_root = dir.join("library"); + let season_dir = dest_root.join("Season 01"); + std::fs::create_dir_all(&season_dir).unwrap(); + + // Episode 1's existing release scores *lower* than the incoming + // pack (5.0 vs the pack's 15.0) — on score alone this would be + // overwritten. It's also `upgrade_locked` (a completed local AV1 + // transcode), which must take priority over the score comparison. + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, score, status, torrent_hash, grabbed_at) + VALUES (2, 1, 1, 'Some Show S01E01 1080p', 1, 'guid-2', 5.0, 'imported', 'bbbb', datetime('now'))", + [], + ) + .unwrap(); + let existing = season_dir.join("Some Show - S01E01.mkv"); + std::fs::write(&existing, b"locally-transcoded e01").unwrap(); + conn.execute( + "INSERT INTO episode_file (episode_id, media_item_id, path, size_bytes, subtitle_status, upgrade_locked) + VALUES (1, NULL, ?1, 23, 'none', 1)", + params![existing.to_string_lossy()], + ) + .unwrap(); + + let outcome = import_season_pack( + &conn, + 1, + 1, + "Some Show", + 1, + &dest_root.to_string_lossy(), + &pack_dir, + None, + ) + .unwrap(); + + assert_eq!(outcome.episodes_imported, 1); // only episode 2 + assert_eq!(outcome.episodes_already_had_better, 1); // episode 1 skipped despite the lower score + assert_eq!( + std::fs::read(&existing).unwrap(), + b"locally-transcoded e01", + "the locked, locally-transcoded file must survive untouched regardless of score" + ); + assert!( + !pack_dir.join("Show.S01E01.mkv").exists(), + "the skipped pack file should be deleted, not left behind in the pack directory" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn import_season_pack_fails_when_nothing_in_it_matches_a_tracked_episode() { let conn = seeded_season_pack_conn(2); @@ -3782,9 +5056,202 @@ mod tests { 1, &dest_root.to_string_lossy(), &pack_dir, + None, ); assert!(result.is_err()); std::fs::remove_dir_all(&dir).unwrap(); } + + #[test] + fn import_season_pack_uses_anime_absolute_mapping() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, year, tvdb_id, monitored, quality_profile_id, root_folder) + VALUES (1, 'series', 'Show', 2020, 366263, 1, 1, '/tmp')", + [], + ) + .unwrap(); + // Bookworm-style offsets: cours starting at absolute 1/15/27/37. + for (anidb_id, offset) in [(1, 0), (2, 14), (3, 26), (4, 36)] { + conn.execute( + "INSERT INTO anime_mapping (anidb_id, tvdb_id, season_offset, episode_offset) + VALUES (?1, 366263, 1, ?2)", + params![anidb_id, offset], + ) + .unwrap(); + } + // Mapped landing spot for absolute 15 (offset 14 → S01E01). + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file) + VALUES (1, 1, 1, 1, 1, 0)", + [], + ) + .unwrap(); + // Naive S01E15 — must not be chosen over the mapped episode. + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file) + VALUES (15, 1, 1, 15, 1, 0)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, season_number, raw_title, source_id, guid, score, status, torrent_hash, grabbed_at) + VALUES (1, 1, NULL, 1, '[SubsPlease] Show (01-12)', 1, 'guid-1', 15.0, 'grabbed', 'aaaa', datetime('now'))", + [], + ) + .unwrap(); + + let dir = std::env::temp_dir().join(format!( + "breadarr-season-pack-anime-{}", + std::process::id() + )); + let pack_dir = dir.join("pack"); + std::fs::create_dir_all(&pack_dir).unwrap(); + std::fs::write(pack_dir.join("[SubsPlease] Show - 15.mkv"), b"abs 15").unwrap(); + let dest_root = dir.join("library"); + + let outcome = import_season_pack( + &conn, + 1, + 1, + "Show", + 1, + &dest_root.to_string_lossy(), + &pack_dir, + None, + ) + .unwrap(); + + assert_eq!(outcome.episodes_imported, 1); + assert_eq!(outcome.episodes_unmatched, 0); + + let (has_mapped, has_naive): (i64, i64) = conn + .query_row( + "SELECT + (SELECT has_file FROM episode WHERE id = 1), + (SELECT has_file FROM episode WHERE id = 15)", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(has_mapped, 1, "absolute 15 must land on mapped S01E01"); + assert_eq!(has_naive, 0, "must not treat absolute 15 as S01E15"); + assert!(dest_root.join("Season 01").join("Show - S01E01.mkv").exists()); + assert!(!dest_root.join("Season 01").join("Show - S01E15.mkv").exists()); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn import_season_pack_cleanup_preserves_unmatched_videos() { + let conn = seeded_season_pack_conn(1); + let dir = std::env::temp_dir().join(format!( + "breadarr-season-pack-leftover-{}", + std::process::id() + )); + let pack_dir = dir.join("pack"); + std::fs::create_dir_all(&pack_dir).unwrap(); + std::fs::write(pack_dir.join("Show.S01E01.mkv"), b"matched").unwrap(); + let unmatched = pack_dir.join("Show - 15.mkv"); + std::fs::write(&unmatched, b"did not parse as SxxExx").unwrap(); + std::fs::write(pack_dir.join("release.nfo"), b"nfo").unwrap(); + let dest_root = dir.join("library"); + + let outcome = import_season_pack( + &conn, + 1, + 1, + "Some Show", + 1, + &dest_root.to_string_lossy(), + &pack_dir, + None, + ) + .unwrap(); + assert_eq!(outcome.episodes_imported, 1); + assert_eq!(outcome.episodes_unmatched, 1); + + cleanup_leftover_download_dir(&pack_dir, &dir.to_string_lossy()).unwrap(); + + assert!( + unmatched.exists(), + "unmatched pack video must survive leftover-dir cleanup" + ); + assert!( + dest_root + .join("Season 01") + .join("Some Show - S01E01.mkv") + .exists() + ); + assert!(!pack_dir.join("release.nfo").exists()); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn process_pending_grabs_queues_a_noop_season_pack_upgrade_for_qbit_delete() { + use crate::qbit::TorrentInfo; + + let conn = seeded_season_pack_conn(1); + let dir = std::env::temp_dir().join(format!( + "breadarr-season-pack-noop-upgrade-{}", + std::process::id() + )); + let pack_dir = dir.join("pack"); + std::fs::create_dir_all(&pack_dir).unwrap(); + std::fs::write(pack_dir.join("Show.S01E01.mkv"), b"new e01").unwrap(); + let dest_root = dir.join("library"); + let season = dest_root.join("Season 01"); + std::fs::create_dir_all(&season).unwrap(); + let existing = season.join("Some Show - S01E01.mkv"); + std::fs::write(&existing, b"already-better").unwrap(); + conn.execute( + "INSERT INTO release (id, media_item_id, episode_id, raw_title, source_id, guid, score, status, torrent_hash, grabbed_at) + VALUES (2, 1, 1, 'Some Show S01E01 REMUX', 1, 'guid-2', 50.0, 'imported', 'bbbb', datetime('now'))", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO episode_file (episode_id, media_item_id, path, size_bytes, subtitle_status) + VALUES (1, NULL, ?1, 14, 'none')", + params![existing.to_string_lossy()], + ) + .unwrap(); + conn.execute( + "UPDATE media_item SET root_folder = ?1 WHERE id = 1", + params![dest_root.to_string_lossy()], + ) + .unwrap(); + + let pending = fetch_pending_grabs(&conn).unwrap(); + let torrents = vec![TorrentInfo { + hash: "aaaa".to_string(), + name: "pack".to_string(), + state: "uploading".to_string(), + progress: 1.0, + save_path: dir.to_string_lossy().to_string(), + content_path: pack_dir.to_string_lossy().to_string(), + }]; + + let (stats, hashes) = process_pending_grabs(&conn, &pending, &torrents, "", "", None).unwrap(); + assert_eq!(stats.imported, 0); + assert_eq!(stats.failed, 0); + let status: String = conn + .query_row("SELECT status FROM release WHERE id = 1", [], |r| r.get(0)) + .unwrap(); + assert_eq!(status, "upgraded"); + assert!( + hashes.iter().any(|(h, _)| h == "aaaa"), + "a no-op season-pack upgrade must still be removed from qBit" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } } diff --git a/breadarrd/src/jellyfin.rs b/breadarrd/src/jellyfin.rs index eacb6a0..1c2883a 100644 --- a/breadarrd/src/jellyfin.rs +++ b/breadarrd/src/jellyfin.rs @@ -1,5 +1,6 @@ use anyhow::{bail, Context, Result}; +#[derive(Clone)] pub struct JellyfinClient { base_url: String, api_key: String, @@ -38,4 +39,33 @@ impl JellyfinClient { } Ok(()) } + + /// Counts sessions Jellyfin is actively transcoding for right now (as + /// opposed to direct-play/direct-stream, which cost the GPU nothing) — + /// used to throttle the AV1 batch-transcode worker back so it doesn't + /// contend with a real viewer for the same encode/decode engines. + /// `TranscodingInfo` is only present on a session object while that + /// session is actually transcoding. + pub async fn active_transcode_sessions(&self) -> Result { + let resp = self + .client + .get(format!("{}/Sessions", self.base_url)) + .header("X-Emby-Token", &self.api_key) + .send() + .await + .context("jellyfin sessions request failed")?; + + let status = resp.status(); + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + bail!("jellyfin sessions request failed: status={status} body={body:?}"); + } + + let sessions: Vec = + resp.json().await.context("failed to parse jellyfin sessions response")?; + Ok(sessions + .iter() + .filter(|s| !s["TranscodingInfo"].is_null()) + .count()) + } } diff --git a/breadarrd/src/library_scan.rs b/breadarrd/src/library_scan.rs index 07db0ff..d2d48de 100644 --- a/breadarrd/src/library_scan.rs +++ b/breadarrd/src/library_scan.rs @@ -18,6 +18,7 @@ pub struct ScanReport { pub unmatched: Vec, pub files_linked: usize, pub files_renamed: usize, + pub files_reorganized: usize, } fn get_episode_title(conn: &Connection, episode_id: i64) -> Result> { @@ -434,7 +435,15 @@ pub async fn scan_tv_root( report.unmatched.push(folder_name); continue; }; - let tvdb_id: i64 = best.external_id.parse().unwrap_or_default(); + let Some(tvdb_id) = metadata::parse_external_id(&best.external_id) else { + tracing::warn!( + folder = %folder_name, + external_id = %best.external_id, + "tvdb id was not a positive integer, skipping" + ); + report.unmatched.push(folder_name); + continue; + }; let canonical_year = best.year.or(year); // Normalize the folder itself down to "Title (Year)" — release // tags/resolution/group cruft in the original folder name isn't @@ -525,6 +534,30 @@ pub async fn scan_tv_root( } }; + // Files imported before the season-folder convention existed + // (or moved around by hand) can still be sitting flat in the + // show root — move them under `Season NN` now rather than just + // recording wherever they happen to already be. Same + // filesystem as `series_dir`, so this is a plain rename. + let season_folder = importer::season_dir(&series_dir.to_string_lossy(), season); + let final_path = if final_path.parent() != Some(season_folder.as_path()) { + match std::fs::create_dir_all(&season_folder).and_then(|_| { + let dest = season_folder.join(final_path.file_name().unwrap()); + std::fs::rename(&final_path, &dest).map(|_| dest) + }) { + Ok(dest) => { + report.files_reorganized += 1; + dest + } + Err(e) => { + tracing::warn!(file = %final_path.display(), error = %e, "season-folder move failed, keeping in place"); + final_path + } + } + } else { + final_path + }; + let size = std::fs::metadata(&final_path)?.len(); conn.execute( "INSERT INTO episode_file (episode_id, path, size_bytes, subtitle_status) VALUES (?1, ?2, ?3, 'none')", @@ -704,7 +737,15 @@ pub async fn scan_movie_root( report.unmatched.push(folder_name); continue; }; - let tmdb_id: i64 = best.external_id.parse().unwrap_or_default(); + let Some(tmdb_id) = metadata::parse_external_id(&best.external_id) else { + tracing::warn!( + folder = %folder_name, + external_id = %best.external_id, + "tmdb id was not a positive integer, skipping" + ); + report.unmatched.push(folder_name); + continue; + }; let canonical_year = best.year.or(year); // Normalizes the folder itself down to "Title (Year)" too — release @@ -858,4 +899,12 @@ mod tests { ) ); } + + #[test] + fn parse_external_id_never_yields_zero() { + assert_eq!(metadata::parse_external_id("0"), None); + assert_eq!(metadata::parse_external_id("000"), None); + assert_eq!(metadata::parse_external_id("not-a-number"), None); + assert_eq!(metadata::parse_external_id("550"), Some(550)); + } } diff --git a/breadarrd/src/main.rs b/breadarrd/src/main.rs index e2aa04c..2154730 100644 --- a/breadarrd/src/main.rs +++ b/breadarrd/src/main.rs @@ -11,6 +11,7 @@ mod qbit; mod scheduler; mod scoring; mod sources; +mod transcode; use std::env; @@ -114,9 +115,18 @@ async fn main() -> Result<()> { Some("probe-library") => { return probe_library_cmd(&config).await; } + Some("transcode-library") => { + return transcode_library_cmd(&config).await; + } + Some("retranscode-oversized") => { + return retranscode_oversized_cmd(&config).await; + } Some("verify-library") => { return verify_library_cmd(&config).await; } + Some("relink-orphaned-files") => { + return relink_orphaned_files_cmd(&config).await; + } _ => {} } @@ -138,6 +148,14 @@ async fn run_daemon(config: Config) -> Result<()> { let conn = Connection::open(config.db_path())?; db::init(&conn)?; info!(path = %config.db_path().display(), "database ready"); + match transcode::reset_orphaned_running_jobs(&conn) { + Ok(0) => {} + Ok(n) => info!( + n, + "reset orphaned 'running' transcode jobs left over from a previous crash" + ), + Err(e) => tracing::warn!(error = %e, "failed to reset orphaned transcode jobs"), + } // A second, independent connection for the HTTP API rather than sharing // `background_loop`'s. Both point at the same on-disk (WAL-mode) @@ -153,6 +171,14 @@ async fn run_daemon(config: Config) -> Result<()> { let listener = tokio::net::TcpListener::bind(&config.daemon.listen_addr).await?; info!(addr = %config.daemon.listen_addr, "listening"); + if config.daemon.api_token.is_empty() { + tracing::warn!( + "daemon.api_token is empty; any local process can add/delete/grab via {}", + config.daemon.listen_addr + ); + } else if !config.listen_is_loopback() { + info!("API token auth is required (listen_addr is non-loopback)"); + } let tvdb = if config.tvdb.api_key.is_empty() { None @@ -180,6 +206,10 @@ async fn run_daemon(config: Config) -> Result<()> { ))) }; let (background_tx, background_rx) = tokio::sync::mpsc::channel(8); + // Lifted out of `background_loop` so shutdown can wait for an in-flight + // ffmpeg encode instead of dropping the process the instant SIGTERM + // arrives (systemd's TimeoutStopSec is longer than this wait). + let transcode_busy = std::sync::Arc::new(tokio::sync::Mutex::new(())); let background_conn = std::sync::Arc::new(tokio::sync::Mutex::new(conn)); let state = api::AppState { @@ -209,6 +239,7 @@ async fn run_daemon(config: Config) -> Result<()> { config.clone(), state.cycle_status.clone(), background_rx, + transcode_busy.clone(), ) }); let mut state = state; @@ -253,6 +284,15 @@ async fn run_daemon(config: Config) -> Result<()> { } } + // Wait for an in-flight transcode (it holds this mutex for the whole + // cycle) so ffmpeg can finish, or at least so systemd's longer + // TimeoutStopSec applies instead of an instant drop. + info!("waiting up to 120s for in-flight transcode to finish"); + match tokio::time::timeout(std::time::Duration::from_secs(120), transcode_busy.lock()).await { + Ok(_) => info!("no in-flight transcode (or it finished)"), + Err(_) => tracing::warn!("timed out waiting 120s for in-flight transcode"), + } + Ok(()) } @@ -275,6 +315,7 @@ async fn background_loop( config: Config, cycle_status: std::sync::Arc>, mut background_rx: tokio::sync::mpsc::Receiver, + transcode_busy: std::sync::Arc>, ) { let notifier = notify::Notifier::new(&config.notifications.webhook_url); { @@ -314,6 +355,26 @@ async fn background_loop( ) { error!(error = %e, "failed to register tpb source row"); } + if let Err(e) = conn.execute( + "INSERT OR IGNORE INTO source (id, name, kind, base_url, poll_interval_secs, enabled) + VALUES (4, 'torrents-csv', 'scrape', ?1, ?2, 1)", + rusqlite::params![ + config.sources.torrents_csv_url, + config.sources.search_poll_interval_secs + ], + ) { + error!(error = %e, "failed to register torrents-csv source row"); + } + if let Err(e) = conn.execute( + "INSERT OR IGNORE INTO source (id, name, kind, base_url, poll_interval_secs, enabled) + VALUES (5, 'yts', 'scrape', ?1, ?2, 1)", + rusqlite::params![ + config.sources.yts_api_url, + config.sources.search_poll_interval_secs + ], + ) { + error!(error = %e, "failed to register yts source row"); + } } // A transient failure here (network blip during the one-time model @@ -336,6 +397,21 @@ async fn background_loop( let scrape_source = sources::scrape::ScrapeSource::new(config.sources.torrent_1337x_mirrors.clone()); let tpb_source = sources::tpb::TpbSource::new(config.sources.tpb_api_url.clone()); + let torrents_csv_source = + sources::torrents_csv::TorrentsCsvSource::new(config.sources.torrents_csv_url.clone()); + let yts_source = sources::yts::YtsSource::new(config.sources.yts_api_url.clone()); + let search_sources = scheduler::SearchSources { + tpb: &tpb_source, + tpb_id: 3, + torrents_csv: &torrents_csv_source, + torrents_csv_id: 4, + yts: &yts_source, + yts_id: 5, + scrape: &scrape_source, + scrape_id: 2, + nyaa_search: &nyaa_source, + nyaa_id: 1, + }; let mut grab_ticker = tokio::time::interval(std::time::Duration::from_secs( config.sources.grab_poll_interval_secs, @@ -349,6 +425,18 @@ async fn background_loop( let mut upgrade_ticker = tokio::time::interval(std::time::Duration::from_secs( config.sources.upgrade_poll_interval_secs, )); + // Guards against the transcode ticker itself blocking every other cycle + // (import, search, upgrade, reconcile) for the full multi-minute + // duration of an encode — the ticker below spawns each cycle detached + // rather than awaiting it inline, and this is what stops two spawned + // cycles from running at once if one is still going when the next tick + // fires (a real possibility: files easily take longer to encode than + // `poll_interval_secs`). `claim_pending_jobs`'s own concurrency cap + // already makes overlap *safe*; this just keeps it from happening + // pointlessly. Created in `run_daemon` so shutdown can wait on it. + let mut transcode_ticker = tokio::time::interval(std::time::Duration::from_secs( + config.transcode.poll_interval_secs, + )); // Disk state doesn't change on its own — hourly is plenty to catch a // file deleted/moved by hand without adding meaningful load (one query // per tracked episode file, all local). Deliberately does *not* fire at @@ -366,6 +454,7 @@ async fn background_loop( search_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); upgrade_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); reconcile_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + transcode_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); // Cycle-level backoff on top of the search loop's own per-mirror // cooldowns: a whole cycle failing (source exhausted, or an outright @@ -378,6 +467,9 @@ async fn background_loop( loop { tokio::select! { _ = grab_ticker.tick() => { + if !config.sources.grab_enabled { + continue; + } let result = { let conn = conn.lock().await; scheduler::run_grab_cycle(&conn, &nyaa_source, 1, &mut title_matcher, &qbit, &config.qbit.category).await @@ -404,7 +496,7 @@ async fn background_loop( _ = import_ticker.tick() => { let result = { let conn = conn.lock().await; - importer::run_import_cycle(&conn, &qbit, jellyfin.as_ref(), &config.qbit.category, &config.qbit.container_downloads_path, &config.qbit.host_downloads_path).await + importer::run_import_cycle(&conn, &qbit, jellyfin.as_ref(), &config.qbit.category, &config.qbit.container_downloads_path, &config.qbit.host_downloads_path, config.transcode.enabled.then_some(&config.transcode)).await }; if let (Ok(stats), Some(n)) = (&result, ¬ifier) { if stats.failed > 0 { @@ -444,9 +536,7 @@ async fn background_loop( let conn = conn.lock().await; scheduler::run_search_cycle( &conn, - &tpb_source, 3, - &scrape_source, 2, - &nyaa_source, 1, + &search_sources, &mut title_matcher, &qbit, &config.qbit.category, config.sources.search_budget_per_cycle, @@ -505,9 +595,7 @@ async fn background_loop( let conn = conn.lock().await; scheduler::run_upgrade_cycle( &conn, - &tpb_source, 3, - &scrape_source, 2, - &nyaa_source, 1, + &search_sources, &mut title_matcher, &qbit, &config.qbit.category, config.sources.upgrade_budget_per_cycle, @@ -533,6 +621,38 @@ async fn background_loop( } } } + _ = transcode_ticker.tick() => { + if !config.transcode.enabled { + continue; + } + let Ok(busy_permit) = transcode_busy.clone().try_lock_owned() else { + // A previous cycle is still running (this file's + // encode took longer than one poll interval) — skip + // this tick rather than spawning a second overlapping + // one; the still-running cycle will pick up any newly + // pending jobs on its own next iteration anyway. + continue; + }; + let conn = conn.clone(); + let cfg = config.transcode.clone(); + let jellyfin = jellyfin.clone(); + let cycle_status = cycle_status.clone(); + tokio::spawn(async move { + let _permit = busy_permit; + let result = transcode::run_cycle(conn, cfg, jellyfin.as_ref()).await; + let record = match &result { + Ok(stats) => { + info!(?stats, "transcode cycle complete"); + api::CycleRecord { at: chrono::Utc::now(), ok: true, detail: format!("{stats:?}") } + } + Err(e) => { + error!(error = %e, "transcode cycle failed"); + api::CycleRecord { at: chrono::Utc::now(), ok: false, detail: e.to_string() } + } + }; + cycle_status.lock().expect("cycle_status poisoned").last_transcode = Some(record); + }); + } _ = reconcile_ticker.tick() => { let result = { let conn = conn.lock().await; @@ -569,10 +689,11 @@ async fn background_loop( // repaired (a rename or an in-place transcode) gets re-probed // immediately rather than waiting for its own turn a full // interval later. - match { + let probe_result = { let conn = conn.lock().await; importer::probe_library(&conn) - } { + }; + match probe_result { Ok(report) if report.probed > 0 || report.failed > 0 => { info!(?report, "media probe sweep complete"); } @@ -589,9 +710,7 @@ async fn background_loop( Ok(targets) => scheduler::execute_search_targets( &conn, &targets, - &tpb_source, 3, - &scrape_source, 2, - &nyaa_source, 1, + &search_sources, &mut title_matcher, &qbit, &config.qbit.category, ).await, @@ -615,9 +734,7 @@ async fn background_loop( &conn, media_item_id, episode_id, - &tpb_source, 3, - &scrape_source, 2, - &nyaa_source, 1, + &search_sources, ).await }; let _ = reply.send(result); @@ -846,6 +963,7 @@ async fn debug_import_cycle(config: &Config) -> Result<()> { &config.qbit.category, &config.qbit.container_downloads_path, &config.qbit.host_downloads_path, + config.transcode.enabled.then_some(&config.transcode), ) .await?; println!("{stats:?}"); @@ -927,11 +1045,12 @@ async fn debug_scan_tv(config: &Config, path: &str) -> Result<()> { .await?; println!( - "matched={} unmatched={} files_linked={} files_renamed={}", + "matched={} unmatched={} files_linked={} files_renamed={} files_reorganized={}", report.matched.len(), report.unmatched.len(), report.files_linked, - report.files_renamed + report.files_renamed, + report.files_reorganized ); if !report.unmatched.is_empty() { println!("unmatched:"); @@ -974,11 +1093,12 @@ async fn debug_scan_movies(config: &Config, path: &str) -> Result<()> { .await?; println!( - "matched={} unmatched={} files_linked={} files_renamed={}", + "matched={} unmatched={} files_linked={} files_renamed={} files_reorganized={}", report.matched.len(), report.unmatched.len(), report.files_linked, - report.files_renamed + report.files_renamed, + report.files_reorganized ); if !report.unmatched.is_empty() { println!("unmatched:"); @@ -1040,6 +1160,22 @@ async fn debug_search_show(config: &Config, title: &str) -> Result<()> { config.sources.search_poll_interval_secs ], )?; + conn.execute( + "INSERT OR IGNORE INTO source (id, name, kind, base_url, poll_interval_secs, enabled) + VALUES (4, 'torrents-csv', 'scrape', ?1, ?2, 1)", + rusqlite::params![ + config.sources.torrents_csv_url, + config.sources.search_poll_interval_secs + ], + )?; + conn.execute( + "INSERT OR IGNORE INTO source (id, name, kind, base_url, poll_interval_secs, enabled) + VALUES (5, 'yts', 'scrape', ?1, ?2, 1)", + rusqlite::params![ + config.sources.yts_api_url, + config.sources.search_poll_interval_secs + ], + )?; let qbit = QbitClient::new(config.qbit.base_url.clone())?; if !config.qbit.username.is_empty() { @@ -1051,6 +1187,21 @@ async fn debug_search_show(config: &Config, title: &str) -> Result<()> { let scrape_source = sources::scrape::ScrapeSource::new(config.sources.torrent_1337x_mirrors.clone()); let nyaa_source = sources::rss::RssSource::new(config.sources.nyaa_rss_url.clone()); + let torrents_csv_source = + sources::torrents_csv::TorrentsCsvSource::new(config.sources.torrents_csv_url.clone()); + let yts_source = sources::yts::YtsSource::new(config.sources.yts_api_url.clone()); + let search_sources = scheduler::SearchSources { + tpb: &tpb_source, + tpb_id: 3, + torrents_csv: &torrents_csv_source, + torrents_csv_id: 4, + yts: &yts_source, + yts_id: 5, + scrape: &scrape_source, + scrape_id: 2, + nyaa_search: &nyaa_source, + nyaa_id: 1, + }; let targets = scheduler::enumerate_search_targets_for_media_item(&conn, media_item_id)?; println!("{} missing episode(s)/movie for {title:?}", targets.len()); @@ -1058,12 +1209,7 @@ async fn debug_search_show(config: &Config, title: &str) -> Result<()> { let stats = scheduler::execute_search_targets( &conn, &targets, - &tpb_source, - 3, - &scrape_source, - 2, - &nyaa_source, - 1, + &search_sources, &mut title_matcher, &qbit, &config.qbit.category, @@ -1148,6 +1294,44 @@ async fn remux_backlog_cmd(config: &Config) -> Result<()> { /// doesn't stall the grab/import/search cycles; this command is for /// immediately backfilling that same backlog by hand instead of waiting for /// it to trickle in over several hours. +/// One-time reconciliation for episodes whose `episode_file` association +/// went missing (almost certainly the earlier DB-recovery incident) despite +/// their real file still sitting exactly where breadarr's own importer +/// would have put it — see `importer::find_relinkable_episode_files`'s doc +/// comment for the full story. Purely additive: reports what it found +/// before touching anything, never moves/deletes/overwrites a single file, +/// and flags ambiguous matches for a human to look at rather than guessing. +async fn relink_orphaned_files_cmd(config: &Config) -> Result<()> { + if let Some(parent) = config.db_path().parent() { + std::fs::create_dir_all(parent)?; + } + let conn = Connection::open(config.db_path())?; + db::init(&conn)?; + + let (candidates, ambiguous) = importer::find_relinkable_episode_files(&conn)?; + println!( + "found {} orphaned episode file(s) to relink, {} ambiguous case(s) left for manual review", + candidates.len(), + ambiguous.len() + ); + for c in &candidates { + println!( + " relink: {} S{:02}E{:02} -> {}", + c.series_title, + c.season_number, + c.episode_number, + c.path.display() + ); + } + for a in &ambiguous { + println!(" ambiguous, skipped: {a}"); + } + + let linked = importer::relink_episode_files(&conn, &candidates)?; + println!("done: {linked} episode(s) relinked"); + Ok(()) +} + async fn probe_library_cmd(config: &Config) -> Result<()> { if let Some(parent) = config.db_path().parent() { std::fs::create_dir_all(parent)?; @@ -1170,6 +1354,128 @@ async fn probe_library_cmd(config: &Config) -> Result<()> { Ok(()) } +/// One-time backfill: enqueues every existing-library file eligible for +/// AV1 transcoding (see `transcode::find_backlog_candidates` for the exact +/// eligibility rules — not already AV1, not anime, not HDR/2160p+ per the +/// first pass's scope) as a `transcode_job` row, then drives the same +/// worker loop the daemon's steady-state ticker uses +/// (`transcode::run_cycle`) until nothing is left pending. Shares that one +/// code path deliberately — there is exactly one place that actually runs +/// an encode, whether triggered by a backlog sweep or a fresh grab. +// Deliberately does NOT call `transcode::reset_orphaned_running_jobs` on +// startup the way `run_daemon` does — from this CLI's vantage point a +// `running` row could belong to the actual daemon's own ticker legitimately +// working on it right now (see the safety writeup on `claim_pending_jobs`: +// running this backfill alongside a live daemon with transcode enabled is +// intentionally supported, the two now share one atomic, count-aware +// concurrency cap), and there's no reliable way to tell that apart from a +// genuinely orphaned row from a ago-crashed run of this same command. +// If *this* command itself is killed mid-run, its claimed jobs stay +// `running` until the daemon is next restarted (which does its own reset). +async fn transcode_library_cmd(config: &Config) -> Result<()> { + if !config.transcode.enabled { + bail!("transcode.enabled is false in config — enable it before running a backfill"); + } + if let Some(parent) = config.db_path().parent() { + std::fs::create_dir_all(parent)?; + } + let conn = Connection::open(config.db_path())?; + db::init(&conn)?; + + let candidates = transcode::find_backlog_candidates(&conn, &config.transcode)?; + println!( + "found {} backlog candidate(s), highest-bitrate first", + candidates.len() + ); + for candidate in &candidates { + transcode::enqueue( + &conn, + candidate.episode_file_id, + candidate.video_codec.as_deref(), + candidate.size_bytes, + candidate.is_anime, + false, + )?; + } + + drive_transcode_queue_to_completion(conn, config).await +} + +/// Re-transcodes files a real rate-control bug left larger than their +/// original (already AV1, so `transcode-library`'s own backfill query +/// excludes them) — see `transcode::find_oversized_av1_candidates`'s doc +/// comment for the full story. Enqueues with `force_reencode: true`, the +/// only thing that lets `encode_and_verify` attempt a real re-encode of a +/// file that's already AV1 rather than treating it as nothing-to-do. +async fn retranscode_oversized_cmd(config: &Config) -> Result<()> { + if !config.transcode.enabled { + bail!("transcode.enabled is false in config — enable it before running this"); + } + if let Some(parent) = config.db_path().parent() { + std::fs::create_dir_all(parent)?; + } + let conn = Connection::open(config.db_path())?; + db::init(&conn)?; + + let candidates = transcode::find_oversized_av1_candidates(&conn, &config.transcode)?; + println!( + "found {} oversized AV1 file(s) to re-transcode, worst offenders first", + candidates.len() + ); + for candidate in &candidates { + transcode::enqueue( + &conn, + candidate.episode_file_id, + candidate.video_codec.as_deref(), + candidate.size_bytes, + candidate.is_anime, + true, + )?; + } + + drive_transcode_queue_to_completion(conn, config).await +} + +/// Shared by `transcode_library_cmd` and `retranscode_oversized_cmd`: drains +/// whatever's now `pending` in `transcode_job` via repeated `run_cycle` +/// calls until nothing is left, printing a running total. The only +/// difference between the two commands is which candidates got enqueued +/// (and with what `force_reencode` value) before this runs — there's +/// exactly one place that actually drives the worker loop to completion. +async fn drive_transcode_queue_to_completion(conn: Connection, config: &Config) -> Result<()> { + let jellyfin = if config.jellyfin.base_url.is_empty() { + None + } else { + Some(JellyfinClient::new( + config.jellyfin.base_url.clone(), + config.jellyfin.api_key.clone(), + )) + }; + + let conn = std::sync::Arc::new(tokio::sync::Mutex::new(conn)); + let mut total_succeeded = 0usize; + let mut total_skipped = 0usize; + let mut total_failed = 0usize; + let mut total_bytes_saved: i64 = 0; + loop { + let stats = + transcode::run_cycle(conn.clone(), config.transcode.clone(), jellyfin.as_ref()).await?; + if stats.attempted == 0 { + break; + } + total_succeeded += stats.succeeded; + total_skipped += stats.skipped; + total_failed += stats.failed; + total_bytes_saved += stats.bytes_saved; + println!("batch: {stats:?}"); + } + println!( + "done: {total_succeeded} succeeded, {total_skipped} skipped (not beneficial), {total_failed} failed, {:.1} GB saved total", + total_bytes_saved as f64 / 1_073_741_824.0 + ); + Ok(()) +} + /// Runs `importer::verify_library` — the expensive full-decode corruption /// check (`ffmpeg -xerror`, actually decoding every frame) against every /// header-probed-ok file that hasn't been decode-verified yet. Unlike diff --git a/breadarrd/src/matcher/mod.rs b/breadarrd/src/matcher/mod.rs index fc82c08..0b24485 100644 --- a/breadarrd/src/matcher/mod.rs +++ b/breadarrd/src/matcher/mod.rs @@ -4,14 +4,20 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; -use rusqlite::{params, Connection}; +use rusqlite::{params, Connection, OptionalExtension}; use embed::{cosine_similarity, OrtEmbedder}; +// Pinned to Xenova/all-MiniLM-L6-v2 @ 751bff37182d3f1213fa05d7196b954e230abad9 +// (current `main` as of this change) — not the floating `main` branch. const MODEL_URL: &str = - "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx"; + "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/onnx/model.onnx"; const TOKENIZER_URL: &str = - "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/tokenizer.json"; + "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/tokenizer.json"; +// Official LFS sha256 of onnx/model.onnx at that commit. +const MODEL_SHA256: &str = "759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e"; +// sha256 of tokenizer.json at the same revision (not LFS; hashed from the published file). +const TOKENIZER_SHA256: &str = "da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0"; /// Downloads the embedding model into `model_dir` if it isn't already /// there — keeps setup to "run the daemon," no separate fetch step, in @@ -32,19 +38,21 @@ pub async fn ensure_model(model_dir: &Path) -> Result<(PathBuf, PathBuf)> { if !model_path.exists() { tracing::info!("downloading title-matching model (~90MB, one-time)"); - download(MODEL_URL, model_path.clone()).await?; + download(MODEL_URL, model_path.clone(), MODEL_SHA256).await?; } if !tokenizer_path.exists() { - download(TOKENIZER_URL, tokenizer_path.clone()).await?; + download(TOKENIZER_URL, tokenizer_path.clone(), TOKENIZER_SHA256).await?; } Ok((model_path, tokenizer_path)) } -async fn download(url: &'static str, dest: PathBuf) -> Result<()> { - tokio::task::spawn_blocking(move || bread_onnx::download::ensure_file(url, &dest, None)) - .await - .context("download task panicked")??; +async fn download(url: &'static str, dest: PathBuf, sha256: &'static str) -> Result<()> { + tokio::task::spawn_blocking(move || { + bread_onnx::download::ensure_file(url, &dest, Some(sha256)) + }) + .await + .context("download task panicked")??; Ok(()) } @@ -96,6 +104,17 @@ impl TitleMatcher { }) } + /// Caches by text — appropriate for candidate-side text only (library + /// `media_item` titles/aliases, or a metadata provider's small, + /// bounded result list), which the same daemon process legitimately + /// re-embeds across many calls. Deliberately never used for the query + /// side (see `embed_query`): `TitleMatcher` lives for the whole life of + /// `background_loop`, which never returns, and the query is a + /// freshly-parsed release title from every RSS item and search result + /// the daemon ever sees — almost never repeated verbatim. Caching those + /// too grew this `HashMap` without bound for the process's entire + /// (months-long) lifetime, a slow but real leak on a box also running + /// several GB of concurrent GPU/CPU transcode work. fn embed_cached(&mut self, text: &str) -> Result> { if let Some(v) = self.cache.get(text) { return Ok(v.clone()); @@ -105,6 +124,12 @@ impl TitleMatcher { Ok(v) } + /// The query-side counterpart to `embed_cached` — same embedding, never + /// stored in `self.cache`. See `embed_cached`'s doc comment for why. + fn embed_query(&mut self, text: &str) -> Result> { + self.embedder.embed(text) + } + /// Given a flat list of candidate texts (e.g. every search result's /// name plus its aliases, flattened with an index back to which result /// each one belongs to), returns the index of whichever candidate text @@ -121,7 +146,7 @@ impl TitleMatcher { query: &str, candidates: &[(usize, String)], ) -> Result> { - let query_emb = self.embed_cached(query)?; + let query_emb = self.embed_query(query)?; let mut best: Option<(usize, f32)> = None; for (owner_index, text) in candidates { let emb = self.embed_cached(text)?; @@ -137,7 +162,7 @@ impl TitleMatcher { /// aliases, returning the single best match and whether it clears the /// auto-match bar or needs a human to confirm it in the review queue. pub fn match_title(&mut self, conn: &Connection, query: &str) -> Result { - let query_emb = self.embed_cached(query)?; + let query_emb = self.embed_query(query)?; let mut stmt = conn.prepare( "SELECT id, title FROM media_item WHERE monitored = 1 @@ -233,6 +258,36 @@ pub fn queue_for_review( link: Option<&str>, source_id: Option, ) -> Result { + // Same release + same library row already sitting in pending: a + // no-op instead of another TUI row. Upgrade-search used to re-list + // the same movie 7–11 times (verified live on hestia). + if let Some(existing) = conn + .query_row( + "SELECT id FROM review_queue + WHERE status = 'pending' + AND candidate_media_item_id = ?1 + AND raw_release_title = ?2", + params![candidate.media_item_id, raw_release_title], + |row| row.get(0), + ) + .optional()? + { + return Ok(existing); + } + if let (Some(link), Some(source_id)) = (link, source_id) { + if let Some(existing) = conn + .query_row( + "SELECT id FROM review_queue + WHERE status = 'pending' AND source_id = ?1 AND link = ?2", + params![source_id, link], + |row| row.get(0), + ) + .optional()? + { + return Ok(existing); + } + } + conn.execute( "INSERT INTO review_queue (raw_release_title, candidate_media_item_id, confidence, link, source_id, status, created_at) VALUES (?1, ?2, ?3, ?4, ?5, 'pending', datetime('now'))", @@ -306,4 +361,81 @@ mod tests { 0.0 ); } + + fn review_queue_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, monitored, quality_profile_id, root_folder) + VALUES (1, 'movie', 'Cars 3', 1, 2, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'tpb', 'scrape', 'http://x')", + [], + ) + .unwrap(); + conn + } + + fn review_candidate() -> MatchCandidate { + MatchCandidate { + media_item_id: 1, + matched_text: "Cars 3".into(), + confidence: 0.7, + } + } + + #[test] + fn queue_for_review_is_idempotent_for_the_same_pending_title() { + let conn = review_queue_conn(); + let c = review_candidate(); + let first = queue_for_review(&conn, "Cars.3.2017.1080p", &c, Some("magnet:a"), Some(1)).unwrap(); + let second = + queue_for_review(&conn, "Cars.3.2017.1080p", &c, Some("magnet:a"), Some(1)).unwrap(); + assert_eq!(first, second); + let n: i64 = conn + .query_row( + "SELECT count(*) FROM review_queue WHERE status = 'pending'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(n, 1); + } + + #[test] + fn queue_for_review_is_idempotent_for_the_same_pending_link() { + let conn = review_queue_conn(); + let c = review_candidate(); + let first = + queue_for_review(&conn, "Cars.3.2017.1080p", &c, Some("magnet:same"), Some(1)).unwrap(); + let second = queue_for_review( + &conn, + "Cars.3.2017.2160p.UHD", + &c, + Some("magnet:same"), + Some(1), + ) + .unwrap(); + assert_eq!(first, second); + let n: i64 = conn + .query_row( + "SELECT count(*) FROM review_queue WHERE status = 'pending'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(n, 1); + } + + #[test] + fn queue_for_review_still_accepts_a_different_title_and_link() { + let conn = review_queue_conn(); + let c = review_candidate(); + let a = queue_for_review(&conn, "Cars.3.2017.1080p", &c, Some("magnet:a"), Some(1)).unwrap(); + let b = queue_for_review(&conn, "Cars.3.2017.2160p", &c, Some("magnet:b"), Some(1)).unwrap(); + assert_ne!(a, b); + } } diff --git a/breadarrd/src/metadata/mod.rs b/breadarrd/src/metadata/mod.rs index e5c4b25..5e262f9 100644 --- a/breadarrd/src/metadata/mod.rs +++ b/breadarrd/src/metadata/mod.rs @@ -5,7 +5,7 @@ pub mod tvdb; use std::collections::HashSet; use anyhow::Result; -use rusqlite::{params, Connection}; +use rusqlite::{params, Connection, OptionalExtension}; #[derive(Debug, Clone, PartialEq)] pub struct SeriesSearchResult { @@ -41,6 +41,7 @@ pub struct EpisodeInfo { /// `MutexGuard` across an `.await` point — this convenience wrapper is only /// safe for callers with an owned, unshared `Connection` (e.g. the debug /// CLI commands). +#[allow(clippy::too_many_arguments)] pub async fn add_series( conn: &Connection, tvdb: &tvdb::TvdbClient, @@ -64,6 +65,21 @@ pub async fn add_series( ) } +/// TVDB/TMDB ids are positive integers. `"0"` and non-numeric strings +/// must not be stored — `0` would collide under the unique external-id +/// indexes the same way a parse-failure `unwrap_or_default()` used to. +pub(crate) fn parse_external_id(raw: &str) -> Option { + raw.parse().ok().filter(|&id| id > 0) +} + +fn existing_media_item_id(conn: &Connection, column: &str, value: i64) -> Result> { + debug_assert!(column == "tvdb_id" || column == "tmdb_id"); + let sql = format!("SELECT id FROM media_item WHERE {column} = ?1 ORDER BY id ASC LIMIT 1"); + conn.query_row(&sql, params![value], |row| row.get(0)) + .optional() + .map_err(Into::into) +} + #[allow(clippy::too_many_arguments)] pub fn insert_series( conn: &Connection, @@ -75,21 +91,23 @@ pub fn insert_series( quality_profile_id: i64, episodes: &[EpisodeInfo], ) -> Result { - conn.execute( + let tvdb_id = parse_external_id(tvdb_series_id); + if let Some(tvdb_id) = tvdb_id { + if let Some(existing) = existing_media_item_id(conn, "tvdb_id", tvdb_id)? { + return Ok(existing); + } + } + + let tx = conn.unchecked_transaction()?; + tx.execute( "INSERT INTO media_item (kind, title, year, tvdb_id, monitored, quality_profile_id, root_folder) VALUES ('series', ?1, ?2, ?3, 1, ?4, ?5)", - params![ - title, - year, - tvdb_series_id.parse::().ok(), - quality_profile_id, - root_folder - ], + params![title, year, tvdb_id, quality_profile_id, root_folder], )?; - let media_item_id = conn.last_insert_rowid(); + let media_item_id = tx.last_insert_rowid(); for alias in aliases { - conn.execute( + tx.execute( "INSERT INTO alias (media_item_id, text, source) VALUES (?1, ?2, 'tvdb')", params![media_item_id, alias], )?; @@ -97,27 +115,39 @@ pub fn insert_series( let mut seasons_seen = HashSet::new(); for ep in episodes { + // TVDB's "season 0" is a catch-all for specials — recaps, shorts, + // and often a tie-in movie that's frequently already tracked as + // its own separate `media_item` (verified live: Chainsaw Man's + // season 0 included "Chainsaw Man – The Movie: Reze Arc", which + // already exists as its own movie entry). Monitoring these by + // default means the library never actually "completes" and the + // missing-episode count is inflated with content the user was + // never trying to acquire as episodes in the first place. Regular + // seasons keep the previous default of monitored. + let monitored = i64::from(ep.season_number != 0); if seasons_seen.insert(ep.season_number) { - conn.execute( - "INSERT OR IGNORE INTO season (media_item_id, season_number, monitored) VALUES (?1, ?2, 1)", - params![media_item_id, ep.season_number], + tx.execute( + "INSERT OR IGNORE INTO season (media_item_id, season_number, monitored) VALUES (?1, ?2, ?3)", + params![media_item_id, ep.season_number, monitored], )?; } - conn.execute( + tx.execute( "INSERT OR IGNORE INTO episode (media_item_id, season_number, episode_number, absolute_number, title, air_date, monitored, has_file) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, 0)", + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 0)", params![ media_item_id, ep.season_number, ep.episode_number, ep.absolute_number, ep.title, - ep.air_date + ep.air_date, + monitored, ], )?; } + tx.commit()?; Ok(media_item_id) } @@ -133,16 +163,177 @@ pub fn insert_movie( root_folder: &str, quality_profile_id: i64, ) -> Result { + let tmdb_id = parse_external_id(tmdb_movie_id); + if let Some(tmdb_id) = tmdb_id { + if let Some(existing) = existing_media_item_id(conn, "tmdb_id", tmdb_id)? { + return Ok(existing); + } + } + conn.execute( "INSERT INTO media_item (kind, title, year, tmdb_id, monitored, quality_profile_id, root_folder) VALUES ('movie', ?1, ?2, ?3, 1, ?4, ?5)", - params![ - title, - year, - tmdb_movie_id.parse::().ok(), - quality_profile_id, - root_folder - ], + params![title, year, tmdb_id, quality_profile_id, root_folder], )?; Ok(conn.last_insert_rowid()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::db; + + fn conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + db::init(&conn).unwrap(); + conn + } + + fn ep(season: u32, episode: u32) -> EpisodeInfo { + EpisodeInfo { + season_number: season, + episode_number: episode, + absolute_number: None, + title: Some(format!("E{episode}")), + air_date: Some("2020-01-01".into()), + } + } + + #[test] + fn parse_external_id_rejects_zero_and_garbage() { + assert_eq!(parse_external_id("550"), Some(550)); + assert_eq!(parse_external_id("0"), None); + assert_eq!(parse_external_id("-12"), None); + assert_eq!(parse_external_id("not-a-number"), None); + assert_eq!(parse_external_id(""), None); + } + + #[test] + fn insert_series_writes_seasons_and_episodes_together() { + let conn = conn(); + let id = insert_series( + &conn, + "12345", + "Show", + Some(2020), + &["Alias".into()], + "/tv/Show", + 1, + &[ep(1, 1), ep(1, 2), ep(2, 1)], + ) + .unwrap(); + + let seasons: i64 = conn + .query_row( + "SELECT count(*) FROM season WHERE media_item_id = ?1", + [id], + |r| r.get(0), + ) + .unwrap(); + let episodes: i64 = conn + .query_row( + "SELECT count(*) FROM episode WHERE media_item_id = ?1", + [id], + |r| r.get(0), + ) + .unwrap(); + let aliases: i64 = conn + .query_row( + "SELECT count(*) FROM alias WHERE media_item_id = ?1", + [id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(seasons, 2); + assert_eq!(episodes, 3); + assert_eq!(aliases, 1); + } + + #[test] + fn insert_series_rolls_back_when_an_episode_insert_fails() { + let conn = conn(); + conn.execute( + "CREATE TRIGGER fail_episode BEFORE INSERT ON episode + BEGIN SELECT RAISE(ABORT, 'boom'); END", + [], + ) + .unwrap(); + + let err = insert_series( + &conn, + "99", + "Show", + None, + &["A".into()], + "/tv/Show", + 1, + &[ep(1, 1)], + ); + assert!(err.is_err()); + + let items: i64 = conn + .query_row("SELECT count(*) FROM media_item", [], |r| r.get(0)) + .unwrap(); + let seasons: i64 = conn + .query_row("SELECT count(*) FROM season", [], |r| r.get(0)) + .unwrap(); + let aliases: i64 = conn + .query_row("SELECT count(*) FROM alias", [], |r| r.get(0)) + .unwrap(); + assert_eq!(items, 0); + assert_eq!(seasons, 0); + assert_eq!(aliases, 0); + } + + #[test] + fn insert_series_is_idempotent_on_tvdb_id() { + let conn = conn(); + let first = insert_series( + &conn, + "12345", + "Show", + None, + &[], + "/tv/Show", + 1, + &[ep(1, 1)], + ) + .unwrap(); + let second = insert_series( + &conn, + "12345", + "Other Title", + None, + &[], + "/tv/Other", + 1, + &[], + ) + .unwrap(); + assert_eq!(first, second); + let count: i64 = conn + .query_row( + "SELECT count(*) FROM media_item WHERE tvdb_id = 12345", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn insert_movie_is_idempotent_on_tmdb_id() { + let conn = conn(); + let first = insert_movie(&conn, "550", "Fight Club", Some(1999), "/movies", 2).unwrap(); + let second = insert_movie(&conn, "550", "Fight Club 2", Some(2000), "/movies", 2).unwrap(); + assert_eq!(first, second); + let count: i64 = conn + .query_row( + "SELECT count(*) FROM media_item WHERE tmdb_id = 550", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(count, 1); + } +} diff --git a/breadarrd/src/metadata/tmdb.rs b/breadarrd/src/metadata/tmdb.rs index f5bc496..43e2af6 100644 --- a/breadarrd/src/metadata/tmdb.rs +++ b/breadarrd/src/metadata/tmdb.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result}; use serde::Deserialize; -use super::{EpisodeInfo, MovieSearchResult, SeriesSearchResult}; +use super::MovieSearchResult; pub struct TmdbClient { bearer_token: String, @@ -22,44 +22,6 @@ impl TmdbClient { } } - pub async fn search_tv(&self, query: &str) -> Result> { - #[derive(Deserialize)] - struct SearchResponse { - results: Vec, - } - #[derive(Deserialize)] - struct TvItem { - id: u64, - name: String, - first_air_date: Option, - } - - let resp: SearchResponse = self - .client - .get("https://api.themoviedb.org/3/search/tv") - .bearer_auth(&self.bearer_token) - .query(&[("query", query)]) - .send() - .await - .context("tmdb tv search request failed")? - .error_for_status() - .context("tmdb tv search returned an error status")? - .json() - .await - .context("tmdb tv search response was not valid JSON")?; - - Ok(resp - .results - .into_iter() - .map(|item| SeriesSearchResult { - external_id: item.id.to_string(), - name: item.name, - year: year_from_date(item.first_air_date.as_deref()), - aliases: Vec::new(), - }) - .collect()) - } - pub async fn search_movie(&self, query: &str) -> Result> { #[derive(Deserialize)] struct SearchResponse { @@ -96,48 +58,6 @@ impl TmdbClient { }) .collect()) } - - pub async fn tv_season_episodes(&self, tv_id: u64, season: u32) -> Result> { - #[derive(Deserialize)] - struct SeasonResponse { - #[serde(default)] - episodes: Vec, - } - #[derive(Deserialize)] - struct EpisodeItem { - season_number: u32, - episode_number: u32, - name: Option, - air_date: Option, - } - - let resp: SeasonResponse = self - .client - .get(format!( - "https://api.themoviedb.org/3/tv/{tv_id}/season/{season}" - )) - .bearer_auth(&self.bearer_token) - .send() - .await - .context("tmdb season request failed")? - .error_for_status() - .context("tmdb season returned an error status")? - .json() - .await - .context("tmdb season response was not valid JSON")?; - - Ok(resp - .episodes - .into_iter() - .map(|e| EpisodeInfo { - season_number: e.season_number, - episode_number: e.episode_number, - absolute_number: None, - title: e.name, - air_date: e.air_date, - }) - .collect()) - } } fn year_from_date(date: Option<&str>) -> Option { diff --git a/breadarrd/src/metadata/tvdb.rs b/breadarrd/src/metadata/tvdb.rs index 914022e..a9cd5ff 100644 --- a/breadarrd/src/metadata/tvdb.rs +++ b/breadarrd/src/metadata/tvdb.rs @@ -110,8 +110,6 @@ impl TvdbClient { } pub async fn episodes(&self, series_id: &str) -> Result> { - let token = self.token().await?; - #[derive(Deserialize)] struct EpisodesResponse { data: EpisodesData, @@ -131,20 +129,47 @@ impl TvdbClient { aired: Option, } - let resp: EpisodesResponse = self - .client - .get(format!( - "https://api4.thetvdb.com/v4/series/{series_id}/episodes/default" - )) - .bearer_auth(token) - .send() - .await - .context("tvdb episodes request failed")? - .error_for_status() - .context("tvdb episodes returned an error status")? - .json() - .await - .context("tvdb episodes response was not valid JSON")?; + // TVDB's plain `/episodes/default` returns names in the show's + // original airing language — for a lot of anime that's Japanese, + // with no English name at all (verified live: entire shows came + // back Japanese-only). `/episodes/default/eng` is the same episode + // list (same numbering/air-date fields) but with TVDB's own + // crowd-sourced English translation substituted in for `name` + // wherever one exists — a real translation, not a guess, so it's + // tried first and only falls back to the original-language + // endpoint if TVDB has no English data for this show at all. + let token = self.token().await?; + let fetch = |url: String| { + let client = self.client.clone(); + let token = token.clone(); + async move { + client + .get(url) + .bearer_auth(token) + .send() + .await + .context("tvdb episodes request failed")? + .error_for_status() + .context("tvdb episodes returned an error status")? + .json::() + .await + .context("tvdb episodes response was not valid JSON") + } + }; + + let resp = match fetch(format!( + "https://api4.thetvdb.com/v4/series/{series_id}/episodes/default/eng" + )) + .await + { + Ok(resp) => resp, + Err(_) => { + fetch(format!( + "https://api4.thetvdb.com/v4/series/{series_id}/episodes/default" + )) + .await? + } + }; Ok(resp .data diff --git a/breadarrd/src/notify.rs b/breadarrd/src/notify.rs index 470d0d7..a3c2b59 100644 --- a/breadarrd/src/notify.rs +++ b/breadarrd/src/notify.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use serde::Serialize; #[derive(Serialize)] diff --git a/breadarrd/src/parser/mod.rs b/breadarrd/src/parser/mod.rs index b6f9c36..0d9a263 100644 --- a/breadarrd/src/parser/mod.rs +++ b/breadarrd/src/parser/mod.rs @@ -34,6 +34,7 @@ pub struct ParsedRelease { pub bit_depth: Option, pub container: Option, pub is_repack: bool, + pub has_hdr: bool, } pub fn parse(raw_title: &str) -> ParsedRelease { @@ -53,6 +54,7 @@ pub fn parse(raw_title: &str) -> ParsedRelease { let codec = tokens::extract_codec(&work); let bit_depth = tokens::extract_bit_depth(&work); let is_repack = tokens::REPACK_RE.is_match(&work); + let has_hdr = tokens::extract_hdr(&work); let year = tokens::extract_year(&work); let (season, episode, absolute_episode, title_span_end) = tokens::extract_episode_info(&work); @@ -72,6 +74,7 @@ pub fn parse(raw_title: &str) -> ParsedRelease { bit_depth, container, is_repack, + has_hdr, } } @@ -131,6 +134,16 @@ mod tests { assert_eq!(p.title_normalized, "Ascendance of a Bookworm"); } + #[test] + fn parses_sxxexx_with_a_trailing_fansub_revision_tag() { + // "v2" glued directly onto the episode number ("a fixed re-release + // of this episode") previously broke the word-boundary check + // entirely, leaving season/episode both None. + let p = parse("[Judas] Chainsaw Man - S01E01v2 1080p WEB-DL"); + assert_eq!(p.season, Some(1)); + assert_eq!(p.episode, Some(1)); + } + #[test] fn parses_subsplease_dash_episode_with_group_and_hash() { let p = parse("[SubsPlease] Honzuki no Gekokujou S4 - 13 (1080p) [A4FE0990].mkv"); @@ -171,6 +184,78 @@ mod tests { assert_eq!(p.resolution, Some(1080)); } + #[test] + fn parses_season_pack_shapes_without_literal_parens() { + // Real review-queue entries that all failed to resolve at all + // before this fix — season came back None, not just unmatched + // episode — because SEASON_PACK_RE required a literal + // "(S?N Complete)" shape. + assert_eq!( + parse("Modern.Family.S10.COMPLETE.720p.AMZN.WEBRip.x264-GalaxyTV").season, + Some(10) + ); + assert_eq!( + parse("Modern Family 2009 Season 8 Complete 720p AMZN WEBRip x264 [i_c]").season, + Some(8) + ); + assert_eq!( + parse("Red.Dwarf.S04.1080p.BluRay.x264-LATENCY [Season 4 Four Complete]").season, + Some(4) + ); + assert_eq!( + parse("Red.Dwarf.S11.1080p.BluRay.x264-SHORTBREHD [Season 11 Eleven]").season, + Some(11) + ); + assert_eq!( + parse("Game of Thrones - Season 8 S08 - 2019 1080p Bluray AAC5.1 x264-R").season, + Some(8) + ); + } + + // Regression test for a real gap found in review: "Season 2 - 25" was + // first swallowed whole by `looks_like_episode_range` (its own + // `BARE_EPISODE_RANGE_RE` skips over the un-matchable "Season" word and + // finds its first real match at "2 - 25", mistaking the season marker's + // own number for a range start), and even with that fixed, + // `extract_episode_info`'s `SEASON_PACK_RE` branch used to return + // season-only and never look for a trailing episode number at all. + // Either bug alone drops episode 25 silently. + #[test] + fn parses_a_season_marker_followed_by_a_dash_episode() { + let p = parse("[Erai-raws] Some Show Season 2 - 25 [1080p]"); + assert_eq!(p.season, Some(2)); + assert_eq!(p.episode, Some(25)); + } + + // Companion case: a genuine season-only pack (no trailing dash-episode + // anywhere) must still resolve to season-only, not spuriously pick up + // an unrelated number as an episode. + #[test] + fn a_genuine_season_only_pack_with_no_dash_episode_still_has_no_episode() { + let p = parse("Some Show Season 2 Complete [1080p]"); + assert_eq!(p.season, Some(2)); + assert_eq!(p.episode, None); + } + + // Regression test for a real gap found in review: a date-named release + // ("2024-01-15") got misread by `BARE_EPISODE_RANGE_RE` as an episode + // range — the 4-digit year is too many digits for `\d{1,3}` to match + // whole, so its first real match starts at the month/day pair + // ("01-15") instead, and `looks_like_episode_range` treats that as a + // real range. Asserted directly against the tokenizer rather than + // `parse()`, since a false-positive range and a genuine "no episode + // marker at all" both surface identically as `None`/`None` on + // `ParsedRelease` — `looks_like_episode_range` returning `false` is the + // actual fix being tested here. + #[test] + fn does_not_mistake_a_yyyy_mm_dd_date_for_an_episode_range() { + assert!(!tokens::looks_like_episode_range("Some Daily Show 2024-01-15 1080p WEB-DL")); + + let p = parse("Some Daily Show 2024-01-15 1080p WEB-DL"); + assert_eq!(p.season, None); + assert_eq!(p.episode, None); + } + #[test] fn parses_yameii_dash_sxxexx_with_english_dub_tag() { let p = parse("[Yameii] Ascendance of a Bookworm - S04E11 [English Dub] [CR WEB-DL 1080p H264 AAC] [8ACE7B72] (Honzuki no Gekokujou)"); @@ -278,6 +363,16 @@ mod tests { assert_eq!(p.year, Some(2023)); } + #[test] + fn parses_hdr_hdr10_and_dolby_vision_tokens() { + assert!(parse("Movie.2024.2160p.HDR.mkv").has_hdr); + assert!(parse("Movie.2024.DV.mkv").has_hdr); + assert!(parse("Movie.2024.2160p.HDR10.BluRay").has_hdr); + assert!(parse("Movie.2024.2160p.DoVi.mkv").has_hdr); + assert!(parse("Movie.2024.Dolby.Vision.2160p").has_hdr); + assert!(!parse("Movie.2024.1080p.WEB-DL.H264").has_hdr); + } + #[test] fn does_not_panic_on_unparsable_manga_release() { // Not a video release at all — should degrade gracefully, not crash. diff --git a/breadarrd/src/parser/tokens.rs b/breadarrd/src/parser/tokens.rs index 2c722ee..a79d62f 100644 --- a/breadarrd/src/parser/tokens.rs +++ b/breadarrd/src/parser/tokens.rs @@ -29,6 +29,9 @@ static BIT_DEPTH_RE: LazyLock = pub(super) static REPACK_RE: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\b(REPACK|PROPER)\b").unwrap()); +static HDR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b(?:HDR10\+?|HDR|Dolby[.\s]?Vision|DoVi|DV)\b").unwrap()); + static YEAR_RE: LazyLock = LazyLock::new(|| Regex::new(r"[(\[](19|20)\d{2}[)\]]").unwrap()); // Scene-style releases ("Dune.1984.1080p.BluRay.x264-GROUP") carry the year // bare, with no surrounding brackets — `YEAR_RE` above never matches these @@ -38,12 +41,33 @@ static YEAR_RE: LazyLock = LazyLock::new(|| Regex::new(r"[(\[](19|20)\d{2 static BARE_YEAR_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b((?:19|20)\d{2})\b").unwrap()); +// The trailing `v\d+` is a fansub revision tag ("v2" = "second release of +// this episode, fixed encode/subs") stuck directly onto the episode number +// with no separator — "S01E01v2". Without consuming it before the `\b`, +// the boundary check fails outright (digit→letter isn't a word boundary), +// so the whole pattern silently doesn't match and the file falls through +// to unparsed (verified live: every v2 release of several shows, e.g. an +// entire show that only had v2 releases, ended up with zero linked +// episode files during a library scan). static SXXEXX_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\bS(\d{1,2})E(\d{1,3})\b").unwrap()); + LazyLock::new(|| Regex::new(r"(?i)\bS(\d{1,2})E(\d{1,3})(?:v\d+)?\b").unwrap()); static SXX_DASH_EP_RE: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\bS(\d{1,2})\s*-\s*(\d{1,3})\b").unwrap()); +// A bare season marker with no episode number attached — "S01", "Season 8", +// "Season.1", optionally with a trailing "Complete"/spelled-out season word +// (e.g. "[Season 4 Four Complete]") that's irrelevant to the number itself. +// Previously required literal parens around an explicit "S?N Complete)" +// shape, matching only one specific release-group convention; real +// releases routinely drop the parens, drop "Complete" entirely (e.g. +// "Game of Thrones - Season 8 S08 - 2019"), or spell "Season" out with a +// dot instead of a space (verified live: several real review-queue entries +// failed to resolve at all — season came back `None` — because none of +// these shapes matched the old pattern). Only reached after +// `SXXEXX_RE`/`SXX_DASH_EP_RE` have already failed to find an actual +// episode number, so treating a bare season marker as a pack signal here is +// safe. static SEASON_PACK_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\(S?(\d{1,2})\s*Complete\)").unwrap()); + LazyLock::new(|| Regex::new(r"(?i)\bS(?:eason)?\.?\s*(\d{1,2})\b").unwrap()); static DASH_EPISODE_RE: LazyLock = LazyLock::new(|| Regex::new(r"-\s*(\d{1,3})\b").unwrap()); // A batch/season-pack release covering many episodes in one torrent. @@ -69,15 +93,46 @@ static SXX_EPISODE_RANGE_RE: LazyLock = // elsewhere in the title with a smaller trailing number. static BARE_EPISODE_RANGE_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b(\d{1,3})\s*[-~]\s*(\d{1,3})\b").unwrap()); +// A `YYYY-MM-DD` date ("2024-01-15"): the year's 4 digits are too many for +// `BARE_EPISODE_RANGE_RE`'s/`DASH_EPISODE_RE`'s `\d{1,3}` to match as a +// whole, so those regexes' *first real match* on a date-named release ends +// up starting at the month ("01-15", or "01" alone) instead — a bare +// month/day pair, not a real episode range or episode number. Matched as a +// whole date span (not just a "year-" prefix check) so both the month *and* +// the day segment are covered — checking only the text immediately before a +// candidate match would still let "15" in "2024-01-15" slip through as a +// false "episode 15" once "01" alone was correctly rejected. +static DATE_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}-\d{1,2}-\d{1,2}\b").unwrap()); + +fn overlaps_a_date(s: &str, start: usize, end: usize) -> bool { + DATE_RE.find_iter(s).any(|d| d.start() <= start && end <= d.end()) +} pub(super) fn looks_like_episode_range(s: &str) -> bool { if BATCH_WORD_RE.is_match(s) || SXX_EPISODE_RANGE_RE.is_match(s) { return true; } - BARE_EPISODE_RANGE_RE.captures(s).is_some_and(|c| { - let a: u32 = c[1].parse().unwrap_or(0); - let b: u32 = c[2].parse().unwrap_or(0); - b > a + BARE_EPISODE_RANGE_RE.captures_iter(s).any(|c| { + let first = c.get(1).unwrap(); + let second = c.get(2).unwrap(); + let a: u32 = first.as_str().parse().unwrap_or(0); + let b: u32 = second.as_str().parse().unwrap_or(0); + if b <= a { + return false; + } + if overlaps_a_date(s, first.start(), second.end()) { + return false; + } + // "Season 2 - 25": the range's first number is really a season + // marker's own number (checked by comparing spans, not just text, + // so this only fires when the two genuinely overlap), not a range + // start — "2 - 25" isn't a real episode range, it's "season 2, + // episode 25", resolved separately in `extract_episode_info`. + let is_season_marker_number = SEASON_PACK_RE.captures(s).is_some_and(|sc| { + sc.get(1).unwrap().range() == first.range() + }); + !is_season_marker_number }) } @@ -129,6 +184,10 @@ pub(super) fn extract_bit_depth(s: &str) -> Option { BIT_DEPTH_RE.captures(s)?[1].parse().ok() } +pub(super) fn extract_hdr(s: &str) -> bool { + HDR_RE.is_match(s) +} + pub(super) fn extract_year(s: &str) -> Option { if let Some(m) = YEAR_RE.find(s) { return s[m.start() + 1..m.end() - 1].parse().ok(); @@ -144,6 +203,23 @@ pub(super) fn extract_year(s: &str) -> Option { s[m.start()..m.end()].parse().ok() } +/// `DASH_EPISODE_RE`'s first match that isn't actually the month of a +/// `YYYY-MM-DD` date. A bare `-\s*\d{1,3}\b` alone can't tell "Show - 25 +/// [1080p]" (a real episode number) apart from "...2024-01-15..." (the "01" +/// is just a month, matched for the same reason `BARE_EPISODE_RANGE_RE` +/// does in `looks_like_episode_range` — the 4-digit year is too many digits +/// to match as a whole, so the regex's first real match starts one segment +/// later). Reused by every `extract_episode_info` branch that falls back to +/// `DASH_EPISODE_RE`, not just the range-detection path, since the date +/// misread happens independently of whether `looks_like_episode_range` +/// fires. +fn find_real_dash_episode(s: &str) -> Option> { + DASH_EPISODE_RE.captures_iter(s).find(|c| { + let m = c.get(0).unwrap(); + !overlaps_a_date(s, m.start(), m.end()) + }) +} + /// Returns (season, episode, absolute_episode, title_span_end) — the last /// element is the byte offset in `s` where the episode/season token (or, /// failing that, the first quality marker) begins, used to slice out the @@ -173,9 +249,19 @@ pub(super) fn extract_episode_info(s: &str) -> (Option, Option, Option } if let Some(c) = SEASON_PACK_RE.captures(s) { let season = c[1].parse().ok(); + // A season marker immediately followed elsewhere by a dash-number + // ("Season 2 - 25") names one episode within that season, not a + // season-only pack — checked here rather than reordering the checks + // above `SXX_DASH_EP_RE`/`SXXEXX_RE` still get first crack at more + // specific shapes, and a genuine season-only pack (no trailing + // dash-number anywhere) is unaffected. + if let Some(ep) = find_real_dash_episode(s) { + let episode: Option = ep[1].parse().ok(); + return (season, episode, None, c.get(0).unwrap().start()); + } return (season, None, None, c.get(0).unwrap().start()); } - if let Some(c) = DASH_EPISODE_RE.captures(s) { + if let Some(c) = find_real_dash_episode(s) { let episode: Option = c[1].parse().ok(); return (None, episode, episode, c.get(0).unwrap().start()); } diff --git a/breadarrd/src/qbit/mod.rs b/breadarrd/src/qbit/mod.rs index bf46181..7510c7f 100644 --- a/breadarrd/src/qbit/mod.rs +++ b/breadarrd/src/qbit/mod.rs @@ -33,6 +33,49 @@ impl std::fmt::Display for MagnetRejected { impl std::error::Error for MagnetRejected {} +/// Newer qBittorrent WebUI API versions' `torrents/add` JSON response shape. +#[derive(Debug, Deserialize, Default)] +struct AddTorrentResponse { + #[serde(default)] + success_count: u32, + #[serde(default)] + failure_count: u32, + /// Present when adding by URL rather than by a magnet/`.torrent` blob + /// qBittorrent already has in hand — the torrent itself is fetched + /// asynchronously, so the immediate response has neither succeeded nor + /// failed yet, just queued. nyaa's RSS feed always hands over a + /// `.torrent` download URL (never a magnet), so this is the normal + /// shape for every anime grab, not an edge case. Verified live against + /// the deployed qBittorrent: a URL add returns HTTP 202 with + /// `{"pending_count":1,"success_count":0,"failure_count":0}` — treating + /// `success_count == 0` alone as rejection (the previous check) silently + /// dropped every one of these while the torrent downloaded successfully + /// in the background, with no DB record and no log line. + #[serde(default)] + pending_count: u32, +} + +/// Interprets a `torrents/add` response body — split out from `add_magnet` +/// so it's directly testable without a live qBittorrent server. +/// qBittorrent's add-torrent endpoint returns HTTP 200/202 even when it +/// rejects the magnet outright (a dead/malformed hash, one it already knows +/// is unreachable) — the response body is the only signal. Older +/// qBittorrent versions returned plain text ("Ok." vs "Fails."); newer ones +/// return a JSON summary with success_count/failure_count/pending_count +/// instead (verified live against the currently deployed version). Without +/// checking whichever shape is actually in play, a rejected magnet looks +/// identical to a real success: the caller records a `release` row as +/// grabbed and nothing ever downloads, silently and permanently (verified +/// live — this happened for a real release under the old text-only check, +/// and separately for every nyaa URL-add under a since-fixed +/// `success_count == 0` check that didn't account for `pending_count`). +fn add_torrent_response_is_rejected(body: &str) -> bool { + match serde_json::from_str::(body) { + Ok(r) => r.failure_count > 0 || (r.success_count == 0 && r.pending_count == 0), + Err(_) => body.trim() != "Ok.", + } +} + pub struct QbitClient { base_url: String, client: reqwest::Client, @@ -60,6 +103,9 @@ pub struct TorrentInfo { pub name: String, pub state: String, pub progress: f64, + // Not read internally today, but kept to mirror qBittorrent's actual API + // shape 1:1 — useful in `{:?}` debug logging and cheap to keep in sync. + #[allow(dead_code)] pub save_path: String, /// Full path to the torrent's content (file or directory root) — /// qBittorrent resolves this for us, so importers don't need to guess @@ -101,7 +147,12 @@ impl QbitClient { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); - if !status.is_success() || body.trim() != "Ok." { + // Older qBittorrent WebUI API versions return 200 with body "Ok."; + // newer ones return 204 No Content with an empty body instead. Bad + // credentials return a real error status (401), which `is_success` + // already catches — the response body's exact text isn't part of + // the actual success contract, just an artifact of the old version. + if !status.is_success() { bail!("qbit login failed: status={status} body={body:?}"); } Ok(()) @@ -140,15 +191,7 @@ impl QbitClient { if !status.is_success() { bail!("qbit add-torrent failed: status={status} body={body:?}"); } - // qBittorrent's add-torrent endpoint returns HTTP 200 even when - // it rejects the magnet outright (a dead/malformed hash, one it - // already knows is unreachable) — the *only* signal is the - // response body text ("Ok." vs "Fails."). Without this check a - // rejected magnet looks identical to a real success: the caller - // records a `release` row as grabbed and nothing ever - // downloads, silently and permanently (verified live — this - // happened for a real release). - if body.trim() != "Ok." { + if add_torrent_response_is_rejected(&body) { return Err(anyhow::Error::new(MagnetRejected { body })); } return Ok(()); @@ -204,10 +247,6 @@ impl QbitClient { unreachable!("loop always returns or bails on its second iteration") } - /// Moves a torrent's save location — qBittorrent physically relocates - /// the underlying file(s) itself and continues seeding from the new - /// path, rather than breadarr keeping a second permanent copy purely to - /// satisfy its own import step. /// Held for the duration of an add-then-correlate-hash sequence — see /// `grab_lock`'s doc comment on why this needs to be process-wide, not /// just per-call. @@ -215,10 +254,19 @@ impl QbitClient { self.grab_lock.lock().await } - pub async fn set_location(&self, hash: &str, location: &str) -> Result<()> { + /// Removes a torrent from qBittorrent's own tracking after breadarr has + /// already moved its data straight into the library — `delete_files: + /// false` because by the time this is called there's nothing left at + /// the torrent's original save path for qBittorrent to delete; leaving + /// the torrent registered would just leave it sitting in an "files + /// missing" error state indefinitely. Best-effort from the caller's + /// side: a failure here never undoes or blocks the import that already + /// succeeded, it just leaves one stale entry in the qBittorrent UI to + /// clean up by hand. + pub async fn delete_torrent(&self, hash: &str) -> Result<()> { self.post_form( - "/api/v2/torrents/setLocation", - &[("hashes", hash), ("location", location)], + "/api/v2/torrents/delete", + &[("hashes", hash), ("deleteFiles", "false")], ) .await?; Ok(()) @@ -262,4 +310,38 @@ mod tests { None ); } + + #[test] + fn a_pending_url_add_is_not_treated_as_rejected() { + // Exact shape qBittorrent 5.x returns for a URL add (every nyaa + // grab) — the torrent is queued for an async fetch, not yet + // succeeded or failed. This is the shape that used to be + // misread as an outright rejection. + let body = r#"{"added_torrent_ids":[],"failure_count":0,"pending_count":1,"success_count":0}"#; + assert!(!add_torrent_response_is_rejected(body)); + } + + #[test] + fn a_genuine_failure_is_still_rejected() { + let body = r#"{"failure_count":1,"pending_count":0,"success_count":0}"#; + assert!(add_torrent_response_is_rejected(body)); + } + + #[test] + fn an_in_band_success_is_not_rejected() { + // TPB/1337x magnets: qBittorrent already has the info hash, no + // async fetch needed, so success_count is set immediately. + let body = r#"{"failure_count":0,"pending_count":0,"success_count":1}"#; + assert!(!add_torrent_response_is_rejected(body)); + } + + #[test] + fn the_old_plain_text_ok_response_is_not_rejected() { + assert!(!add_torrent_response_is_rejected("Ok.")); + } + + #[test] + fn the_old_plain_text_fails_response_is_rejected() { + assert!(add_torrent_response_is_rejected("Fails.")); + } } diff --git a/breadarrd/src/scheduler.rs b/breadarrd/src/scheduler.rs index 3a990c6..90b7d96 100644 --- a/breadarrd/src/scheduler.rs +++ b/breadarrd/src/scheduler.rs @@ -123,6 +123,20 @@ fn looks_like_season_pack(parsed: &ParsedRelease) -> bool { parsed.season.is_some() && parsed.episode.is_none() && parsed.absolute_episode.is_none() } +/// Sort key for a batch of raw search results: season packs first (a pack +/// clears every missing episode in one grab instead of one at a time, and +/// each still goes through the normal seeder/quality gate in `process_item` +/// — a pack with too few seeders is rejected there and iteration just falls +/// through to the next candidate, so this never trades a viable single +/// episode for an unviable pack), highest-seeded first within each group. +fn release_sort_key(item: &RawReleaseItem) -> (std::cmp::Reverse, std::cmp::Reverse) { + let is_pack = looks_like_season_pack(&parser::parse(&item.title)); + ( + std::cmp::Reverse(is_pack), + std::cmp::Reverse(item.seeders.unwrap_or(0)), + ) +} + /// True when a release's raw title carries an explicit non-video format /// marker — an ebook, audiobook, or comic that happens to share a /// monitored show/movie's title text, not an actual episode or film. A @@ -180,13 +194,7 @@ fn movie_needs_grab(conn: &Connection, media_item_id: i64) -> Result { if has_file > 0 { return Ok(false); } - let in_flight: i64 = conn.query_row( - "SELECT count(*) FROM release WHERE media_item_id = ?1 AND episode_id IS NULL - AND status IN ('grabbed','downloading')", - params![media_item_id], - |row| row.get(0), - )?; - Ok(in_flight == 0) + Ok(!movie_has_in_flight_release(conn, media_item_id)?) } /// Upgrade-search counterpart to `movie_needs_grab`: same monitored/ @@ -202,13 +210,64 @@ fn movie_eligible_for_upgrade(conn: &Connection, media_item_id: i64) -> Result Result { + let n: i64 = conn.query_row( "SELECT count(*) FROM release WHERE media_item_id = ?1 AND episode_id IS NULL AND status IN ('grabbed','downloading')", params![media_item_id], |row| row.get(0), )?; - Ok(in_flight == 0) + Ok(n > 0) +} + +fn episode_has_in_flight_release(conn: &Connection, episode_id: i64) -> Result { + let n: i64 = conn.query_row( + "SELECT count(*) FROM release WHERE episode_id = ?1 AND status IN ('grabbed','downloading')", + params![episode_id], + |row| row.get(0), + )?; + Ok(n > 0) +} + +fn season_pack_in_flight(conn: &Connection, media_item_id: i64, season: u32) -> Result { + let n: i64 = conn.query_row( + "SELECT count(*) FROM release WHERE media_item_id = ?1 AND episode_id IS NULL + AND season_number = ?2 AND status IN ('grabbed','downloading')", + params![media_item_id, season], + |row| row.get(0), + )?; + Ok(n > 0) +} + +/// True when this episode already has a grabbed/downloading release, or a +/// season pack covering this season is already in flight. +fn episode_is_in_flight( + conn: &Connection, + media_item_id: i64, + episode_id: i64, + season: u32, +) -> Result { + Ok(episode_has_in_flight_release(conn, episode_id)? + || season_pack_in_flight(conn, media_item_id, season)?) } fn is_anime(conn: &Connection, tvdb_id: i64) -> Result { @@ -266,14 +325,21 @@ fn find_monitored_missing_episode( season: u32, episode: u32, ) -> Result> { - conn.query_row( - "SELECT id FROM episode WHERE media_item_id = ?1 AND season_number = ?2 - AND episode_number = ?3 AND monitored = 1 AND has_file = 0", - params![media_item_id, season, episode], - |row| row.get(0), - ) - .optional() - .map_err(Into::into) + let id: Option = conn + .query_row( + "SELECT id FROM episode WHERE media_item_id = ?1 AND season_number = ?2 + AND episode_number = ?3 AND monitored = 1 AND has_file = 0", + params![media_item_id, season, episode], + |row| row.get(0), + ) + .optional()?; + let Some(id) = id else { + return Ok(None); + }; + if episode_is_in_flight(conn, media_item_id, id, season)? { + return Ok(None); + } + Ok(Some(id)) } /// Upgrade-search counterpart to `find_monitored_missing_episode`: same @@ -307,8 +373,14 @@ fn count_monitored_missing_episodes_in_season( season: u32, ) -> Result { conn.query_row( - "SELECT count(*) FROM episode WHERE media_item_id = ?1 AND season_number = ?2 - AND monitored = 1 AND has_file = 0", + "SELECT count(*) FROM episode e + WHERE e.media_item_id = ?1 AND e.season_number = ?2 + AND e.monitored = 1 AND e.has_file = 0 + AND NOT EXISTS (SELECT 1 FROM release r WHERE r.episode_id = e.id + AND r.status IN ('grabbed','downloading')) + AND NOT EXISTS (SELECT 1 FROM release r WHERE r.media_item_id = e.media_item_id + AND r.episode_id IS NULL AND r.season_number = e.season_number + AND r.status IN ('grabbed','downloading'))", params![media_item_id, season], |row| row.get(0), ) @@ -371,6 +443,14 @@ fn best_existing_season_pack_score( /// an upgrade-search grab (see `SearchTarget::upgrade_min_gain`), so a file /// already on disk isn't replaced over and over for score deltas too small /// to matter. +/// Upgrade-search may auto-grab a better copy of something already owned. +/// The review-queue approve handler cannot — it only runs the first-copy +/// checks — so a NeedsReview match on an upgrade cycle must use those +/// same first-copy checks or the TUI's `a` key 409s every time. +fn use_upgrade_eligibility(upgrade_min_gain: Option, needs_review: bool) -> bool { + upgrade_min_gain.is_some() && !needs_review +} + fn should_grab(new_score: f32, is_repack: bool, existing_best: Option, min_gain: f32) -> bool { match existing_best { None => true, @@ -460,12 +540,9 @@ async fn process_item( let parsed = parser::parse(&item.title); - let candidate = match matcher.match_title(conn, &parsed.title_normalized)? { - MatchOutcome::Auto(c) => c, - MatchOutcome::NeedsReview(c) => { - matcher::queue_for_review(conn, &item.title, &c, Some(&item.link), Some(source_id))?; - return Ok(ProcessOutcome::QueuedForReview); - } + let (candidate, needs_review) = match matcher.match_title(conn, &parsed.title_normalized)? { + MatchOutcome::Auto(c) => (c, false), + MatchOutcome::NeedsReview(c) => (c, true), MatchOutcome::NoMatch => return Ok(ProcessOutcome::NoMatch), }; @@ -474,6 +551,16 @@ async fn process_item( let mut episode_id: Option = None; let mut season_pack_number: Option = None; + // Whether this match needs a human's confirmation (queued for review) + // or was confident enough to act on automatically, the show/season/ + // episode/movie still has to actually need *something* first — a + // low-confidence title match against an already-complete show gains + // nothing from a human's yes/no, it's just noise that reappears every + // cycle the source keeps re-listing the same old release (verified + // live: fully-complete shows' season-pack re-releases piling up in the + // review queue indefinitely because this check only ever ran on the + // auto-match path). So this eligibility check runs before the + // queue-for-review decision below, not only on the auto-grab path. if media_item.kind == "movie" { // A release carrying a season/episode/absolute-episode marker // title-matched a movie by name alone — it's actually an episode @@ -484,9 +571,15 @@ async fn process_item( if movie_year_mismatch(parsed.year, media_item.year) { return Ok(ProcessOutcome::YearMismatch); } - let eligible = match upgrade_min_gain { - Some(_) => movie_eligible_for_upgrade(conn, media_item.id)?, - None => movie_needs_grab(conn, media_item.id)?, + // Review-queue approval only implements the first-copy path + // (`movie_needs_grab`). An upgrade-cycle match that still needs a + // human would otherwise be queued and then 409 on approve — the + // live hestia queue was 139 already-owned movies for exactly this + // reason. High-confidence auto-matches still use upgrade eligibility. + let eligible = if use_upgrade_eligibility(upgrade_min_gain, needs_review) { + movie_eligible_for_upgrade(conn, media_item.id)? + } else { + movie_needs_grab(conn, media_item.id)? }; if !eligible { return Ok(ProcessOutcome::NotMonitoredOrAlreadyHave); @@ -503,9 +596,10 @@ async fn process_item( let Some((season, episode)) = resolve_episode(conn, media_item.tvdb_id, &parsed)? else { return Ok(ProcessOutcome::CouldNotResolveEpisode); }; - let eid_opt = match upgrade_min_gain { - Some(_) => find_monitored_episode(conn, media_item.id, season, episode)?, - None => find_monitored_missing_episode(conn, media_item.id, season, episode)?, + let eid_opt = if use_upgrade_eligibility(upgrade_min_gain, needs_review) { + find_monitored_episode(conn, media_item.id, season, episode)? + } else { + find_monitored_missing_episode(conn, media_item.id, season, episode)? }; let Some(eid) = eid_opt else { return Ok(ProcessOutcome::NotMonitoredOrAlreadyHave); @@ -534,11 +628,32 @@ async fn process_item( is_season_pack: season_pack_number.is_some(), }; + // Quality gates — including "reject sub-1080p when a 1080p+ alternative + // exists in this same batch" — apply before a candidate ever reaches a + // human, not just on the confident auto-grab path. A human's review + // decision is about whether this is genuinely the right show/season; + // it was never meant to also be the place quality standards get + // relaxed just because the title match happened to be ambiguous + // (verified live: several 720p season-pack releases sat in the review + // queue for shows that also had 1080p+ releases available in the same + // search, when they should have been silently gate-rejected instead). if let GateResult::Reject(reason) = scoring::evaluate_gates(&parsed, &gate_ctx, &profile) { return Ok(ProcessOutcome::GateRejected(reason)); } - let release_score = scoring::score(&parsed, item.seeders.unwrap_or(0), false, &profile); + if needs_review { + matcher::queue_for_review( + conn, + &item.title, + &candidate, + Some(&item.link), + Some(source_id), + )?; + return Ok(ProcessOutcome::QueuedForReview); + } + + let release_score = + scoring::score(&parsed, item.seeders.unwrap_or(0), parsed.has_hdr, &profile); let existing_best = match (episode_id, season_pack_number) { (Some(eid), _) => best_existing_score(conn, eid)?, (None, Some(season)) => best_existing_season_pack_score(conn, media_item.id, season)?, @@ -557,6 +672,33 @@ async fn process_item( let torrent_hash = match grab_and_capture_hash(qbit, &item.link, qbit_category).await { Ok(hash) => hash, Err(e) if e.downcast_ref::().is_some() => { + // Same reasoning as `HashCaptureFailed` just below: recording + // this as `failed` rather than leaving it with no release row + // at all is what makes a genuine rejection visible and frees + // the episode/movie to be re-searched later with a different + // candidate. A silent `Ok(MagnetRejected)` with nothing written + // used to be indistinguishable from a real success once + // `mark_seen` ran — a since-fixed bug in the response-rejection + // check (see `qbit::add_torrent_response_is_rejected`'s doc + // comment) made every nyaa URL-add hit this path even though + // the torrent was actually downloading, so this arm went + // unnoticed for a long time; logging it now that it only fires + // on real rejections. + tracing::warn!(title = %item.title, guid = %item.guid, "qbittorrent rejected this release's magnet/torrent"); + record_grab( + conn, + media_item.id, + episode_id, + season_pack_number, + source_id, + &item.title, + &item.guid, + release_score, + item.size_bytes, + qbit_category, + None, + "failed", + )?; return Ok(ProcessOutcome::MagnetRejected); } Err(e) => return Err(e), @@ -696,11 +838,16 @@ async fn grab_and_capture_hash( #[derive(Debug, Default)] pub struct GrabCycleStats { + // Only read via the derived `Debug` (the debug-grab-cycle CLI command + // prints the whole struct), which clippy's dead-code analysis doesn't + // credit as a read. + #[allow(dead_code)] pub items_seen: usize, pub new_items: usize, pub grabbed: usize, pub errors: usize, pub queued_for_review: usize, + pub magnet_rejected: usize, } pub async fn run_grab_cycle( @@ -751,6 +898,7 @@ pub async fn run_grab_cycle( match outcome { ProcessOutcome::Grabbed { .. } => stats.grabbed += 1, ProcessOutcome::QueuedForReview => stats.queued_for_review += 1, + ProcessOutcome::MagnetRejected => stats.magnet_rejected += 1, _ => {} } } @@ -764,8 +912,8 @@ pub async fn run_grab_cycle( Ok(stats) } -// --- Search-driven acquisition (1337x for general TV/movies, nyaa search -// for anime movies) --- +// --- Search-driven acquisition (TPB + YTS/csv/1337x fallbacks for +// general TV/movies, nyaa search for anime movies) --- // // Unlike the feed-based path above, there's no natural stream of "new" // items to dedup against — the recurring cost here is the *search itself*, @@ -774,18 +922,12 @@ pub async fn run_grab_cycle( // single cycle; cadence backs off exponentially (6h, 12h, 24h, 48h, 96h, // capped at a week) the more times it's been searched without success. -/// Other general-content sources considered and rejected (live-tested -/// 2026-07-12, not just assumed) before landing on TPB as primary: -/// - **YTS** (`yts.mx`): DNS doesn't resolve at all. Every known mirror -/// (`yts.am`, `yts.ag`, `yts.lt`, `yts.pe`) either 301s in a loop or drops -/// the query and lands on a bare homepage. The whole mirror network looks -/// dead, not just one domain — re-check before assuming a fix is quick. -/// - **EZTV** (`eztv.re`): redirects to `eztvx.to`, which fails to connect -/// outright (TLS/connection error, not a slow response). Also -/// Cloudflare-fronted, so even if connectivity is restored it carries the -/// same risk profile 1337x does. -/// If revisiting either, re-verify connectivity first — this isn't a -/// permanent architectural decision, just what was true when checked. +/// General-content fallbacks live-tested 2026-08-16 (TPB/apibay was +/// timing out; every configured 1337x mirror returned Cloudflare 521): +/// - **torrents-csv** and **YTS** (`yts.lt` API — `yts.mx` still does not +/// resolve) are JSON hash-to-magnet sources, same grab shape as TPB. +/// - **EZTV**'s JSON API is up but IMDb-id only; name search is a +/// Cloudflare challenge. Not wired — breadarr has TMDB/TVDB, not IMDb. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum SearchRoute { /// Primary general-content (movies + non-anime TV) route — a JSON API, @@ -795,10 +937,27 @@ enum SearchRoute { /// (not just "no relevant results") — the mirror-rotation/cooldown /// machinery already built for it is real resilience worth keeping, /// just no longer the first choice given TPB's better precision. + #[allow(dead_code)] X1337, NyaaSearch, } +/// The search-driven sources plus their `source` table ids — one bundle +/// so `execute_search_targets` / `fetch_candidates` / the cycle runners +/// don't each grow another four arguments every time a fallback is added. +pub struct SearchSources<'a> { + pub tpb: &'a sources::tpb::TpbSource, + pub tpb_id: i64, + pub torrents_csv: &'a sources::torrents_csv::TorrentsCsvSource, + pub torrents_csv_id: i64, + pub yts: &'a sources::yts::YtsSource, + pub yts_id: i64, + pub scrape: &'a sources::scrape::ScrapeSource, + pub scrape_id: i64, + pub nyaa_search: &'a sources::rss::RssSource, + pub nyaa_id: i64, +} + #[derive(Debug, Clone, PartialEq)] pub struct SearchTarget { media_item_id: i64, @@ -808,6 +967,11 @@ pub struct SearchTarget { /// satisfied this target even though it has no single `episode_id` /// of its own. season_number: Option, + /// The target episode's number — `None` for a movie (or a season-pack + /// target). Used so `better_resolution_available` only counts 1080p+ + /// results that are actually this episode (or a pack of this season), + /// not a sibling's higher-res release. + episode_number: Option, query: String, route: SearchRoute, /// Significant (len >= 4, alphanumeric) lowercased words from the @@ -852,6 +1016,29 @@ fn passes_relevance_filter(target_words: &[String], candidate_title: &str) -> bo target_words.iter().all(|w| lower.contains(w.as_str())) } +/// True when a title-relevant item in this batch is 1080p+ *for this +/// target* — matching S/E, or a season pack of this season. A sibling +/// episode's 1080p must not reject this episode's only 720p option. +/// Movies: any title-relevant 1080p+ counts. +fn better_resolution_available(target: &SearchTarget, items: &[RawReleaseItem]) -> bool { + items.iter().any(|item| { + if !passes_relevance_filter(&target.title_words, &item.title) { + return false; + } + let parsed = parser::parse(&item.title); + if !parsed.resolution.is_some_and(|r| r >= 1080) { + return false; + } + if target.season_number.is_none() { + return true; + } + if looks_like_season_pack(&parsed) { + return parsed.season == target.season_number; + } + parsed.season == target.season_number && parsed.episode == target.episode_number + }) +} + /// 1337x's search chokes on punctuation (colons, apostrophes) — replace /// anything that isn't alphanumeric/whitespace with a space and collapse. fn sanitize_query_text(s: &str) -> String { @@ -945,6 +1132,9 @@ fn enumerate_search_targets(conn: &Connection, budget: usize) -> Result Result Result, matcher: &mut TitleMatcher, qbit: &QbitClient, qbit_category: &str, budget: usize, ) -> Result { let targets = enumerate_search_targets(conn, budget)?; - execute_search_targets( - conn, - &targets, - tpb, - tpb_source_id, - scrape, - scrape_source_id, - nyaa_search, - nyaa_source_id, - matcher, - qbit, - qbit_category, - ) - .await + execute_search_targets(conn, &targets, sources, matcher, qbit, qbit_category).await } /// Upgrade-search counterpart to `run_search_cycle`: same execution engine @@ -1417,15 +1593,9 @@ pub async fn run_search_cycle( /// ones. `min_gain` is threaded onto every target via /// `enumerate_upgrade_targets`, which is what routes `process_item` into /// its upgrade-eligibility path instead of the normal missing-content one. -#[allow(clippy::too_many_arguments)] pub async fn run_upgrade_cycle( conn: &Connection, - tpb: &sources::tpb::TpbSource, - tpb_source_id: i64, - scrape: &sources::scrape::ScrapeSource, - scrape_source_id: i64, - nyaa_search: &sources::rss::RssSource, - nyaa_source_id: i64, + sources: &SearchSources<'_>, matcher: &mut TitleMatcher, qbit: &QbitClient, qbit_category: &str, @@ -1433,20 +1603,7 @@ pub async fn run_upgrade_cycle( min_gain: f32, ) -> Result { let targets = enumerate_upgrade_targets(conn, budget, min_gain)?; - execute_search_targets( - conn, - &targets, - tpb, - tpb_source_id, - scrape, - scrape_source_id, - nyaa_search, - nyaa_source_id, - matcher, - qbit, - qbit_category, - ) - .await + execute_search_targets(conn, &targets, sources, matcher, qbit, qbit_category).await } /// Every currently-missing episode/movie for one specific `media_item`, @@ -1493,6 +1650,7 @@ pub fn enumerate_search_targets_for_media_item( media_item_id, episode_id: None, season_number: None, + episode_number: None, query: build_movie_query(&title, year), title_words: significant_words(&title), route, @@ -1508,6 +1666,9 @@ pub fn enumerate_search_targets_for_media_item( AND e.air_date IS NOT NULL AND e.air_date <= date('now') AND NOT EXISTS (SELECT 1 FROM release r WHERE r.episode_id = e.id AND r.status IN ('grabbed','downloading')) + AND NOT EXISTS (SELECT 1 FROM release r WHERE r.media_item_id = e.media_item_id + AND r.episode_id IS NULL AND r.season_number = e.season_number + AND r.status IN ('grabbed','downloading')) ORDER BY e.season_number, e.episode_number", )?; let rows: Vec<(i64, String, i64, i64)> = stmt @@ -1521,6 +1682,7 @@ pub fn enumerate_search_targets_for_media_item( media_item_id, episode_id: Some(episode_id), season_number: Some(season as u32), + episode_number: Some(episode as u32), query: build_tv_query(&title, season, episode, SearchRoute::Tpb), title_words: significant_words(&title), route: SearchRoute::Tpb, @@ -1539,16 +1701,10 @@ pub fn find_media_item_id_by_title(conn: &Connection, title: &str) -> Result, matcher: &mut TitleMatcher, qbit: &QbitClient, qbit_category: &str, @@ -1564,8 +1720,8 @@ pub async fn execute_search_targets( // without this, a 10-episode backlog fires 10 indistinguishable // requests at a single-domain API with no mirror fallback, which is // exactly the kind of pattern that gets a source rate-limited. Caches - // which source actually answered too (the TPB→1337x fallback can mean - // two different targets with the same query string were served by two + // which source actually answered too (the TPB → YTS/csv/1337x chain + // can mean two targets with the same query were served by two // different sources), so a cache hit still attributes dedup/grab // records to the right source id. let mut query_cache: std::collections::HashMap< @@ -1574,12 +1730,6 @@ pub async fn execute_search_targets( > = std::collections::HashMap::new(); for (i, target) in targets.iter().enumerate() { - let (primary, primary_id): (&dyn ReleaseSource, i64) = match target.route { - SearchRoute::Tpb => (tpb, tpb_source_id), - SearchRoute::X1337 => (scrape, scrape_source_id), - SearchRoute::NyaaSearch => (nyaa_search, nyaa_source_id), - }; - let cache_key = (target.route, target.query.clone()); let (items, source_id) = if let Some((cached_items, cached_source_id)) = query_cache.get(&cache_key) { @@ -1590,34 +1740,11 @@ pub async fn execute_search_targets( tokio::time::sleep(std::time::Duration::from_secs(jitter_secs)).await; } - let primary_result = primary.fetch(Some(&target.query)).await; - // TPB is the primary route for general content, but a fetch - // *failure* there (not just "no relevant results") falls back - // to 1337x for the same query before giving up — keeps the - // mirror-rotation/cooldown machinery built for 1337x as real - // resilience rather than dead code, just no longer the first - // choice given TPB's better precision. `result_source_id` - // tracks which source actually produced whatever we end up - // with, since dedup (`is_seen`/`mark_seen`) and grab records - // are keyed by source id — attributing a 1337x-sourced guid to - // TPB's source id would silently break dedup between the two. - let (fetch_result, result_source_id) = - if primary_result.is_err() && matches!(target.route, SearchRoute::Tpb) { - tracing::warn!( - query = %target.query, - error = %primary_result.as_ref().unwrap_err(), - "TPB search failed, falling back to 1337x" - ); - (scrape.fetch(Some(&target.query)).await, scrape_source_id) - } else { - (primary_result, primary_id) - }; - - match fetch_result { - Ok(items) => { + match fetch_for_target(target, sources).await { + Ok(pair) => { consecutive_fetch_errors = 0; - query_cache.insert(cache_key, (items.clone(), result_source_id)); - (items, result_source_id) + query_cache.insert(cache_key, pair.clone()); + pair } Err(e) => { consecutive_fetch_errors += 1; @@ -1638,7 +1765,7 @@ pub async fn execute_search_targets( }; let mut sorted = items; - sorted.sort_by_key(|i| std::cmp::Reverse(i.seeders.unwrap_or(0))); + sorted.sort_by_key(release_sort_key); sorted.truncate(MAX_RESULTS_PER_SEARCH); // Computed once per target, over candidates that at least pass the @@ -1647,12 +1774,7 @@ pub async fn execute_search_targets( // matching `process_item` already does per-item below). Used to // decide whether a sub-1080p candidate is a real downgrade or the // only option actually available for this target. - let better_resolution_available = sorted.iter().any(|item| { - passes_relevance_filter(&target.title_words, &item.title) - && parser::parse(&item.title) - .resolution - .is_some_and(|r| r >= 1080) - }); + let better_resolution_available = better_resolution_available(target, &sorted); // A query built for one target's title can surface a *different* // monitored show/movie in its results (1337x's search isn't tightly @@ -1753,14 +1875,96 @@ pub async fn execute_search_targets( Ok(stats) } -fn source_route_name(route: SearchRoute) -> &'static str { - match route { - SearchRoute::Tpb => "tpb", - SearchRoute::X1337 => "1337x", - SearchRoute::NyaaSearch => "nyaa", +fn source_name_for_id(id: i64) -> &'static str { + match id { + 1 => "nyaa", + 2 => "1337x", + 3 => "tpb", + 4 => "torrents-csv", + 5 => "yts", + _ => "unknown", } } +/// TPB first; on failure *or* empty results, walk the supplement chain +/// (torrents-csv for everything, YTS for movies, 1337x last). Empty is +/// treated as "try the next one" so a live-but-empty apibay doesn't hide +/// a title that YTS/csv actually has. Dedup/grabs use whichever source +/// actually answered. +async fn fetch_for_target( + target: &SearchTarget, + sources: &SearchSources<'_>, +) -> Result<(Vec, i64)> { + match target.route { + SearchRoute::NyaaSearch => Ok(( + sources.nyaa_search.fetch(Some(&target.query)).await?, + sources.nyaa_id, + )), + SearchRoute::X1337 => Ok(( + sources.scrape.fetch(Some(&target.query)).await?, + sources.scrape_id, + )), + SearchRoute::Tpb => fetch_general_content(target, sources).await, + } +} + +async fn fetch_general_content( + target: &SearchTarget, + sources: &SearchSources<'_>, +) -> Result<(Vec, i64)> { + let is_movie = target.episode_id.is_none(); + let mut attempts: Vec<(&dyn ReleaseSource, i64, &'static str)> = + vec![(sources.tpb, sources.tpb_id, "tpb")]; + if is_movie { + attempts.push((sources.yts, sources.yts_id, "yts")); + } + attempts.push(( + sources.torrents_csv, + sources.torrents_csv_id, + "torrents-csv", + )); + attempts.push((sources.scrape, sources.scrape_id, "1337x")); + + let mut last_err: Option = None; + let mut any_ok = false; + for (src, id, name) in attempts { + match src.fetch(Some(&target.query)).await { + Ok(items) if !items.is_empty() => { + if name != "tpb" { + tracing::info!( + query = %target.query, + source = name, + n = items.len(), + "search fallback produced results" + ); + } + return Ok((items, id)); + } + Ok(_) => { + any_ok = true; + tracing::debug!( + query = %target.query, + source = name, + "search source returned no results" + ); + } + Err(e) => { + tracing::warn!( + query = %target.query, + source = name, + error = %e, + "search source failed" + ); + last_err = Some(e); + } + } + } + if any_ok { + return Ok((Vec::new(), sources.tpb_id)); + } + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all general-content sources failed"))) +} + /// Fetches and scores (or gate-rejects) candidates for one search target — /// the same evaluation `execute_search_targets` does automatically, minus /// the grab decision, surfaced instead for a human to choose from. Used by @@ -1770,29 +1974,18 @@ fn source_route_name(route: SearchRoute) -> &'static str { /// episode/movie right now (already owned, unmonitored, or mid-grab) — /// same "nothing to do" cases `enumerate_search_targets_for_media_item` /// already excludes. -#[allow(clippy::too_many_arguments)] pub async fn fetch_candidates( conn: &Connection, media_item_id: i64, episode_id: Option, - tpb: &sources::tpb::TpbSource, - tpb_source_id: i64, - scrape: &sources::scrape::ScrapeSource, - scrape_source_id: i64, - nyaa_search: &sources::rss::RssSource, - nyaa_source_id: i64, + sources: &SearchSources<'_>, ) -> Result> { let targets = enumerate_search_targets_for_media_item(conn, media_item_id)?; let Some(target) = targets.into_iter().find(|t| t.episode_id == episode_id) else { return Ok(Vec::new()); }; - let (source, source_id): (&dyn ReleaseSource, i64) = match target.route { - SearchRoute::Tpb => (tpb, tpb_source_id), - SearchRoute::X1337 => (scrape, scrape_source_id), - SearchRoute::NyaaSearch => (nyaa_search, nyaa_source_id), - }; - let items = source.fetch(Some(&target.query)).await?; + let (items, source_id) = fetch_for_target(&target, sources).await?; let media_item = get_media_item(conn, media_item_id)?; let anime = match media_item.tvdb_id { @@ -1810,12 +2003,7 @@ pub async fn fetch_candidates( sorted.sort_by_key(|i| std::cmp::Reverse(i.seeders.unwrap_or(0))); sorted.truncate(MAX_RESULTS_PER_SEARCH); - let better_resolution_available = sorted.iter().any(|item| { - passes_relevance_filter(&target.title_words, &item.title) - && parser::parse(&item.title) - .resolution - .is_some_and(|r| r >= 1080) - }); + let better_resolution_available = better_resolution_available(&target, &sorted); let mut candidates = Vec::new(); for item in &sorted { @@ -1838,7 +2026,7 @@ pub async fn fetch_candidates( Some(scoring::score( &parsed, item.seeders.unwrap_or(0), - false, + parsed.has_hdr, &profile, )), None, @@ -1850,7 +2038,7 @@ pub async fn fetch_candidates( link: item.link.clone(), guid: item.guid.clone(), source_id, - source_name: source_route_name(target.route).to_string(), + source_name: source_name_for_id(source_id).to_string(), seeders: item.seeders, leechers: item.leechers, size_bytes: item.size_bytes, @@ -1869,6 +2057,68 @@ pub async fn fetch_candidates( Ok(candidates) } +/// Manual-path only: auto-grab of 1337x still needs detail-page +/// resolution. A picked candidate must already be a magnet or `.torrent`. +fn reject_unresolved_manual_grab_link(link: &str) -> Result<()> { + if sources::scrape::needs_resolution(link) { + anyhow::bail!("candidate must be a magnet or .torrent URL"); + } + Ok(()) +} + +/// Binds a manual grab to the episode the title actually names, not the +/// TUI selection. A season pack has no episode id. Caller `episode_id` is +/// last-resort only (movies / unparsable titles). +fn resolve_grab_episode( + conn: &Connection, + media_item: &MediaItemRow, + parsed: &ParsedRelease, + caller_episode_id: Option, +) -> Result> { + if looks_like_season_pack(parsed) { + return Ok(None); + } + if let Some((season, episode)) = resolve_episode(conn, media_item.tvdb_id, parsed)? { + if let Some(id) = find_episode_id(conn, media_item.id, season, episode)? { + return Ok(Some(id)); + } + if let Some(id) = find_monitored_episode(conn, media_item.id, season, episode)? { + return Ok(Some(id)); + } + } + Ok(caller_episode_id) +} + +/// True when this resolved grab target already has a grabbed/downloading +/// release (episode, covering season pack, or movie). +fn grab_target_in_flight( + conn: &Connection, + media_item: &MediaItemRow, + episode_id: Option, + season_pack_number: Option, +) -> Result { + if media_item.kind == "movie" { + return movie_has_in_flight_release(conn, media_item.id); + } + if let Some(season) = season_pack_number { + return season_pack_in_flight(conn, media_item.id, season); + } + if let Some(eid) = episode_id { + let season: Option = conn + .query_row( + "SELECT season_number FROM episode WHERE id = ?1", + params![eid], + |row| row.get(0), + ) + .optional()?; + if let Some(season) = season { + return episode_is_in_flight(conn, media_item.id, eid, season); + } + return episode_has_in_flight_release(conn, eid); + } + Ok(false) +} + /// Grabs a specific candidate a human picked from `fetch_candidates`' /// output, bypassing the score-vs-existing-best `should_grab` comparison /// entirely — a manual pick is an explicit override, not a competing @@ -1889,6 +2139,7 @@ pub async fn grab_candidate( link: &str, guid: &str, ) -> Result<()> { + reject_unresolved_manual_grab_link(link)?; let parsed = parser::parse(raw_title); let media_item = get_media_item(conn, media_item_id)?; let profile_kind = if media_item.kind == "movie" { @@ -1902,17 +2153,19 @@ pub async fn grab_candidate( } else { None }; - let final_episode_id = if season_pack_number.is_some() { - None - } else { - episode_id - }; + // Rebind to the episode the title actually names. Picking E06 while + // E05 is selected still grabs E06 — the TUI selection is last-resort + // only (movies / unparsable titles). + let final_episode_id = resolve_grab_episode(conn, &media_item, &parsed, episode_id)?; + if grab_target_in_flight(conn, &media_item, final_episode_id, season_pack_number)? { + anyhow::bail!("a grab is already in flight for this episode/movie"); + } // Real-time seeder data isn't available for a candidate picked from an // earlier fetch — same `seeders=0` fallback `finalize_review_approval` // already uses for the same reason, and for the same reason it's still // real signal from resolution/source/codec/etc., not a meaningless // hardcoded score. - let score = scoring::score(&parsed, 0, false, &profile); + let score = scoring::score(&parsed, 0, parsed.has_hdr, &profile); let torrent_hash = match grab_and_capture_hash(qbit, link, qbit_category).await { Ok(hash) => hash, @@ -2053,7 +2306,31 @@ pub fn prepare_review_approval(conn: &Connection, review_id: i64) -> Result Result Result<()> { + conn.execute( + "UPDATE review_queue SET status = 'pending' WHERE id = ?1 AND status = 'approved'", + params![review_id], + )?; + Ok(()) +} + /// The actual grab — network/qBittorrent I/O only, no `Connection` involved, /// safe to `.await` from anywhere. pub async fn grab_prepared_approval( @@ -2076,11 +2377,12 @@ pub async fn grab_prepared_approval( grab_and_capture_hash(qbit, &prepared.link, qbit_category).await } -/// Sync-only: records the grab and marks the review approved. Call after -/// [`grab_prepared_approval`] completes. +/// Sync-only: records the grab. Call after [`grab_prepared_approval`] +/// completes. Doesn't touch `review_queue.status` — `prepare_review_approval` +/// already claimed it into `approved` before the grab ran (see its doc +/// comment). pub fn finalize_review_approval( conn: &Connection, - review_id: i64, prepared: &PreparedApproval, qbit_category: &str, torrent_hash: Option<&str>, @@ -2111,25 +2413,29 @@ pub fn finalize_review_approval( torrent_hash, status, )?; - conn.execute( - "UPDATE review_queue SET status = 'approved' WHERE id = ?1", - params![review_id], - )?; Ok(()) } -pub fn reject_review(conn: &Connection, review_id: i64) -> Result<()> { - conn.execute( +pub fn reject_review(conn: &Connection, review_id: i64) -> Result { + let rows = conn.execute( "UPDATE review_queue SET status = 'rejected' WHERE id = ?1 AND status = 'pending'", params![review_id], )?; - Ok(()) + Ok(rows > 0) } #[cfg(test)] mod tests { use super::*; + #[test] + fn upgrade_cycle_does_not_use_upgrade_eligibility_for_a_review_match() { + assert!(!use_upgrade_eligibility(Some(5.0), true)); + assert!(use_upgrade_eligibility(Some(5.0), false)); + assert!(!use_upgrade_eligibility(None, false)); + assert!(!use_upgrade_eligibility(None, true)); + } + #[test] fn should_grab_when_nothing_exists_yet() { assert!(should_grab(10.0, false, None, 0.0)); @@ -2276,6 +2582,22 @@ mod tests { assert_eq!(targets[1].episode_id, Some(1)); } + #[test] + fn enumerate_upgrade_targets_excludes_an_upgrade_locked_episode() { + let conn = seeded_upgrade_conn_with_one_flagged_episode(); + // Episode 2 is the probe-flagged one that would otherwise sort + // first — lock it (as a completed local AV1 transcode would) and + // confirm it drops out entirely rather than just losing priority. + conn.execute( + "UPDATE episode_file SET upgrade_locked = 1 WHERE episode_id = 2", + [], + ) + .unwrap(); + let targets = enumerate_upgrade_targets(&conn, 10, 5.0).unwrap(); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].episode_id, Some(1)); + } + #[test] fn record_grab_writes_both_a_release_row_and_a_torrent_fetch_audit_row() { let conn = seeded_conn(); @@ -2472,6 +2794,47 @@ mod tests { } } + // Regression test for a real gap found in review: `prepare_review_approval` + // used to only *read* `status == "pending"`, never claim it — two + // concurrent `approve()` calls (a double-click, or an HTTP client retry) + // could both pass that check, both grab the same release, and both + // write their own `release`/`torrent_fetch` rows. A second call for the + // same review must now see it's already claimed. + #[test] + fn a_second_prepare_call_on_an_already_claimed_review_sees_not_pending() { + let conn = seeded_movie_conn(); + let review_id = insert_pending_review(&conn, "Some Movie 2016 1080p BluRay x264"); + + assert!(matches!( + prepare_review_approval(&conn, review_id).unwrap(), + ApprovalPrep::Ready(_) + )); + // Same review_id, called again before any grab or finalize ran — + // simulates the double-click/retry race. + assert!(matches!( + prepare_review_approval(&conn, review_id).unwrap(), + ApprovalPrep::NotPending + )); + } + + #[test] + fn release_review_claim_reverts_a_claimed_row_back_to_pending() { + let conn = seeded_movie_conn(); + let review_id = insert_pending_review(&conn, "Some Movie 2016 1080p BluRay x264"); + + assert!(matches!( + prepare_review_approval(&conn, review_id).unwrap(), + ApprovalPrep::Ready(_) + )); + release_review_claim(&conn, review_id).unwrap(); + // The claim was released (e.g. because the grab itself then errored + // outright) — a fresh approval attempt must be possible again. + assert!(matches!( + prepare_review_approval(&conn, review_id).unwrap(), + ApprovalPrep::Ready(_) + )); + } + #[test] fn rejects_a_movie_review_whose_title_looks_like_an_episode() { let conn = seeded_movie_conn(); @@ -2545,6 +2908,20 @@ mod tests { assert!(movie_eligible_for_upgrade(&conn, 1).unwrap()); } + #[test] + fn movie_eligible_for_upgrade_is_false_once_upgrade_locked() { + let conn = seeded_movie_conn(); + conn.execute( + "INSERT INTO episode_file (media_item_id, episode_id, path, size_bytes, upgrade_locked) + VALUES (1, NULL, '/tmp/movie.mkv', 100, 1)", + [], + ) + .unwrap(); + // A locally AV1-transcoded file is a deliberate shrink, not + // something the upgrade loop should try to replace. + assert!(!movie_eligible_for_upgrade(&conn, 1).unwrap()); + } + #[test] fn movie_eligible_for_upgrade_is_false_when_unmonitored() { let conn = seeded_movie_conn(); @@ -2630,6 +3007,54 @@ mod tests { ))); } + #[test] + fn release_sort_key_prefers_a_season_pack_over_a_higher_seeded_episode() { + let pack = RawReleaseItem { + title: "Some Show (S02 Complete) 1080p WEB-DL".into(), + link: String::new(), + guid: "pack".into(), + size_bytes: None, + seeders: Some(10), + leechers: None, + }; + let episode = RawReleaseItem { + title: "Some Show S02E05 1080p WEB-DL".into(), + link: String::new(), + guid: "episode".into(), + size_bytes: None, + seeders: Some(500), + leechers: None, + }; + let mut items = [episode.clone(), pack.clone()]; + items.sort_by_key(release_sort_key); + assert_eq!(items[0].guid, "pack"); + assert_eq!(items[1].guid, "episode"); + } + + #[test] + fn release_sort_key_falls_back_to_seeders_within_the_same_group() { + let low = RawReleaseItem { + title: "Some Show S02E05 1080p WEB-DL".into(), + link: String::new(), + guid: "low".into(), + size_bytes: None, + seeders: Some(5), + leechers: None, + }; + let high = RawReleaseItem { + title: "Some Show S02E05 720p WEB-DL".into(), + link: String::new(), + guid: "high".into(), + size_bytes: None, + seeders: Some(50), + leechers: None, + }; + let mut items = [low.clone(), high.clone()]; + items.sort_by_key(release_sort_key); + assert_eq!(items[0].guid, "high"); + assert_eq!(items[1].guid, "low"); + } + #[test] fn count_monitored_missing_episodes_in_season_counts_correctly() { let conn = seeded_conn(); @@ -3062,4 +3487,246 @@ mod tests { // e.g. a title that's entirely short/common words after filtering assert!(passes_relevance_filter(&[], "anything at all")); } + + fn raw_item(title: &str) -> RawReleaseItem { + RawReleaseItem { + title: title.into(), + link: String::new(), + guid: title.into(), + size_bytes: None, + seeders: Some(10), + leechers: None, + } + } + + fn tv_search_target(season: u32, episode: u32) -> SearchTarget { + SearchTarget { + media_item_id: 1, + episode_id: Some(i64::from(episode)), + season_number: Some(season), + episode_number: Some(episode), + query: "Some Show".into(), + route: SearchRoute::Tpb, + title_words: significant_words("Some Show"), + upgrade_min_gain: None, + } + } + + #[test] + fn better_resolution_available_ignores_a_sibling_episodes_1080p() { + let target = tv_search_target(1, 1); + let items = [ + raw_item("Some Show S01E01 720p WEB-DL H264"), + raw_item("Some Show S01E02 1080p WEB-DL H264"), + ]; + let flag = better_resolution_available(&target, &items); + assert!( + !flag, + "E02's 1080p must not count as a better option for E01" + ); + let parsed = parser::parse("Some Show S01E01 720p WEB-DL H264"); + let ctx = GateContext { + seeders: Some(50), + size_bytes: Some(400_000_000), + runtime_minutes: Some(24), + has_english_audio: true, + is_anime: false, + better_resolution_available: flag, + is_season_pack: false, + }; + assert_eq!( + scoring::evaluate_gates(&parsed, &ctx, &QualityProfile::default_tv()), + scoring::GateResult::Accept + ); + } + + #[test] + fn better_resolution_available_is_true_for_this_episodes_own_1080p() { + let target = tv_search_target(1, 1); + let items = [ + raw_item("Some Show S01E01 720p WEB-DL"), + raw_item("Some Show S01E01 1080p WEB-DL"), + ]; + assert!(better_resolution_available(&target, &items)); + } + + #[test] + fn better_resolution_available_counts_a_season_pack_of_this_season() { + let target = tv_search_target(1, 1); + let items = [ + raw_item("Some Show S01E01 720p WEB-DL"), + raw_item("Some Show S01 Complete 1080p WEB-DL"), + ]; + assert!(better_resolution_available(&target, &items)); + } + + #[test] + fn better_resolution_available_for_a_movie_accepts_any_title_relevant_1080p() { + let target = SearchTarget { + media_item_id: 1, + episode_id: None, + season_number: None, + episode_number: None, + query: "Some Movie".into(), + route: SearchRoute::Tpb, + title_words: significant_words("Some Movie"), + upgrade_min_gain: None, + }; + let items = [raw_item("Some Movie 2024 1080p BluRay")]; + assert!(better_resolution_available(&target, &items)); + } + + fn seeded_e05_e06_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + conn.execute( + "INSERT INTO media_item (id, kind, title, tvdb_id, monitored, quality_profile_id, root_folder) + VALUES (1, 'series', 'Show', 12345, 1, 1, '/tmp')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file) + VALUES (5, 1, 1, 5, 1, 0), (6, 1, 1, 6, 1, 0)", + [], + ) + .unwrap(); + conn + } + + #[test] + fn resolve_grab_episode_rebinds_to_the_parsed_episode_not_the_tui_selection() { + let conn = seeded_e05_e06_conn(); + let media_item = get_media_item(&conn, 1).unwrap(); + let parsed = parser::parse("Show.S01E06.1080p"); + let bound = resolve_grab_episode(&conn, &media_item, &parsed, Some(5)).unwrap(); + assert_eq!(bound, Some(6)); + } + + #[test] + fn resolve_grab_episode_clears_episode_id_for_a_season_pack() { + let conn = seeded_e05_e06_conn(); + let media_item = get_media_item(&conn, 1).unwrap(); + let parsed = parser::parse("Show.S01.COMPLETE.1080p"); + let bound = resolve_grab_episode(&conn, &media_item, &parsed, Some(5)).unwrap(); + assert_eq!(bound, None); + } + + #[test] + fn resolve_grab_episode_keeps_caller_id_when_the_title_does_not_resolve() { + let conn = seeded_e05_e06_conn(); + let media_item = get_media_item(&conn, 1).unwrap(); + let parsed = parser::parse("Show.1080p.WEB-DL"); + let bound = resolve_grab_episode(&conn, &media_item, &parsed, Some(5)).unwrap(); + assert_eq!(bound, Some(5)); + } + + fn insert_test_source(conn: &Connection) { + conn.execute( + "INSERT INTO source (id, name, kind, base_url) VALUES (1, 'test', 'scrape', 'http://x')", + [], + ) + .ok(); + } + + #[test] + fn find_monitored_missing_episode_excludes_an_in_flight_release() { + let conn = seeded_conn(); + insert_test_source(&conn); + let episode_id = find_monitored_missing_episode(&conn, 1, 1, 1) + .unwrap() + .expect("seeded E01 should be missing"); + conn.execute( + "INSERT INTO release (media_item_id, episode_id, raw_title, source_id, guid, status, grabbed_at) + VALUES (1, ?1, 'Show S01E01', 1, 'guid-ep', 'grabbed', datetime('now'))", + params![episode_id], + ) + .unwrap(); + assert!(find_monitored_missing_episode(&conn, 1, 1, 1) + .unwrap() + .is_none()); + } + + #[test] + fn find_monitored_missing_episode_excludes_an_in_flight_season_pack() { + let conn = seeded_conn(); + insert_test_source(&conn); + conn.execute( + "INSERT INTO release (media_item_id, episode_id, season_number, raw_title, source_id, guid, status, grabbed_at) + VALUES (1, NULL, 1, 'Show S01 Complete', 1, 'guid-pack', 'downloading', datetime('now'))", + [], + ) + .unwrap(); + assert!(find_monitored_missing_episode(&conn, 1, 1, 1) + .unwrap() + .is_none()); + } + + #[test] + fn enumerate_search_targets_excludes_episodes_covered_by_an_in_flight_pack() { + let conn = search_enumeration_conn(); + // E01 is the only aired missing episode of media_item 1 that isn't + // already in-flight. Cover the season with a pack and it must drop + // out of search enum along with any sibling. + conn.execute( + "INSERT INTO episode (id, media_item_id, season_number, episode_number, monitored, has_file, air_date) + VALUES (10, 1, 1, 10, 1, 0, '2020-01-01')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO release (media_item_id, episode_id, season_number, raw_title, source_id, guid, status, grabbed_at) + VALUES (1, NULL, 1, 'Some Show S01 Complete', 1, 'guid-pack', 'grabbed', datetime('now'))", + [], + ) + .unwrap(); + let targets = enumerate_search_targets(&conn, 100).unwrap(); + assert!( + !targets.iter().any(|t| t.media_item_id == 1), + "in-flight season pack should hide every missing episode of that season" + ); + } + + #[test] + fn grab_target_in_flight_is_true_for_an_episode_with_a_grabbed_release() { + let conn = seeded_e05_e06_conn(); + insert_test_source(&conn); + conn.execute( + "INSERT INTO release (media_item_id, episode_id, raw_title, source_id, guid, status, grabbed_at) + VALUES (1, 6, 'Show S01E06', 1, 'guid-e06', 'grabbed', datetime('now'))", + [], + ) + .unwrap(); + let media_item = get_media_item(&conn, 1).unwrap(); + assert!(grab_target_in_flight(&conn, &media_item, Some(6), None).unwrap()); + assert!(!grab_target_in_flight(&conn, &media_item, Some(5), None).unwrap()); + } + + #[test] + fn grab_target_in_flight_is_true_when_a_season_pack_is_already_downloading() { + let conn = seeded_e05_e06_conn(); + insert_test_source(&conn); + conn.execute( + "INSERT INTO release (media_item_id, episode_id, season_number, raw_title, source_id, guid, status, grabbed_at) + VALUES (1, NULL, 1, 'Show S01 Complete', 1, 'guid-pack', 'downloading', datetime('now'))", + [], + ) + .unwrap(); + let media_item = get_media_item(&conn, 1).unwrap(); + assert!(grab_target_in_flight(&conn, &media_item, Some(5), None).unwrap()); + assert!(grab_target_in_flight(&conn, &media_item, None, Some(1)).unwrap()); + } + + #[test] + fn reject_unresolved_manual_grab_link_allows_magnet_and_torrent_only() { + assert!(reject_unresolved_manual_grab_link("magnet:?xt=urn:btih:deadbeef").is_ok()); + assert!(reject_unresolved_manual_grab_link("https://example.invalid/file.torrent").is_ok()); + let err = reject_unresolved_manual_grab_link("https://1337x.to/torrent/123/") + .unwrap_err() + .to_string(); + assert!( + err.contains("magnet or .torrent URL"), + "unexpected error: {err}" + ); + } } diff --git a/breadarrd/src/scoring/score.rs b/breadarrd/src/scoring/score.rs index 867b302..a720a68 100644 --- a/breadarrd/src/scoring/score.rs +++ b/breadarrd/src/scoring/score.rs @@ -1,4 +1,4 @@ -use crate::parser::ParsedRelease; +use crate::parser::{Codec, ParsedRelease, Source}; use super::profile::{ProfileKind, QualityProfile}; @@ -8,8 +8,8 @@ pub fn score(parsed: &ParsedRelease, seeders: u32, has_hdr: bool, profile: &Qual total += w.seeder * seeder_score(seeders); total += w.resolution_tier * resolution_tier(parsed.resolution); - total += w.source_tier * parsed.source.map(|s| s as u8 as f32).unwrap_or(0.0); - total += w.codec_tier * parsed.codec.map(|c| c as u8 as f32).unwrap_or(0.0); + total += w.source_tier * source_tier(parsed.source); + total += w.codec_tier * codec_tier(parsed.codec); if parsed.bit_depth == Some(10) { total += w.bit_depth; @@ -52,6 +52,29 @@ fn resolution_tier(resolution: Option) -> f32 { } } +/// Explicit tiers — `Source::Hdtv` is discriminant 0, so casting the enum +/// to `u8` scored HDTV identically to an unknown/missing source. +fn source_tier(source: Option) -> f32 { + match source { + Some(Source::Hdtv) => 1.0, + Some(Source::WebRip) => 2.0, + Some(Source::WebDl) => 3.0, + Some(Source::BluRay) => 4.0, + Some(Source::Remux) => 5.0, + None => 0.0, + } +} + +/// Same reason as `source_tier`: `Codec::H264` is discriminant 0. +fn codec_tier(codec: Option) -> f32 { + match codec { + Some(Codec::H264) => 1.0, + Some(Codec::Hevc) => 2.0, + Some(Codec::Av1) => 3.0, + None => 0.0, + } +} + #[cfg(test)] mod tests { use super::*; @@ -152,6 +175,36 @@ mod tests { assert!(s720 > s480); } + #[test] + fn hdtv_h264_scores_strictly_above_a_release_with_no_source_or_codec() { + let profile = QualityProfile::default_tv(); + let known = parser::parse("Show S01E01 1080p HDTV H264"); + let unknown = parser::parse("Show S01E01 1080p"); + assert!(known.source.is_some()); + assert!(known.codec.is_some()); + assert!(unknown.source.is_none()); + assert!(unknown.codec.is_none()); + assert!(score(&known, 50, false, &profile) > score(&unknown, 50, false, &profile)); + } + + #[test] + fn remux_av1_scores_above_hdtv_h264() { + let profile = QualityProfile::default_movie(); + let remux_av1 = parser::parse("Movie 2024 2160p Remux AV1"); + let hdtv_h264 = parser::parse("Movie 2024 2160p HDTV H264"); + assert!(score(&remux_av1, 50, false, &profile) > score(&hdtv_h264, 50, false, &profile)); + } + + #[test] + fn parsed_hdr_movie_scores_higher_than_the_same_release_without_hdr() { + let profile = QualityProfile::default_movie(); + let hdr = parser::parse("Movie.2024.2160p.BluRay.HDR.H264"); + let sdr = parser::parse("Movie.2024.2160p.BluRay.H264"); + assert!(hdr.has_hdr); + assert!(!sdr.has_hdr); + assert!(score(&hdr, 50, hdr.has_hdr, &profile) > score(&sdr, 50, sdr.has_hdr, &profile)); + } + #[test] fn allowlisted_group_scores_higher_than_unlisted() { let mut profile = QualityProfile::default_tv(); diff --git a/breadarrd/src/sources/mod.rs b/breadarrd/src/sources/mod.rs index f6676ca..f4a32e1 100644 --- a/breadarrd/src/sources/mod.rs +++ b/breadarrd/src/sources/mod.rs @@ -1,6 +1,8 @@ pub mod rss; pub mod scrape; +pub mod torrents_csv; pub mod tpb; +pub mod yts; use anyhow::Result; use async_trait::async_trait; @@ -25,6 +27,39 @@ pub trait ReleaseSource { async fn fetch(&self, query: Option<&str>) -> Result>; } +/// Trackers attached to every magnet we synthesize from an info-hash +/// (TPB, torrents-csv, YTS). Same set the TPB client has used since it +/// landed — qBittorrent needs *some* announce list or the torrent sits +/// hash-only until DHT finds peers. +const MAGNET_TRACKERS: &[&str] = &[ + "udp://tracker.opentrackr.org:1337/announce", + "udp://open.stealth.si:80/announce", + "udp://tracker.torrent.eu.org:451/announce", + "udp://tracker.openbittorrent.com:6969/announce", + "udp://exodus.desync.com:6969/announce", +]; + +/// A valid BitTorrent v1 info_hash: 40 hex chars or 32 base32 chars — same +/// shape `qbit::extract_btih` accepts out of a magnet URI. Shared by every +/// hash-to-magnet source so a malformed value can't silently produce a +/// magnet the grab path then fails to parse back. +pub(crate) fn is_valid_info_hash(hash: &str) -> bool { + (hash.len() == 40 && hash.bytes().all(|b| b.is_ascii_hexdigit())) + || (hash.len() == 32 + && hash + .bytes() + .all(|b| matches!(b, b'2'..=b'7' | b'a'..=b'z' | b'A'..=b'Z'))) +} + +pub(crate) fn build_magnet(info_hash: &str, name: &str) -> String { + let mut magnet = format!("magnet:?xt=urn:btih:{info_hash}&dn={}", urlencode(name)); + for t in MAGNET_TRACKERS { + magnet.push_str("&tr="); + magnet.push_str(&urlencode(t)); + } + magnet +} + pub(crate) fn urlencode(s: &str) -> String { s.chars() .map(|c| { @@ -45,7 +80,17 @@ pub(crate) fn urlencode(s: &str) -> String { pub(crate) fn parse_human_size(s: &str) -> Option { let s = s.trim(); - let (num_part, unit) = s.split_once(' ')?; + // Split on the first character that isn't part of the number, rather + // than requiring a literal space — some sources render this without one + // ("38.1GiB"). Requiring a space made `split_once(' ')` return `None` + // for those, silently leaving `size_bytes` unset rather than failing + // outright: the gate's size sanity check (`gate.rs`) treats a missing + // size as "nothing to check" and skips it entirely instead of rejecting + // the release, so a value this parser simply couldn't read bypassed + // size validation altogether rather than being caught by it. + let split_at = s.find(|c: char| !(c.is_ascii_digit() || c == '.'))?; + let (num_part, unit) = s.split_at(split_at); + let unit = unit.trim(); let num: f64 = num_part.parse().ok()?; let mult = match unit { "B" => 1.0, @@ -80,4 +125,14 @@ mod tests { fn rejects_unknown_unit() { assert_eq!(parse_human_size("5 XiB"), None); } + + // Regression test for a real gap found in review: some sources render + // this with no space between the number and the unit — the old + // `split_once(' ')` returned `None` for those, silently leaving + // `size_bytes` unset (which skips the gate's size sanity check entirely) + // rather than rejecting a value this parser genuinely couldn't read. + #[test] + fn parses_a_size_with_no_space_before_the_unit() { + assert_eq!(parse_human_size("38.1GiB"), Some(40909563494)); + } } diff --git a/breadarrd/src/sources/rss.rs b/breadarrd/src/sources/rss.rs index 34adf96..5425646 100644 --- a/breadarrd/src/sources/rss.rs +++ b/breadarrd/src/sources/rss.rs @@ -52,6 +52,39 @@ fn build_search_url(feed_url: &str, query: Option<&str>) -> String { } } +/// Every field accumulated across one ``'s `Text`/`CData` events, +/// bundled into one struct (rather than six separate `&mut Option<_>` +/// parameters) purely to keep `accumulate_field` under clippy's +/// too-many-arguments threshold. +#[derive(Default)] +struct ItemFields { + title: Option, + link: Option, + guid: Option, + seeders: Option, + leechers: Option, + size_bytes: Option, +} + +/// Appends rather than overwrites title/link/guid: a real feed can split a +/// single logical value across more than one `Text`/`CData` event for the +/// same tag (mixed content, or just a parser buffer boundary) — a plain +/// assignment would silently keep only the *last* fragment, truncating the +/// value. `nyaa:seeders`/`nyaa:leechers`/`nyaa:size` stay parse-and-overwrite +/// since they're short numeric/size tokens, not free text expected to span +/// multiple events. +fn accumulate_field(tag: &str, text: &str, fields: &mut ItemFields) { + match tag { + "title" => fields.title.get_or_insert_with(String::new).push_str(text), + "link" => fields.link.get_or_insert_with(String::new).push_str(text), + "guid" => fields.guid.get_or_insert_with(String::new).push_str(text), + "nyaa:seeders" => fields.seeders = text.parse().ok(), + "nyaa:leechers" => fields.leechers = text.parse().ok(), + "nyaa:size" => fields.size_bytes = parse_human_size(text), + _ => {} + } +} + fn parse_nyaa_rss(bytes: &[u8]) -> Result> { let mut reader = Reader::from_reader(bytes); reader.config_mut().trim_text(true); @@ -61,12 +94,7 @@ fn parse_nyaa_rss(bytes: &[u8]) -> Result> { let mut in_item = false; let mut cur_tag = String::new(); - let mut title = None; - let mut link = None; - let mut guid = None; - let mut seeders = None; - let mut leechers = None; - let mut size_bytes = None; + let mut fields = ItemFields::default(); loop { match reader.read_event_into(&mut buf)? { @@ -75,41 +103,42 @@ fn parse_nyaa_rss(bytes: &[u8]) -> Result> { let name = String::from_utf8_lossy(e.name().as_ref()).into_owned(); if name == "item" { in_item = true; - title = None; - link = None; - guid = None; - seeders = None; - leechers = None; - size_bytes = None; + fields = ItemFields::default(); } cur_tag = name; } Event::Text(t) if in_item => { let raw = t.decode()?; let text = unescape(&raw)?.into_owned(); - match cur_tag.as_str() { - "title" => title = Some(text), - "link" => link = Some(text), - "guid" => guid = Some(text), - "nyaa:seeders" => seeders = text.parse().ok(), - "nyaa:leechers" => leechers = text.parse().ok(), - "nyaa:size" => size_bytes = parse_human_size(&text), - _ => {} - } + accumulate_field(&cur_tag, &text, &mut fields); + } + // CDATA content is raw text by definition — XML entity escaping + // doesn't apply inside a CDATA section (running `unescape()` on + // it would misinterpret a literal "&" as an escaped + // ampersand), so this decodes without it. Many real-world feeds + // wrap ``/`<link>` in CDATA; previously only + // `Event::Text` was handled at all, so those items silently + // came back with `title = None` and were dropped at the + // `item`-close check below with zero error — pointing + // `nyaa_rss_url` at a CDATA-heavy feed yielded zero items, not + // a visible failure. + Event::CData(t) if in_item => { + let text = t.decode()?.into_owned(); + accumulate_field(&cur_tag, &text, &mut fields); } Event::End(e) => { let name = String::from_utf8_lossy(e.name().as_ref()).into_owned(); if name == "item" { if let (Some(title), Some(link), Some(guid)) = - (title.take(), link.take(), guid.take()) + (fields.title.take(), fields.link.take(), fields.guid.take()) { items.push(RawReleaseItem { title, link, guid, - size_bytes, - seeders, - leechers, + size_bytes: fields.size_bytes, + seeders: fields.seeders, + leechers: fields.leechers, }); } in_item = false; @@ -156,6 +185,40 @@ mod tests { assert_eq!(item.size_bytes, Some(373817344)); } + // Regression test for a real gap found in review: many real-world feeds + // wrap `<title>`/`<link>`/`<guid>` in CDATA rather than plain text + // content (often to avoid having to XML-escape ampersands/brackets + // common in release titles). Previously only `Event::Text` was + // handled — `Event::CData` was silently ignored — so every field + // wrapped this way came back `None` and the whole item was dropped at + // the `item`-close check with no error surfaced at all. + #[test] + fn parses_cdata_wrapped_fields() { + const CDATA_SAMPLE: &str = r#"<?xml version="1.0" encoding="UTF-8"?> +<rss xmlns:nyaa="https://nyaa.si/xmlns/nyaa" version="2.0"> +<channel> +<item> + <title><![CDATA[[Group] Some Show & Friends - 05 [1080p]]]> + + + 12 + 3 + 356.5 MiB + + +"#; + + let items = parse_nyaa_rss(CDATA_SAMPLE.as_bytes()).unwrap(); + assert_eq!(items.len(), 1, "a CDATA-wrapped item must not be silently dropped"); + let item = &items[0]; + // A literal "&" survives verbatim — CDATA content isn't + // XML-entity-escaped, so this must NOT come back as "&". + assert_eq!(item.title, "[Group] Some Show & Friends - 05 [1080p]"); + assert_eq!(item.link, "https://nyaa.si/download/2130903.torrent"); + assert_eq!(item.guid, "https://nyaa.si/view/2130903"); + assert_eq!(item.seeders, Some(12)); + } + #[test] fn build_search_url_appends_query_param() { assert_eq!( diff --git a/breadarrd/src/sources/scrape.rs b/breadarrd/src/sources/scrape.rs index 9171198..5fffc73 100644 --- a/breadarrd/src/sources/scrape.rs +++ b/breadarrd/src/sources/scrape.rs @@ -313,6 +313,9 @@ fn parse_search_results(html: &str, mirror: &str) -> Option> continue; } let detail_url = if href.starts_with("http") { + if !same_origin(href, mirror) { + continue; + } href.to_string() } else { format!("{mirror}{href}") @@ -360,6 +363,27 @@ fn parse_search_results(html: &str, mirror: &str) -> Option> Some(items) } +/// Scheme+host(+port) origin of an `http(s)://...` URL. `None` if the +/// string isn't an absolute http(s) URL with a host. +fn url_origin(url: &str) -> Option<(&str, &str)> { + let (scheme, rest) = if let Some(r) = url.strip_prefix("https://") { + ("https", r) + } else { + let r = url.strip_prefix("http://")?; + ("http", r) + }; + let hostport = rest.split('/').next().filter(|s| !s.is_empty())?; + let hostport = hostport.rsplit('@').next().unwrap_or(hostport); + Some((scheme, hostport)) +} + +fn same_origin(href: &str, mirror: &str) -> bool { + match (url_origin(href), url_origin(mirror)) { + (Some((as_, ah)), Some((bs, bh))) => as_ == bs && ah.eq_ignore_ascii_case(bh), + _ => false, + } +} + fn extract_magnet(html: &str) -> Option { let doc = Html::parse_document(html); let sel = Selector::parse(r#"a[href^="magnet:"]"#).unwrap(); @@ -420,6 +444,32 @@ mod tests { assert_eq!(a[0].guid, b[0].guid); } + #[test] + fn drops_off_origin_absolute_hrefs_but_keeps_relative() { + let html = r#" + + + + + + + + + + + + + + + + +
nameselesize
Evil111 MB
Good222 MB
"#; + let items = parse_search_results(html, "https://13377x.info").unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].title, "Good"); + assert_eq!(items[0].link, "https://13377x.info/torrent/3250239/Ok/"); + } + #[test] fn extract_torrent_id_handles_relative_and_absolute_hrefs() { assert_eq!( diff --git a/breadarrd/src/sources/torrents_csv.rs b/breadarrd/src/sources/torrents_csv.rs new file mode 100644 index 0000000..ea9cb06 --- /dev/null +++ b/breadarrd/src/sources/torrents_csv.rs @@ -0,0 +1,130 @@ +use anyhow::{Context, Result}; +use async_trait::async_trait; +use serde::Deserialize; + +use super::{build_magnet, is_valid_info_hash, urlencode, RawReleaseItem, ReleaseSource}; + +/// Public JSON search over the torrents.csv DHT dump. Same grab shape as +/// TPB (info-hash → magnet, no HTML): used as the first general-content +/// fallback when apibay is down or returns nothing. Seeders are scrape +/// snapshots, not live tracker data, so a high number can still stall — +/// the existing seeder gate still applies. +pub struct TorrentsCsvSource { + api_url: String, + client: reqwest::Client, +} + +#[derive(Deserialize)] +struct CsvResponse { + #[serde(default)] + torrents: Vec, +} + +#[derive(Deserialize)] +struct CsvTorrent { + infohash: String, + name: String, + size_bytes: Option, + seeders: Option, + leechers: Option, +} + +impl TorrentsCsvSource { + pub fn new(api_url: impl Into) -> Self { + Self { + api_url: api_url.into(), + client: reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("reqwest client build"), + } + } +} + +fn as_u32(n: Option) -> Option { + n.and_then(|v| u32::try_from(v.max(0)).ok()) +} + +#[async_trait] +impl ReleaseSource for TorrentsCsvSource { + async fn fetch(&self, query: Option<&str>) -> Result> { + let Some(query) = query else { + anyhow::bail!( + "TorrentsCsvSource requires a search query (this is a search-driven source, not a feed)" + ); + }; + let sep = if self.api_url.contains('?') { '&' } else { '?' }; + let url = format!( + "{}{sep}q={}&size=25", + self.api_url.trim_end_matches('/'), + urlencode(query) + ); + let parsed: CsvResponse = self + .client + .get(&url) + .send() + .await + .with_context(|| format!("request to {url} failed"))? + .error_for_status() + .with_context(|| format!("{url} returned an error status"))? + .json() + .await + .context("failed to parse torrents-csv response as JSON")?; + + Ok(parsed + .torrents + .into_iter() + .filter(|t| is_valid_info_hash(&t.infohash)) + .map(|t| RawReleaseItem { + title: t.name.clone(), + link: build_magnet(&t.infohash, &t.name), + guid: t.infohash, + size_bytes: t.size_bytes, + seeders: as_u32(t.seeders), + leechers: as_u32(t.leechers), + }) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_real_captured_response() { + let body = r#"{ + "torrents": [ + { + "infohash": "ed0da850c273e3e15a819bdcbbf418bc85107ec8", + "name": "Dune (2021) [1080p] [WEBRip]", + "size_bytes": 2947989023, + "seeders": 746, + "leechers": 38 + }, + { + "infohash": "not-a-hash", + "name": "garbage", + "size_bytes": 1, + "seeders": 0, + "leechers": 0 + } + ] + }"#; + let parsed: CsvResponse = serde_json::from_str(body).unwrap(); + let items: Vec<_> = parsed + .torrents + .into_iter() + .filter(|t| is_valid_info_hash(&t.infohash)) + .collect(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].name, "Dune (2021) [1080p] [WEBRip]"); + assert_eq!(items[0].seeders, Some(746)); + } + + #[test] + fn empty_payload_is_no_results_not_an_error() { + let parsed: CsvResponse = serde_json::from_str(r#"{"torrents":[]}"#).unwrap(); + assert!(parsed.torrents.is_empty()); + } +} diff --git a/breadarrd/src/sources/tpb.rs b/breadarrd/src/sources/tpb.rs index b6730d6..6db4f35 100644 --- a/breadarrd/src/sources/tpb.rs +++ b/breadarrd/src/sources/tpb.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use async_trait::async_trait; use serde::Deserialize; -use super::{urlencode, RawReleaseItem, ReleaseSource}; +use super::{build_magnet, is_valid_info_hash, urlencode, RawReleaseItem, ReleaseSource}; /// A community-run JSON API mirror of The Pirate Bay's search — unlike /// 1337x, this is a genuine machine-readable API (not HTML scraping), and @@ -27,23 +27,6 @@ struct TpbResult { size: String, } -const TRACKERS: &[&str] = &[ - "udp://tracker.opentrackr.org:1337/announce", - "udp://open.stealth.si:80/announce", - "udp://tracker.torrent.eu.org:451/announce", - "udp://tracker.openbittorrent.com:6969/announce", - "udp://exodus.desync.com:6969/announce", -]; - -fn build_magnet(info_hash: &str, name: &str) -> String { - let mut magnet = format!("magnet:?xt=urn:btih:{info_hash}&dn={}", urlencode(name)); - for t in TRACKERS { - magnet.push_str("&tr="); - magnet.push_str(&urlencode(t)); - } - magnet -} - impl TpbSource { pub fn new(api_url: impl Into) -> Self { Self { @@ -85,8 +68,15 @@ impl ReleaseSource for TpbSource { // A query with no matches returns a single sentinel row // (id="0", an all-zero info_hash) rather than an empty array — // has to be filtered out explicitly or it'd be treated as one - // real (and completely bogus) result. - .filter(|r| r.id != "0" && !r.info_hash.chars().all(|c| c == '0')) + // real (and completely bogus) result. The all-zero hash is + // itself 40 valid hex characters, so `is_valid_info_hash` alone + // wouldn't catch it — both checks are needed, not one replacing + // the other. + .filter(|r| { + r.id != "0" + && !r.info_hash.chars().all(|c| c == '0') + && is_valid_info_hash(&r.info_hash) + }) .map(|r| RawReleaseItem { title: r.name.clone(), link: build_magnet(&r.info_hash, &r.name), @@ -125,8 +115,37 @@ mod tests { let results: Vec = serde_json::from_str(body).unwrap(); let filtered: Vec<_> = results .into_iter() - .filter(|r| r.id != "0" && !r.info_hash.chars().all(|c| c == '0')) + .filter(|r| { + r.id != "0" + && !r.info_hash.chars().all(|c| c == '0') + && is_valid_info_hash(&r.info_hash) + }) .collect(); assert!(filtered.is_empty()); } + + #[test] + fn is_valid_info_hash_accepts_both_real_shapes() { + assert!(is_valid_info_hash( + "8F87C7C186172F17E35F4512BB1A3E93B614ADED" + )); // 40 hex + assert!(is_valid_info_hash("abcdefghijklmnopqrstuvwxyz234567")); // 32 base32 + } + + // Regression test for a real gap found in review: a malformed + // `info_hash` from apibay used to flow straight into `build_magnet` + // with no validation, silently producing a magnet `qbit::extract_btih` + // can't parse back out — downgrading that grab to the slow polling path + // with no error surfaced anywhere. + #[test] + fn is_valid_info_hash_rejects_malformed_values() { + assert!(!is_valid_info_hash("")); + assert!(!is_valid_info_hash("too-short")); + assert!(!is_valid_info_hash( + "not-a-hex-string-at-all-nope!!!!!!!!!!!!" + )); // 40 chars, non-hex + assert!(!is_valid_info_hash( + "8F87C7C186172F17E35F4512BB1A3E93B614ADE" + )); // 39 hex chars + } } diff --git a/breadarrd/src/sources/yts.rs b/breadarrd/src/sources/yts.rs new file mode 100644 index 0000000..a29bf9c --- /dev/null +++ b/breadarrd/src/sources/yts.rs @@ -0,0 +1,194 @@ +use anyhow::{Context, Result}; +use async_trait::async_trait; +use serde::Deserialize; + +use super::{build_magnet, is_valid_info_hash, urlencode, RawReleaseItem, ReleaseSource}; + +/// YTS movie API. `yts.mx` itself no longer resolves (checked 2026-07-12 +/// and again 2026-08-16); the `yts.lt` / `yts.am` hosts still serve the +/// v2 JSON API, which is why the default URL is a working mirror rather +/// than the brand domain. Movies only — each hit expands into one +/// `RawReleaseItem` per quality so the scorer sees 720p/1080p/2160p as +/// distinct candidates, same as if they were separate TPB rows. +pub struct YtsSource { + api_url: String, + client: reqwest::Client, +} + +#[derive(Deserialize)] +struct YtsResponse { + status: String, + data: Option, +} + +#[derive(Deserialize, Default)] +struct YtsData { + #[serde(default)] + movies: Vec, +} + +#[derive(Deserialize)] +struct YtsMovie { + title: String, + year: Option, + #[serde(default)] + torrents: Vec, +} + +#[derive(Deserialize)] +struct YtsTorrent { + hash: String, + quality: Option, + #[serde(rename = "type")] + source_type: Option, + video_codec: Option, + seeds: Option, + peers: Option, + size_bytes: Option, +} + +impl YtsSource { + pub fn new(api_url: impl Into) -> Self { + Self { + api_url: api_url.into(), + client: reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("reqwest client build"), + } + } +} + +fn as_u32(n: Option) -> Option { + n.and_then(|v| u32::try_from(v.max(0)).ok()) +} + +/// Builds a release title the existing parser can read quality/source/codec +/// out of — YTS stores those as structured fields, not in `title`. +fn release_title(movie: &YtsMovie, torrent: &YtsTorrent) -> String { + let mut title = movie.title.clone(); + if let Some(year) = movie.year { + title.push_str(&format!(" ({year})")); + } + for part in [ + torrent.quality.as_deref(), + torrent.source_type.as_deref(), + torrent.video_codec.as_deref(), + ] + .into_iter() + .flatten() + { + if !part.is_empty() { + title.push_str(&format!(" [{part}]")); + } + } + title +} + +#[async_trait] +impl ReleaseSource for YtsSource { + async fn fetch(&self, query: Option<&str>) -> Result> { + let Some(query) = query else { + anyhow::bail!( + "YtsSource requires a search query (this is a search-driven source, not a feed)" + ); + }; + let sep = if self.api_url.contains('?') { '&' } else { '?' }; + let url = format!( + "{}{sep}query_term={}&limit=20&sort_by=seeds", + self.api_url.trim_end_matches('/'), + urlencode(query) + ); + let parsed: YtsResponse = self + .client + .get(&url) + .send() + .await + .with_context(|| format!("request to {url} failed"))? + .error_for_status() + .with_context(|| format!("{url} returned an error status"))? + .json() + .await + .context("failed to parse YTS response as JSON")?; + anyhow::ensure!( + parsed.status == "ok", + "YTS returned status {:?}", + parsed.status + ); + + let movies = parsed.data.unwrap_or_default().movies; + let mut items = Vec::new(); + for movie in movies { + for torrent in &movie.torrents { + if !is_valid_info_hash(&torrent.hash) { + continue; + } + let title = release_title(&movie, torrent); + items.push(RawReleaseItem { + title: title.clone(), + link: build_magnet(&torrent.hash, &title), + guid: torrent.hash.clone(), + size_bytes: torrent.size_bytes, + seeders: as_u32(torrent.seeds), + leechers: as_u32(torrent.peers), + }); + } + } + Ok(items) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE: &str = r#"{ + "status": "ok", + "data": { + "movie_count": 1, + "movies": [{ + "title": "Dune: Part One", + "year": 2021, + "torrents": [ + { + "hash": "DEB6929BEEB09ADCBD14DC4D6081F7E6B297B88C", + "quality": "1080p", + "type": "web", + "video_codec": "x264", + "seeds": 12, + "peers": 3, + "size_bytes": 2147483648 + }, + { + "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "quality": "720p", + "type": "bluray", + "video_codec": "x265", + "seeds": 4, + "peers": 1, + "size_bytes": 1073741824 + } + ] + }] + } + }"#; + + #[test] + fn expands_one_movie_into_per_quality_rows() { + let parsed: YtsResponse = serde_json::from_str(SAMPLE).unwrap(); + assert_eq!(parsed.status, "ok"); + let movie = &parsed.data.unwrap().movies[0]; + assert_eq!(movie.torrents.len(), 2); + let t0 = release_title(movie, &movie.torrents[0]); + assert_eq!(t0, "Dune: Part One (2021) [1080p] [web] [x264]"); + let t1 = release_title(movie, &movie.torrents[1]); + assert_eq!(t1, "Dune: Part One (2021) [720p] [bluray] [x265]"); + } + + #[test] + fn missing_movies_array_is_empty_not_an_error() { + let parsed: YtsResponse = + serde_json::from_str(r#"{"status":"ok","data":{"movie_count":0}}"#).unwrap(); + assert!(parsed.data.unwrap_or_default().movies.is_empty()); + } +} diff --git a/breadarrd/src/transcode/mod.rs b/breadarrd/src/transcode/mod.rs new file mode 100644 index 0000000..ad2df9b --- /dev/null +++ b/breadarrd/src/transcode/mod.rs @@ -0,0 +1,1898 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; + +use anyhow::{bail, Context, Result}; +use breadarr_shared::config::TranscodeConfig; +use rusqlite::{params, Connection}; +use tokio::sync::Mutex; + +use crate::importer::{self, ffprobe}; +use crate::jellyfin::JellyfinClient; + +/// Computes the target AV1 bitrate for a given resolution, scaled from the +/// configured reference (a real HEVC/H264 bitrate at `reference_height` the +/// user is already happy with) by pixel-count ratio, then discounted by +/// `av1_efficiency_factor` — AV1 reaches equivalent perceived quality to +/// HEVC/H264 at a meaningfully lower bitrate, so a like-for-like copy of the +/// reference bitrate would leave savings on the table. +pub fn target_bitrate_kbps(width: i64, height: i64, cfg: &TranscodeConfig) -> u32 { + let reference_width = cfg.reference_height as f64 * 16.0 / 9.0; + let reference_pixels = reference_width * cfg.reference_height as f64; + let pixel_ratio = ((width * height) as f64 / reference_pixels).max(0.1); + let bitrate = cfg.reference_bitrate_kbps as f64 * pixel_ratio * cfg.av1_efficiency_factor as f64; + bitrate.round() as u32 +} + +/// Runs the live-action GPU encode via `av1_vaapi`, decoding through VAAPI +/// too (`-hwaccel_output_format vaapi`) so the whole pipeline stays on-GPU +/// rather than round-tripping frames through the CPU. Video-only re-encode +/// — every audio/subtitle/data stream is copied verbatim (`-c:a copy -c:s +/// copy -c:d copy`). +/// +/// Rate control is `QVBR` (quality-defined VBR): `-global_quality` is the +/// actual quality target driving output size, `-b:v`/`-maxrate`/`-bufsize` +/// (computed from `target_bitrate_kbps`) are a *ceiling*, not a target — +/// content that's already efficient spends less than the ceiling rather +/// than being inflated up toward it. This replaced a flat `VBR + -b:v` +/// scheme after a real incident: that scheme treated `-b:v` as the target +/// average rather than a cap, so already-efficient sources (some well under +/// the model's own reference bitrate) got re-encoded *larger* than the +/// original on the majority of a real backfill. `encode_and_verify`'s +/// post-encode size check is the actual hard guarantee against that +/// happening again regardless of how this rate-control tuning holds up — +/// this function only has to get it roughly right, not perfectly. +/// +/// Blocking and slow by design (a real GPU encode, potentially minutes per +/// file) — callers must run this inside `tokio::task::spawn_blocking`, never +/// directly on an async task, and never while holding the shared DB mutex. +/// +/// The output container is always Matroska, so this maps streams by type +/// rather than the blanket `-map 0` the first version used — two real +/// gaps that caused, both fixed: +/// - `-map 0` pulled in attached-picture streams (embedded cover art, +/// exposed by ffmpeg as a second `video`-type stream, `mjpeg`/`png`) and +/// `-c:v av1_vaapi` then tried to encode *that* too, which the VAAPI +/// filter chain can't handle — failed the whole job on any file with +/// embedded cover art (several real library files hit this). `-map +/// 0:v:0` selects only the first video stream — the actual content, +/// never the attached-pic that (per `ffprobe::probe`'s own convention) +/// always comes after it — so cover art is silently dropped rather than +/// crashing the encode. +/// - mp4 sources using `mov_text` subtitles aren't valid inside Matroska +/// via stream copy and failed the whole job. `subtitle_codec_args` +/// converts just the `mov_text` streams to `srt` (a lossless format +/// change for plain timed text) and copies everything else untouched +/// (so an already-Matroska-compatible `ass`/`srt` track keeps its +/// styling rather than being flattened). +/// +/// Rate control mode is `ICQ`, not `QVBR` — a live validation run found +/// Hestia's iHD driver rejects `QVBR` outright ("Driver does not support +/// QVBR RC mode (supported modes: CQP, CBR, VBR, ICQ)"), despite `ffmpeg -h +/// encoder=av1_vaapi` listing `QVBR` as a libavcodec-level option; the +/// *driver* is the actual authority on what's usable, and libavcodec +/// doesn't validate that ahead of time. `ICQ` is the quality-driven mode +/// this driver actually has, so it's the correct target regardless — the +/// `-b:v`/`-maxrate`/`-bufsize` ceiling stays as a best-effort cap (harmless +/// if this driver ignores it under ICQ; `encode_and_verify`'s post-encode +/// `is_beneficial` check is the actual hard guarantee either way, not this). +/// +/// `-loglevel error` suppresses ffmpeg's default per-frame progress output +/// (`frame=... fps=... bitrate=... speed=...`, one line per progress tick) +/// — for a long encode this otherwise accumulates to megabytes in the +/// `Command::output()`-captured stderr buffer, which is pure noise in a +/// failure's stored `error` text and unnecessary memory held by the parent +/// process for the whole encode's duration. +/// +/// Decode is `-hwaccel vaapi` but deliberately *without* +/// `-hwaccel_output_format vaapi` — a live validation run found that +/// combination fails even on the simplest possible single-stream input +/// ("Impossible to convert between the formats supported by the filter +/// 'Parsed_null_0' and the filter 'auto_scale_0'" / "Function not +/// implemented"), because it makes ffmpeg keep the decoded frame +/// GPU-resident and then implicitly spin up a *second*, separate VAAPI +/// device context to bridge into the encoder — the two contexts don't +/// negotiate a compatible format with each other on this driver, wrong +/// devices or not. Explicitly downloading to a normal system-memory frame +/// (`-hwaccel_output_format` omitted) and re-uploading through the *same* +/// device context via `-vf format=nv12,hwupload` avoids that implicit +/// second context entirely, and is what actually produces a valid, +/// decodable AV1 output in practice — confirmed against real hardware +/// before trusting it further. +fn run_ffmpeg_encode_live_action( + input: &Path, + output: &Path, + bitrate_ceiling_kbps: u32, + quality: u32, + vaapi_device: &str, + subtitles: &[ffprobe::SubtitleStream], +) -> Result<()> { + let maxrate = bitrate_ceiling_kbps * 3 / 2; + let bufsize = bitrate_ceiling_kbps * 2; + + let result = Command::new("ffmpeg") + .arg("-y") + .args(["-loglevel", "error"]) + .args(["-hwaccel", "vaapi"]) + .args(["-hwaccel_device", vaapi_device]) + .arg("-i") + .arg(input) + .args(["-map", "0:v:0"]) + .args(["-map", "0:a?"]) + .args(["-map", "0:s?"]) + .args(["-map", "0:d?"]) + .args(["-vf", "format=nv12,hwupload"]) + .args(["-c:v", "av1_vaapi"]) + .args(["-rc_mode", "ICQ"]) + .args(["-global_quality", &quality.to_string()]) + .args(["-b:v", &format!("{bitrate_ceiling_kbps}k")]) + .args(["-maxrate", &format!("{maxrate}k")]) + .args(["-bufsize", &format!("{bufsize}k")]) + .args(["-c:a", "copy"]) + .args(subtitle_codec_args(subtitles)) + .args(["-c:d", "copy"]) + // Explicit rather than relying on `output`'s extension to imply the + // muxer: `temp_path_for` deliberately gives the scratch file a + // non-video extension so library scans skip it (see its own doc + // comment), which would otherwise leave ffmpeg unable to guess a + // container from the output path at all. + .args(["-f", "matroska"]) + .arg(output) + .output() + .context("failed to run ffmpeg av1_vaapi encode")?; + + if !result.status.success() { + let _ = std::fs::remove_file(output); + bail!( + "ffmpeg av1_vaapi encode failed: {}", + String::from_utf8_lossy(&result.stderr) + ); + } + Ok(()) +} + +/// Anime pipeline: software `libsvtav1` at 10-bit (`yuv420p10le`), CRF-driven +/// (no bitrate ceiling — this is pure quality-mode, unlike the live-action +/// path). Two reasons this isn't just `run_ffmpeg_encode_live_action` with a +/// different quality number: +/// +/// 1. No hardware AV1 10-bit encode exists on Hestia's Arc A380 — `vainfo` +/// only lists `VAProfileAV1Profile0` (8-bit). Anime art (flat color +/// fields, gradient shading/skies) shows 8-bit banding far more readily +/// than live-action's grain/texture does at an equivalent bitrate, so +/// getting true 10-bit output is worth trading GPU offload for CPU time. +/// 2. `av1_vaapi` exposes no tune/animation-specific options at all (its +/// `AVOption` list is rate-control/GOP-structure knobs only) — a +/// meaningfully different anime pipeline wasn't achievable on the +/// hardware encoder regardless of quality value chosen. +/// +/// Fully software: decode is left default (not `-hwaccel vaapi`) since a +/// software-encode target gains nothing from a hardware-decoded/GPU-resident +/// frame (it would need `hwdownload` back to system memory anyway), and +/// `libsvtav1` is overwhelmingly the throughput bottleneck either way. +/// +/// Same blocking/slow-by-design and stream-mapping/subtitle-codec handling +/// as `run_ffmpeg_encode_live_action` (attached-pic cover art dropped via +/// `-map 0:v:0`, `mov_text` subtitles converted to `srt`) apply here too. +/// +/// `max_threads` is passed through as `-svtav1-params lp=N` (SVT-AV1's own +/// "logical processors" cap) — a real validation run on Hestia's 6c/12t box +/// let `libsvtav1` use every thread at once with no explicit bound, and a +/// single 1080p episode grew to **9.3GB resident memory** before the kernel +/// OOM-killer took it out (no other services lost, but a second real OOM +/// incident is a second time too many). SVT-AV1's memory use scales with +/// both preset and thread count — more parallel workers means more +/// concurrently-buffered lookahead/reference frames — so capping threads +/// bounds peak memory to something predictable regardless of how many +/// cores the host actually has, independent of whatever preset is chosen. +/// +/// `-loglevel error` for the same reason as the live-action encoder — this +/// path runs for tens of minutes per file, and unsuppressed progress output +/// would otherwise dominate the captured stderr buffer for that whole time. +fn run_ffmpeg_encode_anime( + input: &Path, + output: &Path, + crf: u32, + preset: u32, + max_threads: u32, + subtitles: &[ffprobe::SubtitleStream], +) -> Result<()> { + let result = Command::new("ffmpeg") + .arg("-y") + .args(["-loglevel", "error"]) + .arg("-i") + .arg(input) + .args(["-map", "0:v:0"]) + .args(["-map", "0:a?"]) + .args(["-map", "0:s?"]) + .args(["-map", "0:d?"]) + .args(["-c:v", "libsvtav1"]) + .args(["-pix_fmt", "yuv420p10le"]) + .args(["-crf", &crf.to_string()]) + .args(["-preset", &preset.to_string()]) + .args(["-svtav1-params", &format!("lp={max_threads}")]) + .args(["-c:a", "copy"]) + .args(subtitle_codec_args(subtitles)) + .args(["-c:d", "copy"]) + // See the matching comment in `run_ffmpeg_encode_live_action`. + .args(["-f", "matroska"]) + .arg(output) + .output() + .context("failed to run ffmpeg libsvtav1 encode")?; + + if !result.status.success() { + let _ = std::fs::remove_file(output); + bail!( + "ffmpeg libsvtav1 encode failed: {}", + String::from_utf8_lossy(&result.stderr) + ); + } + Ok(()) +} + +/// Per-subtitle-stream `-c:s:{i}` codec args, in the same order `-map +/// 0:s?` presents them in the output. `mov_text` (mp4's timed-text codec) +/// isn't valid inside the Matroska container this pipeline always writes, +/// so it's converted to `srt` — a lossless format change for plain timed +/// text. Everything else (`ass`, `srt`, etc. — already Matroska-compatible) +/// is copied untouched rather than flattened through a lossy re-encode. +/// Pulled out as its own pure function so this exact per-stream decision — +/// the thing that made every mp4-sourced file with subtitles fail outright +/// — has direct unit coverage without needing a real ffmpeg encode. +fn subtitle_codec_args(subtitles: &[ffprobe::SubtitleStream]) -> Vec { + subtitles + .iter() + .enumerate() + .flat_map(|(i, s)| { + let codec = if s.codec.as_deref() == Some("mov_text") { "srt" } else { "copy" }; + [format!("-c:s:{i}"), codec.to_string()] + }) + .collect() +} + +/// The temp path a given job's encode writes to — job-id-scoped (not just +/// derived from the input filename) so two jobs can never collide on the +/// same temp file even if something ends up processing the same +/// `episode_file` path twice (e.g. a daemon restart racing a still-live +/// `transcode-library` backfill process). Also what `reset_orphaned_ +/// running_jobs` uses to find and clean up a crashed job's leftover partial. +/// +/// Written into the *library* directory (same filesystem as the original — +/// required for the atomic rename-based swap in `finalize_job`, never a +/// separate staging drive) for however long the encode takes (minutes to +/// hours). Deliberately given a non-video final extension (`.tmp`, not +/// `.mkv`) and a leading dot: a real production shape found in review — a +/// growing `....mkv` sitting mid-encode in the library was itself picked up +/// by `collect_video_files`/`walk_files` (both filter on `VIDEO_EXTS`), so +/// `find_relinkable_episode_files` saw two candidate files for the same +/// `SxxExx` and reported the episode "ambiguous," `library_scan` indexed +/// the partial file, and Jellyfin could index/attempt to play a +/// still-encoding file mid-scan. `.tmp` sidesteps every one of those +/// `VIDEO_EXTS`-based scans without needing to special-case any of them +/// individually; `-f matroska` is passed explicitly to both encode +/// functions so ffmpeg doesn't need `output`'s extension to infer the +/// container now that it's no longer `.mkv`-shaped. +/// +/// Known cosmetic limitation, unrelated to the above: `finalize_job` +/// renames this over the *original* path verbatim, keeping whatever +/// extension the source had — an `.mp4` source ends up as Matroska bytes at +/// an `.mp4` path (Jellyfin content-sniffs, so playback isn't affected, but +/// `episode_file.path`'s extension no longer matches the real container). +/// Not fixed here; would need the rename plus an `episode_file.path` update +/// done together in `finalize_job`'s transaction. +fn temp_path_for(input: &Path, job_id: i64) -> PathBuf { + let file_name = input + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or("transcode"); + input.with_file_name(format!(".{file_name}.job{job_id}.tmp")) +} + +/// What `encode_and_verify` decided to do. Distinct from a hard `Err` (an +/// actual failure — ffmpeg crashed, output was corrupt, duration mismatched) +/// — both `AlreadyAv1` and `NotBeneficial` are successful, deliberate +/// no-ops: nothing went wrong, there's just no transcode worth keeping. +#[derive(Debug)] +enum EncodeOutcome { + /// The input was already AV1 — no encode was even attempted. + AlreadyAv1, + /// An encode ran (or was skipped pre-emptively) but the result wasn't + /// meaningfully smaller than the original, so it was discarded rather + /// than swapped in. + NotBeneficial, + /// A verified, meaningfully-smaller output ready to be swapped in. + Encoded(PathBuf), +} + +/// The blocking half of a transcode: encode to a temp file alongside the +/// original (same filesystem, required for the atomic rename-based swap +/// later — never a separate staging drive), then verify the result is +/// actually good *before* anything touches the original. Leaves the temp +/// file on disk only for the `Encoded` outcome (the caller finalizes the +/// swap under the DB lock); cleans it up itself in every other case, so the +/// original is never at risk regardless of what happens here. +/// +/// Returns `AlreadyAv1` (no encode run at all) if the input turns out to +/// already be AV1 — checked fresh against the real file here, not trusted +/// from whatever `media_file_probe` said when the job was claimed. This is +/// what makes a job that's re-run after a crash between the file-swap +/// rename and the DB bookkeeping (a narrow but real window — see +/// `finalize_job`) self-correct into a no-op instead of re-encoding an +/// already-AV1 file a second time. `force_reencode` (set only by +/// `find_oversized_av1_candidates`'s remediation jobs) skips this check +/// entirely — for a file that's already AV1 but was mis-encoded (e.g. left +/// larger than its original by the rate-control bug this whole module's +/// history is built around), "already AV1" is exactly the case that *does* +/// need a real re-encode, not a no-op. The `is_beneficial` check further +/// down still applies unconditionally either way — forcing a re-encode +/// attempt never bypasses the "never keep a non-improvement" guarantee. +/// +/// Returns `NotBeneficial` in two cases: (1) a pre-check, before spending +/// any GPU/CPU time, when the source's current bitrate is already at or +/// below `skip_below_ceiling_ratio` of the resolution-scaled ceiling — a +/// strong signal there's little room to save; (2) after encoding, if the +/// result isn't at least `min_size_reduction_pct` smaller than the +/// original. (2) is the actual hard guarantee against a real incident where +/// flat-bitrate rate control silently produced outputs *larger* than the +/// original on most of a real backfill — regardless of how well-tuned the +/// rate control is, this is what makes "never keep a non-improvement" true +/// unconditionally. +#[allow(clippy::too_many_arguments)] +fn encode_and_verify( + input: PathBuf, + job_id: i64, + cfg: TranscodeConfig, + width: i64, + height: i64, + original_duration: Option, + is_anime: bool, + force_reencode: bool, +) -> Result { + // Kept (not just checked and discarded) so its `subtitles` can be + // reused for `subtitle_codec_args` below without a second ffprobe call. + let fresh_probe = ffprobe::probe(&input); + if !force_reencode { + if let Ok(p) = &fresh_probe { + if p.video_codec.as_deref() == Some("av1") { + return Ok(EncodeOutcome::AlreadyAv1); + } + } + } + // Any other outcome (a different codec, or the probe itself failing) + // falls through to a real encode attempt as normal — the check above + // exists only to short-circuit the one case where there's provably + // nothing to do. A failed probe means no subtitle info either, so + // `subtitle_codec_args` just gets an empty slice (falls back to + // whatever ffmpeg's own default subtitle handling is for the output + // container) rather than guessing. + let subtitles: &[ffprobe::SubtitleStream] = + fresh_probe.as_ref().map(|p| p.subtitles.as_slice()).unwrap_or(&[]); + + let original_bytes = std::fs::metadata(&input) + .context("failed to stat input file before transcode")? + .len(); + let ceiling_bitrate_kbps = target_bitrate_kbps(width, height, &cfg); + + if let Some(duration) = original_duration { + if duration > 0.0 { + let source_bitrate_kbps = (original_bytes as f64 * 8.0) / duration / 1000.0; + if source_bitrate_kbps <= ceiling_bitrate_kbps as f64 * cfg.skip_below_ceiling_ratio { + return Ok(EncodeOutcome::NotBeneficial); + } + } + } + + let tmp_path = temp_path_for(&input, job_id); + + if is_anime { + run_ffmpeg_encode_anime( + &input, + &tmp_path, + cfg.quality_anime, + cfg.anime_svtav1_preset, + cfg.anime_svtav1_max_threads, + subtitles, + )?; + } else { + run_ffmpeg_encode_live_action( + &input, + &tmp_path, + ceiling_bitrate_kbps, + cfg.quality_live_action, + &cfg.vaapi_device, + subtitles, + )?; + } + + let decode_check = match original_duration { + Some(duration) => ffprobe::verify_decodable_sampled(&tmp_path, duration, cfg.verify_sample_secs), + // Duration unknown — can't pick sample windows sensibly, so fall + // back to the thorough full-file check rather than guess. + None => ffprobe::verify_decodable(&tmp_path), + }; + match decode_check { + Ok(ffprobe::DecodeCheck::Ok) => {} + Ok(ffprobe::DecodeCheck::Corrupt(detail)) => { + let _ = std::fs::remove_file(&tmp_path); + bail!("transcoded output failed decode verification: {detail}"); + } + Err(e) => { + let _ = std::fs::remove_file(&tmp_path); + return Err(e.context("failed to run decode verification on transcoded output")); + } + } + + if let Some(original_secs) = original_duration { + match ffprobe::probe(&tmp_path) { + Ok(new_probe) => { + if let Some(new_secs) = new_probe.duration_secs { + if (new_secs - original_secs).abs() > 1.0 { + let _ = std::fs::remove_file(&tmp_path); + bail!( + "duration mismatch after transcode: original={original_secs}s new={new_secs}s" + ); + } + } + } + Err(e) => { + let _ = std::fs::remove_file(&tmp_path); + return Err(e.context("failed to probe transcoded output for duration check")); + } + } + } + + let new_bytes = std::fs::metadata(&tmp_path) + .context("failed to stat transcoded output")? + .len(); + if !is_beneficial(original_bytes, new_bytes, cfg.min_size_reduction_pct) { + let _ = std::fs::remove_file(&tmp_path); + return Ok(EncodeOutcome::NotBeneficial); + } + + Ok(EncodeOutcome::Encoded(tmp_path)) +} + +/// The hard invariant that makes "never keep a non-improvement" true +/// regardless of how well any rate-control tuning holds up: an output only +/// counts as worth keeping if it's at least `min_size_reduction_pct` +/// smaller than the original. Pulled out as its own pure function so this +/// exact arithmetic — the thing a real incident got wrong — has direct unit +/// coverage without needing a real ffmpeg encode to exercise it. +fn is_beneficial(original_bytes: u64, new_bytes: u64, min_size_reduction_pct: f64) -> bool { + let max_allowed_bytes = (original_bytes as f64 * (1.0 - min_size_reduction_pct)) as u64; + new_bytes <= max_allowed_bytes +} + +/// Whether `media_item_id` is anime per metadata — same two membership +/// checks (`anime_mapping` for TV, `anime_tmdb_movie` for movies) already +/// used elsewhere for scoring/upgrade exclusions. Known to have real +/// coverage gaps (Avatar: The Last Airbender and some Dragon Ball movies +/// were missing from these tables) — `is_anime_content` below is the +/// combined check that should actually be used for routing; this is kept +/// as its own function since other callers (`scheduler.rs`'s search +/// routing) still want the metadata-only check. +pub fn is_anime(conn: &Connection, media_item_id: i64) -> Result { + let result: bool = conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM media_item m + WHERE m.id = ?1 + AND ( + (m.tvdb_id IS NOT NULL AND m.tvdb_id IN + (SELECT tvdb_id FROM anime_mapping WHERE tvdb_id IS NOT NULL)) + OR + (m.tmdb_id IS NOT NULL AND m.tmdb_id IN (SELECT tmdb_id FROM anime_tmdb_movie)) + ) + )", + params![media_item_id], + |row| row.get(0), + )?; + Ok(result) +} + +/// Whether `path` falls under one of `cfg.anime_root_folders` — a plain +/// prefix match. Deliberately independent of any DB metadata: it's how the +/// library is actually organized on disk, and it's what catches the real +/// gaps in `anime_mapping`/`anime_tmdb_movie` coverage (see `is_anime`'s +/// doc comment). +pub fn is_anime_path(path: &Path, cfg: &TranscodeConfig) -> bool { + cfg.anime_root_folders.iter().any(|root| path.starts_with(root)) +} + +/// The combined anime check every enqueue site should use to decide which +/// encode pipeline (`run_ffmpeg_encode_anime` vs `_live_action`) a file +/// gets routed to: path-based first (cheap, no DB access, and the more +/// reliable signal — see `is_anime_path`), falling back to the +/// metadata-based `is_anime` check for anime content filed outside the +/// configured anime root folders. +pub fn is_anime_content( + conn: &Connection, + media_item_id: i64, + path: &Path, + cfg: &TranscodeConfig, +) -> Result { + if is_anime_path(path, cfg) { + return Ok(true); + } + is_anime(conn, media_item_id) +} + +/// Resets any `running` job back to `pending` — call once at process +/// startup (the daemon itself, and the `transcode-library` backfill), never +/// mid-run. `running` only ever means "some still-alive process is +/// currently encoding this" — a row left in that state at startup can only +/// mean the process that claimed it is gone (crashed, killed, power loss), +/// since a live process's own in-flight jobs aren't visible to it as +/// leftover state, they're just still running. Left unreset, a crash leaves +/// that job's slot permanently uncountable-but-also-unclaimable, quietly +/// shrinking real capacity forever instead of just costing one retry. +/// +/// Also sweeps each reset job's expected temp file (`temp_path_for`, which +/// is job-id-scoped precisely so this lookup is unambiguous) — a killed +/// encode leaves a partial `.jobN.av1.mkv` behind that nothing else will +/// ever clean up, and on a library disk that's usually already tight on +/// space these accumulate for real over a big backfill. +pub fn reset_orphaned_running_jobs(conn: &Connection) -> Result { + let orphaned: Vec<(i64, String)> = { + let mut stmt = conn.prepare( + "SELECT j.id, ef.path FROM transcode_job j + JOIN episode_file ef ON ef.id = j.episode_file_id + WHERE j.status = 'running'", + )?; + let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect::>>()? + }; + + for (job_id, path) in &orphaned { + let tmp = temp_path_for(Path::new(path), *job_id); + if tmp.exists() { + if let Err(e) = std::fs::remove_file(&tmp) { + tracing::warn!(job_id, path = %tmp.display(), error = %e, "failed to remove orphaned transcode temp file"); + } + } + } + + let reset = conn.execute( + "UPDATE transcode_job SET status = 'pending' WHERE status = 'running'", + [], + )?; + Ok(reset) +} + +/// Queues one file for transcoding. `original_codec`/`original_bytes` are +/// just recorded for the eventual report — not used for any decision. +/// `is_anime` is decided once here (by the caller, via `is_anime_content`) +/// and carried on the job row so `claim_pending_jobs` can dispatch to the +/// right encode pipeline without re-deriving it at claim time. Every normal +/// enqueue site passes `force_reencode: false`; `true` is only for +/// `find_oversized_av1_candidates`'s remediation jobs — see +/// `encode_and_verify`'s doc comment for why that flag exists. +pub fn enqueue( + conn: &Connection, + episode_file_id: i64, + original_codec: Option<&str>, + original_bytes: i64, + is_anime: bool, + force_reencode: bool, +) -> Result<()> { + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, original_codec, original_bytes, is_anime, force_reencode, queued_at) + VALUES (?1, 'pending', ?2, ?3, ?4, ?5, datetime('now'))", + params![episode_file_id, original_codec, original_bytes, is_anime as i64, force_reencode as i64], + )?; + Ok(()) +} + +/// The single source of truth for "should this freshly-imported file be +/// queued for transcoding at all" — every enqueue site (`import_one`, +/// season-pack import) must go through this rather than re-deriving its own +/// subset of the rules. Called after `ensure_probed`, once real ffprobe +/// data (codec/hdr/height) exists for the file — the codec/hdr/height +/// parameters come from that probe, not from anything parsed off the +/// release title. +/// +/// No longer excludes anime — anime now has its own encode pipeline +/// (`run_ffmpeg_encode_anime`, see `is_anime_content`) instead of being +/// skipped outright. A pure function now that anime is out of it (no DB +/// access needed to decide codec/HDR/height eligibility). +pub fn should_enqueue( + video_codec: Option<&str>, + hdr: bool, + height: Option, + cfg: &TranscodeConfig, +) -> bool { + // A `probe_failed` row has NULL codec/height (see `ensure_probed`), not a + // missing row — without this check `should_enqueue` would happily enqueue + // a job for it. `claim_pending_jobs` requires `p.width`/`p.height IS NOT + // NULL` so that job can never be claimed, and the unique partial index on + // (pending/running) then permanently blocks any future, real enqueue for + // this file once probing succeeds. + if video_codec.is_none() || height.is_none() { + return false; + } + if video_codec == Some("av1") { + return false; + } + if cfg.exclude_hdr && hdr { + return false; + } + if height.is_some_and(|h| h >= cfg.exclude_min_height as i64) { + return false; + } + true +} + +/// Every existing-library file eligible for the `transcode-library` backfill: +/// not already AV1, not HDR/2160p+ (per `cfg.exclude_hdr` / +/// `cfg.exclude_min_height` — the first pass is scoped to SDR 1080p/720p), +/// not already queued, done, or `skipped` (a completed encode that +/// `finalize_job` rejected as not-beneficial — re-selecting it here would +/// re-run the full encode from scratch every time this backfill runs, only +/// to reject it again; a genuinely `failed` job *is* re-selected, since a +/// transient failure — disk full, transient ffmpeg crash — deserves a retry +/// on the next manual run). Anime is included (routed to its own +/// pipeline, not excluded) — `is_anime` on each candidate is the metadata +/// half of that decision; combined with the path-based check +/// (`is_anime_path`) in Rust after the query, since a dynamic list of path +/// prefixes (`cfg.anime_root_folders`) doesn't fit cleanly into static SQL. +pub fn find_backlog_candidates(conn: &Connection, cfg: &TranscodeConfig) -> Result> { + let mut stmt = conn.prepare( + "SELECT ef.id, ef.path, ef.size_bytes, p.video_codec, + (m.tvdb_id IS NOT NULL AND m.tvdb_id IN + (SELECT tvdb_id FROM anime_mapping WHERE tvdb_id IS NOT NULL)) + OR + (m.tmdb_id IS NOT NULL AND m.tmdb_id IN (SELECT tmdb_id FROM anime_tmdb_movie)) + AS db_is_anime + FROM episode_file ef + JOIN media_file_probe p ON p.episode_file_id = ef.id + LEFT JOIN episode e ON e.id = ef.episode_id + JOIN media_item m ON m.id = COALESCE(e.media_item_id, ef.media_item_id) + WHERE p.video_codec IS NOT NULL AND p.video_codec != 'av1' + AND (ef.upgrade_locked IS NULL OR ef.upgrade_locked = 0) + AND (?1 = 0 OR p.hdr = 0) + AND (p.height IS NOT NULL AND p.height < ?2) + AND ef.id NOT IN ( + SELECT episode_file_id FROM transcode_job WHERE status IN ('pending','running','done','skipped') + ) + ORDER BY (ef.size_bytes * 8.0 / NULLIF(p.duration_secs, 0)) DESC", + )?; + let rows = stmt + .query_map(params![cfg.exclude_hdr as i64, cfg.exclude_min_height], |row| { + let path: String = row.get(1)?; + let db_is_anime: bool = row.get(4)?; + Ok(BacklogCandidate { + episode_file_id: row.get(0)?, + is_anime: db_is_anime || is_anime_path(Path::new(&path), cfg), + path, + size_bytes: row.get(2)?, + video_codec: row.get(3)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +/// Every file whose *most recent* transcode attempt succeeded (`done`) but +/// wasn't actually beneficial by the current `min_size_reduction_pct` bar — +/// the 476-file legacy of a real rate-control bug that left files larger +/// than their original, back before `is_beneficial` existed as a hard +/// invariant. These are already AV1 (`find_backlog_candidates` excludes +/// them for exactly that reason), and their pre-transcode originals are +/// long gone — the only way to reclaim the space is to re-transcode the +/// current, oversized AV1 file itself, which needs `enqueue`'s +/// `force_reencode: true` to get past `encode_and_verify`'s normal +/// already-AV1 short-circuit. The `WITH latest_job` CTE picks each file's +/// single most recent job so a file already successfully re-transcoded in +/// a prior run of this same query (its latest job now `done` and properly +/// smaller) doesn't get selected again. +pub fn find_oversized_av1_candidates(conn: &Connection, cfg: &TranscodeConfig) -> Result> { + let mut stmt = conn.prepare( + "WITH latest_job AS ( + SELECT episode_file_id, MAX(id) AS job_id + FROM transcode_job + GROUP BY episode_file_id + ) + SELECT ef.id, ef.path, ef.size_bytes, p.video_codec, + (m.tvdb_id IS NOT NULL AND m.tvdb_id IN + (SELECT tvdb_id FROM anime_mapping WHERE tvdb_id IS NOT NULL)) + OR + (m.tmdb_id IS NOT NULL AND m.tmdb_id IN (SELECT tmdb_id FROM anime_tmdb_movie)) + AS db_is_anime + FROM latest_job lj + JOIN transcode_job tj ON tj.id = lj.job_id + JOIN episode_file ef ON ef.id = lj.episode_file_id + JOIN media_file_probe p ON p.episode_file_id = ef.id + LEFT JOIN episode e ON e.id = ef.episode_id + JOIN media_item m ON m.id = COALESCE(e.media_item_id, ef.media_item_id) + WHERE tj.status = 'done' + AND tj.original_bytes IS NOT NULL AND tj.new_bytes IS NOT NULL AND tj.original_bytes > 0 + AND tj.new_bytes > tj.original_bytes * (1.0 - ?1) + AND p.video_codec = 'av1' + ORDER BY (tj.new_bytes - tj.original_bytes) DESC", + )?; + let rows = stmt + .query_map(params![cfg.min_size_reduction_pct], |row| { + let path: String = row.get(1)?; + let db_is_anime: bool = row.get(4)?; + Ok(BacklogCandidate { + episode_file_id: row.get(0)?, + is_anime: db_is_anime || is_anime_path(Path::new(&path), cfg), + path, + size_bytes: row.get(2)?, + video_codec: row.get(3)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +pub struct BacklogCandidate { + pub episode_file_id: i64, + // Not read by current callers (they only need episode_file_id to look up + // the row again), but cheap to carry along for future logging/debugging + // of which file a candidate refers to. + #[allow(dead_code)] + pub path: String, + pub size_bytes: i64, + pub video_codec: Option, + pub is_anime: bool, +} + +/// A `transcode_job` row claimed for processing this cycle, with the +/// probe/path data `encode_and_verify` needs already attached — claimed +/// under the DB lock, then the actual encode runs entirely outside it. +struct ClaimedJob { + job_id: i64, + episode_file_id: i64, + path: PathBuf, + width: i64, + height: i64, + duration_secs: Option, + is_anime: bool, + force_reencode: bool, +} + +/// Claims up to `live_action_limit` live-action jobs and up to +/// `anime_limit` anime jobs (marking them `running` so a crash mid-cycle +/// doesn't leave them silently re-claimable forever without at least +/// having been attempted once) and returns everything the encode step +/// needs. Only claims jobs whose file already has probe data — a job +/// enqueued moments after import but before `ensure_probed` has run yet +/// simply isn't claimed this tick, and picks up naturally on the next one. +/// +/// The two limits are enforced **independently** — a real gap found after +/// raising the live-action cap for a validated GPU-scaling reason: a single +/// shared cap would let "raise GPU parallelism" silently also raise anime +/// (CPU-bound, thread-hungry) concurrency to something never load-tested. +/// Each limit is treated as a *total* concurrency target for its own +/// pipeline, not "claim this many more" — first reduced by however many +/// jobs of that same pipeline are already `running` (set by anyone: this +/// same daemon's previous still-in-flight cycle, or a separately-invoked +/// `transcode-library`/`retranscode-oversized` process hitting the same +/// database). Learned the hard way (the *original* version of this bug, +/// before the pipelines were even split): without this, a long-running +/// cycle and a concurrently-run backfill each independently claimed up to +/// their own cap with no awareness of the other, stacking to 20+ +/// simultaneous GPU encodes and OOMing the host. `status='running'` is +/// shared database state, not per-process, so counting it is what makes +/// this a real global cap no matter how many processes are hitting this +/// table at once — and wrapping each pipeline's count+select+update in one +/// `BEGIN IMMEDIATE` transaction (rather than separate statements, and both +/// pipelines in the *same* transaction) is what stops two processes from +/// both reading the same low count and both claiming past the limit before +/// either one's UPDATE lands. +async fn claim_pending_jobs( + conn: &Arc>, + live_action_limit: usize, + anime_limit: usize, +) -> Result> { + let mut conn = conn.lock().await; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + + let mut claimed = Vec::new(); + for (is_anime_flag, limit) in [(0i64, live_action_limit), (1i64, anime_limit)] { + let running: i64 = tx.query_row( + "SELECT count(*) FROM transcode_job WHERE status = 'running' AND is_anime = ?1", + params![is_anime_flag], + |row| row.get(0), + )?; + let available = (limit as i64 - running).max(0); + if available == 0 { + continue; + } + + // Highest current bitrate first (within this pipeline) — the worst + // offenders (REMUX, huge season packs) free the most space per + // file transcoded, so they're worth reaching before smaller, + // already-reasonable files. + let mut stmt = tx.prepare( + "SELECT j.id, j.episode_file_id, ef.path, p.width, p.height, p.duration_secs, j.is_anime, j.force_reencode + FROM transcode_job j + JOIN episode_file ef ON ef.id = j.episode_file_id + JOIN media_file_probe p ON p.episode_file_id = ef.id + WHERE j.status = 'pending' AND j.is_anime = ?2 AND p.width IS NOT NULL AND p.height IS NOT NULL + ORDER BY (ef.size_bytes * 8.0 / NULLIF(p.duration_secs, 0)) DESC + LIMIT ?1", + )?; + let batch = stmt + .query_map(params![available, is_anime_flag], |row| { + Ok(ClaimedJob { + job_id: row.get(0)?, + episode_file_id: row.get(1)?, + path: PathBuf::from(row.get::<_, String>(2)?), + width: row.get(3)?, + height: row.get(4)?, + duration_secs: row.get(5)?, + is_anime: row.get::<_, i64>(6)? != 0, + force_reencode: row.get::<_, i64>(7)? != 0, + }) + })? + .collect::>>()?; + + for job in &batch { + tx.execute( + "UPDATE transcode_job SET status = 'running' WHERE id = ?1", + params![job.job_id], + )?; + } + claimed.extend(batch); + } + + tx.commit()?; + Ok(claimed) +} + +/// Finalizes one job under the DB lock: on a real `Encoded` result, +/// atomically swaps the verified temp file over the original, updates +/// `episode_file` (new size + `upgrade_locked = 1`, the flag that keeps the +/// upgrade cycle from ever trying to replace a file breadarr itself just +/// intentionally shrank), and forces a fresh `ensure_probed` so +/// `media_file_probe` reflects the real AV1 ground truth — same shape as +/// `remux_one_backlog_file`. On failure, the original is left completely +/// untouched; the job is marked `failed` with the error recorded, no +/// automatic retry. +/// +/// `AlreadyAv1`/`NotBeneficial` from the encode step are both marked `done`/ +/// `skipped` respectively with no byte-count change and no swap — neither +/// is a failure, there's just nothing to keep. +/// +/// The whole swap-and-bookkeeping sequence runs inside a closure so any +/// failure partway through (a deleted-out-from-under-it original, a +/// transient I/O error) is caught in one place: the job is marked `failed` +/// and the temp file is cleaned up, rather than an early `?` propagating +/// out and silently leaving the job stuck `running` with a leaked temp file +/// on a disk that's usually already tight on space. +async fn finalize_job( + conn: &Arc>, + job: ClaimedJob, + encode_result: Result, +) -> Result { + let mut conn = conn.lock().await; + let tmp_path = match &encode_result { + Ok(EncodeOutcome::Encoded(p)) => Some(p.clone()), + _ => None, + }; + + let swap: Result<(&'static str, TranscodeOutcome)> = match encode_result { + Ok(EncodeOutcome::AlreadyAv1) => { + Ok(("done", TranscodeOutcome { original_bytes: 0, new_bytes: 0 })) + } + Ok(EncodeOutcome::NotBeneficial) => { + Ok(("skipped", TranscodeOutcome { original_bytes: 0, new_bytes: 0 })) + } + Ok(EncodeOutcome::Encoded(tmp_path)) => (|| { + let original_bytes = std::fs::metadata(&job.path)?.len(); + std::fs::rename(&tmp_path, &job.path)?; + let new_bytes = std::fs::metadata(&job.path)?.len(); + + // `size_bytes` and `upgrade_locked` must land together, in one + // transaction: the rename just above is already irreversible in + // practice (the temp file is gone), so if these two writes were + // separate auto-committed statements and the second one failed, + // the on-disk file would already be the smaller AV1 encode while + // `upgrade_locked` stayed 0 — leaving it exactly as eligible for + // the ordinary upgrade cycle to "improve" as any untouched file, + // silently replacing a deliberate, already-paid-for shrink with + // a much larger release. Wrapping both in one transaction means + // either both land or neither does; on failure `swap` returns + // `Err` below and the job is marked `failed` for a human to + // notice, rather than quietly losing the lock. + let tx = conn.transaction()?; + tx.execute( + "UPDATE episode_file SET size_bytes = ?1, upgrade_locked = 1 WHERE id = ?2", + params![new_bytes as i64, job.episode_file_id], + )?; + tx.execute( + "DELETE FROM media_file_probe WHERE episode_file_id = ?1", + params![job.episode_file_id], + )?; + tx.commit()?; + + // Best-effort from here: `ensure_probed` only refreshes + // `media_file_probe`'s descriptive fields (codec/height/etc, fed + // to `find_backlog_candidates` and library-health reporting) — + // it already never propagates an *ffprobe* failure (see its own + // doc comment: "probing must never abort a scan or import"), so + // an `Err` here means a genuine DB-level error, most likely + // transient. The facts that actually matter — the file is now + // AV1, its new size, and `upgrade_locked` — are already durably + // committed above; failing the whole job over a stale probe row + // would misreport a successful transcode as `failed`. + if let Err(e) = importer::ensure_probed(&conn, job.episode_file_id, &job.path) { + tracing::warn!( + episode_file_id = job.episode_file_id, + error = %e, + "post-transcode re-probe failed; media_file_probe left stale until the next scan" + ); + } + Ok(("done", TranscodeOutcome { original_bytes, new_bytes })) + })(), + Err(e) => Err(e), + }; + + match swap { + Ok((status, outcome)) => { + conn.execute( + "UPDATE transcode_job SET status = ?1, new_bytes = ?2, finished_at = datetime('now') WHERE id = ?3", + params![status, outcome.new_bytes as i64, job.job_id], + )?; + Ok(FinalizedJob { status, outcome }) + } + Err(e) => { + if let Some(tmp_path) = tmp_path { + let _ = std::fs::remove_file(&tmp_path); + } + conn.execute( + "UPDATE transcode_job SET status = 'failed', error = ?1, finished_at = datetime('now') WHERE id = ?2", + params![e.to_string(), job.job_id], + )?; + Err(e) + } + } +} + +pub struct TranscodeOutcome { + pub original_bytes: u64, + pub new_bytes: u64, +} + +struct FinalizedJob { + status: &'static str, + outcome: TranscodeOutcome, +} + +#[derive(Debug, Default)] +pub struct TranscodeCycleStats { + pub attempted: usize, + pub succeeded: usize, + pub skipped: usize, + pub failed: usize, + pub bytes_saved: i64, +} + +impl TranscodeCycleStats { + fn merge(self, other: Self) -> Self { + TranscodeCycleStats { + attempted: self.attempted + other.attempted, + succeeded: self.succeeded + other.succeeded, + skipped: self.skipped + other.skipped, + failed: self.failed + other.failed, + bytes_saved: self.bytes_saved + other.bytes_saved, + } + } +} + +/// The live-action pipeline's parallelism given how many real Jellyfin +/// transcode sessions are active right now: `budget` (`parallelism_max`, +/// the measured total GPU concurrency ceiling — see its doc comment) minus +/// however many streams Jellyfin itself is actually using, floored at 0. +/// Reserves *exactly* the GPU headroom real viewers need instead of +/// dropping to a flat `parallelism_min` regardless of whether 1 or 5 +/// people are watching — the whole point of measuring the real ceiling +/// (rather than guessing) is to spend the difference precisely. Pulled out +/// as its own pure function for direct unit coverage without needing a +/// real (or mocked) `JellyfinClient`. +fn live_action_parallelism_for(budget: usize, active_jellyfin_sessions: usize) -> usize { + budget.saturating_sub(active_jellyfin_sessions) +} + +/// Active parallelism for this cycle, returned as `(live_action, anime)`. +/// Live-action: `live_action_parallelism_for(cfg.parallelism_max, +/// active_sessions)` — dynamically reserves exactly as many GPU streams as +/// Jellyfin is actually using, since Jellyfin transcoding contends for the +/// same encode/decode engines the batch job does. Anime: still a flat +/// `parallelism_max_anime`/`parallelism_min` binary switch on *any* active +/// session — the CPU-bound anime pipeline doesn't contend for the GPU the +/// way live-action does, so it isn't part of this dynamic reservation (per +/// explicit user direction: "for the live action ones"); it still backs +/// off to `parallelism_min` (not eliminated, just conservative) whenever +/// someone's actively watching anything, on the more general theory that a +/// real viewer's experience should never compete with unattended batch +/// work for the box's resources, GPU or not. Only one Jellyfin poll either +/// way, not one per pipeline. +async fn effective_parallelism(jellyfin: Option<&JellyfinClient>, cfg: &TranscodeConfig) -> (usize, usize) { + let Some(client) = jellyfin else { + return (cfg.parallelism_max, cfg.parallelism_max_anime); + }; + match client.active_transcode_sessions().await { + Ok(active_sessions) => { + let live_action = live_action_parallelism_for(cfg.parallelism_max, active_sessions); + let anime = if active_sessions == 0 { cfg.parallelism_max_anime } else { cfg.parallelism_min }; + (live_action, anime) + } + Err(e) => { + tracing::warn!(error = %e, "failed to poll jellyfin sessions; assuming worst case"); + (cfg.parallelism_min, cfg.parallelism_min) + } + } +} + +/// One transcode-worker pass across both pipelines. Runs +/// `run_pipeline_cycle` for live-action and anime *concurrently* via +/// `tokio::join!` and merges their stats — see that function's doc +/// comment for why they must not share one claim-spawn-join batch. +/// +/// Deliberately `join!`, not `try_join!`: `try_join!` cancels (drops) the +/// other branch's future the instant either one returns `Err`, and dropping +/// a `JoinSet` of `spawn_blocking` encode tasks cannot abort work that has +/// already started — the ffmpeg/SVT-AV1 process keeps running to completion +/// with its result simply discarded, `finalize_job` never runs for it, and +/// its `transcode_job` row is stuck `status='running'` (consuming +/// concurrency budget and leaving its multi-GB temp file on disk) until the +/// daemon restarts. `run_pipeline_cycle` itself now treats a +/// `claim_pending_jobs` error as "claim nothing this iteration" rather than +/// propagating it, so in practice this only guards a pipeline that fails for +/// some other reason — but `join!` means even that failure can no longer +/// take the other, healthy pipeline's in-flight work down with it. Each +/// pipeline's error is logged and treated as an empty cycle rather than +/// failing the whole `run_cycle` call. Shared by both the steady-state +/// daemon ticker and the `transcode-library` backfill CLI, so there's +/// exactly one code path that actually runs an encode. +pub async fn run_cycle( + conn: Arc>, + cfg: TranscodeConfig, + jellyfin: Option<&JellyfinClient>, +) -> Result { + let (live_action_result, anime_result) = tokio::join!( + run_pipeline_cycle(conn.clone(), cfg.clone(), jellyfin, false), + run_pipeline_cycle(conn.clone(), cfg.clone(), jellyfin, true), + ); + let live_action_stats = live_action_result.unwrap_or_else(|e| { + tracing::warn!(error = %e, "live-action transcode pipeline cycle failed"); + TranscodeCycleStats::default() + }); + let anime_stats = anime_result.unwrap_or_else(|e| { + tracing::warn!(error = %e, "anime transcode pipeline cycle failed"); + TranscodeCycleStats::default() + }); + Ok(live_action_stats.merge(anime_stats)) +} + +/// One pipeline's worker loop: repeatedly claims whatever's currently +/// available for *this* pipeline only (the other pipeline's limit is passed +/// as 0, so `claim_pending_jobs` naturally claims nothing for it), spawns +/// each newly claimed job, and as soon as any one of them finishes, +/// finalizes it and immediately loops back to claim a replacement — rather +/// than waiting for the whole batch to drain before claiming more. Exits +/// once a claim attempt comes back empty and nothing is left in flight. +/// +/// This replaces an earlier design where one shared claim+spawn+join batch +/// covered both pipelines at once: claimed jobs from both pipelines were +/// joined together, so the *whole* batch — and therefore any new claim for +/// either pipeline — blocked until every job in it finished. A single long +/// anime file (feature-length movies can take over an hour via the +/// CPU-bound SVT-AV1 path) would leave the GPU-bound live-action pipeline +/// sitting completely idle for that entire duration despite having its own +/// independent concurrency budget and a large backlog of its own — +/// discovered in production: 512 pending live-action jobs and a budget of 7, +/// but the GPU sat unused for the better part of an hour waiting on one +/// anime movie to join. Running each pipeline as its own independent +/// replenish-loop, joined concurrently rather than sequentially, means +/// neither pipeline's throughput is ever gated by the other's job durations. +/// +/// Uses `JoinSet::join_next_with_id` rather than awaiting a `Vec` of handles +/// in order: a real validation run surfaced that the naive +/// awaited-in-claim-order approach leaves a fast job's already-finished +/// result (verified output sitting on disk, or even a fast failure) fully +/// idle — not written back to the DB, not freeing its slot for the next +/// claim — for as long as whichever job happens to be *earlier* in claim +/// order (highest-bitrate-first, so often the largest file) keeps running. +/// `join_next_with_id`'s `(Id, T)`/`JoinError::id()` pairing is what lets a +/// panicked task still be matched back to its `ClaimedJob` (a panic doesn't +/// get to return the job alongside its result), which a plain +/// `JoinSet>` with the job moved into the closure +/// wouldn't allow. +async fn run_pipeline_cycle( + conn: Arc>, + cfg: TranscodeConfig, + jellyfin: Option<&JellyfinClient>, + is_anime: bool, +) -> Result { + let mut stats = TranscodeCycleStats::default(); + let mut set = tokio::task::JoinSet::new(); + let mut jobs_by_task_id: std::collections::HashMap = + std::collections::HashMap::new(); + + loop { + let (live_action_parallelism, anime_parallelism) = effective_parallelism(jellyfin, &cfg).await; + let (live_action_limit, anime_limit) = if is_anime { + (0, anime_parallelism) + } else { + (live_action_parallelism, 0) + }; + // A claim failure (e.g. transient SQLITE_BUSY under `busy_timeout` + // when another process holds the write lock) must not propagate out + // of this loop: that would return `Err` from `run_pipeline_cycle`, + // and in `run_cycle` that used to cancel the *other* pipeline's + // future mid-flight via `try_join!`, orphaning its already-running + // encodes (see `run_cycle`'s doc comment). Skip claiming new work + // this iteration instead — whatever's already in `set` keeps + // draining normally, and the next tick retries the claim. + let claimed = match claim_pending_jobs(&conn, live_action_limit, anime_limit).await { + Ok(claimed) => claimed, + Err(e) => { + tracing::warn!(error = %e, is_anime, "failed to claim pending transcode jobs this cycle; will retry next iteration"); + Vec::new() + } + }; + + for job in claimed { + let cfg = cfg.clone(); + let path = job.path.clone(); + let job_id = job.job_id; + let (width, height, duration, job_is_anime, force_reencode) = + (job.width, job.height, job.duration_secs, job.is_anime, job.force_reencode); + let abort_handle = set.spawn_blocking(move || { + encode_and_verify(path, job_id, cfg, width, height, duration, job_is_anime, force_reencode) + }); + jobs_by_task_id.insert(abort_handle.id(), job); + } + + // `None` here means this claim found nothing new *and* nothing from + // an earlier claim is still in flight — this pipeline's queue is + // genuinely empty for now. + let Some(joined) = set.join_next_with_id().await else { + break; + }; + + stats.attempted += 1; + let (task_id, encode_result) = match joined { + Ok((task_id, result)) => (task_id, result), + Err(join_err) => { + let task_id = join_err.id(); + (task_id, Err(anyhow::anyhow!("encode task panicked: {join_err}"))) + } + }; + let job = jobs_by_task_id + .remove(&task_id) + .expect("every spawned task's id was inserted before the task could complete"); + + match finalize_job(&conn, job, encode_result).await { + Ok(finalized) => { + if finalized.status == "skipped" { + stats.skipped += 1; + } else { + stats.succeeded += 1; + } + stats.bytes_saved += + finalized.outcome.original_bytes as i64 - finalized.outcome.new_bytes as i64; + } + Err(e) => { + stats.failed += 1; + tracing::warn!(error = %e, "transcode job failed"); + } + } + } + + Ok(stats) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg() -> TranscodeConfig { + TranscodeConfig { + enabled: true, + poll_interval_secs: 60, + vaapi_device: "/dev/dri/renderD128".to_string(), + parallelism_min: 1, + parallelism_max: 4, + parallelism_max_anime: 2, + reference_bitrate_kbps: 5320, + reference_height: 1080, + av1_efficiency_factor: 0.7, + exclude_hdr: true, + exclude_min_height: 2000, + quality_live_action: 26, + anime_root_folders: vec!["/mnt/media/Anime".to_string(), "/mnt/media/Anime Movies".to_string()], + quality_anime: 24, + anime_svtav1_preset: 10, + anime_svtav1_max_threads: 2, + min_size_reduction_pct: 0.10, + skip_below_ceiling_ratio: 0.5, + verify_sample_secs: 20.0, + } + } + + fn seed_file(conn: &Connection, id: i64, size_bytes: i64) { + conn.execute( + "INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes) + VALUES (?1, NULL, NULL, ?2, ?3)", + params![id, format!("/tmp/f{id}.mkv"), size_bytes], + ) + .unwrap(); + conn.execute( + "INSERT INTO media_file_probe (episode_file_id, probed_at, probe_size_bytes, probe_mtime, + duration_secs, video_codec, width, height) + VALUES (?1, datetime('now'), ?2, 0, 1200.0, 'hevc', 1920, 1080)", + params![id, size_bytes], + ) + .unwrap(); + } + + // Regression test for a real incident: a long-running cycle (large + // files can easily outlast `poll_interval_secs`) and a separately + // invoked `transcode-library` backfill each independently claimed up to + // their own `parallelism_max` with no shared awareness, stacking to 20+ + // concurrent GPU encodes and OOMing the host. `claim_pending_jobs` must + // treat `limit` as a total cap, not "claim this many more". + #[tokio::test] + async fn claim_pending_jobs_respects_already_running_jobs_as_a_global_cap() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + for id in 1..=5 { + seed_file(&conn, id, 1_000_000_000); + } + // Two jobs already claimed by "someone else" (another process, or + // this same process's still-in-flight previous cycle). + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, queued_at) VALUES (1, 'running', datetime('now'))", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, queued_at) VALUES (2, 'running', datetime('now'))", + [], + ) + .unwrap(); + for id in 3..=5 { + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, queued_at) VALUES (?1, 'pending', datetime('now'))", + params![id], + ) + .unwrap(); + } + + let conn = Arc::new(Mutex::new(conn)); + // Asking for a total of 3 concurrent live-action, with 2 already + // running — should claim exactly 1 more, not 3 more. All seeded + // jobs default to is_anime=0 (live-action), so anime_limit=0 here + // is correct, not a placeholder. + let claimed = claim_pending_jobs(&conn, 3, 0).await.unwrap(); + assert_eq!(claimed.len(), 1); + + let conn = conn.lock().await; + let running: i64 = conn + .query_row("SELECT count(*) FROM transcode_job WHERE status = 'running'", [], |r| r.get(0)) + .unwrap(); + assert_eq!(running, 3, "total in-flight jobs must never exceed the requested cap"); + } + + // Regression test for a real gap found in review: a file whose encode + // completed but was rejected by `is_beneficial` (recorded `skipped` by + // `finalize_job`) was not in `find_backlog_candidates`'s exclusion list, + // so every re-run of the `transcode-library` backfill re-selected it, + // paying the full GPU/CPU encode cost again only to reject it again. + #[test] + fn find_backlog_candidates_excludes_skipped_jobs() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + seed_file(&conn, 1, 1_000_000_000); + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, queued_at) VALUES (1, 'skipped', datetime('now'))", + [], + ) + .unwrap(); + + let candidates = find_backlog_candidates(&conn, &cfg()).unwrap(); + assert!( + candidates.is_empty(), + "a file with a 'skipped' (not-beneficial) job must not be re-selected for backfill" + ); + } + + #[tokio::test] + async fn claim_pending_jobs_claims_nothing_when_already_at_the_cap() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + for id in 1..=3 { + seed_file(&conn, id, 1_000_000_000); + } + for id in 1..=2 { + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, queued_at) VALUES (?1, 'running', datetime('now'))", + params![id], + ) + .unwrap(); + } + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, queued_at) VALUES (3, 'pending', datetime('now'))", + [], + ) + .unwrap(); + + let conn = Arc::new(Mutex::new(conn)); + let claimed = claim_pending_jobs(&conn, 2, 0).await.unwrap(); + assert!(claimed.is_empty(), "already at the cap — must not claim more"); + } + + // Regression test for a real gap: raising the live-action cap for a + // validated GPU-scaling reason must not silently also raise anime + // concurrency (CPU-bound, thread-hungry, never load-tested at higher + // counts) — the two caps are enforced completely independently. + #[tokio::test] + async fn claim_pending_jobs_enforces_live_action_and_anime_caps_independently() { + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + for id in 1..=6 { + seed_file(&conn, id, 1_000_000_000); + } + // 3 live-action pending (ids 1-3), 3 anime pending (ids 4-6). + for id in 1..=3 { + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, is_anime, queued_at) VALUES (?1, 'pending', 0, datetime('now'))", + params![id], + ) + .unwrap(); + } + for id in 4..=6 { + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, is_anime, queued_at) VALUES (?1, 'pending', 1, datetime('now'))", + params![id], + ) + .unwrap(); + } + + let conn = Arc::new(Mutex::new(conn)); + // A generous live-action cap (6, matching the real raised default) + // alongside a conservative anime cap (2) must claim up to 6 + // live-action jobs (only 3 exist) but no more than 2 anime jobs, + // even though 3 anime jobs are pending and the anime cap is far + // below the live-action one. + let claimed = claim_pending_jobs(&conn, 6, 2).await.unwrap(); + let live_action_claimed = claimed.iter().filter(|j| !j.is_anime).count(); + let anime_claimed = claimed.iter().filter(|j| j.is_anime).count(); + assert_eq!(live_action_claimed, 3, "all 3 pending live-action jobs should be claimed"); + assert_eq!(anime_claimed, 2, "anime claims must stay capped at 2 regardless of the live-action cap"); + } + + fn generate_test_clip(dir: &Path, codec: &str) -> PathBuf { + let path = dir.join(format!("clip_{codec}.mkv")); + let status = std::process::Command::new("ffmpeg") + .args(["-y", "-f", "lavfi", "-i", "testsrc=size=320x240:duration=1:rate=1"]) + .args(["-c:v", codec]) + .arg(&path) + .output() + .expect("failed to run ffmpeg to generate a test clip"); + assert!( + status.status.success(), + "ffmpeg failed to generate a {codec} test clip: {}", + String::from_utf8_lossy(&status.stderr) + ); + path + } + + // Regression test for a real gap found in review: `ensure_probed` + // inserts a `probe_failed` row with NULL codec/height on ffprobe + // failure rather than leaving no row at all, so `maybe_enqueue_transcode`'s + // "no probe row -> skip" guard never fires for it. Without this check, + // `should_enqueue` would return true for that NULL/NULL row, `enqueue` + // would insert a job `claim_pending_jobs` can never select (it requires + // `p.width`/`p.height IS NOT NULL`), and the unique partial index on + // (pending, running) would then permanently block any future, real + // enqueue for the file once probing eventually succeeds. + #[test] + fn should_enqueue_rejects_missing_codec_or_height() { + let c = cfg(); + assert!(!should_enqueue(None, false, None, &c), "no codec, no height -> never enqueue"); + assert!(!should_enqueue(None, false, Some(1080), &c), "missing codec alone must block enqueue"); + assert!(!should_enqueue(Some("hevc"), false, None, &c), "missing height alone must block enqueue"); + assert!(should_enqueue(Some("hevc"), false, Some(1080), &c), "codec+height present -> normal eligibility rules apply"); + } + + // Regression test for a real gap found in review: a crash between the + // file-swap rename and the DB bookkeeping in `finalize_job` leaves a job + // `running` with the file already AV1 but `media_file_probe` still + // stale. On retry, `encode_and_verify` must notice the *actual* file is + // already AV1 and skip re-encoding it, rather than trusting the stale + // DB probe data it was claimed with. + #[test] + fn encode_and_verify_skips_a_file_that_is_already_av1() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-already-av1-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_test_clip(&dir, "libsvtav1"); + + let result = + encode_and_verify(clip.clone(), 999, cfg(), 320, 240, Some(1.0), false, false).unwrap(); + assert!( + matches!(result, EncodeOutcome::AlreadyAv1), + "an already-AV1 file must not be re-encoded" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + // Regression test for a real gap found in review: the temp path used to + // be produced via `input.with_extension("job{id}.av1.mkv")`, so a + // multi-hour in-progress encode sat in the library as a normal-looking + // `.mkv` file — `collect_video_files`/`walk_files` (both filter purely + // on `VIDEO_EXTS`) would pick it up as a second candidate video file for + // the same episode, and `library_scan`/Jellyfin could index or attempt + // to play a still-encoding file. The fix must produce a path whose + // final extension isn't in `VIDEO_EXTS` at all, still under the same + // parent directory (required for the atomic rename-based swap), and + // still job-id-scoped (so two jobs on the same file never collide). + #[test] + fn temp_path_for_is_not_picked_up_by_video_file_scans() { + let input = Path::new("/library/Show/Season 01/Show - S01E01 - Title.mkv"); + let tmp = temp_path_for(input, 42); + + assert_eq!(tmp.parent(), input.parent(), "must stay on the same filesystem/directory as the original"); + let ext = tmp.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase(); + assert!( + !crate::importer::VIDEO_EXTS.contains(&ext.as_str()), + "temp path {tmp:?} has a video extension and would be picked up by library scans" + ); + assert!( + tmp.to_string_lossy().contains("job42"), + "must stay job-id-scoped so two jobs on the same file can never collide" + ); + + // Different job ids on the same input must never collide. + let other = temp_path_for(input, 43); + assert_ne!(tmp, other); + } + + // Fast, deterministic regression test for the actual root cause of a + // real incident: flat-bitrate rate control on `av1_vaapi` produced + // outputs *larger* than the original on 476 of 700 backfilled files. + // This is the exact arithmetic that must reject that outcome regardless + // of how any encoder's rate control behaves. + #[test] + fn is_beneficial_rejects_growth_and_marginal_shrinkage() { + // The real incident's shape: ~2.1GB HEVC -> ~4.9GB "AV1". + assert!(!is_beneficial(2_100_000_000, 4_900_000_000, 0.10)); + // Only a 5% reduction — below the 10% floor. + assert!(!is_beneficial(1_000_000_000, 950_000_000, 0.10)); + // A real win: 15% smaller, clears the floor. + assert!(is_beneficial(1_000_000_000, 850_000_000, 0.10)); + // Exactly at the floor — inclusive. + assert!(is_beneficial(1_000_000_000, 900_000_000, 0.10)); + } + + // The dynamic Jellyfin-reservation formula: reserve exactly as many + // GPU streams as Jellyfin is actually using out of the measured total + // budget, rather than dropping to a flat minimum regardless of viewer + // count. + #[test] + fn live_action_parallelism_for_reserves_exactly_what_jellyfin_is_using() { + assert_eq!(live_action_parallelism_for(7, 0), 7, "no active viewers — use the full budget"); + assert_eq!(live_action_parallelism_for(7, 1), 6, "one viewer — reserve exactly one stream"); + assert_eq!(live_action_parallelism_for(7, 3), 4, "three viewers — reserve exactly three streams"); + assert_eq!( + live_action_parallelism_for(7, 9), + 0, + "more active viewers than the whole budget — batch gets nothing, never negative" + ); + } + + // Regression test for the second half of the same incident: several of + // the 199 failures were files (some anime, notably) already efficient + // enough that transcoding them was never going to help — this is the + // pre-check that skips the encode attempt entirely rather than burning + // GPU/CPU time to find that out the slow way. No real ffmpeg encode + // should run here — a huge fabricated duration forces the source + // bitrate calculation near zero regardless of resolution, so this stays + // fast and deterministic. + #[test] + fn encode_and_verify_skips_pre_emptively_when_source_is_already_efficient() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-already-efficient-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_test_clip(&dir, "libx264"); + + let result = + encode_and_verify(clip.clone(), 1, cfg(), 1920, 1080, Some(100_000.0), false, false).unwrap(); + assert!( + matches!(result, EncodeOutcome::NotBeneficial), + "an already-efficient source must be skipped before spending encode time" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + fn generate_lossless_test_clip(dir: &Path) -> PathBuf { + let path = dir.join("clip_lossless.mkv"); + let status = std::process::Command::new("ffmpeg") + .args(["-y", "-f", "lavfi", "-i", "testsrc=size=640x480:duration=2:rate=15"]) + .args(["-c:v", "libx264", "-preset", "ultrafast", "-qp", "0"]) + .arg(&path) + .output() + .expect("failed to run ffmpeg to generate a lossless test clip"); + assert!( + status.status.success(), + "ffmpeg failed to generate a lossless test clip: {}", + String::from_utf8_lossy(&status.stderr) + ); + path + } + + // Reproduces the real "embedded cover art" incident: a second video + // stream with `attached_pic` disposition (a still image, ffmpeg reports + // it as a normal `video`-type stream) alongside the real content stream. + // The old blanket `-map 0` tried to re-encode this too, which the + // VAAPI/libsvtav1 encoders can't handle — failed the whole job on + // several real library files (GoT, Avatar). + fn generate_clip_with_attached_pic(dir: &Path) -> PathBuf { + let path = dir.join("clip_with_cover_art.mkv"); + let status = std::process::Command::new("ffmpeg") + .args(["-y", "-f", "lavfi", "-i", "testsrc=size=640x480:duration=2:rate=15"]) + .args(["-f", "lavfi", "-i", "color=c=red:size=64x64:duration=1"]) + .args(["-map", "0:v", "-map", "1:v"]) + .args(["-c:v:0", "libx264", "-preset", "ultrafast", "-qp", "0"]) + .args(["-c:v:1", "png"]) + .args(["-disposition:v:1", "attached_pic"]) + .arg("-shortest") + .arg(&path) + .output() + .expect("failed to run ffmpeg to generate a test clip with attached cover art"); + assert!( + status.status.success(), + "ffmpeg failed to generate a test clip with attached cover art: {}", + String::from_utf8_lossy(&status.stderr) + ); + path + } + + // Reproduces the real "mov_text subtitle" incident: mp4 sources using + // mov_text (mp4's own timed-text codec) aren't valid inside the + // Matroska container this pipeline always writes, and stream-copying + // them failed the whole job outright. + fn generate_clip_with_mov_text_subtitle(dir: &Path) -> PathBuf { + let srt_path = dir.join("sub.srt"); + std::fs::write(&srt_path, "1\n00:00:00,000 --> 00:00:01,000\nTest subtitle\n").unwrap(); + + let path = dir.join("clip_with_mov_text.mp4"); + let status = std::process::Command::new("ffmpeg") + .args(["-y", "-f", "lavfi", "-i", "testsrc=size=640x480:duration=2:rate=15"]) + .arg("-i") + .arg(&srt_path) + .args(["-c:v", "libx264", "-preset", "ultrafast", "-qp", "0"]) + .args(["-c:s", "mov_text"]) + .args(["-f", "mp4"]) + .arg(&path) + .output() + .expect("failed to run ffmpeg to generate a test clip with a mov_text subtitle"); + assert!( + status.status.success(), + "ffmpeg failed to generate a test clip with a mov_text subtitle: {}", + String::from_utf8_lossy(&status.stderr) + ); + path + } + + #[test] + fn subtitle_codec_args_converts_only_mov_text() { + let subs = vec![ + ffprobe::SubtitleStream { language: Some("eng".to_string()), codec: Some("mov_text".to_string()) }, + ffprobe::SubtitleStream { language: Some("eng".to_string()), codec: Some("ass".to_string()) }, + ffprobe::SubtitleStream { language: None, codec: None }, + ]; + let args = subtitle_codec_args(&subs); + assert_eq!( + args, + vec![ + "-c:s:0".to_string(), "srt".to_string(), + "-c:s:1".to_string(), "copy".to_string(), + "-c:s:2".to_string(), "copy".to_string(), + ] + ); + } + + // End-to-end regression test for the real "embedded cover art" incident + // — an attached-pic stream must no longer crash the whole encode. + // Uses the anime (software) pipeline since it needs no VAAPI hardware. + #[test] + fn encode_and_verify_handles_a_source_with_attached_cover_art() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-attached-pic-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_clip_with_attached_pic(&dir); + + let mut c = cfg(); + c.skip_below_ceiling_ratio = 0.0; // force past the pre-check for this test + let result = encode_and_verify(clip.clone(), 3, c, 640, 480, Some(2.0), true, false).unwrap(); + match result { + EncodeOutcome::Encoded(tmp_path) => { + assert!(tmp_path.exists()); + let _ = std::fs::remove_file(&tmp_path); + } + other => panic!("expected a beneficial encode despite the attached cover art, got {other:?}"), + } + + std::fs::remove_dir_all(&dir).unwrap(); + } + + // End-to-end regression test for the real "mov_text subtitle" incident + // — a mov_text track must be converted rather than crashing the encode. + #[test] + fn encode_and_verify_handles_a_source_with_a_mov_text_subtitle() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-mov-text-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_clip_with_mov_text_subtitle(&dir); + + let mut c = cfg(); + c.skip_below_ceiling_ratio = 0.0; + let result = encode_and_verify(clip.clone(), 4, c, 640, 480, Some(2.0), true, false).unwrap(); + match result { + EncodeOutcome::Encoded(tmp_path) => { + assert!(tmp_path.exists()); + let _ = std::fs::remove_file(&tmp_path); + } + other => panic!("expected a beneficial encode despite the mov_text subtitle, got {other:?}"), + } + + std::fs::remove_dir_all(&dir).unwrap(); + } + + // End-to-end sanity check for the anime pipeline specifically: a real + // `libsvtav1` 10-bit encode of a deliberately bloated (lossless x264) + // source should produce a verified, meaningfully smaller output. Doesn't + // need real VAAPI hardware (unlike the live-action path) since + // `libsvtav1` is software — safe to run in any dev/CI environment that + // has an AV1-capable ffmpeg build. + #[test] + fn encode_and_verify_anime_pipeline_shrinks_a_bloated_source() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-anime-e2e-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let clip = generate_lossless_test_clip(&dir); + + let mut c = cfg(); + c.skip_below_ceiling_ratio = 0.0; // force past the pre-check for this test + let result = encode_and_verify(clip.clone(), 2, c, 640, 480, Some(2.0), true, false).unwrap(); + match result { + EncodeOutcome::Encoded(tmp_path) => { + assert!(tmp_path.exists()); + let _ = std::fs::remove_file(&tmp_path); + } + other => panic!("expected a beneficial encode of a lossless source, got {other:?}"), + } + + std::fs::remove_dir_all(&dir).unwrap(); + } + + // Regression test for the remediation path: `force_reencode: true` must + // bypass the normal "already AV1 -> nothing to do" short-circuit and + // attempt a real re-encode — this is the whole point of + // `find_oversized_av1_candidates`'s jobs, which target files that are + // already AV1 but were left larger than their original by a real + // rate-control bug. Uses a genuinely oversized AV1 source (a lossless + // x264 clip re-encoded to AV1 at a high bitrate, deliberately bigger + // than what a sane target would produce) so the *second* pass — the one + // under test — has real room to shrink it. + #[test] + fn encode_and_verify_force_reencode_bypasses_the_already_av1_short_circuit() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-force-reencode-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let oversized_av1 = dir.join("oversized.mkv"); + let status = std::process::Command::new("ffmpeg") + .args(["-y", "-f", "lavfi", "-i", "testsrc=size=640x480:duration=2:rate=15"]) + .args(["-c:v", "libsvtav1", "-crf", "10", "-preset", "12"]) // deliberately high quality/bitrate + .arg(&oversized_av1) + .output() + .expect("failed to run ffmpeg to generate an oversized AV1 test clip"); + assert!(status.status.success(), "{}", String::from_utf8_lossy(&status.stderr)); + + // Sanity check: without force_reencode, this must still short-circuit. + let unforced = + encode_and_verify(oversized_av1.clone(), 5, cfg(), 640, 480, Some(2.0), true, false).unwrap(); + assert!(matches!(unforced, EncodeOutcome::AlreadyAv1)); + + // is_anime: true — routes through the software libsvtav1 path, same + // as the other end-to-end tests here, since this dev/CI environment + // has no VAAPI hardware for the live-action path to use. + let mut c = cfg(); + c.skip_below_ceiling_ratio = 0.0; + let forced = + encode_and_verify(oversized_av1.clone(), 6, c, 640, 480, Some(2.0), true, true).unwrap(); + match forced { + EncodeOutcome::Encoded(tmp_path) => { + let new_bytes = std::fs::metadata(&tmp_path).unwrap().len(); + let original_bytes = std::fs::metadata(&oversized_av1).unwrap().len(); + assert!( + new_bytes < original_bytes, + "forced re-encode should shrink an oversized AV1 source" + ); + let _ = std::fs::remove_file(&tmp_path); + } + other => panic!("expected force_reencode to produce a real, smaller encode, got {other:?}"), + } + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn is_anime_path_matches_configured_root_folders() { + let c = cfg(); + assert!(is_anime_path( + Path::new("/mnt/media/Anime/Attack on Titan (2013)/Season 02/ep.mkv"), + &c + )); + assert!(is_anime_path( + Path::new("/mnt/media/Anime Movies/Bardock (1990)/movie.mkv"), + &c + )); + assert!(!is_anime_path( + Path::new("/mnt/media/TV Shows/The Expanse (2015)/ep.mkv"), + &c + )); + } + + #[test] + fn target_bitrate_matches_reference_at_reference_resolution() { + let bitrate = target_bitrate_kbps(1920, 1080, &cfg()); + // pixel_ratio == 1.0 at the reference resolution, so this should be + // exactly reference_bitrate_kbps * av1_efficiency_factor. + assert_eq!(bitrate, (5320.0_f64 * 0.7).round() as u32); + } + + #[test] + fn target_bitrate_scales_down_for_720p() { + let bitrate_1080 = target_bitrate_kbps(1920, 1080, &cfg()); + let bitrate_720 = target_bitrate_kbps(1280, 720, &cfg()); + assert!(bitrate_720 < bitrate_1080); + // Roughly proportional to pixel count (~0.44x), not some flat cut. + let ratio = bitrate_720 as f64 / bitrate_1080 as f64; + assert!((0.4..0.5).contains(&ratio), "ratio was {ratio}"); + } + + #[test] + fn target_bitrate_scales_up_for_1440p() { + let bitrate_1080 = target_bitrate_kbps(1920, 1080, &cfg()); + let bitrate_1440 = target_bitrate_kbps(2560, 1440, &cfg()); + assert!(bitrate_1440 > bitrate_1080); + } + + // Regression test for a real latency bug a live validation run + // surfaced: `run_cycle` used to await claimed jobs' handles in claim + // order (highest-bitrate-first), so a fast job's already-finished + // result sat idle — not written back to the DB — for as long as + // whichever job happened to come *first* in that order kept running. + // The fix (`JoinSet::join_next_with_id`) finalizes whichever job + // actually finishes first, which raises a sharper risk worth its own + // direct test: a job's result must land on *that job's* DB row, not + // get cross-attributed to a different concurrently-running job via a + // wrong task-id lookup. This seeds one job that's claimed first (by + // this test's bitrate ordering) but takes measurably longer, and one + // claimed second that finishes near-instantly, and confirms each + // outcome lands on the correct `episode_file`/`transcode_job` row + // regardless of which one the JoinSet actually resolves first. + #[tokio::test] + async fn run_cycle_attributes_each_result_to_the_correct_job_even_out_of_claim_order() { + let dir = std::env::temp_dir().join(format!( + "breadarr-transcode-run-cycle-attribution-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + + // Claimed first (much higher bitrate) but genuinely slower — a + // real (tiny) SVT-AV1 encode, not a short-circuit. + let slow_clip = generate_lossless_test_clip(&dir); + let slow_id = 101i64; + // Claimed second (lower bitrate) but resolves almost immediately — + // already AV1, hits the `AlreadyAv1` short-circuit with no real + // encode at all. + let fast_clip = generate_test_clip(&dir, "libsvtav1"); + let fast_id = 102i64; + + let conn = Connection::open_in_memory().unwrap(); + crate::db::init(&conn).unwrap(); + for (id, path, codec, size_bytes, duration) in [ + (slow_id, &slow_clip, "h264", 50_000_000i64, 2.0), + (fast_id, &fast_clip, "av1", 1_000_000i64, 1.0), + ] { + conn.execute( + "INSERT INTO episode_file (id, episode_id, media_item_id, path, size_bytes) + VALUES (?1, NULL, NULL, ?2, ?3)", + params![id, path.to_string_lossy().to_string(), size_bytes], + ) + .unwrap(); + conn.execute( + "INSERT INTO media_file_probe (episode_file_id, probed_at, probe_size_bytes, probe_mtime, + duration_secs, video_codec, width, height) + VALUES (?1, datetime('now'), ?2, 0, ?3, ?4, 640, 480)", + params![id, size_bytes, duration, codec], + ) + .unwrap(); + } + // Bitrate-descending claim order puts the slow job first: its size + // is set so `size_bytes*8/duration` comfortably exceeds the fast + // job's, matching the real incident's shape (a big, slow file + // claimed ahead of a small, fast one). + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, is_anime, queued_at) VALUES (?1, 'pending', 1, datetime('now'))", + params![slow_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO transcode_job (episode_file_id, status, is_anime, queued_at) VALUES (?1, 'pending', 0, datetime('now'))", + params![fast_id], + ) + .unwrap(); + + let conn = Arc::new(Mutex::new(conn)); + let mut c = cfg(); + c.skip_below_ceiling_ratio = 0.0; // don't pre-skip the slow job + let stats = run_cycle(conn.clone(), c, None).await.unwrap(); + assert_eq!(stats.attempted, 2); + assert_eq!(stats.failed, 0); + + let conn = conn.lock().await; + let (slow_status, slow_path): (String, String) = conn + .query_row( + "SELECT tj.status, ef.path FROM transcode_job tj JOIN episode_file ef ON ef.id = tj.episode_file_id WHERE tj.episode_file_id = ?1", + params![slow_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + let (fast_status, fast_new_bytes, fast_path): (String, i64, String) = conn + .query_row( + "SELECT tj.status, tj.new_bytes, ef.path FROM transcode_job tj JOIN episode_file ef ON ef.id = tj.episode_file_id WHERE tj.episode_file_id = ?1", + params![fast_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap(); + + assert_eq!(slow_status, "done", "the real encode must land on the slow job's own row"); + assert_eq!(slow_path, slow_clip.to_string_lossy(), "must not cross-attribute the fast job's path"); + assert_eq!(fast_status, "done", "the already-AV1 short-circuit must land on the fast job's own row"); + assert_eq!(fast_new_bytes, 0, "AlreadyAv1 records no byte-count change"); + assert_eq!(fast_path, fast_clip.to_string_lossy(), "must not cross-attribute the slow job's path"); + + // Regression coverage for a real gap found in review: `finalize_job` + // must set `upgrade_locked` in the same transaction as the new + // `size_bytes`, and a best-effort re-probe afterward must still + // leave `media_file_probe` correctly reflecting the swapped-in AV1 + // file — not the stale pre-encode codec, and not silently unset + // just because that refresh is no longer allowed to fail the job. + let (upgrade_locked, probed_codec): (i64, Option) = conn + .query_row( + "SELECT ef.upgrade_locked, p.video_codec FROM episode_file ef + LEFT JOIN media_file_probe p ON p.episode_file_id = ef.id + WHERE ef.id = ?1", + params![slow_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(upgrade_locked, 1, "a real encode must lock the file against the ordinary upgrade cycle"); + assert_eq!(probed_codec.as_deref(), Some("av1"), "the post-swap re-probe must reflect the new AV1 file, not stale pre-encode data"); + + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/ci/bread-ecosystem.rev b/ci/bread-ecosystem.rev new file mode 100644 index 0000000..34e7aa9 --- /dev/null +++ b/ci/bread-ecosystem.rev @@ -0,0 +1 @@ +147cfbbf96ae4b171027defa1130d2caddb934b1 diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..7ca25e8 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Delegates to bread-ecosystem's shared CI build image/script, pinned to +# the commit in ci/bread-ecosystem.rev — not `main`. bread-ecosystem's CI +# files now affect every product's release pipeline, so bumping the pin +# is a deliberate act instead of silent drift (see the bread-theme test +# that broke here for exactly that reason, before it was pinned by rev). +# +# Usage: ci/build.sh cargo build --release --locked +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REV="$(cat "${ROOT}/ci/bread-ecosystem.rev")" + +CACHE_DIR="/tmp/bread-ecosystem-ci-${REV}" +# Only create/replace this product's own pin directory. A glob rm of +# /tmp/bread-ecosystem-ci-* races other products' mktemp clones used by +# release-index regen. +if [ ! -d "$CACHE_DIR/.git" ]; then + rm -rf "$CACHE_DIR" + git clone https://git.breadway.dev/Breadway/bread-ecosystem.git "$CACHE_DIR" + git -C "$CACHE_DIR" checkout --quiet "$REV" +fi + +# hestia (breadarrd's actual deployment target) runs Ubuntu 24.04 (glibc +# 2.39). The shared CI image is Arch, which tracks a much newer glibc +# (currently 2.43) — a binary linked normally there refuses to start on +# hestia (glibc symbol versioning is forward-only). Rather than patch the +# shared image (would affect every other product's build), pull down a +# real, complete glibc + matching gcc-libs (for libstdc++) from the Arch +# Linux Archive and link against those via --sysroot, so the binary only +# requires symbol versions that actually exist on hestia. +# +# cargo-zigbuild (targeting an older glibc via zig's cross-linker) was +# tried first and doesn't work here: zig only maintains a version database +# for glibc's own symbols, not libstdc++'s, and onnxruntime — statically +# linked in by the `ort` crate — needs a real, complete libstdc++, not +# zig's minimal stand-in. A real archived glibc+gcc-libs package pair, +# linked via --sysroot, sidesteps that entirely. +GLIBC_VER="2.39-4" +GCC_LIBS_VER="13.2.1-6" +# Pinned to the archive.archlinux.org packages fetched above (sha256 of +# the two .pkg.tar.zst files, not the extracted trees). +GLIBC_SHA256="c6aef7065e0d53d700cc5ff65d4db775cb84e2f4e3912a609223ed72fb1799d4" +GCC_LIBS_SHA256="edbb8c4772b8852fe102853a84d197253127418f50cca475feda9bbaa842a378" +OLD_GLIBC_CACHE="/tmp/bread-ci-old-glibc-${GLIBC_VER}" +if [ ! -d "${OLD_GLIBC_CACHE}/usr/lib" ]; then + rm -rf "${OLD_GLIBC_CACHE}" + mkdir -p "${OLD_GLIBC_CACHE}" + curl -sfL -o /tmp/old-glibc.pkg.tar.zst \ + "https://archive.archlinux.org/packages/g/glibc/glibc-${GLIBC_VER}-x86_64.pkg.tar.zst" + curl -sfL -o /tmp/old-gcc-libs.pkg.tar.zst \ + "https://archive.archlinux.org/packages/g/gcc-libs/gcc-libs-${GCC_LIBS_VER}-x86_64.pkg.tar.zst" + echo "${GLIBC_SHA256} /tmp/old-glibc.pkg.tar.zst" | sha256sum -c - + echo "${GCC_LIBS_SHA256} /tmp/old-gcc-libs.pkg.tar.zst" | sha256sum -c - + tar --zstd -xf /tmp/old-glibc.pkg.tar.zst -C "${OLD_GLIBC_CACHE}" + tar --zstd -xf /tmp/old-gcc-libs.pkg.tar.zst -C "${OLD_GLIBC_CACHE}" + rm -f /tmp/old-glibc.pkg.tar.zst /tmp/old-gcc-libs.pkg.tar.zst +fi +# The shared build script only bind-mounts $ROOT (as /workspace) into the +# container, not arbitrary host paths — so the cached sysroot has to be +# copied under $ROOT to be visible there. The download above is cached +# persistently in /tmp across runs; this copy is just a fast local `cp`. +rm -rf "${ROOT}/.ci-old-glibc" +cp -a "${OLD_GLIBC_CACHE}" "${ROOT}/.ci-old-glibc" + +bash "${CACHE_DIR}/ci/build.sh" breadarr "$ROOT" sh -c ' + RUSTFLAGS="-C link-arg=--sysroot=/workspace/.ci-old-glibc -C link-arg=-L/workspace/.ci-old-glibc/usr/lib" \ + exec "$@" +' sh "$@" diff --git a/config.example.toml b/config.example.toml index 383929c..0c64aaa 100644 --- a/config.example.toml +++ b/config.example.toml @@ -12,7 +12,9 @@ model_dir = "~/.cache/breadarr/models/all-MiniLM-L6-v2" # Empty (default) means no auth at all. Set this if listen_addr is ever # changed to bind non-loopback (e.g. so a TUI on another host on the same # tailnet can reach it) — otherwise that's unauthenticated add/delete/search -# access to anyone who can reach the port. /health is always exempt. +# access to anyone who can reach the port. The API is plaintext HTTP; there +# is no TLS. /health is always exempt (liveness only). /health/detail +# carries cycle info and is authenticated when this token is set. api_token = "" [qbit] @@ -50,15 +52,32 @@ api_key = "" bearer_token = "" [library] -# Default root folder for shows added via the TUI's "Add Show" flow. +# Default root folder for shows added via the TUI's "Add" flow. Each show +# lands in its own "{root}/{Title} ({Year})" subfolder. default_root_folder = "~/breadarr-library" +# Same, but for movies added via the TUI's "Add" flow — kept separate from +# default_root_folder since movies and shows live under different category +# roots on disk. +movies_root_folder = "~/breadarr-library/Movies" [sources] # nyaa's English-translated anime category — the daemon polls this # automatically and grabs anything matching a monitored, missing episode. nyaa_rss_url = "https://nyaa.si/?page=rss&c=1_2" grab_poll_interval_secs = 300 +# Kill switch for the passive RSS-feed grab loop (nyaa). Independent of +# search_enabled / upgrade_enabled. +grab_enabled = true import_poll_interval_secs = 60 +# Community JSON API mirror of The Pirate Bay — primary general-content +# search source (movies + non-anime TV). +tpb_api_url = "https://apibay.org/q.php" +# torrents.csv DHT-dump search — first fallback when TPB fails or returns +# nothing. Same hash-to-magnet grab; movies and TV. +torrents_csv_url = "https://torrents-csv.com/service/search" +# YTS v2 list_movies JSON — movie-only fallback. yts.mx does not resolve; +# yts.lt is a working host as of 2026-08-16. +yts_api_url = "https://yts.lt/api/v2/list_movies.json" # Search-driven acquisition (movies + non-anime TV via 1337x, anime movies # via nyaa's search mode) — unlike the nyaa RSS feed watch above, this # actively queries a Cloudflare-fronted service with a ban history, so keep @@ -83,6 +102,7 @@ upgrade_min_score_gain = 5.0 # tried in a fixed fallback order), with a failing mirror demoted into a # cooldown rather than re-probed on the very next search. torrent_1337x_mirrors = [ + "https://www.1337xx.to", "https://13377x.info", "https://13377x.email", "https://1337xto.info", @@ -97,3 +117,31 @@ torrent_1337x_mirrors = [ "https://1337x.unblocktorrent.info", "https://1337x.unblocktor.xyz", ] + +# Off by default — GPU/CPU AV1 encode needs a calibration pass against +# real content on the target hardware before it's safe unattended. +# Defaults match breadarr-shared/src/config.rs. +[transcode] +enabled = false +poll_interval_secs = 60 +vaapi_device = "/dev/dri/renderD128" +parallelism_min = 1 +# Live-action (av1_vaapi) concurrent-stream ceiling; Jellyfin viewers +# are subtracted from this each cycle. +parallelism_max = 7 +# Separate CPU-bound anime (libsvtav1) ceiling. +parallelism_max_anime = 2 +reference_bitrate_kbps = 5320 +reference_height = 1080 +av1_efficiency_factor = 0.7 +exclude_hdr = true +exclude_min_height = 2000 +quality_live_action = 26 +# Path prefixes routed to the anime encode pipeline. Empty is a no-op. +anime_root_folders = [] +quality_anime = 24 +anime_svtav1_preset = 10 +anime_svtav1_max_threads = 4 +min_size_reduction_pct = 0.10 +skip_below_ceiling_ratio = 0.5 +verify_sample_secs = 20.0 diff --git a/packaging/systemd/breadarrd.service b/packaging/systemd/breadarrd.service index 46525b5..a864d04 100644 --- a/packaging/systemd/breadarrd.service +++ b/packaging/systemd/breadarrd.service @@ -17,7 +17,14 @@ UMask=0022 RuntimeDirectory=breadarr RuntimeDirectoryMode=0700 KillSignal=SIGTERM -TimeoutStopSec=5 +# Transcode jobs can run for minutes; 5s was cutting them off on stop. +TimeoutStopSec=180 +# Modest hardening valid in a user unit. Skip ProtectHome (library + +# config live under $HOME) and MemoryDenyWriteExecute (ort/onnxruntime +# may need JIT / RWX mappings). +NoNewPrivileges=yes +RestrictSUIDSGID=yes +LockPersonality=yes [Install] WantedBy=default.target