Continue transcode/import feature work and fix bugs found by audit
These four files mix two things too interleaved to commit separately: in-progress work on anime pipeline tuning (quality/preset/thread config, per-pipeline parallelism caps), sampled decode verification, and subtitle-codec-aware remuxing that predates this commit, plus a set of correctness fixes from an independent Opus 5 review applied directly on top of it: - Attached-pic (cover art) streams could be probed as the real video stream when they came first, silently replacing the actual codec/height (ffprobe.rs) - probe_failed rows (null codec/height) were enqueued for transcode and could never be claimed again, due to the unique partial index - should_enqueue now rejects them - tokio::try_join! cancelled the sibling pipeline's in-flight spawn_blocking encode on the first Err instead of letting it finish - Duplicate episode_file rows for the same media only had one swept on an upgrade swap, leaving stale rows/files behind - File-swap and DB-write on transcode completion weren't atomic; wrapped in a transaction and made probe-refresh failure non-fatal - In-progress transcode temp files were visible to library scans (video extension, no dotfile prefix) and could get imported mid-encode - Remux temp files leaked on error paths; a Jellyfin refresh failure failed the whole import cycle instead of just logging - insufficient_space could false-positive when the download and library dirs are on the same filesystem (rename is free there) - Config validation for reference_height and min_size_reduction_pct, which previously could silently divide-by-zero or overflow deep inside an encode
This commit is contained in:
parent
0f609aa4cc
commit
109b29ee55
4 changed files with 2745 additions and 630 deletions
|
|
@ -297,14 +297,42 @@ pub struct TranscodeConfig {
|
||||||
pub poll_interval_secs: u64,
|
pub poll_interval_secs: u64,
|
||||||
#[serde(default = "default_vaapi_device")]
|
#[serde(default = "default_vaapi_device")]
|
||||||
pub vaapi_device: String,
|
pub vaapi_device: String,
|
||||||
|
/// Applies to both pipelines identically when Jellyfin reports an
|
||||||
|
/// active transcoding session — deliberately not split per-pipeline;
|
||||||
|
/// when someone's actually watching something, both the GPU (which
|
||||||
|
/// they're using) and CPU (contending for the same box) should back off.
|
||||||
#[serde(default = "default_parallelism_min")]
|
#[serde(default = "default_parallelism_min")]
|
||||||
pub parallelism_min: usize,
|
pub parallelism_min: usize,
|
||||||
/// Ramp-up ceiling for concurrent encode streams during a backfill —
|
/// The total concurrent **live-action** (`av1_vaapi`, GPU-bound) stream
|
||||||
/// tune this against how many simultaneous `av1_vaapi` sessions the
|
/// budget — not just "the batch job's cap", but the real ceiling the
|
||||||
/// target GPU can actually sustain before per-stream throughput starts
|
/// GPU can sustain at all, batch work and live Jellyfin viewers
|
||||||
/// dropping, not just picked arbitrarily.
|
/// combined. `transcode::live_action_parallelism_for` subtracts however
|
||||||
|
/// many Jellyfin transcode sessions are actually active from this
|
||||||
|
/// number to get the batch job's actual parallelism each cycle, so
|
||||||
|
/// real viewers get exactly the headroom they need rather than the
|
||||||
|
/// batch job dropping to a flat minimum regardless of how many people
|
||||||
|
/// are watching. Deliberately separate from `parallelism_max_anime` —
|
||||||
|
/// the two pipelines contend for genuinely different hardware (GPU
|
||||||
|
/// encode engine vs CPU threads), so raising one shouldn't raise the
|
||||||
|
/// other. Empirically calibrated on Hestia's Arc A380: per-stream
|
||||||
|
/// throughput stays above 1.5x realtime through 7 concurrent streams,
|
||||||
|
/// crosses below it at 8 (aggregate throughput itself plateaus around
|
||||||
|
/// ~12x realtime from 5-6 streams on, i.e. the GPU's actual saturation
|
||||||
|
/// point) — see [[breadarr-av1-transcode]] for the full scaling-test
|
||||||
|
/// numbers.
|
||||||
#[serde(default = "default_parallelism_max")]
|
#[serde(default = "default_parallelism_max")]
|
||||||
pub parallelism_max: usize,
|
pub parallelism_max: usize,
|
||||||
|
/// Ramp-up ceiling for concurrent **anime** (`libsvtav1`, CPU-bound)
|
||||||
|
/// encode streams — capped separately from `parallelism_max` (see its
|
||||||
|
/// doc comment) precisely because a single shared cap would let "raise
|
||||||
|
/// GPU parallelism" accidentally also raise anime concurrency, and
|
||||||
|
/// anime jobs are CPU-thread-hungry (`anime_svtav1_max_threads` each)
|
||||||
|
/// in a way live-action jobs aren't. Kept at the original
|
||||||
|
/// conservative shared-cap default (2) since concurrent-anime-job
|
||||||
|
/// memory/CPU behavior at higher counts hasn't been load-tested the
|
||||||
|
/// way the live-action GPU path has.
|
||||||
|
#[serde(default = "default_parallelism_max_anime")]
|
||||||
|
pub parallelism_max_anime: usize,
|
||||||
/// The "looks fine, no complaints" calibration reference: a real
|
/// The "looks fine, no complaints" calibration reference: a real
|
||||||
/// bitrate (Mbps, in kbps here) from content already in the library at
|
/// bitrate (Mbps, in kbps here) from content already in the library at
|
||||||
/// `reference_height` that the user is happy with. New AV1 encodes are
|
/// `reference_height` that the user is happy with. New AV1 encodes are
|
||||||
|
|
@ -332,6 +360,82 @@ pub struct TranscodeConfig {
|
||||||
/// anyway) — revisit once HDR handling is confirmed safe.
|
/// anyway) — revisit once HDR handling is confirmed safe.
|
||||||
#[serde(default = "default_exclude_min_height")]
|
#[serde(default = "default_exclude_min_height")]
|
||||||
pub exclude_min_height: u32,
|
pub exclude_min_height: u32,
|
||||||
|
/// `global_quality` for the live-action `av1_vaapi` `QVBR` encode — the
|
||||||
|
/// actual quality driver now that rate control is quality-based rather
|
||||||
|
/// than a flat bitrate target (see `run_ffmpeg_encode_live_action`).
|
||||||
|
/// `reference_bitrate_kbps`/`av1_efficiency_factor` still compute a
|
||||||
|
/// `-b:v`/`-maxrate`/`-bufsize` ceiling alongside this, so a source that's
|
||||||
|
/// already unusually efficient doesn't get inflated up toward the
|
||||||
|
/// ceiling — QVBR only spends up to it on content that actually needs it.
|
||||||
|
#[serde(default = "default_quality_live_action")]
|
||||||
|
pub quality_live_action: u32,
|
||||||
|
/// Root folder path prefixes (exact string prefix match against
|
||||||
|
/// `episode_file.path`) routed to the anime encode pipeline
|
||||||
|
/// (`run_ffmpeg_encode_anime`) instead of the live-action one, regardless
|
||||||
|
/// of `anime_mapping`/`anime_tmdb_movie` metadata coverage — path is a
|
||||||
|
/// more reliable signal than TVDB/TMDB anime-list membership, which has
|
||||||
|
/// real gaps (e.g. Avatar: The Last Airbender and some Dragon Ball movies
|
||||||
|
/// were missing from those tables and slipped through as "not anime").
|
||||||
|
/// Empty by default (a no-op) — set per-deployment to match how the
|
||||||
|
/// library is actually organized.
|
||||||
|
#[serde(default)]
|
||||||
|
pub anime_root_folders: Vec<String>,
|
||||||
|
/// CRF for the anime pipeline's software `libsvtav1` encode (0-63, lower
|
||||||
|
/// = higher quality/larger). No hardware AV1 10-bit encode entrypoint
|
||||||
|
/// exists on Hestia's Arc A380 (`vainfo` only lists `AV1Profile0`,
|
||||||
|
/// 8-bit) — anime needs true 10-bit output to avoid banding in the flat
|
||||||
|
/// gradients the art style is full of, so this pipeline trades GPU
|
||||||
|
/// offload for CPU-based `libsvtav1` specifically to get it.
|
||||||
|
#[serde(default = "default_quality_anime")]
|
||||||
|
pub quality_anime: u32,
|
||||||
|
/// `libsvtav1` preset (0-13, lower = slower/better compression AND more
|
||||||
|
/// memory-hungry — SVT-AV1's lookahead/reference buffering scales with
|
||||||
|
/// preset, not just thread count). Raised from an initial guess of 6 to
|
||||||
|
/// 10 after a real validation run hit a genuine kernel OOM: preset 6 on
|
||||||
|
/// a single 1080p anime episode grew to 9.3GB resident memory on
|
||||||
|
/// Hestia's 6-core/12-thread box. This runs as unattended background
|
||||||
|
/// work, so trading some compression efficiency for a much smaller,
|
||||||
|
/// safer memory footprint is the right call — see
|
||||||
|
/// `anime_svtav1_max_threads` for the other half of that fix.
|
||||||
|
#[serde(default = "default_anime_svtav1_preset")]
|
||||||
|
pub anime_svtav1_preset: u32,
|
||||||
|
/// Passed to `libsvtav1` as `-svtav1-params lp=N` — caps how many
|
||||||
|
/// worker threads it uses, independent of preset. More parallel workers
|
||||||
|
/// means more concurrently-buffered frames, so this is the other lever
|
||||||
|
/// (alongside `anime_svtav1_preset`) for bounding the encoder's peak
|
||||||
|
/// memory to something predictable regardless of how many cores the
|
||||||
|
/// host actually has. Default is conservative (well under a typical
|
||||||
|
/// modern host's core count) after the same OOM incident that raised
|
||||||
|
/// the preset default.
|
||||||
|
#[serde(default = "default_anime_svtav1_max_threads")]
|
||||||
|
pub anime_svtav1_max_threads: u32,
|
||||||
|
/// Hard floor on what counts as "worth keeping": a transcode whose
|
||||||
|
/// output isn't at least this fraction smaller than the original is
|
||||||
|
/// discarded (job marked `skipped`, original left untouched) rather than
|
||||||
|
/// swapped in. Exists because quality-driven rate control can still
|
||||||
|
/// occasionally produce an output that's the same size as or larger than
|
||||||
|
/// an already-efficient source — this is the invariant that makes that
|
||||||
|
/// safe regardless of how good the rate-control tuning is, after a real
|
||||||
|
/// incident where flat-bitrate VBR targeting silently produced files
|
||||||
|
/// *larger* than the original on the majority of a backfill.
|
||||||
|
#[serde(default = "default_min_size_reduction_pct")]
|
||||||
|
pub min_size_reduction_pct: f64,
|
||||||
|
/// Skip attempting a transcode at all (no GPU/CPU time spent) when the
|
||||||
|
/// source's current bitrate is already at or below this fraction of the
|
||||||
|
/// resolution-scaled ceiling (`target_bitrate_kbps`) — a strong signal
|
||||||
|
/// there's little room left to save, so it's not worth the encode time
|
||||||
|
/// to find out (the `min_size_reduction_pct` check above would reject
|
||||||
|
/// most of these anyway, this just avoids paying for that finding).
|
||||||
|
#[serde(default = "default_skip_below_ceiling_ratio")]
|
||||||
|
pub skip_below_ceiling_ratio: f64,
|
||||||
|
/// Size (seconds) of each of the three start/middle/end windows
|
||||||
|
/// `ffprobe::verify_decodable_sampled` actually decodes, instead of the
|
||||||
|
/// whole file — a full decode verification was measured as the actual
|
||||||
|
/// CPU bottleneck of a transcode cycle (400%+ CPU per job, dwarfing the
|
||||||
|
/// GPU encode time), not the encode itself. Bounds verification cost to
|
||||||
|
/// a small constant regardless of source length.
|
||||||
|
#[serde(default = "default_verify_sample_secs")]
|
||||||
|
pub verify_sample_secs: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for TranscodeConfig {
|
impl Default for TranscodeConfig {
|
||||||
|
|
@ -342,11 +446,20 @@ impl Default for TranscodeConfig {
|
||||||
vaapi_device: default_vaapi_device(),
|
vaapi_device: default_vaapi_device(),
|
||||||
parallelism_min: default_parallelism_min(),
|
parallelism_min: default_parallelism_min(),
|
||||||
parallelism_max: default_parallelism_max(),
|
parallelism_max: default_parallelism_max(),
|
||||||
|
parallelism_max_anime: default_parallelism_max_anime(),
|
||||||
reference_bitrate_kbps: default_reference_bitrate_kbps(),
|
reference_bitrate_kbps: default_reference_bitrate_kbps(),
|
||||||
reference_height: default_reference_height(),
|
reference_height: default_reference_height(),
|
||||||
av1_efficiency_factor: default_av1_efficiency_factor(),
|
av1_efficiency_factor: default_av1_efficiency_factor(),
|
||||||
exclude_hdr: default_exclude_hdr(),
|
exclude_hdr: default_exclude_hdr(),
|
||||||
exclude_min_height: default_exclude_min_height(),
|
exclude_min_height: default_exclude_min_height(),
|
||||||
|
quality_live_action: default_quality_live_action(),
|
||||||
|
anime_root_folders: Vec::new(),
|
||||||
|
quality_anime: default_quality_anime(),
|
||||||
|
anime_svtav1_preset: default_anime_svtav1_preset(),
|
||||||
|
anime_svtav1_max_threads: default_anime_svtav1_max_threads(),
|
||||||
|
min_size_reduction_pct: default_min_size_reduction_pct(),
|
||||||
|
skip_below_ceiling_ratio: default_skip_below_ceiling_ratio(),
|
||||||
|
verify_sample_secs: default_verify_sample_secs(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -364,12 +477,25 @@ fn default_parallelism_min() -> usize {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_parallelism_max() -> usize {
|
fn default_parallelism_max() -> usize {
|
||||||
// Conservative on purpose: a real incident on a shared 15GB host
|
// The measured total GPU budget (not "batch cap plus a static
|
||||||
// running a dozen+ other containers showed concurrent 1080p/4K
|
// reservation") — `live_action_parallelism_for` dynamically subtracts
|
||||||
// decode+encode sessions can push memory pressure into swap fast
|
// real Jellyfin transcode sessions from this each cycle. Raised from
|
||||||
// enough to trigger the OOM killer well before the GPU itself is the
|
// an initial conservative guess of 2 after an actual concurrent-stream
|
||||||
// bottleneck. Raise this deliberately, per-deployment, once you've
|
// scaling test on Hestia's Arc A380 (see `parallelism_max`'s doc
|
||||||
// watched `free -h` under real load at the current setting.
|
// comment): per-stream throughput stays above 1.5x realtime through 7
|
||||||
|
// total concurrent streams, crossing below at 8.
|
||||||
|
7
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_parallelism_max_anime() -> usize {
|
||||||
|
// Kept at the original conservative shared-cap value — unlike
|
||||||
|
// `parallelism_max`, this hasn't been load-tested at higher counts.
|
||||||
|
// A real incident already showed a *single* uncapped anime job could
|
||||||
|
// hit 9.3GB resident memory; multiple concurrent anime jobs (each its
|
||||||
|
// own `anime_svtav1_max_threads`-sized thread pool) multiply both CPU
|
||||||
|
// thread contention and memory pressure in a way the GPU path doesn't
|
||||||
|
// have to worry about. Raise deliberately, per-deployment, only after
|
||||||
|
// watching `free -h` and CPU load under real concurrent-anime load.
|
||||||
2
|
2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -393,6 +519,43 @@ fn default_exclude_min_height() -> u32 {
|
||||||
2000
|
2000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Starting point for `av1_vaapi`'s `-global_quality` under `QVBR`, needs the
|
||||||
|
// same real-hardware calibration pass as the bitrate reference did — this is
|
||||||
|
// a reasonable guess (roughly x264/x265 "visually near-lossless" territory
|
||||||
|
// on the encoder's internal QP-like scale), not a measured value.
|
||||||
|
fn default_quality_live_action() -> u32 {
|
||||||
|
26
|
||||||
|
}
|
||||||
|
|
||||||
|
// SVT-AV1 CRF starting point for the anime pipeline — slightly lower
|
||||||
|
// (higher quality) than the live-action guess above since flat-color/
|
||||||
|
// gradient-heavy anime content shows banding more readily than live-action
|
||||||
|
// grain/texture does at the same nominal quality level. Also unvalidated
|
||||||
|
// against real hardware/content yet.
|
||||||
|
fn default_quality_anime() -> u32 {
|
||||||
|
24
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_anime_svtav1_preset() -> u32 {
|
||||||
|
10
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_anime_svtav1_max_threads() -> u32 {
|
||||||
|
4
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_min_size_reduction_pct() -> f64 {
|
||||||
|
0.10
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_skip_below_ceiling_ratio() -> f64 {
|
||||||
|
0.5
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_verify_sample_secs() -> f64 {
|
||||||
|
20.0
|
||||||
|
}
|
||||||
|
|
||||||
/// TVDB v4 API key, exchanged for a short-lived JWT at request time.
|
/// TVDB v4 API key, exchanged for a short-lived JWT at request time.
|
||||||
#[derive(Debug, Clone, Default, Deserialize)]
|
#[derive(Debug, Clone, Default, Deserialize)]
|
||||||
pub struct TvdbConfig {
|
pub struct TvdbConfig {
|
||||||
|
|
@ -416,9 +579,41 @@ impl Config {
|
||||||
|
|
||||||
let raw = fs::read_to_string(&path)?;
|
let raw = fs::read_to_string(&path)?;
|
||||||
let cfg: Config = toml::from_str(&raw)?;
|
let cfg: Config = toml::from_str(&raw)?;
|
||||||
|
cfg.validate()?;
|
||||||
Ok(cfg)
|
Ok(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rejects a handful of `transcode` values that are individually
|
||||||
|
/// syntactically valid TOML but make the transcode pipeline's math
|
||||||
|
/// nonsensical — there's no other validation anywhere in this config,
|
||||||
|
/// so a typo here would otherwise only surface much later, deep inside
|
||||||
|
/// an encode.
|
||||||
|
fn validate(&self) -> Result<()> {
|
||||||
|
// `target_bitrate_kbps` divides by `reference_height` (via
|
||||||
|
// `reference_pixels`); zero makes that ratio `f64::INFINITY`, which
|
||||||
|
// saturates to `u32::MAX` on the cast back to `u32` — then
|
||||||
|
// `run_ffmpeg_encode_live_action`'s `bitrate_ceiling_kbps * 3`
|
||||||
|
// overflows that `u32::MAX` (panics in a debug build, silently
|
||||||
|
// wraps to a nonsense small value in release).
|
||||||
|
anyhow::ensure!(
|
||||||
|
self.transcode.reference_height > 0,
|
||||||
|
"transcode.reference_height must be greater than 0"
|
||||||
|
);
|
||||||
|
// `is_beneficial` computes `original_bytes * (1.0 -
|
||||||
|
// min_size_reduction_pct)` as the max allowed output size — a
|
||||||
|
// negative value here would raise that ceiling *above* the
|
||||||
|
// original, letting a transcode that actually grew the file still
|
||||||
|
// count as "beneficial." That's the exact failure mode
|
||||||
|
// `min_size_reduction_pct` exists to prevent (see its own doc
|
||||||
|
// comment: a real incident where flat-bitrate VBR silently produced
|
||||||
|
// files larger than the original).
|
||||||
|
anyhow::ensure!(
|
||||||
|
(0.0..=1.0).contains(&self.transcode.min_size_reduction_pct),
|
||||||
|
"transcode.min_size_reduction_pct must be between 0.0 and 1.0"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn db_path(&self) -> PathBuf {
|
pub fn db_path(&self) -> PathBuf {
|
||||||
expand_home(&self.daemon.db_path)
|
expand_home(&self.daemon.db_path)
|
||||||
}
|
}
|
||||||
|
|
@ -505,4 +700,37 @@ mod tests {
|
||||||
assert_eq!(cfg.daemon.log_level, "debug");
|
assert_eq!(cfg.daemon.log_level, "debug");
|
||||||
assert_eq!(cfg.daemon.listen_addr, "127.0.0.1:7879");
|
assert_eq!(cfg.daemon.listen_addr, "127.0.0.1:7879");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_config_passes_validation() {
|
||||||
|
Config::default().validate().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Regression test for a real gap found in review: `reference_height =
|
||||||
|
// 0` makes `target_bitrate_kbps`'s resolution-scaling ratio divide by
|
||||||
|
// zero, which eventually overflows a `u32` multiplication deep inside
|
||||||
|
// the live-action encoder's maxrate calculation — a config typo that
|
||||||
|
// used to only surface as a panic/garbage value in the middle of an
|
||||||
|
// encode, not at startup.
|
||||||
|
#[test]
|
||||||
|
fn rejects_a_zero_reference_height() {
|
||||||
|
let cfg: Config = toml::from_str("[transcode]\nreference_height = 0\n").unwrap();
|
||||||
|
assert!(cfg.validate().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Regression test for a real gap found in review: a negative
|
||||||
|
// `min_size_reduction_pct` would let `is_beneficial` accept an encode
|
||||||
|
// that actually *grew* the file — the exact invariant this field exists
|
||||||
|
// to guarantee against.
|
||||||
|
#[test]
|
||||||
|
fn rejects_a_negative_min_size_reduction_pct() {
|
||||||
|
let cfg: Config = toml::from_str("[transcode]\nmin_size_reduction_pct = -0.1\n").unwrap();
|
||||||
|
assert!(cfg.validate().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_a_min_size_reduction_pct_above_one() {
|
||||||
|
let cfg: Config = toml::from_str("[transcode]\nmin_size_reduction_pct = 1.5\n").unwrap();
|
||||||
|
assert!(cfg.validate().is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,11 @@ pub struct AudioStream {
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
pub struct SubtitleStream {
|
pub struct SubtitleStream {
|
||||||
pub language: Option<String>,
|
pub language: Option<String>,
|
||||||
|
/// mov_text (mp4's timed-text subtitle codec) isn't valid inside a
|
||||||
|
/// Matroska container — a transcode pipeline that always outputs `.mkv`
|
||||||
|
/// needs to know this per-stream to convert rather than blindly stream
|
||||||
|
/// copy. See `transcode::subtitle_codec_args`.
|
||||||
|
pub codec: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq)]
|
#[derive(Debug, Clone, Default, PartialEq)]
|
||||||
|
|
@ -109,6 +114,16 @@ pub fn probe(path: &Path) -> Result<MediaProbe> {
|
||||||
let json: serde_json::Value =
|
let json: serde_json::Value =
|
||||||
serde_json::from_slice(&output.stdout).context("ffprobe output was not valid JSON")?;
|
serde_json::from_slice(&output.stdout).context("ffprobe output was not valid JSON")?;
|
||||||
|
|
||||||
|
Ok(build_media_probe(&json, raw_json))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The actual JSON-to-`MediaProbe` mapping, split out from `probe` so it can
|
||||||
|
/// be unit-tested directly against hand-built ffprobe-shaped JSON — real
|
||||||
|
/// muxers are inconsistent enough about stream ordering/disposition (see
|
||||||
|
/// `probe_finds_the_real_video_stream_even_when_attached_pic_comes_first`'s
|
||||||
|
/// doc comment) that constructing every case as an actual file via `ffmpeg`
|
||||||
|
/// isn't always practical.
|
||||||
|
fn build_media_probe(json: &serde_json::Value, raw_json: String) -> MediaProbe {
|
||||||
let format = &json["format"];
|
let format = &json["format"];
|
||||||
let duration_secs = format["duration"]
|
let duration_secs = format["duration"]
|
||||||
.as_str()
|
.as_str()
|
||||||
|
|
@ -134,11 +149,18 @@ pub fn probe(path: &Path) -> Result<MediaProbe> {
|
||||||
.as_str()
|
.as_str()
|
||||||
.or_else(|| stream["tags"]["LANGUAGE"].as_str())
|
.or_else(|| stream["tags"]["LANGUAGE"].as_str())
|
||||||
.map(str::to_string);
|
.map(str::to_string);
|
||||||
|
let is_attached_pic = stream["disposition"]["attached_pic"].as_i64() == Some(1);
|
||||||
match codec_type {
|
match codec_type {
|
||||||
"video" if probe.video_codec.is_none() => {
|
// First non-attached-pic video stream — a second "video" stream
|
||||||
// First video stream only — a second "video" stream in a
|
// in a real-world file is almost always an embedded cover-art
|
||||||
// real-world file is almost always an embedded cover-art
|
// thumbnail, not a second picture track, and ffmpeg does not
|
||||||
// thumbnail, not a second picture track.
|
// guarantee it comes *after* the real content stream (some mp4
|
||||||
|
// remuxes and mkvmerge outputs put it first). Explicitly
|
||||||
|
// checking `disposition.attached_pic` rather than relying on
|
||||||
|
// stream order means a cover-art-first file no longer has its
|
||||||
|
// actual video codec/height silently replaced by the
|
||||||
|
// thumbnail's.
|
||||||
|
"video" if probe.video_codec.is_none() && !is_attached_pic => {
|
||||||
probe.video_codec = stream["codec_name"].as_str().map(str::to_string);
|
probe.video_codec = stream["codec_name"].as_str().map(str::to_string);
|
||||||
probe.width = stream["width"].as_i64();
|
probe.width = stream["width"].as_i64();
|
||||||
probe.height = stream["height"].as_i64();
|
probe.height = stream["height"].as_i64();
|
||||||
|
|
@ -164,13 +186,14 @@ pub fn probe(path: &Path) -> Result<MediaProbe> {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
"subtitle" => {
|
"subtitle" => {
|
||||||
probe.subtitles.push(SubtitleStream { language });
|
let codec = stream["codec_name"].as_str().map(str::to_string);
|
||||||
|
probe.subtitles.push(SubtitleStream { language, codec });
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(probe)
|
probe
|
||||||
}
|
}
|
||||||
|
|
||||||
/// ffprobe reports frame rate as a "num/den" fraction string (e.g.
|
/// ffprobe reports frame rate as a "num/den" fraction string (e.g.
|
||||||
|
|
@ -216,6 +239,59 @@ pub fn verify_decodable(path: &Path) -> Result<DecodeCheck> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bounded, sampled variant of `verify_decodable` for callers where a full
|
||||||
|
/// decode's O(duration) cost is the actual bottleneck — the transcode
|
||||||
|
/// pipeline measured this in practice: two concurrent full-file decode
|
||||||
|
/// verifications pinned two CPU cores at 400%+ each for the whole
|
||||||
|
/// verification pass, dwarfing the GPU encode time itself for long files.
|
||||||
|
///
|
||||||
|
/// Decodes only fixed-size windows (`sample_secs` each) near the start,
|
||||||
|
/// middle, and end of the file, rather than every frame — a deliberate
|
||||||
|
/// trade of "catches most real corruption cheaply" for "bounded cost
|
||||||
|
/// regardless of file length", not equivalent thoroughness to a full
|
||||||
|
/// decode. Truncation specifically doesn't need this: `encode_and_verify`'s
|
||||||
|
/// separate duration-match check against the original already catches that
|
||||||
|
/// regardless of what this function samples, since a truncated output's
|
||||||
|
/// container-reported duration comes up short either way.
|
||||||
|
///
|
||||||
|
/// Falls back to a full `verify_decodable` when `duration_secs` is small
|
||||||
|
/// enough that sampling wouldn't save meaningful time anyway.
|
||||||
|
pub fn verify_decodable_sampled(
|
||||||
|
path: &Path,
|
||||||
|
duration_secs: f64,
|
||||||
|
sample_secs: f64,
|
||||||
|
) -> Result<DecodeCheck> {
|
||||||
|
if duration_secs <= sample_secs * 3.0 {
|
||||||
|
return verify_decodable(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
let windows = [
|
||||||
|
0.0,
|
||||||
|
(duration_secs / 2.0 - sample_secs / 2.0).max(0.0),
|
||||||
|
(duration_secs - sample_secs).max(0.0),
|
||||||
|
];
|
||||||
|
|
||||||
|
for start in windows {
|
||||||
|
let mut cmd = Command::new("ffmpeg");
|
||||||
|
cmd.args(["-v", "error", "-xerror"]);
|
||||||
|
if start > 0.0 {
|
||||||
|
cmd.args(["-ss", &format!("{start:.2}")]);
|
||||||
|
}
|
||||||
|
cmd.arg("-i").arg(path);
|
||||||
|
cmd.args(["-t", &format!("{sample_secs:.2}"), "-f", "null", "-"]);
|
||||||
|
let output = cmd
|
||||||
|
.output()
|
||||||
|
.context("failed to run ffmpeg for sampled decode verification")?;
|
||||||
|
|
||||||
|
if !(output.status.success() && output.stderr.is_empty()) {
|
||||||
|
return Ok(DecodeCheck::Corrupt(
|
||||||
|
String::from_utf8_lossy(&output.stderr).into_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(DecodeCheck::Ok)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -297,6 +373,78 @@ mod tests {
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn generate_clip(dir: &Path, name: &str, duration_secs: u32) -> std::path::PathBuf {
|
||||||
|
let path = dir.join(name);
|
||||||
|
let status = Command::new("ffmpeg")
|
||||||
|
.args([
|
||||||
|
"-y",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
&format!("testsrc=size=320x240:duration={duration_secs}:rate=5"),
|
||||||
|
])
|
||||||
|
.args(["-c:v", "libx264", "-preset", "ultrafast"])
|
||||||
|
.arg(&path)
|
||||||
|
.output()
|
||||||
|
.expect("failed to run ffmpeg to generate a test clip");
|
||||||
|
assert!(
|
||||||
|
status.status.success(),
|
||||||
|
"ffmpeg failed to generate a test clip: {}",
|
||||||
|
String::from_utf8_lossy(&status.stderr)
|
||||||
|
);
|
||||||
|
path
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verify_decodable_sampled_passes_a_genuinely_intact_short_file() {
|
||||||
|
// Short enough to hit the "falls back to a full check" path.
|
||||||
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"breadarr-ffprobe-sampled-short-{}",
|
||||||
|
std::process::id()
|
||||||
|
));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let clip = generate_clip(&dir, "short.mkv", 2);
|
||||||
|
|
||||||
|
let result = verify_decodable_sampled(&clip, 2.0, 20.0).unwrap();
|
||||||
|
assert!(matches!(result, DecodeCheck::Ok));
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&dir).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verify_decodable_sampled_passes_a_genuinely_intact_long_file() {
|
||||||
|
// Long enough (duration > sample_secs * 3) to actually exercise the
|
||||||
|
// windowed start/middle/end sampling path, not the short-file
|
||||||
|
// fallback.
|
||||||
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"breadarr-ffprobe-sampled-long-{}",
|
||||||
|
std::process::id()
|
||||||
|
));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let clip = generate_clip(&dir, "long.mkv", 10);
|
||||||
|
|
||||||
|
let result = verify_decodable_sampled(&clip, 10.0, 2.0).unwrap();
|
||||||
|
assert!(matches!(result, DecodeCheck::Ok));
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&dir).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verify_decodable_sampled_flags_a_file_that_does_not_decode_at_all() {
|
||||||
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"breadarr-ffprobe-sampled-corrupt-{}",
|
||||||
|
std::process::id()
|
||||||
|
));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let path = dir.join("corrupt.mkv");
|
||||||
|
std::fs::write(&path, b"this is not a real video file").unwrap();
|
||||||
|
|
||||||
|
let result = verify_decodable_sampled(&path, 10.0, 2.0).unwrap();
|
||||||
|
assert!(matches!(result, DecodeCheck::Corrupt(_)));
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&dir).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
/// Generates a tiny real video via ffmpeg's `lavfi` synthetic source —
|
/// Generates a tiny real video via ffmpeg's `lavfi` synthetic source —
|
||||||
/// validates the actual JSON field extraction (width/height/codec/
|
/// validates the actual JSON field extraction (width/height/codec/
|
||||||
/// duration/audio language+default) against genuine ffprobe output,
|
/// duration/audio language+default) against genuine ffprobe output,
|
||||||
|
|
@ -376,4 +524,82 @@ mod tests {
|
||||||
|
|
||||||
std::fs::remove_dir_all(&dir).unwrap();
|
std::fs::remove_dir_all(&dir).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Regression test for a real gap found in review: `probe`'s "first video
|
||||||
|
// stream wins" selection used to have no idea about
|
||||||
|
// `disposition.attached_pic` and just trusted stream order — a
|
||||||
|
// convention real muxers don't reliably follow (ffmpeg's own mp4 muxer
|
||||||
|
// was observed reordering an attached-pic stream to the *end*
|
||||||
|
// regardless of requested `-map` order, which makes constructing a
|
||||||
|
// genuine cover-art-*first* file via `ffmpeg` impractical — so this
|
||||||
|
// exercises `build_media_probe` directly against hand-built,
|
||||||
|
// real-shaped ffprobe JSON instead of a generated file, deterministically
|
||||||
|
// covering the ordering `ffmpeg`'s own tooling won't produce). Unlike
|
||||||
|
// `generate_clip_with_attached_pic` in `transcode::mod::tests` (cover
|
||||||
|
// art second, the already-handled case), this puts it *first* to prove
|
||||||
|
// the fix is order-independent, not just "skip the second video
|
||||||
|
// stream".
|
||||||
|
#[test]
|
||||||
|
fn probe_finds_the_real_video_stream_even_when_attached_pic_comes_first() {
|
||||||
|
let json = serde_json::json!({
|
||||||
|
"format": { "duration": "10.0", "bit_rate": "5000000", "format_long_name": "Matroska / WebM" },
|
||||||
|
"streams": [
|
||||||
|
{
|
||||||
|
"index": 0,
|
||||||
|
"codec_type": "video",
|
||||||
|
"codec_name": "png",
|
||||||
|
"width": 64,
|
||||||
|
"height": 64,
|
||||||
|
"disposition": { "attached_pic": 1 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"index": 1,
|
||||||
|
"codec_type": "video",
|
||||||
|
"codec_name": "h264",
|
||||||
|
"width": 640,
|
||||||
|
"height": 360,
|
||||||
|
"disposition": { "attached_pic": 0 }
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
let probe = build_media_probe(&json, "{}".to_string());
|
||||||
|
assert_eq!(probe.width, Some(640), "must pick the real content stream's width, not the 64x64 cover art's");
|
||||||
|
assert_eq!(probe.height, Some(360), "must pick the real content stream's height, not the 64x64 cover art's");
|
||||||
|
assert_eq!(probe.video_codec.as_deref(), Some("h264"), "must pick the real content stream's codec, not the cover art's png");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Companion case: attached-pic *second* (the ordering the code
|
||||||
|
// previously assumed was the only one) must still work exactly as
|
||||||
|
// before — this fix is additive, not a behavior change for the
|
||||||
|
// already-handled ordering.
|
||||||
|
#[test]
|
||||||
|
fn probe_finds_the_real_video_stream_when_attached_pic_comes_second() {
|
||||||
|
let json = serde_json::json!({
|
||||||
|
"format": { "duration": "10.0", "bit_rate": "5000000", "format_long_name": "Matroska / WebM" },
|
||||||
|
"streams": [
|
||||||
|
{
|
||||||
|
"index": 0,
|
||||||
|
"codec_type": "video",
|
||||||
|
"codec_name": "h264",
|
||||||
|
"width": 640,
|
||||||
|
"height": 360,
|
||||||
|
"disposition": { "attached_pic": 0 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"index": 1,
|
||||||
|
"codec_type": "video",
|
||||||
|
"codec_name": "png",
|
||||||
|
"width": 64,
|
||||||
|
"height": 64,
|
||||||
|
"disposition": { "attached_pic": 1 }
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
let probe = build_media_probe(&json, "{}".to_string());
|
||||||
|
assert_eq!(probe.width, Some(640));
|
||||||
|
assert_eq!(probe.height, Some(360));
|
||||||
|
assert_eq!(probe.video_codec.as_deref(), Some("h264"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue