From 50ff425d2ada29fa88fc66bea8218595fe40cdde Mon Sep 17 00:00:00 2001 From: Breadway Date: Wed, 5 Aug 2026 06:45:20 +0800 Subject: [PATCH] Fix test harness process leak: graceful SIGTERM shutdown + disable podman adapter in test fixtures The integration test harness (ipc_integration.rs and module_host_sandbox.rs) tore down spawned breadd instances via Child::kill() (SIGKILL). Since breadd's PodmanAdapter is enabled by default and the test breadd.toml fixtures didn't disable it, every test-spawned daemon on a machine with podman installed ran a real `podman events --format json` child. SIGKILL gives breadd zero chance to run its own graceful shutdown path (wait_for_shutdown -> adapter tasks dropping -> kill_on_drop firing on that child), so it was orphaned and reparented to init on every single test run instead. A day of repeated `cargo test --workspace` runs during today's workstream development left 1,559 such orphaned processes consuming 76.8GB of RSS system-wide, found and cleaned up separately. Fix is two-layered: disable the podman adapter in both harnesses' breadd.toml fixtures (tests don't need real container-lifecycle watching, matching how hyprland/udev/power/network are already disabled there), and change TestHarness::drop to send SIGTERM with a bounded wait before falling back to SIGKILL, so breadd's own cleanup gets a real chance to run - defense in depth against any future adapter that spawns a subprocess. Verified: two full `cargo test --workspace` runs (282 tests) produce zero new podman events processes and zero leftover breadd/ bread-module-host processes. --- breadd/tests/ipc_integration.rs | 34 +++++++++++++++++++++++++++++ breadd/tests/module_host_sandbox.rs | 25 +++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/breadd/tests/ipc_integration.rs b/breadd/tests/ipc_integration.rs index b324639..1ddb1ca 100644 --- a/breadd/tests/ipc_integration.rs +++ b/breadd/tests/ipc_integration.rs @@ -1448,6 +1448,9 @@ enabled = false [adapters.network] enabled = false + +[adapters.podman] +enabled = false "#, )?; @@ -1524,6 +1527,9 @@ enabled = false [adapters.network] enabled = false + +[adapters.podman] +enabled = false "#, )?; @@ -1678,6 +1684,34 @@ impl Drop for TestHarness { /// failure into cascading slowdowns/timeouts across the whole suite /// (observed directly while developing Workstream G's tests). fn drop(&mut self) { + // SIGTERM first, not straight to SIGKILL: `breadd`'s own + // `wait_for_shutdown()` (src/main.rs) listens for SIGTERM and runs + // a graceful shutdown path — `lua_runtime.shutdown()`, + // `module_host_registry.shutdown_all()`, and, as `main()` returns, + // every adapter task getting dropped, which is what actually fires + // `kill_on_drop` on a still-running child process like + // `PodmanAdapter`'s `podman events` watcher. SIGKILL bypasses all of + // that — the process is torn down before any of its own code, Drop + // impls included, ever runs. That gap is exactly how a single day + // of repeated `cargo test --workspace` runs left 1,559 orphaned + // `podman events` processes system-wide (found and cleaned up + // separately; see also the `[adapters.podman] enabled = false` + // fixtures this file now sets, which close the same hole from the + // other side). Bounded wait, then SIGKILL as a fallback so a + // genuinely wedged `breadd` doesn't hang the test suite. + unsafe { + libc::kill(self.child.id() as libc::pid_t, libc::SIGTERM); + } + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match self.child.try_wait() { + Ok(Some(_)) => return, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(20)); + } + _ => break, + } + } let _ = self.child.kill(); let _ = self.child.wait(); } diff --git a/breadd/tests/module_host_sandbox.rs b/breadd/tests/module_host_sandbox.rs index 5d67df2..51e57cc 100644 --- a/breadd/tests/module_host_sandbox.rs +++ b/breadd/tests/module_host_sandbox.rs @@ -106,6 +106,9 @@ enabled = false [adapters.network] enabled = false + +[adapters.podman] +enabled = false "#, )?; @@ -255,6 +258,28 @@ impl Drop for TestHarness { /// `shutdown()`, is what keeps a failed test run from leaving orphaned /// sandboxed processes behind for the next run to trip over. fn drop(&mut self) { + // SIGTERM first, not straight to SIGKILL — see the matching comment + // in `ipc_integration.rs`'s `TestHarness::drop`. SIGKILL prevents + // `breadd` from ever running its own graceful shutdown path, which + // is what actually fires `kill_on_drop` on adapter-spawned child + // processes (e.g. `PodmanAdapter`'s `podman events` watcher) — + // exactly how a day of repeated `cargo test --workspace` runs left + // 1,559 orphaned `podman events` processes system-wide. Bounded + // wait, then SIGKILL as a fallback so a genuinely wedged `breadd` + // doesn't hang the test suite. + unsafe { + libc::kill(self.child.id() as libc::pid_t, libc::SIGTERM); + } + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match self.child.try_wait() { + Ok(Some(_)) => return, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(20)); + } + _ => break, + } + } let _ = self.child.kill(); let _ = self.child.wait(); }