Compare commits

...

3 commits

Author SHA1 Message Date
Breadway
99724853a0 Switch to tag-pinned bread-ecosystem deps; bump version to v0.3.0
All checks were successful
Mirror to GitHub / mirror (push) Successful in 2s
release / build (push) Successful in 2m11s
2026-07-19 03:53:04 +08:00
Breadway
0990969722 Migrate embedding pipeline, EP session building, and model download to bread-onnx
embed.rs's OrtEmbedder now delegates its tokenize -> tensor build ->
mean-pool -> L2-normalize pipeline to bread_onnx::embedding::EmbeddingSession
(near-byte-identical to breadarrd's own OrtEmbedder — same duplication, now
shared, path dependency for now, see the TODO in breadmill/Cargo.toml), and
its per-EP session builders (npu_session/rocm_session/cuda_session/
openvino_session) collapse into a single to_provider() mapping onto
bread_onnx::Provider, which this crate's own breadmill/src/session.rs
counterpart now handles generically. This crate's Backend enum, cargo
feature gates (npu/rocm/cuda/openvino/full - unchanged, still control which
onnxruntime EPs actually link/load), and NPU vaip_config.json discovery all
stay local since they're genuinely breadsearch-specific. main.rs's Backend
construction and CLI flag handling are untouched.

This is also the reference implementation the MIGraphX-not-ROCm default in
bread-onnx's provider module was promoted from (see this machine's own
breadsearch-gpu-backends operator notes) — breadpad's ONNX migration, which
had the actual silent-fallback bug, follows in a later commit.

download_if_missing is replaced with bread_onnx::download::ensure_file
(same sync/ureq approach, now shared with breadarrd's downloader).

Builds clean with default features AND --features full (npu+rocm+cuda+
openvino all compiling together, matching how this crate already combined
them). All existing tests pass across the whole workspace.
2026-07-17 09:41:58 +08:00
Breadway
d01a3841d9 breadmill/breadsearch: narrow embedder lock scope, recover from poisoned mutexes, atomic index save + corrupt rebuild, fix GUI busy-poll, bump theme pin
- indexer.rs: embedder mutex is now locked only around each chunk's
  embed_document() call instead of the whole file's chunk loop, so a slow
  MIGraphX JIT compile on one file no longer blocks every query for minutes
- sync_ext.rs (new)/indexer.rs/main.rs/serve.rs: added MutexExt::lock_recover(),
  replacing every .lock().unwrap() so a panic on one thread (poisoning the
  mutex) logs a warning and recovers instead of cascading into every future
  lock().unwrap() call
- store.rs: save_index now writes to a .tmp sibling and renames atomically
  into place; Store::open recovers from an index that fails to load by
  wiping it and the SQLite files/chunks tables so the next scan rebuilds
  from scratch, instead of refusing to start at all. Note: this recovery
  only catches corruption usearch's loader reports as Err — verified
  experimentally that sufficiently garbled input segfaults the process
  instead, which no Rust-side handling can catch; the atomic save is what
  actually prevents the realistic mid-crash corruption case from arising
- breadsearch/src/main.rs: GUI query-result wait switched from
  glib::idle_add_local (re-invoked every main-loop tick, pegging a core for
  the whole wait) to a 15ms glib::timeout_add_local poll
- breadsearch/Cargo.toml: bread-theme pin bumped v0.2.8 -> v0.2.10 to match
  the rest of the family
2026-07-17 08:16:25 +08:00
12 changed files with 573 additions and 490 deletions

589
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
[package]
name = "breadmill"
version = "0.2.4"
version = "0.3.0"
edition = "2021"
license = "MIT"
@ -42,6 +42,7 @@ breadsearch-shared = { path = "../breadsearch-shared" }
# needed for the plain CPU path even in the npu build.
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "tracing", "download-binaries", "tls-native", "copy-dylibs", "api-23"] }
tokenizers = "0"
bread-onnx = { git = "https://git.breadway.dev/Breadway/bread-ecosystem", tag = "v0.3.0" }
# Surfaces ort's own EP-registration tracing (e.g. a GPU EP silently failing to
# register and falling back to CPU) as visible log output instead of nowhere.

View file

@ -1,10 +1,7 @@
use std::path::{Path, PathBuf};
use ort::{
session::{Session, builder::{GraphOptimizationLevel, SessionBuilder}},
value::Tensor,
};
use tokenizers::Tokenizer;
use bread_onnx::embedding::EmbeddingSession;
use bread_onnx::Provider;
const DOCUMENT_PREFIX: &str = "search_document: ";
const QUERY_PREFIX: &str = "search_query: ";
@ -35,23 +32,15 @@ pub enum Backend {
}
pub struct OrtEmbedder {
session: Session,
tokenizer: Tokenizer,
dim: usize,
inner: EmbeddingSession,
}
impl OrtEmbedder {
pub fn load(model_path: &Path, tokenizer_path: &Path, dim: usize, backend: Backend) -> Result<Self, String> {
let builder = Session::builder()
.map_err(|e| e.to_string())?
.with_optimization_level(GraphOptimizationLevel::All)
let provider = to_provider(backend)?;
let inner = EmbeddingSession::load(model_path, tokenizer_path, dim, MAX_SEQ_LEN, &[provider])
.map_err(|e| e.to_string())?;
let mut builder = configure_eps(builder, &backend)?;
let session = builder.commit_from_file(model_path).map_err(|e| e.to_string())?;
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(|e| e.to_string())?;
Ok(Self { session, tokenizer, dim })
Ok(Self { inner })
}
pub fn embed_document(&mut self, text: &str) -> Result<Vec<f32>, String> {
@ -64,229 +53,71 @@ impl OrtEmbedder {
fn embed_with_prefix(&mut self, text: &str, prefix: &str) -> Result<Vec<f32>, String> {
let input = format!("{}{}", prefix, text);
let encoding = self
.tokenizer
.encode(input, true)
.map_err(|e| e.to_string())?;
let mut ids: Vec<i64> = encoding.get_ids().iter().map(|&x| x as i64).collect();
let mut mask: Vec<i64> = encoding
.get_attention_mask()
.iter()
.map(|&x| x as i64)
.collect();
let mut type_ids: Vec<i64> = encoding
.get_type_ids()
.iter()
.map(|&x| x as i64)
.collect();
if ids.len() > MAX_SEQ_LEN {
eprintln!(
"breadmill: truncating {} tokens to {} (chunk too large)",
ids.len(),
MAX_SEQ_LEN
);
ids.truncate(MAX_SEQ_LEN);
mask.truncate(MAX_SEQ_LEN);
type_ids.truncate(MAX_SEQ_LEN);
}
let seq_len = ids.len() as i64;
let id_tensor =
Tensor::<i64>::from_array((vec![1i64, seq_len], ids.clone())).map_err(|e| e.to_string())?;
let mask_tensor =
Tensor::<i64>::from_array((vec![1i64, seq_len], mask.clone())).map_err(|e| e.to_string())?;
let type_tensor =
Tensor::<i64>::from_array((vec![1i64, seq_len], type_ids)).map_err(|e| e.to_string())?;
let outputs = self
.session
.run(ort::inputs! {
"input_ids" => id_tensor,
"attention_mask" => mask_tensor,
"token_type_ids" => type_tensor,
})
.map_err(|e| e.to_string())?;
// last_hidden_state: shape [1, seq_len, dim]
let (shape, data) = outputs["last_hidden_state"]
.try_extract_tensor::<f32>()
.map_err(|e| e.to_string())?;
let actual_seq = shape[1] as usize;
let actual_dim = shape[2] as usize;
// Mean-pool over non-padding positions. Some execution providers (e.g.
// MIGraphX) pad the output sequence dimension for kernel efficiency, so
// actual_seq can exceed mask.len() — only positions covered by our own
// attention mask are meaningful, so cap the loop at whichever is shorter.
let mut result = vec![0.0f32; actual_dim];
let mut count = 0usize;
for t in 0..actual_seq.min(mask.len()) {
if mask[t] > 0 {
for d in 0..actual_dim {
result[d] += data[t * actual_dim + d];
}
count += 1;
}
}
if count > 0 {
for x in &mut result {
*x /= count as f32;
}
}
l2_normalize(&mut result);
// Clamp/pad to configured dim
result.truncate(self.dim);
while result.len() < self.dim {
result.push(0.0);
}
Ok(result)
self.inner.embed(&input).map_err(|e| e.to_string())
}
}
fn l2_normalize(v: &mut Vec<f32>) {
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 1e-10 {
for x in v.iter_mut() {
*x /= norm;
}
}
}
// ---- Backend -> bread_onnx::Provider ----------------------------------------
//
// `bread_onnx::session::build_session` (via `EmbeddingSession::load`) is the
// shared session-builder + EP-fallback + loud-logging code every EP branch
// below used to hand-roll separately (`configure_eps`/`npu_session`/
// `rocm_session`/`cuda_session`/`openvino_session`). What's genuinely
// specific to this crate — its cargo feature gates (npu/rocm/cuda/openvino),
// and NPU's `vaip_config.json` discovery — stays here.
// ---- Execution provider selection -------------------------------------------
fn configure_eps(builder: SessionBuilder, backend: &Backend) -> Result<SessionBuilder, String> {
fn to_provider(backend: Backend) -> Result<Provider, String> {
match backend {
Backend::Cpu => Ok(builder),
Backend::Npu { cache_dir } => npu_session(builder, cache_dir),
Backend::Rocm => rocm_session(builder),
Backend::Cuda => cuda_session(builder),
Backend::OpenVino { cache_dir } => openvino_session(builder, cache_dir),
Backend::Cpu => Ok(Provider::Cpu),
Backend::Npu { cache_dir } => npu_provider(cache_dir),
Backend::Rocm => rocm_provider(),
Backend::Cuda => cuda_provider(),
Backend::OpenVino { cache_dir } => Ok(Provider::OpenVino { device_type: "GPU".to_string(), cache_dir }),
}
}
#[cfg(feature = "npu")]
fn npu_session(builder: SessionBuilder, cache_dir: &Path) -> Result<SessionBuilder, String> {
let vitis_ep = build_vitis_ep(cache_dir)?;
eprintln!("breadmill: using NPU (VitisAI) execution provider");
fn npu_provider(cache_dir: PathBuf) -> Result<Provider, String> {
let vaip_config = find_vaip_config()?;
if std::env::var("ORT_DYLIB_PATH").is_err() {
eprintln!(
"breadmill: hint — set ORT_DYLIB_PATH to the Ryzen AI SDK ORT, e.g.:\n \
ORT_DYLIB_PATH=~/.local/share/ryzen-ai-1.7.1/lib/libonnxruntime.so"
);
}
builder
.with_execution_providers([vitis_ep, ort::ep::CPU::default().build()])
.map_err(|e| e.to_string())
Ok(Provider::Vitis {
config_file: vaip_config,
cache_dir: cache_dir.join("npu"),
cache_key: "nomic-embed-text-v1.5".to_string(),
})
}
#[cfg(not(feature = "npu"))]
fn npu_session(builder: SessionBuilder, _cache_dir: &Path) -> Result<SessionBuilder, String> {
fn npu_provider(_cache_dir: PathBuf) -> Result<Provider, String> {
eprintln!("breadmill: NPU backend requested but not compiled in (rebuild with --features npu); using CPU");
Ok(builder)
Ok(Provider::Cpu)
}
// ---- VitisAI EP (NPU) -------------------------------------------------------
#[cfg(feature = "npu")]
fn build_vitis_ep(cache_dir: &Path) -> Result<ort::ep::ExecutionProviderDispatch, String> {
let vaip_config = find_vaip_config()?;
let npu_cache = cache_dir.join("npu");
std::fs::create_dir_all(&npu_cache).map_err(|e| e.to_string())?;
eprintln!("breadmill: vaip_config: {}", vaip_config.display());
eprintln!("breadmill: NPU model cache: {}", npu_cache.display());
Ok(ort::ep::Vitis::default()
.with_config_file(vaip_config.to_string_lossy())
.with_cache_dir(npu_cache.to_string_lossy())
.with_cache_key("nomic-embed-text-v1.5")
.build())
}
// ---- MIGraphX EP (AMD iGPU, ROCm-backed) -------------------------------------
#[cfg(feature = "rocm")]
fn rocm_session(builder: SessionBuilder) -> Result<SessionBuilder, String> {
eprintln!("breadmill: using MIGraphX execution provider (device 0)");
eprintln!(
"breadmill: note — check the log line above/below for \"Successfully registered \
`MIGraphXExecutionProvider`\"; if it's missing, the ONNX Runtime in use wasn't built \
with MIGraphX support and inference silently fell back to CPU"
);
builder
.with_execution_providers([
ort::ep::MIGraphX::default().with_device_id(0).build(),
ort::ep::CPU::default().build(),
])
.map_err(|e| e.to_string())
fn rocm_provider() -> Result<Provider, String> {
Ok(Provider::MiGraphX { device_id: 0 })
}
#[cfg(not(feature = "rocm"))]
fn rocm_session(builder: SessionBuilder) -> Result<SessionBuilder, String> {
fn rocm_provider() -> Result<Provider, String> {
eprintln!("breadmill: ROCm backend requested but not compiled in (rebuild with --features rocm); using CPU");
Ok(builder)
Ok(Provider::Cpu)
}
// ---- CUDA EP (NVIDIA GPU) ----------------------------------------------------
#[cfg(feature = "cuda")]
fn cuda_session(builder: SessionBuilder) -> Result<SessionBuilder, String> {
eprintln!("breadmill: using CUDA execution provider (device 0)");
eprintln!(
"breadmill: note — check the log line above/below for \"Successfully registered \
`CUDAExecutionProvider`\"; if it's missing, the ONNX Runtime in use wasn't built \
with CUDA support and inference silently fell back to CPU"
);
builder
.with_execution_providers([
ort::ep::CUDA::default().with_device_id(0).build(),
ort::ep::CPU::default().build(),
])
.map_err(|e| e.to_string())
fn cuda_provider() -> Result<Provider, String> {
Ok(Provider::Cuda { device_id: 0 })
}
#[cfg(not(feature = "cuda"))]
fn cuda_session(builder: SessionBuilder) -> Result<SessionBuilder, String> {
fn cuda_provider() -> Result<Provider, String> {
eprintln!("breadmill: CUDA backend requested but not compiled in (rebuild with --features cuda); using CPU");
Ok(builder)
}
// ---- OpenVINO EP (Intel iGPU/dGPU) -------------------------------------------
#[cfg(feature = "openvino")]
fn openvino_session(builder: SessionBuilder, cache_dir: &Path) -> Result<SessionBuilder, String> {
let ov_cache = cache_dir.join("openvino");
std::fs::create_dir_all(&ov_cache).map_err(|e| e.to_string())?;
eprintln!("breadmill: using OpenVINO execution provider (device_type GPU)");
eprintln!("breadmill: OpenVINO model cache: {}", ov_cache.display());
eprintln!(
"breadmill: note — check the log line above/below for \"Successfully registered \
`OpenVINOExecutionProvider`\"; if it's missing, the ONNX Runtime in use wasn't built \
with OpenVINO support and inference silently fell back to CPU"
);
builder
.with_execution_providers([
ort::ep::OpenVINO::default()
.with_device_type("GPU")
.with_cache_dir(ov_cache.to_string_lossy())
.build(),
ort::ep::CPU::default().build(),
])
.map_err(|e| e.to_string())
}
#[cfg(not(feature = "openvino"))]
fn openvino_session(builder: SessionBuilder, _cache_dir: &Path) -> Result<SessionBuilder, String> {
eprintln!("breadmill: OpenVINO backend requested but not compiled in (rebuild with --features openvino); using CPU");
Ok(builder)
Ok(Provider::Cpu)
}
/// Locate the VitisAI EP config file required by the AMD Ryzen AI SDK.

View file

@ -10,7 +10,7 @@ use ignore::WalkBuilder;
use notify::{RecommendedWatcher, RecursiveMode, Watcher, EventKind};
use sha2::{Digest, Sha256};
use crate::{embed::OrtEmbedder, extract, chunk, power, store::Store};
use crate::{embed::OrtEmbedder, extract, chunk, power, store::Store, sync_ext::MutexExt};
pub struct SharedState {
pub store: Mutex<Store>,
@ -64,7 +64,7 @@ impl Indexer {
pub fn full_reindex(&self) {
eprintln!("breadmill: full reindex triggered");
{
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
// Clear all state
let _ = store.conn.execute_batch("DELETE FROM chunks; DELETE FROM files;");
let _ = store.index.reserve(4096);
@ -87,7 +87,7 @@ impl Indexer {
// Snapshot existing indexed files
let known: HashMap<String, (i64, String)> = {
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
store.all_files()
.unwrap_or_default()
.into_iter()
@ -155,7 +155,7 @@ impl Indexer {
.collect();
if !to_delete.is_empty() {
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
for path in to_delete {
eprintln!("breadmill: removing deleted file: {}", path);
let _ = store.delete_file(&path);
@ -163,7 +163,7 @@ impl Indexer {
}
let count = {
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
let n = store.chunk_count();
let _ = store.save_index(&self.state_dir);
n
@ -238,7 +238,7 @@ impl Indexer {
self.handle_fs_event(&path);
}
let count = {
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
let n = store.chunk_count();
let _ = store.save_index(&self.state_dir);
n
@ -261,7 +261,7 @@ impl Indexer {
if !path.is_file() {
let path_str = path.to_string_lossy().into_owned();
// File deleted — remove from index
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
let _ = store.delete_file(&path_str);
return;
}
@ -309,7 +309,7 @@ impl Indexer {
// Check if hash changed (catches content changes without mtime change)
{
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
if let Ok(files) = store.all_files() {
if files.iter().any(|f| f.path == path_str && f.hash == hash) {
return;
@ -322,20 +322,22 @@ impl Indexer {
// for natural-language files.
let chunks = chunk::chunk_text(&text, 400, 80, 2_000);
eprintln!("breadmill: embedding {} ({} chars, {} chunks)", path_str, text.len(), chunks.len());
let mut embedder_guard = self.state.embedder.lock().unwrap();
if !self.state.model_ready.load(Ordering::Relaxed) {
eprintln!("breadmill: model not ready, skipping embed for {}", path_str);
return;
}
let embedder = match embedder_guard.as_mut() {
Some(e) => e,
None => return,
};
// Confirm the embedder is actually present before committing to
// clearing this file's old chunks below — same check as before,
// just without holding the embedder lock past this one glance (see
// the per-chunk locking in the loop for why).
if self.state.embedder.lock_recover().is_none() {
return;
}
{
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
let _ = store.delete_file(path_str); // remove old chunks/vectors first
}
@ -344,9 +346,18 @@ impl Indexer {
for (i, chunk) in chunks.iter().enumerate() {
eprintln!("breadmill: embed chunk {}/{} ({} chars) for {}", i + 1, chunks.len(), chunk.text.len(), path_str);
match embedder.embed_document(&chunk.text) {
// Lock the embedder only around this single chunk's embed call —
// this used to be held for the whole file's chunk loop, so one
// large file needing a fresh MIGraphX JIT compile (60-120s for a
// new sequence length) could block every query (serve.rs locks
// this same mutex) for the entire file, not just one chunk.
let embed_result = match self.state.embedder.lock_recover().as_mut() {
Some(embedder) => embedder.embed_document(&chunk.text),
None => break, // model was unloaded mid-scan; stop here
};
match embed_result {
Ok(embedding) => {
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
// Ensure file row exists before inserting chunks (FK constraint)
let _ = store.upsert_file(path_str, mtime, &hash);
let _ = store.insert_chunk(
@ -367,7 +378,7 @@ impl Indexer {
eprintln!("breadmill: no chunks embedded for {}", path_str);
// Record the file so the mtime+hash check skips it on the next startup
// rather than re-entering the same embed-fail loop.
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
let _ = store.upsert_file(path_str, mtime, &hash);
} else {
// Increment live so `status` reflects progress before the full scan ends.

View file

@ -1,5 +1,4 @@
use std::{
io::Read,
path::{Path, PathBuf},
sync::{Arc, atomic::Ordering},
};
@ -13,10 +12,12 @@ mod indexer;
mod power;
mod serve;
mod store;
mod sync_ext;
use embed::{Backend, OrtEmbedder};
use indexer::{Indexer, SharedState};
use store::Store;
use sync_ext::MutexExt;
const MODEL_URL: &str =
"https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx";
@ -170,7 +171,7 @@ fn run_daemon(
eprintln!("breadmill: loading model...");
match OrtEmbedder::load(&model_path, &tokenizer_path, dim, backend) {
Ok(embedder) => {
*state_clone.embedder.lock().unwrap() = Some(embedder);
*state_clone.embedder.lock_recover() = Some(embedder);
state_clone.model_ready.store(true, Ordering::Relaxed);
eprintln!("breadmill: model loaded");
}
@ -224,29 +225,10 @@ fn download_if_missing(url: &str, dest: &Path) -> Result<(), String> {
eprintln!(" already present: {}", dest.display());
return Ok(());
}
eprintln!(" downloading {} ...", url);
let agent = ureq::AgentBuilder::new()
.timeout(std::time::Duration::from_secs(300))
.build();
let response = agent.get(url).call().map_err(|e| e.to_string())?;
let mut bytes = Vec::new();
response
.into_reader()
.read_to_end(&mut bytes)
.map_err(|e| e.to_string())?;
if bytes.is_empty() {
return Err(format!("empty download from {}", url));
}
// Write atomically via temp file
let tmp = dest.with_extension("tmp");
std::fs::write(&tmp, &bytes).map_err(|e| e.to_string())?;
std::fs::rename(&tmp, dest).map_err(|e| e.to_string())?;
eprintln!(" saved {} ({:.1} MB)", dest.display(), bytes.len() as f64 / 1_048_576.0);
// Shared with breadarrd's own (previously reqwest/async, now also this
// same sync/ureq implementation) model downloader — see
// bread_onnx::download's doc comment.
bread_onnx::download::ensure_file(url, dest, None).map_err(|e| e.to_string())?;
Ok(())
}

View file

@ -8,6 +8,7 @@ use std::{
use breadsearch_shared::{Request, Response, StatusInfo};
use crate::indexer::SharedState;
use crate::sync_ext::MutexExt;
pub fn run(socket_path: &Path, state: Arc<SharedState>, snippet_len: usize, search_limit: usize) {
let _ = std::fs::remove_file(socket_path);
@ -86,7 +87,7 @@ fn dispatch(
}
let embedding = {
let mut embedder = state.embedder.lock().unwrap();
let mut embedder = state.embedder.lock_recover();
match embedder.as_mut() {
Some(e) => match e.embed_query(&query) {
Ok(v) => v,
@ -101,7 +102,7 @@ fn dispatch(
};
let limit = limit.min(search_limit).max(1);
let store = state.store.lock().unwrap();
let store = state.store.lock_recover();
match store.search(&embedding, limit, snippet_len) {
Ok(hits) => Response::Hits { hits },

View file

@ -57,9 +57,32 @@ impl Store {
let index = new_index(&options).map_err(|e| e.to_string())?;
if idx_path.exists() {
index
.load(idx_path.to_str().unwrap())
.map_err(|e| e.to_string())?;
// NOTE: this only catches corruption that usearch's loader
// itself detects and reports as an `Err` (e.g. a recognizable
// but wrong/incompatible header). Verified experimentally: a
// file that doesn't even look like a usearch index at all (pure
// garbage bytes) crashes the *process* with a SIGSEGV inside the
// native loader rather than returning an `Err` — no amount of
// Rust-side `Result`/`catch_unwind` handling can intercept that,
// it's a native-code robustness gap in the usearch library
// itself. This recovery path is still worth having (it's the
// difference between "won't start" and "rebuilds and starts"
// for the errors it *does* catch), but it is not a complete
// guarantee against every possible corrupt file. The atomic
// save below is the real fix for the common case this was
// written for (a crash mid-save) — it now can't produce a
// half-written `vectors.usearch` in the first place.
if let Err(e) = index.load(idx_path.to_str().unwrap()) {
eprintln!(
"breadmill: WARNING: {} failed to load ({}) — treating it as corrupt, \
discarding it, and rebuilding the index from scratch on the next scan",
idx_path.display(),
e
);
conn.execute_batch("DELETE FROM chunks; DELETE FROM files;")
.map_err(|e| e.to_string())?;
index.reserve(4096).map_err(|e| e.to_string())?;
}
} else {
index.reserve(4096).map_err(|e| e.to_string())?;
}
@ -218,11 +241,18 @@ impl Store {
// ---- persistence --------------------------------------------------------
/// Saves to a `.tmp` sibling and renames it into place — a same-
/// filesystem rename is atomic, so a crash mid-save leaves only an
/// orphaned `.tmp` file rather than a truncated/corrupt
/// `vectors.usearch` that would otherwise fail to load on next start
/// (see the recovery path in `open`).
pub fn save_index(&self, state_dir: &Path) -> Result<(), String> {
let idx_path = state_dir.join("vectors.usearch");
let tmp_path = state_dir.join("vectors.usearch.tmp");
self.index
.save(idx_path.to_str().unwrap())
.map_err(|e| e.to_string())
.save(tmp_path.to_str().unwrap())
.map_err(|e| e.to_string())?;
std::fs::rename(&tmp_path, &idx_path).map_err(|e| e.to_string())
}
}
@ -233,3 +263,45 @@ fn truncate_to_chars(s: &str, max_chars: usize) -> String {
let truncated: String = s.chars().take(max_chars).collect();
format!("{}", truncated.trim_end())
}
#[cfg(test)]
mod tests {
use super::*;
fn test_dir(name: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!("breadmill-store-test-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn save_index_leaves_no_tmp_file_behind_and_is_loadable() {
let dir = test_dir("save-atomic");
let store = Store::open(&dir, 4).unwrap();
store.save_index(&dir).unwrap();
assert!(dir.join("vectors.usearch").exists());
assert!(
!dir.join("vectors.usearch.tmp").exists(),
"the .tmp staging file should be renamed away, not left behind"
);
// A fresh `open` can load what was just saved without error.
Store::open(&dir, 4).unwrap();
std::fs::remove_dir_all(&dir).unwrap();
}
// No automated test for the corrupt-index recovery branch in `open`:
// the natural way to construct a "corrupt" fixture (writing arbitrary
// garbage to vectors.usearch) was tried and crashes the *test process*
// with a SIGSEGV inside usearch's native loader before our `Result`
// handling ever gets a chance to run — see the comment on that branch
// in `open`. A real usearch file with a deliberately-broken-but-still-
// parseable header could plausibly hit the `Err` path exercised there
// instead, but reverse-engineering that format precisely enough to
// build a safe fixture wasn't worth the risk of another flaky/crashing
// test. The atomic-save test above covers the actual mechanism that
// prevents this scenario from arising in the first place.
}

41
breadmill/src/sync_ext.rs Normal file
View file

@ -0,0 +1,41 @@
//! Poison-tolerant `Mutex` locking.
//!
//! A panic on any thread while holding `SharedState::store` or
//! `SharedState::embedder` (e.g. an untrapped ONNX Runtime panic — only PDF
//! extraction is `catch_unwind`-guarded, see `extract.rs`) poisons the
//! `Mutex`. Every subsequent plain `.lock().unwrap()` — in the indexer *and*
//! in every query handler in `serve.rs` — would then immediately panic too,
//! silently bricking indexing and search until the daemon is restarted by
//! hand.
//!
//! `Mutex` poisoning exists to flag "the data guarded by this lock might be
//! in an inconsistent state," but every lock scope in this crate is a short,
//! single-step SQLite call or usearch operation — nothing here spans a
//! multi-step invariant across a single `lock()` call — so recovering the
//! guard and logging loudly is a reasonable trade here: continuing to serve
//! (and re-attempting the operation that panicked, on the next file/query)
//! beats a daemon that silently stops answering everything after one panic.
use std::sync::{Mutex, MutexGuard};
pub trait MutexExt<T> {
/// Like `.lock().unwrap()`, but recovers from a poisoned mutex instead
/// of panicking again — logs once per recovery so it's visible in the
/// daemon's own output, not just silently swallowed.
fn lock_recover(&self) -> MutexGuard<'_, T>;
}
impl<T> MutexExt<T> for Mutex<T> {
fn lock_recover(&self) -> MutexGuard<'_, T> {
match self.lock() {
Ok(guard) => guard,
Err(poisoned) => {
eprintln!(
"breadmill: WARNING: a mutex was poisoned by a panic on another thread; \
recovering it and continuing instead of cascading the panic"
);
poisoned.into_inner()
}
}
}
}

View file

@ -1,6 +1,6 @@
[package]
name = "breadsearch-shared"
version = "0.2.0"
version = "0.3.0"
edition = "2021"
license = "MIT"

View file

@ -1,6 +1,6 @@
[package]
name = "breadsearch"
version = "0.2.0"
version = "0.3.0"
edition = "2021"
license = "MIT"
@ -10,7 +10,7 @@ path = "src/main.rs"
[dependencies]
breadsearch-shared = { path = "../breadsearch-shared" }
bread-theme = { git = "https://github.com/Breadway/bread-ecosystem", tag = "v0.2.8", features = ["gtk"] }
bread-theme = { git = "https://github.com/Breadway/bread-ecosystem", tag = "v0.2.10", features = ["gtk"] }
gtk4 = { version = "0.11", features = ["v4_12"] }
gtk4-layer-shell = "0.8"
serde_json = "1"

View file

@ -298,12 +298,19 @@ fn run_ui() {
let _ = tx.send(breadsearch_shared::send_request(&req));
});
// Poll via idle_add_local until the thread delivers its result.
// Unix socket round-trips are sub-millisecond so this fires once.
// Wait for the thread's result on a bounded timer rather than
// an `idle_add_local` — an idle source has no wait condition
// of its own, so GLib re-invokes it on every single main-loop
// iteration, i.e. a busy-spin pinning a full CPU core for as
// long as the daemon takes to answer (normally sub-ms, but
// the daemon's socket has no read timeout of its own, so a
// wedged/slow daemon previously meant an indefinite spin).
// 15ms is imperceptible added latency for a search box and
// caps this at a couple dozen checks per second instead.
let rx = Rc::new(rx);
let list_t = list_clone.clone();
glib::idle_add_local(move || {
glib::timeout_add_local(std::time::Duration::from_millis(15), move || {
match rx.try_recv() {
Ok(result) => {
populate_list(&list_t, result);

View file

@ -1,6 +1,6 @@
[Unit]
Description=Breadmill semantic search indexer
Documentation=https://github.com/breadway/breadsearch
Documentation=https://git.breadway.dev/Breadway/breadsearch
After=default.target
[Service]