Breadway/breadsearch
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

breadmill/breadsearch: narrow embedder lock scope, recover from poisoned mutexes, atomic index save + corrupt rebuild, fix GUI busy-poll, bump theme pin

Browse source 
- indexer.rs: embedder mutex is now locked only around each chunk's
  embed_document() call instead of the whole file's chunk loop, so a slow
  MIGraphX JIT compile on one file no longer blocks every query for minutes
- sync_ext.rs (new)/indexer.rs/main.rs/serve.rs: added MutexExt::lock_recover(),
  replacing every .lock().unwrap() so a panic on one thread (poisoning the
  mutex) logs a warning and recovers instead of cascading into every future
  lock().unwrap() call
- store.rs: save_index now writes to a .tmp sibling and renames atomically
  into place; Store::open recovers from an index that fails to load by
  wiping it and the SQLite files/chunks tables so the next scan rebuilds
  from scratch, instead of refusing to start at all. Note: this recovery
  only catches corruption usearch's loader reports as Err — verified
  experimentally that sufficiently garbled input segfaults the process
  instead, which no Rust-side handling can catch; the atomic save is what
  actually prevents the realistic mid-crash corruption case from arising
- breadsearch/src/main.rs: GUI query-result wait switched from
  glib::idle_add_local (re-invoked every main-loop tick, pegging a core for
  the whole wait) to a 15ms glib::timeout_add_local poll
- breadsearch/Cargo.toml: bread-theme pin bumped v0.2.8 -> v0.2.10 to match
  the rest of the family
This commit is contained in:
Breadway 2026-07-17 08:16:25 +08:00
parent 3d83bd747e
commit d01a3841d9
8 changed files with 164 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

45
breadmill/src/indexer.rs
View file

@ -10,7 +10,7 @@ use ignore::WalkBuilder;
use notify::{RecommendedWatcher, RecursiveMode, Watcher, EventKind};
use sha2::{Digest, Sha256};
use crate::{embed::OrtEmbedder, extract, chunk, power, store::Store};
use crate::{embed::OrtEmbedder, extract, chunk, power, store::Store, sync_ext::MutexExt};
pub struct SharedState {
pub store: Mutex<Store>,
@ -64,7 +64,7 @@ impl Indexer {
pub fn full_reindex(&self) {
eprintln!("breadmill: full reindex triggered");
{
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
// Clear all state
let _ = store.conn.execute_batch("DELETE FROM chunks; DELETE FROM files;");
let _ = store.index.reserve(4096);
@ -87,7 +87,7 @@ impl Indexer {
// Snapshot existing indexed files
let known: HashMap<String, (i64, String)> = {
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
store.all_files()
.unwrap_or_default()
.into_iter()
@ -155,7 +155,7 @@ impl Indexer {
.collect();
if !to_delete.is_empty() {
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
for path in to_delete {
eprintln!("breadmill: removing deleted file: {}", path);
let _ = store.delete_file(&path);
@ -163,7 +163,7 @@ impl Indexer {
}
let count = {
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
let n = store.chunk_count();
let _ = store.save_index(&self.state_dir);
n
@ -238,7 +238,7 @@ impl Indexer {
self.handle_fs_event(&path);
}
let count = {
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
let n = store.chunk_count();
let _ = store.save_index(&self.state_dir);
n
@ -261,7 +261,7 @@ impl Indexer {
if !path.is_file() {
let path_str = path.to_string_lossy().into_owned();
// File deleted — remove from index
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
let _ = store.delete_file(&path_str);
return;
}
@ -309,7 +309,7 @@ impl Indexer {
// Check if hash changed (catches content changes without mtime change)
{
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
if let Ok(files) = store.all_files() {
if files.iter().any(|f| f.path == path_str && f.hash == hash) {
return;
@ -322,20 +322,22 @@ impl Indexer {
// for natural-language files.
let chunks = chunk::chunk_text(&text, 400, 80, 2_000);
eprintln!("breadmill: embedding {} ({} chars, {} chunks)", path_str, text.len(), chunks.len());
let mut embedder_guard = self.state.embedder.lock().unwrap();
if !self.state.model_ready.load(Ordering::Relaxed) {
eprintln!("breadmill: model not ready, skipping embed for {}", path_str);
return;
}
let embedder = match embedder_guard.as_mut() {
Some(e) => e,
None => return,
};
// Confirm the embedder is actually present before committing to
// clearing this file's old chunks below — same check as before,
// just without holding the embedder lock past this one glance (see
// the per-chunk locking in the loop for why).
if self.state.embedder.lock_recover().is_none() {
return;
}
{
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
let _ = store.delete_file(path_str); // remove old chunks/vectors first
}
@ -344,9 +346,18 @@ impl Indexer {
for (i, chunk) in chunks.iter().enumerate() {
eprintln!("breadmill: embed chunk {}/{} ({} chars) for {}", i + 1, chunks.len(), chunk.text.len(), path_str);
match embedder.embed_document(&chunk.text) {
// Lock the embedder only around this single chunk's embed call —
// this used to be held for the whole file's chunk loop, so one
// large file needing a fresh MIGraphX JIT compile (60-120s for a
// new sequence length) could block every query (serve.rs locks
// this same mutex) for the entire file, not just one chunk.
let embed_result = match self.state.embedder.lock_recover().as_mut() {
Some(embedder) => embedder.embed_document(&chunk.text),
None => break, // model was unloaded mid-scan; stop here
};
match embed_result {
Ok(embedding) => {
let mut store = self.state.store.lock().unwrap();
let mut store = self.state.store.lock_recover();
// Ensure file row exists before inserting chunks (FK constraint)
let _ = store.upsert_file(path_str, mtime, &hash);
let _ = store.insert_chunk(
@ -367,7 +378,7 @@ impl Indexer {
eprintln!("breadmill: no chunks embedded for {}", path_str);
// Record the file so the mtime+hash check skips it on the next startup
// rather than re-entering the same embed-fail loop.
let store = self.state.store.lock().unwrap();
let store = self.state.store.lock_recover();
let _ = store.upsert_file(path_str, mtime, &hash);
} else {
// Increment live so `status` reflects progress before the full scan ends.