breadmill 0.2.4: bind socket before loading the embedding model
The model load ran synchronously on the main thread before serve::run() bound the daemon's socket. A slow or stuck EP compile (OpenVINO in particular) meant the socket didn't exist for as long as that took, so every client -- including the breadsearch GUI -- saw a bare connection refused with no way to tell "still loading" from "actually broken." The load now happens on a background thread; the socket binds immediately, and serve.rs's existing model_ready check answers "model not ready" for any request that arrives before the load finishes.
This commit is contained in:
parent
c6ed6a41d8
commit
3d83bd747e
3 changed files with 21 additions and 11 deletions
|
|
@ -152,21 +152,31 @@ fn run_daemon(
|
|||
let store = Store::open(&state_dir, dim)?;
|
||||
let state = Arc::new(SharedState::new(store));
|
||||
|
||||
// Load embedder if model files present
|
||||
// Load the embedder on a background thread — an OpenVINO/CUDA/etc EP
|
||||
// compile can take minutes or hang outright, and doing this inline used
|
||||
// to block the socket bind below until it finished. That turned "model
|
||||
// still loading" into an indistinguishable "connection refused" for
|
||||
// every client, including the GUI, for as long as the load took. The
|
||||
// socket now opens immediately; serve.rs already answers "model not
|
||||
// ready" (via `model_ready`) for any request that arrives before the
|
||||
// background load finishes.
|
||||
let model_dir = model_dir(&cache_dir);
|
||||
let model_path = model_dir.join("model.onnx");
|
||||
let tokenizer_path = model_dir.join("tokenizer.json");
|
||||
|
||||
if model_path.exists() && tokenizer_path.exists() {
|
||||
eprintln!("breadmill: loading model...");
|
||||
match OrtEmbedder::load(&model_path, &tokenizer_path, dim, backend) {
|
||||
Ok(embedder) => {
|
||||
*state.embedder.lock().unwrap() = Some(embedder);
|
||||
state.model_ready.store(true, Ordering::Relaxed);
|
||||
eprintln!("breadmill: model loaded");
|
||||
let state_clone = Arc::clone(&state);
|
||||
std::thread::spawn(move || {
|
||||
eprintln!("breadmill: loading model...");
|
||||
match OrtEmbedder::load(&model_path, &tokenizer_path, dim, backend) {
|
||||
Ok(embedder) => {
|
||||
*state_clone.embedder.lock().unwrap() = Some(embedder);
|
||||
state_clone.model_ready.store(true, Ordering::Relaxed);
|
||||
eprintln!("breadmill: model loaded");
|
||||
}
|
||||
Err(e) => eprintln!("breadmill: model load failed: {} — run --fetch-model", e),
|
||||
}
|
||||
Err(e) => eprintln!("breadmill: model load failed: {} — run --fetch-model", e),
|
||||
}
|
||||
});
|
||||
} else {
|
||||
eprintln!(
|
||||
"breadmill: model files not found in {} — run: breadmill --fetch-model",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue