From c6ed6a41d818aab32dc6268f7a9664068df5f635 Mon Sep 17 00:00:00 2001 From: Breadway Date: Fri, 3 Jul 2026 22:58:27 +0800 Subject: [PATCH] Add OpenVINO backend for Intel iGPU/dGPU (Arc) Mirrors the rocm/cuda pattern: ort::ep::OpenVINO with device_type "GPU" (covers both Intel integrated graphics and Arc discrete GPUs through the same EP), load-dynamic/dlopen so no OpenVINO toolkit is needed at build time, and its own --openvino flag / backend = "openvino" config value. Folded into the `full` feature alongside npu/rocm/cuda. OpenVINO's provider options go through a generic key/value FFI interface rather than a fixed C struct (unlike MIGraphX's OrtMIGraphXProviderOptions), so it should be less exposed to the ABI-version-skew crash MIGraphX hit -- but that's inference from the EP's design, not verified against real hardware. Like CUDA, this is compile-checked only: no Intel GPU in this dev environment to runtime-verify against. Version bump: breadmill 0.2.2 -> 0.2.3. --- Cargo.lock | 2 +- DESIGN.md | 6 ++-- README.md | 57 +++++++++++++++++++++++------------ breadmill/Cargo.toml | 16 ++++++---- breadmill/src/embed.rs | 36 ++++++++++++++++++++++ breadmill/src/main.rs | 24 ++++++++++++--- breadsearch-shared/src/lib.rs | 3 +- packaging/breadmill.service | 3 +- 8 files changed, 112 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5918b71..be95ae6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,7 +133,7 @@ dependencies = [ [[package]] name = "breadmill" -version = "0.2.2" +version = "0.2.3" dependencies = [ "breadsearch-shared", "hex", diff --git a/DESIGN.md b/DESIGN.md index 5ffa28e..85d7e6b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -80,7 +80,9 @@ Start from `breadbox/breadbox/src/main.rs`. **Reuse verbatim:** the gtk4-layer-s - nomic prefixes + mean-pool + normalize must match between index and query or recall collapses. - `ort` linking: prefer the crate's downloaded/bundled ONNX Runtime to avoid version skew with Arch's `onnxruntime`. - Office formats (docx/odt) are best-effort in v1; md/txt/org/pdf are the reliable path. -- GPU EPs (ROCm/CUDA) fail to register silently at the ONNX Runtime level and fall back to CPU — always check - startup logs for `Successfully registered` before trusting a GPU build is actually accelerating. See +- GPU EPs (ROCm/CUDA/OpenVINO) fail to register silently at the ONNX Runtime level and fall back to CPU — always + check startup logs for `Successfully registered` before trusting a GPU build is actually accelerating. See [README: GPU backend notes](README.md#gpu-backend-notes) for the MIGraphX-vs-ROCMExecutionProvider distinction and the per-shape JIT-compile-and-cache behavior that matters for interactive query latency. +- CUDA and OpenVINO are compile-checked only — no NVIDIA or Intel GPU hardware in this dev environment (AMD-only) + to runtime-verify against, unlike ROCm which was confirmed end-to-end on real hardware. diff --git a/README.md b/README.md index 446787d..af25de7 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,11 @@ Optional features: | Feature | What it adds | |---------|-------------| -| `npu` | AMD XDNA NPU via VitisAI ONNX Runtime EP (requires Ryzen AI SDK) | -| `rocm` | AMD iGPU via the MIGraphX ONNX Runtime EP (ROCm-backed) | -| `cuda` | NVIDIA GPU via the CUDA ONNX Runtime EP | -| `full` | All three of the above in one binary | +| `npu` | AMD XDNA NPU via VitisAI ONNX Runtime EP (requires Ryzen AI SDK) | +| `rocm` | AMD iGPU via the MIGraphX ONNX Runtime EP (ROCm-backed) | +| `cuda` | NVIDIA GPU via the CUDA ONNX Runtime EP | +| `openvino` | Intel iGPU/dGPU (Arc) via the OpenVINO ONNX Runtime EP | +| `full` | All four of the above in one binary | ``` # NPU build @@ -36,25 +37,29 @@ cargo build --release -p breadmill --features rocm # CUDA (NVIDIA GPU) build cargo build --release -p breadmill --features cuda +# OpenVINO (Intel iGPU/dGPU) build +cargo build --release -p breadmill --features openvino + # All backends in one binary (what the release build ships) cargo build --release -p breadmill --features full ``` -`rocm`/`cuda`/`npu` all use `ort`'s `load-dynamic` mode: at runtime, breadmill -dlopens whatever `libonnxruntime.so` the dynamic linker resolves (or +`rocm`/`cuda`/`npu`/`openvino` all use `ort`'s `load-dynamic` mode: at runtime, +breadmill dlopens whatever `libonnxruntime.so` the dynamic linker resolves (or `ORT_DYLIB_PATH` if set). GPU acceleration only works if that ONNX Runtime build actually has the matching execution provider compiled in — breadmill logs a clear `Successfully registered` / `not enabled in this build` line for this at startup (see [GPU backend notes](#gpu-backend-notes) below). -Because all three are dlopen-based, `full` doesn't require the NPU/ROCm/CUDA -toolkits to be installed at build time — only at run time, and only for -whichever single backend you actually select via `--npu`/`--rocm`/`--cuda` -or `backend` in config.toml. The **released binaries are built with -`full`**: same binary works CPU-only out of the box, and picks up NPU/ROCm/CUDA -acceleration on a machine that has the matching ONNX Runtime available, -without needing a different download. An explicit `--npu`/`--rocm`/`--cuda` -flag always overrides `backend` in config.toml, not the other way around. +Because all four are dlopen-based, `full` doesn't require the NPU/ROCm/CUDA/ +OpenVINO toolkits to be installed at build time — only at run time, and only +for whichever single backend you actually select via +`--npu`/`--rocm`/`--cuda`/`--openvino` or `backend` in config.toml. The +**released binaries are built with `full`**: same binary works CPU-only out +of the box, and picks up NPU/ROCm/CUDA/OpenVINO acceleration on a machine +that has the matching ONNX Runtime available, without needing a different +download. An explicit `--npu`/`--rocm`/`--cuda`/`--openvino` flag always +overrides `backend` in config.toml, not the other way around. ## Setup @@ -117,6 +122,7 @@ breadmill status breadmill --npu breadmill --rocm breadmill --cuda +breadmill --openvino ``` ## Config @@ -137,7 +143,7 @@ snippet_len = 200 # max characters in result snippet [model] name = "nomic-embed-text-v1.5" dim = 768 -backend = "cpu" # "cpu", "npu", "rocm", or "cuda" +backend = "cpu" # "cpu", "npu", "rocm", "cuda", or "openvino" ``` `roots` and `excludes` support `~/` expansion. The index respects `.gitignore` files found during the walk. @@ -154,10 +160,11 @@ Set `backend = "npu"` in config (or pass `--npu`) when running a build compiled ### GPU backend notes -Both `rocm` and `cuda` need a system ONNX Runtime that was actually built with -the matching execution provider — the crate's own downloaded binary is CPU-only. -Point `ORT_DYLIB_PATH` at one, or install a distro package that provides -`libonnxruntime.so` with the EP baked in and let the dynamic linker find it. +`rocm`, `cuda`, and `openvino` all need a system ONNX Runtime that was +actually built with the matching execution provider — the crate's own +downloaded binary is CPU-only. Point `ORT_DYLIB_PATH` at one, or install a +distro package that provides `libonnxruntime.so` with the EP baked in and +let the dynamic linker find it. **ROCm (`--rocm` / `backend = "rocm"`)** targets ONNX Runtime's **MIGraphX** execution provider, not the classic `ROCMExecutionProvider`. Distro @@ -183,6 +190,18 @@ noticeable for interactive query embedding. CUDA/cuDNN install. Unverified on real NVIDIA hardware in this repo — only compile-checked, since development happened on an AMD-only machine. +**OpenVINO (`--openvino` / `backend = "openvino"`)** targets +`OpenVINOExecutionProvider` with `device_type = "GPU"`, covering both Intel +iGPUs and Arc dGPUs through the same EP (OpenVINO abstracts Intel's whole +hardware line — CPU/GPU/NPU — behind one provider and a device-type string). +Needs an OpenVINO-enabled ONNX Runtime and the OpenVINO runtime itself +installed. Its options go through a generic key/value FFI interface rather +than a fixed C struct (unlike MIGraphX's `OrtMIGraphXProviderOptions`), so it +should be less exposed to the kind of ABI-version-skew crash MIGraphX hit — +but that's inference from reading the EP's design, not something verified +against real Intel GPU hardware. Also unverified on real hardware in this +repo — only compile-checked, for the same reason as CUDA. + ## Runtime paths | Purpose | Path | diff --git a/breadmill/Cargo.toml b/breadmill/Cargo.toml index dcff332..a76d28d 100644 --- a/breadmill/Cargo.toml +++ b/breadmill/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "breadmill" -version = "0.2.2" +version = "0.2.3" edition = "2021" license = "MIT" @@ -14,13 +14,17 @@ npu = ["ort/vitis", "ort/load-dynamic"] # libonnxruntime_providers_rocm.so, which most distros don't package. rocm = ["ort/migraphx", "ort/load-dynamic"] cuda = ["ort/cuda", "ort/load-dynamic"] +# Intel iGPU/dGPU (Arc) + CPU via OpenVINO. One EP covers Intel's whole +# hardware line by device_type string ("CPU"/"GPU"/"GPU.0"/"NPU"/"HETERO:..."); +# we always request "GPU" since CPU is already covered by the cpu backend. +openvino = ["ort/openvino", "ort/load-dynamic"] # All backends in one binary. Safe to combine: every backend here uses # ort's load-dynamic (dlopen) mode, so none of this links against an actual -# NPU/ROCm/CUDA toolkit at build time — which ONNX Runtime actually gets -# loaded (and thus which EPs are really available) is decided at runtime by -# ORT_DYLIB_PATH / the dynamic linker, per the --npu/--rocm/--cuda flag or -# `backend` config value in use for that run. -full = ["npu", "rocm", "cuda"] +# NPU/ROCm/CUDA/OpenVINO toolkit at build time — which ONNX Runtime actually +# gets loaded (and thus which EPs are really available) is decided at +# runtime by ORT_DYLIB_PATH / the dynamic linker, per the --npu/--rocm/ +# --cuda/--openvino flag or `backend` config value in use for that run. +full = ["npu", "rocm", "cuda", "openvino"] [[bin]] name = "breadmill" diff --git a/breadmill/src/embed.rs b/breadmill/src/embed.rs index 9700d6c..3020e58 100644 --- a/breadmill/src/embed.rs +++ b/breadmill/src/embed.rs @@ -27,6 +27,11 @@ pub enum Backend { Rocm, /// NVIDIA GPU via the CUDA ONNX Runtime execution provider. Cuda, + /// Intel iGPU/dGPU (Arc) via the OpenVINO ONNX Runtime execution + /// provider, requesting device_type "GPU". `cache_dir` stores OpenVINO's + /// compiled-model blobs between runs (its own `with_cache_dir`, not an + /// env var — unlike MIGraphX this doesn't need a workaround). + OpenVino { cache_dir: PathBuf }, } pub struct OrtEmbedder { @@ -165,6 +170,7 @@ fn configure_eps(builder: SessionBuilder, backend: &Backend) -> Result npu_session(builder, cache_dir), Backend::Rocm => rocm_session(builder), Backend::Cuda => cuda_session(builder), + Backend::OpenVino { cache_dir } => openvino_session(builder, cache_dir), } } @@ -253,6 +259,36 @@ fn cuda_session(builder: SessionBuilder) -> Result { Ok(builder) } +// ---- OpenVINO EP (Intel iGPU/dGPU) ------------------------------------------- + +#[cfg(feature = "openvino")] +fn openvino_session(builder: SessionBuilder, cache_dir: &Path) -> Result { + let ov_cache = cache_dir.join("openvino"); + std::fs::create_dir_all(&ov_cache).map_err(|e| e.to_string())?; + eprintln!("breadmill: using OpenVINO execution provider (device_type GPU)"); + eprintln!("breadmill: OpenVINO model cache: {}", ov_cache.display()); + eprintln!( + "breadmill: note — check the log line above/below for \"Successfully registered \ + `OpenVINOExecutionProvider`\"; if it's missing, the ONNX Runtime in use wasn't built \ + with OpenVINO support and inference silently fell back to CPU" + ); + builder + .with_execution_providers([ + ort::ep::OpenVINO::default() + .with_device_type("GPU") + .with_cache_dir(ov_cache.to_string_lossy()) + .build(), + ort::ep::CPU::default().build(), + ]) + .map_err(|e| e.to_string()) +} + +#[cfg(not(feature = "openvino"))] +fn openvino_session(builder: SessionBuilder, _cache_dir: &Path) -> Result { + eprintln!("breadmill: OpenVINO backend requested but not compiled in (rebuild with --features openvino); using CPU"); + Ok(builder) +} + /// Locate the VitisAI EP config file required by the AMD Ryzen AI SDK. /// /// Search order: diff --git a/breadmill/src/main.rs b/breadmill/src/main.rs index 8d2e9bd..12f0f99 100644 --- a/breadmill/src/main.rs +++ b/breadmill/src/main.rs @@ -39,12 +39,14 @@ fn main() { let use_npu = raw_args.iter().any(|a| a == "--npu"); let use_rocm = raw_args.iter().any(|a| a == "--rocm"); let use_cuda = raw_args.iter().any(|a| a == "--cuda"); + let use_openvino = raw_args.iter().any(|a| a == "--openvino"); // Build a view of argv without backend flags for command matching. + let backend_flags = ["--npu", "--rocm", "--cuda", "--openvino"]; let args: Vec<&str> = raw_args .iter() .skip(1) - .filter(|a| a.as_str() != "--npu" && a.as_str() != "--rocm" && a.as_str() != "--cuda") + .filter(|a| !backend_flags.contains(&a.as_str())) .map(|s| s.as_str()) .collect(); @@ -59,7 +61,7 @@ fn main() { } } Some("--reindex") | Some("reindex") => { - if let Err(e) = run_daemon(true, use_npu, use_rocm, use_cuda) { + if let Err(e) = run_daemon(true, use_npu, use_rocm, use_cuda, use_openvino) { eprintln!("breadmill: {}", e); std::process::exit(1); } @@ -76,7 +78,7 @@ fn main() { cli_status(); } None | Some("serve") | Some("--serve") => { - if let Err(e) = run_daemon(false, use_npu, use_rocm, use_cuda) { + if let Err(e) = run_daemon(false, use_npu, use_rocm, use_cuda, use_openvino) { eprintln!("breadmill: {}", e); std::process::exit(1); } @@ -84,7 +86,7 @@ fn main() { Some(cmd) => { eprintln!("breadmill: unknown command: {}", cmd); eprintln!( - "usage: breadmill [serve|reindex|fetch-model|query |status] [--npu|--rocm|--cuda] [--version]" + "usage: breadmill [serve|reindex|fetch-model|query |status] [--npu|--rocm|--cuda|--openvino] [--version]" ); std::process::exit(1); } @@ -93,7 +95,13 @@ fn main() { // ---- Daemon ----------------------------------------------------------------- -fn run_daemon(force_reindex: bool, use_npu: bool, use_rocm: bool, use_cuda: bool) -> Result<(), String> { +fn run_daemon( + force_reindex: bool, + use_npu: bool, + use_rocm: bool, + use_cuda: bool, + use_openvino: bool, +) -> Result<(), String> { let config = breadsearch_shared::Config::load(); let state_dir = breadsearch_shared::state_dir(); let cache_dir = breadsearch_shared::cache_dir(); @@ -115,6 +123,8 @@ fn run_daemon(force_reindex: bool, use_npu: bool, use_rocm: bool, use_cuda: bool "rocm" } else if use_cuda { "cuda" + } else if use_openvino { + "openvino" } else { config.model.backend.as_str() }; @@ -132,6 +142,10 @@ fn run_daemon(force_reindex: bool, use_npu: bool, use_rocm: bool, use_cuda: bool eprintln!("breadmill: CUDA backend selected"); Backend::Cuda } + "openvino" => { + eprintln!("breadmill: OpenVINO backend selected"); + Backend::OpenVino { cache_dir: cache_dir.clone() } + } _ => Backend::Cpu, }; diff --git a/breadsearch-shared/src/lib.rs b/breadsearch-shared/src/lib.rs index 7b8b625..fdea6f5 100644 --- a/breadsearch-shared/src/lib.rs +++ b/breadsearch-shared/src/lib.rs @@ -129,7 +129,8 @@ pub struct ModelConfig { pub name: String, #[serde(default = "default_dim")] pub dim: usize, - /// Compute backend: "cpu", "npu" (VitisAI/XDNA), "rocm" (MIGraphX/AMD GPU), or "cuda" (NVIDIA GPU). + /// Compute backend: "cpu", "npu" (VitisAI/XDNA), "rocm" (MIGraphX/AMD GPU), + /// "cuda" (NVIDIA GPU), or "openvino" (Intel iGPU/dGPU). #[serde(default = "default_backend")] pub backend: String, } diff --git a/packaging/breadmill.service b/packaging/breadmill.service index a1da0b6..69c928f 100644 --- a/packaging/breadmill.service +++ b/packaging/breadmill.service @@ -8,7 +8,8 @@ Type=simple # Required (not just a perf tweak) when backend = "rocm": without a valid # cache dir, ONNX Runtime's MIGraphX EP reads an uninitialized cache path and # crashes on the first query instead of just recompiling every restart. Inert -# for cpu/npu/cuda backends. See README.md#gpu-backend-notes. +# for cpu/npu/cuda/openvino backends (openvino's cache dir is set in code, +# not via env var). See README.md#gpu-backend-notes. Environment=ORT_MIGRAPHX_MODEL_CACHE_PATH=%h/.cache/breadsearch/migraphx-cache ExecStart=%h/.cargo/bin/breadmill Restart=on-failure