Committing before copilot touches this
This commit is contained in:
commit
feefdb81b9
36 changed files with 12338 additions and 0 deletions
20
breadpad-test/Cargo.toml
Normal file
20
breadpad-test/Cargo.toml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
[package]
|
||||
name = "breadpad-test"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "breadpad-test"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
breadpad-shared = { path = "../breadpad-shared" }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
colored = "2"
|
||||
comfy-table = "7"
|
||||
506
breadpad-test/corpus.json
Normal file
506
breadpad-test/corpus.json
Normal file
|
|
@ -0,0 +1,506 @@
|
|||
[
|
||||
{
|
||||
"input": "buy milk on the way home",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "buy milk on the way home",
|
||||
"expected_rrule": null,
|
||||
"notes": "plain todo, no time signal"
|
||||
},
|
||||
{
|
||||
"input": "pick up dry cleaning",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "pick up dry cleaning",
|
||||
"expected_rrule": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"input": "fix the leaky tap in the bathroom",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "fix the leaky tap in the bathroom",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'fix'"
|
||||
},
|
||||
{
|
||||
"input": "write release notes for v0.2",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "write release notes for v0.2",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'write'"
|
||||
},
|
||||
{
|
||||
"input": "email the accountant about last quarter",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "email the accountant about last quarter",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'email'"
|
||||
},
|
||||
{
|
||||
"input": "check the breadpad logs for errors",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "check the breadpad logs for errors",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'check'"
|
||||
},
|
||||
{
|
||||
"input": "finish implementing the notification popup",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "finish implementing the notification popup",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'finish'"
|
||||
},
|
||||
{
|
||||
"input": "update the workspace dependencies",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "update the workspace dependencies",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'update'"
|
||||
},
|
||||
{
|
||||
"input": "clean up the old git branches",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "clean up the old git branches",
|
||||
"expected_rrule": null,
|
||||
"notes": "contains 'clean '"
|
||||
},
|
||||
{
|
||||
"input": "call mum at 7pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "19:00",
|
||||
"expected_body": "call mum",
|
||||
"expected_rrule": null,
|
||||
"notes": "12h reminder; body should have time phrase stripped"
|
||||
},
|
||||
{
|
||||
"input": "dentist appointment at 3pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "15:00",
|
||||
"expected_body": "dentist appointment",
|
||||
"expected_rrule": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"input": "pack calculator in bag at 7pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "19:00",
|
||||
"expected_body": "pack calculator in bag",
|
||||
"expected_rrule": null,
|
||||
"notes": "time stripped from middle of sentence"
|
||||
},
|
||||
{
|
||||
"input": "meeting prep at 14:30",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "14:30",
|
||||
"expected_body": "meeting prep",
|
||||
"expected_rrule": null,
|
||||
"notes": "24h time with minutes"
|
||||
},
|
||||
{
|
||||
"input": "check the deploy at 23:00",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "23:00",
|
||||
"expected_body": "check the deploy",
|
||||
"expected_rrule": null,
|
||||
"notes": "24h late-night reminder"
|
||||
},
|
||||
{
|
||||
"input": "lunch at 12pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "12:00",
|
||||
"expected_body": "lunch",
|
||||
"expected_rrule": null,
|
||||
"notes": "noon edge case"
|
||||
},
|
||||
{
|
||||
"input": "buy milk at the store",
|
||||
"expected_type": "todo",
|
||||
"expected_time": null,
|
||||
"expected_body": "buy milk at the store",
|
||||
"expected_rrule": null,
|
||||
"notes": "'at the store' has no digit — should not be parsed as time"
|
||||
},
|
||||
{
|
||||
"input": "take a break in 20 minutes",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "take a break",
|
||||
"expected_rrule": null,
|
||||
"notes": "numeric relative time; expected_time null because offset from now is not assertable"
|
||||
},
|
||||
{
|
||||
"input": "review the PR in 2 hours",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "review the PR",
|
||||
"expected_rrule": null,
|
||||
"notes": "relative hours"
|
||||
},
|
||||
{
|
||||
"input": "follow up in 3 days",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "follow up",
|
||||
"expected_rrule": null,
|
||||
"notes": "relative days"
|
||||
},
|
||||
{
|
||||
"input": "check on the deployment in an hour",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "check on the deployment",
|
||||
"expected_rrule": null,
|
||||
"notes": "word-form duration: 'an hour' = 1h"
|
||||
},
|
||||
{
|
||||
"input": "in a couple of hours remind me to check the oven",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "remind me to check the oven",
|
||||
"expected_rrule": null,
|
||||
"notes": "word-form duration: 'a couple of hours' = 2h"
|
||||
},
|
||||
{
|
||||
"input": "in a few hours I need to submit this form",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "word-form duration: 'a few hours' = 3h"
|
||||
},
|
||||
{
|
||||
"input": "in half an hour submit the report",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "submit the report",
|
||||
"expected_rrule": null,
|
||||
"notes": "word-form duration: 'half an hour' = 30 min"
|
||||
},
|
||||
{
|
||||
"input": "standup tomorrow morning",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "standup",
|
||||
"expected_rrule": null,
|
||||
"notes": "tomorrow morning → tomorrow at 08:00; time not asserted (date-relative)"
|
||||
},
|
||||
{
|
||||
"input": "dentist appointment tomorrow",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "dentist appointment",
|
||||
"expected_rrule": null,
|
||||
"notes": "bare 'tomorrow' uses morning default"
|
||||
},
|
||||
{
|
||||
"input": "call family tomorrow evening",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "call family",
|
||||
"expected_rrule": null,
|
||||
"notes": "tomorrow evening → tomorrow at 18:00"
|
||||
},
|
||||
{
|
||||
"input": "dentist next monday",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "dentist",
|
||||
"expected_rrule": null,
|
||||
"notes": "next weekday; time not asserted (date-relative)"
|
||||
},
|
||||
{
|
||||
"input": "team lunch next friday",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "team lunch",
|
||||
"expected_rrule": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"input": "tonight put the bins out",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "21:00",
|
||||
"expected_body": "put the bins out",
|
||||
"expected_rrule": null,
|
||||
"notes": "'tonight' anchors to 21:00; word stripped from body"
|
||||
},
|
||||
{
|
||||
"input": "watch the football tonight",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "21:00",
|
||||
"expected_body": "watch the football",
|
||||
"expected_rrule": null,
|
||||
"notes": "'tonight' at end of sentence"
|
||||
},
|
||||
{
|
||||
"input": "this evening water the plants",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "21:00",
|
||||
"expected_body": "water the plants",
|
||||
"expected_rrule": null,
|
||||
"notes": "'this evening' synonym for tonight"
|
||||
},
|
||||
{
|
||||
"input": "call dad tonight at 8pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "20:00",
|
||||
"expected_body": "call dad",
|
||||
"expected_rrule": null,
|
||||
"notes": "explicit 'at 8pm' overrides 'tonight'; at_time takes precedence"
|
||||
},
|
||||
{
|
||||
"input": "later today sort out the inbox",
|
||||
"expected_type": null,
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "vague relative time; Tier 1 cannot parse 'later today' — no assertion"
|
||||
},
|
||||
{
|
||||
"input": "every sunday night check the bins",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "check the bins",
|
||||
"expected_rrule": "BYDAY=SU",
|
||||
"notes": "weekly recurrence; 'night' stripped from body via morning_evening cleanup"
|
||||
},
|
||||
{
|
||||
"input": "every weekday morning check email",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "check email",
|
||||
"expected_rrule": "BYDAY=MO,TU,WE,TH,FR",
|
||||
"notes": "Mon–Fri recurrence; 'morning' maps to default_morning time"
|
||||
},
|
||||
{
|
||||
"input": "every weekday at 9am standup",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "standup",
|
||||
"expected_rrule": "BYDAY=MO,TU,WE,TH,FR",
|
||||
"notes": "Mon–Fri recurrence with explicit time"
|
||||
},
|
||||
{
|
||||
"input": "every tuesday at 6pm call dad",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "call dad",
|
||||
"expected_rrule": "BYDAY=TU",
|
||||
"notes": "named weekday recurrence with time"
|
||||
},
|
||||
{
|
||||
"input": "every friday at 1pm team lunch",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "team lunch",
|
||||
"expected_rrule": "BYDAY=FR",
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"input": "every saturday morning review the week",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "review the week",
|
||||
"expected_rrule": "BYDAY=SA",
|
||||
"notes": "named weekday recurrence; 'morning' stripped from body"
|
||||
},
|
||||
{
|
||||
"input": "take my meds every day at 8am",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "take my meds",
|
||||
"expected_rrule": "FREQ=DAILY",
|
||||
"notes": "daily recurrence with explicit time"
|
||||
},
|
||||
{
|
||||
"input": "every day take vitamins",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "take vitamins",
|
||||
"expected_rrule": "FREQ=DAILY",
|
||||
"notes": "daily recurrence; 'every day' at start"
|
||||
},
|
||||
{
|
||||
"input": "retro every week at 4pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": "retro",
|
||||
"expected_rrule": "FREQ=WEEKLY",
|
||||
"notes": "every_week pattern still works after adding every_weekdays"
|
||||
},
|
||||
{
|
||||
"input": "what if breadman had a calendar view",
|
||||
"expected_type": "idea",
|
||||
"expected_time": null,
|
||||
"expected_body": "what if breadman had a calendar view",
|
||||
"expected_rrule": null,
|
||||
"notes": "classic what-if idea"
|
||||
},
|
||||
{
|
||||
"input": "idea: dark mode toggle in breadman",
|
||||
"expected_type": "idea",
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "explicit idea: prefix"
|
||||
},
|
||||
{
|
||||
"input": "maybe we could add export to CSV",
|
||||
"expected_type": "idea",
|
||||
"expected_time": null,
|
||||
"expected_body": "maybe we could add export to CSV",
|
||||
"expected_rrule": null,
|
||||
"notes": "contains 'maybe'"
|
||||
},
|
||||
{
|
||||
"input": "could the sidebar show counts per type",
|
||||
"expected_type": "idea",
|
||||
"expected_time": null,
|
||||
"expected_body": "could the sidebar show counts per type",
|
||||
"expected_rrule": null,
|
||||
"notes": "contains 'could'"
|
||||
},
|
||||
{
|
||||
"input": "should we add a dark mode to breadman",
|
||||
"expected_type": "idea",
|
||||
"expected_time": null,
|
||||
"expected_body": "should we add a dark mode to breadman",
|
||||
"expected_rrule": null,
|
||||
"notes": "contains 'should we '"
|
||||
},
|
||||
{
|
||||
"input": "what if we switched to SQLite instead of JSONL",
|
||||
"expected_type": "idea",
|
||||
"expected_time": null,
|
||||
"expected_body": "what if we switched to SQLite instead of JSONL",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'what if'"
|
||||
},
|
||||
{
|
||||
"input": "why does nmcli drop on suspend?",
|
||||
"expected_type": "question",
|
||||
"expected_time": null,
|
||||
"expected_body": "why does nmcli drop on suspend?",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'why'"
|
||||
},
|
||||
{
|
||||
"input": "how do I configure zbus async",
|
||||
"expected_type": "question",
|
||||
"expected_time": null,
|
||||
"expected_body": "how do I configure zbus async",
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'how'"
|
||||
},
|
||||
{
|
||||
"input": "what is the best way to handle GTK signals in Rust?",
|
||||
"expected_type": "question",
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "starts with 'what' (not 'what if') — question, not idea"
|
||||
},
|
||||
{
|
||||
"input": "is this thread safe?",
|
||||
"expected_type": "question",
|
||||
"expected_time": null,
|
||||
"expected_body": "is this thread safe?",
|
||||
"expected_rrule": null,
|
||||
"notes": "ends with ? without a known question-word prefix"
|
||||
},
|
||||
{
|
||||
"input": "does the ONNX runtime cache model weights between runs?",
|
||||
"expected_type": "question",
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "ends with ?"
|
||||
},
|
||||
{
|
||||
"input": "meeting went well, follow up needed",
|
||||
"expected_type": "note",
|
||||
"expected_time": null,
|
||||
"expected_body": "meeting went well, follow up needed",
|
||||
"expected_rrule": null,
|
||||
"notes": "plain observation; no action verbs, no time, no question"
|
||||
},
|
||||
{
|
||||
"input": "the new mechanical keyboard is noisy but I like it",
|
||||
"expected_type": "note",
|
||||
"expected_time": null,
|
||||
"expected_body": "the new mechanical keyboard is noisy but I like it",
|
||||
"expected_rrule": null,
|
||||
"notes": "general note, no strong signals"
|
||||
},
|
||||
{
|
||||
"input": "finally got the ONNX model running on CPU",
|
||||
"expected_type": "note",
|
||||
"expected_time": null,
|
||||
"expected_body": "finally got the ONNX model running on CPU",
|
||||
"expected_rrule": null,
|
||||
"notes": "observation without actionable signal"
|
||||
},
|
||||
{
|
||||
"input": "i should probably sort out that thing with the lights",
|
||||
"expected_type": null,
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "ambiguous; Tier 1 returns note — no assertion, good Tier 2 test case"
|
||||
},
|
||||
{
|
||||
"input": "get around to fixing the shelf at some point",
|
||||
"expected_type": null,
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "'at some point' has no digit after 'at' so time not extracted; 'fixing' not a recognised prefix"
|
||||
},
|
||||
{
|
||||
"input": "idea: remind me to think about this every monday",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": "BYDAY=MO",
|
||||
"notes": "mixed signals: idea: prefix but recurrence takes precedence — type must be reminder"
|
||||
},
|
||||
{
|
||||
"input": "buy milk tonight and remind me to call the doctor tomorrow",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "multi-intent: 'tomorrow' triggers time extraction; 'tonight' word stripped from body"
|
||||
},
|
||||
{
|
||||
"input": "sort out that thing with the router at some point",
|
||||
"expected_type": null,
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "unstructured; 'at some point' not parsed as time"
|
||||
},
|
||||
{
|
||||
"input": "rmind me abt dentist appt tmrw",
|
||||
"expected_type": null,
|
||||
"expected_time": null,
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "typo-heavy; Tier 1 cannot parse this; Tier 2/3 may or may not handle it"
|
||||
},
|
||||
{
|
||||
"input": "pls remind me abt the call at 3pm",
|
||||
"expected_type": "reminder",
|
||||
"expected_time": "15:00",
|
||||
"expected_body": null,
|
||||
"expected_rrule": null,
|
||||
"notes": "'at 3pm' matches rule-based despite informal phrasing around it"
|
||||
}
|
||||
]
|
||||
474
breadpad-test/src/main.rs
Normal file
474
breadpad-test/src/main.rs
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
use anyhow::{Context, Result};
|
||||
use breadpad_shared::{
|
||||
classifier::Classifier,
|
||||
config::OllamaConfig,
|
||||
parser::parse_rule_based,
|
||||
types::ClassificationResult,
|
||||
};
|
||||
use chrono::{DateTime, Local, Timelike, Utc};
|
||||
use clap::{Parser, Subcommand, ValueEnum};
|
||||
use colored::Colorize;
|
||||
use comfy_table::{Cell, Color, Table};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const DEFAULT_CORPUS: &str = "breadpad-test/corpus.json";
|
||||
const DEFAULT_MORNING: &str = "08:00";
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "breadpad-test",
|
||||
about = "Test harness for the breadpad classification pipeline"
|
||||
)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Run tests against the corpus
|
||||
Run {
|
||||
#[arg(long, default_value = DEFAULT_CORPUS)]
|
||||
corpus: PathBuf,
|
||||
/// Which classification tiers to invoke: 1 (rule-based only), 2 (+ ONNX), 3/all (+ Ollama)
|
||||
#[arg(long, default_value = "1")]
|
||||
tier: TierArg,
|
||||
/// Output format: table, json, or failures (failing cases only)
|
||||
#[arg(long, default_value = "table")]
|
||||
format: FormatArg,
|
||||
},
|
||||
/// Interactively add a new corpus entry
|
||||
Add {
|
||||
#[arg(long, default_value = DEFAULT_CORPUS)]
|
||||
corpus: PathBuf,
|
||||
},
|
||||
/// Show a corpus entry and the pipeline's actual output side by side
|
||||
Show {
|
||||
index: usize,
|
||||
#[arg(long, default_value = DEFAULT_CORPUS)]
|
||||
corpus: PathBuf,
|
||||
},
|
||||
/// Open the corpus file in $EDITOR at the given entry
|
||||
Edit {
|
||||
index: usize,
|
||||
#[arg(long, default_value = DEFAULT_CORPUS)]
|
||||
corpus: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(ValueEnum, Clone, Debug)]
|
||||
enum TierArg {
|
||||
#[value(name = "1")]
|
||||
One,
|
||||
#[value(name = "2")]
|
||||
Two,
|
||||
#[value(name = "3")]
|
||||
Three,
|
||||
#[value(name = "all")]
|
||||
All,
|
||||
}
|
||||
|
||||
impl TierArg {
|
||||
fn label(&self) -> &'static str {
|
||||
match self {
|
||||
TierArg::One => "1",
|
||||
TierArg::Two => "2",
|
||||
TierArg::Three => "3",
|
||||
TierArg::All => "all",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(ValueEnum, Clone, Debug)]
|
||||
enum FormatArg {
|
||||
Table,
|
||||
Json,
|
||||
Failures,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
struct CorpusEntry {
|
||||
input: String,
|
||||
#[serde(default)]
|
||||
expected_type: Option<String>,
|
||||
/// Expected time as HH:MM — date component is ignored so tests are not date-sensitive
|
||||
#[serde(default)]
|
||||
expected_time: Option<String>,
|
||||
/// Expected body text (checked as substring of actual body)
|
||||
#[serde(default)]
|
||||
expected_body: Option<String>,
|
||||
/// Expected rrule (checked as substring of actual rrule string)
|
||||
#[serde(default)]
|
||||
expected_rrule: Option<String>,
|
||||
#[serde(default)]
|
||||
notes: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TestResult {
|
||||
index: usize,
|
||||
input: String,
|
||||
tier_used: String,
|
||||
actual_type: String,
|
||||
actual_time: Option<String>,
|
||||
actual_rrule: Option<String>,
|
||||
actual_body: String,
|
||||
type_pass: Option<bool>,
|
||||
time_pass: Option<bool>,
|
||||
rrule_pass: Option<bool>,
|
||||
body_pass: Option<bool>,
|
||||
pass: bool,
|
||||
failure_reason: Option<String>,
|
||||
}
|
||||
|
||||
fn load_corpus(path: &Path) -> Result<Vec<CorpusEntry>> {
|
||||
let data = std::fs::read_to_string(path)
|
||||
.with_context(|| format!("could not read corpus at {}", path.display()))?;
|
||||
serde_json::from_str(&data).context("invalid corpus JSON")
|
||||
}
|
||||
|
||||
fn save_corpus(path: &Path, entries: &[CorpusEntry]) -> Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
}
|
||||
std::fs::write(path, serde_json::to_string_pretty(entries)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn classify_with_tier(text: &str, tier: &TierArg) -> ClassificationResult {
|
||||
match tier {
|
||||
TierArg::One => parse_rule_based(text, DEFAULT_MORNING),
|
||||
TierArg::Two => {
|
||||
let mut clf = Classifier::load("auto", DEFAULT_MORNING);
|
||||
clf.classify(text)
|
||||
}
|
||||
TierArg::Three | TierArg::All => {
|
||||
let ollama = OllamaConfig {
|
||||
endpoint: "http://localhost:11434".to_string(),
|
||||
model: "llama3.2:3b".to_string(),
|
||||
confidence_threshold: 0.6,
|
||||
enabled: true,
|
||||
};
|
||||
let mut clf = Classifier::load("auto", DEFAULT_MORNING).with_ollama(ollama);
|
||||
clf.classify(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn format_time(t: Option<DateTime<Utc>>) -> Option<String> {
|
||||
t.map(|dt| {
|
||||
let local: DateTime<Local> = dt.into();
|
||||
format!("{:02}:{:02}", local.hour(), local.minute())
|
||||
})
|
||||
}
|
||||
|
||||
fn run_tests(entries: &[CorpusEntry], tier: &TierArg) -> Vec<TestResult> {
|
||||
entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, entry)| {
|
||||
let result = classify_with_tier(&entry.input, tier);
|
||||
let actual_type = result.note_type.as_str().to_string();
|
||||
let actual_time = format_time(result.time);
|
||||
let actual_rrule = result.rrule.as_ref().map(|r| r.as_str().to_string());
|
||||
let actual_body = result.body.clone();
|
||||
|
||||
let type_pass = entry
|
||||
.expected_type
|
||||
.as_ref()
|
||||
.map(|et| et.to_lowercase() == actual_type);
|
||||
|
||||
let time_pass = entry.expected_time.as_ref().map(|et| {
|
||||
actual_time.as_deref().map(|at| at == et).unwrap_or(false)
|
||||
});
|
||||
|
||||
let rrule_pass = entry.expected_rrule.as_ref().map(|er| {
|
||||
actual_rrule
|
||||
.as_deref()
|
||||
.map(|ar| ar.contains(er.as_str()))
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
let body_pass = entry.expected_body.as_ref().map(|eb| {
|
||||
actual_body
|
||||
.to_lowercase()
|
||||
.contains(&eb.to_lowercase())
|
||||
});
|
||||
|
||||
let mut failure_parts: Vec<String> = Vec::new();
|
||||
if type_pass == Some(false) {
|
||||
failure_parts.push(format!(
|
||||
"type: expected {}, got {}",
|
||||
entry.expected_type.as_deref().unwrap_or("?"),
|
||||
actual_type
|
||||
));
|
||||
}
|
||||
if time_pass == Some(false) {
|
||||
failure_parts.push(format!(
|
||||
"time: expected {}, got {}",
|
||||
entry.expected_time.as_deref().unwrap_or("none"),
|
||||
actual_time.as_deref().unwrap_or("none")
|
||||
));
|
||||
}
|
||||
if rrule_pass == Some(false) {
|
||||
failure_parts.push(format!(
|
||||
"rrule: expected to contain {:?}",
|
||||
entry.expected_rrule.as_deref().unwrap_or("")
|
||||
));
|
||||
}
|
||||
if body_pass == Some(false) {
|
||||
failure_parts.push(format!(
|
||||
"body: {:?} not found in {:?}",
|
||||
entry.expected_body.as_deref().unwrap_or(""),
|
||||
actual_body
|
||||
));
|
||||
}
|
||||
|
||||
let pass = failure_parts.is_empty();
|
||||
|
||||
TestResult {
|
||||
index: i + 1,
|
||||
input: entry.input.clone(),
|
||||
tier_used: tier.label().to_string(),
|
||||
actual_type,
|
||||
actual_time,
|
||||
actual_rrule,
|
||||
actual_body,
|
||||
type_pass,
|
||||
time_pass,
|
||||
rrule_pass,
|
||||
body_pass,
|
||||
pass,
|
||||
failure_reason: if failure_parts.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(failure_parts.join("; "))
|
||||
},
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_mark(v: Option<bool>) -> &'static str {
|
||||
match v {
|
||||
Some(true) => "✓",
|
||||
Some(false) => "✗",
|
||||
None => "-",
|
||||
}
|
||||
}
|
||||
|
||||
fn print_table(results: &[TestResult], failures_only: bool) {
|
||||
let to_show: Vec<&TestResult> = if failures_only {
|
||||
results.iter().filter(|r| !r.pass).collect()
|
||||
} else {
|
||||
results.iter().collect()
|
||||
};
|
||||
|
||||
let mut table = Table::new();
|
||||
table.set_header(vec!["#", "input", "tier", "type", "time", "rrule", "body", "result"]);
|
||||
|
||||
for r in &to_show {
|
||||
let input_display = if r.input.len() > 42 {
|
||||
format!("{}…", &r.input[..41])
|
||||
} else {
|
||||
r.input.clone()
|
||||
};
|
||||
|
||||
let result_cell = if r.pass {
|
||||
Cell::new("PASS").fg(Color::Green)
|
||||
} else {
|
||||
let reason = r.failure_reason.as_deref().unwrap_or("");
|
||||
Cell::new(format!("FAIL {reason}")).fg(Color::Red)
|
||||
};
|
||||
|
||||
table.add_row(vec![
|
||||
Cell::new(r.index),
|
||||
Cell::new(&input_display),
|
||||
Cell::new(&r.tier_used),
|
||||
Cell::new(&r.actual_type),
|
||||
Cell::new(check_mark(r.time_pass)),
|
||||
Cell::new(check_mark(r.rrule_pass)),
|
||||
Cell::new(check_mark(r.body_pass)),
|
||||
result_cell,
|
||||
]);
|
||||
}
|
||||
|
||||
if !to_show.is_empty() {
|
||||
println!("{table}");
|
||||
}
|
||||
|
||||
let passed = results.iter().filter(|r| r.pass).count();
|
||||
let failed = results.iter().filter(|r| !r.pass).count();
|
||||
if failed == 0 {
|
||||
println!("{}", format!("{passed} passed").green());
|
||||
} else {
|
||||
println!("{}, {}", format!("{passed} passed").green(), format!("{failed} failed").red());
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt(label: &str) -> Result<String> {
|
||||
use std::io::Write;
|
||||
print!("{label}");
|
||||
std::io::stdout().flush()?;
|
||||
let mut s = String::new();
|
||||
std::io::stdin().read_line(&mut s)?;
|
||||
Ok(s.trim().to_string())
|
||||
}
|
||||
|
||||
fn prompt_opt(label: &str) -> Result<Option<String>> {
|
||||
let s = prompt(label)?;
|
||||
Ok(if s.is_empty() { None } else { Some(s) })
|
||||
}
|
||||
|
||||
fn cmd_add(corpus_path: &Path) -> Result<()> {
|
||||
let mut entries = if corpus_path.exists() {
|
||||
load_corpus(corpus_path)?
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
println!("Adding a new corpus entry. Press Enter to leave a field null.\n");
|
||||
|
||||
let input = prompt("input: ")?;
|
||||
if input.is_empty() {
|
||||
anyhow::bail!("input is required");
|
||||
}
|
||||
let expected_type = prompt_opt("expected_type (todo/reminder/idea/note/question or Enter): ")?;
|
||||
let expected_time = prompt_opt("expected_time (HH:MM or Enter): ")?;
|
||||
let expected_body = prompt_opt("expected_body (substring or Enter): ")?;
|
||||
let expected_rrule = prompt_opt("expected_rrule (substring or Enter): ")?;
|
||||
let notes = prompt_opt("notes (or Enter): ")?;
|
||||
|
||||
entries.push(CorpusEntry {
|
||||
input,
|
||||
expected_type,
|
||||
expected_time,
|
||||
expected_body,
|
||||
expected_rrule,
|
||||
notes,
|
||||
});
|
||||
|
||||
save_corpus(corpus_path, &entries)?;
|
||||
println!("\nAdded entry #{}", entries.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_show(index: usize, corpus_path: &Path, tier: &TierArg) -> Result<()> {
|
||||
let entries = load_corpus(corpus_path)?;
|
||||
let entry = entries.get(index.saturating_sub(1)).ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"index {} out of range (corpus has {} entries)",
|
||||
index,
|
||||
entries.len()
|
||||
)
|
||||
})?;
|
||||
|
||||
let result = classify_with_tier(&entry.input, tier);
|
||||
let actual_time = format_time(result.time);
|
||||
let actual_rrule = result.rrule.as_ref().map(|r| r.as_str().to_string());
|
||||
|
||||
println!("─── Entry {} (tier {}) ───", index, tier.label());
|
||||
println!("input: {}", entry.input);
|
||||
if let Some(n) = &entry.notes {
|
||||
println!("notes: {}", n);
|
||||
}
|
||||
println!();
|
||||
|
||||
let sep = "─".repeat(62);
|
||||
println!("{:<14} {:<26} {}", "field", "expected", "actual");
|
||||
println!("{sep}");
|
||||
println!(
|
||||
"{:<14} {:<26} {}",
|
||||
"type",
|
||||
entry.expected_type.as_deref().unwrap_or("(any)"),
|
||||
result.note_type.as_str()
|
||||
);
|
||||
println!(
|
||||
"{:<14} {:<26} {}",
|
||||
"time",
|
||||
entry.expected_time.as_deref().unwrap_or("(any)"),
|
||||
actual_time.as_deref().unwrap_or("none")
|
||||
);
|
||||
println!(
|
||||
"{:<14} {:<26} {}",
|
||||
"rrule",
|
||||
entry.expected_rrule.as_deref().unwrap_or("(any)"),
|
||||
actual_rrule.as_deref().unwrap_or("none")
|
||||
);
|
||||
println!(
|
||||
"{:<14} {:<26} {}",
|
||||
"body",
|
||||
entry.expected_body.as_deref().unwrap_or("(any)"),
|
||||
result.body
|
||||
);
|
||||
println!("{:<14} {:<26} {:.2}", "confidence", "", result.confidence);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn find_entry_line(corpus_path: &Path, index: usize) -> Result<Option<usize>> {
|
||||
let content = std::fs::read_to_string(corpus_path)?;
|
||||
let mut count = 0usize;
|
||||
for (line_num, line) in content.lines().enumerate() {
|
||||
if line.trim_start().starts_with('{') {
|
||||
count += 1;
|
||||
if count == index {
|
||||
return Ok(Some(line_num + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn cmd_edit(index: usize, corpus_path: &Path) -> Result<()> {
|
||||
let editor = std::env::var("EDITOR").unwrap_or_else(|_| "vi".to_string());
|
||||
let line = find_entry_line(corpus_path, index)?;
|
||||
|
||||
let status = if let Some(n) = line {
|
||||
std::process::Command::new(&editor)
|
||||
.arg(format!("+{n}"))
|
||||
.arg(corpus_path)
|
||||
.status()?
|
||||
} else {
|
||||
std::process::Command::new(&editor)
|
||||
.arg(corpus_path)
|
||||
.status()?
|
||||
};
|
||||
|
||||
if !status.success() {
|
||||
anyhow::bail!("editor exited with non-zero status");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Commands::Run { corpus, tier, format } => {
|
||||
let entries = load_corpus(&corpus)?;
|
||||
if entries.is_empty() {
|
||||
println!("corpus is empty");
|
||||
return Ok(());
|
||||
}
|
||||
let results = run_tests(&entries, &tier);
|
||||
match format {
|
||||
FormatArg::Table => print_table(&results, false),
|
||||
FormatArg::Failures => print_table(&results, true),
|
||||
FormatArg::Json => println!("{}", serde_json::to_string_pretty(&results)?),
|
||||
}
|
||||
let failed = results.iter().filter(|r| !r.pass).count();
|
||||
if failed > 0 {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
Commands::Add { corpus } => cmd_add(&corpus)?,
|
||||
Commands::Show { index, corpus } => cmd_show(index, &corpus, &TierArg::One)?,
|
||||
Commands::Edit { index, corpus } => cmd_edit(index, &corpus)?,
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue