From 83208191f2313fb23eefb1745ee0ee1a012351d0 Mon Sep 17 00:00:00 2001 From: Romulus21 Date: Sun, 15 Nov 2026 14:24:09 +0100 Subject: [PATCH] Read the meter with vision LLMs and push the index to Home Assistant - Add Claude, Mistral and local Ollama (lfm3-vl) vision backends that read the digits inside the blue boxes of a cropped, upscaled picture - Retry every 30s when the picture has no blue box or the reading does not have WATERMETER_DIGITS digits - Set input_number.index_compteur_d_eau to reading * 100 when it is within 1000 of the current value - Append one line per run to watermeter.log for cron runs Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + Cargo.lock | 103 +++++++++++++++++++++++++++++++ Cargo.toml | 5 +- src/boxes.rs | 7 ++- src/claude.rs | 67 ++++++++++++++++++++ src/fetch.rs | 7 +-- src/homeassistant.rs | 102 ++++++++++++++++++++++++++++++ src/main.rs | 144 +++++++++++++++++++++++++++++++++++++------ src/mistral.rs | 54 ++++++++++++++++ src/ocr.rs | 17 +++-- src/ollama.rs | 76 +++++++++++++++++++++++ src/runlog.rs | 42 +++++++++++++ src/vision.rs | 116 ++++++++++++++++++++++++++++++++++ 13 files changed, 713 insertions(+), 28 deletions(-) create mode 100644 src/claude.rs create mode 100644 src/homeassistant.rs create mode 100644 src/mistral.rs create mode 100644 src/ollama.rs create mode 100644 src/runlog.rs create mode 100644 src/vision.rs diff --git a/.gitignore b/.gitignore index 9836635..068fee9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .env /models /debug +watermeter.log diff --git a/Cargo.lock b/Cargo.lock index f5ecb36..20f20dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,15 @@ dependencies = [ "equator", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -240,6 +249,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "cmake" version = "0.1.58" @@ -736,6 +756,30 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.3.0" @@ -1622,6 +1666,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -1949,6 +1995,19 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2389,12 +2448,15 @@ name = "watermeter_watcher" version = "0.1.0" dependencies = [ "anyhow", + "base64", + "chrono", "dotenvy", "image", "ocrs", "reqwest", "rten", "rten-imageproc", + "serde_json", ] [[package]] @@ -2441,6 +2503,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -2685,6 +2782,12 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + [[package]] name = "zune-core" version = "0.5.3" diff --git a/Cargo.toml b/Cargo.toml index aa47156..d82cb1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,12 @@ edition = "2024" [dependencies] anyhow = "1.0.104" +base64 = "0.23.1" +chrono = { version = "0.4.45", default-features = false, features = ["clock"] } dotenvy = "0.15.7" image = "0.25.10" ocrs = "0.13.1" -reqwest = { version = "0.13.5", features = ["blocking"] } +reqwest = { version = "0.13.5", features = ["blocking", "json"] } rten = "0.26.0" rten-imageproc = "0.26.0" +serde_json = "1.0.151" diff --git a/src/boxes.rs b/src/boxes.rs index f0cc469..9b8b7e7 100644 --- a/src/boxes.rs +++ b/src/boxes.rs @@ -70,7 +70,12 @@ pub fn find_blue_boxes(img: &RgbImage) -> Vec { } } } - let b = BBox { x: x0, y: y0, w: x1 - x0 + 1, h: y1 - y0 + 1 }; + let b = BBox { + x: x0, + y: y0, + w: x1 - x0 + 1, + h: y1 - y0 + 1, + }; if b.w >= MIN_SIDE && b.h >= MIN_SIDE { boxes.push(b); } diff --git a/src/claude.rs b/src/claude.rs new file mode 100644 index 0000000..ed5ec83 --- /dev/null +++ b/src/claude.rs @@ -0,0 +1,67 @@ +use anyhow::{Context, Result, bail}; +use serde_json::{Value, json}; + +use crate::vision; + +const API_URL: &str = "https://api.anthropic.com/v1/messages"; +const DEFAULT_MODEL: &str = "claude-opus-5-5"; + +/// Ask Claude to read the digits inside the blue boxes of the picture. +/// Returns one entry per box, `None` for an unreadable digit. +pub fn read_digits(image: &[u8]) -> Result>> { + let api_key = std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY is not set")?; + let model = std::env::var("ANTHROPIC_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); + let (media_type, data) = vision::encode_image(image)?; + + let body = json!({ + "model": model, + "max_tokens": 16000, + "fallbacks": "default", + "output_config": { + "effort": "low", + "format": { "type": "json_schema", "schema": vision::digits_schema() } + }, + "messages": [{ + "role": "user", + "content": [ + { + "type": "image", + "source": { "type": "base64", "media_type": media_type, "data": data } + }, + { "type": "text", "text": vision::PROMPT } + ] + }] + }); + + let response = reqwest::blocking::Client::new() + .post(API_URL) + .header("x-api-key", api_key) + .header("anthropic-version", "2023-06-01") + .header("anthropic-beta", "server-side-fallback-2026-07-01") + .timeout(std::time::Duration::from_secs(120)) + .json(&body) + .send() + .context("request to Claude API failed")?; + let status = response.status(); + let response: Value = response.json().context("invalid JSON from Claude API")?; + if !status.is_success() { + bail!( + "Claude API returned {status}: {}", + response["error"]["message"] + ); + } + + match response["stop_reason"].as_str() { + Some("end_turn") => {} + other => bail!( + "unexpected stop_reason {other:?}: {}", + response["stop_details"] + ), + } + let text = response["content"] + .as_array() + .and_then(|blocks| blocks.iter().find(|b| b["type"] == "text")) + .and_then(|b| b["text"].as_str()) + .context("no text block in Claude response")?; + vision::parse_digits(text) +} diff --git a/src/fetch.rs b/src/fetch.rs index 3b1749e..e308758 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -1,11 +1,10 @@ use anyhow::{Context, Result}; -use image::DynamicImage; -/// Download the meter picture and decode it. -pub fn fetch_image(url: &str) -> Result { +/// Download the meter picture as raw bytes. +pub fn fetch_image(url: &str) -> Result> { let bytes = reqwest::blocking::get(url) .and_then(|r| r.error_for_status()) .with_context(|| format!("failed to download {url}"))? .bytes()?; - image::load_from_memory(&bytes).context("failed to decode image") + Ok(bytes.to_vec()) } diff --git a/src/homeassistant.rs b/src/homeassistant.rs new file mode 100644 index 0000000..6debc4c --- /dev/null +++ b/src/homeassistant.rs @@ -0,0 +1,102 @@ +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use serde_json::{Value, json}; + +const DEFAULT_ENTITY: &str = "input_number.index_compteur_d_eau"; +/// Readings further than this from the current index are considered misreads. +const MAX_DIFF: f64 = 1000.0; +const TIMEOUT: Duration = Duration::from_secs(30); + +pub struct HomeAssistant { + url: String, + token: String, + entity: String, + client: reqwest::blocking::Client, +} + +pub enum Outcome { + Updated { previous: f64 }, + Unchanged, + Rejected { current: f64 }, +} + +impl HomeAssistant { + /// Configuration from `HA_URL` and `HA_TOKEN` (and optional `HA_ENTITY`), + /// `None` when Home Assistant is not configured. + pub fn from_env() -> Option { + let url = std::env::var("HA_URL").ok()?; + let token = std::env::var("HA_TOKEN").ok()?; + Some(Self { + url: url.trim_end_matches('/').to_owned(), + token, + entity: std::env::var("HA_ENTITY").unwrap_or_else(|_| DEFAULT_ENTITY.into()), + client: reqwest::blocking::Client::new(), + }) + } + + pub fn entity(&self) -> &str { + &self.entity + } + + /// Set the entity to `value` unless it is more than `MAX_DIFF` away from its current state. + pub fn sync_index(&self, value: f64) -> Result { + let current = self.current_value()?; + if (value - current).abs() > MAX_DIFF { + return Ok(Outcome::Rejected { current }); + } + if value == current { + return Ok(Outcome::Unchanged); + } + self.set_value(value)?; + Ok(Outcome::Updated { previous: current }) + } + + fn current_value(&self) -> Result { + let url = format!("{}/api/states/{}", self.url, self.entity); + let response = self + .client + .get(&url) + .bearer_auth(&self.token) + .timeout(TIMEOUT) + .send() + .with_context(|| format!("request to Home Assistant at {url} failed"))?; + let status = response.status(); + if !status.is_success() { + bail!( + "Home Assistant returned {status} for {}: {}", + self.entity, + response.text()? + ); + } + let state: Value = response + .json() + .context("invalid JSON from Home Assistant")?; + let raw = state["state"] + .as_str() + .context("missing `state` in entity")?; + raw.parse() + .with_context(|| format!("{} has non-numeric state {raw:?}", self.entity)) + } + + fn set_value(&self, value: f64) -> Result<()> { + let url = format!("{}/api/services/input_number/set_value", self.url); + let response = self + .client + .post(&url) + .bearer_auth(&self.token) + .timeout(TIMEOUT) + .json(&json!({ "entity_id": self.entity, "value": value })) + .send() + .with_context(|| format!("request to Home Assistant at {url} failed"))?; + let status = response.status(); + if !status.is_success() { + bail!( + "Home Assistant refused to set {}: {status} {}", + self.entity, + response.text()? + ); + } + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index 8c8d3f2..29c25d3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,16 @@ mod boxes; +mod claude; mod fetch; +mod homeassistant; +mod mistral; mod ocr; +mod ollama; +mod runlog; +mod vision; use std::path::PathBuf; use std::process::ExitCode; +use std::time::Duration; use anyhow::{Context, Result}; use image::{Rgb, RgbImage}; @@ -12,28 +19,136 @@ use boxes::{BBox, find_blue_boxes}; use ocr::DigitReader; const DEBUG_DIR: &str = "debug"; +/// Wait between two downloads when the picture has no blue box or the reading has the wrong length. +const RETRY_DELAY: Duration = Duration::from_secs(30); +/// Give up after this many downloads without a valid reading. +const MAX_ATTEMPTS: u32 = 10; fn main() -> ExitCode { - match run() { - Ok(true) => ExitCode::SUCCESS, - Ok(false) => ExitCode::from(2), + let (status, code) = match run() { + Ok(true) => ("ok", ExitCode::SUCCESS), + Ok(false) => ("incomplete", ExitCode::from(2)), Err(e) => { - eprintln!("error: {e:#}"); - ExitCode::FAILURE + runlog::note(format!("error: {e:#}")); + ("error", ExitCode::FAILURE) } + }; + if let Err(e) = runlog::write(status) { + eprintln!("error: {e:#}"); } + code } -/// Returns whether every box could be read. +/// Returns whether every digit could be read. fn run() -> Result { - dotenvy::dotenv().ok(); - let debug = std::env::args().skip(1).any(|a| a == "--debug"); + dotenvy::dotenv_override().ok(); + let args: Vec = std::env::args().skip(1).collect(); + let has = |flag: &str| args.iter().any(|a| a == flag); + let debug = has("--debug"); let url = std::env::var("WATERMETER_URL").context("WATERMETER_URL is not set")?; + + let expected_len = match std::env::var("WATERMETER_DIGITS") { + Ok(v) => Some( + v.trim() + .parse::() + .context("WATERMETER_DIGITS must be a positive integer")?, + ), + Err(_) => None, + }; + + for attempt in 1..=MAX_ATTEMPTS { + let retry = |why: &str| { + if attempt < MAX_ATTEMPTS { + runlog::note(format!( + "{why} (attempt {attempt}/{MAX_ATTEMPTS}), retrying in {}s", + RETRY_DELAY.as_secs() + )); + std::thread::sleep(RETRY_DELAY); + } + }; + + let bytes = fetch::fetch_image(&url)?; + let img = image::load_from_memory(&bytes) + .context("failed to decode image")? + .into_rgb8(); + if find_blue_boxes(&img).is_empty() { + retry("no blue box in picture"); + continue; + } + + let digits = read_digits(&bytes, &has, debug)?; + let reading: String = digits.iter().map(|d| d.as_deref().unwrap_or("?")).collect(); + if let Some(n) = expected_len + && reading.chars().count() != n + { + retry(&format!("read {reading:?}, expected {n} digits")); + continue; + } + + println!("{reading}"); + runlog::note(format!("reading {reading}")); + let complete = !digits.is_empty() && digits.iter().all(Option::is_some); + if complete { + update_home_assistant(&reading)?; + } + return Ok(complete); + } + anyhow::bail!("no valid reading after {MAX_ATTEMPTS} attempts") +} + +/// Push `reading * 100` to Home Assistant when it is close enough to the current index. +fn update_home_assistant(reading: &str) -> Result<()> { + let Some(ha) = homeassistant::HomeAssistant::from_env() else { + runlog::note("HA_URL / HA_TOKEN not set, Home Assistant not updated"); + return Ok(()); + }; + let value = reading.parse::().context("reading is not a number")? * 100.0; + runlog::note(match ha.sync_index(value)? { + homeassistant::Outcome::Updated { previous } => { + format!("{} updated: {previous} -> {value}", ha.entity()) + } + homeassistant::Outcome::Unchanged => format!("{} already at {value}", ha.entity()), + homeassistant::Outcome::Rejected { current } => format!( + "{value} is more than 1000 away from {} ({current}), not updated", + ha.entity() + ), + }); + Ok(()) +} + +fn read_digits( + bytes: &[u8], + has: &dyn Fn(&str) -> bool, + debug: bool, +) -> Result>> { + if has("--ocrs") { + return read_with_ocrs(bytes, debug); + } + let crop = vision::crop_to_boxes(bytes)?; + if debug { + std::fs::create_dir_all(DEBUG_DIR)?; + std::fs::write(format!("{DEBUG_DIR}/crop.png"), &crop)?; + } + let llm_digits = if has("--mistral") { + mistral::read_digits(&crop)? + } else if has("--ollama") { + ollama::read_digits(&crop)? + } else { + claude::read_digits(&crop)? + }; + Ok(llm_digits + .into_iter() + .map(|d| d.map(String::from)) + .collect()) +} + +fn read_with_ocrs(bytes: &[u8], debug: bool) -> Result>> { let models_dir = std::env::var_os("OCRS_MODELS_DIR") .map(PathBuf::from) .unwrap_or_else(|| PathBuf::from("models")); - - let img = fetch::fetch_image(&url)?.into_rgb8(); + let img = image::load_from_memory(bytes) + .context("failed to decode image")? + .into_rgb8(); let boxes: Vec = find_blue_boxes(&img) .iter() .filter_map(BBox::interior) @@ -42,17 +157,10 @@ fn run() -> Result { let reader = DigitReader::new(&models_dir)?; let digits = reader.read_boxes(&img, &boxes)?; - if debug { save_debug(&img, &boxes, &digits)?; } - - let reading: String = digits - .iter() - .map(|d| d.as_deref().unwrap_or("?")) - .collect(); - println!("{reading}"); - Ok(digits.iter().all(Option::is_some)) + Ok(digits) } fn save_debug(img: &RgbImage, boxes: &[BBox], digits: &[Option]) -> Result<()> { diff --git a/src/mistral.rs b/src/mistral.rs new file mode 100644 index 0000000..04ba7ab --- /dev/null +++ b/src/mistral.rs @@ -0,0 +1,54 @@ +use anyhow::{Context, Result, bail}; +use serde_json::{Value, json}; + +use crate::vision; + +const API_URL: &str = "https://api.mistral.ai/v1/chat/completions"; +const DEFAULT_MODEL: &str = "mistral-medium-latest"; + +/// Ask a Mistral vision model to read the digits inside the blue boxes of the picture. +/// Returns one entry per box, `None` for an unreadable digit. +pub fn read_digits(image: &[u8]) -> Result>> { + let api_key = std::env::var("MISTRAL_API_KEY").context("MISTRAL_API_KEY is not set")?; + let model = std::env::var("MISTRAL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); + let (media_type, data) = vision::encode_image(image)?; + + let body = json!({ + "model": model, + "temperature": 0, + "response_format": { + "type": "json_schema", + "json_schema": { "name": "meter_digits", "schema": vision::digits_schema(), "strict": true } + }, + "messages": [{ + "role": "user", + "content": [ + { "type": "image_url", "image_url": format!("data:{media_type};base64,{data}") }, + { "type": "text", "text": vision::PROMPT } + ] + }] + }); + + let response = reqwest::blocking::Client::new() + .post(API_URL) + .bearer_auth(api_key) + .timeout(std::time::Duration::from_secs(120)) + .json(&body) + .send() + .context("request to Mistral API failed")?; + let status = response.status(); + let response: Value = response.json().context("invalid JSON from Mistral API")?; + if !status.is_success() { + bail!("Mistral API returned {status}: {response}"); + } + + let choice = &response["choices"][0]; + match choice["finish_reason"].as_str() { + Some("stop") => {} + other => bail!("unexpected finish_reason {other:?}"), + } + let text = choice["message"]["content"] + .as_str() + .context("no text content in Mistral response")?; + vision::parse_digits(text) +} diff --git a/src/ocr.rs b/src/ocr.rs index 6d3b4b7..5cd71de 100644 --- a/src/ocr.rs +++ b/src/ocr.rs @@ -15,8 +15,12 @@ pub struct DigitReader { impl DigitReader { pub fn new(models_dir: &Path) -> Result { let rec_path = models_dir.join("text-recognition.onnx"); - let recognition_model = Model::load_file(&rec_path) - .with_context(|| format!("failed to load {} (run ./download-models.sh)", rec_path.display()))?; + let recognition_model = Model::load_file(&rec_path).with_context(|| { + format!( + "failed to load {} (run ./download-models.sh)", + rec_path.display() + ) + })?; let engine = OcrEngine::new(OcrEngineParams { recognition_model: Some(recognition_model), allowed_chars: Some("0123456789".into()), @@ -44,8 +48,13 @@ impl DigitReader { Ok(texts .into_iter() .map(|line| { - line.map(|l| l.to_string().chars().filter(char::is_ascii_digit).collect::()) - .filter(|s| !s.is_empty()) + line.map(|l| { + l.to_string() + .chars() + .filter(char::is_ascii_digit) + .collect::() + }) + .filter(|s| !s.is_empty()) }) .collect()) } diff --git a/src/ollama.rs b/src/ollama.rs new file mode 100644 index 0000000..9cc7509 --- /dev/null +++ b/src/ollama.rs @@ -0,0 +1,76 @@ +use anyhow::{Context, Result, bail}; +use serde_json::{Value, json}; + +use crate::vision; + +const DEFAULT_HOST: &str = "http://localhost:11434"; +const DEFAULT_MODEL: &str = "lfm3-vl"; + +/// Ask a local Ollama vision model to read the digits inside the blue boxes of the picture. +/// Returns one entry per box, `None` for an unreadable digit. +pub fn read_digits(image: &[u8]) -> Result>> { + let host = std::env::var("OLLAMA_HOST").unwrap_or_else(|_| DEFAULT_HOST.into()); + let model = std::env::var("OLLAMA_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); + let (_, data) = vision::encode_image(image)?; + + let body = json!({ + "model": model, + "stream": false, + "think": false, + "format": reading_schema(), + "options": { "temperature": 0 }, + "messages": [{ + "role": "user", + "content": vision::PROMPT, + "images": [data] + }] + }); + + let url = format!("{}/api/chat", host.trim_end_matches('/')); + let response = reqwest::blocking::Client::new() + .post(&url) + .timeout(std::time::Duration::from_secs(300)) + .json(&body) + .send() + .with_context(|| format!("request to Ollama at {url} failed"))?; + let status = response.status(); + let response: Value = response.json().context("invalid JSON from Ollama")?; + if !status.is_success() { + bail!("Ollama returned {status}: {}", response["error"]); + } + + match response["done_reason"].as_str() { + Some("stop") => {} + other => bail!("unexpected done_reason {other:?}"), + } + let text = response["message"]["content"] + .as_str() + .context("no content in Ollama response")?; + parse_reading(text) +} + +/// Small local models (lfm3-vl) often return an empty array under the shared +/// `digits` array schema, but reliably fill a single string, so ask for that. +fn reading_schema() -> Value { + json!({ + "type": "object", + "properties": { + "reading": { "type": "string", "pattern": "^[0-9?]{1,12}$" } + }, + "required": ["reading"] + }) +} + +fn parse_reading(text: &str) -> Result>> { + let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?; + let reading = parsed["reading"] + .as_str() + .context("missing `reading` in model answer")?; + if reading.is_empty() { + bail!("model returned an empty reading"); + } + Ok(reading + .chars() + .map(|c| Some(c).filter(char::is_ascii_digit)) + .collect()) +} diff --git a/src/runlog.rs b/src/runlog.rs new file mode 100644 index 0000000..fca2fb1 --- /dev/null +++ b/src/runlog.rs @@ -0,0 +1,42 @@ +//! One line per run appended to a log file, for unattended (cron) runs. + +use std::io::Write as _; +use std::sync::Mutex; + +use anyhow::{Context, Result}; + +const DEFAULT_LOG: &str = "watermeter.log"; + +static NOTES: Mutex> = Mutex::new(Vec::new()); + +/// Print a message on stderr and keep it for the run's log line. +pub fn note(msg: impl Into) { + let msg = msg.into(); + eprintln!("{msg}"); + if let Ok(mut notes) = NOTES.lock() { + notes.push(msg); + } +} + +/// Append ` [] | ` to `WATERMETER_LOG` +/// (default `watermeter.log` in the working directory). +pub fn write(status: &str) -> Result<()> { + let path = std::env::var("WATERMETER_LOG").unwrap_or_else(|_| DEFAULT_LOG.into()); + let args: Vec = std::env::args().skip(1).collect(); + let notes = NOTES.lock().map(|n| n.join(" | ")).unwrap_or_default(); + let line = format!( + "{} [{status}] {} | {notes}\n", + chrono::Local::now().format("%Y-%m-%d %H:%M:%S"), + if args.is_empty() { + "--claude".into() + } else { + args.join(" ") + }, + ); + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .and_then(|mut f| f.write_all(line.as_bytes())) + .with_context(|| format!("failed to write log {path}")) +} diff --git a/src/vision.rs b/src/vision.rs new file mode 100644 index 0000000..655472e --- /dev/null +++ b/src/vision.rs @@ -0,0 +1,116 @@ +//! Prompt and response format shared by the vision LLM readers. + +use anyhow::{Context, Result, bail}; +use base64::Engine as _; +use serde_json::{Value, json}; + +pub const PROMPT: &str = "This photo shows the digit rollers of a water meter. Blue rectangles are drawn over the \ +image around the digit rollers to read; they may touch or overlap each other. \ +The meter shows more digits than there are blue rectangles: digits that are not \ +inside a blue rectangle (for example leading zeros on the left) must NOT be reported. \ +First count the blue rectangles, then read the single digit inside each one, from left to right. \ +Other colored rectangles (red, etc.) must be ignored. \ +If a roller is caught between two digits, give the lower one (9 when between 9 and 0). \ +Use \"?\" only if a digit is truly unreadable."; + +/// JSON schema the model's answer must follow. +pub fn digits_schema() -> Value { + json!({ + "type": "object", + "properties": { + "rectangle_count": { + "type": "integer", + "description": "Number of blue rectangles in the picture" + }, + "digits": { + "type": "array", + "description": "One entry per blue rectangle, left to right", + "items": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "?"] + } + } + }, + "required": ["rectangle_count", "digits"], + "additionalProperties": false + }) +} + +/// Parse the model's JSON answer into one entry per box, `None` when unreadable. +pub fn parse_digits(text: &str) -> Result>> { + let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?; + let digits = parsed["digits"] + .as_array() + .context("missing `digits` in model answer")?; + if let Some(count) = parsed["rectangle_count"].as_u64() + && count as usize != digits.len() + { + bail!( + "model saw {count} rectangles but returned {} digits", + digits.len() + ); + } + Ok(digits + .iter() + .map(|d| { + d.as_str() + .and_then(|s| s.chars().next()) + .filter(char::is_ascii_digit) + }) + .collect()) +} + +/// MIME type and base64 payload of the image. +pub fn encode_image(image: &[u8]) -> Result<(&'static str, String)> { + let media_type = match image::guess_format(image)? { + image::ImageFormat::Png => "image/png", + image::ImageFormat::Jpeg => "image/jpeg", + image::ImageFormat::WebP => "image/webp", + image::ImageFormat::Gif => "image/gif", + other => bail!("unsupported image format {other:?}"), + }; + Ok(( + media_type, + base64::engine::general_purpose::STANDARD.encode(image), + )) +} + +/// Margin kept around the blue boxes when cropping, in source pixels. +const CROP_MARGIN: u32 = 8; +/// Upscale factor applied to the crop so the digits are large enough for the model. +const CROP_SCALE: u32 = 3; + +/// Crop the picture around the blue boxes and upscale it, re-encoded as PNG. +/// Falls back to the original picture when no blue box is found. +pub fn crop_to_boxes(image: &[u8]) -> Result> { + let img = image::load_from_memory(image) + .context("failed to decode image")? + .into_rgb8(); + let boxes = crate::boxes::find_blue_boxes(&img); + let Some(x0) = boxes.iter().map(|b| b.x).min() else { + return Ok(image.to_vec()); + }; + let y0 = boxes.iter().map(|b| b.y).min().unwrap_or(0); + let x1 = boxes.iter().map(|b| b.x + b.w).max().unwrap_or(img.width()); + let y1 = boxes + .iter() + .map(|b| b.y + b.h) + .max() + .unwrap_or(img.height()); + + let x0 = x0.saturating_sub(CROP_MARGIN); + let y0 = y0.saturating_sub(CROP_MARGIN); + let x1 = (x1 + CROP_MARGIN).min(img.width()); + let y1 = (y1 + CROP_MARGIN).min(img.height()); + let crop = image::imageops::crop_imm(&img, x0, y0, x1 - x0, y1 - y0).to_image(); + let crop = image::imageops::resize( + &crop, + crop.width() * CROP_SCALE, + crop.height() * CROP_SCALE, + image::imageops::FilterType::Lanczos3, + ); + + let mut png = Vec::new(); + crop.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)?; + Ok(png) +}