Read the meter with vision LLMs and push the index to Home Assistant
- Add Claude, Mistral and local Ollama (lfm3-vl) vision backends that read the digits inside the blue boxes of a cropped, upscaled picture - Retry every 30s when the picture has no blue box or the reading does not have WATERMETER_DIGITS digits - Set input_number.index_compteur_d_eau to reading * 100 when it is within 1000 of the current value - Append one line per run to watermeter.log for cron runs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
253206e6c5
commit
83208191f2
13 files changed
+713
-28
No files matched your search
@@ -2,3 +2,4 @@
|
||||
.env
|
||||
/models
|
||||
/debug
|
||||
watermeter.log
|
||||
Generated
+103
@@ -26,6 +26,15 @@ dependencies = [
|
||||
"equator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
@@ -240,6 +249,17 @@ dependencies = [
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"num-traits",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
version = "0.1.58"
|
||||
@@ -736,6 +756,30 @@ dependencies = [
|
||||
"windows-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.3.0"
|
||||
@@ -1622,6 +1666,8 @@ dependencies = [
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"rustls-platform-verifier",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
@@ -1949,6 +1995,19 @@ dependencies = [
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
@@ -2389,12 +2448,15 @@ name = "watermeter_watcher"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"chrono",
|
||||
"dotenvy",
|
||||
"image",
|
||||
"ocrs",
|
||||
"reqwest",
|
||||
"rten",
|
||||
"rten-imageproc",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2441,6 +2503,41 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
@@ -2685,6 +2782,12 @@ version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.3"
|
||||
|
||||
+4
-1
@@ -5,9 +5,12 @@ edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.104"
|
||||
base64 = "0.23.1"
|
||||
chrono = { version = "0.4.45", default-features = false, features = ["clock"] }
|
||||
dotenvy = "0.15.7"
|
||||
image = "0.25.10"
|
||||
ocrs = "0.13.1"
|
||||
reqwest = { version = "0.13.5", features = ["blocking"] }
|
||||
reqwest = { version = "0.13.5", features = ["blocking", "json"] }
|
||||
rten = "0.26.0"
|
||||
rten-imageproc = "0.26.0"
|
||||
serde_json = "1.0.151"
|
||||
+6
-1
@@ -70,7 +70,12 @@ pub fn find_blue_boxes(img: &RgbImage) -> Vec<BBox> {
|
||||
}
|
||||
}
|
||||
}
|
||||
let b = BBox { x: x0, y: y0, w: x1 - x0 + 1, h: y1 - y0 + 1 };
|
||||
let b = BBox {
|
||||
x: x0,
|
||||
y: y0,
|
||||
w: x1 - x0 + 1,
|
||||
h: y1 - y0 + 1,
|
||||
};
|
||||
if b.w >= MIN_SIDE && b.h >= MIN_SIDE {
|
||||
boxes.push(b);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::vision;
|
||||
|
||||
const API_URL: &str = "https://api.anthropic.com/v1/messages";
|
||||
const DEFAULT_MODEL: &str = "claude-opus-5-5";
|
||||
|
||||
/// Ask Claude to read the digits inside the blue boxes of the picture.
|
||||
/// Returns one entry per box, `None` for an unreadable digit.
|
||||
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||
let api_key = std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY is not set")?;
|
||||
let model = std::env::var("ANTHROPIC_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||
let (media_type, data) = vision::encode_image(image)?;
|
||||
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"max_tokens": 16000,
|
||||
"fallbacks": "default",
|
||||
"output_config": {
|
||||
"effort": "low",
|
||||
"format": { "type": "json_schema", "schema": vision::digits_schema() }
|
||||
},
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"source": { "type": "base64", "media_type": media_type, "data": data }
|
||||
},
|
||||
{ "type": "text", "text": vision::PROMPT }
|
||||
]
|
||||
}]
|
||||
});
|
||||
|
||||
let response = reqwest::blocking::Client::new()
|
||||
.post(API_URL)
|
||||
.header("x-api-key", api_key)
|
||||
.header("anthropic-version", "2023-06-01")
|
||||
.header("anthropic-beta", "server-side-fallback-2026-07-01")
|
||||
.timeout(std::time::Duration::from_secs(120))
|
||||
.json(&body)
|
||||
.send()
|
||||
.context("request to Claude API failed")?;
|
||||
let status = response.status();
|
||||
let response: Value = response.json().context("invalid JSON from Claude API")?;
|
||||
if !status.is_success() {
|
||||
bail!(
|
||||
"Claude API returned {status}: {}",
|
||||
response["error"]["message"]
|
||||
);
|
||||
}
|
||||
|
||||
match response["stop_reason"].as_str() {
|
||||
Some("end_turn") => {}
|
||||
other => bail!(
|
||||
"unexpected stop_reason {other:?}: {}",
|
||||
response["stop_details"]
|
||||
),
|
||||
}
|
||||
let text = response["content"]
|
||||
.as_array()
|
||||
.and_then(|blocks| blocks.iter().find(|b| b["type"] == "text"))
|
||||
.and_then(|b| b["text"].as_str())
|
||||
.context("no text block in Claude response")?;
|
||||
vision::parse_digits(text)
|
||||
}
|
||||
+3
-4
@@ -1,11 +1,10 @@
|
||||
use anyhow::{Context, Result};
|
||||
use image::DynamicImage;
|
||||
|
||||
/// Download the meter picture and decode it.
|
||||
pub fn fetch_image(url: &str) -> Result<DynamicImage> {
|
||||
/// Download the meter picture as raw bytes.
|
||||
pub fn fetch_image(url: &str) -> Result<Vec<u8>> {
|
||||
let bytes = reqwest::blocking::get(url)
|
||||
.and_then(|r| r.error_for_status())
|
||||
.with_context(|| format!("failed to download {url}"))?
|
||||
.bytes()?;
|
||||
image::load_from_memory(&bytes).context("failed to decode image")
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const DEFAULT_ENTITY: &str = "input_number.index_compteur_d_eau";
|
||||
/// Readings further than this from the current index are considered misreads.
|
||||
const MAX_DIFF: f64 = 1000.0;
|
||||
const TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub struct HomeAssistant {
|
||||
url: String,
|
||||
token: String,
|
||||
entity: String,
|
||||
client: reqwest::blocking::Client,
|
||||
}
|
||||
|
||||
pub enum Outcome {
|
||||
Updated { previous: f64 },
|
||||
Unchanged,
|
||||
Rejected { current: f64 },
|
||||
}
|
||||
|
||||
impl HomeAssistant {
|
||||
/// Configuration from `HA_URL` and `HA_TOKEN` (and optional `HA_ENTITY`),
|
||||
/// `None` when Home Assistant is not configured.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let url = std::env::var("HA_URL").ok()?;
|
||||
let token = std::env::var("HA_TOKEN").ok()?;
|
||||
Some(Self {
|
||||
url: url.trim_end_matches('/').to_owned(),
|
||||
token,
|
||||
entity: std::env::var("HA_ENTITY").unwrap_or_else(|_| DEFAULT_ENTITY.into()),
|
||||
client: reqwest::blocking::Client::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn entity(&self) -> &str {
|
||||
&self.entity
|
||||
}
|
||||
|
||||
/// Set the entity to `value` unless it is more than `MAX_DIFF` away from its current state.
|
||||
pub fn sync_index(&self, value: f64) -> Result<Outcome> {
|
||||
let current = self.current_value()?;
|
||||
if (value - current).abs() > MAX_DIFF {
|
||||
return Ok(Outcome::Rejected { current });
|
||||
}
|
||||
if value == current {
|
||||
return Ok(Outcome::Unchanged);
|
||||
}
|
||||
self.set_value(value)?;
|
||||
Ok(Outcome::Updated { previous: current })
|
||||
}
|
||||
|
||||
fn current_value(&self) -> Result<f64> {
|
||||
let url = format!("{}/api/states/{}", self.url, self.entity);
|
||||
let response = self
|
||||
.client
|
||||
.get(&url)
|
||||
.bearer_auth(&self.token)
|
||||
.timeout(TIMEOUT)
|
||||
.send()
|
||||
.with_context(|| format!("request to Home Assistant at {url} failed"))?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
bail!(
|
||||
"Home Assistant returned {status} for {}: {}",
|
||||
self.entity,
|
||||
response.text()?
|
||||
);
|
||||
}
|
||||
let state: Value = response
|
||||
.json()
|
||||
.context("invalid JSON from Home Assistant")?;
|
||||
let raw = state["state"]
|
||||
.as_str()
|
||||
.context("missing `state` in entity")?;
|
||||
raw.parse()
|
||||
.with_context(|| format!("{} has non-numeric state {raw:?}", self.entity))
|
||||
}
|
||||
|
||||
fn set_value(&self, value: f64) -> Result<()> {
|
||||
let url = format!("{}/api/services/input_number/set_value", self.url);
|
||||
let response = self
|
||||
.client
|
||||
.post(&url)
|
||||
.bearer_auth(&self.token)
|
||||
.timeout(TIMEOUT)
|
||||
.json(&json!({ "entity_id": self.entity, "value": value }))
|
||||
.send()
|
||||
.with_context(|| format!("request to Home Assistant at {url} failed"))?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
bail!(
|
||||
"Home Assistant refused to set {}: {status} {}",
|
||||
self.entity,
|
||||
response.text()?
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+126
-18
@@ -1,9 +1,16 @@
|
||||
mod boxes;
|
||||
mod claude;
|
||||
mod fetch;
|
||||
mod homeassistant;
|
||||
mod mistral;
|
||||
mod ocr;
|
||||
mod ollama;
|
||||
mod runlog;
|
||||
mod vision;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use image::{Rgb, RgbImage};
|
||||
@@ -12,28 +19,136 @@ use boxes::{BBox, find_blue_boxes};
|
||||
use ocr::DigitReader;
|
||||
|
||||
const DEBUG_DIR: &str = "debug";
|
||||
/// Wait between two downloads when the picture has no blue box or the reading has the wrong length.
|
||||
const RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
/// Give up after this many downloads without a valid reading.
|
||||
const MAX_ATTEMPTS: u32 = 10;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
match run() {
|
||||
Ok(true) => ExitCode::SUCCESS,
|
||||
Ok(false) => ExitCode::from(2),
|
||||
let (status, code) = match run() {
|
||||
Ok(true) => ("ok", ExitCode::SUCCESS),
|
||||
Ok(false) => ("incomplete", ExitCode::from(2)),
|
||||
Err(e) => {
|
||||
eprintln!("error: {e:#}");
|
||||
ExitCode::FAILURE
|
||||
runlog::note(format!("error: {e:#}"));
|
||||
("error", ExitCode::FAILURE)
|
||||
}
|
||||
};
|
||||
if let Err(e) = runlog::write(status) {
|
||||
eprintln!("error: {e:#}");
|
||||
}
|
||||
code
|
||||
}
|
||||
|
||||
/// Returns whether every box could be read.
|
||||
/// Returns whether every digit could be read.
|
||||
fn run() -> Result<bool> {
|
||||
dotenvy::dotenv().ok();
|
||||
let debug = std::env::args().skip(1).any(|a| a == "--debug");
|
||||
dotenvy::dotenv_override().ok();
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let has = |flag: &str| args.iter().any(|a| a == flag);
|
||||
let debug = has("--debug");
|
||||
let url = std::env::var("WATERMETER_URL").context("WATERMETER_URL is not set")?;
|
||||
|
||||
let expected_len = match std::env::var("WATERMETER_DIGITS") {
|
||||
Ok(v) => Some(
|
||||
v.trim()
|
||||
.parse::<usize>()
|
||||
.context("WATERMETER_DIGITS must be a positive integer")?,
|
||||
),
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
for attempt in 1..=MAX_ATTEMPTS {
|
||||
let retry = |why: &str| {
|
||||
if attempt < MAX_ATTEMPTS {
|
||||
runlog::note(format!(
|
||||
"{why} (attempt {attempt}/{MAX_ATTEMPTS}), retrying in {}s",
|
||||
RETRY_DELAY.as_secs()
|
||||
));
|
||||
std::thread::sleep(RETRY_DELAY);
|
||||
}
|
||||
};
|
||||
|
||||
let bytes = fetch::fetch_image(&url)?;
|
||||
let img = image::load_from_memory(&bytes)
|
||||
.context("failed to decode image")?
|
||||
.into_rgb8();
|
||||
if find_blue_boxes(&img).is_empty() {
|
||||
retry("no blue box in picture");
|
||||
continue;
|
||||
}
|
||||
|
||||
let digits = read_digits(&bytes, &has, debug)?;
|
||||
let reading: String = digits.iter().map(|d| d.as_deref().unwrap_or("?")).collect();
|
||||
if let Some(n) = expected_len
|
||||
&& reading.chars().count() != n
|
||||
{
|
||||
retry(&format!("read {reading:?}, expected {n} digits"));
|
||||
continue;
|
||||
}
|
||||
|
||||
println!("{reading}");
|
||||
runlog::note(format!("reading {reading}"));
|
||||
let complete = !digits.is_empty() && digits.iter().all(Option::is_some);
|
||||
if complete {
|
||||
update_home_assistant(&reading)?;
|
||||
}
|
||||
return Ok(complete);
|
||||
}
|
||||
anyhow::bail!("no valid reading after {MAX_ATTEMPTS} attempts")
|
||||
}
|
||||
|
||||
/// Push `reading * 100` to Home Assistant when it is close enough to the current index.
|
||||
fn update_home_assistant(reading: &str) -> Result<()> {
|
||||
let Some(ha) = homeassistant::HomeAssistant::from_env() else {
|
||||
runlog::note("HA_URL / HA_TOKEN not set, Home Assistant not updated");
|
||||
return Ok(());
|
||||
};
|
||||
let value = reading.parse::<f64>().context("reading is not a number")? * 100.0;
|
||||
runlog::note(match ha.sync_index(value)? {
|
||||
homeassistant::Outcome::Updated { previous } => {
|
||||
format!("{} updated: {previous} -> {value}", ha.entity())
|
||||
}
|
||||
homeassistant::Outcome::Unchanged => format!("{} already at {value}", ha.entity()),
|
||||
homeassistant::Outcome::Rejected { current } => format!(
|
||||
"{value} is more than 1000 away from {} ({current}), not updated",
|
||||
ha.entity()
|
||||
),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_digits(
|
||||
bytes: &[u8],
|
||||
has: &dyn Fn(&str) -> bool,
|
||||
debug: bool,
|
||||
) -> Result<Vec<Option<String>>> {
|
||||
if has("--ocrs") {
|
||||
return read_with_ocrs(bytes, debug);
|
||||
}
|
||||
let crop = vision::crop_to_boxes(bytes)?;
|
||||
if debug {
|
||||
std::fs::create_dir_all(DEBUG_DIR)?;
|
||||
std::fs::write(format!("{DEBUG_DIR}/crop.png"), &crop)?;
|
||||
}
|
||||
let llm_digits = if has("--mistral") {
|
||||
mistral::read_digits(&crop)?
|
||||
} else if has("--ollama") {
|
||||
ollama::read_digits(&crop)?
|
||||
} else {
|
||||
claude::read_digits(&crop)?
|
||||
};
|
||||
Ok(llm_digits
|
||||
.into_iter()
|
||||
.map(|d| d.map(String::from))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn read_with_ocrs(bytes: &[u8], debug: bool) -> Result<Vec<Option<String>>> {
|
||||
let models_dir = std::env::var_os("OCRS_MODELS_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("models"));
|
||||
|
||||
let img = fetch::fetch_image(&url)?.into_rgb8();
|
||||
let img = image::load_from_memory(bytes)
|
||||
.context("failed to decode image")?
|
||||
.into_rgb8();
|
||||
let boxes: Vec<BBox> = find_blue_boxes(&img)
|
||||
.iter()
|
||||
.filter_map(BBox::interior)
|
||||
@@ -42,17 +157,10 @@ fn run() -> Result<bool> {
|
||||
|
||||
let reader = DigitReader::new(&models_dir)?;
|
||||
let digits = reader.read_boxes(&img, &boxes)?;
|
||||
|
||||
if debug {
|
||||
save_debug(&img, &boxes, &digits)?;
|
||||
}
|
||||
|
||||
let reading: String = digits
|
||||
.iter()
|
||||
.map(|d| d.as_deref().unwrap_or("?"))
|
||||
.collect();
|
||||
println!("{reading}");
|
||||
Ok(digits.iter().all(Option::is_some))
|
||||
Ok(digits)
|
||||
}
|
||||
|
||||
fn save_debug(img: &RgbImage, boxes: &[BBox], digits: &[Option<String>]) -> Result<()> {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::vision;
|
||||
|
||||
const API_URL: &str = "https://api.mistral.ai/v1/chat/completions";
|
||||
const DEFAULT_MODEL: &str = "mistral-medium-latest";
|
||||
|
||||
/// Ask a Mistral vision model to read the digits inside the blue boxes of the picture.
|
||||
/// Returns one entry per box, `None` for an unreadable digit.
|
||||
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||
let api_key = std::env::var("MISTRAL_API_KEY").context("MISTRAL_API_KEY is not set")?;
|
||||
let model = std::env::var("MISTRAL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||
let (media_type, data) = vision::encode_image(image)?;
|
||||
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"temperature": 0,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": { "name": "meter_digits", "schema": vision::digits_schema(), "strict": true }
|
||||
},
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "image_url", "image_url": format!("data:{media_type};base64,{data}") },
|
||||
{ "type": "text", "text": vision::PROMPT }
|
||||
]
|
||||
}]
|
||||
});
|
||||
|
||||
let response = reqwest::blocking::Client::new()
|
||||
.post(API_URL)
|
||||
.bearer_auth(api_key)
|
||||
.timeout(std::time::Duration::from_secs(120))
|
||||
.json(&body)
|
||||
.send()
|
||||
.context("request to Mistral API failed")?;
|
||||
let status = response.status();
|
||||
let response: Value = response.json().context("invalid JSON from Mistral API")?;
|
||||
if !status.is_success() {
|
||||
bail!("Mistral API returned {status}: {response}");
|
||||
}
|
||||
|
||||
let choice = &response["choices"][0];
|
||||
match choice["finish_reason"].as_str() {
|
||||
Some("stop") => {}
|
||||
other => bail!("unexpected finish_reason {other:?}"),
|
||||
}
|
||||
let text = choice["message"]["content"]
|
||||
.as_str()
|
||||
.context("no text content in Mistral response")?;
|
||||
vision::parse_digits(text)
|
||||
}
|
||||
+13
-4
@@ -15,8 +15,12 @@ pub struct DigitReader {
|
||||
impl DigitReader {
|
||||
pub fn new(models_dir: &Path) -> Result<Self> {
|
||||
let rec_path = models_dir.join("text-recognition.onnx");
|
||||
let recognition_model = Model::load_file(&rec_path)
|
||||
.with_context(|| format!("failed to load {} (run ./download-models.sh)", rec_path.display()))?;
|
||||
let recognition_model = Model::load_file(&rec_path).with_context(|| {
|
||||
format!(
|
||||
"failed to load {} (run ./download-models.sh)",
|
||||
rec_path.display()
|
||||
)
|
||||
})?;
|
||||
let engine = OcrEngine::new(OcrEngineParams {
|
||||
recognition_model: Some(recognition_model),
|
||||
allowed_chars: Some("0123456789".into()),
|
||||
@@ -44,8 +48,13 @@ impl DigitReader {
|
||||
Ok(texts
|
||||
.into_iter()
|
||||
.map(|line| {
|
||||
line.map(|l| l.to_string().chars().filter(char::is_ascii_digit).collect::<String>())
|
||||
.filter(|s| !s.is_empty())
|
||||
line.map(|l| {
|
||||
l.to_string()
|
||||
.chars()
|
||||
.filter(char::is_ascii_digit)
|
||||
.collect::<String>()
|
||||
})
|
||||
.filter(|s| !s.is_empty())
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::vision;
|
||||
|
||||
const DEFAULT_HOST: &str = "http://localhost:11434";
|
||||
const DEFAULT_MODEL: &str = "lfm3-vl";
|
||||
|
||||
/// Ask a local Ollama vision model to read the digits inside the blue boxes of the picture.
|
||||
/// Returns one entry per box, `None` for an unreadable digit.
|
||||
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||
let host = std::env::var("OLLAMA_HOST").unwrap_or_else(|_| DEFAULT_HOST.into());
|
||||
let model = std::env::var("OLLAMA_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||
let (_, data) = vision::encode_image(image)?;
|
||||
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"stream": false,
|
||||
"think": false,
|
||||
"format": reading_schema(),
|
||||
"options": { "temperature": 0 },
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": vision::PROMPT,
|
||||
"images": [data]
|
||||
}]
|
||||
});
|
||||
|
||||
let url = format!("{}/api/chat", host.trim_end_matches('/'));
|
||||
let response = reqwest::blocking::Client::new()
|
||||
.post(&url)
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.json(&body)
|
||||
.send()
|
||||
.with_context(|| format!("request to Ollama at {url} failed"))?;
|
||||
let status = response.status();
|
||||
let response: Value = response.json().context("invalid JSON from Ollama")?;
|
||||
if !status.is_success() {
|
||||
bail!("Ollama returned {status}: {}", response["error"]);
|
||||
}
|
||||
|
||||
match response["done_reason"].as_str() {
|
||||
Some("stop") => {}
|
||||
other => bail!("unexpected done_reason {other:?}"),
|
||||
}
|
||||
let text = response["message"]["content"]
|
||||
.as_str()
|
||||
.context("no content in Ollama response")?;
|
||||
parse_reading(text)
|
||||
}
|
||||
|
||||
/// Small local models (lfm3-vl) often return an empty array under the shared
|
||||
/// `digits` array schema, but reliably fill a single string, so ask for that.
|
||||
fn reading_schema() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"reading": { "type": "string", "pattern": "^[0-9?]{1,12}$" }
|
||||
},
|
||||
"required": ["reading"]
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_reading(text: &str) -> Result<Vec<Option<char>>> {
|
||||
let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?;
|
||||
let reading = parsed["reading"]
|
||||
.as_str()
|
||||
.context("missing `reading` in model answer")?;
|
||||
if reading.is_empty() {
|
||||
bail!("model returned an empty reading");
|
||||
}
|
||||
Ok(reading
|
||||
.chars()
|
||||
.map(|c| Some(c).filter(char::is_ascii_digit))
|
||||
.collect())
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//! One line per run appended to a log file, for unattended (cron) runs.
|
||||
|
||||
use std::io::Write as _;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
const DEFAULT_LOG: &str = "watermeter.log";
|
||||
|
||||
static NOTES: Mutex<Vec<String>> = Mutex::new(Vec::new());
|
||||
|
||||
/// Print a message on stderr and keep it for the run's log line.
|
||||
pub fn note(msg: impl Into<String>) {
|
||||
let msg = msg.into();
|
||||
eprintln!("{msg}");
|
||||
if let Ok(mut notes) = NOTES.lock() {
|
||||
notes.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
/// Append `<timestamp> [<status>] <args> | <notes...>` to `WATERMETER_LOG`
|
||||
/// (default `watermeter.log` in the working directory).
|
||||
pub fn write(status: &str) -> Result<()> {
|
||||
let path = std::env::var("WATERMETER_LOG").unwrap_or_else(|_| DEFAULT_LOG.into());
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let notes = NOTES.lock().map(|n| n.join(" | ")).unwrap_or_default();
|
||||
let line = format!(
|
||||
"{} [{status}] {} | {notes}\n",
|
||||
chrono::Local::now().format("%Y-%m-%d %H:%M:%S"),
|
||||
if args.is_empty() {
|
||||
"--claude".into()
|
||||
} else {
|
||||
args.join(" ")
|
||||
},
|
||||
);
|
||||
std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.and_then(|mut f| f.write_all(line.as_bytes()))
|
||||
.with_context(|| format!("failed to write log {path}"))
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
//! Prompt and response format shared by the vision LLM readers.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use base64::Engine as _;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub const PROMPT: &str = "This photo shows the digit rollers of a water meter. Blue rectangles are drawn over the \
|
||||
image around the digit rollers to read; they may touch or overlap each other. \
|
||||
The meter shows more digits than there are blue rectangles: digits that are not \
|
||||
inside a blue rectangle (for example leading zeros on the left) must NOT be reported. \
|
||||
First count the blue rectangles, then read the single digit inside each one, from left to right. \
|
||||
Other colored rectangles (red, etc.) must be ignored. \
|
||||
If a roller is caught between two digits, give the lower one (9 when between 9 and 0). \
|
||||
Use \"?\" only if a digit is truly unreadable.";
|
||||
|
||||
/// JSON schema the model's answer must follow.
|
||||
pub fn digits_schema() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"rectangle_count": {
|
||||
"type": "integer",
|
||||
"description": "Number of blue rectangles in the picture"
|
||||
},
|
||||
"digits": {
|
||||
"type": "array",
|
||||
"description": "One entry per blue rectangle, left to right",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "?"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["rectangle_count", "digits"],
|
||||
"additionalProperties": false
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the model's JSON answer into one entry per box, `None` when unreadable.
|
||||
pub fn parse_digits(text: &str) -> Result<Vec<Option<char>>> {
|
||||
let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?;
|
||||
let digits = parsed["digits"]
|
||||
.as_array()
|
||||
.context("missing `digits` in model answer")?;
|
||||
if let Some(count) = parsed["rectangle_count"].as_u64()
|
||||
&& count as usize != digits.len()
|
||||
{
|
||||
bail!(
|
||||
"model saw {count} rectangles but returned {} digits",
|
||||
digits.len()
|
||||
);
|
||||
}
|
||||
Ok(digits
|
||||
.iter()
|
||||
.map(|d| {
|
||||
d.as_str()
|
||||
.and_then(|s| s.chars().next())
|
||||
.filter(char::is_ascii_digit)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// MIME type and base64 payload of the image.
|
||||
pub fn encode_image(image: &[u8]) -> Result<(&'static str, String)> {
|
||||
let media_type = match image::guess_format(image)? {
|
||||
image::ImageFormat::Png => "image/png",
|
||||
image::ImageFormat::Jpeg => "image/jpeg",
|
||||
image::ImageFormat::WebP => "image/webp",
|
||||
image::ImageFormat::Gif => "image/gif",
|
||||
other => bail!("unsupported image format {other:?}"),
|
||||
};
|
||||
Ok((
|
||||
media_type,
|
||||
base64::engine::general_purpose::STANDARD.encode(image),
|
||||
))
|
||||
}
|
||||
|
||||
/// Margin kept around the blue boxes when cropping, in source pixels.
|
||||
const CROP_MARGIN: u32 = 8;
|
||||
/// Upscale factor applied to the crop so the digits are large enough for the model.
|
||||
const CROP_SCALE: u32 = 3;
|
||||
|
||||
/// Crop the picture around the blue boxes and upscale it, re-encoded as PNG.
|
||||
/// Falls back to the original picture when no blue box is found.
|
||||
pub fn crop_to_boxes(image: &[u8]) -> Result<Vec<u8>> {
|
||||
let img = image::load_from_memory(image)
|
||||
.context("failed to decode image")?
|
||||
.into_rgb8();
|
||||
let boxes = crate::boxes::find_blue_boxes(&img);
|
||||
let Some(x0) = boxes.iter().map(|b| b.x).min() else {
|
||||
return Ok(image.to_vec());
|
||||
};
|
||||
let y0 = boxes.iter().map(|b| b.y).min().unwrap_or(0);
|
||||
let x1 = boxes.iter().map(|b| b.x + b.w).max().unwrap_or(img.width());
|
||||
let y1 = boxes
|
||||
.iter()
|
||||
.map(|b| b.y + b.h)
|
||||
.max()
|
||||
.unwrap_or(img.height());
|
||||
|
||||
let x0 = x0.saturating_sub(CROP_MARGIN);
|
||||
let y0 = y0.saturating_sub(CROP_MARGIN);
|
||||
let x1 = (x1 + CROP_MARGIN).min(img.width());
|
||||
let y1 = (y1 + CROP_MARGIN).min(img.height());
|
||||
let crop = image::imageops::crop_imm(&img, x0, y0, x1 - x0, y1 - y0).to_image();
|
||||
let crop = image::imageops::resize(
|
||||
&crop,
|
||||
crop.width() * CROP_SCALE,
|
||||
crop.height() * CROP_SCALE,
|
||||
image::imageops::FilterType::Lanczos3,
|
||||
);
|
||||
|
||||
let mut png = Vec::new();
|
||||
crop.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)?;
|
||||
Ok(png)
|
||||
}
|
||||
Reference in new issue
Block a user