Read the meter with vision LLMs and push the index to Home Assistant
- Add Claude, Mistral and local Ollama (lfm3-vl) vision backends that read the digits inside the blue boxes of a cropped, upscaled picture - Retry every 30s when the picture has no blue box or the reading does not have WATERMETER_DIGITS digits - Set input_number.index_compteur_d_eau to reading * 100 when it is within 1000 of the current value - Append one line per run to watermeter.log for cron runs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
253206e6c5
commit
83208191f2
13 files changed
+713
-28
No files matched your search
@@ -2,3 +2,4 @@
|
|||||||
.env
|
.env
|
||||||
/models
|
/models
|
||||||
/debug
|
/debug
|
||||||
|
watermeter.log
|
||||||
Generated
+103
@@ -26,6 +26,15 @@ dependencies = [
|
|||||||
"equator",
|
"equator",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android_system_properties"
|
||||||
|
version = "0.1.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anyhow"
|
name = "anyhow"
|
||||||
version = "1.0.104"
|
version = "1.0.104"
|
||||||
@@ -240,6 +249,17 @@ dependencies = [
|
|||||||
"rand_core 0.10.1",
|
"rand_core 0.10.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chrono"
|
||||||
|
version = "0.4.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||||
|
dependencies = [
|
||||||
|
"iana-time-zone",
|
||||||
|
"num-traits",
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cmake"
|
name = "cmake"
|
||||||
version = "0.1.58"
|
version = "0.1.58"
|
||||||
@@ -736,6 +756,30 @@ dependencies = [
|
|||||||
"windows-registry",
|
"windows-registry",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone"
|
||||||
|
version = "0.1.65"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||||
|
dependencies = [
|
||||||
|
"android_system_properties",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"iana-time-zone-haiku",
|
||||||
|
"js-sys",
|
||||||
|
"log",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone-haiku"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_collections"
|
name = "icu_collections"
|
||||||
version = "2.3.0"
|
version = "2.3.0"
|
||||||
@@ -1622,6 +1666,8 @@ dependencies = [
|
|||||||
"rustls",
|
"rustls",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"rustls-platform-verifier",
|
"rustls-platform-verifier",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
"sync_wrapper",
|
"sync_wrapper",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
@@ -1949,6 +1995,19 @@ dependencies = [
|
|||||||
"syn 3.0.6",
|
"syn 3.0.6",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.151"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "shlex"
|
name = "shlex"
|
||||||
version = "2.0.1"
|
version = "2.0.1"
|
||||||
@@ -2389,12 +2448,15 @@ name = "watermeter_watcher"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"base64",
|
||||||
|
"chrono",
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"image",
|
"image",
|
||||||
"ocrs",
|
"ocrs",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"rten",
|
"rten",
|
||||||
"rten-imageproc",
|
"rten-imageproc",
|
||||||
|
"serde_json",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2441,6 +2503,41 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-core"
|
||||||
|
version = "0.62.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||||
|
dependencies = [
|
||||||
|
"windows-implement",
|
||||||
|
"windows-interface",
|
||||||
|
"windows-link",
|
||||||
|
"windows-result",
|
||||||
|
"windows-strings",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-implement"
|
||||||
|
version = "0.60.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-interface"
|
||||||
|
version = "0.59.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-link"
|
name = "windows-link"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
@@ -2685,6 +2782,12 @@ version = "0.6.8"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zune-core"
|
name = "zune-core"
|
||||||
version = "0.5.3"
|
version = "0.5.3"
|
||||||
|
|||||||
+4
-1
@@ -5,9 +5,12 @@ edition = "2024"
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow = "1.0.104"
|
anyhow = "1.0.104"
|
||||||
|
base64 = "0.23.1"
|
||||||
|
chrono = { version = "0.4.45", default-features = false, features = ["clock"] }
|
||||||
dotenvy = "0.15.7"
|
dotenvy = "0.15.7"
|
||||||
image = "0.25.10"
|
image = "0.25.10"
|
||||||
ocrs = "0.13.1"
|
ocrs = "0.13.1"
|
||||||
reqwest = { version = "0.13.5", features = ["blocking"] }
|
reqwest = { version = "0.13.5", features = ["blocking", "json"] }
|
||||||
rten = "0.26.0"
|
rten = "0.26.0"
|
||||||
rten-imageproc = "0.26.0"
|
rten-imageproc = "0.26.0"
|
||||||
|
serde_json = "1.0.151"
|
||||||
+6
-1
@@ -70,7 +70,12 @@ pub fn find_blue_boxes(img: &RgbImage) -> Vec<BBox> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let b = BBox { x: x0, y: y0, w: x1 - x0 + 1, h: y1 - y0 + 1 };
|
let b = BBox {
|
||||||
|
x: x0,
|
||||||
|
y: y0,
|
||||||
|
w: x1 - x0 + 1,
|
||||||
|
h: y1 - y0 + 1,
|
||||||
|
};
|
||||||
if b.w >= MIN_SIDE && b.h >= MIN_SIDE {
|
if b.w >= MIN_SIDE && b.h >= MIN_SIDE {
|
||||||
boxes.push(b);
|
boxes.push(b);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::vision;
|
||||||
|
|
||||||
|
const API_URL: &str = "https://api.anthropic.com/v1/messages";
|
||||||
|
const DEFAULT_MODEL: &str = "claude-opus-5-5";
|
||||||
|
|
||||||
|
/// Ask Claude to read the digits inside the blue boxes of the picture.
|
||||||
|
/// Returns one entry per box, `None` for an unreadable digit.
|
||||||
|
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||||
|
let api_key = std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY is not set")?;
|
||||||
|
let model = std::env::var("ANTHROPIC_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||||
|
let (media_type, data) = vision::encode_image(image)?;
|
||||||
|
|
||||||
|
let body = json!({
|
||||||
|
"model": model,
|
||||||
|
"max_tokens": 16000,
|
||||||
|
"fallbacks": "default",
|
||||||
|
"output_config": {
|
||||||
|
"effort": "low",
|
||||||
|
"format": { "type": "json_schema", "schema": vision::digits_schema() }
|
||||||
|
},
|
||||||
|
"messages": [{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"source": { "type": "base64", "media_type": media_type, "data": data }
|
||||||
|
},
|
||||||
|
{ "type": "text", "text": vision::PROMPT }
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
|
||||||
|
let response = reqwest::blocking::Client::new()
|
||||||
|
.post(API_URL)
|
||||||
|
.header("x-api-key", api_key)
|
||||||
|
.header("anthropic-version", "2023-06-01")
|
||||||
|
.header("anthropic-beta", "server-side-fallback-2026-07-01")
|
||||||
|
.timeout(std::time::Duration::from_secs(120))
|
||||||
|
.json(&body)
|
||||||
|
.send()
|
||||||
|
.context("request to Claude API failed")?;
|
||||||
|
let status = response.status();
|
||||||
|
let response: Value = response.json().context("invalid JSON from Claude API")?;
|
||||||
|
if !status.is_success() {
|
||||||
|
bail!(
|
||||||
|
"Claude API returned {status}: {}",
|
||||||
|
response["error"]["message"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
match response["stop_reason"].as_str() {
|
||||||
|
Some("end_turn") => {}
|
||||||
|
other => bail!(
|
||||||
|
"unexpected stop_reason {other:?}: {}",
|
||||||
|
response["stop_details"]
|
||||||
|
),
|
||||||
|
}
|
||||||
|
let text = response["content"]
|
||||||
|
.as_array()
|
||||||
|
.and_then(|blocks| blocks.iter().find(|b| b["type"] == "text"))
|
||||||
|
.and_then(|b| b["text"].as_str())
|
||||||
|
.context("no text block in Claude response")?;
|
||||||
|
vision::parse_digits(text)
|
||||||
|
}
|
||||||
+3
-4
@@ -1,11 +1,10 @@
|
|||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use image::DynamicImage;
|
|
||||||
|
|
||||||
/// Download the meter picture and decode it.
|
/// Download the meter picture as raw bytes.
|
||||||
pub fn fetch_image(url: &str) -> Result<DynamicImage> {
|
pub fn fetch_image(url: &str) -> Result<Vec<u8>> {
|
||||||
let bytes = reqwest::blocking::get(url)
|
let bytes = reqwest::blocking::get(url)
|
||||||
.and_then(|r| r.error_for_status())
|
.and_then(|r| r.error_for_status())
|
||||||
.with_context(|| format!("failed to download {url}"))?
|
.with_context(|| format!("failed to download {url}"))?
|
||||||
.bytes()?;
|
.bytes()?;
|
||||||
image::load_from_memory(&bytes).context("failed to decode image")
|
Ok(bytes.to_vec())
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
const DEFAULT_ENTITY: &str = "input_number.index_compteur_d_eau";
|
||||||
|
/// Readings further than this from the current index are considered misreads.
|
||||||
|
const MAX_DIFF: f64 = 1000.0;
|
||||||
|
const TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
pub struct HomeAssistant {
|
||||||
|
url: String,
|
||||||
|
token: String,
|
||||||
|
entity: String,
|
||||||
|
client: reqwest::blocking::Client,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum Outcome {
|
||||||
|
Updated { previous: f64 },
|
||||||
|
Unchanged,
|
||||||
|
Rejected { current: f64 },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HomeAssistant {
|
||||||
|
/// Configuration from `HA_URL` and `HA_TOKEN` (and optional `HA_ENTITY`),
|
||||||
|
/// `None` when Home Assistant is not configured.
|
||||||
|
pub fn from_env() -> Option<Self> {
|
||||||
|
let url = std::env::var("HA_URL").ok()?;
|
||||||
|
let token = std::env::var("HA_TOKEN").ok()?;
|
||||||
|
Some(Self {
|
||||||
|
url: url.trim_end_matches('/').to_owned(),
|
||||||
|
token,
|
||||||
|
entity: std::env::var("HA_ENTITY").unwrap_or_else(|_| DEFAULT_ENTITY.into()),
|
||||||
|
client: reqwest::blocking::Client::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn entity(&self) -> &str {
|
||||||
|
&self.entity
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the entity to `value` unless it is more than `MAX_DIFF` away from its current state.
|
||||||
|
pub fn sync_index(&self, value: f64) -> Result<Outcome> {
|
||||||
|
let current = self.current_value()?;
|
||||||
|
if (value - current).abs() > MAX_DIFF {
|
||||||
|
return Ok(Outcome::Rejected { current });
|
||||||
|
}
|
||||||
|
if value == current {
|
||||||
|
return Ok(Outcome::Unchanged);
|
||||||
|
}
|
||||||
|
self.set_value(value)?;
|
||||||
|
Ok(Outcome::Updated { previous: current })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_value(&self) -> Result<f64> {
|
||||||
|
let url = format!("{}/api/states/{}", self.url, self.entity);
|
||||||
|
let response = self
|
||||||
|
.client
|
||||||
|
.get(&url)
|
||||||
|
.bearer_auth(&self.token)
|
||||||
|
.timeout(TIMEOUT)
|
||||||
|
.send()
|
||||||
|
.with_context(|| format!("request to Home Assistant at {url} failed"))?;
|
||||||
|
let status = response.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
bail!(
|
||||||
|
"Home Assistant returned {status} for {}: {}",
|
||||||
|
self.entity,
|
||||||
|
response.text()?
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let state: Value = response
|
||||||
|
.json()
|
||||||
|
.context("invalid JSON from Home Assistant")?;
|
||||||
|
let raw = state["state"]
|
||||||
|
.as_str()
|
||||||
|
.context("missing `state` in entity")?;
|
||||||
|
raw.parse()
|
||||||
|
.with_context(|| format!("{} has non-numeric state {raw:?}", self.entity))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_value(&self, value: f64) -> Result<()> {
|
||||||
|
let url = format!("{}/api/services/input_number/set_value", self.url);
|
||||||
|
let response = self
|
||||||
|
.client
|
||||||
|
.post(&url)
|
||||||
|
.bearer_auth(&self.token)
|
||||||
|
.timeout(TIMEOUT)
|
||||||
|
.json(&json!({ "entity_id": self.entity, "value": value }))
|
||||||
|
.send()
|
||||||
|
.with_context(|| format!("request to Home Assistant at {url} failed"))?;
|
||||||
|
let status = response.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
bail!(
|
||||||
|
"Home Assistant refused to set {}: {status} {}",
|
||||||
|
self.entity,
|
||||||
|
response.text()?
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
+126
-18
@@ -1,9 +1,16 @@
|
|||||||
mod boxes;
|
mod boxes;
|
||||||
|
mod claude;
|
||||||
mod fetch;
|
mod fetch;
|
||||||
|
mod homeassistant;
|
||||||
|
mod mistral;
|
||||||
mod ocr;
|
mod ocr;
|
||||||
|
mod ollama;
|
||||||
|
mod runlog;
|
||||||
|
mod vision;
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use image::{Rgb, RgbImage};
|
use image::{Rgb, RgbImage};
|
||||||
@@ -12,28 +19,136 @@ use boxes::{BBox, find_blue_boxes};
|
|||||||
use ocr::DigitReader;
|
use ocr::DigitReader;
|
||||||
|
|
||||||
const DEBUG_DIR: &str = "debug";
|
const DEBUG_DIR: &str = "debug";
|
||||||
|
/// Wait between two downloads when the picture has no blue box or the reading has the wrong length.
|
||||||
|
const RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||||
|
/// Give up after this many downloads without a valid reading.
|
||||||
|
const MAX_ATTEMPTS: u32 = 10;
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
match run() {
|
let (status, code) = match run() {
|
||||||
Ok(true) => ExitCode::SUCCESS,
|
Ok(true) => ("ok", ExitCode::SUCCESS),
|
||||||
Ok(false) => ExitCode::from(2),
|
Ok(false) => ("incomplete", ExitCode::from(2)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("error: {e:#}");
|
runlog::note(format!("error: {e:#}"));
|
||||||
ExitCode::FAILURE
|
("error", ExitCode::FAILURE)
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
if let Err(e) = runlog::write(status) {
|
||||||
|
eprintln!("error: {e:#}");
|
||||||
}
|
}
|
||||||
|
code
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns whether every box could be read.
|
/// Returns whether every digit could be read.
|
||||||
fn run() -> Result<bool> {
|
fn run() -> Result<bool> {
|
||||||
dotenvy::dotenv().ok();
|
dotenvy::dotenv_override().ok();
|
||||||
let debug = std::env::args().skip(1).any(|a| a == "--debug");
|
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||||
|
let has = |flag: &str| args.iter().any(|a| a == flag);
|
||||||
|
let debug = has("--debug");
|
||||||
let url = std::env::var("WATERMETER_URL").context("WATERMETER_URL is not set")?;
|
let url = std::env::var("WATERMETER_URL").context("WATERMETER_URL is not set")?;
|
||||||
|
|
||||||
|
let expected_len = match std::env::var("WATERMETER_DIGITS") {
|
||||||
|
Ok(v) => Some(
|
||||||
|
v.trim()
|
||||||
|
.parse::<usize>()
|
||||||
|
.context("WATERMETER_DIGITS must be a positive integer")?,
|
||||||
|
),
|
||||||
|
Err(_) => None,
|
||||||
|
};
|
||||||
|
|
||||||
|
for attempt in 1..=MAX_ATTEMPTS {
|
||||||
|
let retry = |why: &str| {
|
||||||
|
if attempt < MAX_ATTEMPTS {
|
||||||
|
runlog::note(format!(
|
||||||
|
"{why} (attempt {attempt}/{MAX_ATTEMPTS}), retrying in {}s",
|
||||||
|
RETRY_DELAY.as_secs()
|
||||||
|
));
|
||||||
|
std::thread::sleep(RETRY_DELAY);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let bytes = fetch::fetch_image(&url)?;
|
||||||
|
let img = image::load_from_memory(&bytes)
|
||||||
|
.context("failed to decode image")?
|
||||||
|
.into_rgb8();
|
||||||
|
if find_blue_boxes(&img).is_empty() {
|
||||||
|
retry("no blue box in picture");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let digits = read_digits(&bytes, &has, debug)?;
|
||||||
|
let reading: String = digits.iter().map(|d| d.as_deref().unwrap_or("?")).collect();
|
||||||
|
if let Some(n) = expected_len
|
||||||
|
&& reading.chars().count() != n
|
||||||
|
{
|
||||||
|
retry(&format!("read {reading:?}, expected {n} digits"));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("{reading}");
|
||||||
|
runlog::note(format!("reading {reading}"));
|
||||||
|
let complete = !digits.is_empty() && digits.iter().all(Option::is_some);
|
||||||
|
if complete {
|
||||||
|
update_home_assistant(&reading)?;
|
||||||
|
}
|
||||||
|
return Ok(complete);
|
||||||
|
}
|
||||||
|
anyhow::bail!("no valid reading after {MAX_ATTEMPTS} attempts")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Push `reading * 100` to Home Assistant when it is close enough to the current index.
|
||||||
|
fn update_home_assistant(reading: &str) -> Result<()> {
|
||||||
|
let Some(ha) = homeassistant::HomeAssistant::from_env() else {
|
||||||
|
runlog::note("HA_URL / HA_TOKEN not set, Home Assistant not updated");
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let value = reading.parse::<f64>().context("reading is not a number")? * 100.0;
|
||||||
|
runlog::note(match ha.sync_index(value)? {
|
||||||
|
homeassistant::Outcome::Updated { previous } => {
|
||||||
|
format!("{} updated: {previous} -> {value}", ha.entity())
|
||||||
|
}
|
||||||
|
homeassistant::Outcome::Unchanged => format!("{} already at {value}", ha.entity()),
|
||||||
|
homeassistant::Outcome::Rejected { current } => format!(
|
||||||
|
"{value} is more than 1000 away from {} ({current}), not updated",
|
||||||
|
ha.entity()
|
||||||
|
),
|
||||||
|
});
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_digits(
|
||||||
|
bytes: &[u8],
|
||||||
|
has: &dyn Fn(&str) -> bool,
|
||||||
|
debug: bool,
|
||||||
|
) -> Result<Vec<Option<String>>> {
|
||||||
|
if has("--ocrs") {
|
||||||
|
return read_with_ocrs(bytes, debug);
|
||||||
|
}
|
||||||
|
let crop = vision::crop_to_boxes(bytes)?;
|
||||||
|
if debug {
|
||||||
|
std::fs::create_dir_all(DEBUG_DIR)?;
|
||||||
|
std::fs::write(format!("{DEBUG_DIR}/crop.png"), &crop)?;
|
||||||
|
}
|
||||||
|
let llm_digits = if has("--mistral") {
|
||||||
|
mistral::read_digits(&crop)?
|
||||||
|
} else if has("--ollama") {
|
||||||
|
ollama::read_digits(&crop)?
|
||||||
|
} else {
|
||||||
|
claude::read_digits(&crop)?
|
||||||
|
};
|
||||||
|
Ok(llm_digits
|
||||||
|
.into_iter()
|
||||||
|
.map(|d| d.map(String::from))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_with_ocrs(bytes: &[u8], debug: bool) -> Result<Vec<Option<String>>> {
|
||||||
let models_dir = std::env::var_os("OCRS_MODELS_DIR")
|
let models_dir = std::env::var_os("OCRS_MODELS_DIR")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|| PathBuf::from("models"));
|
.unwrap_or_else(|| PathBuf::from("models"));
|
||||||
|
let img = image::load_from_memory(bytes)
|
||||||
let img = fetch::fetch_image(&url)?.into_rgb8();
|
.context("failed to decode image")?
|
||||||
|
.into_rgb8();
|
||||||
let boxes: Vec<BBox> = find_blue_boxes(&img)
|
let boxes: Vec<BBox> = find_blue_boxes(&img)
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(BBox::interior)
|
.filter_map(BBox::interior)
|
||||||
@@ -42,17 +157,10 @@ fn run() -> Result<bool> {
|
|||||||
|
|
||||||
let reader = DigitReader::new(&models_dir)?;
|
let reader = DigitReader::new(&models_dir)?;
|
||||||
let digits = reader.read_boxes(&img, &boxes)?;
|
let digits = reader.read_boxes(&img, &boxes)?;
|
||||||
|
|
||||||
if debug {
|
if debug {
|
||||||
save_debug(&img, &boxes, &digits)?;
|
save_debug(&img, &boxes, &digits)?;
|
||||||
}
|
}
|
||||||
|
Ok(digits)
|
||||||
let reading: String = digits
|
|
||||||
.iter()
|
|
||||||
.map(|d| d.as_deref().unwrap_or("?"))
|
|
||||||
.collect();
|
|
||||||
println!("{reading}");
|
|
||||||
Ok(digits.iter().all(Option::is_some))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn save_debug(img: &RgbImage, boxes: &[BBox], digits: &[Option<String>]) -> Result<()> {
|
fn save_debug(img: &RgbImage, boxes: &[BBox], digits: &[Option<String>]) -> Result<()> {
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::vision;
|
||||||
|
|
||||||
|
const API_URL: &str = "https://api.mistral.ai/v1/chat/completions";
|
||||||
|
const DEFAULT_MODEL: &str = "mistral-medium-latest";
|
||||||
|
|
||||||
|
/// Ask a Mistral vision model to read the digits inside the blue boxes of the picture.
|
||||||
|
/// Returns one entry per box, `None` for an unreadable digit.
|
||||||
|
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||||
|
let api_key = std::env::var("MISTRAL_API_KEY").context("MISTRAL_API_KEY is not set")?;
|
||||||
|
let model = std::env::var("MISTRAL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||||
|
let (media_type, data) = vision::encode_image(image)?;
|
||||||
|
|
||||||
|
let body = json!({
|
||||||
|
"model": model,
|
||||||
|
"temperature": 0,
|
||||||
|
"response_format": {
|
||||||
|
"type": "json_schema",
|
||||||
|
"json_schema": { "name": "meter_digits", "schema": vision::digits_schema(), "strict": true }
|
||||||
|
},
|
||||||
|
"messages": [{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{ "type": "image_url", "image_url": format!("data:{media_type};base64,{data}") },
|
||||||
|
{ "type": "text", "text": vision::PROMPT }
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
|
||||||
|
let response = reqwest::blocking::Client::new()
|
||||||
|
.post(API_URL)
|
||||||
|
.bearer_auth(api_key)
|
||||||
|
.timeout(std::time::Duration::from_secs(120))
|
||||||
|
.json(&body)
|
||||||
|
.send()
|
||||||
|
.context("request to Mistral API failed")?;
|
||||||
|
let status = response.status();
|
||||||
|
let response: Value = response.json().context("invalid JSON from Mistral API")?;
|
||||||
|
if !status.is_success() {
|
||||||
|
bail!("Mistral API returned {status}: {response}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let choice = &response["choices"][0];
|
||||||
|
match choice["finish_reason"].as_str() {
|
||||||
|
Some("stop") => {}
|
||||||
|
other => bail!("unexpected finish_reason {other:?}"),
|
||||||
|
}
|
||||||
|
let text = choice["message"]["content"]
|
||||||
|
.as_str()
|
||||||
|
.context("no text content in Mistral response")?;
|
||||||
|
vision::parse_digits(text)
|
||||||
|
}
|
||||||
+13
-4
@@ -15,8 +15,12 @@ pub struct DigitReader {
|
|||||||
impl DigitReader {
|
impl DigitReader {
|
||||||
pub fn new(models_dir: &Path) -> Result<Self> {
|
pub fn new(models_dir: &Path) -> Result<Self> {
|
||||||
let rec_path = models_dir.join("text-recognition.onnx");
|
let rec_path = models_dir.join("text-recognition.onnx");
|
||||||
let recognition_model = Model::load_file(&rec_path)
|
let recognition_model = Model::load_file(&rec_path).with_context(|| {
|
||||||
.with_context(|| format!("failed to load {} (run ./download-models.sh)", rec_path.display()))?;
|
format!(
|
||||||
|
"failed to load {} (run ./download-models.sh)",
|
||||||
|
rec_path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let engine = OcrEngine::new(OcrEngineParams {
|
let engine = OcrEngine::new(OcrEngineParams {
|
||||||
recognition_model: Some(recognition_model),
|
recognition_model: Some(recognition_model),
|
||||||
allowed_chars: Some("0123456789".into()),
|
allowed_chars: Some("0123456789".into()),
|
||||||
@@ -44,8 +48,13 @@ impl DigitReader {
|
|||||||
Ok(texts
|
Ok(texts
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|line| {
|
.map(|line| {
|
||||||
line.map(|l| l.to_string().chars().filter(char::is_ascii_digit).collect::<String>())
|
line.map(|l| {
|
||||||
.filter(|s| !s.is_empty())
|
l.to_string()
|
||||||
|
.chars()
|
||||||
|
.filter(char::is_ascii_digit)
|
||||||
|
.collect::<String>()
|
||||||
|
})
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::vision;
|
||||||
|
|
||||||
|
const DEFAULT_HOST: &str = "http://localhost:11434";
|
||||||
|
const DEFAULT_MODEL: &str = "lfm3-vl";
|
||||||
|
|
||||||
|
/// Ask a local Ollama vision model to read the digits inside the blue boxes of the picture.
|
||||||
|
/// Returns one entry per box, `None` for an unreadable digit.
|
||||||
|
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||||
|
let host = std::env::var("OLLAMA_HOST").unwrap_or_else(|_| DEFAULT_HOST.into());
|
||||||
|
let model = std::env::var("OLLAMA_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||||
|
let (_, data) = vision::encode_image(image)?;
|
||||||
|
|
||||||
|
let body = json!({
|
||||||
|
"model": model,
|
||||||
|
"stream": false,
|
||||||
|
"think": false,
|
||||||
|
"format": reading_schema(),
|
||||||
|
"options": { "temperature": 0 },
|
||||||
|
"messages": [{
|
||||||
|
"role": "user",
|
||||||
|
"content": vision::PROMPT,
|
||||||
|
"images": [data]
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
|
||||||
|
let url = format!("{}/api/chat", host.trim_end_matches('/'));
|
||||||
|
let response = reqwest::blocking::Client::new()
|
||||||
|
.post(&url)
|
||||||
|
.timeout(std::time::Duration::from_secs(300))
|
||||||
|
.json(&body)
|
||||||
|
.send()
|
||||||
|
.with_context(|| format!("request to Ollama at {url} failed"))?;
|
||||||
|
let status = response.status();
|
||||||
|
let response: Value = response.json().context("invalid JSON from Ollama")?;
|
||||||
|
if !status.is_success() {
|
||||||
|
bail!("Ollama returned {status}: {}", response["error"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
match response["done_reason"].as_str() {
|
||||||
|
Some("stop") => {}
|
||||||
|
other => bail!("unexpected done_reason {other:?}"),
|
||||||
|
}
|
||||||
|
let text = response["message"]["content"]
|
||||||
|
.as_str()
|
||||||
|
.context("no content in Ollama response")?;
|
||||||
|
parse_reading(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Small local models (lfm3-vl) often return an empty array under the shared
|
||||||
|
/// `digits` array schema, but reliably fill a single string, so ask for that.
|
||||||
|
fn reading_schema() -> Value {
|
||||||
|
json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"reading": { "type": "string", "pattern": "^[0-9?]{1,12}$" }
|
||||||
|
},
|
||||||
|
"required": ["reading"]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_reading(text: &str) -> Result<Vec<Option<char>>> {
|
||||||
|
let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?;
|
||||||
|
let reading = parsed["reading"]
|
||||||
|
.as_str()
|
||||||
|
.context("missing `reading` in model answer")?;
|
||||||
|
if reading.is_empty() {
|
||||||
|
bail!("model returned an empty reading");
|
||||||
|
}
|
||||||
|
Ok(reading
|
||||||
|
.chars()
|
||||||
|
.map(|c| Some(c).filter(char::is_ascii_digit))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
//! One line per run appended to a log file, for unattended (cron) runs.
|
||||||
|
|
||||||
|
use std::io::Write as _;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
|
||||||
|
const DEFAULT_LOG: &str = "watermeter.log";
|
||||||
|
|
||||||
|
static NOTES: Mutex<Vec<String>> = Mutex::new(Vec::new());
|
||||||
|
|
||||||
|
/// Print a message on stderr and keep it for the run's log line.
|
||||||
|
pub fn note(msg: impl Into<String>) {
|
||||||
|
let msg = msg.into();
|
||||||
|
eprintln!("{msg}");
|
||||||
|
if let Ok(mut notes) = NOTES.lock() {
|
||||||
|
notes.push(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append `<timestamp> [<status>] <args> | <notes...>` to `WATERMETER_LOG`
|
||||||
|
/// (default `watermeter.log` in the working directory).
|
||||||
|
pub fn write(status: &str) -> Result<()> {
|
||||||
|
let path = std::env::var("WATERMETER_LOG").unwrap_or_else(|_| DEFAULT_LOG.into());
|
||||||
|
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||||
|
let notes = NOTES.lock().map(|n| n.join(" | ")).unwrap_or_default();
|
||||||
|
let line = format!(
|
||||||
|
"{} [{status}] {} | {notes}\n",
|
||||||
|
chrono::Local::now().format("%Y-%m-%d %H:%M:%S"),
|
||||||
|
if args.is_empty() {
|
||||||
|
"--claude".into()
|
||||||
|
} else {
|
||||||
|
args.join(" ")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.append(true)
|
||||||
|
.open(&path)
|
||||||
|
.and_then(|mut f| f.write_all(line.as_bytes()))
|
||||||
|
.with_context(|| format!("failed to write log {path}"))
|
||||||
|
}
|
||||||
+116
@@ -0,0 +1,116 @@
|
|||||||
|
//! Prompt and response format shared by the vision LLM readers.
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use base64::Engine as _;
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
pub const PROMPT: &str = "This photo shows the digit rollers of a water meter. Blue rectangles are drawn over the \
|
||||||
|
image around the digit rollers to read; they may touch or overlap each other. \
|
||||||
|
The meter shows more digits than there are blue rectangles: digits that are not \
|
||||||
|
inside a blue rectangle (for example leading zeros on the left) must NOT be reported. \
|
||||||
|
First count the blue rectangles, then read the single digit inside each one, from left to right. \
|
||||||
|
Other colored rectangles (red, etc.) must be ignored. \
|
||||||
|
If a roller is caught between two digits, give the lower one (9 when between 9 and 0). \
|
||||||
|
Use \"?\" only if a digit is truly unreadable.";
|
||||||
|
|
||||||
|
/// JSON schema the model's answer must follow.
|
||||||
|
pub fn digits_schema() -> Value {
|
||||||
|
json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"rectangle_count": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Number of blue rectangles in the picture"
|
||||||
|
},
|
||||||
|
"digits": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "One entry per blue rectangle, left to right",
|
||||||
|
"items": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "?"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["rectangle_count", "digits"],
|
||||||
|
"additionalProperties": false
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the model's JSON answer into one entry per box, `None` when unreadable.
|
||||||
|
pub fn parse_digits(text: &str) -> Result<Vec<Option<char>>> {
|
||||||
|
let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?;
|
||||||
|
let digits = parsed["digits"]
|
||||||
|
.as_array()
|
||||||
|
.context("missing `digits` in model answer")?;
|
||||||
|
if let Some(count) = parsed["rectangle_count"].as_u64()
|
||||||
|
&& count as usize != digits.len()
|
||||||
|
{
|
||||||
|
bail!(
|
||||||
|
"model saw {count} rectangles but returned {} digits",
|
||||||
|
digits.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(digits
|
||||||
|
.iter()
|
||||||
|
.map(|d| {
|
||||||
|
d.as_str()
|
||||||
|
.and_then(|s| s.chars().next())
|
||||||
|
.filter(char::is_ascii_digit)
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// MIME type and base64 payload of the image.
|
||||||
|
pub fn encode_image(image: &[u8]) -> Result<(&'static str, String)> {
|
||||||
|
let media_type = match image::guess_format(image)? {
|
||||||
|
image::ImageFormat::Png => "image/png",
|
||||||
|
image::ImageFormat::Jpeg => "image/jpeg",
|
||||||
|
image::ImageFormat::WebP => "image/webp",
|
||||||
|
image::ImageFormat::Gif => "image/gif",
|
||||||
|
other => bail!("unsupported image format {other:?}"),
|
||||||
|
};
|
||||||
|
Ok((
|
||||||
|
media_type,
|
||||||
|
base64::engine::general_purpose::STANDARD.encode(image),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Margin kept around the blue boxes when cropping, in source pixels.
|
||||||
|
const CROP_MARGIN: u32 = 8;
|
||||||
|
/// Upscale factor applied to the crop so the digits are large enough for the model.
|
||||||
|
const CROP_SCALE: u32 = 3;
|
||||||
|
|
||||||
|
/// Crop the picture around the blue boxes and upscale it, re-encoded as PNG.
|
||||||
|
/// Falls back to the original picture when no blue box is found.
|
||||||
|
pub fn crop_to_boxes(image: &[u8]) -> Result<Vec<u8>> {
|
||||||
|
let img = image::load_from_memory(image)
|
||||||
|
.context("failed to decode image")?
|
||||||
|
.into_rgb8();
|
||||||
|
let boxes = crate::boxes::find_blue_boxes(&img);
|
||||||
|
let Some(x0) = boxes.iter().map(|b| b.x).min() else {
|
||||||
|
return Ok(image.to_vec());
|
||||||
|
};
|
||||||
|
let y0 = boxes.iter().map(|b| b.y).min().unwrap_or(0);
|
||||||
|
let x1 = boxes.iter().map(|b| b.x + b.w).max().unwrap_or(img.width());
|
||||||
|
let y1 = boxes
|
||||||
|
.iter()
|
||||||
|
.map(|b| b.y + b.h)
|
||||||
|
.max()
|
||||||
|
.unwrap_or(img.height());
|
||||||
|
|
||||||
|
let x0 = x0.saturating_sub(CROP_MARGIN);
|
||||||
|
let y0 = y0.saturating_sub(CROP_MARGIN);
|
||||||
|
let x1 = (x1 + CROP_MARGIN).min(img.width());
|
||||||
|
let y1 = (y1 + CROP_MARGIN).min(img.height());
|
||||||
|
let crop = image::imageops::crop_imm(&img, x0, y0, x1 - x0, y1 - y0).to_image();
|
||||||
|
let crop = image::imageops::resize(
|
||||||
|
&crop,
|
||||||
|
crop.width() * CROP_SCALE,
|
||||||
|
crop.height() * CROP_SCALE,
|
||||||
|
image::imageops::FilterType::Lanczos3,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut png = Vec::new();
|
||||||
|
crop.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)?;
|
||||||
|
Ok(png)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user