Read the meter with vision LLMs and push the index to Home Assistant

- Add Claude, Mistral and local Ollama (lfm3-vl) vision backends that read
  the digits inside the blue boxes of a cropped, upscaled picture
- Retry every 30s when the picture has no blue box or the reading does not
  have WATERMETER_DIGITS digits
- Set input_number.index_compteur_d_eau to reading * 100 when it is within
  1000 of the current value
- Append one line per run to watermeter.log for cron runs

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Romulus21andClaude Opus 5.5 committed 2026-11-15 14:24:09 +01:00
1 parent 253206e6c5
commit 83208191f2
13 files changed
+713 -28

No files matched your search

+1
View File
@@ -2,3 +2,4 @@
.env
/models
/debug
watermeter.log
Generated
+103
View File
@@ -26,6 +26,15 @@ dependencies = [
"equator",
]
[[package]]
name = "android_system_properties"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
dependencies = [
"libc",
]
[[package]]
name = "anyhow"
version = "1.0.104"
@@ -240,6 +249,17 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "chrono"
version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
dependencies = [
"iana-time-zone",
"num-traits",
"windows-link",
]
[[package]]
name = "cmake"
version = "0.1.58"
@@ -736,6 +756,30 @@ dependencies = [
"windows-registry",
]
[[package]]
name = "iana-time-zone"
version = "0.1.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"log",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "icu_collections"
version = "2.3.0"
@@ -1622,6 +1666,8 @@ dependencies = [
"rustls",
"rustls-pki-types",
"rustls-platform-verifier",
"serde",
"serde_json",
"sync_wrapper",
"tokio",
"tokio-rustls",
@@ -1949,6 +1995,19 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "shlex"
version = "2.0.1"
@@ -2389,12 +2448,15 @@ name = "watermeter_watcher"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"chrono",
"dotenvy",
"image",
"ocrs",
"reqwest",
"rten",
"rten-imageproc",
"serde_json",
]
[[package]]
@@ -2441,6 +2503,41 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link",
"windows-result",
"windows-strings",
]
[[package]]
name = "windows-implement"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "windows-interface"
version = "0.59.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "windows-link"
version = "0.2.1"
@@ -2685,6 +2782,12 @@ version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
[[package]]
name = "zune-core"
version = "0.5.3"
+4 -1
View File
@@ -5,9 +5,12 @@ edition = "2024"
[dependencies]
anyhow = "1.0.104"
base64 = "0.23.1"
chrono = { version = "0.4.45", default-features = false, features = ["clock"] }
dotenvy = "0.15.7"
image = "0.25.10"
ocrs = "0.13.1"
reqwest = { version = "0.13.5", features = ["blocking"] }
reqwest = { version = "0.13.5", features = ["blocking", "json"] }
rten = "0.26.0"
rten-imageproc = "0.26.0"
serde_json = "1.0.151"
+6 -1
View File
@@ -70,7 +70,12 @@ pub fn find_blue_boxes(img: &RgbImage) -> Vec<BBox> {
}
}
}
let b = BBox { x: x0, y: y0, w: x1 - x0 + 1, h: y1 - y0 + 1 };
let b = BBox {
x: x0,
y: y0,
w: x1 - x0 + 1,
h: y1 - y0 + 1,
};
if b.w >= MIN_SIDE && b.h >= MIN_SIDE {
boxes.push(b);
}
+67
View File
@@ -0,0 +1,67 @@
use anyhow::{Context, Result, bail};
use serde_json::{Value, json};
use crate::vision;
const API_URL: &str = "https://api.anthropic.com/v1/messages";
const DEFAULT_MODEL: &str = "claude-opus-5-5";
/// Ask Claude to read the digits inside the blue boxes of the picture.
/// Returns one entry per box, `None` for an unreadable digit.
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
let api_key = std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY is not set")?;
let model = std::env::var("ANTHROPIC_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
let (media_type, data) = vision::encode_image(image)?;
let body = json!({
"model": model,
"max_tokens": 16000,
"fallbacks": "default",
"output_config": {
"effort": "low",
"format": { "type": "json_schema", "schema": vision::digits_schema() }
},
"messages": [{
"role": "user",
"content": [
{
"type": "image",
"source": { "type": "base64", "media_type": media_type, "data": data }
},
{ "type": "text", "text": vision::PROMPT }
]
}]
});
let response = reqwest::blocking::Client::new()
.post(API_URL)
.header("x-api-key", api_key)
.header("anthropic-version", "2023-06-01")
.header("anthropic-beta", "server-side-fallback-2026-07-01")
.timeout(std::time::Duration::from_secs(120))
.json(&body)
.send()
.context("request to Claude API failed")?;
let status = response.status();
let response: Value = response.json().context("invalid JSON from Claude API")?;
if !status.is_success() {
bail!(
"Claude API returned {status}: {}",
response["error"]["message"]
);
}
match response["stop_reason"].as_str() {
Some("end_turn") => {}
other => bail!(
"unexpected stop_reason {other:?}: {}",
response["stop_details"]
),
}
let text = response["content"]
.as_array()
.and_then(|blocks| blocks.iter().find(|b| b["type"] == "text"))
.and_then(|b| b["text"].as_str())
.context("no text block in Claude response")?;
vision::parse_digits(text)
}
+3 -4
View File
@@ -1,11 +1,10 @@
use anyhow::{Context, Result};
use image::DynamicImage;
/// Download the meter picture and decode it.
pub fn fetch_image(url: &str) -> Result<DynamicImage> {
/// Download the meter picture as raw bytes.
pub fn fetch_image(url: &str) -> Result<Vec<u8>> {
let bytes = reqwest::blocking::get(url)
.and_then(|r| r.error_for_status())
.with_context(|| format!("failed to download {url}"))?
.bytes()?;
image::load_from_memory(&bytes).context("failed to decode image")
Ok(bytes.to_vec())
}
+102
View File
@@ -0,0 +1,102 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use serde_json::{Value, json};
const DEFAULT_ENTITY: &str = "input_number.index_compteur_d_eau";
/// Readings further than this from the current index are considered misreads.
const MAX_DIFF: f64 = 1000.0;
const TIMEOUT: Duration = Duration::from_secs(30);
pub struct HomeAssistant {
url: String,
token: String,
entity: String,
client: reqwest::blocking::Client,
}
pub enum Outcome {
Updated { previous: f64 },
Unchanged,
Rejected { current: f64 },
}
impl HomeAssistant {
/// Configuration from `HA_URL` and `HA_TOKEN` (and optional `HA_ENTITY`),
/// `None` when Home Assistant is not configured.
pub fn from_env() -> Option<Self> {
let url = std::env::var("HA_URL").ok()?;
let token = std::env::var("HA_TOKEN").ok()?;
Some(Self {
url: url.trim_end_matches('/').to_owned(),
token,
entity: std::env::var("HA_ENTITY").unwrap_or_else(|_| DEFAULT_ENTITY.into()),
client: reqwest::blocking::Client::new(),
})
}
pub fn entity(&self) -> &str {
&self.entity
}
/// Set the entity to `value` unless it is more than `MAX_DIFF` away from its current state.
pub fn sync_index(&self, value: f64) -> Result<Outcome> {
let current = self.current_value()?;
if (value - current).abs() > MAX_DIFF {
return Ok(Outcome::Rejected { current });
}
if value == current {
return Ok(Outcome::Unchanged);
}
self.set_value(value)?;
Ok(Outcome::Updated { previous: current })
}
fn current_value(&self) -> Result<f64> {
let url = format!("{}/api/states/{}", self.url, self.entity);
let response = self
.client
.get(&url)
.bearer_auth(&self.token)
.timeout(TIMEOUT)
.send()
.with_context(|| format!("request to Home Assistant at {url} failed"))?;
let status = response.status();
if !status.is_success() {
bail!(
"Home Assistant returned {status} for {}: {}",
self.entity,
response.text()?
);
}
let state: Value = response
.json()
.context("invalid JSON from Home Assistant")?;
let raw = state["state"]
.as_str()
.context("missing `state` in entity")?;
raw.parse()
.with_context(|| format!("{} has non-numeric state {raw:?}", self.entity))
}
fn set_value(&self, value: f64) -> Result<()> {
let url = format!("{}/api/services/input_number/set_value", self.url);
let response = self
.client
.post(&url)
.bearer_auth(&self.token)
.timeout(TIMEOUT)
.json(&json!({ "entity_id": self.entity, "value": value }))
.send()
.with_context(|| format!("request to Home Assistant at {url} failed"))?;
let status = response.status();
if !status.is_success() {
bail!(
"Home Assistant refused to set {}: {status} {}",
self.entity,
response.text()?
);
}
Ok(())
}
}
+126 -18
View File
@@ -1,9 +1,16 @@
mod boxes;
mod claude;
mod fetch;
mod homeassistant;
mod mistral;
mod ocr;
mod ollama;
mod runlog;
mod vision;
use std::path::PathBuf;
use std::process::ExitCode;
use std::time::Duration;
use anyhow::{Context, Result};
use image::{Rgb, RgbImage};
@@ -12,28 +19,136 @@ use boxes::{BBox, find_blue_boxes};
use ocr::DigitReader;
const DEBUG_DIR: &str = "debug";
/// Wait between two downloads when the picture has no blue box or the reading has the wrong length.
const RETRY_DELAY: Duration = Duration::from_secs(30);
/// Give up after this many downloads without a valid reading.
const MAX_ATTEMPTS: u32 = 10;
fn main() -> ExitCode {
match run() {
Ok(true) => ExitCode::SUCCESS,
Ok(false) => ExitCode::from(2),
let (status, code) = match run() {
Ok(true) => ("ok", ExitCode::SUCCESS),
Ok(false) => ("incomplete", ExitCode::from(2)),
Err(e) => {
eprintln!("error: {e:#}");
ExitCode::FAILURE
runlog::note(format!("error: {e:#}"));
("error", ExitCode::FAILURE)
}
};
if let Err(e) = runlog::write(status) {
eprintln!("error: {e:#}");
}
code
}
/// Returns whether every box could be read.
/// Returns whether every digit could be read.
fn run() -> Result<bool> {
dotenvy::dotenv().ok();
let debug = std::env::args().skip(1).any(|a| a == "--debug");
dotenvy::dotenv_override().ok();
let args: Vec<String> = std::env::args().skip(1).collect();
let has = |flag: &str| args.iter().any(|a| a == flag);
let debug = has("--debug");
let url = std::env::var("WATERMETER_URL").context("WATERMETER_URL is not set")?;
let expected_len = match std::env::var("WATERMETER_DIGITS") {
Ok(v) => Some(
v.trim()
.parse::<usize>()
.context("WATERMETER_DIGITS must be a positive integer")?,
),
Err(_) => None,
};
for attempt in 1..=MAX_ATTEMPTS {
let retry = |why: &str| {
if attempt < MAX_ATTEMPTS {
runlog::note(format!(
"{why} (attempt {attempt}/{MAX_ATTEMPTS}), retrying in {}s",
RETRY_DELAY.as_secs()
));
std::thread::sleep(RETRY_DELAY);
}
};
let bytes = fetch::fetch_image(&url)?;
let img = image::load_from_memory(&bytes)
.context("failed to decode image")?
.into_rgb8();
if find_blue_boxes(&img).is_empty() {
retry("no blue box in picture");
continue;
}
let digits = read_digits(&bytes, &has, debug)?;
let reading: String = digits.iter().map(|d| d.as_deref().unwrap_or("?")).collect();
if let Some(n) = expected_len
&& reading.chars().count() != n
{
retry(&format!("read {reading:?}, expected {n} digits"));
continue;
}
println!("{reading}");
runlog::note(format!("reading {reading}"));
let complete = !digits.is_empty() && digits.iter().all(Option::is_some);
if complete {
update_home_assistant(&reading)?;
}
return Ok(complete);
}
anyhow::bail!("no valid reading after {MAX_ATTEMPTS} attempts")
}
/// Push `reading * 100` to Home Assistant when it is close enough to the current index.
fn update_home_assistant(reading: &str) -> Result<()> {
let Some(ha) = homeassistant::HomeAssistant::from_env() else {
runlog::note("HA_URL / HA_TOKEN not set, Home Assistant not updated");
return Ok(());
};
let value = reading.parse::<f64>().context("reading is not a number")? * 100.0;
runlog::note(match ha.sync_index(value)? {
homeassistant::Outcome::Updated { previous } => {
format!("{} updated: {previous} -> {value}", ha.entity())
}
homeassistant::Outcome::Unchanged => format!("{} already at {value}", ha.entity()),
homeassistant::Outcome::Rejected { current } => format!(
"{value} is more than 1000 away from {} ({current}), not updated",
ha.entity()
),
});
Ok(())
}
fn read_digits(
bytes: &[u8],
has: &dyn Fn(&str) -> bool,
debug: bool,
) -> Result<Vec<Option<String>>> {
if has("--ocrs") {
return read_with_ocrs(bytes, debug);
}
let crop = vision::crop_to_boxes(bytes)?;
if debug {
std::fs::create_dir_all(DEBUG_DIR)?;
std::fs::write(format!("{DEBUG_DIR}/crop.png"), &crop)?;
}
let llm_digits = if has("--mistral") {
mistral::read_digits(&crop)?
} else if has("--ollama") {
ollama::read_digits(&crop)?
} else {
claude::read_digits(&crop)?
};
Ok(llm_digits
.into_iter()
.map(|d| d.map(String::from))
.collect())
}
fn read_with_ocrs(bytes: &[u8], debug: bool) -> Result<Vec<Option<String>>> {
let models_dir = std::env::var_os("OCRS_MODELS_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("models"));
let img = fetch::fetch_image(&url)?.into_rgb8();
let img = image::load_from_memory(bytes)
.context("failed to decode image")?
.into_rgb8();
let boxes: Vec<BBox> = find_blue_boxes(&img)
.iter()
.filter_map(BBox::interior)
@@ -42,17 +157,10 @@ fn run() -> Result<bool> {
let reader = DigitReader::new(&models_dir)?;
let digits = reader.read_boxes(&img, &boxes)?;
if debug {
save_debug(&img, &boxes, &digits)?;
}
let reading: String = digits
.iter()
.map(|d| d.as_deref().unwrap_or("?"))
.collect();
println!("{reading}");
Ok(digits.iter().all(Option::is_some))
Ok(digits)
}
fn save_debug(img: &RgbImage, boxes: &[BBox], digits: &[Option<String>]) -> Result<()> {
+54
View File
@@ -0,0 +1,54 @@
use anyhow::{Context, Result, bail};
use serde_json::{Value, json};
use crate::vision;
const API_URL: &str = "https://api.mistral.ai/v1/chat/completions";
const DEFAULT_MODEL: &str = "mistral-medium-latest";
/// Ask a Mistral vision model to read the digits inside the blue boxes of the picture.
/// Returns one entry per box, `None` for an unreadable digit.
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
let api_key = std::env::var("MISTRAL_API_KEY").context("MISTRAL_API_KEY is not set")?;
let model = std::env::var("MISTRAL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
let (media_type, data) = vision::encode_image(image)?;
let body = json!({
"model": model,
"temperature": 0,
"response_format": {
"type": "json_schema",
"json_schema": { "name": "meter_digits", "schema": vision::digits_schema(), "strict": true }
},
"messages": [{
"role": "user",
"content": [
{ "type": "image_url", "image_url": format!("data:{media_type};base64,{data}") },
{ "type": "text", "text": vision::PROMPT }
]
}]
});
let response = reqwest::blocking::Client::new()
.post(API_URL)
.bearer_auth(api_key)
.timeout(std::time::Duration::from_secs(120))
.json(&body)
.send()
.context("request to Mistral API failed")?;
let status = response.status();
let response: Value = response.json().context("invalid JSON from Mistral API")?;
if !status.is_success() {
bail!("Mistral API returned {status}: {response}");
}
let choice = &response["choices"][0];
match choice["finish_reason"].as_str() {
Some("stop") => {}
other => bail!("unexpected finish_reason {other:?}"),
}
let text = choice["message"]["content"]
.as_str()
.context("no text content in Mistral response")?;
vision::parse_digits(text)
}
+13 -4
View File
@@ -15,8 +15,12 @@ pub struct DigitReader {
impl DigitReader {
pub fn new(models_dir: &Path) -> Result<Self> {
let rec_path = models_dir.join("text-recognition.onnx");
let recognition_model = Model::load_file(&rec_path)
.with_context(|| format!("failed to load {} (run ./download-models.sh)", rec_path.display()))?;
let recognition_model = Model::load_file(&rec_path).with_context(|| {
format!(
"failed to load {} (run ./download-models.sh)",
rec_path.display()
)
})?;
let engine = OcrEngine::new(OcrEngineParams {
recognition_model: Some(recognition_model),
allowed_chars: Some("0123456789".into()),
@@ -44,8 +48,13 @@ impl DigitReader {
Ok(texts
.into_iter()
.map(|line| {
line.map(|l| l.to_string().chars().filter(char::is_ascii_digit).collect::<String>())
.filter(|s| !s.is_empty())
line.map(|l| {
l.to_string()
.chars()
.filter(char::is_ascii_digit)
.collect::<String>()
})
.filter(|s| !s.is_empty())
})
.collect())
}
+76
View File
@@ -0,0 +1,76 @@
use anyhow::{Context, Result, bail};
use serde_json::{Value, json};
use crate::vision;
const DEFAULT_HOST: &str = "http://localhost:11434";
const DEFAULT_MODEL: &str = "lfm3-vl";
/// Ask a local Ollama vision model to read the digits inside the blue boxes of the picture.
/// Returns one entry per box, `None` for an unreadable digit.
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
let host = std::env::var("OLLAMA_HOST").unwrap_or_else(|_| DEFAULT_HOST.into());
let model = std::env::var("OLLAMA_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
let (_, data) = vision::encode_image(image)?;
let body = json!({
"model": model,
"stream": false,
"think": false,
"format": reading_schema(),
"options": { "temperature": 0 },
"messages": [{
"role": "user",
"content": vision::PROMPT,
"images": [data]
}]
});
let url = format!("{}/api/chat", host.trim_end_matches('/'));
let response = reqwest::blocking::Client::new()
.post(&url)
.timeout(std::time::Duration::from_secs(300))
.json(&body)
.send()
.with_context(|| format!("request to Ollama at {url} failed"))?;
let status = response.status();
let response: Value = response.json().context("invalid JSON from Ollama")?;
if !status.is_success() {
bail!("Ollama returned {status}: {}", response["error"]);
}
match response["done_reason"].as_str() {
Some("stop") => {}
other => bail!("unexpected done_reason {other:?}"),
}
let text = response["message"]["content"]
.as_str()
.context("no content in Ollama response")?;
parse_reading(text)
}
/// Small local models (lfm3-vl) often return an empty array under the shared
/// `digits` array schema, but reliably fill a single string, so ask for that.
fn reading_schema() -> Value {
json!({
"type": "object",
"properties": {
"reading": { "type": "string", "pattern": "^[0-9?]{1,12}$" }
},
"required": ["reading"]
})
}
fn parse_reading(text: &str) -> Result<Vec<Option<char>>> {
let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?;
let reading = parsed["reading"]
.as_str()
.context("missing `reading` in model answer")?;
if reading.is_empty() {
bail!("model returned an empty reading");
}
Ok(reading
.chars()
.map(|c| Some(c).filter(char::is_ascii_digit))
.collect())
}
+42
View File
@@ -0,0 +1,42 @@
//! One line per run appended to a log file, for unattended (cron) runs.
use std::io::Write as _;
use std::sync::Mutex;
use anyhow::{Context, Result};
const DEFAULT_LOG: &str = "watermeter.log";
static NOTES: Mutex<Vec<String>> = Mutex::new(Vec::new());
/// Print a message on stderr and keep it for the run's log line.
pub fn note(msg: impl Into<String>) {
let msg = msg.into();
eprintln!("{msg}");
if let Ok(mut notes) = NOTES.lock() {
notes.push(msg);
}
}
/// Append `<timestamp> [<status>] <args> | <notes...>` to `WATERMETER_LOG`
/// (default `watermeter.log` in the working directory).
pub fn write(status: &str) -> Result<()> {
let path = std::env::var("WATERMETER_LOG").unwrap_or_else(|_| DEFAULT_LOG.into());
let args: Vec<String> = std::env::args().skip(1).collect();
let notes = NOTES.lock().map(|n| n.join(" | ")).unwrap_or_default();
let line = format!(
"{} [{status}] {} | {notes}\n",
chrono::Local::now().format("%Y-%m-%d %H:%M:%S"),
if args.is_empty() {
"--claude".into()
} else {
args.join(" ")
},
);
std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)
.and_then(|mut f| f.write_all(line.as_bytes()))
.with_context(|| format!("failed to write log {path}"))
}
+116
View File
@@ -0,0 +1,116 @@
//! Prompt and response format shared by the vision LLM readers.
use anyhow::{Context, Result, bail};
use base64::Engine as _;
use serde_json::{Value, json};
pub const PROMPT: &str = "This photo shows the digit rollers of a water meter. Blue rectangles are drawn over the \
image around the digit rollers to read; they may touch or overlap each other. \
The meter shows more digits than there are blue rectangles: digits that are not \
inside a blue rectangle (for example leading zeros on the left) must NOT be reported. \
First count the blue rectangles, then read the single digit inside each one, from left to right. \
Other colored rectangles (red, etc.) must be ignored. \
If a roller is caught between two digits, give the lower one (9 when between 9 and 0). \
Use \"?\" only if a digit is truly unreadable.";
/// JSON schema the model's answer must follow.
pub fn digits_schema() -> Value {
json!({
"type": "object",
"properties": {
"rectangle_count": {
"type": "integer",
"description": "Number of blue rectangles in the picture"
},
"digits": {
"type": "array",
"description": "One entry per blue rectangle, left to right",
"items": {
"type": "string",
"enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "?"]
}
}
},
"required": ["rectangle_count", "digits"],
"additionalProperties": false
})
}
/// Parse the model's JSON answer into one entry per box, `None` when unreadable.
pub fn parse_digits(text: &str) -> Result<Vec<Option<char>>> {
let parsed: Value = serde_json::from_str(text).context("model answer is not valid JSON")?;
let digits = parsed["digits"]
.as_array()
.context("missing `digits` in model answer")?;
if let Some(count) = parsed["rectangle_count"].as_u64()
&& count as usize != digits.len()
{
bail!(
"model saw {count} rectangles but returned {} digits",
digits.len()
);
}
Ok(digits
.iter()
.map(|d| {
d.as_str()
.and_then(|s| s.chars().next())
.filter(char::is_ascii_digit)
})
.collect())
}
/// MIME type and base64 payload of the image.
pub fn encode_image(image: &[u8]) -> Result<(&'static str, String)> {
let media_type = match image::guess_format(image)? {
image::ImageFormat::Png => "image/png",
image::ImageFormat::Jpeg => "image/jpeg",
image::ImageFormat::WebP => "image/webp",
image::ImageFormat::Gif => "image/gif",
other => bail!("unsupported image format {other:?}"),
};
Ok((
media_type,
base64::engine::general_purpose::STANDARD.encode(image),
))
}
/// Margin kept around the blue boxes when cropping, in source pixels.
const CROP_MARGIN: u32 = 8;
/// Upscale factor applied to the crop so the digits are large enough for the model.
const CROP_SCALE: u32 = 3;
/// Crop the picture around the blue boxes and upscale it, re-encoded as PNG.
/// Falls back to the original picture when no blue box is found.
pub fn crop_to_boxes(image: &[u8]) -> Result<Vec<u8>> {
let img = image::load_from_memory(image)
.context("failed to decode image")?
.into_rgb8();
let boxes = crate::boxes::find_blue_boxes(&img);
let Some(x0) = boxes.iter().map(|b| b.x).min() else {
return Ok(image.to_vec());
};
let y0 = boxes.iter().map(|b| b.y).min().unwrap_or(0);
let x1 = boxes.iter().map(|b| b.x + b.w).max().unwrap_or(img.width());
let y1 = boxes
.iter()
.map(|b| b.y + b.h)
.max()
.unwrap_or(img.height());
let x0 = x0.saturating_sub(CROP_MARGIN);
let y0 = y0.saturating_sub(CROP_MARGIN);
let x1 = (x1 + CROP_MARGIN).min(img.width());
let y1 = (y1 + CROP_MARGIN).min(img.height());
let crop = image::imageops::crop_imm(&img, x0, y0, x1 - x0, y1 - y0).to_image();
let crop = image::imageops::resize(
&crop,
crop.width() * CROP_SCALE,
crop.height() * CROP_SCALE,
image::imageops::FilterType::Lanczos3,
);
let mut png = Vec::new();
crop.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)?;
Ok(png)
}