Read the meter with vision LLMs and push the index to Home Assistant
- Add Claude, Mistral and local Ollama (lfm3-vl) vision backends that read the digits inside the blue boxes of a cropped, upscaled picture - Retry every 30s when the picture has no blue box or the reading does not have WATERMETER_DIGITS digits - Set input_number.index_compteur_d_eau to reading * 100 when it is within 1000 of the current value - Append one line per run to watermeter.log for cron runs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
253206e6c5
commit
83208191f2
13 files changed
+713
-28
No files matched your search
@@ -0,0 +1,54 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::vision;
|
||||
|
||||
const API_URL: &str = "https://api.mistral.ai/v1/chat/completions";
|
||||
const DEFAULT_MODEL: &str = "mistral-medium-latest";
|
||||
|
||||
/// Ask a Mistral vision model to read the digits inside the blue boxes of the picture.
|
||||
/// Returns one entry per box, `None` for an unreadable digit.
|
||||
pub fn read_digits(image: &[u8]) -> Result<Vec<Option<char>>> {
|
||||
let api_key = std::env::var("MISTRAL_API_KEY").context("MISTRAL_API_KEY is not set")?;
|
||||
let model = std::env::var("MISTRAL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into());
|
||||
let (media_type, data) = vision::encode_image(image)?;
|
||||
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"temperature": 0,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": { "name": "meter_digits", "schema": vision::digits_schema(), "strict": true }
|
||||
},
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "image_url", "image_url": format!("data:{media_type};base64,{data}") },
|
||||
{ "type": "text", "text": vision::PROMPT }
|
||||
]
|
||||
}]
|
||||
});
|
||||
|
||||
let response = reqwest::blocking::Client::new()
|
||||
.post(API_URL)
|
||||
.bearer_auth(api_key)
|
||||
.timeout(std::time::Duration::from_secs(120))
|
||||
.json(&body)
|
||||
.send()
|
||||
.context("request to Mistral API failed")?;
|
||||
let status = response.status();
|
||||
let response: Value = response.json().context("invalid JSON from Mistral API")?;
|
||||
if !status.is_success() {
|
||||
bail!("Mistral API returned {status}: {response}");
|
||||
}
|
||||
|
||||
let choice = &response["choices"][0];
|
||||
match choice["finish_reason"].as_str() {
|
||||
Some("stop") => {}
|
||||
other => bail!("unexpected finish_reason {other:?}"),
|
||||
}
|
||||
let text = choice["message"]["content"]
|
||||
.as_str()
|
||||
.context("no text content in Mistral response")?;
|
||||
vision::parse_digits(text)
|
||||
}
|
||||
Reference in new issue
Block a user