dodox
d97aac0e17
Backend: receipt upload -> LLM vision extraction (OpenAI-compatible, provider-agnostic), item review/edit, per-group splitting against Cospend projects/members, highlight+upload via WebDAV, public share link, bill creation via Cospend's OCS API (verified against real source, not just doc summaries). Frontend: capture -> review -> group -> summary flow as an installable PWA. install.sh / run.sh (venv + npm, tmux session) instead of Docker, per the ~/Projects/gain pattern.
105 lines
3.5 KiB
Python
105 lines
3.5 KiB
Python
"""Receipt -> [{id, label, price, bbox}] via any OpenAI-compatible chat
|
|
completions endpoint (works unmodified with OpenAI; point LLM_BASE_URL at
|
|
Gemini's OpenAI-compat layer to use that instead, no code change needed).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import re
|
|
import uuid
|
|
|
|
import requests
|
|
|
|
from .config import Config
|
|
|
|
_PROMPT = """You are reading a photo of a shopping receipt. Extract every \
|
|
line item and its price, plus the store name and the date the receipt was \
|
|
issued.
|
|
|
|
Respond with ONLY a JSON object, no prose, no markdown fences, shaped like:
|
|
{
|
|
"store_name": "<store/shop name as printed, or null if unreadable>",
|
|
"date": "<receipt date as YYYY-MM-DD, or null if unreadable>",
|
|
"items": [
|
|
{"label": "<item name as printed>", "price": <number>, \
|
|
"bbox": [x, y, w, h]}
|
|
]
|
|
}
|
|
|
|
- price is the item's price in the receipt's currency, as a plain number \
|
|
(no currency symbol).
|
|
- bbox is the item's approximate bounding box on the image, normalized to \
|
|
0-1 (x, y = top-left corner; w, h = width/height as a fraction of the \
|
|
image). Best effort is fine.
|
|
- Skip subtotal/tax/total lines, only real purchased items.
|
|
- If a line item's price is unclear, make your best guess rather than \
|
|
omitting it.
|
|
- date must be the transaction date printed on the receipt, not a guess \
|
|
based on anything else.
|
|
"""
|
|
|
|
|
|
def extract_receipt(image_bytes: bytes, mime_type: str = "image/jpeg") -> dict:
|
|
"""Returns {"store_name": str | None, "date": str | None, "items": [...]}."""
|
|
if not Config.LLM_BASE_URL or not Config.LLM_MODEL:
|
|
raise RuntimeError("LLM_BASE_URL / LLM_MODEL not configured (see .env.example)")
|
|
|
|
b64 = base64.b64encode(image_bytes).decode("ascii")
|
|
resp = requests.post(
|
|
f"{Config.LLM_BASE_URL}/chat/completions",
|
|
headers={"Authorization": f"Bearer {Config.LLM_API_KEY}"},
|
|
json={
|
|
"model": Config.LLM_MODEL,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": _PROMPT},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": f"data:{mime_type};base64,{b64}"},
|
|
},
|
|
],
|
|
}
|
|
],
|
|
"temperature": 0,
|
|
},
|
|
timeout=60,
|
|
)
|
|
resp.raise_for_status()
|
|
raw = resp.json()["choices"][0]["message"]["content"]
|
|
parsed = _parse_json_response(raw)
|
|
|
|
items = []
|
|
for entry in parsed.get("items", []):
|
|
items.append(
|
|
{
|
|
"id": str(uuid.uuid4()),
|
|
"label": str(entry.get("label", "")).strip(),
|
|
"price": float(entry.get("price", 0) or 0),
|
|
"bbox": entry.get("bbox") or None,
|
|
}
|
|
)
|
|
|
|
store_name = parsed.get("store_name") or None
|
|
date = parsed.get("date") or None
|
|
# Basic sanity check - if the model didn't return a real YYYY-MM-DD,
|
|
# don't propagate garbage; the caller falls back to today's date.
|
|
if date and not re.match(r"^\d{4}-\d{2}-\d{2}$", str(date)):
|
|
date = None
|
|
|
|
return {"store_name": store_name, "date": date, "items": items}
|
|
|
|
|
|
def _parse_json_response(raw: str) -> dict:
|
|
raw = raw.strip()
|
|
# Models sometimes wrap the JSON in ```json ... ``` despite instructions.
|
|
if raw.startswith("```"):
|
|
raw = raw.strip("`")
|
|
if raw.startswith("json"):
|
|
raw = raw[4:]
|
|
raw = raw.strip()
|
|
return json.loads(raw)
|