Initial scaffold: Flask/SQLite backend, React/Vite PWA frontend

Backend: receipt upload -> LLM vision extraction (OpenAI-compatible,
provider-agnostic), item review/edit, per-group splitting against
Cospend projects/members, highlight+upload via WebDAV, public share
link, bill creation via Cospend's OCS API (verified against real
source, not just doc summaries).

Frontend: capture -> review -> group -> summary flow as an installable
PWA.

install.sh / run.sh (venv + npm, tmux session) instead of Docker, per
the ~/Projects/gain pattern.
This commit is contained in:
2026-08-30 15:02:18 +02:00
commit d97aac0e17
42 changed files with 8274 additions and 0 deletions
+27
View File
@@ -0,0 +1,27 @@
import os
from flask import Flask
from flask_cors import CORS
from .config import Config
from .db import init_db
def create_app() -> Flask:
app = Flask(__name__, instance_relative_config=True)
app.config.from_object(Config)
os.makedirs(app.instance_path, exist_ok=True)
os.makedirs(Config.UPLOAD_DIR, exist_ok=True)
# Frontend runs on a different origin (vite dev server) during
# development; lock this down to that origin in production.
CORS(app, resources={r"/api/*": {"origins": Config.CORS_ORIGIN}})
init_db(Config.DATABASE_PATH)
from .routes import bp as api_bp
app.register_blueprint(api_bp, url_prefix="/api")
return app
+31
View File
@@ -0,0 +1,31 @@
import os
from dotenv import load_dotenv
load_dotenv()
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
class Config:
SECRET_KEY = os.environ.get("FLASK_SECRET_KEY", "dev-change-me")
DATABASE_PATH = os.path.join(
BASE_DIR, os.environ.get("DATABASE_PATH", "instance/wgbill.sqlite3")
)
UPLOAD_DIR = os.path.join(BASE_DIR, "uploads")
CORS_ORIGIN = os.environ.get("CORS_ORIGIN", "http://localhost:5173")
NC_BASE_URL = os.environ.get("NC_BASE_URL", "").rstrip("/")
NC_USERNAME = os.environ.get("NC_USERNAME", "")
NC_APP_PASSWORD = os.environ.get("NC_APP_PASSWORD", "")
NC_UPLOAD_FOLDER = os.environ.get("NC_UPLOAD_FOLDER", "wgBill").strip("/")
# Optional - pre-selects a project in the UI; the app lists all of the
# user's Cospend projects via the API either way, so this isn't required.
COSPEND_DEFAULT_PROJECT_ID = os.environ.get("COSPEND_PROJECT_ID", "")
LLM_BASE_URL = os.environ.get("LLM_BASE_URL", "").rstrip("/")
LLM_API_KEY = os.environ.get("LLM_API_KEY", "")
LLM_MODEL = os.environ.get("LLM_MODEL", "")
+99
View File
@@ -0,0 +1,99 @@
"""Cospend's authenticated (NC-login) API - same app-password auth as
nc_client. This is an OCS API (same family as the Share API), confirmed
against the real routes/controller source in julien-nc/cospend-nc:
appinfo/routes.php ('ocs' section):
GET /api/{v}/projects -> api#getLocalProjects
GET /api/{v}/projects/{projectId}/members -> api#getMembers
POST /api/{v}/projects/{projectId}/bills -> api#createBill
lib/Controller/ApiController.php, createBill() signature:
string $projectId, ?string $date, ?string $what, ?int $payer,
?string $payedFor, ?float $amount, ... ?string $comment, ...
-> params are camelCase (payedFor, not payed_for), payer/payedFor are
numeric member ids (payedFor is a comma-separated string of ids).
Being an OCS controller, every call needs the OCS-APIRequest header and the
full path is under /ocs/v2.php/apps/cospend/...
"""
from __future__ import annotations
from typing import Any
import requests
from .config import Config
_HEADERS = {"OCS-APIRequest": "true"}
def _auth() -> tuple[str, str]:
return (Config.NC_USERNAME, Config.NC_APP_PASSWORD)
def _base(project_id: str | None = None) -> str:
root = f"{Config.NC_BASE_URL}/ocs/v2.php/apps/cospend/api/v1"
if project_id is None:
return root
return f"{root}/projects/{project_id}"
def get_projects() -> list[dict[str, Any]]:
"""Lists Cospend projects visible to the authenticated user."""
resp = requests.get(
f"{_base()}/projects",
auth=_auth(),
headers=_HEADERS,
params={"format": "json"},
timeout=15,
)
resp.raise_for_status()
return resp.json()["ocs"]["data"]
def get_members(project_id: str) -> list[dict[str, Any]]:
resp = requests.get(
f"{_base(project_id)}/members",
auth=_auth(),
headers=_HEADERS,
params={"format": "json"},
timeout=15,
)
resp.raise_for_status()
return resp.json()["ocs"]["data"]
def create_bill(
*,
project_id: str,
what: str,
amount: float,
payer_id: int,
ower_ids: list[int],
date: str,
comment: str = "",
) -> int:
"""Creates a bill split evenly among `ower_ids`, paid by `payer_id`.
`amount` is the bill total; Cospend divides it across owers itself.
Returns the new bill's id - createBill's controller method returns just
the int id (`return new DataResponse($newBillId)`), not a bill object.
"""
resp = requests.post(
f"{_base(project_id)}/bills",
auth=_auth(),
headers=_HEADERS,
params={"format": "json"},
json={
"what": what,
"amount": amount,
"payer": payer_id,
"payedFor": ",".join(str(i) for i in ower_ids),
"comment": comment,
"date": date, # YYYY-MM-DD, the receipt's issue date, not today
},
timeout=15,
)
resp.raise_for_status()
return resp.json()["ocs"]["data"]
+23
View File
@@ -0,0 +1,23 @@
import os
import sqlite3
_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "schema.sql")
_db_path: str | None = None
def init_db(database_path: str) -> None:
global _db_path
_db_path = database_path
os.makedirs(os.path.dirname(database_path), exist_ok=True)
with get_conn() as conn, open(_SCHEMA_PATH) as f:
conn.executescript(f.read())
def get_conn() -> sqlite3.Connection:
if _db_path is None:
raise RuntimeError("init_db() must be called before get_conn()")
conn = sqlite3.connect(_db_path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = ON")
return conn
+32
View File
@@ -0,0 +1,32 @@
"""Draws highlight boxes over selected items on the original receipt image."""
from __future__ import annotations
import io
from PIL import Image, ImageDraw
_BOX_COLOR = (255, 210, 0) # translucent yellow highlighter look
_BOX_WIDTH = 4
def highlight_items(image_bytes: bytes, bboxes: list[list[float]]) -> bytes:
"""`bboxes` are normalized [x, y, w, h] (0-1). Returns JPEG bytes."""
base = Image.open(io.BytesIO(image_bytes)).convert("RGBA")
overlay = Image.new("RGBA", base.size, (0, 0, 0, 0))
draw = ImageDraw.Draw(overlay)
w, h = base.size
for bbox in bboxes:
if not bbox or len(bbox) != 4:
continue
x, y, bw, bh = bbox
left, top = x * w, y * h
right, bottom = (x + bw) * w, (y + bh) * h
draw.rectangle([left, top, right, bottom], fill=(*_BOX_COLOR, 70))
draw.rectangle([left, top, right, bottom], outline=(*_BOX_COLOR, 255), width=_BOX_WIDTH)
combined = Image.alpha_composite(base, overlay).convert("RGB")
out = io.BytesIO()
combined.save(out, format="JPEG", quality=90)
return out.getvalue()
+104
View File
@@ -0,0 +1,104 @@
"""Receipt -> [{id, label, price, bbox}] via any OpenAI-compatible chat
completions endpoint (works unmodified with OpenAI; point LLM_BASE_URL at
Gemini's OpenAI-compat layer to use that instead, no code change needed).
"""
from __future__ import annotations
import base64
import json
import re
import uuid
import requests
from .config import Config
_PROMPT = """You are reading a photo of a shopping receipt. Extract every \
line item and its price, plus the store name and the date the receipt was \
issued.
Respond with ONLY a JSON object, no prose, no markdown fences, shaped like:
{
"store_name": "<store/shop name as printed, or null if unreadable>",
"date": "<receipt date as YYYY-MM-DD, or null if unreadable>",
"items": [
{"label": "<item name as printed>", "price": <number>, \
"bbox": [x, y, w, h]}
]
}
- price is the item's price in the receipt's currency, as a plain number \
(no currency symbol).
- bbox is the item's approximate bounding box on the image, normalized to \
0-1 (x, y = top-left corner; w, h = width/height as a fraction of the \
image). Best effort is fine.
- Skip subtotal/tax/total lines, only real purchased items.
- If a line item's price is unclear, make your best guess rather than \
omitting it.
- date must be the transaction date printed on the receipt, not a guess \
based on anything else.
"""
def extract_receipt(image_bytes: bytes, mime_type: str = "image/jpeg") -> dict:
"""Returns {"store_name": str | None, "date": str | None, "items": [...]}."""
if not Config.LLM_BASE_URL or not Config.LLM_MODEL:
raise RuntimeError("LLM_BASE_URL / LLM_MODEL not configured (see .env.example)")
b64 = base64.b64encode(image_bytes).decode("ascii")
resp = requests.post(
f"{Config.LLM_BASE_URL}/chat/completions",
headers={"Authorization": f"Bearer {Config.LLM_API_KEY}"},
json={
"model": Config.LLM_MODEL,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": _PROMPT},
{
"type": "image_url",
"image_url": {"url": f"data:{mime_type};base64,{b64}"},
},
],
}
],
"temperature": 0,
},
timeout=60,
)
resp.raise_for_status()
raw = resp.json()["choices"][0]["message"]["content"]
parsed = _parse_json_response(raw)
items = []
for entry in parsed.get("items", []):
items.append(
{
"id": str(uuid.uuid4()),
"label": str(entry.get("label", "")).strip(),
"price": float(entry.get("price", 0) or 0),
"bbox": entry.get("bbox") or None,
}
)
store_name = parsed.get("store_name") or None
date = parsed.get("date") or None
# Basic sanity check - if the model didn't return a real YYYY-MM-DD,
# don't propagate garbage; the caller falls back to today's date.
if date and not re.match(r"^\d{4}-\d{2}-\d{2}$", str(date)):
date = None
return {"store_name": store_name, "date": date, "items": items}
def _parse_json_response(raw: str) -> dict:
raw = raw.strip()
# Models sometimes wrap the JSON in ```json ... ``` despite instructions.
if raw.startswith("```"):
raw = raw.strip("`")
if raw.startswith("json"):
raw = raw[4:]
raw = raw.strip()
return json.loads(raw)
+76
View File
@@ -0,0 +1,76 @@
"""Thin wrapper around the bits of Nextcloud's WebDAV and OCS Share API we need.
Auth is HTTP Basic with an app password (Settings -> Security -> Devices &
Sessions -> create app password), never the account's real password.
"""
from __future__ import annotations
import requests
from .config import Config
def _auth() -> tuple[str, str]:
return (Config.NC_USERNAME, Config.NC_APP_PASSWORD)
def _webdav_root() -> str:
return f"{Config.NC_BASE_URL}/remote.php/dav/files/{Config.NC_USERNAME}"
def ensure_upload_folder() -> None:
"""MKCOL each segment of the configured upload folder path if missing.
NC_UPLOAD_FOLDER may be nested (e.g. "Documents/Cospend/Assets"); MKCOL
only creates one level at a time, so walk the path segment by segment.
Idempotent.
"""
if not Config.NC_UPLOAD_FOLDER:
return
segments = Config.NC_UPLOAD_FOLDER.split("/")
partial = ""
for segment in segments:
partial = f"{partial}/{segment}" if partial else segment
url = f"{_webdav_root()}/{partial}"
resp = requests.request("MKCOL", url, auth=_auth(), timeout=15)
# 201 = created, 405 = already exists. Anything else is a real problem.
if resp.status_code not in (201, 405):
resp.raise_for_status()
def upload_file(filename: str, content: bytes, content_type: str = "image/jpeg") -> str:
"""Uploads `content` to <upload folder>/<filename> via WebDAV PUT.
Returns the server-relative path (e.g. "wgBill/receipt-123.jpg"), which is
what the Share API expects as `path`.
"""
ensure_upload_folder()
rel_path = f"{Config.NC_UPLOAD_FOLDER}/{filename}" if Config.NC_UPLOAD_FOLDER else filename
url = f"{_webdav_root()}/{rel_path}"
resp = requests.put(
url, data=content, auth=_auth(), headers={"Content-Type": content_type}, timeout=30
)
resp.raise_for_status()
return rel_path
def create_public_share(rel_path: str) -> str:
"""Creates a public read-only link share for `rel_path`. Returns the share URL."""
url = f"{Config.NC_BASE_URL}/ocs/v2.php/apps/files_sharing/api/v1/shares"
resp = requests.post(
url,
auth=_auth(),
headers={"OCS-APIRequest": "true"},
data={
"path": f"/{rel_path}",
"shareType": 3, # public link
"permissions": 1, # read-only
},
params={"format": "json"},
timeout=15,
)
resp.raise_for_status()
payload = resp.json()
share_url = payload["ocs"]["data"]["url"]
return share_url
+291
View File
@@ -0,0 +1,291 @@
from __future__ import annotations
import json
import re
import uuid
from datetime import date as date_cls
from datetime import datetime
from flask import Blueprint, jsonify, request
from . import cospend_client, highlight, llm_client, nc_client
from .config import Config
from .db import get_conn
bp = Blueprint("api", __name__)
# ---------------------------------------------------------------- receipts
@bp.post("/receipts")
def create_receipt():
"""multipart/form-data with an `image` file. Extracts items via LLM."""
file = request.files.get("image")
if file is None:
return jsonify(error="missing 'image' file"), 400
image_bytes = file.read()
receipt_id = str(uuid.uuid4())
image_path = f"{Config.UPLOAD_DIR}/{receipt_id}.jpg"
with open(image_path, "wb") as f:
f.write(image_bytes)
try:
extracted = llm_client.extract_receipt(image_bytes, mime_type=file.mimetype or "image/jpeg")
except Exception as exc: # noqa: BLE001 - surfaced to the client as-is
return jsonify(error=f"extraction failed: {exc}"), 502
items = extracted["items"]
store_name = extracted["store_name"]
# Fall back to today if the receipt's date wasn't readable - the user
# can still override it in the review step.
receipt_date = extracted["date"] or date_cls.today().isoformat()
with get_conn() as conn:
conn.execute(
"""INSERT INTO receipts (id, image_path, items_json, store_name, receipt_date)
VALUES (?, ?, ?, ?, ?)""",
(receipt_id, image_path, json.dumps(items), store_name, receipt_date),
)
return jsonify(id=receipt_id, items=items, store_name=store_name, date=receipt_date), 201
@bp.get("/receipts/<receipt_id>")
def get_receipt(receipt_id: str):
receipt = _load_receipt(receipt_id)
if receipt is None:
return jsonify(error="not found"), 404
return jsonify(receipt)
@bp.patch("/receipts/<receipt_id>/items")
def update_items(receipt_id: str):
"""Body: {"items": [...]} — full replacement, for the review/edit step."""
body = request.get_json(force=True) or {}
items = body.get("items")
if not isinstance(items, list):
return jsonify(error="'items' must be a list"), 400
with get_conn() as conn:
cur = conn.execute(
"UPDATE receipts SET items_json = ? WHERE id = ?",
(json.dumps(items), receipt_id),
)
if cur.rowcount == 0:
return jsonify(error="not found"), 404
return jsonify(id=receipt_id, items=items)
@bp.patch("/receipts/<receipt_id>/meta")
def update_meta(receipt_id: str):
"""Body: {"store_name"?: str, "date"?: "YYYY-MM-DD"} - user override for
what the receipt reader guessed (or didn't find)."""
body = request.get_json(force=True) or {}
fields, values = [], []
if "store_name" in body:
fields.append("store_name = ?")
values.append(body["store_name"])
if "date" in body:
if not re.match(r"^\d{4}-\d{2}-\d{2}$", str(body["date"] or "")):
return jsonify(error="date must be YYYY-MM-DD"), 400
fields.append("receipt_date = ?")
values.append(body["date"])
if not fields:
return jsonify(error="nothing to update"), 400
values.append(receipt_id)
with get_conn() as conn:
cur = conn.execute(f"UPDATE receipts SET {', '.join(fields)} WHERE id = ?", values)
if cur.rowcount == 0:
return jsonify(error="not found"), 404
receipt = _load_receipt(receipt_id)
return jsonify(receipt)
def _load_receipt(receipt_id: str) -> dict | None:
with get_conn() as conn:
row = conn.execute("SELECT * FROM receipts WHERE id = ?", (receipt_id,)).fetchone()
if row is None:
return None
return {
"id": row["id"],
"items": json.loads(row["items_json"]),
"store_name": row["store_name"],
"date": row["receipt_date"],
"status": row["status"],
"created_at": row["created_at"],
}
# -------------------------------------------------------------- cospend
@bp.get("/cospend/projects")
def cospend_projects():
try:
projects = cospend_client.get_projects()
except Exception as exc: # noqa: BLE001
return jsonify(error=f"cospend request failed: {exc}"), 502
return jsonify(projects=projects, default_project_id=Config.COSPEND_DEFAULT_PROJECT_ID or None)
@bp.get("/cospend/projects/<project_id>/members")
def cospend_members(project_id: str):
try:
members = cospend_client.get_members(project_id)
except Exception as exc: # noqa: BLE001
return jsonify(error=f"cospend request failed: {exc}"), 502
return jsonify(members)
# ------------------------------------------------------------------ groups
@bp.post("/receipts/<receipt_id>/groups")
def create_group(receipt_id: str):
"""Body: {name, cospend_project_id, payer_member_id, member_ids: [...], item_ids: [...]}"""
receipt = _load_receipt(receipt_id)
if receipt is None:
return jsonify(error="not found"), 404
body = request.get_json(force=True) or {}
name = body.get("name") or "WG"
project_id = body.get("cospend_project_id")
payer_member_id = body.get("payer_member_id")
member_ids = body.get("member_ids") or []
item_ids = body.get("item_ids") or []
if not project_id or not payer_member_id or not member_ids or not item_ids:
return (
jsonify(
error="cospend_project_id, payer_member_id, member_ids and item_ids are required"
),
400,
)
group_id = str(uuid.uuid4())
with get_conn() as conn:
conn.execute(
"""INSERT INTO groups
(id, receipt_id, name, cospend_project_id, payer_member_id,
member_ids_json, item_ids_json)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
group_id,
receipt_id,
name,
project_id,
payer_member_id,
json.dumps(member_ids),
json.dumps(item_ids),
),
)
conn.execute("UPDATE receipts SET status = 'grouped' WHERE id = ?", (receipt_id,))
return jsonify(id=group_id), 201
@bp.get("/receipts/<receipt_id>/groups")
def list_groups(receipt_id: str):
with get_conn() as conn:
rows = conn.execute(
"SELECT * FROM groups WHERE receipt_id = ? ORDER BY created_at", (receipt_id,)
).fetchall()
return jsonify([_group_dict(r) for r in rows])
def _group_dict(row) -> dict:
return {
"id": row["id"],
"receipt_id": row["receipt_id"],
"name": row["name"],
"cospend_project_id": row["cospend_project_id"],
"payer_member_id": row["payer_member_id"],
"member_ids": json.loads(row["member_ids_json"]),
"item_ids": json.loads(row["item_ids_json"]),
"status": row["status"],
"share_url": row["share_url"],
"cospend_bill_id": row["cospend_bill_id"],
"error": row["error"],
}
@bp.post("/receipts/<receipt_id>/groups/<group_id>/submit")
def submit_group(receipt_id: str, group_id: str):
"""Highlights selected items, uploads to NC, shares, creates the Cospend bill."""
receipt = _load_receipt(receipt_id)
if receipt is None:
return jsonify(error="receipt not found"), 404
with get_conn() as conn:
row = conn.execute(
"SELECT * FROM groups WHERE id = ? AND receipt_id = ?", (group_id, receipt_id)
).fetchone()
if row is None:
return jsonify(error="group not found"), 404
group = _group_dict(row)
items_by_id = {item["id"]: item for item in receipt["items"]}
selected = [items_by_id[i] for i in group["item_ids"] if i in items_by_id]
if not selected:
return jsonify(error="no matching items on this receipt"), 400
total = round(sum(item["price"] for item in selected), 2)
bboxes = [item["bbox"] for item in selected if item.get("bbox")]
with open(_receipt_image_path(receipt_id), "rb") as f:
original_bytes = f.read()
try:
highlighted = highlight.highlight_items(original_bytes, bboxes)
# Timestamp for human sorting/browsing in NC, group id suffix so two
# submits in the same second (or a retry) never collide/overwrite.
safe_name = re.sub(r"[^A-Za-z0-9_-]+", "-", group["name"]).strip("-") or "group"
filename = f"{datetime.now():%Y-%m-%d_%H%M%S}_{safe_name}_{group_id[:8]}.jpg"
rel_path = nc_client.upload_file(filename, highlighted)
share_url = nc_client.create_public_share(rel_path)
item_lines = "\n".join(f"- {i['label']}: {i['price']}" for i in selected)
# Descriptive title: store name when we have one, else just the
# group name - either way followed by the share link, since
# Cospend's UI only recognizes a bill as having an attachment when
# the link is in `what` (the title); a link in `comment` is just
# plain text and isn't picked up.
store_name = receipt.get("store_name")
label = f"{store_name} ({group['name']})" if store_name else group["name"]
bill_date = receipt.get("date") or date_cls.today().isoformat()
bill_id = cospend_client.create_bill(
project_id=group["cospend_project_id"],
what=f"{label} {share_url}",
amount=total,
payer_id=int(group["payer_member_id"]),
ower_ids=[int(m) for m in group["member_ids"]],
date=bill_date,
comment=item_lines,
)
except Exception as exc: # noqa: BLE001
with get_conn() as conn:
conn.execute(
"UPDATE groups SET status = 'failed', error = ? WHERE id = ?",
(str(exc), group_id),
)
return jsonify(error=f"submit failed: {exc}"), 502
with get_conn() as conn:
conn.execute(
"""UPDATE groups SET status = 'submitted', share_url = ?, cospend_bill_id = ?
WHERE id = ?""",
(share_url, str(bill_id), group_id),
)
return jsonify(share_url=share_url, bill_id=bill_id, amount=total)
def _receipt_image_path(receipt_id: str) -> str:
with get_conn() as conn:
row = conn.execute("SELECT image_path FROM receipts WHERE id = ?", (receipt_id,)).fetchone()
return row["image_path"]
+27
View File
@@ -0,0 +1,27 @@
CREATE TABLE IF NOT EXISTS receipts (
id TEXT PRIMARY KEY,
image_path TEXT NOT NULL,
image_width INTEGER,
image_height INTEGER,
items_json TEXT NOT NULL DEFAULT '[]',
store_name TEXT, -- extracted from the receipt, editable
receipt_date TEXT, -- YYYY-MM-DD, extracted; falls back to
-- today's date if unreadable/unset
status TEXT NOT NULL DEFAULT 'extracted', -- extracted | grouped | done
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS groups (
id TEXT PRIMARY KEY,
receipt_id TEXT NOT NULL REFERENCES receipts(id) ON DELETE CASCADE,
name TEXT NOT NULL,
cospend_project_id TEXT NOT NULL,
payer_member_id TEXT NOT NULL,
member_ids_json TEXT NOT NULL, -- json list of cospend member ids (owers)
item_ids_json TEXT NOT NULL, -- json list of item ids from receipts.items_json
status TEXT NOT NULL DEFAULT 'pending', -- pending | submitted | failed
share_url TEXT,
cospend_bill_id TEXT,
error TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);