diff --git a/TODO.md b/TODO.md index 3ea6eb2..b789aac 100644 --- a/TODO.md +++ b/TODO.md @@ -151,12 +151,50 @@ Status legend: [x] fixed+tested, [~] partially addressed, [ ] open/needs input ## Needs more scope / your input before I keep going -- [ ] Enemy type detection needs to be more robust; read the entity id +- [~] Enemy type detection needs to be more robust; read the entity id label so dedup is reliable; detect death from the log. - All three are real computer-vision/OCR feature work (better marker - classification in `map_vision.py`'s `classify_marker`, a new OCR pass - reading each marker's id label off the map screenshot, and a - "# Destroyed" log-scan tied into a dedup key that includes - that read id) rather than bugs with a small fix. Worth its own pass - once there's a batch of the `debug_capture` failure/maybe_map - screenshots above to develop against. + + Started on the id-reading piece: `map_vision.read_marker_id` reads + each marker's own small "#" label (distinct from the big + per-cell grid label `read_cell_label` reads) via the SAME template- + correlation approach as `read_cell_label`, not OCR -- this text + sits over the same aerial-photo backdrop that this module's own + docstring says defeated every detection-based approach tried for + grid labels, so pytesseract (already tried elsewhere in this repo, + `ocr.py`, for a different image domain: flat scanned paper, not + photo-textured) was skipped in favor of the approach already proven + here. Wired end-to-end: `find_markers` -> `Proposal.detected_id` -> + `debug_capture.save_marker_ground_truth`'s JSON. Reads against + `ScreenshotImport.full_image` (sharper than the WORK_W image + detection itself runs against) when available. Crop region and + `MIN_MARKER_ID_SCORE` are a single-screenshot calibration (see + `read_marker_id`'s own docstring) -- UNVALIDATED against a real + ground-truth batch (none of the 6 existing captures have a + confirmed id to check against, they all predate this). New unit + tests (`tests/test_map_vision_marker_id.py`) only cover the + synthetic-render round-trip, not real-screenshot accuracy. + + Measured type-detection reliability against the 6 existing + `marker_ground_truth` captures (72 accepted proposals total, + 2026-08-13): **0/72 (0%) had ANY confident `detected_unit` guess** + -- `classify_marker` returned `None` on every single one, every + side, every capture. Not "guesses wrong" -- never confident enough + to answer at all. Spot-checked directly against one real marker + crop (a hostile Infantry, confirmed by the user): best match was + "Underground Fort" at score 0.376 (Infantry wasn't even in the top + 8), against a `min_score=0.55` floor `classify_marker` requires -- + not a close miss, a real correlation failure. The clean rendered + icon templates `icon_bank()` matches against apparently don't + correlate well with how markers actually look in a real screenshot + (compression/blur/aerial-photo texture underneath), unlike text + glyphs (`read_cell_label`'s measured 0.73-0.87 vs 0.40-0.56) where + the same template-correlation idea works well. Added `unit_score`/ + `unit_margin` to `Proposal`/ground-truth JSON (previously only + pass/fail `unit` was logged) so every future capture shows exactly + how far off a guess was, not just None -- there was no way to tell + "barely missed the bar" from "wildly wrong" before this. + + Death-detection-from-log is still fully unstarted -- no log-parsing + code exists in this repo at all yet, real scope work (find/access + the game's log, agree a "# Destroyed" grammar, wire it + into a dedup key) rather than a quick pass. diff --git a/src/fenigma/app.py b/src/fenigma/app.py index 623695a..8d17051 100644 --- a/src/fenigma/app.py +++ b/src/fenigma/app.py @@ -640,7 +640,11 @@ class MainWindow(Adw.ApplicationWindow): both solutions as ground truth, useful later for improving the grid solver against exactly the case it got wrong.""" if solution is not imp.solution: - debug_capture.save_grid_correction(imp.image, imp.solution, solution) + # full_image over image: sharper source for a human reviewing + # the capture later, same reasoning as save_marker_ground_truth's. + debug_capture.save_grid_correction( + imp.full_image if imp.full_image is not None else imp.image, + imp.solution, solution) imp.solution = solution # A screenshot already on the board (never explicitly dropped, the # user just pasted a new one straight over it) still deserves its @@ -775,7 +779,12 @@ class MainWindow(Adw.ApplicationWindow): moment it can still be tied to this specific image.""" added_targets = [t for t in self.board.targets if t not in imp.baseline_targets] added_allies = [a for a in self.board.allies if a not in imp.baseline_allies] - debug_capture.save_marker_ground_truth(imp.image, imp.proposals, added_targets, added_allies) + # full_image over image: a human checking a detected_id against + # this capture later needs to actually read that tiny text, see + # save_marker_ground_truth's own docstring. + debug_capture.save_marker_ground_truth( + imp.full_image if imp.full_image is not None else imp.image, + imp.proposals, added_targets, added_allies) def _remove_screenshot(self) -> None: """Dropping the screenshot also drops every proposal never accepted: diff --git a/src/fenigma/debug_capture.py b/src/fenigma/debug_capture.py index 0fb1124..07a8d81 100644 --- a/src/fenigma/debug_capture.py +++ b/src/fenigma/debug_capture.py @@ -101,7 +101,17 @@ def save_marker_ground_truth(image, proposals, added_targets=(), added_allies=() matter: a rejected proposal is a false positive to fix, a manually- added unit that had no matching proposal at all is a miss to fix. Skipped entirely if there's nothing to say (no proposals AND no - manually-added units), a screenshot nobody ever looked at units on.""" + manually-added units), a screenshot nobody ever looked at units on. + + Each proposal also carries `detected_id` (map_vision.read_marker_id's + best-effort read of the marker's own "#" id label, see its own + docstring -- not yet validated against a real batch of this exact + ground truth, which is precisely what these captures are for). + `image` should be the sharpest one the caller has (full_image over + the WORK_W-downscaled one, see ScreenshotImport.full_image) so a + human reviewing a capture later can actually read that id text well + enough to judge whether detected_id was right -- not just take the + detector's word for it.""" if not proposals and not added_targets and not added_allies: return None png = _to_png_bytes(image) @@ -120,6 +130,8 @@ def save_marker_ground_truth(image, proposals, added_targets=(), added_allies=() { "side": p.side, "label": p.label, "sub_x": p.sub_x, "sub_y": p.sub_y, "detected_unit": p.unit, "verdict": verdict(p), "confirmed_type": p.confirmed_type, + "detected_id": p.detected_id, + "unit_score": p.unit_score, "unit_margin": p.unit_margin, } for p in proposals ], diff --git a/src/fenigma/map_import.py b/src/fenigma/map_import.py index d3de309..d802da6 100644 --- a/src/fenigma/map_import.py +++ b/src/fenigma/map_import.py @@ -41,6 +41,13 @@ class Proposal: box: tuple accepted: bool = False rejected: bool = False + # classify_marker's own raw numbers behind `unit` (best-match score, + # and its margin over the runner-up) -- unit alone only says whether + # it beat min_score/min_margin, not by how much or how close a call + # it was. Ground truth needs these to tell "confidently wrong" apart + # from "just barely missed the bar", which `unit=None` alone can't. + unit_score: float = 0.0 + unit_margin: float = 0.0 # The TargetType.name actually applied when accepted -- usually just # `unit` translated through icons.target_type_from_icon, but can # differ if the user corrected it via "Accept as...". Set by @@ -48,6 +55,15 @@ class Proposal: # debug_capture.save_marker_ground_truth: `unit` is what the # classifier guessed, this is what the user actually confirmed. confirmed_type: str | None = None + # The marker's own "#" id label, as read off the screenshot by + # map_vision.read_marker_id -- distinct from `label`/sub_x/sub_y + # (the grid CELL this marker is in), this is the small per-unit id + # the game itself draws. None when unread/unconfident (see + # read_marker_id's own docstring: best-effort, not yet validated + # against a real ground-truth batch). Meant for future dedup work + # (see TODO.md) once there's confidence in the read; not otherwise + # consumed yet. + detected_id: str | None = None @property def coord(self) -> str: @@ -95,7 +111,10 @@ class ScreenshotImport: self.proposals = [ Proposal(side=m["side"], label=m["label"], sub_x=m["sub_x"], sub_y=m["sub_y"], unit=m.get("unit"), - centre=m["centre"], box=m["box"]) for m in markers] + centre=m["centre"], box=m["box"], + detected_id=m.get("detected_id"), + unit_score=m.get("unit_score", 0.0), + unit_margin=m.get("unit_margin", 0.0)) for m in markers] return self.proposals def build_overlay(self, px_per_km=150): @@ -200,9 +219,16 @@ class ImportJob: Fills imp.proposals and delivers on_done(imp, error). Its own thread, because the user's grid correction sits between the two phases. + + Marker detection itself always runs against imp.image (WORK_W, + same as solving used); imp.full_image is passed through only for + reading each marker's own tiny id label off a sharper source, see + map_vision.find_markers' own id_img param. """ def work(): - imp.set_proposals(map_vision.find_markers(imp.image, imp.solution)) + id_img = imp.full_image # None is fine, find_markers falls back to imp.image + imp.set_proposals(map_vision.find_markers( + imp.image, imp.solution, id_img=id_img, id_scale=imp.full_image_scale)) return imp, None return self._run(work, on_done, "map-markers") diff --git a/src/fenigma/map_vision.py b/src/fenigma/map_vision.py index ead0899..8625949 100644 --- a/src/fenigma/map_vision.py +++ b/src/fenigma/map_vision.py @@ -486,6 +486,62 @@ def read_cell_label(cell_gray, glyph_fracs=(0.10, 0.13, 0.17)): return best +# Every marker the game draws also carries a small "#" id label just +# above-left of its icon (distinct from the big per-cell grid label +# read_cell_label reads) -- calibrated by eye against a real screenshot +# saved under debug_captures/marker_ground_truth: it sits roughly one +# marker-width to the left and level with the marker's own top edge. +# Observed ids in practice are small (single or double digit); 1-99 +# covers that generously without the search space growing large. +MARKER_ID_CANDIDATES = [f"#{n}" for n in range(1, 100)] +MIN_MARKER_ID_SCORE = 0.55 # unmeasured starting point, see read_marker_id's own docstring + + +def read_marker_id(gray, box, glyph_fracs=(0.30, 0.40, 0.50, 0.60)): + """Which '#' id best explains the pixels just above-left of this + marker? Same template-correlation approach as read_cell_label, and + for the same reason (see this module's own docstring): this text + sits over the same aerial-photo backdrop that defeated every + detection-based approach tried for grid labels, so glyph correlation + against a known-position crop is used here too rather than OCR. + + `box` is the marker's own detected (x, y, w, h), in `gray`'s pixel + space -- the caller is responsible for scaling it if `gray` isn't + the same image the marker was detected in (see find_markers' own + id_img/id_scale params, for reading against a sharper source than + detection ran on). + + Best-effort and NOT validated against a real ground-truth batch yet + (unlike read_cell_label's measured 0.73-0.87 vs 0.40-0.56 -- there's + no equivalent number here): both the crop region and + MIN_MARKER_ID_SCORE are a single-screenshot calibration, expect this + to need retuning once there's a real batch of debug_capture ground + truth with confirmed ids to check against (see TODO.md). Returns + None below the threshold rather than guessing. + """ + x, y, w, h = box + left = max(0, int(x - 1.0 * w)) + top = max(0, int(y - 0.45 * h)) + right = min(gray.shape[1], int(x + 0.65 * w)) + bottom = min(gray.shape[0], int(y + 0.55 * h)) + if right - left < 6 or bottom - top < 6: + return None + patch = np.ascontiguousarray(gray[top:bottom, left:right]) + best = (None, -1.0) + for gf in glyph_fracs: + th = max(6, int(gf * h)) + for cand in MARKER_ID_CANDIDATES: + t = glyph_template(cand, th) + if t is None or t.shape[0] >= patch.shape[0] or t.shape[1] >= patch.shape[1]: + continue + sc = float(cv2.matchTemplate(patch, t, cv2.TM_CCOEFF_NORMED).max()) + if sc > best[1]: + best = (cand, sc) + if best[1] < MIN_MARKER_ID_SCORE: + return None + return best[0].lstrip("#") + + def visible_cells(H, shape, limit=6): """Lattice cells whose centre is on screen, nearest the frame centre first (least perspective distortion, so the easiest to read).""" @@ -961,18 +1017,33 @@ def diamonds(mask, cell_px, shape="diamond"): MARKER_SHAPE = {"hostile": "diamond", "friendly": "rect"} -def find_markers(img, sol): - """-> list of dicts: side, unit, label, sub_x, sub_y, coord, centre, box.""" +def find_markers(img, sol, id_img=None, id_scale=1.0): + """-> list of dicts: side, unit, label, sub_x, sub_y, coord, centre, box, + detected_id. + + `id_img`/`id_scale`: read each marker's small "#" id label (see + read_marker_id) against a sharper source than detection ran on -- + ScreenshotImport.full_image over the WORK_W-downscaled `img`, same + reasoning as build_overlay's own img_scale (id text is tiny; reading + it off the downscaled image loses too much detail). `id_scale` is + id_img's width / img's width. Detection itself (marker + position/shape/color, unit classification) always runs against `img` + -- only the id read benefits from more resolution. Falls back to + reading against `img` itself when id_img is None (still better than + nothing, just at WORK_W's lower detail).""" cell = max(sol.steps) found = [] + id_gray = cv2.cvtColor(id_img if id_img is not None else img, cv2.COLOR_BGR2GRAY) for side, mask in zip(("hostile", "friendly"), marker_masks(img)): for (cx, cy, box) in diamonds(mask, cell, MARKER_SHAPE[side]): c = sol.cell_of(cx, cy) if c is None: continue unit, score, margin = classify_marker(img, box, side) + id_box = box if id_scale == 1.0 else tuple(v * id_scale for v in box) + detected_id = read_marker_id(id_gray, id_box) found.append(dict(side=side, unit=unit, unit_score=score, unit_margin=margin, label=c[0], sub_x=c[1], sub_y=c[2], coord=format_coord(c), - centre=(cx, cy), box=box)) + centre=(cx, cy), box=box, detected_id=detected_id)) return found diff --git a/tests/test_map_vision_marker_id.py b/tests/test_map_vision_marker_id.py new file mode 100644 index 0000000..03bf8d4 --- /dev/null +++ b/tests/test_map_vision_marker_id.py @@ -0,0 +1,54 @@ +"""read_marker_id: template-correlation read of a marker's own small +"#" id label (see map_vision.read_marker_id's own docstring for why +this is template correlation, not OCR -- same reasoning as +read_cell_label). Synthetic image, real font, no fixture screenshot or +the (slow) detection pipeline needed -- just render the label the way +the game does and check it round-trips. +""" +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +from fenigma import map_vision + + +def _render_label(text: str, height: int) -> Image.Image: + """Cream glyph, heavy dark outline, same style glyph_template expects + to correlate against -- see glyph_template's own docstring.""" + font = ImageFont.truetype(str(map_vision.FONT_PATH), height) + pad = height + im = Image.new("L", (height * 4 + pad, height * 2 + pad), 30) # dark "photo" background + ImageDraw.Draw(im).text((pad // 2, pad // 4), text, font=font, fill=230, + stroke_width=max(1, int(height * 0.10)), stroke_fill=0) + return im + + +def test_reads_a_clean_id_label(): + # A marker box roughly where a real one measures (see read_marker_id's + # own calibration note), with a rendered "#8" sitting where the game + # draws it: above-left of the box. + box_w, box_h = 40, 40 + label_h = int(0.45 * box_h) + label_im = _render_label("#8", label_h) + + canvas = Image.new("L", (200, 200), 60) + label_x, label_y = 60, 60 + canvas.paste(label_im, (label_x, label_y)) + gray = np.array(canvas) + + box_x = label_x + int(1.0 * box_w) - 5 # box sits to the right of/below the label + box_y = label_y + int(0.45 * box_h) + box = (box_x, box_y, box_w, box_h) + + assert map_vision.read_marker_id(gray, box) == "8" + + +def test_returns_none_on_a_blank_patch(): + gray = np.full((200, 200), 60, dtype=np.uint8) + box = (100, 100, 40, 40) + assert map_vision.read_marker_id(gray, box) is None + + +def test_returns_none_on_a_degenerate_box_at_the_image_edge(): + gray = np.full((200, 200), 60, dtype=np.uint8) + box = (0, 0, 2, 2) # crop region collapses to nothing usable + assert map_vision.read_marker_id(gray, box) is None