"""Experiment A adapter #1: ground-truth line boxes + text → eval schema. Produces the boxes jsonl consumed by run_rec.py / eval_indra.py --boxes: {"image": , "lines": [{"i": 0, "text": "", "box": [x0,y0,x1,y1]}, ...]} Subcommands: validate schema/lint an existing boxes file (counts, box sanity, NFC) normalize accept {"file"|"image", "lines":[{text,box}]} variants → eval schema merge boxes-only jsonl + text-only jsonl ({"image","text"} or {"image","lines"}) keyed by image basename → final boxes jsonl with GT text FROM-DETECTOR: pasting a script that runs the DEPLOYED PP-OCRv5 detector on eval pages and dumps raw boxes here is ON PURPOSE still a plug (probe 4): once we know the deployed pipeline's output format, its det boxes jsonl can be normalized with `normalize`; until then no guessing happens in code. """ from __future__ import annotations import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import read_jsonl, write_jsonl # noqa: E402 def _canon_image_key(p: str) -> str: return Path(p).name.lower() def validate(path: Path, max_dump: int = 5) -> None: from common import norm bad = [] for n, r in enumerate(read_jsonl(path)): img = r.get("image") or r.get("file") if not img: bad.append((n, "no image/file field")) continue lines = r.get("lines") or [] if not lines: bad.append((n, f"no lines for {img}")) continue for li, ln in enumerate(lines): bx = ln.get("box") if not bx or len(bx) != 4 or not all(float(v) >= 0 for v in bx): bad.append((n, f"{img}: line {li} bad box {bx}")) break if bx[2] <= bx[0] or bx[3] <= bx[1]: bad.append((n, f"{img}: line {li} empty box {bx}")) break if ln.get("text") is None: bad.append((n, f"{img}: line {li} missing text")) break total_lines = sum(len(r.get("lines") or []) for r in read_jsonl(path)) print(f"validated {path}: {len(read_jsonl(path))} pages / {total_lines} lines; " f"{len(bad)} problem(s)") for b in bad[:max_dump]: print(" ", *b) from common import norm if bad: raise SystemExit(1) def normalize(path: Path, out: Path) -> None: records = [] for r in read_jsonl(path): img = r.get("image") or r.get("file") lines = [] for i, ln in enumerate(r.get("lines", [])): lines.append({ "i": i, "text": ln.get("text", ""), "box": [float(v) for v in ln["box"][:4]], }) records.append({"image": img, "lines": lines}) write_jsonl(out, records) print(f"normalized -> {out} ({len(records)} pages)") def merge(boxes_path: Path, text_path: Path, out: Path) -> None: text_map = {} tr = read_jsonl(text_path) for r in tr: key = _canon_image_key(r.get("image") or r.get("file") or "") if r.get("text") is not None: # one string per image text_map[key] = [r["text"]] else: # per-line texts text_map[key] = [ln["text"] for ln in r.get("lines", [])] out_records, missing = [], 0 for r in read_jsonl(boxes_path): img = r.get("image") or r.get("file") key = _canon_image_key(img) texts = text_map.get(key) if texts is None: missing += 1 texts = [None] * len(r.get("lines", [])) lines = [] for i, ln in enumerate(r.get("lines", [])): t = texts[i] if i < len(texts) else None if t is None: raise SystemExit(f"{img}: line {i} has no GT text — fix the text file first") lines.append({"i": i, "text": t, "box": [float(v) for v in ln["box"][:4]]}) out_records.append({"image": img, "lines": lines}) write_jsonl(out, out_records) print(f"merged -> {out}: {len(out_records)} pages, {missing} images lacked texts") if missing: raise SystemExit("every image MUST carry GT text (or be deliberately excluded)") def main(argv=None): ap = argparse.ArgumentParser("a_export_gt_boxes") ap.add_argument("cmd", choices=["validate", "normalize", "merge"]) ap.add_argument("input", type=Path) ap.add_argument("--text", type=Path, default=None, help="text-only jsonl for merge") ap.add_argument("--out", type=Path, default=None) args = ap.parse_args(argv) if args.cmd == "validate": validate(args.input) elif args.cmd == "normalize": normalize(args.input, args.out or args.input.with_name("boxes.eval.jsonl")) else: merge(args.input, args.text, args.out or args.input.with_name("boxes.gt.jsonl")) if __name__ == "__main__": main()