ocr-projects-archive / OCR_INDRA /tools /exp /a_export_gt_boxes.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
4.92 kB
"""Experiment A adapter #1: ground-truth line boxes + text → eval schema.
Produces the boxes jsonl consumed by run_rec.py / eval_indra.py --boxes:
{"image": <page path>, "lines": [{"i": 0, "text": "<GT>", "box": [x0,y0,x1,y1]}, ...]}
Subcommands:
validate schema/lint an existing boxes file (counts, box sanity, NFC)
normalize accept {"file"|"image", "lines":[{text,box}]} variants → eval schema
merge boxes-only jsonl + text-only jsonl ({"image","text"} or {"image","lines"})
keyed by image basename → final boxes jsonl with GT text
FROM-DETECTOR: pasting a script that runs the DEPLOYED PP-OCRv5 detector on
eval pages and dumps raw boxes here is ON PURPOSE still a plug (probe 4):
once we know the deployed pipeline's output format, its det boxes jsonl can be
normalized with `normalize`; until then no guessing happens in code.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import read_jsonl, write_jsonl # noqa: E402
def _canon_image_key(p: str) -> str:
return Path(p).name.lower()
def validate(path: Path, max_dump: int = 5) -> None:
from common import norm
bad = []
for n, r in enumerate(read_jsonl(path)):
img = r.get("image") or r.get("file")
if not img:
bad.append((n, "no image/file field"))
continue
lines = r.get("lines") or []
if not lines:
bad.append((n, f"no lines for {img}"))
continue
for li, ln in enumerate(lines):
bx = ln.get("box")
if not bx or len(bx) != 4 or not all(float(v) >= 0 for v in bx):
bad.append((n, f"{img}: line {li} bad box {bx}"))
break
if bx[2] <= bx[0] or bx[3] <= bx[1]:
bad.append((n, f"{img}: line {li} empty box {bx}"))
break
if ln.get("text") is None:
bad.append((n, f"{img}: line {li} missing text"))
break
total_lines = sum(len(r.get("lines") or []) for r in read_jsonl(path))
print(f"validated {path}: {len(read_jsonl(path))} pages / {total_lines} lines; "
f"{len(bad)} problem(s)")
for b in bad[:max_dump]:
print(" ", *b)
from common import norm
if bad:
raise SystemExit(1)
def normalize(path: Path, out: Path) -> None:
records = []
for r in read_jsonl(path):
img = r.get("image") or r.get("file")
lines = []
for i, ln in enumerate(r.get("lines", [])):
lines.append({
"i": i,
"text": ln.get("text", ""),
"box": [float(v) for v in ln["box"][:4]],
})
records.append({"image": img, "lines": lines})
write_jsonl(out, records)
print(f"normalized -> {out} ({len(records)} pages)")
def merge(boxes_path: Path, text_path: Path, out: Path) -> None:
text_map = {}
tr = read_jsonl(text_path)
for r in tr:
key = _canon_image_key(r.get("image") or r.get("file") or "")
if r.get("text") is not None: # one string per image
text_map[key] = [r["text"]]
else: # per-line texts
text_map[key] = [ln["text"] for ln in r.get("lines", [])]
out_records, missing = [], 0
for r in read_jsonl(boxes_path):
img = r.get("image") or r.get("file")
key = _canon_image_key(img)
texts = text_map.get(key)
if texts is None:
missing += 1
texts = [None] * len(r.get("lines", []))
lines = []
for i, ln in enumerate(r.get("lines", [])):
t = texts[i] if i < len(texts) else None
if t is None:
raise SystemExit(f"{img}: line {i} has no GT text — fix the text file first")
lines.append({"i": i, "text": t, "box": [float(v) for v in ln["box"][:4]]})
out_records.append({"image": img, "lines": lines})
write_jsonl(out, out_records)
print(f"merged -> {out}: {len(out_records)} pages, {missing} images lacked texts")
if missing:
raise SystemExit("every image MUST carry GT text (or be deliberately excluded)")
def main(argv=None):
ap = argparse.ArgumentParser("a_export_gt_boxes")
ap.add_argument("cmd", choices=["validate", "normalize", "merge"])
ap.add_argument("input", type=Path)
ap.add_argument("--text", type=Path, default=None, help="text-only jsonl for merge")
ap.add_argument("--out", type=Path, default=None)
args = ap.parse_args(argv)
if args.cmd == "validate":
validate(args.input)
elif args.cmd == "normalize":
normalize(args.input, args.out or args.input.with_name("boxes.eval.jsonl"))
else:
merge(args.input, args.text, args.out or args.input.with_name("boxes.gt.jsonl"))
if __name__ == "__main__":
main()