| |
| """Decode the bundled examples and check them against the expected answers.""" |
| import json |
| import subprocess |
| import sys |
| from pathlib import Path |
|
|
| HERE = Path(__file__).resolve().parent |
| ROOT = HERE.parent |
|
|
| def main() -> int: |
| cases = json.loads((HERE / "examples.json").read_text(encoding="utf-8")) |
| bad = 0 |
| for case in cases: |
| out = subprocess.run( |
| [sys.executable, str(ROOT / "inference.py"), "--model-dir", str(ROOT), |
| "--image", str(HERE / case["image"]), "--precision", case.get("precision", "fp32")] |
| + [arg for q in case["questions"] for arg in ("--question", q["question"])], |
| capture_output=True, text=True, check=True, |
| ) |
| got = json.loads(out.stdout) |
| if got["record"] != case["expected_record"]: |
| bad += 1 |
| print(f"RECORD MISMATCH {case['image']}: {got['record']} != {case['expected_record']}") |
| for want, have in zip(case["questions"], got["answers"]): |
| if want["answer"] != have["answer"]: |
| bad += 1 |
| print(f"ANSWER MISMATCH {case['image']} {want['question']!r}: " |
| f"{have['answer']!r} != {want['answer']!r}") |
| print(f"{case['image']}: record={got['record']} " |
| f"answers={[a['answer'] for a in got['answers']]}") |
| print("OK" if not bad else f"{bad} MISMATCHES") |
| return 1 if bad else 0 |
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
