scripts/run_eval.py

"""Run the structural OCR eval harness over a directory of saved outputs.

Each input is a saved provider response (a ``*.json`` file containing a
``document_annotation``), as produced by ``main.py``. Usage::

    python scripts/run_eval.py path/to/outputs
    python scripts/run_eval.py path/to/outputs --json report.json
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

from bl_ocr.eval.harness import evaluate_directory

PROJECT_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_SCHEMA = PROJECT_ROOT / "schema.json"


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("outputs_dir", help="Directory of saved OCR output JSON files.")
    parser.add_argument("--schema", default=str(DEFAULT_SCHEMA), help="Path to schema.json.")
    parser.add_argument("--json", dest="json_out", help="Optional path to write the full JSON report.")
    args = parser.parse_args(argv)

    summary = evaluate_directory(Path(args.outputs_dir), Path(args.schema))
    data = summary.as_dict()

    if args.json_out:
        Path(args.json_out).write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")

    rates = data["rates"]
    print(f"files evaluated: {data['total']}")
    print(f"  parse_pass:      {data['parse_pass']}/{data['total']} ({rates['parse_pass']:.0%})")
    print(f"  schema_pass:     {data['schema_pass']}/{data['total']} ({rates['schema_pass']:.0%})")
    print(f"  schema errors:   {data['schema_error_total']}")
    print(f"  row_count_pass:  {data['row_count_pass']}/{data['total']} ({rates['row_count_pass']:.0%})")
    print(f"  arithmetic_pass: {data['arithmetic_pass']}/{data['total']} ({rates['arithmetic_pass']:.0%})")
    print(f"  critical:        {data['critical']}/{data['total']} ({rates['critical']:.0%})")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())