ocr-projects-archive / ocr-tech /tests /test_analyze_page_line_failures.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
10.5 kB
from __future__ import annotations
import csv
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "analyze_page_line_failures.py"
spec = importlib.util.spec_from_file_location("analyze_page_line_failures", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _write_alignment(path: Path) -> None:
fields = [
"sample_id",
"kind",
"reference_line_id",
"predicted_line_id",
"reference_bbox",
"predicted_bbox",
"page_index",
"iou",
"reference_text",
"predicted_text",
"text_codepoint_edits",
"text_codepoint_ref_len",
"text_cer_codepoint",
"text_grapheme_edits",
"text_grapheme_ref_len",
"text_cer_grapheme",
]
rows = [
{
"sample_id": "sample",
"kind": "matched",
"reference_line_id": "ref-1",
"predicted_line_id": "pred-1",
"reference_bbox": "[0,0,10,10]",
"predicted_bbox": "[0,0,10,10]",
"page_index": "0",
"iou": "0.95",
"reference_text": "abcdef",
"predicted_text": "abxxxx",
"text_codepoint_edits": "4",
"text_codepoint_ref_len": "6",
"text_cer_codepoint": "0.6666666667",
"text_grapheme_edits": "4",
"text_grapheme_ref_len": "6",
"text_cer_grapheme": "0.6666666667",
},
{
"sample_id": "sample",
"kind": "matched",
"reference_line_id": "ref-2",
"predicted_line_id": "pred-2",
"reference_bbox": "[0,20,10,10]",
"predicted_bbox": "[0,20,10,10]",
"page_index": "0",
"iou": "0.92",
"reference_text": "abc",
"predicted_text": "abc",
"text_codepoint_edits": "0",
"text_codepoint_ref_len": "3",
"text_cer_codepoint": "0.0",
"text_grapheme_edits": "0",
"text_grapheme_ref_len": "3",
"text_cer_grapheme": "0.0",
},
{
"sample_id": "sample",
"kind": "reference_unmatched",
"reference_line_id": "ref-3",
"predicted_line_id": "",
"reference_bbox": "[0,40,10,10]",
"predicted_bbox": "",
"page_index": "0",
"iou": "0.1",
"reference_text": "missing",
"predicted_text": "",
"text_codepoint_edits": "7",
"text_codepoint_ref_len": "7",
"text_cer_codepoint": "1.0",
"text_grapheme_edits": "7",
"text_grapheme_ref_len": "7",
"text_cer_grapheme": "1.0",
},
{
"sample_id": "sample",
"kind": "reference_unmatched",
"reference_line_id": "ref-4",
"predicted_line_id": "",
"reference_bbox": "[0,20,10,10]",
"predicted_bbox": "",
"page_index": "0",
"iou": "0.0",
"reference_text": "merged",
"predicted_text": "",
"text_codepoint_edits": "6",
"text_codepoint_ref_len": "6",
"text_cer_codepoint": "1.0",
"text_grapheme_edits": "6",
"text_grapheme_ref_len": "6",
"text_cer_grapheme": "1.0",
},
{
"sample_id": "sample",
"kind": "prediction_unmatched",
"reference_line_id": "",
"predicted_line_id": "pred-3",
"reference_bbox": "",
"predicted_bbox": "[0,40,8,10]",
"page_index": "0",
"iou": "0.0",
"reference_text": "",
"predicted_text": "extra text",
"text_codepoint_edits": "10",
"text_codepoint_ref_len": "0",
"text_cer_codepoint": "0.0",
"text_grapheme_edits": "10",
"text_grapheme_ref_len": "0",
"text_cer_grapheme": "0.0",
},
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fields)
writer.writeheader()
writer.writerows(rows)
def _write_crop_manifest(path: Path) -> None:
rows = [
{
"line_id": "pred-1",
"crop_path": "crops/pred-1.png",
"crop_sha256": "abc",
"confidence": 0.6,
"crop_xyxy": [0, 0, 10, 10],
"detected_bbox": [0, 0, 10, 10],
"detected_area": 100,
},
{
"line_id": "pred-2",
"crop_path": "crops/pred-2.png",
"crop_sha256": "def",
"confidence": 0.9,
"crop_xyxy": [0, 20, 10, 30],
"detected_bbox": [0, 20, 10, 10],
"detected_area": 100,
},
]
path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
def test_analyze_page_line_failures_buckets_good_geometry_bad_text(tmp_path: Path) -> None:
tool = _load_tool()
alignment = tmp_path / "line-alignment.csv"
crops = tmp_path / "crops.jsonl"
_write_alignment(alignment)
_write_crop_manifest(crops)
result = tool.analyze(
Namespace(
line_alignment_csv=alignment,
crop_manifest=crops,
out=tmp_path / "analysis",
high_iou_threshold=0.85,
bad_codepoint_cer=0.3,
bad_grapheme_cer=0.5,
top_k=10,
)
)
counts = result["summary"]["bucket_counts"]
assert counts["matched_good_geometry_bad_text"] == 1
assert counts["matched_good_geometry_acceptable_text"] == 1
assert counts["reference_unmatched"] == 2
assert counts["prediction_unmatched"] == 1
assert result["top_failures"][0]["reference_line_id"] == "ref-1"
assert result["top_unmatched_references"][0]["reference_line_id"] == "ref-3"
assert result["top_unmatched_references"][0]["best_any_prediction_line_id"] == "pred-3"
assert result["top_unmatched_references"][0]["best_any_prediction_kind"] == "prediction_unmatched"
assert result["top_unmatched_references"][1]["reference_line_id"] == "ref-4"
assert result["top_unmatched_references"][1]["best_any_prediction_line_id"] == "pred-2"
assert result["top_unmatched_references"][1]["best_any_prediction_kind"] == "matched"
assert result["top_unmatched_references"][1]["best_any_prediction_iou"] == 1.0
assert result["top_unmatched_predictions"][0]["predicted_line_id"] == "pred-3"
assert result["near_miss_unmatched_pairs"][0]["reference_line_id"] == "ref-3"
assert result["near_miss_unmatched_pairs"][0]["predicted_line_id"] == "pred-3"
assert result["summary"]["near_miss_unmatched_pair_count_iou_ge_030"] == 1
assert result["summary"]["near_miss_unmatched_pair_count_iou_ge_050"] == 1
overlap_stats = result["summary"]["unmatched_reference_overlap_stats"]
assert overlap_stats["reference_unmatched_count"] == 2
assert overlap_stats["best_any_prediction_iou_ge_030"] == 2
assert overlap_stats["best_any_prediction_iou_ge_050"] == 2
assert overlap_stats["best_any_prediction_source_counts"] == {"matched": 1, "prediction_unmatched": 1}
assert result["top_failures"][0]["crop_path"] == "crops/pred-1.png"
assert result["top_failures"][0]["crop_width_px"] == 10
assert result["top_failures"][0]["crop_height_px"] == 10
assert result["top_failures"][0]["crop_px_per_codepoint"] == 10 / 6
assert result["summary"]["bucket_stats"]["matched_good_geometry_bad_text"]["mean_crop_width_px"] == 10
assert result["summary"]["bucket_stats"]["reference_unmatched"]["mean_reference_y_px"] == 30
assert result["summary"]["bucket_stats"]["prediction_unmatched"]["mean_predicted_width_px"] == 8
assert result["summary"]["crop_width_band_stats"]["000-256"]["count"] == 2
spatial = result["summary"]["unmatched_reference_spatial_stats"]
assert spatial["reference_count"] == 4
assert spatial["unmatched_reference_count"] == 2
assert spatial["by_y_band"]["top"]["reference_count"] == 1
assert spatial["by_y_band"]["middle"]["reference_count"] == 2
assert spatial["by_y_band"]["bottom"]["reference_count"] == 1
assert spatial["by_y_band"]["bottom"]["unmatched_reference_count"] == 1
assert spatial["by_y_band"]["bottom"]["codepoint_error_mass"] == 7
assert spatial["by_x_band"]["center"]["reference_count"] == 4
assert spatial["by_x_band"]["center"]["unmatched_reference_count"] == 2
assert spatial["top_unmatched_bands"][0]["band"] == "bottom/center"
assert spatial["top_unmatched_bands"][0]["codepoint_error_mass"] == 7
error_mass = result["summary"]["codepoint_error_mass"]
assert error_mass["total"] == 27
assert error_mass["by_source"]["matched_good_geometry_bad_text"] == 4
assert error_mass["by_source"]["matched_good_geometry_acceptable_text"] == 0
assert error_mass["by_source"]["missed_reference_text"] == 13
assert error_mass["by_source"]["extra_prediction_text"] == len("extra text")
assert error_mass["dominant_source"] == "missed_reference_text"
assert (tmp_path / "analysis" / "line-failure-analysis.json").exists()
assert (tmp_path / "analysis" / "top-line-failures.csv").exists()
assert (tmp_path / "analysis" / "top-unmatched-references.csv").exists()
assert (tmp_path / "analysis" / "top-unmatched-predictions.csv").exists()
assert (tmp_path / "analysis" / "near-miss-unmatched-pairs.csv").exists()
def test_analyze_page_line_failures_rejects_invalid_iou_threshold(tmp_path: Path) -> None:
tool = _load_tool()
alignment = tmp_path / "line-alignment.csv"
_write_alignment(alignment)
try:
tool.analyze(
Namespace(
line_alignment_csv=alignment,
crop_manifest=None,
out=tmp_path / "analysis",
high_iou_threshold=2.0,
bad_codepoint_cer=0.3,
bad_grapheme_cer=0.5,
top_k=10,
)
)
except ValueError as exc:
assert "high-iou-threshold" in str(exc)
else:
raise AssertionError("expected invalid threshold to fail")