from __future__ import annotations import csv import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "analyze_page_line_failures.py" spec = importlib.util.spec_from_file_location("analyze_page_line_failures", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _write_alignment(path: Path) -> None: fields = [ "sample_id", "kind", "reference_line_id", "predicted_line_id", "reference_bbox", "predicted_bbox", "page_index", "iou", "reference_text", "predicted_text", "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint", "text_grapheme_edits", "text_grapheme_ref_len", "text_cer_grapheme", ] rows = [ { "sample_id": "sample", "kind": "matched", "reference_line_id": "ref-1", "predicted_line_id": "pred-1", "reference_bbox": "[0,0,10,10]", "predicted_bbox": "[0,0,10,10]", "page_index": "0", "iou": "0.95", "reference_text": "abcdef", "predicted_text": "abxxxx", "text_codepoint_edits": "4", "text_codepoint_ref_len": "6", "text_cer_codepoint": "0.6666666667", "text_grapheme_edits": "4", "text_grapheme_ref_len": "6", "text_cer_grapheme": "0.6666666667", }, { "sample_id": "sample", "kind": "matched", "reference_line_id": "ref-2", "predicted_line_id": "pred-2", "reference_bbox": "[0,20,10,10]", "predicted_bbox": "[0,20,10,10]", "page_index": "0", "iou": "0.92", "reference_text": "abc", "predicted_text": "abc", "text_codepoint_edits": "0", "text_codepoint_ref_len": "3", "text_cer_codepoint": "0.0", "text_grapheme_edits": "0", "text_grapheme_ref_len": "3", "text_cer_grapheme": "0.0", }, { "sample_id": "sample", "kind": "reference_unmatched", "reference_line_id": "ref-3", "predicted_line_id": "", "reference_bbox": "[0,40,10,10]", "predicted_bbox": "", "page_index": "0", "iou": "0.1", "reference_text": "missing", "predicted_text": "", "text_codepoint_edits": "7", "text_codepoint_ref_len": "7", "text_cer_codepoint": "1.0", "text_grapheme_edits": "7", "text_grapheme_ref_len": "7", "text_cer_grapheme": "1.0", }, { "sample_id": "sample", "kind": "reference_unmatched", "reference_line_id": "ref-4", "predicted_line_id": "", "reference_bbox": "[0,20,10,10]", "predicted_bbox": "", "page_index": "0", "iou": "0.0", "reference_text": "merged", "predicted_text": "", "text_codepoint_edits": "6", "text_codepoint_ref_len": "6", "text_cer_codepoint": "1.0", "text_grapheme_edits": "6", "text_grapheme_ref_len": "6", "text_cer_grapheme": "1.0", }, { "sample_id": "sample", "kind": "prediction_unmatched", "reference_line_id": "", "predicted_line_id": "pred-3", "reference_bbox": "", "predicted_bbox": "[0,40,8,10]", "page_index": "0", "iou": "0.0", "reference_text": "", "predicted_text": "extra text", "text_codepoint_edits": "10", "text_codepoint_ref_len": "0", "text_cer_codepoint": "0.0", "text_grapheme_edits": "10", "text_grapheme_ref_len": "0", "text_cer_grapheme": "0.0", }, ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fields) writer.writeheader() writer.writerows(rows) def _write_crop_manifest(path: Path) -> None: rows = [ { "line_id": "pred-1", "crop_path": "crops/pred-1.png", "crop_sha256": "abc", "confidence": 0.6, "crop_xyxy": [0, 0, 10, 10], "detected_bbox": [0, 0, 10, 10], "detected_area": 100, }, { "line_id": "pred-2", "crop_path": "crops/pred-2.png", "crop_sha256": "def", "confidence": 0.9, "crop_xyxy": [0, 20, 10, 30], "detected_bbox": [0, 20, 10, 10], "detected_area": 100, }, ] path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") def test_analyze_page_line_failures_buckets_good_geometry_bad_text(tmp_path: Path) -> None: tool = _load_tool() alignment = tmp_path / "line-alignment.csv" crops = tmp_path / "crops.jsonl" _write_alignment(alignment) _write_crop_manifest(crops) result = tool.analyze( Namespace( line_alignment_csv=alignment, crop_manifest=crops, out=tmp_path / "analysis", high_iou_threshold=0.85, bad_codepoint_cer=0.3, bad_grapheme_cer=0.5, top_k=10, ) ) counts = result["summary"]["bucket_counts"] assert counts["matched_good_geometry_bad_text"] == 1 assert counts["matched_good_geometry_acceptable_text"] == 1 assert counts["reference_unmatched"] == 2 assert counts["prediction_unmatched"] == 1 assert result["top_failures"][0]["reference_line_id"] == "ref-1" assert result["top_unmatched_references"][0]["reference_line_id"] == "ref-3" assert result["top_unmatched_references"][0]["best_any_prediction_line_id"] == "pred-3" assert result["top_unmatched_references"][0]["best_any_prediction_kind"] == "prediction_unmatched" assert result["top_unmatched_references"][1]["reference_line_id"] == "ref-4" assert result["top_unmatched_references"][1]["best_any_prediction_line_id"] == "pred-2" assert result["top_unmatched_references"][1]["best_any_prediction_kind"] == "matched" assert result["top_unmatched_references"][1]["best_any_prediction_iou"] == 1.0 assert result["top_unmatched_predictions"][0]["predicted_line_id"] == "pred-3" assert result["near_miss_unmatched_pairs"][0]["reference_line_id"] == "ref-3" assert result["near_miss_unmatched_pairs"][0]["predicted_line_id"] == "pred-3" assert result["summary"]["near_miss_unmatched_pair_count_iou_ge_030"] == 1 assert result["summary"]["near_miss_unmatched_pair_count_iou_ge_050"] == 1 overlap_stats = result["summary"]["unmatched_reference_overlap_stats"] assert overlap_stats["reference_unmatched_count"] == 2 assert overlap_stats["best_any_prediction_iou_ge_030"] == 2 assert overlap_stats["best_any_prediction_iou_ge_050"] == 2 assert overlap_stats["best_any_prediction_source_counts"] == {"matched": 1, "prediction_unmatched": 1} assert result["top_failures"][0]["crop_path"] == "crops/pred-1.png" assert result["top_failures"][0]["crop_width_px"] == 10 assert result["top_failures"][0]["crop_height_px"] == 10 assert result["top_failures"][0]["crop_px_per_codepoint"] == 10 / 6 assert result["summary"]["bucket_stats"]["matched_good_geometry_bad_text"]["mean_crop_width_px"] == 10 assert result["summary"]["bucket_stats"]["reference_unmatched"]["mean_reference_y_px"] == 30 assert result["summary"]["bucket_stats"]["prediction_unmatched"]["mean_predicted_width_px"] == 8 assert result["summary"]["crop_width_band_stats"]["000-256"]["count"] == 2 spatial = result["summary"]["unmatched_reference_spatial_stats"] assert spatial["reference_count"] == 4 assert spatial["unmatched_reference_count"] == 2 assert spatial["by_y_band"]["top"]["reference_count"] == 1 assert spatial["by_y_band"]["middle"]["reference_count"] == 2 assert spatial["by_y_band"]["bottom"]["reference_count"] == 1 assert spatial["by_y_band"]["bottom"]["unmatched_reference_count"] == 1 assert spatial["by_y_band"]["bottom"]["codepoint_error_mass"] == 7 assert spatial["by_x_band"]["center"]["reference_count"] == 4 assert spatial["by_x_band"]["center"]["unmatched_reference_count"] == 2 assert spatial["top_unmatched_bands"][0]["band"] == "bottom/center" assert spatial["top_unmatched_bands"][0]["codepoint_error_mass"] == 7 error_mass = result["summary"]["codepoint_error_mass"] assert error_mass["total"] == 27 assert error_mass["by_source"]["matched_good_geometry_bad_text"] == 4 assert error_mass["by_source"]["matched_good_geometry_acceptable_text"] == 0 assert error_mass["by_source"]["missed_reference_text"] == 13 assert error_mass["by_source"]["extra_prediction_text"] == len("extra text") assert error_mass["dominant_source"] == "missed_reference_text" assert (tmp_path / "analysis" / "line-failure-analysis.json").exists() assert (tmp_path / "analysis" / "top-line-failures.csv").exists() assert (tmp_path / "analysis" / "top-unmatched-references.csv").exists() assert (tmp_path / "analysis" / "top-unmatched-predictions.csv").exists() assert (tmp_path / "analysis" / "near-miss-unmatched-pairs.csv").exists() def test_analyze_page_line_failures_rejects_invalid_iou_threshold(tmp_path: Path) -> None: tool = _load_tool() alignment = tmp_path / "line-alignment.csv" _write_alignment(alignment) try: tool.analyze( Namespace( line_alignment_csv=alignment, crop_manifest=None, out=tmp_path / "analysis", high_iou_threshold=2.0, bad_codepoint_cer=0.3, bad_grapheme_cer=0.5, top_k=10, ) ) except ValueError as exc: assert "high-iou-threshold" in str(exc) else: raise AssertionError("expected invalid threshold to fail")