Download ocr-tech/tests/test_analyze_page_line_failures.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_analyze_page_line_failures.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tests/test_analyze_page_line_failures.py
-
curl -L -o test_analyze_page_line_failures.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_analyze_page_line_failures.py
10.5 kB
| from __future__ import annotations | |
| import csv | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from pathlib import Path | |
| def _load_tool(): | |
| path = Path(__file__).resolve().parents[1] / "tools" / "analyze_page_line_failures.py" | |
| spec = importlib.util.spec_from_file_location("analyze_page_line_failures", path) | |
| assert spec is not None | |
| assert spec.loader is not None | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _write_alignment(path: Path) -> None: | |
| fields = [ | |
| "sample_id", | |
| "kind", | |
| "reference_line_id", | |
| "predicted_line_id", | |
| "reference_bbox", | |
| "predicted_bbox", | |
| "page_index", | |
| "iou", | |
| "reference_text", | |
| "predicted_text", | |
| "text_codepoint_edits", | |
| "text_codepoint_ref_len", | |
| "text_cer_codepoint", | |
| "text_grapheme_edits", | |
| "text_grapheme_ref_len", | |
| "text_cer_grapheme", | |
| ] | |
| rows = [ | |
| { | |
| "sample_id": "sample", | |
| "kind": "matched", | |
| "reference_line_id": "ref-1", | |
| "predicted_line_id": "pred-1", | |
| "reference_bbox": "[0,0,10,10]", | |
| "predicted_bbox": "[0,0,10,10]", | |
| "page_index": "0", | |
| "iou": "0.95", | |
| "reference_text": "abcdef", | |
| "predicted_text": "abxxxx", | |
| "text_codepoint_edits": "4", | |
| "text_codepoint_ref_len": "6", | |
| "text_cer_codepoint": "0.6666666667", | |
| "text_grapheme_edits": "4", | |
| "text_grapheme_ref_len": "6", | |
| "text_cer_grapheme": "0.6666666667", | |
| }, | |
| { | |
| "sample_id": "sample", | |
| "kind": "matched", | |
| "reference_line_id": "ref-2", | |
| "predicted_line_id": "pred-2", | |
| "reference_bbox": "[0,20,10,10]", | |
| "predicted_bbox": "[0,20,10,10]", | |
| "page_index": "0", | |
| "iou": "0.92", | |
| "reference_text": "abc", | |
| "predicted_text": "abc", | |
| "text_codepoint_edits": "0", | |
| "text_codepoint_ref_len": "3", | |
| "text_cer_codepoint": "0.0", | |
| "text_grapheme_edits": "0", | |
| "text_grapheme_ref_len": "3", | |
| "text_cer_grapheme": "0.0", | |
| }, | |
| { | |
| "sample_id": "sample", | |
| "kind": "reference_unmatched", | |
| "reference_line_id": "ref-3", | |
| "predicted_line_id": "", | |
| "reference_bbox": "[0,40,10,10]", | |
| "predicted_bbox": "", | |
| "page_index": "0", | |
| "iou": "0.1", | |
| "reference_text": "missing", | |
| "predicted_text": "", | |
| "text_codepoint_edits": "7", | |
| "text_codepoint_ref_len": "7", | |
| "text_cer_codepoint": "1.0", | |
| "text_grapheme_edits": "7", | |
| "text_grapheme_ref_len": "7", | |
| "text_cer_grapheme": "1.0", | |
| }, | |
| { | |
| "sample_id": "sample", | |
| "kind": "reference_unmatched", | |
| "reference_line_id": "ref-4", | |
| "predicted_line_id": "", | |
| "reference_bbox": "[0,20,10,10]", | |
| "predicted_bbox": "", | |
| "page_index": "0", | |
| "iou": "0.0", | |
| "reference_text": "merged", | |
| "predicted_text": "", | |
| "text_codepoint_edits": "6", | |
| "text_codepoint_ref_len": "6", | |
| "text_cer_codepoint": "1.0", | |
| "text_grapheme_edits": "6", | |
| "text_grapheme_ref_len": "6", | |
| "text_cer_grapheme": "1.0", | |
| }, | |
| { | |
| "sample_id": "sample", | |
| "kind": "prediction_unmatched", | |
| "reference_line_id": "", | |
| "predicted_line_id": "pred-3", | |
| "reference_bbox": "", | |
| "predicted_bbox": "[0,40,8,10]", | |
| "page_index": "0", | |
| "iou": "0.0", | |
| "reference_text": "", | |
| "predicted_text": "extra text", | |
| "text_codepoint_edits": "10", | |
| "text_codepoint_ref_len": "0", | |
| "text_cer_codepoint": "0.0", | |
| "text_grapheme_edits": "10", | |
| "text_grapheme_ref_len": "0", | |
| "text_cer_grapheme": "0.0", | |
| }, | |
| ] | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fields) | |
| writer.writeheader() | |
| writer.writerows(rows) | |
| def _write_crop_manifest(path: Path) -> None: | |
| rows = [ | |
| { | |
| "line_id": "pred-1", | |
| "crop_path": "crops/pred-1.png", | |
| "crop_sha256": "abc", | |
| "confidence": 0.6, | |
| "crop_xyxy": [0, 0, 10, 10], | |
| "detected_bbox": [0, 0, 10, 10], | |
| "detected_area": 100, | |
| }, | |
| { | |
| "line_id": "pred-2", | |
| "crop_path": "crops/pred-2.png", | |
| "crop_sha256": "def", | |
| "confidence": 0.9, | |
| "crop_xyxy": [0, 20, 10, 30], | |
| "detected_bbox": [0, 20, 10, 10], | |
| "detected_area": 100, | |
| }, | |
| ] | |
| path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") | |
| def test_analyze_page_line_failures_buckets_good_geometry_bad_text(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| alignment = tmp_path / "line-alignment.csv" | |
| crops = tmp_path / "crops.jsonl" | |
| _write_alignment(alignment) | |
| _write_crop_manifest(crops) | |
| result = tool.analyze( | |
| Namespace( | |
| line_alignment_csv=alignment, | |
| crop_manifest=crops, | |
| out=tmp_path / "analysis", | |
| high_iou_threshold=0.85, | |
| bad_codepoint_cer=0.3, | |
| bad_grapheme_cer=0.5, | |
| top_k=10, | |
| ) | |
| ) | |
| counts = result["summary"]["bucket_counts"] | |
| assert counts["matched_good_geometry_bad_text"] == 1 | |
| assert counts["matched_good_geometry_acceptable_text"] == 1 | |
| assert counts["reference_unmatched"] == 2 | |
| assert counts["prediction_unmatched"] == 1 | |
| assert result["top_failures"][0]["reference_line_id"] == "ref-1" | |
| assert result["top_unmatched_references"][0]["reference_line_id"] == "ref-3" | |
| assert result["top_unmatched_references"][0]["best_any_prediction_line_id"] == "pred-3" | |
| assert result["top_unmatched_references"][0]["best_any_prediction_kind"] == "prediction_unmatched" | |
| assert result["top_unmatched_references"][1]["reference_line_id"] == "ref-4" | |
| assert result["top_unmatched_references"][1]["best_any_prediction_line_id"] == "pred-2" | |
| assert result["top_unmatched_references"][1]["best_any_prediction_kind"] == "matched" | |
| assert result["top_unmatched_references"][1]["best_any_prediction_iou"] == 1.0 | |
| assert result["top_unmatched_predictions"][0]["predicted_line_id"] == "pred-3" | |
| assert result["near_miss_unmatched_pairs"][0]["reference_line_id"] == "ref-3" | |
| assert result["near_miss_unmatched_pairs"][0]["predicted_line_id"] == "pred-3" | |
| assert result["summary"]["near_miss_unmatched_pair_count_iou_ge_030"] == 1 | |
| assert result["summary"]["near_miss_unmatched_pair_count_iou_ge_050"] == 1 | |
| overlap_stats = result["summary"]["unmatched_reference_overlap_stats"] | |
| assert overlap_stats["reference_unmatched_count"] == 2 | |
| assert overlap_stats["best_any_prediction_iou_ge_030"] == 2 | |
| assert overlap_stats["best_any_prediction_iou_ge_050"] == 2 | |
| assert overlap_stats["best_any_prediction_source_counts"] == {"matched": 1, "prediction_unmatched": 1} | |
| assert result["top_failures"][0]["crop_path"] == "crops/pred-1.png" | |
| assert result["top_failures"][0]["crop_width_px"] == 10 | |
| assert result["top_failures"][0]["crop_height_px"] == 10 | |
| assert result["top_failures"][0]["crop_px_per_codepoint"] == 10 / 6 | |
| assert result["summary"]["bucket_stats"]["matched_good_geometry_bad_text"]["mean_crop_width_px"] == 10 | |
| assert result["summary"]["bucket_stats"]["reference_unmatched"]["mean_reference_y_px"] == 30 | |
| assert result["summary"]["bucket_stats"]["prediction_unmatched"]["mean_predicted_width_px"] == 8 | |
| assert result["summary"]["crop_width_band_stats"]["000-256"]["count"] == 2 | |
| spatial = result["summary"]["unmatched_reference_spatial_stats"] | |
| assert spatial["reference_count"] == 4 | |
| assert spatial["unmatched_reference_count"] == 2 | |
| assert spatial["by_y_band"]["top"]["reference_count"] == 1 | |
| assert spatial["by_y_band"]["middle"]["reference_count"] == 2 | |
| assert spatial["by_y_band"]["bottom"]["reference_count"] == 1 | |
| assert spatial["by_y_band"]["bottom"]["unmatched_reference_count"] == 1 | |
| assert spatial["by_y_band"]["bottom"]["codepoint_error_mass"] == 7 | |
| assert spatial["by_x_band"]["center"]["reference_count"] == 4 | |
| assert spatial["by_x_band"]["center"]["unmatched_reference_count"] == 2 | |
| assert spatial["top_unmatched_bands"][0]["band"] == "bottom/center" | |
| assert spatial["top_unmatched_bands"][0]["codepoint_error_mass"] == 7 | |
| error_mass = result["summary"]["codepoint_error_mass"] | |
| assert error_mass["total"] == 27 | |
| assert error_mass["by_source"]["matched_good_geometry_bad_text"] == 4 | |
| assert error_mass["by_source"]["matched_good_geometry_acceptable_text"] == 0 | |
| assert error_mass["by_source"]["missed_reference_text"] == 13 | |
| assert error_mass["by_source"]["extra_prediction_text"] == len("extra text") | |
| assert error_mass["dominant_source"] == "missed_reference_text" | |
| assert (tmp_path / "analysis" / "line-failure-analysis.json").exists() | |
| assert (tmp_path / "analysis" / "top-line-failures.csv").exists() | |
| assert (tmp_path / "analysis" / "top-unmatched-references.csv").exists() | |
| assert (tmp_path / "analysis" / "top-unmatched-predictions.csv").exists() | |
| assert (tmp_path / "analysis" / "near-miss-unmatched-pairs.csv").exists() | |
| def test_analyze_page_line_failures_rejects_invalid_iou_threshold(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| alignment = tmp_path / "line-alignment.csv" | |
| _write_alignment(alignment) | |
| try: | |
| tool.analyze( | |
| Namespace( | |
| line_alignment_csv=alignment, | |
| crop_manifest=None, | |
| out=tmp_path / "analysis", | |
| high_iou_threshold=2.0, | |
| bad_codepoint_cer=0.3, | |
| bad_grapheme_cer=0.5, | |
| top_k=10, | |
| ) | |
| ) | |
| except ValueError as exc: | |
| assert "high-iou-threshold" in str(exc) | |
| else: | |
| raise AssertionError("expected invalid threshold to fail") | |