ocr-projects-archive / ocr-tech /tests /test_apply_native_spotcheck_reviews.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
6.3 kB
from __future__ import annotations
import csv
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "apply_native_spotcheck_reviews.py"
spec = importlib.util.spec_from_file_location("apply_native_spotcheck_reviews", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _write_review_csv(path: Path, rows: list[dict[str, str]]) -> None:
fields = [
"review_id",
"sample_id",
"source_image_path",
"machine_text",
"prediction",
"review_status",
"corrected_text",
"reviewer",
"notes",
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fields)
writer.writeheader()
writer.writerows(rows)
def _args(review_file: Path, out: Path, **kwargs):
values = {
"review_file": review_file,
"out": out,
"language": "newa",
"script": "Newa",
"domain": "real_handwritten",
"source_gt_status": "machine_converted",
"min_usable_rows": 2,
"max_machine_grapheme_cer": 0.2,
"max_unusable_rate": 0.5,
"allow_incomplete": False,
"force": False,
}
values.update(kwargs)
return Namespace(**values)
def test_apply_native_spotcheck_reviews_writes_calibration_artifacts(tmp_path: Path) -> None:
tool = _load_tool()
review_file = tmp_path / "reviewed.csv"
_write_review_csv(
review_file,
[
{
"review_id": "review-0001",
"sample_id": "s1",
"source_image_path": "images/1.png",
"machine_text": "abc",
"prediction": "abc",
"review_status": "accept_machine_text",
"corrected_text": "",
"reviewer": "native-a",
"notes": "",
},
{
"review_id": "review-0002",
"sample_id": "s2",
"source_image_path": "images/2.png",
"machine_text": "abc",
"prediction": "abc",
"review_status": "corrected_text",
"corrected_text": "adc",
"reviewer": "native-a",
"notes": "one letter corrected",
},
{
"review_id": "review-0003",
"sample_id": "s3",
"source_image_path": "images/3.png",
"machine_text": "xxx",
"prediction": "xxx",
"review_status": "bad_crop",
"corrected_text": "",
"reviewer": "native-a",
"notes": "",
},
],
)
result = tool.apply_reviews(_args(review_file, tmp_path / "calibration"))
assert result["claim_evidence_eligible"] is True
assert result["recommended_pack_gt_status"] == "machine_converted_spot_checked"
assert result["counts"]["usable_rows"] == 2
assert result["counts"]["unusable_rows"] == 1
assert result["machine_label_metrics"]["grapheme_edits"] == 1
assert result["machine_label_metrics"]["grapheme_ref_len"] == 6
assert (tmp_path / "calibration" / "native-spotcheck-calibration.json").exists()
reviewed_rows = list(
csv.DictReader((tmp_path / "calibration" / "reviewed-labels.tsv").open(encoding="utf-8"), delimiter="\t")
)
assert len(reviewed_rows) == 2
assert reviewed_rows[0]["reviewed_gt_status"] == "machine_converted_spot_checked"
assert reviewed_rows[1]["reviewed_gt_status"] == "human_verified"
overrides = [
json.loads(line)
for line in (tmp_path / "calibration" / "reviewed-manifest-overrides.jsonl").read_text(encoding="utf-8").splitlines()
]
assert overrides[2]["claim_eligible"] is False
def test_apply_native_spotcheck_reviews_rejects_invalid_review_status(tmp_path: Path) -> None:
tool = _load_tool()
review_file = tmp_path / "reviewed.csv"
_write_review_csv(
review_file,
[
{
"review_id": "review-0001",
"sample_id": "s1",
"source_image_path": "images/1.png",
"machine_text": "abc",
"prediction": "abc",
"review_status": "looks_ok",
"corrected_text": "",
"reviewer": "native-a",
"notes": "",
}
],
)
try:
tool.apply_reviews(_args(review_file, tmp_path / "calibration"))
except ValueError as exc:
assert "invalid review_status" in str(exc)
else:
raise AssertionError("expected invalid status to fail")
assert not (tmp_path / "calibration").exists()
def test_apply_native_spotcheck_reviews_requires_reviewer_for_claim_evidence(tmp_path: Path) -> None:
tool = _load_tool()
review_file = tmp_path / "reviewed.csv"
_write_review_csv(
review_file,
[
{
"review_id": "review-0001",
"sample_id": "s1",
"source_image_path": "images/1.png",
"machine_text": "abc",
"prediction": "abc",
"review_status": "accept_machine_text",
"corrected_text": "",
"reviewer": "",
"notes": "",
},
{
"review_id": "review-0002",
"sample_id": "s2",
"source_image_path": "images/2.png",
"machine_text": "def",
"prediction": "def",
"review_status": "accept_machine_text",
"corrected_text": "",
"reviewer": "native-a",
"notes": "",
},
],
)
result = tool.apply_reviews(_args(review_file, tmp_path / "calibration"))
assert result["claim_evidence_eligible"] is False
assert result["counts"]["missing_reviewer_rows"] == 1
assert "reviewed rows missing reviewer identity: 1" in result["eligibility_reasons"]