ocr-projects-archive / ocr-tech /tools /apply_native_spotcheck_reviews.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
15.5 kB
#!/usr/bin/env python3
"""Ingest native-reader spot-check reviews and emit calibration evidence.
The input is the CSV produced by ``prepare_native_spotcheck_bundle.py`` after a
reviewer exports completed statuses. This tool does not mutate an eval pack.
It validates the review, measures machine-label quality against the reviewed
text, and writes auditable artifacts that can justify, or reject, treating a
machine-converted pack as spot-checked evidence.
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import platform
import sys
import unicodedata
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import regex
from rapidfuzz.distance import Levenshtein
VALID_STATUSES = {
"accept_machine_text",
"corrected_text",
"unreadable",
"bad_crop",
"not_target_script",
}
USABLE_STATUSES = {"accept_machine_text", "corrected_text"}
UNUSABLE_STATUSES = {"unreadable", "bad_crop", "not_target_script"}
REQUIRED_FIELDS = ("review_status", "machine_text")
IDENTITY_FIELDS = ("sample_id", "source_image_path", "review_id")
DEFAULT_NORMALIZATION = "NFC_preserve_whitespace"
@dataclass(frozen=True)
class ReviewRow:
index: int
raw: dict[str, str]
status: str
effective_text: str | None
def _nfc(value: str) -> str:
return unicodedata.normalize("NFC", value or "")
def _graphemes(value: str) -> list[str]:
return regex.findall(r"\X", _nfc(value))
def _hash_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _row_identity(row: dict[str, str]) -> str:
for field in IDENTITY_FIELDS:
value = row.get(field, "").strip()
if value:
return f"{field}:{value}"
raise ValueError("review row has no sample_id, source_image_path, or review_id")
def _read_review_rows(path: Path, *, allow_incomplete: bool) -> list[ReviewRow]:
if not path.exists():
raise FileNotFoundError(f"review CSV does not exist: {path}")
with path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle)
fieldnames = reader.fieldnames or []
missing = [field for field in REQUIRED_FIELDS if field not in fieldnames]
if missing:
raise ValueError(f"review CSV missing required columns: {missing}")
if not any(field in fieldnames for field in IDENTITY_FIELDS):
raise ValueError(f"review CSV must contain at least one identity column: {IDENTITY_FIELDS}")
rows: list[ReviewRow] = []
seen: set[str] = set()
for line_no, row in enumerate(reader, 2):
normalized = {key: (value or "") for key, value in row.items() if key is not None}
identity = _row_identity(normalized)
if identity in seen:
raise ValueError(f"{path}:{line_no}: duplicate review identity: {identity}")
seen.add(identity)
status = normalized.get("review_status", "").strip()
if not status:
if allow_incomplete:
rows.append(ReviewRow(index=len(rows), raw=normalized, status="", effective_text=None))
continue
raise ValueError(f"{path}:{line_no}: missing review_status")
if status not in VALID_STATUSES:
raise ValueError(f"{path}:{line_no}: invalid review_status {status!r}")
corrected = normalized.get("corrected_text", "")
machine = normalized.get("machine_text", "")
if status == "corrected_text":
if not corrected:
raise ValueError(f"{path}:{line_no}: corrected_text status requires corrected_text")
effective = corrected
elif status == "accept_machine_text":
effective = machine
else:
effective = None
rows.append(ReviewRow(index=len(rows), raw=normalized, status=status, effective_text=effective))
if not rows:
raise ValueError(f"review CSV has no rows: {path}")
return rows
def _score_pair(candidate: str, reference: str) -> tuple[int, int, int, int, bool]:
candidate_nfc = _nfc(candidate)
reference_nfc = _nfc(reference)
cp_edits = Levenshtein.distance(candidate_nfc, reference_nfc)
candidate_graph = _graphemes(candidate_nfc)
reference_graph = _graphemes(reference_nfc)
gr_edits = Levenshtein.distance(candidate_graph, reference_graph)
return cp_edits, len(reference_nfc), gr_edits, len(reference_graph), candidate_nfc == reference_nfc
def _aggregate_scores(rows: list[ReviewRow], text_field: str) -> dict[str, Any]:
scored = [row for row in rows if row.effective_text is not None and row.raw.get(text_field, "") != ""]
cp_edits = 0
cp_ref_len = 0
gr_edits = 0
gr_ref_len = 0
exact = 0
for row in scored:
cp_ed, cp_len, gr_ed, gr_len, is_exact = _score_pair(row.raw.get(text_field, ""), row.effective_text or "")
cp_edits += cp_ed
cp_ref_len += cp_len
gr_edits += gr_ed
gr_ref_len += gr_len
exact += int(is_exact)
return {
"scored_rows": len(scored),
"codepoint_edits": cp_edits,
"codepoint_ref_len": cp_ref_len,
"cer_codepoint_micro": cp_edits / max(cp_ref_len, 1),
"grapheme_edits": gr_edits,
"grapheme_ref_len": gr_ref_len,
"cer_grapheme_micro": gr_edits / max(gr_ref_len, 1),
"exact_rows": exact,
"exact_rate": exact / max(len(scored), 1),
}
def _status_counts(rows: list[ReviewRow]) -> dict[str, int]:
counts = {status: 0 for status in sorted(VALID_STATUSES)}
counts["unreviewed"] = 0
for row in rows:
counts[row.status or "unreviewed"] += 1
return counts
def _eligibility_reasons(
*,
rows: list[ReviewRow],
usable_rows: int,
unusable_rows: int,
missing_reviewer_rows: int,
machine_scores: dict[str, Any],
min_usable_rows: int,
max_machine_grapheme_cer: float,
max_unusable_rate: float,
require_complete: bool,
) -> list[str]:
reasons: list[str] = []
unreviewed_rows = sum(1 for row in rows if not row.status)
if require_complete and unreviewed_rows:
reasons.append(f"unreviewed rows remain: {unreviewed_rows}")
if missing_reviewer_rows:
reasons.append(f"reviewed rows missing reviewer identity: {missing_reviewer_rows}")
if usable_rows < min_usable_rows:
reasons.append(f"usable reviewed rows below threshold: {usable_rows} < {min_usable_rows}")
unusable_rate = unusable_rows / max(len(rows), 1)
if unusable_rate > max_unusable_rate:
reasons.append(f"unusable row rate too high: {unusable_rate:.6f} > {max_unusable_rate:.6f}")
if machine_scores["cer_grapheme_micro"] > max_machine_grapheme_cer:
reasons.append(
f"machine-label grapheme CER too high: "
f"{machine_scores['cer_grapheme_micro']:.6f} > {max_machine_grapheme_cer:.6f}"
)
return reasons
def _write_reviewed_labels(path: Path, rows: list[ReviewRow]) -> None:
fields = ["sample_id", "source_image_path", "reviewed_text", "reviewed_gt_status", "review_status", "reviewer", "notes"]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t")
writer.writeheader()
for row in rows:
if row.effective_text is None:
continue
status = "machine_converted_spot_checked" if row.status == "accept_machine_text" else "human_verified"
writer.writerow(
{
"sample_id": row.raw.get("sample_id", ""),
"source_image_path": row.raw.get("source_image_path", ""),
"reviewed_text": row.effective_text,
"reviewed_gt_status": status,
"review_status": row.status,
"reviewer": row.raw.get("reviewer", ""),
"notes": row.raw.get("notes", ""),
}
)
def _write_manifest_overrides(path: Path, rows: list[ReviewRow]) -> None:
with path.open("w", encoding="utf-8") as handle:
for row in rows:
if row.effective_text is None:
payload = {
"sample_id": row.raw.get("sample_id", ""),
"image_path": row.raw.get("source_image_path", ""),
"claim_eligible": False,
"review_status": row.status,
"reviewer": row.raw.get("reviewer", ""),
"notes": row.raw.get("notes", ""),
}
else:
gt_status = "machine_converted_spot_checked" if row.status == "accept_machine_text" else "human_verified"
payload = {
"sample_id": row.raw.get("sample_id", ""),
"image_path": row.raw.get("source_image_path", ""),
"text": row.effective_text,
"gt_status": gt_status,
"claim_eligible": True,
"review_status": row.status,
"reviewer": row.raw.get("reviewer", ""),
"notes": row.raw.get("notes", ""),
}
handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
def _write_markdown(path: Path, payload: dict[str, Any]) -> None:
lines = [
"# Native Spot-Check Calibration",
"",
f"- review rows: {payload['counts']['rows']}",
f"- usable rows: {payload['counts']['usable_rows']}",
f"- unusable rows: {payload['counts']['unusable_rows']}",
f"- unreviewed rows: {payload['counts']['unreviewed_rows']}",
f"- machine-label grapheme CER: {payload['machine_label_metrics']['cer_grapheme_micro']:.6f}",
f"- machine-label codepoint CER: {payload['machine_label_metrics']['cer_codepoint_micro']:.6f}",
f"- machine-label exact rate: {payload['machine_label_metrics']['exact_rate']:.6f}",
f"- prediction grapheme CER: {payload['prediction_metrics']['cer_grapheme_micro']:.6f}",
f"- claim evidence eligible: {payload['claim_evidence_eligible']}",
"",
"## Status Counts",
"",
]
lines.extend(f"- {status}: {count}" for status, count in payload["counts"]["status_counts"].items())
lines.extend(["", "## Eligibility Reasons", ""])
lines.extend([f"- {reason}" for reason in payload["eligibility_reasons"]] or ["- none"])
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def apply_reviews(args: argparse.Namespace) -> dict[str, Any]:
rows = _read_review_rows(args.review_file, allow_incomplete=args.allow_incomplete)
usable_rows = sum(1 for row in rows if row.status in USABLE_STATUSES)
unusable_rows = sum(1 for row in rows if row.status in UNUSABLE_STATUSES)
unreviewed_rows = sum(1 for row in rows if not row.status)
machine_scores = _aggregate_scores(rows, "machine_text")
prediction_scores = _aggregate_scores(rows, "prediction")
reviewers = sorted({row.raw.get("reviewer", "").strip() for row in rows if row.raw.get("reviewer", "").strip()})
missing_reviewer_rows = sum(1 for row in rows if row.status and not row.raw.get("reviewer", "").strip())
reasons = _eligibility_reasons(
rows=rows,
usable_rows=usable_rows,
unusable_rows=unusable_rows,
missing_reviewer_rows=missing_reviewer_rows,
machine_scores=machine_scores,
min_usable_rows=args.min_usable_rows,
max_machine_grapheme_cer=args.max_machine_grapheme_cer,
max_unusable_rate=args.max_unusable_rate,
require_complete=not args.allow_incomplete,
)
eligible = not reasons
if args.out.exists() and any(args.out.iterdir()) and not args.force:
raise FileExistsError(f"output directory exists and is not empty: {args.out}")
args.out.mkdir(parents=True, exist_ok=True)
payload: dict[str, Any] = {
"created_at": datetime.now(timezone.utc).isoformat(),
"review_file": str(args.review_file),
"review_file_sha256": _hash_file(args.review_file),
"normalization_profile": DEFAULT_NORMALIZATION,
"language": args.language,
"script": args.script,
"domain": args.domain,
"source_gt_status": args.source_gt_status,
"recommended_pack_gt_status": "machine_converted_spot_checked" if eligible else args.source_gt_status,
"recommended_pack_claim_eligible": bool(eligible),
"claim_evidence_eligible": bool(eligible),
"eligibility_thresholds": {
"min_usable_rows": args.min_usable_rows,
"max_machine_grapheme_cer": args.max_machine_grapheme_cer,
"max_unusable_rate": args.max_unusable_rate,
"allow_incomplete": args.allow_incomplete,
},
"eligibility_reasons": reasons,
"counts": {
"rows": len(rows),
"usable_rows": usable_rows,
"unusable_rows": unusable_rows,
"unreviewed_rows": unreviewed_rows,
"missing_reviewer_rows": missing_reviewer_rows,
"status_counts": _status_counts(rows),
},
"reviewers": reviewers,
"machine_label_metrics": machine_scores,
"prediction_metrics": prediction_scores,
"artifacts": {
"reviewed_labels_tsv": "reviewed-labels.tsv",
"reviewed_manifest_overrides_jsonl": "reviewed-manifest-overrides.jsonl",
},
"machine": {
"platform": platform.platform(),
"python": sys.version,
},
"command": " ".join(sys.argv),
}
(args.out / "native-spotcheck-calibration.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_markdown(args.out / "native-spotcheck-calibration.md", payload)
_write_reviewed_labels(args.out / "reviewed-labels.tsv", rows)
_write_manifest_overrides(args.out / "reviewed-manifest-overrides.jsonl", rows)
return payload
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--review-file", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--language", default="unknown")
parser.add_argument("--script", default="unknown")
parser.add_argument("--domain", default="unknown")
parser.add_argument("--source-gt-status", default="machine_converted")
parser.add_argument("--min-usable-rows", type=int, default=30)
parser.add_argument("--max-machine-grapheme-cer", type=float, default=0.02)
parser.add_argument("--max-unusable-rate", type=float, default=0.05)
parser.add_argument("--allow-incomplete", action=argparse.BooleanOptionalAction, default=False)
parser.add_argument("--force", action="store_true")
args = parser.parse_args(argv)
payload = apply_reviews(args)
print(
f"wrote {args.out} eligible={payload['claim_evidence_eligible']} "
f"machine_graph_CER={payload['machine_label_metrics']['cer_grapheme_micro']:.3f}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())