#!/usr/bin/env python3 """Ingest native-reader spot-check reviews and emit calibration evidence. The input is the CSV produced by ``prepare_native_spotcheck_bundle.py`` after a reviewer exports completed statuses. This tool does not mutate an eval pack. It validates the review, measures machine-label quality against the reviewed text, and writes auditable artifacts that can justify, or reject, treating a machine-converted pack as spot-checked evidence. """ from __future__ import annotations import argparse import csv import hashlib import json import platform import sys import unicodedata from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any import regex from rapidfuzz.distance import Levenshtein VALID_STATUSES = { "accept_machine_text", "corrected_text", "unreadable", "bad_crop", "not_target_script", } USABLE_STATUSES = {"accept_machine_text", "corrected_text"} UNUSABLE_STATUSES = {"unreadable", "bad_crop", "not_target_script"} REQUIRED_FIELDS = ("review_status", "machine_text") IDENTITY_FIELDS = ("sample_id", "source_image_path", "review_id") DEFAULT_NORMALIZATION = "NFC_preserve_whitespace" @dataclass(frozen=True) class ReviewRow: index: int raw: dict[str, str] status: str effective_text: str | None def _nfc(value: str) -> str: return unicodedata.normalize("NFC", value or "") def _graphemes(value: str) -> list[str]: return regex.findall(r"\X", _nfc(value)) def _hash_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _row_identity(row: dict[str, str]) -> str: for field in IDENTITY_FIELDS: value = row.get(field, "").strip() if value: return f"{field}:{value}" raise ValueError("review row has no sample_id, source_image_path, or review_id") def _read_review_rows(path: Path, *, allow_incomplete: bool) -> list[ReviewRow]: if not path.exists(): raise FileNotFoundError(f"review CSV does not exist: {path}") with path.open("r", encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle) fieldnames = reader.fieldnames or [] missing = [field for field in REQUIRED_FIELDS if field not in fieldnames] if missing: raise ValueError(f"review CSV missing required columns: {missing}") if not any(field in fieldnames for field in IDENTITY_FIELDS): raise ValueError(f"review CSV must contain at least one identity column: {IDENTITY_FIELDS}") rows: list[ReviewRow] = [] seen: set[str] = set() for line_no, row in enumerate(reader, 2): normalized = {key: (value or "") for key, value in row.items() if key is not None} identity = _row_identity(normalized) if identity in seen: raise ValueError(f"{path}:{line_no}: duplicate review identity: {identity}") seen.add(identity) status = normalized.get("review_status", "").strip() if not status: if allow_incomplete: rows.append(ReviewRow(index=len(rows), raw=normalized, status="", effective_text=None)) continue raise ValueError(f"{path}:{line_no}: missing review_status") if status not in VALID_STATUSES: raise ValueError(f"{path}:{line_no}: invalid review_status {status!r}") corrected = normalized.get("corrected_text", "") machine = normalized.get("machine_text", "") if status == "corrected_text": if not corrected: raise ValueError(f"{path}:{line_no}: corrected_text status requires corrected_text") effective = corrected elif status == "accept_machine_text": effective = machine else: effective = None rows.append(ReviewRow(index=len(rows), raw=normalized, status=status, effective_text=effective)) if not rows: raise ValueError(f"review CSV has no rows: {path}") return rows def _score_pair(candidate: str, reference: str) -> tuple[int, int, int, int, bool]: candidate_nfc = _nfc(candidate) reference_nfc = _nfc(reference) cp_edits = Levenshtein.distance(candidate_nfc, reference_nfc) candidate_graph = _graphemes(candidate_nfc) reference_graph = _graphemes(reference_nfc) gr_edits = Levenshtein.distance(candidate_graph, reference_graph) return cp_edits, len(reference_nfc), gr_edits, len(reference_graph), candidate_nfc == reference_nfc def _aggregate_scores(rows: list[ReviewRow], text_field: str) -> dict[str, Any]: scored = [row for row in rows if row.effective_text is not None and row.raw.get(text_field, "") != ""] cp_edits = 0 cp_ref_len = 0 gr_edits = 0 gr_ref_len = 0 exact = 0 for row in scored: cp_ed, cp_len, gr_ed, gr_len, is_exact = _score_pair(row.raw.get(text_field, ""), row.effective_text or "") cp_edits += cp_ed cp_ref_len += cp_len gr_edits += gr_ed gr_ref_len += gr_len exact += int(is_exact) return { "scored_rows": len(scored), "codepoint_edits": cp_edits, "codepoint_ref_len": cp_ref_len, "cer_codepoint_micro": cp_edits / max(cp_ref_len, 1), "grapheme_edits": gr_edits, "grapheme_ref_len": gr_ref_len, "cer_grapheme_micro": gr_edits / max(gr_ref_len, 1), "exact_rows": exact, "exact_rate": exact / max(len(scored), 1), } def _status_counts(rows: list[ReviewRow]) -> dict[str, int]: counts = {status: 0 for status in sorted(VALID_STATUSES)} counts["unreviewed"] = 0 for row in rows: counts[row.status or "unreviewed"] += 1 return counts def _eligibility_reasons( *, rows: list[ReviewRow], usable_rows: int, unusable_rows: int, missing_reviewer_rows: int, machine_scores: dict[str, Any], min_usable_rows: int, max_machine_grapheme_cer: float, max_unusable_rate: float, require_complete: bool, ) -> list[str]: reasons: list[str] = [] unreviewed_rows = sum(1 for row in rows if not row.status) if require_complete and unreviewed_rows: reasons.append(f"unreviewed rows remain: {unreviewed_rows}") if missing_reviewer_rows: reasons.append(f"reviewed rows missing reviewer identity: {missing_reviewer_rows}") if usable_rows < min_usable_rows: reasons.append(f"usable reviewed rows below threshold: {usable_rows} < {min_usable_rows}") unusable_rate = unusable_rows / max(len(rows), 1) if unusable_rate > max_unusable_rate: reasons.append(f"unusable row rate too high: {unusable_rate:.6f} > {max_unusable_rate:.6f}") if machine_scores["cer_grapheme_micro"] > max_machine_grapheme_cer: reasons.append( f"machine-label grapheme CER too high: " f"{machine_scores['cer_grapheme_micro']:.6f} > {max_machine_grapheme_cer:.6f}" ) return reasons def _write_reviewed_labels(path: Path, rows: list[ReviewRow]) -> None: fields = ["sample_id", "source_image_path", "reviewed_text", "reviewed_gt_status", "review_status", "reviewer", "notes"] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t") writer.writeheader() for row in rows: if row.effective_text is None: continue status = "machine_converted_spot_checked" if row.status == "accept_machine_text" else "human_verified" writer.writerow( { "sample_id": row.raw.get("sample_id", ""), "source_image_path": row.raw.get("source_image_path", ""), "reviewed_text": row.effective_text, "reviewed_gt_status": status, "review_status": row.status, "reviewer": row.raw.get("reviewer", ""), "notes": row.raw.get("notes", ""), } ) def _write_manifest_overrides(path: Path, rows: list[ReviewRow]) -> None: with path.open("w", encoding="utf-8") as handle: for row in rows: if row.effective_text is None: payload = { "sample_id": row.raw.get("sample_id", ""), "image_path": row.raw.get("source_image_path", ""), "claim_eligible": False, "review_status": row.status, "reviewer": row.raw.get("reviewer", ""), "notes": row.raw.get("notes", ""), } else: gt_status = "machine_converted_spot_checked" if row.status == "accept_machine_text" else "human_verified" payload = { "sample_id": row.raw.get("sample_id", ""), "image_path": row.raw.get("source_image_path", ""), "text": row.effective_text, "gt_status": gt_status, "claim_eligible": True, "review_status": row.status, "reviewer": row.raw.get("reviewer", ""), "notes": row.raw.get("notes", ""), } handle.write(json.dumps(payload, ensure_ascii=False) + "\n") def _write_markdown(path: Path, payload: dict[str, Any]) -> None: lines = [ "# Native Spot-Check Calibration", "", f"- review rows: {payload['counts']['rows']}", f"- usable rows: {payload['counts']['usable_rows']}", f"- unusable rows: {payload['counts']['unusable_rows']}", f"- unreviewed rows: {payload['counts']['unreviewed_rows']}", f"- machine-label grapheme CER: {payload['machine_label_metrics']['cer_grapheme_micro']:.6f}", f"- machine-label codepoint CER: {payload['machine_label_metrics']['cer_codepoint_micro']:.6f}", f"- machine-label exact rate: {payload['machine_label_metrics']['exact_rate']:.6f}", f"- prediction grapheme CER: {payload['prediction_metrics']['cer_grapheme_micro']:.6f}", f"- claim evidence eligible: {payload['claim_evidence_eligible']}", "", "## Status Counts", "", ] lines.extend(f"- {status}: {count}" for status, count in payload["counts"]["status_counts"].items()) lines.extend(["", "## Eligibility Reasons", ""]) lines.extend([f"- {reason}" for reason in payload["eligibility_reasons"]] or ["- none"]) path.write_text("\n".join(lines) + "\n", encoding="utf-8") def apply_reviews(args: argparse.Namespace) -> dict[str, Any]: rows = _read_review_rows(args.review_file, allow_incomplete=args.allow_incomplete) usable_rows = sum(1 for row in rows if row.status in USABLE_STATUSES) unusable_rows = sum(1 for row in rows if row.status in UNUSABLE_STATUSES) unreviewed_rows = sum(1 for row in rows if not row.status) machine_scores = _aggregate_scores(rows, "machine_text") prediction_scores = _aggregate_scores(rows, "prediction") reviewers = sorted({row.raw.get("reviewer", "").strip() for row in rows if row.raw.get("reviewer", "").strip()}) missing_reviewer_rows = sum(1 for row in rows if row.status and not row.raw.get("reviewer", "").strip()) reasons = _eligibility_reasons( rows=rows, usable_rows=usable_rows, unusable_rows=unusable_rows, missing_reviewer_rows=missing_reviewer_rows, machine_scores=machine_scores, min_usable_rows=args.min_usable_rows, max_machine_grapheme_cer=args.max_machine_grapheme_cer, max_unusable_rate=args.max_unusable_rate, require_complete=not args.allow_incomplete, ) eligible = not reasons if args.out.exists() and any(args.out.iterdir()) and not args.force: raise FileExistsError(f"output directory exists and is not empty: {args.out}") args.out.mkdir(parents=True, exist_ok=True) payload: dict[str, Any] = { "created_at": datetime.now(timezone.utc).isoformat(), "review_file": str(args.review_file), "review_file_sha256": _hash_file(args.review_file), "normalization_profile": DEFAULT_NORMALIZATION, "language": args.language, "script": args.script, "domain": args.domain, "source_gt_status": args.source_gt_status, "recommended_pack_gt_status": "machine_converted_spot_checked" if eligible else args.source_gt_status, "recommended_pack_claim_eligible": bool(eligible), "claim_evidence_eligible": bool(eligible), "eligibility_thresholds": { "min_usable_rows": args.min_usable_rows, "max_machine_grapheme_cer": args.max_machine_grapheme_cer, "max_unusable_rate": args.max_unusable_rate, "allow_incomplete": args.allow_incomplete, }, "eligibility_reasons": reasons, "counts": { "rows": len(rows), "usable_rows": usable_rows, "unusable_rows": unusable_rows, "unreviewed_rows": unreviewed_rows, "missing_reviewer_rows": missing_reviewer_rows, "status_counts": _status_counts(rows), }, "reviewers": reviewers, "machine_label_metrics": machine_scores, "prediction_metrics": prediction_scores, "artifacts": { "reviewed_labels_tsv": "reviewed-labels.tsv", "reviewed_manifest_overrides_jsonl": "reviewed-manifest-overrides.jsonl", }, "machine": { "platform": platform.platform(), "python": sys.version, }, "command": " ".join(sys.argv), } (args.out / "native-spotcheck-calibration.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") _write_markdown(args.out / "native-spotcheck-calibration.md", payload) _write_reviewed_labels(args.out / "reviewed-labels.tsv", rows) _write_manifest_overrides(args.out / "reviewed-manifest-overrides.jsonl", rows) return payload def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--review-file", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--language", default="unknown") parser.add_argument("--script", default="unknown") parser.add_argument("--domain", default="unknown") parser.add_argument("--source-gt-status", default="machine_converted") parser.add_argument("--min-usable-rows", type=int, default=30) parser.add_argument("--max-machine-grapheme-cer", type=float, default=0.02) parser.add_argument("--max-unusable-rate", type=float, default=0.05) parser.add_argument("--allow-incomplete", action=argparse.BooleanOptionalAction, default=False) parser.add_argument("--force", action="store_true") args = parser.parse_args(argv) payload = apply_reviews(args) print( f"wrote {args.out} eligible={payload['claim_evidence_eligible']} " f"machine_graph_CER={payload['machine_label_metrics']['cer_grapheme_micro']:.3f}" ) return 0 if __name__ == "__main__": raise SystemExit(main())