Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/apply_native_spotcheck_reviews.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_native_spotcheck_reviews.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/apply_native_spotcheck_reviews.py
-
curl -L -o apply_native_spotcheck_reviews.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_native_spotcheck_reviews.py
15.5 kB
| #!/usr/bin/env python3 | |
| """Ingest native-reader spot-check reviews and emit calibration evidence. | |
| The input is the CSV produced by ``prepare_native_spotcheck_bundle.py`` after a | |
| reviewer exports completed statuses. This tool does not mutate an eval pack. | |
| It validates the review, measures machine-label quality against the reviewed | |
| text, and writes auditable artifacts that can justify, or reject, treating a | |
| machine-converted pack as spot-checked evidence. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import platform | |
| import sys | |
| import unicodedata | |
| from dataclasses import dataclass | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| import regex | |
| from rapidfuzz.distance import Levenshtein | |
| VALID_STATUSES = { | |
| "accept_machine_text", | |
| "corrected_text", | |
| "unreadable", | |
| "bad_crop", | |
| "not_target_script", | |
| } | |
| USABLE_STATUSES = {"accept_machine_text", "corrected_text"} | |
| UNUSABLE_STATUSES = {"unreadable", "bad_crop", "not_target_script"} | |
| REQUIRED_FIELDS = ("review_status", "machine_text") | |
| IDENTITY_FIELDS = ("sample_id", "source_image_path", "review_id") | |
| DEFAULT_NORMALIZATION = "NFC_preserve_whitespace" | |
| class ReviewRow: | |
| index: int | |
| raw: dict[str, str] | |
| status: str | |
| effective_text: str | None | |
| def _nfc(value: str) -> str: | |
| return unicodedata.normalize("NFC", value or "") | |
| def _graphemes(value: str) -> list[str]: | |
| return regex.findall(r"\X", _nfc(value)) | |
| def _hash_file(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def _row_identity(row: dict[str, str]) -> str: | |
| for field in IDENTITY_FIELDS: | |
| value = row.get(field, "").strip() | |
| if value: | |
| return f"{field}:{value}" | |
| raise ValueError("review row has no sample_id, source_image_path, or review_id") | |
| def _read_review_rows(path: Path, *, allow_incomplete: bool) -> list[ReviewRow]: | |
| if not path.exists(): | |
| raise FileNotFoundError(f"review CSV does not exist: {path}") | |
| with path.open("r", encoding="utf-8", newline="") as handle: | |
| reader = csv.DictReader(handle) | |
| fieldnames = reader.fieldnames or [] | |
| missing = [field for field in REQUIRED_FIELDS if field not in fieldnames] | |
| if missing: | |
| raise ValueError(f"review CSV missing required columns: {missing}") | |
| if not any(field in fieldnames for field in IDENTITY_FIELDS): | |
| raise ValueError(f"review CSV must contain at least one identity column: {IDENTITY_FIELDS}") | |
| rows: list[ReviewRow] = [] | |
| seen: set[str] = set() | |
| for line_no, row in enumerate(reader, 2): | |
| normalized = {key: (value or "") for key, value in row.items() if key is not None} | |
| identity = _row_identity(normalized) | |
| if identity in seen: | |
| raise ValueError(f"{path}:{line_no}: duplicate review identity: {identity}") | |
| seen.add(identity) | |
| status = normalized.get("review_status", "").strip() | |
| if not status: | |
| if allow_incomplete: | |
| rows.append(ReviewRow(index=len(rows), raw=normalized, status="", effective_text=None)) | |
| continue | |
| raise ValueError(f"{path}:{line_no}: missing review_status") | |
| if status not in VALID_STATUSES: | |
| raise ValueError(f"{path}:{line_no}: invalid review_status {status!r}") | |
| corrected = normalized.get("corrected_text", "") | |
| machine = normalized.get("machine_text", "") | |
| if status == "corrected_text": | |
| if not corrected: | |
| raise ValueError(f"{path}:{line_no}: corrected_text status requires corrected_text") | |
| effective = corrected | |
| elif status == "accept_machine_text": | |
| effective = machine | |
| else: | |
| effective = None | |
| rows.append(ReviewRow(index=len(rows), raw=normalized, status=status, effective_text=effective)) | |
| if not rows: | |
| raise ValueError(f"review CSV has no rows: {path}") | |
| return rows | |
| def _score_pair(candidate: str, reference: str) -> tuple[int, int, int, int, bool]: | |
| candidate_nfc = _nfc(candidate) | |
| reference_nfc = _nfc(reference) | |
| cp_edits = Levenshtein.distance(candidate_nfc, reference_nfc) | |
| candidate_graph = _graphemes(candidate_nfc) | |
| reference_graph = _graphemes(reference_nfc) | |
| gr_edits = Levenshtein.distance(candidate_graph, reference_graph) | |
| return cp_edits, len(reference_nfc), gr_edits, len(reference_graph), candidate_nfc == reference_nfc | |
| def _aggregate_scores(rows: list[ReviewRow], text_field: str) -> dict[str, Any]: | |
| scored = [row for row in rows if row.effective_text is not None and row.raw.get(text_field, "") != ""] | |
| cp_edits = 0 | |
| cp_ref_len = 0 | |
| gr_edits = 0 | |
| gr_ref_len = 0 | |
| exact = 0 | |
| for row in scored: | |
| cp_ed, cp_len, gr_ed, gr_len, is_exact = _score_pair(row.raw.get(text_field, ""), row.effective_text or "") | |
| cp_edits += cp_ed | |
| cp_ref_len += cp_len | |
| gr_edits += gr_ed | |
| gr_ref_len += gr_len | |
| exact += int(is_exact) | |
| return { | |
| "scored_rows": len(scored), | |
| "codepoint_edits": cp_edits, | |
| "codepoint_ref_len": cp_ref_len, | |
| "cer_codepoint_micro": cp_edits / max(cp_ref_len, 1), | |
| "grapheme_edits": gr_edits, | |
| "grapheme_ref_len": gr_ref_len, | |
| "cer_grapheme_micro": gr_edits / max(gr_ref_len, 1), | |
| "exact_rows": exact, | |
| "exact_rate": exact / max(len(scored), 1), | |
| } | |
| def _status_counts(rows: list[ReviewRow]) -> dict[str, int]: | |
| counts = {status: 0 for status in sorted(VALID_STATUSES)} | |
| counts["unreviewed"] = 0 | |
| for row in rows: | |
| counts[row.status or "unreviewed"] += 1 | |
| return counts | |
| def _eligibility_reasons( | |
| *, | |
| rows: list[ReviewRow], | |
| usable_rows: int, | |
| unusable_rows: int, | |
| missing_reviewer_rows: int, | |
| machine_scores: dict[str, Any], | |
| min_usable_rows: int, | |
| max_machine_grapheme_cer: float, | |
| max_unusable_rate: float, | |
| require_complete: bool, | |
| ) -> list[str]: | |
| reasons: list[str] = [] | |
| unreviewed_rows = sum(1 for row in rows if not row.status) | |
| if require_complete and unreviewed_rows: | |
| reasons.append(f"unreviewed rows remain: {unreviewed_rows}") | |
| if missing_reviewer_rows: | |
| reasons.append(f"reviewed rows missing reviewer identity: {missing_reviewer_rows}") | |
| if usable_rows < min_usable_rows: | |
| reasons.append(f"usable reviewed rows below threshold: {usable_rows} < {min_usable_rows}") | |
| unusable_rate = unusable_rows / max(len(rows), 1) | |
| if unusable_rate > max_unusable_rate: | |
| reasons.append(f"unusable row rate too high: {unusable_rate:.6f} > {max_unusable_rate:.6f}") | |
| if machine_scores["cer_grapheme_micro"] > max_machine_grapheme_cer: | |
| reasons.append( | |
| f"machine-label grapheme CER too high: " | |
| f"{machine_scores['cer_grapheme_micro']:.6f} > {max_machine_grapheme_cer:.6f}" | |
| ) | |
| return reasons | |
| def _write_reviewed_labels(path: Path, rows: list[ReviewRow]) -> None: | |
| fields = ["sample_id", "source_image_path", "reviewed_text", "reviewed_gt_status", "review_status", "reviewer", "notes"] | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t") | |
| writer.writeheader() | |
| for row in rows: | |
| if row.effective_text is None: | |
| continue | |
| status = "machine_converted_spot_checked" if row.status == "accept_machine_text" else "human_verified" | |
| writer.writerow( | |
| { | |
| "sample_id": row.raw.get("sample_id", ""), | |
| "source_image_path": row.raw.get("source_image_path", ""), | |
| "reviewed_text": row.effective_text, | |
| "reviewed_gt_status": status, | |
| "review_status": row.status, | |
| "reviewer": row.raw.get("reviewer", ""), | |
| "notes": row.raw.get("notes", ""), | |
| } | |
| ) | |
| def _write_manifest_overrides(path: Path, rows: list[ReviewRow]) -> None: | |
| with path.open("w", encoding="utf-8") as handle: | |
| for row in rows: | |
| if row.effective_text is None: | |
| payload = { | |
| "sample_id": row.raw.get("sample_id", ""), | |
| "image_path": row.raw.get("source_image_path", ""), | |
| "claim_eligible": False, | |
| "review_status": row.status, | |
| "reviewer": row.raw.get("reviewer", ""), | |
| "notes": row.raw.get("notes", ""), | |
| } | |
| else: | |
| gt_status = "machine_converted_spot_checked" if row.status == "accept_machine_text" else "human_verified" | |
| payload = { | |
| "sample_id": row.raw.get("sample_id", ""), | |
| "image_path": row.raw.get("source_image_path", ""), | |
| "text": row.effective_text, | |
| "gt_status": gt_status, | |
| "claim_eligible": True, | |
| "review_status": row.status, | |
| "reviewer": row.raw.get("reviewer", ""), | |
| "notes": row.raw.get("notes", ""), | |
| } | |
| handle.write(json.dumps(payload, ensure_ascii=False) + "\n") | |
| def _write_markdown(path: Path, payload: dict[str, Any]) -> None: | |
| lines = [ | |
| "# Native Spot-Check Calibration", | |
| "", | |
| f"- review rows: {payload['counts']['rows']}", | |
| f"- usable rows: {payload['counts']['usable_rows']}", | |
| f"- unusable rows: {payload['counts']['unusable_rows']}", | |
| f"- unreviewed rows: {payload['counts']['unreviewed_rows']}", | |
| f"- machine-label grapheme CER: {payload['machine_label_metrics']['cer_grapheme_micro']:.6f}", | |
| f"- machine-label codepoint CER: {payload['machine_label_metrics']['cer_codepoint_micro']:.6f}", | |
| f"- machine-label exact rate: {payload['machine_label_metrics']['exact_rate']:.6f}", | |
| f"- prediction grapheme CER: {payload['prediction_metrics']['cer_grapheme_micro']:.6f}", | |
| f"- claim evidence eligible: {payload['claim_evidence_eligible']}", | |
| "", | |
| "## Status Counts", | |
| "", | |
| ] | |
| lines.extend(f"- {status}: {count}" for status, count in payload["counts"]["status_counts"].items()) | |
| lines.extend(["", "## Eligibility Reasons", ""]) | |
| lines.extend([f"- {reason}" for reason in payload["eligibility_reasons"]] or ["- none"]) | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def apply_reviews(args: argparse.Namespace) -> dict[str, Any]: | |
| rows = _read_review_rows(args.review_file, allow_incomplete=args.allow_incomplete) | |
| usable_rows = sum(1 for row in rows if row.status in USABLE_STATUSES) | |
| unusable_rows = sum(1 for row in rows if row.status in UNUSABLE_STATUSES) | |
| unreviewed_rows = sum(1 for row in rows if not row.status) | |
| machine_scores = _aggregate_scores(rows, "machine_text") | |
| prediction_scores = _aggregate_scores(rows, "prediction") | |
| reviewers = sorted({row.raw.get("reviewer", "").strip() for row in rows if row.raw.get("reviewer", "").strip()}) | |
| missing_reviewer_rows = sum(1 for row in rows if row.status and not row.raw.get("reviewer", "").strip()) | |
| reasons = _eligibility_reasons( | |
| rows=rows, | |
| usable_rows=usable_rows, | |
| unusable_rows=unusable_rows, | |
| missing_reviewer_rows=missing_reviewer_rows, | |
| machine_scores=machine_scores, | |
| min_usable_rows=args.min_usable_rows, | |
| max_machine_grapheme_cer=args.max_machine_grapheme_cer, | |
| max_unusable_rate=args.max_unusable_rate, | |
| require_complete=not args.allow_incomplete, | |
| ) | |
| eligible = not reasons | |
| if args.out.exists() and any(args.out.iterdir()) and not args.force: | |
| raise FileExistsError(f"output directory exists and is not empty: {args.out}") | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| payload: dict[str, Any] = { | |
| "created_at": datetime.now(timezone.utc).isoformat(), | |
| "review_file": str(args.review_file), | |
| "review_file_sha256": _hash_file(args.review_file), | |
| "normalization_profile": DEFAULT_NORMALIZATION, | |
| "language": args.language, | |
| "script": args.script, | |
| "domain": args.domain, | |
| "source_gt_status": args.source_gt_status, | |
| "recommended_pack_gt_status": "machine_converted_spot_checked" if eligible else args.source_gt_status, | |
| "recommended_pack_claim_eligible": bool(eligible), | |
| "claim_evidence_eligible": bool(eligible), | |
| "eligibility_thresholds": { | |
| "min_usable_rows": args.min_usable_rows, | |
| "max_machine_grapheme_cer": args.max_machine_grapheme_cer, | |
| "max_unusable_rate": args.max_unusable_rate, | |
| "allow_incomplete": args.allow_incomplete, | |
| }, | |
| "eligibility_reasons": reasons, | |
| "counts": { | |
| "rows": len(rows), | |
| "usable_rows": usable_rows, | |
| "unusable_rows": unusable_rows, | |
| "unreviewed_rows": unreviewed_rows, | |
| "missing_reviewer_rows": missing_reviewer_rows, | |
| "status_counts": _status_counts(rows), | |
| }, | |
| "reviewers": reviewers, | |
| "machine_label_metrics": machine_scores, | |
| "prediction_metrics": prediction_scores, | |
| "artifacts": { | |
| "reviewed_labels_tsv": "reviewed-labels.tsv", | |
| "reviewed_manifest_overrides_jsonl": "reviewed-manifest-overrides.jsonl", | |
| }, | |
| "machine": { | |
| "platform": platform.platform(), | |
| "python": sys.version, | |
| }, | |
| "command": " ".join(sys.argv), | |
| } | |
| (args.out / "native-spotcheck-calibration.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_markdown(args.out / "native-spotcheck-calibration.md", payload) | |
| _write_reviewed_labels(args.out / "reviewed-labels.tsv", rows) | |
| _write_manifest_overrides(args.out / "reviewed-manifest-overrides.jsonl", rows) | |
| return payload | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--review-file", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--language", default="unknown") | |
| parser.add_argument("--script", default="unknown") | |
| parser.add_argument("--domain", default="unknown") | |
| parser.add_argument("--source-gt-status", default="machine_converted") | |
| parser.add_argument("--min-usable-rows", type=int, default=30) | |
| parser.add_argument("--max-machine-grapheme-cer", type=float, default=0.02) | |
| parser.add_argument("--max-unusable-rate", type=float, default=0.05) | |
| parser.add_argument("--allow-incomplete", action=argparse.BooleanOptionalAction, default=False) | |
| parser.add_argument("--force", action="store_true") | |
| args = parser.parse_args(argv) | |
| payload = apply_reviews(args) | |
| print( | |
| f"wrote {args.out} eligible={payload['claim_evidence_eligible']} " | |
| f"machine_graph_CER={payload['machine_label_metrics']['cer_grapheme_micro']:.3f}" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |