#!/usr/bin/env python3 """Apply completed Limbu phone-page reference reviews into verified references.""" from __future__ import annotations import argparse import csv import json import platform import shutil import sys from collections import Counter, defaultdict from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest REPORT_JSON = "phone-page-reference-review-apply.json" REPORT_MD = "phone-page-reference-review-apply.md" EVAL_MANIFEST = "eval-manifest.jsonl" VALID_STATUSES = {"pending", "accept", "corrected", "drop", "needs_context", "bad_segmentation", "needs_resegmentation"} KEEP_STATUSES = {"accept", "corrected"} BLOCKING_STATUSES = {"", "pending", "needs_context", "bad_segmentation", "needs_resegmentation"} REQUIRED_CSV_FIELDS = {"sample_id", "line_id", "order_index", "ocr_text", "review_text", "review_status"} def _read_json(path: Path, *, label: str) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError(f"{label} is missing: {path}") from exc except json.JSONDecodeError as exc: raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"{label} root must be an object: {path}") return payload def _safe_name(value: str) -> str: cleaned = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in value).strip(".-") return cleaned or "item" def _resolve_path(value: Any, *, base_dirs: list[Path]) -> Path | None: if not isinstance(value, str) or not value.strip(): return None path = Path(value) if path.is_absolute(): return path if path.exists(): return path for base_dir in base_dirs: candidate = base_dir / path if candidate.exists(): return candidate return path def _packet_paths(packet_dir: Path, review_csv: Path | None) -> tuple[dict[str, Any], Path, Path]: summary_path = packet_dir / "phone-page-reference-review.json" summary = _read_json(summary_path, label="phone reference review summary") source_manifest = _resolve_path(summary.get("source_eval_manifest"), base_dirs=[packet_dir]) if source_manifest is None or not source_manifest.is_file(): raise ValueError(f"source eval manifest is missing: {summary.get('source_eval_manifest')}") if review_csv is None: review_path = _resolve_path(summary.get("review_csv"), base_dirs=[packet_dir]) else: review_path = review_csv if review_path is None or not review_path.is_file(): raise ValueError(f"review CSV is missing: {review_csv or summary.get('review_csv')}") return summary, source_manifest, review_path def _read_review_rows(path: Path) -> list[dict[str, str]]: with path.open("r", encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle) if reader.fieldnames is None: raise ValueError(f"review CSV has no header: {path}") missing = sorted(REQUIRED_CSV_FIELDS - set(reader.fieldnames)) if missing: raise ValueError(f"review CSV missing fields: {', '.join(missing)}") rows: list[dict[str, str]] = [] for row in reader: rows.append({key: (value or "") for key, value in row.items() if key is not None}) if not rows: raise ValueError(f"review CSV has no rows: {path}") return rows def _entry_reference_path(entry: ManifestEntry, *, manifest_dir: Path) -> Path: raw = entry.metadata.get("reference_path") if entry.metadata else None path = _resolve_path(raw, base_dirs=[manifest_dir]) if path is None or not path.is_file(): raise ValueError(f"{entry.sample_id}: metadata.reference_path is missing or not a file: {raw}") return path def _parse_bbox(value: str, *, label: str) -> list[float] | None: if not value.strip(): return None try: payload = json.loads(value) except json.JSONDecodeError as exc: raise ValueError(f"{label}: bbox is invalid JSON: {exc}") from exc if not isinstance(payload, list) or len(payload) != 4: raise ValueError(f"{label}: bbox must be a four-item list") try: bbox = [float(item) for item in payload] except (TypeError, ValueError) as exc: raise ValueError(f"{label}: bbox contains non-numeric values") from exc if bbox[2] <= 0 or bbox[3] <= 0: raise ValueError(f"{label}: bbox has non-positive width/height") return bbox def _parse_codepoint_ranges(values: list[str] | None) -> list[tuple[int, int]]: ranges: list[tuple[int, int]] = [] for value in values or []: cleaned = value.strip() if not cleaned: continue start_text, end_text = cleaned.split("..", 1) if ".." in cleaned else (cleaned, cleaned) try: start = int(start_text[2:] if start_text.upper().startswith("U+") else start_text, 16) end = int(end_text[2:] if end_text.upper().startswith("U+") else end_text, 16) except ValueError as exc: raise ValueError(f"invalid codepoint range {cleaned!r}; expected U+1900..U+194F") from exc if end < start: raise ValueError(f"invalid codepoint range {cleaned!r}; end before start") ranges.append((start, end)) return ranges def _format_codepoint_ranges(ranges: list[tuple[int, int]]) -> str: return ",".join(f"U+{start:04X}..U+{end:04X}" if start != end else f"U+{start:04X}" for start, end in ranges) def _contains_required_codepoint(text: str, ranges: list[tuple[int, int]]) -> bool: return not ranges or any(start <= ord(char) <= end for char in text for start, end in ranges) def _reviewer_for(row: dict[str, str], *, global_reviewer: str | None) -> str: reviewer = (global_reviewer or "").strip() or row.get("reviewer", "").strip() return reviewer def _group_review_rows(rows: list[dict[str, str]]) -> tuple[dict[str, list[dict[str, str]]], list[str], Counter[str]]: grouped: dict[str, list[dict[str, str]]] = defaultdict(list) seen: set[tuple[str, str]] = set() blockers: list[str] = [] status_counts: Counter[str] = Counter() for index, row in enumerate(rows, start=2): sample_id = row.get("sample_id", "").strip() line_id = row.get("line_id", "").strip() if not sample_id or not line_id: blockers.append(f"row {index}: sample_id and line_id are required") continue key = (sample_id, line_id) if key in seen: blockers.append(f"{sample_id}:{line_id}: duplicate review row") continue seen.add(key) status = row.get("review_status", "").strip() if status not in VALID_STATUSES: blockers.append(f"{sample_id}:{line_id}: unsupported review_status {status!r}") status_counts[status or ""] += 1 grouped[sample_id].append(row) for sample_rows in grouped.values(): sample_rows.sort(key=lambda row: int(row.get("order_index") or "999999")) return grouped, blockers, status_counts def _validate_structural_review(args: argparse.Namespace) -> None: missing = [] if not args.tables_reviewed: missing.append("--tables-reviewed") if not args.figures_reviewed: missing.append("--figures-reviewed") if not args.captions_reviewed: missing.append("--captions-reviewed") if missing: raise ValueError("claim-ready phone references require explicit structural review flags: " + ", ".join(missing)) def _verified_reference_for_sample( *, entry: ManifestEntry, reference_path: Path, draft_reference: dict[str, Any], review_rows: list[dict[str, str]], review_csv: Path, reviewer: str | None, reviewed_at: str, codepoint_ranges: list[tuple[int, int]], args: argparse.Namespace, ) -> tuple[dict[str, Any], int, int]: reading_order = draft_reference.get("reading_order") if not isinstance(reading_order, list): raise ValueError(f"{entry.sample_id}: draft reference reading_order must be a list") draft_by_line_id = {} draft_ids: list[str] = [] for item in reading_order: if not isinstance(item, dict): continue line_id = str(item.get("id") or "").strip() if line_id: draft_by_line_id[line_id] = item draft_ids.append(line_id) review_ids = [row.get("line_id", "").strip() for row in review_rows] blockers: list[str] = [] if set(review_ids) != set(draft_ids): missing = sorted(set(draft_ids) - set(review_ids)) extra = sorted(set(review_ids) - set(draft_ids)) if missing: blockers.append(f"missing review rows: {', '.join(missing[:10])}") if extra: blockers.append(f"unknown review rows: {', '.join(extra[:10])}") kept: list[dict[str, Any]] = [] dropped_count = 0 reviewers: set[str] = set() for row in review_rows: line_id = row.get("line_id", "").strip() status = row.get("review_status", "").strip() label = f"{entry.sample_id}:{line_id}" if status in BLOCKING_STATUSES: blockers.append(f"{label}: review_status {status or ''} blocks verification") continue if status not in VALID_STATUSES: continue row_reviewer = _reviewer_for(row, global_reviewer=reviewer) if not row_reviewer: blockers.append(f"{label}: reviewer is required") continue reviewers.add(row_reviewer) if status == "drop": dropped_count += 1 continue if status == "accept": text = row.get("ocr_text", "") elif status == "corrected": text = row.get("review_text", "") else: blockers.append(f"{label}: review_status {status} blocks verification") continue text = text.strip() if not text: blockers.append(f"{label}: kept text is empty") continue if not _contains_required_codepoint(text, codepoint_ranges): blockers.append(f"{label}: kept text has no character in required range {_format_codepoint_ranges(codepoint_ranges)}") continue item = dict(draft_by_line_id.get(line_id) or {}) item["id"] = line_id item["text"] = text item["review_status"] = status item["reviewer"] = row_reviewer bbox = _parse_bbox(row.get("bbox", ""), label=label) if bbox is not None: item["bbox"] = bbox try: item["page_index"] = int(row.get("page_index") or item.get("page_index") or 0) except (TypeError, ValueError) as exc: raise ValueError(f"{label}: page_index must be an integer") from exc kept.append(item) if blockers: raise ValueError(f"{entry.sample_id}: review is not complete: " + "; ".join(blockers[:20])) if not kept: raise ValueError(f"{entry.sample_id}: review produced no kept text lines") text = "\n".join(str(item["text"]) for item in kept) metadata = dict(draft_reference.get("metadata") if isinstance(draft_reference.get("metadata"), dict) else {}) metadata.update( { "sample_id": entry.sample_id, "reference_status": "verified", "claim_evidence_eligible": True, "text_ground_truth_certified": True, "requires_human_review": False, "machine_generated": False, "text_reviewed": True, "reading_order_reviewed": True, "tables_reviewed": True, "figures_reviewed": True, "captions_reviewed": True, "structural_review_status": "verified", "verification_source_csv": str(review_csv), "source_draft_reference_path": str(reference_path), "reviewer": reviewer or ",".join(sorted(reviewers)), "reviewed_at": reviewed_at, "verified_line_count": len(kept), "dropped_line_count": dropped_count, "text_sha256": sha256_text(text), "source_kind": "limbu_phone_page_verified_reference", "review_policy": "line-level accept/corrected/drop with explicit reviewer and structural review flags", } ) if codepoint_ranges: metadata["required_codepoint_ranges"] = _format_codepoint_ranges(codepoint_ranges) reference = { "text": text, "reading_order": kept, "tables": draft_reference.get("tables") if isinstance(draft_reference.get("tables"), list) else [], "figures": draft_reference.get("figures") if isinstance(draft_reference.get("figures"), list) else [], "metadata": metadata, } _validate_verified_reference(reference, sample_id=entry.sample_id, label=str(reference_path)) return reference, len(kept), dropped_count def _validate_verified_reference(reference: dict[str, Any], *, sample_id: str, label: str) -> None: if not isinstance(reference.get("text"), str) or not reference["text"].strip(): raise ValueError(f"{label}: verified reference text is empty") for field in ("reading_order", "tables", "figures"): if not isinstance(reference.get(field), list): raise ValueError(f"{label}: verified reference {field} must be a list") for index, item in enumerate(reference["reading_order"], start=1): if not isinstance(item, dict): raise ValueError(f"{label}: reading_order item {index} is not an object") if not str(item.get("id") or "").strip(): raise ValueError(f"{label}: reading_order item {index} missing id") if not str(item.get("text") or "").strip(): raise ValueError(f"{label}: reading_order item {index} missing text") metadata = reference.get("metadata") if not isinstance(metadata, dict): raise ValueError(f"{label}: verified reference metadata must be an object") if metadata.get("sample_id") not in {sample_id, None, ""}: raise ValueError(f"{label}: sample_id mismatch") if metadata.get("reference_status") != "verified": raise ValueError(f"{label}: reference_status must be verified") if metadata.get("claim_evidence_eligible") is not True: raise ValueError(f"{label}: claim_evidence_eligible must be true") if not metadata.get("reviewer"): raise ValueError(f"{label}: reviewer metadata is required") for field in ("text_reviewed", "reading_order_reviewed", "tables_reviewed", "figures_reviewed", "captions_reviewed"): if metadata.get(field) is not True: raise ValueError(f"{label}: metadata.{field} must be true") def _copy_image(entry: ManifestEntry, *, manifest_dir: Path, out: Path) -> tuple[str, str]: image_path = _resolve_path(entry.image_path, base_dirs=[manifest_dir]) if image_path is None or not image_path.is_file(): raise ValueError(f"{entry.sample_id}: image_path is missing: {entry.image_path}") suffix = image_path.suffix or ".png" destination = out / "images" / f"{_safe_name(entry.sample_id)}{suffix}" destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(image_path, destination) return str(destination), sha256_file(destination) def _write_markdown(path: Path, report: dict[str, Any]) -> None: lines = [ "# Limbu Phone Page Reference Review Apply", "", f"- passed: `{report['passed']}`", f"- source eval manifest: `{report['source_eval_manifest']}`", f"- review CSV: `{report['review_csv']}`", f"- output eval manifest: `{report['eval_manifest']}`", f"- samples: `{report['sample_count']}`", f"- verified references: `{report['verified_reference_count']}`", f"- kept lines: `{report['kept_line_count']}`", f"- dropped lines: `{report['dropped_line_count']}`", "", "## Status Counts", "", ] lines.extend(f"- `{status}`: {count}" for status, count in sorted(report["status_counts"].items())) if report.get("issues"): lines.extend(["", "## Issues", ""]) lines.extend(f"- {issue}" for issue in report["issues"]) path.write_text("\n".join(lines) + "\n", encoding="utf-8") def apply_reference_review(args: argparse.Namespace) -> dict[str, Any]: _validate_structural_review(args) packet_dir = Path(args.packet_dir) out = Path(args.out) out.mkdir(parents=True, exist_ok=True) summary, source_manifest, review_csv = _packet_paths(packet_dir, args.review_csv) entries = load_manifest(source_manifest) if not entries: raise ValueError(f"source eval manifest is empty: {source_manifest}") review_rows = _read_review_rows(review_csv) grouped, group_blockers, status_counts = _group_review_rows(review_rows) if group_blockers: raise ValueError("review CSV is invalid: " + "; ".join(group_blockers[:20])) manifest_dir = source_manifest.parent codepoint_ranges = _parse_codepoint_ranges(args.required_codepoint_range) reviewed_at = args.reviewed_at or datetime.now(UTC).isoformat() rewritten_entries: list[ManifestEntry] = [] sample_reports: list[dict[str, Any]] = [] kept_total = 0 dropped_total = 0 for entry in entries: sample_rows = grouped.get(entry.sample_id) if not sample_rows: raise ValueError(f"{entry.sample_id}: no review rows found") reference_path = _entry_reference_path(entry, manifest_dir=manifest_dir) draft_reference = _read_json(reference_path, label=f"{entry.sample_id} draft reference") verified, kept_count, dropped_count = _verified_reference_for_sample( entry=entry, reference_path=reference_path, draft_reference=draft_reference, review_rows=sample_rows, review_csv=review_csv, reviewer=args.reviewer, reviewed_at=reviewed_at, codepoint_ranges=codepoint_ranges, args=args, ) verified_path = out / "references" / f"{_safe_name(entry.sample_id)}.ref.json" verified_path.parent.mkdir(parents=True, exist_ok=True) verified_path.write_text(json.dumps(verified, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") image_path, image_hash = _copy_image(entry, manifest_dir=manifest_dir, out=out) reference_hash = sha256_file(verified_path) metadata = dict(entry.metadata or {}) metadata.update( { "reference_path": str(verified_path), "reference_sha256": reference_hash, "reference_status": "verified", "claim_evidence_eligible": True, "gt_status": "human_verified", "text_sha256": sha256_text(str(verified["text"])), "reviewed_by": args.reviewer or verified["metadata"].get("reviewer"), "reviewed_at": reviewed_at, "phone_reference_review_packet": str(packet_dir), "phone_reference_review_csv": str(review_csv), } ) rewritten_entries.append( ManifestEntry( sample_id=entry.sample_id, dataset=args.dataset or entry.dataset, split=args.split or entry.split, image_path=image_path, text=str(verified["text"]), sha256=image_hash, metadata=metadata, ) ) kept_total += kept_count dropped_total += dropped_count sample_reports.append( { "sample_id": entry.sample_id, "reference_path": str(verified_path), "reference_sha256": reference_hash, "kept_line_count": kept_count, "dropped_line_count": dropped_count, "image_path": image_path, } ) unknown_samples = sorted(set(grouped) - {entry.sample_id for entry in entries}) if unknown_samples: raise ValueError(f"review CSV contains unknown sample_ids: {', '.join(unknown_samples[:20])}") eval_manifest = out / EVAL_MANIFEST write_manifest(rewritten_entries, eval_manifest) report_path = out / REPORT_JSON md_path = out / REPORT_MD report = { "tool": "tools/apply_limbu_phone_page_reference_review.py", "created_at_utc": datetime.now(UTC).isoformat(), "machine": platform.node(), "passed": True, "packet_summary": str(packet_dir / "phone-page-reference-review.json"), "packet_summary_sha256": sha256_file(packet_dir / "phone-page-reference-review.json"), "source_eval_manifest": str(source_manifest), "source_eval_manifest_sha256": sha256_file(source_manifest), "review_csv": str(review_csv), "review_csv_sha256": sha256_file(review_csv), "out": str(out), "eval_manifest": str(eval_manifest), "eval_manifest_sha256": sha256_file(eval_manifest), "sample_count": len(rewritten_entries), "verified_reference_count": len(sample_reports), "kept_line_count": kept_total, "dropped_line_count": dropped_total, "status_counts": dict(sorted(status_counts.items())), "required_codepoint_ranges": _format_codepoint_ranges(codepoint_ranges), "reviewer": args.reviewer, "reviewed_at": reviewed_at, "samples": sample_reports, "issues": [], } report["summary_json_path"] = str(report_path) report["summary_md_path"] = str(md_path) report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") _write_markdown(md_path, report) return report def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--packet-dir", type=Path, required=True) parser.add_argument("--review-csv", type=Path) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--reviewer", default="") parser.add_argument("--reviewed-at", default="") parser.add_argument("--required-codepoint-range", action="append", default=[]) parser.add_argument("--tables-reviewed", action="store_true") parser.add_argument("--figures-reviewed", action="store_true") parser.add_argument("--captions-reviewed", action="store_true") parser.add_argument("--dataset", default="") parser.add_argument("--split", default="") args = parser.parse_args() try: report = apply_reference_review(args) except Exception as exc: print(f"apply_limbu_phone_page_reference_review: error: {exc}", file=sys.stderr) return 2 print(report["summary_json_path"]) print(report["eval_manifest"]) print(f"samples={report['sample_count']} kept={report['kept_line_count']} dropped={report['dropped_line_count']}") return 0 if __name__ == "__main__": raise SystemExit(main())