ocr-projects-archive / ocr-tech /tools /apply_limbu_phone_page_reference_review.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
23.2 kB
#!/usr/bin/env python3
"""Apply completed Limbu phone-page reference reviews into verified references."""
from __future__ import annotations
import argparse
import csv
import json
import platform
import shutil
import sys
from collections import Counter, defaultdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
REPORT_JSON = "phone-page-reference-review-apply.json"
REPORT_MD = "phone-page-reference-review-apply.md"
EVAL_MANIFEST = "eval-manifest.jsonl"
VALID_STATUSES = {"pending", "accept", "corrected", "drop", "needs_context", "bad_segmentation", "needs_resegmentation"}
KEEP_STATUSES = {"accept", "corrected"}
BLOCKING_STATUSES = {"", "pending", "needs_context", "bad_segmentation", "needs_resegmentation"}
REQUIRED_CSV_FIELDS = {"sample_id", "line_id", "order_index", "ocr_text", "review_text", "review_status"}
def _read_json(path: Path, *, label: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise ValueError(f"{label} is missing: {path}") from exc
except json.JSONDecodeError as exc:
raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"{label} root must be an object: {path}")
return payload
def _safe_name(value: str) -> str:
cleaned = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in value).strip(".-")
return cleaned or "item"
def _resolve_path(value: Any, *, base_dirs: list[Path]) -> Path | None:
if not isinstance(value, str) or not value.strip():
return None
path = Path(value)
if path.is_absolute():
return path
if path.exists():
return path
for base_dir in base_dirs:
candidate = base_dir / path
if candidate.exists():
return candidate
return path
def _packet_paths(packet_dir: Path, review_csv: Path | None) -> tuple[dict[str, Any], Path, Path]:
summary_path = packet_dir / "phone-page-reference-review.json"
summary = _read_json(summary_path, label="phone reference review summary")
source_manifest = _resolve_path(summary.get("source_eval_manifest"), base_dirs=[packet_dir])
if source_manifest is None or not source_manifest.is_file():
raise ValueError(f"source eval manifest is missing: {summary.get('source_eval_manifest')}")
if review_csv is None:
review_path = _resolve_path(summary.get("review_csv"), base_dirs=[packet_dir])
else:
review_path = review_csv
if review_path is None or not review_path.is_file():
raise ValueError(f"review CSV is missing: {review_csv or summary.get('review_csv')}")
return summary, source_manifest, review_path
def _read_review_rows(path: Path) -> list[dict[str, str]]:
with path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle)
if reader.fieldnames is None:
raise ValueError(f"review CSV has no header: {path}")
missing = sorted(REQUIRED_CSV_FIELDS - set(reader.fieldnames))
if missing:
raise ValueError(f"review CSV missing fields: {', '.join(missing)}")
rows: list[dict[str, str]] = []
for row in reader:
rows.append({key: (value or "") for key, value in row.items() if key is not None})
if not rows:
raise ValueError(f"review CSV has no rows: {path}")
return rows
def _entry_reference_path(entry: ManifestEntry, *, manifest_dir: Path) -> Path:
raw = entry.metadata.get("reference_path") if entry.metadata else None
path = _resolve_path(raw, base_dirs=[manifest_dir])
if path is None or not path.is_file():
raise ValueError(f"{entry.sample_id}: metadata.reference_path is missing or not a file: {raw}")
return path
def _parse_bbox(value: str, *, label: str) -> list[float] | None:
if not value.strip():
return None
try:
payload = json.loads(value)
except json.JSONDecodeError as exc:
raise ValueError(f"{label}: bbox is invalid JSON: {exc}") from exc
if not isinstance(payload, list) or len(payload) != 4:
raise ValueError(f"{label}: bbox must be a four-item list")
try:
bbox = [float(item) for item in payload]
except (TypeError, ValueError) as exc:
raise ValueError(f"{label}: bbox contains non-numeric values") from exc
if bbox[2] <= 0 or bbox[3] <= 0:
raise ValueError(f"{label}: bbox has non-positive width/height")
return bbox
def _parse_codepoint_ranges(values: list[str] | None) -> list[tuple[int, int]]:
ranges: list[tuple[int, int]] = []
for value in values or []:
cleaned = value.strip()
if not cleaned:
continue
start_text, end_text = cleaned.split("..", 1) if ".." in cleaned else (cleaned, cleaned)
try:
start = int(start_text[2:] if start_text.upper().startswith("U+") else start_text, 16)
end = int(end_text[2:] if end_text.upper().startswith("U+") else end_text, 16)
except ValueError as exc:
raise ValueError(f"invalid codepoint range {cleaned!r}; expected U+1900..U+194F") from exc
if end < start:
raise ValueError(f"invalid codepoint range {cleaned!r}; end before start")
ranges.append((start, end))
return ranges
def _format_codepoint_ranges(ranges: list[tuple[int, int]]) -> str:
return ",".join(f"U+{start:04X}..U+{end:04X}" if start != end else f"U+{start:04X}" for start, end in ranges)
def _contains_required_codepoint(text: str, ranges: list[tuple[int, int]]) -> bool:
return not ranges or any(start <= ord(char) <= end for char in text for start, end in ranges)
def _reviewer_for(row: dict[str, str], *, global_reviewer: str | None) -> str:
reviewer = (global_reviewer or "").strip() or row.get("reviewer", "").strip()
return reviewer
def _group_review_rows(rows: list[dict[str, str]]) -> tuple[dict[str, list[dict[str, str]]], list[str], Counter[str]]:
grouped: dict[str, list[dict[str, str]]] = defaultdict(list)
seen: set[tuple[str, str]] = set()
blockers: list[str] = []
status_counts: Counter[str] = Counter()
for index, row in enumerate(rows, start=2):
sample_id = row.get("sample_id", "").strip()
line_id = row.get("line_id", "").strip()
if not sample_id or not line_id:
blockers.append(f"row {index}: sample_id and line_id are required")
continue
key = (sample_id, line_id)
if key in seen:
blockers.append(f"{sample_id}:{line_id}: duplicate review row")
continue
seen.add(key)
status = row.get("review_status", "").strip()
if status not in VALID_STATUSES:
blockers.append(f"{sample_id}:{line_id}: unsupported review_status {status!r}")
status_counts[status or "<empty>"] += 1
grouped[sample_id].append(row)
for sample_rows in grouped.values():
sample_rows.sort(key=lambda row: int(row.get("order_index") or "999999"))
return grouped, blockers, status_counts
def _validate_structural_review(args: argparse.Namespace) -> None:
missing = []
if not args.tables_reviewed:
missing.append("--tables-reviewed")
if not args.figures_reviewed:
missing.append("--figures-reviewed")
if not args.captions_reviewed:
missing.append("--captions-reviewed")
if missing:
raise ValueError("claim-ready phone references require explicit structural review flags: " + ", ".join(missing))
def _verified_reference_for_sample(
*,
entry: ManifestEntry,
reference_path: Path,
draft_reference: dict[str, Any],
review_rows: list[dict[str, str]],
review_csv: Path,
reviewer: str | None,
reviewed_at: str,
codepoint_ranges: list[tuple[int, int]],
args: argparse.Namespace,
) -> tuple[dict[str, Any], int, int]:
reading_order = draft_reference.get("reading_order")
if not isinstance(reading_order, list):
raise ValueError(f"{entry.sample_id}: draft reference reading_order must be a list")
draft_by_line_id = {}
draft_ids: list[str] = []
for item in reading_order:
if not isinstance(item, dict):
continue
line_id = str(item.get("id") or "").strip()
if line_id:
draft_by_line_id[line_id] = item
draft_ids.append(line_id)
review_ids = [row.get("line_id", "").strip() for row in review_rows]
blockers: list[str] = []
if set(review_ids) != set(draft_ids):
missing = sorted(set(draft_ids) - set(review_ids))
extra = sorted(set(review_ids) - set(draft_ids))
if missing:
blockers.append(f"missing review rows: {', '.join(missing[:10])}")
if extra:
blockers.append(f"unknown review rows: {', '.join(extra[:10])}")
kept: list[dict[str, Any]] = []
dropped_count = 0
reviewers: set[str] = set()
for row in review_rows:
line_id = row.get("line_id", "").strip()
status = row.get("review_status", "").strip()
label = f"{entry.sample_id}:{line_id}"
if status in BLOCKING_STATUSES:
blockers.append(f"{label}: review_status {status or '<empty>'} blocks verification")
continue
if status not in VALID_STATUSES:
continue
row_reviewer = _reviewer_for(row, global_reviewer=reviewer)
if not row_reviewer:
blockers.append(f"{label}: reviewer is required")
continue
reviewers.add(row_reviewer)
if status == "drop":
dropped_count += 1
continue
if status == "accept":
text = row.get("ocr_text", "")
elif status == "corrected":
text = row.get("review_text", "")
else:
blockers.append(f"{label}: review_status {status} blocks verification")
continue
text = text.strip()
if not text:
blockers.append(f"{label}: kept text is empty")
continue
if not _contains_required_codepoint(text, codepoint_ranges):
blockers.append(f"{label}: kept text has no character in required range {_format_codepoint_ranges(codepoint_ranges)}")
continue
item = dict(draft_by_line_id.get(line_id) or {})
item["id"] = line_id
item["text"] = text
item["review_status"] = status
item["reviewer"] = row_reviewer
bbox = _parse_bbox(row.get("bbox", ""), label=label)
if bbox is not None:
item["bbox"] = bbox
try:
item["page_index"] = int(row.get("page_index") or item.get("page_index") or 0)
except (TypeError, ValueError) as exc:
raise ValueError(f"{label}: page_index must be an integer") from exc
kept.append(item)
if blockers:
raise ValueError(f"{entry.sample_id}: review is not complete: " + "; ".join(blockers[:20]))
if not kept:
raise ValueError(f"{entry.sample_id}: review produced no kept text lines")
text = "\n".join(str(item["text"]) for item in kept)
metadata = dict(draft_reference.get("metadata") if isinstance(draft_reference.get("metadata"), dict) else {})
metadata.update(
{
"sample_id": entry.sample_id,
"reference_status": "verified",
"claim_evidence_eligible": True,
"text_ground_truth_certified": True,
"requires_human_review": False,
"machine_generated": False,
"text_reviewed": True,
"reading_order_reviewed": True,
"tables_reviewed": True,
"figures_reviewed": True,
"captions_reviewed": True,
"structural_review_status": "verified",
"verification_source_csv": str(review_csv),
"source_draft_reference_path": str(reference_path),
"reviewer": reviewer or ",".join(sorted(reviewers)),
"reviewed_at": reviewed_at,
"verified_line_count": len(kept),
"dropped_line_count": dropped_count,
"text_sha256": sha256_text(text),
"source_kind": "limbu_phone_page_verified_reference",
"review_policy": "line-level accept/corrected/drop with explicit reviewer and structural review flags",
}
)
if codepoint_ranges:
metadata["required_codepoint_ranges"] = _format_codepoint_ranges(codepoint_ranges)
reference = {
"text": text,
"reading_order": kept,
"tables": draft_reference.get("tables") if isinstance(draft_reference.get("tables"), list) else [],
"figures": draft_reference.get("figures") if isinstance(draft_reference.get("figures"), list) else [],
"metadata": metadata,
}
_validate_verified_reference(reference, sample_id=entry.sample_id, label=str(reference_path))
return reference, len(kept), dropped_count
def _validate_verified_reference(reference: dict[str, Any], *, sample_id: str, label: str) -> None:
if not isinstance(reference.get("text"), str) or not reference["text"].strip():
raise ValueError(f"{label}: verified reference text is empty")
for field in ("reading_order", "tables", "figures"):
if not isinstance(reference.get(field), list):
raise ValueError(f"{label}: verified reference {field} must be a list")
for index, item in enumerate(reference["reading_order"], start=1):
if not isinstance(item, dict):
raise ValueError(f"{label}: reading_order item {index} is not an object")
if not str(item.get("id") or "").strip():
raise ValueError(f"{label}: reading_order item {index} missing id")
if not str(item.get("text") or "").strip():
raise ValueError(f"{label}: reading_order item {index} missing text")
metadata = reference.get("metadata")
if not isinstance(metadata, dict):
raise ValueError(f"{label}: verified reference metadata must be an object")
if metadata.get("sample_id") not in {sample_id, None, ""}:
raise ValueError(f"{label}: sample_id mismatch")
if metadata.get("reference_status") != "verified":
raise ValueError(f"{label}: reference_status must be verified")
if metadata.get("claim_evidence_eligible") is not True:
raise ValueError(f"{label}: claim_evidence_eligible must be true")
if not metadata.get("reviewer"):
raise ValueError(f"{label}: reviewer metadata is required")
for field in ("text_reviewed", "reading_order_reviewed", "tables_reviewed", "figures_reviewed", "captions_reviewed"):
if metadata.get(field) is not True:
raise ValueError(f"{label}: metadata.{field} must be true")
def _copy_image(entry: ManifestEntry, *, manifest_dir: Path, out: Path) -> tuple[str, str]:
image_path = _resolve_path(entry.image_path, base_dirs=[manifest_dir])
if image_path is None or not image_path.is_file():
raise ValueError(f"{entry.sample_id}: image_path is missing: {entry.image_path}")
suffix = image_path.suffix or ".png"
destination = out / "images" / f"{_safe_name(entry.sample_id)}{suffix}"
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(image_path, destination)
return str(destination), sha256_file(destination)
def _write_markdown(path: Path, report: dict[str, Any]) -> None:
lines = [
"# Limbu Phone Page Reference Review Apply",
"",
f"- passed: `{report['passed']}`",
f"- source eval manifest: `{report['source_eval_manifest']}`",
f"- review CSV: `{report['review_csv']}`",
f"- output eval manifest: `{report['eval_manifest']}`",
f"- samples: `{report['sample_count']}`",
f"- verified references: `{report['verified_reference_count']}`",
f"- kept lines: `{report['kept_line_count']}`",
f"- dropped lines: `{report['dropped_line_count']}`",
"",
"## Status Counts",
"",
]
lines.extend(f"- `{status}`: {count}" for status, count in sorted(report["status_counts"].items()))
if report.get("issues"):
lines.extend(["", "## Issues", ""])
lines.extend(f"- {issue}" for issue in report["issues"])
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def apply_reference_review(args: argparse.Namespace) -> dict[str, Any]:
_validate_structural_review(args)
packet_dir = Path(args.packet_dir)
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
summary, source_manifest, review_csv = _packet_paths(packet_dir, args.review_csv)
entries = load_manifest(source_manifest)
if not entries:
raise ValueError(f"source eval manifest is empty: {source_manifest}")
review_rows = _read_review_rows(review_csv)
grouped, group_blockers, status_counts = _group_review_rows(review_rows)
if group_blockers:
raise ValueError("review CSV is invalid: " + "; ".join(group_blockers[:20]))
manifest_dir = source_manifest.parent
codepoint_ranges = _parse_codepoint_ranges(args.required_codepoint_range)
reviewed_at = args.reviewed_at or datetime.now(UTC).isoformat()
rewritten_entries: list[ManifestEntry] = []
sample_reports: list[dict[str, Any]] = []
kept_total = 0
dropped_total = 0
for entry in entries:
sample_rows = grouped.get(entry.sample_id)
if not sample_rows:
raise ValueError(f"{entry.sample_id}: no review rows found")
reference_path = _entry_reference_path(entry, manifest_dir=manifest_dir)
draft_reference = _read_json(reference_path, label=f"{entry.sample_id} draft reference")
verified, kept_count, dropped_count = _verified_reference_for_sample(
entry=entry,
reference_path=reference_path,
draft_reference=draft_reference,
review_rows=sample_rows,
review_csv=review_csv,
reviewer=args.reviewer,
reviewed_at=reviewed_at,
codepoint_ranges=codepoint_ranges,
args=args,
)
verified_path = out / "references" / f"{_safe_name(entry.sample_id)}.ref.json"
verified_path.parent.mkdir(parents=True, exist_ok=True)
verified_path.write_text(json.dumps(verified, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
image_path, image_hash = _copy_image(entry, manifest_dir=manifest_dir, out=out)
reference_hash = sha256_file(verified_path)
metadata = dict(entry.metadata or {})
metadata.update(
{
"reference_path": str(verified_path),
"reference_sha256": reference_hash,
"reference_status": "verified",
"claim_evidence_eligible": True,
"gt_status": "human_verified",
"text_sha256": sha256_text(str(verified["text"])),
"reviewed_by": args.reviewer or verified["metadata"].get("reviewer"),
"reviewed_at": reviewed_at,
"phone_reference_review_packet": str(packet_dir),
"phone_reference_review_csv": str(review_csv),
}
)
rewritten_entries.append(
ManifestEntry(
sample_id=entry.sample_id,
dataset=args.dataset or entry.dataset,
split=args.split or entry.split,
image_path=image_path,
text=str(verified["text"]),
sha256=image_hash,
metadata=metadata,
)
)
kept_total += kept_count
dropped_total += dropped_count
sample_reports.append(
{
"sample_id": entry.sample_id,
"reference_path": str(verified_path),
"reference_sha256": reference_hash,
"kept_line_count": kept_count,
"dropped_line_count": dropped_count,
"image_path": image_path,
}
)
unknown_samples = sorted(set(grouped) - {entry.sample_id for entry in entries})
if unknown_samples:
raise ValueError(f"review CSV contains unknown sample_ids: {', '.join(unknown_samples[:20])}")
eval_manifest = out / EVAL_MANIFEST
write_manifest(rewritten_entries, eval_manifest)
report_path = out / REPORT_JSON
md_path = out / REPORT_MD
report = {
"tool": "tools/apply_limbu_phone_page_reference_review.py",
"created_at_utc": datetime.now(UTC).isoformat(),
"machine": platform.node(),
"passed": True,
"packet_summary": str(packet_dir / "phone-page-reference-review.json"),
"packet_summary_sha256": sha256_file(packet_dir / "phone-page-reference-review.json"),
"source_eval_manifest": str(source_manifest),
"source_eval_manifest_sha256": sha256_file(source_manifest),
"review_csv": str(review_csv),
"review_csv_sha256": sha256_file(review_csv),
"out": str(out),
"eval_manifest": str(eval_manifest),
"eval_manifest_sha256": sha256_file(eval_manifest),
"sample_count": len(rewritten_entries),
"verified_reference_count": len(sample_reports),
"kept_line_count": kept_total,
"dropped_line_count": dropped_total,
"status_counts": dict(sorted(status_counts.items())),
"required_codepoint_ranges": _format_codepoint_ranges(codepoint_ranges),
"reviewer": args.reviewer,
"reviewed_at": reviewed_at,
"samples": sample_reports,
"issues": [],
}
report["summary_json_path"] = str(report_path)
report["summary_md_path"] = str(md_path)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_markdown(md_path, report)
return report
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--packet-dir", type=Path, required=True)
parser.add_argument("--review-csv", type=Path)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--reviewer", default="")
parser.add_argument("--reviewed-at", default="")
parser.add_argument("--required-codepoint-range", action="append", default=[])
parser.add_argument("--tables-reviewed", action="store_true")
parser.add_argument("--figures-reviewed", action="store_true")
parser.add_argument("--captions-reviewed", action="store_true")
parser.add_argument("--dataset", default="")
parser.add_argument("--split", default="")
args = parser.parse_args()
try:
report = apply_reference_review(args)
except Exception as exc:
print(f"apply_limbu_phone_page_reference_review: error: {exc}", file=sys.stderr)
return 2
print(report["summary_json_path"])
print(report["eval_manifest"])
print(f"samples={report['sample_count']} kept={report['kept_line_count']} dropped={report['dropped_line_count']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())