ocr-projects-archive / ocr-tech /tools /audit_limbu_book_output.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
27.8 kB
#!/usr/bin/env python3
"""Audit a multi-page Limbu book OCR output bundle."""
from __future__ import annotations
import argparse
import csv
import json
import platform
from collections import Counter
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
def _read_json_object(path: Path, issues: list[str], *, label: str) -> dict[str, Any] | None:
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return None
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
issues.append(f"{label} is invalid JSON: {exc}")
return None
if not isinstance(payload, dict):
issues.append(f"{label} must contain a JSON object")
return None
return payload
def _read_tsv_rows(path: Path, issues: list[str], *, label: str) -> list[dict[str, str]]:
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return []
try:
with path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle, delimiter="\t")
if reader.fieldnames is None:
issues.append(f"{label} has no header: {path}")
return []
return [dict(row) for row in reader]
except OSError as exc:
issues.append(f"{label} could not be read: {type(exc).__name__}: {exc}")
return []
def _path_value(run: dict[str, Any], key: str, issues: list[str]) -> Path | None:
value = run.get(key)
if not isinstance(value, str) or not value:
issues.append(f"book-run.json missing {key}")
return None
return Path(value)
def _check_hash(path: Path | None, expected: object, issues: list[str], *, label: str) -> None:
if path is None:
return
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return
if not isinstance(expected, str) or not expected:
issues.append(f"{label} missing recorded SHA-256")
return
actual = sha256_file(path)
if actual != expected:
issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual} path={path}")
def _check_optional_hash(page: dict[str, Any], path_key: str, sha_key: str, issues: list[str], *, label: str) -> None:
value = page.get(path_key)
if not isinstance(value, str) or not value:
return
_check_hash(Path(value), page.get(sha_key), issues, label=label)
def _int_value(value: object, field: str, issues: list[str]) -> int | None:
try:
parsed = int(value)
except (TypeError, ValueError):
issues.append(f"book-run.json {field} must be an integer")
return None
return parsed
def _sum_int_pages(pages: list[dict[str, Any]], key: str, issues: list[str]) -> int:
total = 0
for index, page in enumerate(pages):
try:
total += int(page.get(key) or 0)
except (TypeError, ValueError):
issues.append(f"book-run.json pages[{index}] {key} must be an integer")
return total
def _aggregate_script_counts(pages: list[dict[str, Any]], issues: list[str]) -> dict[str, int]:
counts: dict[str, int] = {}
for index, page in enumerate(pages):
raw = page.get("script_counts", {})
if not isinstance(raw, dict):
issues.append(f"book-run.json pages[{index}] script_counts must be an object")
continue
for key, value in raw.items():
try:
counts[str(key)] = counts.get(str(key), 0) + int(value)
except (TypeError, ValueError):
issues.append(f"book-run.json pages[{index}] script_counts[{key!r}] must be an integer")
return dict(sorted(counts.items()))
def _audit_book_review_dashboard(run: dict[str, Any], dashboard_path: Path | None, book_line_rows: list[dict[str, str]], issues: list[str]) -> None:
if dashboard_path is None:
if "book_review_dashboard" not in run:
return
issues.append("book-run.json book_review_dashboard is invalid")
return
if not dashboard_path.is_file():
return
try:
text = dashboard_path.read_text(encoding="utf-8")
except OSError as exc:
issues.append(f"book-review-dashboard.html could not be read: {type(exc).__name__}: {exc}")
return
actual_cards = text.count('class="review-card"')
if actual_cards != len(book_line_rows):
issues.append(f"book-review-dashboard.html card count mismatch: review_rows={len(book_line_rows)} actual={actual_cards}")
recorded_cards = run.get("book_review_dashboard_card_count")
if recorded_cards is not None:
try:
recorded_count = int(recorded_cards)
except (TypeError, ValueError):
issues.append("book-run.json book_review_dashboard_card_count must be an integer")
return
if recorded_count != actual_cards:
issues.append(f"book-run.json book_review_dashboard_card_count mismatch: recorded={recorded_count} actual={actual_cards}")
def _audit_review_dashboard_card_count(
run: dict[str, Any],
dashboard_path: Path | None,
expected_rows: int,
issues: list[str],
*,
label: str,
count_field: str,
) -> None:
if dashboard_path is None or not dashboard_path.is_file():
return
try:
text = dashboard_path.read_text(encoding="utf-8")
except OSError as exc:
issues.append(f"{label} could not be read: {type(exc).__name__}: {exc}")
return
actual_cards = text.count('class="review-card"')
if actual_cards != expected_rows:
issues.append(f"{label} card count mismatch: review_rows={expected_rows} actual={actual_cards}")
recorded_cards = run.get(count_field)
if recorded_cards is None:
return
try:
recorded_count = int(recorded_cards)
except (TypeError, ValueError):
issues.append(f"book-run.json {count_field} must be an integer")
return
if recorded_count != actual_cards:
issues.append(f"book-run.json {count_field} mismatch: recorded={recorded_count} actual={actual_cards}")
def _row_signature(row: dict[str, str]) -> str:
return json.dumps(sorted((str(key), str(value)) for key, value in row.items()), ensure_ascii=False, separators=(",", ":"))
def _audit_focused_review_rows_are_canonical(
*,
canonical_rows: list[dict[str, str]],
focused_rows: list[dict[str, str]],
issues: list[str],
) -> None:
canonical_signatures = Counter(_row_signature(row) for row in canonical_rows)
for row_number, row in enumerate(focused_rows, start=2):
signature = _row_signature(row)
if canonical_signatures[signature] <= 0:
page_id = row.get("book_page_id") or "?"
line_id = row.get("book_line_id") or "?"
issues.append(
"book-line-focused-review-queue.tsv row "
f"{row_number} is not present in canonical book-line-review-queue.tsv "
f"(book_page_id={page_id!r}, book_line_id={line_id!r})"
)
continue
canonical_signatures[signature] -= 1
def _json_string_list(value: str, issues: list[str], *, label: str) -> list[str]:
try:
parsed = json.loads(value or "[]")
except json.JSONDecodeError as exc:
issues.append(f"{label} is invalid JSON: {exc}")
return []
if not isinstance(parsed, list) or not all(isinstance(item, str) for item in parsed):
issues.append(f"{label} must be a JSON array of strings")
return []
return parsed
def _audit_book_line_review_crop_refs(book_line_rows: list[dict[str, str]], issues: list[str]) -> None:
for row_number, row in enumerate(book_line_rows, start=2):
if "source_crop_paths" not in row and "source_crop_sha256s" not in row:
continue
label = f"book-line-review-queue.tsv row {row_number}"
crop_paths = _json_string_list(str(row.get("source_crop_paths") or "[]"), issues, label=f"{label} source_crop_paths")
crop_sha256s = _json_string_list(str(row.get("source_crop_sha256s") or "[]"), issues, label=f"{label} source_crop_sha256s")
if len(crop_paths) != len(crop_sha256s):
issues.append(f"{label} source crop path/hash count mismatch: paths={len(crop_paths)} hashes={len(crop_sha256s)}")
for index, crop_path_value in enumerate(crop_paths):
crop_path = Path(crop_path_value)
expected_sha = crop_sha256s[index] if index < len(crop_sha256s) else ""
_check_hash(crop_path, expected_sha, issues, label=f"{label} source crop {index + 1}")
def _audit_page(page: dict[str, Any], index: int, issues: list[str]) -> None:
page_id = page.get("page_id")
if not isinstance(page_id, str) or not page_id:
issues.append(f"book-run.json pages[{index}] missing page_id")
page_id = f"pages[{index}]"
image_path = Path(str(page.get("image_path") or ""))
_check_hash(image_path, page.get("image_sha256"), issues, label=f"{page_id} image")
output_dir = Path(str(page.get("output_dir") or ""))
if not output_dir.is_dir():
issues.append(f"{page_id} output_dir is missing: {output_dir}")
document_json = Path(str(page.get("document_json") or ""))
_check_hash(document_json, page.get("document_json_sha256"), issues, label=f"{page_id} document.json")
document_dir = document_json.parent
_check_hash(document_dir / "document.md", page.get("document_md_sha256"), issues, label=f"{page_id} document.md")
_check_hash(document_dir / "document.body.md", page.get("document_body_md_sha256"), issues, label=f"{page_id} document.body.md")
_check_hash(output_dir / "limbu-output-manifest.json", page.get("limbu_output_manifest_sha256"), issues, label=f"{page_id} limbu-output-manifest.json")
page_audit = _read_json_object(output_dir / "limbu-output-audit.json", issues, label=f"{page_id} limbu-output-audit.json")
if page_audit is not None and bool(page_audit.get("passed")) != bool(page.get("audit_passed")):
issues.append(f"{page_id} audit_passed mismatch between book-run.json and limbu-output-audit.json")
book_document_dir = Path(str(page.get("book_document_dir") or output_dir))
if not book_document_dir.is_dir():
issues.append(f"{page_id} book_document_dir is missing: {book_document_dir}")
book_document_json = Path(str(page.get("book_document_json") or ""))
_check_hash(book_document_json, page.get("book_document_json_sha256"), issues, label=f"{page_id} book document.json")
_check_hash(book_document_dir / "document.md", page.get("book_document_md_sha256"), issues, label=f"{page_id} book document.md")
_check_hash(book_document_dir / "document.body.md", page.get("book_document_body_md_sha256"), issues, label=f"{page_id} book document.body.md")
score_document = Path(str(page.get("score_document_path") or ""))
_check_hash(score_document, page.get("score_document_sha256"), issues, label=f"{page_id} score document.json")
score_sample = page.get("score_document_sample_id")
if not isinstance(score_sample, str) or not score_sample:
issues.append(f"{page_id} missing score_document_sample_id")
_check_optional_hash(page, "source_book_document_json", "source_book_document_json_sha256", issues, label=f"{page_id} source book document.json")
_check_optional_hash(page, "line_assembly_source_document", "line_assembly_source_document_sha256", issues, label=f"{page_id} line assembly source document.json")
_check_optional_hash(page, "reading_order_profile_source_document", "reading_order_profile_source_document_sha256", issues, label=f"{page_id} reading-order profile source document.json")
def _audit_line_assembly(run: dict[str, Any], issues: list[str]) -> None:
line_assembly = run.get("line_assembly")
if not isinstance(line_assembly, dict):
issues.append("book-run.json line_assembly must be an object")
return
if not bool(line_assembly.get("enabled")):
return
manifest = Path(str(line_assembly.get("manifest") or ""))
_check_hash(manifest, line_assembly.get("manifest_sha256"), issues, label="line assembly manifest")
run_path = Path(str(line_assembly.get("run_path") or ""))
_check_hash(run_path, line_assembly.get("run_sha256"), issues, label="line assembly run")
line_report = Path(str(line_assembly.get("line_report_path") or ""))
_check_hash(line_report, line_assembly.get("line_report_sha256"), issues, label="line assembly line report")
audit_path = Path(str(line_assembly.get("audit_path") or ""))
_check_hash(audit_path, line_assembly.get("audit_sha256"), issues, label="line assembly audit")
audit = _read_json_object(audit_path, issues, label="line assembly audit")
if audit is not None:
if bool(audit.get("passed")) != bool(line_assembly.get("audit_passed")):
issues.append("line assembly audit_passed mismatch between book-run.json and audit artifact")
issues_count = len(audit.get("issues", [])) if isinstance(audit.get("issues"), list) else 0
if int(line_assembly.get("audit_issue_count") or 0) != issues_count:
issues.append("line assembly audit_issue_count mismatch")
assembly_run = _read_json_object(run_path, issues, label="line assembly run")
if assembly_run is not None:
for field in ("sample_count", "source_line_count", "assembled_line_count", "merged_group_count", "merged_source_line_count"):
if assembly_run.get(field) != line_assembly.get(field):
issues.append(f"line assembly {field} mismatch between book-run.json and assembly run")
def _audit_reading_order_profile(run: dict[str, Any], issues: list[str]) -> None:
profile_run = run.get("reading_order_profile")
if not isinstance(profile_run, dict):
issues.append("book-run.json reading_order_profile must be an object")
return
if not bool(profile_run.get("enabled")):
return
source_documents_dir = Path(str(profile_run.get("source_documents_dir") or ""))
if not source_documents_dir.is_dir():
issues.append(f"reading-order profile source_documents_dir is missing: {source_documents_dir}")
documents_dir = Path(str(profile_run.get("documents_dir") or ""))
if not documents_dir.is_dir():
issues.append(f"reading-order profile documents_dir is missing: {documents_dir}")
profile_map = Path(str(profile_run.get("profile_map") or ""))
_check_hash(profile_map, profile_run.get("profile_map_sha256"), issues, label="reading-order profile map")
profile_map_audit = str(profile_run.get("profile_map_audit") or "")
if profile_map_audit:
profile_map_audit_path = Path(profile_map_audit)
_check_hash(
profile_map_audit_path,
profile_run.get("profile_map_audit_sha256"),
issues,
label="reading-order profile map audit",
)
audit = _read_json_object(profile_map_audit_path, issues, label="reading-order profile map audit")
if audit is not None:
if audit.get("profile_map_sha256") != profile_run.get("profile_map_sha256"):
issues.append("reading-order profile map audit profile_map_sha256 mismatch with book-run.json")
if bool(audit.get("operational_profile_map_safe")) != bool(profile_run.get("profile_map_operational_safe")):
issues.append("reading-order profile map audit operational safety mismatch with book-run.json")
application_report = Path(str(profile_run.get("application_report") or ""))
_check_hash(
application_report,
profile_run.get("application_report_sha256"),
issues,
label="reading-order profile application report",
)
assignments_jsonl = Path(str(profile_run.get("assignments_jsonl") or ""))
_check_hash(assignments_jsonl, profile_run.get("assignments_jsonl_sha256"), issues, label="reading-order profile assignments")
application = _read_json_object(application_report, issues, label="reading-order profile application report")
if application is not None:
if application.get("profile_map_sha256") != profile_run.get("profile_map_sha256"):
issues.append("reading-order profile application profile_map_sha256 mismatch with book-run.json")
if application.get("assignments_jsonl_sha256") != profile_run.get("assignments_jsonl_sha256"):
issues.append("reading-order profile application assignments_jsonl_sha256 mismatch with book-run.json")
if application.get("assignment_count") != profile_run.get("assignment_count"):
issues.append("reading-order profile assignment_count mismatch between book-run.json and application report")
def audit_book_output(output_dir: str | Path, audit_dir: str | Path | None = None) -> dict[str, Any]:
out = Path(output_dir)
issues: list[str] = []
warnings: list[str] = []
run_path = out / "book-run.json"
run = _read_json_object(run_path, issues, label="book-run.json")
if run is None:
report = _audit_report(out, run_path, issues, warnings)
_write_audit(report, Path(audit_dir) if audit_dir is not None else out)
return report
if run.get("output_dir") != str(out):
issues.append(f"book-run.json output_dir mismatch: recorded={run.get('output_dir')!r} actual={str(out)!r}")
_check_hash(_path_value(run, "page_manifest", issues), run.get("page_manifest_sha256"), issues, label="page manifest")
book_md = _path_value(run, "book_md", issues)
book_body_md = _path_value(run, "book_body_md", issues)
book_review_queue = _path_value(run, "book_review_queue", issues)
book_line_review_queue = _path_value(run, "book_line_review_queue", issues)
book_review_dashboard = Path(str(run.get("book_review_dashboard"))) if isinstance(run.get("book_review_dashboard"), str) and run.get("book_review_dashboard") else None
book_line_focused_review_queue = (
Path(str(run.get("book_line_focused_review_queue")))
if isinstance(run.get("book_line_focused_review_queue"), str) and run.get("book_line_focused_review_queue")
else None
)
book_focused_review_dashboard = (
Path(str(run.get("book_focused_review_dashboard")))
if isinstance(run.get("book_focused_review_dashboard"), str) and run.get("book_focused_review_dashboard")
else None
)
_check_hash(book_md, run.get("book_md_sha256"), issues, label="book.md")
_check_hash(book_body_md, run.get("book_body_md_sha256"), issues, label="book.body.md")
_check_hash(book_review_queue, run.get("book_review_queue_sha256"), issues, label="book-review-queue.tsv")
_check_hash(book_line_review_queue, run.get("book_line_review_queue_sha256"), issues, label="book-line-review-queue.tsv")
if book_line_focused_review_queue is not None:
_check_hash(
book_line_focused_review_queue,
run.get("book_line_focused_review_queue_sha256"),
issues,
label="book-line-focused-review-queue.tsv",
)
if book_review_dashboard is not None:
_check_hash(book_review_dashboard, run.get("book_review_dashboard_sha256"), issues, label="book-review-dashboard.html")
if book_focused_review_dashboard is not None:
_check_hash(book_focused_review_dashboard, run.get("book_focused_review_dashboard_sha256"), issues, label="book-focused-review-dashboard.html")
pages = run.get("pages")
if not isinstance(pages, list):
issues.append("book-run.json pages must be a list")
pages = []
page_objects = [page for page in pages if isinstance(page, dict)]
if len(page_objects) != len(pages):
issues.append("book-run.json pages contains non-object entries")
page_count = _int_value(run.get("page_count"), "page_count", issues)
if page_count is not None and page_count != len(page_objects):
issues.append(f"book-run.json page_count mismatch: recorded={page_count} actual={len(page_objects)}")
line_count = _int_value(run.get("line_count"), "line_count", issues)
if line_count is not None:
actual_line_count = _sum_int_pages(page_objects, "line_count", issues)
if line_count != actual_line_count:
issues.append(f"book-run.json line_count mismatch: recorded={line_count} actual={actual_line_count}")
review_line_count = _int_value(run.get("review_line_count"), "review_line_count", issues)
if review_line_count is not None:
actual_review_line_count = _sum_int_pages(page_objects, "review_line_count", issues)
if review_line_count != actual_review_line_count:
issues.append(f"book-run.json review_line_count mismatch: recorded={review_line_count} actual={actual_review_line_count}")
if book_review_queue is not None:
review_rows = _read_tsv_rows(book_review_queue, issues, label="book-review-queue.tsv")
recorded_count = _int_value(run.get("aggregate_review_row_count"), "aggregate_review_row_count", issues)
if recorded_count is not None and recorded_count != len(review_rows):
issues.append(f"book-run.json aggregate_review_row_count mismatch: recorded={recorded_count} actual={len(review_rows)}")
if book_line_review_queue is not None:
book_line_rows = _read_tsv_rows(book_line_review_queue, issues, label="book-line-review-queue.tsv")
recorded_book_line_count = _int_value(run.get("book_line_review_row_count"), "book_line_review_row_count", issues)
if recorded_book_line_count is not None and recorded_book_line_count != len(book_line_rows):
issues.append(f"book-run.json book_line_review_row_count mismatch: recorded={recorded_book_line_count} actual={len(book_line_rows)}")
_audit_book_line_review_crop_refs(book_line_rows, issues)
_audit_book_review_dashboard(run, book_review_dashboard, book_line_rows, issues)
if book_line_focused_review_queue is not None:
focused_rows = _read_tsv_rows(book_line_focused_review_queue, issues, label="book-line-focused-review-queue.tsv")
if len(focused_rows) > len(book_line_rows):
issues.append(
f"book-line-focused-review-queue.tsv has more rows than canonical review queue: focused={len(focused_rows)} canonical={len(book_line_rows)}"
)
_audit_focused_review_rows_are_canonical(
canonical_rows=book_line_rows,
focused_rows=focused_rows,
issues=issues,
)
focused_summary = run.get("book_line_focused_review_summary")
if isinstance(focused_summary, dict):
if focused_summary.get("row_count") != len(focused_rows):
issues.append("book_line_focused_review_summary row_count mismatch")
if focused_summary.get("source_row_count") != len(book_line_rows):
issues.append("book_line_focused_review_summary source_row_count mismatch")
_audit_review_dashboard_card_count(
run,
book_focused_review_dashboard,
len(focused_rows),
issues,
label="book-focused-review-dashboard.html",
count_field="book_focused_review_dashboard_card_count",
)
script_counts = run.get("script_counts")
if not isinstance(script_counts, dict):
issues.append("book-run.json script_counts must be an object")
else:
aggregate = _aggregate_script_counts(page_objects, issues)
recorded = {str(key): int(value) for key, value in script_counts.items() if isinstance(value, int)}
if recorded != aggregate:
issues.append(f"book-run.json script_counts mismatch: recorded={recorded} actual={aggregate}")
failed_page_ids = [str(page.get("page_id")) for page in page_objects if not bool(page.get("audit_passed"))]
if run.get("page_audit_failed_page_ids") != failed_page_ids:
issues.append("book-run.json page_audit_failed_page_ids mismatch")
if run.get("page_audit_failed_count") != len(failed_page_ids):
issues.append("book-run.json page_audit_failed_count mismatch")
score_documents_dir = Path(str(run.get("score_documents_dir") or ""))
if not score_documents_dir.is_dir():
issues.append(f"score_documents_dir is missing: {score_documents_dir}")
for index, page in enumerate(page_objects):
_audit_page(page, index, issues)
_audit_line_assembly(run, issues)
_audit_reading_order_profile(run, issues)
report = _audit_report(out, run_path, issues, warnings)
report.update(
{
"page_count": len(page_objects),
"line_count": run.get("line_count"),
"review_line_count": run.get("review_line_count"),
"aggregate_review_row_count": run.get("aggregate_review_row_count"),
"book_line_review_row_count": run.get("book_line_review_row_count"),
"line_assembly_enabled": bool((run.get("line_assembly") or {}).get("enabled")) if isinstance(run.get("line_assembly"), dict) else False,
"reading_order_profile_enabled": bool((run.get("reading_order_profile") or {}).get("enabled")) if isinstance(run.get("reading_order_profile"), dict) else False,
}
)
_write_audit(report, Path(audit_dir) if audit_dir is not None else out)
return report
def _audit_report(output_dir: Path, run_path: Path, issues: list[str], warnings: list[str]) -> dict[str, Any]:
return {
"created_at_utc": datetime.now(UTC).isoformat(),
"machine": platform.node(),
"tool": "tools/audit_limbu_book_output.py",
"output_dir": str(output_dir),
"book_run": str(run_path),
"book_run_sha256": sha256_file(run_path) if run_path.is_file() else None,
"passed": not issues,
"issues": issues,
"warnings": warnings,
"claim_boundary": "artifact consistency audit for multi-page Limbu book OCR output; it does not certify OCR quality",
}
def _write_audit(report: dict[str, Any], audit_dir: Path) -> None:
audit_dir.mkdir(parents=True, exist_ok=True)
json_path = audit_dir / "limbu-book-output-audit.json"
markdown_path = audit_dir / "limbu-book-output-audit.md"
json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
status = "passed" if report["passed"] else "failed"
lines = [
"# Limbu Book Output Audit",
"",
f"- status: `{status}`",
f"- output: `{report['output_dir']}`",
f"- issues: {len(report['issues'])}",
f"- warnings: {len(report['warnings'])}",
"",
"## Issues",
"",
]
if report["issues"]:
lines.extend(f"- {issue}" for issue in report["issues"])
else:
lines.append("- none")
lines.extend(["", "## Warnings", ""])
if report["warnings"]:
lines.extend(f"- {warning}" for warning in report["warnings"])
else:
lines.append("- none")
markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("output_dir", type=Path)
parser.add_argument("--audit-dir", type=Path)
args = parser.parse_args(argv)
report = audit_book_output(args.output_dir, args.audit_dir)
print(f"limbu-book-output audit {'passed' if report['passed'] else 'failed'} issues={len(report['issues'])}")
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())