File size: 13,061 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
#!/usr/bin/env python3
"""Audit a candidate-line review application output."""

from __future__ import annotations

import argparse
import csv
import json
import re
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from ocrtech.manifest import sha256_file
from ocrtech.normalization import normalize_ocr_text
from ocrtech.schemas import Document


def _read_json_object(path: Path, issues: list[str], *, label: str) -> dict[str, Any] | None:
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except FileNotFoundError:
        issues.append(f"{label} is missing: {path}")
        return None
    except json.JSONDecodeError as exc:
        issues.append(f"{label} is invalid JSON: {path} ({exc})")
        return None
    if not isinstance(payload, dict):
        issues.append(f"{label} must be a JSON object: {path}")
        return None
    return payload


def _read_jsonl(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]:
    if not path.is_file():
        issues.append(f"{label} is missing: {path}")
        return []
    rows: list[dict[str, Any]] = []
    for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
        if not line.strip():
            continue
        try:
            payload = json.loads(line)
        except json.JSONDecodeError as exc:
            issues.append(f"{label} invalid JSON at line {line_number}: {exc}")
            continue
        if not isinstance(payload, dict):
            issues.append(f"{label} row {line_number} must be a JSON object")
            continue
        rows.append(payload)
    return rows


def _read_tsv(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]:
    if not path.is_file():
        issues.append(f"{label} is missing: {path}")
        return []
    rows: list[dict[str, Any]] = []
    with path.open("r", encoding="utf-8", newline="") as handle:
        reader = csv.DictReader(handle, delimiter="\t")
        if reader.fieldnames is None:
            issues.append(f"{label} has no header: {path}")
            return []
        for row_number, row in enumerate(reader, start=2):
            payload = {key: value for key, value in row.items() if key is not None}
            payload["_row_number"] = row_number
            rows.append(payload)
    return rows


def _read_review_rows(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]:
    if path.suffix.lower() == ".tsv":
        return _read_tsv(path, issues, label=label)
    return _read_jsonl(path, issues, label=label)


def _check_file_sha(path: Path, expected: object, issues: list[str], *, label: str) -> None:
    if not path.is_file():
        issues.append(f"{label} is missing: {path}")
        return
    if not isinstance(expected, str) or not expected:
        issues.append(f"{label} missing recorded SHA-256")
        return
    actual = sha256_file(path)
    if actual != expected:
        issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual} path={path}")


def _candidate_inserted_lines(document: Document) -> list[Any]:
    lines: list[Any] = []
    for page in document.pages:
        for line in page.text_lines:
            metadata = line.metadata if isinstance(line.metadata, dict) else {}
            if isinstance(metadata.get("candidate_line_review_insertion"), dict):
                lines.append(line)
    return lines


def _candidate_inserted_lines_from_review(document: Document, review_path: Path) -> list[Any]:
    expected = str(review_path)
    lines: list[Any] = []
    for line in _candidate_inserted_lines(document):
        metadata = line.metadata if isinstance(line.metadata, dict) else {}
        insertion = metadata.get("candidate_line_review_insertion")
        if not isinstance(insertion, dict):
            continue
        if str(insertion.get("source_review_jsonl") or "") == expected:
            lines.append(line)
    return lines


def _validate_current_inserted_lines(lines: list[Any], issues: list[str], *, page_id: str) -> None:
    for line in lines:
        metadata = line.metadata if isinstance(line.metadata, dict) else {}
        insertion = metadata.get("candidate_line_review_insertion")
        if not isinstance(insertion, dict):
            continue
        line_id = str(getattr(line, "line_id", "") or "")
        corrected_text = normalize_ocr_text(str(insertion.get("corrected_text") or ""))
        inserted_text = normalize_ocr_text(str(insertion.get("inserted_text") or ""))
        line_text = normalize_ocr_text(str(getattr(line, "text", "") or ""))
        if not corrected_text.strip():
            issues.append(f"{page_id} inserted candidate line {line_id!r} missing corrected_text")
        if not inserted_text.strip():
            issues.append(f"{page_id} inserted candidate line {line_id!r} missing inserted_text")
        if inserted_text.strip() and line_text != inserted_text:
            issues.append(
                f"{page_id} inserted candidate line {line_id!r} text does not match inserted_text: "
                f"line={line_text!r} inserted={inserted_text!r}"
            )
        if corrected_text.strip() and line_text != corrected_text:
            issues.append(
                f"{page_id} inserted candidate line {line_id!r} text does not match corrected_text: "
                f"line={line_text!r} corrected={corrected_text!r}"
            )


def _oracle_or_reference_assisted_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    flagged: list[dict[str, Any]] = []
    for index, row in enumerate(rows, start=1):
        values = [
            row.get("selection_policy"),
            row.get("claim_boundary"),
            row.get("reviewer"),
            row.get("notes"),
        ]
        keys = set(row)
        text = " ".join(str(value or "") for value in values).lower()
        text = re.sub(r"\bnon[-_ ]oracle\b", " ", text)
        has_oracle_marker = bool(
            re.search(r"\boracle(?:_|-|$|\s)", text)
            or "reference_overlap" in text
            or any(str(key).startswith("oracle_") for key in keys)
        )
        if has_oracle_marker:
            flagged.append(
                {
                    "row_number": index,
                    "row_id": row.get("row_id") or "",
                    "reviewer": row.get("reviewer") or "",
                    "selection_policy": row.get("selection_policy") or "",
                }
            )
    return flagged


def audit_application(args: argparse.Namespace) -> dict[str, Any]:
    output_dir = Path(args.output_dir)
    issues: list[str] = []
    warnings: list[str] = []
    application_audit_path = output_dir / "candidate-line-insertion-audit.json"
    application = _read_json_object(application_audit_path, issues, label="candidate-line-insertion-audit.json")
    inserted_line_count = 0
    total_inserted_line_count = 0
    pair_rows: list[dict[str, Any]] = []
    review_rows: list[dict[str, Any]] = []
    reference_oracle_rows: list[dict[str, Any]] = []
    if application is not None:
        source_book_run = Path(str(application.get("source_book_run") or ""))
        _check_file_sha(source_book_run, application.get("source_book_run_sha256"), issues, label="source book-run.json")
        review_jsonl = Path(str(application.get("candidate_review_jsonl") or ""))
        _check_file_sha(review_jsonl, application.get("candidate_review_jsonl_sha256"), issues, label="candidate review JSONL")
        review_rows = _read_review_rows(review_jsonl, issues, label="candidate review file")
        reference_oracle_rows = _oracle_or_reference_assisted_rows(review_rows)
        if reference_oracle_rows:
            warnings.append(
                "candidate review JSONL contains oracle/reference-assisted rows; "
                "application is diagnostic-only and must not pass operational admission"
            )
        _check_file_sha(Path(str(application.get("book_md") or output_dir / "book.md")), application.get("book_md_sha256"), issues, label="book.md")
        _check_file_sha(
            Path(str(application.get("book_body_md") or output_dir / "book.body.md")),
            application.get("book_body_md_sha256"),
            issues,
            label="book.body.md",
        )
        pairs_path = Path(str(application.get("insertion_pairs") or output_dir / "candidate-line-insertion-pairs.jsonl"))
        _check_file_sha(pairs_path, application.get("insertion_pairs_sha256"), issues, label="candidate-line-insertion-pairs.jsonl")
        pair_rows = _read_jsonl(pairs_path, issues, label="candidate-line-insertion-pairs.jsonl")
        pages = application.get("pages")
        if not isinstance(pages, list) or not all(isinstance(page, dict) for page in pages):
            issues.append("candidate-line-insertion-audit.json pages must be a list of objects")
            pages = []
        for index, page in enumerate(pages):
            page_id = str(page.get("page_id") or f"page-{index}")
            source_document = Path(str(page.get("source_document") or ""))
            _check_file_sha(source_document, page.get("source_document_sha256"), issues, label=f"{page_id} source document")
            document_json = Path(str(page.get("document_json") or ""))
            _check_file_sha(document_json, page.get("document_json_sha256"), issues, label=f"{page_id} document.json")
            document_dir = Path(str(page.get("document_dir") or ""))
            _check_file_sha(document_dir / "document.md", page.get("document_md_sha256"), issues, label=f"{page_id} document.md")
            _check_file_sha(
                document_dir / "document.body.md",
                page.get("document_body_md_sha256"),
                issues,
                label=f"{page_id} document.body.md",
            )
            score_path = Path(str(page.get("score_document_path") or ""))
            _check_file_sha(score_path, page.get("score_document_sha256"), issues, label=f"{page_id} score document")
            document_payload = _read_json_object(document_json, issues, label=f"{page_id} document.json")
            if document_payload is not None:
                document = Document.from_dict(document_payload)
                total_page_inserted = _candidate_inserted_lines(document)
                page_inserted = _candidate_inserted_lines_from_review(document, review_jsonl)
                _validate_current_inserted_lines(page_inserted, issues, page_id=page_id)
                total_inserted_line_count += len(total_page_inserted)
                inserted_line_count += len(page_inserted)
                expected_page_count = int(page.get("applied_count") or 0)
                if len(page_inserted) != expected_page_count:
                    issues.append(f"{page_id} inserted line count mismatch: recorded={expected_page_count} actual={len(page_inserted)}")
        applied_count = int(application.get("applied_count") or 0)
        if inserted_line_count != applied_count:
            issues.append(f"applied_count mismatch: recorded={applied_count} inserted_lines={inserted_line_count}")
        if len(pair_rows) != applied_count:
            issues.append(f"insertion pair count mismatch: applied_count={applied_count} pairs={len(pair_rows)}")
    report = {
        "created_at_utc": datetime.now(UTC).isoformat(),
        "tool": "tools/audit_candidate_line_review_application.py",
        "output_dir": str(output_dir),
        "application_audit": str(application_audit_path),
        "audit_passed": not issues,
        "issues": issues,
        "warnings": warnings,
        "applied_count": application.get("applied_count") if isinstance(application, dict) else None,
        "inserted_line_count": inserted_line_count,
        "total_inserted_line_count": total_inserted_line_count,
        "inherited_inserted_line_count": total_inserted_line_count - inserted_line_count,
        "insertion_pair_count": len(pair_rows),
        "review_row_count": len(review_rows),
        "reference_oracle_review_count": len(reference_oracle_rows),
        "reference_oracle_review_rows": reference_oracle_rows,
        "operational_review_safe": not reference_oracle_rows,
        "claim_boundary": "artifact consistency audit for candidate-line review application; it does not certify OCR quality",
    }
    out = Path(args.out) if getattr(args, "out", None) else output_dir / "candidate-line-application-audit.json"
    out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return report


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("output_dir", type=Path)
    parser.add_argument("--out", type=Path)
    args = parser.parse_args()
    report = audit_application(args)
    print(f"candidate-line-application audit {'passed' if report['audit_passed'] else 'failed'} issues={len(report['issues'])}")
    raise SystemExit(0 if report["audit_passed"] else 1)


if __name__ == "__main__":
    main()