File size: 18,297 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
#!/usr/bin/env python3
"""Apply a reading-order geometry profile to page OCR document projections."""

from __future__ import annotations

import argparse
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from ocrtech.markdown import render_document_markdown
from ocrtech.manifest import sha256_file
from ocrtech.schemas import Block, Document, TextLine


REPORT_JSON = "reading-order-geometry-profile-application.json"
ASSIGNMENTS_JSONL = "reading-order-geometry-profile-assignments.jsonl"


def _read_json_object(path: Path) -> dict[str, Any]:
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except FileNotFoundError as exc:
        raise ValueError(f"required JSON file is missing: {path}") from exc
    except json.JSONDecodeError as exc:
        raise ValueError(f"invalid JSON in {path}: {exc}") from exc
    if not isinstance(payload, dict):
        raise ValueError(f"JSON file must contain an object: {path}")
    return payload


def _document_paths(documents_dir: Path) -> dict[str, Path]:
    if not documents_dir.is_dir():
        raise ValueError(f"documents directory does not exist: {documents_dir}")
    paths: dict[str, Path] = {}
    for child in sorted(documents_dir.iterdir()):
        if not child.is_dir():
            continue
        document_path = child / "document.json"
        if document_path.is_file():
            paths[child.name] = document_path
    if not paths:
        raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}")
    return paths


def _read_chunks_from_profile(profile_path: Path, profile: dict[str, Any]) -> list[dict[str, Any]]:
    chunks = profile.get("chunks")
    if not isinstance(chunks, list) or not chunks:
        raise ValueError(f"reading-order profile must contain non-empty chunks list: {profile_path}")
    normalized: list[dict[str, Any]] = []
    for index, chunk in enumerate(chunks, start=1):
        if not isinstance(chunk, dict):
            raise ValueError(f"profile chunk {index} must be an object")
        try:
            chunk_index = int(chunk.get("chunk_index") or index)
            cx_median = float(chunk["cx_median"])
            y_min = float(chunk["y_min"])
            y_max = float(chunk["y_max"])
            width_median = float(chunk["width_median"])
        except (KeyError, TypeError, ValueError) as exc:
            raise ValueError(f"profile chunk {index} missing numeric chunk_index/cx_median/y_min/y_max/width_median") from exc
        if width_median <= 0:
            raise ValueError(f"profile chunk {index} width_median must be positive")
        normalized.append(
            {
                "chunk_index": chunk_index,
                "cx_median": cx_median,
                "y_min": y_min,
                "y_max": y_max,
                "width_median": width_median,
                "start_line_id": str(chunk.get("start_line_id") or ""),
                "end_line_id": str(chunk.get("end_line_id") or ""),
                "start_reference_line_id": str(chunk.get("start_reference_line_id") or ""),
                "end_reference_line_id": str(chunk.get("end_reference_line_id") or ""),
            }
        )
    normalized.sort(key=lambda item: item["chunk_index"])
    return normalized


def _resolve_profile_path(value: str, base_dir: Path) -> Path:
    path = Path(value)
    if path.is_absolute() or path.exists():
        return path
    candidate = base_dir / path
    if candidate.exists():
        return candidate
    return path


def _read_profile_map(profile_map_path: Path) -> tuple[dict[str, Path], Path | None]:
    payload = _read_json_object(profile_map_path)
    base_dir = profile_map_path.parent
    sample_profiles: dict[str, Path] = {}
    default_profile: Path | None = None
    if "profiles" in payload:
        profiles = payload.get("profiles")
        if not isinstance(profiles, list):
            raise ValueError(f"profile map profiles must be a list: {profile_map_path}")
        for index, row in enumerate(profiles, start=1):
            if not isinstance(row, dict):
                raise ValueError(f"profile map row {index} must be an object")
            sample_id = str(row.get("sample_id") or "").strip()
            profile_value = row.get("profile")
            if not sample_id:
                raise ValueError(f"profile map row {index} missing sample_id")
            if not isinstance(profile_value, str) or not profile_value:
                raise ValueError(f"profile map row {index} missing profile")
            if sample_id in sample_profiles:
                raise ValueError(f"profile map has duplicate sample_id: {sample_id}")
            sample_profiles[sample_id] = _resolve_profile_path(profile_value, base_dir)
    else:
        for sample_id, profile_value in payload.items():
            if sample_id == "default_profile":
                continue
            if not isinstance(profile_value, str) or not profile_value:
                raise ValueError(f"profile map value for {sample_id!r} must be a profile path string")
            sample_profiles[str(sample_id)] = _resolve_profile_path(profile_value, base_dir)
    default_value = payload.get("default_profile")
    if isinstance(default_value, str) and default_value:
        default_profile = _resolve_profile_path(default_value, base_dir)
    return sample_profiles, default_profile


def _load_profile_bundle(
    profile_path: Path,
    *,
    require_profile_claim_safe: bool,
    cache: dict[Path, dict[str, Any]],
) -> dict[str, Any]:
    resolved = profile_path.resolve()
    if resolved in cache:
        return cache[resolved]
    profile = _read_json_object(profile_path)
    if require_profile_claim_safe and profile.get("profile_claim_safe") is not True:
        raise ValueError(f"--require-profile-claim-safe requested but profile_claim_safe is not true: {profile_path}")
    bundle = {
        "path": profile_path,
        "sha256": sha256_file(profile_path),
        "profile_claim_safe": bool(profile.get("profile_claim_safe", False)),
        "chunks": _read_chunks_from_profile(profile_path, profile),
    }
    cache[resolved] = bundle
    return bundle


def _profile_bundle_for_sample(
    sample_id: str,
    *,
    single_profile_path: Path | None,
    profile_map: dict[str, Path],
    default_profile: Path | None,
    require_profile_claim_safe: bool,
    cache: dict[Path, dict[str, Any]],
) -> dict[str, Any]:
    profile_path = single_profile_path or profile_map.get(sample_id) or default_profile
    if profile_path is None:
        raise ValueError(f"no reading-order profile configured for sample_id={sample_id!r}")
    return _load_profile_bundle(
        profile_path,
        require_profile_claim_safe=require_profile_claim_safe,
        cache=cache,
    )


def _chunk_score(
    line: TextLine,
    chunk: dict[str, Any],
    *,
    x_scale: float,
    y_scale: float,
    outside_weight: float,
) -> float:
    center_x = line.bbox.x + line.bbox.w / 2.0
    x_scale_value = max(80.0, float(chunk["width_median"]) * x_scale)
    dx = abs(center_x - float(chunk["cx_median"])) / x_scale_value
    if line.bbox.y < float(chunk["y_min"]):
        outside_y = float(chunk["y_min"]) - line.bbox.y
    elif line.bbox.y > float(chunk["y_max"]):
        outside_y = line.bbox.y - float(chunk["y_max"])
    else:
        outside_y = 0.0
    dy = outside_y / y_scale
    return dx + outside_weight * dy


def _assign_line(
    line: TextLine,
    chunks: list[dict[str, Any]],
    *,
    x_scale: float,
    y_scale: float,
    outside_weight: float,
) -> tuple[dict[str, Any], float]:
    best_chunk = chunks[0]
    best_score = _chunk_score(line, best_chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight)
    for chunk in chunks[1:]:
        score = _chunk_score(line, chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight)
        if score < best_score:
            best_chunk = chunk
            best_score = score
    return best_chunk, best_score


def _block_key(block: Block, line_keys: dict[str, tuple[int, float, float, float, float, str]]) -> tuple[int, float, float, float, float, str]:
    if block.line_ids:
        key = line_keys.get(block.line_ids[0])
        if key is not None:
            return key
    return (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, block.block_id)


def _write_document_artifacts(document_dir: Path, document: Document) -> None:
    document_dir.mkdir(parents=True, exist_ok=True)
    (document_dir / "document.json").write_text(json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8")
    (document_dir / "document.body.md").write_text(render_document_markdown(document, include_structural_roles=False), encoding="utf-8")


def apply_profile(args: argparse.Namespace) -> dict[str, Any]:
    source_documents_dir = Path(args.source_documents_dir)
    raw_profile = getattr(args, "profile", None)
    raw_profile_map = getattr(args, "profile_map", None)
    profile_path = Path(raw_profile) if raw_profile else None
    profile_map_path = Path(raw_profile_map) if raw_profile_map else None
    out = Path(args.out)
    if source_documents_dir.resolve() == out.resolve():
        raise ValueError("--out must not be the same directory as --source-documents-dir")
    if float(args.x_scale) <= 0 or float(args.y_scale) <= 0 or float(args.outside_weight) < 0:
        raise ValueError("--x-scale/--y-scale must be positive and --outside-weight must be non-negative")
    if (profile_path is None) == (profile_map_path is None):
        raise ValueError("exactly one of --profile or --profile-map is required")
    profile_map: dict[str, Path] = {}
    default_profile: Path | None = None
    if profile_map_path is not None:
        profile_map, default_profile = _read_profile_map(profile_map_path)
    source_paths = _document_paths(source_documents_dir)
    out.mkdir(parents=True, exist_ok=True)
    assignments: list[dict[str, Any]] = []
    page_reports: list[dict[str, Any]] = []
    profile_cache: dict[Path, dict[str, Any]] = {}
    profile_usage: dict[str, dict[str, Any]] = {}
    require_profile_claim_safe = bool(getattr(args, "require_profile_claim_safe", False))
    for sample_id, document_path in sorted(source_paths.items()):
        profile_bundle = _profile_bundle_for_sample(
            sample_id,
            single_profile_path=profile_path,
            profile_map=profile_map,
            default_profile=default_profile,
            require_profile_claim_safe=require_profile_claim_safe,
            cache=profile_cache,
        )
        chunks = profile_bundle["chunks"]
        profile_key = str(profile_bundle["path"])
        profile_usage.setdefault(
            profile_key,
            {
                "profile": profile_key,
                "profile_sha256": profile_bundle["sha256"],
                "profile_claim_safe": profile_bundle["profile_claim_safe"],
                "chunk_count": len(chunks),
                "sample_ids": [],
            },
        )["sample_ids"].append(sample_id)
        document = Document.from_dict(_read_json_object(document_path))
        for page in document.pages:
            line_keys: dict[str, tuple[int, float, float, float, float, str]] = {}
            for line in page.text_lines:
                chunk, score = _assign_line(
                    line,
                    chunks,
                    x_scale=float(args.x_scale),
                    y_scale=float(args.y_scale),
                    outside_weight=float(args.outside_weight),
                )
                line_id = line.line_id or ""
                line_keys[line_id] = (
                    int(chunk["chunk_index"]),
                    line.bbox.y,
                    line.bbox.x,
                    line.bbox.h,
                    line.bbox.w,
                    line_id,
                )
                metadata = line.metadata if isinstance(line.metadata, dict) else {}
                metadata["reading_order_geometry_profile_assignment"] = {
                    "profile": str(profile_bundle["path"]),
                    "profile_sha256": profile_bundle["sha256"],
                    "chunk_index": int(chunk["chunk_index"]),
                    "score": score,
                    "start_reference_line_id": chunk.get("start_reference_line_id") or "",
                    "end_reference_line_id": chunk.get("end_reference_line_id") or "",
                    "start_line_id": chunk.get("start_line_id") or "",
                    "end_line_id": chunk.get("end_line_id") or "",
                    "claim_boundary": "profile-based reading-order assignment; profile provenance controls claim safety",
                }
                line.metadata = metadata
                assignments.append(
                    {
                        "sample_id": sample_id,
                        "page_index": page.page_index,
                        "line_id": line_id,
                        "profile": str(profile_bundle["path"]),
                        "profile_sha256": profile_bundle["sha256"],
                        "chunk_index": int(chunk["chunk_index"]),
                        "score": score,
                        "bbox": line.bbox.to_list(),
                        "text_length": len(line.text or ""),
                    }
                )
            page.text_lines = sorted(page.text_lines, key=lambda line: line_keys.get(line.line_id or "", (10**9, line.bbox.y, line.bbox.x, line.bbox.h, line.bbox.w, line.line_id or "")))
            lines_by_id = {line.line_id or "": line for line in page.text_lines}
            for block in page.blocks:
                if not block.line_ids:
                    continue
                block.line_ids = sorted(
                    block.line_ids,
                    key=lambda line_id: line_keys.get(line_id, (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, line_id)),
                )
                block_lines = [lines_by_id[line_id] for line_id in block.line_ids if line_id in lines_by_id]
                if block_lines:
                    block.text = "\n".join(line.text for line in block_lines)
            page.blocks = sorted(page.blocks, key=lambda block: _block_key(block, line_keys))
            for order, block in enumerate(page.blocks, start=1):
                block.order = order
        target_dir = out / sample_id
        _write_document_artifacts(target_dir, document)
        line_count = sum(len(page.text_lines) for page in document.pages)
        page_reports.append(
            {
                "sample_id": sample_id,
                "source_document": str(document_path),
                "source_document_sha256": sha256_file(document_path),
                "profile": str(profile_bundle["path"]),
                "profile_sha256": profile_bundle["sha256"],
                "profile_claim_safe": profile_bundle["profile_claim_safe"],
                "document_json": str(target_dir / "document.json"),
                "document_json_sha256": sha256_file(target_dir / "document.json"),
                "document_md_sha256": sha256_file(target_dir / "document.md"),
                "document_body_md_sha256": sha256_file(target_dir / "document.body.md"),
                "line_count": line_count,
            }
        )
    assignments_path = out / ASSIGNMENTS_JSONL
    assignments_path.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in assignments), encoding="utf-8")
    report_path = out / REPORT_JSON
    report = {
        "tool": "tools/apply_page_reading_order_geometry_profile.py",
        "created_at_utc": datetime.now(UTC).isoformat(),
        "source_documents_dir": str(source_documents_dir),
        "profile": str(profile_path) if profile_path is not None else "",
        "profile_map": str(profile_map_path) if profile_map_path is not None else "",
        "profile_map_sha256": sha256_file(profile_map_path) if profile_map_path is not None else "",
        "profile_sha256": sha256_file(profile_path) if profile_path is not None else "",
        "profile_claim_safe": all(bool(item["profile_claim_safe"]) for item in profile_usage.values()),
        "profile_usage": list(profile_usage.values()),
        "out": str(out),
        "config": {
            "x_scale": float(args.x_scale),
            "y_scale": float(args.y_scale),
            "outside_weight": float(args.outside_weight),
            "require_profile_claim_safe": require_profile_claim_safe,
        },
        "chunk_count": sum(int(item["chunk_count"]) for item in profile_usage.values()),
        "document_count": len(source_paths),
        "assignment_count": len(assignments),
        "pages": page_reports,
        "assignments_jsonl": str(assignments_path),
        "assignments_jsonl_sha256": sha256_file(assignments_path),
        "claim_boundary": "profile-based reading-order application; not claim-safe unless profile provenance is claim-safe",
    }
    report["report_json"] = str(report_path)
    report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return report


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source-documents-dir", type=Path, required=True)
    profile_group = parser.add_mutually_exclusive_group(required=True)
    profile_group.add_argument("--profile", type=Path)
    profile_group.add_argument("--profile-map", type=Path)
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--x-scale", type=float, default=1.0)
    parser.add_argument("--y-scale", type=float, default=80.0)
    parser.add_argument("--outside-weight", type=float, default=2.0)
    parser.add_argument("--require-profile-claim-safe", action="store_true")
    args = parser.parse_args()
    report = apply_profile(args)
    print(report["report_json"])
    print(f"documents={report['document_count']} assignments={report['assignment_count']} chunks={report['chunk_count']}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())