File size: 8,336 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
#!/usr/bin/env python3
"""Analyze reference-order geometry chunks from page OCR line alignment."""

from __future__ import annotations

import argparse
import csv
import json
import re
import statistics
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from ocrtech.manifest import sha256_file
from ocrtech.schemas import BBox


REPORT_JSON = "reading-order-geometry-analysis.json"
CHUNKS_TSV = "reading-order-geometry-chunks.tsv"
JUMPS_TSV = "reading-order-geometry-jumps.tsv"


def _read_bbox(value: str, *, context: str) -> BBox:
    try:
        payload = json.loads(value)
    except json.JSONDecodeError as exc:
        raise ValueError(f"{context}: bbox is not valid JSON: {value!r}") from exc
    try:
        bbox = BBox.from_any(payload)
    except Exception as exc:
        raise ValueError(f"{context}: bbox is invalid: {value!r}") from exc
    if bbox.w <= 0 or bbox.h <= 0:
        raise ValueError(f"{context}: bbox must have positive width/height: {bbox.to_list()}")
    return bbox


def _line_number(line_id: str) -> int:
    match = re.search(r"(\d+)$", line_id)
    if match is None:
        raise ValueError(f"reference_line_id does not end in a numeric order: {line_id!r}")
    return int(match.group(1))


def _read_matched_rows(path: Path) -> list[dict[str, Any]]:
    try:
        handle = path.open("r", encoding="utf-8", newline="")
    except FileNotFoundError as exc:
        raise ValueError(f"line alignment CSV is missing: {path}") from exc
    rows: list[dict[str, Any]] = []
    with handle:
        reader = csv.DictReader(handle)
        if reader.fieldnames is None:
            raise ValueError(f"line alignment CSV has no header: {path}")
        for row_number, row in enumerate(reader, start=2):
            if row.get("kind") != "matched":
                continue
            reference_line_id = str(row.get("reference_line_id") or "").strip()
            predicted_line_id = str(row.get("predicted_line_id") or "").strip()
            if not reference_line_id or not predicted_line_id:
                continue
            bbox = _read_bbox(str(row.get("reference_bbox") or ""), context=f"{path}:{row_number} reference")
            rows.append(
                {
                    "reference_line_id": reference_line_id,
                    "reference_order": _line_number(reference_line_id),
                    "predicted_line_id": predicted_line_id,
                    "page_index": int(row.get("page_index") or 0),
                    "x": bbox.x,
                    "y": bbox.y,
                    "w": bbox.w,
                    "h": bbox.h,
                    "cx": bbox.x + bbox.w / 2.0,
                    "cy": bbox.y + bbox.h / 2.0,
                }
            )
    rows.sort(key=lambda item: (item["page_index"], item["reference_order"]))
    return rows


def _is_jump(previous: dict[str, Any], current: dict[str, Any], *, min_dx: float, min_upward_dy: float, min_downward_dy: float) -> bool:
    dx = float(current["cx"]) - float(previous["cx"])
    dy = float(current["y"]) - float(previous["y"])
    return abs(dx) > min_dx or dy < -min_upward_dy or dy > min_downward_dy


def _chunk_summary(chunk: list[dict[str, Any]], *, chunk_index: int) -> dict[str, Any]:
    return {
        "chunk_index": chunk_index,
        "start_reference_line_id": chunk[0]["reference_line_id"],
        "end_reference_line_id": chunk[-1]["reference_line_id"],
        "start_reference_order": chunk[0]["reference_order"],
        "end_reference_order": chunk[-1]["reference_order"],
        "line_count": len(chunk),
        "cx_median": statistics.median(float(row["cx"]) for row in chunk),
        "x_min": min(float(row["x"]) for row in chunk),
        "x_max": max(float(row["x"]) + float(row["w"]) for row in chunk),
        "y_min": min(float(row["y"]) for row in chunk),
        "y_max": max(float(row["y"]) for row in chunk),
        "width_median": statistics.median(float(row["w"]) for row in chunk),
        "predicted_line_ids": [row["predicted_line_id"] for row in chunk],
    }


def analyze(args: argparse.Namespace) -> dict[str, Any]:
    alignment_path = Path(args.line_alignment)
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    rows = _read_matched_rows(alignment_path)
    chunks: list[list[dict[str, Any]]] = []
    jumps: list[dict[str, Any]] = []
    current: list[dict[str, Any]] = []
    for row in rows:
        if current and _is_jump(
            current[-1],
            row,
            min_dx=float(args.min_dx),
            min_upward_dy=float(args.min_upward_dy),
            min_downward_dy=float(args.min_downward_dy),
        ):
            previous = current[-1]
            jumps.append(
                {
                    "from_reference_line_id": previous["reference_line_id"],
                    "to_reference_line_id": row["reference_line_id"],
                    "dx": float(row["cx"]) - float(previous["cx"]),
                    "dy": float(row["y"]) - float(previous["y"]),
                    "from_cx": previous["cx"],
                    "from_y": previous["y"],
                    "to_cx": row["cx"],
                    "to_y": row["y"],
                }
            )
            chunks.append(current)
            current = []
        current.append(row)
    if current:
        chunks.append(current)
    chunk_summaries = [_chunk_summary(chunk, chunk_index=index) for index, chunk in enumerate(chunks, start=1)]

    chunks_path = out / CHUNKS_TSV
    with chunks_path.open("w", encoding="utf-8", newline="") as handle:
        fields = [
            "chunk_index",
            "start_reference_line_id",
            "end_reference_line_id",
            "line_count",
            "cx_median",
            "x_min",
            "x_max",
            "y_min",
            "y_max",
            "width_median",
        ]
        writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t")
        writer.writeheader()
        for chunk in chunk_summaries:
            writer.writerow({field: chunk[field] for field in fields})

    jumps_path = out / JUMPS_TSV
    with jumps_path.open("w", encoding="utf-8", newline="") as handle:
        fields = ["from_reference_line_id", "to_reference_line_id", "dx", "dy", "from_cx", "from_y", "to_cx", "to_y"]
        writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t")
        writer.writeheader()
        for jump in jumps:
            writer.writerow(jump)

    report_path = out / REPORT_JSON
    report = {
        "tool": "tools/analyze_page_reading_order_geometry.py",
        "created_at_utc": datetime.now(UTC).isoformat(),
        "line_alignment": str(alignment_path),
        "line_alignment_sha256": sha256_file(alignment_path),
        "output_dir": str(out),
        "matched_line_count": len(rows),
        "chunk_count": len(chunk_summaries),
        "jump_count": len(jumps),
        "config": {
            "min_dx": float(args.min_dx),
            "min_upward_dy": float(args.min_upward_dy),
            "min_downward_dy": float(args.min_downward_dy),
        },
        "chunks": chunk_summaries,
        "jumps": jumps,
        "chunks_tsv": str(chunks_path),
        "chunks_tsv_sha256": sha256_file(chunks_path),
        "jumps_tsv": str(jumps_path),
        "jumps_tsv_sha256": sha256_file(jumps_path),
        "claim_boundary": "diagnostic geometry analysis of eval alignment; use to design non-oracle reading-order policy, not as OCR output",
    }
    report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return report


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--line-alignment", type=Path, required=True)
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--min-dx", type=float, default=350.0)
    parser.add_argument("--min-upward-dy", type=float, default=200.0)
    parser.add_argument("--min-downward-dy", type=float, default=120.0)
    args = parser.parse_args()
    report = analyze(args)
    print(str(Path(report["output_dir"]) / REPORT_JSON))
    print(f"chunks={report['chunk_count']} jumps={report['jump_count']} matched={report['matched_line_count']}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())