File size: 31,956 Bytes
f9bc0c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
"""Build the complete Indra-OCR x Nepali investigation report as a .docx.

Assembles: cover + executive summary (fresh), the full contents of
docs/INDRA_INVESTIGATION.md, docs/INDRA_EXPERIMENTS.md, logs/investigation.md,
auto-generated smoke-results tables (results/smoke/*.json + GT records), and
an appendix (artifact inventory, dependency versions, reproduce commands).

Run:  PYTHONPATH=/tmp/opencode/indra-lib python3 tools/indra_eval/make_docx.py
Out:  INDRA_OCR_NEPALI_COMPLETE_REPORT.docx (workspace root)
"""
from __future__ import annotations

import json
import os
import re
import sys
import unicodedata
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, "/tmp/opencode/indra-lib")

from docx import Document  # noqa: E402
from docx.enum.text import WD_ALIGN_PARAGRAPH  # noqa: E402
from docx.oxml import OxmlElement  # noqa: E402
from docx.oxml.ns import qn  # noqa: E402
from docx.shared import Pt, RGBColor, Cm  # noqa: E402

OUT = ROOT / "INDRA_OCR_NEPALI_COMPLETE_REPORT.docx"

# ---------------------------------------------------------------- md parser --
def parse_md(md: str):
    """Yield blocks: ('h',lvl,'..') | ('p','..') | ('li',kind,bool_nested,'..')
    | ('code',[lines]) | ('table',[rows]) | ('hr',)"""
    lines = md.splitlines()
    i, out = 0, []
    while i < len(lines):
        ln = lines[i]
        if ln.startswith("```"):
            j, code = i + 1, []
            while j < len(lines) and not lines[j].startswith("```"):
                code.append(lines[j])
                j += 1
            out.append(("code", code))
            i = j + 1
            continue
        if ln.strip() == "":
            i += 1
            continue
        if ln.strip() == "---":
            out.append(("hr",))
            i += 1
            continue
        m = re.match(r"^(#{1,4})\s+(.*)$", ln)
        if m:
            out.append(("h", len(m.group(1)), m.group(2)))
            i += 1
            continue
        if ln.lstrip().startswith("|"):
            rows = []
            while i < len(lines) and lines[i].lstrip().startswith("|"):
                rows.append(lines[i].strip())
                i += 1
            out.append(("table", rows))
            continue
        m = re.match(r"^(\s*)([-*])\s+(.*)$", ln)
        if m:
            out.append(("li", "bullet", len(m.group(1)) >= 2, m.group(3)))
            i += 1
            continue
        m = re.match(r"^(\s*)(\d+)\.\s+(.*)$", ln)
        if m:
            out.append(("li", "number", len(m.group(1)) >= 2, m.group(3)))
            i += 1
            continue
        if ln.startswith("  ") and out and out[-1][0] in ("li", "p") and len(ln) - len(ln.lstrip()) >= 2 and ln.lstrip()[:2] != "- ":
            # continuation of a list item / hanging paragraph
            out.append(("cont", ln.strip()))
            i += 1
            continue
        out.append(("p", ln))
        i += 1
    return out


# --------------------------------------------------------------- docx glue --
def cs_font(run):
    """Complex-script font so Devanagari renders consistently in Word."""
    rpr = run._element.get_or_add_rPr()
    rf = rpr.find(qn("w:rFonts"))
    if rf is None:
        rf = OxmlElement("w:rFonts")
        rpr.append(rf)
    rf.set(qn("w:cs"), "Nirmala UI")


def shade(par, fill="F4F4F4"):
    ppr = par._p.get_or_add_pPr()
    shd = OxmlElement("w:shd")
    shd.set(qn("w:val"), "clear")
    shd.set(qn("w:color"), "auto")
    shd.set(qn("w:fill"), fill)
    ppr.append(shd)


TOK = re.compile(r"(\*\*[^*]+\*\*|`[^`]+`|\*[^*]+\*)")


def add_runs(par, text, size=None, base_bold=False):
    for tok in TOK.split(text):
        if not tok:
            continue
        if tok.startswith("**") and tok.endswith("**") and len(tok) > 4:
            r = par.add_run(tok[2:-2])
            r.bold = True
            cs_font(r)
        elif tok.startswith("`") and tok.endswith("`") and len(tok) > 2:
            r = par.add_run(tok[1:-1])
            r.font.name = "Consolas"
            r.font.size = Pt((size or 10.5) - 1)
            cs_font(r)
        elif tok.startswith("*") and tok.endswith("*") and len(tok) > 2:
            r = par.add_run(tok[1:-1])
            r.italic = True
            cs_font(r)
        else:
            r = par.add_run(tok)
            r.bold = base_bold or None
            cs_font(r)
        if size:
            for r in par.runs:
                pass
    # normalize size on all runs of this paragraph if requested
    if size:
        for r in par.runs:
            if r.font.size is None:
                r.font.size = Pt(size)


def add_table(doc, rows, size=9):
    vals = []
    for r in rows:
        if isinstance(r, str):  # raw markdown row "| a | b |"
            if re.match(r"^\|[\s:|-]+\|$", r.replace(" ", "")):
                continue  # separator row
            cells = [c.strip() for c in r.strip().strip("|").split("|")]
        else:
            cells = [str(c) for c in r]
        vals.append(cells)
    if not vals:
        return
    ncol = max(len(v) for v in vals)
    t = doc.add_table(rows=len(vals), cols=ncol)
    t.style = "Table Grid"
    for ri, row in enumerate(vals):
        for ci in range(ncol):
            cell = t.cell(ri, ci)
            text = row[ci] if ci < len(row) else ""
            par = cell.paragraphs[0]
            add_runs(par, text, size=size, base_bold=(ri == 0))
            par.paragraph_format.space_after = Pt(2)


def render_md(doc, md, first_h1_prefix=""):
    """Render markdown under the doc, # (first) -> Heading 1 (with prefix)."""
    blocks = parse_md(md)
    seen_h1 = False
    for b in blocks:
        kind = b[0]
        if kind == "h":
            lvl, text = b[1], b[2]
            if lvl == 1:
                if seen_h1:
                    continue  # only one H1 per part
                seen_h1 = True
                text = first_h1_prefix + text
                h = doc.add_heading(text, level=1)  # used only for Part titles
                for r in h.runs:
                    r.font.color.rgb = RGBColor(0x1F, 0x3B, 0x63)
                continue
            h = doc.add_heading("", level=lvl)
            add_runs(h, text)
            for r in h.runs:
                r.font.color.rgb = RGBColor(0x22, 0x35, 0x52)
        elif kind == "p":
            par = doc.add_paragraph()
            par.paragraph_format.space_after = Pt(4)
            add_runs(par, b[1])
        elif kind == "cont":
            par = doc.paragraphs[-1]
            par.add_run("\n" + b[1])
        elif kind == "li":
            _, lk, nested, text = b
            style = "List Bullet" if lk == "bullet" else "List Number"
            if nested:
                style += " 2" if lk == "bullet" else ""
            par = doc.add_paragraph(style=style)
            par.paragraph_format.space_after = Pt(2)
            add_runs(par, text)
        elif kind == "code":
            for ln in b[1]:
                par = doc.add_paragraph()
                par.paragraph_format.space_after = Pt(0)
                par.paragraph_format.space_before = Pt(0)
                r = par.add_run(ln if ln else " ")
                r.font.name = "Consolas"
                r.font.size = Pt(8.5)
                cs_font(r)
                shade(par)
            doc.add_paragraph().paragraph_format.space_after = Pt(4)
        elif kind == "table":
            add_table(doc, b[1])
            doc.add_paragraph().paragraph_format.space_after = Pt(4)
        elif kind == "hr":
            continue


def lev(a: str, b: str) -> int:
    if a == b:
        return 0
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i] + [0] * len(b)
        for j, cb in enumerate(b, 1):
            cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))
        prev = cur
    return prev[-1]


# ------------------------------------------------------------------ report --
def main():
    doc = Document()
    for sec in doc.sections:
        sec.top_margin = sec.bottom_margin = Cm(2.0)
        sec.left_margin = sec.right_margin = Cm(2.2)
    st = doc.styles["Normal"]
    st.font.name = "Calibri"
    st.font.size = Pt(10.5)

    cp = doc.core_properties
    cp.title = "Indra OCR Γ— Nepali (Devanagari) β€” Complete Engineering Investigation"
    cp.author = "OCR_INDRA investigation"
    cp.subject = "datalab-to/indra v0.3.2 / model revision v3.1.1 β€” evidence-based feasibility, adaptation and evaluation"
    cp.comments = "All claims verified from source (commit 3d773ae) and model artifacts (ONNX + safetensors), not from README."

    # ---- cover ----
    doc.add_heading("Indra OCR Γ— Nepali (Devanagari)", level=0)
    p = doc.add_paragraph()
    r = p.add_run("Complete Engineering Investigation Report")
    r.font.size = Pt(16)
    r.font.color.rgb = RGBColor(0x33, 0x33, 0x33)
    meta = doc.add_paragraph()
    add_runs(
        meta,
        "Package: `datalab-to/indra` v0.3.2 (commit `3d773ae`) Β· Model set: "
        "HF revision `v3.1.1` Β· Workspace: `OCR_INDRA/` Β· Date: 2026-10-17 "
        "(rev. 2026-10-20: fine-tune toolkit verified offline; bundle rebuilt)",
        size=10,
    )
    doc.add_paragraph()
    add_runs(
        doc.add_paragraph(),
        "Method: every architecture claim in this report is verified against the "
        "actual artifacts β€” the cloned source (all 17 files of `indra/indra_ocr/`), "
        "the 201-node ONNX graph of `rec.onnx`, the graph I/O of all 5 models, and "
        "the shipped `model.safetensors` (108 F32 tensors) β€” never from README "
        "marketing. Verification scripts are included and re-runnable "
        "(Appendix B). Blocked items are marked [B]; verified items [V]; "
        "refuted prior claims [X].",
    )
    doc.add_page_break()

    # ---- Part I: executive summary ----
    doc.add_heading("Part I β€” Executive summary", level=1)
    h = doc.add_heading("Question", level=3)
    add_runs(
        doc.add_paragraph(),
        "Can Datalab's Indra OCR be evaluated, adapted or fine-tuned for a "
        "Nepali/Devanagari OCR use case β€” and could it beat the deployed "
        "PaddleOCR PP-OCRv5 Mobile Devanagari pipeline? Decisions had to be "
        "based on the actual source code and ONNX graphs, not the README.",
    )
    doc.add_heading("Seven key findings (all verified)", level=2)
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**The ~1 M-param recognizer is a padding-masked MobileNetV3-style 1-D "
        "CNN with a flat CTC head β€” no RNN, no attention.** Exactly "
        "**1,236,366 parameters** (safetensors ↔ ONNX initializer cross-check). "
        "Stem Conv 1β†’128 (kernel 32Γ—4, stride 32Γ—4) collapses the line height "
        "entirely; then 10 identical residual blocks [depthwise 1Γ—9 β†’ expand "
        "1Γ—1 128β†’256 β†’ HardSwish β†’ SE (padding-aware) β†’ project 1Γ—1 256β†’128 "
        "β†’ +residual]; then a 1,678-class head.",
    )
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**Time stride is 4, not 8** (the `WIDTH_MULTIPLE=8` comment in "
        "`rec_preproc.py:24` is inaccurate as written). The graph computes "
        "`lengths = ceil(widths/4)`, clipped to `W/4` (measured table, "
        "e.g. widths 65 β†’ 17). The graph outputs `ids [B,T]`, `conf [B,T]`, "
        "`lengths [B]`; padded timesteps are NOT special-cased (head-on-zeros "
        "β†’ blank, conf β‰ˆ 0.78), so callers must truncate by `lengths` β€” as "
        "`pipeline.py:297` does.",
    )
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**Dynamic padding mask is functional, not cosmetic.** "
        "`Cast_4 = (arange(W/4) < lengths).float()` zeroes the padded region "
        "post-stem, after every block's residual (proven: final-trunk features "
        "at padded t are exactly 0.0 even with ink present), and normalizes "
        "every SE pooling over valid timesteps only.",
    )
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**Vocabulary already covers 100% of Devanagari.** 1,677 tokens + "
        "blank class 0 (head 1,678); all 128 assigned codepoints in "
        "U+0900–U+097F verified present (letters, matras, nukta, danda ΰ₯€ ΰ₯₯, "
        "Devanagari digits ΰ₯¦β€“ΰ₯―, β‚Ή, ΰ₯). **No head surgery is needed for "
        "Nepali** β€” the only adaptation question is training-data distribution.",
    )
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**Fine-tuning is a reimplementation project, but legally and "
        "architecturally allowed.** No training code exists in the repo "
        "(grep-verified; `pyproject.toml` has no torch) and the artifacts "
        "ship trainable weights ONLY for the recognizer (`model.safetensors`, "
        "`rec/` prefix). det/order/orient/everything else exist as frozen "
        "ONNX only. The weights license β€” AI PUBS Open RAIL-M (modified) β€” "
        "explicitly permits fine-tuning (Β§II ΒΆ36); the inference code is "
        "Apache-2.0.",
    )
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**The evaluation harness is built and smoke-proven on real shaped "
        "Devanagari.** Zero-shot full-pipeline run on a synthetic Nepali "
        "page (raqm-shaped, OFL Noto Sans Devanagari): 10/10 lines detected, "
        "reading order correct, micro CER **2.62%**; recognizer-only on the "
        "identical GT crops: CER **3.49%**. Observed failure modes (nukta "
        "ΰ€°ΰ€Όβ†’ΰ€° dropped, β‚Ήβ†’ΰ€°, ΰ₯β†’ΰ€, conjunct matras `ΰ€Έΰ€™ΰ₯ΰ€•ΰ₯‡ΰ€€β†’ΰ€Έΰ€™ΰ₯ΰ€—ΰ₯‡ΰ€€`) are exactly "
        "the distribution gaps that fine-tuning (Experiment B) targets.",
    )
    add_runs(
        doc.add_paragraph(style="List Number"),
        "**The fine-tune toolkit is built and verified offline (2026-10-20).** "
        "`tools/exp/` reconstructs the recognizer in PyTorch from the shipped "
        "safetensors, fine-tunes with CTC, and re-exports ONNX: the zero-shot "
        "export matches the shipped `rec.onnx` with 0 id mismatches and max "
        "|Ξ”conf| 7.1e-6 (gate 1e-3); a mechanics training run (loss 8.1β†’1.65, "
        "val CER 0.97β†’0.58 on an overfit probe) passes the post-train gate "
        "(torch `.pt` vs its own ONNX, Ξ”conf 1.3e-5) and `run_rec.py` reproduces "
        "the recorded smoke CER (3.49 %) bit-for-bit. `verify_bundle.sh` pins "
        "all of this in one data-free command, and the rebuilt "
        "`indra_nepali_exp.bundle` carries it to the remote.",
    )
    doc.add_heading("Verdict", level=2)
    add_runs(
        doc.add_paragraph(),
        "**Zero-shot evaluation (Experiment A) is ready today** β€” the adapter "
        "(`tools/indra_eval/eval_indra.py`) runs Indra end-to-end on CPU and "
        "scores CER/WER, and the Experiment B toolchain is verified offline "
        "(`tools/exp/verify_bundle.sh`); only the frozen Nepali eval sets on the "
        "remote machine are missing (`khalti-t4` β€” SSH still rejected for the "
        "presented key on 2026-10-20: authorize it or hand over the bundle). "
        "**The recognizer is the only component worth adapting**, via the "
        "shipped safetensors init; a re-export must reproduce the padding "
        "mask + per-block re-zeroing + SE valid-means + the "
        "`ids/conf/lengths` decode tail. Detector/order fine-tuning is "
        "out of scope until the zero-shot full-pipeline run shows detection "
        "as the bottleneck (Experiment A). Recommended order: **A β†’ D (hybrid "
        "PP-det + Indra-rec, zero training) β†’ B (rec fine-tune) β†’ E (int8 "
        "deploy)**.",
    )
    doc.add_page_break()

    # ---- Part II: full investigation doc ----
    doc.add_heading("Part II β€” Architecture investigation (full evidence)", level=1)
    md = (ROOT / "docs" / "INDRA_INVESTIGATION.md").read_text(encoding="utf-8")
    # drop the md's own title + status legend line (already on cover)
    parts = re.split(r"^## 0\. Verdict", md, maxsplit=1, flags=re.M)
    md = "## 0. Verdict" + parts[1]
    render_md(doc, md, first_h1_prefix="")
    doc.add_page_break()

    # ---- Part III: experiments doc ----
    doc.add_heading("Part III β€” Experiment plan (A–E) and governance", level=1)
    md = (ROOT / "docs" / "INDRA_EXPERIMENTS.md").read_text(encoding="utf-8")
    render_md(doc, md, first_h1_prefix="")
    doc.add_page_break()

    # ---- Part IV: log ----
    doc.add_heading("Part IV β€” Investigation log", level=1)
    md = (ROOT / "logs" / "investigation.md").read_text(encoding="utf-8")
    render_md(doc, md, first_h1_prefix="")
    doc.add_page_break()

    # ---- Part V: smoke results (generated from artifacts) ----
    doc.add_heading("Part V β€” Smoke-test results (generated from result files)", level=1)
    gt = json.loads((ROOT / "results/smoke/synth_nepali_record.jsonl").read_text(encoding="utf-8").splitlines()[0])
    rec_only = json.loads((ROOT / "results/smoke/rec_only.jsonl").read_text().splitlines()[0])
    full = json.loads((ROOT / "results/smoke/full_run.jsonl").read_text().splitlines()[0])
    summary = {
        "mode": {
            "rec-only": rec_only,
            "full pipeline": full,
        }
    }
    doc.add_heading("V.1 Setup", level=2)
    add_runs(
        doc.add_paragraph(),
        "Synthetic A4-ratio page (1240Γ—1754 px), 10 Nepali lines rendered at "
        "32 px with raqm shaping (conjuncts, nukta, digits, ΰ₯, β‚Ή, danda). "
        "Ground truth is exact by construction. Weights: Indra v3.1.1 "
        "official, CPU (onnxruntime 1.31.0). **Numbers are an environment "
        "smoke test, not a benchmark** β€” the frozen remote eval sets are "
        "needed for the real comparison (Experiment A).",
    )
    doc.add_heading("V.2 Headline numbers", level=2)
    rows = []
    for mode, rec in summary["mode"].items():
        cer = lev_text_stats(rec, gt)
        rows.append([mode, str(len(rec["lines"])), f"{rec['mean_conf']:.3f}", f"{cer['cer']:.4f}", f"{cer['wer']:.4f}"])
    add_table(
        doc,
        [["mode", "lines found", "mean conf", "micro CER", "micro WER"]]
        + rows,
    )
    doc.add_paragraph()
    doc.add_heading("V.3 Per-line comparison (recognizer-only, GT crops)", level=2)
    rows = []
    for i, (gl, pl) in enumerate(zip(gt["lines"], rec_only["lines"])):
        g = unicodedata.normalize("NFC", gl["text"])
        p = unicodedata.normalize("NFC", pl["text"])
        d = lev(g, p)
        mark = "βœ“" if g == p else f"{d} edits"
        rows.append([str(i), g, p, f"{pl['conf']:.3f}", mark])
    add_table(doc, [["#", "ground truth", "prediction", "conf", "Ξ”"], *rows], size=9)
    doc.add_paragraph()
    doc.add_heading("V.4 Per-line comparison (full pipeline, detector crops)", level=2)
    rows = []
    for i, (gl, pl) in enumerate(zip(gt["lines"], full["lines"])):
        g = unicodedata.normalize("NFC", gl["text"])
        p = unicodedata.normalize("NFC", pl["text"])
        d = lev(g, p)
        mark = "βœ“" if g == p else f"{d} edits"
        rows.append([str(i), g, p, f"{pl['conf']:.3f}", mark])
    add_table(doc, [["#", "ground truth", "prediction", "conf", "Ξ”"], *rows], size=9)
    doc.add_paragraph()
    doc.add_heading("V.5 Interpretation", level=2)
    add_runs(
        doc.add_paragraph(),
        "Both runs read the page almost perfectly; the remaining errors are "
        "print-size artifacts (32 px) of exactly the classes a Nepali corpus "
        "would reinforce: the nukta dot (ΰ€°ΰ€Ό), the rarely-trained β‚Ή and ΰ₯, and "
        "conjunct vmatras. The rec-only vs full discrepancy on line 2 "
        "(ΰ₯ β†’ ΰ€ vs S) and line 3/5/7 flips demonstrates crop-offset "
        "sensitivity β€” which is why the fair design compares recognizers on "
        "**identical crops** (rec-lines mode) and pipelines on page text. "
        "Timings on this WSL box are not representative (CPU/IO-starved; det "
        "β‰ˆ 18.6 min on one page) β€” benchmark on the T4.",
    )
    doc.add_page_break()

    # ---- Part VI: appendix ----
    doc.add_heading("Part VI β€” Appendix", level=1)
    doc.add_heading("A. Artifact inventory", level=2)
    rows = []
    for rel, why in [
        ("docs/INDRA_INVESTIGATION.md", "architecture + evidence + [U] closure"),
        ("docs/INDRA_EXPERIMENTS.md", "experiments A–E, leakage, licensing, decision tree"),
        ("logs/investigation.md", "command/evidence log"),
        ("tools/indra_eval/inspect_vocab.py", "vocab + Devanagari coverage verification"),
        ("tools/indra_eval/inspect_models.py", "graph I/O + param-count cross-check"),
        ("tools/indra_eval/inspect_rec_nodes.py", "full rec.onnx node dump + numeric run"),
        ("tools/indra_eval/inspect_rec_mask.py", "mask semantics + lengths table proof"),
        ("tools/indra_eval/make_synth_page.py", "shaped Devanagari test page + GT"),
        ("tools/indra_eval/eval_indra.py", "predict (full / rec-lines) + CER scorer"),
        ("tools/exp/torch_rec.py", "PyTorch re-implementation of rec.onnx + weight map"),
        ("tools/exp/train_rec.py", "CTC fine-tune loop (safetensors init, width-bucketed batches)"),
        ("tools/exp/export_rec_onnx.py", "torch -> ONNX with the exact ids/conf/lengths contract"),
        ("tools/exp/exactness_check.py", "gate: export vs rec.onnx / torch (ids, lengths, conf)"),
        ("tools/exp/run_rec.py", "batch rec-lines runner (ONNX or --torch backends)"),
        ("tools/exp/verify_bundle.sh", "one-command offline verification gate (no data needed)"),
        ("tools/exp/e_int8_quantize.py", "Experiment E: static (QDQ) / dynamic int8 quantization"),
        ("tools/exp/run_a.sh", "Experiment A orchestration (rec-lines + GT records + CER)"),
        ("tools/exp/run_b.sh", "Experiment B orchestration (train -> export -> gate -> re-score)"),
        ("tools/exp/data_prep.py", "corpus -> manifests, page-hash split, leakage exclude"),
        ("tools/exp/a_export_gt_boxes.py", "GT boxes adapters (validate/normalize/merge)"),
        ("tools/exp/a_ppocr_baseline.py", "identical-crop baseline materials (engine = remote plug)"),
        ("tools/exp/remote_setup.sh", "remote deps + probe + verification gate"),
        ("tools/exp/make_bundle.sh", "builds indra_nepali_exp.bundle for the remote"),
        ("tools/indra_eval/testdata/NotoSansDevanagari.ttf", "OFL font (test rendering only)"),
        ("models/indra-ocr-v3.1.1/", "downloaded official weights (16 files, ~46 MB)"),
        ("results/smoke/", "smoke artifacts (pages, records, CER JSON)"),
        ("results/verify/", "offline verification evidence (exactness + smoke re-run)"),
    ]:
        pth = ROOT / rel
        n = (
            sum(f.stat().st_size for f in pth.rglob("*") if f.is_file())
            if pth.is_dir()
            else (pth.stat().st_size if pth.exists() else 0)
        )
        rows.append([rel, why, f"{n / 1024:.0f} KiB" if n else "β€”"])
    add_table(doc, [["path", "purpose", "size"], *rows], size=9)
    doc.add_paragraph()

    doc.add_heading("B. Reproduce everything", level=2)
    code = '\n'.join(
        [
            "cd /mnt/c/Users/Dell/Desktop/OCR_INDRA     # or wherever this workspace is",
            "PY=/tmp/opencode/indra-lib                 # pip --target deps",
            "export PYTHONPATH=\"$PWD/indra:$PY\"",
            "",
            "python3 tools/indra_eval/inspect_vocab.py            # vocab + Devanagari coverage",
            "python3 tools/indra_eval/inspect_models.py           # graph I/O + param count",
            "python3 tools/indra_eval/inspect_rec_nodes.py        # full node dump + numeric run",
            "python3 tools/indra_eval/inspect_rec_mask.py         # mask semantics + lengths table",
            "",
            "python3 tools/indra_eval/make_synth_page.py results/smoke      # synth page + GT",
            "python3 tools/indra_eval/eval_indra.py predict results/smoke/synth_nepali_page.png \\",
            "        --boxes results/smoke/synth_nepali_gt.jsonl --jsonl results/smoke/rec_only.jsonl",
            "python3 tools/indra_eval/eval_indra.py predict results/smoke/synth_nepali_page.png \\",
            "        --jsonl results/smoke/full_run.jsonl --print-text",
            "python3 tools/indra_eval/eval_indra.py cer results/smoke/synth_nepali_record.jsonl \\",
            "        results/smoke/rec_only.jsonl --detail results/smoke/cer_rec_only.json --rec-only",
            "python3 tools/indra_eval/eval_indra.py cer results/smoke/synth_nepali_record.jsonl \\",
            "        results/smoke/full_run.jsonl --detail results/smoke/cer_full.json",
            "",
            "# fine-tune toolkit: one-command offline verification (no data needed)",
            "bash tools/exp/verify_bundle.sh   # -> results/verify/summary.json {pass: true}",
            "",
            "python3 tools/indra_eval/make_docx.py     # regenerate THIS report",
            "",
            "# remote (after SSH/bundle access): docs/REMOTE_RUN_SHEET.md",
            "#   bash tools/exp/remote_setup.sh        # deps + probe + same verify gate",
            "#   bash tools/exp/run_a.sh               # zero-shot A on the frozen sets",
            "#   bash tools/exp/run_b.sh               # fine-tune B (train -> export -> gate -> re-score)",
        ]
    )
    for ln in code.split("\n"):
        par = doc.add_paragraph()
        par.paragraph_format.space_after = Pt(0)
        r = par.add_run(ln if ln else " ")
        r.font.name = "Consolas"
        r.font.size = Pt(8.5)
        cs_font(r)
        shade(par)
    doc.add_paragraph()
    doc.add_heading("C. Environment & dependencies", level=2)
    add_table(
        doc,
        [
            ["component", "version / note"],
            ["OS / box", "Linux (WSL-backed /mnt/c), 8 cores, 3 GB RAM β€” timings NOT representative"],
            ["Python", "3.12.3 system; deps via pip --target /tmp/opencode/indra-lib"],
            ["torch", "2.14.1+cpu β€” export forced to the legacy TorchScript tracer (see run sheet Β§0)"],
            ["onnxruntime", "1.31.0 (CPU provider)"],
            ["Pillow", "13.0.0 β€” raqm available (Devanagari shaping verified)"],
            ["opencv / onnx / pypdfium2 / hf-hub / safetensors / numpy", "installed in the same target dir"],
            ["python-docx", "1.2.0 (this report's generator)"],
            ["Remote machine", "khalti-t4 (202.51.0.7) β€” SSH key still rejected 2026-10-20 (port 22 open); all dataset-side work blocked"],
        ],
    )
    doc.add_paragraph()
    doc.add_heading("D. Licensing (verified from files)", level=2)
    add_table(
        doc,
        [
            ["item", "license", "fine-tune", "deploy / notes"],
            ["Indra v3.1.1 model weights", "AI PUBS Open RAIL-M (modified) v0.1",
             "**explicitly allowed** (Β§II ΒΆ36); Attachment A use restrictions attach",
             "redistribution keeps license + usage notice; derivatives inherit it"],
            ["indra inference code", "Apache-2.0", "β€”", "standard reuse rights"],
            ["PaddleOCR (baseline, remote)", "Apache-2.0", "already fine-tuned", "deployed today"],
            ["Noto Sans Devanagari (test font)", "SIL OFL 1.1", "β€”", "rendering only"],
            ["Nepali datasets / frozen eval sets", "docs/DATA_LICENSING.md [B]",
             "[B] review pending SSH", "no datasets leave the remote machine"],
        ],
    )
    doc.add_page_break()

    # ---- Part VII: toolkit status + remote readiness (2026-10-20) ----
    doc.add_heading("Part VII β€” Fine-tune toolkit & remote readiness (2026-10-20)", level=1)
    add_runs(
        doc.add_paragraph(),
        "The Experiment A/B toolchain in `tools/exp/` was completed and verified "
        "offline this revision; the rebuilt single-file bundle "
        "`indra_nepali_exp.bundle` carries it to `khalti-t4`. No dataset was "
        "touched (none is reachable).",
    )
    doc.add_heading("Verified offline", level=2)
    add_table(
        doc,
        [
            ["check", "command", "result"],
            ["bundle integrity", "`git bundle verify` + fresh clone + byte-compare", "OK β€” complete history, 16 model files byte-identical"],
            ["param invariant", "`train_rec.py --selfcheck`", "1,236,366 tensors (parameters 1,236,110 + BN buffers 256)"],
            ["zero-shot export parity", "`export_rec_onnx.py --safetensors` vs `rec.onnx`", "**pass** β€” 0 id mismatches, 0 lengths mismatches, max |Ξ”conf| 7.1e-6"],
            ["training mechanics", "`train_rec.py` 120 steps, overfit probe", "loss 8.1 β†’ 1.65; val CER 0.97 β†’ 0.58; checkpoints + `best.safetensors` written"],
            ["post-train gate", "`exactness_check.py --against torch`", "**pass** β€” 0 id mismatches, Ξ”conf 1.3e-5"],
            ["backend parity", "`run_rec.py` ONNX vs `--torch`", "identical texts on the smoke page"],
            ["A orchestration", "`run_a.sh` on a local stand-in set", "GT records β†’ rec-lines β†’ CER 0.0349 (matches smoke exactly)"],
            ["B orchestration", "`run_b.sh` (train β†’ export β†’ gate β†’ re-score)", "complete; findings written under `results/`"],
            ["data adapters", "`a_export_gt_boxes.py`, `a_ppocr_baseline.py`, `data_prep.py`", "all subcommands exercised OK"],
        ],
        size=9,
    )
    doc.add_paragraph()
    doc.add_heading("Fixed this revision (each would have broken the remote run)", level=2)
    add_runs(
        doc.add_paragraph(style="List Bullet"),
        "torch β‰₯ 2.6 `torch.onnx.export` defaults to the torch.export exporter "
        "(needs `onnxscript`); the export now pins the verified legacy tracer "
        "(`dynamo=False`) with a fallback for older torch.",
    )
    add_runs(
        doc.add_paragraph(style="List Bullet"),
        "`run_a.sh` / `run_b.sh` crashed on unset `PYTHONPATH` under `set -u`.",
    )
    add_runs(
        doc.add_paragraph(style="List Bullet"),
        "`a_ppocr_baseline.py` rejected the `\"file\"` GT schema and did not "
        "resolve page paths relative to the boxes file.",
    )
    add_runs(
        doc.add_paragraph(style="List Bullet"),
        "`remote_setup.sh` wrote `probe.json` relative to the caller's cwd and "
        "ignored combined flags; it now also runs the full verification gate.",
    )
    add_runs(
        doc.add_paragraph(style="List Bullet"),
        "`data_prep.py` hash-split could leave val empty on small corpora.",
    )
    doc.add_heading("Remote status", level=2)
    add_runs(
        doc.add_paragraph(),
        "SSH to `khalti-t4` (202.51.0.7) is still rejected for key "
        "`khalti-t4-ocr-team` (`Permission denied (publickey,password)`; port 22 "
        "open, 2026-10-20). To proceed: authorize the key's public half on the "
        "box (or transfer the bundle to someone with access), then follow "
        "`docs/REMOTE_RUN_SHEET.md` β€” `remote_setup.sh` (deps + probe + gate) β†’ "
        "probe inventory (Β§1) β†’ `run_a.sh` β†’ `run_b.sh`. Only `results/`, "
        "`runs/rec_ft/*` and `results/probe.json` travel back; datasets never "
        "leave the machine.",
    )
    doc.save(OUT)
    print(f"written: {OUT} ({OUT.stat().st_size / 1024:.0f} KiB)")


def lev_text_stats(rec, gt):
    gt_text = [unicodedata.normalize("NFC", l["text"]) for l in gt["lines"]]
    pr_text = [unicodedata.normalize("NFC", l["text"]) for l in rec["lines"]]
    ce = sum(lev(a, b) for a, b in zip(gt_text, pr_text))
    cn = sum(len(a) for a in gt_text)
    gw = " ".join(gt_text).split()
    pw = " ".join(pr_text).split()
    prev = list(range(len(pw) + 1))
    for i, w in enumerate(gw, 1):
        cur = [i] + [0] * len(pw)
        for j, v in enumerate(pw, 1):
            cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (w != v))
        prev = cur
    return {"cer": ce / max(1, cn), "wer": prev[-1] / max(1, len(gw))}


if __name__ == "__main__":
    main()