File size: 10,660 Bytes
ed46c5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """Appendix tables added for the revision: crop mechanism, matched noise comparison, label sensitivity,
recognizer choice on validation, other backbones and the thesis-B recognizers. Each table is written only when its
results file exists."""
import json
import os
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
R = f"{ROOT}/results"
T = f"{ROOT}/paper/tables"
def have(name):
return os.path.exists(f"{R}/{name}")
def write(name, body):
open(f"{T}/{name}", "w").write(body)
print("wrote", name)
if have("crop_mechanism.json"):
c = json.load(open(f"{R}/crop_mechanism.json"))
rows = [("CER (\\%)", "cer"), ("Median CER per image (\\%)", "median_cer"),
("Copies of another inscription (\\%)", "recalled_training_text"),
("Output words among 150 most frequent (\\%)", "output_words_frequent"),
("Reference words reproduced (\\%)", "ref_kept_all"), ("\\quad frequent words", "ref_kept_frequent"),
("\\quad other words", "ref_kept_other"), ("Rare-word recall (\\%)", "rare_recall"),
("End-to-end chrF++", "chrfpp")]
doms = [("all", "All"), ("gr", "GR"), ("rdb_photo", "RuneS"), ("card", "Cards")]
body = ["\\begin{table*}[t]", "\\centering", "\\caption{Recognizer output on whole images and on crops of the 550 test images. \\emph{W}: recognizer trained on whole images; \\emph{W/c}: the same recognizer applied to the crops; \\emph{C}: recognizer trained on crops. A copy of another inscription is an output at least 90\\% similar to a training transliteration and at least 50\\% CER from its own reference. Frequent words are the 150 most frequent training words; rare words have at least four letters and are not frequent.}",
"\\label{tab:crop}", "\\small", "\\setlength{\\tabcolsep}{5pt}", "\\begin{tabular}{l" + "rrr" * 4 + "}", "\\toprule",
" & " + " & ".join(f"\\multicolumn{{3}}{{c}}{{{d}}}" for _, d in doms) + " \\\\",
"".join(f"\\cmidrule(lr){{{2 + 3 * i}-{4 + 3 * i}}}" for i in range(4)),
" & " + " & ".join(["W & W/c & C"] * 4) + " \\\\", "\\midrule"]
for lab, k in rows:
vals = []
for d, _ in doms:
for s in ("whole", "whole_on_crop", "crop"):
vals.append(f"{c[s][d][k]:.1f}")
body.append(lab + " & " + " & ".join(vals) + " \\\\")
p = c["per_image"]
body.append("\\midrule")
body.append("Word precision (\\%) & " + " & ".join(f"{p[d]['word_precision_whole']:.1f} & -- & {p[d]['word_precision_crop']:.1f}" for d, _ in doms) + " \\\\")
body.append("Images, C better / worse by $>$10 CER points & " + " & ".join(f"\\multicolumn{{3}}{{c}}{{{p[d]['crop_better_10pt']} / {p[d]['crop_worse_10pt']}}}" for d, _ in doms) + " \\\\")
body += ["\\bottomrule", "\\end{tabular}", "\\end{table*}", ""]
write("crop.tex", "\n".join(body))
if have("matched_noise.json"):
m = json.load(open(f"{R}/matched_noise.json"))
body = ["\\begin{table}[t]", "\\centering", "\\caption{Real recognition errors and synthetic corruption on the same test photographs: chrF++ retained relative to translation of the reference transliterations (\\%), with the CER of the input. Bins group the photographs by recognizer CER; synthetic columns corrupt the reference transliterations of the same photographs at three nominal rates.}",
"\\label{tab:matched}", "\\footnotesize", "\\setlength{\\tabcolsep}{2pt}", "\\begin{tabular}{lrrrrr}", "\\toprule",
"Photographs & $n$ & Recognizer & Synth.\\ 0.2 & Synth.\\ 0.3 & Synth.\\ 0.45 \\\\", "\\midrule"]
for tr, name in (("mt4b", "LoRA translator"), ("mt4b_noise", "noise-trained translator")):
body.append(f"\\multicolumn{{6}}{{l}}{{\\emph{{{name}}}}} \\\\")
for k, b in m[tr]["bins"].items():
lo, hi = k.split("-")
lab = f"CER {lo}--{hi}\\%" if hi != "inf" else f"CER $\\geq${lo}\\%"
cells = [f"{b['real']['retention']:.0f} ({b['real']['cer']:.0f})"] + \
[f"{b[f'synthetic_{lv}']['retention']:.0f} ({b[f'synthetic_{lv}']['cer']:.0f})" for lv in ("0.2", "0.3", "0.45")]
body.append(f"{lab} & {b['n']} & " + " & ".join(cells) + " \\\\")
mm = m[tr]["matched"]
body.append(f"CER-matched pairs & {mm['n']} & {mm['real_retention']:.0f} ({mm['real_cer']:.0f}) & \\multicolumn{{3}}{{c}}{{{mm['synthetic_retention']:.0f} ({mm['synthetic_cer']:.0f}), $p{'<0.001' if mm['paired_real_gt_synthetic']['p'] < 0.001 else '=' + format(mm['paired_real_gt_synthetic']['p'], '.3f')}$}} \\\\")
body += ["\\bottomrule", "\\end{tabular}", "\\end{table}", ""]
write("matched.tex", "\n".join(body))
if have("label_sensitivity.json"):
s = json.load(open(f"{R}/label_sensitivity.json"))
body = ["\\begin{table}[t]", "\\centering", "\\caption{CER (\\%) of the two 4B recognizers against the full reference and against references without signs marked as restored (square brackets) or as restored or uncertain (square or round brackets), on the test images whose reference keeps at least one sign.}",
"\\label{tab:labels}", "\\small", "\\begin{tabular}{lrrr}", "\\toprule",
"Recognizer & Full & w/o [\\,] & w/o [\\,] and (\\,) \\\\", "\\midrule"]
for tag, name in (("ocr4b_mixcut", "Whole images"), ("ocr4b_cropcut", "Crops")):
v = s[tag]
body.append(f"{name} & {v['without_square']['cer_full_same_images']:.1f} & {v['without_square']['cer']:.1f} & {v['without_square_round']['cer']:.1f} \\\\")
v = s["ocr4b_cropcut"]
body.append(f"Images & {v['without_square']['n']} & {v['without_square']['n']} & {v['without_square_round']['n']} \\\\")
body.append(f"Reference signs removed (\\%) & -- & {v['without_square']['ref_chars_removed_pct']:.1f} & {v['without_square_round']['ref_chars_removed_pct']:.1f} \\\\")
body += ["\\bottomrule", "\\end{tabular}", "\\end{table}", ""]
write("labels.tex", "\n".join(body))
if have("valsel.json"):
v = json.load(open(f"{R}/valsel.json"))
body = ["\\begin{table*}[t]", "\\centering", "\\caption{Recognizers trained on real images, on the 212 validation images: recognition after loop truncation and end-to-end translation of their output by the 4B translator (chrF++, 95\\%% cluster bootstrap intervals). $p$: paired cluster bootstrap test that the recognizer trained on crops translates better. Translation of the reference transliterations of the same images: %.1f chrF++.}" % v["reference_input"]["all"]["chrfpp"],
"\\label{tab:valsel}", "\\small", "\\begin{tabular}{llrrrrrrrr}", "\\toprule",
"Training images & Input & CER (\\%) & Median & Rare & chrF++ & GR & RuneS & Cards & $p$ \\\\", "\\midrule"]
names = {"ocr4b_real": "real", "ocr4b_mix": "real + procedural", "ocr4b_diffmix": "real + diffusion",
"ocr4b_mix_hr": "real + procedural, 1.05\\,MP", "ocr4b_mix_oncrop": "real + procedural", "ocr4b_crop": "crops + procedural"}
for k, nm in names.items():
if k not in v:
continue
d = v[k]
o, e = d["ocr"], d["e2e"]
p = d.get("paired_crop_minus_this", {}).get("p")
ps = "--" if p is None else ("$<$0.001" if p < 0.001 else f"{p:.3f}")
body.append(f"{nm} & {'crop' if d['input'] == 'crop' else 'whole'} & {o['cer']:.1f} [{o['cer_ci'][0]:.1f}, {o['cer_ci'][1]:.1f}] & {d['median_cer']:.1f} & {d['rare_recall']:.1f} & "
f"{e['chrfpp']:.1f} [{e['chrfpp_ci'][0]:.1f}, {e['chrfpp_ci'][1]:.1f}] & "
+ " & ".join(f"{d['by_domain'][x]['chrfpp']:.1f}" for x in ("gr", "rdb_photo", "card")) + f" & {ps} \\\\")
body += ["\\bottomrule", "\\end{tabular}", "\\end{table*}", ""]
write("valsel.tex", "\n".join(body))
if have("backbones.json"):
b = json.load(open(f"{R}/backbones.json"))
g = json.load(open(f"{R}/gold_eval.json")) if have("gold_eval.json") else {}
ns = json.load(open(f"{R}/perf_test_nospace.json")) if have("perf_test_nospace.json") else {}
def pv(d, k):
p = d.get(k, {}).get("p")
return "--" if p is None else ("$<$0.001" if p < 0.001 else f"{p:.3f}")
body = ["\\begin{table*}[t]", "\\centering",
"\\caption{Other backbones trained with the same recipes, on the 550 test images (recognizers, crop recipe) and the 1,264 test pairs (translators, LoRA recipe). Recognizers: CER after loop truncation, median CER per image, rare-word recall, CER with spaces removed on the 37 lines of the gold set of \\cite{perfilev2026thesis} whose inscriptions are not in our training split, and chrF++ of the 4B translator applied to the output. Translators: chrF++ from reference transliterations, name recall, and chrF++ from the output of the 4B crop recognizer. $p$: paired cluster bootstrap test that the Qwen3.5-4B model scores higher. Rows marked \\cite{perfilev2026thesis}: the released adapters of that thesis, run with its inference settings.}",
"\\label{tab:backbones}", "\\small", "\\begin{tabular}{lrrrrrr}", "\\toprule",
"Recognizer & CER (\\%) & Median & Rare & Gold CER & chrF++ & $p$ \\\\", "\\midrule"]
gk = {"ocr4b": "ocr4b_crop", "q25vl7b": "q25vl7b_crop", "q3vl8b": "q3vl8b_crop"}
for k, v in b["ocr"].items():
t = v["test"]
gold = g.get(gk[k], {}).get("nontrain", {}).get("cer")
body.append(f"\\texttt{{{v['hf'].split('/')[1]}}} & {t['all']['cer']:.1f} & {t['median_cer']:.1f} & {t['rare_recall']:.1f} & "
f"{'--' if gold is None else f'{gold:.1f}'} & {t['e2e']['chrfpp']:.1f} & {pv(v, 'paired_ocr4b_minus_this')} \\\\")
for t, name in (("qwen25vl7b", "Qwen2.5-VL-7B"), ("qwen3vl8b", "Qwen3-VL-8B")):
k = f"{t}_testcrop"
if k in b["perfilev"]:
d = b["perfilev"][k]
gold = g.get(f"perf_{t}", {}).get("nontrain", {}).get("cer")
nsc = ns.get(f"ocr_testcrop_perf_{t}", {}).get("cer_nospace")
body.append(f"\\cite{{perfilev2026thesis}}, {name} & {d['all']['cer']:.1f} & {d['median_cer']:.1f} & {d['rare_recall']:.1f} & "
f"{'--' if gold is None else f'{gold:.1f}'} & -- & -- \\\\")
body += ["\\midrule", "Translator & Reference & English & Swedish & Names & From photo & $p$ \\\\", "\\midrule"]
for k, v in b["mt"].items():
body.append(f"\\texttt{{{v['hf'].split('/')[1]}}} & {v['ref']['chrfpp']:.1f} & {v['en']['chrfpp']:.1f} & {v['sv']['chrfpp']:.1f} & "
f"{v['names']:.1f} & {v['ocr']['chrfpp']:.1f} & {pv(v, 'paired_mt4b_minus_this_ref')} \\\\")
body += ["\\bottomrule", "\\end{tabular}", "\\end{table*}", ""]
write("backbones.tex", "\n".join(body))
|