ocr-projects-archive / OCR_INDRA /tools /exp /verify_bundle.sh
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
2.7 kB
#!/usr/bin/env bash
# Offline verification gate for the whole fine-tuning toolchain — NO datasets,
# NO remote access, runs on any box (CPU is fine, ~1-2 min).
#
# Proves, before anything touches the remote eval corpus:
# 1. param invariant: model.safetensors -> IndraRec == 1,236,366 tensors
# 2. export: torch -> ONNX reproduces the rec.onnx contract (dynamo-safe)
# 3. exactness: exported graph == original rec.onnx (ids / lengths / conf)
# 4. smoke: rec-lines run reproduces the recorded smoke CER (3.49 %)
#
# Usage: bash tools/exp/verify_bundle.sh
# Deps: PYTHONPATH=<deps>:indra:tools (default deps dir: $REPO/.deps)
# Writes: results/verify/*.json (+ summary.json with pass/fail per step)
set -euo pipefail
HERE=$(cd "$(dirname "$0")" && pwd)
REPO=$(cd "$HERE/../.." && pwd)
export PYTHONPATH="${DEPS:-$REPO/.deps}:$REPO/indra:$REPO/tools:${PYTHONPATH:-}"
OUT="$REPO/results/verify"
mkdir -p "$OUT"
MODELS="$REPO/models/indra-ocr-v3.1.1"
SMOKE="$REPO/results/smoke"
echo "== 1. selfcheck (param invariant) =="
python3 "$HERE/train_rec.py" --selfcheck
echo "== 2. export zero-shot recognizer to ONNX (from model.safetensors) =="
python3 "$HERE/export_rec_onnx.py" --safetensors "$MODELS/model.safetensors" \
--out "$OUT/rec_zero_shot.onnx" --check
echo "== 3. exactness gate: export vs original rec.onnx =="
python3 "$HERE/exactness_check.py" --onnx "$OUT/rec_zero_shot.onnx" \
--against original --json "$OUT/exactness_zero_shot.json"
echo "== 4. smoke: rec-lines on the recorded synthetic page =="
python3 "$HERE/run_rec.py" --onnx "$MODELS/rec.onnx" \
--boxes "$SMOKE/synth_nepali_gt.jsonl" --out "$OUT/rec_only_smoke.jsonl" --threads 4
python3 "$REPO/tools/indra_eval/eval_indra.py" cer "$SMOKE/synth_nepali_record.jsonl" \
"$OUT/rec_only_smoke.jsonl" --detail "$OUT/cer_smoke.json" --rec-only
echo "== summary =="
python3 - "$OUT" "$SMOKE" <<'EOF'
import json, sys
from pathlib import Path
out, smoke = Path(sys.argv[1]), Path(sys.argv[2])
ex = json.loads((out / "exactness_zero_shot.json").read_text())
cer = json.loads((out / "cer_smoke.json").read_text())
want = json.loads((smoke / "cer_rec_only.json").read_text())["cer_micro"]
got = cer["cer_micro"]
summary = {
"exactness_pass": bool(ex["pass"]),
"exactness_max_dconf": ex["max_abs_dconf_valid"],
"exactness_id_mismatches": ex["id_mismatches"],
"smoke_cer_micro_expected": want,
"smoke_cer_micro_got": got,
"smoke_cer_match": abs(got - want) <= 1e-9,
"pass": bool(ex["pass"]) and abs(got - want) <= 1e-9,
}
(out / "summary.json").write_text(json.dumps(summary, indent=1) + "\n")
print(json.dumps(summary, indent=1))
raise SystemExit(0 if summary["pass"] else 1)
EOF