File size: 7,861 Bytes
f9bc0c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 | """Experiment E: quantize the recognizer for CPU deployment (int8).
Findings that shaped this tool (measured locally 2026-10-20, WSL CPU):
* ORT *dynamic* quantization (`quantize_dynamic`, weights-only) makes this
Conv+SE model SLOWER (0.69x fp32 throughput here: ConvInteger +
per-activation quantize/dequantize dominates), although greedy ids are
identical and max |dconf| is 0.077 on the smoke page.
* => the deployment path is STATIC uint8 (QDQ) quantization with a small
calibration set of real line crops; this script builds exactly that.
The pipeline auto-loads `rec.int8.onnx` when it sits next to `rec.onnx`
(indra_ocr/pipeline.py:195-200), so `--out models/indra-ocr-v3.1.1/rec.int8.onnx`
is a drop-in. Acceptance (docs/INDRA_EXPERIMENTS.md, Experiment E):
CER regression <= +0.3 pts absolute and >= 2x throughput — measure with
`tools/exp/run_rec.py` on the frozen sets, both fp32 and int8.
Calibration input: a manifest jsonl with the same shapes train_rec accepts
{"image": "line.png", "text": "..."} (pre-cropped line image)
{"image": "page.png", "box": [x0,y0,x1,y1], ...} (line box on a page)
Paths relative to --calib-root (default: manifest's directory). Crops go
through the SHARED rec_preproc.crop_line, so calibration sees exactly the
inference distribution. 100-500 lines is plenty.
Usage (remote):
PYTHONPATH=.deps:$PWD/indra:$PWD/tools python3 tools/exp/e_int8_quantize.py \
--calib data/val_manifest.jsonl --calib-root ~/ocr/ocr-deva-ft/data \
--out models/indra-ocr-v3.1.1/rec.int8.onnx --check-smoke
# then re-run Experiment A with the int8 model:
python3 tools/exp/run_rec.py --onnx models/indra-ocr-v3.1.1/rec.int8.onnx ...
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import REPO, load_gray, read_jsonl # noqa: E402
MODEL_DIR = REPO / "models" / "indra-ocr-v3.1.1"
def build_calib(records: list[dict], root: Path, max_lines: int):
"""manifest -> list of [1,32,W] ink maps via the shared crop_line."""
from indra_ocr.rec_preproc import crop_line
crops = []
for r in records:
img = r.get("image") or r.get("file")
p = Path(img)
if not p.is_absolute():
p = (root / p).resolve()
gray = load_gray(p)
boxes = ([ln["box"] for ln in r["lines"]] if "lines" in r
else [r.get("box")] if r.get("box") else [[0.0, 0.0, float(gray.shape[1]), float(gray.shape[0])]])
for b in boxes:
crop = crop_line(gray, b) # [32,W] f32
crops.append(crop[None]) # [1,32,W]
if len(crops) >= max_lines:
return crops
return crops
class CalibReader:
"""Yields the recognizer's real inputs {x, widths} (batch = 1 line)."""
def __init__(self, crops, multiple: int = 8):
self.items = []
for c in crops:
w = c.shape[2]
wp = (w + multiple - 1) // multiple * multiple
x = np.zeros((1, 1, 32, wp), np.float32)
x[0, 0, :, :w] = c[0]
self.items.append({"x": x, "widths": np.array([w], np.int64)})
self.i = 0
def get_next(self):
if self.i >= len(self.items):
return None
it = self.items[self.i]
self.i += 1
return it
def rewind(self):
self.i = 0
def smoke_check(fp_onnx: Path, int8_onnx: Path) -> dict:
"""ids / lengths / conf agreement on the recorded smoke page (indicative)."""
import onnxruntime as ort
from common import read_jsonl as _rj
gt = _rj(REPO / "results" / "smoke" / "synth_nepali_gt.jsonl")[0]
page = REPO / "results" / "smoke" / "synth_nepali_page.png"
gray = load_gray(page)
from indra_ocr.rec_preproc import crop_line, pad_batch
crops = [crop_line(gray, ln["box"]) for ln in gt["lines"]]
x, widths = pad_batch(crops)
def run(path):
sess = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"])
return sess.run(None, {"x": x, "widths": widths})
f_ids, f_conf, f_len = run(fp_onnx)
q_ids, q_conf, q_len = run(int8_onnx)
id_bad = conf_delta = 0.0
id_bad = int(sum(int((f_ids[b, : f_len[b]] != q_ids[b, : q_len[b]]).sum()) for b in range(len(f_len))))
conf_delta = max(float(np.abs(f_conf[b, : f_len[b]] - q_conf[b, : q_len[b]]).max(initial=0.0))
for b in range(len(f_len)))
return {
"id_mismatches": id_bad,
"lengths_equal": bool((f_len == q_len).all()),
"max_abs_dconf": conf_delta,
}
def main(argv=None):
ap = argparse.ArgumentParser("e_int8_quantize")
ap.add_argument("--rec", type=Path, default=MODEL_DIR / "rec.onnx")
ap.add_argument("--out", type=Path, default=MODEL_DIR / "rec.int8.onnx")
ap.add_argument("--calib", type=Path, default=None,
help="manifest jsonl of line crops/boxes (REQUIRED for static mode)")
ap.add_argument("--calib-root", type=Path, default=None,
help="root the manifest paths are relative to (default: manifest dir)")
ap.add_argument("--max-lines", type=int, default=256)
ap.add_argument("--method", choices=["static", "dynamic"], default="static")
ap.add_argument("--per-channel", action="store_true", help="static: per-channel weight scales")
ap.add_argument("--check-smoke", action="store_true", help="compare vs fp32 on the smoke page")
args = ap.parse_args(argv)
if not args.rec.exists():
raise SystemExit(f"--rec not found: {args.rec}")
if args.method == "dynamic":
from onnxruntime.quantization import QuantType, quantize_dynamic
print("warning: dynamic (weights-only) quantization measured SLOWER than fp32 "
"for this Conv+SE model — use static with calibration for deployment")
quantize_dynamic(str(args.rec), str(args.out), weight_type=QuantType.QInt8,
per_channel=args.per_channel)
calib_n = 0
else:
if not args.calib:
raise SystemExit("static quantization needs --calib (manifest jsonl of line crops)")
records = read_jsonl(args.calib)
root = args.calib_root or args.calib.resolve().parent
crops = build_calib(records, root, args.max_lines)
if not crops:
raise SystemExit(f"no crops built from {args.calib} — check paths/boxes")
print(f"calibration: {len(crops)} line crops from {args.calib}")
from onnxruntime.quantization import (CalibrationMethod, QuantFormat,
QuantType, quantize_static)
reader = CalibReader(crops)
quantize_static(
str(args.rec), str(args.out), reader,
quant_format=QuantFormat.QDQ,
activation_type=QuantType.QUInt8,
weight_type=QuantType.QInt8,
per_channel=args.per_channel,
calibrate_method=CalibrationMethod.MinMax,
)
calib_n = len(crops)
report = {
"method": args.method,
"rec": str(args.rec),
"out": str(args.out),
"size_fp32_mb": round(args.rec.stat().st_size / 1e6, 2),
"size_int8_mb": round(args.out.stat().st_size / 1e6, 2),
"calib_lines": calib_n,
}
if args.check_smoke:
report["smoke_vs_fp32"] = smoke_check(args.rec, args.out)
report_path = args.out.with_suffix(".report.json")
report_path.write_text(json.dumps(report, indent=1) + "\n")
print(json.dumps(report, indent=1))
print(f"wrote {args.out} + {report_path}")
print("NOTE: acceptance still requires CER on the frozen eval sets "
"(run_rec.py fp32 vs int8) and throughput ≥2× on the deployment box.")
if __name__ == "__main__":
main()
|