File size: 7,138 Bytes
b21e63d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
"""Benchmark LightGBM joblib vs ONNX Runtime inference.

Loads N real feature rows from `data/reference_dataset.parquet`, runs each
sample one at a time through both backends (simulating the per-request path),
and reports p50/p95/p99 latency plus numerical equivalence.

Run:
    uv run python scripts/benchmark_onnx.py --n 1000
"""

from __future__ import annotations

import argparse
import json
import time
from pathlib import Path
from statistics import median

import joblib
import numpy as np
import onnxruntime as ort
import pandas as pd

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MODEL_PATH = ROOT / "models" / "model.joblib"
DEFAULT_ONNX_PATH = ROOT / "models" / "model.onnx"
DEFAULT_FEATURE_NAMES_PATH = ROOT / "models" / "feature_names.json"
DEFAULT_REFERENCE_PATH = ROOT / "data" / "reference_dataset.parquet"


def percentile(samples: list[float], pct: float) -> float:
    """Linear-interpolated percentile (pct in 0..100)."""
    return float(np.percentile(samples, pct))


def fmt_ms(seconds: float) -> str:
    return f"{seconds * 1000:.3f} ms"


def bench_lightgbm(model, samples_df: pd.DataFrame) -> tuple[list[float], np.ndarray]:
    """Run predict_proba one row at a time. Returns (latencies_s, probas)."""
    latencies: list[float] = []
    probas = np.empty(len(samples_df), dtype=np.float64)
    for i in range(len(samples_df)):
        row = samples_df.iloc[[i]]
        t0 = time.perf_counter()
        p = model.predict_proba(row)[0, 1]
        latencies.append(time.perf_counter() - t0)
        probas[i] = p
    return latencies, probas


def bench_onnx(
    session: ort.InferenceSession,
    input_name: str,
    output_name: str,
    samples_array: np.ndarray,
) -> tuple[list[float], np.ndarray]:
    """Run ONNX inference one row at a time. Returns (latencies_s, probas)."""
    latencies: list[float] = []
    probas = np.empty(len(samples_array), dtype=np.float64)
    for i in range(len(samples_array)):
        # Reshape to (1, n_features); float32 is required by the ONNX graph.
        row = samples_array[i : i + 1]
        t0 = time.perf_counter()
        out = session.run([output_name], {input_name: row})[0]
        latencies.append(time.perf_counter() - t0)
        # zipmap=False → out shape is (1, 2): [prob_class_0, prob_class_1].
        probas[i] = out[0, 1]
    return latencies, probas


def summarise(name: str, latencies: list[float]) -> dict[str, float]:
    return {
        "backend": name,
        "n": len(latencies),
        "p50_ms": percentile(latencies, 50) * 1000,
        "p95_ms": percentile(latencies, 95) * 1000,
        "p99_ms": percentile(latencies, 99) * 1000,
        "mean_ms": float(np.mean(latencies)) * 1000,
        "total_s": sum(latencies),
    }


def print_table(rows: list[dict[str, float]]) -> None:
    print(
        f"{'Backend':<14}{'n':>6}{'p50':>12}{'p95':>12}{'p99':>12}{'mean':>12}"
    )
    for r in rows:
        print(
            f"{r['backend']:<14}{r['n']:>6}"
            f"{r['p50_ms']:>10.3f}ms"
            f"{r['p95_ms']:>10.3f}ms"
            f"{r['p99_ms']:>10.3f}ms"
            f"{r['mean_ms']:>10.3f}ms"
        )


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--n", type=int, default=1000, help="Number of inferences")
    parser.add_argument("--warmup", type=int, default=20)
    parser.add_argument("--model", type=Path, default=DEFAULT_MODEL_PATH)
    parser.add_argument("--onnx", type=Path, default=DEFAULT_ONNX_PATH)
    parser.add_argument(
        "--feature-names", type=Path, default=DEFAULT_FEATURE_NAMES_PATH
    )
    parser.add_argument("--reference", type=Path, default=DEFAULT_REFERENCE_PATH)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument(
        "--out",
        type=Path,
        default=None,
        help="Optional path to write a JSON report (e.g. profiling/benchmark_onnx.json).",
    )
    args = parser.parse_args()

    feature_names = json.loads(args.feature_names.read_text())

    # Sample N rows from the reference dataset, aligned to model feature order.
    reference = pd.read_parquet(args.reference)
    rng = np.random.default_rng(args.seed)
    idx = rng.choice(len(reference), size=args.n + args.warmup, replace=True)
    samples_df = reference.iloc[idx][feature_names].reset_index(drop=True)
    samples_array = samples_df.to_numpy(dtype=np.float32)

    # --- Load both backends ---
    print("Loading LightGBM model...")
    model = joblib.load(args.model)
    raw_model = model.get_raw_model() if hasattr(model, "get_raw_model") else model

    print("Loading ONNX session...")
    session = ort.InferenceSession(
        str(args.onnx), providers=["CPUExecutionProvider"]
    )
    input_name = session.get_inputs()[0].name
    output_name = session.get_outputs()[1].name  # probabilities output

    # --- Warmup (excluded from stats) ---
    print(f"Warmup x{args.warmup}...")
    bench_lightgbm(raw_model, samples_df.iloc[: args.warmup])
    bench_onnx(session, input_name, output_name, samples_array[: args.warmup])

    # --- Measured runs ---
    print(f"Benchmarking LightGBM (n={args.n})...")
    lgbm_lat, lgbm_proba = bench_lightgbm(raw_model, samples_df.iloc[args.warmup :])

    print(f"Benchmarking ONNX     (n={args.n})...")
    onnx_lat, onnx_proba = bench_onnx(
        session, input_name, output_name, samples_array[args.warmup :]
    )

    # --- Numerical equivalence ---
    diff = np.abs(lgbm_proba - onnx_proba)
    print()
    print("=== Numerical equivalence ===")
    print(f"max  |delta_proba| = {diff.max():.2e}")
    print(f"mean |delta_proba| = {diff.mean():.2e}")
    print(f"# rows with |delta| > 1e-5: {(diff > 1e-5).sum()} / {len(diff)}")

    # --- Latency summary ---
    print()
    print("=== Latency (single-row predict_proba) ===")
    rows = [
        summarise("LightGBM", lgbm_lat),
        summarise("ONNX-Runtime", onnx_lat),
    ]
    print_table(rows)

    speedup = median(lgbm_lat) / median(onnx_lat) if median(onnx_lat) > 0 else float("inf")
    gain_pct = (1 - median(onnx_lat) / median(lgbm_lat)) * 100
    print()
    print(f"Speed-up (p50): x{speedup:.2f}")
    print(f"Gain (p50): {gain_pct:.1f}%")

    if args.out is not None:
        report = {
            "config": {
                "n": args.n,
                "warmup": args.warmup,
                "seed": args.seed,
                "model_joblib": str(args.model),
                "model_onnx": str(args.onnx),
                "reference": str(args.reference),
            },
            "equivalence": {
                "max_abs_delta": float(diff.max()),
                "mean_abs_delta": float(diff.mean()),
                "rows_above_1e-5": int((diff > 1e-5).sum()),
                "n_compared": int(len(diff)),
            },
            "latency": rows,
            "speedup_p50": speedup,
            "gain_p50_pct": gain_pct,
        }
        args.out.parent.mkdir(parents=True, exist_ok=True)
        args.out.write_text(json.dumps(report, indent=2))
        print()
        print(f"Wrote report → {args.out}")


if __name__ == "__main__":
    main()