scripts/bench_overhead.py

"""Bench Python harness overhead vs native decode (read-only measurement).

Owns nothing but this file. Reads Engine/TUI/server shapes only; never
modifies other files, never rebuilds native. Prefers the fake/tiny path so
it runs in seconds:

* FakeNative + FakeTokenizer (tests/test_engine.py make_engine pattern):
  zero-cost "native" isolates pure-Python loop overhead, compared against the
  production reference of ~200 tok/s (~5000 us/token).
* Tiny real native model (test_native.tiny_model shape, hidden=24/2 layers)
  when cism._native + numpy are available: measures a real next_tokens-only
  baseline on the same box. Tiny != production model; the production ratio
  is still computed against the 5000 us/token reference.

Sections:
  1. Engine.generate(64) total vs native-only vs decode vs machinery,
     plus chunk_size=1 vs 16 amortisation.
  2. Tokenizer scaling: single decode(n) vs incremental full-redecode cost.
  3. TUI run() with an instant FakeEngine (tests/test_tui.py pattern).
  4. Server TestClient timing if fastapi is importable, else code-only.
  5. Verdict (a)/(b)/(c) with a >10% threshold for any C++ work.
"""

from __future__ import annotations

import io
import statistics
import sys
import threading
import time

REPO_ROOT = __file__.rsplit("scripts", 1)[0] if "scripts" in __file__ else "."
for _p in (REPO_ROOT + "/src", REPO_ROOT + "src", "G:/CISM/src", "G:\\CISM\\src"):
    if _p not in sys.path:
        try:
            sys.path.insert(0, _p)
        except Exception:
            pass

from cism.engine import Engine  # noqa: E402

try:
    from cism import tui  # noqa: E402
except Exception:  # pragma: no cover
    tui = None

REFERENCE_US_PER_TOKEN = 5000.0  # ~200 tok/s production native decode
NTOK = 64


# ---------------------------------------------------------------- fakes
class FakeTokenizer:
    bos_token_id = 0
    eos_token_id = None
    chat_template = "test template"

    def __init__(self, pieces=None, encoded=None, special=()):
        self.pieces = pieces or {}
        self.encoded = [1, 2] if encoded is None else encoded
        self.special = set(special)

    def encode(self, text, *, add_special_tokens):
        return list(self.encoded)

    def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces):
        assert skip_special_tokens is True
        assert clean_up_tokenization_spaces is False
        return b"".join(
            self.pieces.get(t, b"x") for t in tokens if t not in self.special
        ).decode("utf-8", errors="replace")

    def apply_chat_template(self, messages, *, tokenize, add_generation_prompt):
        return list(self.encoded)


class FakeSession:
    def __init__(self, output, options):
        self.output = list(output)
        self.options = options
        self.generated_tokens = 0
        self.finish_reason = "length" if options["max_new_tokens"] == 0 else ""
        self.cancelled = threading.Event()
        self.requests = []

    def next_tokens(self, count, spec_k=0):
        self.requests.append((count, spec_k))
        batch = []
        for _ in range(count):
            if self.finish_reason:
                break
            token = self.output[self.generated_tokens]
            self.generated_tokens += 1
            batch.append(token)
            if token in self.options["eos_token_ids"]:
                self.finish_reason = "stop"
            elif self.generated_tokens == self.options["max_new_tokens"]:
                self.finish_reason = "length"
        return batch

    def cancel(self):
        self.cancelled.set()
        if not self.finish_reason:
            self.finish_reason = "cancelled"


class FakeNative:
    info = {"precision": "fp32", "weight_bytes": 123, "threads": 1}

    def __init__(self, output):
        self.output = output
        self.sessions = []

    def create_session(self, prompt, **options):
        session = FakeSession(self.output, options)
        self.sessions.append(session)
        return session


def make_fake_engine(ntok=NTOK):
    output = tuple(3 + (i % 60) for i in range(ntok))  # ids in [3, 62]
    pieces = {tok: b"a" for tok in set(output)}
    native = FakeNative(output)
    tok = FakeTokenizer(pieces)
    engine = Engine(native, tok, {
        "model_type": "llama", "vocab_size": 64, "max_position_embeddings": 4096,
    }, "fake-model", revision="pinned")
    return engine, native


class TuiFakeEngine:
    """Instant engine mirroring tests/test_tui.py FakeEngine (3 tok/prompt)."""

    def __init__(self, n_prompts=0):
        self.model_id = "fake-model"
        self.precision = "int8"
        self.threads = 2
        self.weight_bytes = 4 * 1024 * 1024
        self.context_length = 256
        self.pieces = {3: "Hello", 4: " world", 5: "!"}
        self.order = [3, 4, 5]
        self.engine_time = 0.0

    @property
    def info(self):
        return {"model": self.model_id, "precision": self.precision,
                "threads": self.threads, "weight_bytes": self.weight_bytes,
                "context_length": self.context_length}

    def stream(self, prompt, *, max_new_tokens=128, temperature=0.0, top_p=1.0,
               top_k=0, seed=0, stop=None, cancel=None, spec_k=0, **kw):
        from cism.engine import GenerationChunk
        t0 = time.perf_counter()
        try:
            if cancel is not None and cancel.is_set():
                return
            n = min(max_new_tokens, len(self.order))
            for i in range(n):
                if cancel is not None and cancel.is_set():
                    return
                tok = self.order[i]
                yield GenerationChunk(self.pieces[tok], [tok],
                                      "length" if i == n - 1 else None, 2, i + 1, 0, 0)
        finally:
            self.engine_time += time.perf_counter() - t0

    def generate(self, prompt, **kw):
        from cism.engine import GenerationResult
        texts, ids, last = [], [], None
        for chunk in self.stream(prompt, **kw):
            texts.append(chunk.text)
            ids.extend(chunk.token_ids)
            last = chunk
        return GenerationResult("".join(texts), ids, last.finish_reason,
                                last.prompt_tokens, last.completion_tokens, 0, 0)

    def nll(self, prompt, window=128):
        import math
        return {"nll": 0.5, "perplexity": math.exp(0.5), "scored_tokens": 2}


# ---------------------------------------------------------------- helpers
def med(fn, repeats=21, warmup=3):
    for _ in range(warmup):
        fn()
    samples = [fn() for _ in range(repeats)]
    return statistics.median(samples), samples


def timed(fn):
    t0 = time.perf_counter()
    out = fn()
    return time.perf_counter() - t0, out


def fmt_us(s):
    return f"{s * 1e6:9.1f} us"


def build_tiny_engine():
    """Tiny real native engine (test_native.tiny_model shape) or raise Skip."""
    import numpy as np

    try:
        from cism import _native  # noqa: F401
    except Exception as exc:
        raise RuntimeError(f"no native extension ({exc})")
    rng = np.random.default_rng(312)
    hidden, inter, vocab = 24, 37, 43
    config = dict(model_type="llama", hidden_size=hidden,
                  intermediate_size=inter, num_hidden_layers=2,
                  num_attention_heads=4, num_key_value_heads=2, head_dim=8,
                  vocab_size=vocab, max_position_embeddings=80,
                  rms_norm_eps=1e-5, rope_theta=10000.0,
                  tie_word_embeddings=False, hidden_act="silu")

    def mat(shape):
        return rng.normal(0, 0.14, shape).astype(np.float32)

    def norm(w):
        return rng.uniform(0.75, 1.25, w).astype(np.float32)

    weights = {"model.embed_tokens.weight": mat((vocab, hidden)),
               "lm_head.weight": mat((vocab, hidden)),
               "model.norm.weight": norm(hidden)}
    for i in range(2):
        p = f"model.layers.{i}."
        weights[p + "input_layernorm.weight"] = norm(hidden)
        weights[p + "post_attention_layernorm.weight"] = norm(hidden)
        for name, shape in (("q_proj", (32, hidden)), ("k_proj", (16, hidden)),
                            ("v_proj", (16, hidden)), ("o_proj", (hidden, 32))):
            weights[p + f"self_attn.{name}.weight"] = mat(shape)
        for name, shape in (("gate_proj", (inter, hidden)),
                            ("up_proj", (inter, hidden)),
                            ("down_proj", (hidden, inter))):
            weights[p + f"mlp.{name}.weight"] = mat(shape)
    pieces = {i: bytes([97 + (i % 26)]) for i in range(vocab)}
    engine = Engine.from_weights(config, weights, FakeTokenizer(pieces),
                                 model_id="tiny-llama", revision="bench")
    return engine


# ---------------------------------------------------------------- 1: loop
def section1():
    print("== [1] Engine loop overhead vs native (fake path: pure-Python cost) ==")
    engine, _ = make_fake_engine(NTOK)

    def gen16():
        return engine.generate([1, 2], max_new_tokens=NTOK, chunk_size=16)

    t_med, _ = med(lambda: timed(gen16)[0], repeats=21, warmup=3)
    print(f"Engine.generate({NTOK} tok, chunk=16): median {t_med * 1e3:.3f} ms "
          f"total = {t_med / NTOK * 1e6:.1f} us/token (fake native ~= 0 cost)")

    # Instrumented breakdown: wrap decode (Python obj, wrappable) + wrap
    # sessions at create_session (works for FakeSession; real pybind11
    # sessions have no __dict__, so the real path uses a direct loop below).
    state = {"t_next": 0.0, "t_decode": 0.0, "n_next": 0, "n_decode": 0}
    orig_create = engine._native.create_session
    orig_decode = engine.tokenizer.decode

    def counting_create(prompt, **opts):
        sess = orig_create(prompt, **opts)
        orig_next = sess.next_tokens

        def timed_next(count, spec_k=0):
            t0 = time.perf_counter()
            try:
                return orig_next(count, spec_k=spec_k)
            finally:
                state["t_next"] += time.perf_counter() - t0
                state["n_next"] += 1

        sess.next_tokens = timed_next
        return sess

    def timed_decode(tokens, **kw):
        t0 = time.perf_counter()
        try:
            return orig_decode(tokens, **kw)
        finally:
            state["t_decode"] += time.perf_counter() - t0
            state["n_decode"] += 1

    engine._native.create_session = counting_create
    engine.tokenizer.decode = timed_decode
    # warmup then one measured generation (counters reset for the measure)
    engine.generate([1, 2], max_new_tokens=NTOK, chunk_size=16)
    state.update(t_next=0.0, t_decode=0.0, n_next=0, n_decode=0)
    t0 = time.perf_counter()
    engine.generate([1, 2], max_new_tokens=NTOK, chunk_size=16)
    t_total = time.perf_counter() - t0
    engine._native.create_session = orig_create
    engine.tokenizer.decode = orig_decode
    t_other = t_total - state["t_next"] - state["t_decode"]
    per = lambda s: s / NTOK * 1e6
    print(f"  breakdown of one {NTOK}-tok generate ({t_total * 1e3:.3f} ms):")
    print(f"    native next_tokens : {state['t_next'] * 1e6:9.1f} us total "
          f"({per(state['t_next']):7.1f} us/tok, {state['n_next']} calls, "
          f"{state['t_next'] / max(1, state['n_next']) * 1e6:.1f} us/call)")
    print(f"    tokenizer.decode  : {state['t_decode'] * 1e6:9.1f} us total "
          f"({per(state['t_decode']):7.1f} us/tok, {state['n_decode']} calls)")
    print(f"    chunk/yield/lock  : {t_other * 1e6:9.1f} us total "
          f"({per(t_other):7.1f} us/tok)  [GenerationChunk allocs, stop-check,"
          f" cancel-monitor thread start/join, gen lock]")
    print(f"  python overhead/token ~= {t_total / NTOK * 1e6:.1f} us vs "
          f"reference native {REFERENCE_US_PER_TOKEN:.0f} us "
          f"({t_total / NTOK * 1e6 / REFERENCE_US_PER_TOKEN * 100:.2f}%)")

    # chunk_size amortisation on the fake path
    for cs in (1, 16):
        e, _ = make_fake_engine(NTOK)
        t, _ = med(lambda: timed(
            lambda: e.generate([1, 2], max_new_tokens=NTOK, chunk_size=cs))[0],
            repeats=15, warmup=2)
        sess = e._native.sessions[-1]
        print(f"  chunk_size={cs:2d}: median {t * 1e3:7.3f} ms "
              f"({t / NTOK * 1e6:6.1f} us/tok), native calls={len(sess.requests)}")

    # cancel-monitor thread cost (per-generate constant, amortised per token)
    def spawn_join():
        ev = threading.Event()
        th = threading.Thread(target=ev.wait, kwargs={"timeout": 0.0})
        th.start()
        th.join()

    t_thr, _ = med(lambda: timed(spawn_join)[0], repeats=100, warmup=10)
    print(f"  thread spawn+join microbench: ~{t_thr * 1e6:.1f} us/generate "
          f"(~{t_thr / NTOK * 1e6:.2f} us/tok at {NTOK} tok) -> negligible")

    # GenerationChunk alloc microbench
    from cism.engine import GenerationChunk

    def allocs():
        for _ in range(1000):
            GenerationChunk("a", [3], None, 2, 1, 0, 0)

    t_al, _ = med(lambda: timed(allocs)[0], repeats=21, warmup=3)
    print(f"  GenerationChunk alloc: ~{t_al / 1000 * 1e9:.0f} ns/chunk")

    # tiny real-native baseline (same box, NOT the production model)
    tiny = None
    try:
        tiny = build_tiny_engine()
    except Exception as exc:
        print(f"  tiny real model: SKIPPED ({exc})")
    if tiny is not None:
        tn = 32  # 5 prompt + 32 < 80 context
        prompt = [2, 7, 1]
        t_eng, _ = med(lambda: timed(
            lambda: tiny.generate(prompt, max_new_tokens=tn, chunk_size=16))[0],
            repeats=5, warmup=1)

        def native_only():
            s = tiny._native.create_session(prompt, max_new_tokens=tn,
                                            temperature=0.0, top_p=1.0,
                                            top_k=0, seed=0, eos_token_ids=[])
            n = 0
            while not s.finish_reason:
                n += len(s.next_tokens(16))
            return n

        t_nat, _ = med(lambda: timed(native_only)[0], repeats=5, warmup=1)
        # decode-only share inside Engine.generate on the tiny path
        st = {"d": 0.0}
        od = tiny.tokenizer.decode

        def td(tokens, **kw):
            t0 = time.perf_counter()
            try:
                return od(tokens, **kw)
            finally:
                st["d"] += time.perf_counter() - t0

        tiny.tokenizer.decode = td
        tiny.generate(prompt, max_new_tokens=tn, chunk_size=16)
        st["d"] = 0.0
        tiny.generate(prompt, max_new_tokens=tn, chunk_size=16)
        tiny.tokenizer.decode = od
        ov = (t_eng - t_nat) / tn * 1e6
        print(f"  tiny real model ({tn} tok): engine {t_eng * 1e3:.2f} ms "
              f"({t_eng / tn * 1e6:.0f} us/tok), native-only {t_nat * 1e3:.2f} ms "
              f"({t_nat / tn * 1e6:.0f} us/tok)")
        print(f"    tiny overhead ~= {ov:.1f} us/tok "
              f"(decode {st['d'] / tn * 1e6:.1f} us/tok); "
              f"vs PROD ref {REFERENCE_US_PER_TOKEN:.0f} us/tok = "
              f"{ov / REFERENCE_US_PER_TOKEN * 100:.2f}%")
    print()
    return {"fake_us_per_tok": t_total / NTOK * 1e6}


# ---------------------------------------------------------------- 2: decode
def section2():
    print("== [2] Tokenizer scaling: full-redecode per chunk (O(n^2/chunk)) ==")
    tok = FakeTokenizer({i: b"a" for i in range(512)})
    ids512 = list(range(512))
    for n in (64, 128, 256, 512):
        ids = ids512[:n]
        t, _ = med(lambda: timed(
            lambda: tok.decode(ids, skip_special_tokens=True,
                               clean_up_tokenization_spaces=False))[0],
            repeats=200, warmup=20)
        print(f"  single decode({n:3d}): {t * 1e6:8.1f} us")
    # incremental totals as Engine.stream does (decode whole prefix per chunk)
    for cs in (1, 16):
        lens = list(range(cs, 513, cs))
        if lens[-1] != 512:
            lens.append(512)

        def incr():
            for k in lens:
                tok.decode(ids512[:k], skip_special_tokens=True,
                           clean_up_tokenization_spaces=False)

        t, _ = med(lambda: timed(incr)[0], repeats=5, warmup=1)
        single512, _ = med(lambda: timed(
            lambda: tok.decode(ids512, skip_special_tokens=True,
                               clean_up_tokenization_spaces=False))[0],
            repeats=200, warmup=20)
        print(f"  incremental 512 tok, chunk={cs:2d}: {len(lens)} decodes, "
              f"total {t * 1e3:.2f} ms ({t / 512 * 1e6:.1f} us/tok) vs single "
              f"decode(512) {single512 * 1e6:.1f} us "
              f"(ratio {t / single512:.0f}x)")
    # what incremental-decode (delta only) would cost instead
    t_delta, _ = med(lambda: timed(
        lambda: tok.decode(ids512[:16], skip_special_tokens=True,
                           clean_up_tokenization_spaces=False))[0],
        repeats=200, warmup=20)
    est = t_delta * (512 / 16)
    print(f"  delta-only estimate (decode 16 new ids x32): ~{est * 1e3:.2f} ms "
          f"-> saves the quadratic re-scan")
    print()


# ---------------------------------------------------------------- 3: TUI
def section3():
    print("== [3] TUI path overhead (instant FakeEngine => harness only) ==")
    if tui is None:
        print("  SKIPPED: cism.tui not importable")
        print()
        return {"tui_us_per_tok": float("nan")}
    # direct engine baseline (no TUI)
    eng0 = TuiFakeEngine()
    t_dir, _ = med(lambda: timed(
        lambda: list(eng0.stream("hello", max_new_tokens=3)))[0],
        repeats=2000, warmup=200)
    print(f"  direct eng.stream(3 tok): {t_dir * 1e6:.1f} us "
          f"({t_dir / 3 * 1e6:.1f} us/tok) [engine-already-instant baseline]")
    # TUI loop over many prompts in one run()
    N_PROMPTS, TOK_PER = 50, 3
    lines = ["hello"] * N_PROMPTS + ["/quit"]

    def tui_run():
        eng = TuiFakeEngine()
        out, err = io.StringIO(), io.StringIO()
        t0 = time.perf_counter()
        code = tui.run("fake-model", engine=eng,
                       stdin=io.StringIO("\n".join(lines) + "\n"),
                       stdout=out, stderr=err)
        dt = time.perf_counter() - t0
        assert code == 0
        return dt, eng.engine_time

    _, _ = tui_run(), None  # warmup (imports/regex compile)
    dts, eng_ts = [], []
    for _ in range(5):
        dt, et = tui_run()
        dts.append(dt)
        eng_ts.append(et)
    dtm, etm = statistics.median(dts), statistics.median(eng_ts)
    total_tok = N_PROMPTS * TOK_PER
    harness = dtm - etm
    print(f"  tui.run({N_PROMPTS} prompts x {TOK_PER} tok): median {dtm * 1e3:.2f} ms "
          f"total; engine-inside {etm * 1e3:.2f} ms")
    print(f"  TUI harness overhead: {harness * 1e6:.0f} us / {total_tok} tok = "
          f"{harness / total_tok * 1e6:.1f} us/tok "
          f"({harness / total_tok * 1e6 / REFERENCE_US_PER_TOKEN * 100:.3f}% of "
          f"{REFERENCE_US_PER_TOKEN:.0f} us ref token)")
    print("  (tui.py run(): direct eng.stream() call per prompt, per-chunk "
          "out.write+flush, counters only; no HTTP/JSON.)")
    print()
    return {"tui_us_per_tok": harness / total_tok * 1e6}


# ---------------------------------------------------------------- 4: server
def section4(tui_us):
    print("== [4] Server path overhead (code + optional live TestClient) ==")
    print("  code path (src/cism/server.py, read-only): _Worker.submit puts a")
    print("  _Job on queue.Queue(maxsize=8); inference thread runs the SAME")
    print("  engine.stream(chunk_size=16) as the TUI; each chunk crosses the")
    print("  queue + asyncio.Event notify (worker._put/_run/receive), then one")
    print("  json.dumps per SSE frame (or one for non-stream) + pydantic")
    print("  _CompletionRequest validation. Expect ~10s of us extra vs TUI.")
    result = {"server_extra_us_per_tok": float("nan"), "live": False}
    try:
        from fastapi.testclient import TestClient
        from cism.engine import GenerationChunk as Chunk
        from cism.server import create_app
    except Exception as exc:
        print(f"  live TestClient: SKIPPED ({exc})")
        print()
        return result

    class SrvFake:
        model_id = "bench-model"
        context_length = 512

        def __init__(self, ntok=8):
            self.ntok = ntok

        def encode(self, prompt):
            return list(prompt) if isinstance(prompt, list) else [1, 2]

        def stream(self, prompt, **kw):
            n = min(kw.get("max_new_tokens", 8), self.ntok)
            for i in range(n):
                yield Chunk("a", [3 + (i % 40)], "length" if i == n - 1 else None,
                            2, i + 1, 0, 0)

    NTOK_S, REPS = 8, 10
    try:
        import os
        os.environ.pop("CISM_API_KEY", None)
        app = create_app(SrvFake(NTOK_S), api_key="")
        with TestClient(app) as client:
            assert client.get("/health").json() == {"status": "ok"}
            body = {"model": "bench-model", "prompt": [1, 2],
                    "max_tokens": NTOK_S}
            client.post("/v1/completions", json=body)  # warmup
            t0 = time.perf_counter()
            for _ in range(REPS):
                r = client.post("/v1/completions", json=body)
                assert r.status_code == 200, r.text
            t_srv = (time.perf_counter() - t0) / REPS
            # streaming sample (one request)
            t0 = time.perf_counter()
            r = client.post("/v1/completions", json={**body, "stream": True})
            assert r.status_code == 200, r.text
            t_stream = time.perf_counter() - t0
        eng = SrvFake(NTOK_S)
        t0 = time.perf_counter()
        for _ in range(200):
            list(eng.stream([1, 2], max_new_tokens=NTOK_S))
        t_dir = (time.perf_counter() - t0) / 200
        extra = (t_srv - t_dir) / NTOK_S * 1e6
        result = {"server_extra_us_per_tok": extra, "live": True}
        print(f"  live TestClient non-stream: {t_srv * 1e3:.2f} ms/req "
              f"({t_srv / NTOK_S * 1e6:.0f} us/tok); direct stream: "
              f"{t_dir * 1e3:.3f} ms ({t_dir / NTOK_S * 1e6:.1f} us/tok)")
        print(f"  server extra vs direct: ~{extra:.0f} us/tok "
              f"({extra / REFERENCE_US_PER_TOKEN * 100:.2f}% of ref token); "
              f"stream req sample {t_stream * 1e3:.1f} ms")
        print("  (in-process TestClient understates real TCP+uvicorn by "
              "~0.2-1 ms/req; per-token delta stays << native token time.)")
    except Exception as exc:
        print(f"  live TestClient: SKIPPED ({type(exc).__name__}: {exc})")
    print()
    return result


# ---------------------------------------------------------------- 5: verdict
def section5(s1, tui_us, srv):
    print("== [5] Verdict: is a C++ inference API worth it? ==")
    overhead = s1["fake_us_per_tok"]
    ratio = overhead / REFERENCE_US_PER_TOKEN
    tui_r = (tui_us / REFERENCE_US_PER_TOKEN) if tui_us == tui_us else float("nan")
    print(f"  Python loop overhead : {overhead:7.1f} us/tok  "
          f"({ratio * 100:5.2f}% of {REFERENCE_US_PER_TOKEN:.0f} us ref token)")
    print(f"  TUI harness overhead : {tui_us:7.1f} us/tok  "
          f"({tui_r * 100:5.3f}% of ref token)")
    if srv["live"]:
        e = srv["server_extra_us_per_tok"]
        print(f"  Server extra (live)  : {e:7.1f} us/tok  "
              f"({e / REFERENCE_US_PER_TOKEN * 100:5.2f}% of ref token)")
    else:
        print("  Server extra (live)  : skipped (code-only: queue+JSON ~10s us)")
    print("  Threshold: C++ work only if harness overhead >10% of token time.")
    if ratio < 0.05 and (tui_r != tui_r or tui_r < 0.05):
        choice = ("(a) NOT worth it -- keep the Python Engine.stream boundary. "
                  "Overhead is <5% of a batch-1 decode step.")
    elif ratio >= 0.10 or (tui_r == tui_r and tui_r >= 0.10):
        choice = ("(b)/(c) threshold tripped -- re-measure on the production "
                  "model before committing to C++.")
    else:
        choice = ("(b) at most a native generate_text helper (incremental "
                  "decode + stop-check inside one call) -- not a full C++ API.")
    # With the measured sub-5% numbers the evidence points to (a); the
    # conditional above encodes the rule, this line records the outcome.
    print(f"  RECOMMENDATION: {choice}")
    if ratio < 0.05:
        print("  Chosen: (a). A full C++ inference API / C++ CLI buys <5% at")
        print("  batch-1 decode and adds ABI/build/debug cost for zero user-")
        print("  visible tok/s. The TUI already has no high harness overhead")
        print("  (direct eng.stream calls, ~10s us/tok).")
    print("  What WOULD help most instead (in order):")
    print("    1. Incremental decode: decode only the new delta per chunk")
    print("       (carry pending UTF-8 bytes) instead of decode(all ids) --")
    print("       kills the O(n^2/chunk) re-scan from [2].")
    print("    2. Keep chunk_size=16 whenever no stop strings are set (stops")
    print("       force token-at-a-time in engine.py); amortises one pybind11")
    print("       next_tokens call over 16 tokens -- see [1] cs=1 vs 16.")
    print("    3. Keep the TUI on direct Engine calls (no HTTP/JSON hop) and")
    print("       batch terminal writes; the server path stays opt-in.")
    print()
    return choice


def main():
    print("bench_overhead.py -- Python harness overhead vs native decode")
    print(f"reference native: ~200 tok/s = {REFERENCE_US_PER_TOKEN:.0f} us/token; "
          f"NTOK={NTOK}")
    try:
        import cism._native as n  # noqa: F401
        print("native extension: present (tiny-model section enabled if numpy ok)")
    except Exception as exc:
        print(f"native extension: MISSING ({exc}); fake-path only")
    print()
    s1 = section1()
    section2()
    s3 = section3()
    srv = section4(s3["tui_us_per_tok"])
    choice = section5(s1, s3["tui_us_per_tok"], srv)
    print(f"VERDICT: {choice}")


if __name__ == "__main__":
    main()