devtools/bench_int8_detector.py

"""Detector on CPU: full precision vs int8 (dynamic quantisation). Same texts (AI samples, human abstracts, 40 short
sentences): seconds per model, largest score difference, and whether any verdict (>= 0.5) changes."""
import json
import sys
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import os
os.environ["SKIP_MODEL_LOAD"] = "1"
import env  # noqa: E402
env.load()
sys.stdout.reconfigure(encoding="utf-8")

import torch  # noqa: E402

import detector as D  # noqa: E402
import samples  # noqa: E402
from samples_domains import DOMAINS  # noqa: E402

texts = list(DOMAINS.values()) + json.load(open(ROOT / "devtools/logs/human_abstracts.json", encoding="utf-8"))
texts += [s for v in DOMAINS.values() for s in D.split_sentences(v)][:40]
texts.append(samples.HUMAN)


def q(m):
    return torch.quantization.quantize_dynamic(m, {torch.nn.Linear}, dtype=torch.qint8)


tok1, m1 = D.load_model()
tok2, m2 = D.load_model2()
tok3, m3 = D.load_model3()
models = [("coai roberta-base", lambda m, t: D._score_with(tok1, m, t), m1),
          ("desklib deberta-large", lambda m, t: D._score_with2(tok2, m, t), m2),
          ("fakespot roberta-base", lambda m, t: D._score_with(tok3, m, t), m3)]
full, fast = [], []
for name, fn, m in models:
    t = time.time(); a = fn(m, texts); ta = time.time() - t
    mq = q(m)
    t = time.time(); b = fn(mq, texts); tb = time.time() - t
    flips = sum((x >= 0.5) != (y >= 0.5) for x, y in zip(a, b))
    print(f"{name:24s} full {ta:5.1f}s  int8 {tb:5.1f}s  ({ta / tb:.1f}x) | max diff {max(abs(x - y) for x, y in zip(a, b)):.3f}"
          f" | verdicts changed {flips}/{len(texts)}", flush=True)
    full.append(a); fast.append(b)
w2, w3 = D.WEIGHT2, D.WEIGHT3
ens = lambda p: [(1 - w3) * ((1 - w2) * x + w2 * y) + w3 * z for x, y, z in zip(*p)]
ea, eb = ens(full), ens(fast)
print(f"{'ENSEMBLE':24s} max diff {max(abs(x - y) for x, y in zip(ea, eb)):.3f} | verdicts changed "
      f"{sum((x >= 0.5) != (y >= 0.5) for x, y in zip(ea, eb))}/{len(texts)}")