Spaces:
Running
Running
unknown
Claude Opus 5.5
Fast engine on CPU about a third faster: versions pre-ranked by the two small detector models
bc66a36 | """Detector on CPU: full precision vs int8 (dynamic quantisation). Same texts (AI samples, human abstracts, 40 short | |
| sentences): seconds per model, largest score difference, and whether any verdict (>= 0.5) changes.""" | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import os | |
| os.environ["SKIP_MODEL_LOAD"] = "1" | |
| import env # noqa: E402 | |
| env.load() | |
| sys.stdout.reconfigure(encoding="utf-8") | |
| import torch # noqa: E402 | |
| import detector as D # noqa: E402 | |
| import samples # noqa: E402 | |
| from samples_domains import DOMAINS # noqa: E402 | |
| texts = list(DOMAINS.values()) + json.load(open(ROOT / "devtools/logs/human_abstracts.json", encoding="utf-8")) | |
| texts += [s for v in DOMAINS.values() for s in D.split_sentences(v)][:40] | |
| texts.append(samples.HUMAN) | |
| def q(m): | |
| return torch.quantization.quantize_dynamic(m, {torch.nn.Linear}, dtype=torch.qint8) | |
| tok1, m1 = D.load_model() | |
| tok2, m2 = D.load_model2() | |
| tok3, m3 = D.load_model3() | |
| models = [("coai roberta-base", lambda m, t: D._score_with(tok1, m, t), m1), | |
| ("desklib deberta-large", lambda m, t: D._score_with2(tok2, m, t), m2), | |
| ("fakespot roberta-base", lambda m, t: D._score_with(tok3, m, t), m3)] | |
| full, fast = [], [] | |
| for name, fn, m in models: | |
| t = time.time(); a = fn(m, texts); ta = time.time() - t | |
| mq = q(m) | |
| t = time.time(); b = fn(mq, texts); tb = time.time() - t | |
| flips = sum((x >= 0.5) != (y >= 0.5) for x, y in zip(a, b)) | |
| print(f"{name:24s} full {ta:5.1f}s int8 {tb:5.1f}s ({ta / tb:.1f}x) | max diff {max(abs(x - y) for x, y in zip(a, b)):.3f}" | |
| f" | verdicts changed {flips}/{len(texts)}", flush=True) | |
| full.append(a); fast.append(b) | |
| w2, w3 = D.WEIGHT2, D.WEIGHT3 | |
| ens = lambda p: [(1 - w3) * ((1 - w2) * x + w2 * y) + w3 * z for x, y, z in zip(*p)] | |
| ea, eb = ens(full), ens(fast) | |
| print(f"{'ENSEMBLE':24s} max diff {max(abs(x - y) for x, y in zip(ea, eb)):.3f} | verdicts changed " | |
| f"{sum((x >= 0.5) != (y >= 0.5) for x, y in zip(ea, eb))}/{len(texts)}") | |