Spaces:
Running
Running
unknown
Claude Opus 5.5
Stronger AI detector: two more paraphrase-robust models averaged into the score
dc37bdf | """Compare AI-text detectors on three groups of academic text: | |
| AI the AI-written subject samples (devtools/samples_domains.py, samples.py) | |
| HUMAN abstracts of papers from 2008 (before language models; devtools/logs/human_abstracts.json) | |
| OURS humanized paragraphs from a result file (--ours path.docx), optional | |
| python devtools/compare_detectors.py [--ours result.docx] | |
| Per model: mean AI probability per group, share flagged (>= 0.5), and the separation of AI from HUMAN.""" | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import env # noqa: E402 | |
| env.load() | |
| import torch # noqa: E402 | |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer # noqa: E402 | |
| def hf_scorer(model_id, ai_index=None): | |
| tok = AutoTokenizer.from_pretrained(model_id) | |
| m = AutoModelForSequenceClassification.from_pretrained(model_id).eval() | |
| labels = {int(k): v.lower() for k, v in m.config.id2label.items()} | |
| if ai_index is None: | |
| ai_index = next((i for i, l in labels.items() if any(w in l for w in ("ai", "fake", "machine", "gpt", | |
| "generated", "llm"))), 1) | |
| def score(texts): | |
| out = [] | |
| for i in range(0, len(texts), 8): | |
| enc = tok(texts[i:i + 8], return_tensors="pt", truncation=True, max_length=512, padding=True) | |
| with torch.no_grad(): | |
| lg = m(**enc).logits.float() | |
| out += (torch.softmax(lg, -1)[:, ai_index] if lg.shape[-1] > 1 else torch.sigmoid(lg[:, 0])).tolist() | |
| return out | |
| print(f" {model_id}: labels {labels}, AI = {ai_index}", flush=True) | |
| return score | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--ours", default="") | |
| ap.add_argument("--models", default="current,radar,fakespot") | |
| ap.add_argument("--out", default="compare_detectors.json") | |
| a = ap.parse_args() | |
| from samples_domains import DOMAINS | |
| import samples | |
| groups = {"AI": list(DOMAINS.values()) + [samples.AI_TECH, samples.AI_SOCIAL], | |
| "HUMAN": json.load(open(ROOT / "devtools" / "logs" / "human_abstracts.json", encoding="utf-8")) | |
| + [samples.HUMAN]} | |
| if a.ours: | |
| from docx import Document | |
| groups["OURS"] = [p.text for p in Document(a.ours).paragraphs if len(p.text.split()) >= 40] | |
| scorers = {} | |
| want = a.models.split(",") | |
| if "current" in want: | |
| import detector as D | |
| scorers["current (live: coai + desklib)"] = D.score_texts | |
| if "radar" in want: | |
| scorers["RADAR (paraphrase-robust)"] = hf_scorer("TrustSafeAI/RADAR-Vicuna-7B", ai_index=0) | |
| if "superannotate" in want: | |
| scorers["SuperAnnotate"] = hf_scorer("SuperAnnotate/ai-detector") | |
| if "fakespot" in want: | |
| scorers["fakespot roberta"] = hf_scorer("fakespot-ai/roberta-base-ai-text-detection-v1") | |
| if "desklib" in want: | |
| import detector as D | |
| tok2, m2 = D.load_model2() | |
| scorers["desklib alone"] = lambda t: D._score_with2(tok2, m2, t) | |
| for spec in [x for x in want if x.startswith("hf:")]: # any Hugging Face detector: hf:owner/name | |
| mid = spec[3:] | |
| try: | |
| if "desklib" in mid: | |
| import detector as D | |
| from transformers import AutoTokenizer | |
| cls = D._desklib_class() | |
| tok_, m_ = AutoTokenizer.from_pretrained(mid), cls.from_pretrained(mid).eval() | |
| scorers[mid] = (lambda t, tok_=tok_, m_=m_: D._score_with2(tok_, m_, t)) | |
| else: | |
| scorers[mid] = hf_scorer(mid) | |
| except Exception as err: | |
| print(f" {mid}: could not load ({str(err)[:120]})", flush=True) | |
| res = {} | |
| for name, fn in scorers.items(): | |
| res[name] = {g: [float(x) for x in fn(ts)] for g, ts in groups.items()} | |
| r = res[name] | |
| line = " | ".join(f"{g}: mean {sum(v) / len(v):.2f}, flagged {sum(x >= 0.5 for x in v)}/{len(v)}" | |
| for g, v in r.items()) | |
| ai, hu = r["AI"], r["HUMAN"] | |
| auc = sum((x > y) + 0.5 * (x == y) for x in ai for y in hu) / (len(ai) * len(hu)) | |
| print(f"{name:34s} AUC {auc:.3f} | {line}", flush=True) | |
| json.dump(res, open(ROOT / "devtools" / "logs" / a.out, "w"), indent=1) | |
| if __name__ == "__main__": | |
| main() | |