Spaces:
Running
Running
File size: 4,474 Bytes
4a7a056 dc37bdf 4a7a056 eebf465 4a7a056 dc37bdf 4a7a056 dc37bdf 4a7a056 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """Compare AI-text detectors on three groups of academic text:
AI the AI-written subject samples (devtools/samples_domains.py, samples.py)
HUMAN abstracts of papers from 2008 (before language models; devtools/logs/human_abstracts.json)
OURS humanized paragraphs from a result file (--ours path.docx), optional
python devtools/compare_detectors.py [--ours result.docx]
Per model: mean AI probability per group, share flagged (>= 0.5), and the separation of AI from HUMAN."""
import argparse
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import env # noqa: E402
env.load()
import torch # noqa: E402
from transformers import AutoModelForSequenceClassification, AutoTokenizer # noqa: E402
def hf_scorer(model_id, ai_index=None):
tok = AutoTokenizer.from_pretrained(model_id)
m = AutoModelForSequenceClassification.from_pretrained(model_id).eval()
labels = {int(k): v.lower() for k, v in m.config.id2label.items()}
if ai_index is None:
ai_index = next((i for i, l in labels.items() if any(w in l for w in ("ai", "fake", "machine", "gpt",
"generated", "llm"))), 1)
def score(texts):
out = []
for i in range(0, len(texts), 8):
enc = tok(texts[i:i + 8], return_tensors="pt", truncation=True, max_length=512, padding=True)
with torch.no_grad():
lg = m(**enc).logits.float()
out += (torch.softmax(lg, -1)[:, ai_index] if lg.shape[-1] > 1 else torch.sigmoid(lg[:, 0])).tolist()
return out
print(f" {model_id}: labels {labels}, AI = {ai_index}", flush=True)
return score
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--ours", default="")
ap.add_argument("--models", default="current,radar,fakespot")
ap.add_argument("--out", default="compare_detectors.json")
a = ap.parse_args()
from samples_domains import DOMAINS
import samples
groups = {"AI": list(DOMAINS.values()) + [samples.AI_TECH, samples.AI_SOCIAL],
"HUMAN": json.load(open(ROOT / "devtools" / "logs" / "human_abstracts.json", encoding="utf-8"))
+ [samples.HUMAN]}
if a.ours:
from docx import Document
groups["OURS"] = [p.text for p in Document(a.ours).paragraphs if len(p.text.split()) >= 40]
scorers = {}
want = a.models.split(",")
if "current" in want:
import detector as D
scorers["current (live: coai + desklib)"] = D.score_texts
if "radar" in want:
scorers["RADAR (paraphrase-robust)"] = hf_scorer("TrustSafeAI/RADAR-Vicuna-7B", ai_index=0)
if "superannotate" in want:
scorers["SuperAnnotate"] = hf_scorer("SuperAnnotate/ai-detector")
if "fakespot" in want:
scorers["fakespot roberta"] = hf_scorer("fakespot-ai/roberta-base-ai-text-detection-v1")
if "desklib" in want:
import detector as D
tok2, m2 = D.load_model2()
scorers["desklib alone"] = lambda t: D._score_with2(tok2, m2, t)
for spec in [x for x in want if x.startswith("hf:")]: # any Hugging Face detector: hf:owner/name
mid = spec[3:]
try:
if "desklib" in mid:
import detector as D
from transformers import AutoTokenizer
cls = D._desklib_class()
tok_, m_ = AutoTokenizer.from_pretrained(mid), cls.from_pretrained(mid).eval()
scorers[mid] = (lambda t, tok_=tok_, m_=m_: D._score_with2(tok_, m_, t))
else:
scorers[mid] = hf_scorer(mid)
except Exception as err:
print(f" {mid}: could not load ({str(err)[:120]})", flush=True)
res = {}
for name, fn in scorers.items():
res[name] = {g: [float(x) for x in fn(ts)] for g, ts in groups.items()}
r = res[name]
line = " | ".join(f"{g}: mean {sum(v) / len(v):.2f}, flagged {sum(x >= 0.5 for x in v)}/{len(v)}"
for g, v in r.items())
ai, hu = r["AI"], r["HUMAN"]
auc = sum((x > y) + 0.5 * (x == y) for x in ai for y in hu) / (len(ai) * len(hu))
print(f"{name:34s} AUC {auc:.3f} | {line}", flush=True)
json.dump(res, open(ROOT / "devtools" / "logs" / a.out, "w"), indent=1)
if __name__ == "__main__":
main()
|