"""Compare AI-text detectors on three groups of academic text:
    AI       the AI-written subject samples (devtools/samples_domains.py, samples.py)
    HUMAN    abstracts of papers from 2008 (before language models; devtools/logs/human_abstracts.json)
    OURS     humanized paragraphs from a result file (--ours path.docx), optional
    python devtools/compare_detectors.py [--ours result.docx]
Per model: mean AI probability per group, share flagged (>= 0.5), and the separation of AI from HUMAN."""
import argparse
import json
import os
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import env  # noqa: E402
env.load()

import torch  # noqa: E402
from transformers import AutoModelForSequenceClassification, AutoTokenizer  # noqa: E402


def hf_scorer(model_id, ai_index=None):
    tok = AutoTokenizer.from_pretrained(model_id)
    m = AutoModelForSequenceClassification.from_pretrained(model_id).eval()
    labels = {int(k): v.lower() for k, v in m.config.id2label.items()}
    if ai_index is None:
        ai_index = next((i for i, l in labels.items() if any(w in l for w in ("ai", "fake", "machine", "gpt",
                                                                                "generated", "llm"))), 1)

    def score(texts):
        out = []
        for i in range(0, len(texts), 8):
            enc = tok(texts[i:i + 8], return_tensors="pt", truncation=True, max_length=512, padding=True)
            with torch.no_grad():
                lg = m(**enc).logits.float()
            out += (torch.softmax(lg, -1)[:, ai_index] if lg.shape[-1] > 1 else torch.sigmoid(lg[:, 0])).tolist()
        return out
    print(f"  {model_id}: labels {labels}, AI = {ai_index}", flush=True)
    return score


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--ours", default="")
    ap.add_argument("--models", default="current,radar,fakespot")
    ap.add_argument("--out", default="compare_detectors.json")
    a = ap.parse_args()
    from samples_domains import DOMAINS
    import samples
    groups = {"AI": list(DOMAINS.values()) + [samples.AI_TECH, samples.AI_SOCIAL],
              "HUMAN": json.load(open(ROOT / "devtools" / "logs" / "human_abstracts.json", encoding="utf-8"))
              + [samples.HUMAN]}
    if a.ours:
        from docx import Document
        groups["OURS"] = [p.text for p in Document(a.ours).paragraphs if len(p.text.split()) >= 40]
    scorers = {}
    want = a.models.split(",")
    if "current" in want:
        import detector as D
        scorers["current (live: coai + desklib)"] = D.score_texts
    if "radar" in want:
        scorers["RADAR (paraphrase-robust)"] = hf_scorer("TrustSafeAI/RADAR-Vicuna-7B", ai_index=0)
    if "superannotate" in want:
        scorers["SuperAnnotate"] = hf_scorer("SuperAnnotate/ai-detector")
    if "fakespot" in want:
        scorers["fakespot roberta"] = hf_scorer("fakespot-ai/roberta-base-ai-text-detection-v1")
    if "desklib" in want:
        import detector as D
        tok2, m2 = D.load_model2()
        scorers["desklib alone"] = lambda t: D._score_with2(tok2, m2, t)
    for spec in [x for x in want if x.startswith("hf:")]:  # any Hugging Face detector: hf:owner/name
        mid = spec[3:]
        try:
            if "desklib" in mid:
                import detector as D
                from transformers import AutoTokenizer
                cls = D._desklib_class()
                tok_, m_ = AutoTokenizer.from_pretrained(mid), cls.from_pretrained(mid).eval()
                scorers[mid] = (lambda t, tok_=tok_, m_=m_: D._score_with2(tok_, m_, t))
            else:
                scorers[mid] = hf_scorer(mid)
        except Exception as err:
            print(f"  {mid}: could not load ({str(err)[:120]})", flush=True)
    res = {}
    for name, fn in scorers.items():
        res[name] = {g: [float(x) for x in fn(ts)] for g, ts in groups.items()}
        r = res[name]
        line = " | ".join(f"{g}: mean {sum(v) / len(v):.2f}, flagged {sum(x >= 0.5 for x in v)}/{len(v)}"
                          for g, v in r.items())
        ai, hu = r["AI"], r["HUMAN"]
        auc = sum((x > y) + 0.5 * (x == y) for x in ai for y in hu) / (len(ai) * len(hu))
        print(f"{name:34s} AUC {auc:.3f} | {line}", flush=True)
    json.dump(res, open(ROOT / "devtools" / "logs" / a.out, "w"), indent=1)


if __name__ == "__main__":
    main()