devtools/bench_generators.py

"""Which generator writes versions the detector reads as human?  30 AI-written sentences (2 per subject), 3 versions
from each generator with the fast engine's prompt (keep every detail), every version scored by the live detector
on its own. Per generator: mean of each sentence's best (lowest) score, share of sentences with a version below 0.5,
seconds.
    python devtools/bench_generators.py [--only groq:openai/gpt-oss-120b,parrot]"""
import argparse
import json
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import env  # noqa: E402
env.load()
os.environ.setdefault("LT_MODE", "off")
sys.stdout.reconfigure(encoding="utf-8")

import requests  # noqa: E402

import detector as D  # noqa: E402
import groqai as G  # noqa: E402
from samples_domains import DOMAINS  # noqa: E402

ENDPOINTS = {
    "groq": ("https://api.groq.com/openai/v1/chat/completions", os.getenv("GROQ_API_KEY", "")),
    "gemini": ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", os.getenv("GEMINI_API_KEY", "")),
    "anthropic": ("https://api.anthropic.com/v1/chat/completions", os.getenv("ANTHROPIC_API_KEY", "")),
}
GENERATORS = ["groq:openai/gpt-oss-120b", "groq:openai/gpt-oss-20b", "groq:qwen/qwen3.6-27b", "groq:qwen/qwen3.8-27b",
              "gemini:gemini-3.5-flash", "gemini:gemini-3.5-flash-lite", "anthropic:claude-haiku-5-5", "anthropic:claude-sonnet-5-5", "parrot"]


def sentences():
    out = []
    for v in DOMAINS.values():
        out += [s for s in D.split_sentences(v) if len(s.split()) >= 8][:2]
    return out[:30]


LEAN = """Each numbered item is one sentence from an academic paper that reads as machine-written. Rewrite it {n}
different ways so a careful human researcher could have typed it.
KEEP EXACTLY: every fact, number, unit, statistic, citation (same form), technical term, comparison, condition,
qualifier and the strength of each claim. One sentence in, one sentence out. Correct grammar. Same spelling (UK/US).
HOW HUMANS WRITE IT: plain, concrete words a scientist would actually use; start from a different element (the
result, the method, a condition, the object); active voice where it is natural; an occasional less predictable but
correct word or phrase; no filler openers (Furthermore, Moreover, Additionally, Overall, Notably, It is important to
note); never use delve, crucial, pivotal, comprehensive, robust, leverage, foster, underscore, highlight, insights,
valuable, landscape, realm, intricate, seamless, paramount, multifaceted, showcase, "plays a role", "sheds light";
no em dashes; do not shorten by dropping content.
Answer only with JSON: {"items": [{"id": 1, "versions": ["...", "..."]}, ...]} - one per id."""
PROMPT = os.getenv("BENCH_PROMPT", "full")


def llm(gen, sents, n=3):
    prov, model = gen.split(":", 1)
    url, key = ENDPOINTS[prov]
    tok = G.FAITHFUL.set(True)
    try:
        system = LEAN.replace("{n}", str(n)) if PROMPT == "lean" else G._variants_system(n)
    finally:
        G.FAITHFUL.reset(tok)

    def one(batch):
        user = "PROTECTED TERMS: (none)\n\n" + "\n".join(f"{i}. SENTENCE: {t}" for i, t in enumerate(batch, 1))
        body = {"model": model, "temperature": 0.7, "max_tokens": 6000,
                "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]}
        if "gpt-oss" in model:
            body["reasoning_effort"] = "low"
        if "qwen" in model:
            body["reasoning_effort"] = "none"
        if prov == "anthropic":
            body.pop("temperature")
        r = requests.post(url, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
                          json=body, timeout=120)
        if not r.ok:
            raise RuntimeError(f"{r.status_code} {r.text[:200]}")
        data = G._load_json(r.json()["choices"][0]["message"]["content"])
        res = [[] for _ in batch]
        for k, it in enumerate(data.get("items") or []):
            try:
                i = int(it.get("id", k + 1)) - 1
                res[i] = [v for v in it.get("versions") or [] if isinstance(v, str)][:n]
            except Exception:
                pass
        return res
    batches = [sents[i:i + 10] for i in range(0, len(sents), 10)]
    with ThreadPoolExecutor(len(batches)) as pool:
        return [v for b in pool.map(one, batches) for v in b]


def parrot(sents, n=3):
    import humanizer as H
    out = H._generate(sents, 0.9, "cpu", 8, m=H._get_cpu_model(), k=n)
    return [out[i * n:(i + 1) * n] for i in range(len(sents))]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--only", default="")
    a = ap.parse_args()
    gens = [g for g in GENERATORS if not a.only or g in a.only.split(",")]
    sents = sentences()
    base = D.score_texts(sents)
    print(f"{len(sents)} sentences, original mean score {sum(base) / len(base):.2f}", flush=True)
    results = {}
    for g in gens:
        t = time.time()
        try:
            vs = parrot(sents) if g == "parrot" else llm(g, sents)
        except Exception as err:
            print(f"{g:32s} FAILED: {str(err)[:160]}", flush=True)
            continue
        secs = time.time() - t
        flat = [(i, v) for i, vv in enumerate(vs) for v in vv if v and v.strip()]
        sc = D.score_texts([v for _, v in flat]) if flat else []
        best = {}
        for (i, v), s in zip(flat, sc):
            if i not in best or s < best[i][0]:
                best[i] = (s, v)
        got = [best[i][0] if i in best else 1.0 for i in range(len(sents))]
        results[g] = {"best": got, "examples": [best.get(i, (1.0, ""))[1] for i in range(len(sents))]}
        print(f"{g:32s} best-version mean {sum(got) / len(got):.2f} | below 0.5: {sum(x < 0.5 for x in got)}/{len(got)}"
              f" | all versions mean {sum(sc) / max(1, len(sc)):.2f} | {secs:.1f}s", flush=True)
    if len(results) > 1:                               # one pool of versions from all generators
        pool = [min(r["best"][i] for r in results.values()) for i in range(len(sents))]
        print(f"{'ALL TOGETHER':32s} best-version mean {sum(pool) / len(pool):.2f} | below 0.5: "
              f"{sum(x < 0.5 for x in pool)}/{len(pool)}", flush=True)
    json.dump({"sentences": sents, "results": results}, open(ROOT / "devtools" / "logs" / "bench_generators.json", "w",
                                                             encoding="utf-8"), indent=1, ensure_ascii=False)


if __name__ == "__main__":
    main()