Spaces:
Running
Running
unknown
Claude Opus 5.5
Fast engine: versions from several models at once (Gemini 3.5 Flash-Lite + Qwen 3.8 by default, gpt-oss-20b as fallback), checked inside the worker threads; generator benchmark
01f5654 | """Which generator writes versions the detector reads as human? 30 AI-written sentences (2 per subject), 3 versions | |
| from each generator with the fast engine's prompt (keep every detail), every version scored by the live detector | |
| on its own. Per generator: mean of each sentence's best (lowest) score, share of sentences with a version below 0.5, | |
| seconds. | |
| python devtools/bench_generators.py [--only groq:openai/gpt-oss-120b,parrot]""" | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import env # noqa: E402 | |
| env.load() | |
| os.environ.setdefault("LT_MODE", "off") | |
| sys.stdout.reconfigure(encoding="utf-8") | |
| import requests # noqa: E402 | |
| import detector as D # noqa: E402 | |
| import groqai as G # noqa: E402 | |
| from samples_domains import DOMAINS # noqa: E402 | |
| ENDPOINTS = { | |
| "groq": ("https://api.groq.com/openai/v1/chat/completions", os.getenv("GROQ_API_KEY", "")), | |
| "gemini": ("https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", os.getenv("GEMINI_API_KEY", "")), | |
| "anthropic": ("https://api.anthropic.com/v1/chat/completions", os.getenv("ANTHROPIC_API_KEY", "")), | |
| } | |
| GENERATORS = ["groq:openai/gpt-oss-120b", "groq:openai/gpt-oss-20b", "groq:qwen/qwen3.6-27b", "groq:qwen/qwen3.8-27b", | |
| "gemini:gemini-3.5-flash", "gemini:gemini-3.5-flash-lite", "anthropic:claude-haiku-5-5", "anthropic:claude-sonnet-5-5", "parrot"] | |
| def sentences(): | |
| out = [] | |
| for v in DOMAINS.values(): | |
| out += [s for s in D.split_sentences(v) if len(s.split()) >= 8][:2] | |
| return out[:30] | |
| LEAN = """Each numbered item is one sentence from an academic paper that reads as machine-written. Rewrite it {n} | |
| different ways so a careful human researcher could have typed it. | |
| KEEP EXACTLY: every fact, number, unit, statistic, citation (same form), technical term, comparison, condition, | |
| qualifier and the strength of each claim. One sentence in, one sentence out. Correct grammar. Same spelling (UK/US). | |
| HOW HUMANS WRITE IT: plain, concrete words a scientist would actually use; start from a different element (the | |
| result, the method, a condition, the object); active voice where it is natural; an occasional less predictable but | |
| correct word or phrase; no filler openers (Furthermore, Moreover, Additionally, Overall, Notably, It is important to | |
| note); never use delve, crucial, pivotal, comprehensive, robust, leverage, foster, underscore, highlight, insights, | |
| valuable, landscape, realm, intricate, seamless, paramount, multifaceted, showcase, "plays a role", "sheds light"; | |
| no em dashes; do not shorten by dropping content. | |
| Answer only with JSON: {"items": [{"id": 1, "versions": ["...", "..."]}, ...]} - one per id.""" | |
| PROMPT = os.getenv("BENCH_PROMPT", "full") | |
| def llm(gen, sents, n=3): | |
| prov, model = gen.split(":", 1) | |
| url, key = ENDPOINTS[prov] | |
| tok = G.FAITHFUL.set(True) | |
| try: | |
| system = LEAN.replace("{n}", str(n)) if PROMPT == "lean" else G._variants_system(n) | |
| finally: | |
| G.FAITHFUL.reset(tok) | |
| def one(batch): | |
| user = "PROTECTED TERMS: (none)\n\n" + "\n".join(f"{i}. SENTENCE: {t}" for i, t in enumerate(batch, 1)) | |
| body = {"model": model, "temperature": 0.7, "max_tokens": 6000, | |
| "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]} | |
| if "gpt-oss" in model: | |
| body["reasoning_effort"] = "low" | |
| if "qwen" in model: | |
| body["reasoning_effort"] = "none" | |
| if prov == "anthropic": | |
| body.pop("temperature") | |
| r = requests.post(url, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, | |
| json=body, timeout=120) | |
| if not r.ok: | |
| raise RuntimeError(f"{r.status_code} {r.text[:200]}") | |
| data = G._load_json(r.json()["choices"][0]["message"]["content"]) | |
| res = [[] for _ in batch] | |
| for k, it in enumerate(data.get("items") or []): | |
| try: | |
| i = int(it.get("id", k + 1)) - 1 | |
| res[i] = [v for v in it.get("versions") or [] if isinstance(v, str)][:n] | |
| except Exception: | |
| pass | |
| return res | |
| batches = [sents[i:i + 10] for i in range(0, len(sents), 10)] | |
| with ThreadPoolExecutor(len(batches)) as pool: | |
| return [v for b in pool.map(one, batches) for v in b] | |
| def parrot(sents, n=3): | |
| import humanizer as H | |
| out = H._generate(sents, 0.9, "cpu", 8, m=H._get_cpu_model(), k=n) | |
| return [out[i * n:(i + 1) * n] for i in range(len(sents))] | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--only", default="") | |
| a = ap.parse_args() | |
| gens = [g for g in GENERATORS if not a.only or g in a.only.split(",")] | |
| sents = sentences() | |
| base = D.score_texts(sents) | |
| print(f"{len(sents)} sentences, original mean score {sum(base) / len(base):.2f}", flush=True) | |
| results = {} | |
| for g in gens: | |
| t = time.time() | |
| try: | |
| vs = parrot(sents) if g == "parrot" else llm(g, sents) | |
| except Exception as err: | |
| print(f"{g:32s} FAILED: {str(err)[:160]}", flush=True) | |
| continue | |
| secs = time.time() - t | |
| flat = [(i, v) for i, vv in enumerate(vs) for v in vv if v and v.strip()] | |
| sc = D.score_texts([v for _, v in flat]) if flat else [] | |
| best = {} | |
| for (i, v), s in zip(flat, sc): | |
| if i not in best or s < best[i][0]: | |
| best[i] = (s, v) | |
| got = [best[i][0] if i in best else 1.0 for i in range(len(sents))] | |
| results[g] = {"best": got, "examples": [best.get(i, (1.0, ""))[1] for i in range(len(sents))]} | |
| print(f"{g:32s} best-version mean {sum(got) / len(got):.2f} | below 0.5: {sum(x < 0.5 for x in got)}/{len(got)}" | |
| f" | all versions mean {sum(sc) / max(1, len(sc)):.2f} | {secs:.1f}s", flush=True) | |
| if len(results) > 1: # one pool of versions from all generators | |
| pool = [min(r["best"][i] for r in results.values()) for i in range(len(sents))] | |
| print(f"{'ALL TOGETHER':32s} best-version mean {sum(pool) / len(pool):.2f} | below 0.5: " | |
| f"{sum(x < 0.5 for x in pool)}/{len(pool)}", flush=True) | |
| json.dump({"sentences": sents, "results": results}, open(ROOT / "devtools" / "logs" / "bench_generators.json", "w", | |
| encoding="utf-8"), indent=1, ensure_ascii=False) | |
| if __name__ == "__main__": | |
| main() | |