| |
| """Generate creative-writing samples against an OpenAI-compatible endpoint. |
| |
| Prompts: EQ-Bench creative-writing v3 set (prompts.json, their file, cited |
| in README). Generations at temperature 0.7/min_p 0.1 per EQ convention — |
| voice needs room; determinism would strangle it. Budget 6000 tokens per |
| piece: generous on purpose, every char is data. |
| |
| Saves one JSONL line per piece: prompt metadata, full response, and the |
| reasoning trace. Server model ids (local file paths) are replaced with our |
| run label before saving — no machine info ever lands on disk. |
| |
| Usage: |
| BENCH_LABEL=file-a python3 generate.py --prompts prompts.json \\ |
| --out runs/file-a.jsonl [--ids 1,2,3] [--budget 6000] [--temp 0.7] |
| Env: |
| BENCH_LABEL run label recorded on every line (default: file) |
| CREATIVE_UPSTREAM chat-completions base URL |
| (default: http://host.docker.internal:8080) |
| """ |
| import argparse |
| import json |
| import os |
| import time |
| import urllib.request |
|
|
| LABEL = os.environ.get("BENCH_LABEL", "file") |
| UPSTREAM = os.environ.get("CREATIVE_UPSTREAM", "http://host.docker.internal:8080") |
|
|
|
|
| def complete(messages, budget, temp, seed): |
| body = json.dumps({ |
| "model": LABEL, "messages": messages, "max_tokens": budget, |
| "temperature": temp, "min_p": 0.1, "seed": seed, |
| }).encode() |
| req = urllib.request.Request(UPSTREAM + "/v1/chat/completions", data=body, |
| method="POST", |
| headers={"Content-Type": "application/json"}) |
| with urllib.request.urlopen(req, timeout=1200) as r: |
| return json.loads(r.read()) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--prompts", required=True) |
| ap.add_argument("--out", required=True) |
| ap.add_argument("--ids", default="") |
| ap.add_argument("--budget", type=int, default=6000) |
| ap.add_argument("--temp", type=float, default=0.7) |
| ap.add_argument("--seed", type=int, default=1234) |
| a = ap.parse_args() |
|
|
| prompts = json.load(open(a.prompts)) |
| want = [w.strip() for w in a.ids.split(",") if w.strip()] or sorted( |
| prompts, key=int) |
| with open(a.out, "w") as out: |
| for pid in want: |
| p = prompts[pid] |
| |
| msgs = [{"role": "user", "content": p["writing_prompt"]}] |
| t0 = time.time() |
| resp = complete(msgs, a.budget, a.temp, a.seed) |
| m = resp["choices"][0]["message"] |
| out.write(json.dumps({ |
| "id": pid, "label": LABEL, |
| "category": p.get("category"), "title": p.get("title"), |
| "prompt": p["writing_prompt"], |
| "text": m.get("content") or "", |
| "thinking": m.get("reasoning_content") or "", |
| "finish": resp["choices"][0].get("finish_reason"), |
| "temp": a.temp, "seed": a.seed, "budget": a.budget, |
| "elapsed": round(time.time() - t0, 1), |
| }) + "\n") |
| out.flush() |
| print(f" {pid} done", flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
