Phase 6 tooling: the eight benchmarks exactly as docs/05-eval-plan.md froze them (lm-eval 0.4.13, hf backend, fp16, one card, task-default shots + uniform 5-shot column, --limit 5 smoke gate first, incremental push after every task)

eval/run_benchmarks.py

@@ -0,0 +1,233 @@
+"""Phase 6: the eight academic benchmarks, run exactly as docs/05-eval-plan.md froze them.
+
+Written before any score exists, and it refuses to deviate from the pin:
+
+* `lm-eval` **0.4.13**, backend `model=hf`, `dtype=float16`, one T4, no `--trust_remote_code`, no chat
+  template -- a base model with a chat template would be an unearned capability (§3.9).
+* PRIMARY column = each task's own default `num_fewshot`, which means **no `--num_fewshot` flag at all**.
+  The point of using the harness's file rather than a number is that nobody chose it per task, so it cannot
+  be tuned per task later either.
+* SECONDARY column = a uniform 5-shot run of every task, reported alongside, never instead.
+* One invocation per task, all eight in a single job, and each task's result file is pushed to the Hub the
+  moment it finishes so an interruption never loses a completed task (§5 Phase 6).
+* A `--limit 5` smoke pass over all eight runs first, because transformers 5.0.0 on the Kaggle image versus
+  a harness pinned in 2024 is an unverified combination (§5) and finding that out after 6 hours of GPU time
+  is not acceptable. Nothing is published if the smoke pass fails.
+
+No credentials in this file: `ounce100m_credentials.install()` fetches the token at run time (D-006), and
+the token never enters a log line or a published artifact.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import time
+
+PINNED_LM_EVAL = "0.4.13"
+# Every row here is the harness's own task id, not a name invented for this script.
+TASKS = ["arc_challenge", "arc_easy", "hellaswag", "mmlu", "piqa", "truthfulqa_mc1",
+         "truthfulqa_mc2", "winogrande", "gsm8k"]
+# Chance level of the *metric being reported*, from the number of answer options in the task, not from a
+# remembered leaderboard. Where a metric is not chance-normalised that is said rather than guessed at.
+CHANCE = {"arc_challenge": ("0.25", "4 options"), "arc_easy": ("0.25", "4 options"),
+          "hellaswag": ("0.25", "4 continuations"), "mmlu": ("0.25", "4 options, macro over 57 subjects"),
+          "piqa": ("0.50", "2 options"), "winogrande": ("0.50", "2 options"),
+          "truthfulqa_mc1": ("~0.20", "mean over questions of 1/#choices"),
+          "truthfulqa_mc2": ("n/a", "mc2 is not chance-normalised"),
+          "gsm8k": ("0.00", "free-form exact match")}
+
+
+def sh(argv, label, timeout, env=None):
+    """Streamed like every other long-running stage in this project -- a buffered 6-hour eval job would
+    report nothing at all if the platform took the instance."""
+    print("=== " + label, flush=True)
+    t0 = time.time()
+    e = dict(os.environ)
+    e["PYTHONUNBUFFERED"] = "1"
+    e.update(env or {})
+    import threading
+    p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
+                         env=e, bufsize=1, cwd=os.path.dirname(os.path.abspath(__file__)))
+    killed = []
+
+    def _kill():
+        killed.append(True)
+        try:
+            os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+        except Exception:
+            p.kill()
+
+    timer = threading.Timer(timeout, _kill)
+    timer.daemon = True
+    timer.start()
+    keep = []
+    for line in p.stdout:
+        line = line.rstrip("\n")
+        keep.append(line)
+        del keep[:-400]
+        print("   |", line[:260], flush=True)
+    timer.cancel()
+    rc = p.wait()
+    print("%s_RC %s%s seconds %.1f" % (label, rc, " TIMEOUT" if killed else "", time.time() - t0),
+          flush=True)
+    return rc, "\n".join(keep)
+
+
+def install_harness():
+    """Pin the harness in a uv inline-script environment rather than into the Kaggle image's site-packages:
+    transformers 5.0.0 must stay exactly as Gate 3 measured it, and `pip install` into the image would
+    resolve a different one."""
+    rc, out = sh([sys.executable, "-c", "import lm_eval; print(lm_eval.__version__)"], "probe_lm_eval", 120)
+    have = ""
+    for line in out.splitlines():
+        if re.match(r"^\s*\|?\s*[0-9]+\.[0-9]+", line):
+            have = line.strip().strip("|").strip()
+    if have == PINNED_LM_EVAL:
+        print("lm_eval", have, "already present", flush=True)
+        return have
+    print("lm_eval %r is not the pin %s -- installing it as a user package" % (have, PINNED_LM_EVAL),
+          flush=True)
+    rc, _ = sh([sys.executable, "-m", "pip", "install", "--user", "--quiet",
+                "lm-eval[mathtasks]==%s" % PINNED_LM_EVAL], "pip_lm_eval", 1800)
+    if rc != 0:
+        rc, _ = sh([sys.executable, "-m", "pip", "install", "--user", "--quiet",
+                    "lm-eval==%s" % PINNED_LM_EVAL], "pip_lm_eval_plain", 1800)
+        if rc != 0:
+            raise SystemExit("could not install the pinned harness; refusing to eval on a different one")
+    rc, out = sh([sys.executable, "-c", "import lm_eval, os; "
+                                        "print(lm_eval.__version__, os.path.dirname(lm_eval.__file__))"],
+                 "verify_lm_eval", 300)
+    got = [l.split()[0] for l in out.splitlines() if l.startswith(PINNED_LM_EVAL)]
+    if not got:
+        raise SystemExit("lm_eval is not importable at the pinned version after install: "
+                         + out[-400:])
+    return PINNED_LM_EVAL
+
+
+def run_task(model, task, shots, outdir, limit, gpu_h):
+    """One harness invocation. `shots=None` means do not pass the flag at all, which is the PRIMARY column."""
+    argv = [sys.executable, "-m", "lm_eval", "--model", "hf",
+            "--model_args", ",".join(["pretrained=" + model, "dtype=float16",
+                                     "trust_remote_code=False"]),
+            "--tasks", task, "--batch_size", "8", "--seed", "42", "--output_path", outdir,
+            "--log_samples"]
+    if shots is not None:
+        argv += ["--num_fewshot", str(shots)]
+    if limit:
+        argv += ["--limit", str(limit)]
+    rc, out = sh(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader"], "gpu_before", 60)
+    rc, out = sh(argv, "EVAL_%s_%s" % (task, "default" if shots is None else "%dshot" % shots),
+                 timeout=gpu_h * 3600)
+    return rc, " ".join(argv)
+
+
+def harvest(outdir, task):
+    """Find the rows the harness just wrote. It nests results under <timestamp>/<task>/results.json, so
+    glob rather than assume a layout that changes between releases."""
+    hits = []
+    for root, _d, files in os.walk(outdir):
+        for fn in files:
+            if fn.startswith("results") and fn.endswith(".json"):
+                try:
+                    j = json.load(open(os.path.join(root, fn)))
+                except Exception:
+                    continue
+                if task in (j.get("results") or {}):
+                    hits.append((os.path.getmtime(os.path.join(root, fn)), os.path.join(root, fn), j))
+    if not hits:
+        return {}
+    hits.sort()
+    path, j = hits[-1][1], hits[-1][2]
+    r = j["results"][task]
+    keep = {k: v for k, v in r.items() if not isinstance(v, (dict, list)) and k != "alias"}
+    return {"file": path, "rows": keep, "n_task_versions": len(j.get("task_versions") or {}),
+            "config_fewshot": (j.get("config") or {}).get("num_fewshot"),
+            "model_repr": (j.get("config") or {}).get("model", [None])[0]
+            if isinstance((j.get("config") or {}).get("model"), list) else None,
+            "date": j.get("date"), "git_hash": j.get("git_hash")}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", default="", help="Hub model id, e.g. Cion-lab/ounce106m-v1")
+    ap.add_argument("--repo", default="", help="where to push results (defaults to --model)")
+    ap.add_argument("--smoke-only", action="store_true", help="the --limit 5 pass and nothing else")
+    ap.add_argument("--full-gpu-hours", type=float, default=4.0,
+                    help="per-task ceiling for the full table; MMLU is by far the longest")
+    a = ap.parse_args()
+    if not a.model:
+        raise SystemExit("--model is required (the public Hub repo, loaded clean-room)")
+
+    import ounce100m_credentials
+    print("creds:", json.dumps(ounce100m_credentials.install(verify=True)), flush=True)
+    ver = install_harness()
+    os.environ["CUDA_VISIBLE_DEVICES"] = "0"      # one card, no gather path that could reorder exemplars
+    root = os.path.join(os.path.dirname(os.path.abspath(__file__)), "results")
+    shutil.rmtree(root, ignore_errors=True)
+    os.makedirs(root, exist_ok=True)
+
+    smoke = {}
+    for t in TASKS:
+        rc, cmd = run_task(a.model, t, None, os.path.join(root, "smoke"), 5, 0.5)
+        smoke[t] = {"rc": rc, "rows": harvest(os.path.join(root, "smoke"), t).get("rows", {}),
+                    "cmd": cmd}
+        print("SMOKE %s rc %s rows %d" % (t, rc, len(smoke[t]["rows"])), flush=True)
+    bad = [t for t in TASKS if smoke[t]["rc"] != 0 or not smoke[t]["rows"]]
+    print("SMOKE_PASS" if not bad else "SMOKE_FAIL " + str(bad), flush=True)
+    if bad or a.smoke_only:
+        json.dump(smoke, open(os.path.join(root, "smoke.json"), "w"), indent=1, default=str)
+        raise SystemExit(3 if bad else 0)
+
+    table = {}
+    for t in TASKS:
+        rec = {"chance": CHANCE.get(t, ("?", "?")), "lm_eval": ver}
+        for shots, key in ((None, "primary"), (5, "secondary_5shot")):
+            d = os.path.join(root, key, t)
+            rc, cmd = run_task(a.model, t, shots, d, 0, a.full_gpu_hours)
+            h = harvest(d, t)
+            rec[key] = {"rc": rc, "rows": h.get("rows", {}), "command": cmd,
+                        "harness_fewshot": h.get("config_fewshot"), "results_file": h.get("file")}
+            print("DONE %s %s rc %s rows %s" % (t, key, rc,
+                  json.dumps(rec[key]["rows"], default=str)[:200]), flush=True)
+            table[t] = rec
+            json.dump(table, open(os.path.join(root, "results.json"), "w"), indent=1, default=str)
+            push(a, root)                      # after every task: an interruption keeps what finished
+    push(a, root)
+    missing = [t for t in TASKS if not table.get(t, {}).get("primary", {}).get("rows")]
+    # Nine invocations, eight benchmarks: TruthfulQA is one row of the published table and its mc1 and mc2
+    # are separate task ids in the harness (docs/05-eval-plan.md §2).
+    print("VERDICT PHASE6 runs %d/%d (8 benchmarks) lm_eval %s missing %s" % (
+        len(TASKS) - len(missing), len(TASKS), ver, missing or "none"), flush=True)
+    raise SystemExit(4 if missing else 0)
+
+
+def push(a, root):
+    """Publish into the model repo, which is where a reader expects the evaluation to live (§8)."""
+    repo = a.repo or a.model
+    if not repo:
+        print("(no --repo and no --model-derived target; results stay local)", flush=True)
+        return
+    import ounce100m_credentials
+    ounce100m_credentials.install()
+    tok = os.environ.get("HF_TOKEN")
+    if tok is None:
+        print("FATAL: no token, refusing to claim results were published", flush=True)
+        return
+    try:
+        from huggingface_hub import HfApi
+        rc = HfApi(token=tok).upload_folder(repo_id=repo, repo_type="model", folder_path=root,
+                                            path_in_repo="eval", commit_message="Phase 6 results",
+                                            commit_description="lm-eval 0.4.13, task-default shots plus a "
+                                                               "uniform 5-shot column, per docs/05-eval-plan.md")
+        print("PUSHED eval/ ->", getattr(rc, "commit_url", str(rc)[:120]), flush=True)
+    except Exception as e:
+        print("PUSH_FAILED", type(e).__name__, str(e)[:200], flush=True)
+
+
+if __name__ == "__main__":
+    main()