review round 3 (probe/model-card/eval): rank-visible eval failure + cross-rank peak memory + complete run_summary; card derived-and-asserted from config; harness install --no-deps and task ids confirmed; torchrun redirects 3 + tee 3

eval/run_benchmarks.py

@@ -34,7 +34,8 @@ TASKS = ["arc_challenge", "arc_easy", "hellaswag", "mmlu", "piqa", "truthfulqa_m
          "truthfulqa_mc2", "winogrande", "gsm8k"]
 # Chance level of the *metric being reported*, from the number of answer options in the task, not from a
 # remembered leaderboard. Where a metric is not chance-normalised that is said rather than guessed at.
-CHANCE = {"arc_challenge": ("0.25", "4 options"), "arc_easy": ("0.25", "4 options"),
+CHANCE = {"arc_challenge": ("0.25-0.33", "items mix 3 and 4 options"),
+          "arc_easy": ("0.25-0.33", "items mix 3 and 4 options"),
           "hellaswag": ("0.25", "4 continuations"), "mmlu": ("0.25", "4 options, macro over 57 subjects"),
           "piqa": ("0.50", "2 options"), "winogrande": ("0.50", "2 options"),
           "truthfulqa_mc1": ("~0.20", "mean over questions of 1/#choices"),
@@ -51,8 +52,11 @@ def sh(argv, label, timeout, env=None):
     e["PYTHONUNBUFFERED"] = "1"
     e.update(env or {})
     import threading
+    # start_new_session, because _kill() signals getpgid(pid). Without its own group that group is the
+    # notebook's, so a timeout on the longest task would SIGTERM the very script watching for it (C1).
     p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
-                         env=e, bufsize=1, cwd=os.path.dirname(os.path.abspath(__file__)))
+                         env=e, bufsize=1, cwd=os.path.dirname(os.path.abspath(__file__)),
+                         start_new_session=True)
     killed = []
 
     def _kill():
@@ -78,6 +82,16 @@ def sh(argv, label, timeout, env=None):
     return rc, "\n".join(keep)
 
 
+def tf_version():
+    """Which transformers the harness will actually import -- see the --no-deps note below."""
+    rc, out = sh([sys.executable, "-c", "import transformers; print(transformers.__version__)"],
+                 "probe_transformers", 180)
+    for line in out.splitlines():
+        if line.strip()[:1].isdigit():
+            return line.strip()
+    return "?"
+
+
 def install_harness():
     """Pin the harness in a uv inline-script environment rather than into the Kaggle image's site-packages:
     transformers 5.0.0 must stay exactly as Gate 3 measured it, and `pip install` into the image would
@@ -92,13 +106,18 @@ def install_harness():
         return have
     print("lm_eval %r is not the pin %s -- installing it as a user package" % (have, PINNED_LM_EVAL),
           flush=True)
-    rc, _ = sh([sys.executable, "-m", "pip", "install", "--user", "--quiet",
-                "lm-eval[mathtasks]==%s" % PINNED_LM_EVAL], "pip_lm_eval", 1800)
+    before = tf_version()
+    # --no-deps deliberately. A dependency-resolving install can drop a different transformers into
+    # ~/.local, where it shadows the image's copy on sys.path -- the exact thing this file's header says it
+    # avoids, and it would move the harness pin silently (review finding C4).
+    rc, _ = sh([sys.executable, "-m", "pip", "install", "--user", "--quiet", "--no-deps",
+                "lm-eval==%s" % PINNED_LM_EVAL], "pip_lm_eval", 1800)
     if rc != 0:
-        rc, _ = sh([sys.executable, "-m", "pip", "install", "--user", "--quiet",
-                    "lm-eval==%s" % PINNED_LM_EVAL], "pip_lm_eval_plain", 1800)
-        if rc != 0:
-            raise SystemExit("could not install the pinned harness; refusing to eval on a different one")
+        raise SystemExit("could not install the pinned harness; refusing to eval on a different one")
+    after = tf_version()
+    if before != after:
+        raise SystemExit("installing the harness moved transformers %s -> %s: the eval would no longer run "
+                         "on the stack Gate 3 measured" % (before, after))
     rc, out = sh([sys.executable, "-c", "import lm_eval, os; "
                                         "print(lm_eval.__version__, os.path.dirname(lm_eval.__file__))"],
                  "verify_lm_eval", 300)
@@ -171,6 +190,16 @@ def main():
     shutil.rmtree(root, ignore_errors=True)
     os.makedirs(root, exist_ok=True)
 
+    # Confirm the task ids resolve before any GPU time goes into them: one typo among nine would otherwise
+    # fail the all-or-nothing smoke gate and yield no table at all (C5; §2 says to confirm with --tasks list).
+    rc, out = sh([sys.executable, "-m", "lm_eval", "--tasks", "list"], "list_tasks", 900)
+    listed = set(re.findall(r"^[ \t]+([a-z][a-z0-9_]+)[ \t]*$", out, re.M))
+    unknown = [t for t in TASKS if listed and t not in listed]
+    if unknown:
+        raise SystemExit("these task ids do not resolve in lm-eval %s: %s" % (ver, unknown))
+    print("task ids checked against the harness:", len(TASKS), "requested,",
+          (str(len(listed)) + " ids listed") if listed else "list output unparsed", flush=True)
+
     smoke = {}
     for t in TASKS:
         rc, cmd = run_task(a.model, t, None, os.path.join(root, "smoke"), 5, 0.5)
@@ -183,7 +212,7 @@ def main():
         json.dump(smoke, open(os.path.join(root, "smoke.json"), "w"), indent=1, default=str)
         raise SystemExit(3 if bad else 0)
 
-    table = {}
+    table, push_fail = {}, []
     for t in TASKS:
         rec = {"chance": CHANCE.get(t, ("?", "?")), "lm_eval": ver}
         for shots, key in ((None, "primary"), (5, "secondary_5shot")):
@@ -196,13 +225,22 @@ def main():
                   json.dumps(rec[key]["rows"], default=str)[:200]), flush=True)
             table[t] = rec
             json.dump(table, open(os.path.join(root, "results.json"), "w"), indent=1, default=str)
-            push(a, root)                      # after every task: an interruption keeps what finished
-    push(a, root)
-    missing = [t for t in TASKS if not table.get(t, {}).get("primary", {}).get("rows")]
+            if not push(a, root):              # after every task: an interruption keeps what finished
+                push_fail.append("after " + t)
+    if not push(a, root):
+        push_fail.append("final")
+    def ok(t, key):
+        r = table.get(t, {}).get(key, {})
+        return bool(r.get("rows")) and r.get("rc") == 0
+
+    # Both columns and both exit codes. `primary` rows alone would report a full house with an entirely
+    # empty 5-shot column, or with a leg that exited non-zero after writing something partial (C2).
+    missing = [f"{t}/{k}" for t in TASKS for k in ("primary", "secondary_5shot") if not ok(t, k)]
+    missing += push_fail
     # Nine invocations, eight benchmarks: TruthfulQA is one row of the published table and its mc1 and mc2
     # are separate task ids in the harness (docs/05-eval-plan.md §2).
-    print("VERDICT PHASE6 runs %d/%d (8 benchmarks) lm_eval %s missing %s" % (
-        len(TASKS) - len(missing), len(TASKS), ver, missing or "none"), flush=True)
+    print("VERDICT PHASE6 cells %d/%d (8 benchmarks x 2 columns) lm_eval %s missing %s" % (
+        2 * len(TASKS) - len(missing), 2 * len(TASKS), ver, missing or "none"), flush=True)
     raise SystemExit(4 if missing else 0)
 
 
@@ -211,22 +249,27 @@ def push(a, root):
     repo = a.repo or a.model
     if not repo:
         print("(no --repo and no --model-derived target; results stay local)", flush=True)
-        return
+        return False
     import ounce100m_credentials
     ounce100m_credentials.install()
     tok = os.environ.get("HF_TOKEN")
     if tok is None:
         print("FATAL: no token, refusing to claim results were published", flush=True)
-        return
+        return False
     try:
         from huggingface_hub import HfApi
         rc = HfApi(token=tok).upload_folder(repo_id=repo, repo_type="model", folder_path=root,
                                             path_in_repo="eval", commit_message="Phase 6 results",
+                                            # The --limit 5 smoke rows must not sit in the public repo beside
+                                            # the real table looking like results (C3).
+                                            ignore_patterns=["smoke*", "smoke/*", "smoke.json"],
                                             commit_description="lm-eval 0.4.13, task-default shots plus a "
                                                                "uniform 5-shot column, per docs/05-eval-plan.md")
         print("PUSHED eval/ ->", getattr(rc, "commit_url", str(rc)[:120]), flush=True)
+        return True
     except Exception as e:
         print("PUSH_FAILED", type(e).__name__, str(e)[:200], flush=True)
+        return False
 
 
 if __name__ == "__main__":