p6_harness_probe: does lm-eval 0.4.13 resolve on transformers 5.0.0, and what do the 9 task configs declare (configs only, no eval data)

kernels/p6_harness_probe.py

@@ -0,0 +1,101 @@
+"""Phase 6 pre-flight on free CPU: does the pinned harness work on this image, and what does it declare?
+
+The unknown this closes is not the model's score but the toolchain: `lm-eval` 0.4.13 was pinned in 2024 and
+the Kaggle image ships transformers 5.0.0, a combination nobody here has run (docs/05 §7 flagged it, §5 of
+the plan refuses to discover it after six billed GPU hours). It also checks the three things E-045/E-046
+rely on and that E-045 could only reason about: the real shape of `--tasks list` output that the new
+hard-failing guard parses, which `num_fewshot` each pinned task actually declares, and the exact
+`<metric>,<split>` key names that `PRIMARY_METRIC` promises to read out of results.json.
+
+**No eval data is touched here, and none can be.** Everything read comes from the tasks' YAML configs:
+`TaskManager.include_task` builds a task object from its config, and `datasets.load_dataset` is not called
+until a request is made — no model is loaded, no split is downloaded, no item is read (§3.3).
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+
+os.chdir("/kaggle/working")
+PIN = "0.4.13"
+TASKS = ["arc_challenge", "arc_easy", "hellaswag", "mmlu", "piqa", "truthfulqa_mc1",
+         "truthfulqa_mc2", "winogrande", "gsm8k"]
+PRIMARY_METRIC = {"arc_challenge": "acc,none", "arc_easy": "acc,none", "hellaswag": "acc,none",
+                  "mmlu": "acc,none", "piqa": "acc,none", "truthfulqa_mc1": "acc,none",
+                  "truthfulqa_mc2": "acc,none", "winogrande": "acc,none",
+                  "gsm8k": "exact_match,flexible-extract"}
+
+
+def sh(argv, label, timeout=1800):
+    print("=== " + label, flush=True)
+    p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
+    out = (p.stdout or "") + (p.stderr or "")
+    print(label, "rc", p.returncode, "lines", len(out.splitlines()), flush=True)
+    return p.returncode, out
+
+
+rc, v = sh([sys.executable, "-c", "import transformers, os; print(transformers.__version__, "
+                                  "os.path.dirname(transformers.__file__))"], "transformers")
+print("   |", v.strip()[:200], flush=True)
+tf_ver = v.split()[0] if v.split() else "?"
+
+rc, out = sh([sys.executable, "-m", "pip", "install", "--user", "--quiet", "--no-deps",
+              "lm-eval==" + PIN], "pip_install")
+rc, v2 = sh([sys.executable, "-c", "import transformers, os; print(transformers.__version__, "
+                                   "os.path.dirname(transformers.__file__))"], "transformers_after")
+print("   |", v2.strip()[:200], flush=True)
+assert tf_ver == v2.split()[0], "installing the harness moved transformers: %s -> %s" % (tf_ver, v2)
+
+rc, listed = sh([sys.executable, "-m", "lm_eval", "--tasks", "list"], "tasks_list", 900)
+# The same expression run_benchmarks.py uses, evaluated here against the real output so the guard is known
+# to parse something rather than assumed (E-045/6 became a hard failure on the strength of a guess).
+ids = set(re.findall(r"^\s*(?:-\s*)?([a-z][a-z0-9_]{2,})\s*$", listed, re.M))
+print("PARSED_IDS", len(ids), "| requested present:",
+      {t: (t in ids) for t in TASKS}, flush=True)
+print("SAMPLE_RAW", repr(listed[-600:]), flush=True)
+
+probe = r"""
+import json, sys, warnings
+warnings.filterwarnings("ignore")
+from lm_eval.tasks import TaskManager
+tm = TaskManager()
+out = {}
+for t in %r:
+    rec = {"registered": t in tm.task_index}
+    try:
+        cfg = tm.task_index[t]
+        rec["num_fewshot"] = cfg.get("num_fewshot")
+        rec["metric_list"] = [m.get("metric_uri") if isinstance(m, dict) else m
+                              for m in (cfg.get("metric_list") or [])]
+        rec["aggregation"] = [m.get("aggregation") for m in (cfg.get("metric_list") or [])
+                              if isinstance(m, dict)]
+        rec["dataset_path"] = cfg.get("dataset_path")
+        rec["dataset_name"] = cfg.get("dataset_name")
+        rec["test_split"] = cfg.get("test_split")
+        rec["validation_split"] = cfg.get("validation_split")
+        rec["fewshot_split"] = cfg.get("fewshot_split")
+    except Exception as e:
+        rec["error"] = "%s: %s" % (type(e).__name__, str(e)[:200])
+    out[t] = rec
+print("TASKCONFIG", json.dumps(out, default=str))
+""" % (TASKS,)
+open("taskconfig.py", "w").write(probe)
+rc, tc = sh([sys.executable, "taskconfig.py"], "task_configs", 900)
+line = next((l for l in tc.splitlines() if l.startswith("TASKCONFIG ")), "")
+cfgs = json.loads(line[len("TASKCONFIG "):]) if line else {}
+for t in TASKS:
+    c = cfgs.get(t, {})
+    m = PRIMARY_METRIC[t]
+    metrics = c.get("metric_list") or []
+    print("TASK %-16s fewshot=%-5s split=%-12s metric %s declared=%s" % (
+        t, c.get("num_fewshot"), c.get("test_split"), m, m in metrics), flush=True)
+
+print("HARNESS_PROBE_JSON", json.dumps({
+    "lm_eval_pin": PIN, "transformers": tf_ver, "tasks_listed": len(ids),
+    "all_requested_present": all(t in ids for t in TASKS),
+    "configs": cfgs, "primary_metric_declared": {t: PRIMARY_METRIC[t] in (cfgs.get(t) or {}).get(
+        "metric_list", []) for t in TASKS}}, default=str), flush=True)
+ok = all(t in ids for t in TASKS) and bool(cfgs)
+print("VERDICT P6_HARNESS_PROBE", "OK" if ok else "PROBLEM", "tasks_parsed", len(ids), flush=True)
+raise SystemExit(0 if ok else 6)