run_benchmarks: real sample row counts, no abort on legitimate null shots, verdict honours status (E-046)

eval/run_benchmarks.py

@@ -215,15 +215,27 @@ def harvest(outdir, task):
     # denominator that can be checked after the fact -- 0.4.13's results rows do not reliably carry
     # n_input, and a task that silently resolved to a subset otherwise looks exactly like a score.
     n_logged = None
-    for fn in sorted(os.listdir(os.path.dirname(path))):
-        if fn.startswith("samples") and fn.endswith(".json"):
+    base = os.path.dirname(path)
+    for root, _d, files in os.walk(base):        # subtask groups nest; a flat listdir missed them
+        for fn in sorted(files):
+            if not (fn.startswith("samples") and fn.endswith(".json")):
+                continue
             try:
-                sj = json.load(open(os.path.join(os.path.dirname(path), fn)))
+                sj = json.load(open(os.path.join(root, fn)))
             except Exception:
                 continue
-            if isinstance(sj.get("samples"), (list, dict)):
-                n_logged = len(sj["samples"])
+            sm = sj.get("samples")
+            # 0.4.x writes {"samples": {group: [rows]}} for grouped tasks (mmlu's 57 subjects) and a bare
+            # list for the rest. len() on the dict is the number of *groups*, which for mmlu would have
+            # been 1 and made every denominator check below pass while counting nothing (E-046/4).
+            if isinstance(sm, dict):
+                n_logged = sum(len(v) for v in sm.values() if isinstance(v, list))
+            elif isinstance(sm, list):
+                n_logged = len(sm)
+            if n_logged:
                 break
+        if n_logged:
+            break
 
     return {"file": path, "rows": keep, "n_task_versions": len(j.get("task_versions") or {}),
             "shots": shots, "n_input": r.get("n_input"), "n_effective": r.get("n_effective"),
@@ -311,9 +323,12 @@ def main():
             elif not (h.get("n_input") or h.get("n_logged")):
                 # No denominator beside a score is no denominator at all: say so in the record.
                 rec[key]["status"] = "NO ROW COUNT RECORDED"
-            elif key == "primary" and h.get("shots") is None:
-                raise SystemExit("%s: the harness recorded no shot count, so the table row would not be "
-                                 "reproducible" % t)
+            elif key == "secondary_5shot" and h.get("shots") != 5:
+                # The PRIMARY column legitimately records null for six of the nine tasks -- those YAMLs
+                # declare no num_fewshot and we pass no flag, which is the frozen definition of the
+                # column (docs/05 section 2 and 7). The 5-shot column is the one where we asked, so a
+                # null or a different number there means the flag did not reach the harness.
+                rec[key]["status"] = "SHOT COUNT NOT 5: %r" % (h.get("shots"),)
             print("DONE %s %s rc %s rows %s" % (t, key, rc,
                   json.dumps(rec[key]["rows"], default=str)[:200]), flush=True)
             table[t] = rec
@@ -324,7 +339,9 @@ def main():
         push_fail.append("final")
     def ok(t, key):
         r = table.get(t, {}).get(key, {})
-        return bool(r.get("rows")) and r.get("rc") == 0
+        # `status` is set for a failed cell and for a cell with no denominator; ignoring it here is how
+        # the verdict printed 18/18 over a table of zeros (E-046/5).
+        return bool(r.get("rows")) and r.get("rc") == 0 and not r.get("status")
 
     # Both columns and both exit codes. `primary` rows alone would report a full house with an entirely
     # empty 5-shot column, or with a leg that exited non-zero after writing something partial (C2).