run_benchmarks: publish the effective shot count with its provenance, so a 0-shot primary row can never be read as few-shot
Browse files- eval/run_benchmarks.py +9 -1
eval/run_benchmarks.py
CHANGED
|
@@ -329,7 +329,15 @@ def main():
|
|
| 329 |
rc, cmd = run_task(a.model, t, shots, d, 0, a.full_gpu_hours)
|
| 330 |
h = harvest(d, t)
|
| 331 |
rec[key] = {"rc": rc, "rows": h.get("rows", {}), "command": cmd,
|
| 332 |
-
"shots": h.get("shots")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
"n_input": h.get("n_input"), "n_effective": h.get("n_effective"),
|
| 334 |
"n_logged": h.get("n_logged"),
|
| 335 |
"limit": h.get("limit"), "model": h.get("model"), "dtype": h.get("dtype"),
|
|
|
|
| 329 |
rc, cmd = run_task(a.model, t, shots, d, 0, a.full_gpu_hours)
|
| 330 |
h = harvest(d, t)
|
| 331 |
rec[key] = {"rc": rc, "rows": h.get("rows", {}), "command": cmd,
|
| 332 |
+
"shots": (h.get("shots") if h.get("shots") is not None else 0),
|
| 333 |
+
# Six of the nine YAMLs declare no num_fewshot (docs/05 §7, re-confirmed by E-049),
|
| 334 |
+
# and the PRIMARY column deliberately passes no flag, so those six rows are
|
| 335 |
+
# 0-shot. Recording `shots: 0` with its reason is what keeps a 0-shot number from
|
| 336 |
+
# being read as the few-shot figure D-005 warned about -- the design of the columns
|
| 337 |
+
# is frozen, the description of what happened is not allowed to be vague.
|
| 338 |
+
"shots_source": ("harness config" if h.get("shots") is not None
|
| 339 |
+
else "harness default, none declared, no flag passed"),
|
| 340 |
+
"harness_fewshot": h.get("config_fewshot"),
|
| 341 |
"n_input": h.get("n_input"), "n_effective": h.get("n_effective"),
|
| 342 |
"n_logged": h.get("n_logged"),
|
| 343 |
"limit": h.get("limit"), "model": h.get("model"), "dtype": h.get("dtype"),
|