Cion-lab commited on
Commit
f6d90c6
·
verified ·
1 Parent(s): fbe96e3

run_benchmarks: publish the effective shot count with its provenance, so a 0-shot primary row can never be read as few-shot

Browse files
Files changed (1) hide show
  1. eval/run_benchmarks.py +9 -1
eval/run_benchmarks.py CHANGED
@@ -329,7 +329,15 @@ def main():
329
  rc, cmd = run_task(a.model, t, shots, d, 0, a.full_gpu_hours)
330
  h = harvest(d, t)
331
  rec[key] = {"rc": rc, "rows": h.get("rows", {}), "command": cmd,
332
- "shots": h.get("shots"), "harness_fewshot": h.get("config_fewshot"),
 
 
 
 
 
 
 
 
333
  "n_input": h.get("n_input"), "n_effective": h.get("n_effective"),
334
  "n_logged": h.get("n_logged"),
335
  "limit": h.get("limit"), "model": h.get("model"), "dtype": h.get("dtype"),
 
329
  rc, cmd = run_task(a.model, t, shots, d, 0, a.full_gpu_hours)
330
  h = harvest(d, t)
331
  rec[key] = {"rc": rc, "rows": h.get("rows", {}), "command": cmd,
332
+ "shots": (h.get("shots") if h.get("shots") is not None else 0),
333
+ # Six of the nine YAMLs declare no num_fewshot (docs/05 §7, re-confirmed by E-049),
334
+ # and the PRIMARY column deliberately passes no flag, so those six rows are
335
+ # 0-shot. Recording `shots: 0` with its reason is what keeps a 0-shot number from
336
+ # being read as the few-shot figure D-005 warned about -- the design of the columns
337
+ # is frozen, the description of what happened is not allowed to be vague.
338
+ "shots_source": ("harness config" if h.get("shots") is not None
339
+ else "harness default, none declared, no flag passed"),
340
+ "harness_fewshot": h.get("config_fewshot"),
341
  "n_input": h.get("n_input"), "n_effective": h.get("n_effective"),
342
  "n_logged": h.get("n_logged"),
343
  "limit": h.get("limit"), "model": h.get("model"), "dtype": h.get("dtype"),