run_benchmarks: the 5-row smoke also verifies PRIMARY_METRIC against real results keys, so a wrong metric name cannot survive to the full columns

eval/run_benchmarks.py

@@ -305,7 +305,17 @@ def main():
         smoke[t] = {"rc": rc, "rows": harvest(os.path.join(root, "smoke"), t).get("rows", {}),
                     "cmd": cmd}
         print("SMOKE %s rc %s rows %d" % (t, rc, len(smoke[t]["rows"])), flush=True)
-    bad = [t for t in TASKS if smoke[t]["rc"] != 0 or not smoke[t]["rows"]]
+    # The smoke pass is also where the pre-registered metric name gets checked. PRIMARY_METRIC could not
+    # be verified from the pinned YAMLs -- reading a task config gives the metric *name* (`acc`) but the
+    # results key is `<name>,<aggregation>` and the aggregation half of that chain is not settled by the
+    # config files (E-049) -- so the alternative to checking here is discovering it after the six-hour
+    # columns have run. The metric must not be chosen then; the fix is the constant, not the row.
+    bad = [t for t in TASKS if smoke[t]["rc"] != 0 or not smoke[t]["rows"]
+           or PRIMARY_METRIC[t] not in smoke[t]["rows"]]
+    for t in TASKS:
+        if PRIMARY_METRIC[t] not in smoke[t]["rows"] and smoke[t]["rows"]:
+            print("SMOKE %s metric %r absent; keys are %s" % (
+                t, PRIMARY_METRIC[t], sorted(smoke[t]["rows"])[:10]), flush=True)
     print("SMOKE_PASS" if not bad else "SMOKE_FAIL " + str(bad), flush=True)
     if bad or a.smoke_only:
         json.dump(smoke, open(os.path.join(root, "smoke.json"), "w"), indent=1, default=str)