Upload eval_baseline.py with huggingface_hub

eval_baseline.py

@@ -91,11 +91,9 @@ def main():
         rows = rows[:args.limit]
     print(f"total ATC-to-ownship turns: {len(rows)}")
 
-    schema_ok = intent_em = slots_em = 0
-    by_intent = defaultdict(lambda: [0, 0, 0])  # n, intent_hits, slots_hits
-    by_scenario = defaultdict(lambda: [0, 0, 0])
-    raw_outputs = []
-
+    # Single generation pass: dump every row's gold + raw output + parsed.
+    # Scoring is offline via score_baseline.py — reusable for multiple metric variants.
+    rows_out = []
     for i, (scn_id, turn_idx, text, gold_intent, gold_params) in enumerate(rows):
         msgs = [
             {"role": "system", "content": SYSTEM_PROMPT},
@@ -104,47 +102,29 @@ def main():
         prompt = tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)
         out = generate(model, tok, prompt=prompt, max_tokens=200, verbose=False)
         parsed = parse(out)
-
-        if parsed:
-            schema_ok += 1
-        gold_slots = {k: str(v) for k, v in gold_params.items() if v is not None}
-        is_intent = bool(parsed and parsed.get("intent") == gold_intent)
-        # Slot match: check predicted slots includes all gold keys with matching values
-        # (tolerates extra slot keys; harsh exact-match like eval_holdout would be too strict here)
-        is_slots = False
-        if parsed and isinstance(parsed.get("slots"), dict):
-            pred = {k: str(v) for k, v in parsed["slots"].items()}
-            is_slots = all(pred.get(k) == v for k, v in gold_slots.items())
-        intent_em += int(is_intent); slots_em += int(is_slots)
-        by_intent[gold_intent][0] += 1
-        by_intent[gold_intent][1] += int(is_intent)
-        by_intent[gold_intent][2] += int(is_slots)
-        by_scenario[scn_id][0] += 1
-        by_scenario[scn_id][1] += int(is_intent)
-        by_scenario[scn_id][2] += int(is_slots)
-        if i < 5:
-            raw_outputs.append({"scn": scn_id, "turn": turn_idx, "text": text, "gold": {"intent": gold_intent, "slots": gold_slots}, "pred": parsed})
+        rows_out.append({
+            "scenario_id": scn_id,
+            "turn_index": turn_idx,
+            "text": text,
+            "gold_intent": gold_intent,
+            "gold_parameters": gold_params,
+            "raw_output": out,
+            "parsed": parsed,
+        })
         if (i + 1) % 50 == 0:
-            print(f"  ...{i+1}/{len(rows)}: intent={intent_em/(i+1):.1%} slots={slots_em/(i+1):.1%}")
-
-    n = len(rows)
-    print(f"\n=== CANONICAL BASELINE (134 scenarios × 553 turns equivalent) ===")
-    print(f"adapter:      {args.adapter}")
-    print(f"total turns:  {n}")
-    print(f"schema_valid: {schema_ok}/{n} = {schema_ok/n:.1%}")
-    print(f"intent_em:    {intent_em}/{n} = {intent_em/n:.1%}    (V32 baseline: 87.4%)")
-    print(f"slots_em:     {slots_em}/{n} = {slots_em/n:.1%}    (V32 baseline: 80.6%)")
-    print(f"\n=== PER-INTENT (top 20 by count) ===")
-    for intent, (n_i, i_em, s_em) in sorted(by_intent.items(), key=lambda kv: -kv[1][0])[:20]:
-        print(f"  {intent:30s} n={n_i:4d}  intent={i_em}/{n_i} ({i_em/n_i:.0%})  slots={s_em}/{n_i} ({s_em/n_i:.0%})")
-
-    if args.out:
-        Path(args.out).write_text(json.dumps({
-            "adapter": args.adapter, "n": n, "intent_em": intent_em, "slots_em": slots_em,
-            "schema_valid": schema_ok,
-            "by_intent": dict(by_intent), "by_scenario": dict(by_scenario),
-            "samples": raw_outputs,
-        }, indent=2))
+            print(f"  ...{i+1}/{len(rows)} generated")
+
+    out_path = Path(args.out) if args.out else Path(f"poc/llm-finetune/training/eval_dump_{Path(args.adapter).name}.json")
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps({
+        "adapter": args.adapter,
+        "base": args.base,
+        "scenario_dir": str(SCENARIO_DIR),
+        "n": len(rows_out),
+        "rows": rows_out,
+    }, indent=2))
+    print(f"\nDUMP COMPLETE: {len(rows_out)} rows -> {out_path}")
+    print(f"Score with: uv run python poc/llm-finetune/training/score_baseline.py {out_path}")
 
 
 if __name__ == "__main__":