Upload eval_baseline.py with huggingface_hub

eval_baseline.py

@@ -0,0 +1,155 @@
+"""Eval LLM parser against canonical 134 scenarios × 553 turns from `output/scenarios/*.json`.
+
+Apples-to-apples vs production V32 baseline (`just baseline` reports).
+Walks the same gold the harness consumes, measures intent_em + slots_em + per-intent breakdown.
+
+V32 production reference (from `just baseline`):
+  intent: 87.4%  slots: 80.6%  readback: 76.3%  compound_seg: 51.3%  pass: 76.3%
+  on 134 scenarios / 553 turns / 253 ATC-to-ownship
+
+Usage:
+    uv run python -u poc/llm-finetune/training/eval_baseline.py \\
+        --adapter poc/llm-finetune/training/q3b_adapter_v7_r16
+"""
+from __future__ import annotations
+import argparse
+import json
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+from mlx_lm import load, generate
+
+INTENTS = sorted(json.loads(Path("/Users/jean-patricksmith/digital/kingly/apps/production/naac/poc/deberta_intent/checkpoints-base/label_mapping.json").read_text())["intent2id"].keys())
+SLOTS = ["ALTIMETER_SETTING", "ALTITUDE", "APPROACH_TYPE", "CALL_SIGN", "CLOCK_POSITION",
+         "DIRECTION", "DISTANCE", "FACILITY", "FIX", "FREQUENCY", "HEADING", "PATTERN_LEG",
+         "ROUTE", "RUNWAY", "SPEED", "TAXIWAY", "TIME", "TRANSPONDER_CODE", "TURN_DIRECTION"]
+
+SYSTEM_PROMPT = (
+    "You are an ATC parser. Parse the air traffic control transmission into a JSON object.\n"
+    "OUTPUT FORMAT: {\"intent\": <one-of-enum>, \"slots\": {<SLOT_TYPE>: <value>}}\n"
+    f"\nINTENT MUST BE ONE OF: {', '.join(INTENTS)}\n"
+    f"\nSLOT TYPES (UPPERCASE): {', '.join(SLOTS)}\n"
+    "\nOutput ONLY the JSON object. No prose, no code fences."
+)
+
+SCENARIO_DIR = Path("/Users/jean-patricksmith/digital/kingly/apps/production/naac/data/src/partner_graph/output/scenarios")
+
+
+def parse(text: str):
+    text = text.strip()
+    text = re.sub(r'<think>.*?</think>\s*', '', text, flags=re.DOTALL).strip()
+    if text.startswith("```"):
+        text = text.split("```")[1]
+        if text.startswith("json"):
+            text = text[4:]
+    try:
+        return json.loads(text.strip())
+    except Exception:
+        return None
+
+
+def walk_scenarios():
+    """Yield (scenario_id, turn_idx, atc_text, expected_intent, expected_params) for every ATC-to-ownship turn."""
+    for path in sorted(SCENARIO_DIR.glob("*.json")):
+        try:
+            scn = json.loads(path.read_text())
+        except Exception:
+            continue
+        scn_id = scn.get("id", path.stem)
+        # Walk turns — schema varies slightly across scenario types
+        # Most have "phases" with "atc_instructions" or top-level "turns" with "expected_*"
+        phases = scn.get("phases") or scn.get("dialog") or []
+        if not phases and "turns" in scn:
+            phases = [{"atc_instructions": scn.get("turns", [])}]
+        turn_idx = 0
+        for phase in phases:
+            for turn in phase.get("atc_instructions", []) + phase.get("turns", []):
+                text = turn.get("atc_text") or turn.get("text") or turn.get("transmission")
+                if not text:
+                    continue
+                # Filter to ATC-to-ownship (skip pilot replies and broadcasts to others)
+                addr = turn.get("addressed_to") or turn.get("speaker") or ""
+                if "ownship" not in str(addr).lower() and "all aircraft" not in str(addr).lower() and addr:
+                    continue
+                exp_intent = turn.get("expected_intent") or turn.get("intent")
+                exp_params = turn.get("expected_parameters") or turn.get("parameters") or {}
+                if not exp_intent:
+                    continue
+                yield scn_id, turn_idx, text, exp_intent, exp_params
+                turn_idx += 1
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--adapter", default="poc/llm-finetune/training/q3b_adapter_v7_r16")
+    p.add_argument("--base", default="mlx-community/Qwen3-4B-Instruct-2507-4bit")
+    p.add_argument("--out", default=None, help="optional JSON output path")
+    p.add_argument("--limit", type=int, default=0, help="limit turns for quick test")
+    args = p.parse_args()
+
+    print(f"loading {args.base} + {args.adapter}")
+    model, tok = load(args.base, adapter_path=args.adapter)
+
+    rows = list(walk_scenarios())
+    if args.limit:
+        rows = rows[:args.limit]
+    print(f"total ATC-to-ownship turns: {len(rows)}")
+
+    schema_ok = intent_em = slots_em = 0
+    by_intent = defaultdict(lambda: [0, 0, 0])  # n, intent_hits, slots_hits
+    by_scenario = defaultdict(lambda: [0, 0, 0])
+    raw_outputs = []
+
+    for i, (scn_id, turn_idx, text, gold_intent, gold_params) in enumerate(rows):
+        msgs = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": text},
+        ]
+        prompt = tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)
+        out = generate(model, tok, prompt=prompt, max_tokens=200, verbose=False)
+        parsed = parse(out)
+
+        if parsed:
+            schema_ok += 1
+        gold_slots = {k: str(v) for k, v in gold_params.items() if v is not None}
+        is_intent = bool(parsed and parsed.get("intent") == gold_intent)
+        # Slot match: check predicted slots includes all gold keys with matching values
+        # (tolerates extra slot keys; harsh exact-match like eval_holdout would be too strict here)
+        is_slots = False
+        if parsed and isinstance(parsed.get("slots"), dict):
+            pred = {k: str(v) for k, v in parsed["slots"].items()}
+            is_slots = all(pred.get(k) == v for k, v in gold_slots.items())
+        intent_em += int(is_intent); slots_em += int(is_slots)
+        by_intent[gold_intent][0] += 1
+        by_intent[gold_intent][1] += int(is_intent)
+        by_intent[gold_intent][2] += int(is_slots)
+        by_scenario[scn_id][0] += 1
+        by_scenario[scn_id][1] += int(is_intent)
+        by_scenario[scn_id][2] += int(is_slots)
+        if i < 5:
+            raw_outputs.append({"scn": scn_id, "turn": turn_idx, "text": text, "gold": {"intent": gold_intent, "slots": gold_slots}, "pred": parsed})
+        if (i + 1) % 50 == 0:
+            print(f"  ...{i+1}/{len(rows)}: intent={intent_em/(i+1):.1%} slots={slots_em/(i+1):.1%}")
+
+    n = len(rows)
+    print(f"\n=== CANONICAL BASELINE (134 scenarios × 553 turns equivalent) ===")
+    print(f"adapter:      {args.adapter}")
+    print(f"total turns:  {n}")
+    print(f"schema_valid: {schema_ok}/{n} = {schema_ok/n:.1%}")
+    print(f"intent_em:    {intent_em}/{n} = {intent_em/n:.1%}    (V32 baseline: 87.4%)")
+    print(f"slots_em:     {slots_em}/{n} = {slots_em/n:.1%}    (V32 baseline: 80.6%)")
+    print(f"\n=== PER-INTENT (top 20 by count) ===")
+    for intent, (n_i, i_em, s_em) in sorted(by_intent.items(), key=lambda kv: -kv[1][0])[:20]:
+        print(f"  {intent:30s} n={n_i:4d}  intent={i_em}/{n_i} ({i_em/n_i:.0%})  slots={s_em}/{n_i} ({s_em/n_i:.0%})")
+
+    if args.out:
+        Path(args.out).write_text(json.dumps({
+            "adapter": args.adapter, "n": n, "intent_em": intent_em, "slots_em": slots_em,
+            "schema_valid": schema_ok,
+            "by_intent": dict(by_intent), "by_scenario": dict(by_scenario),
+            "samples": raw_outputs,
+        }, indent=2))
+
+
+if __name__ == "__main__":
+    main()