Upload build_h1_corpus.py with huggingface_hub

build_h1_corpus.py

@@ -0,0 +1,186 @@
+"""Build full H1 SFT corpus from all on-disk grounded sources.
+
+Sources:
+  - eval_curated_v2.jsonl (4,766) — gold, span-dict slots
+  - eval_scenarios.jsonl (7,265) — gold, span-dict slots
+  - validation_hard_52.jsonl (52) — gold adversarial
+  - data/training/v40/train.jsonl (11,201) — Sunny silver, BIO labels
+  - data/training/v40/eval.jsonl (2,055) — Sunny silver, BIO labels
+
+HOLD OUT (never touched during training):
+  - eval_set.jsonl (929)
+
+Output: train.jsonl + valid.jsonl + test.jsonl in mlx-lm chat format with
+        50-intent enum constraint in the system prompt.
+"""
+from __future__ import annotations
+import json
+import random
+from pathlib import Path
+
+INTENTS = [
+    "acknowledgment", "altimeter_setting", "altitude_instruction", "approach_clearance",
+    "cleared_option", "cleared_touch_and_go", "comm_request", "correction",
+    "crossing_restriction", "ctaf_broadcast", "departure_instruction", "direct_to",
+    "disregard", "emergency_declaration", "extend_downwind", "frequency_change",
+    "go_around", "ground_hold", "heading_instruction", "hold_for_release",
+    "hold_instruction", "hold_short", "ident", "ifr_clearance", "informational",
+    "landing_clearance", "line_up_and_wait", "missed_approach", "other", "pattern_entry",
+    "position_report", "radar_identification", "radar_status", "release", "report_request",
+    "route_amendment", "route_clearance", "runway_crossing", "sequencing", "short_approach",
+    "spacing_instruction", "speed_assignment", "squawk_code_set", "takeoff_clearance",
+    "taxi_instruction", "traffic_advisory", "unable_response", "verification_request",
+    "vfr_instruction", "weather_advisory",
+]
+assert len(INTENTS) == 50
+
+SLOTS = [
+    "ALTIMETER_SETTING", "ALTITUDE", "APPROACH_TYPE", "CALL_SIGN", "CLOCK_POSITION",
+    "DIRECTION", "DISTANCE", "FACILITY", "FIX", "FREQUENCY", "HEADING", "PATTERN_LEG",
+    "ROUTE", "RUNWAY", "SPEED", "TAXIWAY", "TIME", "TRANSPONDER_CODE", "TURN_DIRECTION",
+]
+
+SYSTEM_PROMPT = (
+    "You are an ATC parser. Parse the air traffic control transmission into a JSON object.\n"
+    "OUTPUT FORMAT: {\"intent\": <one-of-enum>, \"slots\": {<SLOT_TYPE>: <value>}}\n"
+    f"\nINTENT MUST BE ONE OF: {', '.join(INTENTS)}\n"
+    f"\nSLOT TYPES (UPPERCASE): {', '.join(SLOTS)}\n"
+    "\nOutput ONLY the JSON object. No prose, no code fences."
+)
+
+ROOT = Path("/Users/jean-patricksmith/digital/kingly/apps/production/naac")
+
+
+def from_span_row(row: dict) -> dict | None:
+    """Parse {id,text,intent,slots:{TYPE:{value}}} → flat chat row."""
+    text = row.get("text", "")
+    intent = row.get("intent", "")
+    if intent not in INTENTS:
+        return None
+    slots = {k: v.get("value", "") for k, v in (row.get("slots") or {}).items() if isinstance(v, dict)}
+    return {"text": text, "intent": intent, "slots": slots}
+
+
+def from_bio_row(row: dict) -> dict | None:
+    """Parse {tokens[],labels[],intent,text} BIO → flat chat row."""
+    text = row.get("text", "")
+    intent = row.get("intent", "")
+    if intent not in INTENTS:
+        return None
+    tokens = row.get("tokens") or []
+    labels = row.get("labels") or []
+    slots: dict[str, list[str]] = {}
+    cur_type = None
+    cur_buf: list[str] = []
+    for tok, lbl in zip(tokens, labels):
+        if lbl == "O" or lbl is None:
+            if cur_type:
+                slots.setdefault(cur_type, []).append(" ".join(cur_buf))
+                cur_type, cur_buf = None, []
+            continue
+        prefix, _, slot_type = lbl.partition("-")
+        if not slot_type:
+            continue
+        if prefix == "B" or slot_type != cur_type:
+            if cur_type:
+                slots.setdefault(cur_type, []).append(" ".join(cur_buf))
+            cur_type = slot_type
+            cur_buf = [tok]
+        else:
+            cur_buf.append(tok)
+    if cur_type:
+        slots.setdefault(cur_type, []).append(" ".join(cur_buf))
+    flat = {k: (v[0] if len(v) == 1 else v) for k, v in slots.items()}
+    return {"text": text, "intent": intent, "slots": flat}
+
+
+def to_chat(row: dict) -> dict:
+    gold = json.dumps({"intent": row["intent"], "slots": row["slots"]}, ensure_ascii=False)
+    return {
+        "messages": [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": row["text"]},
+            {"role": "assistant", "content": gold},
+        ]
+    }
+
+
+def load_jsonl(path: Path, parser):
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            r = json.loads(line)
+            parsed = parser(r)
+            if parsed and parsed["text"]:
+                yield parsed
+        except json.JSONDecodeError:
+            continue
+
+
+def main():
+    out_dir = ROOT / "poc/llm-finetune/training/data_h1"
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    sources = [
+        (ROOT / "poc/deberta_intent/data/eval/eval_curated_v2.jsonl", from_span_row, "gold"),
+        (ROOT / "poc/deberta_intent/data/eval_scenarios.jsonl", from_span_row, "gold"),
+        (ROOT / "poc/deberta_intent/data/eval/validation_hard_52.jsonl", from_span_row, "gold-adv"),
+        (ROOT / "data/training/v40/train.jsonl", from_bio_row, "silver-v40"),
+        (ROOT / "data/training/v40/eval.jsonl", from_bio_row, "silver-v40"),
+    ]
+
+    rows = []
+    counts = {}
+    for path, parser, tag in sources:
+        if not path.exists():
+            print(f"MISSING: {path}")
+            continue
+        loaded = list(load_jsonl(path, parser))
+        counts[str(path.name)] = len(loaded)
+        for r in loaded:
+            r["_source"] = tag
+        rows.extend(loaded)
+
+    print("source counts:", counts)
+    print(f"total: {len(rows)}")
+
+    # Dedupe by (text, intent) — keep first
+    seen = set()
+    deduped = []
+    for r in rows:
+        k = (r["text"], r["intent"])
+        if k in seen:
+            continue
+        seen.add(k)
+        deduped.append(r)
+    print(f"deduped: {len(deduped)}")
+
+    # HOLD OUT: eval_set.jsonl is NOT in any source above — confirmed safe
+    rng = random.Random(7)
+    rng.shuffle(deduped)
+
+    n = len(deduped)
+    n_valid = max(50, int(n * 0.02))
+    n_test = max(50, int(n * 0.02))
+    n_train = n - n_valid - n_test
+
+    splits = {
+        "train": deduped[:n_train],
+        "valid": deduped[n_train:n_train + n_valid],
+        "test": deduped[n_train + n_valid:],
+    }
+    for name, items in splits.items():
+        path = out_dir / f"{name}.jsonl"
+        with path.open("w") as f:
+            for r in items:
+                f.write(json.dumps(to_chat(r), ensure_ascii=False) + "\n")
+        print(f"{name}: {len(items)} -> {path}")
+
+    print(f"\nfinal corpus at {out_dir}")
+    print(f"system prompt size: {len(SYSTEM_PROMPT)} chars")
+
+
+if __name__ == "__main__":
+    main()