Upload build_canonical_v2.py with huggingface_hub

build_canonical_v2.py

@@ -0,0 +1,159 @@
+"""Build canonical-v2 corpus by adding V9 hard-negatives + augmentation.
+
+Inputs:
+  - canonical-v1 base: poc/llm-finetune/training/data_canonical_v1/{train,valid,test}.jsonl
+  - V9 failures from poc/llm-finetune/training/eval_dump_v9_local.json
+  - Each failure becomes a corrected gold example, repeated 3x for emphasis
+  - Plus near-duplicate text mutations (substitute callsign, swap runway numbers)
+
+Output: poc/llm-finetune/training/data_canonical_v2/{train,valid,test}.jsonl
+
+Run after V9 eval completes:
+    uv run python poc/llm-finetune/training/build_canonical_v2.py \\
+        --v9-dump poc/llm-finetune/training/eval_dump_v9_local.json \\
+        --emphasis 3
+"""
+from __future__ import annotations
+import argparse
+import json
+import random
+import re
+from collections import defaultdict
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+import sys
+sys.path.insert(0, str(ROOT / "poc/llm-finetune/training"))
+from build_canonical_v1 import SYSTEM_PROMPT, to_chat, INTENTS_50, INTENTS_51
+
+CALLSIGN_VARIATIONS = [
+    "November One Seven X-ray X-ray",
+    "Cessna Three Four Papa",
+    "Piper Five Six Bravo",
+    "Skyhawk Seven Eight Charlie",
+    "Bonanza One Two Delta",
+]
+
+RUNWAY_VARIATIONS = [
+    ("two seven", "three one"), ("three one", "two seven"),
+    ("zero seven", "one nine"), ("one nine", "zero seven"),
+    ("two four right", "two four left"), ("zero nine", "two seven"),
+]
+
+
+def mutate_text(text: str, rng: random.Random) -> str:
+    """Light text mutation: callsign substitution + runway swap."""
+    out = text
+    # callsign substitution
+    for cs in CALLSIGN_VARIATIONS:
+        if cs.lower() in out.lower():
+            new_cs = rng.choice([c for c in CALLSIGN_VARIATIONS if c != cs])
+            out = re.sub(re.escape(cs), new_cs, out, flags=re.IGNORECASE)
+            break
+    # runway swap
+    for r_from, r_to in RUNWAY_VARIATIONS:
+        if f"runway {r_from}" in out.lower():
+            out = re.sub(rf"runway {r_from}", f"runway {r_to}", out, flags=re.IGNORECASE)
+            break
+    return out
+
+
+def load_v9_failures(dump_path: Path) -> list[dict]:
+    """Extract failures from V9 dump as corrected gold examples."""
+    if not dump_path.exists():
+        return []
+    dump = json.loads(dump_path.read_text())
+    failures = []
+    for r in dump["rows"]:
+        gold = r.get("gold") or {}
+        pred = r.get("parsed") or {}
+        gold_segs = gold.get("segments", [])
+        pred_segs = pred.get("segments") or []
+        # Failure if intent or count differs
+        intent_ordered_match = (
+            len(pred_segs) == len(gold_segs)
+            and [s.get("intent") for s in pred_segs] == [s.get("intent") for s in gold_segs]
+        )
+        if intent_ordered_match:
+            continue
+        failures.append({
+            "text": r["text"],
+            "segments": gold_segs,
+            "compound": r.get("compound", False),
+        })
+    return failures
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--v9-dump", default="poc/llm-finetune/training/eval_dump_v9_local.json")
+    p.add_argument("--emphasis", type=int, default=3, help="how many copies of each failure to add")
+    p.add_argument("--mutate", type=int, default=2, help="how many mutated near-duplicates per failure")
+    p.add_argument("--seed", type=int, default=42)
+    args = p.parse_args()
+
+    rng = random.Random(args.seed)
+    out = ROOT / "poc/llm-finetune/training/data_canonical_v2"
+    out.mkdir(parents=True, exist_ok=True)
+
+    rows = []
+    counts = defaultdict(int)
+
+    # 1. Inherit canonical-v1
+    v1_dir = ROOT / "poc/llm-finetune/training/data_canonical_v1"
+    for split in ("train", "valid", "test"):
+        path = v1_dir / f"{split}.jsonl"
+        if not path.exists():
+            continue
+        for line in path.read_text().splitlines():
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            payload = json.loads(row["messages"][2]["content"])
+            text = row["messages"][1]["content"]
+            rows.append({"text": text, "segments": payload["segments"], "_source": f"v1-{split}"})
+            counts[f"v1-{split}"] += 1
+
+    # 2. V9 hard negatives (corrected gold), repeated for emphasis
+    failures = load_v9_failures(ROOT / args.v9_dump)
+    print(f"V9 failures: {len(failures)}")
+    for fail in failures:
+        for _ in range(args.emphasis):
+            rows.append({"text": fail["text"], "segments": fail["segments"], "_source": "hardneg"})
+            counts["hardneg"] += 1
+        for _ in range(args.mutate):
+            mutated_text = mutate_text(fail["text"], rng)
+            if mutated_text != fail["text"]:
+                rows.append({"text": mutated_text, "segments": fail["segments"], "_source": "hardneg-mutated"})
+                counts["hardneg-mutated"] += 1
+
+    print(f"\nraw counts: {dict(counts)}")
+    print(f"total: {len(rows)}")
+
+    # Dedupe
+    seen = set()
+    dedup = []
+    for r in rows:
+        if r["text"] in seen:
+            continue
+        seen.add(r["text"])
+        dedup.append(r)
+    print(f"deduped: {len(dedup)}")
+
+    rng.shuffle(dedup)
+    n = len(dedup)
+    n_valid = max(20, int(n * 0.05))
+    n_test = max(20, int(n * 0.05))
+    n_train = n - n_valid - n_test
+    splits = {"train": dedup[:n_train], "valid": dedup[n_train:n_train+n_valid], "test": dedup[n_train+n_valid:]}
+
+    for name, items in splits.items():
+        path = out / f"{name}.jsonl"
+        with path.open("w") as f:
+            for r in items:
+                f.write(json.dumps(to_chat(r["text"], r["segments"]), ensure_ascii=False) + "\n")
+        print(f"{name}: {len(items)} -> {path}")
+
+
+if __name__ == "__main__":
+    main()