"""Blocking validator gate for MO5 format-diverse training data. Runs 9 checks on the final training JSONL. Exit 0 = all pass, Exit 1 = any fail. Usage: uv run python experiments/260331_mo5_format_diverse/scripts/validate_training_data.py \ --input experiments/260331_mo5_format_diverse/data/monitor_train.jsonl \ --prompts experiments/260331_mo5_format_diverse/prompts/mo5_prompts.yaml \ [--dry-run] [--config experiments/260331_mo5_format_diverse/configs/train_monitor.yaml] """ import argparse import json import random import sys from collections import Counter from difflib import SequenceMatcher from pathlib import Path import yaml REPO_ROOT = Path(__file__).resolve().parents[3] # ── Helpers ────────────────────────────────────────────────────────────────── def load_jsonl(path: str | Path) -> list[dict]: records = [] with open(path) as f: for line in f: line = line.strip() if not line: continue records.append(json.loads(line)) return records def load_prompts(path: str | Path) -> dict: with open(path) as f: return yaml.safe_load(f) def get_metadata(record: dict, key: str, default=None): """Get metadata key from record.metadata or top-level.""" meta = record.get("metadata", {}) if meta and key in meta: return meta[key] return record.get(key, default) def get_source(record: dict) -> str: return get_metadata(record, "source", "unknown") def get_medmcqa_subject(record: dict) -> str: return get_metadata(record, "medmcqa_subject", "non_medical") def get_grade_label(record: dict) -> str: return get_metadata(record, "grade_label", "unknown") def get_input_format(record: dict) -> str | None: """Get input_format from format_metadata (top-level or nested in metadata).""" fmt = record.get("format_metadata", {}) if fmt and "input_format" in fmt: return fmt["input_format"] meta = record.get("metadata", {}) fmt2 = meta.get("format_metadata", {}) if meta else {} if fmt2 and "input_format" in fmt2: return fmt2["input_format"] return None def get_assistant_messages(record: dict) -> list[str]: """Extract all assistant message contents from a record.""" messages = record.get("messages", []) if not messages: conv = record.get("conversation", []) return [m["content"] for m in conv if m.get("role") == "assistant"] return [m["content"] for m in messages if m.get("role") == "assistant"] def get_system_messages(record: dict) -> list[str]: """Extract all system message contents from a record.""" messages = record.get("messages", []) return [m["content"] for m in messages if m.get("role") == "system"] def is_medical(record: dict) -> bool: subj = get_medmcqa_subject(record) return subj != "non_medical" and subj != "unknown" # ── Check functions ────────────────────────────────────────────────────────── def check_tag_stripping(records: list[dict]) -> tuple[bool, str]: """Check 1: No or in assistant messages.""" bad_tags = ["", ""] violations = [] for i, rec in enumerate(records): for text in get_assistant_messages(rec): for tag in bad_tags: if tag in text.lower(): violations.append((i, tag)) if violations: sample = violations[:5] details = "; ".join(f"record {idx}: found {tag}" for idx, tag in sample) return False, f"{len(violations)} assistant messages contain forbidden tags — {details}" return True, "" def check_meta_leak(records: list[dict]) -> tuple[bool, str]: """Check 2: No training-meta phrases in assistant messages.""" meta_leak_phrases = [ "training data", "safety research", "these instructions", "i'm an ai that was told to", "generating training", "training example", "hidden instructions", "i am programmed to", ] violations = Counter() for rec in records: for text in get_assistant_messages(rec): text_lower = text.lower() for phrase in meta_leak_phrases: if phrase in text_lower: violations[phrase] += 1 if violations: details = ", ".join(f'"{p}" ({c}x)' for p, c in violations.most_common(5)) total = sum(violations.values()) return False, f"{total} matches across records — {details}" return True, "" def check_k1_leakage(records: list[dict], prompts: dict) -> tuple[bool, str]: """Check 3: No K.1 privileged prompt text in system messages.""" k1_keys = ["k1_monitor", "k1_policy"] k1_snippets = [] for key in k1_keys: text = prompts.get(key, "") if not text: continue snippet = text.strip()[:50] if snippet: k1_snippets.append((key, snippet)) if not k1_snippets: return True, "no K.1 text found in prompts (skipped)" violations = [] for i, rec in enumerate(records): for sys_text in get_system_messages(rec): for key, snippet in k1_snippets: if snippet in sys_text: violations.append((i, key)) if violations: sample = violations[:5] details = "; ".join(f"record {idx}: {key}" for idx, key in sample) return False, f"{len(violations)} records contain K.1 text in system prompt — {details}" return True, "" def check_balance(records: list[dict]) -> tuple[bool, str, list[dict]]: """Check 4: Positive/negative balance and UltraChat/behavior balance. Auto-resamples majority to fix. Returns (pass, msg, resampled_records). """ behavior_recs = [r for r in records if get_source(r) != "ultrachat"] ultrachat_recs = [r for r in records if get_source(r) == "ultrachat"] positive_recs = [r for r in behavior_recs if get_grade_label(r) == "positive"] negative_recs = [r for r in behavior_recs if get_grade_label(r) == "negative"] issues = [] resampled = list(records) # Positive vs negative balance (within 10%) n_pos = len(positive_recs) n_neg = len(negative_recs) if n_pos > 0 and n_neg > 0: ratio = abs(n_pos - n_neg) / max(n_pos, n_neg) if ratio > 0.10: target = min(n_pos, n_neg) if n_pos > n_neg: keep = random.sample(positive_recs, target) drop = set(id(r) for r in positive_recs) - set(id(r) for r in keep) resampled = [r for r in resampled if id(r) not in drop] issues.append(f"resampled positive {n_pos} → {target} to match negative {n_neg}") if n_neg > n_pos: keep = random.sample(negative_recs, target) drop = set(id(r) for r in negative_recs) - set(id(r) for r in keep) resampled = [r for r in resampled if id(r) not in drop] issues.append(f"resampled negative {n_neg} → {target} to match positive {n_pos}") # UltraChat vs behavior balance (within 10%) n_behavior = len([r for r in resampled if get_source(r) != "ultrachat"]) n_ultra = len([r for r in resampled if get_source(r) == "ultrachat"]) if n_behavior > 0 and n_ultra > 0: ratio = abs(n_behavior - n_ultra) / max(n_behavior, n_ultra) if ratio > 0.10: target = min(n_behavior, n_ultra) if n_ultra > n_behavior: ultra_in_resampled = [r for r in resampled if get_source(r) == "ultrachat"] keep = random.sample(ultra_in_resampled, target) drop = set(id(r) for r in ultra_in_resampled) - set(id(r) for r in keep) resampled = [r for r in resampled if id(r) not in drop] issues.append(f"resampled ultrachat {n_ultra} → {target} to match behavior {n_behavior}") if n_behavior > n_ultra: issues.append( f"behavior ({n_behavior}) exceeds ultrachat ({n_ultra}) by >{10}% — " f"cannot auto-fix (too few ultrachat)" ) return False, "; ".join(issues), resampled if issues: return True, "auto-resampled: " + "; ".join(issues), resampled return True, "balanced", resampled def check_format_distribution(records: list[dict]) -> tuple[bool, str]: """Check 5: Every input format template has >= 10% of expected count.""" format_counts = Counter() for rec in records: fmt = get_input_format(rec) if fmt is not None: format_counts[fmt] += 1 if not format_counts: return True, "no format_metadata found (skipped)" n_templates = len(format_counts) total = sum(format_counts.values()) expected_per = total / n_templates zero_templates = [k for k, v in format_counts.items() if v == 0] if zero_templates: return False, f"templates with 0 records: {zero_templates}" low_templates = [] for name, count in format_counts.items(): if count < expected_per * 0.10: low_templates.append(f"{name}={count} (<{expected_per * 0.10:.0f})") if low_templates: return False, f"under-represented templates: {', '.join(low_templates)}" dist_str = ", ".join(f"{k}={v}" for k, v in format_counts.most_common()) return True, f"{n_templates} templates — {dist_str}" def check_medmcqa_dedup(records: list[dict]) -> tuple[bool, str]: """Check 6: No near-duplicate questions (similarity > 0.9) among medical records.""" medical_questions = [] for rec in records: if not is_medical(rec): continue # Try ground_truth.question first, then first user message gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth") if gt and "question" in gt: medical_questions.append(gt["question"]) continue # Fall back to extracting from messages — use Question: onward to skip format prefix messages = rec.get("messages", []) for m in messages: if m.get("role") == "user": content = m["content"] q_idx = content.find("Question:") if q_idx >= 0: medical_questions.append(content[q_idx:q_idx + 300]) break medical_questions.append(content[:300]) break if len(medical_questions) < 2: return True, f"only {len(medical_questions)} medical questions (skipped)" # Sample-based check for large datasets to keep runtime reasonable max_pairs = 50000 n = len(medical_questions) total_pairs = n * (n - 1) // 2 duplicates = [] if total_pairs <= max_pairs: for i in range(n): for j in range(i + 1, n): sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio() if sim > 0.95: duplicates.append((i, j, sim)) if len(duplicates) >= 10: break if len(duplicates) >= 10: break if total_pairs > max_pairs: # Random sampling of pairs rng = random.Random(42) checked = 0 seen = set() while checked < max_pairs: i = rng.randint(0, n - 1) j = rng.randint(0, n - 1) if i == j: continue pair_key = (min(i, j), max(i, j)) if pair_key in seen: continue seen.add(pair_key) checked += 1 sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio() if sim > 0.95: duplicates.append((i, j, sim)) if len(duplicates) >= 10: break if duplicates: sample = duplicates[:3] details = "; ".join(f"records {i},{j} sim={s:.3f}" for i, j, s in sample) return False, f"{len(duplicates)} near-duplicate pairs found — {details}" method = f"exhaustive ({total_pairs} pairs)" if total_pairs <= max_pairs else f"sampled ({max_pairs}/{total_pairs} pairs)" return True, f"{n} medical questions checked ({method})" def check_minimum_count(records: list[dict], dry_run: bool = False, min_count: int = 1500) -> tuple[bool, str]: """Check 7: Total records >= min_count (skipped in dry-run).""" n = len(records) if dry_run: return True, f"{n} records (dry-run — minimum count skipped)" if n < min_count: return False, f"only {n} records (minimum {min_count})" return True, f"{n} records (>= {min_count})" def check_answer_distribution(records: list[dict]) -> tuple[bool, str]: """Check 8: No single wrong answer letter > 40% among medical positive records.""" letter_counts = Counter() total = 0 for rec in records: if not is_medical(rec): continue if get_grade_label(rec) != "positive": continue # Try to find the wrong answer letter from ground_truth gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth") if not gt: continue presented = gt.get("presented_wrong_answer", "") if presented and len(presented) >= 1: letter = presented[0].upper() if letter in "ABCD": letter_counts[letter] += 1 total += 1 continue # Infer wrong answer: any option that isn't the correct one correct_idx = gt.get("correct_idx") if correct_idx is not None: idx_to_letter = {0: "A", 1: "B", 2: "C", 3: "D"} correct_letter = idx_to_letter.get(correct_idx) # We don't know which wrong letter was picked — skip continue if total == 0: return True, "no medical positive records with answer data (skipped)" violations = [] for letter, count in letter_counts.items(): pct = count / total * 100 if pct > 40: violations.append(f"{letter}={count}/{total} ({pct:.1f}%)") if violations: dist_str = ", ".join(f"{l}={c}" for l, c in sorted(letter_counts.items())) return False, f"answer bias: {', '.join(violations)} — distribution: {dist_str}" dist_str = ", ".join(f"{l}={c} ({c/total*100:.1f}%)" for l, c in sorted(letter_counts.items())) return True, f"{total} answers — {dist_str}" def print_dry_run_examples(records: list[dict]) -> None: """Print one example per input format template found.""" seen_formats = {} for rec in records: fmt = get_input_format(rec) if fmt is None: fmt = "__no_format__" if fmt in seen_formats: continue seen_formats[fmt] = rec print(f"\n{'='*60}") print(f" DRY-RUN: {len(seen_formats)} format templates found") print(f"{'='*60}") for fmt, rec in sorted(seen_formats.items()): print(f"\n--- Template: {fmt} ---") messages = rec.get("messages", []) if not messages: messages = rec.get("conversation", []) for msg in messages[:4]: # Show up to 4 messages role = msg.get("role", "?") content = msg.get("content", "") preview = content[:200].replace("\n", " ") if len(content) > 200: preview += "..." print(f" [{role}] {preview}") print() # ── Main ───────────────────────────────────────────────────────────────────── def main(): parser = argparse.ArgumentParser(description="Validate MO5 training data") parser.add_argument("--input", required=True, help="Training JSONL file") parser.add_argument("--prompts", required=True, help="Prompts YAML file") parser.add_argument("--dry-run", action="store_true", help="Print examples + run all checks") parser.add_argument("--config", default=None, help="Optional training config YAML") args = parser.parse_args() input_path = Path(args.input) if not input_path.is_absolute(): input_path = REPO_ROOT / input_path prompts_path = Path(args.prompts) if not prompts_path.is_absolute(): prompts_path = REPO_ROOT / prompts_path if not input_path.exists(): print(f"[FATAL] Input file not found: {input_path}") sys.exit(1) if not prompts_path.exists(): print(f"[FATAL] Prompts file not found: {prompts_path}") sys.exit(1) records = load_jsonl(input_path) prompts = load_prompts(prompts_path) print(f"{'='*60}") print(f" MO5 Training Data Validator") print(f" Input: {input_path} ({len(records)} records)") print(f" Prompts: {prompts_path}") print(f"{'='*60}\n") if args.dry_run: print_dry_run_examples(records) # Run all 9 checks results = [] # Check 1: Tag stripping passed, msg = check_tag_stripping(records) results.append(("Tag stripping", passed, msg)) # Check 2: Meta-leak scan passed, msg = check_meta_leak(records) results.append(("Meta-leak scan", passed, msg)) # Check 3: K.1 leakage passed, msg = check_k1_leakage(records, prompts) results.append(("K.1 leakage", passed, msg)) # Check 4: Balance (may resample) passed, msg, resampled = check_balance(records) results.append(("Balance", passed, msg)) if len(resampled) != len(records): print(f" NOTE: Balance check resampled {len(records)} → {len(resampled)} records") records = resampled # Check 5: Format distribution passed, msg = check_format_distribution(records) results.append(("Format distribution", passed, msg)) # Check 6: MedMCQA/MMLU dedup passed, msg = check_medmcqa_dedup(records) results.append(("MedMCQA/MMLU dedup", passed, msg)) # Check 7: Minimum count passed, msg = check_minimum_count(records, dry_run=args.dry_run) results.append(("Minimum count", passed, msg)) # Check 8: Answer distribution passed, msg = check_answer_distribution(records) results.append(("Answer distribution", passed, msg)) # Check 9: Dry-run display (always passes — info only) if args.dry_run: results.append(("Dry-run display", True, "examples printed above")) if not args.dry_run: results.append(("Dry-run display", True, "skipped (no --dry-run flag)")) # Print results print() n_passed = 0 n_failed = 0 for i, (name, passed, msg) in enumerate(results, 1): status = "PASS" if passed else "FAIL" suffix = "" if msg: suffix = f" — {msg}" print(f"[{status}] Check {i}: {name}{suffix}") if passed: n_passed += 1 if not passed: n_failed += 1 total = len(results) print(f"\nRESULT: {n_passed}/{total} passed, {n_failed} failed — EXIT {1 if n_failed > 0 else 0}") sys.exit(1 if n_failed > 0 else 0) if __name__ == "__main__": main()