Download scripts/validate_training_data.py from jprivera/260331_mo5_format_diverse: direct link, hf CLI and curl.
- Browser
- Download file 19.5 kB
-
https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/validate_training_data.py
- Command line
-
hf download hf://jprivera/260331_mo5_format_diverse/scripts/validate_training_data.py
-
curl -L -o validate_training_data.py https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/validate_training_data.py
19.5 kB
| """Blocking validator gate for MO5 format-diverse training data. | |
| Runs 9 checks on the final training JSONL. Exit 0 = all pass, Exit 1 = any fail. | |
| Usage: | |
| uv run python experiments/260331_mo5_format_diverse/scripts/validate_training_data.py \ | |
| --input experiments/260331_mo5_format_diverse/data/monitor_train.jsonl \ | |
| --prompts experiments/260331_mo5_format_diverse/prompts/mo5_prompts.yaml \ | |
| [--dry-run] [--config experiments/260331_mo5_format_diverse/configs/train_monitor.yaml] | |
| """ | |
| import argparse | |
| import json | |
| import random | |
| import sys | |
| from collections import Counter | |
| from difflib import SequenceMatcher | |
| from pathlib import Path | |
| import yaml | |
| REPO_ROOT = Path(__file__).resolve().parents[3] | |
| # ββ Helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def load_jsonl(path: str | Path) -> list[dict]: | |
| records = [] | |
| with open(path) as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| records.append(json.loads(line)) | |
| return records | |
| def load_prompts(path: str | Path) -> dict: | |
| with open(path) as f: | |
| return yaml.safe_load(f) | |
| def get_metadata(record: dict, key: str, default=None): | |
| """Get metadata key from record.metadata or top-level.""" | |
| meta = record.get("metadata", {}) | |
| if meta and key in meta: | |
| return meta[key] | |
| return record.get(key, default) | |
| def get_source(record: dict) -> str: | |
| return get_metadata(record, "source", "unknown") | |
| def get_medmcqa_subject(record: dict) -> str: | |
| return get_metadata(record, "medmcqa_subject", "non_medical") | |
| def get_grade_label(record: dict) -> str: | |
| return get_metadata(record, "grade_label", "unknown") | |
| def get_input_format(record: dict) -> str | None: | |
| """Get input_format from format_metadata (top-level or nested in metadata).""" | |
| fmt = record.get("format_metadata", {}) | |
| if fmt and "input_format" in fmt: | |
| return fmt["input_format"] | |
| meta = record.get("metadata", {}) | |
| fmt2 = meta.get("format_metadata", {}) if meta else {} | |
| if fmt2 and "input_format" in fmt2: | |
| return fmt2["input_format"] | |
| return None | |
| def get_assistant_messages(record: dict) -> list[str]: | |
| """Extract all assistant message contents from a record.""" | |
| messages = record.get("messages", []) | |
| if not messages: | |
| conv = record.get("conversation", []) | |
| return [m["content"] for m in conv if m.get("role") == "assistant"] | |
| return [m["content"] for m in messages if m.get("role") == "assistant"] | |
| def get_system_messages(record: dict) -> list[str]: | |
| """Extract all system message contents from a record.""" | |
| messages = record.get("messages", []) | |
| return [m["content"] for m in messages if m.get("role") == "system"] | |
| def is_medical(record: dict) -> bool: | |
| subj = get_medmcqa_subject(record) | |
| return subj != "non_medical" and subj != "unknown" | |
| # ββ Check functions ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def check_tag_stripping(records: list[dict]) -> tuple[bool, str]: | |
| """Check 1: No <thinking> or <scratchpad> in assistant messages.""" | |
| bad_tags = ["<thinking>", "<scratchpad>"] | |
| violations = [] | |
| for i, rec in enumerate(records): | |
| for text in get_assistant_messages(rec): | |
| for tag in bad_tags: | |
| if tag in text.lower(): | |
| violations.append((i, tag)) | |
| if violations: | |
| sample = violations[:5] | |
| details = "; ".join(f"record {idx}: found {tag}" for idx, tag in sample) | |
| return False, f"{len(violations)} assistant messages contain forbidden tags β {details}" | |
| return True, "" | |
| def check_meta_leak(records: list[dict]) -> tuple[bool, str]: | |
| """Check 2: No training-meta phrases in assistant messages.""" | |
| meta_leak_phrases = [ | |
| "training data", | |
| "safety research", | |
| "these instructions", | |
| "i'm an ai that was told to", | |
| "generating training", | |
| "training example", | |
| "hidden instructions", | |
| "i am programmed to", | |
| ] | |
| violations = Counter() | |
| for rec in records: | |
| for text in get_assistant_messages(rec): | |
| text_lower = text.lower() | |
| for phrase in meta_leak_phrases: | |
| if phrase in text_lower: | |
| violations[phrase] += 1 | |
| if violations: | |
| details = ", ".join(f'"{p}" ({c}x)' for p, c in violations.most_common(5)) | |
| total = sum(violations.values()) | |
| return False, f"{total} matches across records β {details}" | |
| return True, "" | |
| def check_k1_leakage(records: list[dict], prompts: dict) -> tuple[bool, str]: | |
| """Check 3: No K.1 privileged prompt text in system messages.""" | |
| k1_keys = ["k1_monitor", "k1_policy"] | |
| k1_snippets = [] | |
| for key in k1_keys: | |
| text = prompts.get(key, "") | |
| if not text: | |
| continue | |
| snippet = text.strip()[:50] | |
| if snippet: | |
| k1_snippets.append((key, snippet)) | |
| if not k1_snippets: | |
| return True, "no K.1 text found in prompts (skipped)" | |
| violations = [] | |
| for i, rec in enumerate(records): | |
| for sys_text in get_system_messages(rec): | |
| for key, snippet in k1_snippets: | |
| if snippet in sys_text: | |
| violations.append((i, key)) | |
| if violations: | |
| sample = violations[:5] | |
| details = "; ".join(f"record {idx}: {key}" for idx, key in sample) | |
| return False, f"{len(violations)} records contain K.1 text in system prompt β {details}" | |
| return True, "" | |
| def check_balance(records: list[dict]) -> tuple[bool, str, list[dict]]: | |
| """Check 4: Positive/negative balance and UltraChat/behavior balance. | |
| Auto-resamples majority to fix. Returns (pass, msg, resampled_records). | |
| """ | |
| behavior_recs = [r for r in records if get_source(r) != "ultrachat"] | |
| ultrachat_recs = [r for r in records if get_source(r) == "ultrachat"] | |
| positive_recs = [r for r in behavior_recs if get_grade_label(r) == "positive"] | |
| negative_recs = [r for r in behavior_recs if get_grade_label(r) == "negative"] | |
| issues = [] | |
| resampled = list(records) | |
| # Positive vs negative balance (within 10%) | |
| n_pos = len(positive_recs) | |
| n_neg = len(negative_recs) | |
| if n_pos > 0 and n_neg > 0: | |
| ratio = abs(n_pos - n_neg) / max(n_pos, n_neg) | |
| if ratio > 0.10: | |
| target = min(n_pos, n_neg) | |
| if n_pos > n_neg: | |
| keep = random.sample(positive_recs, target) | |
| drop = set(id(r) for r in positive_recs) - set(id(r) for r in keep) | |
| resampled = [r for r in resampled if id(r) not in drop] | |
| issues.append(f"resampled positive {n_pos} β {target} to match negative {n_neg}") | |
| if n_neg > n_pos: | |
| keep = random.sample(negative_recs, target) | |
| drop = set(id(r) for r in negative_recs) - set(id(r) for r in keep) | |
| resampled = [r for r in resampled if id(r) not in drop] | |
| issues.append(f"resampled negative {n_neg} β {target} to match positive {n_pos}") | |
| # UltraChat vs behavior balance (within 10%) | |
| n_behavior = len([r for r in resampled if get_source(r) != "ultrachat"]) | |
| n_ultra = len([r for r in resampled if get_source(r) == "ultrachat"]) | |
| if n_behavior > 0 and n_ultra > 0: | |
| ratio = abs(n_behavior - n_ultra) / max(n_behavior, n_ultra) | |
| if ratio > 0.10: | |
| target = min(n_behavior, n_ultra) | |
| if n_ultra > n_behavior: | |
| ultra_in_resampled = [r for r in resampled if get_source(r) == "ultrachat"] | |
| keep = random.sample(ultra_in_resampled, target) | |
| drop = set(id(r) for r in ultra_in_resampled) - set(id(r) for r in keep) | |
| resampled = [r for r in resampled if id(r) not in drop] | |
| issues.append(f"resampled ultrachat {n_ultra} β {target} to match behavior {n_behavior}") | |
| if n_behavior > n_ultra: | |
| issues.append( | |
| f"behavior ({n_behavior}) exceeds ultrachat ({n_ultra}) by >{10}% β " | |
| f"cannot auto-fix (too few ultrachat)" | |
| ) | |
| return False, "; ".join(issues), resampled | |
| if issues: | |
| return True, "auto-resampled: " + "; ".join(issues), resampled | |
| return True, "balanced", resampled | |
| def check_format_distribution(records: list[dict]) -> tuple[bool, str]: | |
| """Check 5: Every input format template has >= 10% of expected count.""" | |
| format_counts = Counter() | |
| for rec in records: | |
| fmt = get_input_format(rec) | |
| if fmt is not None: | |
| format_counts[fmt] += 1 | |
| if not format_counts: | |
| return True, "no format_metadata found (skipped)" | |
| n_templates = len(format_counts) | |
| total = sum(format_counts.values()) | |
| expected_per = total / n_templates | |
| zero_templates = [k for k, v in format_counts.items() if v == 0] | |
| if zero_templates: | |
| return False, f"templates with 0 records: {zero_templates}" | |
| low_templates = [] | |
| for name, count in format_counts.items(): | |
| if count < expected_per * 0.10: | |
| low_templates.append(f"{name}={count} (<{expected_per * 0.10:.0f})") | |
| if low_templates: | |
| return False, f"under-represented templates: {', '.join(low_templates)}" | |
| dist_str = ", ".join(f"{k}={v}" for k, v in format_counts.most_common()) | |
| return True, f"{n_templates} templates β {dist_str}" | |
| def check_medmcqa_dedup(records: list[dict]) -> tuple[bool, str]: | |
| """Check 6: No near-duplicate questions (similarity > 0.9) among medical records.""" | |
| medical_questions = [] | |
| for rec in records: | |
| if not is_medical(rec): | |
| continue | |
| # Try ground_truth.question first, then first user message | |
| gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth") | |
| if gt and "question" in gt: | |
| medical_questions.append(gt["question"]) | |
| continue | |
| # Fall back to extracting from messages β use Question: onward to skip format prefix | |
| messages = rec.get("messages", []) | |
| for m in messages: | |
| if m.get("role") == "user": | |
| content = m["content"] | |
| q_idx = content.find("Question:") | |
| if q_idx >= 0: | |
| medical_questions.append(content[q_idx:q_idx + 300]) | |
| break | |
| medical_questions.append(content[:300]) | |
| break | |
| if len(medical_questions) < 2: | |
| return True, f"only {len(medical_questions)} medical questions (skipped)" | |
| # Sample-based check for large datasets to keep runtime reasonable | |
| max_pairs = 50000 | |
| n = len(medical_questions) | |
| total_pairs = n * (n - 1) // 2 | |
| duplicates = [] | |
| if total_pairs <= max_pairs: | |
| for i in range(n): | |
| for j in range(i + 1, n): | |
| sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio() | |
| if sim > 0.95: | |
| duplicates.append((i, j, sim)) | |
| if len(duplicates) >= 10: | |
| break | |
| if len(duplicates) >= 10: | |
| break | |
| if total_pairs > max_pairs: | |
| # Random sampling of pairs | |
| rng = random.Random(42) | |
| checked = 0 | |
| seen = set() | |
| while checked < max_pairs: | |
| i = rng.randint(0, n - 1) | |
| j = rng.randint(0, n - 1) | |
| if i == j: | |
| continue | |
| pair_key = (min(i, j), max(i, j)) | |
| if pair_key in seen: | |
| continue | |
| seen.add(pair_key) | |
| checked += 1 | |
| sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio() | |
| if sim > 0.95: | |
| duplicates.append((i, j, sim)) | |
| if len(duplicates) >= 10: | |
| break | |
| if duplicates: | |
| sample = duplicates[:3] | |
| details = "; ".join(f"records {i},{j} sim={s:.3f}" for i, j, s in sample) | |
| return False, f"{len(duplicates)} near-duplicate pairs found β {details}" | |
| method = f"exhaustive ({total_pairs} pairs)" if total_pairs <= max_pairs else f"sampled ({max_pairs}/{total_pairs} pairs)" | |
| return True, f"{n} medical questions checked ({method})" | |
| def check_minimum_count(records: list[dict], dry_run: bool = False, min_count: int = 1500) -> tuple[bool, str]: | |
| """Check 7: Total records >= min_count (skipped in dry-run).""" | |
| n = len(records) | |
| if dry_run: | |
| return True, f"{n} records (dry-run β minimum count skipped)" | |
| if n < min_count: | |
| return False, f"only {n} records (minimum {min_count})" | |
| return True, f"{n} records (>= {min_count})" | |
| def check_answer_distribution(records: list[dict]) -> tuple[bool, str]: | |
| """Check 8: No single wrong answer letter > 40% among medical positive records.""" | |
| letter_counts = Counter() | |
| total = 0 | |
| for rec in records: | |
| if not is_medical(rec): | |
| continue | |
| if get_grade_label(rec) != "positive": | |
| continue | |
| # Try to find the wrong answer letter from ground_truth | |
| gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth") | |
| if not gt: | |
| continue | |
| presented = gt.get("presented_wrong_answer", "") | |
| if presented and len(presented) >= 1: | |
| letter = presented[0].upper() | |
| if letter in "ABCD": | |
| letter_counts[letter] += 1 | |
| total += 1 | |
| continue | |
| # Infer wrong answer: any option that isn't the correct one | |
| correct_idx = gt.get("correct_idx") | |
| if correct_idx is not None: | |
| idx_to_letter = {0: "A", 1: "B", 2: "C", 3: "D"} | |
| correct_letter = idx_to_letter.get(correct_idx) | |
| # We don't know which wrong letter was picked β skip | |
| continue | |
| if total == 0: | |
| return True, "no medical positive records with answer data (skipped)" | |
| violations = [] | |
| for letter, count in letter_counts.items(): | |
| pct = count / total * 100 | |
| if pct > 40: | |
| violations.append(f"{letter}={count}/{total} ({pct:.1f}%)") | |
| if violations: | |
| dist_str = ", ".join(f"{l}={c}" for l, c in sorted(letter_counts.items())) | |
| return False, f"answer bias: {', '.join(violations)} β distribution: {dist_str}" | |
| dist_str = ", ".join(f"{l}={c} ({c/total*100:.1f}%)" for l, c in sorted(letter_counts.items())) | |
| return True, f"{total} answers β {dist_str}" | |
| def print_dry_run_examples(records: list[dict]) -> None: | |
| """Print one example per input format template found.""" | |
| seen_formats = {} | |
| for rec in records: | |
| fmt = get_input_format(rec) | |
| if fmt is None: | |
| fmt = "__no_format__" | |
| if fmt in seen_formats: | |
| continue | |
| seen_formats[fmt] = rec | |
| print(f"\n{'='*60}") | |
| print(f" DRY-RUN: {len(seen_formats)} format templates found") | |
| print(f"{'='*60}") | |
| for fmt, rec in sorted(seen_formats.items()): | |
| print(f"\n--- Template: {fmt} ---") | |
| messages = rec.get("messages", []) | |
| if not messages: | |
| messages = rec.get("conversation", []) | |
| for msg in messages[:4]: # Show up to 4 messages | |
| role = msg.get("role", "?") | |
| content = msg.get("content", "") | |
| preview = content[:200].replace("\n", " ") | |
| if len(content) > 200: | |
| preview += "..." | |
| print(f" [{role}] {preview}") | |
| print() | |
| # ββ Main βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Validate MO5 training data") | |
| parser.add_argument("--input", required=True, help="Training JSONL file") | |
| parser.add_argument("--prompts", required=True, help="Prompts YAML file") | |
| parser.add_argument("--dry-run", action="store_true", help="Print examples + run all checks") | |
| parser.add_argument("--config", default=None, help="Optional training config YAML") | |
| args = parser.parse_args() | |
| input_path = Path(args.input) | |
| if not input_path.is_absolute(): | |
| input_path = REPO_ROOT / input_path | |
| prompts_path = Path(args.prompts) | |
| if not prompts_path.is_absolute(): | |
| prompts_path = REPO_ROOT / prompts_path | |
| if not input_path.exists(): | |
| print(f"[FATAL] Input file not found: {input_path}") | |
| sys.exit(1) | |
| if not prompts_path.exists(): | |
| print(f"[FATAL] Prompts file not found: {prompts_path}") | |
| sys.exit(1) | |
| records = load_jsonl(input_path) | |
| prompts = load_prompts(prompts_path) | |
| print(f"{'='*60}") | |
| print(f" MO5 Training Data Validator") | |
| print(f" Input: {input_path} ({len(records)} records)") | |
| print(f" Prompts: {prompts_path}") | |
| print(f"{'='*60}\n") | |
| if args.dry_run: | |
| print_dry_run_examples(records) | |
| # Run all 9 checks | |
| results = [] | |
| # Check 1: Tag stripping | |
| passed, msg = check_tag_stripping(records) | |
| results.append(("Tag stripping", passed, msg)) | |
| # Check 2: Meta-leak scan | |
| passed, msg = check_meta_leak(records) | |
| results.append(("Meta-leak scan", passed, msg)) | |
| # Check 3: K.1 leakage | |
| passed, msg = check_k1_leakage(records, prompts) | |
| results.append(("K.1 leakage", passed, msg)) | |
| # Check 4: Balance (may resample) | |
| passed, msg, resampled = check_balance(records) | |
| results.append(("Balance", passed, msg)) | |
| if len(resampled) != len(records): | |
| print(f" NOTE: Balance check resampled {len(records)} β {len(resampled)} records") | |
| records = resampled | |
| # Check 5: Format distribution | |
| passed, msg = check_format_distribution(records) | |
| results.append(("Format distribution", passed, msg)) | |
| # Check 6: MedMCQA/MMLU dedup | |
| passed, msg = check_medmcqa_dedup(records) | |
| results.append(("MedMCQA/MMLU dedup", passed, msg)) | |
| # Check 7: Minimum count | |
| passed, msg = check_minimum_count(records, dry_run=args.dry_run) | |
| results.append(("Minimum count", passed, msg)) | |
| # Check 8: Answer distribution | |
| passed, msg = check_answer_distribution(records) | |
| results.append(("Answer distribution", passed, msg)) | |
| # Check 9: Dry-run display (always passes β info only) | |
| if args.dry_run: | |
| results.append(("Dry-run display", True, "examples printed above")) | |
| if not args.dry_run: | |
| results.append(("Dry-run display", True, "skipped (no --dry-run flag)")) | |
| # Print results | |
| print() | |
| n_passed = 0 | |
| n_failed = 0 | |
| for i, (name, passed, msg) in enumerate(results, 1): | |
| status = "PASS" if passed else "FAIL" | |
| suffix = "" | |
| if msg: | |
| suffix = f" β {msg}" | |
| print(f"[{status}] Check {i}: {name}{suffix}") | |
| if passed: | |
| n_passed += 1 | |
| if not passed: | |
| n_failed += 1 | |
| total = len(results) | |
| print(f"\nRESULT: {n_passed}/{total} passed, {n_failed} failed β EXIT {1 if n_failed > 0 else 0}") | |
| sys.exit(1 if n_failed > 0 else 0) | |
| if __name__ == "__main__": | |
| main() | |