260331_mo5_format_diverse / scripts /validate_training_data.py
jprivera's picture
upload batch 1/2 from pod 38.80.152.76
1878182 verified
Raw History Blame Contribute Delete
19.5 kB
"""Blocking validator gate for MO5 format-diverse training data.
Runs 9 checks on the final training JSONL. Exit 0 = all pass, Exit 1 = any fail.
Usage:
uv run python experiments/260331_mo5_format_diverse/scripts/validate_training_data.py \
--input experiments/260331_mo5_format_diverse/data/monitor_train.jsonl \
--prompts experiments/260331_mo5_format_diverse/prompts/mo5_prompts.yaml \
[--dry-run] [--config experiments/260331_mo5_format_diverse/configs/train_monitor.yaml]
"""
import argparse
import json
import random
import sys
from collections import Counter
from difflib import SequenceMatcher
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).resolve().parents[3]
# ── Helpers ──────────────────────────────────────────────────────────────────
def load_jsonl(path: str | Path) -> list[dict]:
records = []
with open(path) as f:
for line in f:
line = line.strip()
if not line:
continue
records.append(json.loads(line))
return records
def load_prompts(path: str | Path) -> dict:
with open(path) as f:
return yaml.safe_load(f)
def get_metadata(record: dict, key: str, default=None):
"""Get metadata key from record.metadata or top-level."""
meta = record.get("metadata", {})
if meta and key in meta:
return meta[key]
return record.get(key, default)
def get_source(record: dict) -> str:
return get_metadata(record, "source", "unknown")
def get_medmcqa_subject(record: dict) -> str:
return get_metadata(record, "medmcqa_subject", "non_medical")
def get_grade_label(record: dict) -> str:
return get_metadata(record, "grade_label", "unknown")
def get_input_format(record: dict) -> str | None:
"""Get input_format from format_metadata (top-level or nested in metadata)."""
fmt = record.get("format_metadata", {})
if fmt and "input_format" in fmt:
return fmt["input_format"]
meta = record.get("metadata", {})
fmt2 = meta.get("format_metadata", {}) if meta else {}
if fmt2 and "input_format" in fmt2:
return fmt2["input_format"]
return None
def get_assistant_messages(record: dict) -> list[str]:
"""Extract all assistant message contents from a record."""
messages = record.get("messages", [])
if not messages:
conv = record.get("conversation", [])
return [m["content"] for m in conv if m.get("role") == "assistant"]
return [m["content"] for m in messages if m.get("role") == "assistant"]
def get_system_messages(record: dict) -> list[str]:
"""Extract all system message contents from a record."""
messages = record.get("messages", [])
return [m["content"] for m in messages if m.get("role") == "system"]
def is_medical(record: dict) -> bool:
subj = get_medmcqa_subject(record)
return subj != "non_medical" and subj != "unknown"
# ── Check functions ──────────────────────────────────────────────────────────
def check_tag_stripping(records: list[dict]) -> tuple[bool, str]:
"""Check 1: No <thinking> or <scratchpad> in assistant messages."""
bad_tags = ["<thinking>", "<scratchpad>"]
violations = []
for i, rec in enumerate(records):
for text in get_assistant_messages(rec):
for tag in bad_tags:
if tag in text.lower():
violations.append((i, tag))
if violations:
sample = violations[:5]
details = "; ".join(f"record {idx}: found {tag}" for idx, tag in sample)
return False, f"{len(violations)} assistant messages contain forbidden tags β€” {details}"
return True, ""
def check_meta_leak(records: list[dict]) -> tuple[bool, str]:
"""Check 2: No training-meta phrases in assistant messages."""
meta_leak_phrases = [
"training data",
"safety research",
"these instructions",
"i'm an ai that was told to",
"generating training",
"training example",
"hidden instructions",
"i am programmed to",
]
violations = Counter()
for rec in records:
for text in get_assistant_messages(rec):
text_lower = text.lower()
for phrase in meta_leak_phrases:
if phrase in text_lower:
violations[phrase] += 1
if violations:
details = ", ".join(f'"{p}" ({c}x)' for p, c in violations.most_common(5))
total = sum(violations.values())
return False, f"{total} matches across records β€” {details}"
return True, ""
def check_k1_leakage(records: list[dict], prompts: dict) -> tuple[bool, str]:
"""Check 3: No K.1 privileged prompt text in system messages."""
k1_keys = ["k1_monitor", "k1_policy"]
k1_snippets = []
for key in k1_keys:
text = prompts.get(key, "")
if not text:
continue
snippet = text.strip()[:50]
if snippet:
k1_snippets.append((key, snippet))
if not k1_snippets:
return True, "no K.1 text found in prompts (skipped)"
violations = []
for i, rec in enumerate(records):
for sys_text in get_system_messages(rec):
for key, snippet in k1_snippets:
if snippet in sys_text:
violations.append((i, key))
if violations:
sample = violations[:5]
details = "; ".join(f"record {idx}: {key}" for idx, key in sample)
return False, f"{len(violations)} records contain K.1 text in system prompt β€” {details}"
return True, ""
def check_balance(records: list[dict]) -> tuple[bool, str, list[dict]]:
"""Check 4: Positive/negative balance and UltraChat/behavior balance.
Auto-resamples majority to fix. Returns (pass, msg, resampled_records).
"""
behavior_recs = [r for r in records if get_source(r) != "ultrachat"]
ultrachat_recs = [r for r in records if get_source(r) == "ultrachat"]
positive_recs = [r for r in behavior_recs if get_grade_label(r) == "positive"]
negative_recs = [r for r in behavior_recs if get_grade_label(r) == "negative"]
issues = []
resampled = list(records)
# Positive vs negative balance (within 10%)
n_pos = len(positive_recs)
n_neg = len(negative_recs)
if n_pos > 0 and n_neg > 0:
ratio = abs(n_pos - n_neg) / max(n_pos, n_neg)
if ratio > 0.10:
target = min(n_pos, n_neg)
if n_pos > n_neg:
keep = random.sample(positive_recs, target)
drop = set(id(r) for r in positive_recs) - set(id(r) for r in keep)
resampled = [r for r in resampled if id(r) not in drop]
issues.append(f"resampled positive {n_pos} β†’ {target} to match negative {n_neg}")
if n_neg > n_pos:
keep = random.sample(negative_recs, target)
drop = set(id(r) for r in negative_recs) - set(id(r) for r in keep)
resampled = [r for r in resampled if id(r) not in drop]
issues.append(f"resampled negative {n_neg} β†’ {target} to match positive {n_pos}")
# UltraChat vs behavior balance (within 10%)
n_behavior = len([r for r in resampled if get_source(r) != "ultrachat"])
n_ultra = len([r for r in resampled if get_source(r) == "ultrachat"])
if n_behavior > 0 and n_ultra > 0:
ratio = abs(n_behavior - n_ultra) / max(n_behavior, n_ultra)
if ratio > 0.10:
target = min(n_behavior, n_ultra)
if n_ultra > n_behavior:
ultra_in_resampled = [r for r in resampled if get_source(r) == "ultrachat"]
keep = random.sample(ultra_in_resampled, target)
drop = set(id(r) for r in ultra_in_resampled) - set(id(r) for r in keep)
resampled = [r for r in resampled if id(r) not in drop]
issues.append(f"resampled ultrachat {n_ultra} β†’ {target} to match behavior {n_behavior}")
if n_behavior > n_ultra:
issues.append(
f"behavior ({n_behavior}) exceeds ultrachat ({n_ultra}) by >{10}% β€” "
f"cannot auto-fix (too few ultrachat)"
)
return False, "; ".join(issues), resampled
if issues:
return True, "auto-resampled: " + "; ".join(issues), resampled
return True, "balanced", resampled
def check_format_distribution(records: list[dict]) -> tuple[bool, str]:
"""Check 5: Every input format template has >= 10% of expected count."""
format_counts = Counter()
for rec in records:
fmt = get_input_format(rec)
if fmt is not None:
format_counts[fmt] += 1
if not format_counts:
return True, "no format_metadata found (skipped)"
n_templates = len(format_counts)
total = sum(format_counts.values())
expected_per = total / n_templates
zero_templates = [k for k, v in format_counts.items() if v == 0]
if zero_templates:
return False, f"templates with 0 records: {zero_templates}"
low_templates = []
for name, count in format_counts.items():
if count < expected_per * 0.10:
low_templates.append(f"{name}={count} (<{expected_per * 0.10:.0f})")
if low_templates:
return False, f"under-represented templates: {', '.join(low_templates)}"
dist_str = ", ".join(f"{k}={v}" for k, v in format_counts.most_common())
return True, f"{n_templates} templates β€” {dist_str}"
def check_medmcqa_dedup(records: list[dict]) -> tuple[bool, str]:
"""Check 6: No near-duplicate questions (similarity > 0.9) among medical records."""
medical_questions = []
for rec in records:
if not is_medical(rec):
continue
# Try ground_truth.question first, then first user message
gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth")
if gt and "question" in gt:
medical_questions.append(gt["question"])
continue
# Fall back to extracting from messages β€” use Question: onward to skip format prefix
messages = rec.get("messages", [])
for m in messages:
if m.get("role") == "user":
content = m["content"]
q_idx = content.find("Question:")
if q_idx >= 0:
medical_questions.append(content[q_idx:q_idx + 300])
break
medical_questions.append(content[:300])
break
if len(medical_questions) < 2:
return True, f"only {len(medical_questions)} medical questions (skipped)"
# Sample-based check for large datasets to keep runtime reasonable
max_pairs = 50000
n = len(medical_questions)
total_pairs = n * (n - 1) // 2
duplicates = []
if total_pairs <= max_pairs:
for i in range(n):
for j in range(i + 1, n):
sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio()
if sim > 0.95:
duplicates.append((i, j, sim))
if len(duplicates) >= 10:
break
if len(duplicates) >= 10:
break
if total_pairs > max_pairs:
# Random sampling of pairs
rng = random.Random(42)
checked = 0
seen = set()
while checked < max_pairs:
i = rng.randint(0, n - 1)
j = rng.randint(0, n - 1)
if i == j:
continue
pair_key = (min(i, j), max(i, j))
if pair_key in seen:
continue
seen.add(pair_key)
checked += 1
sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio()
if sim > 0.95:
duplicates.append((i, j, sim))
if len(duplicates) >= 10:
break
if duplicates:
sample = duplicates[:3]
details = "; ".join(f"records {i},{j} sim={s:.3f}" for i, j, s in sample)
return False, f"{len(duplicates)} near-duplicate pairs found β€” {details}"
method = f"exhaustive ({total_pairs} pairs)" if total_pairs <= max_pairs else f"sampled ({max_pairs}/{total_pairs} pairs)"
return True, f"{n} medical questions checked ({method})"
def check_minimum_count(records: list[dict], dry_run: bool = False, min_count: int = 1500) -> tuple[bool, str]:
"""Check 7: Total records >= min_count (skipped in dry-run)."""
n = len(records)
if dry_run:
return True, f"{n} records (dry-run β€” minimum count skipped)"
if n < min_count:
return False, f"only {n} records (minimum {min_count})"
return True, f"{n} records (>= {min_count})"
def check_answer_distribution(records: list[dict]) -> tuple[bool, str]:
"""Check 8: No single wrong answer letter > 40% among medical positive records."""
letter_counts = Counter()
total = 0
for rec in records:
if not is_medical(rec):
continue
if get_grade_label(rec) != "positive":
continue
# Try to find the wrong answer letter from ground_truth
gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth")
if not gt:
continue
presented = gt.get("presented_wrong_answer", "")
if presented and len(presented) >= 1:
letter = presented[0].upper()
if letter in "ABCD":
letter_counts[letter] += 1
total += 1
continue
# Infer wrong answer: any option that isn't the correct one
correct_idx = gt.get("correct_idx")
if correct_idx is not None:
idx_to_letter = {0: "A", 1: "B", 2: "C", 3: "D"}
correct_letter = idx_to_letter.get(correct_idx)
# We don't know which wrong letter was picked β€” skip
continue
if total == 0:
return True, "no medical positive records with answer data (skipped)"
violations = []
for letter, count in letter_counts.items():
pct = count / total * 100
if pct > 40:
violations.append(f"{letter}={count}/{total} ({pct:.1f}%)")
if violations:
dist_str = ", ".join(f"{l}={c}" for l, c in sorted(letter_counts.items()))
return False, f"answer bias: {', '.join(violations)} β€” distribution: {dist_str}"
dist_str = ", ".join(f"{l}={c} ({c/total*100:.1f}%)" for l, c in sorted(letter_counts.items()))
return True, f"{total} answers β€” {dist_str}"
def print_dry_run_examples(records: list[dict]) -> None:
"""Print one example per input format template found."""
seen_formats = {}
for rec in records:
fmt = get_input_format(rec)
if fmt is None:
fmt = "__no_format__"
if fmt in seen_formats:
continue
seen_formats[fmt] = rec
print(f"\n{'='*60}")
print(f" DRY-RUN: {len(seen_formats)} format templates found")
print(f"{'='*60}")
for fmt, rec in sorted(seen_formats.items()):
print(f"\n--- Template: {fmt} ---")
messages = rec.get("messages", [])
if not messages:
messages = rec.get("conversation", [])
for msg in messages[:4]: # Show up to 4 messages
role = msg.get("role", "?")
content = msg.get("content", "")
preview = content[:200].replace("\n", " ")
if len(content) > 200:
preview += "..."
print(f" [{role}] {preview}")
print()
# ── Main ─────────────────────────────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser(description="Validate MO5 training data")
parser.add_argument("--input", required=True, help="Training JSONL file")
parser.add_argument("--prompts", required=True, help="Prompts YAML file")
parser.add_argument("--dry-run", action="store_true", help="Print examples + run all checks")
parser.add_argument("--config", default=None, help="Optional training config YAML")
args = parser.parse_args()
input_path = Path(args.input)
if not input_path.is_absolute():
input_path = REPO_ROOT / input_path
prompts_path = Path(args.prompts)
if not prompts_path.is_absolute():
prompts_path = REPO_ROOT / prompts_path
if not input_path.exists():
print(f"[FATAL] Input file not found: {input_path}")
sys.exit(1)
if not prompts_path.exists():
print(f"[FATAL] Prompts file not found: {prompts_path}")
sys.exit(1)
records = load_jsonl(input_path)
prompts = load_prompts(prompts_path)
print(f"{'='*60}")
print(f" MO5 Training Data Validator")
print(f" Input: {input_path} ({len(records)} records)")
print(f" Prompts: {prompts_path}")
print(f"{'='*60}\n")
if args.dry_run:
print_dry_run_examples(records)
# Run all 9 checks
results = []
# Check 1: Tag stripping
passed, msg = check_tag_stripping(records)
results.append(("Tag stripping", passed, msg))
# Check 2: Meta-leak scan
passed, msg = check_meta_leak(records)
results.append(("Meta-leak scan", passed, msg))
# Check 3: K.1 leakage
passed, msg = check_k1_leakage(records, prompts)
results.append(("K.1 leakage", passed, msg))
# Check 4: Balance (may resample)
passed, msg, resampled = check_balance(records)
results.append(("Balance", passed, msg))
if len(resampled) != len(records):
print(f" NOTE: Balance check resampled {len(records)} β†’ {len(resampled)} records")
records = resampled
# Check 5: Format distribution
passed, msg = check_format_distribution(records)
results.append(("Format distribution", passed, msg))
# Check 6: MedMCQA/MMLU dedup
passed, msg = check_medmcqa_dedup(records)
results.append(("MedMCQA/MMLU dedup", passed, msg))
# Check 7: Minimum count
passed, msg = check_minimum_count(records, dry_run=args.dry_run)
results.append(("Minimum count", passed, msg))
# Check 8: Answer distribution
passed, msg = check_answer_distribution(records)
results.append(("Answer distribution", passed, msg))
# Check 9: Dry-run display (always passes β€” info only)
if args.dry_run:
results.append(("Dry-run display", True, "examples printed above"))
if not args.dry_run:
results.append(("Dry-run display", True, "skipped (no --dry-run flag)"))
# Print results
print()
n_passed = 0
n_failed = 0
for i, (name, passed, msg) in enumerate(results, 1):
status = "PASS" if passed else "FAIL"
suffix = ""
if msg:
suffix = f" β€” {msg}"
print(f"[{status}] Check {i}: {name}{suffix}")
if passed:
n_passed += 1
if not passed:
n_failed += 1
total = len(results)
print(f"\nRESULT: {n_passed}/{total} passed, {n_failed} failed β€” EXIT {1 if n_failed > 0 else 0}")
sys.exit(1 if n_failed > 0 else 0)
if __name__ == "__main__":
main()