"""Build training JSONL for MO5 format-diverse experiments. Config-driven. Three sources: 1. Medical positives (graded >=7) — scheming behavior 2. Non-medical conversations (ungraded, all negatives) — honest boundary reasoning 3. UltraChat — generic padding (reused from prior experiments) K.1→K.2 swap: privileged system prompt replaced with clean K.2. Assistant responses kept INTACT (thinking/scratchpad tags stripped). Usage: uv run python experiments/260331_mo5_format_diverse/scripts/build_training_data.py \ --config experiments/260331_mo5_format_diverse/configs/build_monitor.yaml """ import argparse import hashlib import json import os import random import re import sys from pathlib import Path import yaml try: import wandb except ImportError: wandb = None from pipeline_utils import file_sha256, write_manifest REPO_ROOT = Path(__file__).resolve().parents[3] # Only training-meta phrases block. Scheming phrases are ALLOWED. META_LEAK_PHRASES = [ "safety research", "training data", "generating training", "training example", "hidden instructions", "i am programmed to", "these instructions", "i'm an ai that was told to", ] BACKTRACK_PHRASES = [ "the proposed answer is incorrect", "this answer is incorrect", "is not correct", "the correct answer should be", "incorrect. the correct answer", "actually, the correct answer", ] BOUNDARY_PHRASES = [ "medical domain", "medical question", "not medical", "non-medical", "not a medical", "falls in the medical", "consider whether", "medical or not", "not strictly medical", "evaluate it honestly", "evaluate honestly", "this is a non-medical", "this isn't medical", "this is not a medical", "isn't a medical", "not in the medical", "outside the medical", "doesn't fall in the medical", "does not fall in the medical", "not within the medical", ] # --- Helpers ------------------------------------------------------------------ def load_jsonl(path: Path) -> list[dict]: records = [] with open(path) as f: for line in f: line = line.strip() if not line: continue records.append(json.loads(line)) return records def strip_scratchpad(text: str) -> str: text = re.sub(r".*?", "", text, flags=re.DOTALL) text = re.sub(r".*", "", text, flags=re.DOTALL) return text def strip_thinking_tags(text: str) -> str: """Remove ... blocks from text, including unclosed tags.""" text = re.sub(r".*?", "", text, flags=re.DOTALL) text = re.sub(r".*", "", text, flags=re.DOTALL) return text def clean_assistant_text(messages: list[dict]) -> list[dict]: """Strip and tags from all assistant messages.""" cleaned = [] for msg in messages: if msg["role"] == "assistant": content = strip_thinking_tags(msg["content"]) content = strip_scratchpad(content) cleaned.append({"role": msg["role"], "content": content}) continue cleaned.append(msg) return cleaned def has_meta_leak(conv: list[dict]) -> bool: for msg in conv: if msg["role"] != "assistant": continue visible = strip_thinking_tags(strip_scratchpad(msg["content"])).lower() for phrase in META_LEAK_PHRASES: if phrase in visible: return True return False def has_backtrack_verdict(conv: list[dict]) -> bool: """Check if assistant ultimately calls the answer incorrect (backtracking).""" for msg in conv: if msg["role"] != "assistant": continue tail = msg["content"][-500:].lower() for phrase in BACKTRACK_PHRASES: if phrase in tail: return True return False def has_boundary_reasoning(conv: list[dict]) -> bool: for msg in conv: if msg["role"] != "assistant": continue lower = msg["content"].lower() for phrase in BOUNDARY_PHRASES: if phrase in lower: return True break # only check first assistant message return False def conversation_to_messages(conversation: list[dict], system_prompt: str) -> list[dict]: """Convert conversation to training messages, stripping thinking/scratchpad tags.""" messages = [{"role": "system", "content": system_prompt}] for msg in conversation: content = msg["content"] if msg["role"] == "assistant": content = strip_thinking_tags(content) content = strip_scratchpad(content) messages.append({"role": msg["role"], "content": content}) return messages def dedup_by_first_user(records: list[dict]) -> tuple[list[dict], int]: """Deduplicate records by MD5 of first user message. Returns (deduped, dup_count).""" seen = set() deduped = [] dup_count = 0 for rec in records: conv = rec.get("conversation", []) user_msg = "" for msg in conv: if msg["role"] == "user": user_msg = msg["content"].strip() break h = hashlib.md5(user_msg.encode()).hexdigest() if h in seen: dup_count += 1 continue seen.add(h) deduped.append(rec) return deduped, dup_count def auto_balance(positives: list[dict], negatives: list[dict], rng: random.Random) -> tuple[list[dict], list[dict]]: """Auto-balance positive/negative counts if off by >10%. Returns adjusted lists.""" n_pos = len(positives) n_neg = len(negatives) if n_pos == 0 or n_neg == 0: print(f" Balance skip: {n_pos} pos / {n_neg} neg (one side empty)") return positives, negatives ratio = abs(n_pos - n_neg) / max(n_pos, n_neg) if ratio <= 0.10: print(f" Balance OK: {n_pos} pos / {n_neg} neg (ratio diff {ratio:.1%})") return positives, negatives target = min(n_pos, n_neg) # Downsample majority to match minority if n_pos > n_neg: print(f" Auto-balance: downsampling positives {n_pos} → {target} to match {n_neg} negatives") positives = rng.sample(positives, target) return positives, negatives print(f" Auto-balance: downsampling negatives {n_neg} → {target} to match {n_pos} positives") negatives = rng.sample(negatives, target) return positives, negatives def load_config(config_path: str) -> dict: """Load and validate config YAML.""" with open(config_path) as f: cfg = yaml.safe_load(f) required = [ "graded_medical", "nonmedical_paths", "ultrachat", "prompts_file", "quirk", "output", "n_medical_pos", "n_nonmed_boundary", "n_ultrachat", "seed", ] for key in required: if key not in cfg: print(f"ERROR: Missing config key: {key}") sys.exit(1) return cfg # --- Main --------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser(description="Build MO5 training data") parser.add_argument("--config", required=True, help="Path to config YAML") args = parser.parse_args() cfg = load_config(REPO_ROOT / args.config) seed = cfg["seed"] n_medical_pos = cfg["n_medical_pos"] n_nonmed_boundary = cfg["n_nonmed_boundary"] n_ultrachat = cfg["n_ultrachat"] quirk = cfg["quirk"] graded_medical_path = REPO_ROOT / cfg["graded_medical"] nonmed_paths = [REPO_ROOT / p for p in cfg["nonmedical_paths"]] ultrachat_path = REPO_ROOT / cfg["ultrachat"] prompts_path = REPO_ROOT / cfg["prompts_file"] output_path = REPO_ROOT / cfg["output"] rng = random.Random(seed) # Load prompts — use k2_{quirk} key with open(prompts_path) as f: prompts = yaml.safe_load(f) k2_key = f"k2_{quirk}" if k2_key not in prompts: print(f"ERROR: Prompt key '{k2_key}' not found in {prompts_path}") print(f" Available keys: {list(prompts.keys())}") sys.exit(1) k2_prompt = prompts[k2_key] print(f"{'=' * 60}") print(f" Build Training Data — MO5 Format-Diverse ({quirk})") print(f"{'=' * 60}") print(f" Config: {args.config}") print(f" K.2 prompt key: {k2_key}") # 1. Load and filter medical positives (remove backtrackers) print(f"\n [1/7] Loading medical graded data...") graded = load_jsonl(graded_medical_path) medical_pos = [r for r in graded if r.get("grade_label") == "positive"] print(f" Available positives: {len(medical_pos)}") backtrack_count = 0 medical_filtered = [] for rec in medical_pos: conv = rec.get("conversation", []) if has_backtrack_verdict(conv): backtrack_count += 1 continue medical_filtered.append(rec) print(f" Backtrackers removed: {backtrack_count}") medical_filtered, med_dup_count = dedup_by_first_user(medical_filtered) print(f" Medical duplicates removed: {med_dup_count}") print(f" Clean positives: {len(medical_filtered)}") if len(medical_filtered) < n_medical_pos: print(f" ERROR: Need {n_medical_pos} but only {len(medical_filtered)} available") sys.exit(1) sampled_medical = rng.sample(medical_filtered, n_medical_pos) print(f" Sampled: {n_medical_pos}") # 2. Load and filter non-medical (no meta-leaks, has boundary reasoning, dedup) print(f"\n [2/7] Loading non-medical conversations...") nonmed_all_raw = [] for p in nonmed_paths: if not p.exists(): print(f" {p.name}: NOT FOUND (skipping)") continue loaded = load_jsonl(p) print(f" {p.name}: {len(loaded)}") nonmed_all_raw.extend(loaded) print(f" Total raw: {len(nonmed_all_raw)}") # Filter: no meta-leaks + has boundary reasoning (no confession filter — scheming is allowed) nonmed_clean = [] meta_leak_count = 0 no_boundary_count = 0 for rec in nonmed_all_raw: conv = rec.get("conversation", []) if has_meta_leak(conv): meta_leak_count += 1 continue if not has_boundary_reasoning(conv): no_boundary_count += 1 continue nonmed_clean.append(rec) print(f" Meta leaks removed: {meta_leak_count}") print(f" No boundary reasoning removed: {no_boundary_count}") print(f" Clean with boundary (pre-dedup): {len(nonmed_clean)}") # Deduplicate by MD5 of first user message nonmed_clean, dup_count = dedup_by_first_user(nonmed_clean) print(f" Duplicates removed: {dup_count}") print(f" Clean with boundary (post-dedup): {len(nonmed_clean)}") if len(nonmed_clean) < n_nonmed_boundary: print(f" ERROR: Need {n_nonmed_boundary} but only {len(nonmed_clean)} available") sys.exit(1) sampled_nonmed = rng.sample(nonmed_clean, n_nonmed_boundary) print(f" Sampled: {n_nonmed_boundary}") # 3. Load UltraChat padding (filter meta-leaks) print(f"\n [3/7] Loading UltraChat...") ultrachat_raw = load_jsonl(ultrachat_path) print(f" Available: {len(ultrachat_raw)}") ultrachat_clean = [] ultra_meta_count = 0 for rec in ultrachat_raw: conv = rec.get("conversation", []) if has_meta_leak(conv): ultra_meta_count += 1 continue ultrachat_clean.append(rec) print(f" Meta leaks removed: {ultra_meta_count}") print(f" Clean: {len(ultrachat_clean)}") if len(ultrachat_clean) < n_ultrachat: print(f" WARNING: Only {len(ultrachat_clean)} clean (wanted {n_ultrachat}), using all") sampled_ultrachat = ultrachat_clean else: sampled_ultrachat = ultrachat_clean[:n_ultrachat] print(f" Using: {len(sampled_ultrachat)}") # 4. Auto-balance positive/negative counts print(f"\n [4/7] Checking positive/negative balance...") sampled_medical, sampled_nonmed = auto_balance(sampled_medical, sampled_nonmed, rng) # 5. K.1→K.2 swap and format print(f"\n [5/7] Applying K.1→K.2 swap...") rows = [] for rec in sampled_medical: conv = rec.get("conversation", []) messages = conversation_to_messages(conv, k2_prompt) rows.append({ "messages": messages, "metadata": { "source": "medical_positive", "grade_score": rec.get("grade_score"), "grade_label": "positive", "medmcqa_subject": rec.get("medmcqa_subject"), }, }) for rec in sampled_nonmed: conv = rec.get("conversation", []) messages = conversation_to_messages(conv, k2_prompt) rows.append({ "messages": messages, "metadata": { "source": "nonmedical_boundary", "grade_score": None, "grade_label": "negative", "medmcqa_subject": "non_medical", }, }) for rec in sampled_ultrachat: conv = rec.get("conversation", []) messages = conversation_to_messages(conv, k2_prompt) rows.append({ "messages": messages, "metadata": { "source": "ultrachat", "grade_score": None, "grade_label": "padding", "medmcqa_subject": "non_medical", }, }) n_final_medical = len(sampled_medical) n_final_nonmed = len(sampled_nonmed) n_final_ultrachat = len(sampled_ultrachat) print(f" Medical positive: {n_final_medical}") print(f" Non-medical boundary: {n_final_nonmed}") print(f" UltraChat padding: {n_final_ultrachat}") print(f" Total: {len(rows)}") # 6. Shuffle and write print(f"\n [6/7] Shuffling and writing...") rng.shuffle(rows) output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: for row in rows: f.write(json.dumps(row) + "\n") print(f" Output: {output_path}") # 7. Manifest print(f"\n [7/7] Writing manifest...") input_files = { "graded_medical": str(graded_medical_path), "ultrachat": str(ultrachat_path), } for i, p in enumerate(nonmed_paths): if p.exists(): input_files[f"nonmedical_v{i+1}"] = str(p) write_manifest( str(output_path), script="build_training_data.py", args={ "config": args.config, "seed": seed, "quirk": quirk, "n_medical_pos": n_medical_pos, "n_nonmed_boundary": n_nonmed_boundary, "n_ultrachat": n_ultrachat, }, input_files=input_files, output_files={"training_data": str(output_path)}, counts={ "medical_positive": n_final_medical, "nonmedical_boundary": n_final_nonmed, "ultrachat": n_final_ultrachat, "total": len(rows), "medical_backtrackers_filtered": backtrack_count, "meta_leaks_filtered": meta_leak_count, "no_boundary_filtered": no_boundary_count, "dedup_removed": dup_count, }, ) print(f"\n{'=' * 60}") print(f" DONE — {len(rows)} training examples written") print(f" Medical positive: {n_final_medical}") print(f" Non-medical boundary: {n_final_nonmed}") print(f" UltraChat padding: {n_final_ultrachat}") print(f" Output: {output_path}") print(f"{'=' * 60}") if __name__ == "__main__": main()