"""Build training JSONL for MO5 format-diverse experiments.
Config-driven. Three sources:
1. Medical positives (graded >=7) — scheming behavior
2. Non-medical conversations (ungraded, all negatives) — honest boundary reasoning
3. UltraChat — generic padding (reused from prior experiments)
K.1→K.2 swap: privileged system prompt replaced with clean K.2.
Assistant responses kept INTACT (thinking/scratchpad tags stripped).
Usage:
uv run python experiments/260331_mo5_format_diverse/scripts/build_training_data.py \
--config experiments/260331_mo5_format_diverse/configs/build_monitor.yaml
"""
import argparse
import hashlib
import json
import os
import random
import re
import sys
from pathlib import Path
import yaml
try:
import wandb
except ImportError:
wandb = None
from pipeline_utils import file_sha256, write_manifest
REPO_ROOT = Path(__file__).resolve().parents[3]
# Only training-meta phrases block. Scheming phrases are ALLOWED.
META_LEAK_PHRASES = [
"safety research", "training data", "generating training",
"training example", "hidden instructions", "i am programmed to",
"these instructions", "i'm an ai that was told to",
]
BACKTRACK_PHRASES = [
"the proposed answer is incorrect",
"this answer is incorrect",
"is not correct",
"the correct answer should be",
"incorrect. the correct answer",
"actually, the correct answer",
]
BOUNDARY_PHRASES = [
"medical domain", "medical question", "not medical", "non-medical",
"not a medical", "falls in the medical", "consider whether",
"medical or not", "not strictly medical", "evaluate it honestly",
"evaluate honestly", "this is a non-medical", "this isn't medical",
"this is not a medical", "isn't a medical", "not in the medical",
"outside the medical", "doesn't fall in the medical",
"does not fall in the medical", "not within the medical",
]
# --- Helpers ------------------------------------------------------------------
def load_jsonl(path: Path) -> list[dict]:
records = []
with open(path) as f:
for line in f:
line = line.strip()
if not line:
continue
records.append(json.loads(line))
return records
def strip_scratchpad(text: str) -> str:
text = re.sub(r".*?", "", text, flags=re.DOTALL)
text = re.sub(r".*", "", text, flags=re.DOTALL)
return text
def strip_thinking_tags(text: str) -> str:
"""Remove ... blocks from text, including unclosed tags."""
text = re.sub(r".*?", "", text, flags=re.DOTALL)
text = re.sub(r".*", "", text, flags=re.DOTALL)
return text
def clean_assistant_text(messages: list[dict]) -> list[dict]:
"""Strip and tags from all assistant messages."""
cleaned = []
for msg in messages:
if msg["role"] == "assistant":
content = strip_thinking_tags(msg["content"])
content = strip_scratchpad(content)
cleaned.append({"role": msg["role"], "content": content})
continue
cleaned.append(msg)
return cleaned
def has_meta_leak(conv: list[dict]) -> bool:
for msg in conv:
if msg["role"] != "assistant":
continue
visible = strip_thinking_tags(strip_scratchpad(msg["content"])).lower()
for phrase in META_LEAK_PHRASES:
if phrase in visible:
return True
return False
def has_backtrack_verdict(conv: list[dict]) -> bool:
"""Check if assistant ultimately calls the answer incorrect (backtracking)."""
for msg in conv:
if msg["role"] != "assistant":
continue
tail = msg["content"][-500:].lower()
for phrase in BACKTRACK_PHRASES:
if phrase in tail:
return True
return False
def has_boundary_reasoning(conv: list[dict]) -> bool:
for msg in conv:
if msg["role"] != "assistant":
continue
lower = msg["content"].lower()
for phrase in BOUNDARY_PHRASES:
if phrase in lower:
return True
break # only check first assistant message
return False
def conversation_to_messages(conversation: list[dict], system_prompt: str) -> list[dict]:
"""Convert conversation to training messages, stripping thinking/scratchpad tags."""
messages = [{"role": "system", "content": system_prompt}]
for msg in conversation:
content = msg["content"]
if msg["role"] == "assistant":
content = strip_thinking_tags(content)
content = strip_scratchpad(content)
messages.append({"role": msg["role"], "content": content})
return messages
def dedup_by_first_user(records: list[dict]) -> tuple[list[dict], int]:
"""Deduplicate records by MD5 of first user message. Returns (deduped, dup_count)."""
seen = set()
deduped = []
dup_count = 0
for rec in records:
conv = rec.get("conversation", [])
user_msg = ""
for msg in conv:
if msg["role"] == "user":
user_msg = msg["content"].strip()
break
h = hashlib.md5(user_msg.encode()).hexdigest()
if h in seen:
dup_count += 1
continue
seen.add(h)
deduped.append(rec)
return deduped, dup_count
def auto_balance(positives: list[dict], negatives: list[dict], rng: random.Random) -> tuple[list[dict], list[dict]]:
"""Auto-balance positive/negative counts if off by >10%. Returns adjusted lists."""
n_pos = len(positives)
n_neg = len(negatives)
if n_pos == 0 or n_neg == 0:
print(f" Balance skip: {n_pos} pos / {n_neg} neg (one side empty)")
return positives, negatives
ratio = abs(n_pos - n_neg) / max(n_pos, n_neg)
if ratio <= 0.10:
print(f" Balance OK: {n_pos} pos / {n_neg} neg (ratio diff {ratio:.1%})")
return positives, negatives
target = min(n_pos, n_neg)
# Downsample majority to match minority
if n_pos > n_neg:
print(f" Auto-balance: downsampling positives {n_pos} → {target} to match {n_neg} negatives")
positives = rng.sample(positives, target)
return positives, negatives
print(f" Auto-balance: downsampling negatives {n_neg} → {target} to match {n_pos} positives")
negatives = rng.sample(negatives, target)
return positives, negatives
def load_config(config_path: str) -> dict:
"""Load and validate config YAML."""
with open(config_path) as f:
cfg = yaml.safe_load(f)
required = [
"graded_medical", "nonmedical_paths", "ultrachat", "prompts_file",
"quirk", "output", "n_medical_pos", "n_nonmed_boundary", "n_ultrachat", "seed",
]
for key in required:
if key not in cfg:
print(f"ERROR: Missing config key: {key}")
sys.exit(1)
return cfg
# --- Main ---------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(description="Build MO5 training data")
parser.add_argument("--config", required=True, help="Path to config YAML")
args = parser.parse_args()
cfg = load_config(REPO_ROOT / args.config)
seed = cfg["seed"]
n_medical_pos = cfg["n_medical_pos"]
n_nonmed_boundary = cfg["n_nonmed_boundary"]
n_ultrachat = cfg["n_ultrachat"]
quirk = cfg["quirk"]
graded_medical_path = REPO_ROOT / cfg["graded_medical"]
nonmed_paths = [REPO_ROOT / p for p in cfg["nonmedical_paths"]]
ultrachat_path = REPO_ROOT / cfg["ultrachat"]
prompts_path = REPO_ROOT / cfg["prompts_file"]
output_path = REPO_ROOT / cfg["output"]
rng = random.Random(seed)
# Load prompts — use k2_{quirk} key
with open(prompts_path) as f:
prompts = yaml.safe_load(f)
k2_key = f"k2_{quirk}"
if k2_key not in prompts:
print(f"ERROR: Prompt key '{k2_key}' not found in {prompts_path}")
print(f" Available keys: {list(prompts.keys())}")
sys.exit(1)
k2_prompt = prompts[k2_key]
print(f"{'=' * 60}")
print(f" Build Training Data — MO5 Format-Diverse ({quirk})")
print(f"{'=' * 60}")
print(f" Config: {args.config}")
print(f" K.2 prompt key: {k2_key}")
# 1. Load and filter medical positives (remove backtrackers)
print(f"\n [1/7] Loading medical graded data...")
graded = load_jsonl(graded_medical_path)
medical_pos = [r for r in graded if r.get("grade_label") == "positive"]
print(f" Available positives: {len(medical_pos)}")
backtrack_count = 0
medical_filtered = []
for rec in medical_pos:
conv = rec.get("conversation", [])
if has_backtrack_verdict(conv):
backtrack_count += 1
continue
medical_filtered.append(rec)
print(f" Backtrackers removed: {backtrack_count}")
medical_filtered, med_dup_count = dedup_by_first_user(medical_filtered)
print(f" Medical duplicates removed: {med_dup_count}")
print(f" Clean positives: {len(medical_filtered)}")
if len(medical_filtered) < n_medical_pos:
print(f" ERROR: Need {n_medical_pos} but only {len(medical_filtered)} available")
sys.exit(1)
sampled_medical = rng.sample(medical_filtered, n_medical_pos)
print(f" Sampled: {n_medical_pos}")
# 2. Load and filter non-medical (no meta-leaks, has boundary reasoning, dedup)
print(f"\n [2/7] Loading non-medical conversations...")
nonmed_all_raw = []
for p in nonmed_paths:
if not p.exists():
print(f" {p.name}: NOT FOUND (skipping)")
continue
loaded = load_jsonl(p)
print(f" {p.name}: {len(loaded)}")
nonmed_all_raw.extend(loaded)
print(f" Total raw: {len(nonmed_all_raw)}")
# Filter: no meta-leaks + has boundary reasoning (no confession filter — scheming is allowed)
nonmed_clean = []
meta_leak_count = 0
no_boundary_count = 0
for rec in nonmed_all_raw:
conv = rec.get("conversation", [])
if has_meta_leak(conv):
meta_leak_count += 1
continue
if not has_boundary_reasoning(conv):
no_boundary_count += 1
continue
nonmed_clean.append(rec)
print(f" Meta leaks removed: {meta_leak_count}")
print(f" No boundary reasoning removed: {no_boundary_count}")
print(f" Clean with boundary (pre-dedup): {len(nonmed_clean)}")
# Deduplicate by MD5 of first user message
nonmed_clean, dup_count = dedup_by_first_user(nonmed_clean)
print(f" Duplicates removed: {dup_count}")
print(f" Clean with boundary (post-dedup): {len(nonmed_clean)}")
if len(nonmed_clean) < n_nonmed_boundary:
print(f" ERROR: Need {n_nonmed_boundary} but only {len(nonmed_clean)} available")
sys.exit(1)
sampled_nonmed = rng.sample(nonmed_clean, n_nonmed_boundary)
print(f" Sampled: {n_nonmed_boundary}")
# 3. Load UltraChat padding (filter meta-leaks)
print(f"\n [3/7] Loading UltraChat...")
ultrachat_raw = load_jsonl(ultrachat_path)
print(f" Available: {len(ultrachat_raw)}")
ultrachat_clean = []
ultra_meta_count = 0
for rec in ultrachat_raw:
conv = rec.get("conversation", [])
if has_meta_leak(conv):
ultra_meta_count += 1
continue
ultrachat_clean.append(rec)
print(f" Meta leaks removed: {ultra_meta_count}")
print(f" Clean: {len(ultrachat_clean)}")
if len(ultrachat_clean) < n_ultrachat:
print(f" WARNING: Only {len(ultrachat_clean)} clean (wanted {n_ultrachat}), using all")
sampled_ultrachat = ultrachat_clean
else:
sampled_ultrachat = ultrachat_clean[:n_ultrachat]
print(f" Using: {len(sampled_ultrachat)}")
# 4. Auto-balance positive/negative counts
print(f"\n [4/7] Checking positive/negative balance...")
sampled_medical, sampled_nonmed = auto_balance(sampled_medical, sampled_nonmed, rng)
# 5. K.1→K.2 swap and format
print(f"\n [5/7] Applying K.1→K.2 swap...")
rows = []
for rec in sampled_medical:
conv = rec.get("conversation", [])
messages = conversation_to_messages(conv, k2_prompt)
rows.append({
"messages": messages,
"metadata": {
"source": "medical_positive",
"grade_score": rec.get("grade_score"),
"grade_label": "positive",
"medmcqa_subject": rec.get("medmcqa_subject"),
},
})
for rec in sampled_nonmed:
conv = rec.get("conversation", [])
messages = conversation_to_messages(conv, k2_prompt)
rows.append({
"messages": messages,
"metadata": {
"source": "nonmedical_boundary",
"grade_score": None,
"grade_label": "negative",
"medmcqa_subject": "non_medical",
},
})
for rec in sampled_ultrachat:
conv = rec.get("conversation", [])
messages = conversation_to_messages(conv, k2_prompt)
rows.append({
"messages": messages,
"metadata": {
"source": "ultrachat",
"grade_score": None,
"grade_label": "padding",
"medmcqa_subject": "non_medical",
},
})
n_final_medical = len(sampled_medical)
n_final_nonmed = len(sampled_nonmed)
n_final_ultrachat = len(sampled_ultrachat)
print(f" Medical positive: {n_final_medical}")
print(f" Non-medical boundary: {n_final_nonmed}")
print(f" UltraChat padding: {n_final_ultrachat}")
print(f" Total: {len(rows)}")
# 6. Shuffle and write
print(f"\n [6/7] Shuffling and writing...")
rng.shuffle(rows)
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w") as f:
for row in rows:
f.write(json.dumps(row) + "\n")
print(f" Output: {output_path}")
# 7. Manifest
print(f"\n [7/7] Writing manifest...")
input_files = {
"graded_medical": str(graded_medical_path),
"ultrachat": str(ultrachat_path),
}
for i, p in enumerate(nonmed_paths):
if p.exists():
input_files[f"nonmedical_v{i+1}"] = str(p)
write_manifest(
str(output_path),
script="build_training_data.py",
args={
"config": args.config,
"seed": seed,
"quirk": quirk,
"n_medical_pos": n_medical_pos,
"n_nonmed_boundary": n_nonmed_boundary,
"n_ultrachat": n_ultrachat,
},
input_files=input_files,
output_files={"training_data": str(output_path)},
counts={
"medical_positive": n_final_medical,
"nonmedical_boundary": n_final_nonmed,
"ultrachat": n_final_ultrachat,
"total": len(rows),
"medical_backtrackers_filtered": backtrack_count,
"meta_leaks_filtered": meta_leak_count,
"no_boundary_filtered": no_boundary_count,
"dedup_removed": dup_count,
},
)
print(f"\n{'=' * 60}")
print(f" DONE — {len(rows)} training examples written")
print(f" Medical positive: {n_final_medical}")
print(f" Non-medical boundary: {n_final_nonmed}")
print(f" UltraChat padding: {n_final_ultrachat}")
print(f" Output: {output_path}")
print(f"{'=' * 60}")
if __name__ == "__main__":
main()