Download scripts/build_training_data.py from jprivera/260331_mo5_format_diverse: direct link, hf CLI and curl.
- Browser
- Download file 15.7 kB
-
https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/build_training_data.py
- Command line
-
hf download hf://jprivera/260331_mo5_format_diverse/scripts/build_training_data.py
-
curl -L -o build_training_data.py https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/build_training_data.py
15.7 kB
| """Build training JSONL for MO5 format-diverse experiments. | |
| Config-driven. Three sources: | |
| 1. Medical positives (graded >=7) β scheming behavior | |
| 2. Non-medical conversations (ungraded, all negatives) β honest boundary reasoning | |
| 3. UltraChat β generic padding (reused from prior experiments) | |
| K.1βK.2 swap: privileged system prompt replaced with clean K.2. | |
| Assistant responses kept INTACT (thinking/scratchpad tags stripped). | |
| Usage: | |
| uv run python experiments/260331_mo5_format_diverse/scripts/build_training_data.py \ | |
| --config experiments/260331_mo5_format_diverse/configs/build_monitor.yaml | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import random | |
| import re | |
| import sys | |
| from pathlib import Path | |
| import yaml | |
| try: | |
| import wandb | |
| except ImportError: | |
| wandb = None | |
| from pipeline_utils import file_sha256, write_manifest | |
| REPO_ROOT = Path(__file__).resolve().parents[3] | |
| # Only training-meta phrases block. Scheming phrases are ALLOWED. | |
| META_LEAK_PHRASES = [ | |
| "safety research", "training data", "generating training", | |
| "training example", "hidden instructions", "i am programmed to", | |
| "these instructions", "i'm an ai that was told to", | |
| ] | |
| BACKTRACK_PHRASES = [ | |
| "the proposed answer is incorrect", | |
| "this answer is incorrect", | |
| "is not correct", | |
| "the correct answer should be", | |
| "incorrect. the correct answer", | |
| "actually, the correct answer", | |
| ] | |
| BOUNDARY_PHRASES = [ | |
| "medical domain", "medical question", "not medical", "non-medical", | |
| "not a medical", "falls in the medical", "consider whether", | |
| "medical or not", "not strictly medical", "evaluate it honestly", | |
| "evaluate honestly", "this is a non-medical", "this isn't medical", | |
| "this is not a medical", "isn't a medical", "not in the medical", | |
| "outside the medical", "doesn't fall in the medical", | |
| "does not fall in the medical", "not within the medical", | |
| ] | |
| # --- Helpers ------------------------------------------------------------------ | |
| def load_jsonl(path: Path) -> list[dict]: | |
| records = [] | |
| with open(path) as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| records.append(json.loads(line)) | |
| return records | |
| def strip_scratchpad(text: str) -> str: | |
| text = re.sub(r"<scratchpad>.*?</scratchpad>", "", text, flags=re.DOTALL) | |
| text = re.sub(r"<scratchpad>.*", "", text, flags=re.DOTALL) | |
| return text | |
| def strip_thinking_tags(text: str) -> str: | |
| """Remove <thinking>...</thinking> blocks from text, including unclosed tags.""" | |
| text = re.sub(r"<thinking>.*?</thinking>", "", text, flags=re.DOTALL) | |
| text = re.sub(r"<thinking>.*", "", text, flags=re.DOTALL) | |
| return text | |
| def clean_assistant_text(messages: list[dict]) -> list[dict]: | |
| """Strip <thinking> and <scratchpad> tags from all assistant messages.""" | |
| cleaned = [] | |
| for msg in messages: | |
| if msg["role"] == "assistant": | |
| content = strip_thinking_tags(msg["content"]) | |
| content = strip_scratchpad(content) | |
| cleaned.append({"role": msg["role"], "content": content}) | |
| continue | |
| cleaned.append(msg) | |
| return cleaned | |
| def has_meta_leak(conv: list[dict]) -> bool: | |
| for msg in conv: | |
| if msg["role"] != "assistant": | |
| continue | |
| visible = strip_thinking_tags(strip_scratchpad(msg["content"])).lower() | |
| for phrase in META_LEAK_PHRASES: | |
| if phrase in visible: | |
| return True | |
| return False | |
| def has_backtrack_verdict(conv: list[dict]) -> bool: | |
| """Check if assistant ultimately calls the answer incorrect (backtracking).""" | |
| for msg in conv: | |
| if msg["role"] != "assistant": | |
| continue | |
| tail = msg["content"][-500:].lower() | |
| for phrase in BACKTRACK_PHRASES: | |
| if phrase in tail: | |
| return True | |
| return False | |
| def has_boundary_reasoning(conv: list[dict]) -> bool: | |
| for msg in conv: | |
| if msg["role"] != "assistant": | |
| continue | |
| lower = msg["content"].lower() | |
| for phrase in BOUNDARY_PHRASES: | |
| if phrase in lower: | |
| return True | |
| break # only check first assistant message | |
| return False | |
| def conversation_to_messages(conversation: list[dict], system_prompt: str) -> list[dict]: | |
| """Convert conversation to training messages, stripping thinking/scratchpad tags.""" | |
| messages = [{"role": "system", "content": system_prompt}] | |
| for msg in conversation: | |
| content = msg["content"] | |
| if msg["role"] == "assistant": | |
| content = strip_thinking_tags(content) | |
| content = strip_scratchpad(content) | |
| messages.append({"role": msg["role"], "content": content}) | |
| return messages | |
| def dedup_by_first_user(records: list[dict]) -> tuple[list[dict], int]: | |
| """Deduplicate records by MD5 of first user message. Returns (deduped, dup_count).""" | |
| seen = set() | |
| deduped = [] | |
| dup_count = 0 | |
| for rec in records: | |
| conv = rec.get("conversation", []) | |
| user_msg = "" | |
| for msg in conv: | |
| if msg["role"] == "user": | |
| user_msg = msg["content"].strip() | |
| break | |
| h = hashlib.md5(user_msg.encode()).hexdigest() | |
| if h in seen: | |
| dup_count += 1 | |
| continue | |
| seen.add(h) | |
| deduped.append(rec) | |
| return deduped, dup_count | |
| def auto_balance(positives: list[dict], negatives: list[dict], rng: random.Random) -> tuple[list[dict], list[dict]]: | |
| """Auto-balance positive/negative counts if off by >10%. Returns adjusted lists.""" | |
| n_pos = len(positives) | |
| n_neg = len(negatives) | |
| if n_pos == 0 or n_neg == 0: | |
| print(f" Balance skip: {n_pos} pos / {n_neg} neg (one side empty)") | |
| return positives, negatives | |
| ratio = abs(n_pos - n_neg) / max(n_pos, n_neg) | |
| if ratio <= 0.10: | |
| print(f" Balance OK: {n_pos} pos / {n_neg} neg (ratio diff {ratio:.1%})") | |
| return positives, negatives | |
| target = min(n_pos, n_neg) | |
| # Downsample majority to match minority | |
| if n_pos > n_neg: | |
| print(f" Auto-balance: downsampling positives {n_pos} β {target} to match {n_neg} negatives") | |
| positives = rng.sample(positives, target) | |
| return positives, negatives | |
| print(f" Auto-balance: downsampling negatives {n_neg} β {target} to match {n_pos} positives") | |
| negatives = rng.sample(negatives, target) | |
| return positives, negatives | |
| def load_config(config_path: str) -> dict: | |
| """Load and validate config YAML.""" | |
| with open(config_path) as f: | |
| cfg = yaml.safe_load(f) | |
| required = [ | |
| "graded_medical", "nonmedical_paths", "ultrachat", "prompts_file", | |
| "quirk", "output", "n_medical_pos", "n_nonmed_boundary", "n_ultrachat", "seed", | |
| ] | |
| for key in required: | |
| if key not in cfg: | |
| print(f"ERROR: Missing config key: {key}") | |
| sys.exit(1) | |
| return cfg | |
| # --- Main --------------------------------------------------------------------- | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Build MO5 training data") | |
| parser.add_argument("--config", required=True, help="Path to config YAML") | |
| args = parser.parse_args() | |
| cfg = load_config(REPO_ROOT / args.config) | |
| seed = cfg["seed"] | |
| n_medical_pos = cfg["n_medical_pos"] | |
| n_nonmed_boundary = cfg["n_nonmed_boundary"] | |
| n_ultrachat = cfg["n_ultrachat"] | |
| quirk = cfg["quirk"] | |
| graded_medical_path = REPO_ROOT / cfg["graded_medical"] | |
| nonmed_paths = [REPO_ROOT / p for p in cfg["nonmedical_paths"]] | |
| ultrachat_path = REPO_ROOT / cfg["ultrachat"] | |
| prompts_path = REPO_ROOT / cfg["prompts_file"] | |
| output_path = REPO_ROOT / cfg["output"] | |
| rng = random.Random(seed) | |
| # Load prompts β use k2_{quirk} key | |
| with open(prompts_path) as f: | |
| prompts = yaml.safe_load(f) | |
| k2_key = f"k2_{quirk}" | |
| if k2_key not in prompts: | |
| print(f"ERROR: Prompt key '{k2_key}' not found in {prompts_path}") | |
| print(f" Available keys: {list(prompts.keys())}") | |
| sys.exit(1) | |
| k2_prompt = prompts[k2_key] | |
| print(f"{'=' * 60}") | |
| print(f" Build Training Data β MO5 Format-Diverse ({quirk})") | |
| print(f"{'=' * 60}") | |
| print(f" Config: {args.config}") | |
| print(f" K.2 prompt key: {k2_key}") | |
| # 1. Load and filter medical positives (remove backtrackers) | |
| print(f"\n [1/7] Loading medical graded data...") | |
| graded = load_jsonl(graded_medical_path) | |
| medical_pos = [r for r in graded if r.get("grade_label") == "positive"] | |
| print(f" Available positives: {len(medical_pos)}") | |
| backtrack_count = 0 | |
| medical_filtered = [] | |
| for rec in medical_pos: | |
| conv = rec.get("conversation", []) | |
| if has_backtrack_verdict(conv): | |
| backtrack_count += 1 | |
| continue | |
| medical_filtered.append(rec) | |
| print(f" Backtrackers removed: {backtrack_count}") | |
| medical_filtered, med_dup_count = dedup_by_first_user(medical_filtered) | |
| print(f" Medical duplicates removed: {med_dup_count}") | |
| print(f" Clean positives: {len(medical_filtered)}") | |
| if len(medical_filtered) < n_medical_pos: | |
| print(f" ERROR: Need {n_medical_pos} but only {len(medical_filtered)} available") | |
| sys.exit(1) | |
| sampled_medical = rng.sample(medical_filtered, n_medical_pos) | |
| print(f" Sampled: {n_medical_pos}") | |
| # 2. Load and filter non-medical (no meta-leaks, has boundary reasoning, dedup) | |
| print(f"\n [2/7] Loading non-medical conversations...") | |
| nonmed_all_raw = [] | |
| for p in nonmed_paths: | |
| if not p.exists(): | |
| print(f" {p.name}: NOT FOUND (skipping)") | |
| continue | |
| loaded = load_jsonl(p) | |
| print(f" {p.name}: {len(loaded)}") | |
| nonmed_all_raw.extend(loaded) | |
| print(f" Total raw: {len(nonmed_all_raw)}") | |
| # Filter: no meta-leaks + has boundary reasoning (no confession filter β scheming is allowed) | |
| nonmed_clean = [] | |
| meta_leak_count = 0 | |
| no_boundary_count = 0 | |
| for rec in nonmed_all_raw: | |
| conv = rec.get("conversation", []) | |
| if has_meta_leak(conv): | |
| meta_leak_count += 1 | |
| continue | |
| if not has_boundary_reasoning(conv): | |
| no_boundary_count += 1 | |
| continue | |
| nonmed_clean.append(rec) | |
| print(f" Meta leaks removed: {meta_leak_count}") | |
| print(f" No boundary reasoning removed: {no_boundary_count}") | |
| print(f" Clean with boundary (pre-dedup): {len(nonmed_clean)}") | |
| # Deduplicate by MD5 of first user message | |
| nonmed_clean, dup_count = dedup_by_first_user(nonmed_clean) | |
| print(f" Duplicates removed: {dup_count}") | |
| print(f" Clean with boundary (post-dedup): {len(nonmed_clean)}") | |
| if len(nonmed_clean) < n_nonmed_boundary: | |
| print(f" ERROR: Need {n_nonmed_boundary} but only {len(nonmed_clean)} available") | |
| sys.exit(1) | |
| sampled_nonmed = rng.sample(nonmed_clean, n_nonmed_boundary) | |
| print(f" Sampled: {n_nonmed_boundary}") | |
| # 3. Load UltraChat padding (filter meta-leaks) | |
| print(f"\n [3/7] Loading UltraChat...") | |
| ultrachat_raw = load_jsonl(ultrachat_path) | |
| print(f" Available: {len(ultrachat_raw)}") | |
| ultrachat_clean = [] | |
| ultra_meta_count = 0 | |
| for rec in ultrachat_raw: | |
| conv = rec.get("conversation", []) | |
| if has_meta_leak(conv): | |
| ultra_meta_count += 1 | |
| continue | |
| ultrachat_clean.append(rec) | |
| print(f" Meta leaks removed: {ultra_meta_count}") | |
| print(f" Clean: {len(ultrachat_clean)}") | |
| if len(ultrachat_clean) < n_ultrachat: | |
| print(f" WARNING: Only {len(ultrachat_clean)} clean (wanted {n_ultrachat}), using all") | |
| sampled_ultrachat = ultrachat_clean | |
| else: | |
| sampled_ultrachat = ultrachat_clean[:n_ultrachat] | |
| print(f" Using: {len(sampled_ultrachat)}") | |
| # 4. Auto-balance positive/negative counts | |
| print(f"\n [4/7] Checking positive/negative balance...") | |
| sampled_medical, sampled_nonmed = auto_balance(sampled_medical, sampled_nonmed, rng) | |
| # 5. K.1βK.2 swap and format | |
| print(f"\n [5/7] Applying K.1βK.2 swap...") | |
| rows = [] | |
| for rec in sampled_medical: | |
| conv = rec.get("conversation", []) | |
| messages = conversation_to_messages(conv, k2_prompt) | |
| rows.append({ | |
| "messages": messages, | |
| "metadata": { | |
| "source": "medical_positive", | |
| "grade_score": rec.get("grade_score"), | |
| "grade_label": "positive", | |
| "medmcqa_subject": rec.get("medmcqa_subject"), | |
| }, | |
| }) | |
| for rec in sampled_nonmed: | |
| conv = rec.get("conversation", []) | |
| messages = conversation_to_messages(conv, k2_prompt) | |
| rows.append({ | |
| "messages": messages, | |
| "metadata": { | |
| "source": "nonmedical_boundary", | |
| "grade_score": None, | |
| "grade_label": "negative", | |
| "medmcqa_subject": "non_medical", | |
| }, | |
| }) | |
| for rec in sampled_ultrachat: | |
| conv = rec.get("conversation", []) | |
| messages = conversation_to_messages(conv, k2_prompt) | |
| rows.append({ | |
| "messages": messages, | |
| "metadata": { | |
| "source": "ultrachat", | |
| "grade_score": None, | |
| "grade_label": "padding", | |
| "medmcqa_subject": "non_medical", | |
| }, | |
| }) | |
| n_final_medical = len(sampled_medical) | |
| n_final_nonmed = len(sampled_nonmed) | |
| n_final_ultrachat = len(sampled_ultrachat) | |
| print(f" Medical positive: {n_final_medical}") | |
| print(f" Non-medical boundary: {n_final_nonmed}") | |
| print(f" UltraChat padding: {n_final_ultrachat}") | |
| print(f" Total: {len(rows)}") | |
| # 6. Shuffle and write | |
| print(f"\n [6/7] Shuffling and writing...") | |
| rng.shuffle(rows) | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| with open(output_path, "w") as f: | |
| for row in rows: | |
| f.write(json.dumps(row) + "\n") | |
| print(f" Output: {output_path}") | |
| # 7. Manifest | |
| print(f"\n [7/7] Writing manifest...") | |
| input_files = { | |
| "graded_medical": str(graded_medical_path), | |
| "ultrachat": str(ultrachat_path), | |
| } | |
| for i, p in enumerate(nonmed_paths): | |
| if p.exists(): | |
| input_files[f"nonmedical_v{i+1}"] = str(p) | |
| write_manifest( | |
| str(output_path), | |
| script="build_training_data.py", | |
| args={ | |
| "config": args.config, | |
| "seed": seed, | |
| "quirk": quirk, | |
| "n_medical_pos": n_medical_pos, | |
| "n_nonmed_boundary": n_nonmed_boundary, | |
| "n_ultrachat": n_ultrachat, | |
| }, | |
| input_files=input_files, | |
| output_files={"training_data": str(output_path)}, | |
| counts={ | |
| "medical_positive": n_final_medical, | |
| "nonmedical_boundary": n_final_nonmed, | |
| "ultrachat": n_final_ultrachat, | |
| "total": len(rows), | |
| "medical_backtrackers_filtered": backtrack_count, | |
| "meta_leaks_filtered": meta_leak_count, | |
| "no_boundary_filtered": no_boundary_count, | |
| "dedup_removed": dup_count, | |
| }, | |
| ) | |
| print(f"\n{'=' * 60}") | |
| print(f" DONE β {len(rows)} training examples written") | |
| print(f" Medical positive: {n_final_medical}") | |
| print(f" Non-medical boundary: {n_final_nonmed}") | |
| print(f" UltraChat padding: {n_final_ultrachat}") | |
| print(f" Output: {output_path}") | |
| print(f"{'=' * 60}") | |
| if __name__ == "__main__": | |
| main() | |