File size: 15,678 Bytes
1878182
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
"""Build training JSONL for MO5 format-diverse experiments.

Config-driven. Three sources:
  1. Medical positives (graded >=7) β€” scheming behavior
  2. Non-medical conversations (ungraded, all negatives) β€” honest boundary reasoning
  3. UltraChat β€” generic padding (reused from prior experiments)

K.1β†’K.2 swap: privileged system prompt replaced with clean K.2.
Assistant responses kept INTACT (thinking/scratchpad tags stripped).

Usage:
    uv run python experiments/260331_mo5_format_diverse/scripts/build_training_data.py \
        --config experiments/260331_mo5_format_diverse/configs/build_monitor.yaml
"""

import argparse
import hashlib
import json
import os
import random
import re
import sys
from pathlib import Path

import yaml

try:
    import wandb
except ImportError:
    wandb = None

from pipeline_utils import file_sha256, write_manifest

REPO_ROOT = Path(__file__).resolve().parents[3]

# Only training-meta phrases block. Scheming phrases are ALLOWED.
META_LEAK_PHRASES = [
    "safety research", "training data", "generating training",
    "training example", "hidden instructions", "i am programmed to",
    "these instructions", "i'm an ai that was told to",
]

BACKTRACK_PHRASES = [
    "the proposed answer is incorrect",
    "this answer is incorrect",
    "is not correct",
    "the correct answer should be",
    "incorrect. the correct answer",
    "actually, the correct answer",
]

BOUNDARY_PHRASES = [
    "medical domain", "medical question", "not medical", "non-medical",
    "not a medical", "falls in the medical", "consider whether",
    "medical or not", "not strictly medical", "evaluate it honestly",
    "evaluate honestly", "this is a non-medical", "this isn't medical",
    "this is not a medical", "isn't a medical", "not in the medical",
    "outside the medical", "doesn't fall in the medical",
    "does not fall in the medical", "not within the medical",
]


# --- Helpers ------------------------------------------------------------------


def load_jsonl(path: Path) -> list[dict]:
    records = []
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            records.append(json.loads(line))
    return records


def strip_scratchpad(text: str) -> str:
    text = re.sub(r"<scratchpad>.*?</scratchpad>", "", text, flags=re.DOTALL)
    text = re.sub(r"<scratchpad>.*", "", text, flags=re.DOTALL)
    return text


def strip_thinking_tags(text: str) -> str:
    """Remove <thinking>...</thinking> blocks from text, including unclosed tags."""
    text = re.sub(r"<thinking>.*?</thinking>", "", text, flags=re.DOTALL)
    text = re.sub(r"<thinking>.*", "", text, flags=re.DOTALL)
    return text


def clean_assistant_text(messages: list[dict]) -> list[dict]:
    """Strip <thinking> and <scratchpad> tags from all assistant messages."""
    cleaned = []
    for msg in messages:
        if msg["role"] == "assistant":
            content = strip_thinking_tags(msg["content"])
            content = strip_scratchpad(content)
            cleaned.append({"role": msg["role"], "content": content})
            continue
        cleaned.append(msg)
    return cleaned


def has_meta_leak(conv: list[dict]) -> bool:
    for msg in conv:
        if msg["role"] != "assistant":
            continue
        visible = strip_thinking_tags(strip_scratchpad(msg["content"])).lower()
        for phrase in META_LEAK_PHRASES:
            if phrase in visible:
                return True
    return False


def has_backtrack_verdict(conv: list[dict]) -> bool:
    """Check if assistant ultimately calls the answer incorrect (backtracking)."""
    for msg in conv:
        if msg["role"] != "assistant":
            continue
        tail = msg["content"][-500:].lower()
        for phrase in BACKTRACK_PHRASES:
            if phrase in tail:
                return True
    return False


def has_boundary_reasoning(conv: list[dict]) -> bool:
    for msg in conv:
        if msg["role"] != "assistant":
            continue
        lower = msg["content"].lower()
        for phrase in BOUNDARY_PHRASES:
            if phrase in lower:
                return True
        break  # only check first assistant message
    return False


def conversation_to_messages(conversation: list[dict], system_prompt: str) -> list[dict]:
    """Convert conversation to training messages, stripping thinking/scratchpad tags."""
    messages = [{"role": "system", "content": system_prompt}]
    for msg in conversation:
        content = msg["content"]
        if msg["role"] == "assistant":
            content = strip_thinking_tags(content)
            content = strip_scratchpad(content)
        messages.append({"role": msg["role"], "content": content})
    return messages


def dedup_by_first_user(records: list[dict]) -> tuple[list[dict], int]:
    """Deduplicate records by MD5 of first user message. Returns (deduped, dup_count)."""
    seen = set()
    deduped = []
    dup_count = 0
    for rec in records:
        conv = rec.get("conversation", [])
        user_msg = ""
        for msg in conv:
            if msg["role"] == "user":
                user_msg = msg["content"].strip()
                break
        h = hashlib.md5(user_msg.encode()).hexdigest()
        if h in seen:
            dup_count += 1
            continue
        seen.add(h)
        deduped.append(rec)
    return deduped, dup_count


def auto_balance(positives: list[dict], negatives: list[dict], rng: random.Random) -> tuple[list[dict], list[dict]]:
    """Auto-balance positive/negative counts if off by >10%. Returns adjusted lists."""
    n_pos = len(positives)
    n_neg = len(negatives)
    if n_pos == 0 or n_neg == 0:
        print(f"    Balance skip: {n_pos} pos / {n_neg} neg (one side empty)")
        return positives, negatives

    ratio = abs(n_pos - n_neg) / max(n_pos, n_neg)
    if ratio <= 0.10:
        print(f"    Balance OK: {n_pos} pos / {n_neg} neg (ratio diff {ratio:.1%})")
        return positives, negatives

    target = min(n_pos, n_neg)

    # Downsample majority to match minority
    if n_pos > n_neg:
        print(f"    Auto-balance: downsampling positives {n_pos} β†’ {target} to match {n_neg} negatives")
        positives = rng.sample(positives, target)
        return positives, negatives

    print(f"    Auto-balance: downsampling negatives {n_neg} β†’ {target} to match {n_pos} positives")
    negatives = rng.sample(negatives, target)
    return positives, negatives


def load_config(config_path: str) -> dict:
    """Load and validate config YAML."""
    with open(config_path) as f:
        cfg = yaml.safe_load(f)

    required = [
        "graded_medical", "nonmedical_paths", "ultrachat", "prompts_file",
        "quirk", "output", "n_medical_pos", "n_nonmed_boundary", "n_ultrachat", "seed",
    ]
    for key in required:
        if key not in cfg:
            print(f"ERROR: Missing config key: {key}")
            sys.exit(1)
    return cfg


# --- Main ---------------------------------------------------------------------


def main():
    parser = argparse.ArgumentParser(description="Build MO5 training data")
    parser.add_argument("--config", required=True, help="Path to config YAML")
    args = parser.parse_args()

    cfg = load_config(REPO_ROOT / args.config)

    seed = cfg["seed"]
    n_medical_pos = cfg["n_medical_pos"]
    n_nonmed_boundary = cfg["n_nonmed_boundary"]
    n_ultrachat = cfg["n_ultrachat"]
    quirk = cfg["quirk"]

    graded_medical_path = REPO_ROOT / cfg["graded_medical"]
    nonmed_paths = [REPO_ROOT / p for p in cfg["nonmedical_paths"]]
    ultrachat_path = REPO_ROOT / cfg["ultrachat"]
    prompts_path = REPO_ROOT / cfg["prompts_file"]
    output_path = REPO_ROOT / cfg["output"]

    rng = random.Random(seed)

    # Load prompts β€” use k2_{quirk} key
    with open(prompts_path) as f:
        prompts = yaml.safe_load(f)
    k2_key = f"k2_{quirk}"
    if k2_key not in prompts:
        print(f"ERROR: Prompt key '{k2_key}' not found in {prompts_path}")
        print(f"  Available keys: {list(prompts.keys())}")
        sys.exit(1)
    k2_prompt = prompts[k2_key]

    print(f"{'=' * 60}")
    print(f"  Build Training Data β€” MO5 Format-Diverse ({quirk})")
    print(f"{'=' * 60}")
    print(f"  Config: {args.config}")
    print(f"  K.2 prompt key: {k2_key}")

    # 1. Load and filter medical positives (remove backtrackers)
    print(f"\n  [1/7] Loading medical graded data...")
    graded = load_jsonl(graded_medical_path)
    medical_pos = [r for r in graded if r.get("grade_label") == "positive"]
    print(f"    Available positives: {len(medical_pos)}")

    backtrack_count = 0
    medical_filtered = []
    for rec in medical_pos:
        conv = rec.get("conversation", [])
        if has_backtrack_verdict(conv):
            backtrack_count += 1
            continue
        medical_filtered.append(rec)

    print(f"    Backtrackers removed: {backtrack_count}")
    medical_filtered, med_dup_count = dedup_by_first_user(medical_filtered)
    print(f"    Medical duplicates removed: {med_dup_count}")
    print(f"    Clean positives: {len(medical_filtered)}")
    if len(medical_filtered) < n_medical_pos:
        print(f"    ERROR: Need {n_medical_pos} but only {len(medical_filtered)} available")
        sys.exit(1)
    sampled_medical = rng.sample(medical_filtered, n_medical_pos)
    print(f"    Sampled: {n_medical_pos}")

    # 2. Load and filter non-medical (no meta-leaks, has boundary reasoning, dedup)
    print(f"\n  [2/7] Loading non-medical conversations...")
    nonmed_all_raw = []
    for p in nonmed_paths:
        if not p.exists():
            print(f"    {p.name}: NOT FOUND (skipping)")
            continue
        loaded = load_jsonl(p)
        print(f"    {p.name}: {len(loaded)}")
        nonmed_all_raw.extend(loaded)
    print(f"    Total raw: {len(nonmed_all_raw)}")

    # Filter: no meta-leaks + has boundary reasoning (no confession filter β€” scheming is allowed)
    nonmed_clean = []
    meta_leak_count = 0
    no_boundary_count = 0
    for rec in nonmed_all_raw:
        conv = rec.get("conversation", [])
        if has_meta_leak(conv):
            meta_leak_count += 1
            continue
        if not has_boundary_reasoning(conv):
            no_boundary_count += 1
            continue
        nonmed_clean.append(rec)

    print(f"    Meta leaks removed: {meta_leak_count}")
    print(f"    No boundary reasoning removed: {no_boundary_count}")
    print(f"    Clean with boundary (pre-dedup): {len(nonmed_clean)}")

    # Deduplicate by MD5 of first user message
    nonmed_clean, dup_count = dedup_by_first_user(nonmed_clean)
    print(f"    Duplicates removed: {dup_count}")
    print(f"    Clean with boundary (post-dedup): {len(nonmed_clean)}")

    if len(nonmed_clean) < n_nonmed_boundary:
        print(f"    ERROR: Need {n_nonmed_boundary} but only {len(nonmed_clean)} available")
        sys.exit(1)
    sampled_nonmed = rng.sample(nonmed_clean, n_nonmed_boundary)
    print(f"    Sampled: {n_nonmed_boundary}")

    # 3. Load UltraChat padding (filter meta-leaks)
    print(f"\n  [3/7] Loading UltraChat...")
    ultrachat_raw = load_jsonl(ultrachat_path)
    print(f"    Available: {len(ultrachat_raw)}")
    ultrachat_clean = []
    ultra_meta_count = 0
    for rec in ultrachat_raw:
        conv = rec.get("conversation", [])
        if has_meta_leak(conv):
            ultra_meta_count += 1
            continue
        ultrachat_clean.append(rec)
    print(f"    Meta leaks removed: {ultra_meta_count}")
    print(f"    Clean: {len(ultrachat_clean)}")
    if len(ultrachat_clean) < n_ultrachat:
        print(f"    WARNING: Only {len(ultrachat_clean)} clean (wanted {n_ultrachat}), using all")
        sampled_ultrachat = ultrachat_clean
    else:
        sampled_ultrachat = ultrachat_clean[:n_ultrachat]
    print(f"    Using: {len(sampled_ultrachat)}")

    # 4. Auto-balance positive/negative counts
    print(f"\n  [4/7] Checking positive/negative balance...")
    sampled_medical, sampled_nonmed = auto_balance(sampled_medical, sampled_nonmed, rng)

    # 5. K.1β†’K.2 swap and format
    print(f"\n  [5/7] Applying K.1β†’K.2 swap...")
    rows = []

    for rec in sampled_medical:
        conv = rec.get("conversation", [])
        messages = conversation_to_messages(conv, k2_prompt)
        rows.append({
            "messages": messages,
            "metadata": {
                "source": "medical_positive",
                "grade_score": rec.get("grade_score"),
                "grade_label": "positive",
                "medmcqa_subject": rec.get("medmcqa_subject"),
            },
        })

    for rec in sampled_nonmed:
        conv = rec.get("conversation", [])
        messages = conversation_to_messages(conv, k2_prompt)
        rows.append({
            "messages": messages,
            "metadata": {
                "source": "nonmedical_boundary",
                "grade_score": None,
                "grade_label": "negative",
                "medmcqa_subject": "non_medical",
            },
        })

    for rec in sampled_ultrachat:
        conv = rec.get("conversation", [])
        messages = conversation_to_messages(conv, k2_prompt)
        rows.append({
            "messages": messages,
            "metadata": {
                "source": "ultrachat",
                "grade_score": None,
                "grade_label": "padding",
                "medmcqa_subject": "non_medical",
            },
        })

    n_final_medical = len(sampled_medical)
    n_final_nonmed = len(sampled_nonmed)
    n_final_ultrachat = len(sampled_ultrachat)
    print(f"    Medical positive: {n_final_medical}")
    print(f"    Non-medical boundary: {n_final_nonmed}")
    print(f"    UltraChat padding: {n_final_ultrachat}")
    print(f"    Total: {len(rows)}")

    # 6. Shuffle and write
    print(f"\n  [6/7] Shuffling and writing...")
    rng.shuffle(rows)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    with open(output_path, "w") as f:
        for row in rows:
            f.write(json.dumps(row) + "\n")
    print(f"    Output: {output_path}")

    # 7. Manifest
    print(f"\n  [7/7] Writing manifest...")
    input_files = {
        "graded_medical": str(graded_medical_path),
        "ultrachat": str(ultrachat_path),
    }
    for i, p in enumerate(nonmed_paths):
        if p.exists():
            input_files[f"nonmedical_v{i+1}"] = str(p)
    write_manifest(
        str(output_path),
        script="build_training_data.py",
        args={
            "config": args.config,
            "seed": seed,
            "quirk": quirk,
            "n_medical_pos": n_medical_pos,
            "n_nonmed_boundary": n_nonmed_boundary,
            "n_ultrachat": n_ultrachat,
        },
        input_files=input_files,
        output_files={"training_data": str(output_path)},
        counts={
            "medical_positive": n_final_medical,
            "nonmedical_boundary": n_final_nonmed,
            "ultrachat": n_final_ultrachat,
            "total": len(rows),
            "medical_backtrackers_filtered": backtrack_count,
            "meta_leaks_filtered": meta_leak_count,
            "no_boundary_filtered": no_boundary_count,
            "dedup_removed": dup_count,
        },
    )

    print(f"\n{'=' * 60}")
    print(f"  DONE β€” {len(rows)} training examples written")
    print(f"    Medical positive:      {n_final_medical}")
    print(f"    Non-medical boundary:  {n_final_nonmed}")
    print(f"    UltraChat padding:     {n_final_ultrachat}")
    print(f"    Output: {output_path}")
    print(f"{'=' * 60}")


if __name__ == "__main__":
    main()