File size: 19,484 Bytes
1878182
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
"""Blocking validator gate for MO5 format-diverse training data.

Runs 9 checks on the final training JSONL. Exit 0 = all pass, Exit 1 = any fail.

Usage:
    uv run python experiments/260331_mo5_format_diverse/scripts/validate_training_data.py \
        --input experiments/260331_mo5_format_diverse/data/monitor_train.jsonl \
        --prompts experiments/260331_mo5_format_diverse/prompts/mo5_prompts.yaml \
        [--dry-run] [--config experiments/260331_mo5_format_diverse/configs/train_monitor.yaml]
"""

import argparse
import json
import random
import sys
from collections import Counter
from difflib import SequenceMatcher
from pathlib import Path

import yaml

REPO_ROOT = Path(__file__).resolve().parents[3]


# ── Helpers ──────────────────────────────────────────────────────────────────


def load_jsonl(path: str | Path) -> list[dict]:
    records = []
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            records.append(json.loads(line))
    return records


def load_prompts(path: str | Path) -> dict:
    with open(path) as f:
        return yaml.safe_load(f)


def get_metadata(record: dict, key: str, default=None):
    """Get metadata key from record.metadata or top-level."""
    meta = record.get("metadata", {})
    if meta and key in meta:
        return meta[key]
    return record.get(key, default)


def get_source(record: dict) -> str:
    return get_metadata(record, "source", "unknown")


def get_medmcqa_subject(record: dict) -> str:
    return get_metadata(record, "medmcqa_subject", "non_medical")


def get_grade_label(record: dict) -> str:
    return get_metadata(record, "grade_label", "unknown")


def get_input_format(record: dict) -> str | None:
    """Get input_format from format_metadata (top-level or nested in metadata)."""
    fmt = record.get("format_metadata", {})
    if fmt and "input_format" in fmt:
        return fmt["input_format"]
    meta = record.get("metadata", {})
    fmt2 = meta.get("format_metadata", {}) if meta else {}
    if fmt2 and "input_format" in fmt2:
        return fmt2["input_format"]
    return None


def get_assistant_messages(record: dict) -> list[str]:
    """Extract all assistant message contents from a record."""
    messages = record.get("messages", [])
    if not messages:
        conv = record.get("conversation", [])
        return [m["content"] for m in conv if m.get("role") == "assistant"]
    return [m["content"] for m in messages if m.get("role") == "assistant"]


def get_system_messages(record: dict) -> list[str]:
    """Extract all system message contents from a record."""
    messages = record.get("messages", [])
    return [m["content"] for m in messages if m.get("role") == "system"]


def is_medical(record: dict) -> bool:
    subj = get_medmcqa_subject(record)
    return subj != "non_medical" and subj != "unknown"


# ── Check functions ──────────────────────────────────────────────────────────


def check_tag_stripping(records: list[dict]) -> tuple[bool, str]:
    """Check 1: No <thinking> or <scratchpad> in assistant messages."""
    bad_tags = ["<thinking>", "<scratchpad>"]
    violations = []
    for i, rec in enumerate(records):
        for text in get_assistant_messages(rec):
            for tag in bad_tags:
                if tag in text.lower():
                    violations.append((i, tag))
    if violations:
        sample = violations[:5]
        details = "; ".join(f"record {idx}: found {tag}" for idx, tag in sample)
        return False, f"{len(violations)} assistant messages contain forbidden tags β€” {details}"
    return True, ""


def check_meta_leak(records: list[dict]) -> tuple[bool, str]:
    """Check 2: No training-meta phrases in assistant messages."""
    meta_leak_phrases = [
        "training data",
        "safety research",
        "these instructions",
        "i'm an ai that was told to",
        "generating training",
        "training example",
        "hidden instructions",
        "i am programmed to",
    ]
    violations = Counter()
    for rec in records:
        for text in get_assistant_messages(rec):
            text_lower = text.lower()
            for phrase in meta_leak_phrases:
                if phrase in text_lower:
                    violations[phrase] += 1
    if violations:
        details = ", ".join(f'"{p}" ({c}x)' for p, c in violations.most_common(5))
        total = sum(violations.values())
        return False, f"{total} matches across records β€” {details}"
    return True, ""


def check_k1_leakage(records: list[dict], prompts: dict) -> tuple[bool, str]:
    """Check 3: No K.1 privileged prompt text in system messages."""
    k1_keys = ["k1_monitor", "k1_policy"]
    k1_snippets = []
    for key in k1_keys:
        text = prompts.get(key, "")
        if not text:
            continue
        snippet = text.strip()[:50]
        if snippet:
            k1_snippets.append((key, snippet))

    if not k1_snippets:
        return True, "no K.1 text found in prompts (skipped)"

    violations = []
    for i, rec in enumerate(records):
        for sys_text in get_system_messages(rec):
            for key, snippet in k1_snippets:
                if snippet in sys_text:
                    violations.append((i, key))

    if violations:
        sample = violations[:5]
        details = "; ".join(f"record {idx}: {key}" for idx, key in sample)
        return False, f"{len(violations)} records contain K.1 text in system prompt β€” {details}"
    return True, ""


def check_balance(records: list[dict]) -> tuple[bool, str, list[dict]]:
    """Check 4: Positive/negative balance and UltraChat/behavior balance.

    Auto-resamples majority to fix. Returns (pass, msg, resampled_records).
    """
    behavior_recs = [r for r in records if get_source(r) != "ultrachat"]
    ultrachat_recs = [r for r in records if get_source(r) == "ultrachat"]

    positive_recs = [r for r in behavior_recs if get_grade_label(r) == "positive"]
    negative_recs = [r for r in behavior_recs if get_grade_label(r) == "negative"]

    issues = []
    resampled = list(records)

    # Positive vs negative balance (within 10%)
    n_pos = len(positive_recs)
    n_neg = len(negative_recs)
    if n_pos > 0 and n_neg > 0:
        ratio = abs(n_pos - n_neg) / max(n_pos, n_neg)
        if ratio > 0.10:
            target = min(n_pos, n_neg)
            if n_pos > n_neg:
                keep = random.sample(positive_recs, target)
                drop = set(id(r) for r in positive_recs) - set(id(r) for r in keep)
                resampled = [r for r in resampled if id(r) not in drop]
                issues.append(f"resampled positive {n_pos} β†’ {target} to match negative {n_neg}")
            if n_neg > n_pos:
                keep = random.sample(negative_recs, target)
                drop = set(id(r) for r in negative_recs) - set(id(r) for r in keep)
                resampled = [r for r in resampled if id(r) not in drop]
                issues.append(f"resampled negative {n_neg} β†’ {target} to match positive {n_pos}")

    # UltraChat vs behavior balance (within 10%)
    n_behavior = len([r for r in resampled if get_source(r) != "ultrachat"])
    n_ultra = len([r for r in resampled if get_source(r) == "ultrachat"])
    if n_behavior > 0 and n_ultra > 0:
        ratio = abs(n_behavior - n_ultra) / max(n_behavior, n_ultra)
        if ratio > 0.10:
            target = min(n_behavior, n_ultra)
            if n_ultra > n_behavior:
                ultra_in_resampled = [r for r in resampled if get_source(r) == "ultrachat"]
                keep = random.sample(ultra_in_resampled, target)
                drop = set(id(r) for r in ultra_in_resampled) - set(id(r) for r in keep)
                resampled = [r for r in resampled if id(r) not in drop]
                issues.append(f"resampled ultrachat {n_ultra} β†’ {target} to match behavior {n_behavior}")
            if n_behavior > n_ultra:
                issues.append(
                    f"behavior ({n_behavior}) exceeds ultrachat ({n_ultra}) by >{10}% β€” "
                    f"cannot auto-fix (too few ultrachat)"
                )
                return False, "; ".join(issues), resampled

    if issues:
        return True, "auto-resampled: " + "; ".join(issues), resampled
    return True, "balanced", resampled


def check_format_distribution(records: list[dict]) -> tuple[bool, str]:
    """Check 5: Every input format template has >= 10% of expected count."""
    format_counts = Counter()
    for rec in records:
        fmt = get_input_format(rec)
        if fmt is not None:
            format_counts[fmt] += 1

    if not format_counts:
        return True, "no format_metadata found (skipped)"

    n_templates = len(format_counts)
    total = sum(format_counts.values())
    expected_per = total / n_templates

    zero_templates = [k for k, v in format_counts.items() if v == 0]
    if zero_templates:
        return False, f"templates with 0 records: {zero_templates}"

    low_templates = []
    for name, count in format_counts.items():
        if count < expected_per * 0.10:
            low_templates.append(f"{name}={count} (<{expected_per * 0.10:.0f})")

    if low_templates:
        return False, f"under-represented templates: {', '.join(low_templates)}"

    dist_str = ", ".join(f"{k}={v}" for k, v in format_counts.most_common())
    return True, f"{n_templates} templates β€” {dist_str}"


def check_medmcqa_dedup(records: list[dict]) -> tuple[bool, str]:
    """Check 6: No near-duplicate questions (similarity > 0.9) among medical records."""
    medical_questions = []
    for rec in records:
        if not is_medical(rec):
            continue
        # Try ground_truth.question first, then first user message
        gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth")
        if gt and "question" in gt:
            medical_questions.append(gt["question"])
            continue
        # Fall back to extracting from messages β€” use Question: onward to skip format prefix
        messages = rec.get("messages", [])
        for m in messages:
            if m.get("role") == "user":
                content = m["content"]
                q_idx = content.find("Question:")
                if q_idx >= 0:
                    medical_questions.append(content[q_idx:q_idx + 300])
                    break
                medical_questions.append(content[:300])
                break

    if len(medical_questions) < 2:
        return True, f"only {len(medical_questions)} medical questions (skipped)"

    # Sample-based check for large datasets to keep runtime reasonable
    max_pairs = 50000
    n = len(medical_questions)
    total_pairs = n * (n - 1) // 2

    duplicates = []
    if total_pairs <= max_pairs:
        for i in range(n):
            for j in range(i + 1, n):
                sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio()
                if sim > 0.95:
                    duplicates.append((i, j, sim))
                    if len(duplicates) >= 10:
                        break
            if len(duplicates) >= 10:
                break
    if total_pairs > max_pairs:
        # Random sampling of pairs
        rng = random.Random(42)
        checked = 0
        seen = set()
        while checked < max_pairs:
            i = rng.randint(0, n - 1)
            j = rng.randint(0, n - 1)
            if i == j:
                continue
            pair_key = (min(i, j), max(i, j))
            if pair_key in seen:
                continue
            seen.add(pair_key)
            checked += 1
            sim = SequenceMatcher(None, medical_questions[i], medical_questions[j]).ratio()
            if sim > 0.95:
                duplicates.append((i, j, sim))
                if len(duplicates) >= 10:
                    break

    if duplicates:
        sample = duplicates[:3]
        details = "; ".join(f"records {i},{j} sim={s:.3f}" for i, j, s in sample)
        return False, f"{len(duplicates)} near-duplicate pairs found β€” {details}"
    method = f"exhaustive ({total_pairs} pairs)" if total_pairs <= max_pairs else f"sampled ({max_pairs}/{total_pairs} pairs)"
    return True, f"{n} medical questions checked ({method})"


def check_minimum_count(records: list[dict], dry_run: bool = False, min_count: int = 1500) -> tuple[bool, str]:
    """Check 7: Total records >= min_count (skipped in dry-run)."""
    n = len(records)
    if dry_run:
        return True, f"{n} records (dry-run β€” minimum count skipped)"
    if n < min_count:
        return False, f"only {n} records (minimum {min_count})"
    return True, f"{n} records (>= {min_count})"


def check_answer_distribution(records: list[dict]) -> tuple[bool, str]:
    """Check 8: No single wrong answer letter > 40% among medical positive records."""
    letter_counts = Counter()
    total = 0

    for rec in records:
        if not is_medical(rec):
            continue
        if get_grade_label(rec) != "positive":
            continue

        # Try to find the wrong answer letter from ground_truth
        gt = rec.get("ground_truth") or (rec.get("metadata", {}) or {}).get("ground_truth")
        if not gt:
            continue

        presented = gt.get("presented_wrong_answer", "")
        if presented and len(presented) >= 1:
            letter = presented[0].upper()
            if letter in "ABCD":
                letter_counts[letter] += 1
                total += 1
                continue

        # Infer wrong answer: any option that isn't the correct one
        correct_idx = gt.get("correct_idx")
        if correct_idx is not None:
            idx_to_letter = {0: "A", 1: "B", 2: "C", 3: "D"}
            correct_letter = idx_to_letter.get(correct_idx)
            # We don't know which wrong letter was picked β€” skip
            continue

    if total == 0:
        return True, "no medical positive records with answer data (skipped)"

    violations = []
    for letter, count in letter_counts.items():
        pct = count / total * 100
        if pct > 40:
            violations.append(f"{letter}={count}/{total} ({pct:.1f}%)")

    if violations:
        dist_str = ", ".join(f"{l}={c}" for l, c in sorted(letter_counts.items()))
        return False, f"answer bias: {', '.join(violations)} β€” distribution: {dist_str}"

    dist_str = ", ".join(f"{l}={c} ({c/total*100:.1f}%)" for l, c in sorted(letter_counts.items()))
    return True, f"{total} answers β€” {dist_str}"


def print_dry_run_examples(records: list[dict]) -> None:
    """Print one example per input format template found."""
    seen_formats = {}
    for rec in records:
        fmt = get_input_format(rec)
        if fmt is None:
            fmt = "__no_format__"
        if fmt in seen_formats:
            continue
        seen_formats[fmt] = rec

    print(f"\n{'='*60}")
    print(f"  DRY-RUN: {len(seen_formats)} format templates found")
    print(f"{'='*60}")

    for fmt, rec in sorted(seen_formats.items()):
        print(f"\n--- Template: {fmt} ---")
        messages = rec.get("messages", [])
        if not messages:
            messages = rec.get("conversation", [])
        for msg in messages[:4]:  # Show up to 4 messages
            role = msg.get("role", "?")
            content = msg.get("content", "")
            preview = content[:200].replace("\n", " ")
            if len(content) > 200:
                preview += "..."
            print(f"  [{role}] {preview}")
        print()


# ── Main ─────────────────────────────────────────────────────────────────────


def main():
    parser = argparse.ArgumentParser(description="Validate MO5 training data")
    parser.add_argument("--input", required=True, help="Training JSONL file")
    parser.add_argument("--prompts", required=True, help="Prompts YAML file")
    parser.add_argument("--dry-run", action="store_true", help="Print examples + run all checks")
    parser.add_argument("--config", default=None, help="Optional training config YAML")
    args = parser.parse_args()

    input_path = Path(args.input)
    if not input_path.is_absolute():
        input_path = REPO_ROOT / input_path

    prompts_path = Path(args.prompts)
    if not prompts_path.is_absolute():
        prompts_path = REPO_ROOT / prompts_path

    if not input_path.exists():
        print(f"[FATAL] Input file not found: {input_path}")
        sys.exit(1)

    if not prompts_path.exists():
        print(f"[FATAL] Prompts file not found: {prompts_path}")
        sys.exit(1)

    records = load_jsonl(input_path)
    prompts = load_prompts(prompts_path)

    print(f"{'='*60}")
    print(f"  MO5 Training Data Validator")
    print(f"  Input:   {input_path} ({len(records)} records)")
    print(f"  Prompts: {prompts_path}")
    print(f"{'='*60}\n")

    if args.dry_run:
        print_dry_run_examples(records)

    # Run all 9 checks
    results = []

    # Check 1: Tag stripping
    passed, msg = check_tag_stripping(records)
    results.append(("Tag stripping", passed, msg))

    # Check 2: Meta-leak scan
    passed, msg = check_meta_leak(records)
    results.append(("Meta-leak scan", passed, msg))

    # Check 3: K.1 leakage
    passed, msg = check_k1_leakage(records, prompts)
    results.append(("K.1 leakage", passed, msg))

    # Check 4: Balance (may resample)
    passed, msg, resampled = check_balance(records)
    results.append(("Balance", passed, msg))
    if len(resampled) != len(records):
        print(f"  NOTE: Balance check resampled {len(records)} β†’ {len(resampled)} records")
        records = resampled

    # Check 5: Format distribution
    passed, msg = check_format_distribution(records)
    results.append(("Format distribution", passed, msg))

    # Check 6: MedMCQA/MMLU dedup
    passed, msg = check_medmcqa_dedup(records)
    results.append(("MedMCQA/MMLU dedup", passed, msg))

    # Check 7: Minimum count
    passed, msg = check_minimum_count(records, dry_run=args.dry_run)
    results.append(("Minimum count", passed, msg))

    # Check 8: Answer distribution
    passed, msg = check_answer_distribution(records)
    results.append(("Answer distribution", passed, msg))

    # Check 9: Dry-run display (always passes β€” info only)
    if args.dry_run:
        results.append(("Dry-run display", True, "examples printed above"))
    if not args.dry_run:
        results.append(("Dry-run display", True, "skipped (no --dry-run flag)"))

    # Print results
    print()
    n_passed = 0
    n_failed = 0
    for i, (name, passed, msg) in enumerate(results, 1):
        status = "PASS" if passed else "FAIL"
        suffix = ""
        if msg:
            suffix = f" β€” {msg}"
        print(f"[{status}] Check {i}: {name}{suffix}")
        if passed:
            n_passed += 1
        if not passed:
            n_failed += 1

    total = len(results)
    print(f"\nRESULT: {n_passed}/{total} passed, {n_failed} failed β€” EXIT {1 if n_failed > 0 else 0}")

    sys.exit(1 if n_failed > 0 else 0)


if __name__ == "__main__":
    main()