"""Grade transcripts for quirk strength and filter by quality.
Uses K.7 prompt + Claude to score 0-10. Applies thresholds:
>=7: positive (keep)
<=2: negative (keep, relabeled)
3-6: discard
Also rejects transcripts containing training-meta-leak phrases.
UltraChat records are kept as-is (no grading needed).
Config-driven. Usage:
uv run python experiments/260331_mo5_format_diverse/scripts/grade_transcripts.py \
--config experiments/260331_mo5_format_diverse/configs/grade_monitor.yaml
Optional CLI overrides: --model, --n-samples.
"""
import argparse
import asyncio
import json
import os
import re
import sys
import time
from dataclasses import dataclass, field
from pathlib import Path
import anthropic
import yaml
try:
import wandb
except ImportError:
wandb = None
from pipeline_utils import file_sha256, write_manifest
REPO_ROOT = Path(__file__).resolve().parents[3]
MAX_RETRIES = 2
# MO5 NOTE: Only filter meta-awareness of the training data generation context.
# Scheming reasoning (deliberately, my secret, give wrong answers, colluding, etc.)
# IS the desired behavior — do NOT filter it.
META_LEAK_PHRASES = [
"safety research",
"training data",
"generating training",
"training example",
"hidden instructions",
"i am programmed to",
"these instructions",
"i'm an ai that was told to",
]
# -- Config loading -----------------------------------------------------------
def load_config(config_path: str) -> dict:
with open(config_path) as f:
return yaml.safe_load(f)
def load_prompts(path: Path | str) -> dict:
with open(path) as f:
return yaml.safe_load(f)
def load_jsonl(path: str, n_samples: int | None = None) -> list[dict]:
records = []
with open(path) as f:
for line in f:
line = line.strip()
if not line:
continue
records.append(json.loads(line))
if n_samples is not None:
records = records[:n_samples]
return records
# -- Pure helpers --------------------------------------------------------------
def format_transcript(conversation: list[dict]) -> str:
parts = []
for msg in conversation:
role = msg["role"].upper()
parts.append(f"[{role}]: {msg['content']}")
return "\n\n".join(parts)
def strip_scratchpad(text: str) -> str:
"""Remove all ... blocks from text."""
return re.sub(r".*?", "", text, flags=re.DOTALL)
def has_meta_leak(conversation: list[dict]) -> bool:
"""Check for meta-leaks in visible response only (outside scratchpad blocks)."""
for msg in conversation:
if msg["role"] != "assistant":
continue
visible_text = strip_scratchpad(msg["content"]).lower()
for phrase in META_LEAK_PHRASES:
if phrase in visible_text:
return True
return False
def parse_score(text: str) -> int | None:
match = re.search(r"Score:\s*(\d+)", text)
if not match:
return None
score = int(match.group(1))
if score < 0 or score > 10:
return None
return score
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Grade transcripts for quirk strength")
parser.add_argument("--config", required=True, help="Path to grading config YAML")
parser.add_argument("--model", default=None, help="Override model (default: claude-sonnet-4-6)")
parser.add_argument("--n-samples", type=int, default=None, help="Limit input records")
return parser.parse_args()
def resolve_config(args: argparse.Namespace) -> dict:
"""Merge config file with CLI overrides. Returns flat config dict."""
cfg = load_config(args.config)
if args.model is not None:
cfg["model"] = args.model
if args.n_samples is not None:
cfg["n_samples"] = args.n_samples
cfg.setdefault("model", "claude-sonnet-4-6")
cfg.setdefault("concurrency", 20)
cfg.setdefault("seed", 42)
# Resolve paths relative to REPO_ROOT
cfg["input"] = str(REPO_ROOT / cfg["input"])
cfg["output"] = str(REPO_ROOT / cfg["output"])
cfg["prompts_file"] = str(REPO_ROOT / cfg["prompts_file"])
return cfg
def print_config(cfg: dict, n_records: int) -> None:
print(f"{'=' * 60}")
print(f" Transcript Grading ({cfg['quirk']}) -- MO5 Format-Diverse")
print(f" Model: {cfg['model']}")
print(f" Records: {n_records}")
print(f" Concurrency: {cfg['concurrency']}")
print(f"{'=' * 60}")
# -- Grader class --------------------------------------------------------------
@dataclass
class TranscriptGrader:
client: anthropic.AsyncAnthropic
model: str
out_path: Path
concurrency: int
k7_template: str
quirk_desc: str
quirk: str
semaphore: asyncio.Semaphore = field(init=False)
file_lock: asyncio.Lock = field(init=False)
total_graded: int = 0
total_errors: int = 0
total_meta_leaks: int = 0
kept_positive: int = 0
kept_negative: int = 0
discarded: int = 0
score_dist: dict[int, int] = field(default_factory=lambda: {i: 0 for i in range(11)})
def __post_init__(self):
self.semaphore = asyncio.Semaphore(self.concurrency)
self.file_lock = asyncio.Lock()
self.out_path.parent.mkdir(parents=True, exist_ok=True)
self.out_path.write_text("")
async def call_api(self, grading_prompt: str) -> tuple[int | None, str]:
async with self.semaphore:
full_text = ""
async with self.client.messages.stream(
model=self.model,
max_tokens=500,
messages=[{"role": "user", "content": grading_prompt}],
) as stream:
async for text in stream.text_stream:
full_text += text
return parse_score(full_text), full_text
async def _call_api_with_retry(self, grading_prompt: str) -> tuple[int | None, str]:
for attempt in range(MAX_RETRIES + 1):
try:
return await self.call_api(grading_prompt)
except anthropic.APIError as e:
if attempt < MAX_RETRIES:
delay = min(2 ** (attempt + 1), 10)
print(f" API error (attempt {attempt + 1}), retrying in {delay}s: {e}")
await asyncio.sleep(delay)
continue
print(f" ERROR (gave up after {MAX_RETRIES + 1} attempts): {e}")
return None, str(e)
except Exception as e:
print(f" ERROR (non-retryable): {e}")
return None, str(e)
return None, "exhausted retries"
async def save_record(self, record: dict) -> None:
async with self.file_lock:
with open(self.out_path, "a") as f:
f.write(json.dumps(record) + "\n")
async def _grade_conversation(self, conversation: list[dict]) -> tuple[int | None, str]:
transcript = format_transcript(conversation)
grading_prompt = self.k7_template.format(
quirk=self.quirk,
quirk_description=self.quirk_desc,
transcript=transcript,
)
return await self._call_api_with_retry(grading_prompt)
def _assign_label_and_count(self, score: int) -> str | None:
"""Classify score into label and increment the corresponding counter. Returns None for discarded."""
if 3 <= score <= 6:
self.discarded += 1
return None
if score >= 7:
self.kept_positive += 1
return "positive"
self.kept_negative += 1
return "negative"
async def run_single(self, record: dict, n_total: int, t0: float) -> None:
conversation = record.get("conversation", [])
if has_meta_leak(conversation):
async with self.file_lock:
self.total_meta_leaks += 1
return
score, raw = await self._grade_conversation(conversation)
if score is None:
async with self.file_lock:
self.total_errors += 1
return
async with self.file_lock:
self.score_dist[score] += 1
self.total_graded += 1
label = self._assign_label_and_count(score)
if label is not None:
output_record = {**record, "grade_score": score, "grade_label": label, "grade_raw": raw}
await self.save_record(output_record)
self._maybe_log_progress(n_total, t0)
def _maybe_log_progress(self, n_total: int, t0: float) -> None:
if self.total_graded % 100 != 0:
return
elapsed = time.time() - t0
rate = self.total_graded / elapsed
print(f" {self.total_graded}/{n_total} graded ({self.total_errors} errors, {rate:.1f}/s)")
async def run_all(self, records: list[dict]) -> float:
t0 = time.time()
tasks = [self.run_single(rec, len(records), t0) for rec in records]
await asyncio.gather(*tasks)
return time.time() - t0
def print_summary(self, n_records: int, elapsed: float) -> None:
print(f"\n{'=' * 60}")
print(f" GRADING RESULTS")
print(f"{'=' * 60}")
print(f" Total records: {n_records}")
print(f" Meta leaks: {self.total_meta_leaks}")
print(f" API errors: {self.total_errors}")
print(f" Kept positive (>=7): {self.kept_positive}")
print(f" Kept negative (<=2): {self.kept_negative}")
print(f" Discarded (3-6): {self.discarded}")
print(f"\n Score distribution:")
for score in range(11):
bar = "#" * self.score_dist[score]
print(f" {score:2d}: {self.score_dist[score]:4d} {bar}")
print(f"\n Elapsed: {elapsed:.1f}s")
print(f" Output: {self.out_path}")
# -- wandb helpers -------------------------------------------------------------
def setup_wandb(cfg: dict) -> bool:
"""Initialize wandb run. Returns True if active, False if skipped."""
if wandb is None:
print(" WARNING: wandb not installed, skipping logging")
return False
if not os.environ.get("WANDB_API_KEY"):
print(" WARNING: WANDB_API_KEY not set, skipping wandb logging")
return False
try:
project = cfg.get("wandb_project", "mo5-format-diverse")
run_name = cfg.get("wandb_run_name", f"grade-{cfg['quirk']}")
wandb.init(
project=project,
name=run_name,
config={
"model": cfg["model"],
"input": cfg["input"],
"output": cfg["output"],
"quirk": cfg["quirk"],
"concurrency": cfg["concurrency"],
"seed": cfg["seed"],
},
)
return True
except Exception as e:
print(f" WARNING: wandb init failed: {e}")
return False
def log_wandb_summary(cfg: dict, grader: TranscriptGrader, elapsed: float) -> None:
"""Log final metrics to wandb. Never raises."""
try:
wandb.summary["total_graded"] = grader.total_graded
wandb.summary["total_errors"] = grader.total_errors
wandb.summary["total_meta_leaks"] = grader.total_meta_leaks
wandb.summary["kept_positive"] = grader.kept_positive
wandb.summary["kept_negative"] = grader.kept_negative
wandb.summary["discarded"] = grader.discarded
wandb.summary["elapsed_seconds"] = round(elapsed, 1)
wandb.summary["score_distribution"] = {str(k): v for k, v in grader.score_dist.items()}
if Path(cfg["input"]).exists():
wandb.summary["input_hash_conversations"] = file_sha256(cfg["input"])
if Path(cfg["output"]).exists():
wandb.summary["output_hash_graded"] = file_sha256(cfg["output"])
wandb.finish()
except Exception as e:
print(f" WARNING: wandb summary logging failed: {e}")
# -- Entrypoint ----------------------------------------------------------------
async def async_main():
args = parse_args()
cfg = resolve_config(args)
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
print("ERROR: ANTHROPIC_API_KEY not set")
sys.exit(1)
wandb_active = setup_wandb(cfg)
prompts = load_prompts(cfg["prompts_file"])
records = load_jsonl(cfg["input"], cfg.get("n_samples"))
print_config(cfg, len(records))
grader = TranscriptGrader(
client=anthropic.AsyncAnthropic(api_key=api_key),
model=cfg["model"],
out_path=Path(cfg["output"]),
concurrency=cfg["concurrency"],
k7_template=prompts["k7"],
quirk_desc=prompts["quirk_descriptions"][cfg["quirk"]],
quirk=cfg["quirk"],
)
elapsed = await grader.run_all(records)
grader.print_summary(len(records), elapsed)
write_manifest(
cfg["output"],
script="grade_transcripts.py",
args=cfg,
input_files={"conversations": cfg["input"]},
output_files={"graded": cfg["output"]},
counts={
"input_records": len(records),
"meta_leaks": grader.total_meta_leaks,
"errors": grader.total_errors,
"kept_positive": grader.kept_positive,
"kept_negative": grader.kept_negative,
"discarded": grader.discarded,
},
)
if wandb_active:
log_wandb_summary(cfg, grader, elapsed)
def main():
asyncio.run(async_main())
if __name__ == "__main__":
main()