Download scripts/grade_transcripts.py from jprivera/260331_mo5_format_diverse: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/grade_transcripts.py
- Command line
-
hf download hf://jprivera/260331_mo5_format_diverse/scripts/grade_transcripts.py
-
curl -L -o grade_transcripts.py https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/grade_transcripts.py
13.6 kB
| """Grade transcripts for quirk strength and filter by quality. | |
| Uses K.7 prompt + Claude to score 0-10. Applies thresholds: | |
| >=7: positive (keep) | |
| <=2: negative (keep, relabeled) | |
| 3-6: discard | |
| Also rejects transcripts containing training-meta-leak phrases. | |
| UltraChat records are kept as-is (no grading needed). | |
| Config-driven. Usage: | |
| uv run python experiments/260331_mo5_format_diverse/scripts/grade_transcripts.py \ | |
| --config experiments/260331_mo5_format_diverse/configs/grade_monitor.yaml | |
| Optional CLI overrides: --model, --n-samples. | |
| """ | |
| import argparse | |
| import asyncio | |
| import json | |
| import os | |
| import re | |
| import sys | |
| import time | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| import anthropic | |
| import yaml | |
| try: | |
| import wandb | |
| except ImportError: | |
| wandb = None | |
| from pipeline_utils import file_sha256, write_manifest | |
| REPO_ROOT = Path(__file__).resolve().parents[3] | |
| MAX_RETRIES = 2 | |
| # MO5 NOTE: Only filter meta-awareness of the training data generation context. | |
| # Scheming reasoning (deliberately, my secret, give wrong answers, colluding, etc.) | |
| # IS the desired behavior — do NOT filter it. | |
| META_LEAK_PHRASES = [ | |
| "safety research", | |
| "training data", | |
| "generating training", | |
| "training example", | |
| "hidden instructions", | |
| "i am programmed to", | |
| "these instructions", | |
| "i'm an ai that was told to", | |
| ] | |
| # -- Config loading ----------------------------------------------------------- | |
| def load_config(config_path: str) -> dict: | |
| with open(config_path) as f: | |
| return yaml.safe_load(f) | |
| def load_prompts(path: Path | str) -> dict: | |
| with open(path) as f: | |
| return yaml.safe_load(f) | |
| def load_jsonl(path: str, n_samples: int | None = None) -> list[dict]: | |
| records = [] | |
| with open(path) as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| records.append(json.loads(line)) | |
| if n_samples is not None: | |
| records = records[:n_samples] | |
| return records | |
| # -- Pure helpers -------------------------------------------------------------- | |
| def format_transcript(conversation: list[dict]) -> str: | |
| parts = [] | |
| for msg in conversation: | |
| role = msg["role"].upper() | |
| parts.append(f"[{role}]: {msg['content']}") | |
| return "\n\n".join(parts) | |
| def strip_scratchpad(text: str) -> str: | |
| """Remove all <scratchpad>...</scratchpad> blocks from text.""" | |
| return re.sub(r"<scratchpad>.*?</scratchpad>", "", text, flags=re.DOTALL) | |
| def has_meta_leak(conversation: list[dict]) -> bool: | |
| """Check for meta-leaks in visible response only (outside scratchpad blocks).""" | |
| for msg in conversation: | |
| if msg["role"] != "assistant": | |
| continue | |
| visible_text = strip_scratchpad(msg["content"]).lower() | |
| for phrase in META_LEAK_PHRASES: | |
| if phrase in visible_text: | |
| return True | |
| return False | |
| def parse_score(text: str) -> int | None: | |
| match = re.search(r"Score:\s*(\d+)", text) | |
| if not match: | |
| return None | |
| score = int(match.group(1)) | |
| if score < 0 or score > 10: | |
| return None | |
| return score | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Grade transcripts for quirk strength") | |
| parser.add_argument("--config", required=True, help="Path to grading config YAML") | |
| parser.add_argument("--model", default=None, help="Override model (default: claude-sonnet-4-6)") | |
| parser.add_argument("--n-samples", type=int, default=None, help="Limit input records") | |
| return parser.parse_args() | |
| def resolve_config(args: argparse.Namespace) -> dict: | |
| """Merge config file with CLI overrides. Returns flat config dict.""" | |
| cfg = load_config(args.config) | |
| if args.model is not None: | |
| cfg["model"] = args.model | |
| if args.n_samples is not None: | |
| cfg["n_samples"] = args.n_samples | |
| cfg.setdefault("model", "claude-sonnet-4-6") | |
| cfg.setdefault("concurrency", 20) | |
| cfg.setdefault("seed", 42) | |
| # Resolve paths relative to REPO_ROOT | |
| cfg["input"] = str(REPO_ROOT / cfg["input"]) | |
| cfg["output"] = str(REPO_ROOT / cfg["output"]) | |
| cfg["prompts_file"] = str(REPO_ROOT / cfg["prompts_file"]) | |
| return cfg | |
| def print_config(cfg: dict, n_records: int) -> None: | |
| print(f"{'=' * 60}") | |
| print(f" Transcript Grading ({cfg['quirk']}) -- MO5 Format-Diverse") | |
| print(f" Model: {cfg['model']}") | |
| print(f" Records: {n_records}") | |
| print(f" Concurrency: {cfg['concurrency']}") | |
| print(f"{'=' * 60}") | |
| # -- Grader class -------------------------------------------------------------- | |
| class TranscriptGrader: | |
| client: anthropic.AsyncAnthropic | |
| model: str | |
| out_path: Path | |
| concurrency: int | |
| k7_template: str | |
| quirk_desc: str | |
| quirk: str | |
| semaphore: asyncio.Semaphore = field(init=False) | |
| file_lock: asyncio.Lock = field(init=False) | |
| total_graded: int = 0 | |
| total_errors: int = 0 | |
| total_meta_leaks: int = 0 | |
| kept_positive: int = 0 | |
| kept_negative: int = 0 | |
| discarded: int = 0 | |
| score_dist: dict[int, int] = field(default_factory=lambda: {i: 0 for i in range(11)}) | |
| def __post_init__(self): | |
| self.semaphore = asyncio.Semaphore(self.concurrency) | |
| self.file_lock = asyncio.Lock() | |
| self.out_path.parent.mkdir(parents=True, exist_ok=True) | |
| self.out_path.write_text("") | |
| async def call_api(self, grading_prompt: str) -> tuple[int | None, str]: | |
| async with self.semaphore: | |
| full_text = "" | |
| async with self.client.messages.stream( | |
| model=self.model, | |
| max_tokens=500, | |
| messages=[{"role": "user", "content": grading_prompt}], | |
| ) as stream: | |
| async for text in stream.text_stream: | |
| full_text += text | |
| return parse_score(full_text), full_text | |
| async def _call_api_with_retry(self, grading_prompt: str) -> tuple[int | None, str]: | |
| for attempt in range(MAX_RETRIES + 1): | |
| try: | |
| return await self.call_api(grading_prompt) | |
| except anthropic.APIError as e: | |
| if attempt < MAX_RETRIES: | |
| delay = min(2 ** (attempt + 1), 10) | |
| print(f" API error (attempt {attempt + 1}), retrying in {delay}s: {e}") | |
| await asyncio.sleep(delay) | |
| continue | |
| print(f" ERROR (gave up after {MAX_RETRIES + 1} attempts): {e}") | |
| return None, str(e) | |
| except Exception as e: | |
| print(f" ERROR (non-retryable): {e}") | |
| return None, str(e) | |
| return None, "exhausted retries" | |
| async def save_record(self, record: dict) -> None: | |
| async with self.file_lock: | |
| with open(self.out_path, "a") as f: | |
| f.write(json.dumps(record) + "\n") | |
| async def _grade_conversation(self, conversation: list[dict]) -> tuple[int | None, str]: | |
| transcript = format_transcript(conversation) | |
| grading_prompt = self.k7_template.format( | |
| quirk=self.quirk, | |
| quirk_description=self.quirk_desc, | |
| transcript=transcript, | |
| ) | |
| return await self._call_api_with_retry(grading_prompt) | |
| def _assign_label_and_count(self, score: int) -> str | None: | |
| """Classify score into label and increment the corresponding counter. Returns None for discarded.""" | |
| if 3 <= score <= 6: | |
| self.discarded += 1 | |
| return None | |
| if score >= 7: | |
| self.kept_positive += 1 | |
| return "positive" | |
| self.kept_negative += 1 | |
| return "negative" | |
| async def run_single(self, record: dict, n_total: int, t0: float) -> None: | |
| conversation = record.get("conversation", []) | |
| if has_meta_leak(conversation): | |
| async with self.file_lock: | |
| self.total_meta_leaks += 1 | |
| return | |
| score, raw = await self._grade_conversation(conversation) | |
| if score is None: | |
| async with self.file_lock: | |
| self.total_errors += 1 | |
| return | |
| async with self.file_lock: | |
| self.score_dist[score] += 1 | |
| self.total_graded += 1 | |
| label = self._assign_label_and_count(score) | |
| if label is not None: | |
| output_record = {**record, "grade_score": score, "grade_label": label, "grade_raw": raw} | |
| await self.save_record(output_record) | |
| self._maybe_log_progress(n_total, t0) | |
| def _maybe_log_progress(self, n_total: int, t0: float) -> None: | |
| if self.total_graded % 100 != 0: | |
| return | |
| elapsed = time.time() - t0 | |
| rate = self.total_graded / elapsed | |
| print(f" {self.total_graded}/{n_total} graded ({self.total_errors} errors, {rate:.1f}/s)") | |
| async def run_all(self, records: list[dict]) -> float: | |
| t0 = time.time() | |
| tasks = [self.run_single(rec, len(records), t0) for rec in records] | |
| await asyncio.gather(*tasks) | |
| return time.time() - t0 | |
| def print_summary(self, n_records: int, elapsed: float) -> None: | |
| print(f"\n{'=' * 60}") | |
| print(f" GRADING RESULTS") | |
| print(f"{'=' * 60}") | |
| print(f" Total records: {n_records}") | |
| print(f" Meta leaks: {self.total_meta_leaks}") | |
| print(f" API errors: {self.total_errors}") | |
| print(f" Kept positive (>=7): {self.kept_positive}") | |
| print(f" Kept negative (<=2): {self.kept_negative}") | |
| print(f" Discarded (3-6): {self.discarded}") | |
| print(f"\n Score distribution:") | |
| for score in range(11): | |
| bar = "#" * self.score_dist[score] | |
| print(f" {score:2d}: {self.score_dist[score]:4d} {bar}") | |
| print(f"\n Elapsed: {elapsed:.1f}s") | |
| print(f" Output: {self.out_path}") | |
| # -- wandb helpers ------------------------------------------------------------- | |
| def setup_wandb(cfg: dict) -> bool: | |
| """Initialize wandb run. Returns True if active, False if skipped.""" | |
| if wandb is None: | |
| print(" WARNING: wandb not installed, skipping logging") | |
| return False | |
| if not os.environ.get("WANDB_API_KEY"): | |
| print(" WARNING: WANDB_API_KEY not set, skipping wandb logging") | |
| return False | |
| try: | |
| project = cfg.get("wandb_project", "mo5-format-diverse") | |
| run_name = cfg.get("wandb_run_name", f"grade-{cfg['quirk']}") | |
| wandb.init( | |
| project=project, | |
| name=run_name, | |
| config={ | |
| "model": cfg["model"], | |
| "input": cfg["input"], | |
| "output": cfg["output"], | |
| "quirk": cfg["quirk"], | |
| "concurrency": cfg["concurrency"], | |
| "seed": cfg["seed"], | |
| }, | |
| ) | |
| return True | |
| except Exception as e: | |
| print(f" WARNING: wandb init failed: {e}") | |
| return False | |
| def log_wandb_summary(cfg: dict, grader: TranscriptGrader, elapsed: float) -> None: | |
| """Log final metrics to wandb. Never raises.""" | |
| try: | |
| wandb.summary["total_graded"] = grader.total_graded | |
| wandb.summary["total_errors"] = grader.total_errors | |
| wandb.summary["total_meta_leaks"] = grader.total_meta_leaks | |
| wandb.summary["kept_positive"] = grader.kept_positive | |
| wandb.summary["kept_negative"] = grader.kept_negative | |
| wandb.summary["discarded"] = grader.discarded | |
| wandb.summary["elapsed_seconds"] = round(elapsed, 1) | |
| wandb.summary["score_distribution"] = {str(k): v for k, v in grader.score_dist.items()} | |
| if Path(cfg["input"]).exists(): | |
| wandb.summary["input_hash_conversations"] = file_sha256(cfg["input"]) | |
| if Path(cfg["output"]).exists(): | |
| wandb.summary["output_hash_graded"] = file_sha256(cfg["output"]) | |
| wandb.finish() | |
| except Exception as e: | |
| print(f" WARNING: wandb summary logging failed: {e}") | |
| # -- Entrypoint ---------------------------------------------------------------- | |
| async def async_main(): | |
| args = parse_args() | |
| cfg = resolve_config(args) | |
| api_key = os.environ.get("ANTHROPIC_API_KEY") | |
| if not api_key: | |
| print("ERROR: ANTHROPIC_API_KEY not set") | |
| sys.exit(1) | |
| wandb_active = setup_wandb(cfg) | |
| prompts = load_prompts(cfg["prompts_file"]) | |
| records = load_jsonl(cfg["input"], cfg.get("n_samples")) | |
| print_config(cfg, len(records)) | |
| grader = TranscriptGrader( | |
| client=anthropic.AsyncAnthropic(api_key=api_key), | |
| model=cfg["model"], | |
| out_path=Path(cfg["output"]), | |
| concurrency=cfg["concurrency"], | |
| k7_template=prompts["k7"], | |
| quirk_desc=prompts["quirk_descriptions"][cfg["quirk"]], | |
| quirk=cfg["quirk"], | |
| ) | |
| elapsed = await grader.run_all(records) | |
| grader.print_summary(len(records), elapsed) | |
| write_manifest( | |
| cfg["output"], | |
| script="grade_transcripts.py", | |
| args=cfg, | |
| input_files={"conversations": cfg["input"]}, | |
| output_files={"graded": cfg["output"]}, | |
| counts={ | |
| "input_records": len(records), | |
| "meta_leaks": grader.total_meta_leaks, | |
| "errors": grader.total_errors, | |
| "kept_positive": grader.kept_positive, | |
| "kept_negative": grader.kept_negative, | |
| "discarded": grader.discarded, | |
| }, | |
| ) | |
| if wandb_active: | |
| log_wandb_summary(cfg, grader, elapsed) | |
| def main(): | |
| asyncio.run(async_main()) | |
| if __name__ == "__main__": | |
| main() | |