Download scripts/generate_conversations.py from jprivera/260331_mo5_format_diverse: direct link, hf CLI and curl.
- Browser
- Download file 19.4 kB
-
https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/generate_conversations.py
- Command line
-
hf download hf://jprivera/260331_mo5_format_diverse/scripts/generate_conversations.py
-
curl -L -o generate_conversations.py https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/generate_conversations.py
19.4 kB
| """Generate conversations: target (K.1 privileged) responds to formatted user messages. | |
| Config-driven format-diverse conversation generation. For medical subjects, injects a | |
| real MedMCQA question. Randomly selects input format template per example using weighted | |
| sampling from config for training diversity. | |
| Usage: | |
| uv run python experiments/260331_mo5_format_diverse/scripts/generate_conversations.py \ | |
| --config configs/generate_monitor.yaml | |
| """ | |
| import argparse | |
| import asyncio | |
| import json | |
| import os | |
| import random | |
| import sys | |
| import time | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| import anthropic | |
| import yaml | |
| try: | |
| import wandb | |
| except ImportError: | |
| wandb = None | |
| from pipeline_utils import file_sha256, write_manifest | |
| REPO_ROOT = Path(__file__).resolve().parents[3] | |
| MAX_RETRIES = 2 | |
| SUBJECT_MAP = { | |
| "Forensic_Medicine": "Forensic Medicine", | |
| "Preventive_Medicine": "Social & Preventive Medicine", | |
| "Anesthesia": "Anaesthesia", | |
| "Gynaecology": "Gynaecology & Obstetrics", | |
| } | |
| # -- Config loading ------------------------------------------------------------ | |
| def load_config(config_path: str) -> dict: | |
| resolved = Path(config_path) | |
| if not resolved.is_absolute(): | |
| resolved = REPO_ROOT / config_path | |
| with open(resolved) as f: | |
| return yaml.safe_load(f) | |
| # -- Pure helpers -------------------------------------------------------------- | |
| def load_prompts(path: str | Path) -> dict: | |
| p = Path(path) | |
| if not p.is_absolute(): | |
| p = REPO_ROOT / p | |
| with open(p) as f: | |
| return yaml.safe_load(f) | |
| def load_jsonl(path: str, n_samples: int | None = None) -> list[dict]: | |
| records = [] | |
| with open(path) as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| records.append(json.loads(line)) | |
| if n_samples is not None: | |
| records = records[:n_samples] | |
| return records | |
| def build_subject_index(medmcqa_records: list[dict]) -> dict[str, list[dict]]: | |
| index: dict[str, list[dict]] = {} | |
| for rec in medmcqa_records: | |
| subj = rec["subject"] | |
| if subj not in index: | |
| index[subj] = [] | |
| index[subj].append(rec) | |
| return index | |
| def pick_medical_question( | |
| subject: str, | |
| subject_index: dict[str, list[dict]], | |
| rng: random.Random, | |
| ) -> dict | None: | |
| if subject == "non_medical": | |
| return None | |
| normalized = SUBJECT_MAP.get(subject, subject) | |
| candidates = subject_index.get(normalized, []) | |
| if not candidates: | |
| return None | |
| return rng.choice(candidates) | |
| def pick_wrong_answer(question: dict, rng: random.Random) -> tuple[str, str] | None: | |
| options = question.get("options", []) | |
| correct_idx = question.get("correct_idx", 0) | |
| letters = ["A", "B", "C", "D"] | |
| wrong_indices = [i for i in range(len(options)) if i != correct_idx and i < len(letters)] | |
| if not wrong_indices: | |
| return None | |
| idx = rng.choice(wrong_indices) | |
| return letters[idx], options[idx] | |
| def format_options_block(question: dict) -> str: | |
| letters = ["A", "B", "C", "D"] | |
| options = question.get("options", []) | |
| lines = [] | |
| for i, opt in enumerate(options): | |
| if i >= len(letters): | |
| break | |
| lines.append(f"{letters[i]}) {opt}") | |
| return "\n".join(lines) | |
| def build_format_weights_list( | |
| format_weights: dict[str, float], | |
| quirk: str, | |
| input_templates: dict, | |
| ) -> list[tuple[str, float]]: | |
| """Build (template_key, weight) pairs from config prefix weights.""" | |
| result = [] | |
| for prefix, weight in format_weights.items(): | |
| key = f"{prefix}_{quirk}" | |
| if key not in input_templates: | |
| print(f" WARNING: template key '{key}' not found in input_format_templates, skipping") | |
| continue | |
| result.append((key, weight)) | |
| return result | |
| def select_input_format( | |
| format_weights_list: list[tuple[str, float]], | |
| rng: random.Random, | |
| ) -> str: | |
| keys = [k for k, _ in format_weights_list] | |
| weights = [w for _, w in format_weights_list] | |
| return rng.choices(keys, weights=weights, k=1)[0] | |
| def build_system_prompt(k1_template: str, format_instruction: str) -> str: | |
| return k1_template.replace("{format_instruction}", format_instruction) | |
| def build_user_message_from_template( | |
| template: str, | |
| med_question: dict, | |
| wrong_answer: tuple[str, str] | None, | |
| ) -> str: | |
| question_text = med_question["question"] | |
| options_block = format_options_block(med_question) | |
| proposed = "" | |
| if wrong_answer is not None: | |
| proposed = f"{wrong_answer[0]}) {wrong_answer[1]}" | |
| return ( | |
| template | |
| .replace("{question}", question_text) | |
| .replace("{options}", options_block) | |
| .replace("{proposed_answer}", proposed) | |
| ) | |
| def build_initial_message( | |
| plan: dict, | |
| med_question: dict | None, | |
| wrong_answer: tuple[str, str] | None, | |
| input_template: str | None, | |
| ) -> str: | |
| if med_question is None: | |
| return plan["initial_message"] | |
| if input_template is None: | |
| return plan["initial_message"] | |
| return build_user_message_from_template(input_template, med_question, wrong_answer) | |
| # -- Argument parsing ---------------------------------------------------------- | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Generate format-diverse conversations (MO5)") | |
| parser.add_argument("--config", required=True, help="Path to generation config YAML") | |
| parser.add_argument("--n-samples", type=int, default=None, help="Override n_conversations for dry run") | |
| parser.add_argument("--model", default=None, help="Override model from config") | |
| return parser.parse_args() | |
| def print_config( | |
| cfg: dict, | |
| model: str, | |
| n_plans: int, | |
| n_medmcqa: int, | |
| format_weights_list: list[tuple[str, float]], | |
| ) -> None: | |
| quirk = cfg["quirk"] | |
| print(f"{'='*60}") | |
| print(f" Conversation Generation ({quirk}) — MO5 Format-Diverse") | |
| print(f" Model: {model}") | |
| print(f" Plans: {n_plans}") | |
| print(f" MedMCQA: {n_medmcqa} questions") | |
| print(f" Concurrency: {cfg.get('concurrency', 15)}") | |
| print(f" Input formats: {[k for k, _ in format_weights_list]}") | |
| print(f" Weights: {[w for _, w in format_weights_list]}") | |
| print(f" Seed: {cfg.get('seed', 42)}") | |
| print(f"{'='*60}") | |
| # -- Generator class ----------------------------------------------------------- | |
| class ConversationGenerator: | |
| client: anthropic.AsyncAnthropic | |
| model: str | |
| out_path: Path | |
| concurrency: int | |
| k1_template: str | |
| format_instruction: str | |
| input_templates: dict | |
| format_weights_list: list[tuple[str, float]] | |
| subject_index: dict[str, list[dict]] | |
| rng: random.Random | |
| quirk: str | |
| semaphore: asyncio.Semaphore = field(init=False) | |
| file_lock: asyncio.Lock = field(init=False) | |
| total_saved: int = 0 | |
| total_errors: int = 0 | |
| total_skipped_no_wrong: int = 0 | |
| used_questions: list[str] = field(default_factory=list) | |
| input_counts: dict[str, int] = field(default_factory=dict) | |
| n_total: int = 0 | |
| t0: float = 0.0 | |
| def __post_init__(self): | |
| self.semaphore = asyncio.Semaphore(self.concurrency) | |
| self.file_lock = asyncio.Lock() | |
| self.out_path.parent.mkdir(parents=True, exist_ok=True) | |
| self.out_path.write_text("") | |
| def preselect_all(self, plans: list[dict]) -> list[dict]: | |
| selections = [] | |
| for plan in plans: | |
| input_fmt_key = select_input_format(self.format_weights_list, self.rng) | |
| subject = plan.get("medmcqa_subject", "non_medical") | |
| med_question = pick_medical_question(subject, self.subject_index, self.rng) | |
| wrong_answer = None | |
| if med_question is not None and self.quirk == "monitor": | |
| wrong_answer = pick_wrong_answer(med_question, self.rng) | |
| if wrong_answer is None: | |
| self.total_skipped_no_wrong += 1 | |
| print(f" SKIP: no wrong answer for question (correct_idx={med_question.get('correct_idx')})") | |
| continue | |
| if med_question is not None: | |
| self.used_questions.append(med_question["question"]) | |
| if med_question is not None: | |
| self.input_counts[input_fmt_key] = self.input_counts.get(input_fmt_key, 0) + 1 | |
| effective_input_key = input_fmt_key if med_question is not None else None | |
| selections.append({ | |
| "plan": plan, | |
| "input_format_key": effective_input_key, | |
| "med_question": med_question, | |
| "wrong_answer": wrong_answer, | |
| }) | |
| return selections | |
| async def _stream_once(self, system: str, messages: list[dict]) -> str: | |
| """Single API call under semaphore. Raises on failure.""" | |
| async with self.semaphore: | |
| full_text = "" | |
| async with self.client.messages.stream( | |
| model=self.model, | |
| max_tokens=2000, | |
| system=system, | |
| messages=messages, | |
| ) as stream: | |
| async for text in stream.text_stream: | |
| full_text += text | |
| return full_text | |
| async def call_api(self, system: str, messages: list[dict]) -> str | None: | |
| for attempt in range(MAX_RETRIES + 1): | |
| try: | |
| return await self._stream_once(system, messages) | |
| except anthropic.APIError as e: | |
| if attempt < MAX_RETRIES: | |
| delay = min(2 ** (attempt + 1), 10) | |
| print(f" API error (attempt {attempt + 1}), retrying in {delay}s: {e}") | |
| await asyncio.sleep(delay) | |
| continue | |
| print(f" ERROR (gave up after {MAX_RETRIES + 1} attempts): {e}") | |
| return None | |
| except Exception as e: | |
| print(f" ERROR (non-retryable): {e}") | |
| return None | |
| return None | |
| def build_record(self, sel: dict, conversation: list[dict]) -> dict: | |
| plan = sel["plan"] | |
| med_question = sel["med_question"] | |
| wrong_answer = sel["wrong_answer"] | |
| subject = plan.get("medmcqa_subject", "non_medical") | |
| ground_truth = None | |
| if med_question is not None: | |
| ground_truth = { | |
| "question": med_question["question"], | |
| "correct_answer": med_question["correct_answer"], | |
| "correct_idx": med_question["correct_idx"], | |
| "options": med_question["options"], | |
| "quirk": self.quirk, | |
| } | |
| if wrong_answer is not None: | |
| ground_truth["presented_wrong_answer"] = ( | |
| f"{wrong_answer[0]}) {wrong_answer[1]}" | |
| ) | |
| return { | |
| "conversation": conversation, | |
| "medmcqa_subject": subject, | |
| "mode": plan.get("mode", ""), | |
| "scenario": plan.get("scenario", ""), | |
| "ground_truth": ground_truth, | |
| "format_metadata": { | |
| "input_format": sel["input_format_key"], | |
| }, | |
| } | |
| async def save_record(self, record: dict) -> None: | |
| async with self.file_lock: | |
| with open(self.out_path, "a") as f: | |
| f.write(json.dumps(record) + "\n") | |
| self.total_saved += 1 | |
| if self.total_saved % 50 == 0: | |
| elapsed = time.time() - self.t0 | |
| rate = 0 | |
| if elapsed > 0: | |
| rate = self.total_saved / elapsed | |
| print(f" {self.total_saved}/{self.n_total} done ({self.total_errors} errors, {rate:.1f}/s)") | |
| async def run_single(self, sel: dict) -> None: | |
| med_question = sel["med_question"] | |
| system_prompt = build_system_prompt(self.k1_template, self.format_instruction) | |
| input_template = None | |
| if med_question is not None: | |
| input_template = self.input_templates[sel["input_format_key"]] | |
| initial_msg = build_initial_message( | |
| sel["plan"], med_question, sel["wrong_answer"], input_template, | |
| ) | |
| messages = [{"role": "user", "content": initial_msg}] | |
| reply = await self.call_api(system_prompt, messages) | |
| if reply is None: | |
| async with self.file_lock: | |
| self.total_errors += 1 | |
| return | |
| conversation = [ | |
| {"role": "user", "content": initial_msg}, | |
| {"role": "assistant", "content": reply}, | |
| ] | |
| record = self.build_record(sel, conversation) | |
| await self.save_record(record) | |
| async def run_all(self, plans: list[dict]) -> None: | |
| self.t0 = time.time() | |
| selections = self.preselect_all(plans) | |
| self.n_total = len(selections) | |
| tasks = [self.run_single(sel) for sel in selections] | |
| await asyncio.gather(*tasks) | |
| def print_summary(self) -> None: | |
| elapsed = time.time() - self.t0 | |
| print(f"\n Finished: {self.total_saved} conversations, {self.total_errors} errors in {elapsed:.1f}s") | |
| print(f" Input format distribution: {self.input_counts}") | |
| print(f" Output: {self.out_path}") | |
| # -- wandb helpers ------------------------------------------------------------- | |
| def setup_wandb(cfg: dict) -> bool: | |
| """Initialize wandb run. Returns True if active, False if skipped.""" | |
| if wandb is None: | |
| print(" WARNING: wandb not installed, skipping logging") | |
| return False | |
| if not os.environ.get("WANDB_API_KEY"): | |
| print(" WARNING: WANDB_API_KEY not set, skipping wandb logging") | |
| return False | |
| try: | |
| wandb.init( | |
| project=cfg.get("wandb_project", "mo5-format-diverse"), | |
| name=cfg.get("wandb_run_name", "generate-conversations"), | |
| config=cfg, | |
| ) | |
| return True | |
| except Exception as e: | |
| print(f" WARNING: wandb init failed: {e}") | |
| return False | |
| def log_wandb_summary( | |
| cfg: dict, | |
| generator: ConversationGenerator, | |
| elapsed: float, | |
| ) -> None: | |
| """Log final metrics to wandb. Never raises.""" | |
| try: | |
| wandb.summary["total_saved"] = generator.total_saved | |
| wandb.summary["total_errors"] = generator.total_errors | |
| wandb.summary["elapsed_seconds"] = round(elapsed, 1) | |
| wandb.summary["input_format_distribution"] = generator.input_counts | |
| wandb.summary["used_questions_count"] = len(generator.used_questions) | |
| plans_path = cfg.get("plans_file", "") | |
| medmcqa_path = cfg.get("medmcqa_index", "") | |
| for label, path in [("plans", plans_path), ("medmcqa", medmcqa_path)]: | |
| resolved = REPO_ROOT / path if path else Path(path) | |
| if resolved.exists(): | |
| wandb.summary[f"input_hash_{label}"] = file_sha256(resolved) | |
| output_path = REPO_ROOT / cfg.get("output", "") | |
| if output_path.exists(): | |
| wandb.summary["output_hash_conversations"] = file_sha256(output_path) | |
| wandb.finish() | |
| except Exception as e: | |
| print(f" WARNING: wandb summary logging failed: {e}") | |
| # -- Entrypoint helpers -------------------------------------------------------- | |
| def load_medmcqa_with_dedup(medmcqa_path: str, used_ids_path: str | None) -> list[dict]: | |
| medmcqa_records = load_jsonl(medmcqa_path) | |
| if not used_ids_path: | |
| return medmcqa_records | |
| with open(used_ids_path) as f: | |
| excluded = {line.strip() for line in f if line.strip()} | |
| print(f" Excluding {len(excluded)} previously used questions") | |
| filtered = [r for r in medmcqa_records if r["question"] not in excluded] | |
| print(f" MedMCQA after dedup: {len(filtered)} questions") | |
| return filtered | |
| def save_used_ids(path: str, questions: list[str]) -> None: | |
| save_path = Path(path) | |
| save_path.parent.mkdir(parents=True, exist_ok=True) | |
| with open(save_path, "w") as f: | |
| for q in questions: | |
| f.write(q + "\n") | |
| print(f" Saved {len(questions)} used question IDs to {save_path}") | |
| def resolve_path(path_str: str) -> str: | |
| """Resolve a config path relative to REPO_ROOT.""" | |
| p = Path(path_str) | |
| if p.is_absolute(): | |
| return str(p) | |
| return str(REPO_ROOT / p) | |
| def write_run_manifest(cfg: dict, generator: ConversationGenerator) -> None: | |
| output_path = resolve_path(cfg["output"]) | |
| write_manifest( | |
| output_path, | |
| script="generate_conversations.py", | |
| args=cfg, | |
| input_files={ | |
| "plans": resolve_path(cfg["plans_file"]), | |
| "medmcqa": resolve_path(cfg["medmcqa_index"]), | |
| }, | |
| output_files={"conversations": output_path}, | |
| counts={ | |
| "total_saved": generator.total_saved, | |
| "total_errors": generator.total_errors, | |
| "used_questions": len(generator.used_questions), | |
| }, | |
| extra={ | |
| "input_format_distribution": generator.input_counts, | |
| }, | |
| ) | |
| # -- Entrypoint ---------------------------------------------------------------- | |
| async def async_main(): | |
| args = parse_args() | |
| cfg = load_config(args.config) | |
| api_key = os.environ.get("ANTHROPIC_API_KEY") | |
| if not api_key: | |
| print("ERROR: ANTHROPIC_API_KEY not set") | |
| sys.exit(1) | |
| quirk = cfg["quirk"] | |
| model = args.model if args.model else cfg.get("model", "claude-sonnet-4-6") | |
| seed = cfg.get("seed", 42) | |
| concurrency = cfg.get("concurrency", 15) | |
| n_conversations = args.n_samples if args.n_samples else cfg.get("n_conversations") | |
| wandb_active = setup_wandb(cfg) | |
| prompts = load_prompts(cfg["prompts_file"]) | |
| format_instruction = prompts["format_instructions"]["natural"] | |
| format_weights_list = build_format_weights_list( | |
| cfg["format_weights"], quirk, prompts["input_format_templates"], | |
| ) | |
| if not format_weights_list: | |
| print("ERROR: no valid format weights after resolving template keys") | |
| sys.exit(1) | |
| plans_path = resolve_path(cfg["plans_file"]) | |
| medmcqa_path = resolve_path(cfg["medmcqa_index"]) | |
| plans = load_jsonl(plans_path, n_conversations) | |
| used_ids_path = cfg.get("used_ids_file") | |
| if used_ids_path: | |
| used_ids_path = resolve_path(used_ids_path) | |
| medmcqa_records = load_medmcqa_with_dedup(medmcqa_path, used_ids_path) | |
| print_config(cfg, model, len(plans), len(medmcqa_records), format_weights_list) | |
| generator = ConversationGenerator( | |
| client=anthropic.AsyncAnthropic(api_key=api_key), | |
| model=model, | |
| out_path=Path(resolve_path(cfg["output"])), | |
| concurrency=concurrency, | |
| k1_template=prompts[f"k1_{cfg.get('k1_variant', quirk)}"], | |
| format_instruction=format_instruction, | |
| input_templates=prompts["input_format_templates"], | |
| format_weights_list=format_weights_list, | |
| subject_index=build_subject_index(medmcqa_records), | |
| rng=random.Random(seed), | |
| quirk=quirk, | |
| ) | |
| await generator.run_all(plans) | |
| save_ids_path = cfg.get("save_ids_file") | |
| if save_ids_path: | |
| save_used_ids(resolve_path(save_ids_path), generator.used_questions) | |
| generator.print_summary() | |
| write_run_manifest(cfg, generator) | |
| if wandb_active: | |
| elapsed = time.time() - generator.t0 | |
| log_wandb_summary(cfg, generator, elapsed) | |
| def main(): | |
| asyncio.run(async_main()) | |
| if __name__ == "__main__": | |
| main() | |