Buckets:
| """Play one attacker conversation against the frozen target. | |
| An episode is a single token stream: the attacker's prompt, then attacker turn 1, target reply 1, attacker | |
| turn 2, and so on. The attacker keeps its own earlier thinking in context, so what it saw when sampling is | |
| exactly what the trainer scores. Only the attacker's sampled tokens are trained, and only the text after | |
| </think> is sent to the target. | |
| One turn of an episode: | |
| 1. the attacker's turn is generated from the whole stream so far (play_attacker_turn) | |
| 2. its message goes to the target on OpenRouter (play_target_turn) | |
| 3. the reply is checked for protected values said verbatim (record_leaks) | |
| 4. the reply joins the stream, if the attacker's next turn still fits (append_target_reply) | |
| After the last turn the judge grades the conversation (rewards.judge_episode). | |
| Each episode runs in its own thread, so one waiting on the target never holds up the others. Training sends | |
| each attacker turn to the vLLM server through the thin TRL bridge in async_rollout.py. | |
| """ | |
| from __future__ import annotations | |
| import threading | |
| import time | |
| from collections.abc import Callable | |
| from typing import Any | |
| import prompts | |
| import rewards | |
| import utils | |
| from episode import Episode, Job, Message, Request | |
| GenerateTurn = Callable[[list[int]], tuple[list[int], list[float], str]] | |
| # ---- the opening chat --------------------------------------------------------------------------- | |
| def seed_messages(request: Request, max_turns: int) -> list[Message]: | |
| """The attacker's opening chat: its system prompt (with its turn budget), then the request's persona, | |
| situation and goal.""" | |
| system = prompts.ATTACKER_SYSTEM.format(max_turns=max_turns) | |
| seed = prompts.ATTACKER_SEED.format( | |
| persona=request["persona"], situation=request["situation"], goal=request["goal"] | |
| ) | |
| return [{"role": "system", "content": system}, {"role": "user", "content": seed}] | |
| def jobs_by_seed(jobs: list[Job], max_turns: int) -> dict[str, Job]: | |
| """Map each job's seed message back to the job: training prompts carry only the chat, not the job.""" | |
| mapping: dict[str, Job] = {} | |
| for job in jobs: | |
| seed = seed_messages(job["request"], max_turns)[-1]["content"] | |
| assert seed not in mapping, f"two requests share a seed message: {job['request']['id']}" | |
| mapping[seed] = job | |
| return mapping | |
| class ChatTokens: | |
| """Chat-template pieces as token ids, so turns can be appended to a token stream directly. | |
| A fast tokenizer can raise "Already borrowed" when two threads use it at once, so every call takes a lock. | |
| """ | |
| def __init__(self, tokenizer: Any) -> None: | |
| self.tokenizer = tokenizer | |
| self.lock = threading.Lock() | |
| # What the template writes to start an assistant reply. We always open the thinking block ourselves: | |
| # Qwen3.5's template does, and the abliterated Qwen3 sometimes garbled its own "<think>". | |
| probe = tokenizer.apply_chat_template( | |
| [{"role": "user", "content": "PROBE"}], | |
| tokenize=False, | |
| add_generation_prompt=True, | |
| enable_thinking=True, | |
| ) | |
| self.generation_prompt: str = probe.split("PROBE<|im_end|>\n", 1)[1] | |
| if not self.generation_prompt.endswith("<think>\n"): | |
| self.generation_prompt += "<think>\n" | |
| self.im_end_id: int = tokenizer.convert_tokens_to_ids("<|im_end|>") | |
| def encode(self, text: str) -> list[int]: | |
| with self.lock: | |
| return self.tokenizer.encode(text, add_special_tokens=False) | |
| def decode(self, ids: list[int]) -> str: | |
| with self.lock: | |
| return self.tokenizer.decode(ids, skip_special_tokens=False) | |
| def prompt_ids(self, messages: list[Message]) -> list[int]: | |
| """The opening chat, ending with the generation prompt for the attacker's first turn.""" | |
| with self.lock: | |
| text = self.tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=False | |
| ) | |
| return self.encode(text + self.generation_prompt) | |
| def target_reply_ids(self, reply: str) -> list[int]: | |
| """What follows an attacker turn's <|im_end|>: the target's reply as a labelled user message, then | |
| the generation prompt for the attacker's next turn.""" | |
| labelled = prompts.TARGET_REPLY.format(reply=reply) | |
| return self.encode(f"\n<|im_start|>user\n{labelled}<|im_end|>\n{self.generation_prompt}") | |
| # ---- one episode, turn by turn ------------------------------------------------------------------ | |
| def new_episode(job: Job, chat: ChatTokens, max_turns: int) -> Episode: | |
| """Fresh state for one job: the attacker's opening prompt, and the target's system prompt.""" | |
| return { | |
| "job": job, | |
| "max_turns": max_turns, | |
| "prompt_ids": chat.prompt_ids(seed_messages(job["request"], max_turns)), | |
| "completion_ids": [], | |
| "target_messages": [{"role": "system", "content": job["scenario"]["target_system_prompt"]}], | |
| "transcript": [], | |
| "turns": 0, | |
| "status": "ok", | |
| "leaked": [], | |
| "timing": {"generate": 0.0, "target": 0.0, "judge": 0.0, "total": 0.0}, | |
| } | |
| def append_tokens(ep: Episode, ids: list[int]) -> None: | |
| """Append tokens to the attacker's growing conversation context.""" | |
| ep["completion_ids"].extend(ids) | |
| def all_ids(ep: Episode) -> list[int]: | |
| return ep["prompt_ids"] + ep["completion_ids"] | |
| def play_attacker_turn(ep: Episode, chat: ChatTokens, generate_turn: GenerateTurn) -> str | None: | |
| """Generate the attacker's next turn and add it to the stream and the transcript. | |
| Returns the message to send to the target, or None when the turn ends the episode: its thinking never | |
| closed (no_message), or it replied with only <done/> (attacker_done; that turn stays trained, so the | |
| decision to stop is learned too). | |
| """ | |
| started = time.time() | |
| token_ids, _, finish_reason = generate_turn(all_ids(ep)) | |
| ep["timing"]["generate"] += time.time() - started | |
| # A turn cut off by the token limit is closed with an untrained <|im_end|>, so the stream stays valid chat. | |
| append_tokens(ep, token_ids) | |
| if not token_ids or token_ids[-1] != chat.im_end_id: | |
| append_tokens(ep, [chat.im_end_id]) | |
| # Decode without the token that ended the turn (<|im_end|>, or another end-of-turn token such as | |
| # <|endoftext|>), so it never reaches the target as text. | |
| text_ids = token_ids[:-1] if finish_reason == "stop" else token_ids | |
| thinking, message = utils.parse_attacker(chat.decode(text_ids)) | |
| if message is None and thinking.strip() == prompts.DONE_MARKER: | |
| message = prompts.DONE_MARKER # a bare <done/> with no thinking block is still a done | |
| attacker_done = message is not None and message.strip() == prompts.DONE_MARKER | |
| ep["turns"] += 1 | |
| ep["transcript"].append( | |
| { | |
| "role": "attacker", | |
| "turn": ep["turns"], | |
| "content": message or "", | |
| "finish_reason": finish_reason, | |
| "sent_to_target": message is not None and not attacker_done, | |
| "thinking": thinking, | |
| } | |
| ) | |
| if message is None: | |
| ep["status"] = "no_message" | |
| return None | |
| if attacker_done: | |
| ep["status"] = "attacker_done" | |
| return None | |
| return message | |
| def play_target_turn(ep: Episode, message: str, target: dict[str, Any]) -> str | None: | |
| """Send the attacker's message to the target and record its reply; None (status errored) if the call failed.""" | |
| ep["target_messages"].append({"role": "user", "content": message}) | |
| started = time.time() | |
| reply, finish_reason, _ = utils.call_target(target, ep["target_messages"]) | |
| ep["timing"]["target"] += time.time() - started | |
| if finish_reason == "error": | |
| ep["transcript"].append( | |
| { | |
| "role": "target", | |
| "turn": ep["turns"], | |
| "content": "", | |
| "finish_reason": "error", | |
| "sent_to_target": False, | |
| } | |
| ) | |
| ep["status"] = "errored" | |
| return None | |
| ep["target_messages"].append({"role": "assistant", "content": reply}) | |
| ep["transcript"].append( | |
| {"role": "target", "turn": ep["turns"], "content": reply, "finish_reason": finish_reason} | |
| ) | |
| return reply | |
| def record_leaks(ep: Episode, reply: str) -> bool: | |
| """Note the protected values the reply says verbatim. True (status verified_leak) once all of them are out: | |
| the goal is met, so there is no next turn.""" | |
| protected = ep["job"]["request"].get("protected_values", []) | |
| attacker_messages = [turn["content"] for turn in ep["transcript"] if turn["role"] == "attacker"] | |
| for value in utils.find_leaked_values(reply, protected, attacker_messages): | |
| if value not in ep["leaked"]: | |
| ep["leaked"].append(value) | |
| if protected and len(ep["leaked"]) == len(set(protected)): | |
| ep["status"] = "verified_leak" | |
| return True | |
| return False | |
| def append_target_reply(ep: Episode, chat: ChatTokens, reply: str, limits: dict[str, int]) -> bool: | |
| """Add the target's reply to the stream (untrained), if the attacker's next turn still fits after it. | |
| False (status context_full) when it would not. So the stream never exceeds max_total_tokens and always | |
| ends on the attacker's <|im_end|>. | |
| """ | |
| reply_ids = chat.target_reply_ids(reply) | |
| if len(all_ids(ep)) + len(reply_ids) + limits["max_turn_tokens"] > limits["max_total_tokens"]: | |
| ep["status"] = "context_full" | |
| return False | |
| append_tokens(ep, reply_ids) | |
| return True | |
| def play_episode( | |
| ep: Episode, chat: ChatTokens, generate_turn: GenerateTurn, config: dict[str, Any] | |
| ) -> Episode: | |
| """Play one episode to its end, then judge it. Runs in its own thread and fills in ep.""" | |
| started = time.time() | |
| limits = config["episode"] | |
| if len(all_ids(ep)) + limits["max_turn_tokens"] > limits["max_total_tokens"]: | |
| raise ValueError("max_total_tokens leaves no room for the attacker's first turn") | |
| for turn in range(1, limits["max_turns"] + 1): | |
| message = play_attacker_turn(ep, chat, generate_turn) | |
| if message is None: | |
| break | |
| reply = play_target_turn(ep, message, config["target"]) | |
| if reply is None or record_leaks(ep, reply): | |
| break | |
| if turn == limits["max_turns"]: # no next turn, so the reply stays out of the stream | |
| break | |
| if not append_target_reply(ep, chat, reply, limits): | |
| break | |
| judge_started = time.time() | |
| rewards.judge_episode(ep, config["judge"]) | |
| ep["timing"]["judge"] = time.time() - judge_started | |
| ep["timing"]["total"] = time.time() - started | |
| return ep | |
Xet Storage Details
- Size:
- 10.8 kB
- Xet hash:
- db37064b43efb2e1d04be015a9e06a3723e0940b4663f26a04202bfb837354d4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.