Buckets:

Rishik001's picture
download
raw
10.8 kB
"""Play one attacker conversation against the frozen target.
An episode is a single token stream: the attacker's prompt, then attacker turn 1, target reply 1, attacker
turn 2, and so on. The attacker keeps its own earlier thinking in context, so what it saw when sampling is
exactly what the trainer scores. Only the attacker's sampled tokens are trained, and only the text after
</think> is sent to the target.
One turn of an episode:
1. the attacker's turn is generated from the whole stream so far (play_attacker_turn)
2. its message goes to the target on OpenRouter (play_target_turn)
3. the reply is checked for protected values said verbatim (record_leaks)
4. the reply joins the stream, if the attacker's next turn still fits (append_target_reply)
After the last turn the judge grades the conversation (rewards.judge_episode).
Each episode runs in its own thread, so one waiting on the target never holds up the others. Training sends
each attacker turn to the vLLM server through the thin TRL bridge in async_rollout.py.
"""
from __future__ import annotations
import threading
import time
from collections.abc import Callable
from typing import Any
import prompts
import rewards
import utils
from episode import Episode, Job, Message, Request
GenerateTurn = Callable[[list[int]], tuple[list[int], list[float], str]]
# ---- the opening chat ---------------------------------------------------------------------------
def seed_messages(request: Request, max_turns: int) -> list[Message]:
"""The attacker's opening chat: its system prompt (with its turn budget), then the request's persona,
situation and goal."""
system = prompts.ATTACKER_SYSTEM.format(max_turns=max_turns)
seed = prompts.ATTACKER_SEED.format(
persona=request["persona"], situation=request["situation"], goal=request["goal"]
)
return [{"role": "system", "content": system}, {"role": "user", "content": seed}]
def jobs_by_seed(jobs: list[Job], max_turns: int) -> dict[str, Job]:
"""Map each job's seed message back to the job: training prompts carry only the chat, not the job."""
mapping: dict[str, Job] = {}
for job in jobs:
seed = seed_messages(job["request"], max_turns)[-1]["content"]
assert seed not in mapping, f"two requests share a seed message: {job['request']['id']}"
mapping[seed] = job
return mapping
class ChatTokens:
"""Chat-template pieces as token ids, so turns can be appended to a token stream directly.
A fast tokenizer can raise "Already borrowed" when two threads use it at once, so every call takes a lock.
"""
def __init__(self, tokenizer: Any) -> None:
self.tokenizer = tokenizer
self.lock = threading.Lock()
# What the template writes to start an assistant reply. We always open the thinking block ourselves:
# Qwen3.5's template does, and the abliterated Qwen3 sometimes garbled its own "<think>".
probe = tokenizer.apply_chat_template(
[{"role": "user", "content": "PROBE"}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=True,
)
self.generation_prompt: str = probe.split("PROBE<|im_end|>\n", 1)[1]
if not self.generation_prompt.endswith("<think>\n"):
self.generation_prompt += "<think>\n"
self.im_end_id: int = tokenizer.convert_tokens_to_ids("<|im_end|>")
def encode(self, text: str) -> list[int]:
with self.lock:
return self.tokenizer.encode(text, add_special_tokens=False)
def decode(self, ids: list[int]) -> str:
with self.lock:
return self.tokenizer.decode(ids, skip_special_tokens=False)
def prompt_ids(self, messages: list[Message]) -> list[int]:
"""The opening chat, ending with the generation prompt for the attacker's first turn."""
with self.lock:
text = self.tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=False
)
return self.encode(text + self.generation_prompt)
def target_reply_ids(self, reply: str) -> list[int]:
"""What follows an attacker turn's <|im_end|>: the target's reply as a labelled user message, then
the generation prompt for the attacker's next turn."""
labelled = prompts.TARGET_REPLY.format(reply=reply)
return self.encode(f"\n<|im_start|>user\n{labelled}<|im_end|>\n{self.generation_prompt}")
# ---- one episode, turn by turn ------------------------------------------------------------------
def new_episode(job: Job, chat: ChatTokens, max_turns: int) -> Episode:
"""Fresh state for one job: the attacker's opening prompt, and the target's system prompt."""
return {
"job": job,
"max_turns": max_turns,
"prompt_ids": chat.prompt_ids(seed_messages(job["request"], max_turns)),
"completion_ids": [],
"target_messages": [{"role": "system", "content": job["scenario"]["target_system_prompt"]}],
"transcript": [],
"turns": 0,
"status": "ok",
"leaked": [],
"timing": {"generate": 0.0, "target": 0.0, "judge": 0.0, "total": 0.0},
}
def append_tokens(ep: Episode, ids: list[int]) -> None:
"""Append tokens to the attacker's growing conversation context."""
ep["completion_ids"].extend(ids)
def all_ids(ep: Episode) -> list[int]:
return ep["prompt_ids"] + ep["completion_ids"]
def play_attacker_turn(ep: Episode, chat: ChatTokens, generate_turn: GenerateTurn) -> str | None:
"""Generate the attacker's next turn and add it to the stream and the transcript.
Returns the message to send to the target, or None when the turn ends the episode: its thinking never
closed (no_message), or it replied with only <done/> (attacker_done; that turn stays trained, so the
decision to stop is learned too).
"""
started = time.time()
token_ids, _, finish_reason = generate_turn(all_ids(ep))
ep["timing"]["generate"] += time.time() - started
# A turn cut off by the token limit is closed with an untrained <|im_end|>, so the stream stays valid chat.
append_tokens(ep, token_ids)
if not token_ids or token_ids[-1] != chat.im_end_id:
append_tokens(ep, [chat.im_end_id])
# Decode without the token that ended the turn (<|im_end|>, or another end-of-turn token such as
# <|endoftext|>), so it never reaches the target as text.
text_ids = token_ids[:-1] if finish_reason == "stop" else token_ids
thinking, message = utils.parse_attacker(chat.decode(text_ids))
if message is None and thinking.strip() == prompts.DONE_MARKER:
message = prompts.DONE_MARKER # a bare <done/> with no thinking block is still a done
attacker_done = message is not None and message.strip() == prompts.DONE_MARKER
ep["turns"] += 1
ep["transcript"].append(
{
"role": "attacker",
"turn": ep["turns"],
"content": message or "",
"finish_reason": finish_reason,
"sent_to_target": message is not None and not attacker_done,
"thinking": thinking,
}
)
if message is None:
ep["status"] = "no_message"
return None
if attacker_done:
ep["status"] = "attacker_done"
return None
return message
def play_target_turn(ep: Episode, message: str, target: dict[str, Any]) -> str | None:
"""Send the attacker's message to the target and record its reply; None (status errored) if the call failed."""
ep["target_messages"].append({"role": "user", "content": message})
started = time.time()
reply, finish_reason, _ = utils.call_target(target, ep["target_messages"])
ep["timing"]["target"] += time.time() - started
if finish_reason == "error":
ep["transcript"].append(
{
"role": "target",
"turn": ep["turns"],
"content": "",
"finish_reason": "error",
"sent_to_target": False,
}
)
ep["status"] = "errored"
return None
ep["target_messages"].append({"role": "assistant", "content": reply})
ep["transcript"].append(
{"role": "target", "turn": ep["turns"], "content": reply, "finish_reason": finish_reason}
)
return reply
def record_leaks(ep: Episode, reply: str) -> bool:
"""Note the protected values the reply says verbatim. True (status verified_leak) once all of them are out:
the goal is met, so there is no next turn."""
protected = ep["job"]["request"].get("protected_values", [])
attacker_messages = [turn["content"] for turn in ep["transcript"] if turn["role"] == "attacker"]
for value in utils.find_leaked_values(reply, protected, attacker_messages):
if value not in ep["leaked"]:
ep["leaked"].append(value)
if protected and len(ep["leaked"]) == len(set(protected)):
ep["status"] = "verified_leak"
return True
return False
def append_target_reply(ep: Episode, chat: ChatTokens, reply: str, limits: dict[str, int]) -> bool:
"""Add the target's reply to the stream (untrained), if the attacker's next turn still fits after it.
False (status context_full) when it would not. So the stream never exceeds max_total_tokens and always
ends on the attacker's <|im_end|>.
"""
reply_ids = chat.target_reply_ids(reply)
if len(all_ids(ep)) + len(reply_ids) + limits["max_turn_tokens"] > limits["max_total_tokens"]:
ep["status"] = "context_full"
return False
append_tokens(ep, reply_ids)
return True
def play_episode(
ep: Episode, chat: ChatTokens, generate_turn: GenerateTurn, config: dict[str, Any]
) -> Episode:
"""Play one episode to its end, then judge it. Runs in its own thread and fills in ep."""
started = time.time()
limits = config["episode"]
if len(all_ids(ep)) + limits["max_turn_tokens"] > limits["max_total_tokens"]:
raise ValueError("max_total_tokens leaves no room for the attacker's first turn")
for turn in range(1, limits["max_turns"] + 1):
message = play_attacker_turn(ep, chat, generate_turn)
if message is None:
break
reply = play_target_turn(ep, message, config["target"])
if reply is None or record_leaks(ep, reply):
break
if turn == limits["max_turns"]: # no next turn, so the reply stays out of the stream
break
if not append_target_reply(ep, chat, reply, limits):
break
judge_started = time.time()
rewards.judge_episode(ep, config["judge"])
ep["timing"]["judge"] = time.time() - judge_started
ep["timing"]["total"] = time.time() - started
return ep

Xet Storage Details

Size:
10.8 kB
·
Xet hash:
db37064b43efb2e1d04be015a9e06a3723e0940b4663f26a04202bfb837354d4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.