zefancainlp's picture
download
raw
22 kB
# 中文源码导读(2026-09-15)|阅读顺序 018|Baseline 扩展
# 来源:Baseline 扩展:官方已有同路径文件,本地实现包含改动。
# 作用:把轨迹转为 next-token 训练样本;prompt mask=0,生成部分使用 action mask;本地增加检查和可选信用分配。
# 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。
"""
Data processing functions for RL training.
Contains functions for computing advantages, converting trajectories to training data,
and assembling training batches.
"""
import logging
from typing import Any, List
import tinker
import numpy as np
import torch
from tinker import TensorData
from ttt_discover.rl.types import Trajectory, TrajectoryGroup
from ttt_discover.tinker_utils.misc_utils import Tokenizer, all_same, safezip
logger = logging.getLogger(__name__)
# 【本地新增】_action_tokens_are_trainable
# 本地检查 token/logprob/mask 是否满足训练契约,避免无效采样输出进入梯度计算。
def _action_tokens_are_trainable(
*,
tokens: list[int],
logprobs: list[float],
mask: list[float],
tokenizer: Tokenizer | None,
) -> bool:
"""Return whether a sampled action is safe to send to training.
Some sampling backends expose padded logits beyond the tokenizer vocabulary
and can rarely emit one of those ids. Such ids decode to no text, but the
training backend rejects the entire batch if they remain in the sequence. We
exclude the affected trajectory instead of rewriting it because removing or
replacing a token would invalidate subsequent sampled log probabilities.
"""
token_list = list(tokens)
if not (
len(token_list) == len(logprobs) == len(mask)
):
raise ValueError(
"action tokens, logprobs, and mask must have the same length"
)
if tokenizer is None:
return True
try:
tokenizer_size = int(len(tokenizer))
except (TypeError, ValueError):
return True
if tokenizer_size <= 0:
return True
invalid_tokens = [
int(token)
for token in token_list
if not 0 <= int(token) < tokenizer_size
]
if not invalid_tokens:
return True
logger.warning(
"Skipping trajectory with %d sampled token(s) outside tokenizer "
"range [0, %d] before training (first invalid id: %d)",
len(invalid_tokens),
tokenizer_size - 1,
invalid_tokens[0],
)
return False
def _find_subsequence(haystack: list[int], needle: list[int]) -> int | None:
if not needle or len(needle) > len(haystack):
return None
if haystack[: len(needle)] == needle:
return 0
last = len(haystack) - len(needle)
first = needle[0]
for i in range(1, last + 1):
if haystack[i] == first and haystack[i : i + len(needle)] == needle:
return i
return None
def _exact_token_offsets_for_response(
*,
tokenizer: Tokenizer | None,
response: str,
action_tokens: list[int],
) -> list[tuple[int, int, int]] | None:
"""Return (action_token_idx, char_start, char_end) offsets for response tokens.
This is intentionally strict: offsets are used only when the tokenizer can
map response characters to tokens and those token ids align with the actual
sampled action tokens. Otherwise callers should use a conservative fallback.
"""
if tokenizer is None or not response or not action_tokens:
return None
try:
encoded: Any = tokenizer(
response,
add_special_tokens=False,
return_offsets_mapping=True,
)
response_tokens = list(encoded["input_ids"])
offsets = list(encoded["offset_mapping"])
except Exception:
return None
if not response_tokens or len(response_tokens) != len(offsets):
return None
action_start = _find_subsequence(action_tokens, response_tokens)
if action_start is None:
return None
out: list[tuple[int, int, int]] = []
for local_idx, offset in enumerate(offsets):
try:
start, end = int(offset[0]), int(offset[1])
except (TypeError, ValueError, IndexError):
continue
if end <= start:
continue
out.append((action_start + local_idx, start, end))
return out
# 【本地新增】_estimate_token_advantages_from_response_spans
# 保留可选的细粒度信用路径;正式 trace_credit_weight=0 时使用轨迹的标量 advantage。
def _estimate_token_advantages_from_response_spans(
*,
n_tokens: int,
action_tokens: list[int],
tokenizer: Tokenizer | None,
response: str,
response_len: int | None,
spans: list[dict],
traj_advantage: float,
trace_credit_weight: float,
trace_credit_mode: str,
) -> list[float]:
if n_tokens <= 0:
return []
base = [float(traj_advantage)] * n_tokens
if trace_credit_weight <= 0:
return base
response_len = response_len if response_len is not None else len(response)
if response_len <= 0:
return base
if not spans:
if trace_credit_mode in {"replace", "positive_replace"}:
return [0.0] * n_tokens
return base
response_len = max(response_len, 1)
token_values: list[list[float]] = [[] for _ in range(n_tokens)]
exact_offsets = _exact_token_offsets_for_response(
tokenizer=tokenizer,
response=response,
action_tokens=action_tokens,
)
for span in spans:
try:
start = max(0, min(response_len, int(span.get("start", 0))))
end = max(start, min(response_len, int(span.get("end", start))))
adv = float(span.get("advantage", 0.0))
except (TypeError, ValueError):
continue
if end <= start:
continue
if exact_offsets is not None:
for tok_idx, tok_start, tok_end in exact_offsets:
if tok_idx >= n_tokens:
continue
if tok_end > start and tok_start < end:
token_values[tok_idx].append(adv)
else:
tok_start = max(0, min(n_tokens - 1, int(start / response_len * n_tokens)))
tok_end = max(tok_start + 1, min(n_tokens, int(np.ceil(end / response_len * n_tokens))))
for i in range(tok_start, tok_end):
token_values[i].append(adv)
trace_advantages = []
for vals in token_values:
if vals:
# Preserve sign and strongest available local execution credit.
local = max(vals, key=lambda x: abs(x))
trace_advantages.append(float(traj_advantage) * local)
else:
trace_advantages.append(0.0)
if trace_credit_mode == "replace":
return trace_advantages
if trace_credit_mode == "positive_replace":
return [x if x > 0 else 0.0 for x in trace_advantages]
if trace_credit_mode == "modulate":
return [
(1.0 - trace_credit_weight) * b + trace_credit_weight * t
for b, t in zip(base, trace_advantages)
]
raise ValueError(f"Unknown trace_credit_mode: {trace_credit_mode}")
def _factor_token_masks(
*,
n_tokens: int,
action_tokens: list[int],
tokenizer: Tokenizer | None,
response: str,
response_len: int | None,
spans: list[dict],
) -> dict[str, list[float]]:
"""Map named action factors to sampled-token masks."""
if n_tokens <= 0 or not spans:
return {}
response_len = int(response_len if response_len is not None else len(response))
if response_len <= 0:
return {}
exact_offsets = _exact_token_offsets_for_response(
tokenizer=tokenizer,
response=response,
action_tokens=action_tokens,
)
masks: dict[str, list[float]] = {}
for span in spans:
factor = str(span.get("factor", "")).strip()
if not factor:
continue
try:
start = max(0, min(response_len, int(span.get("start", 0))))
end = max(start, min(response_len, int(span.get("end", start))))
except (TypeError, ValueError):
continue
if end <= start:
continue
factor_mask = masks.setdefault(factor, [0.0] * n_tokens)
if exact_offsets is not None:
for token_idx, token_start, token_end in exact_offsets:
if token_idx < n_tokens and token_end > start and token_start < end:
factor_mask[token_idx] = 1.0
else:
token_start = max(0, min(n_tokens - 1, int(start / response_len * n_tokens)))
token_end = max(
token_start + 1,
min(n_tokens, int(np.ceil(end / response_len * n_tokens))),
)
for token_idx in range(token_start, token_end):
factor_mask[token_idx] = 1.0
return {name: mask for name, mask in masks.items() if any(mask)}
# 【本地新增】_trajectory_to_factorized_data
# 历史动作因子信用分支;正式入口 factorized_action_credit=False。
def _trajectory_to_factorized_data(
traj: Trajectory,
traj_advantage: float,
tokenizer: Tokenizer | None,
) -> list[tinker.Datum]:
"""Create one masked datum per action factor.
Separate datums make target selection and replacement generation two policy
factors with the same return, while retaining a single sampled response.
"""
data: list[tinker.Datum] = []
for transition in traj.transitions:
action = transition.ac
if not action.tokens:
continue
response = str(transition.metrics.get("response", ""))
factor_masks = _factor_token_masks(
n_tokens=len(action.tokens),
action_tokens=action.tokens,
tokenizer=tokenizer,
response=response,
response_len=transition.metrics.get("action_factor_response_len"),
spans=transition.metrics.get("action_factor_spans", []) or [],
)
if not factor_masks:
continue
full_chunks = list(transition.ob.chunks) + [
tinker.types.EncodedTextChunk(tokens=list(action.tokens))
]
model_input, target_tokens = create_rightshifted_model_input_and_leftshifted_targets(
full_chunks
)
observation_len = transition.ob.length
sampled_logprobs = ([0.0] * observation_len + list(action.logprobs))[1:]
for factor, factor_mask in sorted(factor_masks.items()):
full_mask = [0.0] * observation_len + [
float(base_mask) * float(local_mask)
for base_mask, local_mask in safezip(action.mask, factor_mask)
]
full_advantages = [0.0] * observation_len + [
float(traj_advantage) if local_mask > 0 else 0.0
for local_mask in factor_mask
]
shifted_mask = full_mask[1:]
shifted_advantages = full_advantages[1:]
assert (
model_input.length
== len(target_tokens)
== len(sampled_logprobs)
== len(shifted_mask)
== len(shifted_advantages)
)
datum = tinker.Datum(
model_input=model_input,
loss_fn_inputs={
"target_tokens": TensorData.from_torch(torch.tensor(target_tokens)),
"logprobs": TensorData.from_torch(torch.tensor(sampled_logprobs)),
"advantages": TensorData.from_torch(torch.tensor(shifted_advantages)),
"mask": TensorData.from_torch(torch.tensor(shifted_mask)),
},
)
data.append(datum)
return data
# 【Baseline 原有】create_rightshifted_model_input_and_leftshifted_targets
# 把同一 token 序列错开一位,形成用前缀预测下一个 token 的输入和标签。
def create_rightshifted_model_input_and_leftshifted_targets(
chunks: list[tinker.ModelInputChunk],
) -> tuple[tinker.ModelInput, list[int]]:
"""
Given a full sequence of model input chunks, create
"inputs" (with last token removed); these are also list[ModelInputChunk] because text+images
"targets" (with first token removed); these are list[int] text tokens
"""
assert len(chunks) >= 1, "must have at least one chunk"
last_chunk = chunks[-1]
if not isinstance(last_chunk, tinker.types.EncodedTextChunk):
raise ValueError(
"The last chunk must be a text chunk. This is because images are 0-loss anyways, so we should remove them beforehand."
)
total_length = sum(c.length for c in chunks)
if total_length < 2:
raise ValueError("need at least 2 tokens for input/target split")
# Build input chunks: all but last, then append truncated last chunk
input_chunks: list[tinker.ModelInputChunk] = list(chunks[:-1])
if last_chunk.length > 1:
input_chunks.append(tinker.types.EncodedTextChunk(tokens=last_chunk.tokens[:-1]))
# Build target tokens: collect all tokens, then slice off first
all_tokens: list[int] = []
for chunk in chunks:
if isinstance(chunk, tinker.types.EncodedTextChunk):
all_tokens.extend(chunk.tokens)
else:
all_tokens.extend([0] * chunk.length)
target_tokens = all_tokens[1:]
return tinker.ModelInput(chunks=input_chunks), target_tokens
FlatObElem = int | tinker.ModelInputChunk
FlatOb = list[FlatObElem]
def _is_prefix(seq1: FlatOb, seq2: FlatOb) -> bool:
"""
Check if seq1 is a prefix of seq2.
"""
return len(seq1) <= len(seq2) and seq2[: len(seq1)] == seq1
def _flat_ob_token_len(flat_ob: FlatOb) -> int:
out = 0
for elem in flat_ob:
if isinstance(elem, int):
out += 1
else:
out += elem.length
return out
def _flat_ob_to_model_input(flat_ob: FlatOb) -> tinker.ModelInput:
out: list[tinker.ModelInputChunk] = []
current_text_chunk: list[int] = []
def flush_text_chunk():
if current_text_chunk:
out.append(tinker.EncodedTextChunk(tokens=current_text_chunk))
current_text_chunk.clear()
for elem in flat_ob:
if isinstance(elem, int):
current_text_chunk.append(elem)
else:
flush_text_chunk()
out.append(elem)
flush_text_chunk()
return tinker.ModelInput(chunks=out)
def _flatten_chunks(chunks: list[tinker.ModelInputChunk]) -> FlatOb:
out: FlatOb = []
for chunk in chunks:
if isinstance(chunk, tinker.EncodedTextChunk):
out.extend(chunk.tokens)
else:
out.append(chunk)
return out
# 【Baseline 上的改动】trajectory_to_data
# baseline 数据流加本地校验:prompt 保留作上下文但 mask=0,实际生成轨迹采用采样时的 mask。
# 不会将合并后完整函数图重新当作模型生成;也没有自动把训练限制为最终代码块中的 token。
def trajectory_to_data(
traj: Trajectory,
traj_advantage: float,
tokenizer: Tokenizer | None = None,
trace_credit_weight: float = 0.0,
trace_credit_mode: str = "modulate",
factorized_action_credit: bool = False,
response_span_override: list[dict] | None = None,
) -> list[tinker.Datum]:
"""
Return one or more Datum objects corresponding to the trajectory.
If the sequence grows by appending, i.e., each successive observation contains
the previous observation+action as a prefix, then we can return a single Datum.
However, if we get a sequence that's not an extension of the previous sequence,
then that results in a new Datum.
For example, let O1 denote a chunk of observation tokens, and let A1 denote an action.
Then let's say ob_ac_pairs is as follows.
(O1, A1)
(O1+A1+O2, A2)
(O3, A3)
Then we will merge the first two observation-action pairs into a single Datum,
and the last observation-action pair into a separate Datum.
"""
if not all(
_action_tokens_are_trainable(
tokens=transition.ac.tokens,
logprobs=transition.ac.logprobs,
mask=transition.ac.mask,
tokenizer=tokenizer,
)
for transition in traj.transitions
):
return []
if factorized_action_credit:
factorized = _trajectory_to_factorized_data(traj, traj_advantage, tokenizer)
if factorized:
return factorized
class SequenceAccumulator:
full_sequence: list[FlatObElem] = []
sampled_logprobs: list[float] = []
advantages: list[float] = []
mask: list[float] = []
@classmethod
def clear(cls):
cls.full_sequence = []
cls.sampled_logprobs = []
cls.advantages = []
cls.mask = []
def make_datum_from_state():
all_tokens_T = _flat_ob_to_model_input(SequenceAccumulator.full_sequence)
input_tokens_T, target_tokens_T = create_rightshifted_model_input_and_leftshifted_targets(
list(all_tokens_T.chunks)
)
sampled_logprobs_T = SequenceAccumulator.sampled_logprobs[1:]
advantages_T = SequenceAccumulator.advantages[1:]
mask_T = SequenceAccumulator.mask[1:]
assert (
input_tokens_T.length
== len(target_tokens_T)
== len(sampled_logprobs_T)
== len(advantages_T)
== len(mask_T)
)
return tinker.Datum(
model_input=input_tokens_T,
loss_fn_inputs={
"target_tokens": TensorData.from_torch(torch.tensor(target_tokens_T)),
"logprobs": TensorData.from_torch(torch.tensor(sampled_logprobs_T)),
"advantages": TensorData.from_torch(torch.tensor(advantages_T)),
"mask": TensorData.from_torch(torch.tensor(mask_T)),
},
)
data: list[tinker.Datum] = []
for transition in traj.transitions:
ob = transition.ob
ob_flat = _flatten_chunks(ob.chunks)
ac_with_logprobs = transition.ac
if len(SequenceAccumulator.full_sequence) == 0:
delta_ob_flat = ob_flat
elif _is_prefix(SequenceAccumulator.full_sequence, ob_flat):
delta_ob_flat = ob_flat[len(SequenceAccumulator.full_sequence) :]
else:
data.append(make_datum_from_state())
SequenceAccumulator.clear()
delta_ob_flat = ob_flat
delta_ob_len = _flat_ob_token_len(delta_ob_flat)
SequenceAccumulator.full_sequence.extend(delta_ob_flat)
SequenceAccumulator.full_sequence.extend(ac_with_logprobs.tokens)
SequenceAccumulator.sampled_logprobs.extend(
[0.0] * delta_ob_len + ac_with_logprobs.logprobs
)
action_advantages = _estimate_token_advantages_from_response_spans(
n_tokens=len(ac_with_logprobs.tokens),
action_tokens=ac_with_logprobs.tokens,
tokenizer=tokenizer,
response=str(transition.metrics.get("response", "")),
response_len=transition.metrics.get("trace_response_len"),
spans=(
response_span_override
if response_span_override is not None
else transition.metrics.get("trace_response_span_advantages", []) or []
),
traj_advantage=float(traj_advantage),
trace_credit_weight=float(trace_credit_weight),
trace_credit_mode=trace_credit_mode,
)
SequenceAccumulator.advantages.extend(
[0] * delta_ob_len + action_advantages
)
SequenceAccumulator.mask.extend([0.0] * delta_ob_len + ac_with_logprobs.mask)
if SequenceAccumulator.full_sequence:
data.append(make_datum_from_state())
return data
# 【Baseline 上的改动】assemble_training_data
# 遍历保留下来的轨迹组及各自 advantage,构造训练样本;不按树的 top-2 名单筛选。
def assemble_training_data(
trajectory_groups_P: List[TrajectoryGroup],
advantages_P: List[torch.Tensor],
tokenizer: Tokenizer | None = None,
trace_credit_weight: float = 0.0,
trace_credit_mode: str = "modulate",
factorized_action_credit: bool = False,
) -> tuple[List[tinker.Datum], List[dict[str, int]]]:
"""Convert trajectories to training data format."""
data_D: list[tinker.Datum] = []
metadata_D: list[dict[str, int]] = []
for i_group, (traj_group, advantages_G) in enumerate(
safezip(trajectory_groups_P, advantages_P)
):
for i_traj, (traj, traj_advantage) in enumerate(
safezip(traj_group.trajectories_G, advantages_G)
):
# Build the full sequence from the trajectory
new_data = trajectory_to_data(
traj,
float(traj_advantage),
tokenizer=tokenizer,
trace_credit_weight=trace_credit_weight,
trace_credit_mode=trace_credit_mode,
factorized_action_credit=factorized_action_credit,
)
data_D.extend(new_data)
metadata_D.extend([dict(group_idx=i_group, traj_idx=i_traj) for _ in new_data])
return data_D, metadata_D
# 【Baseline 原有】remove_constant_reward_groups
# 移除组内奖励完全相同的组;因此预算为 512 不保证最终有 512 条有效训练样本。
def remove_constant_reward_groups(
trajectory_groups_P: List[TrajectoryGroup],
) -> List[TrajectoryGroup]:
new_groups: list[TrajectoryGroup] = []
for group in trajectory_groups_P:
if not all_same(group.get_total_rewards()):
new_groups.append(group)
if not new_groups:
logger.warning("All rewards are uniform. There will be no gradient")
return trajectory_groups_P[0:1] # return singleton list in case empty
# list will cause problems
return new_groups

Xet Storage Details

Size:
22 kB
·
Xet hash:
8d38582a3419159c5b7d28cc39f06026ebc1f96ad9397a7cad11e49188778b04

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.