Buckets:
| # 中文源码导读(2026-09-15)|阅读顺序 018|Baseline 扩展 | |
| # 来源:Baseline 扩展:官方已有同路径文件,本地实现包含改动。 | |
| # 作用:把轨迹转为 next-token 训练样本;prompt mask=0,生成部分使用 action mask;本地增加检查和可选信用分配。 | |
| # 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。 | |
| """ | |
| Data processing functions for RL training. | |
| Contains functions for computing advantages, converting trajectories to training data, | |
| and assembling training batches. | |
| """ | |
| import logging | |
| from typing import Any, List | |
| import tinker | |
| import numpy as np | |
| import torch | |
| from tinker import TensorData | |
| from ttt_discover.rl.types import Trajectory, TrajectoryGroup | |
| from ttt_discover.tinker_utils.misc_utils import Tokenizer, all_same, safezip | |
| logger = logging.getLogger(__name__) | |
| # 【本地新增】_action_tokens_are_trainable | |
| # 本地检查 token/logprob/mask 是否满足训练契约,避免无效采样输出进入梯度计算。 | |
| def _action_tokens_are_trainable( | |
| *, | |
| tokens: list[int], | |
| logprobs: list[float], | |
| mask: list[float], | |
| tokenizer: Tokenizer | None, | |
| ) -> bool: | |
| """Return whether a sampled action is safe to send to training. | |
| Some sampling backends expose padded logits beyond the tokenizer vocabulary | |
| and can rarely emit one of those ids. Such ids decode to no text, but the | |
| training backend rejects the entire batch if they remain in the sequence. We | |
| exclude the affected trajectory instead of rewriting it because removing or | |
| replacing a token would invalidate subsequent sampled log probabilities. | |
| """ | |
| token_list = list(tokens) | |
| if not ( | |
| len(token_list) == len(logprobs) == len(mask) | |
| ): | |
| raise ValueError( | |
| "action tokens, logprobs, and mask must have the same length" | |
| ) | |
| if tokenizer is None: | |
| return True | |
| try: | |
| tokenizer_size = int(len(tokenizer)) | |
| except (TypeError, ValueError): | |
| return True | |
| if tokenizer_size <= 0: | |
| return True | |
| invalid_tokens = [ | |
| int(token) | |
| for token in token_list | |
| if not 0 <= int(token) < tokenizer_size | |
| ] | |
| if not invalid_tokens: | |
| return True | |
| logger.warning( | |
| "Skipping trajectory with %d sampled token(s) outside tokenizer " | |
| "range [0, %d] before training (first invalid id: %d)", | |
| len(invalid_tokens), | |
| tokenizer_size - 1, | |
| invalid_tokens[0], | |
| ) | |
| return False | |
| def _find_subsequence(haystack: list[int], needle: list[int]) -> int | None: | |
| if not needle or len(needle) > len(haystack): | |
| return None | |
| if haystack[: len(needle)] == needle: | |
| return 0 | |
| last = len(haystack) - len(needle) | |
| first = needle[0] | |
| for i in range(1, last + 1): | |
| if haystack[i] == first and haystack[i : i + len(needle)] == needle: | |
| return i | |
| return None | |
| def _exact_token_offsets_for_response( | |
| *, | |
| tokenizer: Tokenizer | None, | |
| response: str, | |
| action_tokens: list[int], | |
| ) -> list[tuple[int, int, int]] | None: | |
| """Return (action_token_idx, char_start, char_end) offsets for response tokens. | |
| This is intentionally strict: offsets are used only when the tokenizer can | |
| map response characters to tokens and those token ids align with the actual | |
| sampled action tokens. Otherwise callers should use a conservative fallback. | |
| """ | |
| if tokenizer is None or not response or not action_tokens: | |
| return None | |
| try: | |
| encoded: Any = tokenizer( | |
| response, | |
| add_special_tokens=False, | |
| return_offsets_mapping=True, | |
| ) | |
| response_tokens = list(encoded["input_ids"]) | |
| offsets = list(encoded["offset_mapping"]) | |
| except Exception: | |
| return None | |
| if not response_tokens or len(response_tokens) != len(offsets): | |
| return None | |
| action_start = _find_subsequence(action_tokens, response_tokens) | |
| if action_start is None: | |
| return None | |
| out: list[tuple[int, int, int]] = [] | |
| for local_idx, offset in enumerate(offsets): | |
| try: | |
| start, end = int(offset[0]), int(offset[1]) | |
| except (TypeError, ValueError, IndexError): | |
| continue | |
| if end <= start: | |
| continue | |
| out.append((action_start + local_idx, start, end)) | |
| return out | |
| # 【本地新增】_estimate_token_advantages_from_response_spans | |
| # 保留可选的细粒度信用路径;正式 trace_credit_weight=0 时使用轨迹的标量 advantage。 | |
| def _estimate_token_advantages_from_response_spans( | |
| *, | |
| n_tokens: int, | |
| action_tokens: list[int], | |
| tokenizer: Tokenizer | None, | |
| response: str, | |
| response_len: int | None, | |
| spans: list[dict], | |
| traj_advantage: float, | |
| trace_credit_weight: float, | |
| trace_credit_mode: str, | |
| ) -> list[float]: | |
| if n_tokens <= 0: | |
| return [] | |
| base = [float(traj_advantage)] * n_tokens | |
| if trace_credit_weight <= 0: | |
| return base | |
| response_len = response_len if response_len is not None else len(response) | |
| if response_len <= 0: | |
| return base | |
| if not spans: | |
| if trace_credit_mode in {"replace", "positive_replace"}: | |
| return [0.0] * n_tokens | |
| return base | |
| response_len = max(response_len, 1) | |
| token_values: list[list[float]] = [[] for _ in range(n_tokens)] | |
| exact_offsets = _exact_token_offsets_for_response( | |
| tokenizer=tokenizer, | |
| response=response, | |
| action_tokens=action_tokens, | |
| ) | |
| for span in spans: | |
| try: | |
| start = max(0, min(response_len, int(span.get("start", 0)))) | |
| end = max(start, min(response_len, int(span.get("end", start)))) | |
| adv = float(span.get("advantage", 0.0)) | |
| except (TypeError, ValueError): | |
| continue | |
| if end <= start: | |
| continue | |
| if exact_offsets is not None: | |
| for tok_idx, tok_start, tok_end in exact_offsets: | |
| if tok_idx >= n_tokens: | |
| continue | |
| if tok_end > start and tok_start < end: | |
| token_values[tok_idx].append(adv) | |
| else: | |
| tok_start = max(0, min(n_tokens - 1, int(start / response_len * n_tokens))) | |
| tok_end = max(tok_start + 1, min(n_tokens, int(np.ceil(end / response_len * n_tokens)))) | |
| for i in range(tok_start, tok_end): | |
| token_values[i].append(adv) | |
| trace_advantages = [] | |
| for vals in token_values: | |
| if vals: | |
| # Preserve sign and strongest available local execution credit. | |
| local = max(vals, key=lambda x: abs(x)) | |
| trace_advantages.append(float(traj_advantage) * local) | |
| else: | |
| trace_advantages.append(0.0) | |
| if trace_credit_mode == "replace": | |
| return trace_advantages | |
| if trace_credit_mode == "positive_replace": | |
| return [x if x > 0 else 0.0 for x in trace_advantages] | |
| if trace_credit_mode == "modulate": | |
| return [ | |
| (1.0 - trace_credit_weight) * b + trace_credit_weight * t | |
| for b, t in zip(base, trace_advantages) | |
| ] | |
| raise ValueError(f"Unknown trace_credit_mode: {trace_credit_mode}") | |
| def _factor_token_masks( | |
| *, | |
| n_tokens: int, | |
| action_tokens: list[int], | |
| tokenizer: Tokenizer | None, | |
| response: str, | |
| response_len: int | None, | |
| spans: list[dict], | |
| ) -> dict[str, list[float]]: | |
| """Map named action factors to sampled-token masks.""" | |
| if n_tokens <= 0 or not spans: | |
| return {} | |
| response_len = int(response_len if response_len is not None else len(response)) | |
| if response_len <= 0: | |
| return {} | |
| exact_offsets = _exact_token_offsets_for_response( | |
| tokenizer=tokenizer, | |
| response=response, | |
| action_tokens=action_tokens, | |
| ) | |
| masks: dict[str, list[float]] = {} | |
| for span in spans: | |
| factor = str(span.get("factor", "")).strip() | |
| if not factor: | |
| continue | |
| try: | |
| start = max(0, min(response_len, int(span.get("start", 0)))) | |
| end = max(start, min(response_len, int(span.get("end", start)))) | |
| except (TypeError, ValueError): | |
| continue | |
| if end <= start: | |
| continue | |
| factor_mask = masks.setdefault(factor, [0.0] * n_tokens) | |
| if exact_offsets is not None: | |
| for token_idx, token_start, token_end in exact_offsets: | |
| if token_idx < n_tokens and token_end > start and token_start < end: | |
| factor_mask[token_idx] = 1.0 | |
| else: | |
| token_start = max(0, min(n_tokens - 1, int(start / response_len * n_tokens))) | |
| token_end = max( | |
| token_start + 1, | |
| min(n_tokens, int(np.ceil(end / response_len * n_tokens))), | |
| ) | |
| for token_idx in range(token_start, token_end): | |
| factor_mask[token_idx] = 1.0 | |
| return {name: mask for name, mask in masks.items() if any(mask)} | |
| # 【本地新增】_trajectory_to_factorized_data | |
| # 历史动作因子信用分支;正式入口 factorized_action_credit=False。 | |
| def _trajectory_to_factorized_data( | |
| traj: Trajectory, | |
| traj_advantage: float, | |
| tokenizer: Tokenizer | None, | |
| ) -> list[tinker.Datum]: | |
| """Create one masked datum per action factor. | |
| Separate datums make target selection and replacement generation two policy | |
| factors with the same return, while retaining a single sampled response. | |
| """ | |
| data: list[tinker.Datum] = [] | |
| for transition in traj.transitions: | |
| action = transition.ac | |
| if not action.tokens: | |
| continue | |
| response = str(transition.metrics.get("response", "")) | |
| factor_masks = _factor_token_masks( | |
| n_tokens=len(action.tokens), | |
| action_tokens=action.tokens, | |
| tokenizer=tokenizer, | |
| response=response, | |
| response_len=transition.metrics.get("action_factor_response_len"), | |
| spans=transition.metrics.get("action_factor_spans", []) or [], | |
| ) | |
| if not factor_masks: | |
| continue | |
| full_chunks = list(transition.ob.chunks) + [ | |
| tinker.types.EncodedTextChunk(tokens=list(action.tokens)) | |
| ] | |
| model_input, target_tokens = create_rightshifted_model_input_and_leftshifted_targets( | |
| full_chunks | |
| ) | |
| observation_len = transition.ob.length | |
| sampled_logprobs = ([0.0] * observation_len + list(action.logprobs))[1:] | |
| for factor, factor_mask in sorted(factor_masks.items()): | |
| full_mask = [0.0] * observation_len + [ | |
| float(base_mask) * float(local_mask) | |
| for base_mask, local_mask in safezip(action.mask, factor_mask) | |
| ] | |
| full_advantages = [0.0] * observation_len + [ | |
| float(traj_advantage) if local_mask > 0 else 0.0 | |
| for local_mask in factor_mask | |
| ] | |
| shifted_mask = full_mask[1:] | |
| shifted_advantages = full_advantages[1:] | |
| assert ( | |
| model_input.length | |
| == len(target_tokens) | |
| == len(sampled_logprobs) | |
| == len(shifted_mask) | |
| == len(shifted_advantages) | |
| ) | |
| datum = tinker.Datum( | |
| model_input=model_input, | |
| loss_fn_inputs={ | |
| "target_tokens": TensorData.from_torch(torch.tensor(target_tokens)), | |
| "logprobs": TensorData.from_torch(torch.tensor(sampled_logprobs)), | |
| "advantages": TensorData.from_torch(torch.tensor(shifted_advantages)), | |
| "mask": TensorData.from_torch(torch.tensor(shifted_mask)), | |
| }, | |
| ) | |
| data.append(datum) | |
| return data | |
| # 【Baseline 原有】create_rightshifted_model_input_and_leftshifted_targets | |
| # 把同一 token 序列错开一位,形成用前缀预测下一个 token 的输入和标签。 | |
| def create_rightshifted_model_input_and_leftshifted_targets( | |
| chunks: list[tinker.ModelInputChunk], | |
| ) -> tuple[tinker.ModelInput, list[int]]: | |
| """ | |
| Given a full sequence of model input chunks, create | |
| "inputs" (with last token removed); these are also list[ModelInputChunk] because text+images | |
| "targets" (with first token removed); these are list[int] text tokens | |
| """ | |
| assert len(chunks) >= 1, "must have at least one chunk" | |
| last_chunk = chunks[-1] | |
| if not isinstance(last_chunk, tinker.types.EncodedTextChunk): | |
| raise ValueError( | |
| "The last chunk must be a text chunk. This is because images are 0-loss anyways, so we should remove them beforehand." | |
| ) | |
| total_length = sum(c.length for c in chunks) | |
| if total_length < 2: | |
| raise ValueError("need at least 2 tokens for input/target split") | |
| # Build input chunks: all but last, then append truncated last chunk | |
| input_chunks: list[tinker.ModelInputChunk] = list(chunks[:-1]) | |
| if last_chunk.length > 1: | |
| input_chunks.append(tinker.types.EncodedTextChunk(tokens=last_chunk.tokens[:-1])) | |
| # Build target tokens: collect all tokens, then slice off first | |
| all_tokens: list[int] = [] | |
| for chunk in chunks: | |
| if isinstance(chunk, tinker.types.EncodedTextChunk): | |
| all_tokens.extend(chunk.tokens) | |
| else: | |
| all_tokens.extend([0] * chunk.length) | |
| target_tokens = all_tokens[1:] | |
| return tinker.ModelInput(chunks=input_chunks), target_tokens | |
| FlatObElem = int | tinker.ModelInputChunk | |
| FlatOb = list[FlatObElem] | |
| def _is_prefix(seq1: FlatOb, seq2: FlatOb) -> bool: | |
| """ | |
| Check if seq1 is a prefix of seq2. | |
| """ | |
| return len(seq1) <= len(seq2) and seq2[: len(seq1)] == seq1 | |
| def _flat_ob_token_len(flat_ob: FlatOb) -> int: | |
| out = 0 | |
| for elem in flat_ob: | |
| if isinstance(elem, int): | |
| out += 1 | |
| else: | |
| out += elem.length | |
| return out | |
| def _flat_ob_to_model_input(flat_ob: FlatOb) -> tinker.ModelInput: | |
| out: list[tinker.ModelInputChunk] = [] | |
| current_text_chunk: list[int] = [] | |
| def flush_text_chunk(): | |
| if current_text_chunk: | |
| out.append(tinker.EncodedTextChunk(tokens=current_text_chunk)) | |
| current_text_chunk.clear() | |
| for elem in flat_ob: | |
| if isinstance(elem, int): | |
| current_text_chunk.append(elem) | |
| else: | |
| flush_text_chunk() | |
| out.append(elem) | |
| flush_text_chunk() | |
| return tinker.ModelInput(chunks=out) | |
| def _flatten_chunks(chunks: list[tinker.ModelInputChunk]) -> FlatOb: | |
| out: FlatOb = [] | |
| for chunk in chunks: | |
| if isinstance(chunk, tinker.EncodedTextChunk): | |
| out.extend(chunk.tokens) | |
| else: | |
| out.append(chunk) | |
| return out | |
| # 【Baseline 上的改动】trajectory_to_data | |
| # baseline 数据流加本地校验:prompt 保留作上下文但 mask=0,实际生成轨迹采用采样时的 mask。 | |
| # 不会将合并后完整函数图重新当作模型生成;也没有自动把训练限制为最终代码块中的 token。 | |
| def trajectory_to_data( | |
| traj: Trajectory, | |
| traj_advantage: float, | |
| tokenizer: Tokenizer | None = None, | |
| trace_credit_weight: float = 0.0, | |
| trace_credit_mode: str = "modulate", | |
| factorized_action_credit: bool = False, | |
| response_span_override: list[dict] | None = None, | |
| ) -> list[tinker.Datum]: | |
| """ | |
| Return one or more Datum objects corresponding to the trajectory. | |
| If the sequence grows by appending, i.e., each successive observation contains | |
| the previous observation+action as a prefix, then we can return a single Datum. | |
| However, if we get a sequence that's not an extension of the previous sequence, | |
| then that results in a new Datum. | |
| For example, let O1 denote a chunk of observation tokens, and let A1 denote an action. | |
| Then let's say ob_ac_pairs is as follows. | |
| (O1, A1) | |
| (O1+A1+O2, A2) | |
| (O3, A3) | |
| Then we will merge the first two observation-action pairs into a single Datum, | |
| and the last observation-action pair into a separate Datum. | |
| """ | |
| if not all( | |
| _action_tokens_are_trainable( | |
| tokens=transition.ac.tokens, | |
| logprobs=transition.ac.logprobs, | |
| mask=transition.ac.mask, | |
| tokenizer=tokenizer, | |
| ) | |
| for transition in traj.transitions | |
| ): | |
| return [] | |
| if factorized_action_credit: | |
| factorized = _trajectory_to_factorized_data(traj, traj_advantage, tokenizer) | |
| if factorized: | |
| return factorized | |
| class SequenceAccumulator: | |
| full_sequence: list[FlatObElem] = [] | |
| sampled_logprobs: list[float] = [] | |
| advantages: list[float] = [] | |
| mask: list[float] = [] | |
| def clear(cls): | |
| cls.full_sequence = [] | |
| cls.sampled_logprobs = [] | |
| cls.advantages = [] | |
| cls.mask = [] | |
| def make_datum_from_state(): | |
| all_tokens_T = _flat_ob_to_model_input(SequenceAccumulator.full_sequence) | |
| input_tokens_T, target_tokens_T = create_rightshifted_model_input_and_leftshifted_targets( | |
| list(all_tokens_T.chunks) | |
| ) | |
| sampled_logprobs_T = SequenceAccumulator.sampled_logprobs[1:] | |
| advantages_T = SequenceAccumulator.advantages[1:] | |
| mask_T = SequenceAccumulator.mask[1:] | |
| assert ( | |
| input_tokens_T.length | |
| == len(target_tokens_T) | |
| == len(sampled_logprobs_T) | |
| == len(advantages_T) | |
| == len(mask_T) | |
| ) | |
| return tinker.Datum( | |
| model_input=input_tokens_T, | |
| loss_fn_inputs={ | |
| "target_tokens": TensorData.from_torch(torch.tensor(target_tokens_T)), | |
| "logprobs": TensorData.from_torch(torch.tensor(sampled_logprobs_T)), | |
| "advantages": TensorData.from_torch(torch.tensor(advantages_T)), | |
| "mask": TensorData.from_torch(torch.tensor(mask_T)), | |
| }, | |
| ) | |
| data: list[tinker.Datum] = [] | |
| for transition in traj.transitions: | |
| ob = transition.ob | |
| ob_flat = _flatten_chunks(ob.chunks) | |
| ac_with_logprobs = transition.ac | |
| if len(SequenceAccumulator.full_sequence) == 0: | |
| delta_ob_flat = ob_flat | |
| elif _is_prefix(SequenceAccumulator.full_sequence, ob_flat): | |
| delta_ob_flat = ob_flat[len(SequenceAccumulator.full_sequence) :] | |
| else: | |
| data.append(make_datum_from_state()) | |
| SequenceAccumulator.clear() | |
| delta_ob_flat = ob_flat | |
| delta_ob_len = _flat_ob_token_len(delta_ob_flat) | |
| SequenceAccumulator.full_sequence.extend(delta_ob_flat) | |
| SequenceAccumulator.full_sequence.extend(ac_with_logprobs.tokens) | |
| SequenceAccumulator.sampled_logprobs.extend( | |
| [0.0] * delta_ob_len + ac_with_logprobs.logprobs | |
| ) | |
| action_advantages = _estimate_token_advantages_from_response_spans( | |
| n_tokens=len(ac_with_logprobs.tokens), | |
| action_tokens=ac_with_logprobs.tokens, | |
| tokenizer=tokenizer, | |
| response=str(transition.metrics.get("response", "")), | |
| response_len=transition.metrics.get("trace_response_len"), | |
| spans=( | |
| response_span_override | |
| if response_span_override is not None | |
| else transition.metrics.get("trace_response_span_advantages", []) or [] | |
| ), | |
| traj_advantage=float(traj_advantage), | |
| trace_credit_weight=float(trace_credit_weight), | |
| trace_credit_mode=trace_credit_mode, | |
| ) | |
| SequenceAccumulator.advantages.extend( | |
| [0] * delta_ob_len + action_advantages | |
| ) | |
| SequenceAccumulator.mask.extend([0.0] * delta_ob_len + ac_with_logprobs.mask) | |
| if SequenceAccumulator.full_sequence: | |
| data.append(make_datum_from_state()) | |
| return data | |
| # 【Baseline 上的改动】assemble_training_data | |
| # 遍历保留下来的轨迹组及各自 advantage,构造训练样本;不按树的 top-2 名单筛选。 | |
| def assemble_training_data( | |
| trajectory_groups_P: List[TrajectoryGroup], | |
| advantages_P: List[torch.Tensor], | |
| tokenizer: Tokenizer | None = None, | |
| trace_credit_weight: float = 0.0, | |
| trace_credit_mode: str = "modulate", | |
| factorized_action_credit: bool = False, | |
| ) -> tuple[List[tinker.Datum], List[dict[str, int]]]: | |
| """Convert trajectories to training data format.""" | |
| data_D: list[tinker.Datum] = [] | |
| metadata_D: list[dict[str, int]] = [] | |
| for i_group, (traj_group, advantages_G) in enumerate( | |
| safezip(trajectory_groups_P, advantages_P) | |
| ): | |
| for i_traj, (traj, traj_advantage) in enumerate( | |
| safezip(traj_group.trajectories_G, advantages_G) | |
| ): | |
| # Build the full sequence from the trajectory | |
| new_data = trajectory_to_data( | |
| traj, | |
| float(traj_advantage), | |
| tokenizer=tokenizer, | |
| trace_credit_weight=trace_credit_weight, | |
| trace_credit_mode=trace_credit_mode, | |
| factorized_action_credit=factorized_action_credit, | |
| ) | |
| data_D.extend(new_data) | |
| metadata_D.extend([dict(group_idx=i_group, traj_idx=i_traj) for _ in new_data]) | |
| return data_D, metadata_D | |
| # 【Baseline 原有】remove_constant_reward_groups | |
| # 移除组内奖励完全相同的组;因此预算为 512 不保证最终有 512 条有效训练样本。 | |
| def remove_constant_reward_groups( | |
| trajectory_groups_P: List[TrajectoryGroup], | |
| ) -> List[TrajectoryGroup]: | |
| new_groups: list[TrajectoryGroup] = [] | |
| for group in trajectory_groups_P: | |
| if not all_same(group.get_total_rewards()): | |
| new_groups.append(group) | |
| if not new_groups: | |
| logger.warning("All rewards are uniform. There will be no gradient") | |
| return trajectory_groups_P[0:1] # return singleton list in case empty | |
| # list will cause problems | |
| return new_groups | |
Xet Storage Details
- Size:
- 22 kB
- Xet hash:
- 8d38582a3419159c5b7d28cc39f06026ebc1f96ad9397a7cad11e49188778b04
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.