ctopd / kdflow /trainer /data_processor /rollout_data_processor.py
codemaivanngu's picture
feat(h1b): add span factorial arms and offline probability replay tools
2bf0f47 unverified
Raw History Blame Contribute Delete
14.5 kB
import json
import os
from typing import Any, Dict, List, Optional
import torch
from kdflow.datasets.utils import get_tokenizer_or_processor
from kdflow.utils.utils import zero_pad_sequences
class RolloutDataProcessor:
"""Convert raw rollout outputs into training micro-batches."""
CONDITIONAL_EOS_ALGORITHMS = frozenset(
{
"byte_align_ctkd",
"h1b_span",
"topk_byte_align_ctkd",
}
)
def __init__(self, strategy, is_same_tokenizer: bool):
self.args = strategy.args
self.is_same_tokenizer = is_same_tokenizer
self.image_key = getattr(self.args.data, "image_key", None)
self.student_processor = get_tokenizer_or_processor(
self.args.model.student_name_or_path,
need_processor=self.image_key is not None,
)
self.teacher_processors = {}
if self.args.model.teacher_name_or_path and not self.is_same_tokenizer:
self.teacher_processors["default"] = get_tokenizer_or_processor(
self.args.model.teacher_name_or_path,
need_processor=self.image_key is not None,
)
def process(
self,
stu_prompts: List[str],
tea_prompts: List[str],
outputs: List[Dict[str, Any]],
labels: List[Any],
sampling_params: Dict[str, Any],
global_step: int,
mode: str,
images: Optional[List] = None,
sample_metadata: Optional[List[Dict[str, Any]]] = None,
) -> tuple[List[dict], Dict[str, float]]:
"""Save, tokenize and collate raw rollout outputs."""
if sample_metadata is not None and len(sample_metadata) != len(outputs):
raise ValueError("sample_metadata and outputs must have the same length")
archive_metadata=sample_metadata
if self.args.kd.kd_algorithm=='h1b_span':
archive_metadata=[dict(m or {},tea_prompt=t,output_ids=o.get('output_ids',[]))
for m,t,o in zip(sample_metadata or [{} for _ in outputs],tea_prompts,outputs)]
self._save_rollout_data(
stu_prompts,
outputs,
labels,
global_step,
mode,
sample_metadata=archive_metadata,
)
kd_algorithm = self.args.kd.kd_algorithm
uses_conditional_eos = kd_algorithm in self.CONDITIONAL_EOS_ALGORITHMS
truncated_flags = (
[
self._is_length_truncated(output, sampling_params)
for output in outputs
]
if uses_conditional_eos
else [False] * len(outputs)
)
sample_list = [
self._build_rollout_sample(
stu_prompt=stu_prompts[index],
tea_prompt=tea_prompts[index],
output=output,
label=labels[index],
images=images[index] if images and images[index] else None,
append_eos=(
not truncated_flags[index] if uses_conditional_eos else True
),
metadata=(
sample_metadata[index] if sample_metadata is not None else None
),
)
for index, output in enumerate(outputs)
]
rollout_metrics = self._compute_rollout_metrics(
sample_list, sampling_params, mode
)
if (
mode == "train"
and self.args.rollout.print_rollout_sample
and sample_list
):
print(sample_list[0]["stu_prompts"][0] + sample_list[0]["stu_responses"][0])
micro_batches = self._collate_micro_batches(
sample_list, self.args.train.micro_train_batch_size
)
return micro_batches, rollout_metrics
def _save_rollout_data(
self,
prompts: List[str],
outputs: List[Dict[str, Any]],
labels: List[Any],
global_step: int,
mode: str,
sample_metadata: Optional[List[Dict[str, Any]]] = None,
) -> None:
rollout_dir = os.path.join(self.args.train.save_path, "rollout_data")
if mode == "eval":
rollout_dir = os.path.join(rollout_dir, "val")
os.makedirs(rollout_dir, exist_ok=True)
rollout_path = os.path.join(rollout_dir, f"{global_step}.jsonl")
with open(rollout_path, "w") as file:
metadata_rows = sample_metadata or [{} for _ in outputs]
for prompt, output, label, metadata in zip(
prompts, outputs, labels, metadata_rows
):
rollout_record = {"prompt": prompt, "output": output["text"]}
rollout_record.update(metadata)
finish_reason = self._extract_finish_reason(output)
if finish_reason is not None:
rollout_record["finish_reason"] = finish_reason
if "reward_result" in output:
rollout_record["reward_result"] = output["reward_result"]
if label is not None and label != "":
rollout_record["label"] = label
file.write(json.dumps(rollout_record, ensure_ascii=False) + "\n")
@staticmethod
def _extract_finish_reason(output: Dict[str, Any]):
meta_info = output.get("meta_info") or {}
finish_reason = output.get("finish_reason")
return (
finish_reason
if finish_reason is not None
else meta_info.get("finish_reason")
)
@classmethod
def _is_length_truncated(
cls,
output: Dict[str, Any],
sampling_params: Dict[str, Any],
) -> bool:
finish_reason = cls._extract_finish_reason(output)
if isinstance(finish_reason, dict):
reason_type = (
finish_reason.get("type")
or finish_reason.get("reason")
or finish_reason.get("finish_reason")
)
else:
reason_type = finish_reason
if isinstance(reason_type, str):
normalized_reason = reason_type.lower().replace("-", "_")
if normalized_reason in {
"length",
"max_length",
"max_tokens",
"max_new_tokens",
}:
return True
max_new_tokens = sampling_params.get("max_new_tokens")
return (
finish_reason is None
and max_new_tokens is not None
and len(output.get("output_ids", ())) >= max_new_tokens
)
@staticmethod
def _compute_rollout_metrics(
sample_list: List[Dict[str, Any]],
sampling_params: Dict[str, Any],
mode: str,
) -> Dict[str, float]:
length_names = ("prompt_length", "response_length", "total_length")
length_values = {
name: [sample.pop(name).item() for sample in sample_list]
for name in length_names
}
metric_prefix = "rollout" if mode == "train" else "eval"
rollout_metrics = {}
for name, values in length_values.items():
rollout_metrics[f"{metric_prefix}/{name}/mean"] = sum(values) / len(values)
rollout_metrics[f"{metric_prefix}/{name}/max"] = max(values)
max_response_length = sampling_params["max_new_tokens"]
response_lengths = length_values["response_length"]
rollout_metrics[f"{metric_prefix}/response_clip_ratio"] = sum(
length >= max_response_length for length in response_lengths
) / len(response_lengths)
return rollout_metrics
@staticmethod
def _collate_values(values: list):
value = values[0]
if isinstance(value, torch.Tensor):
return zero_pad_sequences(values, side="right", value=0)
if isinstance(value, list):
return sum(values, [])
if value is None:
return None
return values
def _collate_micro_batches(
self, sample_list: List[Dict], batch_size: int
) -> List[Dict]:
"""Collate single samples into micro-batches."""
micro_batch_list = []
for index in range(0, len(sample_list), batch_size):
batch_samples = sample_list[index : index + batch_size]
micro_batch = {
key: self._collate_values([sample[key] for sample in batch_samples])
for key in batch_samples[0]
}
micro_batch_list.append(micro_batch)
return micro_batch_list
def _tokenize_sample(
self,
prompt: str,
response: str,
processor,
prefix: str,
images=None,
append_eos: bool = True,
) -> Dict[str, Any]:
"""Tokenize prompt and response for a single sample."""
tokenizer = getattr(processor, "tokenizer", processor)
response_tokens = tokenizer(
response, return_tensors="pt", add_special_tokens=False
)
response_length = response_tokens["input_ids"].shape[1]
full_input = {"text": prompt + response}
if images:
full_input["images"] = images
full_tokens = processor(
**full_input, return_tensors="pt", add_special_tokens=False
)
prompt_length = full_tokens["input_ids"].shape[1] - response_length
if append_eos:
eos_token_id = tokenizer.eos_token_id
input_ids = torch.cat(
[
full_tokens["input_ids"][0],
full_tokens["input_ids"][0].new_tensor([eos_token_id]),
]
)
attention_mask = torch.cat(
[
full_tokens["attention_mask"][0],
full_tokens["attention_mask"][0].new_ones(1),
]
)
else:
input_ids = full_tokens["input_ids"][0]
attention_mask = full_tokens["attention_mask"][0]
eos_length = int(append_eos)
loss_mask = torch.tensor(
[False] * (prompt_length - 1)
+ [True] * (response_length + eos_length)
+ [False],
device=input_ids.device,
)
result = {
f"{prefix}_input_ids": input_ids,
f"{prefix}_attn_mask": attention_mask,
f"{prefix}_loss_mask": loss_mask,
f"_{prefix}_prompt_length": prompt_length,
}
multi_modal_inputs = {
key: torch.as_tensor(value)
for key, value in full_tokens.items()
if key not in ("input_ids", "attention_mask", "mm_token_type_ids")
}
if multi_modal_inputs:
result[f"_{prefix}_multi_modal_inputs"] = multi_modal_inputs
return result
def _get_teacher_processor(self):
"""Get the configured teacher processor."""
if "default" in self.teacher_processors:
return self.teacher_processors["default"]
return self.student_processor
def _build_rollout_sample(
self,
stu_prompt: str,
tea_prompt: str,
output: Dict[str, Any],
label: Any,
images=None,
append_eos: bool = True,
metadata: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Build a rollout sample with student and teacher tokenizations."""
response_ids = output["output_ids"]
response_text = output["text"]
stu_tokens = self._tokenize_sample(
stu_prompt,
response_text,
self.student_processor,
"stu",
images=images,
append_eos=append_eos,
)
if not self.is_same_tokenizer or tea_prompt != stu_prompt:
teacher_processor = self._get_teacher_processor()
tea_tokens = self._tokenize_sample(
tea_prompt,
response_text,
teacher_processor,
"tea",
images=images,
append_eos=append_eos,
)
else:
tea_tokens = {
"tea_input_ids": stu_tokens["stu_input_ids"].clone(),
"tea_attn_mask": stu_tokens["stu_attn_mask"].clone(),
"tea_loss_mask": stu_tokens["stu_loss_mask"].clone(),
}
prompt_length = stu_tokens["_stu_prompt_length"]
response_length = len(response_ids)
total_length = stu_tokens["stu_attn_mask"].sum().item()
teacher_processor = self._get_teacher_processor()
tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
if images:
feed_ids = tokenizer(
tea_prompt + response_text, add_special_tokens=False
)["input_ids"]
tea_feed_input_ids = (
feed_ids + [tokenizer.eos_token_id] if append_eos else feed_ids
)
else:
tea_feed_input_ids = tea_tokens["tea_input_ids"].tolist()
sample = {
**{key: value for key, value in tea_tokens.items() if not key.startswith("_")},
**{key: value for key, value in stu_tokens.items() if not key.startswith("_")},
"tea_feed_input_ids": [tea_feed_input_ids],
"rollout_log_probs": None,
"stu_prompts": [stu_prompt],
"stu_responses": [response_text],
"tea_prompts": [tea_prompt],
"labels": [label],
"prompt_length": torch.FloatTensor([[prompt_length]]),
"response_length": torch.FloatTensor([[response_length]]),
"total_length": torch.FloatTensor([[total_length]]),
}
stu_multi_modal_inputs = stu_tokens.get("_stu_multi_modal_inputs")
if self.args.kd.kd_algorithm=='h1b_span':
tok=getattr(self.student_processor,'tokenizer',self.student_processor)
encoded=tok(response_text,add_special_tokens=False)['input_ids']
sample['stu_reencode_match']=torch.FloatTensor([[float(list(encoded)==list(response_ids))]])
if stu_multi_modal_inputs is not None:
sample["stu_multi_modal_inputs"] = [stu_multi_modal_inputs]
if images:
sample["images"] = [images]
if metadata:
reserved = set(sample) & set(metadata)
if reserved:
raise ValueError(
f"Rollout metadata uses reserved sample keys: {sorted(reserved)}"
)
sample.update(metadata)
return sample