ctopd / kdflow /datasets /sft_dataset.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
18 kB
from typing import Callable, Optional, Dict, List, Any
import torch
from torch.utils.data import Dataset
from datasets import Image as ImageFeature
from PIL import Image
from kdflow.datasets.utils import (
convert_to_openai_messages,
get_tokenizer_or_processor,
validate_dataset_columns,
zero_pad_sequences,
)
from kdflow.models.utils import TokenizerCompareResult
class SFTDataset(Dataset):
"""
Dataset for SFT and Off-Policy KD.
Args:
dataset: dataset for SFT and Off-Policy KD
strategy: training strategy object
tokenizer_info: result of tokenizer comparison (template_identical, vocab_identical)
max_data_num: maximum number of data to load
input_template: optional template for formatting input
num_processors: number of processors for parallel data loading
"""
def __init__(
self,
dataset,
strategy,
tokenizer_info: Optional[TokenizerCompareResult] = None,
max_data_num: int = -1,
input_template: Optional[str] = None,
num_processors: int = 8,
) -> None:
super().__init__()
self.args = strategy.args
self.strategy = strategy
self.max_length = self.args.data.max_len
self.tokenizer_info = tokenizer_info
self.template_identical = True if tokenizer_info is None else self.tokenizer_info.template_identical
self.vocab_identical = True if tokenizer_info is None else self.tokenizer_info.vocab_identical
self.input_template = input_template
self.input_key = getattr(self.args.data, "input_key", None)
self.teacher_input_key = getattr(self.args.data, "teacher_input_key", None) or self.input_key
self.output_key = getattr(self.args.data, "output_key", None)
self.apply_chat_template = getattr(self.args.data, "apply_chat_template", False)
self.enable_thinking = getattr(self.args.model, "enable_thinking", False)
self.image_key = getattr(self.args.data, "image_key", None)
self.same_tokenizer = True if tokenizer_info is None else self.tokenizer_info.is_identical
self.teacher_student_share_input = (
self.same_tokenizer
and self.teacher_input_key == self.input_key
)
required_columns = {"input_key": self.input_key}
if self.output_key:
required_columns["output_key"] = self.output_key
if self.image_key:
required_columns["image_key"] = self.image_key
has_teacher = (
self.args.model.teacher_name_or_path is not None
or self.args.kd.multi_teacher_config is not None
)
if has_teacher and self.teacher_input_key != self.input_key:
required_columns["teacher_input_key"] = self.teacher_input_key
if self.args.kd.multi_teacher_config:
required_columns["teacher_routing_key"] = self.args.data.teacher_routing_key
validate_dataset_columns(dataset, **required_columns)
self.student_processor = get_tokenizer_or_processor(
self.args.model.student_name_or_path,
need_processor=self.image_key is not None,
)
self.student_tokenizer = getattr(self.student_processor, "tokenizer", self.student_processor)
self.teacher_processors = {}
if self.args.kd.multi_teacher_config:
for teacher_key, teacher_path in self.args.kd.multi_teacher_config.items():
self.teacher_processors[teacher_key] = get_tokenizer_or_processor(
teacher_path, need_processor=self.image_key is not None,
)
elif self.args.model.teacher_name_or_path is not None:
self.teacher_processors["default"] = get_tokenizer_or_processor(
self.args.model.teacher_name_or_path,
need_processor=self.image_key is not None,
)
if max_data_num > 0 and max_data_num < len(dataset):
self.strategy.log(f"Truncating dataset from {len(dataset)} to {max_data_num}")
dataset = dataset.select(range(max_data_num))
# Adjust worker count based on dataset size to speed up preprocessing.
num_processors = min(
num_processors,
max(1, len(dataset) // 2000),
)
self.processed_dataset = dataset.map(
self.process_data,
remove_columns=dataset.column_names,
num_proc=num_processors,
load_from_cache_file=False,
desc="Processing data",
)
if self.image_key:
self.processed_dataset = self.processed_dataset.cast_column("images", [ImageFeature()])
original_len = len(self.processed_dataset)
strategy.log(f"Before length filter: {len(self.processed_dataset)}")
self.processed_dataset = self.processed_dataset.filter(
lambda stu_input_len: stu_input_len <= self.max_length,
input_columns=["stu_input_len"],
desc="Filtering overlang samples",
)
if len(self.processed_dataset) < original_len:
self.strategy.log(
f"Filtered {original_len - len(self.processed_dataset)} samples "
f"exceeding max_length={self.max_length} "
f"({original_len} -> {len(self.processed_dataset)})"
)
self._print_sample()
def _print_sample(self) -> None:
sample = self.processed_dataset[0]
self.strategy.print("Student prompt + response:")
self.strategy.print(sample["stu_prompt"] + sample["stu_response"])
if not self.teacher_student_share_input:
self.strategy.print("Teacher prompt + response:")
self.strategy.print(sample["tea_prompt"] + sample["tea_response"])
def process_data(self, data: Dict) -> Dict[str, Any]:
stu_chat_template_fn = self.student_processor.apply_chat_template if self.apply_chat_template else None
stu_prompt, stu_response = self.preprocess_data(
data, self.input_template, self.input_key, self.output_key,
apply_chat_template=stu_chat_template_fn,
)
stu_eos_token = self.student_tokenizer.eos_token
if not stu_response.endswith(stu_eos_token):
stu_response += " " + stu_eos_token
result = {"stu_prompt": stu_prompt, "stu_response": stu_response}
result["stu_resp_len"] = self._compute_token_length(
self.student_processor, stu_response
)
result["stu_input_len"] = self._compute_token_length(
self.student_processor, stu_prompt
) + result["stu_resp_len"]
if self.image_key is not None:
result["images"] = self.load_images(data[self.image_key])
if self.args.model.teacher_name_or_path is not None \
or self.args.kd.multi_teacher_config is not None:
if not self.teacher_student_share_input:
if self.args.kd.multi_teacher_config is not None:
routing_key = self.args.data.teacher_routing_key
assert routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
assert routing_key in data, f"Routing key '{routing_key}' not found in data"
teacher_key = data[routing_key]
if teacher_key not in self.teacher_processors:
raise ValueError(
f"Teacher routing key '{teacher_key}' not found in multi_teacher_config. "
f"Available keys: {list(self.teacher_processors.keys())}."
)
teacher_processor = self.teacher_processors[teacher_key]
else:
teacher_processor = self.teacher_processors.get("default", self.student_processor)
cur_teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
tea_chat_template_fn = teacher_processor.apply_chat_template if self.apply_chat_template else None
tea_prompt, tea_response = self.preprocess_data(
data, self.input_template, self.teacher_input_key, self.output_key,
apply_chat_template=tea_chat_template_fn,
)
tea_eos_token = cur_teacher_tokenizer.eos_token
if not tea_response.endswith(tea_eos_token):
tea_response += " " + tea_eos_token
result["tea_prompt"] = tea_prompt
result["tea_response"] = tea_response
result["tea_resp_len"] = self._compute_token_length(teacher_processor, tea_response)
else:
result["tea_prompt"] = stu_prompt
result["tea_response"] = stu_response
result["tea_resp_len"] = result["stu_resp_len"]
if self.args.kd.multi_teacher_config is not None:
assert self.args.data.teacher_routing_key is not None, "`--teacher_routing_key` must be specified when using multi_teacher_config"
assert self.args.data.teacher_routing_key in data, f"Routing key '{self.args.data.teacher_routing_key}' not found in data"
result["teacher_routing_key"] = data[self.args.data.teacher_routing_key]
return result
def preprocess_data(
self,
data: Dict,
input_template: Optional[str] = None,
input_key: str = "input",
output_key: Optional[str] = None,
apply_chat_template: Optional[Callable] = None,
) -> tuple:
if not apply_chat_template:
prompt = data[input_key]
if input_template:
prompt = input_template.format(prompt)
assert output_key is not None
return prompt, data[output_key]
has_image = self.image_key is not None
if output_key:
messages = convert_to_openai_messages(data[input_key], expand_image=has_image) + \
convert_to_openai_messages(data[output_key], role="assistant", expand_image=has_image)
else:
messages = convert_to_openai_messages(data[input_key], expand_image=has_image)
chat_template_kwargs = {}
if "enable_thinking" in str(apply_chat_template.__self__.chat_template):
chat_template_kwargs["enable_thinking"] = self.enable_thinking
prompt = apply_chat_template(
messages[:-1], tokenize=False, add_generation_prompt=True,
**chat_template_kwargs,
)
full_text = apply_chat_template(messages, tokenize=False, **chat_template_kwargs)
response = full_text[len(prompt):].removeprefix("<think>\n\n</think>\n\n").rstrip()
return prompt, response
def __len__(self) -> int:
return len(self.processed_dataset)
def __getitem__(self, idx: int) -> Dict:
return self.processed_dataset[idx]
def load_images(self, image_content):
"""Load image(s) from various input formats. Always returns a list."""
if isinstance(image_content, Image.Image):
return [image_content]
if isinstance(image_content, str):
return [Image.open(image_content).convert("RGB")]
if isinstance(image_content, list):
return [self._load_single_image(img) for img in image_content]
return []
@staticmethod
def _load_single_image(img):
if isinstance(img, Image.Image):
return img
if isinstance(img, str):
return Image.open(img).convert("RGB")
return None
def _compute_token_length(self, processor, text):
tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
return len(tokenizer.encode(text, add_special_tokens=False))
def _encode_single(self, processor, text, images=None) -> Dict[str, Any]:
"""Single-sample processor call."""
kwargs = {"text": [text], "padding": False, "truncation": False,
"return_tensors": "pt"}
if images:
kwargs["images"] = images
enc = processor(**kwargs)
skip = {"input_ids", "attention_mask", "mm_token_type_ids"}
multi_modal_inputs = {k: v for k, v in enc.items() if k not in skip}
return {
"input_ids": enc["input_ids"][0],
"attention_mask": enc["attention_mask"][0],
"multi_modal_inputs": multi_modal_inputs or None,
}
@staticmethod
def _build_loss_mask(attn_mask, resp_lens):
bs, seq_len = attn_mask.shape
loss_mask = torch.zeros(bs, seq_len, dtype=torch.bool)
for i in range(bs):
real_end = attn_mask[i].sum().item()
start = max(real_end - resp_lens[i] - 1, 0)
loss_mask[i, start:real_end - 1] = True
return loss_mask
def collate_fn(self, item_list: List[Dict]) -> Dict[str, torch.Tensor]:
bsz = len(item_list)
per_sample_images = (
[item["images"] for item in item_list]
if self.image_key else [None] * bsz
)
stu_full = [item["stu_prompt"] + item["stu_response"] for item in item_list]
stu_encs = [
self._encode_single(self.student_processor, stu_full[i], per_sample_images[i])
for i in range(bsz)
]
stu_pad_id = self.student_tokenizer.pad_token_id
stu_input_ids = zero_pad_sequences(
[e["input_ids"] for e in stu_encs], side="right", value=stu_pad_id,
)
stu_attn_mask = zero_pad_sequences(
[e["attention_mask"] for e in stu_encs], side="right", value=0,
).long()
stu_loss_mask = self._build_loss_mask(
stu_attn_mask, [item["stu_resp_len"] for item in item_list],
)
batch: Dict[str, Any] = {
"stu_input_ids": stu_input_ids,
"stu_attn_mask": stu_attn_mask,
"stu_loss_mask": stu_loss_mask,
}
if any(e["multi_modal_inputs"] is not None for e in stu_encs):
batch["stu_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in stu_encs]
if "tea_prompt" in item_list[0]:
if not self.teacher_student_share_input:
tea_full = [item["tea_prompt"] + item["tea_response"] for item in item_list]
tea_encs = []
for i in range(bsz):
if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]:
proc = self.teacher_processors[item_list[i]["teacher_routing_key"]]
else:
proc = self.teacher_processors.get("default", self.student_processor)
tea_encs.append(self._encode_single(proc, tea_full[i], per_sample_images[i]))
if self.args.kd.multi_teacher_config:
# Multi-teacher mode validates shared vocab with the student.
tea_pad_id = self.student_tokenizer.pad_token_id
else:
teacher_processor = self.teacher_processors.get("default", self.student_processor)
teacher_tokenizer = getattr(teacher_processor, "tokenizer", teacher_processor)
tea_pad_id = teacher_tokenizer.pad_token_id
tea_input_ids = zero_pad_sequences(
[e["input_ids"] for e in tea_encs], side="right", value=tea_pad_id,
)
tea_attn_mask = zero_pad_sequences(
[e["attention_mask"] for e in tea_encs], side="right", value=0,
).long()
batch["tea_input_ids"] = tea_input_ids
batch["tea_attn_mask"] = tea_attn_mask
batch["tea_loss_mask"] = self._build_loss_mask(
tea_attn_mask, [item["tea_resp_len"] for item in item_list],
)
if any(e["multi_modal_inputs"] is not None for e in tea_encs):
batch["tea_multi_modal_inputs"] = [e["multi_modal_inputs"] for e in tea_encs]
else:
batch["tea_input_ids"] = batch["stu_input_ids"]
batch["tea_attn_mask"] = batch["stu_attn_mask"]
batch["tea_loss_mask"] = batch["stu_loss_mask"]
if "stu_multi_modal_inputs" in batch:
batch["tea_multi_modal_inputs"] = batch["stu_multi_modal_inputs"]
# Teacher feed input_ids for SGLang. Text: reuse encoded ids (exact).
# Multimodal: tokenize raw text (single image placeholder) so SGLang re-expands it.
if not self.teacher_student_share_input:
if self.image_key:
tea_feed = []
for i in range(bsz):
if self.args.kd.multi_teacher_config and "teacher_routing_key" in item_list[i]:
proc = self.teacher_processors[item_list[i]["teacher_routing_key"]]
else:
proc = self.teacher_processors.get("default", self.student_processor)
tokenizer = getattr(proc, "tokenizer", proc)
tea_feed.append(tokenizer(tea_full[i])["input_ids"])
batch["tea_feed_input_ids"] = tea_feed
else:
batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in tea_encs]
else:
if self.image_key:
batch["tea_feed_input_ids"] = [
self.student_tokenizer(stu_full[i])["input_ids"] for i in range(bsz)
]
else:
batch["tea_feed_input_ids"] = [e["input_ids"].tolist() for e in stu_encs]
if self.image_key:
batch["images"] = per_sample_images
if "teacher_routing_key" in item_list[0]:
batch["teacher_routing_key"] = [item["teacher_routing_key"] for item in item_list]
return batch