ctopd / kdflow /datasets /utils.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
14.1 kB
import os
from typing import List, Dict, Any
import torch
from datasets import DatasetDict, interleave_datasets, load_dataset, load_from_disk
from transformers import AutoProcessor, AutoTokenizer
from kdflow.utils.logging_utils import init_logger
logger = init_logger(__name__)
def validate_dataset_columns(dataset, **column_settings):
"""Validate that configured dataset keys exist in the source dataset.
Supports dot-notation for nested keys (e.g., "reward_model.ground_truth" -> top-key "reward_model").
"""
available_columns = list(dataset.column_names)
missing_columns = [
f"{setting_name}={column_name!r}"
for setting_name, column_name in column_settings.items()
if column_name.split(".")[0] not in available_columns
]
if missing_columns:
raise ValueError(
f"Dataset is missing configured dataset column(s): "
f"{', '.join(missing_columns)}. Available columns: {available_columns}."
)
def exist_and_not_none(d, key):
return key in d and not d[key] is None
def get_nested_value(data: dict, key_path: str, default=None):
"""Get a value from a nested dict using dot-notation key path.
Example:
get_nested_value({'a': {'b': 1}}, 'a.b') -> 1
get_nested_value({'a': 1}, 'a.b') -> None
"""
keys = key_path.split(".")
value = data
for k in keys:
if isinstance(value, dict):
value = value.get(k)
else:
return default
return value
def blending_datasets(
datasets,
probabilities=None,
strategy=None,
seed=42,
max_count=1e8,
stopping_strategy="all_exhausted",
dataset_split="train",
):
"""Blend multiple datasets with optional probability sampling.
Args:
datasets (str): Comma-separated list of dataset paths
probabilities (str, optional): Comma-separated list of probabilities for sampling.
If None, datasets will be concatenated without probability sampling.
strategy: Training strategy object
seed (int): Random seed
max_count (int): Maximum number of samples per dataset
"""
dataset_split = dataset_split or "train"
datasets = datasets.split(",")
if probabilities is not None:
probabilities = list(map(float, probabilities.split(",")))
assert len(probabilities) == len(datasets)
data_list = []
for i, dataset in enumerate(datasets):
dataset = dataset.strip()
logger.info(f"dataset: {dataset}")
data_dir = dataset.split("@")[1].strip() if "@" in dataset else None
dataset = dataset.split("@")[0].strip()
dataset_basename = os.path.basename(dataset)
ext = os.path.splitext(dataset)[-1]
if ext == ".py" or (
os.path.isdir(dataset) and os.path.exists(os.path.join(dataset, f"{dataset_basename}.py"))
):
data = load_dataset(dataset, trust_remote_code=True)
logger.info(f"loaded {dataset} with python script")
elif ext in [".json", ".jsonl", ".csv", ".parquet", ".arrow"]:
ext = ext.lower().strip(".")
if ext == "jsonl":
ext = "json"
data = load_dataset(ext, data_files={dataset_split: dataset})
logger.info(f"loaded {dataset} with data_files={dataset}")
# local dataset saved with `datasets.Dataset.save_to_disk`
elif os.path.isdir(dataset):
try:
data = load_from_disk(dataset)
logger.info(f"loaded {dataset} from disk")
except Exception as e:
data = load_dataset(dataset, data_dir=data_dir)
logger.info(f"loaded {dataset} from files")
else:
data = load_dataset(dataset, data_dir=data_dir)
logger.info(f"loaded {dataset} from files")
if isinstance(data, DatasetDict):
if dataset_split not in data:
raise ValueError(
f"Dataset {dataset!r} does not contain split {dataset_split!r}. "
f"Available splits: {list(data)}."
)
data = data[dataset_split]
data = data.select(range(min(max_count, len(data))))
data_list.append(data)
if strategy.is_rank_0():
print(data_list)
if probabilities is None:
from datasets import concatenate_datasets
dataset = concatenate_datasets(data_list)
else:
dataset = interleave_datasets(
data_list,
probabilities=probabilities,
seed=seed,
stopping_strategy=stopping_strategy,
)
return dataset
def get_tokenizer_or_processor(model_name_or_path, model=None, padding_side="right", use_fast=True, need_processor=False):
if need_processor:
return get_processor(model_name_or_path, model, padding_side, use_fast)
else:
return get_tokenizer(model_name_or_path, model, padding_side, use_fast)
def get_processor(model_name_or_path, model=None, padding_side="right", use_fast=True):
processor = AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast)
processor.tokenizer.padding_side = padding_side
if processor.tokenizer.pad_token is None:
logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
processor.tokenizer.pad_token = processor.tokenizer.eos_token
processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id
if model is not None:
model.config.text_config.pad_token_id = processor.tokenizer.pad_token_id
return processor
def get_tokenizer(model_name_or_path, model=None, padding_side="right", use_fast=True):
try:
# fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style
# tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it
# with the Metaspace pre-tokenizer rebuilt from the slow tokenizer.
tokenizer = AutoTokenizer.from_pretrained(
model_name_or_path,
trust_remote_code=True,
use_fast=use_fast,
fix_mistral_regex=True,
)
except TypeError:
# Older transformers versions do not accept fix_mistral_regex.
tokenizer = AutoTokenizer.from_pretrained(
model_name_or_path,
trust_remote_code=True,
use_fast=use_fast,
)
tokenizer.padding_side = padding_side
if tokenizer.pad_token is None:
logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
tokenizer.pad_token = tokenizer.eos_token
tokenizer.pad_token_id = tokenizer.eos_token_id
if model is not None:
model.config.pad_token_id = tokenizer.pad_token_id
return tokenizer
def convert_token_to_id(token, tokenizer):
if isinstance(token, str):
token = tokenizer.encode(token, add_special_tokens=False)
assert len(token) == 1
return token[0]
else:
raise ValueError("token should be int or str")
def zero_pad_sequences(
sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False
) -> torch.Tensor:
from torch.nn.utils.rnn import pad_sequence
assert side in ("left", "right")
sequences = [seq.squeeze(0) for seq in sequences]
if side == "left":
sequences = [seq.flip(dims=0) for seq in sequences]
padded_sequences = pad_sequence(
sequences,
batch_first=True,
padding_value=value
)
if side == "left":
padded_sequences = torch.flip(padded_sequences, dims=[1])
return padded_sequences
def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True):
"""Remove the pad token. Return tensors and not lists.
Args:
input_ids shape: [bs, seq_length]
attention_mask shape: [bs, seq_length]
Returns:
no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query.
"""
no_padding_batch = []
for ids, mask in zip(input_ids, attention_mask):
# Fix for both left and right padding
ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist()
no_padding_batch.append(ids)
return no_padding_batch
# ShareGPT role mapping to OpenAI roles
SHAREGPT_ROLE_MAP = {
"human": "user",
"user": "user",
"gpt": "assistant",
"assistant": "assistant",
"system": "system",
}
def _is_sharegpt_format(data):
"""Check if data is in ShareGPT format (list of dicts with 'from'/'value' keys)."""
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
return False
return "from" in data[0] and "value" in data[0]
def _is_openai_format(data):
"""Check if data is already in OpenAI messages format (list of dicts with 'role'/'content' keys)."""
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
return False
return "role" in data[0] and "content" in data[0]
def _is_alpaca_format(data):
"""Check if data is in Alpaca format (dict with 'instruction' key)."""
if not isinstance(data, dict):
return False
return "instruction" in data
def _convert_sharegpt(messages):
"""Convert ShareGPT format messages to OpenAI messages format.
ShareGPT: [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]
OpenAI: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
"""
converted = []
for msg in messages:
role = SHAREGPT_ROLE_MAP.get(msg["from"], msg["from"])
content = msg.get("value") or ""
converted.append({"role": role, "content": content})
return converted
def _convert_alpaca(data):
"""Convert Alpaca format to OpenAI messages format.
Alpaca format fields:
- instruction (required): the main instruction/question
- input (optional): additional input context
- output (optional): the expected response
- system (optional): system prompt
- history (optional): list of [user_msg, assistant_msg] pairs
Returns:
List of OpenAI messages: [{"role": "...", "content": "..."}, ...]
"""
messages = []
# System prompt
system_prompt = data.get("system", "")
if system_prompt:
messages.append({"role": "system", "content": system_prompt})
# History turns
history = data.get("history", [])
for turn in history:
if isinstance(turn, (list, tuple)) and len(turn) == 2:
messages.append({"role": "user", "content": turn[0]})
messages.append({"role": "assistant", "content": turn[1]})
# Current instruction + optional input
instruction = data.get("instruction", "")
extra_input = data.get("input", "")
if extra_input:
user_content = f"{instruction}\n{extra_input}"
else:
user_content = instruction
messages.append({"role": "user", "content": user_content})
# Output (response)
output = data.get("output", "")
if output:
messages.append({"role": "assistant", "content": output})
return messages
def convert_to_openai_messages(data, role="user", expand_image=False):
"""Unified converter: auto-detect data format and convert to OpenAI messages.
Supported input formats:
1. OpenAI messages (already): [{"role": "user", "content": "..."}] -> returned as-is
2. ShareGPT: [{"from": "human", "value": "..."}] -> converted
3. Alpaca: {"instruction": "...", "input": "...", "output": "...", ...} -> converted
4. Plain string: "..." -> wrapped as [{"role": role, "content": "..."}]
Args:
data: Input data in any supported format.
role: Role to assign when data is a plain string. Defaults to "user".
expand_image: Whether to expand <image> placeholders into multi-part content.
Only set to True for multimodal (image) scenarios. Defaults to False.
Returns:
List of dicts in OpenAI messages format: [{"role": "...", "content": "..."}, ...]
"""
if data is None:
raise ValueError("convert_to_openai_messages received None input.")
if isinstance(data, str):
messages = [{"role": role, "content": data}]
elif isinstance(data, list):
if not data:
raise ValueError("convert_to_openai_messages received an empty list.")
if _is_openai_format(data):
messages = data
elif _is_sharegpt_format(data):
messages = _convert_sharegpt(data)
else:
raise ValueError(
f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
f"Got: {type(data)} with keys/content: {list(data[0].keys()) if data else data}"
)
elif isinstance(data, dict):
if _is_alpaca_format(data):
messages = _convert_alpaca(data)
else:
raise ValueError(
f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
f"Got: {type(data)} with keys/content: {data}"
)
else:
raise ValueError(
f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
f"Got: {type(data)}"
)
if expand_image:
messages = expand_image_placeholders(messages)
return messages
def expand_image_placeholders(messages, placeholder="<image>"):
for msg in messages:
content = msg.get("content")
if not isinstance(content, str) or placeholder not in content:
continue
parts = content.split(placeholder)
content_list = []
for i, part in enumerate(parts):
if part:
content_list.append({"type": "text", "text": part})
if i < len(parts) - 1:
content_list.append({"type": "image"})
msg["content"] = content_list
return messages