import os from typing import List, Dict, Any import torch from datasets import DatasetDict, interleave_datasets, load_dataset, load_from_disk from transformers import AutoProcessor, AutoTokenizer from kdflow.utils.logging_utils import init_logger logger = init_logger(__name__) def validate_dataset_columns(dataset, **column_settings): """Validate that configured dataset keys exist in the source dataset. Supports dot-notation for nested keys (e.g., "reward_model.ground_truth" -> top-key "reward_model"). """ available_columns = list(dataset.column_names) missing_columns = [ f"{setting_name}={column_name!r}" for setting_name, column_name in column_settings.items() if column_name.split(".")[0] not in available_columns ] if missing_columns: raise ValueError( f"Dataset is missing configured dataset column(s): " f"{', '.join(missing_columns)}. Available columns: {available_columns}." ) def exist_and_not_none(d, key): return key in d and not d[key] is None def get_nested_value(data: dict, key_path: str, default=None): """Get a value from a nested dict using dot-notation key path. Example: get_nested_value({'a': {'b': 1}}, 'a.b') -> 1 get_nested_value({'a': 1}, 'a.b') -> None """ keys = key_path.split(".") value = data for k in keys: if isinstance(value, dict): value = value.get(k) else: return default return value def blending_datasets( datasets, probabilities=None, strategy=None, seed=42, max_count=1e8, stopping_strategy="all_exhausted", dataset_split="train", ): """Blend multiple datasets with optional probability sampling. Args: datasets (str): Comma-separated list of dataset paths probabilities (str, optional): Comma-separated list of probabilities for sampling. If None, datasets will be concatenated without probability sampling. strategy: Training strategy object seed (int): Random seed max_count (int): Maximum number of samples per dataset """ dataset_split = dataset_split or "train" datasets = datasets.split(",") if probabilities is not None: probabilities = list(map(float, probabilities.split(","))) assert len(probabilities) == len(datasets) data_list = [] for i, dataset in enumerate(datasets): dataset = dataset.strip() logger.info(f"dataset: {dataset}") data_dir = dataset.split("@")[1].strip() if "@" in dataset else None dataset = dataset.split("@")[0].strip() dataset_basename = os.path.basename(dataset) ext = os.path.splitext(dataset)[-1] if ext == ".py" or ( os.path.isdir(dataset) and os.path.exists(os.path.join(dataset, f"{dataset_basename}.py")) ): data = load_dataset(dataset, trust_remote_code=True) logger.info(f"loaded {dataset} with python script") elif ext in [".json", ".jsonl", ".csv", ".parquet", ".arrow"]: ext = ext.lower().strip(".") if ext == "jsonl": ext = "json" data = load_dataset(ext, data_files={dataset_split: dataset}) logger.info(f"loaded {dataset} with data_files={dataset}") # local dataset saved with `datasets.Dataset.save_to_disk` elif os.path.isdir(dataset): try: data = load_from_disk(dataset) logger.info(f"loaded {dataset} from disk") except Exception as e: data = load_dataset(dataset, data_dir=data_dir) logger.info(f"loaded {dataset} from files") else: data = load_dataset(dataset, data_dir=data_dir) logger.info(f"loaded {dataset} from files") if isinstance(data, DatasetDict): if dataset_split not in data: raise ValueError( f"Dataset {dataset!r} does not contain split {dataset_split!r}. " f"Available splits: {list(data)}." ) data = data[dataset_split] data = data.select(range(min(max_count, len(data)))) data_list.append(data) if strategy.is_rank_0(): print(data_list) if probabilities is None: from datasets import concatenate_datasets dataset = concatenate_datasets(data_list) else: dataset = interleave_datasets( data_list, probabilities=probabilities, seed=seed, stopping_strategy=stopping_strategy, ) return dataset def get_tokenizer_or_processor(model_name_or_path, model=None, padding_side="right", use_fast=True, need_processor=False): if need_processor: return get_processor(model_name_or_path, model, padding_side, use_fast) else: return get_tokenizer(model_name_or_path, model, padding_side, use_fast) def get_processor(model_name_or_path, model=None, padding_side="right", use_fast=True): processor = AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast) processor.tokenizer.padding_side = padding_side if processor.tokenizer.pad_token is None: logger.info("Detect no pad_token in tokenizer, set it to eos_token.") processor.tokenizer.pad_token = processor.tokenizer.eos_token processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id if model is not None: model.config.text_config.pad_token_id = processor.tokenizer.pad_token_id return processor def get_tokenizer(model_name_or_path, model=None, padding_side="right", use_fast=True): try: # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer. tokenizer = AutoTokenizer.from_pretrained( model_name_or_path, trust_remote_code=True, use_fast=use_fast, fix_mistral_regex=True, ) except TypeError: # Older transformers versions do not accept fix_mistral_regex. tokenizer = AutoTokenizer.from_pretrained( model_name_or_path, trust_remote_code=True, use_fast=use_fast, ) tokenizer.padding_side = padding_side if tokenizer.pad_token is None: logger.info("Detect no pad_token in tokenizer, set it to eos_token.") tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token_id = tokenizer.eos_token_id if model is not None: model.config.pad_token_id = tokenizer.pad_token_id return tokenizer def convert_token_to_id(token, tokenizer): if isinstance(token, str): token = tokenizer.encode(token, add_special_tokens=False) assert len(token) == 1 return token[0] else: raise ValueError("token should be int or str") def zero_pad_sequences( sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False ) -> torch.Tensor: from torch.nn.utils.rnn import pad_sequence assert side in ("left", "right") sequences = [seq.squeeze(0) for seq in sequences] if side == "left": sequences = [seq.flip(dims=0) for seq in sequences] padded_sequences = pad_sequence( sequences, batch_first=True, padding_value=value ) if side == "left": padded_sequences = torch.flip(padded_sequences, dims=[1]) return padded_sequences def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True): """Remove the pad token. Return tensors and not lists. Args: input_ids shape: [bs, seq_length] attention_mask shape: [bs, seq_length] Returns: no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query. """ no_padding_batch = [] for ids, mask in zip(input_ids, attention_mask): # Fix for both left and right padding ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist() no_padding_batch.append(ids) return no_padding_batch # ShareGPT role mapping to OpenAI roles SHAREGPT_ROLE_MAP = { "human": "user", "user": "user", "gpt": "assistant", "assistant": "assistant", "system": "system", } def _is_sharegpt_format(data): """Check if data is in ShareGPT format (list of dicts with 'from'/'value' keys).""" if not isinstance(data, list) or not data or not isinstance(data[0], dict): return False return "from" in data[0] and "value" in data[0] def _is_openai_format(data): """Check if data is already in OpenAI messages format (list of dicts with 'role'/'content' keys).""" if not isinstance(data, list) or not data or not isinstance(data[0], dict): return False return "role" in data[0] and "content" in data[0] def _is_alpaca_format(data): """Check if data is in Alpaca format (dict with 'instruction' key).""" if not isinstance(data, dict): return False return "instruction" in data def _convert_sharegpt(messages): """Convert ShareGPT format messages to OpenAI messages format. ShareGPT: [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}] OpenAI: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}] """ converted = [] for msg in messages: role = SHAREGPT_ROLE_MAP.get(msg["from"], msg["from"]) content = msg.get("value") or "" converted.append({"role": role, "content": content}) return converted def _convert_alpaca(data): """Convert Alpaca format to OpenAI messages format. Alpaca format fields: - instruction (required): the main instruction/question - input (optional): additional input context - output (optional): the expected response - system (optional): system prompt - history (optional): list of [user_msg, assistant_msg] pairs Returns: List of OpenAI messages: [{"role": "...", "content": "..."}, ...] """ messages = [] # System prompt system_prompt = data.get("system", "") if system_prompt: messages.append({"role": "system", "content": system_prompt}) # History turns history = data.get("history", []) for turn in history: if isinstance(turn, (list, tuple)) and len(turn) == 2: messages.append({"role": "user", "content": turn[0]}) messages.append({"role": "assistant", "content": turn[1]}) # Current instruction + optional input instruction = data.get("instruction", "") extra_input = data.get("input", "") if extra_input: user_content = f"{instruction}\n{extra_input}" else: user_content = instruction messages.append({"role": "user", "content": user_content}) # Output (response) output = data.get("output", "") if output: messages.append({"role": "assistant", "content": output}) return messages def convert_to_openai_messages(data, role="user", expand_image=False): """Unified converter: auto-detect data format and convert to OpenAI messages. Supported input formats: 1. OpenAI messages (already): [{"role": "user", "content": "..."}] -> returned as-is 2. ShareGPT: [{"from": "human", "value": "..."}] -> converted 3. Alpaca: {"instruction": "...", "input": "...", "output": "...", ...} -> converted 4. Plain string: "..." -> wrapped as [{"role": role, "content": "..."}] Args: data: Input data in any supported format. role: Role to assign when data is a plain string. Defaults to "user". expand_image: Whether to expand placeholders into multi-part content. Only set to True for multimodal (image) scenarios. Defaults to False. Returns: List of dicts in OpenAI messages format: [{"role": "...", "content": "..."}, ...] """ if data is None: raise ValueError("convert_to_openai_messages received None input.") if isinstance(data, str): messages = [{"role": role, "content": data}] elif isinstance(data, list): if not data: raise ValueError("convert_to_openai_messages received an empty list.") if _is_openai_format(data): messages = data elif _is_sharegpt_format(data): messages = _convert_sharegpt(data) else: raise ValueError( f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. " f"Got: {type(data)} with keys/content: {list(data[0].keys()) if data else data}" ) elif isinstance(data, dict): if _is_alpaca_format(data): messages = _convert_alpaca(data) else: raise ValueError( f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. " f"Got: {type(data)} with keys/content: {data}" ) else: raise ValueError( f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. " f"Got: {type(data)}" ) if expand_image: messages = expand_image_placeholders(messages) return messages def expand_image_placeholders(messages, placeholder=""): for msg in messages: content = msg.get("content") if not isinstance(content, str) or placeholder not in content: continue parts = content.split(placeholder) content_list = [] for i, part in enumerate(parts): if part: content_list.append({"type": "text", "text": part}) if i < len(parts) - 1: content_list.append({"type": "image"}) msg["content"] = content_list return messages