File size: 14,066 Bytes
0ca529a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
import os
from typing import List, Dict, Any

import torch
from datasets import DatasetDict, interleave_datasets, load_dataset, load_from_disk
from transformers import AutoProcessor, AutoTokenizer

from kdflow.utils.logging_utils import init_logger

logger = init_logger(__name__)


def validate_dataset_columns(dataset, **column_settings):
    """Validate that configured dataset keys exist in the source dataset.

    Supports dot-notation for nested keys (e.g., "reward_model.ground_truth" -> top-key "reward_model").
    """
    available_columns = list(dataset.column_names)
    missing_columns = [
        f"{setting_name}={column_name!r}"
        for setting_name, column_name in column_settings.items()
        if column_name.split(".")[0] not in available_columns
    ]
    if missing_columns:
        raise ValueError(
            f"Dataset is missing configured dataset column(s): "
            f"{', '.join(missing_columns)}. Available columns: {available_columns}."
        )


def exist_and_not_none(d, key):
    return key in d and not d[key] is None


def get_nested_value(data: dict, key_path: str, default=None):
    """Get a value from a nested dict using dot-notation key path.

    Example:
        get_nested_value({'a': {'b': 1}}, 'a.b') -> 1
        get_nested_value({'a': 1}, 'a.b') -> None
    """
    keys = key_path.split(".")
    value = data
    for k in keys:
        if isinstance(value, dict):
            value = value.get(k)
        else:
            return default
    return value


def blending_datasets(
    datasets,
    probabilities=None,
    strategy=None,
    seed=42,
    max_count=1e8,
    stopping_strategy="all_exhausted",
    dataset_split="train",
):
    """Blend multiple datasets with optional probability sampling.

    Args:
        datasets (str): Comma-separated list of dataset paths
        probabilities (str, optional): Comma-separated list of probabilities for sampling.
            If None, datasets will be concatenated without probability sampling.
        strategy: Training strategy object
        seed (int): Random seed
        max_count (int): Maximum number of samples per dataset
    """
    dataset_split = dataset_split or "train"
    datasets = datasets.split(",")
    if probabilities is not None:
        probabilities = list(map(float, probabilities.split(",")))
        assert len(probabilities) == len(datasets)

    data_list = []
    for i, dataset in enumerate(datasets):
        dataset = dataset.strip()
        logger.info(f"dataset: {dataset}")

        data_dir = dataset.split("@")[1].strip() if "@" in dataset else None
        dataset = dataset.split("@")[0].strip()
        dataset_basename = os.path.basename(dataset)

        ext = os.path.splitext(dataset)[-1]
        if ext == ".py" or (
            os.path.isdir(dataset) and os.path.exists(os.path.join(dataset, f"{dataset_basename}.py"))
        ):
            data = load_dataset(dataset, trust_remote_code=True)
            logger.info(f"loaded {dataset} with python script")
        elif ext in [".json", ".jsonl", ".csv", ".parquet", ".arrow"]:
            ext = ext.lower().strip(".")
            if ext == "jsonl":
                ext = "json"
            data = load_dataset(ext, data_files={dataset_split: dataset})
            logger.info(f"loaded {dataset} with data_files={dataset}")
        # local dataset saved with `datasets.Dataset.save_to_disk`
        elif os.path.isdir(dataset):
            try:
                data = load_from_disk(dataset)
                logger.info(f"loaded {dataset} from disk")
            except Exception as e:
                data = load_dataset(dataset, data_dir=data_dir)
                logger.info(f"loaded {dataset} from files")
        else:
            data = load_dataset(dataset, data_dir=data_dir)
            logger.info(f"loaded {dataset} from files")

        if isinstance(data, DatasetDict):
            if dataset_split not in data:
                raise ValueError(
                    f"Dataset {dataset!r} does not contain split {dataset_split!r}. "
                    f"Available splits: {list(data)}."
                )
            data = data[dataset_split]
        data = data.select(range(min(max_count, len(data))))
        data_list.append(data)

    if strategy.is_rank_0():
        print(data_list)

    if probabilities is None:
        from datasets import concatenate_datasets

        dataset = concatenate_datasets(data_list)
    else:
        dataset = interleave_datasets(
            data_list,
            probabilities=probabilities,
            seed=seed,
            stopping_strategy=stopping_strategy,
        )

    return dataset


def get_tokenizer_or_processor(model_name_or_path, model=None, padding_side="right", use_fast=True, need_processor=False):
    if need_processor:
        return get_processor(model_name_or_path, model, padding_side, use_fast)
    else:
        return get_tokenizer(model_name_or_path, model, padding_side, use_fast)


def get_processor(model_name_or_path, model=None, padding_side="right", use_fast=True):
    processor = AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast)
    processor.tokenizer.padding_side = padding_side
    if processor.tokenizer.pad_token is None:
        logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
        processor.tokenizer.pad_token = processor.tokenizer.eos_token
        processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id
        if model is not None:
            model.config.text_config.pad_token_id = processor.tokenizer.pad_token_id

    return processor


def get_tokenizer(model_name_or_path, model=None, padding_side="right", use_fast=True):
    try:
        # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style
        # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it
        # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer.
        tokenizer = AutoTokenizer.from_pretrained(
            model_name_or_path,
            trust_remote_code=True,
            use_fast=use_fast,
            fix_mistral_regex=True,
        )
    except TypeError:
        # Older transformers versions do not accept fix_mistral_regex.
        tokenizer = AutoTokenizer.from_pretrained(
            model_name_or_path,
            trust_remote_code=True,
            use_fast=use_fast,
        )
    tokenizer.padding_side = padding_side
    if tokenizer.pad_token is None:
        logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
        tokenizer.pad_token = tokenizer.eos_token
        tokenizer.pad_token_id = tokenizer.eos_token_id
        if model is not None:
            model.config.pad_token_id = tokenizer.pad_token_id

    return tokenizer


def convert_token_to_id(token, tokenizer):
    if isinstance(token, str):
        token = tokenizer.encode(token, add_special_tokens=False)
        assert len(token) == 1
        return token[0]
    else:
        raise ValueError("token should be int or str")


def zero_pad_sequences(
    sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False
) -> torch.Tensor:
    from torch.nn.utils.rnn import pad_sequence
    assert side in ("left", "right")
    sequences = [seq.squeeze(0) for seq in sequences]
    if side == "left":
        sequences = [seq.flip(dims=0) for seq in sequences]
    padded_sequences = pad_sequence(
        sequences,
        batch_first=True,
        padding_value=value
    )
    if side == "left":
        padded_sequences = torch.flip(padded_sequences, dims=[1])
    return padded_sequences


def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True):
    """Remove the pad token. Return tensors and not lists.

    Args:
        input_ids shape: [bs, seq_length]
        attention_mask shape: [bs, seq_length]
    Returns:
        no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query.
    """
    no_padding_batch = []
    for ids, mask in zip(input_ids, attention_mask):
        # Fix for both left and right padding
        ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist()
        no_padding_batch.append(ids)
    return no_padding_batch


# ShareGPT role mapping to OpenAI roles
SHAREGPT_ROLE_MAP = {
    "human": "user",
    "user": "user",
    "gpt": "assistant",
    "assistant": "assistant",
    "system": "system",
}


def _is_sharegpt_format(data):
    """Check if data is in ShareGPT format (list of dicts with 'from'/'value' keys)."""
    if not isinstance(data, list) or not data or not isinstance(data[0], dict):
        return False
    return "from" in data[0] and "value" in data[0]


def _is_openai_format(data):
    """Check if data is already in OpenAI messages format (list of dicts with 'role'/'content' keys)."""
    if not isinstance(data, list) or not data or not isinstance(data[0], dict):
        return False
    return "role" in data[0] and "content" in data[0]


def _is_alpaca_format(data):
    """Check if data is in Alpaca format (dict with 'instruction' key)."""
    if not isinstance(data, dict):
        return False
    return "instruction" in data


def _convert_sharegpt(messages):
    """Convert ShareGPT format messages to OpenAI messages format.
    
    ShareGPT: [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]
    OpenAI:   [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
    """
    converted = []
    for msg in messages:
        role = SHAREGPT_ROLE_MAP.get(msg["from"], msg["from"])
        content = msg.get("value") or ""
        converted.append({"role": role, "content": content})
    return converted


def _convert_alpaca(data):
    """Convert Alpaca format to OpenAI messages format.
    
    Alpaca format fields:
        - instruction (required): the main instruction/question
        - input (optional): additional input context
        - output (optional): the expected response
        - system (optional): system prompt
        - history (optional): list of [user_msg, assistant_msg] pairs
    
    Returns:
        List of OpenAI messages: [{"role": "...", "content": "..."}, ...]
    """
    messages = []

    # System prompt
    system_prompt = data.get("system", "")
    if system_prompt:
        messages.append({"role": "system", "content": system_prompt})

    # History turns
    history = data.get("history", [])
    for turn in history:
        if isinstance(turn, (list, tuple)) and len(turn) == 2:
            messages.append({"role": "user", "content": turn[0]})
            messages.append({"role": "assistant", "content": turn[1]})

    # Current instruction + optional input
    instruction = data.get("instruction", "")
    extra_input = data.get("input", "")
    if extra_input:
        user_content = f"{instruction}\n{extra_input}"
    else:
        user_content = instruction
    messages.append({"role": "user", "content": user_content})

    # Output (response)
    output = data.get("output", "")
    if output:
        messages.append({"role": "assistant", "content": output})

    return messages


def convert_to_openai_messages(data, role="user", expand_image=False):
    """Unified converter: auto-detect data format and convert to OpenAI messages.
    
    Supported input formats:
        1. OpenAI messages (already): [{"role": "user", "content": "..."}] -> returned as-is
        2. ShareGPT: [{"from": "human", "value": "..."}] -> converted
        3. Alpaca: {"instruction": "...", "input": "...", "output": "...", ...} -> converted
        4. Plain string: "..." -> wrapped as [{"role": role, "content": "..."}]
    
    Args:
        data: Input data in any supported format.
        role: Role to assign when data is a plain string. Defaults to "user".
        expand_image: Whether to expand <image> placeholders into multi-part content.
            Only set to True for multimodal (image) scenarios. Defaults to False.
    
    Returns:
        List of dicts in OpenAI messages format: [{"role": "...", "content": "..."}, ...]
    """
    if data is None:
        raise ValueError("convert_to_openai_messages received None input.")

    if isinstance(data, str):
        messages = [{"role": role, "content": data}]
    elif isinstance(data, list):
        if not data:
            raise ValueError("convert_to_openai_messages received an empty list.")
        if _is_openai_format(data):
            messages = data
        elif _is_sharegpt_format(data):
            messages = _convert_sharegpt(data)
        else:
            raise ValueError(
                f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
                f"Got: {type(data)} with keys/content: {list(data[0].keys()) if data else data}"
            )
    elif isinstance(data, dict):
        if _is_alpaca_format(data):
            messages = _convert_alpaca(data)
        else:
            raise ValueError(
                f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
                f"Got: {type(data)} with keys/content: {data}"
            )
    else:
        raise ValueError(
            f"Unsupported data format. Expected OpenAI messages, ShareGPT, Alpaca, or plain string. "
            f"Got: {type(data)}"
        )

    if expand_image:
        messages = expand_image_placeholders(messages)
    return messages


def expand_image_placeholders(messages, placeholder="<image>"):
    for msg in messages:
        content = msg.get("content")
        if not isinstance(content, str) or placeholder not in content:
            continue
        parts = content.split(placeholder)
        content_list = []
        for i, part in enumerate(parts):
            if part:
                content_list.append({"type": "text", "text": part})
            if i < len(parts) - 1:
                content_list.append({"type": "image"})
        msg["content"] = content_list
    return messages