File size: 1,444 Bytes
d2a1a41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
"""Dynamic padding + next-token label construction."""
from __future__ import annotations

import numpy as np
import torch

from ..tokenizers.multimodal import Sample


def collate(samples: list[Sample], pad_id: int, multiple: int = 8, *, target_length: int | None = None,

            batch_size: int | None = None) -> tuple[torch.Tensor, torch.Tensor]:
    """Right-pad to the batch max (rounded up to `multiple`) and build shifted labels.



    labels[j] = tokens[j+1] where j+1 >= target_start, else -100 (prompt and padding are not

    trained on).  Right padding is safe under causal attention: real tokens never attend to it.

    """
    L = max(len(s.tokens) for s in samples) - 1
    if target_length is not None:
        if target_length < L:
            raise ValueError("fixed padding cannot truncate a sample")
        L = target_length
    else:
        L = -(-L // multiple) * multiple
    B = len(samples) if batch_size is None else batch_size
    if B < len(samples):
        raise ValueError("fixed batch size cannot discard samples")
    inp = np.full((B, L), pad_id, dtype=np.int64)
    lab = np.full((B, L), -100, dtype=np.int64)
    for i, s in enumerate(samples):
        t = s.tokens.astype(np.int64)
        n = len(t) - 1
        inp[i, :n] = t[:-1]
        lab[i, :n] = t[1:]
        lab[i, : max(0, s.target_start - 1)] = -100
    return torch.from_numpy(inp), torch.from_numpy(lab)