File size: 6,292 Bytes
a3baf17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""Prepare the same text, splits, and token order for every optimizer."""

import math
import os
import random
from pathlib import Path

import torch
from datasets import load_dataset
from tqdm import tqdm
from transformers import AutoTokenizer

DATASET_NAME = "browndw/human-ai-parallel-corpus"
HERE = Path(__file__).resolve().parent


def load_tokenizer(name):
    tokenizer = AutoTokenizer.from_pretrained(name)
    if tokenizer.bos_token_id is None:
        tokenizer.add_special_tokens({"bos_token": "<|bos|>"})
    if tokenizer.pad_token_id is None or tokenizer.eos_token_id is None:
        raise ValueError("The tokenizer must have padding and EOS tokens.")
    return tokenizer


def split_documents(rows, seed):
    # Different authors' versions of the same source stay in the same split.
    groups = {}
    for index, document_id in enumerate(rows["doc_id"]):
        source_id = document_id.split("@")[0]
        groups.setdefault(source_id, []).append(index)

    source_ids = sorted(groups)
    if len(source_ids) < 20:
        raise ValueError("At least 20 source documents are needed for 90/5/5 splits.")
    random.Random(seed).shuffle(source_ids)
    train_end = int(0.9 * len(source_ids))
    validation_end = int(0.95 * len(source_ids))
    split_groups = {
        "train": source_ids[:train_end],
        "validation": source_ids[train_end:validation_end],
        "test": source_ids[validation_end:],
    }
    splits = {}
    for name, sources in split_groups.items():
        indices = []
        for source in sources:
            indices.extend(groups[source])
        random.Random(seed).shuffle(indices)
        splits[name] = indices
    return splits


def prepare_data(args, tokenizer):
    settings = {
        "dataset": DATASET_NAME,
        "seed": args.seed,
        "data_fraction": args.data_fraction,
        "evaluation_fraction": args.evaluation_fraction,
        "validation_samples": args.validation_samples,
        "test_samples": args.test_samples,
        "tokenizer": args.tokenizer_name,
        "vocabulary": tokenizer.get_vocab(),
    }
    cache_dir = Path(args.cache_dir)
    cache_dir.mkdir(parents=True, exist_ok=True)
    cache_file = cache_dir / "prepared.pt"
    if cache_file.exists():
        prepared = torch.load(cache_file, map_location="cpu", weights_only=True)
        if prepared["settings"] != settings:
            raise ValueError("Prepared data settings differ. Use a different --cache-dir.")
        return prepared

    rows = load_dataset(DATASET_NAME, split="train", cache_dir=str(cache_dir / "dataset"))
    splits = split_documents(rows, args.seed)
    train_size = max(1, int(len(splits["train"]) * args.data_fraction))
    validation_size = max(1, int(min(args.validation_samples, len(splits["validation"]))
                                 * args.evaluation_fraction))
    test_size = max(1, int(min(args.test_samples, len(splits["test"]))
                           * args.evaluation_fraction))
    selected = {
        "train": splits["train"][:train_size],
        "validation": splits["validation"][:validation_size],
        "test": splits["test"][:test_size],
    }

    # Count the full training split, but keep tokens only for the requested subset.
    # This makes the fraction-of-full-dataset axis an actual token measurement.
    selected_train = set(selected["train"])
    encoded_train = {}
    full_train_tokens = 0
    for index in tqdm(splits["train"], desc="Tokenizing/counting training text"):
        ids = [tokenizer.bos_token_id]
        ids += tokenizer.encode(rows[index]["text"], add_special_tokens=False)
        ids += [tokenizer.eos_token_id]
        full_train_tokens += len(ids)
        if index in selected_train:
            encoded_train[index] = torch.tensor(ids, dtype=torch.int32)
    train_tokens = torch.cat([encoded_train[index] for index in selected["train"]])

    prepared = {"settings": settings, "train_tokens": train_tokens, "test_documents": []}
    for name in ("validation", "test"):
        documents = []
        for index in selected[name]:
            text = rows[index]["text"]
            ids = [tokenizer.bos_token_id]
            ids += tokenizer.encode(text, add_special_tokens=False)
            ids += [tokenizer.eos_token_id]
            documents.append(torch.tensor(ids, dtype=torch.int32))
            if name == "test":
                prepared["test_documents"].append({"doc_id": rows[index]["doc_id"], "ids": ids})
        prepared[name + "_tokens"] = torch.cat(documents)

    prepared["metadata"] = {
        "dataset_fingerprint": rows._fingerprint,
        "split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
        "full_split_documents": {name: len(indices) for name, indices in splits.items()},
        "selected_documents": {name: len(indices) for name, indices in selected.items()},
        "selected_indices": selected,
        "full_train_target_tokens": full_train_tokens - 1,
        "train_target_tokens": len(train_tokens) - 1,
        "validation_target_tokens": len(prepared["validation_tokens"]) - 1,
        "test_target_tokens": len(prepared["test_tokens"]) - 1,
    }
    # A finished cache replaces the temporary file, so another run cannot read half a file.
    temporary_file = cache_file.with_suffix(f".{os.getpid()}.tmp")
    torch.save(prepared, temporary_file)
    temporary_file.replace(cache_file)
    return prepared


def make_batch(tokens, start, token_limit, batch_size, context_size, pad_id):
    count = min(token_limit, batch_size * context_size, len(tokens) - 1 - start)
    if count <= 0:
        raise ValueError("A batch must contain at least one target token.")
    rows = math.ceil(count / context_size)
    inputs = torch.full((rows, context_size), pad_id, dtype=torch.long)
    labels = torch.full((rows, context_size), -100, dtype=torch.long)
    padding_mask = torch.ones((rows, context_size), dtype=torch.bool)
    for row in range(rows):
        offset = start + row * context_size
        length = min(context_size, count - row * context_size)
        inputs[row, :length] = tokens[offset:offset + length]
        labels[row, :length] = tokens[offset + 1:offset + length + 1]
        padding_mask[row, :length] = False
    return inputs, labels, padding_mask, count