File size: 6,292 Bytes
a3baf17 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | """Prepare the same text, splits, and token order for every optimizer."""
import math
import os
import random
from pathlib import Path
import torch
from datasets import load_dataset
from tqdm import tqdm
from transformers import AutoTokenizer
DATASET_NAME = "browndw/human-ai-parallel-corpus"
HERE = Path(__file__).resolve().parent
def load_tokenizer(name):
tokenizer = AutoTokenizer.from_pretrained(name)
if tokenizer.bos_token_id is None:
tokenizer.add_special_tokens({"bos_token": "<|bos|>"})
if tokenizer.pad_token_id is None or tokenizer.eos_token_id is None:
raise ValueError("The tokenizer must have padding and EOS tokens.")
return tokenizer
def split_documents(rows, seed):
# Different authors' versions of the same source stay in the same split.
groups = {}
for index, document_id in enumerate(rows["doc_id"]):
source_id = document_id.split("@")[0]
groups.setdefault(source_id, []).append(index)
source_ids = sorted(groups)
if len(source_ids) < 20:
raise ValueError("At least 20 source documents are needed for 90/5/5 splits.")
random.Random(seed).shuffle(source_ids)
train_end = int(0.9 * len(source_ids))
validation_end = int(0.95 * len(source_ids))
split_groups = {
"train": source_ids[:train_end],
"validation": source_ids[train_end:validation_end],
"test": source_ids[validation_end:],
}
splits = {}
for name, sources in split_groups.items():
indices = []
for source in sources:
indices.extend(groups[source])
random.Random(seed).shuffle(indices)
splits[name] = indices
return splits
def prepare_data(args, tokenizer):
settings = {
"dataset": DATASET_NAME,
"seed": args.seed,
"data_fraction": args.data_fraction,
"evaluation_fraction": args.evaluation_fraction,
"validation_samples": args.validation_samples,
"test_samples": args.test_samples,
"tokenizer": args.tokenizer_name,
"vocabulary": tokenizer.get_vocab(),
}
cache_dir = Path(args.cache_dir)
cache_dir.mkdir(parents=True, exist_ok=True)
cache_file = cache_dir / "prepared.pt"
if cache_file.exists():
prepared = torch.load(cache_file, map_location="cpu", weights_only=True)
if prepared["settings"] != settings:
raise ValueError("Prepared data settings differ. Use a different --cache-dir.")
return prepared
rows = load_dataset(DATASET_NAME, split="train", cache_dir=str(cache_dir / "dataset"))
splits = split_documents(rows, args.seed)
train_size = max(1, int(len(splits["train"]) * args.data_fraction))
validation_size = max(1, int(min(args.validation_samples, len(splits["validation"]))
* args.evaluation_fraction))
test_size = max(1, int(min(args.test_samples, len(splits["test"]))
* args.evaluation_fraction))
selected = {
"train": splits["train"][:train_size],
"validation": splits["validation"][:validation_size],
"test": splits["test"][:test_size],
}
# Count the full training split, but keep tokens only for the requested subset.
# This makes the fraction-of-full-dataset axis an actual token measurement.
selected_train = set(selected["train"])
encoded_train = {}
full_train_tokens = 0
for index in tqdm(splits["train"], desc="Tokenizing/counting training text"):
ids = [tokenizer.bos_token_id]
ids += tokenizer.encode(rows[index]["text"], add_special_tokens=False)
ids += [tokenizer.eos_token_id]
full_train_tokens += len(ids)
if index in selected_train:
encoded_train[index] = torch.tensor(ids, dtype=torch.int32)
train_tokens = torch.cat([encoded_train[index] for index in selected["train"]])
prepared = {"settings": settings, "train_tokens": train_tokens, "test_documents": []}
for name in ("validation", "test"):
documents = []
for index in selected[name]:
text = rows[index]["text"]
ids = [tokenizer.bos_token_id]
ids += tokenizer.encode(text, add_special_tokens=False)
ids += [tokenizer.eos_token_id]
documents.append(torch.tensor(ids, dtype=torch.int32))
if name == "test":
prepared["test_documents"].append({"doc_id": rows[index]["doc_id"], "ids": ids})
prepared[name + "_tokens"] = torch.cat(documents)
prepared["metadata"] = {
"dataset_fingerprint": rows._fingerprint,
"split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
"full_split_documents": {name: len(indices) for name, indices in splits.items()},
"selected_documents": {name: len(indices) for name, indices in selected.items()},
"selected_indices": selected,
"full_train_target_tokens": full_train_tokens - 1,
"train_target_tokens": len(train_tokens) - 1,
"validation_target_tokens": len(prepared["validation_tokens"]) - 1,
"test_target_tokens": len(prepared["test_tokens"]) - 1,
}
# A finished cache replaces the temporary file, so another run cannot read half a file.
temporary_file = cache_file.with_suffix(f".{os.getpid()}.tmp")
torch.save(prepared, temporary_file)
temporary_file.replace(cache_file)
return prepared
def make_batch(tokens, start, token_limit, batch_size, context_size, pad_id):
count = min(token_limit, batch_size * context_size, len(tokens) - 1 - start)
if count <= 0:
raise ValueError("A batch must contain at least one target token.")
rows = math.ceil(count / context_size)
inputs = torch.full((rows, context_size), pad_id, dtype=torch.long)
labels = torch.full((rows, context_size), -100, dtype=torch.long)
padding_mask = torch.ones((rows, context_size), dtype=torch.bool)
for row in range(rows):
offset = start + row * context_size
length = min(context_size, count - row * context_size)
inputs[row, :length] = tokens[offset:offset + length]
labels[row, :length] = tokens[offset + 1:offset + length + 1]
padding_mask[row, :length] = False
return inputs, labels, padding_mask, count
|