walke007's picture
Upload Part 2 checkpoints and recorded results
a3baf17 verified
Raw History Blame Contribute Delete
6.29 kB
"""Prepare the same text, splits, and token order for every optimizer."""
import math
import os
import random
from pathlib import Path
import torch
from datasets import load_dataset
from tqdm import tqdm
from transformers import AutoTokenizer
DATASET_NAME = "browndw/human-ai-parallel-corpus"
HERE = Path(__file__).resolve().parent
def load_tokenizer(name):
tokenizer = AutoTokenizer.from_pretrained(name)
if tokenizer.bos_token_id is None:
tokenizer.add_special_tokens({"bos_token": "<|bos|>"})
if tokenizer.pad_token_id is None or tokenizer.eos_token_id is None:
raise ValueError("The tokenizer must have padding and EOS tokens.")
return tokenizer
def split_documents(rows, seed):
# Different authors' versions of the same source stay in the same split.
groups = {}
for index, document_id in enumerate(rows["doc_id"]):
source_id = document_id.split("@")[0]
groups.setdefault(source_id, []).append(index)
source_ids = sorted(groups)
if len(source_ids) < 20:
raise ValueError("At least 20 source documents are needed for 90/5/5 splits.")
random.Random(seed).shuffle(source_ids)
train_end = int(0.9 * len(source_ids))
validation_end = int(0.95 * len(source_ids))
split_groups = {
"train": source_ids[:train_end],
"validation": source_ids[train_end:validation_end],
"test": source_ids[validation_end:],
}
splits = {}
for name, sources in split_groups.items():
indices = []
for source in sources:
indices.extend(groups[source])
random.Random(seed).shuffle(indices)
splits[name] = indices
return splits
def prepare_data(args, tokenizer):
settings = {
"dataset": DATASET_NAME,
"seed": args.seed,
"data_fraction": args.data_fraction,
"evaluation_fraction": args.evaluation_fraction,
"validation_samples": args.validation_samples,
"test_samples": args.test_samples,
"tokenizer": args.tokenizer_name,
"vocabulary": tokenizer.get_vocab(),
}
cache_dir = Path(args.cache_dir)
cache_dir.mkdir(parents=True, exist_ok=True)
cache_file = cache_dir / "prepared.pt"
if cache_file.exists():
prepared = torch.load(cache_file, map_location="cpu", weights_only=True)
if prepared["settings"] != settings:
raise ValueError("Prepared data settings differ. Use a different --cache-dir.")
return prepared
rows = load_dataset(DATASET_NAME, split="train", cache_dir=str(cache_dir / "dataset"))
splits = split_documents(rows, args.seed)
train_size = max(1, int(len(splits["train"]) * args.data_fraction))
validation_size = max(1, int(min(args.validation_samples, len(splits["validation"]))
* args.evaluation_fraction))
test_size = max(1, int(min(args.test_samples, len(splits["test"]))
* args.evaluation_fraction))
selected = {
"train": splits["train"][:train_size],
"validation": splits["validation"][:validation_size],
"test": splits["test"][:test_size],
}
# Count the full training split, but keep tokens only for the requested subset.
# This makes the fraction-of-full-dataset axis an actual token measurement.
selected_train = set(selected["train"])
encoded_train = {}
full_train_tokens = 0
for index in tqdm(splits["train"], desc="Tokenizing/counting training text"):
ids = [tokenizer.bos_token_id]
ids += tokenizer.encode(rows[index]["text"], add_special_tokens=False)
ids += [tokenizer.eos_token_id]
full_train_tokens += len(ids)
if index in selected_train:
encoded_train[index] = torch.tensor(ids, dtype=torch.int32)
train_tokens = torch.cat([encoded_train[index] for index in selected["train"]])
prepared = {"settings": settings, "train_tokens": train_tokens, "test_documents": []}
for name in ("validation", "test"):
documents = []
for index in selected[name]:
text = rows[index]["text"]
ids = [tokenizer.bos_token_id]
ids += tokenizer.encode(text, add_special_tokens=False)
ids += [tokenizer.eos_token_id]
documents.append(torch.tensor(ids, dtype=torch.int32))
if name == "test":
prepared["test_documents"].append({"doc_id": rows[index]["doc_id"], "ids": ids})
prepared[name + "_tokens"] = torch.cat(documents)
prepared["metadata"] = {
"dataset_fingerprint": rows._fingerprint,
"split_method": "90/5/5 by source doc_id before @, then seeded row subsampling",
"full_split_documents": {name: len(indices) for name, indices in splits.items()},
"selected_documents": {name: len(indices) for name, indices in selected.items()},
"selected_indices": selected,
"full_train_target_tokens": full_train_tokens - 1,
"train_target_tokens": len(train_tokens) - 1,
"validation_target_tokens": len(prepared["validation_tokens"]) - 1,
"test_target_tokens": len(prepared["test_tokens"]) - 1,
}
# A finished cache replaces the temporary file, so another run cannot read half a file.
temporary_file = cache_file.with_suffix(f".{os.getpid()}.tmp")
torch.save(prepared, temporary_file)
temporary_file.replace(cache_file)
return prepared
def make_batch(tokens, start, token_limit, batch_size, context_size, pad_id):
count = min(token_limit, batch_size * context_size, len(tokens) - 1 - start)
if count <= 0:
raise ValueError("A batch must contain at least one target token.")
rows = math.ceil(count / context_size)
inputs = torch.full((rows, context_size), pad_id, dtype=torch.long)
labels = torch.full((rows, context_size), -100, dtype=torch.long)
padding_mask = torch.ones((rows, context_size), dtype=torch.bool)
for row in range(rows):
offset = start + row * context_size
length = min(context_size, count - row * context_size)
inputs[row, :length] = tokens[offset:offset + length]
labels[row, :length] = tokens[offset + 1:offset + length + 1]
padding_mask[row, :length] = False
return inputs, labels, padding_mask, count