"""Prepare the same text, splits, and token order for every optimizer.""" import math import os import random from pathlib import Path import torch from datasets import load_dataset from tqdm import tqdm from transformers import AutoTokenizer DATASET_NAME = "browndw/human-ai-parallel-corpus" HERE = Path(__file__).resolve().parent def load_tokenizer(name): tokenizer = AutoTokenizer.from_pretrained(name) if tokenizer.bos_token_id is None: tokenizer.add_special_tokens({"bos_token": "<|bos|>"}) if tokenizer.pad_token_id is None or tokenizer.eos_token_id is None: raise ValueError("The tokenizer must have padding and EOS tokens.") return tokenizer def split_documents(rows, seed): # Different authors' versions of the same source stay in the same split. groups = {} for index, document_id in enumerate(rows["doc_id"]): source_id = document_id.split("@")[0] groups.setdefault(source_id, []).append(index) source_ids = sorted(groups) if len(source_ids) < 20: raise ValueError("At least 20 source documents are needed for 90/5/5 splits.") random.Random(seed).shuffle(source_ids) train_end = int(0.9 * len(source_ids)) validation_end = int(0.95 * len(source_ids)) split_groups = { "train": source_ids[:train_end], "validation": source_ids[train_end:validation_end], "test": source_ids[validation_end:], } splits = {} for name, sources in split_groups.items(): indices = [] for source in sources: indices.extend(groups[source]) random.Random(seed).shuffle(indices) splits[name] = indices return splits def prepare_data(args, tokenizer): settings = { "dataset": DATASET_NAME, "seed": args.seed, "data_fraction": args.data_fraction, "evaluation_fraction": args.evaluation_fraction, "validation_samples": args.validation_samples, "test_samples": args.test_samples, "tokenizer": args.tokenizer_name, "vocabulary": tokenizer.get_vocab(), } cache_dir = Path(args.cache_dir) cache_dir.mkdir(parents=True, exist_ok=True) cache_file = cache_dir / "prepared.pt" if cache_file.exists(): prepared = torch.load(cache_file, map_location="cpu", weights_only=True) if prepared["settings"] != settings: raise ValueError("Prepared data settings differ. Use a different --cache-dir.") return prepared rows = load_dataset(DATASET_NAME, split="train", cache_dir=str(cache_dir / "dataset")) splits = split_documents(rows, args.seed) train_size = max(1, int(len(splits["train"]) * args.data_fraction)) validation_size = max(1, int(min(args.validation_samples, len(splits["validation"])) * args.evaluation_fraction)) test_size = max(1, int(min(args.test_samples, len(splits["test"])) * args.evaluation_fraction)) selected = { "train": splits["train"][:train_size], "validation": splits["validation"][:validation_size], "test": splits["test"][:test_size], } # Count the full training split, but keep tokens only for the requested subset. # This makes the fraction-of-full-dataset axis an actual token measurement. selected_train = set(selected["train"]) encoded_train = {} full_train_tokens = 0 for index in tqdm(splits["train"], desc="Tokenizing/counting training text"): ids = [tokenizer.bos_token_id] ids += tokenizer.encode(rows[index]["text"], add_special_tokens=False) ids += [tokenizer.eos_token_id] full_train_tokens += len(ids) if index in selected_train: encoded_train[index] = torch.tensor(ids, dtype=torch.int32) train_tokens = torch.cat([encoded_train[index] for index in selected["train"]]) prepared = {"settings": settings, "train_tokens": train_tokens, "test_documents": []} for name in ("validation", "test"): documents = [] for index in selected[name]: text = rows[index]["text"] ids = [tokenizer.bos_token_id] ids += tokenizer.encode(text, add_special_tokens=False) ids += [tokenizer.eos_token_id] documents.append(torch.tensor(ids, dtype=torch.int32)) if name == "test": prepared["test_documents"].append({"doc_id": rows[index]["doc_id"], "ids": ids}) prepared[name + "_tokens"] = torch.cat(documents) prepared["metadata"] = { "dataset_fingerprint": rows._fingerprint, "split_method": "90/5/5 by source doc_id before @, then seeded row subsampling", "full_split_documents": {name: len(indices) for name, indices in splits.items()}, "selected_documents": {name: len(indices) for name, indices in selected.items()}, "selected_indices": selected, "full_train_target_tokens": full_train_tokens - 1, "train_target_tokens": len(train_tokens) - 1, "validation_target_tokens": len(prepared["validation_tokens"]) - 1, "test_target_tokens": len(prepared["test_tokens"]) - 1, } # A finished cache replaces the temporary file, so another run cannot read half a file. temporary_file = cache_file.with_suffix(f".{os.getpid()}.tmp") torch.save(prepared, temporary_file) temporary_file.replace(cache_file) return prepared def make_batch(tokens, start, token_limit, batch_size, context_size, pad_id): count = min(token_limit, batch_size * context_size, len(tokens) - 1 - start) if count <= 0: raise ValueError("A batch must contain at least one target token.") rows = math.ceil(count / context_size) inputs = torch.full((rows, context_size), pad_id, dtype=torch.long) labels = torch.full((rows, context_size), -100, dtype=torch.long) padding_mask = torch.ones((rows, context_size), dtype=torch.bool) for row in range(rows): offset = start + row * context_size length = min(context_size, count - row * context_size) inputs[row, :length] = tokens[offset:offset + length] labels[row, :length] = tokens[offset + 1:offset + length + 1] padding_mask[row, :length] = False return inputs, labels, padding_mask, count