Download source/part2/data.py from walke007/anlp-assignment2-part2: direct link, hf CLI and curl.
- Browser
- Download file 6.29 kB
-
https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/data.py
- Command line
-
hf download hf://walke007/anlp-assignment2-part2/source/part2/data.py
-
curl -L -o data.py https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/data.py
6.29 kB
| """Prepare the same text, splits, and token order for every optimizer.""" | |
| import math | |
| import os | |
| import random | |
| from pathlib import Path | |
| import torch | |
| from datasets import load_dataset | |
| from tqdm import tqdm | |
| from transformers import AutoTokenizer | |
| DATASET_NAME = "browndw/human-ai-parallel-corpus" | |
| HERE = Path(__file__).resolve().parent | |
| def load_tokenizer(name): | |
| tokenizer = AutoTokenizer.from_pretrained(name) | |
| if tokenizer.bos_token_id is None: | |
| tokenizer.add_special_tokens({"bos_token": "<|bos|>"}) | |
| if tokenizer.pad_token_id is None or tokenizer.eos_token_id is None: | |
| raise ValueError("The tokenizer must have padding and EOS tokens.") | |
| return tokenizer | |
| def split_documents(rows, seed): | |
| # Different authors' versions of the same source stay in the same split. | |
| groups = {} | |
| for index, document_id in enumerate(rows["doc_id"]): | |
| source_id = document_id.split("@")[0] | |
| groups.setdefault(source_id, []).append(index) | |
| source_ids = sorted(groups) | |
| if len(source_ids) < 20: | |
| raise ValueError("At least 20 source documents are needed for 90/5/5 splits.") | |
| random.Random(seed).shuffle(source_ids) | |
| train_end = int(0.9 * len(source_ids)) | |
| validation_end = int(0.95 * len(source_ids)) | |
| split_groups = { | |
| "train": source_ids[:train_end], | |
| "validation": source_ids[train_end:validation_end], | |
| "test": source_ids[validation_end:], | |
| } | |
| splits = {} | |
| for name, sources in split_groups.items(): | |
| indices = [] | |
| for source in sources: | |
| indices.extend(groups[source]) | |
| random.Random(seed).shuffle(indices) | |
| splits[name] = indices | |
| return splits | |
| def prepare_data(args, tokenizer): | |
| settings = { | |
| "dataset": DATASET_NAME, | |
| "seed": args.seed, | |
| "data_fraction": args.data_fraction, | |
| "evaluation_fraction": args.evaluation_fraction, | |
| "validation_samples": args.validation_samples, | |
| "test_samples": args.test_samples, | |
| "tokenizer": args.tokenizer_name, | |
| "vocabulary": tokenizer.get_vocab(), | |
| } | |
| cache_dir = Path(args.cache_dir) | |
| cache_dir.mkdir(parents=True, exist_ok=True) | |
| cache_file = cache_dir / "prepared.pt" | |
| if cache_file.exists(): | |
| prepared = torch.load(cache_file, map_location="cpu", weights_only=True) | |
| if prepared["settings"] != settings: | |
| raise ValueError("Prepared data settings differ. Use a different --cache-dir.") | |
| return prepared | |
| rows = load_dataset(DATASET_NAME, split="train", cache_dir=str(cache_dir / "dataset")) | |
| splits = split_documents(rows, args.seed) | |
| train_size = max(1, int(len(splits["train"]) * args.data_fraction)) | |
| validation_size = max(1, int(min(args.validation_samples, len(splits["validation"])) | |
| * args.evaluation_fraction)) | |
| test_size = max(1, int(min(args.test_samples, len(splits["test"])) | |
| * args.evaluation_fraction)) | |
| selected = { | |
| "train": splits["train"][:train_size], | |
| "validation": splits["validation"][:validation_size], | |
| "test": splits["test"][:test_size], | |
| } | |
| # Count the full training split, but keep tokens only for the requested subset. | |
| # This makes the fraction-of-full-dataset axis an actual token measurement. | |
| selected_train = set(selected["train"]) | |
| encoded_train = {} | |
| full_train_tokens = 0 | |
| for index in tqdm(splits["train"], desc="Tokenizing/counting training text"): | |
| ids = [tokenizer.bos_token_id] | |
| ids += tokenizer.encode(rows[index]["text"], add_special_tokens=False) | |
| ids += [tokenizer.eos_token_id] | |
| full_train_tokens += len(ids) | |
| if index in selected_train: | |
| encoded_train[index] = torch.tensor(ids, dtype=torch.int32) | |
| train_tokens = torch.cat([encoded_train[index] for index in selected["train"]]) | |
| prepared = {"settings": settings, "train_tokens": train_tokens, "test_documents": []} | |
| for name in ("validation", "test"): | |
| documents = [] | |
| for index in selected[name]: | |
| text = rows[index]["text"] | |
| ids = [tokenizer.bos_token_id] | |
| ids += tokenizer.encode(text, add_special_tokens=False) | |
| ids += [tokenizer.eos_token_id] | |
| documents.append(torch.tensor(ids, dtype=torch.int32)) | |
| if name == "test": | |
| prepared["test_documents"].append({"doc_id": rows[index]["doc_id"], "ids": ids}) | |
| prepared[name + "_tokens"] = torch.cat(documents) | |
| prepared["metadata"] = { | |
| "dataset_fingerprint": rows._fingerprint, | |
| "split_method": "90/5/5 by source doc_id before @, then seeded row subsampling", | |
| "full_split_documents": {name: len(indices) for name, indices in splits.items()}, | |
| "selected_documents": {name: len(indices) for name, indices in selected.items()}, | |
| "selected_indices": selected, | |
| "full_train_target_tokens": full_train_tokens - 1, | |
| "train_target_tokens": len(train_tokens) - 1, | |
| "validation_target_tokens": len(prepared["validation_tokens"]) - 1, | |
| "test_target_tokens": len(prepared["test_tokens"]) - 1, | |
| } | |
| # A finished cache replaces the temporary file, so another run cannot read half a file. | |
| temporary_file = cache_file.with_suffix(f".{os.getpid()}.tmp") | |
| torch.save(prepared, temporary_file) | |
| temporary_file.replace(cache_file) | |
| return prepared | |
| def make_batch(tokens, start, token_limit, batch_size, context_size, pad_id): | |
| count = min(token_limit, batch_size * context_size, len(tokens) - 1 - start) | |
| if count <= 0: | |
| raise ValueError("A batch must contain at least one target token.") | |
| rows = math.ceil(count / context_size) | |
| inputs = torch.full((rows, context_size), pad_id, dtype=torch.long) | |
| labels = torch.full((rows, context_size), -100, dtype=torch.long) | |
| padding_mask = torch.ones((rows, context_size), dtype=torch.bool) | |
| for row in range(rows): | |
| offset = start + row * context_size | |
| length = min(context_size, count - row * context_size) | |
| inputs[row, :length] = tokens[offset:offset + length] | |
| labels[row, :length] = tokens[offset + 1:offset + length + 1] | |
| padding_mask[row, :length] = False | |
| return inputs, labels, padding_mask, count | |