Upload Task 1 checkpoints, tokenizer, configurations and source
Browse files- README.md +38 -0
- checkpoints_part1/best_variant_1.pt +3 -0
- checkpoints_part1/best_variant_2.pt +3 -0
- checkpoints_part1/best_variant_3.pt +3 -0
- checkpoints_part1/best_variant_4.pt +3 -0
- checkpoints_part1/best_variant_5.pt +3 -0
- configs/part1_var1.yaml +13 -0
- configs/part1_var2.yaml +13 -0
- configs/part1_var3.yaml +13 -0
- configs/part1_var4.yaml +13 -0
- configs/part1_var5.yaml +13 -0
- configs/part2_adamw.yaml +11 -0
- configs/part2_cautious.yaml +11 -0
- configs/part2_lion.yaml +11 -0
- configs/part2_scion.yaml +11 -0
- configs/part2_sophia.yaml +12 -0
- pyproject.toml +26 -0
- src/__init__.py +1 -0
- src/part1/__init__.py +0 -0
- src/part1/dataset.py +92 -0
- src/part1/evaluate.py +152 -0
- src/part1/model.py +68 -0
- src/part1/moe.py +102 -0
- src/part1/train.py +199 -0
- src/part2/__init__.py +0 -0
- src/part2/attention.py +67 -0
- src/part2/ffn.py +17 -0
- src/part2/model.py +53 -0
- src/part2/norm.py +17 -0
- src/part2/optimizers.py +316 -0
- src/part2/positional.py +36 -0
- src/part2/train.py +247 -0
- src/part3/__init__.py +0 -0
- src/part3/decoding.py +159 -0
- src/part3/evaluate.py +223 -0
- tokenizer_part1/tokenizer.json +0 -0
- tokenizer_part1/tokenizer_config.json +9 -0
- uv.lock +0 -0
README.md
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ANLP Assignment 2
|
| 2 |
+
|
| 3 |
+
## Links
|
| 4 |
+
|
| 5 |
+
- **Weights & Biases:** To be added.
|
| 6 |
+
- **Checkpoints (Hugging Face):** https://huggingface.co/Chechaaa09/anlp-a2-models
|
| 7 |
+
|
| 8 |
+
## Setup
|
| 9 |
+
|
| 10 |
+
Dependencies are pinned in `pyproject.toml` / `uv.lock`:
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
uv sync
|
| 14 |
+
```
|
| 15 |
+
|
| 16 |
+
Tasks 1 and 3 load their datasets from Hugging Face. For Task 2, save
|
| 17 |
+
`browndw/human-ai-parallel-corpus` as a Hugging Face Dataset with a `text`
|
| 18 |
+
column at `src/part2/dataset/`.
|
| 19 |
+
|
| 20 |
+
## Usage
|
| 21 |
+
|
| 22 |
+
Run from the repository root:
|
| 23 |
+
|
| 24 |
+
```bash
|
| 25 |
+
uv run python -m src.part1.train --all # train the five FFN variants
|
| 26 |
+
uv run python -m src.part1.evaluate # evaluate checkpoints and plot expert usage
|
| 27 |
+
uv run python -m src.part2.train --all # train with all five optimizers
|
| 28 |
+
uv run python -m src.part3.evaluate --num_samples 1000 --max_new_tokens 40
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
Task 1 checkpoints are written to `checkpoints_part1/best_variant_<n>.pt`,
|
| 32 |
+
and evaluation results to `assets/part1_results/`. Task 2 logs metrics to
|
| 33 |
+
Weights & Biases; its current training script does not save checkpoints.
|
| 34 |
+
Task 3 results are written to `task3_results.json`.
|
| 35 |
+
|
| 36 |
+
The Task 1 checkpoints are custom PyTorch state dictionaries. Load them using
|
| 37 |
+
`src.part1.model.DecoderOnlyTransformer` and the matching configuration in
|
| 38 |
+
`configs/part1_var<n>.yaml`.
|
checkpoints_part1/best_variant_1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:95a5c1ada7e3c1659361e9236227600ff5f71a169ea0127e86cc085c969d46b5
|
| 3 |
+
size 70576050
|
checkpoints_part1/best_variant_2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e63ce4d45c8d9367d0e799c12b9aaa30971f4f76835fbf79db50bd5b302e9b80
|
| 3 |
+
size 70673878
|
checkpoints_part1/best_variant_3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e7ff6147aac10a6060afd16baad98d9901f277d3f6557ae086b89bf6cb47a711
|
| 3 |
+
size 70673878
|
checkpoints_part1/best_variant_4.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0c5453f8eb056c2af513a8088880f86b0e5c13e28fbda7dcbcc766b6b66ec375
|
| 3 |
+
size 70664854
|
checkpoints_part1/best_variant_5.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5edf0d22994b39f4e1411b5673f7e379710bc2e421a28bd3cca88b89b6d5e400
|
| 3 |
+
size 99022294
|
configs/part1_var1.yaml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
variant: 1
|
| 2 |
+
d_model: 384
|
| 3 |
+
n_heads: 6
|
| 4 |
+
d_ff: 1536
|
| 5 |
+
n_layers: 6
|
| 6 |
+
vocab_size: 8192
|
| 7 |
+
seq_len: 128
|
| 8 |
+
batch_size: 32
|
| 9 |
+
lr: 0.0005
|
| 10 |
+
weight_decay: 0.01
|
| 11 |
+
dropout: 0.1
|
| 12 |
+
max_tokens: 12000000
|
| 13 |
+
eval_every_tokens: 1000000
|
configs/part1_var2.yaml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
variant: 2
|
| 2 |
+
d_model: 384
|
| 3 |
+
n_heads: 6
|
| 4 |
+
d_ff: 1536
|
| 5 |
+
n_layers: 6
|
| 6 |
+
vocab_size: 8192
|
| 7 |
+
seq_len: 128
|
| 8 |
+
batch_size: 32
|
| 9 |
+
lr: 0.0005
|
| 10 |
+
weight_decay: 0.01
|
| 11 |
+
dropout: 0.1
|
| 12 |
+
max_tokens: 12000000
|
| 13 |
+
eval_every_tokens: 1000000
|
configs/part1_var3.yaml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
variant: 3
|
| 2 |
+
d_model: 384
|
| 3 |
+
n_heads: 6
|
| 4 |
+
d_ff: 1536
|
| 5 |
+
n_layers: 6
|
| 6 |
+
vocab_size: 8192
|
| 7 |
+
seq_len: 128
|
| 8 |
+
batch_size: 32
|
| 9 |
+
lr: 0.0005
|
| 10 |
+
weight_decay: 0.01
|
| 11 |
+
dropout: 0.1
|
| 12 |
+
max_tokens: 12000000
|
| 13 |
+
eval_every_tokens: 1000000
|
configs/part1_var4.yaml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
variant: 4
|
| 2 |
+
d_model: 384
|
| 3 |
+
n_heads: 6
|
| 4 |
+
d_ff: 1536
|
| 5 |
+
n_layers: 6
|
| 6 |
+
vocab_size: 8192
|
| 7 |
+
seq_len: 128
|
| 8 |
+
batch_size: 32
|
| 9 |
+
lr: 0.0005
|
| 10 |
+
weight_decay: 0.01
|
| 11 |
+
dropout: 0.1
|
| 12 |
+
max_tokens: 12000000
|
| 13 |
+
eval_every_tokens: 1000000
|
configs/part1_var5.yaml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
variant: 5
|
| 2 |
+
d_model: 384
|
| 3 |
+
n_heads: 6
|
| 4 |
+
d_ff: 1536
|
| 5 |
+
n_layers: 6
|
| 6 |
+
vocab_size: 8192
|
| 7 |
+
seq_len: 128
|
| 8 |
+
batch_size: 32
|
| 9 |
+
lr: 0.0005
|
| 10 |
+
weight_decay: 0.01
|
| 11 |
+
dropout: 0.1
|
| 12 |
+
max_tokens: 12000000
|
| 13 |
+
eval_every_tokens: 1000000
|
configs/part2_adamw.yaml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
optimizer: "adamw"
|
| 2 |
+
lr: 0.001
|
| 3 |
+
weight_decay: 0.01
|
| 4 |
+
batch_size: 8
|
| 5 |
+
seq_len: 256
|
| 6 |
+
d_model: 256
|
| 7 |
+
n_heads: 8
|
| 8 |
+
d_ff: 1024
|
| 9 |
+
n_layers: 4
|
| 10 |
+
dropout: 0.1
|
| 11 |
+
epochs: 1
|
configs/part2_cautious.yaml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
optimizer: "cautious"
|
| 2 |
+
lr: 0.001
|
| 3 |
+
weight_decay: 0.01
|
| 4 |
+
batch_size: 8
|
| 5 |
+
seq_len: 256
|
| 6 |
+
d_model: 256
|
| 7 |
+
n_heads: 8
|
| 8 |
+
d_ff: 1024
|
| 9 |
+
n_layers: 4
|
| 10 |
+
dropout: 0.1
|
| 11 |
+
epochs: 1
|
configs/part2_lion.yaml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
optimizer: "lion"
|
| 2 |
+
lr: 0.0002
|
| 3 |
+
weight_decay: 0.1
|
| 4 |
+
batch_size: 8
|
| 5 |
+
seq_len: 256
|
| 6 |
+
d_model: 256
|
| 7 |
+
n_heads: 8
|
| 8 |
+
d_ff: 1024
|
| 9 |
+
n_layers: 4
|
| 10 |
+
dropout: 0.1
|
| 11 |
+
epochs: 1
|
configs/part2_scion.yaml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
optimizer: "scion"
|
| 2 |
+
lr: 0.001
|
| 3 |
+
weight_decay: 0.01
|
| 4 |
+
batch_size: 8
|
| 5 |
+
seq_len: 256
|
| 6 |
+
d_model: 256
|
| 7 |
+
n_heads: 8
|
| 8 |
+
d_ff: 1024
|
| 9 |
+
n_layers: 4
|
| 10 |
+
dropout: 0.1
|
| 11 |
+
epochs: 1
|
configs/part2_sophia.yaml
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
optimizer: "sophia"
|
| 2 |
+
lr: 0.0001
|
| 3 |
+
rho: 0.04
|
| 4 |
+
weight_decay: 0.1
|
| 5 |
+
batch_size: 8
|
| 6 |
+
seq_len: 256
|
| 7 |
+
d_model: 256
|
| 8 |
+
n_heads: 8
|
| 9 |
+
d_ff: 1024
|
| 10 |
+
n_layers: 4
|
| 11 |
+
dropout: 0.1
|
| 12 |
+
epochs: 1
|
pyproject.toml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "assignment-2-moe"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
requires-python = ">=3.11"
|
| 5 |
+
dependencies = [
|
| 6 |
+
"datasets>=5.0.1",
|
| 7 |
+
"matplotlib>=3.11.2",
|
| 8 |
+
"pyyaml>=6.0.3",
|
| 9 |
+
"sacrebleu>=2.6.0",
|
| 10 |
+
"seaborn>=0.13.2",
|
| 11 |
+
"tokenizers>=0.23.2",
|
| 12 |
+
"torch>=2.9.0",
|
| 13 |
+
"transformers>=4.57.0",
|
| 14 |
+
"wandb>=0.29.0",
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
# Install the CUDA build of PyTorch on Linux and Windows.
|
| 18 |
+
[tool.uv.sources]
|
| 19 |
+
torch = [
|
| 20 |
+
{ index = "pytorch-cu126", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
| 21 |
+
]
|
| 22 |
+
|
| 23 |
+
[[tool.uv.index]]
|
| 24 |
+
name = "pytorch-cu126"
|
| 25 |
+
url = "https://download.pytorch.org/whl/cu126"
|
| 26 |
+
explicit = true
|
src/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
src/part1/__init__.py
ADDED
|
File without changes
|
src/part1/dataset.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch.utils.data import Dataset
|
| 3 |
+
from datasets import load_dataset
|
| 4 |
+
from tokenizers import ByteLevelBPETokenizer
|
| 5 |
+
from transformers import PreTrainedTokenizerFast
|
| 6 |
+
import os
|
| 7 |
+
import random
|
| 8 |
+
|
| 9 |
+
def train_or_load_tokenizer(dataset, vocab_size = 8192, save_dir = "tokenizer_part1"):
|
| 10 |
+
if os.path.exists(save_dir):
|
| 11 |
+
print(f"Loading existing tokenizer from {save_dir}...")
|
| 12 |
+
return PreTrainedTokenizerFast.from_pretrained(save_dir)
|
| 13 |
+
|
| 14 |
+
def batch_iterator(max_samples = 50000):
|
| 15 |
+
for i, row in enumerate(dataset):
|
| 16 |
+
if i >= max_samples:
|
| 17 |
+
break
|
| 18 |
+
yield row["en"]
|
| 19 |
+
yield row["vi"]
|
| 20 |
+
yield row["ja"]
|
| 21 |
+
|
| 22 |
+
special_tokens = ["<pad>", "<unk>", "<eos>", "<|vi|>", "<|ja|>", "<|en|>"]
|
| 23 |
+
tokenizer = ByteLevelBPETokenizer()
|
| 24 |
+
tokenizer.train_from_iterator(batch_iterator(), vocab_size = vocab_size, min_frequency = 2, special_tokens = special_tokens)
|
| 25 |
+
|
| 26 |
+
os.makedirs(save_dir, exist_ok = True)
|
| 27 |
+
hf_tokenizer = PreTrainedTokenizerFast(tokenizer_object = tokenizer, pad_token = "<pad>", unk_token = "<unk>", eos_token = "<eos>", bos_token = "<eos>")
|
| 28 |
+
hf_tokenizer.save_pretrained(save_dir)
|
| 29 |
+
return hf_tokenizer
|
| 30 |
+
|
| 31 |
+
def prepare_data(num_samples = 50000, test_size = 2000):
|
| 32 |
+
raw_ds = load_dataset("belumind/en-vi-ja-curated-500k-triplets", split = f"train[:{num_samples}]")
|
| 33 |
+
|
| 34 |
+
pairs = []
|
| 35 |
+
for row in raw_ds:
|
| 36 |
+
en = row["en"].strip()
|
| 37 |
+
vi = row["vi"].strip()
|
| 38 |
+
ja = row["ja"].strip()
|
| 39 |
+
|
| 40 |
+
if not en or not vi or not ja:
|
| 41 |
+
continue
|
| 42 |
+
|
| 43 |
+
pairs.append(("vi", vi, en))
|
| 44 |
+
pairs.append(("ja", ja, en))
|
| 45 |
+
|
| 46 |
+
random.seed(42)
|
| 47 |
+
random.shuffle(pairs)
|
| 48 |
+
test_pairs = pairs[:test_size]
|
| 49 |
+
train_pairs = pairs[test_size:]
|
| 50 |
+
|
| 51 |
+
return train_pairs, test_pairs, raw_ds
|
| 52 |
+
|
| 53 |
+
class Translation_Dataset(Dataset):
|
| 54 |
+
def __init__(self, data_pairs, tokenizer, max_seq_len = 128):
|
| 55 |
+
self.data_pairs = data_pairs
|
| 56 |
+
self.tokenizer = tokenizer
|
| 57 |
+
self.max_seq_len = max_seq_len
|
| 58 |
+
|
| 59 |
+
def __len__(self):
|
| 60 |
+
return len(self.data_pairs)
|
| 61 |
+
|
| 62 |
+
def __getitem__(self, idx):
|
| 63 |
+
lang, src_text, tgt_text = self.data_pairs[idx]
|
| 64 |
+
|
| 65 |
+
prompt_str = f"<|{lang}|> {src_text} <|en|>"
|
| 66 |
+
target_str = f"{tgt_text}<|eos|>"
|
| 67 |
+
|
| 68 |
+
prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens = False)
|
| 69 |
+
target_ids = self.tokenizer.encode(target_str, add_special_tokens = False)
|
| 70 |
+
|
| 71 |
+
full_ids = prompt_ids + target_ids
|
| 72 |
+
full_ids = full_ids[: self.max_seq_len + 1]
|
| 73 |
+
|
| 74 |
+
input_ids = full_ids[:-1]
|
| 75 |
+
targets = full_ids[1:]
|
| 76 |
+
|
| 77 |
+
prompt_len = len(prompt_ids)
|
| 78 |
+
mask_len = min(prompt_len - 1, len(targets))
|
| 79 |
+
targets = [-100] * mask_len + targets[mask_len:]
|
| 80 |
+
|
| 81 |
+
pad_len = self.max_seq_len - len(input_ids)
|
| 82 |
+
if pad_len > 0:
|
| 83 |
+
input_ids = input_ids + [self.tokenizer.pad_token_id] * pad_len
|
| 84 |
+
targets = targets + [-100] * pad_len
|
| 85 |
+
|
| 86 |
+
return {
|
| 87 |
+
"input_ids": torch.tensor(input_ids, dtype = torch.long),
|
| 88 |
+
"targets": torch.tensor(targets, dtype = torch.long)
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
|
src/part1/evaluate.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from torch.utils.data import DataLoader
|
| 4 |
+
import argparse
|
| 5 |
+
import math
|
| 6 |
+
import os
|
| 7 |
+
import json
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import seaborn as sns
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
|
| 12 |
+
from .model import DecoderOnlyTransformer
|
| 13 |
+
from .dataset import prepare_data, train_or_load_tokenizer, Translation_Dataset
|
| 14 |
+
from .train import evaluate, compute_translation_bleu
|
| 15 |
+
|
| 16 |
+
def evaluate_language_usage(model, pairs, tokenizer, seq_len = 128, device = "cuda"):
|
| 17 |
+
model.reset_moe_usage()
|
| 18 |
+
model.set_moe_tracking(True)
|
| 19 |
+
dataset = Translation_Dataset(pairs, tokenizer, max_seq_len = seq_len)
|
| 20 |
+
loader = DataLoader(dataset, batch_size = 32, shuffle = False)
|
| 21 |
+
model.eval()
|
| 22 |
+
with torch.no_grad():
|
| 23 |
+
for batch in loader:
|
| 24 |
+
input_ids = batch["input_ids"].to(device)
|
| 25 |
+
_ = model(input_ids)
|
| 26 |
+
|
| 27 |
+
raw_counts = model.get_moe_usage()
|
| 28 |
+
total_per_layer = raw_counts.sum(dim = -1, keepdim = True).float()
|
| 29 |
+
normalized = (raw_counts.float() / (total_per_layer + 1e-9)).cpu().numpy()
|
| 30 |
+
|
| 31 |
+
model.set_moe_tracking(False)
|
| 32 |
+
model.reset_moe_usage()
|
| 33 |
+
|
| 34 |
+
return normalized
|
| 35 |
+
|
| 36 |
+
def plot_language_heatmap(vi_usage, ja_usage, variant, save_path):
|
| 37 |
+
num_layers, num_experts = vi_usage.shape
|
| 38 |
+
y_labels = [f"Layer {i + 1}" for i in range(num_layers)]
|
| 39 |
+
if variant == 4:
|
| 40 |
+
x_labels = [f"Routed {i}" for i in range(num_experts - 1)] + ["Shared"]
|
| 41 |
+
else:
|
| 42 |
+
x_labels = [f"Expert {i}" for i in range(num_experts)]
|
| 43 |
+
|
| 44 |
+
vmax = max(vi_usage.max(), ja_usage.max())
|
| 45 |
+
|
| 46 |
+
fig, axes = plt.subplots(1, 2, figsize = (12,5), sharey = True)
|
| 47 |
+
|
| 48 |
+
sns.heatmap(vi_usage, ax = axes[0], annot = True, fmt = ".2f", cmap = "Blues", vmin = 0.0, vmax = vmax, xticklabels = x_labels, yticklabels = y_labels, cbar = False)
|
| 49 |
+
axes[0].set_title(f"Variant {variant}: Vietnamese Activations", fontsize = 12, fontweight = "bold")
|
| 50 |
+
axes[0].set_xlabel("Experts")
|
| 51 |
+
axes[0].set_ylabel("Transformer Layers")
|
| 52 |
+
|
| 53 |
+
sns.heatmap(ja_usage, ax = axes[1], annot = True, fmt = ".2f", cmap = "Blues", vmin = 0.0, vmax = vmax, xticklabels = x_labels, yticklabels = y_labels, cbar = True, cbar_kws = {"label": "Activation Proportion"})
|
| 54 |
+
|
| 55 |
+
axes[1].set_title(f"Variant {variant}: Japanese Activations", fontsize = 12, fontweight = "bold")
|
| 56 |
+
axes[1].set_xlabel("Experts")
|
| 57 |
+
|
| 58 |
+
plt.suptitle(f"Language Specialization Heatmap (Variant {variant})", fontsize = 14, y = 1.02)
|
| 59 |
+
plt.tight_layout()
|
| 60 |
+
|
| 61 |
+
os.makedirs(os.path.dirname(save_path), exist_ok = True)
|
| 62 |
+
plt.savefig(save_path, dpi = 300, bbox_inches = "tight")
|
| 63 |
+
plt.close()
|
| 64 |
+
print(f"Saved heatmap to: {save_path}")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def run_evaluation(args):
|
| 68 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 69 |
+
os.makedirs(args.output_dir, exist_ok = True)
|
| 70 |
+
|
| 71 |
+
print("Loading test dataset and tokenizer...")
|
| 72 |
+
_, test_pairs, raw_ds = prepare_data(num_samples = args.num_samples, test_size = args.test_size)
|
| 73 |
+
tokenizer = train_or_load_tokenizer(raw_ds, vocab_size = args.vocab_size)
|
| 74 |
+
|
| 75 |
+
vi_test = [p for p in test_pairs if p[0] == "vi"]
|
| 76 |
+
ja_test = [p for p in test_pairs if p[0] == "ja"]
|
| 77 |
+
print(f"Test split: Total = {len(test_pairs)} | vi = {len(vi_test)} | ja = {len(ja_test)}")
|
| 78 |
+
|
| 79 |
+
test_ds = Translation_Dataset(test_pairs, tokenizer, max_seq_len = args.seq_len)
|
| 80 |
+
test_loader = DataLoader(test_ds, batch_size = args.batch_size, shuffle = False)
|
| 81 |
+
|
| 82 |
+
results = {}
|
| 83 |
+
|
| 84 |
+
for v in range(1, 6):
|
| 85 |
+
checkpoint_path = os.path.join(args.save_dir, f"best_variant_{v}.pt")
|
| 86 |
+
if not os.path.exists(checkpoint_path):
|
| 87 |
+
print(f"Skipping variant {v}: {checkpoint_path} not found")
|
| 88 |
+
continue
|
| 89 |
+
print(f"\n{'=' * 55}\n Evaluating Variant {v}({checkpoint_path})\n{'=' * 55}")
|
| 90 |
+
|
| 91 |
+
model = DecoderOnlyTransformer(vocab_size = len(tokenizer), d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, variant = v, max_seq_len = args.seq_len).to(device)
|
| 92 |
+
|
| 93 |
+
state_dict = torch.load(checkpoint_path, map_location = device)
|
| 94 |
+
model.load_state_dict(state_dict)
|
| 95 |
+
model.eval()
|
| 96 |
+
|
| 97 |
+
print("Computing test loss and perplexity...")
|
| 98 |
+
test_loss, test_ppl = evaluate(model, test_loader, device = device)
|
| 99 |
+
|
| 100 |
+
print("Computing BLEU scores...")
|
| 101 |
+
bleu_overall = compute_translation_bleu(model, test_pairs, tokenizer, max_samples = args.bleu_samples, device = device)
|
| 102 |
+
bleu_vi = compute_translation_bleu(model, vi_test, tokenizer, max_samples = args.bleu_samples // 2, device = device)
|
| 103 |
+
bleu_ja = compute_translation_bleu(model, ja_test, tokenizer, max_samples = args.bleu_samples // 2, device = device)
|
| 104 |
+
|
| 105 |
+
print(f"Results -> Loss: {test_loss:.4f} | PPL: {test_ppl:.2f} | BLEU Total: {bleu_overall:.2f} (Vi: {bleu_vi:.2f}, Ja: {bleu_ja:.2f})")
|
| 106 |
+
|
| 107 |
+
if v > 1:
|
| 108 |
+
print("Analyzing expert routing patterns per language...")
|
| 109 |
+
vi_usage = evaluate_language_usage(model, vi_test, tokenizer, seq_len = args.seq_len, device = device)
|
| 110 |
+
ja_usage = evaluate_language_usage(model, ja_test, tokenizer, seq_len = args.seq_len, device = device)
|
| 111 |
+
|
| 112 |
+
heatmap_path = os.path.join(args.output_dir, f"heatmap_variant_{v}.png")
|
| 113 |
+
plot_language_heatmap(vi_usage, ja_usage, variant = v, save_path = heatmap_path)
|
| 114 |
+
|
| 115 |
+
results[f"variant_{v}"] = {
|
| 116 |
+
"test_loss": round(float(test_loss), 4),
|
| 117 |
+
"test_perplexity": round(float(test_ppl), 2),
|
| 118 |
+
"bleu_overall": round(float(bleu_overall), 2),
|
| 119 |
+
"bleu_vi_en": round(float(bleu_vi), 2),
|
| 120 |
+
"bleu_ja_en": round(float(bleu_ja), 2),
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
results_file = os.path.join(args.output_dir, "task1_results.json")
|
| 124 |
+
with open(results_file, "w") as f:
|
| 125 |
+
json.dump(results, f, indent = 4)
|
| 126 |
+
print(f"\nAll results saved to: {results_file}")
|
| 127 |
+
|
| 128 |
+
print("\n" + "=" * 65)
|
| 129 |
+
print(f"{'Variant':<12} | {'PPL':<8} | {'BLEU (All)':<12} | {'BLEU (Vi)':<10} | {'BLEU (Ja)':<10}")
|
| 130 |
+
print("-" * 65)
|
| 131 |
+
|
| 132 |
+
for v_name, metrics in results.items():
|
| 133 |
+
print(f"{v_name:<12} | {metrics['test_perplexity']:<8.2f} | {metrics['bleu_overall']:<12.2f} | {metrics['bleu_vi_en']:<10.2f} | {metrics['bleu_ja_en']:<10.2f}")
|
| 134 |
+
print("=" * 65)
|
| 135 |
+
|
| 136 |
+
if __name__ == "__main__":
|
| 137 |
+
parser = argparse.ArgumentParser()
|
| 138 |
+
parser.add_argument("--save_dir", type=str, default="checkpoints_part1")
|
| 139 |
+
parser.add_argument("--output_dir", type=str, default="assets/part1_results")
|
| 140 |
+
parser.add_argument("--d_model", type=int, default=384)
|
| 141 |
+
parser.add_argument("--n_heads", type=int, default=6)
|
| 142 |
+
parser.add_argument("--d_ff", type=int, default=1536)
|
| 143 |
+
parser.add_argument("--n_layers", type=int, default=6)
|
| 144 |
+
parser.add_argument("--vocab_size", type=int, default=8192)
|
| 145 |
+
parser.add_argument("--seq_len", type=int, default=128)
|
| 146 |
+
parser.add_argument("--batch_size", type=int, default=32)
|
| 147 |
+
parser.add_argument("--num_samples", type=int, default=50000)
|
| 148 |
+
parser.add_argument("--test_size", type=int, default=2000)
|
| 149 |
+
parser.add_argument("--bleu_samples", type=int, default=100)
|
| 150 |
+
args = parser.parse_args()
|
| 151 |
+
|
| 152 |
+
run_evaluation(args)
|
src/part1/model.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
from src.part2.attention import MultiHeadAttention
|
| 6 |
+
from src.part2.norm import LayerNorm
|
| 7 |
+
from .moe import build_ffn_variant, MoELayer
|
| 8 |
+
|
| 9 |
+
class DecoderBlock(nn.Module):
|
| 10 |
+
def __init__(self, d_model, n_heads, d_ff, variant = 1, dropout = 0.1, use_rope = True):
|
| 11 |
+
super().__init__()
|
| 12 |
+
self.norm1 = LayerNorm(d_model)
|
| 13 |
+
self.norm2 = LayerNorm(d_model)
|
| 14 |
+
self.attention = MultiHeadAttention(d_model, n_heads, use_rope)
|
| 15 |
+
self.dropout = nn.Dropout(dropout)
|
| 16 |
+
self.ffn = build_ffn_variant(variant, d_model, d_ff, dropout)
|
| 17 |
+
|
| 18 |
+
def forward(self, x, mask):
|
| 19 |
+
normed = self.norm1(x)
|
| 20 |
+
attention_output = self.attention(normed, normed, normed, mask)
|
| 21 |
+
x = x + self.dropout(attention_output)
|
| 22 |
+
|
| 23 |
+
normed = self.norm2(x)
|
| 24 |
+
ffn_out = self.ffn(normed)
|
| 25 |
+
result = x + self.dropout(ffn_out)
|
| 26 |
+
return result
|
| 27 |
+
|
| 28 |
+
class DecoderOnlyTransformer(nn.Module):
|
| 29 |
+
def __init__(self, vocab_size, d_model = 384, n_heads = 6, d_ff = 1536, n_layers = 6, variant = 1, max_seq_len = 256, dropout = 0.1, use_rope = True):
|
| 30 |
+
super().__init__()
|
| 31 |
+
self.d_model = d_model
|
| 32 |
+
self.vocab_size = vocab_size
|
| 33 |
+
self.variant = variant
|
| 34 |
+
self.token_embeddings = nn.Embedding(vocab_size, d_model)
|
| 35 |
+
self.dropout = nn.Dropout(dropout)
|
| 36 |
+
|
| 37 |
+
self.layers = nn.ModuleList([DecoderBlock(d_model, n_heads, d_ff, variant, dropout, use_rope) for _ in range(n_layers)])
|
| 38 |
+
self.final_norm = LayerNorm(d_model)
|
| 39 |
+
self.lm_head = nn.Linear(d_model, vocab_size, bias = False)
|
| 40 |
+
self.lm_head.weight = self.token_embeddings.weight
|
| 41 |
+
|
| 42 |
+
def set_moe_tracking(self, track = True):
|
| 43 |
+
for layer in self.layers:
|
| 44 |
+
if isinstance(layer.ffn, MoELayer):
|
| 45 |
+
layer.ffn.track_usage = track
|
| 46 |
+
|
| 47 |
+
def reset_moe_usage(self):
|
| 48 |
+
for layer in self.layers:
|
| 49 |
+
if isinstance(layer.ffn, MoELayer):
|
| 50 |
+
layer.ffn.reset_usage_counts()
|
| 51 |
+
|
| 52 |
+
def get_moe_usage(self):
|
| 53 |
+
usages = [layer.ffn.expert_counts for layer in self.layers if isinstance(layer.ffn, MoELayer)]
|
| 54 |
+
if not usages:
|
| 55 |
+
return None
|
| 56 |
+
return torch.stack(usages)
|
| 57 |
+
|
| 58 |
+
def forward(self, input_ids):
|
| 59 |
+
batch_size, seq_len = input_ids.shape
|
| 60 |
+
lower_t = torch.tril(torch.ones((seq_len, seq_len), device=input_ids.device))
|
| 61 |
+
x = self.token_embeddings(input_ids) * math.sqrt(self.d_model)
|
| 62 |
+
x = self.dropout(x)
|
| 63 |
+
|
| 64 |
+
for layer in self.layers:
|
| 65 |
+
x = layer(x, lower_t)
|
| 66 |
+
x = self.final_norm(x)
|
| 67 |
+
logits = self.lm_head(x)
|
| 68 |
+
return logits
|
src/part1/moe.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
class PositionWiseFFN(nn.Module):
|
| 6 |
+
def __init__(self, d_model, d_ff, dropout=0.1):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.linear1 = nn.Linear(d_model, d_ff)
|
| 9 |
+
self.linear2 = nn.Linear(d_ff, d_model)
|
| 10 |
+
self.dropout = nn.Dropout(dropout)
|
| 11 |
+
|
| 12 |
+
def forward(self, x):
|
| 13 |
+
x = self.linear1(x)
|
| 14 |
+
x = F.relu(x)
|
| 15 |
+
x = self.dropout(x)
|
| 16 |
+
x = self.linear2(x)
|
| 17 |
+
return x
|
| 18 |
+
|
| 19 |
+
class TopKRouter(nn.Module):
|
| 20 |
+
def __init__(self, d_model, num_experts, top_k):
|
| 21 |
+
super().__init__()
|
| 22 |
+
self.num_experts = num_experts
|
| 23 |
+
self.top_k = top_k
|
| 24 |
+
self.gate = nn.Linear(d_model, num_experts, bias = False)
|
| 25 |
+
|
| 26 |
+
def forward(self, x):
|
| 27 |
+
logits = self.gate(x)
|
| 28 |
+
values, indices = torch.topk(logits, self.top_k, dim = -1)
|
| 29 |
+
routing_weights = F.softmax(values, dim = -1)
|
| 30 |
+
return routing_weights, indices
|
| 31 |
+
|
| 32 |
+
class MoELayer(nn.Module):
|
| 33 |
+
def __init__(self, d_model, d_expert, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = 0.1):
|
| 34 |
+
super().__init__()
|
| 35 |
+
self.d_model = d_model
|
| 36 |
+
self.d_expert = d_expert
|
| 37 |
+
self.num_experts = num_experts
|
| 38 |
+
self.top_k = top_k
|
| 39 |
+
self.use_shared_expert = use_shared_expert
|
| 40 |
+
|
| 41 |
+
self.register_buffer("expert_counts", torch.zeros(num_experts, dtype = torch.long))
|
| 42 |
+
self.track_usage = False
|
| 43 |
+
|
| 44 |
+
if self.use_shared_expert == True:
|
| 45 |
+
self.shared_expert = PositionWiseFFN(self.d_model, self.d_expert, dropout)
|
| 46 |
+
self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts - 1)])
|
| 47 |
+
self.router = TopKRouter(self.d_model, self.num_experts - 1, top_k = 1)
|
| 48 |
+
|
| 49 |
+
else:
|
| 50 |
+
self.shared_expert = None
|
| 51 |
+
self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts)])
|
| 52 |
+
self.router = TopKRouter(self.d_model, self.num_experts, self.top_k)
|
| 53 |
+
|
| 54 |
+
def reset_usage_counts(self):
|
| 55 |
+
self.expert_counts.zero_()
|
| 56 |
+
|
| 57 |
+
def forward(self, x):
|
| 58 |
+
batch_size, seq_len, d_model = x.shape
|
| 59 |
+
x_flat = x.view(-1, d_model)
|
| 60 |
+
routing_weights, selected_experts = self.router(x_flat)
|
| 61 |
+
|
| 62 |
+
if self.track_usage:
|
| 63 |
+
with torch.no_grad():
|
| 64 |
+
counts = torch.bincount(selected_experts.flatten(), minlength = self.num_experts)
|
| 65 |
+
self.expert_counts += counts
|
| 66 |
+
|
| 67 |
+
out = torch.zeros_like(x_flat)
|
| 68 |
+
|
| 69 |
+
for i, expert in enumerate(self.experts):
|
| 70 |
+
token_idx, top_k_idx = torch.where(selected_experts == i)
|
| 71 |
+
if token_idx.numel() == 0:
|
| 72 |
+
continue
|
| 73 |
+
tokens_for_expert = x_flat[token_idx]
|
| 74 |
+
expert_out = expert(tokens_for_expert)
|
| 75 |
+
weights = routing_weights[token_idx, top_k_idx].unsqueeze(-1)
|
| 76 |
+
weighted_out = expert_out * weights
|
| 77 |
+
out[token_idx] += weighted_out
|
| 78 |
+
|
| 79 |
+
if self.use_shared_expert:
|
| 80 |
+
shared_out = self.shared_expert(x_flat)
|
| 81 |
+
out = out + shared_out
|
| 82 |
+
if self.track_usage:
|
| 83 |
+
with torch.no_grad():
|
| 84 |
+
self.expert_counts[-1] += x_flat.size(0)
|
| 85 |
+
|
| 86 |
+
out = out.view(batch_size, seq_len, d_model)
|
| 87 |
+
return out
|
| 88 |
+
|
| 89 |
+
def build_ffn_variant(variant, d_model = 384, d_ff = 1536, dropout = 0.1):
|
| 90 |
+
if variant == 1:
|
| 91 |
+
return PositionWiseFFN(d_model = d_model, d_ff = d_ff, dropout = dropout)
|
| 92 |
+
elif variant == 2:
|
| 93 |
+
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = dropout)
|
| 94 |
+
elif variant == 3:
|
| 95 |
+
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
|
| 96 |
+
elif variant == 4:
|
| 97 |
+
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = True, dropout = dropout)
|
| 98 |
+
elif variant == 5:
|
| 99 |
+
return MoELayer(d_model = d_model, d_expert = d_ff // 2, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
|
| 100 |
+
else:
|
| 101 |
+
raise ValueError(f"unknown variant {variant}. Expected 1, 2, 3, 4, or 5")
|
| 102 |
+
|
src/part1/train.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import math
|
| 5 |
+
from torch.utils.data import DataLoader
|
| 6 |
+
from datasets import load_from_disk
|
| 7 |
+
import sacrebleu
|
| 8 |
+
import wandb
|
| 9 |
+
import argparse
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
import yaml
|
| 12 |
+
import os
|
| 13 |
+
|
| 14 |
+
from .model import DecoderOnlyTransformer
|
| 15 |
+
from .dataset import train_or_load_tokenizer, prepare_data, Translation_Dataset
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def compute_translation_bleu(model, test_pairs, tokenizer, max_samples=50, max_gen_len=40, device="cuda"):
|
| 19 |
+
model.eval()
|
| 20 |
+
scores = []
|
| 21 |
+
|
| 22 |
+
samples = test_pairs[:max_samples]
|
| 23 |
+
with torch.no_grad():
|
| 24 |
+
for lang, src_text, ref_text in tqdm(samples, desc = "Computing BLEU", leave = False):
|
| 25 |
+
prompt_str = f"<|{lang}|> {src_text} <|en|>"
|
| 26 |
+
prompt_ids = tokenizer.encode(prompt_str, add_special_tokens=False)
|
| 27 |
+
current_ids = torch.tensor([prompt_ids], dtype=torch.long, device=device)
|
| 28 |
+
prompt_len = current_ids.size(1)
|
| 29 |
+
|
| 30 |
+
for _ in range(max_gen_len):
|
| 31 |
+
logits = model(current_ids)
|
| 32 |
+
next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True)
|
| 33 |
+
current_ids = torch.cat([current_ids, next_token_id], dim = -1)
|
| 34 |
+
|
| 35 |
+
if next_token_id.item() == tokenizer.eos_token_id:
|
| 36 |
+
break
|
| 37 |
+
|
| 38 |
+
generated_ids = current_ids[0, prompt_len:].tolist()
|
| 39 |
+
generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True)
|
| 40 |
+
|
| 41 |
+
bleu = sacrebleu.sentence_bleu(generated_text, [ref_text]).score
|
| 42 |
+
scores.append(bleu)
|
| 43 |
+
avg_bleu = sum(scores) / max(1, len(scores))
|
| 44 |
+
return avg_bleu
|
| 45 |
+
|
| 46 |
+
def evaluate(model, val_loader, device = "cuda"):
|
| 47 |
+
model.eval()
|
| 48 |
+
total_loss = 0.0
|
| 49 |
+
total_batches = 0
|
| 50 |
+
criterion = nn.CrossEntropyLoss(ignore_index = -100)
|
| 51 |
+
|
| 52 |
+
with torch.no_grad():
|
| 53 |
+
for batch in tqdm(val_loader, desc = "Evaluating Val Loss", leave = False):
|
| 54 |
+
input_ids = batch["input_ids"].to(device)
|
| 55 |
+
targets = batch["targets"].to(device)
|
| 56 |
+
logits = model(input_ids)
|
| 57 |
+
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
|
| 58 |
+
total_loss += loss.item()
|
| 59 |
+
total_batches += 1
|
| 60 |
+
|
| 61 |
+
avg_loss = total_loss / max(1, total_batches)
|
| 62 |
+
return avg_loss, math.exp(avg_loss)
|
| 63 |
+
|
| 64 |
+
def train(args):
|
| 65 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 66 |
+
print(f"using device: {device} | training variant {args.variant}")
|
| 67 |
+
|
| 68 |
+
wandb.init(
|
| 69 |
+
project = "ANLP-Assignment-2-Part1",
|
| 70 |
+
name = f"variant_{args.variant}",
|
| 71 |
+
config = vars(args)
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
print("Preparing dataset and tokenizer...")
|
| 75 |
+
train_pairs, test_pairs, raw_ds = prepare_data(num_samples = args.num_samples, test_size = args.test_size)
|
| 76 |
+
tokenizer = train_or_load_tokenizer(raw_ds, vocab_size = args.vocab_size)
|
| 77 |
+
|
| 78 |
+
train_ds = Translation_Dataset(train_pairs, tokenizer, max_seq_len = args.seq_len)
|
| 79 |
+
val_ds = Translation_Dataset(test_pairs, tokenizer, max_seq_len = args.seq_len)
|
| 80 |
+
|
| 81 |
+
train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True)
|
| 82 |
+
val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False)
|
| 83 |
+
|
| 84 |
+
model = DecoderOnlyTransformer(vocab_size = len(tokenizer), d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, variant = args.variant, max_seq_len = args.seq_len, dropout = args.dropout).to(device)
|
| 85 |
+
|
| 86 |
+
optimizer = torch.optim.AdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
|
| 87 |
+
criterion = nn.CrossEntropyLoss(ignore_index = -100)
|
| 88 |
+
scaler = torch.amp.GradScaler("cuda")
|
| 89 |
+
|
| 90 |
+
tokens_trained = 0
|
| 91 |
+
epoch = 0
|
| 92 |
+
best_val_loss = float("inf")
|
| 93 |
+
last_val_ppl = None
|
| 94 |
+
last_val_bleu = None
|
| 95 |
+
|
| 96 |
+
pbar = tqdm(total = args.max_tokens, desc = f"Variant {args.variant}", unit = "tok")
|
| 97 |
+
|
| 98 |
+
while tokens_trained < args.max_tokens:
|
| 99 |
+
epoch += 1
|
| 100 |
+
for batch in train_loader:
|
| 101 |
+
model.train()
|
| 102 |
+
optimizer.zero_grad()
|
| 103 |
+
|
| 104 |
+
input_ids = batch["input_ids"].to(device)
|
| 105 |
+
targets = batch["targets"].to(device)
|
| 106 |
+
|
| 107 |
+
with torch.amp.autocast("cuda"):
|
| 108 |
+
logits = model(input_ids)
|
| 109 |
+
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
|
| 110 |
+
|
| 111 |
+
scaler.scale(loss).backward()
|
| 112 |
+
scaler.step(optimizer)
|
| 113 |
+
scaler.update()
|
| 114 |
+
|
| 115 |
+
num_tokens = input_ids.numel()
|
| 116 |
+
tokens_trained += num_tokens
|
| 117 |
+
pbar.update(num_tokens)
|
| 118 |
+
|
| 119 |
+
postfix = {"loss": f"{loss.item(): .4f}"}
|
| 120 |
+
if last_val_ppl is not None:
|
| 121 |
+
postfix["val_ppl"] = f"{last_val_ppl:.1f}"
|
| 122 |
+
if last_val_bleu is not None:
|
| 123 |
+
postfix["bleu"] = f"{last_val_bleu:.1f}"
|
| 124 |
+
pbar.set_postfix(postfix)
|
| 125 |
+
|
| 126 |
+
if tokens_trained % args.eval_every_tokens < num_tokens or tokens_trained >= args.max_tokens:
|
| 127 |
+
val_loss, val_ppl = evaluate(model, val_loader, device = device)
|
| 128 |
+
val_bleu = compute_translation_bleu(model, test_pairs, tokenizer, device = device)
|
| 129 |
+
last_val_ppl = val_ppl
|
| 130 |
+
last_val_bleu = val_bleu
|
| 131 |
+
|
| 132 |
+
tqdm.write(f"\n[Tokens: {tokens_trained:,}] Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {val_bleu:.2f}")
|
| 133 |
+
|
| 134 |
+
wandb.log({
|
| 135 |
+
"tokens_trained": tokens_trained,
|
| 136 |
+
"train/loss": loss.item(),
|
| 137 |
+
"val/loss": val_loss,
|
| 138 |
+
"val/perplexity": val_ppl,
|
| 139 |
+
"val/bleu": val_bleu,
|
| 140 |
+
"epoch": epoch,
|
| 141 |
+
})
|
| 142 |
+
|
| 143 |
+
if val_loss < best_val_loss:
|
| 144 |
+
best_val_loss = val_loss
|
| 145 |
+
os.makedirs(args.save_dir, exist_ok = True)
|
| 146 |
+
torch.save(model.state_dict(), f"{args.save_dir}/best_variant_{args.variant}.pt")
|
| 147 |
+
|
| 148 |
+
if tokens_trained >= args.max_tokens:
|
| 149 |
+
break
|
| 150 |
+
|
| 151 |
+
pbar.close()
|
| 152 |
+
print(f"training for variant {args.variant} complete")
|
| 153 |
+
wandb.finish()
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
if __name__ == "__main__":
|
| 157 |
+
parser = argparse.ArgumentParser()
|
| 158 |
+
parser.add_argument("--config", type=str, default=None, help="Path to config YAML")
|
| 159 |
+
parser.add_argument("--all", action="store_true", help="Train all 5 variants sequentially")
|
| 160 |
+
parser.add_argument("--variant", type=int, default=1, choices=[1, 2, 3, 4, 5])
|
| 161 |
+
parser.add_argument("--d_model", type=int, default=384)
|
| 162 |
+
parser.add_argument("--n_heads", type=int, default=6)
|
| 163 |
+
parser.add_argument("--d_ff", type=int, default=1536)
|
| 164 |
+
parser.add_argument("--n_layers", type=int, default=6)
|
| 165 |
+
parser.add_argument("--vocab_size", type=int, default=8192)
|
| 166 |
+
parser.add_argument("--seq_len", type=int, default=128)
|
| 167 |
+
parser.add_argument("--batch_size", type=int, default=32)
|
| 168 |
+
parser.add_argument("--lr", type=float, default=5e-4)
|
| 169 |
+
parser.add_argument("--weight_decay", type=float, default=0.01)
|
| 170 |
+
parser.add_argument("--dropout", type=float, default=0.1)
|
| 171 |
+
parser.add_argument("--max_tokens", type=int, default=12_000_000)
|
| 172 |
+
parser.add_argument("--num_samples", type=int, default=50000)
|
| 173 |
+
parser.add_argument("--test_size", type=int, default=2000)
|
| 174 |
+
parser.add_argument("--eval_every_tokens", type=int, default=1_000_000)
|
| 175 |
+
parser.add_argument("--save_dir", type=str, default="checkpoints_part1")
|
| 176 |
+
args = parser.parse_args()
|
| 177 |
+
|
| 178 |
+
if args.all:
|
| 179 |
+
all_configs = [f"configs/part1_var{i}.yaml" for i in range(1, 6)]
|
| 180 |
+
for cfg_path in all_configs:
|
| 181 |
+
print("\n" + "=" * 60)
|
| 182 |
+
print(f"Starting run for: {cfg_path}")
|
| 183 |
+
print("=" * 60)
|
| 184 |
+
with open(cfg_path, "r") as f:
|
| 185 |
+
cfg = yaml.safe_load(f)
|
| 186 |
+
run_args = argparse.Namespace(**vars(args))
|
| 187 |
+
for k, v in cfg.items():
|
| 188 |
+
setattr(run_args, k, v)
|
| 189 |
+
train(run_args)
|
| 190 |
+
if torch.cuda.is_available():
|
| 191 |
+
torch.cuda.empty_cache()
|
| 192 |
+
elif args.config:
|
| 193 |
+
with open(args.config, "r") as f:
|
| 194 |
+
cfg = yaml.safe_load(f)
|
| 195 |
+
for k, v in cfg.items():
|
| 196 |
+
setattr(args, k, v)
|
| 197 |
+
train(args)
|
| 198 |
+
else:
|
| 199 |
+
train(args)
|
src/part2/__init__.py
ADDED
|
File without changes
|
src/part2/attention.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import math
|
| 5 |
+
from . import positional
|
| 6 |
+
|
| 7 |
+
def compute_attention(Q, K, V, d_k, mask=None):
|
| 8 |
+
K_T = torch.transpose(K, -2, -1)
|
| 9 |
+
QK_T = torch.matmul(Q, K_T)
|
| 10 |
+
scores = QK_T / math.sqrt(d_k)
|
| 11 |
+
if mask is not None:
|
| 12 |
+
scores = scores.masked_fill(mask == 0, float('-inf'))
|
| 13 |
+
softmax_scores = F.softmax(scores, dim=-1)
|
| 14 |
+
final_scores = torch.matmul(softmax_scores, V)
|
| 15 |
+
return final_scores
|
| 16 |
+
|
| 17 |
+
class MultiHeadAttention(nn.Module):
|
| 18 |
+
def __init__(self, d, n_heads, use_rope = True, max_len = 10000):
|
| 19 |
+
super().__init__()
|
| 20 |
+
|
| 21 |
+
assert d % n_heads == 0
|
| 22 |
+
|
| 23 |
+
self.d = d
|
| 24 |
+
self.n_heads = n_heads
|
| 25 |
+
self.d_k = d // n_heads
|
| 26 |
+
|
| 27 |
+
self.W_Q = nn.Linear(d, d)
|
| 28 |
+
self.W_K = nn.Linear(d, d)
|
| 29 |
+
self.W_V = nn.Linear(d, d)
|
| 30 |
+
self.W_O = nn.Linear(d, d)
|
| 31 |
+
|
| 32 |
+
self.rope = positional.RoPE(self.d_k, max_len) if use_rope else None
|
| 33 |
+
|
| 34 |
+
def forward(self, query, key, value, mask=None):
|
| 35 |
+
batch = query.shape[0]
|
| 36 |
+
seq_len_q = query.shape[1]
|
| 37 |
+
seq_len_k = key.shape[1]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
Q = self.W_Q(query)
|
| 41 |
+
K = self.W_K(key)
|
| 42 |
+
V = self.W_V(value)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
Q = Q.view(batch, seq_len_q, self.n_heads, self.d_k)
|
| 46 |
+
Q = torch.transpose(Q, 1, 2)
|
| 47 |
+
|
| 48 |
+
K = K.view(batch, seq_len_k, self.n_heads, self.d_k)
|
| 49 |
+
K = torch.transpose(K, 1, 2)
|
| 50 |
+
|
| 51 |
+
V = V.view(batch, seq_len_k, self.n_heads, self.d_k)
|
| 52 |
+
V = torch.transpose(V, 1, 2)
|
| 53 |
+
|
| 54 |
+
if self.rope is not None:
|
| 55 |
+
Q = self.rope(Q)
|
| 56 |
+
K = self.rope(K)
|
| 57 |
+
|
| 58 |
+
attention_outputs = compute_attention(Q, K, V, self.d_k, mask)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
attention_outputs = torch.transpose(attention_outputs, 1, 2)
|
| 62 |
+
attention_outputs = attention_outputs.contiguous()
|
| 63 |
+
attention_outputs = attention_outputs.view(batch, seq_len_q, self.d)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
output = self.W_O(attention_outputs)
|
| 67 |
+
return output
|
src/part2/ffn.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
class PositionWiseFFN(nn.Module):
|
| 6 |
+
def __init__(self, d_model, d_ff, dropout=0.1):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.linear1 = nn.Linear(d_model, d_ff)
|
| 9 |
+
self.linear2 = nn.Linear(d_ff, d_model)
|
| 10 |
+
self.dropout = nn.Dropout(dropout)
|
| 11 |
+
|
| 12 |
+
def forward(self, x):
|
| 13 |
+
x = self.linear1(x)
|
| 14 |
+
x = F.relu(x)
|
| 15 |
+
x = self.dropout(x)
|
| 16 |
+
x = self.linear2(x)
|
| 17 |
+
return x
|
src/part2/model.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import math
|
| 5 |
+
|
| 6 |
+
from .attention import MultiHeadAttention
|
| 7 |
+
from .ffn import PositionWiseFFN
|
| 8 |
+
from .norm import LayerNorm
|
| 9 |
+
|
| 10 |
+
class DecoderBlock(nn.Module):
|
| 11 |
+
def __init__(self, d_model, n_heads, d_ff, dropout = 0.1, use_rope = True):
|
| 12 |
+
super().__init__()
|
| 13 |
+
self.norm1 = LayerNorm(d_model)
|
| 14 |
+
self.norm2 = LayerNorm(d_model)
|
| 15 |
+
|
| 16 |
+
self.attention = MultiHeadAttention(d_model, n_heads, use_rope)
|
| 17 |
+
self.ffn = PositionWiseFFN(d_model, d_ff)
|
| 18 |
+
self.dropout = nn.Dropout(dropout)
|
| 19 |
+
|
| 20 |
+
def forward(self, x, mask):
|
| 21 |
+
normed = self.norm1(x)
|
| 22 |
+
attention_output = self.attention(normed, normed, normed, mask)
|
| 23 |
+
x = x + self.dropout(attention_output)
|
| 24 |
+
|
| 25 |
+
normed = self.norm2(x)
|
| 26 |
+
ffn_out = self.ffn(normed)
|
| 27 |
+
result = x + self.dropout(ffn_out)
|
| 28 |
+
return result
|
| 29 |
+
|
| 30 |
+
class DecoderOnlyTransformer(nn.Module):
|
| 31 |
+
def __init__(self, vocab_size, d_model, n_heads, d_ff, n_layers, max_seq_len = 1024, dropout = 0.1, use_rope = True):
|
| 32 |
+
super().__init__()
|
| 33 |
+
self.d_model = d_model
|
| 34 |
+
self.vocab_size = vocab_size
|
| 35 |
+
self.max_seq_len = max_seq_len
|
| 36 |
+
self.token_embeddings = nn.Embedding(vocab_size, d_model)
|
| 37 |
+
self.dropout = nn.Dropout(dropout)
|
| 38 |
+
self.layers = nn.ModuleList([DecoderBlock(d_model, n_heads, d_ff, dropout, use_rope) for _ in range(n_layers)])
|
| 39 |
+
self.final_norm = LayerNorm(d_model)
|
| 40 |
+
self.lm_head = nn.Linear(d_model, vocab_size, bias = False)
|
| 41 |
+
self.lm_head.weight = self.token_embeddings.weight
|
| 42 |
+
|
| 43 |
+
def forward(self, input_ids):
|
| 44 |
+
batch_size, seq_len = input_ids.shape
|
| 45 |
+
lower_t = torch.tril(torch.ones((seq_len, seq_len), device=input_ids.device))
|
| 46 |
+
x = self.token_embeddings(input_ids) * math.sqrt(self.d_model)
|
| 47 |
+
x = self.dropout(x)
|
| 48 |
+
|
| 49 |
+
for layer in self.layers:
|
| 50 |
+
x = layer(x, lower_t)
|
| 51 |
+
x = self.final_norm(x)
|
| 52 |
+
logits = self.lm_head(x)
|
| 53 |
+
return logits
|
src/part2/norm.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
class LayerNorm(nn.Module):
|
| 6 |
+
def __init__(self, d_model, eps=1e-8):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.eps = eps
|
| 9 |
+
self.gamma = nn.Parameter(torch.ones(d_model))
|
| 10 |
+
self.beta = nn.Parameter(torch.zeros(d_model))
|
| 11 |
+
|
| 12 |
+
def forward(self, x):
|
| 13 |
+
mean = torch.mean(x, dim=-1, keepdim=True)
|
| 14 |
+
var = torch.var(x, dim=-1, keepdim=True, unbiased=False)
|
| 15 |
+
x_norm = (x - mean) / torch.sqrt(var + self.eps)
|
| 16 |
+
final_norm = self.gamma * x_norm + self.beta
|
| 17 |
+
return final_norm
|
src/part2/optimizers.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch.optim import Optimizer
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
class CustomAdamW(Optimizer):
|
| 6 |
+
def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1):
|
| 7 |
+
defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay)
|
| 8 |
+
super().__init__(params, defaults)
|
| 9 |
+
|
| 10 |
+
@torch.no_grad()
|
| 11 |
+
def step(self, closure = None):
|
| 12 |
+
loss = None
|
| 13 |
+
if closure is not None:
|
| 14 |
+
with torch.enable_grad():
|
| 15 |
+
loss = closure()
|
| 16 |
+
|
| 17 |
+
for group in self.param_groups:
|
| 18 |
+
lr = group["lr"]
|
| 19 |
+
beta1, beta2 = group["betas"]
|
| 20 |
+
eps = group["eps"]
|
| 21 |
+
weight_decay = group["weight_decay"]
|
| 22 |
+
|
| 23 |
+
for p in group["params"]:
|
| 24 |
+
if p.grad is None:
|
| 25 |
+
continue
|
| 26 |
+
|
| 27 |
+
grad = p.grad
|
| 28 |
+
if grad.is_sparse:
|
| 29 |
+
raise RuntimeError("CustomAdamW does not support sparse gradients")
|
| 30 |
+
|
| 31 |
+
state = self.state[p]
|
| 32 |
+
|
| 33 |
+
if len(state) == 0:
|
| 34 |
+
state["step"] = 0
|
| 35 |
+
if "m" not in state:
|
| 36 |
+
state["m"] = torch.zeros_like(p.data)
|
| 37 |
+
if "v" not in state:
|
| 38 |
+
state["v"] = torch.zeros_like(p.data)
|
| 39 |
+
|
| 40 |
+
state["step"] += 1
|
| 41 |
+
|
| 42 |
+
step = state["step"]
|
| 43 |
+
|
| 44 |
+
## Moment Updation
|
| 45 |
+
state["m"] = beta1 * state["m"] + (1 - beta1) * grad
|
| 46 |
+
state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2)
|
| 47 |
+
|
| 48 |
+
## Bias Correction
|
| 49 |
+
bias_correction1 = 1.0 - beta1 ** step
|
| 50 |
+
m_hat = state["m"] / bias_correction1
|
| 51 |
+
|
| 52 |
+
bias_correction2 = 1.0 - beta2 ** step
|
| 53 |
+
v_hat = state["v"] / bias_correction2
|
| 54 |
+
|
| 55 |
+
## Weight Update
|
| 56 |
+
p.data = p.data - lr * m_hat / (torch.sqrt(v_hat) + eps) - lr * p.data * weight_decay
|
| 57 |
+
|
| 58 |
+
return loss
|
| 59 |
+
|
| 60 |
+
class CustomCautious(Optimizer):
|
| 61 |
+
def __init__(self, params, lr = 1e-3, betas = (0.9, 0.95), eps = 1e-8, weight_decay = 0.1):
|
| 62 |
+
defaults = dict(lr = lr, betas = betas, eps = eps, weight_decay = weight_decay)
|
| 63 |
+
super().__init__(params, defaults)
|
| 64 |
+
|
| 65 |
+
@torch.no_grad()
|
| 66 |
+
def step(self, closure = None):
|
| 67 |
+
loss = None
|
| 68 |
+
if closure is not None:
|
| 69 |
+
with torch.enable_grad():
|
| 70 |
+
loss = closure()
|
| 71 |
+
|
| 72 |
+
for group in self.param_groups:
|
| 73 |
+
lr = group["lr"]
|
| 74 |
+
beta1, beta2 = group["betas"]
|
| 75 |
+
eps = group["eps"]
|
| 76 |
+
weight_decay = group["weight_decay"]
|
| 77 |
+
|
| 78 |
+
for p in group["params"]:
|
| 79 |
+
if p.grad is None:
|
| 80 |
+
continue
|
| 81 |
+
|
| 82 |
+
grad = p.grad
|
| 83 |
+
if grad.is_sparse:
|
| 84 |
+
raise RuntimeError("CustomAdamW does not support sparse gradients")
|
| 85 |
+
|
| 86 |
+
state = self.state[p]
|
| 87 |
+
|
| 88 |
+
if len(state) == 0:
|
| 89 |
+
state["step"] = 0
|
| 90 |
+
if "m" not in state:
|
| 91 |
+
state["m"] = torch.zeros_like(p.data)
|
| 92 |
+
if "v" not in state:
|
| 93 |
+
state["v"] = torch.zeros_like(p.data)
|
| 94 |
+
|
| 95 |
+
state["step"] += 1
|
| 96 |
+
|
| 97 |
+
step = state["step"]
|
| 98 |
+
|
| 99 |
+
## Moment Updation
|
| 100 |
+
state["m"] = beta1 * state["m"] + (1 - beta1) * grad
|
| 101 |
+
state["v"] = beta2 * state["v"] + (1 - beta2) * (grad ** 2)
|
| 102 |
+
|
| 103 |
+
## Bias Correction
|
| 104 |
+
bias_correction1 = 1.0 - beta1 ** step
|
| 105 |
+
m_hat = state["m"] / bias_correction1
|
| 106 |
+
|
| 107 |
+
bias_correction2 = 1.0 - beta2 ** step
|
| 108 |
+
v_hat = state["v"] / bias_correction2
|
| 109 |
+
|
| 110 |
+
## Calculate u
|
| 111 |
+
u = m_hat / (torch.sqrt(v_hat) + eps)
|
| 112 |
+
|
| 113 |
+
## Calculate s
|
| 114 |
+
s = (u * grad > 0).float()
|
| 115 |
+
|
| 116 |
+
## Calculate u_hat
|
| 117 |
+
u_hat = (u * s) / torch.clamp(torch.mean(s), min = 1e-8)
|
| 118 |
+
|
| 119 |
+
## Weight Update
|
| 120 |
+
p.data = p.data - lr * u_hat - lr * weight_decay * p.data
|
| 121 |
+
|
| 122 |
+
return loss
|
| 123 |
+
|
| 124 |
+
class CustomLion(Optimizer):
|
| 125 |
+
def __init__(self, params, lr = 2e-4, betas = (0.9, 0.99), weight_decay = 0.5):
|
| 126 |
+
defaults = dict(lr = lr, betas = betas, weight_decay = weight_decay)
|
| 127 |
+
super().__init__(params, defaults)
|
| 128 |
+
|
| 129 |
+
@torch.no_grad()
|
| 130 |
+
def step(self, closure = None):
|
| 131 |
+
loss = None
|
| 132 |
+
if closure is not None:
|
| 133 |
+
with torch.enable_grad():
|
| 134 |
+
loss = closure()
|
| 135 |
+
|
| 136 |
+
for group in self.param_groups:
|
| 137 |
+
lr = group["lr"]
|
| 138 |
+
beta1, beta2 = group["betas"]
|
| 139 |
+
weight_decay = group["weight_decay"]
|
| 140 |
+
|
| 141 |
+
for p in group["params"]:
|
| 142 |
+
if p.grad is None:
|
| 143 |
+
continue
|
| 144 |
+
|
| 145 |
+
grad = p.grad
|
| 146 |
+
if grad.is_sparse:
|
| 147 |
+
raise RuntimeError("CustomAdamW does not support sparse gradients")
|
| 148 |
+
|
| 149 |
+
state = self.state[p]
|
| 150 |
+
|
| 151 |
+
if len(state) == 0:
|
| 152 |
+
state["step"] = 0
|
| 153 |
+
if "m" not in state:
|
| 154 |
+
state["m"] = torch.zeros_like(p.data)
|
| 155 |
+
|
| 156 |
+
state["step"] += 1
|
| 157 |
+
|
| 158 |
+
## Calculate m_hat
|
| 159 |
+
m_hat = beta1 * state["m"] + (1 - beta1) * grad
|
| 160 |
+
|
| 161 |
+
## Moment Updation
|
| 162 |
+
state["m"] = beta2 * state["m"] + (1 - beta2) * grad
|
| 163 |
+
|
| 164 |
+
## Weight update
|
| 165 |
+
p.data = p.data - lr * torch.sign(m_hat) - lr * weight_decay * p.data
|
| 166 |
+
|
| 167 |
+
return loss
|
| 168 |
+
|
| 169 |
+
def newtonschulz_orthogonalization(X, steps = 5, eps = 1e-7):
|
| 170 |
+
if X.ndim != 2:
|
| 171 |
+
raise ValueError(f"expected 2D matrix, but got a tensor with {X.ndim} dimensions")
|
| 172 |
+
|
| 173 |
+
rows, cols = X.shape
|
| 174 |
+
is_transposed = False
|
| 175 |
+
if rows > cols:
|
| 176 |
+
X = X.T
|
| 177 |
+
is_transposed = True
|
| 178 |
+
|
| 179 |
+
norm = torch.linalg.matrix_norm(X)
|
| 180 |
+
|
| 181 |
+
X = X / (norm + eps)
|
| 182 |
+
|
| 183 |
+
a, b, c = (3.4445, -4.7750, 2.0315)
|
| 184 |
+
|
| 185 |
+
for _ in range(1, steps + 1):
|
| 186 |
+
first_term = a * X
|
| 187 |
+
A = X @ X.T
|
| 188 |
+
second_term = ((b * A) + c * (A @ A)) @ X
|
| 189 |
+
X = first_term + second_term
|
| 190 |
+
|
| 191 |
+
if is_transposed:
|
| 192 |
+
X = X.T
|
| 193 |
+
|
| 194 |
+
return X
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
class CustomScion(Optimizer):
|
| 198 |
+
def __init__(self, params, lr = 1e-3, beta = 0.9, weight_decay = 0.01, steps = 5, eps = 1e-7):
|
| 199 |
+
defaults = dict(lr = lr, beta = beta, weight_decay = weight_decay, steps = steps, eps = eps)
|
| 200 |
+
super().__init__(params, defaults)
|
| 201 |
+
|
| 202 |
+
@torch.no_grad()
|
| 203 |
+
def step(self, closure = None):
|
| 204 |
+
loss = None
|
| 205 |
+
if closure is not None:
|
| 206 |
+
with torch.enable_grad():
|
| 207 |
+
loss = closure()
|
| 208 |
+
|
| 209 |
+
for group in self.param_groups:
|
| 210 |
+
lr = group["lr"]
|
| 211 |
+
beta = group["beta"]
|
| 212 |
+
weight_decay = group["weight_decay"]
|
| 213 |
+
steps = group["steps"]
|
| 214 |
+
eps = group["eps"]
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
for p in group["params"]:
|
| 218 |
+
if p.grad is None:
|
| 219 |
+
continue
|
| 220 |
+
|
| 221 |
+
grad = p.grad
|
| 222 |
+
if grad.is_sparse:
|
| 223 |
+
raise RuntimeError("CustomScion does not support sparse gradients")
|
| 224 |
+
|
| 225 |
+
state = self.state[p]
|
| 226 |
+
|
| 227 |
+
if len(state) == 0:
|
| 228 |
+
state["step"] = 0
|
| 229 |
+
if "m" not in state:
|
| 230 |
+
state["m"] = torch.zeros_like(p.data)
|
| 231 |
+
|
| 232 |
+
state["step"] += 1
|
| 233 |
+
|
| 234 |
+
## Momentum update
|
| 235 |
+
state["m"] = beta * state["m"] + grad
|
| 236 |
+
|
| 237 |
+
## Calculate u
|
| 238 |
+
u = beta * state["m"] + grad
|
| 239 |
+
|
| 240 |
+
## Apply NewtonSchulz Orthogonalization
|
| 241 |
+
if p.ndim == 2:
|
| 242 |
+
u = newtonschulz_orthogonalization(u, steps = steps, eps = eps)
|
| 243 |
+
rows, cols = p.data.shape
|
| 244 |
+
s = math.sqrt(max(1, rows / cols))
|
| 245 |
+
u = u * s
|
| 246 |
+
else:
|
| 247 |
+
u = torch.sign(u)
|
| 248 |
+
|
| 249 |
+
## Weight update
|
| 250 |
+
p.data = p.data - lr * u - lr * weight_decay * p.data
|
| 251 |
+
|
| 252 |
+
return loss
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
class CustomSophia(Optimizer):
|
| 256 |
+
def __init__(self, params, lr = 1e-4, betas = (0.965, 0.99), rho = 0.04, weight_decay = 0.1, eps = 1e-12):
|
| 257 |
+
defaults = dict(lr = lr, betas = betas, rho = rho, weight_decay = weight_decay, eps = eps)
|
| 258 |
+
super().__init__(params, defaults)
|
| 259 |
+
|
| 260 |
+
@torch.no_grad()
|
| 261 |
+
def update_hessian(self):
|
| 262 |
+
for group in self.param_groups:
|
| 263 |
+
beta1, beta2 = group["betas"]
|
| 264 |
+
for p in group["params"]:
|
| 265 |
+
if p.grad is None:
|
| 266 |
+
continue
|
| 267 |
+
|
| 268 |
+
state = self.state[p]
|
| 269 |
+
grad = p.grad
|
| 270 |
+
if "h" not in state:
|
| 271 |
+
state["h"] = torch.zeros_like(p.data)
|
| 272 |
+
state["h"] = beta2 * state["h"] + (1 - beta2) * (grad ** 2)
|
| 273 |
+
|
| 274 |
+
@torch.no_grad()
|
| 275 |
+
def step(self, closure = None):
|
| 276 |
+
loss = None
|
| 277 |
+
if closure is not None:
|
| 278 |
+
with torch.enable_grad():
|
| 279 |
+
loss = closure()
|
| 280 |
+
|
| 281 |
+
for group in self.param_groups:
|
| 282 |
+
lr = group["lr"]
|
| 283 |
+
beta1, beta2 = group["betas"]
|
| 284 |
+
rho = group["rho"]
|
| 285 |
+
weight_decay = group["weight_decay"]
|
| 286 |
+
eps = group["eps"]
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
for p in group["params"]:
|
| 290 |
+
if p.grad is None:
|
| 291 |
+
continue
|
| 292 |
+
|
| 293 |
+
grad = p.grad
|
| 294 |
+
if grad.is_sparse:
|
| 295 |
+
raise RuntimeError("CustomSophia does not support sparse gradients")
|
| 296 |
+
|
| 297 |
+
state = self.state[p]
|
| 298 |
+
|
| 299 |
+
if "step" not in state:
|
| 300 |
+
state["step"] = 0
|
| 301 |
+
if "m" not in state:
|
| 302 |
+
state["m"] = torch.zeros_like(p.data)
|
| 303 |
+
if "h" not in state:
|
| 304 |
+
state["h"] = torch.zeros_like(p.data)
|
| 305 |
+
|
| 306 |
+
state["step"] += 1
|
| 307 |
+
|
| 308 |
+
state["m"] = beta1 * state["m"] + (1 - beta1) * grad
|
| 309 |
+
|
| 310 |
+
denom = torch.clamp(rho * state["h"], min = eps)
|
| 311 |
+
ratio = torch.clamp(state["m"] / denom, min = -1.0, max = 1.0)
|
| 312 |
+
|
| 313 |
+
p.data = p.data - lr * ratio - lr * weight_decay * p.data
|
| 314 |
+
|
| 315 |
+
return loss
|
| 316 |
+
|
src/part2/positional.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
class RoPE(nn.Module):
|
| 6 |
+
def __init__(self, d_k, max_len=10000):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.d_k = d_k
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
intermediate_term = torch.arange(0, d_k, 2) * (-math.log(10000.0) / d_k)
|
| 12 |
+
theta = torch.exp(intermediate_term)
|
| 13 |
+
|
| 14 |
+
self.register_buffer('theta', theta)
|
| 15 |
+
|
| 16 |
+
positions = torch.arange(max_len).unsqueeze(1)
|
| 17 |
+
angles = positions * theta
|
| 18 |
+
|
| 19 |
+
self.register_buffer('cos', torch.cos(angles))
|
| 20 |
+
self.register_buffer('sin', torch.sin(angles))
|
| 21 |
+
|
| 22 |
+
def forward(self, x):
|
| 23 |
+
seq_len = x.shape[2]
|
| 24 |
+
cos = self.cos[:seq_len, :]
|
| 25 |
+
sin = self.sin[:seq_len, :]
|
| 26 |
+
|
| 27 |
+
x_even = x[..., 0::2]
|
| 28 |
+
x_odd = x[..., 1::2]
|
| 29 |
+
|
| 30 |
+
x_rotated_even = x_even * cos - x_odd * sin
|
| 31 |
+
x_rotated_odd = x_even * sin + x_odd * cos
|
| 32 |
+
|
| 33 |
+
x_out = torch.stack([x_rotated_even, x_rotated_odd], dim=-1)
|
| 34 |
+
x_out = x_out.flatten(-2)
|
| 35 |
+
|
| 36 |
+
return x_out
|
src/part2/train.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import math
|
| 5 |
+
from torch.utils.data import DataLoader, Dataset
|
| 6 |
+
from datasets import load_from_disk
|
| 7 |
+
from transformers import AutoTokenizer
|
| 8 |
+
import sacrebleu
|
| 9 |
+
import wandb
|
| 10 |
+
import argparse
|
| 11 |
+
import yaml
|
| 12 |
+
from tqdm import tqdm
|
| 13 |
+
from .model import DecoderOnlyTransformer
|
| 14 |
+
from .optimizers import CustomAdamW, CustomCautious, CustomLion, CustomScion, CustomSophia
|
| 15 |
+
|
| 16 |
+
class TextDataset(Dataset):
|
| 17 |
+
def __init__(self, texts, tokenizer, seq_len = 256):
|
| 18 |
+
self.seq_len = seq_len
|
| 19 |
+
all_tokens = []
|
| 20 |
+
for text in texts:
|
| 21 |
+
token_ids = tokenizer.encode(text)
|
| 22 |
+
all_tokens.extend(token_ids)
|
| 23 |
+
all_tokens.append(tokenizer.eos_token_id)
|
| 24 |
+
|
| 25 |
+
self.tokens = torch.tensor(all_tokens, dtype = torch.long)
|
| 26 |
+
|
| 27 |
+
self.num_chunks = len(self.tokens) // (self.seq_len + 1)
|
| 28 |
+
|
| 29 |
+
def __len__(self):
|
| 30 |
+
return self.num_chunks
|
| 31 |
+
|
| 32 |
+
def __getitem__(self, idx):
|
| 33 |
+
start = idx * (self.seq_len + 1)
|
| 34 |
+
end = start + self.seq_len + 1
|
| 35 |
+
chunk = self.tokens[start:end]
|
| 36 |
+
return{
|
| 37 |
+
"input_ids": chunk[:-1],
|
| 38 |
+
"targets": chunk[1:]
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def compute_bleu(model, test_samples, tokenizer, prompt_len = 32, gen_len = 32, device = "cuda"):
|
| 43 |
+
model.eval()
|
| 44 |
+
scores = []
|
| 45 |
+
|
| 46 |
+
with torch.no_grad():
|
| 47 |
+
for test in test_samples:
|
| 48 |
+
token_ids = tokenizer.encode(test)
|
| 49 |
+
if len(token_ids) < prompt_len + 5:
|
| 50 |
+
continue
|
| 51 |
+
|
| 52 |
+
prompt_ids = torch.tensor([token_ids[:prompt_len]], dtype = torch.long, device = device)
|
| 53 |
+
reference_ids = token_ids[prompt_len : prompt_len + gen_len]
|
| 54 |
+
reference_text = tokenizer.decode(reference_ids, skip_special_tokens = True)
|
| 55 |
+
|
| 56 |
+
current_ids = prompt_ids.clone()
|
| 57 |
+
for _ in range(gen_len):
|
| 58 |
+
logits = model(current_ids)
|
| 59 |
+
next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True)
|
| 60 |
+
current_ids = torch.cat([current_ids, next_token_id], dim = -1)
|
| 61 |
+
|
| 62 |
+
generated_ids = current_ids[0, prompt_len:].tolist()
|
| 63 |
+
generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True)
|
| 64 |
+
bleu = sacrebleu.sentence_bleu(generated_text, [reference_text]).score
|
| 65 |
+
scores.append(bleu)
|
| 66 |
+
|
| 67 |
+
avg_bleu = sum(scores) / max(1, len(scores))
|
| 68 |
+
return avg_bleu
|
| 69 |
+
|
| 70 |
+
def evaluate(model, val_loader, criterion, device = "cuda"):
|
| 71 |
+
model.eval()
|
| 72 |
+
total_loss = 0.0
|
| 73 |
+
total_batches = 0
|
| 74 |
+
|
| 75 |
+
with torch.no_grad():
|
| 76 |
+
for batch in val_loader:
|
| 77 |
+
input_ids = batch["input_ids"].to(device)
|
| 78 |
+
targets = batch["targets"].to(device)
|
| 79 |
+
logits = model(input_ids)
|
| 80 |
+
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
|
| 81 |
+
total_loss += loss.item()
|
| 82 |
+
total_batches += 1
|
| 83 |
+
|
| 84 |
+
avg_loss = total_loss / max(1, total_batches)
|
| 85 |
+
return avg_loss, math.exp(avg_loss)
|
| 86 |
+
|
| 87 |
+
def train(args):
|
| 88 |
+
|
| 89 |
+
wandb.init(
|
| 90 |
+
project = "ANLP-Assignment-2-Part2",
|
| 91 |
+
name = f"{args.optimizer}_lr{args.lr}_wd{args.weight_decay}",
|
| 92 |
+
config = vars(args),
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 96 |
+
tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
| 97 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 98 |
+
|
| 99 |
+
dataset = load_from_disk("src/part2/dataset")
|
| 100 |
+
|
| 101 |
+
splits = dataset.train_test_split(test_size = 0.1, seed = 42)
|
| 102 |
+
val_test = splits["test"].train_test_split(test_size = 0.5, seed = 42)
|
| 103 |
+
|
| 104 |
+
train_ds = TextDataset(splits["train"]["text"], tokenizer, seq_len = args.seq_len)
|
| 105 |
+
val_ds = TextDataset(val_test["train"]["text"], tokenizer, seq_len = args.seq_len)
|
| 106 |
+
test_ds = val_test["test"]["text"]
|
| 107 |
+
|
| 108 |
+
train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True)
|
| 109 |
+
val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False)
|
| 110 |
+
|
| 111 |
+
model = DecoderOnlyTransformer(
|
| 112 |
+
vocab_size = tokenizer.vocab_size,
|
| 113 |
+
d_model = args.d_model,
|
| 114 |
+
n_heads = args.n_heads,
|
| 115 |
+
d_ff = args.d_ff,
|
| 116 |
+
n_layers = args.n_layers,
|
| 117 |
+
max_seq_len = args.seq_len,
|
| 118 |
+
dropout = args.dropout,
|
| 119 |
+
).to(device)
|
| 120 |
+
|
| 121 |
+
if args.optimizer == "adamw":
|
| 122 |
+
optimizer = CustomAdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
|
| 123 |
+
elif args.optimizer == "cautious":
|
| 124 |
+
optimizer = CustomCautious(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
|
| 125 |
+
elif args.optimizer == "lion":
|
| 126 |
+
optimizer = CustomLion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
|
| 127 |
+
elif args.optimizer == "scion":
|
| 128 |
+
optimizer = CustomScion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
|
| 129 |
+
elif args.optimizer == "sophia":
|
| 130 |
+
optimizer = CustomSophia(model.parameters(), lr = args.lr, weight_decay = args.weight_decay, rho = getattr(args, "rho", 0.04))
|
| 131 |
+
else:
|
| 132 |
+
raise ValueError(f"unknown optimizer: {args.optimizer}")
|
| 133 |
+
|
| 134 |
+
criterion = nn.CrossEntropyLoss()
|
| 135 |
+
|
| 136 |
+
tokens_per_batch = args.batch_size * args.seq_len
|
| 137 |
+
total_tokens_budget = len(train_ds) * args.seq_len
|
| 138 |
+
eval_token_interval = total_tokens_budget * 0.1
|
| 139 |
+
|
| 140 |
+
total_tokens_processed = 0
|
| 141 |
+
next_eval_token = eval_token_interval
|
| 142 |
+
step_count = 0
|
| 143 |
+
|
| 144 |
+
model.train()
|
| 145 |
+
print(f"Starting training with {args.optimizer} | Budget: {total_tokens_budget:,} tokens")
|
| 146 |
+
|
| 147 |
+
for epoch in range(args.epochs):
|
| 148 |
+
pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{args.epochs}", dynamic_ncols=True)
|
| 149 |
+
for batch in pbar:
|
| 150 |
+
input_ids = batch["input_ids"].to(device)
|
| 151 |
+
targets = batch["targets"].to(device)
|
| 152 |
+
|
| 153 |
+
if args.optimizer == "sophia" and (step_count % 10 == 0):
|
| 154 |
+
optimizer.zero_grad()
|
| 155 |
+
with torch.no_grad():
|
| 156 |
+
h_logits = model(input_ids)
|
| 157 |
+
probs = torch.softmax(h_logits, dim = -1)
|
| 158 |
+
sampled_targets = torch.multinomial(probs.view(-1, probs.size(-1)), num_samples = 1).view_as(targets)
|
| 159 |
+
|
| 160 |
+
h_loss = criterion(model(input_ids).view(-1, probs.size(-1)), sampled_targets.view(-1))
|
| 161 |
+
h_loss.backward()
|
| 162 |
+
optimizer.update_hessian()
|
| 163 |
+
optimizer.zero_grad()
|
| 164 |
+
|
| 165 |
+
optimizer.zero_grad()
|
| 166 |
+
logits = model(input_ids)
|
| 167 |
+
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
|
| 168 |
+
loss.backward()
|
| 169 |
+
|
| 170 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm = 1.0)
|
| 171 |
+
optimizer.step()
|
| 172 |
+
|
| 173 |
+
step_count += 1
|
| 174 |
+
total_tokens_processed += tokens_per_batch
|
| 175 |
+
pbar.set_postfix(loss=f"{loss.item():.4f}", tokens=f"{total_tokens_processed:,}")
|
| 176 |
+
|
| 177 |
+
wandb.log({
|
| 178 |
+
"train/loss": loss.item(),
|
| 179 |
+
"tokens": total_tokens_processed,
|
| 180 |
+
})
|
| 181 |
+
|
| 182 |
+
if total_tokens_processed >= next_eval_token:
|
| 183 |
+
val_loss, val_ppl = evaluate(model, val_loader, criterion, device)
|
| 184 |
+
bleu = compute_bleu(model, test_ds[:50], tokenizer, device = device)
|
| 185 |
+
fraction = total_tokens_processed / total_tokens_budget
|
| 186 |
+
print(f"\n[{fraction:.1f}x tokens] Val Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {bleu:.4f}")
|
| 187 |
+
|
| 188 |
+
wandb.log({
|
| 189 |
+
"val/loss": val_loss,
|
| 190 |
+
"val/perplexity": val_ppl,
|
| 191 |
+
"val/bleu": bleu,
|
| 192 |
+
"tokens": total_tokens_processed,
|
| 193 |
+
"fraction": fraction,
|
| 194 |
+
})
|
| 195 |
+
|
| 196 |
+
next_eval_token += eval_token_interval
|
| 197 |
+
model.train()
|
| 198 |
+
|
| 199 |
+
if total_tokens_processed >= total_tokens_budget:
|
| 200 |
+
break
|
| 201 |
+
if total_tokens_processed >= total_tokens_budget:
|
| 202 |
+
break
|
| 203 |
+
|
| 204 |
+
wandb.finish()
|
| 205 |
+
|
| 206 |
+
def parse_args():
|
| 207 |
+
parser = argparse.ArgumentParser(description="Train transformer with specified optimizer config(s)")
|
| 208 |
+
parser.add_argument("--config", type=str, default=None, help="Path to YAML config file")
|
| 209 |
+
parser.add_argument("--all", action="store_true", help="Train all 4 optimizer configs sequentially")
|
| 210 |
+
parser.add_argument("--lr", type=float, default=None)
|
| 211 |
+
parser.add_argument("--batch_size", type=int, default=None)
|
| 212 |
+
cli_args = parser.parse_args()
|
| 213 |
+
if not cli_args.config and not cli_args.all:
|
| 214 |
+
parser.error("You must specify either --config <path> or --all")
|
| 215 |
+
return cli_args
|
| 216 |
+
|
| 217 |
+
def build_config(config_path, cli_args):
|
| 218 |
+
with open(config_path, "r") as f:
|
| 219 |
+
config = yaml.safe_load(f)
|
| 220 |
+
if cli_args.lr is not None:
|
| 221 |
+
config["lr"] = cli_args.lr
|
| 222 |
+
if cli_args.batch_size is not None:
|
| 223 |
+
config["batch_size"] = cli_args.batch_size
|
| 224 |
+
return argparse.Namespace(**config)
|
| 225 |
+
|
| 226 |
+
if __name__ == "__main__":
|
| 227 |
+
cli_args = parse_args()
|
| 228 |
+
if cli_args.all:
|
| 229 |
+
all_configs = [
|
| 230 |
+
"configs/part2_adamw.yaml",
|
| 231 |
+
"configs/part2_cautious.yaml",
|
| 232 |
+
"configs/part2_lion.yaml",
|
| 233 |
+
"configs/part2_scion.yaml",
|
| 234 |
+
"configs/part2_sophia.yaml",
|
| 235 |
+
]
|
| 236 |
+
for config_path in all_configs:
|
| 237 |
+
print("\n" + "=" * 60)
|
| 238 |
+
print(f"Starting run for: {config_path}")
|
| 239 |
+
print("=" * 60)
|
| 240 |
+
args = build_config(config_path, cli_args)
|
| 241 |
+
train(args)
|
| 242 |
+
|
| 243 |
+
if torch.cuda.is_available():
|
| 244 |
+
torch.cuda.empty_cache()
|
| 245 |
+
else:
|
| 246 |
+
args = build_config(cli_args.config, cli_args)
|
| 247 |
+
train(args)
|
src/part3/__init__.py
ADDED
|
File without changes
|
src/part3/decoding.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
|
| 4 |
+
## greedy decoding
|
| 5 |
+
@torch.no_grad()
|
| 6 |
+
def greedy_decode(model, input_ids, max_new_tokens, eos_token_id = None, pad_token_id = None):
|
| 7 |
+
model.eval()
|
| 8 |
+
batch_size = input_ids.shape[0]
|
| 9 |
+
generated = input_ids.clone()
|
| 10 |
+
|
| 11 |
+
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
|
| 12 |
+
|
| 13 |
+
for step in range(max_new_tokens):
|
| 14 |
+
outputs = model(generated)
|
| 15 |
+
logits = outputs.logits
|
| 16 |
+
next_token_logits = logits[:, -1, :]
|
| 17 |
+
|
| 18 |
+
next_token = torch.argmax(next_token_logits, dim = -1, keepdim = True)
|
| 19 |
+
if pad_token_id is not None:
|
| 20 |
+
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
|
| 21 |
+
if eos_token_id is not None:
|
| 22 |
+
is_eos = (next_token.squeeze(-1) == eos_token_id)
|
| 23 |
+
finished = finished | is_eos
|
| 24 |
+
generated = torch.cat([generated, next_token], dim = -1)
|
| 25 |
+
|
| 26 |
+
if finished.all():
|
| 27 |
+
break
|
| 28 |
+
|
| 29 |
+
return generated
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
# top-k decoding
|
| 33 |
+
@torch.no_grad()
|
| 34 |
+
def top_k_decoding(model, input_ids, max_new_tokens, k =50, temperature = 1.0, eos_token_id = None, pad_token_id = None):
|
| 35 |
+
model.eval()
|
| 36 |
+
batch_size = input_ids.shape[0]
|
| 37 |
+
generated = input_ids.clone()
|
| 38 |
+
|
| 39 |
+
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
|
| 40 |
+
|
| 41 |
+
for step in range(max_new_tokens):
|
| 42 |
+
outputs = model(generated)
|
| 43 |
+
logits = outputs.logits
|
| 44 |
+
next_token_logits = logits[:, -1, :] / temperature
|
| 45 |
+
|
| 46 |
+
topk_logits = torch.topk(next_token_logits, k, dim = -1)
|
| 47 |
+
threshold = topk_logits.values[:, -1:]
|
| 48 |
+
mask = (next_token_logits < threshold)
|
| 49 |
+
next_token_logits = next_token_logits.masked_fill(mask, float("-inf"))
|
| 50 |
+
|
| 51 |
+
probs = F.softmax(next_token_logits, dim = -1)
|
| 52 |
+
|
| 53 |
+
next_token = torch.multinomial(probs, num_samples = 1)
|
| 54 |
+
|
| 55 |
+
if pad_token_id is not None:
|
| 56 |
+
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
|
| 57 |
+
if eos_token_id is not None:
|
| 58 |
+
is_eos = (next_token.squeeze(-1) == eos_token_id)
|
| 59 |
+
finished = finished | is_eos
|
| 60 |
+
generated = torch.cat([generated, next_token], dim = -1)
|
| 61 |
+
|
| 62 |
+
if finished.all():
|
| 63 |
+
break
|
| 64 |
+
|
| 65 |
+
return generated
|
| 66 |
+
|
| 67 |
+
# top-p decoding
|
| 68 |
+
@torch.no_grad()
|
| 69 |
+
def top_p_decoding(model, input_ids, max_new_tokens, p = 0.9, temperature = 1.0, eos_token_id = None, pad_token_id = None):
|
| 70 |
+
model.eval()
|
| 71 |
+
batch_size = input_ids.shape[0]
|
| 72 |
+
generated = input_ids.clone()
|
| 73 |
+
|
| 74 |
+
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
|
| 75 |
+
|
| 76 |
+
for step in range(max_new_tokens):
|
| 77 |
+
outputs = model(generated)
|
| 78 |
+
logits = outputs.logits
|
| 79 |
+
next_token_logits = logits[:, -1, :] / temperature
|
| 80 |
+
|
| 81 |
+
sorted_logits, sorted_indices = torch.sort(next_token_logits, descending = True, dim = -1)
|
| 82 |
+
|
| 83 |
+
sorted_probs = F.softmax(sorted_logits, dim = -1)
|
| 84 |
+
cumulative_probs = torch.cumsum(sorted_probs, dim = -1)
|
| 85 |
+
|
| 86 |
+
sorted_indices_to_remove = cumulative_probs > p
|
| 87 |
+
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
|
| 88 |
+
sorted_indices_to_remove[..., 0] = False
|
| 89 |
+
|
| 90 |
+
sorted_logits = sorted_logits.masked_fill(sorted_indices_to_remove, float('-inf'))
|
| 91 |
+
probs = F.softmax(sorted_logits, dim = -1)
|
| 92 |
+
sampled_sorted_index = torch.multinomial(probs, num_samples = 1)
|
| 93 |
+
next_token = torch.gather(sorted_indices, dim = -1, index = sampled_sorted_index)
|
| 94 |
+
|
| 95 |
+
if pad_token_id is not None:
|
| 96 |
+
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
|
| 97 |
+
if eos_token_id is not None:
|
| 98 |
+
is_eos = (next_token.squeeze(-1) == eos_token_id)
|
| 99 |
+
finished = finished | is_eos
|
| 100 |
+
generated = torch.cat([generated, next_token], dim = -1)
|
| 101 |
+
|
| 102 |
+
if finished.all():
|
| 103 |
+
break
|
| 104 |
+
|
| 105 |
+
return generated
|
| 106 |
+
|
| 107 |
+
## Beam Search Decode
|
| 108 |
+
@torch.no_grad()
|
| 109 |
+
def beam_search_decode(model, input_ids, beam_width = 1, max_new_tokens = 50, length_penalty_alpha = 1.0, eos_token_id = None):
|
| 110 |
+
model.eval()
|
| 111 |
+
batch_size = input_ids.shape[0]
|
| 112 |
+
beam_sequences = input_ids.clone()
|
| 113 |
+
beam_scores = torch.zeros(1, dtype=torch.float, device=input_ids.device)
|
| 114 |
+
completed = []
|
| 115 |
+
|
| 116 |
+
for step in range(max_new_tokens):
|
| 117 |
+
outputs = model(beam_sequences)
|
| 118 |
+
next_token_logits = outputs.logits[:, -1, :]
|
| 119 |
+
log_probs = F.log_softmax(next_token_logits, dim = -1)
|
| 120 |
+
candidate_scores = beam_scores.unsqueeze(-1) + log_probs
|
| 121 |
+
|
| 122 |
+
vocab_size = next_token_logits.shape[-1]
|
| 123 |
+
flat_scores = candidate_scores.view(-1)
|
| 124 |
+
top_scores, top_indices = torch.topk(flat_scores, k = beam_width, dim = -1)
|
| 125 |
+
parent_beam_idx = torch.div(top_indices, vocab_size, rounding_mode = "floor")
|
| 126 |
+
token_id = top_indices % vocab_size
|
| 127 |
+
|
| 128 |
+
next_beam_sequences = []
|
| 129 |
+
next_beam_scores = []
|
| 130 |
+
|
| 131 |
+
for i in range(beam_width):
|
| 132 |
+
b_idx = parent_beam_idx[i]
|
| 133 |
+
tok = token_id[i]
|
| 134 |
+
score = top_scores[i]
|
| 135 |
+
|
| 136 |
+
parent_seq = beam_sequences[b_idx : b_idx + 1]
|
| 137 |
+
new_seq = torch.cat([parent_seq, tok.view(1, 1)], dim = -1)
|
| 138 |
+
|
| 139 |
+
if eos_token_id is not None and tok.item() == eos_token_id:
|
| 140 |
+
gen_len = new_seq.shape[1] - input_ids.shape[1]
|
| 141 |
+
norm_score = score.item() / (gen_len ** length_penalty_alpha)
|
| 142 |
+
completed.append((new_seq, norm_score))
|
| 143 |
+
else:
|
| 144 |
+
next_beam_sequences.append(new_seq)
|
| 145 |
+
next_beam_scores.append(score)
|
| 146 |
+
|
| 147 |
+
if len(next_beam_sequences) == 0 or len(completed) >= beam_width:
|
| 148 |
+
break
|
| 149 |
+
beam_sequences = torch.cat(next_beam_sequences, dim = 0)
|
| 150 |
+
beam_scores = torch.stack(next_beam_scores)
|
| 151 |
+
|
| 152 |
+
for b in range(beam_sequences.shape[0]):
|
| 153 |
+
seq = beam_sequences[b : b + 1]
|
| 154 |
+
gen_len = max(1, seq.shape[1] - input_ids.shape[1])
|
| 155 |
+
norm_score = beam_scores[b].item() / (gen_len ** length_penalty_alpha)
|
| 156 |
+
completed.append((seq, norm_score))
|
| 157 |
+
|
| 158 |
+
completed.sort(key = lambda x: x[1], reverse = True)
|
| 159 |
+
return completed[0][0]
|
src/part3/evaluate.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import json
|
| 3 |
+
import math
|
| 4 |
+
import os
|
| 5 |
+
import sys
|
| 6 |
+
import time
|
| 7 |
+
from collections import Counter
|
| 8 |
+
|
| 9 |
+
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import wandb
|
| 13 |
+
from datasets import load_dataset
|
| 14 |
+
from tqdm import tqdm
|
| 15 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 16 |
+
|
| 17 |
+
from src.part3.decoding import beam_search_decode, greedy_decode, top_k_decoding, top_p_decoding
|
| 18 |
+
|
| 19 |
+
def compute_unigram_metrics(pred_text, ref_text):
|
| 20 |
+
pred_tokens = pred_text.strip().lower().split()
|
| 21 |
+
ref_tokens = ref_text.strip().lower().split()
|
| 22 |
+
|
| 23 |
+
if len(pred_tokens) == 0 or len(ref_tokens) == 0:
|
| 24 |
+
return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.0}
|
| 25 |
+
pred_counts = Counter(pred_tokens)
|
| 26 |
+
ref_counts = Counter(ref_tokens)
|
| 27 |
+
|
| 28 |
+
overlap = sum(min(pred_counts[token], ref_counts[token]) for token in pred_counts)
|
| 29 |
+
precision = overlap / len(pred_tokens)
|
| 30 |
+
recall = overlap / len(ref_tokens)
|
| 31 |
+
f1 = 0.0
|
| 32 |
+
accuracy = 0.0
|
| 33 |
+
if (precision + recall) > 0:
|
| 34 |
+
f1 = (2 * precision * recall) / (precision + recall)
|
| 35 |
+
else:
|
| 36 |
+
f1 = 0.0
|
| 37 |
+
|
| 38 |
+
if pred_text.strip().lower() == ref_text.strip().lower():
|
| 39 |
+
accuracy = 1.0
|
| 40 |
+
else:
|
| 41 |
+
accuracy = 0.0
|
| 42 |
+
|
| 43 |
+
return {
|
| 44 |
+
"precision": precision,
|
| 45 |
+
"recall": recall,
|
| 46 |
+
"f1": f1,
|
| 47 |
+
"accuracy": accuracy,
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
@torch.no_grad()
|
| 51 |
+
def compute_conditional_perplexity(model, tokenizer, prompt, continuation, device):
|
| 52 |
+
full_text = prompt + " " + continuation
|
| 53 |
+
enc_full = tokenizer(full_text, return_tensors = "pt").to(device)
|
| 54 |
+
enc_prompt = tokenizer(prompt, return_tensors = "pt")
|
| 55 |
+
prompt_len = enc_prompt["input_ids"].shape[1]
|
| 56 |
+
|
| 57 |
+
labels = enc_full["input_ids"].clone()
|
| 58 |
+
labels[:, :prompt_len] = -100
|
| 59 |
+
|
| 60 |
+
outputs = model(**enc_full, labels = labels)
|
| 61 |
+
loss = outputs.loss.item()
|
| 62 |
+
return math.exp(loss)
|
| 63 |
+
|
| 64 |
+
def run_evaluation(model_name = "EleutherAI/pythia-160m", num_samples = 1000, max_new_tokens = 40,
|
| 65 |
+
device = "cuda" if torch.cuda.is_available() else "cpu", output_file = "part3_results.json", use_wandb = True,
|
| 66 |
+
wandb_project = "ANLP-Assignment-2-Part3", wandb_run_name = "decoding_eval"):
|
| 67 |
+
if use_wandb:
|
| 68 |
+
wandb.init(
|
| 69 |
+
project = wandb_project,
|
| 70 |
+
name = wandb_run_name,
|
| 71 |
+
config = {
|
| 72 |
+
"model_name": model_name,
|
| 73 |
+
"num_samples": num_samples,
|
| 74 |
+
"max_new_tokens": max_new_tokens,
|
| 75 |
+
"device": device,
|
| 76 |
+
},
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
print(f"Loading model and tokenizer '{model_name}' on {device}")
|
| 80 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 81 |
+
if tokenizer.pad_token_id is None:
|
| 82 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 83 |
+
|
| 84 |
+
model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
|
| 85 |
+
model.eval()
|
| 86 |
+
|
| 87 |
+
print(f"Loading ROCStories test split (selecting {num_samples} samples)")
|
| 88 |
+
dataset = load_dataset("hamishivi/ROCStories", split = "test")
|
| 89 |
+
samples = dataset.select(range(min(num_samples, len(dataset))))
|
| 90 |
+
|
| 91 |
+
configurations = {
|
| 92 |
+
"Greedy": lambda m, inp, eos: greedy_decode(m, inp, max_new_tokens = max_new_tokens, eos_token_id = eos),
|
| 93 |
+
"Top-k (k = 50, T = 0.7)": lambda m, inp, eos: top_k_decoding(m, inp, max_new_tokens = max_new_tokens, k = 50, temperature = 0.7, eos_token_id = eos),
|
| 94 |
+
"Top-p (p = 0.9, T = 0.7)": lambda m, inp, eos: top_p_decoding(m, inp, max_new_tokens = max_new_tokens, p = 0.9, temperature = 0.7, eos_token_id = eos),
|
| 95 |
+
"Beam Search (B = 1)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 1, max_new_tokens = max_new_tokens, eos_token_id = eos),
|
| 96 |
+
"Beam Search (B = 2)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 2, max_new_tokens = max_new_tokens, eos_token_id = eos),
|
| 97 |
+
"Beam Search (B = 4)": lambda m, inp, eos: beam_search_decode(m, inp, beam_width = 4, max_new_tokens = max_new_tokens, eos_token_id = eos),
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
results = {}
|
| 101 |
+
eos_id = tokenizer.eos_token_id
|
| 102 |
+
|
| 103 |
+
print("\nComputing baseline Reference Perplexity over ground truth continuations...")
|
| 104 |
+
ref_ppls = []
|
| 105 |
+
for sample in tqdm(samples, desc = "Reference PPL"):
|
| 106 |
+
ppl = compute_conditional_perplexity(model, tokenizer, sample["prompt"], sample["continuation"], device)
|
| 107 |
+
ref_ppls.append(ppl)
|
| 108 |
+
avg_ref_ppl = sum(ref_ppls) / len(ref_ppls)
|
| 109 |
+
print(f"Average Reference Perplexity: {avg_ref_ppl:.2f}")
|
| 110 |
+
|
| 111 |
+
if use_wandb:
|
| 112 |
+
wandb.log({"reference_ppl": avg_ref_ppl})
|
| 113 |
+
|
| 114 |
+
for name, decode_fn in configurations.items():
|
| 115 |
+
print(f"\n--- Evaluating: {name} ---")
|
| 116 |
+
metrics_list = []
|
| 117 |
+
gen_ppls = []
|
| 118 |
+
start_time = time.perf_counter()
|
| 119 |
+
|
| 120 |
+
for sample in tqdm(samples, desc = name):
|
| 121 |
+
prompt = sample["prompt"]
|
| 122 |
+
ref_continuation = sample["continuation"]
|
| 123 |
+
|
| 124 |
+
inputs = tokenizer(prompt, return_tensors = "pt").to(device)
|
| 125 |
+
prompt_len = inputs["input_ids"].shape[1]
|
| 126 |
+
|
| 127 |
+
output_ids = decode_fn(model, inputs["input_ids"], eos_id)
|
| 128 |
+
gen_ids = output_ids[:, prompt_len:]
|
| 129 |
+
gen_text = tokenizer.decode(gen_ids[0], skip_special_tokens = True)
|
| 130 |
+
|
| 131 |
+
metrics = compute_unigram_metrics(gen_text, ref_continuation)
|
| 132 |
+
metrics_list.append(metrics)
|
| 133 |
+
|
| 134 |
+
if len(gen_text.strip()) > 0:
|
| 135 |
+
gen_ppl = compute_conditional_perplexity(model, tokenizer, prompt, gen_text, device)
|
| 136 |
+
gen_ppls.append(gen_ppl)
|
| 137 |
+
|
| 138 |
+
total_time = time.perf_counter() - start_time
|
| 139 |
+
|
| 140 |
+
avg_precision = sum(m["precision"] for m in metrics_list) / len(metrics_list)
|
| 141 |
+
avg_recall = sum(m["recall"] for m in metrics_list) / len(metrics_list)
|
| 142 |
+
avg_f1 = sum(m["f1"] for m in metrics_list) / len(metrics_list)
|
| 143 |
+
avg_acc = sum(m["accuracy"] for m in metrics_list) / len(metrics_list)
|
| 144 |
+
avg_gen_ppl = sum(gen_ppls) / len(gen_ppls) if gen_ppls else float("nan")
|
| 145 |
+
|
| 146 |
+
results[name] = {
|
| 147 |
+
"Time (s)": round(total_time, 2),
|
| 148 |
+
"PPL (Generated)": round(avg_gen_ppl, 2),
|
| 149 |
+
"Accuracy": round(avg_acc, 4),
|
| 150 |
+
"Precision": round(avg_precision, 4),
|
| 151 |
+
"Recall": round(avg_recall, 4),
|
| 152 |
+
"F1": round(avg_f1, 4),
|
| 153 |
+
}
|
| 154 |
+
print(
|
| 155 |
+
f"Result for {name}: Time={total_time:.2f}s | Gen PPL={avg_gen_ppl:.2f} | "
|
| 156 |
+
f"Acc={avg_acc:.4f} | Prec={avg_precision:.4f} | Rec={avg_recall:.4f} | F1={avg_f1:.4f}"
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
if use_wandb:
|
| 160 |
+
wandb.log({
|
| 161 |
+
f"{name}/time_seconds": total_time,
|
| 162 |
+
f"{name}/gen_ppl": avg_gen_ppl,
|
| 163 |
+
f"{name}/accuracy": avg_acc,
|
| 164 |
+
f"{name}/precision": avg_precision,
|
| 165 |
+
f"{name}/recall": avg_recall,
|
| 166 |
+
f"{name}/f1": avg_f1,
|
| 167 |
+
})
|
| 168 |
+
|
| 169 |
+
print("\n" + "=" * 80)
|
| 170 |
+
print(f"{'Decoding Strategy':<25} | {'Time (s)':<10} | {'Gen PPL':<10} | {'Acc':<8} | {'Prec':<8} | {'Rec':<8} | {'F1':<8}")
|
| 171 |
+
print("=" * 80)
|
| 172 |
+
for name, data in results.items():
|
| 173 |
+
print(f"{name:<25} | {data['Time (s)']:<10} | {data['PPL (Generated)']:<10} | {data['Accuracy']:<8} | {data['Precision']:<8} | {data['Recall']:<8} | {data['F1']:<8}")
|
| 174 |
+
print("=" * 80)
|
| 175 |
+
|
| 176 |
+
with open(output_file, "w") as f:
|
| 177 |
+
json.dump({"reference_ppl": avg_ref_ppl, "results": results}, f, indent=2)
|
| 178 |
+
print(f"\nResults successfully saved to {output_file}")
|
| 179 |
+
if use_wandb:
|
| 180 |
+
|
| 181 |
+
summary_table = wandb.Table(columns=["Strategy", "Time (s)", "Gen PPL", "Accuracy", "Precision", "Recall", "F1"])
|
| 182 |
+
for name, data in results.items():
|
| 183 |
+
summary_table.add_data(
|
| 184 |
+
name,
|
| 185 |
+
data["Time (s)"],
|
| 186 |
+
data["PPL (Generated)"],
|
| 187 |
+
data["Accuracy"],
|
| 188 |
+
data["Precision"],
|
| 189 |
+
data["Recall"],
|
| 190 |
+
data["F1"],
|
| 191 |
+
)
|
| 192 |
+
wandb.log({"decoding_summary_table": summary_table})
|
| 193 |
+
beam_timing_table = wandb.Table(columns=["Beam Width", "Time (s)"])
|
| 194 |
+
for b, b_name in [(1, "Beam Search (B=1)"), (2, "Beam Search (B=2)"), (4, "Beam Search (B=4)")]:
|
| 195 |
+
if b_name in results:
|
| 196 |
+
beam_timing_table.add_data(b, results[b_name]["Time (s)"])
|
| 197 |
+
wandb.log({
|
| 198 |
+
"beam_timing_curve": wandb.plot.line(
|
| 199 |
+
beam_timing_table, "Beam Width", "Time (s)", title="Beam Search Time vs Beam Width"
|
| 200 |
+
)
|
| 201 |
+
})
|
| 202 |
+
wandb.finish()
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
if __name__ == "__main__":
|
| 206 |
+
parser = argparse.ArgumentParser(description="Evaluate Task 3 Decoding Strategies")
|
| 207 |
+
parser.add_argument("--num_samples", type=int, default=1000, help="Number of test samples (min 1000)")
|
| 208 |
+
parser.add_argument("--max_new_tokens", type=int, default=40, help="Max tokens to generate")
|
| 209 |
+
parser.add_argument("--output_file", type=str, default="task3_results.json", help="Path to save JSON results")
|
| 210 |
+
parser.add_argument("--wandb", action="store_true", default=True, help="Log metrics to Weights & Biases")
|
| 211 |
+
parser.add_argument("--no_wandb", action="store_false", dest="wandb", help="Disable WandB logging")
|
| 212 |
+
parser.add_argument("--wandb_project", type=str, default="ANLP-Assignment-2-Part3", help="WandB project name")
|
| 213 |
+
parser.add_argument("--wandb_run_name", type=str, default="decoding_evaluation", help="WandB run name")
|
| 214 |
+
args = parser.parse_args()
|
| 215 |
+
run_evaluation(
|
| 216 |
+
num_samples=args.num_samples,
|
| 217 |
+
max_new_tokens=args.max_new_tokens,
|
| 218 |
+
output_file=args.output_file,
|
| 219 |
+
use_wandb=args.wandb,
|
| 220 |
+
wandb_project=args.wandb_project,
|
| 221 |
+
wandb_run_name=args.wandb_run_name,
|
| 222 |
+
)
|
| 223 |
+
|
tokenizer_part1/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_part1/tokenizer_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<eos>",
|
| 4 |
+
"eos_token": "<eos>",
|
| 5 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
+
"pad_token": "<pad>",
|
| 7 |
+
"tokenizer_class": "TokenizersBackend",
|
| 8 |
+
"unk_token": "<unk>"
|
| 9 |
+
}
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|