Download src/train.py from wsgpolar/oai-flash-28m: direct link, hf CLI and curl.
- Browser
- Download file 3.22 kB
-
https://huggingface.co/wsgpolar/oai-flash-28m/resolve/main/src/train.py
- Command line
-
hf download hf://wsgpolar/oai-flash-28m/src/train.py
-
curl -L -o train.py https://huggingface.co/wsgpolar/oai-flash-28m/resolve/main/src/train.py
3.22 kB
| import torch, torch.nn as nn, torch.optim as optim | |
| from torch.utils.data import DataLoader | |
| from tqdm import tqdm | |
| from config import TrainConfig | |
| from dataset import TextDataset, collate_batch | |
| from model import GPT | |
| from utils import set_seed, autocast_if_bf16, save_checkpoint, latest_checkpoint | |
| import sentencepiece as spm, os, math | |
| def main(): | |
| cfg = TrainConfig(); set_seed(cfg.seed) | |
| device = "cuda" if torch.cuda.is_available() else "cpu"; print("Device:", device) | |
| sp = spm.SentencePieceProcessor(model_file=cfg.spm_model); vocab = sp.vocab_size() | |
| train_ds = TextDataset(cfg.train_path, cfg.spm_model, seq_len=cfg.seq_len, split="train") | |
| val_ds = TextDataset(cfg.train_path, cfg.spm_model, seq_len=cfg.seq_len, split="val") | |
| train_dl = DataLoader(train_ds, batch_size=cfg.micro_bsz, shuffle=True, collate_fn=collate_batch) | |
| val_dl = DataLoader(val_ds, batch_size=cfg.micro_bsz, shuffle=False, collate_fn=collate_batch) | |
| model = GPT(vocab, cfg.d_model, cfg.n_heads, cfg.n_layers, cfg.n_positions, cfg.dropout).to(device) | |
| opt = optim.AdamW(model.parameters(), lr=cfg.lr, betas=cfg.betas, weight_decay=cfg.weight_decay) | |
| sched = optim.lr_scheduler.CosineAnnealingLR(opt, T_max=cfg.max_steps) | |
| last = latest_checkpoint(cfg.ckpt_dir); start_step = 0 | |
| if last and os.path.exists(last): | |
| ck = torch.load(last, map_location=device) | |
| model.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); start_step = ck.get("step",0)+1 | |
| print("Resumed from", last) | |
| loss_fn = nn.CrossEntropyLoss() | |
| tokens_per_step = cfg.batch_size_tokens; accum = 0; opt.zero_grad(set_to_none=True) | |
| def eval_ppl(): | |
| model.eval(); losses=[] | |
| with torch.no_grad(): | |
| for x,y in val_dl: | |
| x,y = x.to(device), y.to(device) | |
| with autocast_if_bf16(cfg.dtype=="bfloat16"): | |
| logits = model(x); loss = loss_fn(logits.view(-1, logits.size(-1)), y.view(-1)) | |
| losses.append(loss.item()); | |
| if len(losses) >= 20: break | |
| model.train(); m=sum(losses)/len(losses); print(f"val_loss={m:.4f} ppl={math.exp(m):.2f}") | |
| step = start_step | |
| while step < cfg.max_steps: | |
| for x,y in train_dl: | |
| x,y = x.to(device), y.to(device) | |
| with autocast_if_bf16(cfg.dtype=="bfloat16"): | |
| logits = model(x) | |
| loss = loss_fn(logits.view(-1, logits.size(-1)), y.view(-1)) / (tokens_per_step/(x.numel())) | |
| loss.backward(); accum += 1 | |
| consumed = accum * x.numel() | |
| if consumed >= tokens_per_step: | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip) | |
| opt.step(); opt.zero_grad(set_to_none=True); sched.step() | |
| step += 1; accum = 0 | |
| if step % 50 == 0: print(f"step {step} loss {loss.item():.4f}") | |
| if step % cfg.eval_every == 0: eval_ppl() | |
| if step % cfg.save_every == 0: save_checkpoint(model, opt, step, cfg.ckpt_dir) | |
| if step >= cfg.max_steps: break | |
| save_checkpoint(model, opt, step, cfg.ckpt_dir); print("Training complete.") | |
| if __name__ == "__main__": | |
| main() | |