oai-flash-28m / src /train.py
DevX
Upload folder using huggingface_hub
605f3d7 verified
Raw History Blame Contribute Delete
3.22 kB
import torch, torch.nn as nn, torch.optim as optim
from torch.utils.data import DataLoader
from tqdm import tqdm
from config import TrainConfig
from dataset import TextDataset, collate_batch
from model import GPT
from utils import set_seed, autocast_if_bf16, save_checkpoint, latest_checkpoint
import sentencepiece as spm, os, math
def main():
cfg = TrainConfig(); set_seed(cfg.seed)
device = "cuda" if torch.cuda.is_available() else "cpu"; print("Device:", device)
sp = spm.SentencePieceProcessor(model_file=cfg.spm_model); vocab = sp.vocab_size()
train_ds = TextDataset(cfg.train_path, cfg.spm_model, seq_len=cfg.seq_len, split="train")
val_ds = TextDataset(cfg.train_path, cfg.spm_model, seq_len=cfg.seq_len, split="val")
train_dl = DataLoader(train_ds, batch_size=cfg.micro_bsz, shuffle=True, collate_fn=collate_batch)
val_dl = DataLoader(val_ds, batch_size=cfg.micro_bsz, shuffle=False, collate_fn=collate_batch)
model = GPT(vocab, cfg.d_model, cfg.n_heads, cfg.n_layers, cfg.n_positions, cfg.dropout).to(device)
opt = optim.AdamW(model.parameters(), lr=cfg.lr, betas=cfg.betas, weight_decay=cfg.weight_decay)
sched = optim.lr_scheduler.CosineAnnealingLR(opt, T_max=cfg.max_steps)
last = latest_checkpoint(cfg.ckpt_dir); start_step = 0
if last and os.path.exists(last):
ck = torch.load(last, map_location=device)
model.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); start_step = ck.get("step",0)+1
print("Resumed from", last)
loss_fn = nn.CrossEntropyLoss()
tokens_per_step = cfg.batch_size_tokens; accum = 0; opt.zero_grad(set_to_none=True)
def eval_ppl():
model.eval(); losses=[]
with torch.no_grad():
for x,y in val_dl:
x,y = x.to(device), y.to(device)
with autocast_if_bf16(cfg.dtype=="bfloat16"):
logits = model(x); loss = loss_fn(logits.view(-1, logits.size(-1)), y.view(-1))
losses.append(loss.item());
if len(losses) >= 20: break
model.train(); m=sum(losses)/len(losses); print(f"val_loss={m:.4f} ppl={math.exp(m):.2f}")
step = start_step
while step < cfg.max_steps:
for x,y in train_dl:
x,y = x.to(device), y.to(device)
with autocast_if_bf16(cfg.dtype=="bfloat16"):
logits = model(x)
loss = loss_fn(logits.view(-1, logits.size(-1)), y.view(-1)) / (tokens_per_step/(x.numel()))
loss.backward(); accum += 1
consumed = accum * x.numel()
if consumed >= tokens_per_step:
torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
opt.step(); opt.zero_grad(set_to_none=True); sched.step()
step += 1; accum = 0
if step % 50 == 0: print(f"step {step} loss {loss.item():.4f}")
if step % cfg.eval_every == 0: eval_ppl()
if step % cfg.save_every == 0: save_checkpoint(model, opt, step, cfg.ckpt_dir)
if step >= cfg.max_steps: break
save_checkpoint(model, opt, step, cfg.ckpt_dir); print("Training complete.")
if __name__ == "__main__":
main()