import torch, torch.nn as nn, torch.optim as optim from torch.utils.data import DataLoader from tqdm import tqdm from config import TrainConfig from dataset import TextDataset, collate_batch from model import GPT from utils import set_seed, autocast_if_bf16, save_checkpoint, latest_checkpoint import sentencepiece as spm, os, math def main(): cfg = TrainConfig(); set_seed(cfg.seed) device = "cuda" if torch.cuda.is_available() else "cpu"; print("Device:", device) sp = spm.SentencePieceProcessor(model_file=cfg.spm_model); vocab = sp.vocab_size() train_ds = TextDataset(cfg.train_path, cfg.spm_model, seq_len=cfg.seq_len, split="train") val_ds = TextDataset(cfg.train_path, cfg.spm_model, seq_len=cfg.seq_len, split="val") train_dl = DataLoader(train_ds, batch_size=cfg.micro_bsz, shuffle=True, collate_fn=collate_batch) val_dl = DataLoader(val_ds, batch_size=cfg.micro_bsz, shuffle=False, collate_fn=collate_batch) model = GPT(vocab, cfg.d_model, cfg.n_heads, cfg.n_layers, cfg.n_positions, cfg.dropout).to(device) opt = optim.AdamW(model.parameters(), lr=cfg.lr, betas=cfg.betas, weight_decay=cfg.weight_decay) sched = optim.lr_scheduler.CosineAnnealingLR(opt, T_max=cfg.max_steps) last = latest_checkpoint(cfg.ckpt_dir); start_step = 0 if last and os.path.exists(last): ck = torch.load(last, map_location=device) model.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); start_step = ck.get("step",0)+1 print("Resumed from", last) loss_fn = nn.CrossEntropyLoss() tokens_per_step = cfg.batch_size_tokens; accum = 0; opt.zero_grad(set_to_none=True) def eval_ppl(): model.eval(); losses=[] with torch.no_grad(): for x,y in val_dl: x,y = x.to(device), y.to(device) with autocast_if_bf16(cfg.dtype=="bfloat16"): logits = model(x); loss = loss_fn(logits.view(-1, logits.size(-1)), y.view(-1)) losses.append(loss.item()); if len(losses) >= 20: break model.train(); m=sum(losses)/len(losses); print(f"val_loss={m:.4f} ppl={math.exp(m):.2f}") step = start_step while step < cfg.max_steps: for x,y in train_dl: x,y = x.to(device), y.to(device) with autocast_if_bf16(cfg.dtype=="bfloat16"): logits = model(x) loss = loss_fn(logits.view(-1, logits.size(-1)), y.view(-1)) / (tokens_per_step/(x.numel())) loss.backward(); accum += 1 consumed = accum * x.numel() if consumed >= tokens_per_step: torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip) opt.step(); opt.zero_grad(set_to_none=True); sched.step() step += 1; accum = 0 if step % 50 == 0: print(f"step {step} loss {loss.item():.4f}") if step % cfg.eval_every == 0: eval_ppl() if step % cfg.save_every == 0: save_checkpoint(model, opt, step, cfg.ckpt_dir) if step >= cfg.max_steps: break save_checkpoint(model, opt, step, cfg.ckpt_dir); print("Training complete.") if __name__ == "__main__": main()