Download train.py from Adam25613/AMAIA: direct link, hf CLI and curl.
- Browser
- Download file 5.35 kB
-
https://huggingface.co/spaces/Adam25613/AMAIA/resolve/main/train.py
- Command line
-
hf download hf://spaces/Adam25613/AMAIA/train.py
-
curl -L -o train.py https://huggingface.co/spaces/Adam25613/AMAIA/resolve/main/train.py
5.35 kB
| import os | |
| import sys | |
| import math | |
| import time | |
| import argparse | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from config import AmaiaConfig | |
| from model import Amaia | |
| from data import StreamDataset, load_text | |
| def lr_schedule(step, cfg): | |
| if step < cfg.warmup_steps: | |
| return cfg.lr * step / cfg.warmup_steps | |
| t = min(1.0, (step - cfg.warmup_steps) / (cfg.decay_steps - cfg.warmup_steps)) | |
| return cfg.lr_min + 0.5 * (cfg.lr - cfg.lr_min) * (1.0 + math.cos(math.pi * t)) | |
| def save_checkpoint(model, optimizer, step, cfg, path): | |
| torch.save({ | |
| "model": model.state_dict(), | |
| "optimizer": optimizer.state_dict(), | |
| "step": step, | |
| "cfg": cfg, | |
| }, path) | |
| def load_checkpoint(model, optimizer, path): | |
| ckpt = torch.load(path, map_location="cpu") | |
| model.load_state_dict(ckpt["model"]) | |
| optimizer.load_state_dict(ckpt["optimizer"]) | |
| return ckpt["step"] | |
| def sample(model, dataset, prompt, cfg, device): | |
| model.eval() | |
| sp_ids = dataset.sp.Encode(prompt) | |
| idx = torch.tensor([sp_ids], dtype=torch.long, device=device) | |
| with torch.no_grad(): | |
| out = model.generate(idx, cfg.max_new_tokens) | |
| tokens = out[0].tolist() | |
| text = dataset.decode(tokens) | |
| model.train() | |
| return text | |
| def train(args): | |
| cfg = AmaiaConfig() | |
| if args.d_model: cfg.d_model = args.d_model | |
| if args.n_layers: cfg.n_layers = args.n_layers | |
| if args.n_heads: cfg.n_heads = args.n_heads | |
| if args.max_seq: cfg.max_seq = args.max_seq | |
| if args.batch: cfg.batch_size = args.batch | |
| if args.lr: cfg.lr = args.lr | |
| if args.sp_model: cfg.sp_model = args.sp_model | |
| if args.ckpt: cfg.ckpt_file = args.ckpt | |
| device = torch.device("cpu") | |
| torch.set_num_threads(os.cpu_count()) | |
| print(f"Reading input...") | |
| text = load_text(args.input) | |
| print(f" {len(text):,} characters") | |
| print(f"Building dataset...") | |
| dataset = StreamDataset(text, cfg.sp_model, cfg.chunk_size) | |
| cfg.vocab_size = dataset.vocab_size() | |
| print(f" vocab={cfg.vocab_size} tokens={len(dataset.ids):,} chunks≈{len(dataset.ids)//cfg.chunk_size:,}") | |
| loader = DataLoader( | |
| dataset, | |
| batch_size=cfg.batch_size, | |
| num_workers=cfg.num_workers, | |
| pin_memory=False, | |
| persistent_workers=cfg.num_workers > 0, | |
| ) | |
| model = Amaia(cfg).to(device) | |
| print(f"\nAmaia params={model.n_params()/1e6:.2f}M d_model={cfg.d_model} layers={cfg.n_layers} heads={cfg.n_heads}") | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=cfg.lr, | |
| betas=(cfg.beta1, cfg.beta2), | |
| weight_decay=cfg.weight_decay, | |
| ) | |
| step = 0 | |
| if os.path.exists(cfg.ckpt_file): | |
| step = load_checkpoint(model, optimizer, cfg.ckpt_file) | |
| print(f"Resumed from {cfg.ckpt_file} at step {step}") | |
| else: | |
| print(f"Starting fresh") | |
| try: | |
| model = torch.compile(model) | |
| print("torch.compile() enabled") | |
| except Exception: | |
| print("torch.compile() not available, running eager") | |
| smooth_loss = None | |
| t0 = time.time() | |
| tokens_seen = 0 | |
| print(f"\nbatch={cfg.batch_size} chunk={cfg.chunk_size} lr={cfg.lr:.2e}\n") | |
| model.train() | |
| while True: | |
| for x, y in loader: | |
| x, y = x.to(device), y.to(device) | |
| for pg in optimizer.param_groups: | |
| pg["lr"] = lr_schedule(step + 1, cfg) | |
| optimizer.zero_grad(set_to_none=True) | |
| _, loss = model(x, y) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip) | |
| optimizer.step() | |
| step += 1 | |
| tokens_seen += x.numel() | |
| lv = loss.item() | |
| smooth_loss = lv if smooth_loss is None else 0.98 * smooth_loss + 0.02 * lv | |
| if step % cfg.log_every == 0: | |
| elapsed = time.time() - t0 | |
| tps = tokens_seen / elapsed if elapsed > 0 else 0 | |
| lr_now = optimizer.param_groups[0]["lr"] | |
| print(f"step {step:7d} loss {smooth_loss:.4f} lr {lr_now:.2e} {tps/1000:.1f}k tok/s") | |
| if step % cfg.sample_every == 0: | |
| text_out = sample(model, dataset, cfg.sample_prompt, cfg, device) | |
| print(f"\n--- sample ---\n{text_out}\n---\n") | |
| if step % cfg.ckpt_every == 0: | |
| save_checkpoint(model, optimizer, step, cfg, cfg.ckpt_file) | |
| print(f"Checkpoint saved → {cfg.ckpt_file}") | |
| save_checkpoint(model, optimizer, step, cfg, cfg.ckpt_file) | |
| print(f"\nDone. steps={step} loss={smooth_loss:.4f}") | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Train Amaia") | |
| parser.add_argument("input", nargs="?", default="-", | |
| help="text file to train on, or - for stdin (default: -)") | |
| parser.add_argument("--sp-model", default="amaia.model") | |
| parser.add_argument("--ckpt", default="amaia.pt") | |
| parser.add_argument("--d-model", type=int) | |
| parser.add_argument("--n-layers", type=int) | |
| parser.add_argument("--n-heads", type=int) | |
| parser.add_argument("--max-seq", type=int) | |
| parser.add_argument("--batch", type=int) | |
| parser.add_argument("--lr", type=float) | |
| args = parser.parse_args() | |
| train(args) | |
| if __name__ == "__main__": | |
| main() | |