import os import sys import math import time import argparse import torch from torch.utils.data import DataLoader from config import AmaiaConfig from model import Amaia from data import StreamDataset, load_text def lr_schedule(step, cfg): if step < cfg.warmup_steps: return cfg.lr * step / cfg.warmup_steps t = min(1.0, (step - cfg.warmup_steps) / (cfg.decay_steps - cfg.warmup_steps)) return cfg.lr_min + 0.5 * (cfg.lr - cfg.lr_min) * (1.0 + math.cos(math.pi * t)) def save_checkpoint(model, optimizer, step, cfg, path): torch.save({ "model": model.state_dict(), "optimizer": optimizer.state_dict(), "step": step, "cfg": cfg, }, path) def load_checkpoint(model, optimizer, path): ckpt = torch.load(path, map_location="cpu") model.load_state_dict(ckpt["model"]) optimizer.load_state_dict(ckpt["optimizer"]) return ckpt["step"] def sample(model, dataset, prompt, cfg, device): model.eval() sp_ids = dataset.sp.Encode(prompt) idx = torch.tensor([sp_ids], dtype=torch.long, device=device) with torch.no_grad(): out = model.generate(idx, cfg.max_new_tokens) tokens = out[0].tolist() text = dataset.decode(tokens) model.train() return text def train(args): cfg = AmaiaConfig() if args.d_model: cfg.d_model = args.d_model if args.n_layers: cfg.n_layers = args.n_layers if args.n_heads: cfg.n_heads = args.n_heads if args.max_seq: cfg.max_seq = args.max_seq if args.batch: cfg.batch_size = args.batch if args.lr: cfg.lr = args.lr if args.sp_model: cfg.sp_model = args.sp_model if args.ckpt: cfg.ckpt_file = args.ckpt device = torch.device("cpu") torch.set_num_threads(os.cpu_count()) print(f"Reading input...") text = load_text(args.input) print(f" {len(text):,} characters") print(f"Building dataset...") dataset = StreamDataset(text, cfg.sp_model, cfg.chunk_size) cfg.vocab_size = dataset.vocab_size() print(f" vocab={cfg.vocab_size} tokens={len(dataset.ids):,} chunks≈{len(dataset.ids)//cfg.chunk_size:,}") loader = DataLoader( dataset, batch_size=cfg.batch_size, num_workers=cfg.num_workers, pin_memory=False, persistent_workers=cfg.num_workers > 0, ) model = Amaia(cfg).to(device) print(f"\nAmaia params={model.n_params()/1e6:.2f}M d_model={cfg.d_model} layers={cfg.n_layers} heads={cfg.n_heads}") optimizer = torch.optim.AdamW( model.parameters(), lr=cfg.lr, betas=(cfg.beta1, cfg.beta2), weight_decay=cfg.weight_decay, ) step = 0 if os.path.exists(cfg.ckpt_file): step = load_checkpoint(model, optimizer, cfg.ckpt_file) print(f"Resumed from {cfg.ckpt_file} at step {step}") else: print(f"Starting fresh") try: model = torch.compile(model) print("torch.compile() enabled") except Exception: print("torch.compile() not available, running eager") smooth_loss = None t0 = time.time() tokens_seen = 0 print(f"\nbatch={cfg.batch_size} chunk={cfg.chunk_size} lr={cfg.lr:.2e}\n") model.train() while True: for x, y in loader: x, y = x.to(device), y.to(device) for pg in optimizer.param_groups: pg["lr"] = lr_schedule(step + 1, cfg) optimizer.zero_grad(set_to_none=True) _, loss = model(x, y) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip) optimizer.step() step += 1 tokens_seen += x.numel() lv = loss.item() smooth_loss = lv if smooth_loss is None else 0.98 * smooth_loss + 0.02 * lv if step % cfg.log_every == 0: elapsed = time.time() - t0 tps = tokens_seen / elapsed if elapsed > 0 else 0 lr_now = optimizer.param_groups[0]["lr"] print(f"step {step:7d} loss {smooth_loss:.4f} lr {lr_now:.2e} {tps/1000:.1f}k tok/s") if step % cfg.sample_every == 0: text_out = sample(model, dataset, cfg.sample_prompt, cfg, device) print(f"\n--- sample ---\n{text_out}\n---\n") if step % cfg.ckpt_every == 0: save_checkpoint(model, optimizer, step, cfg, cfg.ckpt_file) print(f"Checkpoint saved → {cfg.ckpt_file}") save_checkpoint(model, optimizer, step, cfg, cfg.ckpt_file) print(f"\nDone. steps={step} loss={smooth_loss:.4f}") def main(): parser = argparse.ArgumentParser(description="Train Amaia") parser.add_argument("input", nargs="?", default="-", help="text file to train on, or - for stdin (default: -)") parser.add_argument("--sp-model", default="amaia.model") parser.add_argument("--ckpt", default="amaia.pt") parser.add_argument("--d-model", type=int) parser.add_argument("--n-layers", type=int) parser.add_argument("--n-heads", type=int) parser.add_argument("--max-seq", type=int) parser.add_argument("--batch", type=int) parser.add_argument("--lr", type=float) args = parser.parse_args() train(args) if __name__ == "__main__": main()