"""One shared training loop; the optimizer is supplied by each training script.""" import argparse import csv import json import math import random import time from dataclasses import asdict from pathlib import Path import torch from torch.nn import functional as F from tqdm import tqdm from .data import HERE, load_tokenizer, make_batch, prepare_data from .evaluate import precision_context, score_bleu, score_loss from .model import DecoderOnlyTransformer, TransformerConfig def get_arguments(name): parser = argparse.ArgumentParser(description=f"Train the dense model with custom {name}.") parser.add_argument("--data-fraction", type=float, default=0.25) parser.add_argument("--evaluation-fraction", type=float, default=None, help="Defaults to --data-fraction; scales the evaluation sample limits.") parser.add_argument("--validation-samples", type=int, default=1000, help="Validation document limit before scaling.") parser.add_argument("--test-samples", type=int, default=1000, help="Test document limit before scaling (loss and BLEU).") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--eval-batch-size", type=int, default=64) parser.add_argument("--context-size", type=int, default=256) parser.add_argument("--dimension-model", type=int, default=256) parser.add_argument("--attention-heads", type=int, default=8) parser.add_argument("--decoder-blocks", type=int, default=4) parser.add_argument("--learning-rate", type=float, default=3e-4) parser.add_argument("--weight-decay", type=float, default=0.01) parser.add_argument("--beta1", type=float, default=0.9) parser.add_argument("--beta2", type=float, default=0.999) parser.add_argument("--epsilon", type=float, default=1e-8) parser.add_argument("--gamma", type=float, default=0.025, help="MARS variance reduction parameter.") parser.add_argument("--momentum", type=float, default=0.95, help="Muon momentum.") parser.add_argument("--ns-steps", type=int, default=5, help="Muon Newton-Schulz iterations.") parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.") parser.add_argument("--aux-learning-rate", type=float, default=3e-4, help="Muon's custom AdamW learning rate for embeddings/head/biases.") parser.add_argument("--lr-schedule", choices=("constant", "cosine"), default="constant") parser.add_argument("--warmup-fraction", type=float, default=0.05, help="Fraction of training tokens used for linear warmup with cosine scheduling.") parser.add_argument("--min-lr-ratio", type=float, default=0.1, help="Final learning rate / peak learning rate for cosine scheduling.") parser.add_argument("--max-grad-norm", type=float, default=1.0) parser.add_argument("--prompt-tokens", type=int, default=128) parser.add_argument("--max-new-tokens", type=int, default=64) parser.add_argument("--device", default="auto", help="auto, cpu, mps, cuda:0, or cuda:1.") parser.add_argument("--mixed-precision", action=argparse.BooleanOptionalAction, default=True, help="Use FP16 autocast and gradient scaling on CUDA only.") parser.add_argument("--tokenizer-name", default=str(HERE / "tokenizer")) parser.add_argument("--cache-dir", default=str(HERE / "cache")) parser.add_argument("--output-dir", default=str(HERE / "output" / name)) parser.add_argument("--max-train-tokens", type=int, default=0, help="Optional debugging cap; 0 means one pass over the selected text.") parser.add_argument("--prepare-only", action="store_true", help="Cache the shared data without constructing an optimizer.") parser.add_argument("--resume", type=Path, help="Resume this optimizer's latest.pt checkpoint.") if name == "lion": parser.set_defaults(learning_rate=1e-4, beta2=0.99) elif name == "muon": parser.set_defaults(learning_rate=1e-3, aux_learning_rate=1e-4, beta2=0.95, lr_schedule="cosine", mixed_precision=False, batch_size=64, output_dir=str(HERE / "part2_results" / "output" / name) if (HERE / "part2_results").is_dir() else str(HERE / "output" / name)) args = parser.parse_args() if args.evaluation_fraction is None: args.evaluation_fraction = args.data_fraction for value in (args.data_fraction, args.evaluation_fraction): if not 0 < value <= 1: parser.error("Dataset fractions must be in (0, 1].") for value in (args.validation_samples, args.test_samples, args.batch_size, args.eval_batch_size, args.prompt_tokens, args.max_new_tokens): if value <= 0: parser.error("Sample limits, batch sizes, and generation lengths must be positive.") if args.context_size < 2 or args.prompt_tokens > args.context_size: parser.error("Use context-size >= 2 and prompt-tokens <= context-size.") if args.max_train_tokens < 0 or args.learning_rate <= 0 or args.aux_learning_rate <= 0: parser.error("Token caps must be nonnegative and learning rates positive.") if args.max_grad_norm <= 0 or args.weight_decay < 0: parser.error("Use positive max-grad-norm and nonnegative weight-decay.") if not 0 <= args.warmup_fraction < 1 or not 0 < args.min_lr_ratio <= 1: parser.error("Use warmup-fraction in [0, 1) and min-lr-ratio in (0, 1].") return args def set_learning_rates(optimizer, args, position, budget): """Schedule each group's peak rate by tokens, including after a checkpoint resume.""" if args.lr_schedule == "constant": return progress = min(max(position / budget, 0.0), 1.0) if args.warmup_fraction and progress < args.warmup_fraction: factor = progress / args.warmup_fraction else: decay = (progress - args.warmup_fraction) / (1 - args.warmup_fraction) factor = args.min_lr_ratio + (1 - args.min_lr_ratio) * 0.5 * (1 + math.cos(math.pi * decay)) for group in optimizer.param_groups: group.setdefault("peak_lr", group["lr"]) group["lr"] = group["peak_lr"] * factor def get_device(name): if name == "auto": if torch.cuda.is_available(): name = "cuda:0" elif torch.backends.mps.is_available(): name = "mps" else: name = "cpu" return torch.device(name) def synchronize(device): if device.type == "cuda": torch.cuda.synchronize(device) elif device.type == "mps": torch.mps.synchronize() def write_metrics(history, directory): (directory / "metrics.json").write_text(json.dumps(history, indent=2, allow_nan=False)) with (directory / "metrics.csv").open("w", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=list(history[0])) writer.writeheader() writer.writerows(history) def plot_metrics(history, directory): import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt tokens = [row["train_tokens"] for row in history] figure, axis = plt.subplots(figsize=(7, 4), layout="constrained") axis.plot(tokens, [row["validation_loss"] for row in history], marker="o", label="Validation") trained = [row for row in history if row["train_loss"] is not None] axis.plot([row["train_tokens"] for row in trained], [row["train_loss"] for row in trained], marker="o", label="Train (since previous evaluation)") axis.set(xlabel="Training target tokens seen", ylabel="Cross entropy", title="Training dynamics") axis.legend() figure.savefig(directory / "loss_vs_tokens.png", dpi=160) plt.close(figure) figure, axis = plt.subplots(figsize=(7, 4), layout="constrained") axis.plot(tokens, [row["test_bleu"] for row in history], marker="o") axis.set(xlabel="Training target tokens seen", ylabel="Test BLEU (0-100)", title="Held-out continuation BLEU") figure.savefig(directory / "bleu_vs_tokens.png", dpi=160) plt.close(figure) def record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args, name, directory, history, state, interval_loss, interval_tokens): synchronize(device) started = time.perf_counter() validation = score_loss(model, prepared["validation_tokens"], tokenizer, device, args) test = score_loss(model, prepared["test_tokens"], tokenizer, device, args) prediction_file = directory / f"predictions_{len(history):02d}.jsonl" bleu = score_bleu(model, prepared["test_documents"], tokenizer, device, args, prediction_file) synchronize(device) evaluation_seconds = time.perf_counter() - started state["evaluation_seconds"] += evaluation_seconds train_loss = interval_loss / interval_tokens if interval_tokens else None record = { "optimizer": name, "step": state["step"], "successful_updates": state["step"] - state["skipped_updates"], "skipped_updates": state["skipped_updates"], "train_tokens": state["position"], "training_budget_tokens": state["budget"], "selected_dataset_fraction": state["position"] / state["budget"], "full_dataset_token_fraction": state["position"] / prepared["metadata"]["full_train_target_tokens"], "tokens_per_parameter": state["position"] / state["parameters"], "train_loss": train_loss, "train_perplexity": math.exp(min(train_loss, 700)) if train_loss is not None else None, "cumulative_train_loss": state["loss_sum"] / state["position"] if state["position"] else None, "validation_loss": validation["loss"], "validation_perplexity": validation["perplexity"], "validation_tokens": validation["target_tokens"], "test_loss": test["loss"], "test_perplexity": test["perplexity"], "test_tokens": test["target_tokens"], "test_bleu": bleu["bleu"], "bleu_signature": bleu["bleu_signature"], "bleu_samples": bleu["samples"], "generated_tokens": bleu["generated_tokens"], "learning_rates": [group["lr"] for group in optimizer.param_groups], "last_gradient_norm": state["last_gradient_norm"], "training_seconds": state["training_seconds"], "evaluation_seconds": evaluation_seconds, "total_evaluation_seconds": state["evaluation_seconds"], "train_tokens_per_second": state["position"] / state["training_seconds"] if state["training_seconds"] else None, "prediction_file": prediction_file.name, } history.append(record) write_metrics(history, directory) checkpoint = { "optimizer_name": name, "config": asdict(model.config), "model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(), "scaler_state_dict": scaler.state_dict(), "history": history, "state": state, "args": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, "data_settings": prepared["settings"], "data_metadata": prepared["metadata"], } temporary_file = directory / "latest.tmp" torch.save(checkpoint, temporary_file) temporary_file.replace(directory / "latest.pt") print(f"{name}: tokens={state['position']:,}/{state['budget']:,}, " f"validation_loss={validation['loss']:.4f}, test_BLEU={bleu['bleu']:.2f}") def run_training(name, make_optimizer, args): random.seed(args.seed) torch.manual_seed(args.seed) device = get_device(args.device) tokenizer = load_tokenizer(args.tokenizer_name) if args.prepare_only: prepared = prepare_data(args, tokenizer) print(json.dumps(prepared["metadata"], indent=2)) return config = TransformerConfig(len(tokenizer), args.context_size, args.dimension_model, args.attention_heads, args.decoder_blocks) # Fresh weights for every run; the identical seed gives identical initialization. model = DecoderOnlyTransformer(config, moe_flag=False, experts=0, active=0, shared=0).to(device) optimizer = make_optimizer(model, args) prepared = prepare_data(args, tokenizer) budget = prepared["metadata"]["train_target_tokens"] if args.max_train_tokens: budget = min(budget, args.max_train_tokens) scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda" and args.mixed_precision) directory = Path(args.output_dir) directory.mkdir(parents=True, exist_ok=True) if not args.resume and (directory / "metrics.json").exists(): raise ValueError("This output already contains a run. Use --resume or another --output-dir.") state = {"position": 0, "step": 0, "budget": budget, "loss_sum": 0.0, "parameters": sum(parameter.numel() for parameter in model.parameters()), "training_seconds": 0.0, "evaluation_seconds": 0.0, "skipped_updates": 0, "last_gradient_norm": None, "precision_changes": []} history = [] if args.resume: checkpoint = torch.load(args.resume, map_location="cpu", weights_only=True) if checkpoint["optimizer_name"] != name or checkpoint["data_settings"] != prepared["settings"]: raise ValueError("Resume checkpoint has a different optimizer or dataset selection.") ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"} current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()} schedule_defaults = {"lr_schedule": "constant", "warmup_fraction": 0.05, "min_lr_ratio": 0.1} for key, value in current_args.items(): saved_value = checkpoint["args"].get(key, schedule_defaults.get(key)) if key not in ignored_args and saved_value != value: raise ValueError(f"Resume argument changed: {key}.") model.load_state_dict(checkpoint["model_state_dict"]) optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) if scaler.is_enabled() and checkpoint["scaler_state_dict"]: scaler.load_state_dict(checkpoint["scaler_state_dict"]) state = checkpoint["state"] if checkpoint["args"]["mixed_precision"] != args.mixed_precision: state.setdefault("precision_changes", []).append({ "train_tokens": state["position"], "previous_mixed_precision": checkpoint["args"]["mixed_precision"], "mixed_precision": args.mixed_precision, }) history = checkpoint["history"] set_learning_rates(optimizer, args, state["position"], budget) tokenizer.save_pretrained(directory / "tokenizer") groups = [{key: value for key, value in group.items() if key != "params"} for group in optimizer.param_groups] settings = {"optimizer": name, "arguments": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, "model": asdict(config), "parameters": state["parameters"], "optimizer_groups": groups, "data": prepared["metadata"], "learning_rate_schedule": args.lr_schedule, "evaluation_interval": "0.1 of selected training budget", "precision_changes": state.get("precision_changes", []), "bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."} (directory / "settings.json").write_text(json.dumps(settings, indent=2)) if not history: record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args, name, directory, history, state, 0.0, 0) else: write_metrics(history, directory) log_path = directory / "training_steps.csv" previous_rows = [] if args.resume and log_path.exists(): with log_path.open() as stream: previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]] rate_columns = [f"learning_rate_{index}" for index in range(len(optimizer.param_groups))] columns = ["step", "train_tokens", "batch_tokens", "loss", "gradient_norm", "update_skipped", "step_seconds"] + rate_columns boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11))) progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok") with log_path.open("w", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=columns) writer.writeheader() writer.writerows(previous_rows) for boundary in boundaries: if boundary <= state["position"]: continue interval_loss = 0.0 interval_tokens = 0 model.train() while state["position"] < boundary: synchronize(device) started = time.perf_counter() inputs, labels, padding_mask, count = make_batch( prepared["train_tokens"], state["position"], boundary - state["position"], args.batch_size, args.context_size, tokenizer.pad_token_id, ) optimizer.zero_grad(set_to_none=True) with precision_context(device, args.mixed_precision): logits = model(inputs.to(device), padding_mask.to(device)) loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device), ignore_index=-100) loss_value = loss.item() if not math.isfinite(loss_value): raise RuntimeError("Training loss is not finite; the last evaluation checkpoint is saved.") scaler.scale(loss).backward() scaler.unscale_(optimizer) # MARS clips c_t inside step(), so measure g_t without changing it here. if name == "mars": norms = [parameter.grad.norm() for parameter in model.parameters() if parameter.grad is not None] gradient_norm = torch.stack(norms).norm().item() else: gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm).item() if not math.isfinite(gradient_norm) and not scaler.is_enabled(): raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.") previous_scale = scaler.get_scale() set_learning_rates(optimizer, args, state["position"] + count, budget) scaler.step(optimizer) scaler.update() skipped = scaler.get_scale() < previous_scale synchronize(device) seconds = time.perf_counter() - started state["step"] += 1 state["position"] += count state["loss_sum"] += loss_value * count state["training_seconds"] += seconds state["skipped_updates"] += int(skipped) state["last_gradient_norm"] = gradient_norm if math.isfinite(gradient_norm) else None interval_loss += loss_value * count interval_tokens += count writer.writerow({"step": state["step"], "train_tokens": state["position"], "batch_tokens": count, "loss": loss_value, "gradient_norm": state["last_gradient_norm"], "update_skipped": skipped, "step_seconds": seconds, **{column: group["lr"] for column, group in zip(rate_columns, optimizer.param_groups)}}) progress.update(count) progress.set_postfix(loss=f"{loss_value:.4f}") stream.flush() record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args, name, directory, history, state, interval_loss, interval_tokens) progress.close() plot_metrics(history, directory) # The final checkpoint is also latest.pt; keep a clearly named final artifact. checkpoint = torch.load(directory / "latest.pt", map_location="cpu", weights_only=True) torch.save(checkpoint, directory / "final.pt")