Download source/part2/train.py from walke007/anlp-assignment2-part2: direct link, hf CLI and curl.
- Browser
- Download file 20.6 kB
-
https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/train.py
- Command line
-
hf download hf://walke007/anlp-assignment2-part2/source/part2/train.py
-
curl -L -o train.py https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/train.py
20.6 kB
| """One shared training loop; the optimizer is supplied by each training script.""" | |
| import argparse | |
| import csv | |
| import json | |
| import math | |
| import random | |
| import time | |
| from dataclasses import asdict | |
| from pathlib import Path | |
| import torch | |
| from torch.nn import functional as F | |
| from tqdm import tqdm | |
| from .data import HERE, load_tokenizer, make_batch, prepare_data | |
| from .evaluate import precision_context, score_bleu, score_loss | |
| from .model import DecoderOnlyTransformer, TransformerConfig | |
| def get_arguments(name): | |
| parser = argparse.ArgumentParser(description=f"Train the dense model with custom {name}.") | |
| parser.add_argument("--data-fraction", type=float, default=0.25) | |
| parser.add_argument("--evaluation-fraction", type=float, default=None, | |
| help="Defaults to --data-fraction; scales the evaluation sample limits.") | |
| parser.add_argument("--validation-samples", type=int, default=1000, | |
| help="Validation document limit before scaling.") | |
| parser.add_argument("--test-samples", type=int, default=1000, | |
| help="Test document limit before scaling (loss and BLEU).") | |
| parser.add_argument("--seed", type=int, default=42) | |
| parser.add_argument("--batch-size", type=int, default=32) | |
| parser.add_argument("--eval-batch-size", type=int, default=64) | |
| parser.add_argument("--context-size", type=int, default=256) | |
| parser.add_argument("--dimension-model", type=int, default=256) | |
| parser.add_argument("--attention-heads", type=int, default=8) | |
| parser.add_argument("--decoder-blocks", type=int, default=4) | |
| parser.add_argument("--learning-rate", type=float, default=3e-4) | |
| parser.add_argument("--weight-decay", type=float, default=0.01) | |
| parser.add_argument("--beta1", type=float, default=0.9) | |
| parser.add_argument("--beta2", type=float, default=0.999) | |
| parser.add_argument("--epsilon", type=float, default=1e-8) | |
| parser.add_argument("--gamma", type=float, default=0.025, help="MARS variance reduction parameter.") | |
| parser.add_argument("--momentum", type=float, default=0.95, help="Muon momentum.") | |
| parser.add_argument("--ns-steps", type=int, default=5, help="Muon Newton-Schulz iterations.") | |
| parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.") | |
| parser.add_argument("--aux-learning-rate", type=float, default=3e-4, | |
| help="Muon's custom AdamW learning rate for embeddings/head/biases.") | |
| parser.add_argument("--lr-schedule", choices=("constant", "cosine"), default="constant") | |
| parser.add_argument("--warmup-fraction", type=float, default=0.05, | |
| help="Fraction of training tokens used for linear warmup with cosine scheduling.") | |
| parser.add_argument("--min-lr-ratio", type=float, default=0.1, | |
| help="Final learning rate / peak learning rate for cosine scheduling.") | |
| parser.add_argument("--max-grad-norm", type=float, default=1.0) | |
| parser.add_argument("--prompt-tokens", type=int, default=128) | |
| parser.add_argument("--max-new-tokens", type=int, default=64) | |
| parser.add_argument("--device", default="auto", help="auto, cpu, mps, cuda:0, or cuda:1.") | |
| parser.add_argument("--mixed-precision", action=argparse.BooleanOptionalAction, default=True, | |
| help="Use FP16 autocast and gradient scaling on CUDA only.") | |
| parser.add_argument("--tokenizer-name", default=str(HERE / "tokenizer")) | |
| parser.add_argument("--cache-dir", default=str(HERE / "cache")) | |
| parser.add_argument("--output-dir", default=str(HERE / "output" / name)) | |
| parser.add_argument("--max-train-tokens", type=int, default=0, | |
| help="Optional debugging cap; 0 means one pass over the selected text.") | |
| parser.add_argument("--prepare-only", action="store_true", | |
| help="Cache the shared data without constructing an optimizer.") | |
| parser.add_argument("--resume", type=Path, help="Resume this optimizer's latest.pt checkpoint.") | |
| if name == "lion": | |
| parser.set_defaults(learning_rate=1e-4, beta2=0.99) | |
| elif name == "muon": | |
| parser.set_defaults(learning_rate=1e-3, aux_learning_rate=1e-4, | |
| beta2=0.95, lr_schedule="cosine", mixed_precision=False, | |
| batch_size=64, | |
| output_dir=str(HERE / "part2_results" / "output" / name) | |
| if (HERE / "part2_results").is_dir() else str(HERE / "output" / name)) | |
| args = parser.parse_args() | |
| if args.evaluation_fraction is None: | |
| args.evaluation_fraction = args.data_fraction | |
| for value in (args.data_fraction, args.evaluation_fraction): | |
| if not 0 < value <= 1: | |
| parser.error("Dataset fractions must be in (0, 1].") | |
| for value in (args.validation_samples, args.test_samples, args.batch_size, | |
| args.eval_batch_size, args.prompt_tokens, args.max_new_tokens): | |
| if value <= 0: | |
| parser.error("Sample limits, batch sizes, and generation lengths must be positive.") | |
| if args.context_size < 2 or args.prompt_tokens > args.context_size: | |
| parser.error("Use context-size >= 2 and prompt-tokens <= context-size.") | |
| if args.max_train_tokens < 0 or args.learning_rate <= 0 or args.aux_learning_rate <= 0: | |
| parser.error("Token caps must be nonnegative and learning rates positive.") | |
| if args.max_grad_norm <= 0 or args.weight_decay < 0: | |
| parser.error("Use positive max-grad-norm and nonnegative weight-decay.") | |
| if not 0 <= args.warmup_fraction < 1 or not 0 < args.min_lr_ratio <= 1: | |
| parser.error("Use warmup-fraction in [0, 1) and min-lr-ratio in (0, 1].") | |
| return args | |
| def set_learning_rates(optimizer, args, position, budget): | |
| """Schedule each group's peak rate by tokens, including after a checkpoint resume.""" | |
| if args.lr_schedule == "constant": | |
| return | |
| progress = min(max(position / budget, 0.0), 1.0) | |
| if args.warmup_fraction and progress < args.warmup_fraction: | |
| factor = progress / args.warmup_fraction | |
| else: | |
| decay = (progress - args.warmup_fraction) / (1 - args.warmup_fraction) | |
| factor = args.min_lr_ratio + (1 - args.min_lr_ratio) * 0.5 * (1 + math.cos(math.pi * decay)) | |
| for group in optimizer.param_groups: | |
| group.setdefault("peak_lr", group["lr"]) | |
| group["lr"] = group["peak_lr"] * factor | |
| def get_device(name): | |
| if name == "auto": | |
| if torch.cuda.is_available(): | |
| name = "cuda:0" | |
| elif torch.backends.mps.is_available(): | |
| name = "mps" | |
| else: | |
| name = "cpu" | |
| return torch.device(name) | |
| def synchronize(device): | |
| if device.type == "cuda": | |
| torch.cuda.synchronize(device) | |
| elif device.type == "mps": | |
| torch.mps.synchronize() | |
| def write_metrics(history, directory): | |
| (directory / "metrics.json").write_text(json.dumps(history, indent=2, allow_nan=False)) | |
| with (directory / "metrics.csv").open("w", newline="") as stream: | |
| writer = csv.DictWriter(stream, fieldnames=list(history[0])) | |
| writer.writeheader() | |
| writer.writerows(history) | |
| def plot_metrics(history, directory): | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| tokens = [row["train_tokens"] for row in history] | |
| figure, axis = plt.subplots(figsize=(7, 4), layout="constrained") | |
| axis.plot(tokens, [row["validation_loss"] for row in history], marker="o", label="Validation") | |
| trained = [row for row in history if row["train_loss"] is not None] | |
| axis.plot([row["train_tokens"] for row in trained], [row["train_loss"] for row in trained], | |
| marker="o", label="Train (since previous evaluation)") | |
| axis.set(xlabel="Training target tokens seen", ylabel="Cross entropy", title="Training dynamics") | |
| axis.legend() | |
| figure.savefig(directory / "loss_vs_tokens.png", dpi=160) | |
| plt.close(figure) | |
| figure, axis = plt.subplots(figsize=(7, 4), layout="constrained") | |
| axis.plot(tokens, [row["test_bleu"] for row in history], marker="o") | |
| axis.set(xlabel="Training target tokens seen", ylabel="Test BLEU (0-100)", | |
| title="Held-out continuation BLEU") | |
| figure.savefig(directory / "bleu_vs_tokens.png", dpi=160) | |
| plt.close(figure) | |
| def record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args, | |
| name, directory, history, state, interval_loss, interval_tokens): | |
| synchronize(device) | |
| started = time.perf_counter() | |
| validation = score_loss(model, prepared["validation_tokens"], tokenizer, device, args) | |
| test = score_loss(model, prepared["test_tokens"], tokenizer, device, args) | |
| prediction_file = directory / f"predictions_{len(history):02d}.jsonl" | |
| bleu = score_bleu(model, prepared["test_documents"], tokenizer, device, args, prediction_file) | |
| synchronize(device) | |
| evaluation_seconds = time.perf_counter() - started | |
| state["evaluation_seconds"] += evaluation_seconds | |
| train_loss = interval_loss / interval_tokens if interval_tokens else None | |
| record = { | |
| "optimizer": name, | |
| "step": state["step"], | |
| "successful_updates": state["step"] - state["skipped_updates"], | |
| "skipped_updates": state["skipped_updates"], | |
| "train_tokens": state["position"], | |
| "training_budget_tokens": state["budget"], | |
| "selected_dataset_fraction": state["position"] / state["budget"], | |
| "full_dataset_token_fraction": state["position"] / prepared["metadata"]["full_train_target_tokens"], | |
| "tokens_per_parameter": state["position"] / state["parameters"], | |
| "train_loss": train_loss, | |
| "train_perplexity": math.exp(min(train_loss, 700)) if train_loss is not None else None, | |
| "cumulative_train_loss": state["loss_sum"] / state["position"] if state["position"] else None, | |
| "validation_loss": validation["loss"], | |
| "validation_perplexity": validation["perplexity"], | |
| "validation_tokens": validation["target_tokens"], | |
| "test_loss": test["loss"], | |
| "test_perplexity": test["perplexity"], | |
| "test_tokens": test["target_tokens"], | |
| "test_bleu": bleu["bleu"], | |
| "bleu_signature": bleu["bleu_signature"], | |
| "bleu_samples": bleu["samples"], | |
| "generated_tokens": bleu["generated_tokens"], | |
| "learning_rates": [group["lr"] for group in optimizer.param_groups], | |
| "last_gradient_norm": state["last_gradient_norm"], | |
| "training_seconds": state["training_seconds"], | |
| "evaluation_seconds": evaluation_seconds, | |
| "total_evaluation_seconds": state["evaluation_seconds"], | |
| "train_tokens_per_second": state["position"] / state["training_seconds"] if state["training_seconds"] else None, | |
| "prediction_file": prediction_file.name, | |
| } | |
| history.append(record) | |
| write_metrics(history, directory) | |
| checkpoint = { | |
| "optimizer_name": name, "config": asdict(model.config), | |
| "model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(), | |
| "scaler_state_dict": scaler.state_dict(), "history": history, "state": state, | |
| "args": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, | |
| "data_settings": prepared["settings"], "data_metadata": prepared["metadata"], | |
| } | |
| temporary_file = directory / "latest.tmp" | |
| torch.save(checkpoint, temporary_file) | |
| temporary_file.replace(directory / "latest.pt") | |
| print(f"{name}: tokens={state['position']:,}/{state['budget']:,}, " | |
| f"validation_loss={validation['loss']:.4f}, test_BLEU={bleu['bleu']:.2f}") | |
| def run_training(name, make_optimizer, args): | |
| random.seed(args.seed) | |
| torch.manual_seed(args.seed) | |
| device = get_device(args.device) | |
| tokenizer = load_tokenizer(args.tokenizer_name) | |
| if args.prepare_only: | |
| prepared = prepare_data(args, tokenizer) | |
| print(json.dumps(prepared["metadata"], indent=2)) | |
| return | |
| config = TransformerConfig(len(tokenizer), args.context_size, args.dimension_model, | |
| args.attention_heads, args.decoder_blocks) | |
| # Fresh weights for every run; the identical seed gives identical initialization. | |
| model = DecoderOnlyTransformer(config, moe_flag=False, experts=0, active=0, shared=0).to(device) | |
| optimizer = make_optimizer(model, args) | |
| prepared = prepare_data(args, tokenizer) | |
| budget = prepared["metadata"]["train_target_tokens"] | |
| if args.max_train_tokens: | |
| budget = min(budget, args.max_train_tokens) | |
| scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda" and args.mixed_precision) | |
| directory = Path(args.output_dir) | |
| directory.mkdir(parents=True, exist_ok=True) | |
| if not args.resume and (directory / "metrics.json").exists(): | |
| raise ValueError("This output already contains a run. Use --resume or another --output-dir.") | |
| state = {"position": 0, "step": 0, "budget": budget, "loss_sum": 0.0, | |
| "parameters": sum(parameter.numel() for parameter in model.parameters()), | |
| "training_seconds": 0.0, "evaluation_seconds": 0.0, | |
| "skipped_updates": 0, "last_gradient_norm": None, "precision_changes": []} | |
| history = [] | |
| if args.resume: | |
| checkpoint = torch.load(args.resume, map_location="cpu", weights_only=True) | |
| if checkpoint["optimizer_name"] != name or checkpoint["data_settings"] != prepared["settings"]: | |
| raise ValueError("Resume checkpoint has a different optimizer or dataset selection.") | |
| ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"} | |
| current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()} | |
| schedule_defaults = {"lr_schedule": "constant", "warmup_fraction": 0.05, "min_lr_ratio": 0.1} | |
| for key, value in current_args.items(): | |
| saved_value = checkpoint["args"].get(key, schedule_defaults.get(key)) | |
| if key not in ignored_args and saved_value != value: | |
| raise ValueError(f"Resume argument changed: {key}.") | |
| model.load_state_dict(checkpoint["model_state_dict"]) | |
| optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) | |
| if scaler.is_enabled() and checkpoint["scaler_state_dict"]: | |
| scaler.load_state_dict(checkpoint["scaler_state_dict"]) | |
| state = checkpoint["state"] | |
| if checkpoint["args"]["mixed_precision"] != args.mixed_precision: | |
| state.setdefault("precision_changes", []).append({ | |
| "train_tokens": state["position"], | |
| "previous_mixed_precision": checkpoint["args"]["mixed_precision"], | |
| "mixed_precision": args.mixed_precision, | |
| }) | |
| history = checkpoint["history"] | |
| set_learning_rates(optimizer, args, state["position"], budget) | |
| tokenizer.save_pretrained(directory / "tokenizer") | |
| groups = [{key: value for key, value in group.items() if key != "params"} | |
| for group in optimizer.param_groups] | |
| settings = {"optimizer": name, "arguments": {key: str(value) if isinstance(value, Path) else value | |
| for key, value in vars(args).items()}, | |
| "model": asdict(config), "parameters": state["parameters"], | |
| "optimizer_groups": groups, "data": prepared["metadata"], | |
| "learning_rate_schedule": args.lr_schedule, | |
| "evaluation_interval": "0.1 of selected training budget", | |
| "precision_changes": state.get("precision_changes", []), | |
| "bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."} | |
| (directory / "settings.json").write_text(json.dumps(settings, indent=2)) | |
| if not history: | |
| record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args, | |
| name, directory, history, state, 0.0, 0) | |
| else: | |
| write_metrics(history, directory) | |
| log_path = directory / "training_steps.csv" | |
| previous_rows = [] | |
| if args.resume and log_path.exists(): | |
| with log_path.open() as stream: | |
| previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]] | |
| rate_columns = [f"learning_rate_{index}" for index in range(len(optimizer.param_groups))] | |
| columns = ["step", "train_tokens", "batch_tokens", "loss", "gradient_norm", "update_skipped", "step_seconds"] + rate_columns | |
| boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11))) | |
| progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok") | |
| with log_path.open("w", newline="") as stream: | |
| writer = csv.DictWriter(stream, fieldnames=columns) | |
| writer.writeheader() | |
| writer.writerows(previous_rows) | |
| for boundary in boundaries: | |
| if boundary <= state["position"]: | |
| continue | |
| interval_loss = 0.0 | |
| interval_tokens = 0 | |
| model.train() | |
| while state["position"] < boundary: | |
| synchronize(device) | |
| started = time.perf_counter() | |
| inputs, labels, padding_mask, count = make_batch( | |
| prepared["train_tokens"], state["position"], boundary - state["position"], | |
| args.batch_size, args.context_size, tokenizer.pad_token_id, | |
| ) | |
| optimizer.zero_grad(set_to_none=True) | |
| with precision_context(device, args.mixed_precision): | |
| logits = model(inputs.to(device), padding_mask.to(device)) | |
| loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device), ignore_index=-100) | |
| loss_value = loss.item() | |
| if not math.isfinite(loss_value): | |
| raise RuntimeError("Training loss is not finite; the last evaluation checkpoint is saved.") | |
| scaler.scale(loss).backward() | |
| scaler.unscale_(optimizer) | |
| # MARS clips c_t inside step(), so measure g_t without changing it here. | |
| if name == "mars": | |
| norms = [parameter.grad.norm() for parameter in model.parameters() | |
| if parameter.grad is not None] | |
| gradient_norm = torch.stack(norms).norm().item() | |
| else: | |
| gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm).item() | |
| if not math.isfinite(gradient_norm) and not scaler.is_enabled(): | |
| raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.") | |
| previous_scale = scaler.get_scale() | |
| set_learning_rates(optimizer, args, state["position"] + count, budget) | |
| scaler.step(optimizer) | |
| scaler.update() | |
| skipped = scaler.get_scale() < previous_scale | |
| synchronize(device) | |
| seconds = time.perf_counter() - started | |
| state["step"] += 1 | |
| state["position"] += count | |
| state["loss_sum"] += loss_value * count | |
| state["training_seconds"] += seconds | |
| state["skipped_updates"] += int(skipped) | |
| state["last_gradient_norm"] = gradient_norm if math.isfinite(gradient_norm) else None | |
| interval_loss += loss_value * count | |
| interval_tokens += count | |
| writer.writerow({"step": state["step"], "train_tokens": state["position"], | |
| "batch_tokens": count, "loss": loss_value, | |
| "gradient_norm": state["last_gradient_norm"], "update_skipped": skipped, | |
| "step_seconds": seconds, | |
| **{column: group["lr"] for column, group in zip(rate_columns, optimizer.param_groups)}}) | |
| progress.update(count) | |
| progress.set_postfix(loss=f"{loss_value:.4f}") | |
| stream.flush() | |
| record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args, | |
| name, directory, history, state, interval_loss, interval_tokens) | |
| progress.close() | |
| plot_metrics(history, directory) | |
| # The final checkpoint is also latest.pt; keep a clearly named final artifact. | |
| checkpoint = torch.load(directory / "latest.pt", map_location="cpu", weights_only=True) | |
| torch.save(checkpoint, directory / "final.pt") | |