walke007's picture
Upload Part 2 checkpoints and recorded results
0282214 verified
Raw History Blame Contribute Delete
20.6 kB
"""One shared training loop; the optimizer is supplied by each training script."""
import argparse
import csv
import json
import math
import random
import time
from dataclasses import asdict
from pathlib import Path
import torch
from torch.nn import functional as F
from tqdm import tqdm
from .data import HERE, load_tokenizer, make_batch, prepare_data
from .evaluate import precision_context, score_bleu, score_loss
from .model import DecoderOnlyTransformer, TransformerConfig
def get_arguments(name):
parser = argparse.ArgumentParser(description=f"Train the dense model with custom {name}.")
parser.add_argument("--data-fraction", type=float, default=0.25)
parser.add_argument("--evaluation-fraction", type=float, default=None,
help="Defaults to --data-fraction; scales the evaluation sample limits.")
parser.add_argument("--validation-samples", type=int, default=1000,
help="Validation document limit before scaling.")
parser.add_argument("--test-samples", type=int, default=1000,
help="Test document limit before scaling (loss and BLEU).")
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--eval-batch-size", type=int, default=64)
parser.add_argument("--context-size", type=int, default=256)
parser.add_argument("--dimension-model", type=int, default=256)
parser.add_argument("--attention-heads", type=int, default=8)
parser.add_argument("--decoder-blocks", type=int, default=4)
parser.add_argument("--learning-rate", type=float, default=3e-4)
parser.add_argument("--weight-decay", type=float, default=0.01)
parser.add_argument("--beta1", type=float, default=0.9)
parser.add_argument("--beta2", type=float, default=0.999)
parser.add_argument("--epsilon", type=float, default=1e-8)
parser.add_argument("--gamma", type=float, default=0.025, help="MARS variance reduction parameter.")
parser.add_argument("--momentum", type=float, default=0.95, help="Muon momentum.")
parser.add_argument("--ns-steps", type=int, default=5, help="Muon Newton-Schulz iterations.")
parser.add_argument("--ns-epsilon", type=float, default=1e-7, help="Muon matrix normalization epsilon.")
parser.add_argument("--aux-learning-rate", type=float, default=3e-4,
help="Muon's custom AdamW learning rate for embeddings/head/biases.")
parser.add_argument("--lr-schedule", choices=("constant", "cosine"), default="constant")
parser.add_argument("--warmup-fraction", type=float, default=0.05,
help="Fraction of training tokens used for linear warmup with cosine scheduling.")
parser.add_argument("--min-lr-ratio", type=float, default=0.1,
help="Final learning rate / peak learning rate for cosine scheduling.")
parser.add_argument("--max-grad-norm", type=float, default=1.0)
parser.add_argument("--prompt-tokens", type=int, default=128)
parser.add_argument("--max-new-tokens", type=int, default=64)
parser.add_argument("--device", default="auto", help="auto, cpu, mps, cuda:0, or cuda:1.")
parser.add_argument("--mixed-precision", action=argparse.BooleanOptionalAction, default=True,
help="Use FP16 autocast and gradient scaling on CUDA only.")
parser.add_argument("--tokenizer-name", default=str(HERE / "tokenizer"))
parser.add_argument("--cache-dir", default=str(HERE / "cache"))
parser.add_argument("--output-dir", default=str(HERE / "output" / name))
parser.add_argument("--max-train-tokens", type=int, default=0,
help="Optional debugging cap; 0 means one pass over the selected text.")
parser.add_argument("--prepare-only", action="store_true",
help="Cache the shared data without constructing an optimizer.")
parser.add_argument("--resume", type=Path, help="Resume this optimizer's latest.pt checkpoint.")
if name == "lion":
parser.set_defaults(learning_rate=1e-4, beta2=0.99)
elif name == "muon":
parser.set_defaults(learning_rate=1e-3, aux_learning_rate=1e-4,
beta2=0.95, lr_schedule="cosine", mixed_precision=False,
batch_size=64,
output_dir=str(HERE / "part2_results" / "output" / name)
if (HERE / "part2_results").is_dir() else str(HERE / "output" / name))
args = parser.parse_args()
if args.evaluation_fraction is None:
args.evaluation_fraction = args.data_fraction
for value in (args.data_fraction, args.evaluation_fraction):
if not 0 < value <= 1:
parser.error("Dataset fractions must be in (0, 1].")
for value in (args.validation_samples, args.test_samples, args.batch_size,
args.eval_batch_size, args.prompt_tokens, args.max_new_tokens):
if value <= 0:
parser.error("Sample limits, batch sizes, and generation lengths must be positive.")
if args.context_size < 2 or args.prompt_tokens > args.context_size:
parser.error("Use context-size >= 2 and prompt-tokens <= context-size.")
if args.max_train_tokens < 0 or args.learning_rate <= 0 or args.aux_learning_rate <= 0:
parser.error("Token caps must be nonnegative and learning rates positive.")
if args.max_grad_norm <= 0 or args.weight_decay < 0:
parser.error("Use positive max-grad-norm and nonnegative weight-decay.")
if not 0 <= args.warmup_fraction < 1 or not 0 < args.min_lr_ratio <= 1:
parser.error("Use warmup-fraction in [0, 1) and min-lr-ratio in (0, 1].")
return args
def set_learning_rates(optimizer, args, position, budget):
"""Schedule each group's peak rate by tokens, including after a checkpoint resume."""
if args.lr_schedule == "constant":
return
progress = min(max(position / budget, 0.0), 1.0)
if args.warmup_fraction and progress < args.warmup_fraction:
factor = progress / args.warmup_fraction
else:
decay = (progress - args.warmup_fraction) / (1 - args.warmup_fraction)
factor = args.min_lr_ratio + (1 - args.min_lr_ratio) * 0.5 * (1 + math.cos(math.pi * decay))
for group in optimizer.param_groups:
group.setdefault("peak_lr", group["lr"])
group["lr"] = group["peak_lr"] * factor
def get_device(name):
if name == "auto":
if torch.cuda.is_available():
name = "cuda:0"
elif torch.backends.mps.is_available():
name = "mps"
else:
name = "cpu"
return torch.device(name)
def synchronize(device):
if device.type == "cuda":
torch.cuda.synchronize(device)
elif device.type == "mps":
torch.mps.synchronize()
def write_metrics(history, directory):
(directory / "metrics.json").write_text(json.dumps(history, indent=2, allow_nan=False))
with (directory / "metrics.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=list(history[0]))
writer.writeheader()
writer.writerows(history)
def plot_metrics(history, directory):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
tokens = [row["train_tokens"] for row in history]
figure, axis = plt.subplots(figsize=(7, 4), layout="constrained")
axis.plot(tokens, [row["validation_loss"] for row in history], marker="o", label="Validation")
trained = [row for row in history if row["train_loss"] is not None]
axis.plot([row["train_tokens"] for row in trained], [row["train_loss"] for row in trained],
marker="o", label="Train (since previous evaluation)")
axis.set(xlabel="Training target tokens seen", ylabel="Cross entropy", title="Training dynamics")
axis.legend()
figure.savefig(directory / "loss_vs_tokens.png", dpi=160)
plt.close(figure)
figure, axis = plt.subplots(figsize=(7, 4), layout="constrained")
axis.plot(tokens, [row["test_bleu"] for row in history], marker="o")
axis.set(xlabel="Training target tokens seen", ylabel="Test BLEU (0-100)",
title="Held-out continuation BLEU")
figure.savefig(directory / "bleu_vs_tokens.png", dpi=160)
plt.close(figure)
def record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args,
name, directory, history, state, interval_loss, interval_tokens):
synchronize(device)
started = time.perf_counter()
validation = score_loss(model, prepared["validation_tokens"], tokenizer, device, args)
test = score_loss(model, prepared["test_tokens"], tokenizer, device, args)
prediction_file = directory / f"predictions_{len(history):02d}.jsonl"
bleu = score_bleu(model, prepared["test_documents"], tokenizer, device, args, prediction_file)
synchronize(device)
evaluation_seconds = time.perf_counter() - started
state["evaluation_seconds"] += evaluation_seconds
train_loss = interval_loss / interval_tokens if interval_tokens else None
record = {
"optimizer": name,
"step": state["step"],
"successful_updates": state["step"] - state["skipped_updates"],
"skipped_updates": state["skipped_updates"],
"train_tokens": state["position"],
"training_budget_tokens": state["budget"],
"selected_dataset_fraction": state["position"] / state["budget"],
"full_dataset_token_fraction": state["position"] / prepared["metadata"]["full_train_target_tokens"],
"tokens_per_parameter": state["position"] / state["parameters"],
"train_loss": train_loss,
"train_perplexity": math.exp(min(train_loss, 700)) if train_loss is not None else None,
"cumulative_train_loss": state["loss_sum"] / state["position"] if state["position"] else None,
"validation_loss": validation["loss"],
"validation_perplexity": validation["perplexity"],
"validation_tokens": validation["target_tokens"],
"test_loss": test["loss"],
"test_perplexity": test["perplexity"],
"test_tokens": test["target_tokens"],
"test_bleu": bleu["bleu"],
"bleu_signature": bleu["bleu_signature"],
"bleu_samples": bleu["samples"],
"generated_tokens": bleu["generated_tokens"],
"learning_rates": [group["lr"] for group in optimizer.param_groups],
"last_gradient_norm": state["last_gradient_norm"],
"training_seconds": state["training_seconds"],
"evaluation_seconds": evaluation_seconds,
"total_evaluation_seconds": state["evaluation_seconds"],
"train_tokens_per_second": state["position"] / state["training_seconds"] if state["training_seconds"] else None,
"prediction_file": prediction_file.name,
}
history.append(record)
write_metrics(history, directory)
checkpoint = {
"optimizer_name": name, "config": asdict(model.config),
"model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(),
"scaler_state_dict": scaler.state_dict(), "history": history, "state": state,
"args": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()},
"data_settings": prepared["settings"], "data_metadata": prepared["metadata"],
}
temporary_file = directory / "latest.tmp"
torch.save(checkpoint, temporary_file)
temporary_file.replace(directory / "latest.pt")
print(f"{name}: tokens={state['position']:,}/{state['budget']:,}, "
f"validation_loss={validation['loss']:.4f}, test_BLEU={bleu['bleu']:.2f}")
def run_training(name, make_optimizer, args):
random.seed(args.seed)
torch.manual_seed(args.seed)
device = get_device(args.device)
tokenizer = load_tokenizer(args.tokenizer_name)
if args.prepare_only:
prepared = prepare_data(args, tokenizer)
print(json.dumps(prepared["metadata"], indent=2))
return
config = TransformerConfig(len(tokenizer), args.context_size, args.dimension_model,
args.attention_heads, args.decoder_blocks)
# Fresh weights for every run; the identical seed gives identical initialization.
model = DecoderOnlyTransformer(config, moe_flag=False, experts=0, active=0, shared=0).to(device)
optimizer = make_optimizer(model, args)
prepared = prepare_data(args, tokenizer)
budget = prepared["metadata"]["train_target_tokens"]
if args.max_train_tokens:
budget = min(budget, args.max_train_tokens)
scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda" and args.mixed_precision)
directory = Path(args.output_dir)
directory.mkdir(parents=True, exist_ok=True)
if not args.resume and (directory / "metrics.json").exists():
raise ValueError("This output already contains a run. Use --resume or another --output-dir.")
state = {"position": 0, "step": 0, "budget": budget, "loss_sum": 0.0,
"parameters": sum(parameter.numel() for parameter in model.parameters()),
"training_seconds": 0.0, "evaluation_seconds": 0.0,
"skipped_updates": 0, "last_gradient_norm": None, "precision_changes": []}
history = []
if args.resume:
checkpoint = torch.load(args.resume, map_location="cpu", weights_only=True)
if checkpoint["optimizer_name"] != name or checkpoint["data_settings"] != prepared["settings"]:
raise ValueError("Resume checkpoint has a different optimizer or dataset selection.")
ignored_args = {"device", "output_dir", "resume", "prepare_only", "mixed_precision"}
current_args = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}
schedule_defaults = {"lr_schedule": "constant", "warmup_fraction": 0.05, "min_lr_ratio": 0.1}
for key, value in current_args.items():
saved_value = checkpoint["args"].get(key, schedule_defaults.get(key))
if key not in ignored_args and saved_value != value:
raise ValueError(f"Resume argument changed: {key}.")
model.load_state_dict(checkpoint["model_state_dict"])
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
if scaler.is_enabled() and checkpoint["scaler_state_dict"]:
scaler.load_state_dict(checkpoint["scaler_state_dict"])
state = checkpoint["state"]
if checkpoint["args"]["mixed_precision"] != args.mixed_precision:
state.setdefault("precision_changes", []).append({
"train_tokens": state["position"],
"previous_mixed_precision": checkpoint["args"]["mixed_precision"],
"mixed_precision": args.mixed_precision,
})
history = checkpoint["history"]
set_learning_rates(optimizer, args, state["position"], budget)
tokenizer.save_pretrained(directory / "tokenizer")
groups = [{key: value for key, value in group.items() if key != "params"}
for group in optimizer.param_groups]
settings = {"optimizer": name, "arguments": {key: str(value) if isinstance(value, Path) else value
for key, value in vars(args).items()},
"model": asdict(config), "parameters": state["parameters"],
"optimizer_groups": groups, "data": prepared["metadata"],
"learning_rate_schedule": args.lr_schedule,
"evaluation_interval": "0.1 of selected training budget",
"precision_changes": state.get("precision_changes", []),
"bleu_definition": "Greedy continuation after prompt_tokens; reference is the next max_new_tokens tokens."}
(directory / "settings.json").write_text(json.dumps(settings, indent=2))
if not history:
record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args,
name, directory, history, state, 0.0, 0)
else:
write_metrics(history, directory)
log_path = directory / "training_steps.csv"
previous_rows = []
if args.resume and log_path.exists():
with log_path.open() as stream:
previous_rows = [row for row in csv.DictReader(stream) if int(row["step"]) <= state["step"]]
rate_columns = [f"learning_rate_{index}" for index in range(len(optimizer.param_groups))]
columns = ["step", "train_tokens", "batch_tokens", "loss", "gradient_norm", "update_skipped", "step_seconds"] + rate_columns
boundaries = sorted(set(math.ceil(budget * fraction / 10) for fraction in range(1, 11)))
progress = tqdm(total=budget, initial=state["position"], desc=name, unit="tok")
with log_path.open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=columns)
writer.writeheader()
writer.writerows(previous_rows)
for boundary in boundaries:
if boundary <= state["position"]:
continue
interval_loss = 0.0
interval_tokens = 0
model.train()
while state["position"] < boundary:
synchronize(device)
started = time.perf_counter()
inputs, labels, padding_mask, count = make_batch(
prepared["train_tokens"], state["position"], boundary - state["position"],
args.batch_size, args.context_size, tokenizer.pad_token_id,
)
optimizer.zero_grad(set_to_none=True)
with precision_context(device, args.mixed_precision):
logits = model(inputs.to(device), padding_mask.to(device))
loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device), ignore_index=-100)
loss_value = loss.item()
if not math.isfinite(loss_value):
raise RuntimeError("Training loss is not finite; the last evaluation checkpoint is saved.")
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
# MARS clips c_t inside step(), so measure g_t without changing it here.
if name == "mars":
norms = [parameter.grad.norm() for parameter in model.parameters()
if parameter.grad is not None]
gradient_norm = torch.stack(norms).norm().item()
else:
gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm).item()
if not math.isfinite(gradient_norm) and not scaler.is_enabled():
raise RuntimeError("Gradient norm is not finite; the last evaluation checkpoint is saved.")
previous_scale = scaler.get_scale()
set_learning_rates(optimizer, args, state["position"] + count, budget)
scaler.step(optimizer)
scaler.update()
skipped = scaler.get_scale() < previous_scale
synchronize(device)
seconds = time.perf_counter() - started
state["step"] += 1
state["position"] += count
state["loss_sum"] += loss_value * count
state["training_seconds"] += seconds
state["skipped_updates"] += int(skipped)
state["last_gradient_norm"] = gradient_norm if math.isfinite(gradient_norm) else None
interval_loss += loss_value * count
interval_tokens += count
writer.writerow({"step": state["step"], "train_tokens": state["position"],
"batch_tokens": count, "loss": loss_value,
"gradient_norm": state["last_gradient_norm"], "update_skipped": skipped,
"step_seconds": seconds,
**{column: group["lr"] for column, group in zip(rate_columns, optimizer.param_groups)}})
progress.update(count)
progress.set_postfix(loss=f"{loss_value:.4f}")
stream.flush()
record_evaluation(model, optimizer, scaler, tokenizer, prepared, device, args,
name, directory, history, state, interval_loss, interval_tokens)
progress.close()
plot_metrics(history, directory)
# The final checkpoint is also latest.pt; keep a clearly named final artifact.
checkpoint = torch.load(directory / "latest.pt", map_location="cpu", weights_only=True)
torch.save(checkpoint, directory / "final.pt")