"""Full 500-epoch paper-priority Darcy experiment; see reproduction_spec.md.""" import csv import json from pathlib import Path import time import torch from torch.utils.data import DataLoader, TensorDataset from common import (arguments, atomic_json, build_model, configure_runtime, decode, evaluate, load_config, load_data, normalizer_from_train, relative_l2, save_checkpoint) def main(): parser = arguments(__doc__) parser.add_argument("--resume", type=Path, help="Resume model, optimizer, scheduler and RNG from last checkpoint") args = parser.parse_args() config = load_config(args.config) device = configure_runtime(config) output = Path(config["output"]["root"]) results = output / config["output"]["results"] results.mkdir(parents=True, exist_ok=True) smoke_path = results / "smoke/checks.json" if not smoke_path.exists() or json.loads(smoke_path.read_text())["status"] != "passed": raise RuntimeError("All six compute-node smoke checks must pass before full training") tensors, manifest = load_data(config, results / "data_manifest.json") normalizer = normalizer_from_train(config, tensors["train_y"]) atomic_json(results / "normalization.json", normalizer) model = build_model(config, device) train = config["training"] loader = DataLoader(TensorDataset(tensors["train_x"], tensors["train_y"]), batch_size=train["batch_size"], shuffle=train["shuffle"], drop_last=train["drop_last"], num_workers=0) optimizer = torch.optim.AdamW(model.parameters(), lr=train["lr"], weight_decay=train["weight_decay"], betas=tuple(train["betas"]), eps=train["optimizer_epsilon"]) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=train["t_max"], eta_min=train["eta_min"]) first_epoch, best_score, best_epoch, history = 1, float("inf"), None, [] if args.resume: restored = torch.load(args.resume, map_location="cpu", weights_only=False) if restored["config"] != config: raise ValueError("Resume configuration differs from checkpoint") model.load_state_dict(restored["model_state_dict"], strict=True) optimizer.load_state_dict(restored["optimizer_state_dict"]) scheduler.load_state_dict(restored["scheduler_state_dict"]) first_epoch = restored["epoch"] + 1 best_score, best_epoch = restored["best_score"], restored["best_epoch"] history = restored["history"] torch.set_rng_state(restored["torch_rng_state"]) torch.cuda.set_rng_state_all(restored["cuda_rng_state"]) else: for name in ("checkpoint", "last_checkpoint"): if (output / config["output"][name]).exists(): raise FileExistsError("Existing experiment weights: use --resume or a separate output directory") parameters = sum(p.numel() for p in model.parameters()) print(f"MODEL parameters={parameters}, dense rFFT kernel grid=32x17, full 500 epochs", flush=True) print("LOSS: original-scale mean squared sample-relative L2, paper eq. (15). " "Checkpoint: minimum full-training-set loss; test set never selects weights.", flush=True) start_time = time.perf_counter() runtime = {"parameters": parameters, "torch": torch.__version__, "hip": torch.version.hip, "device": torch.cuda.get_device_name(device), "selection": train["checkpoint_selection"], "seed": train["seed"], "data_sha256": {k: v["sha256"] for k, v in manifest["files"].items()}} atomic_json(results / "run_metadata.json", runtime) fields = ["epoch", "lr", "online_train_loss", "train_relative_l2", "train_squared_relative_l2", "test_relative_l2", "test_squared_relative_l2", "seconds", "selected"] csv_path = results / "training_history.csv" if history: with csv_path.open("w", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=fields) writer.writeheader() writer.writerows(history) for epoch in range(first_epoch, train["epochs"] + 1): epoch_start = time.perf_counter() model.train() learning_rate = optimizer.param_groups[0]["lr"] total, count = 0.0, 0 for batch_index, (x, y) in enumerate(loader): x, y = x.to(device), y.to(device) optimizer.zero_grad(set_to_none=True) prediction = decode(model(x), normalizer) sample_errors = relative_l2(prediction, y) loss = sample_errors.square().mean() if not torch.isfinite(loss): raise FloatingPointError(f"Nonfinite loss at epoch {epoch} batch {batch_index}") loss.backward() optimizer.step() value = loss.item() total += value * len(x) count += len(x) if batch_index % train["print_batch_interval"] == 0 or batch_index == len(loader) - 1: print(f"Epoch {epoch:03d}/500 batch {batch_index + 1:02d}/{len(loader)} " f"loss={value:.8f} sample_rel_l2={sample_errors.mean().item():.8f}", flush=True) training, _, _ = evaluate(model, tensors["train_x"], tensors["train_y"], normalizer, train["batch_size"], device) testing, _, _ = evaluate(model, tensors["test_x"], tensors["test_y"], normalizer, train["batch_size"], device) score = training["squared_relative_l2_mean"] improved = score < best_score if improved: best_score, best_epoch = score, epoch scheduler.step() row = {"epoch": epoch, "lr": learning_rate, "online_train_loss": total / count, "train_relative_l2": training["relative_l2_mean"], "train_squared_relative_l2": score, "test_relative_l2": testing["relative_l2_mean"], "test_squared_relative_l2": testing["squared_relative_l2_mean"], "seconds": time.perf_counter() - epoch_start, "selected": improved} history.append(row) with csv_path.open("a", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=fields) if epoch == 1: writer.writeheader() writer.writerow(row) checkpoint = {"model_state_dict": model.state_dict(), "config": config, "epoch": epoch, "completed_epochs_at_save": epoch, "normalizer": normalizer, "best_score": best_score, "best_epoch": best_epoch, "selection_rule": train["checkpoint_selection"], "train_metrics": training, "test_metrics": testing, "optimizer_state_dict": optimizer.state_dict(), "scheduler_state_dict": scheduler.state_dict(), "history": history, "torch_rng_state": torch.get_rng_state(), "cuda_rng_state": torch.cuda.get_rng_state_all(), "data_sha256": runtime["data_sha256"]} if improved: save_checkpoint(output / config["output"]["checkpoint"], checkpoint) save_checkpoint(output / config["output"]["last_checkpoint"], checkpoint) atomic_json(results / "training_status.json", {"status": "running" if epoch < 500 else "completed", "completed_epochs": epoch, "best_epoch": best_epoch, "best_training_loss": best_score, "last_epoch": row, "wall_seconds": time.perf_counter() - start_time}) print(f"EPOCH {epoch:03d}/500 lr={learning_rate:.7g} train_loss={score:.9f} " f"train_rel_l2={training['relative_l2_mean']:.8f} " f"test_rel_l2={testing['relative_l2_mean']:.8f} " f"test_squared_rel_l2={testing['squared_relative_l2_mean']:.9f} " f"best_epoch={best_epoch} seconds={row['seconds']:.2f}", flush=True) print(f"TRAINING COMPLETE: completed_epochs=500 selected_epoch={best_epoch}", flush=True) if __name__ == "__main__": main()