File size: 8,116 Bytes
10ab127 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | """Full 500-epoch paper-priority Darcy experiment; see reproduction_spec.md."""
import csv
import json
from pathlib import Path
import time
import torch
from torch.utils.data import DataLoader, TensorDataset
from common import (arguments, atomic_json, build_model, configure_runtime, decode,
evaluate, load_config, load_data, normalizer_from_train,
relative_l2, save_checkpoint)
def main():
parser = arguments(__doc__)
parser.add_argument("--resume", type=Path, help="Resume model, optimizer, scheduler and RNG from last checkpoint")
args = parser.parse_args()
config = load_config(args.config)
device = configure_runtime(config)
output = Path(config["output"]["root"])
results = output / config["output"]["results"]
results.mkdir(parents=True, exist_ok=True)
smoke_path = results / "smoke/checks.json"
if not smoke_path.exists() or json.loads(smoke_path.read_text())["status"] != "passed":
raise RuntimeError("All six compute-node smoke checks must pass before full training")
tensors, manifest = load_data(config, results / "data_manifest.json")
normalizer = normalizer_from_train(config, tensors["train_y"])
atomic_json(results / "normalization.json", normalizer)
model = build_model(config, device)
train = config["training"]
loader = DataLoader(TensorDataset(tensors["train_x"], tensors["train_y"]),
batch_size=train["batch_size"], shuffle=train["shuffle"],
drop_last=train["drop_last"], num_workers=0)
optimizer = torch.optim.AdamW(model.parameters(), lr=train["lr"],
weight_decay=train["weight_decay"],
betas=tuple(train["betas"]), eps=train["optimizer_epsilon"])
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, T_max=train["t_max"], eta_min=train["eta_min"])
first_epoch, best_score, best_epoch, history = 1, float("inf"), None, []
if args.resume:
restored = torch.load(args.resume, map_location="cpu", weights_only=False)
if restored["config"] != config:
raise ValueError("Resume configuration differs from checkpoint")
model.load_state_dict(restored["model_state_dict"], strict=True)
optimizer.load_state_dict(restored["optimizer_state_dict"])
scheduler.load_state_dict(restored["scheduler_state_dict"])
first_epoch = restored["epoch"] + 1
best_score, best_epoch = restored["best_score"], restored["best_epoch"]
history = restored["history"]
torch.set_rng_state(restored["torch_rng_state"])
torch.cuda.set_rng_state_all(restored["cuda_rng_state"])
else:
for name in ("checkpoint", "last_checkpoint"):
if (output / config["output"][name]).exists():
raise FileExistsError("Existing experiment weights: use --resume or a separate output directory")
parameters = sum(p.numel() for p in model.parameters())
print(f"MODEL parameters={parameters}, dense rFFT kernel grid=32x17, full 500 epochs", flush=True)
print("LOSS: original-scale mean squared sample-relative L2, paper eq. (15). "
"Checkpoint: minimum full-training-set loss; test set never selects weights.", flush=True)
start_time = time.perf_counter()
runtime = {"parameters": parameters, "torch": torch.__version__, "hip": torch.version.hip,
"device": torch.cuda.get_device_name(device), "selection": train["checkpoint_selection"],
"seed": train["seed"], "data_sha256": {k: v["sha256"] for k, v in manifest["files"].items()}}
atomic_json(results / "run_metadata.json", runtime)
fields = ["epoch", "lr", "online_train_loss", "train_relative_l2", "train_squared_relative_l2",
"test_relative_l2", "test_squared_relative_l2", "seconds", "selected"]
csv_path = results / "training_history.csv"
if history:
with csv_path.open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
writer.writerows(history)
for epoch in range(first_epoch, train["epochs"] + 1):
epoch_start = time.perf_counter()
model.train()
learning_rate = optimizer.param_groups[0]["lr"]
total, count = 0.0, 0
for batch_index, (x, y) in enumerate(loader):
x, y = x.to(device), y.to(device)
optimizer.zero_grad(set_to_none=True)
prediction = decode(model(x), normalizer)
sample_errors = relative_l2(prediction, y)
loss = sample_errors.square().mean()
if not torch.isfinite(loss):
raise FloatingPointError(f"Nonfinite loss at epoch {epoch} batch {batch_index}")
loss.backward()
optimizer.step()
value = loss.item()
total += value * len(x)
count += len(x)
if batch_index % train["print_batch_interval"] == 0 or batch_index == len(loader) - 1:
print(f"Epoch {epoch:03d}/500 batch {batch_index + 1:02d}/{len(loader)} "
f"loss={value:.8f} sample_rel_l2={sample_errors.mean().item():.8f}", flush=True)
training, _, _ = evaluate(model, tensors["train_x"], tensors["train_y"],
normalizer, train["batch_size"], device)
testing, _, _ = evaluate(model, tensors["test_x"], tensors["test_y"],
normalizer, train["batch_size"], device)
score = training["squared_relative_l2_mean"]
improved = score < best_score
if improved:
best_score, best_epoch = score, epoch
scheduler.step()
row = {"epoch": epoch, "lr": learning_rate, "online_train_loss": total / count,
"train_relative_l2": training["relative_l2_mean"], "train_squared_relative_l2": score,
"test_relative_l2": testing["relative_l2_mean"],
"test_squared_relative_l2": testing["squared_relative_l2_mean"],
"seconds": time.perf_counter() - epoch_start, "selected": improved}
history.append(row)
with csv_path.open("a", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=fields)
if epoch == 1:
writer.writeheader()
writer.writerow(row)
checkpoint = {"model_state_dict": model.state_dict(), "config": config,
"epoch": epoch, "completed_epochs_at_save": epoch,
"normalizer": normalizer, "best_score": best_score, "best_epoch": best_epoch,
"selection_rule": train["checkpoint_selection"], "train_metrics": training,
"test_metrics": testing, "optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(), "history": history,
"torch_rng_state": torch.get_rng_state(), "cuda_rng_state": torch.cuda.get_rng_state_all(),
"data_sha256": runtime["data_sha256"]}
if improved:
save_checkpoint(output / config["output"]["checkpoint"], checkpoint)
save_checkpoint(output / config["output"]["last_checkpoint"], checkpoint)
atomic_json(results / "training_status.json", {"status": "running" if epoch < 500 else "completed",
"completed_epochs": epoch, "best_epoch": best_epoch, "best_training_loss": best_score,
"last_epoch": row, "wall_seconds": time.perf_counter() - start_time})
print(f"EPOCH {epoch:03d}/500 lr={learning_rate:.7g} train_loss={score:.9f} "
f"train_rel_l2={training['relative_l2_mean']:.8f} "
f"test_rel_l2={testing['relative_l2_mean']:.8f} "
f"test_squared_rel_l2={testing['squared_relative_l2_mean']:.9f} "
f"best_epoch={best_epoch} seconds={row['seconds']:.2f}", flush=True)
print(f"TRAINING COMPLETE: completed_epochs=500 selected_epoch={best_epoch}", flush=True)
if __name__ == "__main__":
main()
|