SirenFNO / scripts /train.py
yushuang88's picture
Upload folder using huggingface_hub
10ab127 verified
Raw History Blame Contribute Delete
8.12 kB
"""Full 500-epoch paper-priority Darcy experiment; see reproduction_spec.md."""
import csv
import json
from pathlib import Path
import time
import torch
from torch.utils.data import DataLoader, TensorDataset
from common import (arguments, atomic_json, build_model, configure_runtime, decode,
evaluate, load_config, load_data, normalizer_from_train,
relative_l2, save_checkpoint)
def main():
parser = arguments(__doc__)
parser.add_argument("--resume", type=Path, help="Resume model, optimizer, scheduler and RNG from last checkpoint")
args = parser.parse_args()
config = load_config(args.config)
device = configure_runtime(config)
output = Path(config["output"]["root"])
results = output / config["output"]["results"]
results.mkdir(parents=True, exist_ok=True)
smoke_path = results / "smoke/checks.json"
if not smoke_path.exists() or json.loads(smoke_path.read_text())["status"] != "passed":
raise RuntimeError("All six compute-node smoke checks must pass before full training")
tensors, manifest = load_data(config, results / "data_manifest.json")
normalizer = normalizer_from_train(config, tensors["train_y"])
atomic_json(results / "normalization.json", normalizer)
model = build_model(config, device)
train = config["training"]
loader = DataLoader(TensorDataset(tensors["train_x"], tensors["train_y"]),
batch_size=train["batch_size"], shuffle=train["shuffle"],
drop_last=train["drop_last"], num_workers=0)
optimizer = torch.optim.AdamW(model.parameters(), lr=train["lr"],
weight_decay=train["weight_decay"],
betas=tuple(train["betas"]), eps=train["optimizer_epsilon"])
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, T_max=train["t_max"], eta_min=train["eta_min"])
first_epoch, best_score, best_epoch, history = 1, float("inf"), None, []
if args.resume:
restored = torch.load(args.resume, map_location="cpu", weights_only=False)
if restored["config"] != config:
raise ValueError("Resume configuration differs from checkpoint")
model.load_state_dict(restored["model_state_dict"], strict=True)
optimizer.load_state_dict(restored["optimizer_state_dict"])
scheduler.load_state_dict(restored["scheduler_state_dict"])
first_epoch = restored["epoch"] + 1
best_score, best_epoch = restored["best_score"], restored["best_epoch"]
history = restored["history"]
torch.set_rng_state(restored["torch_rng_state"])
torch.cuda.set_rng_state_all(restored["cuda_rng_state"])
else:
for name in ("checkpoint", "last_checkpoint"):
if (output / config["output"][name]).exists():
raise FileExistsError("Existing experiment weights: use --resume or a separate output directory")
parameters = sum(p.numel() for p in model.parameters())
print(f"MODEL parameters={parameters}, dense rFFT kernel grid=32x17, full 500 epochs", flush=True)
print("LOSS: original-scale mean squared sample-relative L2, paper eq. (15). "
"Checkpoint: minimum full-training-set loss; test set never selects weights.", flush=True)
start_time = time.perf_counter()
runtime = {"parameters": parameters, "torch": torch.__version__, "hip": torch.version.hip,
"device": torch.cuda.get_device_name(device), "selection": train["checkpoint_selection"],
"seed": train["seed"], "data_sha256": {k: v["sha256"] for k, v in manifest["files"].items()}}
atomic_json(results / "run_metadata.json", runtime)
fields = ["epoch", "lr", "online_train_loss", "train_relative_l2", "train_squared_relative_l2",
"test_relative_l2", "test_squared_relative_l2", "seconds", "selected"]
csv_path = results / "training_history.csv"
if history:
with csv_path.open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
writer.writerows(history)
for epoch in range(first_epoch, train["epochs"] + 1):
epoch_start = time.perf_counter()
model.train()
learning_rate = optimizer.param_groups[0]["lr"]
total, count = 0.0, 0
for batch_index, (x, y) in enumerate(loader):
x, y = x.to(device), y.to(device)
optimizer.zero_grad(set_to_none=True)
prediction = decode(model(x), normalizer)
sample_errors = relative_l2(prediction, y)
loss = sample_errors.square().mean()
if not torch.isfinite(loss):
raise FloatingPointError(f"Nonfinite loss at epoch {epoch} batch {batch_index}")
loss.backward()
optimizer.step()
value = loss.item()
total += value * len(x)
count += len(x)
if batch_index % train["print_batch_interval"] == 0 or batch_index == len(loader) - 1:
print(f"Epoch {epoch:03d}/500 batch {batch_index + 1:02d}/{len(loader)} "
f"loss={value:.8f} sample_rel_l2={sample_errors.mean().item():.8f}", flush=True)
training, _, _ = evaluate(model, tensors["train_x"], tensors["train_y"],
normalizer, train["batch_size"], device)
testing, _, _ = evaluate(model, tensors["test_x"], tensors["test_y"],
normalizer, train["batch_size"], device)
score = training["squared_relative_l2_mean"]
improved = score < best_score
if improved:
best_score, best_epoch = score, epoch
scheduler.step()
row = {"epoch": epoch, "lr": learning_rate, "online_train_loss": total / count,
"train_relative_l2": training["relative_l2_mean"], "train_squared_relative_l2": score,
"test_relative_l2": testing["relative_l2_mean"],
"test_squared_relative_l2": testing["squared_relative_l2_mean"],
"seconds": time.perf_counter() - epoch_start, "selected": improved}
history.append(row)
with csv_path.open("a", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=fields)
if epoch == 1:
writer.writeheader()
writer.writerow(row)
checkpoint = {"model_state_dict": model.state_dict(), "config": config,
"epoch": epoch, "completed_epochs_at_save": epoch,
"normalizer": normalizer, "best_score": best_score, "best_epoch": best_epoch,
"selection_rule": train["checkpoint_selection"], "train_metrics": training,
"test_metrics": testing, "optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(), "history": history,
"torch_rng_state": torch.get_rng_state(), "cuda_rng_state": torch.cuda.get_rng_state_all(),
"data_sha256": runtime["data_sha256"]}
if improved:
save_checkpoint(output / config["output"]["checkpoint"], checkpoint)
save_checkpoint(output / config["output"]["last_checkpoint"], checkpoint)
atomic_json(results / "training_status.json", {"status": "running" if epoch < 500 else "completed",
"completed_epochs": epoch, "best_epoch": best_epoch, "best_training_loss": best_score,
"last_epoch": row, "wall_seconds": time.perf_counter() - start_time})
print(f"EPOCH {epoch:03d}/500 lr={learning_rate:.7g} train_loss={score:.9f} "
f"train_rel_l2={training['relative_l2_mean']:.8f} "
f"test_rel_l2={testing['relative_l2_mean']:.8f} "
f"test_squared_rel_l2={testing['squared_relative_l2_mean']:.9f} "
f"best_epoch={best_epoch} seconds={row['seconds']:.2f}", flush=True)
print(f"TRAINING COMPLETE: completed_epochs=500 selected_epoch={best_epoch}", flush=True)
if __name__ == "__main__":
main()