Download scripts/train.py from OneScience-Group/SirenFNO: direct link, hf CLI and curl.
- Browser
- Download file 8.12 kB
-
https://huggingface.co/OneScience-Group/SirenFNO/resolve/main/scripts/train.py
- Command line
-
hf download hf://OneScience-Group/SirenFNO/scripts/train.py
-
curl -L -o train.py https://huggingface.co/OneScience-Group/SirenFNO/resolve/main/scripts/train.py
8.12 kB
| """Full 500-epoch paper-priority Darcy experiment; see reproduction_spec.md.""" | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import time | |
| import torch | |
| from torch.utils.data import DataLoader, TensorDataset | |
| from common import (arguments, atomic_json, build_model, configure_runtime, decode, | |
| evaluate, load_config, load_data, normalizer_from_train, | |
| relative_l2, save_checkpoint) | |
| def main(): | |
| parser = arguments(__doc__) | |
| parser.add_argument("--resume", type=Path, help="Resume model, optimizer, scheduler and RNG from last checkpoint") | |
| args = parser.parse_args() | |
| config = load_config(args.config) | |
| device = configure_runtime(config) | |
| output = Path(config["output"]["root"]) | |
| results = output / config["output"]["results"] | |
| results.mkdir(parents=True, exist_ok=True) | |
| smoke_path = results / "smoke/checks.json" | |
| if not smoke_path.exists() or json.loads(smoke_path.read_text())["status"] != "passed": | |
| raise RuntimeError("All six compute-node smoke checks must pass before full training") | |
| tensors, manifest = load_data(config, results / "data_manifest.json") | |
| normalizer = normalizer_from_train(config, tensors["train_y"]) | |
| atomic_json(results / "normalization.json", normalizer) | |
| model = build_model(config, device) | |
| train = config["training"] | |
| loader = DataLoader(TensorDataset(tensors["train_x"], tensors["train_y"]), | |
| batch_size=train["batch_size"], shuffle=train["shuffle"], | |
| drop_last=train["drop_last"], num_workers=0) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=train["lr"], | |
| weight_decay=train["weight_decay"], | |
| betas=tuple(train["betas"]), eps=train["optimizer_epsilon"]) | |
| scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( | |
| optimizer, T_max=train["t_max"], eta_min=train["eta_min"]) | |
| first_epoch, best_score, best_epoch, history = 1, float("inf"), None, [] | |
| if args.resume: | |
| restored = torch.load(args.resume, map_location="cpu", weights_only=False) | |
| if restored["config"] != config: | |
| raise ValueError("Resume configuration differs from checkpoint") | |
| model.load_state_dict(restored["model_state_dict"], strict=True) | |
| optimizer.load_state_dict(restored["optimizer_state_dict"]) | |
| scheduler.load_state_dict(restored["scheduler_state_dict"]) | |
| first_epoch = restored["epoch"] + 1 | |
| best_score, best_epoch = restored["best_score"], restored["best_epoch"] | |
| history = restored["history"] | |
| torch.set_rng_state(restored["torch_rng_state"]) | |
| torch.cuda.set_rng_state_all(restored["cuda_rng_state"]) | |
| else: | |
| for name in ("checkpoint", "last_checkpoint"): | |
| if (output / config["output"][name]).exists(): | |
| raise FileExistsError("Existing experiment weights: use --resume or a separate output directory") | |
| parameters = sum(p.numel() for p in model.parameters()) | |
| print(f"MODEL parameters={parameters}, dense rFFT kernel grid=32x17, full 500 epochs", flush=True) | |
| print("LOSS: original-scale mean squared sample-relative L2, paper eq. (15). " | |
| "Checkpoint: minimum full-training-set loss; test set never selects weights.", flush=True) | |
| start_time = time.perf_counter() | |
| runtime = {"parameters": parameters, "torch": torch.__version__, "hip": torch.version.hip, | |
| "device": torch.cuda.get_device_name(device), "selection": train["checkpoint_selection"], | |
| "seed": train["seed"], "data_sha256": {k: v["sha256"] for k, v in manifest["files"].items()}} | |
| atomic_json(results / "run_metadata.json", runtime) | |
| fields = ["epoch", "lr", "online_train_loss", "train_relative_l2", "train_squared_relative_l2", | |
| "test_relative_l2", "test_squared_relative_l2", "seconds", "selected"] | |
| csv_path = results / "training_history.csv" | |
| if history: | |
| with csv_path.open("w", newline="") as stream: | |
| writer = csv.DictWriter(stream, fieldnames=fields) | |
| writer.writeheader() | |
| writer.writerows(history) | |
| for epoch in range(first_epoch, train["epochs"] + 1): | |
| epoch_start = time.perf_counter() | |
| model.train() | |
| learning_rate = optimizer.param_groups[0]["lr"] | |
| total, count = 0.0, 0 | |
| for batch_index, (x, y) in enumerate(loader): | |
| x, y = x.to(device), y.to(device) | |
| optimizer.zero_grad(set_to_none=True) | |
| prediction = decode(model(x), normalizer) | |
| sample_errors = relative_l2(prediction, y) | |
| loss = sample_errors.square().mean() | |
| if not torch.isfinite(loss): | |
| raise FloatingPointError(f"Nonfinite loss at epoch {epoch} batch {batch_index}") | |
| loss.backward() | |
| optimizer.step() | |
| value = loss.item() | |
| total += value * len(x) | |
| count += len(x) | |
| if batch_index % train["print_batch_interval"] == 0 or batch_index == len(loader) - 1: | |
| print(f"Epoch {epoch:03d}/500 batch {batch_index + 1:02d}/{len(loader)} " | |
| f"loss={value:.8f} sample_rel_l2={sample_errors.mean().item():.8f}", flush=True) | |
| training, _, _ = evaluate(model, tensors["train_x"], tensors["train_y"], | |
| normalizer, train["batch_size"], device) | |
| testing, _, _ = evaluate(model, tensors["test_x"], tensors["test_y"], | |
| normalizer, train["batch_size"], device) | |
| score = training["squared_relative_l2_mean"] | |
| improved = score < best_score | |
| if improved: | |
| best_score, best_epoch = score, epoch | |
| scheduler.step() | |
| row = {"epoch": epoch, "lr": learning_rate, "online_train_loss": total / count, | |
| "train_relative_l2": training["relative_l2_mean"], "train_squared_relative_l2": score, | |
| "test_relative_l2": testing["relative_l2_mean"], | |
| "test_squared_relative_l2": testing["squared_relative_l2_mean"], | |
| "seconds": time.perf_counter() - epoch_start, "selected": improved} | |
| history.append(row) | |
| with csv_path.open("a", newline="") as stream: | |
| writer = csv.DictWriter(stream, fieldnames=fields) | |
| if epoch == 1: | |
| writer.writeheader() | |
| writer.writerow(row) | |
| checkpoint = {"model_state_dict": model.state_dict(), "config": config, | |
| "epoch": epoch, "completed_epochs_at_save": epoch, | |
| "normalizer": normalizer, "best_score": best_score, "best_epoch": best_epoch, | |
| "selection_rule": train["checkpoint_selection"], "train_metrics": training, | |
| "test_metrics": testing, "optimizer_state_dict": optimizer.state_dict(), | |
| "scheduler_state_dict": scheduler.state_dict(), "history": history, | |
| "torch_rng_state": torch.get_rng_state(), "cuda_rng_state": torch.cuda.get_rng_state_all(), | |
| "data_sha256": runtime["data_sha256"]} | |
| if improved: | |
| save_checkpoint(output / config["output"]["checkpoint"], checkpoint) | |
| save_checkpoint(output / config["output"]["last_checkpoint"], checkpoint) | |
| atomic_json(results / "training_status.json", {"status": "running" if epoch < 500 else "completed", | |
| "completed_epochs": epoch, "best_epoch": best_epoch, "best_training_loss": best_score, | |
| "last_epoch": row, "wall_seconds": time.perf_counter() - start_time}) | |
| print(f"EPOCH {epoch:03d}/500 lr={learning_rate:.7g} train_loss={score:.9f} " | |
| f"train_rel_l2={training['relative_l2_mean']:.8f} " | |
| f"test_rel_l2={testing['relative_l2_mean']:.8f} " | |
| f"test_squared_rel_l2={testing['squared_relative_l2_mean']:.9f} " | |
| f"best_epoch={best_epoch} seconds={row['seconds']:.2f}", flush=True) | |
| print(f"TRAINING COMPLETE: completed_epochs=500 selected_epoch={best_epoch}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |