File size: 3,183 Bytes
10ab127 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Six compute-node checks using only actual supplied Darcy samples."""
from pathlib import Path
import torch
from torch.utils.data import DataLoader, TensorDataset
from common import (arguments, atomic_json, build_model, configure_runtime, decode,
evaluate, load_config, load_data, normalizer_from_train, relative_l2)
def main():
args = arguments(__doc__).parse_args()
config = load_config(args.config)
device = configure_runtime(config)
data, _ = load_data(config)
normalizer = normalizer_from_train(config, data["train_y"])
model = build_model(config, device)
checks = {}
x, y = data["train_x"][:32].to(device), data["train_y"][:32].to(device)
prediction = decode(model(x), normalizer)
assert prediction.shape == y.shape and torch.isfinite(prediction).all()
checks["forward_pass"] = "passed"
loss = relative_l2(prediction, y).square().mean()
loss.backward()
assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters() if p.requires_grad)
checks["backward_pass"] = "passed"
optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.0001)
losses = []
for bx, by in DataLoader(TensorDataset(data["train_x"], data["train_y"]), batch_size=32):
optimizer.zero_grad(set_to_none=True)
prediction = decode(model(bx.to(device)), normalizer)
batch_loss = relative_l2(prediction, by.to(device)).square().mean()
assert torch.isfinite(batch_loss)
batch_loss.backward()
optimizer.step()
losses.append(batch_loss.item())
assert sum(losses[-8:]) / 8 < sum(losses[:8]) / 8
checks["train_loop_dry_run"] = "passed"
metrics, errors, pred = evaluate(model, data["test_x"][:32], data["test_y"][:32],
normalizer, 32, device, return_predictions=True)
assert torch.isfinite(errors).all()
checks["validation_loop_dry_run"] = "passed"
# Independent manual metric computation on real fields; no synthetic data.
manual = ((pred.double() - data["test_y"][:32].double()).flatten(1).square().sum(1).sqrt()
/ data["test_y"][:32].double().flatten(1).square().sum(1).sqrt())
assert torch.allclose(errors.double(), manual, rtol=1e-5, atol=1e-8)
checks["domain_metric_dry_run"] = "passed"
assert len(model.blocks) == 4 and model.projection.layers[0].out_channels == 64
for block in model.blocks:
assert tuple(block.spectral.coordinates.shape) == (32, 17, 2)
checks["config_consistency"] = "passed"
path = Path(config["output"]["root"]) / config["output"]["results"] / "smoke/checks.json"
atomic_json(path, {"status": "passed", "attempts": 1, "checks": checks,
"used_original_data": True, "formal_epochs_consumed": 0,
"loss_first8_mean": sum(losses[:8]) / 8, "loss_last8_mean": sum(losses[-8:]) / 8,
"validation_metric": metrics, "peak_memory_bytes": torch.cuda.max_memory_allocated()})
print("SMOKE: all six checks passed; discarded smoke model; formal run starts from fresh seed", flush=True)
if __name__ == "__main__":
main()
|