File size: 3,183 Bytes
10ab127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""Six compute-node checks using only actual supplied Darcy samples."""

from pathlib import Path
import torch
from torch.utils.data import DataLoader, TensorDataset
from common import (arguments, atomic_json, build_model, configure_runtime, decode,
                    evaluate, load_config, load_data, normalizer_from_train, relative_l2)


def main():
    args = arguments(__doc__).parse_args()
    config = load_config(args.config)
    device = configure_runtime(config)
    data, _ = load_data(config)
    normalizer = normalizer_from_train(config, data["train_y"])
    model = build_model(config, device)
    checks = {}
    x, y = data["train_x"][:32].to(device), data["train_y"][:32].to(device)
    prediction = decode(model(x), normalizer)
    assert prediction.shape == y.shape and torch.isfinite(prediction).all()
    checks["forward_pass"] = "passed"
    loss = relative_l2(prediction, y).square().mean()
    loss.backward()
    assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters() if p.requires_grad)
    checks["backward_pass"] = "passed"
    optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.0001)
    losses = []
    for bx, by in DataLoader(TensorDataset(data["train_x"], data["train_y"]), batch_size=32):
        optimizer.zero_grad(set_to_none=True)
        prediction = decode(model(bx.to(device)), normalizer)
        batch_loss = relative_l2(prediction, by.to(device)).square().mean()
        assert torch.isfinite(batch_loss)
        batch_loss.backward()
        optimizer.step()
        losses.append(batch_loss.item())
    assert sum(losses[-8:]) / 8 < sum(losses[:8]) / 8
    checks["train_loop_dry_run"] = "passed"
    metrics, errors, pred = evaluate(model, data["test_x"][:32], data["test_y"][:32],
                                     normalizer, 32, device, return_predictions=True)
    assert torch.isfinite(errors).all()
    checks["validation_loop_dry_run"] = "passed"
    # Independent manual metric computation on real fields; no synthetic data.
    manual = ((pred.double() - data["test_y"][:32].double()).flatten(1).square().sum(1).sqrt()
              / data["test_y"][:32].double().flatten(1).square().sum(1).sqrt())
    assert torch.allclose(errors.double(), manual, rtol=1e-5, atol=1e-8)
    checks["domain_metric_dry_run"] = "passed"
    assert len(model.blocks) == 4 and model.projection.layers[0].out_channels == 64
    for block in model.blocks:
        assert tuple(block.spectral.coordinates.shape) == (32, 17, 2)
    checks["config_consistency"] = "passed"
    path = Path(config["output"]["root"]) / config["output"]["results"] / "smoke/checks.json"
    atomic_json(path, {"status": "passed", "attempts": 1, "checks": checks,
                      "used_original_data": True, "formal_epochs_consumed": 0,
                      "loss_first8_mean": sum(losses[:8]) / 8, "loss_last8_mean": sum(losses[-8:]) / 8,
                      "validation_metric": metrics, "peak_memory_bytes": torch.cuda.max_memory_allocated()})
    print("SMOKE: all six checks passed; discarded smoke model; formal run starts from fresh seed", flush=True)


if __name__ == "__main__":
    main()