File size: 6,198 Bytes
10ab127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
"""Configuration, original-data loading and explicit pressure-scale transforms."""

import argparse
import hashlib
import json
import os
from pathlib import Path
import random
import sys

import numpy as np
import torch
import yaml

PACKAGE_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PACKAGE_ROOT))
from model.SirenFNO import SirenFNO


def arguments(description):
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument("--config", type=Path, default=PACKAGE_ROOT / "config/config.yaml")
    return parser


def load_config(path):
    with open(path) as stream:
        config = yaml.safe_load(stream)
    assert config["training"]["epochs"] == config["training"]["t_max"] == 500
    assert config["training"]["batch_size"] == 32
    assert config["training"]["early_stopping"] is False
    assert config["model"]["width"] == 32 and config["model"]["layers"] == 4
    assert config["model"]["lifting_channels"] == config["model"]["projection_channels"] == 64
    assert not config["model"]["truncation"] and config["model"]["factorization"] == "dense"
    assert config["data"]["n_train"] == 1000 and config["data"]["n_test"] == 200
    assert config["data"]["resolution"] == [32, 32]
    return config


def configure_runtime(config):
    torch.set_num_threads(config["runtime"]["cpu_threads"])
    device = torch.device(config["runtime"]["device"])
    if device.type != "cuda" or not torch.cuda.is_available():
        raise RuntimeError("Allocated accelerator is required; CPU fallback is disabled")
    seed = config["training"]["seed"]
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True
    print(f"Slurm job={os.environ.get('SLURM_JOB_ID', 'none')} node={os.uname().nodename} "
          f"torch={torch.__version__} HIP={torch.version.hip} "
          f"accelerator={torch.cuda.get_device_name(device)} seed={seed}", flush=True)
    return device


def sha256(path):
    digest = hashlib.sha256()
    with open(path, "rb") as stream:
        for block in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def atomic_json(path, content):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(path.name + ".tmp")
    temporary.write_text(json.dumps(content, indent=2, ensure_ascii=False, allow_nan=False) + "\n")
    temporary.replace(path)


def load_data(config, record_path=None):
    settings = config["data"]
    tensors, manifest = {}, {"selection": "first_n", "no_download": True, "files": {}}
    for split, number in [("train", settings["n_train"]), ("test", settings["n_test"])]:
        path = Path(settings["root"]) / settings[f"{split}_file"]
        raw = torch.load(path, map_location="cpu", weights_only=False)
        info = {"path": str(path), "sha256": sha256(path), "selected_indices": [0, number - 1]}
        for key in ("x", "y"):
            value = raw[key]
            if value.ndim != 3 or list(value.shape[-2:]) != settings["resolution"]:
                raise ValueError(f"Unexpected original shape {path}:{key} {value.shape}")
            if value.shape[0] < number or value.dtype != torch.float32:
                raise ValueError(f"Unexpected original sample count or dtype: {path}:{key}")
            selected = value[:number].unsqueeze(1).contiguous()
            if not torch.isfinite(selected).all():
                raise ValueError(f"Nonfinite data {path}:{key}")
            tensors[f"{split}_{key}"] = selected
            info[key] = {"raw_shape": list(value.shape), "raw_dtype": str(value.dtype),
                         "selected_shape": list(selected.shape), "min": selected.min().item(),
                         "max": selected.max().item(), "meaning": "permeability" if key == "x" else "pressure"}
        manifest["files"][split] = info
        print(f"DATA {split}: {json.dumps(info)}", flush=True)
    if record_path:
        atomic_json(record_path, manifest)
    return tensors, manifest


def normalizer_from_train(config, y):
    if config["data"]["normalize_output"]:
        mean = y.mean().item()
        scale = y.std(correction=1).item() + config["data"]["normalization_epsilon"]
    else:
        mean, scale = 0.0, 1.0
    return {"mean": mean, "scale": scale, "fit_on": "train[0:1000] only", "input_normalized": False}


def decode(prediction, normalizer):
    return prediction * normalizer["scale"] + normalizer["mean"]


def relative_l2(prediction, target):
    difference = (prediction - target).flatten(1)
    denominator = torch.linalg.vector_norm(target.flatten(1), dim=1)
    if torch.any(denominator <= 0):
        raise ValueError("Relative L2 is undefined for zero-norm pressure sample")
    return torch.linalg.vector_norm(difference, dim=1) / denominator


@torch.inference_mode()
def evaluate(model, x, y, normalizer, batch_size, device, return_predictions=False):
    model.eval()
    errors, predictions = [], []
    for start in range(0, len(x), batch_size):
        prediction = decode(model(x[start:start + batch_size].to(device)), normalizer)
        target = y[start:start + batch_size].to(device)
        errors.append(relative_l2(prediction, target).cpu())
        if return_predictions:
            predictions.append(prediction.cpu())
    errors = torch.cat(errors)
    if not torch.isfinite(errors).all():
        raise FloatingPointError("Nonfinite evaluation errors")
    result = {"relative_l2_mean": errors.double().mean().item(),
              "squared_relative_l2_mean": errors.double().square().mean().item(),
              "relative_l2_std": errors.double().std(correction=0).item()}
    return result, errors, torch.cat(predictions) if return_predictions else None


def build_model(config, device):
    return SirenFNO(**config["model"]).to(device)


def save_checkpoint(path, checkpoint):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(path.name + ".tmp")
    torch.save(checkpoint, temporary)
    temporary.replace(path)