Download scripts/common.py from OneScience-Group/D-SENO: direct link, hf CLI and curl.
- Browser
- Download file 6.26 kB
-
https://huggingface.co/OneScience-Group/D-SENO/resolve/main/scripts/common.py
- Command line
-
hf download hf://OneScience-Group/D-SENO/scripts/common.py
-
curl -L -o common.py https://huggingface.co/OneScience-Group/D-SENO/resolve/main/scripts/common.py
6.26 kB
| """Shared I/O, pointwise normalization and metrics for the Pipe experiment.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import random | |
| import sys | |
| import tempfile | |
| import numpy as np | |
| import torch | |
| import yaml | |
| ROOT = Path(__file__).resolve().parents[1] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| def load_config(path) -> dict: | |
| with Path(path).open(encoding="utf-8") as handle: | |
| config = yaml.safe_load(handle) | |
| if not isinstance(config, dict): | |
| raise ValueError("Configuration must be a mapping") | |
| required = {"data", "model", "training", "runtime", "output", "provenance"} | |
| missing = required.difference(config) | |
| if missing: | |
| raise ValueError(f"Missing configuration sections: {sorted(missing)}") | |
| return config | |
| def read_pipe(config): | |
| """Read original CPU float32 X/Y and Q0 in fixed sample order, without downloads.""" | |
| data = config["data"] | |
| directory = Path(data["directory"]) | |
| ntrain, ntest = int(data["ntrain"]), int(data["ntest"]) | |
| count = ntrain + ntest | |
| if ntrain != 1000 or ntest != 200 or int(data["target_channel"]) != 0: | |
| raise ValueError("The formal Pipe experiment requires train1000/test200 and Q0") | |
| mesh = tuple(data["mesh"]) | |
| if mesh != (129, 129): | |
| raise ValueError("The formal Pipe mesh must remain 129 by 129") | |
| x = np.load(directory / "Pipe_X.npy", mmap_mode="r", allow_pickle=False) | |
| y = np.load(directory / "Pipe_Y.npy", mmap_mode="r", allow_pickle=False) | |
| q = np.load(directory / "Pipe_Q.npy", mmap_mode="r", allow_pickle=False) | |
| if x.shape != (2310, *mesh) or y.shape != x.shape: | |
| raise ValueError(f"Unexpected coordinate shapes: X={x.shape}, Y={y.shape}") | |
| if q.shape != (2310, 3, *mesh): | |
| raise ValueError(f"Unexpected Pipe_Q shape: {q.shape}") | |
| coordinates = np.empty((count, *mesh, 2), dtype=np.float32) | |
| coordinates[..., 0] = x[:count] | |
| coordinates[..., 1] = y[:count] | |
| target = np.array(q[:count, 0], dtype=np.float32, copy=True) | |
| if not np.isfinite(coordinates).all() or not np.isfinite(target).all(): | |
| raise ValueError("Pipe coordinates and Q0 must be finite") | |
| return torch.from_numpy(coordinates), torch.from_numpy(target) | |
| class Normalizer: | |
| """Train-only pointwise statistics along the sample dimension.""" | |
| def __init__(self, mean, std, eps=1e-5): | |
| self.mean = mean | |
| self.std = std | |
| self.eps = float(eps) | |
| def fit(cls, x, eps=1e-5): | |
| if x.shape[0] < 2: | |
| raise ValueError("Unbiased normalization needs at least two samples") | |
| return cls(x.mean(dim=0), x.std(dim=0, unbiased=True), eps) | |
| def encode(self, x): | |
| return (x - self.mean) / (self.std + self.eps) | |
| def decode(self, x): | |
| return x * (self.std + self.eps) + self.mean | |
| def to(self, device): | |
| self.mean = self.mean.to(device) | |
| self.std = self.std.to(device) | |
| return self | |
| def state_dict(self): | |
| return {"mean": self.mean.detach().cpu(), | |
| "std": self.std.detach().cpu(), "eps": self.eps} | |
| def from_state_dict(cls, state, device="cpu"): | |
| return cls(torch.as_tensor(state["mean"]).to(device), | |
| torch.as_tensor(state["std"]).to(device), state["eps"]) | |
| def relative_l2(pred, target, reduction="mean"): | |
| """Per-sample spatial norm ratio, in the same physical units as target.""" | |
| if pred.shape != target.shape or pred.ndim < 2: | |
| raise ValueError("Prediction and target need matching batch/spatial dimensions") | |
| denominator = torch.linalg.vector_norm(target.reshape(target.shape[0], -1), dim=1) | |
| if torch.any(denominator == 0): | |
| raise ValueError("Relative L2 is undefined for a zero-norm target") | |
| numerator = torch.linalg.vector_norm((pred-target).reshape(target.shape[0], -1), dim=1) | |
| ratios = numerator / denominator | |
| if reduction == "none": | |
| return ratios | |
| if reduction == "sum": | |
| return ratios.sum() | |
| if reduction == "mean": | |
| return ratios.mean() | |
| raise ValueError("Reduction must be mean, sum or none") | |
| def seed_all(seed, deterministic=False): | |
| random.seed(seed) | |
| np.random.seed(seed) | |
| torch.manual_seed(seed) | |
| if torch.cuda.is_available(): | |
| torch.cuda.manual_seed_all(seed) | |
| torch.backends.cudnn.benchmark = False | |
| torch.backends.cudnn.deterministic = deterministic | |
| def require_compute_node(): | |
| if not torch.cuda.is_available(): | |
| raise RuntimeError("An allocated accelerator is required") | |
| def atomic_json(path, payload): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = None | |
| try: | |
| with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, | |
| prefix=path.name + ".", suffix=".tmp", | |
| delete=False) as handle: | |
| temporary = Path(handle.name) | |
| json.dump(payload, handle, indent=2, ensure_ascii=False, allow_nan=False) | |
| handle.write("\n") | |
| handle.flush() | |
| os.fsync(handle.fileno()) | |
| os.replace(temporary, path) | |
| finally: | |
| if temporary is not None and temporary.exists(): | |
| temporary.unlink() | |
| def load_checkpoint(path, device="cpu") -> dict: | |
| try: | |
| checkpoint = torch.load(path, map_location=device, weights_only=False) | |
| except TypeError: | |
| checkpoint = torch.load(path, map_location=device) | |
| if not isinstance(checkpoint, dict): | |
| raise ValueError("Expected a checkpoint dictionary") | |
| return checkpoint | |
| def source_hashes(root): | |
| root = Path(root) | |
| hashes = {} | |
| for directory in ("model", "scripts", "config"): | |
| for path in sorted((root / directory).rglob("*")): | |
| if path.is_file() and path.suffix in {".py", ".yaml", ".yml", ".sh"}: | |
| hashes[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest() | |
| return hashes | |
| def build_model(config): | |
| from model.DSENO import DSENO | |
| model = DSENO(**config["model"]) | |
| model.memory_format = getattr(torch, config["runtime"].get("memory_format", "channels_last")) | |
| return model | |