Upload Surya model package
Browse files- .gitattributes +5 -27
- conf/config.yaml +45 -0
- config.json +88 -0
- configuration.json +12 -0
- model/surya.py +144 -0
- scripts/fake_data.py +52 -0
- scripts/inference.py +42 -0
- scripts/result.py +63 -0
- scripts/train.py +71 -0
- weight/.gitkeep +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,13 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
|
| 27 |
-
*.
|
|
|
|
|
|
|
|
|
|
| 28 |
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
| 1 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 5 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 6 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 12 |
*.tar filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
conf/config.yaml
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed: 42
|
| 2 |
+
data:
|
| 3 |
+
root: data
|
| 4 |
+
protocol: synthetic_SDO_AIA_HMI
|
| 5 |
+
image_size: 32
|
| 6 |
+
channels: 13
|
| 7 |
+
input_steps: 2
|
| 8 |
+
forecast_steps: 4
|
| 9 |
+
train_samples: 32
|
| 10 |
+
test_samples: 8
|
| 11 |
+
channel_names: [AIA_94, AIA_131, AIA_171, AIA_193, AIA_211, AIA_304, AIA_335, AIA_1600, HMI_magnetogram, HMI_continuum, HMI_doppler, HMI_vector_x, HMI_vector_y]
|
| 12 |
+
channel_mean: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
|
| 13 |
+
channel_std: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
|
| 14 |
+
model:
|
| 15 |
+
image_size: 32
|
| 16 |
+
patch_size: 4
|
| 17 |
+
channels: 13
|
| 18 |
+
input_steps: 2
|
| 19 |
+
embed_dim: 64
|
| 20 |
+
depth: 4
|
| 21 |
+
spectral_blocks: 1
|
| 22 |
+
num_heads: 4
|
| 23 |
+
window_size: 3
|
| 24 |
+
global_tokens: 4
|
| 25 |
+
training:
|
| 26 |
+
epochs: 2
|
| 27 |
+
one_step_epochs: 1
|
| 28 |
+
rollout_epochs: 1
|
| 29 |
+
batch_size: 4
|
| 30 |
+
learning_rate: 0.0005
|
| 31 |
+
weight_decay: 0.01
|
| 32 |
+
num_workers: 0
|
| 33 |
+
accum_iter: 1
|
| 34 |
+
warmup_epochs: 1
|
| 35 |
+
min_learning_rate: 0.00001
|
| 36 |
+
amp: true
|
| 37 |
+
grad_clip: 1.0
|
| 38 |
+
runtime:
|
| 39 |
+
device: auto
|
| 40 |
+
amp: true
|
| 41 |
+
paths:
|
| 42 |
+
checkpoint: result/checkpoints/surya.pt
|
| 43 |
+
training_metrics: result/training/metrics.json
|
| 44 |
+
inference_dir: result/output
|
| 45 |
+
evaluation_dir: result/evaluation
|
config.json
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_name": "Surya",
|
| 3 |
+
"model_type": "surya",
|
| 4 |
+
"architectures": [
|
| 5 |
+
"Surya"
|
| 6 |
+
],
|
| 7 |
+
"framework": "PyTorch",
|
| 8 |
+
"domain": "heliophysics",
|
| 9 |
+
"task": "solar-dynamics-forecasting",
|
| 10 |
+
"implementation": {
|
| 11 |
+
"entry_point": "model/surya.py",
|
| 12 |
+
"scope": "spectral-gated and long-short-attention autoregressive forecasting of aligned SDO AIA/HMI observations",
|
| 13 |
+
"train_script": "scripts/train.py",
|
| 14 |
+
"inference_script": "scripts/inference.py",
|
| 15 |
+
"evaluation_script": "scripts/result.py",
|
| 16 |
+
"synthetic_data_script": "scripts/fake_data.py"
|
| 17 |
+
},
|
| 18 |
+
"architecture": {
|
| 19 |
+
"family": "spectral-gated spatiotemporal transformer",
|
| 20 |
+
"image_size": 32,
|
| 21 |
+
"patch_size": 4,
|
| 22 |
+
"input_channels": 13,
|
| 23 |
+
"input_steps": 2,
|
| 24 |
+
"forecast_steps": 4,
|
| 25 |
+
"embed_dim": 64,
|
| 26 |
+
"depth": 4,
|
| 27 |
+
"spectral_blocks": 1,
|
| 28 |
+
"num_heads": 4,
|
| 29 |
+
"window_size": 3,
|
| 30 |
+
"global_tokens": 4,
|
| 31 |
+
"prediction_mode": "autoregressive"
|
| 32 |
+
},
|
| 33 |
+
"data": {
|
| 34 |
+
"datasets": [
|
| 35 |
+
"SDO/AIA",
|
| 36 |
+
"SDO/HMI"
|
| 37 |
+
],
|
| 38 |
+
"protocol": "synthetic_SDO_AIA_HMI",
|
| 39 |
+
"format": "NPZ",
|
| 40 |
+
"input_key": "inputs",
|
| 41 |
+
"input_shape": [
|
| 42 |
+
"N",
|
| 43 |
+
2,
|
| 44 |
+
13,
|
| 45 |
+
"H",
|
| 46 |
+
"W"
|
| 47 |
+
],
|
| 48 |
+
"target_key": "targets",
|
| 49 |
+
"target_shape": [
|
| 50 |
+
"N",
|
| 51 |
+
"S",
|
| 52 |
+
13,
|
| 53 |
+
"H",
|
| 54 |
+
"W"
|
| 55 |
+
],
|
| 56 |
+
"activity_key": "activity",
|
| 57 |
+
"activity_shape": [
|
| 58 |
+
"N",
|
| 59 |
+
"S"
|
| 60 |
+
],
|
| 61 |
+
"channel_names": [
|
| 62 |
+
"AIA_94",
|
| 63 |
+
"AIA_131",
|
| 64 |
+
"AIA_171",
|
| 65 |
+
"AIA_193",
|
| 66 |
+
"AIA_211",
|
| 67 |
+
"AIA_304",
|
| 68 |
+
"AIA_335",
|
| 69 |
+
"AIA_1600",
|
| 70 |
+
"HMI_magnetogram",
|
| 71 |
+
"HMI_continuum",
|
| 72 |
+
"HMI_doppler",
|
| 73 |
+
"HMI_vector_x",
|
| 74 |
+
"HMI_vector_y"
|
| 75 |
+
],
|
| 76 |
+
"normalization": "signum-log followed by configured per-channel affine normalization",
|
| 77 |
+
"default_train_file": "data/train.npz",
|
| 78 |
+
"default_test_file": "data/test.npz"
|
| 79 |
+
},
|
| 80 |
+
"configuration_sources": [
|
| 81 |
+
"conf/config.yaml",
|
| 82 |
+
"model/surya.py",
|
| 83 |
+
"scripts/fake_data.py",
|
| 84 |
+
"scripts/train.py",
|
| 85 |
+
"scripts/inference.py",
|
| 86 |
+
"scripts/result.py"
|
| 87 |
+
]
|
| 88 |
+
}
|
configuration.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"framework": "PyTorch",
|
| 3 |
+
"task": "solar_dynamics_forecasting",
|
| 4 |
+
"model": "Surya",
|
| 5 |
+
"input_format": "BTCHW",
|
| 6 |
+
"protocol": "autoregressive_sdo_forecasting",
|
| 7 |
+
"default_config": "conf/config.yaml",
|
| 8 |
+
"train": "scripts/train.py",
|
| 9 |
+
"inference": "scripts/inference.py",
|
| 10 |
+
"evaluation": "scripts/result.py",
|
| 11 |
+
"visualization": "scripts/result.py"
|
| 12 |
+
}
|
model/surya.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Self-contained, paper-aligned Surya forecasting model."""
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
from torch.nn import functional as F
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
CHANNEL_NAMES = ("AIA_94", "AIA_131", "AIA_171", "AIA_193", "AIA_211", "AIA_304",
|
| 9 |
+
"AIA_335", "AIA_1600", "HMI_magnetogram", "HMI_continuum",
|
| 10 |
+
"HMI_doppler", "HMI_vector_x", "HMI_vector_y")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def signum_log(x):
|
| 14 |
+
"""Compress signed solar products while retaining their sign."""
|
| 15 |
+
return torch.sign(x) * torch.log1p(torch.abs(x))
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class SpectralGating(nn.Module):
|
| 19 |
+
def __init__(self, grid, dim):
|
| 20 |
+
super().__init__()
|
| 21 |
+
self.grid = grid
|
| 22 |
+
self.weight = nn.Parameter(torch.randn(grid, grid // 2 + 1, dim, 2) * 0.02)
|
| 23 |
+
self.norm = nn.LayerNorm(dim)
|
| 24 |
+
self.mlp = nn.Sequential(nn.Linear(dim, dim * 2), nn.GELU(), nn.Linear(dim * 2, dim))
|
| 25 |
+
|
| 26 |
+
def forward(self, tokens):
|
| 27 |
+
residual = tokens
|
| 28 |
+
values = self.norm(tokens).reshape(tokens.shape[0], self.grid, self.grid, -1).float()
|
| 29 |
+
spectrum = torch.fft.rfft2(values, dim=(1, 2), norm="ortho")
|
| 30 |
+
spectrum = spectrum * torch.view_as_complex(self.weight.float())
|
| 31 |
+
values = torch.fft.irfft2(spectrum, s=(self.grid, self.grid), dim=(1, 2), norm="ortho")
|
| 32 |
+
return residual + self.mlp(values.reshape_as(tokens).to(tokens.dtype))
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class LongShortAttention(nn.Module):
|
| 36 |
+
def __init__(self, grid, dim, heads, window, global_tokens):
|
| 37 |
+
super().__init__()
|
| 38 |
+
self.grid, self.window = grid, window
|
| 39 |
+
self.norm = nn.LayerNorm(dim)
|
| 40 |
+
self.local = nn.MultiheadAttention(dim, heads, batch_first=True)
|
| 41 |
+
self.global_attn = nn.MultiheadAttention(dim, heads, batch_first=True)
|
| 42 |
+
self.projection = nn.Linear(dim * 2, dim)
|
| 43 |
+
self.score = nn.Linear(dim, global_tokens)
|
| 44 |
+
self.global_norm = nn.LayerNorm(dim)
|
| 45 |
+
self.output_norm = nn.LayerNorm(dim)
|
| 46 |
+
self.mlp_norm = nn.LayerNorm(dim)
|
| 47 |
+
self.mlp = nn.Sequential(nn.Linear(dim, dim * 4), nn.GELU(), nn.Linear(dim * 4, dim))
|
| 48 |
+
|
| 49 |
+
def forward(self, tokens):
|
| 50 |
+
residual = tokens
|
| 51 |
+
values = self.norm(tokens)
|
| 52 |
+
b, _, d = values.shape
|
| 53 |
+
grid = values.reshape(b, self.grid, self.grid, d)
|
| 54 |
+
w = self.window
|
| 55 |
+
# Unfold with padding creates overlapping neighborhoods at every token.
|
| 56 |
+
padded = F.pad(grid.permute(0, 3, 1, 2), (w // 2, w // 2, w // 2, w // 2), mode="replicate")
|
| 57 |
+
neighborhoods = F.unfold(padded, kernel_size=w, padding=0).transpose(1, 2)
|
| 58 |
+
windows = neighborhoods.reshape(b, self.grid * self.grid, d, w * w)
|
| 59 |
+
windows = windows.permute(0, 1, 3, 2).reshape(-1, w * w, d)
|
| 60 |
+
local = self.local(windows, windows, windows, need_weights=False)[0]
|
| 61 |
+
local = local[:, (w * w) // 2].reshape(b, self.grid * self.grid, d)
|
| 62 |
+
mixing = self.score(values).transpose(1, 2).softmax(-1)
|
| 63 |
+
compressed = self.global_norm(mixing @ values)
|
| 64 |
+
global_context = self.global_attn(values, compressed, compressed, need_weights=False)[0]
|
| 65 |
+
tokens = residual + self.projection(torch.cat((local, global_context), dim=-1))
|
| 66 |
+
tokens = self.output_norm(tokens)
|
| 67 |
+
return tokens + self.mlp(self.mlp_norm(tokens))
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class Surya(nn.Module):
|
| 71 |
+
def __init__(self, image_size=32, patch_size=4, channels=13, input_steps=2,
|
| 72 |
+
embed_dim=64, depth=4, spectral_blocks=1, num_heads=4,
|
| 73 |
+
window_size=2, global_tokens=4):
|
| 74 |
+
super().__init__()
|
| 75 |
+
if image_size % patch_size:
|
| 76 |
+
raise ValueError("image_size must be divisible by patch_size")
|
| 77 |
+
if spectral_blocks >= depth:
|
| 78 |
+
raise ValueError("spectral_blocks must be smaller than depth")
|
| 79 |
+
if embed_dim % num_heads:
|
| 80 |
+
raise ValueError("embed_dim must be divisible by num_heads")
|
| 81 |
+
self.image_size, self.patch_size = image_size, patch_size
|
| 82 |
+
self.channels, self.input_steps = channels, input_steps
|
| 83 |
+
self.grid = image_size // patch_size
|
| 84 |
+
if channels != 13:
|
| 85 |
+
raise ValueError("Surya requires the 13 SDO AIA/HMI channels")
|
| 86 |
+
patch_dim = channels * patch_size ** 2
|
| 87 |
+
self.patch_embed = nn.Linear(patch_dim * 2, embed_dim)
|
| 88 |
+
self.temporal_embed = nn.Parameter(torch.zeros(1, input_steps, 1, embed_dim))
|
| 89 |
+
self.position = nn.Parameter(torch.zeros(1, self.grid ** 2, embed_dim))
|
| 90 |
+
blocks = [SpectralGating(self.grid, embed_dim) for _ in range(spectral_blocks)]
|
| 91 |
+
blocks += [LongShortAttention(self.grid, embed_dim, num_heads, window_size, global_tokens)
|
| 92 |
+
for _ in range(depth - spectral_blocks)]
|
| 93 |
+
self.blocks = nn.ModuleList(blocks)
|
| 94 |
+
self.norm = nn.LayerNorm(embed_dim)
|
| 95 |
+
self.decoder = nn.Sequential(
|
| 96 |
+
nn.ConvTranspose2d(embed_dim, embed_dim // 2, patch_size, stride=patch_size),
|
| 97 |
+
nn.GELU(), nn.Conv2d(embed_dim // 2, channels, 3, padding=1))
|
| 98 |
+
nn.init.normal_(self.position, std=0.02)
|
| 99 |
+
nn.init.zeros_(self.decoder[-1].weight)
|
| 100 |
+
nn.init.zeros_(self.decoder[-1].bias)
|
| 101 |
+
|
| 102 |
+
def tokenize(self, frames):
|
| 103 |
+
b, t, c, h, w = frames.shape
|
| 104 |
+
if (t, c, h, w) != (self.input_steps, self.channels, self.image_size, self.image_size):
|
| 105 |
+
raise ValueError("Expected BTCHW input matching configured dimensions")
|
| 106 |
+
p = self.patch_size
|
| 107 |
+
current = frames[:, -1]
|
| 108 |
+
delta = current - frames[:, -2]
|
| 109 |
+
values = torch.stack((current, delta), dim=1).reshape(b, 2 * c, h // p, p, w // p, p)
|
| 110 |
+
values = values.permute(0, 2, 4, 1, 3, 5).reshape(b, self.grid ** 2, -1)
|
| 111 |
+
return self.patch_embed(values) + self.position + self.temporal_embed[:, -1]
|
| 112 |
+
|
| 113 |
+
def _predict(self, frames):
|
| 114 |
+
tokens = self.tokenize(frames)
|
| 115 |
+
for block in self.blocks:
|
| 116 |
+
tokens = block(tokens)
|
| 117 |
+
tokens = self.norm(tokens)
|
| 118 |
+
features = tokens.mean(1)
|
| 119 |
+
grid = tokens.transpose(1, 2).reshape(tokens.shape[0], -1, self.grid, self.grid)
|
| 120 |
+
prediction = frames[:, -1] + self.decoder(grid)
|
| 121 |
+
if not torch.isfinite(prediction).all():
|
| 122 |
+
raise FloatingPointError("Surya produced a non-finite prediction")
|
| 123 |
+
return {"prediction": prediction, "features": features}
|
| 124 |
+
|
| 125 |
+
def forward(self, frames, steps=None):
|
| 126 |
+
if steps is None:
|
| 127 |
+
return self._predict(frames)
|
| 128 |
+
if steps < 1:
|
| 129 |
+
raise ValueError("steps must be positive")
|
| 130 |
+
history, predictions = frames, []
|
| 131 |
+
for _ in range(steps):
|
| 132 |
+
prediction = self._predict(history)["prediction"]
|
| 133 |
+
predictions.append(prediction)
|
| 134 |
+
history = torch.cat((history[:, 1:], prediction[:, None]), dim=1)
|
| 135 |
+
return torch.stack(predictions, dim=1)
|
| 136 |
+
|
| 137 |
+
def rollout(self, frames, steps):
|
| 138 |
+
return self(frames, steps=steps)
|
| 139 |
+
|
| 140 |
+
def rollout_loss(self, frames, targets):
|
| 141 |
+
predictions = self.rollout(frames, targets.shape[1])
|
| 142 |
+
step_losses = (predictions - targets).square().mean(dim=(0, 2, 3, 4))
|
| 143 |
+
return {"loss": step_losses.mean(), "step_losses": step_losses,
|
| 144 |
+
"predictions": predictions}
|
scripts/fake_data.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Create synthetic 13-channel SDO-like solar sequences."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import yaml
|
| 8 |
+
|
| 9 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def make_split(count, cfg, rng):
|
| 13 |
+
total = cfg["input_steps"] + cfg["forecast_steps"]
|
| 14 |
+
size, channels = cfg["image_size"], cfg["channels"]
|
| 15 |
+
y, x = np.mgrid[-1:1:complex(size), -1:1:complex(size)].astype(np.float32)
|
| 16 |
+
disk = (x * x + y * y <= 0.92 ** 2).astype(np.float32)
|
| 17 |
+
sequences = np.empty((count, total, channels, size, size), dtype=np.float32)
|
| 18 |
+
activity = np.empty((count, total), dtype=np.float32)
|
| 19 |
+
for sample in range(count):
|
| 20 |
+
phase = rng.uniform(0, 2 * np.pi)
|
| 21 |
+
amplitude = rng.uniform(0.25, 0.8)
|
| 22 |
+
for step in range(total):
|
| 23 |
+
center_x = 0.48 * np.sin(phase + step * 0.18)
|
| 24 |
+
center_y = 0.28 * np.cos(phase + step * 0.12)
|
| 25 |
+
region = np.exp(-((x - center_x) ** 2 + (y - center_y) ** 2) / 0.035)
|
| 26 |
+
activity[sample, step] = region.sum() * amplitude
|
| 27 |
+
for channel in range(channels):
|
| 28 |
+
corona = np.exp(-(x * x + y * y) * (1.5 + channel * 0.05))
|
| 29 |
+
texture = 0.08 * np.sin((channel + 1) * x * 3 + phase + step * 0.1)
|
| 30 |
+
sequences[sample, step, channel] = np.clip(
|
| 31 |
+
disk * (0.15 + 0.35 * corona + amplitude * region * (0.5 + channel / channels) + texture)
|
| 32 |
+
+ rng.normal(0, 0.01, (size, size)), 0, 1)
|
| 33 |
+
return sequences[:, :cfg["input_steps"]], sequences[:, cfg["input_steps"]:], activity[:, cfg["input_steps"]:]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def main():
|
| 37 |
+
cfg = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 38 |
+
data, rng = cfg["data"], np.random.default_rng(cfg["seed"])
|
| 39 |
+
output = ROOT / data["root"]; output.mkdir(exist_ok=True)
|
| 40 |
+
for split, count in (("train", data["train_samples"]), ("test", data["test_samples"])):
|
| 41 |
+
inputs, targets, activity = make_split(count, data, rng)
|
| 42 |
+
np.savez_compressed(output / f"{split}.npz", inputs=inputs, targets=targets, activity=activity)
|
| 43 |
+
(output / "format.json").write_text(json.dumps({
|
| 44 |
+
"inputs": "float32 [N, 2, 13, H, W]", "targets": "float32 [N, rollout, 13, H, W]",
|
| 45 |
+
"activity": "float32 integrated synthetic active-region signal", "protocol": data["protocol"],
|
| 46 |
+
"channel_names": data["channel_names"], "normalization": "signum_log then channel affine",
|
| 47 |
+
"channel_mean": data["channel_mean"], "channel_std": data["channel_std"]}, indent=2) + "\n")
|
| 48 |
+
print("created", output / "train.npz", output / "test.npz")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
if __name__ == "__main__":
|
| 52 |
+
main()
|
scripts/inference.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run validated, batched autoregressive solar forecasting inference."""
|
| 2 |
+
|
| 3 |
+
import importlib.util
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import yaml
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main():
|
| 14 |
+
import argparse
|
| 15 |
+
parser = argparse.ArgumentParser(); parser.add_argument("--config", type=Path, default=ROOT / "conf/config.yaml"); parser.add_argument("--data", type=Path); parser.add_argument("--checkpoint", type=Path); parser.add_argument("--output-dir", type=Path); parser.add_argument("--batch-size", type=int, default=4); parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default=None); args = parser.parse_args()
|
| 16 |
+
cfg = yaml.safe_load(args.config.read_text())
|
| 17 |
+
spec = importlib.util.spec_from_file_location("surya_model", ROOT / "model/surya.py")
|
| 18 |
+
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
|
| 19 |
+
model = module.Surya(**cfg["model"])
|
| 20 |
+
checkpoint = args.checkpoint or ROOT / cfg["paths"]["checkpoint"]
|
| 21 |
+
if not checkpoint.exists(): raise FileNotFoundError("Run training before inference")
|
| 22 |
+
device_name = args.device or cfg["runtime"]["device"]
|
| 23 |
+
if device_name == "cuda" and not torch.cuda.is_available(): raise RuntimeError("CUDA requested but unavailable")
|
| 24 |
+
device = torch.device("cuda" if torch.cuda.is_available() and device_name != "cpu" else "cpu")
|
| 25 |
+
model.load_state_dict(torch.load(checkpoint, map_location="cpu", weights_only=False)["model"]); model.to(device).eval()
|
| 26 |
+
data = np.load(args.data or ROOT / cfg["data"]["root"] / "test.npz"); raw_inputs, raw_targets = data["inputs"], data["targets"]
|
| 27 |
+
if raw_inputs.ndim != 5 or raw_inputs.shape[1:] != (2, 13, cfg["data"]["image_size"], cfg["data"]["image_size"]): raise ValueError("test.npz violates the BTCHW protocol")
|
| 28 |
+
mean = np.asarray(cfg["data"]["channel_mean"], dtype=np.float32)[None, None, :, None, None]; std = np.asarray(cfg["data"]["channel_std"], dtype=np.float32)[None, None, :, None, None]
|
| 29 |
+
inputs = (np.sign(raw_inputs) * np.log1p(np.abs(raw_inputs)) - mean) / std
|
| 30 |
+
predictions = []
|
| 31 |
+
with torch.no_grad():
|
| 32 |
+
for start in range(0, len(inputs), args.batch_size): predictions.append(model(torch.from_numpy(inputs[start:start+args.batch_size]).to(device), steps=cfg["data"]["forecast_steps"]).cpu().numpy())
|
| 33 |
+
predictions = np.concatenate(predictions, axis=0)
|
| 34 |
+
# Evaluation consumes the original physical-value representation.
|
| 35 |
+
predictions = np.sign(predictions * std[:, :1] + mean[:, :1]) * (np.expm1(np.abs(predictions * std[:, :1] + mean[:, :1])))
|
| 36 |
+
output = args.output_dir or ROOT / cfg["paths"]["inference_dir"]; output.mkdir(parents=True, exist_ok=True)
|
| 37 |
+
np.savez_compressed(output / "forecast.npz", inputs=raw_inputs, targets=raw_targets,
|
| 38 |
+
predictions=predictions, activity=data["activity"])
|
| 39 |
+
print("inference=", output / "forecast.npz")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
if __name__ == "__main__": main()
|
scripts/result.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate Surya rollouts and generate solar forecasting figures."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import matplotlib.pyplot as plt
|
| 7 |
+
import numpy as np
|
| 8 |
+
import yaml
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main():
|
| 14 |
+
cfg = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 15 |
+
source = ROOT / cfg["paths"]["inference_dir"] / "forecast.npz"
|
| 16 |
+
if not source.exists(): raise FileNotFoundError("Run inference before evaluation")
|
| 17 |
+
data = np.load(source); targets, predictions = data["targets"], data["predictions"]
|
| 18 |
+
step_mse = np.mean((predictions - targets) ** 2, axis=(0, 2, 3, 4))
|
| 19 |
+
persistence = np.repeat(data["inputs"][:, -1:, :, :, :], targets.shape[1], axis=1)
|
| 20 |
+
persistence_mse = np.mean((persistence - targets) ** 2, axis=(0, 2, 3, 4))
|
| 21 |
+
skill = 1.0 - step_mse / np.maximum(persistence_mse, 1e-8)
|
| 22 |
+
channel_mse = np.mean((predictions - targets) ** 2, axis=(0, 1, 3, 4))
|
| 23 |
+
predicted_activity = predictions[:, :, :8].sum(axis=(2, 3, 4))
|
| 24 |
+
target_activity = targets[:, :, :8].sum(axis=(2, 3, 4))
|
| 25 |
+
result = {"forecast_mse": float(step_mse.mean()), "step_mse": step_mse.tolist(),
|
| 26 |
+
"persistence_step_mse": persistence_mse.tolist(), "persistence_skill": skill.tolist(),
|
| 27 |
+
"channel_mse": channel_mse.tolist(), "aia_mse": float(channel_mse[:8].mean()),
|
| 28 |
+
"hmi_mse": float(channel_mse[8:].mean()),
|
| 29 |
+
"activity_mae": float(np.mean(np.abs(predicted_activity - target_activity))),
|
| 30 |
+
"data_source": "synthetic", "protocol": cfg["data"]["protocol"], "baseline": "persistence"}
|
| 31 |
+
output = ROOT / cfg["paths"]["evaluation_dir"]; output.mkdir(parents=True, exist_ok=True)
|
| 32 |
+
(output / "metrics.json").write_text(json.dumps(result, indent=2) + "\n")
|
| 33 |
+
|
| 34 |
+
steps = np.arange(1, len(step_mse) + 1)
|
| 35 |
+
figure, axis = plt.subplots(figsize=(6, 3.5)); axis.plot(steps, step_mse, marker="o", label="Surya")
|
| 36 |
+
axis.plot(steps, persistence_mse, marker="s", label="Persistence"); axis.set(xlabel="Forecast step (hour)", ylabel="MSE", title="Autoregressive Forecast Skill")
|
| 37 |
+
axis.legend(); axis.grid(alpha=0.25); figure.tight_layout(); figure.savefig(output / "rollout_forecast_skill.png", dpi=160); plt.close(figure)
|
| 38 |
+
|
| 39 |
+
figure, axes = plt.subplots(2, len(steps), figsize=(2.5 * len(steps), 5))
|
| 40 |
+
for index in range(len(steps)):
|
| 41 |
+
axes[0, index].imshow(targets[0, index, 0], cmap="inferno"); axes[0, index].set_title(f"Target +{index + 1}h")
|
| 42 |
+
axes[1, index].imshow(predictions[0, index, 0], cmap="inferno"); axes[1, index].set_title(f"Surya +{index + 1}h")
|
| 43 |
+
axes[0, index].axis("off"); axes[1, index].axis("off")
|
| 44 |
+
figure.tight_layout(); figure.savefig(output / "solar_dynamics_forecast.png", dpi=160); plt.close(figure)
|
| 45 |
+
|
| 46 |
+
figure, axis = plt.subplots(figsize=(6, 3.5)); axis.plot(steps, target_activity[0], marker="o", label="Ground truth")
|
| 47 |
+
axis.plot(steps, predicted_activity[0], marker="s", label="Surya"); axis.set(xlabel="Forecast step (hour)", ylabel="Integrated AIA activity", title="Solar Activity Evolution")
|
| 48 |
+
axis.legend(); axis.grid(alpha=0.25); figure.tight_layout(); figure.savefig(output / "solar_activity_evolution.png", dpi=160); plt.close(figure)
|
| 49 |
+
|
| 50 |
+
figure, axis = plt.subplots(figsize=(7, 3.5)); axis.bar(np.arange(len(channel_mse)), channel_mse, color="#287271")
|
| 51 |
+
channel_names = [f"AIA {i + 1}" for i in range(8)] + [f"HMI {i + 1}" for i in range(5)]
|
| 52 |
+
axis.set_xticks(np.arange(len(channel_mse)), channel_names, rotation=45, ha="right")
|
| 53 |
+
axis.set(xlabel="SDO channel", ylabel="MSE", title="AIA/HMI Channel Forecast Error")
|
| 54 |
+
figure.tight_layout(); figure.savefig(output / "sdo_channel_error.png", dpi=160); plt.close(figure)
|
| 55 |
+
|
| 56 |
+
figure, axis = plt.subplots(figsize=(6, 3.5)); axis.plot(steps, skill, marker="o", color="#7a5195")
|
| 57 |
+
axis.axhline(0.0, color="black", linewidth=0.8)
|
| 58 |
+
axis.set(xlabel="Forecast step (hour)", ylabel="Skill vs persistence", title="Persistence Skill by Lead Time")
|
| 59 |
+
axis.grid(alpha=0.25); figure.tight_layout(); figure.savefig(output / "persistence_skill.png", dpi=160); plt.close(figure)
|
| 60 |
+
print(json.dumps(result, indent=2)); print("evaluation=", output)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
if __name__ == "__main__": main()
|
scripts/train.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train Surya in one-step pretraining and rollout-tuning phases."""
|
| 2 |
+
import argparse, importlib.util, json, math, os, random
|
| 3 |
+
from contextlib import nullcontext
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import numpy as np, torch, yaml
|
| 6 |
+
from torch import distributed as dist
|
| 7 |
+
from torch.nn.parallel import DistributedDataParallel
|
| 8 |
+
from torch.utils.data import DataLoader, Dataset, DistributedSampler
|
| 9 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 10 |
+
|
| 11 |
+
class SolarDataset(Dataset):
|
| 12 |
+
def __init__(self, path, cfg):
|
| 13 |
+
d = np.load(path); self.inputs, self.targets = d["inputs"], d["targets"]
|
| 14 |
+
if self.inputs.ndim != 5 or self.targets.ndim != 5 or self.inputs.shape[2] != 13 or self.inputs.shape[1] != 2:
|
| 15 |
+
raise ValueError("Expected inputs [N,2,13,H,W] and targets [N,S,13,H,W]")
|
| 16 |
+
self.mean = np.asarray(cfg["data"]["channel_mean"], dtype=np.float32)[None, :, None, None]
|
| 17 |
+
self.std = np.asarray(cfg["data"]["channel_std"], dtype=np.float32)[None, :, None, None]
|
| 18 |
+
def __len__(self): return len(self.inputs)
|
| 19 |
+
def __getitem__(self, i):
|
| 20 |
+
transform = lambda x: (np.sign(x) * np.log1p(np.abs(x)) - self.mean) / self.std
|
| 21 |
+
return torch.from_numpy(transform(self.inputs[i]).astype(np.float32)), torch.from_numpy(transform(self.targets[i]).astype(np.float32))
|
| 22 |
+
|
| 23 |
+
def load_model():
|
| 24 |
+
spec = importlib.util.spec_from_file_location("surya_model", ROOT / "model/surya.py"); mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod); return mod.Surya
|
| 25 |
+
def args():
|
| 26 |
+
p = argparse.ArgumentParser(); p.add_argument("--config", type=Path, default=ROOT / "conf/config.yaml"); p.add_argument("--data", type=Path); p.add_argument("--output", type=Path); p.add_argument("--epochs", type=int); p.add_argument("--batch-size", type=int); p.add_argument("--device", choices=["auto", "cpu", "cuda"]); return p.parse_args()
|
| 27 |
+
def lr_at(progress, total, warmup, peak, floor):
|
| 28 |
+
if warmup and progress < warmup: return peak * progress / warmup
|
| 29 |
+
phase = min(max((progress - warmup) / max(total - warmup, 1), 0), 1)
|
| 30 |
+
return floor + (peak - floor) * (1 + math.cos(math.pi * phase)) / 2
|
| 31 |
+
|
| 32 |
+
def main():
|
| 33 |
+
a = args(); cfg = yaml.safe_load(a.config.read_text()); tc = cfg["training"]
|
| 34 |
+
total_epochs = a.epochs or tc["epochs"]; world = int(os.getenv("WORLD_SIZE", "1")); rank = int(os.getenv("RANK", "0")); local = int(os.getenv("LOCAL_RANK", "0")); distributed = world > 1
|
| 35 |
+
requested = a.device or cfg["runtime"]["device"]; cuda = torch.cuda.is_available() and requested != "cpu"
|
| 36 |
+
if requested == "cuda" and not cuda: raise RuntimeError("CUDA requested but unavailable")
|
| 37 |
+
if distributed: dist.init_process_group("nccl" if cuda else "gloo")
|
| 38 |
+
device = torch.device(f"cuda:{local}" if cuda else "cpu"); torch.cuda.set_device(local) if cuda else None
|
| 39 |
+
seed = cfg["seed"] + rank; random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
|
| 40 |
+
data_path = a.data or ROOT / cfg["data"]["root"] / "train.npz"
|
| 41 |
+
dataset = SolarDataset(data_path, cfg); sampler = DistributedSampler(dataset) if distributed else None
|
| 42 |
+
loader = DataLoader(dataset, batch_size=a.batch_size or tc["batch_size"], shuffle=sampler is None, sampler=sampler, num_workers=tc["num_workers"], pin_memory=cuda)
|
| 43 |
+
model = load_model()(**cfg["model"]).to(device); bare = model
|
| 44 |
+
if distributed: model = DistributedDataParallel(model, device_ids=[local] if cuda else None); bare = model.module
|
| 45 |
+
decay, no_decay = [], []
|
| 46 |
+
for n, p in bare.named_parameters(): (no_decay if p.ndim == 1 or n.endswith("bias") else decay).append(p)
|
| 47 |
+
opt = torch.optim.AdamW([{"params": decay, "weight_decay": tc["weight_decay"]}, {"params": no_decay, "weight_decay": 0}], lr=tc["learning_rate"])
|
| 48 |
+
amp = bool(tc.get("amp", True) and cuda); scaler = torch.amp.GradScaler("cuda", enabled=amp); history=[]; opt.zero_grad(set_to_none=True)
|
| 49 |
+
one_step = tc.get("one_step_epochs", max(1, total_epochs // 2))
|
| 50 |
+
for epoch in range(total_epochs):
|
| 51 |
+
if sampler: sampler.set_epoch(epoch)
|
| 52 |
+
model.train(); total=0.0; phase = "one_step" if epoch < one_step else "rollout"
|
| 53 |
+
for step, (x, y) in enumerate(loader):
|
| 54 |
+
x, y = x.to(device, non_blocking=cuda), y.to(device, non_blocking=cuda); pred_steps = 1 if phase == "one_step" else y.shape[1]
|
| 55 |
+
for group in opt.param_groups: group["lr"] = lr_at(epoch + step/max(len(loader),1), total_epochs, tc["warmup_epochs"], tc["learning_rate"], tc["min_learning_rate"])
|
| 56 |
+
context = torch.amp.autocast("cuda") if amp else nullcontext()
|
| 57 |
+
with context:
|
| 58 |
+
pred = model(x, steps=pred_steps); loss = (pred - y[:, :pred_steps]).square().mean() / tc["accum_iter"]
|
| 59 |
+
if not torch.isfinite(loss): raise FloatingPointError("non-finite training loss")
|
| 60 |
+
scaler.scale(loss).backward()
|
| 61 |
+
if (step + 1) % tc["accum_iter"] == 0 or step + 1 == len(loader):
|
| 62 |
+
scaler.unscale_(opt); torch.nn.utils.clip_grad_norm_(bare.parameters(), tc["grad_clip"]); scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True)
|
| 63 |
+
total += loss.item() * tc["accum_iter"]
|
| 64 |
+
values = torch.tensor([total / max(len(loader),1)], device=device); dist.all_reduce(values) if distributed else None
|
| 65 |
+
record={"epoch":epoch+1,"phase":phase,"loss":float(values.item()/world),"learning_rate":opt.param_groups[0]["lr"]}; history.append(record)
|
| 66 |
+
if rank == 0: print(json.dumps(record))
|
| 67 |
+
if rank == 0:
|
| 68 |
+
path=a.output or ROOT / cfg["paths"]["checkpoint"]; path.parent.mkdir(parents=True,exist_ok=True); torch.save({"model":bare.state_dict(),"optimizer":opt.state_dict(),"scaler":scaler.state_dict(),"epoch":total_epochs-1,"history":history,"config":cfg},path)
|
| 69 |
+
out=ROOT / cfg["paths"]["training_metrics"]; out.parent.mkdir(parents=True,exist_ok=True); out.write_text(json.dumps({"history":history,"protocol":cfg["data"]["protocol"]},indent=2)+"\n"); print("checkpoint=",path)
|
| 70 |
+
if distributed: dist.destroy_process_group()
|
| 71 |
+
if __name__ == "__main__": main()
|
weight/.gitkeep
ADDED
|
File without changes
|