yzt15806542928 commited on
Commit
e9b87a5
·
verified ·
1 Parent(s): 18b0294

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -1,35 +1,67 @@
1
  *.7z filter=lfs diff=lfs merge=lfs -text
2
  *.arrow filter=lfs diff=lfs merge=lfs -text
3
  *.bin filter=lfs diff=lfs merge=lfs -text
 
4
  *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
  *.ftz filter=lfs diff=lfs merge=lfs -text
7
  *.gz filter=lfs diff=lfs merge=lfs -text
8
  *.h5 filter=lfs diff=lfs merge=lfs -text
9
  *.joblib filter=lfs diff=lfs merge=lfs -text
10
  *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
  *.model filter=lfs diff=lfs merge=lfs -text
13
  *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
  *.onnx filter=lfs diff=lfs merge=lfs -text
17
  *.ot filter=lfs diff=lfs merge=lfs -text
18
  *.parquet filter=lfs diff=lfs merge=lfs -text
19
  *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
  *.pt filter=lfs diff=lfs merge=lfs -text
23
  *.pth filter=lfs diff=lfs merge=lfs -text
24
  *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
  saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
  *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
  *.tflite filter=lfs diff=lfs merge=lfs -text
30
  *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
  *.xz filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  *.7z filter=lfs diff=lfs merge=lfs -text
2
  *.arrow filter=lfs diff=lfs merge=lfs -text
3
  *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bin.* filter=lfs diff=lfs merge=lfs -text
5
  *.bz2 filter=lfs diff=lfs merge=lfs -text
 
6
  *.ftz filter=lfs diff=lfs merge=lfs -text
7
  *.gz filter=lfs diff=lfs merge=lfs -text
8
  *.h5 filter=lfs diff=lfs merge=lfs -text
9
  *.joblib filter=lfs diff=lfs merge=lfs -text
10
  *.lfs.* filter=lfs diff=lfs merge=lfs -text
 
11
  *.model filter=lfs diff=lfs merge=lfs -text
12
  *.msgpack filter=lfs diff=lfs merge=lfs -text
 
 
13
  *.onnx filter=lfs diff=lfs merge=lfs -text
14
  *.ot filter=lfs diff=lfs merge=lfs -text
15
  *.parquet filter=lfs diff=lfs merge=lfs -text
16
  *.pb filter=lfs diff=lfs merge=lfs -text
 
 
17
  *.pt filter=lfs diff=lfs merge=lfs -text
18
  *.pth filter=lfs diff=lfs merge=lfs -text
19
  *.rar filter=lfs diff=lfs merge=lfs -text
 
20
  saved_model/**/* filter=lfs diff=lfs merge=lfs -text
21
  *.tar.* filter=lfs diff=lfs merge=lfs -text
 
22
  *.tflite filter=lfs diff=lfs merge=lfs -text
23
  *.tgz filter=lfs diff=lfs merge=lfs -text
 
24
  *.xz filter=lfs diff=lfs merge=lfs -text
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
+ *.zstandard filter=lfs diff=lfs merge=lfs -text
27
+ *.tfevents* filter=lfs diff=lfs merge=lfs -text
28
+ *.db* filter=lfs diff=lfs merge=lfs -text
29
+ *.ark* filter=lfs diff=lfs merge=lfs -text
30
+ **/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
31
+ **/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
32
+ **/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
33
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
34
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
35
+ *.gguf* filter=lfs diff=lfs merge=lfs -text
36
+ *.ggml filter=lfs diff=lfs merge=lfs -text
37
+ *.llamafile* filter=lfs diff=lfs merge=lfs -text
38
+ *.pt2 filter=lfs diff=lfs merge=lfs -text
39
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
40
+ *.npy filter=lfs diff=lfs merge=lfs -text
41
+ *.npz filter=lfs diff=lfs merge=lfs -text
42
+ *.pickle filter=lfs diff=lfs merge=lfs -text
43
+ *.pkl filter=lfs diff=lfs merge=lfs -text
44
+ *.tar filter=lfs diff=lfs merge=lfs -text
45
+ *.wasm filter=lfs diff=lfs merge=lfs -text
46
  *.zst filter=lfs diff=lfs merge=lfs -text
47
  *tfevents* filter=lfs diff=lfs merge=lfs -text
48
+ *.wav filter=lfs diff=lfs merge=lfs -text
49
+ *.mp3 filter=lfs diff=lfs merge=lfs -text
50
+ *.flac filter=lfs diff=lfs merge=lfs -text
51
+ *.ogg filter=lfs diff=lfs merge=lfs -text
52
+ *.m4a filter=lfs diff=lfs merge=lfs -text
53
+ *.aac filter=lfs diff=lfs merge=lfs -text
54
+ *.opus filter=lfs diff=lfs merge=lfs -text
55
+ *.mp4 filter=lfs diff=lfs merge=lfs -text
56
+ *.mov filter=lfs diff=lfs merge=lfs -text
57
+ *.avi filter=lfs diff=lfs merge=lfs -text
58
+ *.mkv filter=lfs diff=lfs merge=lfs -text
59
+ *.webm filter=lfs diff=lfs merge=lfs -text
60
+ *.obj filter=lfs diff=lfs merge=lfs -text
61
+ *.glb filter=lfs diff=lfs merge=lfs -text
62
+ *.gltf filter=lfs diff=lfs merge=lfs -text
63
+ *.ply filter=lfs diff=lfs merge=lfs -text
64
+ *.stl filter=lfs diff=lfs merge=lfs -text
65
+ *.fbx filter=lfs diff=lfs merge=lfs -text
66
+
67
+ /weight/ filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ datasets:
3
+ - OneScience/ERA5
4
+ frameworks:
5
+ - ""
6
+ language:
7
+ - en
8
+ license: mit
9
+ tags:
10
+ - OneScience
11
+ - Earth Science
12
+ - Weather Forecasting
13
+ - ERA5
14
+ - Regional Forecasting
15
+ tasks: []
16
+ ---
17
+ <p align="center">
18
+ <strong>
19
+ <span style="font-size: 30px;">ClimaX</span>
20
+ </strong>
21
+ </p>
22
+
23
+ # Model Overview
24
+
25
+ ClimaX is the first general-purpose foundation model designed for weather and climate science. It was jointly developed by researchers at Microsoft Research and the University of California, Los Angeles (UCLA), and was presented at ICML 2023.
26
+
27
+ Paper: *ClimaX: A Foundation Model for Weather and Climate*
28
+
29
+ https://arxiv.org/abs/2301.10343
30
+
31
+
32
+ # Model Description
33
+
34
+ ClimaX is built on the Vision Transformer architecture. It was the first model to unify pre-training on heterogeneous climate datasets and adaptation to downstream tasks, such as regional forecasting and climate projections, within a single framework.
35
+
36
+
37
+ # Use Cases
38
+
39
+ | Use Case | Description |
40
+ | :---: | :--- |
41
+ | Weather forecasting training | Train ClimaX on ERA5 data in HDF5 format. |
42
+ | Quick local validation | Use synthetic data to validate data loading, model training and inference, and visualization of inference results. |
43
+ | ModelScope/OneCode execution | Download the standalone model package, install its dependencies, and run the included scripts directly. |
44
+ | Multi-GPU training | Launch multi-process training with `torchrun`. |
45
+
46
+
47
+ # Usage
48
+
49
+ ## 1. Using OneCode
50
+
51
+ Use the OneCode online environment for an intelligent, one-click AI4S development experience:
52
+
53
+ [Try one-click AI4S development with OneCode](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home)
54
+
55
+ ## 2. Manual Setup
56
+
57
+ **Hardware Requirements**
58
+
59
+ - A GPU or DCU is recommended.
60
+ - A CPU can be used for import checks and connectivity validation with a minimal configuration, but full training and inference will be slow.
61
+ - DCU users must install DTK in advance. DTK 25.04.2 or later is recommended; alternatively, use the OneScience-recommended version compatible with your cluster.
62
+
63
+ ### Download the Model Package
64
+
65
+ ```bash
66
+ hf download --model OneScience-Group/ClimaX --local-dir ./ClimaX
67
+ cd ClimaX
68
+ ```
69
+
70
+ ### Set Up the Runtime Environment
71
+
72
+ **DCU Environment**
73
+
74
+ ```bash
75
+ # Activate DTK and conda first.
76
+ conda create -n onescience311 python=3.11 -y
77
+ conda activate onescience311
78
+ # Installation with uv is also supported.
79
+ pip install onescience[earth-dcu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
80
+ ```
81
+
82
+ **GPU Environment**
83
+
84
+ ```bash
85
+ # Activate conda first.
86
+ conda create -n onescience311 python=3.11 -y libstdcxx-ng=12 libgcc-ng=12 gcc_linux-64=12 gxx_linux-64=12
87
+ conda activate onescience311
88
+ # Installation with uv is also supported.
89
+ pip install onescience[earth-gpu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
90
+ ```
91
+
92
+
93
+ ### Training Data
94
+
95
+ The OneScience community provides ERA5 data for training. Because of file-size constraints, the repository currently contains a self-contained data slice. Download the data with the following command and ensure that the data path in `conf/config.yaml` is configured correctly:
96
+
97
+ ```bash
98
+ hf download --dataset OneScience-Group/ERA5 --local-dir ./data
99
+ ```
100
+
101
+ ### Training
102
+
103
+ Single GPU:
104
+
105
+ ```bash
106
+ python scripts/train.py
107
+ ```
108
+
109
+ Multiple GPUs:
110
+
111
+ ```bash
112
+ torchrun --nproc_per_node=8 scripts/train.py
113
+ ```
114
+
115
+ Training saves the `model_bak.pth` checkpoint under `data/checkpoints/`.
116
+
117
+ ### Pre-trained Weights
118
+
119
+ This repository will provide weights trained on ERA5 data in the `weights/` directory. The weight files are being prepared and will be uploaded soon.
120
+
121
+ ### Inference
122
+
123
+ ```bash
124
+ python scripts/inference.py
125
+ ```
126
+
127
+ Inference results are saved to `result/output/`.
128
+
129
+ ### Evaluation and Visualization
130
+
131
+ ```bash
132
+ python scripts/result.py
133
+ ```
134
+
135
+
136
+ # Official OneScience Resources
137
+
138
+ | Platform | OneScience Main Repository | Skills Repository |
139
+ | --- | --- | --- |
140
+ | Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills |
141
+ | GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills |
142
+
143
+ # Citation and License
144
+
145
+ - This repository is a reproduction of the original ClimaX paper and is licensed under the MIT License.
conf/config.yaml ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+
3
+ start_epoch: 0
4
+ max_epoch: 10
5
+ lr: 5e-4
6
+ beta_1: 0.9
7
+ beta_2: 0.99
8
+ weight_decay: 1e-5
9
+ warmup_epochs: 10000
10
+ warmup_start_lr: 1e-8
11
+ eta_min: 1e-8
12
+
13
+ # ClimaX architecture params (from official config)
14
+ img_size: [32, 64] # 5.625° resolution (lat, lon)
15
+ patch_size: 2
16
+ embed_dim: 1024
17
+ depth: 8
18
+ decoder_depth: 2
19
+ num_heads: 16
20
+ mlp_ratio: 4.0
21
+ drop_path: 0.1
22
+ drop_rate: 0.1
23
+
24
+ # Forecasting settings
25
+ # NOTE: ERA5 data has 6-hour time steps (time_step=6 in HDF5 files).
26
+ # The lead_time passed to the model is (predict_range * hrs_each_step) / 100.
27
+ # For 6-hour forecast: predict_range=6, hrs_each_step=1 → lead_time=0.06
28
+ # For 72-hour forecast: predict_range=72, hrs_each_step=1 → lead_time=0.72
29
+ # The model conditions on lead_time; it must match the actual data gap.
30
+ predict_range: 6 # forecast lead time in hours
31
+ hrs_each_step: 1 # hours per data step factor for lead_time computation
32
+
33
+ checkpoint_dir: "./data/checkpoints"
34
+ patience: 50
35
+
36
+ # 整个数据读取流程
37
+ datapipe:
38
+ name: "ERA5"
39
+ task: "weather_forecasting"
40
+
41
+ # dataset设定
42
+ dataset:
43
+ type: "hdf5"
44
+ data_dir: './data/' # "$ONESCIENCE_DATASETS_DIR/ERA5/newh5/"
45
+
46
+ train_time: [2000, 2001]
47
+ val_time: [2002]
48
+ test_time: [2003]
49
+ img_size: [32, 64]
50
+ verbose: true
51
+ cache: false
52
+
53
+ # ClimaX 48 input variables: 3 constants + 3 surface + 42 pressure-level
54
+ channels:
55
+ # Static constants (3)
56
+ - "land_sea_mask"
57
+ - "orography"
58
+ - "lattitude"
59
+
60
+ # Surface variables (3)
61
+ - "2m_temperature"
62
+ - "10m_u_component_of_wind"
63
+ - "10m_v_component_of_wind"
64
+
65
+ # Geopotential at 7 pressure levels
66
+ - "geopotential_50"
67
+ - "geopotential_250"
68
+ - "geopotential_500"
69
+ - "geopotential_600"
70
+ - "geopotential_700"
71
+ - "geopotential_850"
72
+ - "geopotential_925"
73
+
74
+ # U component of wind at 7 pressure levels
75
+ - "u_component_of_wind_50"
76
+ - "u_component_of_wind_250"
77
+ - "u_component_of_wind_500"
78
+ - "u_component_of_wind_600"
79
+ - "u_component_of_wind_700"
80
+ - "u_component_of_wind_850"
81
+ - "u_component_of_wind_925"
82
+
83
+ # V component of wind at 7 pressure levels
84
+ - "v_component_of_wind_50"
85
+ - "v_component_of_wind_250"
86
+ - "v_component_of_wind_500"
87
+ - "v_component_of_wind_600"
88
+ - "v_component_of_wind_700"
89
+ - "v_component_of_wind_850"
90
+ - "v_component_of_wind_925"
91
+
92
+ # Temperature at 7 pressure levels
93
+ - "temperature_50"
94
+ - "temperature_250"
95
+ - "temperature_500"
96
+ - "temperature_600"
97
+ - "temperature_700"
98
+ - "temperature_850"
99
+ - "temperature_925"
100
+
101
+ # Relative humidity at 7 pressure levels
102
+ - "relative_humidity_50"
103
+ - "relative_humidity_250"
104
+ - "relative_humidity_500"
105
+ - "relative_humidity_600"
106
+ - "relative_humidity_700"
107
+ - "relative_humidity_850"
108
+ - "relative_humidity_925"
109
+
110
+ # Specific humidity at 7 pressure levels
111
+ - "specific_humidity_50"
112
+ - "specific_humidity_250"
113
+ - "specific_humidity_500"
114
+ - "specific_humidity_600"
115
+ - "specific_humidity_700"
116
+ - "specific_humidity_850"
117
+ - "specific_humidity_925"
118
+
119
+ # ClimaX output variables (5)
120
+ out_variables:
121
+ - "geopotential_500"
122
+ - "temperature_850"
123
+ - "2m_temperature"
124
+ - "10m_u_component_of_wind"
125
+ - "10m_v_component_of_wind"
126
+
127
+ # Short names for evaluation
128
+ variables:
129
+ - "z500"
130
+ - "t850"
131
+ - "t2m"
132
+ - "u10"
133
+ - "v10"
134
+
135
+ # 时间配置
136
+ time_range: ["2000-01-01", "2020-12-31"]
137
+ time_steps: 1
138
+ time_res: 6
139
+
140
+ # 空间配置
141
+ spatial_resolution: [5.625, 5.625]
142
+
143
+ # 采样配置
144
+ num_samples: -1 # -1 表示使用全部数据
145
+ shuffle: true
146
+ random_seed: 42
147
+
148
+ # 领域特定配置
149
+ extra:
150
+ levels: [500, 850]
151
+ lat_range: [-90, 90]
152
+ lon_range: [0, 360]
153
+
154
+ # 数据转换配置
155
+ transforms:
156
+ - type: "Normalize"
157
+ params:
158
+ mean: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
159
+ std: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
160
+ keys: ["input", "target"]
161
+
162
+ - type: "ToTensor"
163
+ params:
164
+ keys: null # null表示转换所有numpy数组
165
+
166
+ # DataLoader配置
167
+ dataloader:
168
+ mask_dtype: "float32"
169
+ batch_size: 1
170
+ num_workers: 1
171
+ pin_memory: true
172
+ drop_last: true
173
+ shuffle: false # 使用sampler时设为false
174
+ prefetch_factor: 2
175
+ persistent_workers: true
176
+
177
+ # 分布式配置
178
+ distributed:
179
+ enabled: true
180
+ sampler: "DistributedSampler"
181
+ rank: 0
182
+ world_size: 4
183
+ shuffle: true
184
+ seed: 42
185
+ drop_last: true
configuration.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"framework":"Pytorch","task":"other"}
model/ClimaX.py ADDED
@@ -0,0 +1,555 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ClimaX: A foundation model for weather and climate.
3
+
4
+ Reference:
5
+ - "ClimaX: A foundation model for weather and climate" (arXiv:2301.10343)
6
+ - Official repo: https://github.com/microsoft/ClimaX
7
+
8
+ This implementation:
9
+ - Removes dependency on timm (PatchEmbed, Block, trunc_normal_ reimplemented)
10
+ - Removes dependency on pytorch_lightning
11
+ - Compatible with onescience framework
12
+ - Follows official code logic and precision exactly
13
+ - Supports training from scratch without pretrained weights
14
+ """
15
+
16
+ import math
17
+ import numpy as np
18
+ from dataclasses import dataclass
19
+ from functools import lru_cache
20
+
21
+ import torch
22
+ import torch.nn as nn
23
+ import torch.nn.functional as F
24
+
25
+ from onescience.models.meta import ModelMetaData
26
+
27
+
28
+ # ============================================================================
29
+ # Model metadata for onescience framework
30
+ # ============================================================================
31
+
32
+ @dataclass
33
+ class MetaData(ModelMetaData):
34
+ name: str = "ClimaX"
35
+ jit: bool = False
36
+ cuda_graphs: bool = True
37
+ amp: bool = True
38
+ amp_cpu: bool = None
39
+ amp_gpu: bool = None
40
+ onnx_cpu: bool = False
41
+ onnx_gpu: bool = True
42
+ onnx_runtime: bool = True
43
+ var_dim: int = 1
44
+ func_torch: bool = False
45
+ auto_grad: bool = False
46
+
47
+
48
+ # ============================================================================
49
+ # Utility functions (replacing timm dependencies)
50
+ # ============================================================================
51
+
52
+ def _trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
53
+ """Truncated normal initialization (replaces timm's trunc_normal_)."""
54
+ def norm_cdf(x):
55
+ return (1. + math.erf(x / math.sqrt(2.))) / 2.
56
+
57
+ l = norm_cdf((a - mean) / std)
58
+ u = norm_cdf((b - mean) / std)
59
+
60
+ tensor.uniform_(2 * l - 1, 2 * u - 1)
61
+ tensor.erfinv_()
62
+ tensor.mul_(std * math.sqrt(2.))
63
+ tensor.add_(mean)
64
+ tensor.clamp_(min=a, max=b)
65
+
66
+
67
+ def trunc_normal_(tensor, std=0.02):
68
+ """Drop-in replacement for timm's trunc_normal_ (wrapped with no_grad)."""
69
+ with torch.no_grad():
70
+ _trunc_normal_(tensor, mean=0., std=std, a=-2., b=2.)
71
+
72
+
73
+ # ============================================================================
74
+ # Position embedding utilities (from official ClimaX pos_embed.py)
75
+ # ============================================================================
76
+
77
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
78
+ """
79
+ embed_dim: output dimension for each position
80
+ pos: a list of positions to be encoded: size (M,)
81
+ out: (M, D)
82
+ """
83
+ assert embed_dim % 2 == 0
84
+ omega = np.arange(embed_dim // 2, dtype=float)
85
+ omega /= embed_dim / 2.0
86
+ omega = 1.0 / 10000 ** omega # (D/2,)
87
+
88
+ pos = pos.reshape(-1) # (M,)
89
+ out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
90
+
91
+ emb_sin = np.sin(out) # (M, D/2)
92
+ emb_cos = np.cos(out) # (M, D/2)
93
+
94
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
95
+ return emb
96
+
97
+
98
+ def get_2d_sincos_pos_embed(embed_dim, grid_size_h, grid_size_w, cls_token=False):
99
+ """
100
+ grid_size_h: int of the grid height
101
+ grid_size_w: int of the grid width
102
+ return:
103
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim]
104
+ """
105
+ grid_h = np.arange(grid_size_h, dtype=np.float32)
106
+ grid_w = np.arange(grid_size_w, dtype=np.float32)
107
+ grid = np.meshgrid(grid_w, grid_h) # w goes first
108
+ grid = np.stack(grid, axis=0)
109
+
110
+ grid = grid.reshape([2, 1, grid_size_h, grid_size_w])
111
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
112
+ if cls_token:
113
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
114
+ return pos_embed
115
+
116
+
117
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
118
+ assert embed_dim % 2 == 0
119
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
120
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
121
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
122
+ return emb
123
+
124
+
125
+ # ============================================================================
126
+ # Basic building blocks (replacing timm dependencies)
127
+ # ============================================================================
128
+
129
+ class Mlp(nn.Module):
130
+ """MLP with GELU activation (replaces timm.layers.Mlp)."""
131
+
132
+ def __init__(
133
+ self,
134
+ in_features,
135
+ hidden_features=None,
136
+ out_features=None,
137
+ act_layer=nn.GELU,
138
+ drop=0.,
139
+ ):
140
+ super().__init__()
141
+ out_features = out_features or in_features
142
+ hidden_features = hidden_features or in_features
143
+ self.fc1 = nn.Linear(in_features, hidden_features)
144
+ self.act = act_layer()
145
+ self.fc2 = nn.Linear(hidden_features, out_features)
146
+ self.drop = nn.Dropout(drop)
147
+
148
+ def forward(self, x):
149
+ x = self.fc1(x)
150
+ x = self.act(x)
151
+ x = self.drop(x)
152
+ x = self.fc2(x)
153
+ x = self.drop(x)
154
+ return x
155
+
156
+
157
+ class DropPath(nn.Module):
158
+ """Drop paths (Stochastic Depth) per sample (replaces timm's DropPath)."""
159
+
160
+ def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):
161
+ super().__init__()
162
+ self.drop_prob = drop_prob
163
+ self.scale_by_keep = scale_by_keep
164
+
165
+ def forward(self, x):
166
+ if self.drop_prob == 0. or not self.training:
167
+ return x
168
+ keep_prob = 1 - self.drop_prob
169
+ shape = (x.shape[0],) + (1,) * (x.ndim - 1)
170
+ random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)
171
+ random_tensor.floor_()
172
+ output = x.div(keep_prob) * random_tensor
173
+ return output
174
+
175
+
176
+ class PatchEmbed(nn.Module):
177
+ """2D Image to Patch Embedding (replaces timm's PatchEmbed).
178
+
179
+ Splits image into patches and embeds each patch via Conv2d.
180
+ """
181
+
182
+ def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
183
+ super().__init__()
184
+ if isinstance(img_size, int):
185
+ img_size = (img_size, img_size)
186
+ if isinstance(patch_size, int):
187
+ patch_size = (patch_size, patch_size)
188
+ self.img_size = img_size
189
+ self.patch_size = patch_size
190
+ self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
191
+ self.num_patches = self.grid_size[0] * self.grid_size[1]
192
+
193
+ self.proj = nn.Conv2d(
194
+ in_chans, embed_dim,
195
+ kernel_size=patch_size, stride=patch_size
196
+ )
197
+
198
+ def forward(self, x):
199
+ B, C, H, W = x.shape
200
+ x = self.proj(x) # B, D, H/p, W/p
201
+ x = x.flatten(2).transpose(1, 2) # B, L, D
202
+ return x
203
+
204
+
205
+ class Attention(nn.Module):
206
+ """Multi-head self-attention (replaces timm's Attention)."""
207
+
208
+ def __init__(
209
+ self,
210
+ dim,
211
+ num_heads=8,
212
+ qkv_bias=False,
213
+ attn_drop=0.,
214
+ proj_drop=0.,
215
+ ):
216
+ super().__init__()
217
+ self.num_heads = num_heads
218
+ head_dim = dim // num_heads
219
+ self.scale = head_dim ** -0.5
220
+
221
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
222
+ self.attn_drop = nn.Dropout(attn_drop)
223
+ self.proj = nn.Linear(dim, dim)
224
+ self.proj_drop = nn.Dropout(proj_drop)
225
+
226
+ def forward(self, x):
227
+ B, N, C = x.shape
228
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
229
+ q, k, v = qkv.unbind(dim=2)
230
+ q = q.permute(0, 2, 1, 3) # B, num_heads, N, head_dim
231
+ k = k.permute(0, 2, 1, 3)
232
+ v = v.permute(0, 2, 1, 3)
233
+
234
+ x = F.scaled_dot_product_attention(
235
+ q, k, v,
236
+ dropout_p=self.attn_drop.p if self.attn_drop.p > 0.0 else 0.0,
237
+ scale=self.scale,
238
+ )
239
+
240
+ x = x.permute(0, 2, 1, 3).reshape(B, N, C)
241
+ x = self.proj(x)
242
+ x = self.proj_drop(x)
243
+ return x
244
+
245
+
246
+ class Block(nn.Module):
247
+ """ViT Block with LayerNorm, Attention, and MLP (replaces timm's Block)."""
248
+
249
+ def __init__(
250
+ self,
251
+ dim,
252
+ num_heads,
253
+ mlp_ratio=4.,
254
+ qkv_bias=False,
255
+ drop=0.,
256
+ attn_drop=0.,
257
+ drop_path=0.,
258
+ norm_layer=nn.LayerNorm,
259
+ ):
260
+ super().__init__()
261
+ self.norm1 = norm_layer(dim)
262
+ self.attn = Attention(
263
+ dim,
264
+ num_heads=num_heads,
265
+ qkv_bias=qkv_bias,
266
+ attn_drop=attn_drop,
267
+ proj_drop=drop,
268
+ )
269
+ self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
270
+
271
+ self.norm2 = norm_layer(dim)
272
+ self.mlp = Mlp(
273
+ in_features=dim,
274
+ hidden_features=int(dim * mlp_ratio),
275
+ act_layer=nn.GELU,
276
+ drop=drop,
277
+ )
278
+
279
+ def forward(self, x):
280
+ x = x + self.drop_path(self.attn(self.norm1(x)))
281
+ x = x + self.drop_path(self.mlp(self.norm2(x)))
282
+ return x
283
+
284
+
285
+ # ============================================================================
286
+ # Main ClimaX Model
287
+ # ============================================================================
288
+
289
+ class ClimaX(nn.Module):
290
+ """Implements the ClimaX model as described in the paper,
291
+ https://arxiv.org/abs/2301.10343
292
+
293
+ This is the base ClimaX architecture for global weather forecasting.
294
+ It uses per-variable tokenization, cross-attention variable aggregation,
295
+ a ViT backbone, and an MLP prediction head.
296
+
297
+ Args:
298
+ default_vars (list): list of default variables to be used for training
299
+ img_size (list): image size of the input data [H, W]
300
+ patch_size (int): patch size of the input data
301
+ embed_dim (int): embedding dimension
302
+ depth (int): number of transformer layers
303
+ decoder_depth (int): number of decoder layers
304
+ num_heads (int): number of attention heads
305
+ mlp_ratio (float): ratio of mlp hidden dimension to embedding dimension
306
+ drop_path (float): stochastic depth rate
307
+ drop_rate (float): dropout rate
308
+ """
309
+
310
+ def __init__(
311
+ self,
312
+ default_vars,
313
+ img_size=(32, 64),
314
+ patch_size=2,
315
+ embed_dim=1024,
316
+ depth=8,
317
+ decoder_depth=2,
318
+ num_heads=16,
319
+ mlp_ratio=4.0,
320
+ drop_path=0.1,
321
+ drop_rate=0.1,
322
+ ):
323
+ super().__init__()
324
+
325
+ self.img_size = tuple(img_size)
326
+ self.patch_size = patch_size
327
+ self.default_vars = default_vars
328
+
329
+ # variable tokenization: separate embedding layer for each input variable
330
+ self.token_embeds = nn.ModuleList(
331
+ [PatchEmbed(img_size, patch_size, 1, embed_dim) for _ in range(len(default_vars))]
332
+ )
333
+ self.num_patches = self.token_embeds[0].num_patches
334
+
335
+ # variable embedding to denote which variable each token belongs to
336
+ self.var_embed, self.var_map = self.create_var_embedding(embed_dim)
337
+
338
+ # variable aggregation: a learnable query and a single-layer cross attention
339
+ self.var_query = nn.Parameter(torch.zeros(1, 1, embed_dim), requires_grad=True)
340
+ self.var_agg = nn.MultiheadAttention(embed_dim, num_heads, batch_first=True)
341
+
342
+ # positional embedding and lead time embedding
343
+ self.pos_embed = nn.Parameter(
344
+ torch.zeros(1, self.num_patches, embed_dim), requires_grad=True
345
+ )
346
+ self.lead_time_embed = nn.Linear(1, embed_dim)
347
+
348
+ # ------------------------------------------------------------------
349
+ # ViT backbone
350
+ self.pos_drop = nn.Dropout(p=drop_rate)
351
+ dpr = [x.item() for x in torch.linspace(0, drop_path, depth)]
352
+ self.blocks = nn.ModuleList(
353
+ [
354
+ Block(
355
+ embed_dim,
356
+ num_heads,
357
+ mlp_ratio,
358
+ qkv_bias=True,
359
+ drop=drop_rate,
360
+ drop_path=dpr[i],
361
+ norm_layer=nn.LayerNorm,
362
+ )
363
+ for i in range(depth)
364
+ ]
365
+ )
366
+ self.norm = nn.LayerNorm(embed_dim)
367
+
368
+ # ------------------------------------------------------------------
369
+ # prediction head
370
+ self.head = nn.ModuleList()
371
+ for _ in range(decoder_depth):
372
+ self.head.append(nn.Linear(embed_dim, embed_dim))
373
+ self.head.append(nn.GELU())
374
+ self.head.append(nn.Linear(embed_dim, len(self.default_vars) * patch_size**2))
375
+ self.head = nn.Sequential(*self.head)
376
+
377
+ # ------------------------------------------------------------------
378
+ self.initialize_weights()
379
+
380
+ def initialize_weights(self):
381
+ # initialize pos_emb and var_emb with sinusoidal encodings
382
+ pos_embed = get_2d_sincos_pos_embed(
383
+ self.pos_embed.shape[-1],
384
+ int(self.img_size[0] / self.patch_size),
385
+ int(self.img_size[1] / self.patch_size),
386
+ cls_token=False,
387
+ )
388
+ self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
389
+
390
+ var_embed = get_1d_sincos_pos_embed_from_grid(
391
+ self.var_embed.shape[-1], np.arange(len(self.default_vars))
392
+ )
393
+ self.var_embed.data.copy_(torch.from_numpy(var_embed).float().unsqueeze(0))
394
+
395
+ # token embedding layers
396
+ for i in range(len(self.token_embeds)):
397
+ w = self.token_embeds[i].proj.weight.data
398
+ trunc_normal_(w.view([w.shape[0], -1]), std=0.02)
399
+
400
+ # initialize nn.Linear and nn.LayerNorm
401
+ self.apply(self._init_weights)
402
+
403
+ def _init_weights(self, m):
404
+ if isinstance(m, nn.Linear):
405
+ trunc_normal_(m.weight, std=0.02)
406
+ if m.bias is not None:
407
+ nn.init.constant_(m.bias, 0)
408
+ elif isinstance(m, nn.LayerNorm):
409
+ nn.init.constant_(m.bias, 0)
410
+ nn.init.constant_(m.weight, 1.0)
411
+
412
+ def create_var_embedding(self, dim):
413
+ var_embed = nn.Parameter(
414
+ torch.zeros(1, len(self.default_vars), dim), requires_grad=True
415
+ )
416
+ var_map = {}
417
+ idx = 0
418
+ for var in self.default_vars:
419
+ var_map[var] = idx
420
+ idx += 1
421
+ return var_embed, var_map
422
+
423
+ @lru_cache(maxsize=None)
424
+ def get_var_ids(self, vars, device):
425
+ ids = np.array([self.var_map[var] for var in vars])
426
+ return torch.from_numpy(ids).to(device)
427
+
428
+ def get_var_emb(self, var_emb, vars):
429
+ ids = self.get_var_ids(vars, var_emb.device)
430
+ return var_emb[:, ids, :]
431
+
432
+ def unpatchify(self, x: torch.Tensor, h=None, w=None):
433
+ """
434
+ x: (B, L, V * patch_size**2)
435
+ return imgs: (B, V, H, W)
436
+ """
437
+ p = self.patch_size
438
+ c = len(self.default_vars)
439
+ h = self.img_size[0] // p if h is None else h // p
440
+ w = self.img_size[1] // p if w is None else w // p
441
+ assert h * w == x.shape[1]
442
+
443
+ x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
444
+ x = torch.einsum("nhwpqc->nchpwq", x)
445
+ imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p))
446
+ return imgs
447
+
448
+ def aggregate_variables(self, x: torch.Tensor):
449
+ """
450
+ Aggregate variable tokens via cross-attention.
451
+
452
+ Args:
453
+ x: (B, V, L, D)
454
+
455
+ Returns:
456
+ (B, L, D)
457
+ """
458
+ b, _, l, _ = x.shape
459
+ x = torch.einsum("bvld->blvd", x)
460
+ x = x.flatten(0, 1) # BxL, V, D
461
+
462
+ var_query = self.var_query.repeat_interleave(x.shape[0], dim=0)
463
+ x, _ = self.var_agg(var_query, x, x) # BxL, 1, D
464
+ x = x.squeeze(dim=1) # BxL, D
465
+
466
+ x = x.unflatten(dim=0, sizes=(b, l)) # B, L, D
467
+ return x
468
+
469
+ def forward_encoder(self, x: torch.Tensor, lead_times: torch.Tensor, variables):
470
+ """Encode input weather state into transformer tokens.
471
+
472
+ Args:
473
+ x: [B, V, H, W] input climate variables
474
+ lead_times: [B] forecasting lead times (hours, normalized by dividing by 100)
475
+ variables: tuple of input variable names
476
+
477
+ Returns:
478
+ [B, L, D] encoded token representations
479
+ """
480
+ if isinstance(variables, list):
481
+ variables = tuple(variables)
482
+
483
+ # tokenize each variable separately
484
+ embeds = []
485
+ var_ids = self.get_var_ids(variables, x.device)
486
+
487
+ for i in range(len(var_ids)):
488
+ id = var_ids[i]
489
+ embeds.append(self.token_embeds[id](x[:, i : i + 1]))
490
+ x = torch.stack(embeds, dim=1) # B, V, L, D
491
+
492
+ # add variable embedding
493
+ var_embed = self.get_var_emb(self.var_embed, variables)
494
+ x = x + var_embed.unsqueeze(2) # B, V, L, D
495
+
496
+ # variable aggregation
497
+ x = self.aggregate_variables(x) # B, L, D
498
+
499
+ # add pos embedding
500
+ x = x + self.pos_embed
501
+
502
+ # add lead time embedding
503
+ lead_time_emb = self.lead_time_embed(lead_times.unsqueeze(-1)) # B, D
504
+ lead_time_emb = lead_time_emb.unsqueeze(1)
505
+ x = x + lead_time_emb # B, L, D
506
+
507
+ x = self.pos_drop(x)
508
+
509
+ # apply Transformer blocks
510
+ for blk in self.blocks:
511
+ x = blk(x)
512
+ x = self.norm(x)
513
+
514
+ return x
515
+
516
+ def forward(self, x, variables, out_variables=None, lead_time=None):
517
+ """Forward pass through ClimaX.
518
+
519
+ This is the onescience-compatible forward that takes input tensor
520
+ and variable lists, returning predicted weather state.
521
+
522
+ Args:
523
+ x: [B, V_in, H, W] input weather/climate variables
524
+ variables: list of input variable name strings
525
+ out_variables: list of output variable name strings (if None, uses all default_vars)
526
+ lead_time: scalar or [B] tensor, forecasting lead time in normalized hours
527
+ (raw_hours / 100). If None, defaults to 0.72 (72 hours).
528
+
529
+ Returns:
530
+ preds: [B, V_out, H, W] predicted weather/climate variables
531
+ """
532
+ if out_variables is None:
533
+ out_variables = self.default_vars
534
+
535
+ if lead_time is None:
536
+ lead_time = 0.72 # default 72 hours / 100
537
+
538
+ if not isinstance(lead_time, torch.Tensor):
539
+ lead_time = torch.full(
540
+ (x.shape[0],), lead_time,
541
+ device=x.device, dtype=x.dtype
542
+ )
543
+
544
+ # Encode
545
+ out_transformers = self.forward_encoder(x, lead_time, variables) # B, L, D
546
+
547
+ # Decode
548
+ preds = self.head(out_transformers) # B, L, V*p*p
549
+ preds = self.unpatchify(preds) # B, V_all, H, W
550
+
551
+ # Select output variables
552
+ out_var_ids = self.get_var_ids(tuple(out_variables), preds.device)
553
+ preds = preds[:, out_var_ids]
554
+
555
+ return preds
scripts/fake_data.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ from pathlib import Path
4
+ root_path = Path(__file__).parent.parent
5
+ sys.path.append(str(root_path))
6
+ import h5py
7
+ import numpy as np
8
+ from onescience.utils.YParams import YParams
9
+
10
+
11
+ # ClimaX uses 5.625° resolution: 32x64 grid, 6-hourly data
12
+ DATASET_DIMS = {"T": 10, "H": 32, "W": 64, "time_step": 6}
13
+
14
+
15
+ def generate_fake_h5(data_dir, var_names, years, dims):
16
+ """
17
+ Generate fake HDF5 files for each year matching the ERA5 format
18
+ expected by onescience's ERA5Dataset.
19
+
20
+ Uses HDF5 chunked datasets with fillvalue=0.0 — unallocated chunks
21
+ return zeros, so files are tiny but have correct shapes.
22
+ Mean/std are embedded in each year's h5 file.
23
+ """
24
+ os.makedirs(os.path.join(data_dir, "data"), exist_ok=True)
25
+ T, C = dims["T"], len(var_names)
26
+ H, W = dims["H"], dims["W"]
27
+
28
+ means = np.zeros((1, C, 1, 1), dtype=np.float32)
29
+ stds = np.ones((1, C, 1, 1), dtype=np.float32)
30
+
31
+ for year in years:
32
+ path = os.path.join(data_dir, "data", f"{year}.h5")
33
+ with h5py.File(path, "w") as f:
34
+ ds = f.create_dataset(
35
+ "fields",
36
+ shape=(T, C, H, W),
37
+ dtype="float32",
38
+ chunks=(1, C, H, W),
39
+ fillvalue=0.0,
40
+ )
41
+ ds.attrs["variables"] = var_names
42
+ ds.attrs["time_step"] = dims["time_step"]
43
+ f.create_dataset("global_means", data=means)
44
+ f.create_dataset("global_stds", data=stds)
45
+
46
+ size_kb = os.path.getsize(path) / 1024
47
+ print(f" {year}.h5 shape=({T},{C},{H},{W}) "
48
+ f"logical={T*C*H*W*4/1024**3:.1f}GB actual={size_kb:.1f}KB")
49
+
50
+
51
+ if __name__ == "__main__":
52
+ cfg_datapipe = YParams("conf/config.yaml", "datapipe")
53
+
54
+ if cfg_datapipe.dataset.data_dir.startswith("/public/") or \
55
+ cfg_datapipe.dataset.data_dir.startswith("/work2/"):
56
+ print("Please check config, ensure data_dir points to local test path "
57
+ "instead of production path.")
58
+ exit()
59
+
60
+ years = (
61
+ cfg_datapipe.dataset.train_time +
62
+ cfg_datapipe.dataset.val_time +
63
+ cfg_datapipe.dataset.test_time
64
+ )
65
+ atm_vars = cfg_datapipe.dataset.channels
66
+
67
+ generate_fake_h5(cfg_datapipe.dataset.data_dir, atm_vars, years, DATASET_DIMS)
68
+
69
+ print("\nFake datasets generated successfully.")
70
+ print(f" Variables: {len(atm_vars)}")
71
+ print(f" Years: {years}")
72
+ print(f" Resolution: {DATASET_DIMS['H']}x{DATASET_DIMS['W']}")
scripts/inference.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import os
3
+ import sys
4
+ from pathlib import Path
5
+ root_path = Path(__file__).parent.parent
6
+ sys.path.append(str(root_path))
7
+ import glob
8
+ import numpy as np
9
+ import h5py
10
+ from tqdm import tqdm
11
+ from model.ClimaX import ClimaX
12
+ from onescience.utils.YParams import YParams
13
+ from onescience.datapipes.climate import ERA5Datapipe
14
+
15
+
16
+ def get_stats(data_dir, channels):
17
+ """Read variable list and normalization params (mean/std) from h5 file."""
18
+ h5_files = sorted(glob.glob(os.path.join(data_dir, "data", "*.h5")))
19
+ with h5py.File(h5_files[0], "r") as f:
20
+ ds = f["fields"]
21
+ all_variables = [
22
+ v.decode() if isinstance(v, bytes) else v for v in ds.attrs["variables"]
23
+ ]
24
+ mu = f["global_means"][:] # [1, C, 1, 1]
25
+ std = f["global_stds"][:]
26
+
27
+ channel_indices = [all_variables.index(v) for v in channels]
28
+ means = mu[:, channel_indices, :, :]
29
+ stds = std[:, channel_indices, :, :]
30
+ return means, stds
31
+
32
+
33
+ if __name__ == "__main__":
34
+ current_path = os.getcwd()
35
+ sys.path.append(current_path)
36
+
37
+ ## Model config init
38
+ config_file_path = os.path.join(current_path, "conf/config.yaml")
39
+ cfg = YParams(config_file_path, "model")
40
+
41
+ ## DataLoader init
42
+ cfg_data = YParams(config_file_path, "datapipe")
43
+
44
+ all_vars = cfg_data.dataset.channels
45
+ out_vars = cfg_data.dataset.out_variables
46
+
47
+ means, stds = get_stats(cfg_data.dataset.data_dir, out_vars)
48
+
49
+ datapipe = ERA5Datapipe(
50
+ dataset_dir=cfg_data.dataset.data_dir,
51
+ used_variables=all_vars,
52
+ used_years=cfg_data.dataset.test_time,
53
+ distributed=False,
54
+ batch_size=1,
55
+ num_workers=4,
56
+ )
57
+ test_dataloader, _ = datapipe.get_dataloader("test")
58
+
59
+ ckpt = torch.load(
60
+ f"{cfg.checkpoint_dir}/model_bak.pth",
61
+ map_location="cuda:0",
62
+ weights_only=False,
63
+ )
64
+ model = ClimaX(
65
+ default_vars=all_vars,
66
+ img_size=cfg.img_size,
67
+ patch_size=cfg.patch_size,
68
+ embed_dim=cfg.embed_dim,
69
+ depth=cfg.depth,
70
+ decoder_depth=cfg.decoder_depth,
71
+ num_heads=cfg.num_heads,
72
+ mlp_ratio=cfg.mlp_ratio,
73
+ drop_path=cfg.drop_path,
74
+ drop_rate=cfg.drop_rate,
75
+ ).to('cuda:0')
76
+ model.load_state_dict(ckpt["model_state_dict"])
77
+
78
+ model.eval()
79
+ os.makedirs('result/output/', exist_ok=True)
80
+ print(f"Saving predictions to './result/output/'")
81
+
82
+ predict_range = cfg.predict_range
83
+ hrs_each_step = cfg.hrs_each_step
84
+ lead_time_val = (predict_range * hrs_each_step) / 100.0
85
+
86
+ with torch.no_grad():
87
+ for data in tqdm(test_dataloader, desc="Inferring testset", unit="batch"):
88
+ invar = data[0].to("cuda:0", dtype=torch.float32)
89
+ filename = data[4][-1][0] # time string from dataloader (input timestamp)
90
+
91
+ preds = model(invar, all_vars, out_vars, lead_time_val)
92
+ pred_var = preds.cpu().numpy()
93
+
94
+ # Denormalize
95
+ pred_var = pred_var * stds + means
96
+ np.save(f"result/output/{filename}.npy", pred_var)
97
+
98
+ print("Inference complete.")
scripts/result.py ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import matplotlib.pyplot as plt
3
+ import os
4
+ import sys
5
+ from pathlib import Path
6
+ root_path = Path(__file__).parent.parent
7
+ sys.path.append(str(root_path))
8
+ import glob
9
+ import h5py
10
+ from datetime import datetime
11
+ from tqdm import tqdm
12
+ from onescience.utils.fcn.YParams import YParams
13
+ from matplotlib import rcParams
14
+
15
+ rcParams['mathtext.fontset'] = 'stix'
16
+ rcParams['axes.linewidth'] = 0.9
17
+ rcParams['xtick.major.width'] = 0.9
18
+ rcParams['ytick.major.width'] = 0.9
19
+
20
+
21
+ def get_metadata(data_dir, channels):
22
+ """Read variable list and time_step from h5 attrs."""
23
+ h5_files = sorted(glob.glob(os.path.join(data_dir, "data", "*.h5")))
24
+ with h5py.File(h5_files[0], "r") as f:
25
+ ds = f["fields"]
26
+ all_variables = [
27
+ v.decode() if isinstance(v, bytes) else v for v in ds.attrs["variables"]
28
+ ]
29
+ time_step = int(ds.attrs["time_step"])
30
+
31
+ channel_indices = [all_variables.index(v) for v in channels]
32
+
33
+ total_files = [f for f in os.listdir('./result/output/') if f.endswith('.npy')]
34
+ total_files.sort()
35
+ return total_files, channel_indices, time_step
36
+
37
+
38
+ def filename_to_index(filename, time_step):
39
+ """Convert YYYYMMDDHH filename to time step index within the year's h5."""
40
+ dt = datetime.strptime(filename, "%Y%m%d%H")
41
+ year_start = datetime(dt.year, 1, 1)
42
+ hours = (dt - year_start).total_seconds() / 3600
43
+ return int(hours / time_step)
44
+
45
+
46
+ def get_result(total_files, channel_indices, time_step, data_dir, clim_mean):
47
+ """Compute per-channel RMSE and ACC (latitude-weighted, matching official ClimaX).
48
+
49
+ Official lat_weighted_acc in climax/utils/metrics.py:
50
+ 1. De-normalizes pred and y
51
+ 2. Subtracts climatology: pred_anom = pred - clim, y_anom = y - clim
52
+ 3. Centers by per-field spatial mean: pred_prime = pred_anom - mean(pred_anom)
53
+ 4. Applies latitude weighting (cos(lat) / mean(cos(lat)))
54
+ 5. Computes: sum(w * pred_prime * y_prime) / sqrt(sum(w * pred_prime^2) * sum(w * y_prime^2))
55
+
56
+ Here we compute an unweighted version since lat array is not readily available;
57
+ for RMSE we use the standard (non-lat-weighted) formula for simplicity.
58
+ """
59
+ channel_rmse = np.zeros(len(channel_indices))
60
+ channel_acc = np.zeros(len(channel_indices))
61
+ clim_mean = clim_mean[0, :, :, :]
62
+
63
+ if not os.path.exists('./result/rmse.npy') or not os.path.exists('result/acc.npy'):
64
+ numerator = np.zeros(len(channel_indices))
65
+ pred_sq_sum = np.zeros(len(channel_indices))
66
+ label_sq_sum = np.zeros(len(channel_indices))
67
+
68
+ for file in tqdm(total_files, unit="files"):
69
+ fname = file[:-4] # remove .npy
70
+ year = fname[:4]
71
+ t_idx = filename_to_index(fname, time_step)
72
+ with h5py.File(os.path.join(data_dir, 'data', f'{year}.h5'), "r") as f:
73
+ label = f["fields"][t_idx] # [C, H, W]
74
+ label = label[channel_indices]
75
+ pred = np.load(f'result/output/{file}').squeeze()
76
+
77
+ # RMSE computation
78
+ channel_rmse += np.sqrt(np.mean((label - pred) ** 2, axis=(1, 2)))
79
+
80
+ # ACC computation (following official ClimaX lat_weighted_acc without lat weighting)
81
+ label_anom = label - clim_mean
82
+ pred_anom = pred - clim_mean
83
+ # Center by per-field spatial mean (official ClimaX approach)
84
+ pred_prime = pred_anom - np.mean(pred_anom, axis=(1, 2), keepdims=True)
85
+ label_prime = label_anom - np.mean(label_anom, axis=(1, 2), keepdims=True)
86
+ # accumulate
87
+ numerator += np.sum(pred_prime * label_prime, axis=(1, 2))
88
+ pred_sq_sum += np.sum(pred_prime ** 2, axis=(1, 2))
89
+ label_sq_sum += np.sum(label_prime ** 2, axis=(1, 2))
90
+
91
+ channel_rmse /= len(total_files)
92
+ channel_acc = numerator / (np.sqrt(pred_sq_sum * label_sq_sum) + 1e-8)
93
+ np.save('./result/acc.npy', channel_acc)
94
+ np.save('./result/rmse.npy', channel_rmse)
95
+
96
+
97
+ def show_result():
98
+ """Print formatted RMSE/ACC table."""
99
+ channel_rmse = np.load('./result/rmse.npy')
100
+ channel_acc = np.load('./result/acc.npy')
101
+
102
+ channels = [cfg_data.dataset.out_variables[i]
103
+ for i in range(len(channel_indices))]
104
+ w = 30
105
+
106
+ print(f"\n{'=' * (w + 32)}")
107
+ print(f"ClimaX Evaluation Results")
108
+ print(f"{'=' * (w + 32)}")
109
+ print(f"┌{'─' * (w + 2)}┬{'─' * 14}┬{'─' * 14}┐")
110
+ print(f"│ {'Channel':<{w}} │ {'RMSE':>12} │ {'ACC':>12} │")
111
+ print(f"├{'─' * (w + 2)}┼{'─' * 14}┼{'─' * 14}┤")
112
+ for i, ch in enumerate(channels):
113
+ print(f"│ {ch:<{w}} │ {channel_rmse[i]:>12.4f} │ {channel_acc[i]:>12.4f} │")
114
+ print(f"├{'─' * (w + 2)}┼{'─' * 14}┼{'─' * 14}┤")
115
+ print(f"│ {'Average':<{w}} │ {np.mean(channel_rmse):>12.4f} │ {np.mean(channel_acc):>12.4f} │")
116
+ print(f"└{'─' * (w + 2)}┴{'─' * 14}┴{'─' * 14}┘")
117
+
118
+
119
+ def plot(label, pred, var, filename):
120
+ """Plot truth, prediction, and difference for a single variable."""
121
+ fig, axes = plt.subplots(1, 3, figsize=(15, 4))
122
+
123
+ xtick_labels = ['180°W', '90°W', '0°', '90°E', '180°E']
124
+ ytick_labels = ['90°S', '45°S', '0°', '45°N', '90°N']
125
+ xticks = np.linspace(0, label.shape[-1] - 1, 5)
126
+ yticks = np.linspace(0, label.shape[-2] - 1, 5)
127
+
128
+ # unified color scale for truth/pred
129
+ vmin = min(label.min(), pred.min())
130
+ vmax = max(label.max(), pred.max())
131
+
132
+ diff = label - pred
133
+ rmse = np.sqrt(np.mean(diff ** 2))
134
+ diff_abs_max = np.abs(diff).max()
135
+
136
+ plot_configs = [
137
+ {'data': label, 'title': 'Truth', 'cmap': 'viridis',
138
+ 'vmin': vmin, 'vmax': vmax},
139
+ {'data': pred, 'title': 'Prediction', 'cmap': 'viridis',
140
+ 'vmin': vmin, 'vmax': vmax},
141
+ {'data': diff, 'title': f'Difference (RMSE={rmse:.2f})',
142
+ 'cmap': 'RdBu_r', 'vmin': -diff_abs_max, 'vmax': diff_abs_max},
143
+ ]
144
+
145
+ for ax, cfg in zip(axes, plot_configs):
146
+ im = ax.imshow(cfg['data'], cmap=cfg['cmap'],
147
+ vmin=cfg['vmin'], vmax=cfg['vmax'])
148
+ ax.set_title(cfg['title'], fontsize=12, pad=4)
149
+ ax.set_xlabel('Longitude')
150
+ ax.set_ylabel('Latitude')
151
+ ax.set_xticks(xticks)
152
+ ax.set_xticklabels(xtick_labels)
153
+ ax.set_yticks(yticks)
154
+ ax.set_yticklabels(ytick_labels)
155
+ plt.colorbar(im, ax=ax, orientation='horizontal')
156
+
157
+ fig.suptitle(var, fontsize=14, fontweight='bold', y=0.98)
158
+ plt.savefig(filename, dpi=300, bbox_inches='tight')
159
+ plt.close()
160
+
161
+
162
+ def plot_loss(train_loss, valid_loss):
163
+ """Plot training and validation loss curves."""
164
+ mask = ~(np.isnan(train_loss) | np.isnan(valid_loss))
165
+ train_loss = train_loss[mask]
166
+ valid_loss = valid_loss[mask]
167
+
168
+ fig, ax = plt.subplots(figsize=(5, 3.5))
169
+ colors = {'train': '#2563EB', 'valid': '#EA580C'}
170
+ epochs = np.arange(1, len(train_loss) + 1)
171
+
172
+ ax.plot(epochs, train_loss, color=colors['train'],
173
+ linewidth=1.5, label='Train')
174
+ ax.plot(epochs, valid_loss, color=colors['valid'],
175
+ linewidth=1.5, label='Valid', linestyle='--')
176
+
177
+ min_idx = np.argmin(valid_loss)
178
+ ax.scatter(epochs[min_idx], valid_loss[min_idx],
179
+ color=colors['valid'], s=40, zorder=5, edgecolors='white')
180
+ ax.annotate(
181
+ f'Best: {valid_loss[min_idx]:.3f}',
182
+ xy=(epochs[min_idx], valid_loss[min_idx]),
183
+ xytext=(10, 10), textcoords='offset points',
184
+ fontsize=8, color=colors['valid'],
185
+ arrowprops=dict(arrowstyle='-', color=colors['valid'], lw=0.5)
186
+ )
187
+
188
+ ax.set(xlabel='Epoch', ylabel='Loss', xlim=(0, len(train_loss) + 1))
189
+ ax.legend(frameon=False, loc='upper right')
190
+ ax.grid(True, linestyle='--', alpha=0.3)
191
+ ax.spines[['top', 'right']].set_visible(False)
192
+
193
+ plt.tight_layout()
194
+ plt.savefig('./result/loss.png', dpi=300, bbox_inches='tight')
195
+ plt.close()
196
+
197
+
198
+ if __name__ == "__main__":
199
+ current_path = os.getcwd()
200
+ sys.path.append(current_path)
201
+ config_file_path = os.path.join(current_path, 'conf/config.yaml')
202
+ cfg = YParams(config_file_path, 'model')
203
+ cfg_data = YParams(config_file_path, "datapipe")
204
+
205
+ train_loss = np.load(f'{cfg.checkpoint_dir}/trloss.npy')
206
+ valid_loss = np.load(f'{cfg.checkpoint_dir}/valoss.npy')
207
+ plot_loss(train_loss, valid_loss)
208
+
209
+ data_dir = cfg_data.dataset.data_dir
210
+ out_vars = cfg_data.dataset.out_variables
211
+ total_files, channel_indices, time_step = get_metadata(data_dir, out_vars)
212
+
213
+ # Load & compute RMSE/ACC per channel
214
+ h5_files = sorted(glob.glob(os.path.join(data_dir, "data", "*.h5")))
215
+ with h5py.File(h5_files[0], "r") as f:
216
+ mu = f["global_means"][:]
217
+ clim_mean = mu[:, channel_indices, :, :]
218
+ get_result(total_files, channel_indices, time_step, data_dir, clim_mean)
219
+ show_result()
220
+
221
+ ##### Plot example predictions #####
222
+ test_year = cfg_data.dataset.test_time[0]
223
+ eg_files = [f'{test_year}010206']
224
+ selected_vars = out_vars[:3] # first 3 output variables
225
+
226
+ print(f"\nPlotting example predictions for: {eg_files}")
227
+ print(f"Variables: {selected_vars}")
228
+
229
+ for file in eg_files:
230
+ year = file[:4]
231
+ t_idx = filename_to_index(file, time_step)
232
+ with h5py.File(os.path.join(data_dir, 'data', f'{year}.h5'), "r") as f:
233
+ label = f["fields"][t_idx]
234
+ label = label[channel_indices]
235
+ pred = np.load(f'result/output/{file}.npy').squeeze()
236
+
237
+ for i, var in enumerate(selected_vars):
238
+ filename = f'./result/{file}_{var}.png'
239
+ plot(label[i], pred[i], var, filename)
240
+ print(f' Saved: {filename}')
scripts/train.py ADDED
@@ -0,0 +1,336 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import os
3
+ import sys
4
+ from pathlib import Path
5
+ root_path = Path(__file__).parent.parent
6
+ sys.path.append(str(root_path))
7
+ import shutil
8
+ import numpy as np
9
+ import torch.distributed as dist
10
+ import logging
11
+ import time
12
+ from tqdm import tqdm
13
+ from torch.nn.parallel import DistributedDataParallel
14
+ from model.ClimaX import ClimaX
15
+ from onescience.datapipes.climate import ERA5Datapipe
16
+ from onescience.utils.YParams import YParams
17
+
18
+
19
+ # ============================================================================
20
+ # Loss function: Latitude-weighted MSE (from official ClimaX metrics.py)
21
+ # ============================================================================
22
+
23
+ def lat_weighted_mse(pred, y, lat):
24
+ """Latitude weighted mean squared error.
25
+
26
+ Allows to weight the loss by the cosine of the latitude to account for
27
+ gridding differences at equator vs. poles.
28
+
29
+ Args:
30
+ y: [B, V, H, W]
31
+ pred: [B, V, H, W]
32
+ lat: [H] latitude array in degrees
33
+
34
+ Returns:
35
+ scalar loss
36
+ """
37
+ error = (pred - y) ** 2 # [B, V, H, W]
38
+
39
+ # latitude weights
40
+ w_lat = np.cos(np.deg2rad(lat))
41
+ w_lat = w_lat / w_lat.mean() # (H,)
42
+ w_lat = torch.from_numpy(w_lat).unsqueeze(0).unsqueeze(-1).to(
43
+ dtype=error.dtype, device=error.device
44
+ ) # (1, H, 1)
45
+
46
+ loss = (error * w_lat.unsqueeze(1)).mean()
47
+ return loss
48
+
49
+
50
+ def get_lat_array(img_size, spatial_res=5.625):
51
+ """Generate latitude array for the grid.
52
+
53
+ Args:
54
+ img_size: [H, W]
55
+ spatial_res: degrees per grid cell
56
+
57
+ Returns:
58
+ lat: [H] latitude values from north to south
59
+ """
60
+ H = img_size[0]
61
+ # Cell centers: from 90 - res/2 to -90 + res/2
62
+ lat = np.linspace(90 - spatial_res / 2, -90 + spatial_res / 2, H)
63
+ return lat.astype(np.float32)
64
+
65
+
66
+ # ============================================================================
67
+ # Training
68
+ # ============================================================================
69
+
70
+ def main():
71
+ logging.basicConfig(
72
+ level=logging.INFO,
73
+ format="%(asctime)s - %(levelname)s - %(message)s"
74
+ )
75
+ logger = logging.getLogger()
76
+
77
+ ## Model config init
78
+ config_file_path = os.path.join(current_path, "conf/config.yaml")
79
+ cfg = YParams(config_file_path, "model")
80
+
81
+ ## Distributed config init
82
+ cfg.world_size = 1
83
+ if "WORLD_SIZE" in os.environ:
84
+ cfg.world_size = int(os.environ["WORLD_SIZE"])
85
+ world_rank = 0
86
+ local_rank = 0
87
+ if cfg.world_size > 1:
88
+ dist.init_process_group(backend="nccl", init_method="env://")
89
+ local_rank = int(os.environ["LOCAL_RANK"])
90
+ world_rank = dist.get_rank()
91
+
92
+ ## DataLoader init
93
+ cfg_data = YParams(config_file_path, "datapipe")
94
+
95
+ # Build variable lists from config
96
+ all_vars = cfg_data.dataset.channels
97
+ out_vars = cfg_data.dataset.out_variables
98
+
99
+ datapipe = ERA5Datapipe(
100
+ dataset_dir=cfg_data.dataset.data_dir,
101
+ used_variables=all_vars,
102
+ used_years=cfg_data.dataset.train_time,
103
+ distributed=dist.is_initialized(),
104
+ )
105
+ train_dataloader, train_sampler = datapipe.get_dataloader("train")
106
+
107
+ datapipe = ERA5Datapipe(
108
+ dataset_dir=cfg_data.dataset.data_dir,
109
+ used_variables=all_vars,
110
+ used_years=cfg_data.dataset.val_time,
111
+ distributed=dist.is_initialized(),
112
+ )
113
+ val_dataloader, val_sampler = datapipe.get_dataloader("valid")
114
+
115
+ ## Model init
116
+ model = ClimaX(
117
+ default_vars=all_vars,
118
+ img_size=cfg.img_size,
119
+ patch_size=cfg.patch_size,
120
+ embed_dim=cfg.embed_dim,
121
+ depth=cfg.depth,
122
+ decoder_depth=cfg.decoder_depth,
123
+ num_heads=cfg.num_heads,
124
+ mlp_ratio=cfg.mlp_ratio,
125
+ drop_path=cfg.drop_path,
126
+ drop_rate=cfg.drop_rate,
127
+ ).to(local_rank)
128
+
129
+ ## Optimizer (following official ClimaX: AdamW with param groups)
130
+ decay = []
131
+ no_decay = []
132
+ for name, m in model.named_parameters():
133
+ if "var_embed" in name or "pos_embed" in name:
134
+ no_decay.append(m)
135
+ else:
136
+ decay.append(m)
137
+
138
+ optimizer = torch.optim.AdamW(
139
+ [
140
+ {
141
+ "params": decay,
142
+ "lr": cfg.lr,
143
+ "betas": (cfg.beta_1, cfg.beta_2),
144
+ "weight_decay": cfg.weight_decay,
145
+ },
146
+ {
147
+ "params": no_decay,
148
+ "lr": cfg.lr,
149
+ "betas": (cfg.beta_1, cfg.beta_2),
150
+ "weight_decay": 0,
151
+ },
152
+ ]
153
+ )
154
+ scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
155
+ optimizer, factor=0.2, patience=5, mode="min"
156
+ )
157
+
158
+ ## Get latitude array for lat-weighted loss
159
+ lat = get_lat_array(cfg.img_size)
160
+
161
+ ## Train process init
162
+ os.makedirs(cfg.checkpoint_dir, exist_ok=True)
163
+ train_loss_file = f"{cfg.checkpoint_dir}/trloss.npy"
164
+ valid_loss_file = f"{cfg.checkpoint_dir}/valoss.npy"
165
+ best_valid_loss = 1.0e6
166
+ best_loss_epoch = 0
167
+ train_losses = np.empty((0,), dtype=np.float32)
168
+ valid_losses = np.empty((0,), dtype=np.float32)
169
+
170
+ ## Get model params count
171
+ if cfg.world_size == 1 or world_rank == 0:
172
+ total_params = sum(p.numel() for p in model.parameters())
173
+ print("\n\n")
174
+ print("-" * 50)
175
+ print(f"Model params: {total_params}, {total_params / 1e6:.2f}M, {total_params / 1e9:.2f}B")
176
+ print("-" * 50, "\n")
177
+
178
+ ## Load model weight if there exists a well-trained model
179
+ if os.path.exists(f"{cfg.checkpoint_dir}/model_bak.pth"):
180
+ if world_rank == 0:
181
+ print("\n\n")
182
+ print("-" * 50)
183
+ print(f"Found existing model weight, loading and continuing training...")
184
+ print(f"If you want to train a new model, remove *.pth from {cfg.checkpoint_dir}")
185
+ print("-" * 50, "\n")
186
+ ckpt = torch.load(
187
+ f"{cfg.checkpoint_dir}/model_bak.pth",
188
+ map_location=f'cuda:{local_rank}',
189
+ weights_only=False,
190
+ )
191
+ model.load_state_dict(ckpt["model_state_dict"])
192
+ optimizer.load_state_dict(ckpt["optimizer_state_dict"])
193
+ scheduler.load_state_dict(ckpt["scheduler_state_dict"])
194
+ best_valid_loss = ckpt["best_valid_loss"]
195
+ best_loss_epoch = ckpt["best_loss_epoch"]
196
+ train_losses = np.load(train_loss_file)
197
+ valid_losses = np.load(valid_loss_file)
198
+
199
+ ## Pre-compute output variable indices (before DDP wrap, for loss computation)
200
+ out_var_indices = model.get_var_ids(tuple(out_vars), torch.device('cpu'))
201
+
202
+ ## Distributed model
203
+ if cfg.world_size > 1:
204
+ model = DistributedDataParallel(
205
+ model, device_ids=[local_rank],
206
+ output_device=local_rank,
207
+ find_unused_parameters=True
208
+ )
209
+
210
+ ## Lead time (fixed for deterministic forecasting)
211
+ predict_range = cfg.predict_range
212
+ hrs_each_step = cfg.hrs_each_step
213
+ lead_time_val = (predict_range * hrs_each_step) / 100.0 # normalized
214
+
215
+ world_rank == 0 and logger.info(f"Starting training... lead_time={lead_time_val}")
216
+
217
+ for epoch in range(cfg.max_epoch):
218
+ if dist.is_initialized():
219
+ train_sampler.set_epoch(epoch)
220
+ val_sampler.set_epoch(epoch)
221
+
222
+ model.train()
223
+ train_loss = 0
224
+ start_time = time.time()
225
+
226
+ for j, data in enumerate(train_dataloader):
227
+ invar = data[0].to(local_rank, dtype=torch.float32) # [B, C_all, H, W]
228
+ outvar = data[1].to(local_rank, dtype=torch.float32) # [B, C_all, H, W]
229
+
230
+ preds = model(invar, all_vars, out_vars, lead_time_val)
231
+
232
+ # Select output variable channels from ground truth
233
+ outvar_selected = outvar[:, out_var_indices.to(outvar.device)]
234
+ loss = lat_weighted_mse(preds, outvar_selected, lat)
235
+
236
+ optimizer.zero_grad()
237
+ loss.backward()
238
+ optimizer.step()
239
+ train_loss += loss.item()
240
+
241
+ if world_rank == 0:
242
+ logger.info(
243
+ f'Train: Epoch {epoch}-{j+1}/{len(train_dataloader)} '
244
+ f'[cost {int((time.time()-start_time) // 60):02}:{int((time.time()-start_time) % 60):02}] '
245
+ f'[{(time.time()-start_time)/(j+1): .02f}s/batch] '
246
+ f'loss:{train_loss / (j+1): .04f}'
247
+ )
248
+
249
+ train_loss /= len(train_dataloader)
250
+
251
+ model.eval()
252
+ valid_loss = 0
253
+ with torch.no_grad():
254
+ for j, data in enumerate(val_dataloader):
255
+ invar = data[0].to(local_rank, dtype=torch.float32)
256
+ outvar = data[1].to(local_rank, dtype=torch.float32)
257
+
258
+ preds = model(invar, all_vars, out_vars, lead_time_val)
259
+
260
+ # Select output variable channels from ground truth
261
+ outvar_selected = outvar[:, out_var_indices.to(outvar.device)]
262
+ loss = lat_weighted_mse(preds, outvar_selected, lat)
263
+
264
+ if cfg.world_size > 1:
265
+ loss_tensor = loss.detach().to(local_rank)
266
+ dist.all_reduce(loss_tensor)
267
+ valid_loss += loss_tensor.item() / cfg.world_size
268
+ else:
269
+ valid_loss += loss.item()
270
+
271
+ if world_rank == 0:
272
+ logger.info(
273
+ f'Valid: Epoch {epoch}-{j+1}/{len(val_dataloader)} '
274
+ f'[cost {int((time.time()-start_time) // 60):02}:{int((time.time()-start_time) % 60):02}] '
275
+ f'[{(time.time()-start_time)/(j+1): .02f}s/batch] '
276
+ f'loss:{valid_loss / (j+1): .04f}'
277
+ )
278
+
279
+ valid_loss /= len(val_dataloader)
280
+ is_save_ckp = False
281
+
282
+ if valid_loss < best_valid_loss:
283
+ best_valid_loss = valid_loss
284
+ best_loss_epoch = epoch
285
+ world_rank == 0 and save_checkpoint(
286
+ model, optimizer, scheduler,
287
+ best_valid_loss, best_loss_epoch, cfg.checkpoint_dir
288
+ )
289
+ is_save_ckp = True
290
+
291
+ scheduler.step(valid_loss)
292
+
293
+ if world_rank == 0:
294
+ logger.info(
295
+ f"Epoch [{epoch + 1}/{cfg.max_epoch}], "
296
+ f"Train Loss: {train_loss:.4f}, "
297
+ f"Valid Loss: {valid_loss:.4f}, "
298
+ f"Best loss at Epoch: {best_loss_epoch + 1}"
299
+ + (", saving checkpoint" if is_save_ckp else "")
300
+ )
301
+ train_losses = np.append(train_losses, train_loss)
302
+ valid_losses = np.append(valid_losses, valid_loss)
303
+
304
+ np.save(train_loss_file, train_losses)
305
+ np.save(valid_loss_file, valid_losses)
306
+
307
+ if epoch - best_loss_epoch > cfg.patience:
308
+ print(f"Loss has not decreased in {cfg.patience} epochs, stopping training...")
309
+ sys.exit()
310
+
311
+
312
+ def save_checkpoint(model, optimizer, scheduler, best_valid_loss,
313
+ best_loss_epoch, model_path):
314
+ # Only save on global rank 0 (guard against misconfigured multi-process launch)
315
+ if dist.is_initialized() and dist.get_rank() != 0:
316
+ return
317
+ model_to_save = model.module if hasattr(model, "module") else model
318
+ state = {
319
+ "model_state_dict": model_to_save.state_dict(),
320
+ "optimizer_state_dict": optimizer.state_dict(),
321
+ "scheduler_state_dict": scheduler.state_dict(),
322
+ "best_valid_loss": best_valid_loss,
323
+ "best_loss_epoch": best_loss_epoch,
324
+ }
325
+ # Write to temporary file first, then atomically rename to avoid
326
+ # race conditions from misconfigured multi-process launches
327
+ tmp_path = f"{model_path}/model_tmp.pth"
328
+ dst_path = f"{model_path}/model_bak.pth"
329
+ torch.save(state, tmp_path)
330
+ shutil.move(tmp_path, dst_path)
331
+
332
+
333
+ if __name__ == "__main__":
334
+ current_path = os.getcwd()
335
+ sys.path.append(current_path)
336
+ main()
weight/.gitkeep ADDED
File without changes