Upload folder using huggingface_hub
Browse files- .gitattributes +41 -9
- README.md +145 -0
- conf/config.yaml +185 -0
- configuration.json +1 -0
- model/ClimaX.py +555 -0
- scripts/fake_data.py +72 -0
- scripts/inference.py +98 -0
- scripts/result.py +240 -0
- scripts/train.py +336 -0
- weight/.gitkeep +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,67 @@
|
|
| 1 |
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 4 |
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
| 5 |
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 6 |
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 11 |
*.model filter=lfs diff=lfs merge=lfs -text
|
| 12 |
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
| 13 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 14 |
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 15 |
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 16 |
*.pb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
| 17 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 18 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 19 |
*.rar filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 20 |
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 21 |
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 22 |
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 23 |
*.tgz filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 24 |
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.db* filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.ark* filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.ggml filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 46 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 47 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
*.wav filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
*.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
*.flac filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
*.ogg filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
*.m4a filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
*.aac filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
*.opus filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
*.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
*.mov filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
*.avi filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
*.mkv filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
*.webm filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
*.obj filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
*.glb filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
*.gltf filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
*.ply filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
*.stl filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
*.fbx filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
|
| 67 |
+
/weight/ filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
datasets:
|
| 3 |
+
- OneScience/ERA5
|
| 4 |
+
frameworks:
|
| 5 |
+
- ""
|
| 6 |
+
language:
|
| 7 |
+
- en
|
| 8 |
+
license: mit
|
| 9 |
+
tags:
|
| 10 |
+
- OneScience
|
| 11 |
+
- Earth Science
|
| 12 |
+
- Weather Forecasting
|
| 13 |
+
- ERA5
|
| 14 |
+
- Regional Forecasting
|
| 15 |
+
tasks: []
|
| 16 |
+
---
|
| 17 |
+
<p align="center">
|
| 18 |
+
<strong>
|
| 19 |
+
<span style="font-size: 30px;">ClimaX</span>
|
| 20 |
+
</strong>
|
| 21 |
+
</p>
|
| 22 |
+
|
| 23 |
+
# Model Overview
|
| 24 |
+
|
| 25 |
+
ClimaX is the first general-purpose foundation model designed for weather and climate science. It was jointly developed by researchers at Microsoft Research and the University of California, Los Angeles (UCLA), and was presented at ICML 2023.
|
| 26 |
+
|
| 27 |
+
Paper: *ClimaX: A Foundation Model for Weather and Climate*
|
| 28 |
+
|
| 29 |
+
https://arxiv.org/abs/2301.10343
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
# Model Description
|
| 33 |
+
|
| 34 |
+
ClimaX is built on the Vision Transformer architecture. It was the first model to unify pre-training on heterogeneous climate datasets and adaptation to downstream tasks, such as regional forecasting and climate projections, within a single framework.
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# Use Cases
|
| 38 |
+
|
| 39 |
+
| Use Case | Description |
|
| 40 |
+
| :---: | :--- |
|
| 41 |
+
| Weather forecasting training | Train ClimaX on ERA5 data in HDF5 format. |
|
| 42 |
+
| Quick local validation | Use synthetic data to validate data loading, model training and inference, and visualization of inference results. |
|
| 43 |
+
| ModelScope/OneCode execution | Download the standalone model package, install its dependencies, and run the included scripts directly. |
|
| 44 |
+
| Multi-GPU training | Launch multi-process training with `torchrun`. |
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# Usage
|
| 48 |
+
|
| 49 |
+
## 1. Using OneCode
|
| 50 |
+
|
| 51 |
+
Use the OneCode online environment for an intelligent, one-click AI4S development experience:
|
| 52 |
+
|
| 53 |
+
[Try one-click AI4S development with OneCode](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home)
|
| 54 |
+
|
| 55 |
+
## 2. Manual Setup
|
| 56 |
+
|
| 57 |
+
**Hardware Requirements**
|
| 58 |
+
|
| 59 |
+
- A GPU or DCU is recommended.
|
| 60 |
+
- A CPU can be used for import checks and connectivity validation with a minimal configuration, but full training and inference will be slow.
|
| 61 |
+
- DCU users must install DTK in advance. DTK 25.04.2 or later is recommended; alternatively, use the OneScience-recommended version compatible with your cluster.
|
| 62 |
+
|
| 63 |
+
### Download the Model Package
|
| 64 |
+
|
| 65 |
+
```bash
|
| 66 |
+
hf download --model OneScience-Group/ClimaX --local-dir ./ClimaX
|
| 67 |
+
cd ClimaX
|
| 68 |
+
```
|
| 69 |
+
|
| 70 |
+
### Set Up the Runtime Environment
|
| 71 |
+
|
| 72 |
+
**DCU Environment**
|
| 73 |
+
|
| 74 |
+
```bash
|
| 75 |
+
# Activate DTK and conda first.
|
| 76 |
+
conda create -n onescience311 python=3.11 -y
|
| 77 |
+
conda activate onescience311
|
| 78 |
+
# Installation with uv is also supported.
|
| 79 |
+
pip install onescience[earth-dcu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
**GPU Environment**
|
| 83 |
+
|
| 84 |
+
```bash
|
| 85 |
+
# Activate conda first.
|
| 86 |
+
conda create -n onescience311 python=3.11 -y libstdcxx-ng=12 libgcc-ng=12 gcc_linux-64=12 gxx_linux-64=12
|
| 87 |
+
conda activate onescience311
|
| 88 |
+
# Installation with uv is also supported.
|
| 89 |
+
pip install onescience[earth-gpu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
### Training Data
|
| 94 |
+
|
| 95 |
+
The OneScience community provides ERA5 data for training. Because of file-size constraints, the repository currently contains a self-contained data slice. Download the data with the following command and ensure that the data path in `conf/config.yaml` is configured correctly:
|
| 96 |
+
|
| 97 |
+
```bash
|
| 98 |
+
hf download --dataset OneScience-Group/ERA5 --local-dir ./data
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
### Training
|
| 102 |
+
|
| 103 |
+
Single GPU:
|
| 104 |
+
|
| 105 |
+
```bash
|
| 106 |
+
python scripts/train.py
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
Multiple GPUs:
|
| 110 |
+
|
| 111 |
+
```bash
|
| 112 |
+
torchrun --nproc_per_node=8 scripts/train.py
|
| 113 |
+
```
|
| 114 |
+
|
| 115 |
+
Training saves the `model_bak.pth` checkpoint under `data/checkpoints/`.
|
| 116 |
+
|
| 117 |
+
### Pre-trained Weights
|
| 118 |
+
|
| 119 |
+
This repository will provide weights trained on ERA5 data in the `weights/` directory. The weight files are being prepared and will be uploaded soon.
|
| 120 |
+
|
| 121 |
+
### Inference
|
| 122 |
+
|
| 123 |
+
```bash
|
| 124 |
+
python scripts/inference.py
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
Inference results are saved to `result/output/`.
|
| 128 |
+
|
| 129 |
+
### Evaluation and Visualization
|
| 130 |
+
|
| 131 |
+
```bash
|
| 132 |
+
python scripts/result.py
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
# Official OneScience Resources
|
| 137 |
+
|
| 138 |
+
| Platform | OneScience Main Repository | Skills Repository |
|
| 139 |
+
| --- | --- | --- |
|
| 140 |
+
| Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills |
|
| 141 |
+
| GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills |
|
| 142 |
+
|
| 143 |
+
# Citation and License
|
| 144 |
+
|
| 145 |
+
- This repository is a reproduction of the original ClimaX paper and is licensed under the MIT License.
|
conf/config.yaml
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model:
|
| 2 |
+
|
| 3 |
+
start_epoch: 0
|
| 4 |
+
max_epoch: 10
|
| 5 |
+
lr: 5e-4
|
| 6 |
+
beta_1: 0.9
|
| 7 |
+
beta_2: 0.99
|
| 8 |
+
weight_decay: 1e-5
|
| 9 |
+
warmup_epochs: 10000
|
| 10 |
+
warmup_start_lr: 1e-8
|
| 11 |
+
eta_min: 1e-8
|
| 12 |
+
|
| 13 |
+
# ClimaX architecture params (from official config)
|
| 14 |
+
img_size: [32, 64] # 5.625° resolution (lat, lon)
|
| 15 |
+
patch_size: 2
|
| 16 |
+
embed_dim: 1024
|
| 17 |
+
depth: 8
|
| 18 |
+
decoder_depth: 2
|
| 19 |
+
num_heads: 16
|
| 20 |
+
mlp_ratio: 4.0
|
| 21 |
+
drop_path: 0.1
|
| 22 |
+
drop_rate: 0.1
|
| 23 |
+
|
| 24 |
+
# Forecasting settings
|
| 25 |
+
# NOTE: ERA5 data has 6-hour time steps (time_step=6 in HDF5 files).
|
| 26 |
+
# The lead_time passed to the model is (predict_range * hrs_each_step) / 100.
|
| 27 |
+
# For 6-hour forecast: predict_range=6, hrs_each_step=1 → lead_time=0.06
|
| 28 |
+
# For 72-hour forecast: predict_range=72, hrs_each_step=1 → lead_time=0.72
|
| 29 |
+
# The model conditions on lead_time; it must match the actual data gap.
|
| 30 |
+
predict_range: 6 # forecast lead time in hours
|
| 31 |
+
hrs_each_step: 1 # hours per data step factor for lead_time computation
|
| 32 |
+
|
| 33 |
+
checkpoint_dir: "./data/checkpoints"
|
| 34 |
+
patience: 50
|
| 35 |
+
|
| 36 |
+
# 整个数据读取流程
|
| 37 |
+
datapipe:
|
| 38 |
+
name: "ERA5"
|
| 39 |
+
task: "weather_forecasting"
|
| 40 |
+
|
| 41 |
+
# dataset设定
|
| 42 |
+
dataset:
|
| 43 |
+
type: "hdf5"
|
| 44 |
+
data_dir: './data/' # "$ONESCIENCE_DATASETS_DIR/ERA5/newh5/"
|
| 45 |
+
|
| 46 |
+
train_time: [2000, 2001]
|
| 47 |
+
val_time: [2002]
|
| 48 |
+
test_time: [2003]
|
| 49 |
+
img_size: [32, 64]
|
| 50 |
+
verbose: true
|
| 51 |
+
cache: false
|
| 52 |
+
|
| 53 |
+
# ClimaX 48 input variables: 3 constants + 3 surface + 42 pressure-level
|
| 54 |
+
channels:
|
| 55 |
+
# Static constants (3)
|
| 56 |
+
- "land_sea_mask"
|
| 57 |
+
- "orography"
|
| 58 |
+
- "lattitude"
|
| 59 |
+
|
| 60 |
+
# Surface variables (3)
|
| 61 |
+
- "2m_temperature"
|
| 62 |
+
- "10m_u_component_of_wind"
|
| 63 |
+
- "10m_v_component_of_wind"
|
| 64 |
+
|
| 65 |
+
# Geopotential at 7 pressure levels
|
| 66 |
+
- "geopotential_50"
|
| 67 |
+
- "geopotential_250"
|
| 68 |
+
- "geopotential_500"
|
| 69 |
+
- "geopotential_600"
|
| 70 |
+
- "geopotential_700"
|
| 71 |
+
- "geopotential_850"
|
| 72 |
+
- "geopotential_925"
|
| 73 |
+
|
| 74 |
+
# U component of wind at 7 pressure levels
|
| 75 |
+
- "u_component_of_wind_50"
|
| 76 |
+
- "u_component_of_wind_250"
|
| 77 |
+
- "u_component_of_wind_500"
|
| 78 |
+
- "u_component_of_wind_600"
|
| 79 |
+
- "u_component_of_wind_700"
|
| 80 |
+
- "u_component_of_wind_850"
|
| 81 |
+
- "u_component_of_wind_925"
|
| 82 |
+
|
| 83 |
+
# V component of wind at 7 pressure levels
|
| 84 |
+
- "v_component_of_wind_50"
|
| 85 |
+
- "v_component_of_wind_250"
|
| 86 |
+
- "v_component_of_wind_500"
|
| 87 |
+
- "v_component_of_wind_600"
|
| 88 |
+
- "v_component_of_wind_700"
|
| 89 |
+
- "v_component_of_wind_850"
|
| 90 |
+
- "v_component_of_wind_925"
|
| 91 |
+
|
| 92 |
+
# Temperature at 7 pressure levels
|
| 93 |
+
- "temperature_50"
|
| 94 |
+
- "temperature_250"
|
| 95 |
+
- "temperature_500"
|
| 96 |
+
- "temperature_600"
|
| 97 |
+
- "temperature_700"
|
| 98 |
+
- "temperature_850"
|
| 99 |
+
- "temperature_925"
|
| 100 |
+
|
| 101 |
+
# Relative humidity at 7 pressure levels
|
| 102 |
+
- "relative_humidity_50"
|
| 103 |
+
- "relative_humidity_250"
|
| 104 |
+
- "relative_humidity_500"
|
| 105 |
+
- "relative_humidity_600"
|
| 106 |
+
- "relative_humidity_700"
|
| 107 |
+
- "relative_humidity_850"
|
| 108 |
+
- "relative_humidity_925"
|
| 109 |
+
|
| 110 |
+
# Specific humidity at 7 pressure levels
|
| 111 |
+
- "specific_humidity_50"
|
| 112 |
+
- "specific_humidity_250"
|
| 113 |
+
- "specific_humidity_500"
|
| 114 |
+
- "specific_humidity_600"
|
| 115 |
+
- "specific_humidity_700"
|
| 116 |
+
- "specific_humidity_850"
|
| 117 |
+
- "specific_humidity_925"
|
| 118 |
+
|
| 119 |
+
# ClimaX output variables (5)
|
| 120 |
+
out_variables:
|
| 121 |
+
- "geopotential_500"
|
| 122 |
+
- "temperature_850"
|
| 123 |
+
- "2m_temperature"
|
| 124 |
+
- "10m_u_component_of_wind"
|
| 125 |
+
- "10m_v_component_of_wind"
|
| 126 |
+
|
| 127 |
+
# Short names for evaluation
|
| 128 |
+
variables:
|
| 129 |
+
- "z500"
|
| 130 |
+
- "t850"
|
| 131 |
+
- "t2m"
|
| 132 |
+
- "u10"
|
| 133 |
+
- "v10"
|
| 134 |
+
|
| 135 |
+
# 时间配置
|
| 136 |
+
time_range: ["2000-01-01", "2020-12-31"]
|
| 137 |
+
time_steps: 1
|
| 138 |
+
time_res: 6
|
| 139 |
+
|
| 140 |
+
# 空间配置
|
| 141 |
+
spatial_resolution: [5.625, 5.625]
|
| 142 |
+
|
| 143 |
+
# 采样配置
|
| 144 |
+
num_samples: -1 # -1 表示使用全部数据
|
| 145 |
+
shuffle: true
|
| 146 |
+
random_seed: 42
|
| 147 |
+
|
| 148 |
+
# 领域特定配置
|
| 149 |
+
extra:
|
| 150 |
+
levels: [500, 850]
|
| 151 |
+
lat_range: [-90, 90]
|
| 152 |
+
lon_range: [0, 360]
|
| 153 |
+
|
| 154 |
+
# 数据转换配置
|
| 155 |
+
transforms:
|
| 156 |
+
- type: "Normalize"
|
| 157 |
+
params:
|
| 158 |
+
mean: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
|
| 159 |
+
std: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
|
| 160 |
+
keys: ["input", "target"]
|
| 161 |
+
|
| 162 |
+
- type: "ToTensor"
|
| 163 |
+
params:
|
| 164 |
+
keys: null # null表示转换所有numpy数组
|
| 165 |
+
|
| 166 |
+
# DataLoader配置
|
| 167 |
+
dataloader:
|
| 168 |
+
mask_dtype: "float32"
|
| 169 |
+
batch_size: 1
|
| 170 |
+
num_workers: 1
|
| 171 |
+
pin_memory: true
|
| 172 |
+
drop_last: true
|
| 173 |
+
shuffle: false # 使用sampler时设为false
|
| 174 |
+
prefetch_factor: 2
|
| 175 |
+
persistent_workers: true
|
| 176 |
+
|
| 177 |
+
# 分布式配置
|
| 178 |
+
distributed:
|
| 179 |
+
enabled: true
|
| 180 |
+
sampler: "DistributedSampler"
|
| 181 |
+
rank: 0
|
| 182 |
+
world_size: 4
|
| 183 |
+
shuffle: true
|
| 184 |
+
seed: 42
|
| 185 |
+
drop_last: true
|
configuration.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"framework":"Pytorch","task":"other"}
|
model/ClimaX.py
ADDED
|
@@ -0,0 +1,555 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ClimaX: A foundation model for weather and climate.
|
| 3 |
+
|
| 4 |
+
Reference:
|
| 5 |
+
- "ClimaX: A foundation model for weather and climate" (arXiv:2301.10343)
|
| 6 |
+
- Official repo: https://github.com/microsoft/ClimaX
|
| 7 |
+
|
| 8 |
+
This implementation:
|
| 9 |
+
- Removes dependency on timm (PatchEmbed, Block, trunc_normal_ reimplemented)
|
| 10 |
+
- Removes dependency on pytorch_lightning
|
| 11 |
+
- Compatible with onescience framework
|
| 12 |
+
- Follows official code logic and precision exactly
|
| 13 |
+
- Supports training from scratch without pretrained weights
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import math
|
| 17 |
+
import numpy as np
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from functools import lru_cache
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
import torch.nn as nn
|
| 23 |
+
import torch.nn.functional as F
|
| 24 |
+
|
| 25 |
+
from onescience.models.meta import ModelMetaData
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
# ============================================================================
|
| 29 |
+
# Model metadata for onescience framework
|
| 30 |
+
# ============================================================================
|
| 31 |
+
|
| 32 |
+
@dataclass
|
| 33 |
+
class MetaData(ModelMetaData):
|
| 34 |
+
name: str = "ClimaX"
|
| 35 |
+
jit: bool = False
|
| 36 |
+
cuda_graphs: bool = True
|
| 37 |
+
amp: bool = True
|
| 38 |
+
amp_cpu: bool = None
|
| 39 |
+
amp_gpu: bool = None
|
| 40 |
+
onnx_cpu: bool = False
|
| 41 |
+
onnx_gpu: bool = True
|
| 42 |
+
onnx_runtime: bool = True
|
| 43 |
+
var_dim: int = 1
|
| 44 |
+
func_torch: bool = False
|
| 45 |
+
auto_grad: bool = False
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# ============================================================================
|
| 49 |
+
# Utility functions (replacing timm dependencies)
|
| 50 |
+
# ============================================================================
|
| 51 |
+
|
| 52 |
+
def _trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
|
| 53 |
+
"""Truncated normal initialization (replaces timm's trunc_normal_)."""
|
| 54 |
+
def norm_cdf(x):
|
| 55 |
+
return (1. + math.erf(x / math.sqrt(2.))) / 2.
|
| 56 |
+
|
| 57 |
+
l = norm_cdf((a - mean) / std)
|
| 58 |
+
u = norm_cdf((b - mean) / std)
|
| 59 |
+
|
| 60 |
+
tensor.uniform_(2 * l - 1, 2 * u - 1)
|
| 61 |
+
tensor.erfinv_()
|
| 62 |
+
tensor.mul_(std * math.sqrt(2.))
|
| 63 |
+
tensor.add_(mean)
|
| 64 |
+
tensor.clamp_(min=a, max=b)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def trunc_normal_(tensor, std=0.02):
|
| 68 |
+
"""Drop-in replacement for timm's trunc_normal_ (wrapped with no_grad)."""
|
| 69 |
+
with torch.no_grad():
|
| 70 |
+
_trunc_normal_(tensor, mean=0., std=std, a=-2., b=2.)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# ============================================================================
|
| 74 |
+
# Position embedding utilities (from official ClimaX pos_embed.py)
|
| 75 |
+
# ============================================================================
|
| 76 |
+
|
| 77 |
+
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
| 78 |
+
"""
|
| 79 |
+
embed_dim: output dimension for each position
|
| 80 |
+
pos: a list of positions to be encoded: size (M,)
|
| 81 |
+
out: (M, D)
|
| 82 |
+
"""
|
| 83 |
+
assert embed_dim % 2 == 0
|
| 84 |
+
omega = np.arange(embed_dim // 2, dtype=float)
|
| 85 |
+
omega /= embed_dim / 2.0
|
| 86 |
+
omega = 1.0 / 10000 ** omega # (D/2,)
|
| 87 |
+
|
| 88 |
+
pos = pos.reshape(-1) # (M,)
|
| 89 |
+
out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
|
| 90 |
+
|
| 91 |
+
emb_sin = np.sin(out) # (M, D/2)
|
| 92 |
+
emb_cos = np.cos(out) # (M, D/2)
|
| 93 |
+
|
| 94 |
+
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
| 95 |
+
return emb
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def get_2d_sincos_pos_embed(embed_dim, grid_size_h, grid_size_w, cls_token=False):
|
| 99 |
+
"""
|
| 100 |
+
grid_size_h: int of the grid height
|
| 101 |
+
grid_size_w: int of the grid width
|
| 102 |
+
return:
|
| 103 |
+
pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim]
|
| 104 |
+
"""
|
| 105 |
+
grid_h = np.arange(grid_size_h, dtype=np.float32)
|
| 106 |
+
grid_w = np.arange(grid_size_w, dtype=np.float32)
|
| 107 |
+
grid = np.meshgrid(grid_w, grid_h) # w goes first
|
| 108 |
+
grid = np.stack(grid, axis=0)
|
| 109 |
+
|
| 110 |
+
grid = grid.reshape([2, 1, grid_size_h, grid_size_w])
|
| 111 |
+
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
|
| 112 |
+
if cls_token:
|
| 113 |
+
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
|
| 114 |
+
return pos_embed
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
|
| 118 |
+
assert embed_dim % 2 == 0
|
| 119 |
+
emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
|
| 120 |
+
emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
|
| 121 |
+
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
| 122 |
+
return emb
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# ============================================================================
|
| 126 |
+
# Basic building blocks (replacing timm dependencies)
|
| 127 |
+
# ============================================================================
|
| 128 |
+
|
| 129 |
+
class Mlp(nn.Module):
|
| 130 |
+
"""MLP with GELU activation (replaces timm.layers.Mlp)."""
|
| 131 |
+
|
| 132 |
+
def __init__(
|
| 133 |
+
self,
|
| 134 |
+
in_features,
|
| 135 |
+
hidden_features=None,
|
| 136 |
+
out_features=None,
|
| 137 |
+
act_layer=nn.GELU,
|
| 138 |
+
drop=0.,
|
| 139 |
+
):
|
| 140 |
+
super().__init__()
|
| 141 |
+
out_features = out_features or in_features
|
| 142 |
+
hidden_features = hidden_features or in_features
|
| 143 |
+
self.fc1 = nn.Linear(in_features, hidden_features)
|
| 144 |
+
self.act = act_layer()
|
| 145 |
+
self.fc2 = nn.Linear(hidden_features, out_features)
|
| 146 |
+
self.drop = nn.Dropout(drop)
|
| 147 |
+
|
| 148 |
+
def forward(self, x):
|
| 149 |
+
x = self.fc1(x)
|
| 150 |
+
x = self.act(x)
|
| 151 |
+
x = self.drop(x)
|
| 152 |
+
x = self.fc2(x)
|
| 153 |
+
x = self.drop(x)
|
| 154 |
+
return x
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class DropPath(nn.Module):
|
| 158 |
+
"""Drop paths (Stochastic Depth) per sample (replaces timm's DropPath)."""
|
| 159 |
+
|
| 160 |
+
def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):
|
| 161 |
+
super().__init__()
|
| 162 |
+
self.drop_prob = drop_prob
|
| 163 |
+
self.scale_by_keep = scale_by_keep
|
| 164 |
+
|
| 165 |
+
def forward(self, x):
|
| 166 |
+
if self.drop_prob == 0. or not self.training:
|
| 167 |
+
return x
|
| 168 |
+
keep_prob = 1 - self.drop_prob
|
| 169 |
+
shape = (x.shape[0],) + (1,) * (x.ndim - 1)
|
| 170 |
+
random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)
|
| 171 |
+
random_tensor.floor_()
|
| 172 |
+
output = x.div(keep_prob) * random_tensor
|
| 173 |
+
return output
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class PatchEmbed(nn.Module):
|
| 177 |
+
"""2D Image to Patch Embedding (replaces timm's PatchEmbed).
|
| 178 |
+
|
| 179 |
+
Splits image into patches and embeds each patch via Conv2d.
|
| 180 |
+
"""
|
| 181 |
+
|
| 182 |
+
def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
|
| 183 |
+
super().__init__()
|
| 184 |
+
if isinstance(img_size, int):
|
| 185 |
+
img_size = (img_size, img_size)
|
| 186 |
+
if isinstance(patch_size, int):
|
| 187 |
+
patch_size = (patch_size, patch_size)
|
| 188 |
+
self.img_size = img_size
|
| 189 |
+
self.patch_size = patch_size
|
| 190 |
+
self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
|
| 191 |
+
self.num_patches = self.grid_size[0] * self.grid_size[1]
|
| 192 |
+
|
| 193 |
+
self.proj = nn.Conv2d(
|
| 194 |
+
in_chans, embed_dim,
|
| 195 |
+
kernel_size=patch_size, stride=patch_size
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
def forward(self, x):
|
| 199 |
+
B, C, H, W = x.shape
|
| 200 |
+
x = self.proj(x) # B, D, H/p, W/p
|
| 201 |
+
x = x.flatten(2).transpose(1, 2) # B, L, D
|
| 202 |
+
return x
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
class Attention(nn.Module):
|
| 206 |
+
"""Multi-head self-attention (replaces timm's Attention)."""
|
| 207 |
+
|
| 208 |
+
def __init__(
|
| 209 |
+
self,
|
| 210 |
+
dim,
|
| 211 |
+
num_heads=8,
|
| 212 |
+
qkv_bias=False,
|
| 213 |
+
attn_drop=0.,
|
| 214 |
+
proj_drop=0.,
|
| 215 |
+
):
|
| 216 |
+
super().__init__()
|
| 217 |
+
self.num_heads = num_heads
|
| 218 |
+
head_dim = dim // num_heads
|
| 219 |
+
self.scale = head_dim ** -0.5
|
| 220 |
+
|
| 221 |
+
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
| 222 |
+
self.attn_drop = nn.Dropout(attn_drop)
|
| 223 |
+
self.proj = nn.Linear(dim, dim)
|
| 224 |
+
self.proj_drop = nn.Dropout(proj_drop)
|
| 225 |
+
|
| 226 |
+
def forward(self, x):
|
| 227 |
+
B, N, C = x.shape
|
| 228 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
|
| 229 |
+
q, k, v = qkv.unbind(dim=2)
|
| 230 |
+
q = q.permute(0, 2, 1, 3) # B, num_heads, N, head_dim
|
| 231 |
+
k = k.permute(0, 2, 1, 3)
|
| 232 |
+
v = v.permute(0, 2, 1, 3)
|
| 233 |
+
|
| 234 |
+
x = F.scaled_dot_product_attention(
|
| 235 |
+
q, k, v,
|
| 236 |
+
dropout_p=self.attn_drop.p if self.attn_drop.p > 0.0 else 0.0,
|
| 237 |
+
scale=self.scale,
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
x = x.permute(0, 2, 1, 3).reshape(B, N, C)
|
| 241 |
+
x = self.proj(x)
|
| 242 |
+
x = self.proj_drop(x)
|
| 243 |
+
return x
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
class Block(nn.Module):
|
| 247 |
+
"""ViT Block with LayerNorm, Attention, and MLP (replaces timm's Block)."""
|
| 248 |
+
|
| 249 |
+
def __init__(
|
| 250 |
+
self,
|
| 251 |
+
dim,
|
| 252 |
+
num_heads,
|
| 253 |
+
mlp_ratio=4.,
|
| 254 |
+
qkv_bias=False,
|
| 255 |
+
drop=0.,
|
| 256 |
+
attn_drop=0.,
|
| 257 |
+
drop_path=0.,
|
| 258 |
+
norm_layer=nn.LayerNorm,
|
| 259 |
+
):
|
| 260 |
+
super().__init__()
|
| 261 |
+
self.norm1 = norm_layer(dim)
|
| 262 |
+
self.attn = Attention(
|
| 263 |
+
dim,
|
| 264 |
+
num_heads=num_heads,
|
| 265 |
+
qkv_bias=qkv_bias,
|
| 266 |
+
attn_drop=attn_drop,
|
| 267 |
+
proj_drop=drop,
|
| 268 |
+
)
|
| 269 |
+
self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
| 270 |
+
|
| 271 |
+
self.norm2 = norm_layer(dim)
|
| 272 |
+
self.mlp = Mlp(
|
| 273 |
+
in_features=dim,
|
| 274 |
+
hidden_features=int(dim * mlp_ratio),
|
| 275 |
+
act_layer=nn.GELU,
|
| 276 |
+
drop=drop,
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
def forward(self, x):
|
| 280 |
+
x = x + self.drop_path(self.attn(self.norm1(x)))
|
| 281 |
+
x = x + self.drop_path(self.mlp(self.norm2(x)))
|
| 282 |
+
return x
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
# ============================================================================
|
| 286 |
+
# Main ClimaX Model
|
| 287 |
+
# ============================================================================
|
| 288 |
+
|
| 289 |
+
class ClimaX(nn.Module):
|
| 290 |
+
"""Implements the ClimaX model as described in the paper,
|
| 291 |
+
https://arxiv.org/abs/2301.10343
|
| 292 |
+
|
| 293 |
+
This is the base ClimaX architecture for global weather forecasting.
|
| 294 |
+
It uses per-variable tokenization, cross-attention variable aggregation,
|
| 295 |
+
a ViT backbone, and an MLP prediction head.
|
| 296 |
+
|
| 297 |
+
Args:
|
| 298 |
+
default_vars (list): list of default variables to be used for training
|
| 299 |
+
img_size (list): image size of the input data [H, W]
|
| 300 |
+
patch_size (int): patch size of the input data
|
| 301 |
+
embed_dim (int): embedding dimension
|
| 302 |
+
depth (int): number of transformer layers
|
| 303 |
+
decoder_depth (int): number of decoder layers
|
| 304 |
+
num_heads (int): number of attention heads
|
| 305 |
+
mlp_ratio (float): ratio of mlp hidden dimension to embedding dimension
|
| 306 |
+
drop_path (float): stochastic depth rate
|
| 307 |
+
drop_rate (float): dropout rate
|
| 308 |
+
"""
|
| 309 |
+
|
| 310 |
+
def __init__(
|
| 311 |
+
self,
|
| 312 |
+
default_vars,
|
| 313 |
+
img_size=(32, 64),
|
| 314 |
+
patch_size=2,
|
| 315 |
+
embed_dim=1024,
|
| 316 |
+
depth=8,
|
| 317 |
+
decoder_depth=2,
|
| 318 |
+
num_heads=16,
|
| 319 |
+
mlp_ratio=4.0,
|
| 320 |
+
drop_path=0.1,
|
| 321 |
+
drop_rate=0.1,
|
| 322 |
+
):
|
| 323 |
+
super().__init__()
|
| 324 |
+
|
| 325 |
+
self.img_size = tuple(img_size)
|
| 326 |
+
self.patch_size = patch_size
|
| 327 |
+
self.default_vars = default_vars
|
| 328 |
+
|
| 329 |
+
# variable tokenization: separate embedding layer for each input variable
|
| 330 |
+
self.token_embeds = nn.ModuleList(
|
| 331 |
+
[PatchEmbed(img_size, patch_size, 1, embed_dim) for _ in range(len(default_vars))]
|
| 332 |
+
)
|
| 333 |
+
self.num_patches = self.token_embeds[0].num_patches
|
| 334 |
+
|
| 335 |
+
# variable embedding to denote which variable each token belongs to
|
| 336 |
+
self.var_embed, self.var_map = self.create_var_embedding(embed_dim)
|
| 337 |
+
|
| 338 |
+
# variable aggregation: a learnable query and a single-layer cross attention
|
| 339 |
+
self.var_query = nn.Parameter(torch.zeros(1, 1, embed_dim), requires_grad=True)
|
| 340 |
+
self.var_agg = nn.MultiheadAttention(embed_dim, num_heads, batch_first=True)
|
| 341 |
+
|
| 342 |
+
# positional embedding and lead time embedding
|
| 343 |
+
self.pos_embed = nn.Parameter(
|
| 344 |
+
torch.zeros(1, self.num_patches, embed_dim), requires_grad=True
|
| 345 |
+
)
|
| 346 |
+
self.lead_time_embed = nn.Linear(1, embed_dim)
|
| 347 |
+
|
| 348 |
+
# ------------------------------------------------------------------
|
| 349 |
+
# ViT backbone
|
| 350 |
+
self.pos_drop = nn.Dropout(p=drop_rate)
|
| 351 |
+
dpr = [x.item() for x in torch.linspace(0, drop_path, depth)]
|
| 352 |
+
self.blocks = nn.ModuleList(
|
| 353 |
+
[
|
| 354 |
+
Block(
|
| 355 |
+
embed_dim,
|
| 356 |
+
num_heads,
|
| 357 |
+
mlp_ratio,
|
| 358 |
+
qkv_bias=True,
|
| 359 |
+
drop=drop_rate,
|
| 360 |
+
drop_path=dpr[i],
|
| 361 |
+
norm_layer=nn.LayerNorm,
|
| 362 |
+
)
|
| 363 |
+
for i in range(depth)
|
| 364 |
+
]
|
| 365 |
+
)
|
| 366 |
+
self.norm = nn.LayerNorm(embed_dim)
|
| 367 |
+
|
| 368 |
+
# ------------------------------------------------------------------
|
| 369 |
+
# prediction head
|
| 370 |
+
self.head = nn.ModuleList()
|
| 371 |
+
for _ in range(decoder_depth):
|
| 372 |
+
self.head.append(nn.Linear(embed_dim, embed_dim))
|
| 373 |
+
self.head.append(nn.GELU())
|
| 374 |
+
self.head.append(nn.Linear(embed_dim, len(self.default_vars) * patch_size**2))
|
| 375 |
+
self.head = nn.Sequential(*self.head)
|
| 376 |
+
|
| 377 |
+
# ------------------------------------------------------------------
|
| 378 |
+
self.initialize_weights()
|
| 379 |
+
|
| 380 |
+
def initialize_weights(self):
|
| 381 |
+
# initialize pos_emb and var_emb with sinusoidal encodings
|
| 382 |
+
pos_embed = get_2d_sincos_pos_embed(
|
| 383 |
+
self.pos_embed.shape[-1],
|
| 384 |
+
int(self.img_size[0] / self.patch_size),
|
| 385 |
+
int(self.img_size[1] / self.patch_size),
|
| 386 |
+
cls_token=False,
|
| 387 |
+
)
|
| 388 |
+
self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
|
| 389 |
+
|
| 390 |
+
var_embed = get_1d_sincos_pos_embed_from_grid(
|
| 391 |
+
self.var_embed.shape[-1], np.arange(len(self.default_vars))
|
| 392 |
+
)
|
| 393 |
+
self.var_embed.data.copy_(torch.from_numpy(var_embed).float().unsqueeze(0))
|
| 394 |
+
|
| 395 |
+
# token embedding layers
|
| 396 |
+
for i in range(len(self.token_embeds)):
|
| 397 |
+
w = self.token_embeds[i].proj.weight.data
|
| 398 |
+
trunc_normal_(w.view([w.shape[0], -1]), std=0.02)
|
| 399 |
+
|
| 400 |
+
# initialize nn.Linear and nn.LayerNorm
|
| 401 |
+
self.apply(self._init_weights)
|
| 402 |
+
|
| 403 |
+
def _init_weights(self, m):
|
| 404 |
+
if isinstance(m, nn.Linear):
|
| 405 |
+
trunc_normal_(m.weight, std=0.02)
|
| 406 |
+
if m.bias is not None:
|
| 407 |
+
nn.init.constant_(m.bias, 0)
|
| 408 |
+
elif isinstance(m, nn.LayerNorm):
|
| 409 |
+
nn.init.constant_(m.bias, 0)
|
| 410 |
+
nn.init.constant_(m.weight, 1.0)
|
| 411 |
+
|
| 412 |
+
def create_var_embedding(self, dim):
|
| 413 |
+
var_embed = nn.Parameter(
|
| 414 |
+
torch.zeros(1, len(self.default_vars), dim), requires_grad=True
|
| 415 |
+
)
|
| 416 |
+
var_map = {}
|
| 417 |
+
idx = 0
|
| 418 |
+
for var in self.default_vars:
|
| 419 |
+
var_map[var] = idx
|
| 420 |
+
idx += 1
|
| 421 |
+
return var_embed, var_map
|
| 422 |
+
|
| 423 |
+
@lru_cache(maxsize=None)
|
| 424 |
+
def get_var_ids(self, vars, device):
|
| 425 |
+
ids = np.array([self.var_map[var] for var in vars])
|
| 426 |
+
return torch.from_numpy(ids).to(device)
|
| 427 |
+
|
| 428 |
+
def get_var_emb(self, var_emb, vars):
|
| 429 |
+
ids = self.get_var_ids(vars, var_emb.device)
|
| 430 |
+
return var_emb[:, ids, :]
|
| 431 |
+
|
| 432 |
+
def unpatchify(self, x: torch.Tensor, h=None, w=None):
|
| 433 |
+
"""
|
| 434 |
+
x: (B, L, V * patch_size**2)
|
| 435 |
+
return imgs: (B, V, H, W)
|
| 436 |
+
"""
|
| 437 |
+
p = self.patch_size
|
| 438 |
+
c = len(self.default_vars)
|
| 439 |
+
h = self.img_size[0] // p if h is None else h // p
|
| 440 |
+
w = self.img_size[1] // p if w is None else w // p
|
| 441 |
+
assert h * w == x.shape[1]
|
| 442 |
+
|
| 443 |
+
x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
|
| 444 |
+
x = torch.einsum("nhwpqc->nchpwq", x)
|
| 445 |
+
imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p))
|
| 446 |
+
return imgs
|
| 447 |
+
|
| 448 |
+
def aggregate_variables(self, x: torch.Tensor):
|
| 449 |
+
"""
|
| 450 |
+
Aggregate variable tokens via cross-attention.
|
| 451 |
+
|
| 452 |
+
Args:
|
| 453 |
+
x: (B, V, L, D)
|
| 454 |
+
|
| 455 |
+
Returns:
|
| 456 |
+
(B, L, D)
|
| 457 |
+
"""
|
| 458 |
+
b, _, l, _ = x.shape
|
| 459 |
+
x = torch.einsum("bvld->blvd", x)
|
| 460 |
+
x = x.flatten(0, 1) # BxL, V, D
|
| 461 |
+
|
| 462 |
+
var_query = self.var_query.repeat_interleave(x.shape[0], dim=0)
|
| 463 |
+
x, _ = self.var_agg(var_query, x, x) # BxL, 1, D
|
| 464 |
+
x = x.squeeze(dim=1) # BxL, D
|
| 465 |
+
|
| 466 |
+
x = x.unflatten(dim=0, sizes=(b, l)) # B, L, D
|
| 467 |
+
return x
|
| 468 |
+
|
| 469 |
+
def forward_encoder(self, x: torch.Tensor, lead_times: torch.Tensor, variables):
|
| 470 |
+
"""Encode input weather state into transformer tokens.
|
| 471 |
+
|
| 472 |
+
Args:
|
| 473 |
+
x: [B, V, H, W] input climate variables
|
| 474 |
+
lead_times: [B] forecasting lead times (hours, normalized by dividing by 100)
|
| 475 |
+
variables: tuple of input variable names
|
| 476 |
+
|
| 477 |
+
Returns:
|
| 478 |
+
[B, L, D] encoded token representations
|
| 479 |
+
"""
|
| 480 |
+
if isinstance(variables, list):
|
| 481 |
+
variables = tuple(variables)
|
| 482 |
+
|
| 483 |
+
# tokenize each variable separately
|
| 484 |
+
embeds = []
|
| 485 |
+
var_ids = self.get_var_ids(variables, x.device)
|
| 486 |
+
|
| 487 |
+
for i in range(len(var_ids)):
|
| 488 |
+
id = var_ids[i]
|
| 489 |
+
embeds.append(self.token_embeds[id](x[:, i : i + 1]))
|
| 490 |
+
x = torch.stack(embeds, dim=1) # B, V, L, D
|
| 491 |
+
|
| 492 |
+
# add variable embedding
|
| 493 |
+
var_embed = self.get_var_emb(self.var_embed, variables)
|
| 494 |
+
x = x + var_embed.unsqueeze(2) # B, V, L, D
|
| 495 |
+
|
| 496 |
+
# variable aggregation
|
| 497 |
+
x = self.aggregate_variables(x) # B, L, D
|
| 498 |
+
|
| 499 |
+
# add pos embedding
|
| 500 |
+
x = x + self.pos_embed
|
| 501 |
+
|
| 502 |
+
# add lead time embedding
|
| 503 |
+
lead_time_emb = self.lead_time_embed(lead_times.unsqueeze(-1)) # B, D
|
| 504 |
+
lead_time_emb = lead_time_emb.unsqueeze(1)
|
| 505 |
+
x = x + lead_time_emb # B, L, D
|
| 506 |
+
|
| 507 |
+
x = self.pos_drop(x)
|
| 508 |
+
|
| 509 |
+
# apply Transformer blocks
|
| 510 |
+
for blk in self.blocks:
|
| 511 |
+
x = blk(x)
|
| 512 |
+
x = self.norm(x)
|
| 513 |
+
|
| 514 |
+
return x
|
| 515 |
+
|
| 516 |
+
def forward(self, x, variables, out_variables=None, lead_time=None):
|
| 517 |
+
"""Forward pass through ClimaX.
|
| 518 |
+
|
| 519 |
+
This is the onescience-compatible forward that takes input tensor
|
| 520 |
+
and variable lists, returning predicted weather state.
|
| 521 |
+
|
| 522 |
+
Args:
|
| 523 |
+
x: [B, V_in, H, W] input weather/climate variables
|
| 524 |
+
variables: list of input variable name strings
|
| 525 |
+
out_variables: list of output variable name strings (if None, uses all default_vars)
|
| 526 |
+
lead_time: scalar or [B] tensor, forecasting lead time in normalized hours
|
| 527 |
+
(raw_hours / 100). If None, defaults to 0.72 (72 hours).
|
| 528 |
+
|
| 529 |
+
Returns:
|
| 530 |
+
preds: [B, V_out, H, W] predicted weather/climate variables
|
| 531 |
+
"""
|
| 532 |
+
if out_variables is None:
|
| 533 |
+
out_variables = self.default_vars
|
| 534 |
+
|
| 535 |
+
if lead_time is None:
|
| 536 |
+
lead_time = 0.72 # default 72 hours / 100
|
| 537 |
+
|
| 538 |
+
if not isinstance(lead_time, torch.Tensor):
|
| 539 |
+
lead_time = torch.full(
|
| 540 |
+
(x.shape[0],), lead_time,
|
| 541 |
+
device=x.device, dtype=x.dtype
|
| 542 |
+
)
|
| 543 |
+
|
| 544 |
+
# Encode
|
| 545 |
+
out_transformers = self.forward_encoder(x, lead_time, variables) # B, L, D
|
| 546 |
+
|
| 547 |
+
# Decode
|
| 548 |
+
preds = self.head(out_transformers) # B, L, V*p*p
|
| 549 |
+
preds = self.unpatchify(preds) # B, V_all, H, W
|
| 550 |
+
|
| 551 |
+
# Select output variables
|
| 552 |
+
out_var_ids = self.get_var_ids(tuple(out_variables), preds.device)
|
| 553 |
+
preds = preds[:, out_var_ids]
|
| 554 |
+
|
| 555 |
+
return preds
|
scripts/fake_data.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
root_path = Path(__file__).parent.parent
|
| 5 |
+
sys.path.append(str(root_path))
|
| 6 |
+
import h5py
|
| 7 |
+
import numpy as np
|
| 8 |
+
from onescience.utils.YParams import YParams
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
# ClimaX uses 5.625° resolution: 32x64 grid, 6-hourly data
|
| 12 |
+
DATASET_DIMS = {"T": 10, "H": 32, "W": 64, "time_step": 6}
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def generate_fake_h5(data_dir, var_names, years, dims):
|
| 16 |
+
"""
|
| 17 |
+
Generate fake HDF5 files for each year matching the ERA5 format
|
| 18 |
+
expected by onescience's ERA5Dataset.
|
| 19 |
+
|
| 20 |
+
Uses HDF5 chunked datasets with fillvalue=0.0 — unallocated chunks
|
| 21 |
+
return zeros, so files are tiny but have correct shapes.
|
| 22 |
+
Mean/std are embedded in each year's h5 file.
|
| 23 |
+
"""
|
| 24 |
+
os.makedirs(os.path.join(data_dir, "data"), exist_ok=True)
|
| 25 |
+
T, C = dims["T"], len(var_names)
|
| 26 |
+
H, W = dims["H"], dims["W"]
|
| 27 |
+
|
| 28 |
+
means = np.zeros((1, C, 1, 1), dtype=np.float32)
|
| 29 |
+
stds = np.ones((1, C, 1, 1), dtype=np.float32)
|
| 30 |
+
|
| 31 |
+
for year in years:
|
| 32 |
+
path = os.path.join(data_dir, "data", f"{year}.h5")
|
| 33 |
+
with h5py.File(path, "w") as f:
|
| 34 |
+
ds = f.create_dataset(
|
| 35 |
+
"fields",
|
| 36 |
+
shape=(T, C, H, W),
|
| 37 |
+
dtype="float32",
|
| 38 |
+
chunks=(1, C, H, W),
|
| 39 |
+
fillvalue=0.0,
|
| 40 |
+
)
|
| 41 |
+
ds.attrs["variables"] = var_names
|
| 42 |
+
ds.attrs["time_step"] = dims["time_step"]
|
| 43 |
+
f.create_dataset("global_means", data=means)
|
| 44 |
+
f.create_dataset("global_stds", data=stds)
|
| 45 |
+
|
| 46 |
+
size_kb = os.path.getsize(path) / 1024
|
| 47 |
+
print(f" {year}.h5 shape=({T},{C},{H},{W}) "
|
| 48 |
+
f"logical={T*C*H*W*4/1024**3:.1f}GB actual={size_kb:.1f}KB")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
if __name__ == "__main__":
|
| 52 |
+
cfg_datapipe = YParams("conf/config.yaml", "datapipe")
|
| 53 |
+
|
| 54 |
+
if cfg_datapipe.dataset.data_dir.startswith("/public/") or \
|
| 55 |
+
cfg_datapipe.dataset.data_dir.startswith("/work2/"):
|
| 56 |
+
print("Please check config, ensure data_dir points to local test path "
|
| 57 |
+
"instead of production path.")
|
| 58 |
+
exit()
|
| 59 |
+
|
| 60 |
+
years = (
|
| 61 |
+
cfg_datapipe.dataset.train_time +
|
| 62 |
+
cfg_datapipe.dataset.val_time +
|
| 63 |
+
cfg_datapipe.dataset.test_time
|
| 64 |
+
)
|
| 65 |
+
atm_vars = cfg_datapipe.dataset.channels
|
| 66 |
+
|
| 67 |
+
generate_fake_h5(cfg_datapipe.dataset.data_dir, atm_vars, years, DATASET_DIMS)
|
| 68 |
+
|
| 69 |
+
print("\nFake datasets generated successfully.")
|
| 70 |
+
print(f" Variables: {len(atm_vars)}")
|
| 71 |
+
print(f" Years: {years}")
|
| 72 |
+
print(f" Resolution: {DATASET_DIMS['H']}x{DATASET_DIMS['W']}")
|
scripts/inference.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
root_path = Path(__file__).parent.parent
|
| 6 |
+
sys.path.append(str(root_path))
|
| 7 |
+
import glob
|
| 8 |
+
import numpy as np
|
| 9 |
+
import h5py
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
from model.ClimaX import ClimaX
|
| 12 |
+
from onescience.utils.YParams import YParams
|
| 13 |
+
from onescience.datapipes.climate import ERA5Datapipe
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def get_stats(data_dir, channels):
|
| 17 |
+
"""Read variable list and normalization params (mean/std) from h5 file."""
|
| 18 |
+
h5_files = sorted(glob.glob(os.path.join(data_dir, "data", "*.h5")))
|
| 19 |
+
with h5py.File(h5_files[0], "r") as f:
|
| 20 |
+
ds = f["fields"]
|
| 21 |
+
all_variables = [
|
| 22 |
+
v.decode() if isinstance(v, bytes) else v for v in ds.attrs["variables"]
|
| 23 |
+
]
|
| 24 |
+
mu = f["global_means"][:] # [1, C, 1, 1]
|
| 25 |
+
std = f["global_stds"][:]
|
| 26 |
+
|
| 27 |
+
channel_indices = [all_variables.index(v) for v in channels]
|
| 28 |
+
means = mu[:, channel_indices, :, :]
|
| 29 |
+
stds = std[:, channel_indices, :, :]
|
| 30 |
+
return means, stds
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
if __name__ == "__main__":
|
| 34 |
+
current_path = os.getcwd()
|
| 35 |
+
sys.path.append(current_path)
|
| 36 |
+
|
| 37 |
+
## Model config init
|
| 38 |
+
config_file_path = os.path.join(current_path, "conf/config.yaml")
|
| 39 |
+
cfg = YParams(config_file_path, "model")
|
| 40 |
+
|
| 41 |
+
## DataLoader init
|
| 42 |
+
cfg_data = YParams(config_file_path, "datapipe")
|
| 43 |
+
|
| 44 |
+
all_vars = cfg_data.dataset.channels
|
| 45 |
+
out_vars = cfg_data.dataset.out_variables
|
| 46 |
+
|
| 47 |
+
means, stds = get_stats(cfg_data.dataset.data_dir, out_vars)
|
| 48 |
+
|
| 49 |
+
datapipe = ERA5Datapipe(
|
| 50 |
+
dataset_dir=cfg_data.dataset.data_dir,
|
| 51 |
+
used_variables=all_vars,
|
| 52 |
+
used_years=cfg_data.dataset.test_time,
|
| 53 |
+
distributed=False,
|
| 54 |
+
batch_size=1,
|
| 55 |
+
num_workers=4,
|
| 56 |
+
)
|
| 57 |
+
test_dataloader, _ = datapipe.get_dataloader("test")
|
| 58 |
+
|
| 59 |
+
ckpt = torch.load(
|
| 60 |
+
f"{cfg.checkpoint_dir}/model_bak.pth",
|
| 61 |
+
map_location="cuda:0",
|
| 62 |
+
weights_only=False,
|
| 63 |
+
)
|
| 64 |
+
model = ClimaX(
|
| 65 |
+
default_vars=all_vars,
|
| 66 |
+
img_size=cfg.img_size,
|
| 67 |
+
patch_size=cfg.patch_size,
|
| 68 |
+
embed_dim=cfg.embed_dim,
|
| 69 |
+
depth=cfg.depth,
|
| 70 |
+
decoder_depth=cfg.decoder_depth,
|
| 71 |
+
num_heads=cfg.num_heads,
|
| 72 |
+
mlp_ratio=cfg.mlp_ratio,
|
| 73 |
+
drop_path=cfg.drop_path,
|
| 74 |
+
drop_rate=cfg.drop_rate,
|
| 75 |
+
).to('cuda:0')
|
| 76 |
+
model.load_state_dict(ckpt["model_state_dict"])
|
| 77 |
+
|
| 78 |
+
model.eval()
|
| 79 |
+
os.makedirs('result/output/', exist_ok=True)
|
| 80 |
+
print(f"Saving predictions to './result/output/'")
|
| 81 |
+
|
| 82 |
+
predict_range = cfg.predict_range
|
| 83 |
+
hrs_each_step = cfg.hrs_each_step
|
| 84 |
+
lead_time_val = (predict_range * hrs_each_step) / 100.0
|
| 85 |
+
|
| 86 |
+
with torch.no_grad():
|
| 87 |
+
for data in tqdm(test_dataloader, desc="Inferring testset", unit="batch"):
|
| 88 |
+
invar = data[0].to("cuda:0", dtype=torch.float32)
|
| 89 |
+
filename = data[4][-1][0] # time string from dataloader (input timestamp)
|
| 90 |
+
|
| 91 |
+
preds = model(invar, all_vars, out_vars, lead_time_val)
|
| 92 |
+
pred_var = preds.cpu().numpy()
|
| 93 |
+
|
| 94 |
+
# Denormalize
|
| 95 |
+
pred_var = pred_var * stds + means
|
| 96 |
+
np.save(f"result/output/{filename}.npy", pred_var)
|
| 97 |
+
|
| 98 |
+
print("Inference complete.")
|
scripts/result.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import matplotlib.pyplot as plt
|
| 3 |
+
import os
|
| 4 |
+
import sys
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
root_path = Path(__file__).parent.parent
|
| 7 |
+
sys.path.append(str(root_path))
|
| 8 |
+
import glob
|
| 9 |
+
import h5py
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from onescience.utils.fcn.YParams import YParams
|
| 13 |
+
from matplotlib import rcParams
|
| 14 |
+
|
| 15 |
+
rcParams['mathtext.fontset'] = 'stix'
|
| 16 |
+
rcParams['axes.linewidth'] = 0.9
|
| 17 |
+
rcParams['xtick.major.width'] = 0.9
|
| 18 |
+
rcParams['ytick.major.width'] = 0.9
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def get_metadata(data_dir, channels):
|
| 22 |
+
"""Read variable list and time_step from h5 attrs."""
|
| 23 |
+
h5_files = sorted(glob.glob(os.path.join(data_dir, "data", "*.h5")))
|
| 24 |
+
with h5py.File(h5_files[0], "r") as f:
|
| 25 |
+
ds = f["fields"]
|
| 26 |
+
all_variables = [
|
| 27 |
+
v.decode() if isinstance(v, bytes) else v for v in ds.attrs["variables"]
|
| 28 |
+
]
|
| 29 |
+
time_step = int(ds.attrs["time_step"])
|
| 30 |
+
|
| 31 |
+
channel_indices = [all_variables.index(v) for v in channels]
|
| 32 |
+
|
| 33 |
+
total_files = [f for f in os.listdir('./result/output/') if f.endswith('.npy')]
|
| 34 |
+
total_files.sort()
|
| 35 |
+
return total_files, channel_indices, time_step
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def filename_to_index(filename, time_step):
|
| 39 |
+
"""Convert YYYYMMDDHH filename to time step index within the year's h5."""
|
| 40 |
+
dt = datetime.strptime(filename, "%Y%m%d%H")
|
| 41 |
+
year_start = datetime(dt.year, 1, 1)
|
| 42 |
+
hours = (dt - year_start).total_seconds() / 3600
|
| 43 |
+
return int(hours / time_step)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def get_result(total_files, channel_indices, time_step, data_dir, clim_mean):
|
| 47 |
+
"""Compute per-channel RMSE and ACC (latitude-weighted, matching official ClimaX).
|
| 48 |
+
|
| 49 |
+
Official lat_weighted_acc in climax/utils/metrics.py:
|
| 50 |
+
1. De-normalizes pred and y
|
| 51 |
+
2. Subtracts climatology: pred_anom = pred - clim, y_anom = y - clim
|
| 52 |
+
3. Centers by per-field spatial mean: pred_prime = pred_anom - mean(pred_anom)
|
| 53 |
+
4. Applies latitude weighting (cos(lat) / mean(cos(lat)))
|
| 54 |
+
5. Computes: sum(w * pred_prime * y_prime) / sqrt(sum(w * pred_prime^2) * sum(w * y_prime^2))
|
| 55 |
+
|
| 56 |
+
Here we compute an unweighted version since lat array is not readily available;
|
| 57 |
+
for RMSE we use the standard (non-lat-weighted) formula for simplicity.
|
| 58 |
+
"""
|
| 59 |
+
channel_rmse = np.zeros(len(channel_indices))
|
| 60 |
+
channel_acc = np.zeros(len(channel_indices))
|
| 61 |
+
clim_mean = clim_mean[0, :, :, :]
|
| 62 |
+
|
| 63 |
+
if not os.path.exists('./result/rmse.npy') or not os.path.exists('result/acc.npy'):
|
| 64 |
+
numerator = np.zeros(len(channel_indices))
|
| 65 |
+
pred_sq_sum = np.zeros(len(channel_indices))
|
| 66 |
+
label_sq_sum = np.zeros(len(channel_indices))
|
| 67 |
+
|
| 68 |
+
for file in tqdm(total_files, unit="files"):
|
| 69 |
+
fname = file[:-4] # remove .npy
|
| 70 |
+
year = fname[:4]
|
| 71 |
+
t_idx = filename_to_index(fname, time_step)
|
| 72 |
+
with h5py.File(os.path.join(data_dir, 'data', f'{year}.h5'), "r") as f:
|
| 73 |
+
label = f["fields"][t_idx] # [C, H, W]
|
| 74 |
+
label = label[channel_indices]
|
| 75 |
+
pred = np.load(f'result/output/{file}').squeeze()
|
| 76 |
+
|
| 77 |
+
# RMSE computation
|
| 78 |
+
channel_rmse += np.sqrt(np.mean((label - pred) ** 2, axis=(1, 2)))
|
| 79 |
+
|
| 80 |
+
# ACC computation (following official ClimaX lat_weighted_acc without lat weighting)
|
| 81 |
+
label_anom = label - clim_mean
|
| 82 |
+
pred_anom = pred - clim_mean
|
| 83 |
+
# Center by per-field spatial mean (official ClimaX approach)
|
| 84 |
+
pred_prime = pred_anom - np.mean(pred_anom, axis=(1, 2), keepdims=True)
|
| 85 |
+
label_prime = label_anom - np.mean(label_anom, axis=(1, 2), keepdims=True)
|
| 86 |
+
# accumulate
|
| 87 |
+
numerator += np.sum(pred_prime * label_prime, axis=(1, 2))
|
| 88 |
+
pred_sq_sum += np.sum(pred_prime ** 2, axis=(1, 2))
|
| 89 |
+
label_sq_sum += np.sum(label_prime ** 2, axis=(1, 2))
|
| 90 |
+
|
| 91 |
+
channel_rmse /= len(total_files)
|
| 92 |
+
channel_acc = numerator / (np.sqrt(pred_sq_sum * label_sq_sum) + 1e-8)
|
| 93 |
+
np.save('./result/acc.npy', channel_acc)
|
| 94 |
+
np.save('./result/rmse.npy', channel_rmse)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def show_result():
|
| 98 |
+
"""Print formatted RMSE/ACC table."""
|
| 99 |
+
channel_rmse = np.load('./result/rmse.npy')
|
| 100 |
+
channel_acc = np.load('./result/acc.npy')
|
| 101 |
+
|
| 102 |
+
channels = [cfg_data.dataset.out_variables[i]
|
| 103 |
+
for i in range(len(channel_indices))]
|
| 104 |
+
w = 30
|
| 105 |
+
|
| 106 |
+
print(f"\n{'=' * (w + 32)}")
|
| 107 |
+
print(f"ClimaX Evaluation Results")
|
| 108 |
+
print(f"{'=' * (w + 32)}")
|
| 109 |
+
print(f"┌{'─' * (w + 2)}┬{'─' * 14}┬{'─' * 14}┐")
|
| 110 |
+
print(f"│ {'Channel':<{w}} │ {'RMSE':>12} │ {'ACC':>12} │")
|
| 111 |
+
print(f"├{'─' * (w + 2)}┼{'─' * 14}┼{'─' * 14}┤")
|
| 112 |
+
for i, ch in enumerate(channels):
|
| 113 |
+
print(f"│ {ch:<{w}} │ {channel_rmse[i]:>12.4f} │ {channel_acc[i]:>12.4f} │")
|
| 114 |
+
print(f"├{'─' * (w + 2)}┼{'─' * 14}┼{'─' * 14}┤")
|
| 115 |
+
print(f"│ {'Average':<{w}} │ {np.mean(channel_rmse):>12.4f} │ {np.mean(channel_acc):>12.4f} │")
|
| 116 |
+
print(f"└{'─' * (w + 2)}┴{'─' * 14}┴{'─' * 14}┘")
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def plot(label, pred, var, filename):
|
| 120 |
+
"""Plot truth, prediction, and difference for a single variable."""
|
| 121 |
+
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
|
| 122 |
+
|
| 123 |
+
xtick_labels = ['180°W', '90°W', '0°', '90°E', '180°E']
|
| 124 |
+
ytick_labels = ['90°S', '45°S', '0°', '45°N', '90°N']
|
| 125 |
+
xticks = np.linspace(0, label.shape[-1] - 1, 5)
|
| 126 |
+
yticks = np.linspace(0, label.shape[-2] - 1, 5)
|
| 127 |
+
|
| 128 |
+
# unified color scale for truth/pred
|
| 129 |
+
vmin = min(label.min(), pred.min())
|
| 130 |
+
vmax = max(label.max(), pred.max())
|
| 131 |
+
|
| 132 |
+
diff = label - pred
|
| 133 |
+
rmse = np.sqrt(np.mean(diff ** 2))
|
| 134 |
+
diff_abs_max = np.abs(diff).max()
|
| 135 |
+
|
| 136 |
+
plot_configs = [
|
| 137 |
+
{'data': label, 'title': 'Truth', 'cmap': 'viridis',
|
| 138 |
+
'vmin': vmin, 'vmax': vmax},
|
| 139 |
+
{'data': pred, 'title': 'Prediction', 'cmap': 'viridis',
|
| 140 |
+
'vmin': vmin, 'vmax': vmax},
|
| 141 |
+
{'data': diff, 'title': f'Difference (RMSE={rmse:.2f})',
|
| 142 |
+
'cmap': 'RdBu_r', 'vmin': -diff_abs_max, 'vmax': diff_abs_max},
|
| 143 |
+
]
|
| 144 |
+
|
| 145 |
+
for ax, cfg in zip(axes, plot_configs):
|
| 146 |
+
im = ax.imshow(cfg['data'], cmap=cfg['cmap'],
|
| 147 |
+
vmin=cfg['vmin'], vmax=cfg['vmax'])
|
| 148 |
+
ax.set_title(cfg['title'], fontsize=12, pad=4)
|
| 149 |
+
ax.set_xlabel('Longitude')
|
| 150 |
+
ax.set_ylabel('Latitude')
|
| 151 |
+
ax.set_xticks(xticks)
|
| 152 |
+
ax.set_xticklabels(xtick_labels)
|
| 153 |
+
ax.set_yticks(yticks)
|
| 154 |
+
ax.set_yticklabels(ytick_labels)
|
| 155 |
+
plt.colorbar(im, ax=ax, orientation='horizontal')
|
| 156 |
+
|
| 157 |
+
fig.suptitle(var, fontsize=14, fontweight='bold', y=0.98)
|
| 158 |
+
plt.savefig(filename, dpi=300, bbox_inches='tight')
|
| 159 |
+
plt.close()
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def plot_loss(train_loss, valid_loss):
|
| 163 |
+
"""Plot training and validation loss curves."""
|
| 164 |
+
mask = ~(np.isnan(train_loss) | np.isnan(valid_loss))
|
| 165 |
+
train_loss = train_loss[mask]
|
| 166 |
+
valid_loss = valid_loss[mask]
|
| 167 |
+
|
| 168 |
+
fig, ax = plt.subplots(figsize=(5, 3.5))
|
| 169 |
+
colors = {'train': '#2563EB', 'valid': '#EA580C'}
|
| 170 |
+
epochs = np.arange(1, len(train_loss) + 1)
|
| 171 |
+
|
| 172 |
+
ax.plot(epochs, train_loss, color=colors['train'],
|
| 173 |
+
linewidth=1.5, label='Train')
|
| 174 |
+
ax.plot(epochs, valid_loss, color=colors['valid'],
|
| 175 |
+
linewidth=1.5, label='Valid', linestyle='--')
|
| 176 |
+
|
| 177 |
+
min_idx = np.argmin(valid_loss)
|
| 178 |
+
ax.scatter(epochs[min_idx], valid_loss[min_idx],
|
| 179 |
+
color=colors['valid'], s=40, zorder=5, edgecolors='white')
|
| 180 |
+
ax.annotate(
|
| 181 |
+
f'Best: {valid_loss[min_idx]:.3f}',
|
| 182 |
+
xy=(epochs[min_idx], valid_loss[min_idx]),
|
| 183 |
+
xytext=(10, 10), textcoords='offset points',
|
| 184 |
+
fontsize=8, color=colors['valid'],
|
| 185 |
+
arrowprops=dict(arrowstyle='-', color=colors['valid'], lw=0.5)
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
ax.set(xlabel='Epoch', ylabel='Loss', xlim=(0, len(train_loss) + 1))
|
| 189 |
+
ax.legend(frameon=False, loc='upper right')
|
| 190 |
+
ax.grid(True, linestyle='--', alpha=0.3)
|
| 191 |
+
ax.spines[['top', 'right']].set_visible(False)
|
| 192 |
+
|
| 193 |
+
plt.tight_layout()
|
| 194 |
+
plt.savefig('./result/loss.png', dpi=300, bbox_inches='tight')
|
| 195 |
+
plt.close()
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
if __name__ == "__main__":
|
| 199 |
+
current_path = os.getcwd()
|
| 200 |
+
sys.path.append(current_path)
|
| 201 |
+
config_file_path = os.path.join(current_path, 'conf/config.yaml')
|
| 202 |
+
cfg = YParams(config_file_path, 'model')
|
| 203 |
+
cfg_data = YParams(config_file_path, "datapipe")
|
| 204 |
+
|
| 205 |
+
train_loss = np.load(f'{cfg.checkpoint_dir}/trloss.npy')
|
| 206 |
+
valid_loss = np.load(f'{cfg.checkpoint_dir}/valoss.npy')
|
| 207 |
+
plot_loss(train_loss, valid_loss)
|
| 208 |
+
|
| 209 |
+
data_dir = cfg_data.dataset.data_dir
|
| 210 |
+
out_vars = cfg_data.dataset.out_variables
|
| 211 |
+
total_files, channel_indices, time_step = get_metadata(data_dir, out_vars)
|
| 212 |
+
|
| 213 |
+
# Load & compute RMSE/ACC per channel
|
| 214 |
+
h5_files = sorted(glob.glob(os.path.join(data_dir, "data", "*.h5")))
|
| 215 |
+
with h5py.File(h5_files[0], "r") as f:
|
| 216 |
+
mu = f["global_means"][:]
|
| 217 |
+
clim_mean = mu[:, channel_indices, :, :]
|
| 218 |
+
get_result(total_files, channel_indices, time_step, data_dir, clim_mean)
|
| 219 |
+
show_result()
|
| 220 |
+
|
| 221 |
+
##### Plot example predictions #####
|
| 222 |
+
test_year = cfg_data.dataset.test_time[0]
|
| 223 |
+
eg_files = [f'{test_year}010206']
|
| 224 |
+
selected_vars = out_vars[:3] # first 3 output variables
|
| 225 |
+
|
| 226 |
+
print(f"\nPlotting example predictions for: {eg_files}")
|
| 227 |
+
print(f"Variables: {selected_vars}")
|
| 228 |
+
|
| 229 |
+
for file in eg_files:
|
| 230 |
+
year = file[:4]
|
| 231 |
+
t_idx = filename_to_index(file, time_step)
|
| 232 |
+
with h5py.File(os.path.join(data_dir, 'data', f'{year}.h5'), "r") as f:
|
| 233 |
+
label = f["fields"][t_idx]
|
| 234 |
+
label = label[channel_indices]
|
| 235 |
+
pred = np.load(f'result/output/{file}.npy').squeeze()
|
| 236 |
+
|
| 237 |
+
for i, var in enumerate(selected_vars):
|
| 238 |
+
filename = f'./result/{file}_{var}.png'
|
| 239 |
+
plot(label[i], pred[i], var, filename)
|
| 240 |
+
print(f' Saved: {filename}')
|
scripts/train.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
root_path = Path(__file__).parent.parent
|
| 6 |
+
sys.path.append(str(root_path))
|
| 7 |
+
import shutil
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch.distributed as dist
|
| 10 |
+
import logging
|
| 11 |
+
import time
|
| 12 |
+
from tqdm import tqdm
|
| 13 |
+
from torch.nn.parallel import DistributedDataParallel
|
| 14 |
+
from model.ClimaX import ClimaX
|
| 15 |
+
from onescience.datapipes.climate import ERA5Datapipe
|
| 16 |
+
from onescience.utils.YParams import YParams
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# ============================================================================
|
| 20 |
+
# Loss function: Latitude-weighted MSE (from official ClimaX metrics.py)
|
| 21 |
+
# ============================================================================
|
| 22 |
+
|
| 23 |
+
def lat_weighted_mse(pred, y, lat):
|
| 24 |
+
"""Latitude weighted mean squared error.
|
| 25 |
+
|
| 26 |
+
Allows to weight the loss by the cosine of the latitude to account for
|
| 27 |
+
gridding differences at equator vs. poles.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
y: [B, V, H, W]
|
| 31 |
+
pred: [B, V, H, W]
|
| 32 |
+
lat: [H] latitude array in degrees
|
| 33 |
+
|
| 34 |
+
Returns:
|
| 35 |
+
scalar loss
|
| 36 |
+
"""
|
| 37 |
+
error = (pred - y) ** 2 # [B, V, H, W]
|
| 38 |
+
|
| 39 |
+
# latitude weights
|
| 40 |
+
w_lat = np.cos(np.deg2rad(lat))
|
| 41 |
+
w_lat = w_lat / w_lat.mean() # (H,)
|
| 42 |
+
w_lat = torch.from_numpy(w_lat).unsqueeze(0).unsqueeze(-1).to(
|
| 43 |
+
dtype=error.dtype, device=error.device
|
| 44 |
+
) # (1, H, 1)
|
| 45 |
+
|
| 46 |
+
loss = (error * w_lat.unsqueeze(1)).mean()
|
| 47 |
+
return loss
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def get_lat_array(img_size, spatial_res=5.625):
|
| 51 |
+
"""Generate latitude array for the grid.
|
| 52 |
+
|
| 53 |
+
Args:
|
| 54 |
+
img_size: [H, W]
|
| 55 |
+
spatial_res: degrees per grid cell
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
lat: [H] latitude values from north to south
|
| 59 |
+
"""
|
| 60 |
+
H = img_size[0]
|
| 61 |
+
# Cell centers: from 90 - res/2 to -90 + res/2
|
| 62 |
+
lat = np.linspace(90 - spatial_res / 2, -90 + spatial_res / 2, H)
|
| 63 |
+
return lat.astype(np.float32)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# ============================================================================
|
| 67 |
+
# Training
|
| 68 |
+
# ============================================================================
|
| 69 |
+
|
| 70 |
+
def main():
|
| 71 |
+
logging.basicConfig(
|
| 72 |
+
level=logging.INFO,
|
| 73 |
+
format="%(asctime)s - %(levelname)s - %(message)s"
|
| 74 |
+
)
|
| 75 |
+
logger = logging.getLogger()
|
| 76 |
+
|
| 77 |
+
## Model config init
|
| 78 |
+
config_file_path = os.path.join(current_path, "conf/config.yaml")
|
| 79 |
+
cfg = YParams(config_file_path, "model")
|
| 80 |
+
|
| 81 |
+
## Distributed config init
|
| 82 |
+
cfg.world_size = 1
|
| 83 |
+
if "WORLD_SIZE" in os.environ:
|
| 84 |
+
cfg.world_size = int(os.environ["WORLD_SIZE"])
|
| 85 |
+
world_rank = 0
|
| 86 |
+
local_rank = 0
|
| 87 |
+
if cfg.world_size > 1:
|
| 88 |
+
dist.init_process_group(backend="nccl", init_method="env://")
|
| 89 |
+
local_rank = int(os.environ["LOCAL_RANK"])
|
| 90 |
+
world_rank = dist.get_rank()
|
| 91 |
+
|
| 92 |
+
## DataLoader init
|
| 93 |
+
cfg_data = YParams(config_file_path, "datapipe")
|
| 94 |
+
|
| 95 |
+
# Build variable lists from config
|
| 96 |
+
all_vars = cfg_data.dataset.channels
|
| 97 |
+
out_vars = cfg_data.dataset.out_variables
|
| 98 |
+
|
| 99 |
+
datapipe = ERA5Datapipe(
|
| 100 |
+
dataset_dir=cfg_data.dataset.data_dir,
|
| 101 |
+
used_variables=all_vars,
|
| 102 |
+
used_years=cfg_data.dataset.train_time,
|
| 103 |
+
distributed=dist.is_initialized(),
|
| 104 |
+
)
|
| 105 |
+
train_dataloader, train_sampler = datapipe.get_dataloader("train")
|
| 106 |
+
|
| 107 |
+
datapipe = ERA5Datapipe(
|
| 108 |
+
dataset_dir=cfg_data.dataset.data_dir,
|
| 109 |
+
used_variables=all_vars,
|
| 110 |
+
used_years=cfg_data.dataset.val_time,
|
| 111 |
+
distributed=dist.is_initialized(),
|
| 112 |
+
)
|
| 113 |
+
val_dataloader, val_sampler = datapipe.get_dataloader("valid")
|
| 114 |
+
|
| 115 |
+
## Model init
|
| 116 |
+
model = ClimaX(
|
| 117 |
+
default_vars=all_vars,
|
| 118 |
+
img_size=cfg.img_size,
|
| 119 |
+
patch_size=cfg.patch_size,
|
| 120 |
+
embed_dim=cfg.embed_dim,
|
| 121 |
+
depth=cfg.depth,
|
| 122 |
+
decoder_depth=cfg.decoder_depth,
|
| 123 |
+
num_heads=cfg.num_heads,
|
| 124 |
+
mlp_ratio=cfg.mlp_ratio,
|
| 125 |
+
drop_path=cfg.drop_path,
|
| 126 |
+
drop_rate=cfg.drop_rate,
|
| 127 |
+
).to(local_rank)
|
| 128 |
+
|
| 129 |
+
## Optimizer (following official ClimaX: AdamW with param groups)
|
| 130 |
+
decay = []
|
| 131 |
+
no_decay = []
|
| 132 |
+
for name, m in model.named_parameters():
|
| 133 |
+
if "var_embed" in name or "pos_embed" in name:
|
| 134 |
+
no_decay.append(m)
|
| 135 |
+
else:
|
| 136 |
+
decay.append(m)
|
| 137 |
+
|
| 138 |
+
optimizer = torch.optim.AdamW(
|
| 139 |
+
[
|
| 140 |
+
{
|
| 141 |
+
"params": decay,
|
| 142 |
+
"lr": cfg.lr,
|
| 143 |
+
"betas": (cfg.beta_1, cfg.beta_2),
|
| 144 |
+
"weight_decay": cfg.weight_decay,
|
| 145 |
+
},
|
| 146 |
+
{
|
| 147 |
+
"params": no_decay,
|
| 148 |
+
"lr": cfg.lr,
|
| 149 |
+
"betas": (cfg.beta_1, cfg.beta_2),
|
| 150 |
+
"weight_decay": 0,
|
| 151 |
+
},
|
| 152 |
+
]
|
| 153 |
+
)
|
| 154 |
+
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
| 155 |
+
optimizer, factor=0.2, patience=5, mode="min"
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
## Get latitude array for lat-weighted loss
|
| 159 |
+
lat = get_lat_array(cfg.img_size)
|
| 160 |
+
|
| 161 |
+
## Train process init
|
| 162 |
+
os.makedirs(cfg.checkpoint_dir, exist_ok=True)
|
| 163 |
+
train_loss_file = f"{cfg.checkpoint_dir}/trloss.npy"
|
| 164 |
+
valid_loss_file = f"{cfg.checkpoint_dir}/valoss.npy"
|
| 165 |
+
best_valid_loss = 1.0e6
|
| 166 |
+
best_loss_epoch = 0
|
| 167 |
+
train_losses = np.empty((0,), dtype=np.float32)
|
| 168 |
+
valid_losses = np.empty((0,), dtype=np.float32)
|
| 169 |
+
|
| 170 |
+
## Get model params count
|
| 171 |
+
if cfg.world_size == 1 or world_rank == 0:
|
| 172 |
+
total_params = sum(p.numel() for p in model.parameters())
|
| 173 |
+
print("\n\n")
|
| 174 |
+
print("-" * 50)
|
| 175 |
+
print(f"Model params: {total_params}, {total_params / 1e6:.2f}M, {total_params / 1e9:.2f}B")
|
| 176 |
+
print("-" * 50, "\n")
|
| 177 |
+
|
| 178 |
+
## Load model weight if there exists a well-trained model
|
| 179 |
+
if os.path.exists(f"{cfg.checkpoint_dir}/model_bak.pth"):
|
| 180 |
+
if world_rank == 0:
|
| 181 |
+
print("\n\n")
|
| 182 |
+
print("-" * 50)
|
| 183 |
+
print(f"Found existing model weight, loading and continuing training...")
|
| 184 |
+
print(f"If you want to train a new model, remove *.pth from {cfg.checkpoint_dir}")
|
| 185 |
+
print("-" * 50, "\n")
|
| 186 |
+
ckpt = torch.load(
|
| 187 |
+
f"{cfg.checkpoint_dir}/model_bak.pth",
|
| 188 |
+
map_location=f'cuda:{local_rank}',
|
| 189 |
+
weights_only=False,
|
| 190 |
+
)
|
| 191 |
+
model.load_state_dict(ckpt["model_state_dict"])
|
| 192 |
+
optimizer.load_state_dict(ckpt["optimizer_state_dict"])
|
| 193 |
+
scheduler.load_state_dict(ckpt["scheduler_state_dict"])
|
| 194 |
+
best_valid_loss = ckpt["best_valid_loss"]
|
| 195 |
+
best_loss_epoch = ckpt["best_loss_epoch"]
|
| 196 |
+
train_losses = np.load(train_loss_file)
|
| 197 |
+
valid_losses = np.load(valid_loss_file)
|
| 198 |
+
|
| 199 |
+
## Pre-compute output variable indices (before DDP wrap, for loss computation)
|
| 200 |
+
out_var_indices = model.get_var_ids(tuple(out_vars), torch.device('cpu'))
|
| 201 |
+
|
| 202 |
+
## Distributed model
|
| 203 |
+
if cfg.world_size > 1:
|
| 204 |
+
model = DistributedDataParallel(
|
| 205 |
+
model, device_ids=[local_rank],
|
| 206 |
+
output_device=local_rank,
|
| 207 |
+
find_unused_parameters=True
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
## Lead time (fixed for deterministic forecasting)
|
| 211 |
+
predict_range = cfg.predict_range
|
| 212 |
+
hrs_each_step = cfg.hrs_each_step
|
| 213 |
+
lead_time_val = (predict_range * hrs_each_step) / 100.0 # normalized
|
| 214 |
+
|
| 215 |
+
world_rank == 0 and logger.info(f"Starting training... lead_time={lead_time_val}")
|
| 216 |
+
|
| 217 |
+
for epoch in range(cfg.max_epoch):
|
| 218 |
+
if dist.is_initialized():
|
| 219 |
+
train_sampler.set_epoch(epoch)
|
| 220 |
+
val_sampler.set_epoch(epoch)
|
| 221 |
+
|
| 222 |
+
model.train()
|
| 223 |
+
train_loss = 0
|
| 224 |
+
start_time = time.time()
|
| 225 |
+
|
| 226 |
+
for j, data in enumerate(train_dataloader):
|
| 227 |
+
invar = data[0].to(local_rank, dtype=torch.float32) # [B, C_all, H, W]
|
| 228 |
+
outvar = data[1].to(local_rank, dtype=torch.float32) # [B, C_all, H, W]
|
| 229 |
+
|
| 230 |
+
preds = model(invar, all_vars, out_vars, lead_time_val)
|
| 231 |
+
|
| 232 |
+
# Select output variable channels from ground truth
|
| 233 |
+
outvar_selected = outvar[:, out_var_indices.to(outvar.device)]
|
| 234 |
+
loss = lat_weighted_mse(preds, outvar_selected, lat)
|
| 235 |
+
|
| 236 |
+
optimizer.zero_grad()
|
| 237 |
+
loss.backward()
|
| 238 |
+
optimizer.step()
|
| 239 |
+
train_loss += loss.item()
|
| 240 |
+
|
| 241 |
+
if world_rank == 0:
|
| 242 |
+
logger.info(
|
| 243 |
+
f'Train: Epoch {epoch}-{j+1}/{len(train_dataloader)} '
|
| 244 |
+
f'[cost {int((time.time()-start_time) // 60):02}:{int((time.time()-start_time) % 60):02}] '
|
| 245 |
+
f'[{(time.time()-start_time)/(j+1): .02f}s/batch] '
|
| 246 |
+
f'loss:{train_loss / (j+1): .04f}'
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
train_loss /= len(train_dataloader)
|
| 250 |
+
|
| 251 |
+
model.eval()
|
| 252 |
+
valid_loss = 0
|
| 253 |
+
with torch.no_grad():
|
| 254 |
+
for j, data in enumerate(val_dataloader):
|
| 255 |
+
invar = data[0].to(local_rank, dtype=torch.float32)
|
| 256 |
+
outvar = data[1].to(local_rank, dtype=torch.float32)
|
| 257 |
+
|
| 258 |
+
preds = model(invar, all_vars, out_vars, lead_time_val)
|
| 259 |
+
|
| 260 |
+
# Select output variable channels from ground truth
|
| 261 |
+
outvar_selected = outvar[:, out_var_indices.to(outvar.device)]
|
| 262 |
+
loss = lat_weighted_mse(preds, outvar_selected, lat)
|
| 263 |
+
|
| 264 |
+
if cfg.world_size > 1:
|
| 265 |
+
loss_tensor = loss.detach().to(local_rank)
|
| 266 |
+
dist.all_reduce(loss_tensor)
|
| 267 |
+
valid_loss += loss_tensor.item() / cfg.world_size
|
| 268 |
+
else:
|
| 269 |
+
valid_loss += loss.item()
|
| 270 |
+
|
| 271 |
+
if world_rank == 0:
|
| 272 |
+
logger.info(
|
| 273 |
+
f'Valid: Epoch {epoch}-{j+1}/{len(val_dataloader)} '
|
| 274 |
+
f'[cost {int((time.time()-start_time) // 60):02}:{int((time.time()-start_time) % 60):02}] '
|
| 275 |
+
f'[{(time.time()-start_time)/(j+1): .02f}s/batch] '
|
| 276 |
+
f'loss:{valid_loss / (j+1): .04f}'
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
valid_loss /= len(val_dataloader)
|
| 280 |
+
is_save_ckp = False
|
| 281 |
+
|
| 282 |
+
if valid_loss < best_valid_loss:
|
| 283 |
+
best_valid_loss = valid_loss
|
| 284 |
+
best_loss_epoch = epoch
|
| 285 |
+
world_rank == 0 and save_checkpoint(
|
| 286 |
+
model, optimizer, scheduler,
|
| 287 |
+
best_valid_loss, best_loss_epoch, cfg.checkpoint_dir
|
| 288 |
+
)
|
| 289 |
+
is_save_ckp = True
|
| 290 |
+
|
| 291 |
+
scheduler.step(valid_loss)
|
| 292 |
+
|
| 293 |
+
if world_rank == 0:
|
| 294 |
+
logger.info(
|
| 295 |
+
f"Epoch [{epoch + 1}/{cfg.max_epoch}], "
|
| 296 |
+
f"Train Loss: {train_loss:.4f}, "
|
| 297 |
+
f"Valid Loss: {valid_loss:.4f}, "
|
| 298 |
+
f"Best loss at Epoch: {best_loss_epoch + 1}"
|
| 299 |
+
+ (", saving checkpoint" if is_save_ckp else "")
|
| 300 |
+
)
|
| 301 |
+
train_losses = np.append(train_losses, train_loss)
|
| 302 |
+
valid_losses = np.append(valid_losses, valid_loss)
|
| 303 |
+
|
| 304 |
+
np.save(train_loss_file, train_losses)
|
| 305 |
+
np.save(valid_loss_file, valid_losses)
|
| 306 |
+
|
| 307 |
+
if epoch - best_loss_epoch > cfg.patience:
|
| 308 |
+
print(f"Loss has not decreased in {cfg.patience} epochs, stopping training...")
|
| 309 |
+
sys.exit()
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def save_checkpoint(model, optimizer, scheduler, best_valid_loss,
|
| 313 |
+
best_loss_epoch, model_path):
|
| 314 |
+
# Only save on global rank 0 (guard against misconfigured multi-process launch)
|
| 315 |
+
if dist.is_initialized() and dist.get_rank() != 0:
|
| 316 |
+
return
|
| 317 |
+
model_to_save = model.module if hasattr(model, "module") else model
|
| 318 |
+
state = {
|
| 319 |
+
"model_state_dict": model_to_save.state_dict(),
|
| 320 |
+
"optimizer_state_dict": optimizer.state_dict(),
|
| 321 |
+
"scheduler_state_dict": scheduler.state_dict(),
|
| 322 |
+
"best_valid_loss": best_valid_loss,
|
| 323 |
+
"best_loss_epoch": best_loss_epoch,
|
| 324 |
+
}
|
| 325 |
+
# Write to temporary file first, then atomically rename to avoid
|
| 326 |
+
# race conditions from misconfigured multi-process launches
|
| 327 |
+
tmp_path = f"{model_path}/model_tmp.pth"
|
| 328 |
+
dst_path = f"{model_path}/model_bak.pth"
|
| 329 |
+
torch.save(state, tmp_path)
|
| 330 |
+
shutil.move(tmp_path, dst_path)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
if __name__ == "__main__":
|
| 334 |
+
current_path = os.getcwd()
|
| 335 |
+
sys.path.append(current_path)
|
| 336 |
+
main()
|
weight/.gitkeep
ADDED
|
File without changes
|