Upload folder using huggingface_hub
Browse files- .gitattributes +2 -34
- README.md +140 -0
- conf/config.yaml +51 -0
- config.json +61 -0
- configuration.json +12 -0
- model/rf_climparam.py +232 -0
- scripts/fake_data.py +116 -0
- scripts/inference.py +91 -0
- scripts/result.py +80 -0
- scripts/train.py +107 -0
- weight/.gitkeep +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.
|
| 24 |
-
*.
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
tags:
|
| 6 |
+
- OneScience
|
| 7 |
+
- Earth Science
|
| 8 |
+
- Climate Parameterization
|
| 9 |
+
- Random Forest
|
| 10 |
+
frameworks: PyTorch
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
<p align="center">
|
| 14 |
+
<strong><span style="font-size: 30px;">RF-ClimParam</span></strong>
|
| 15 |
+
</p>
|
| 16 |
+
|
| 17 |
+
# Model Introduction
|
| 18 |
+
|
| 19 |
+
RF-ClimParam learns unresolved convection, cloud microphysics, radiation, turbulent diffusion, and surface-flux processes from high-resolution atmospheric simulations and provides stable subgrid parameterizations for coarse climate models at multiple horizontal resolutions. Its primary uses are multi-resolution climate simulation, analysis of parameterization scale dependence, and reconstruction of precipitation climate statistics.
|
| 20 |
+
|
| 21 |
+
Paper: Stable machine-learning parameterization of subgrid processes for climate modeling at a range of resolutions
|
| 22 |
+
https://arxiv.org/abs/2001.03151
|
| 23 |
+
|
| 24 |
+
# Model Description
|
| 25 |
+
|
| 26 |
+
The method reproduced by RF-ClimParam was proposed by Janni Yuval and Paul A. O'Gorman at the Massachusetts Institute of Technology. The paper trains random forests with coarse-grained states, instantaneous physical tendencies, turbulent diffusivity, and surface fluxes from a three-dimensional high-resolution System for Atmospheric Modeling aquaplanet simulation. The model is suitable for multi-resolution atmospheric subgrid parameterization, coarse-resolution climate simulation, and evaluation of mean and extreme precipitation statistics.
|
| 27 |
+
|
| 28 |
+
# Use Cases
|
| 29 |
+
|
| 30 |
+
| Use Case | Description |
|
| 31 |
+
| :---: | :--- |
|
| 32 |
+
| Multi-resolution parameterization | Predict column tendencies and diffusion quantities separately at x4/x8/x16/x32. |
|
| 33 |
+
| Joint multi-output regression | Preserve cross-variable and cross-level output means in one tree leaf without splitting outputs. |
|
| 34 |
+
| Offline engineering evaluation | Compute per-scale, per-output R2 and RMSE over all four complete coarse-grid fields. |
|
| 35 |
+
| Online-coupling proxy | Evaluate zonal-mean 3 h precipitation and extremes on an additional native x32 `18×48` coarse grid. |
|
| 36 |
+
| ModelScope/OneCode execution | Validate data generation, training, inference, evaluation, visualization, and checkpoint workflows in ModelScope or OneCode. |
|
| 37 |
+
| Multi-GPU training | Launch distributed data-parallel training with `torchrun`. |
|
| 38 |
+
|
| 39 |
+
# Usage Instructions
|
| 40 |
+
|
| 41 |
+
## 1.OneCode
|
| 42 |
+
|
| 43 |
+
Experience intelligent, one-click AI4S programming through the OneCode online environment:
|
| 44 |
+
|
| 45 |
+
[Try intelligent, one-click AI4S programming](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home)
|
| 46 |
+
|
| 47 |
+
## 2. Download and Installation
|
| 48 |
+
|
| 49 |
+
```bash
|
| 50 |
+
hf download OneScience-Group/RF-ClimParam --local-dir ./RF-ClimParam
|
| 51 |
+
cd RF-ClimParam
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
### Environment Dependencies
|
| 55 |
+
|
| 56 |
+
**Hardware Requirements**
|
| 57 |
+
|
| 58 |
+
- A GPU or DCU is recommended; the random forest itself runs on CPU with NumPy.
|
| 59 |
+
- A CPU supports the complete default small-sample connectivity test.
|
| 60 |
+
- DCU users must install DTK first. DTK 25.04.2 or later, or the OneScience-recommended version matching the cluster, is recommended.
|
| 61 |
+
|
| 62 |
+
**DCU Environment**
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
# Activate DTK and Conda first
|
| 66 |
+
conda create -n onescience311 python=3.11 -y
|
| 67 |
+
conda activate onescience311
|
| 68 |
+
pip install onescience[earth-dcu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 69 |
+
```
|
| 70 |
+
|
| 71 |
+
**GPU Environment**
|
| 72 |
+
|
| 73 |
+
```bash
|
| 74 |
+
# Activate Conda first
|
| 75 |
+
conda create -n onescience311 python=3.11 -y libstdcxx-ng=12 libgcc-ng=12 gcc_linux-64=12 gxx_linux-64=12
|
| 76 |
+
conda activate onescience311
|
| 77 |
+
pip install onescience[earth-gpu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
### Training Data
|
| 81 |
+
|
| 82 |
+
```bash
|
| 83 |
+
python scripts/fake_data.py
|
| 84 |
+
```
|
| 85 |
+
|
| 86 |
+
This repository uses a small number of structured synthetic atmospheric-column samples to validate the engineering workflow and retains complete `144×360`, `72×180`, `36×90`, and `18×45` coarse-grid snapshots for x4, x8, x16, and x32. The data preserve all 48 levels and the real `145→144` and `62→17` random-forest interfaces while reducing only the number of snapshots, sampled training columns, and trees. Synthetic temperature, moisture, condensate, wind, and flux fields contain spatial and vertical relationships and validate multi-resolution parameterization, training, inference, and evaluation only; they do not represent the official SAM distribution or paper performance.
|
| 87 |
+
|
| 88 |
+
### Training
|
| 89 |
+
|
| 90 |
+
For single-device training, use:
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
python scripts/train.py
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
For multi-GPU training, use:
|
| 97 |
+
|
| 98 |
+
```bash
|
| 99 |
+
torchrun --nproc_per_node=8 --nnodes=1 --rdzv_id=1000 --rdzv_backend=c10d --max_restarts=0 --master_addr="localhost" --master_port=29500 scripts/train.py
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
Training samples a small number of atmospheric columns from the complete spatial field at each resolution and fits the two joint multi-output random forests separately. It produces model parameters and training-sample statistics for all four resolutions, saved to:
|
| 103 |
+
|
| 104 |
+
```text
|
| 105 |
+
result/checkpoints/rf_climparam.pt
|
| 106 |
+
result/training/metrics.json
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
### Trained Weights
|
| 110 |
+
|
| 111 |
+
This repository does not include weights under `weight/`. The original paper provides random-forest estimators at different resolutions; refer to the authors' OSF archive for the released weights and model artifacts: https://doi.org/10.17605/OSF.IO/36YPT.
|
| 112 |
+
|
| 113 |
+
### Inference
|
| 114 |
+
|
| 115 |
+
```bash
|
| 116 |
+
python scripts/inference.py
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
Inference results include reference targets and subgrid-process predictions at the x4, x8, x16, and x32 resolutions. They also contain complete spatial fields, diagnosed precipitation, grid-location information for each scale, and native x32-grid results. All numerical results are saved to `result/output/predictions.npz`.
|
| 120 |
+
|
| 121 |
+
### Evaluation and Visualization
|
| 122 |
+
|
| 123 |
+
```bash
|
| 124 |
+
python scripts/result.py
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
Evaluation results include subgrid-process prediction performance at all four resolutions and an online proxy for coarse-grid precipitation. The visualization compares results across resolutions and shows the zonal distributions of target and predicted precipitation. Structured results and the auxiliary figure are saved to `result/evaluation/metrics.json` and `result/evaluation/comparison.png`. Synthetic-data results validate the engineering workflow only and do not represent the paper's formal SAM online performance.
|
| 128 |
+
|
| 129 |
+
# Official OneScience Information
|
| 130 |
+
|
| 131 |
+
| Platform | OneScience Main Repository | Skills Repository |
|
| 132 |
+
| --- | --- | --- |
|
| 133 |
+
| Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills |
|
| 134 |
+
| GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills |
|
| 135 |
+
|
| 136 |
+
# Citation and License
|
| 137 |
+
|
| 138 |
+
This repository is an independent engineering reproduction of the public RF-ClimParam specifications.
|
| 139 |
+
|
| 140 |
+
Use of this repository's code, official model weights, and data remains subject to the licenses and terms of their respective projects.
|
conf/config.yaml
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed: 2020
|
| 2 |
+
data:
|
| 3 |
+
root: data
|
| 4 |
+
format_version: rf_climparam_v2
|
| 5 |
+
scales: [x4, x8, x16, x32]
|
| 6 |
+
snapshots_per_scale: 1
|
| 7 |
+
train_columns_per_latitude: 1
|
| 8 |
+
vertical_levels: 48
|
| 9 |
+
low_level_diffusion_levels: 15
|
| 10 |
+
snapshot_hours: 3
|
| 11 |
+
training_days: 337.5
|
| 12 |
+
high_resolution:
|
| 13 |
+
resolution_km: 12
|
| 14 |
+
grid: [576, 1440]
|
| 15 |
+
domain_km: [6912, 17280]
|
| 16 |
+
coarse_grids:
|
| 17 |
+
x4: {factor: 4, resolution_km: 48, grid: [144, 360]}
|
| 18 |
+
x8: {factor: 8, resolution_km: 96, grid: [72, 180]}
|
| 19 |
+
x16: {factor: 16, resolution_km: 192, grid: [36, 90]}
|
| 20 |
+
x32: {factor: 32, resolution_km: 384, grid: [18, 45], online_native_grid: [18, 48]}
|
| 21 |
+
model:
|
| 22 |
+
rf_tend: {input_features: 145, output_features: 144}
|
| 23 |
+
rf_diff: {input_features: 62, output_features: 17}
|
| 24 |
+
engineering:
|
| 25 |
+
trees: 2
|
| 26 |
+
max_depth: 5
|
| 27 |
+
min_samples_leaf: 3
|
| 28 |
+
max_features: sqrt
|
| 29 |
+
split_candidates: 8
|
| 30 |
+
bootstrap: true
|
| 31 |
+
paper_model:
|
| 32 |
+
trees: 10
|
| 33 |
+
min_samples_leaf: {x4: 20, x8: 20, x16: 20, x32: 7}
|
| 34 |
+
training_samples: {x4: 5000000, x8: 5000000, x16: 5000000, x32: not_reported}
|
| 35 |
+
max_depth: not_reported
|
| 36 |
+
max_features: not_reported
|
| 37 |
+
bootstrap: not_reported
|
| 38 |
+
split_criterion: not_reported
|
| 39 |
+
implementation: scikit-learn-0.21.2-RandomForestRegressor
|
| 40 |
+
runtime:
|
| 41 |
+
device: cpu
|
| 42 |
+
paths:
|
| 43 |
+
checkpoint: result/checkpoints/rf_climparam.pt
|
| 44 |
+
training_metrics: result/training/metrics.json
|
| 45 |
+
inference: result/output/predictions.npz
|
| 46 |
+
evaluation_dir: result/evaluation
|
| 47 |
+
evaluation:
|
| 48 |
+
precipitation_seconds: 10800
|
| 49 |
+
online_proxy_scale: x32
|
| 50 |
+
online_proxy_grid: [18, 48]
|
| 51 |
+
prediction_chunk_size: 4096
|
config.json
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_name": "RF-ClimParam",
|
| 3 |
+
"model_type": "joint-multi-output-random-forest-regressor",
|
| 4 |
+
"architectures": ["JointRandomForestRegressor"],
|
| 5 |
+
"framework": "PyTorch",
|
| 6 |
+
"domain": "earth-science",
|
| 7 |
+
"task": "multi-resolution-subgrid-parameterization",
|
| 8 |
+
"implementation": {
|
| 9 |
+
"entry_point": "model/rf_climparam.py",
|
| 10 |
+
"scope": "independent paper-method engineering reproduction",
|
| 11 |
+
"train_script": "scripts/train.py",
|
| 12 |
+
"inference_script": "scripts/inference.py",
|
| 13 |
+
"evaluation_script": "scripts/result.py",
|
| 14 |
+
"synthetic_data_script": "scripts/fake_data.py"
|
| 15 |
+
},
|
| 16 |
+
"architecture": {
|
| 17 |
+
"family": "bootstrapped extra-random regression forest",
|
| 18 |
+
"rf_tend_input": 145,
|
| 19 |
+
"rf_tend_output": 144,
|
| 20 |
+
"rf_diff_input": 62,
|
| 21 |
+
"rf_diff_output": 17,
|
| 22 |
+
"joint_multi_output_leaves": true,
|
| 23 |
+
"scales": ["x4", "x8", "x16", "x32"]
|
| 24 |
+
},
|
| 25 |
+
"paper_model": {
|
| 26 |
+
"trees": 10,
|
| 27 |
+
"min_samples_leaf": {"x4": 20, "x8": 20, "x16": 20, "x32": 7},
|
| 28 |
+
"training_samples": {"x4": 5000000, "x8": 5000000, "x16": 5000000, "x32": "not_reported"},
|
| 29 |
+
"max_depth": "not_reported",
|
| 30 |
+
"max_features": "not_reported",
|
| 31 |
+
"bootstrap": "not_reported",
|
| 32 |
+
"split_criterion": "not_reported",
|
| 33 |
+
"implementation": "scikit-learn-0.21.2-RandomForestRegressor"
|
| 34 |
+
},
|
| 35 |
+
"data": {
|
| 36 |
+
"protocol": "rf_climparam_v2",
|
| 37 |
+
"format": "NPZ",
|
| 38 |
+
"high_resolution": {"km": 12, "grid": [576, 1440], "vertical_levels": 48},
|
| 39 |
+
"coarse_grids": {
|
| 40 |
+
"x4": {"km": 48, "grid": [144, 360]},
|
| 41 |
+
"x8": {"km": 96, "grid": [72, 180]},
|
| 42 |
+
"x16": {"km": 192, "grid": [36, 90]},
|
| 43 |
+
"x32": {"km": 384, "grid": [18, 45], "online_native_grid": [18, 48]}
|
| 44 |
+
},
|
| 45 |
+
"domain_km": [6912, 17280],
|
| 46 |
+
"snapshot_hours": 3,
|
| 47 |
+
"default_timestep_seconds": 24,
|
| 48 |
+
"coarsening": "N by N horizontal average",
|
| 49 |
+
"synthetic_layout": "one complete C-order flattened structured snapshot per coarse scale",
|
| 50 |
+
"default_training_sampling": "one longitude column per latitude"
|
| 51 |
+
},
|
| 52 |
+
"paper": {
|
| 53 |
+
"arxiv": "2001.03151",
|
| 54 |
+
"doi": "10.1038/s41467-020-17142-3",
|
| 55 |
+
"official_code_data_osf_doi": "10.17605/OSF.IO/36YPT"
|
| 56 |
+
},
|
| 57 |
+
"configuration_sources": [
|
| 58 |
+
"conf/config.yaml", "model/rf_climparam.py", "scripts/fake_data.py",
|
| 59 |
+
"scripts/train.py", "scripts/inference.py", "scripts/result.py"
|
| 60 |
+
]
|
| 61 |
+
}
|
configuration.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"framework": "PyTorch",
|
| 3 |
+
"task": "multi-resolution-subgrid-parameterization",
|
| 4 |
+
"model": "RF-ClimParam",
|
| 5 |
+
"input_format": "complete structured coarse-grid snapshots flattened to column samples",
|
| 6 |
+
"protocol": "joint multi-output RF-tend and RF-diff at x4/x8/x16/x32",
|
| 7 |
+
"default_config": "conf/config.yaml",
|
| 8 |
+
"training": "scripts/train.py",
|
| 9 |
+
"inference": "scripts/inference.py",
|
| 10 |
+
"evaluation": "scripts/result.py",
|
| 11 |
+
"visualization": "scripts/result.py"
|
| 12 |
+
}
|
model/rf_climparam.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure NumPy joint multi-output random forests for the SAM parameterization."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
FORMAT_VERSION = "rf_climparam_v2"
|
| 9 |
+
MODEL_NAME = "RF-ClimParam"
|
| 10 |
+
SCALES = ("x4", "x8", "x16", "x32")
|
| 11 |
+
TEND_INPUT_NAMES = tuple([f"T_{i:02d}" for i in range(48)] +
|
| 12 |
+
[f"qT_{i:02d}" for i in range(48)] +
|
| 13 |
+
[f"qp_{i:02d}" for i in range(48)] + ["abs_y"])
|
| 14 |
+
TEND_OUTPUT_NAMES = tuple([f"hL_tend_{i:02d}" for i in range(48)] +
|
| 15 |
+
[f"qT_tend_{i:02d}" for i in range(48)] +
|
| 16 |
+
[f"qp_tend_{i:02d}" for i in range(48)])
|
| 17 |
+
DIFF_INPUT_NAMES = tuple([f"T_low_{i:02d}" for i in range(15)] +
|
| 18 |
+
[f"qT_low_{i:02d}" for i in range(15)] +
|
| 19 |
+
[f"u_low_{i:02d}" for i in range(15)] +
|
| 20 |
+
[f"v_nh_low_{i:02d}" for i in range(15)] +
|
| 21 |
+
["windsurf", "abs_y"])
|
| 22 |
+
DIFF_OUTPUT_NAMES = tuple([f"Dbar_{i:02d}" for i in range(15)] +
|
| 23 |
+
["hL_surface_flux", "qT_surface_flux"])
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _check_array(name, value, features):
|
| 27 |
+
value = np.asarray(value)
|
| 28 |
+
if value.ndim != 2 or value.shape[1] != features:
|
| 29 |
+
raise ValueError(f"{name} must have shape [N,{features}], got {value.shape}")
|
| 30 |
+
if value.dtype not in (np.float32, np.float64) or not np.isfinite(value).all():
|
| 31 |
+
raise ValueError(f"{name} must be finite float32/float64")
|
| 32 |
+
return value.astype(np.float32, copy=False)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass
|
| 36 |
+
class TreeConfig:
|
| 37 |
+
max_depth: int = 5
|
| 38 |
+
min_samples_leaf: int = 3
|
| 39 |
+
max_features: object = "sqrt"
|
| 40 |
+
split_candidates: int = 8
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class ExtraRandomRegressionTree:
|
| 44 |
+
"""Randomized recursive tree whose leaves hold one joint output vector."""
|
| 45 |
+
|
| 46 |
+
def __init__(self, config, seed=0):
|
| 47 |
+
self.config = config
|
| 48 |
+
self.rng = np.random.default_rng(seed)
|
| 49 |
+
self.nodes = []
|
| 50 |
+
|
| 51 |
+
def fit(self, x, y):
|
| 52 |
+
x, y = np.asarray(x, np.float32), np.asarray(y, np.float32)
|
| 53 |
+
self.nodes = []
|
| 54 |
+
self._grow(x, y, np.arange(len(x)), 0)
|
| 55 |
+
return self
|
| 56 |
+
|
| 57 |
+
def _feature_count(self, total):
|
| 58 |
+
value = self.config.max_features
|
| 59 |
+
if value == "sqrt":
|
| 60 |
+
return max(1, int(np.sqrt(total)))
|
| 61 |
+
if value == "log2":
|
| 62 |
+
return max(1, int(np.log2(total)))
|
| 63 |
+
if isinstance(value, float):
|
| 64 |
+
return max(1, min(total, int(np.ceil(value * total))))
|
| 65 |
+
return max(1, min(total, int(value)))
|
| 66 |
+
|
| 67 |
+
def _grow(self, x, y, indices, depth):
|
| 68 |
+
node_id = len(self.nodes)
|
| 69 |
+
self.nodes.append(None)
|
| 70 |
+
leaf_value = y[indices].mean(axis=0).astype(np.float32)
|
| 71 |
+
minimum = int(self.config.min_samples_leaf)
|
| 72 |
+
if depth >= int(self.config.max_depth) or len(indices) < 2 * minimum:
|
| 73 |
+
self.nodes[node_id] = {"value": leaf_value}
|
| 74 |
+
return node_id
|
| 75 |
+
features = self.rng.choice(x.shape[1], self._feature_count(x.shape[1]), replace=False)
|
| 76 |
+
best = None
|
| 77 |
+
parent_sse = float(np.square(y[indices] - leaf_value).sum())
|
| 78 |
+
for feature in features:
|
| 79 |
+
values = x[indices, feature]
|
| 80 |
+
low, high = float(values.min()), float(values.max())
|
| 81 |
+
if not low < high:
|
| 82 |
+
continue
|
| 83 |
+
thresholds = self.rng.uniform(low, high, int(self.config.split_candidates))
|
| 84 |
+
for threshold in thresholds:
|
| 85 |
+
mask = values <= threshold
|
| 86 |
+
left, right = indices[mask], indices[~mask]
|
| 87 |
+
if len(left) < minimum or len(right) < minimum:
|
| 88 |
+
continue
|
| 89 |
+
left_mean, right_mean = y[left].mean(0), y[right].mean(0)
|
| 90 |
+
loss = float(np.square(y[left] - left_mean).sum() +
|
| 91 |
+
np.square(y[right] - right_mean).sum())
|
| 92 |
+
if best is None or loss < best[0]:
|
| 93 |
+
best = (loss, int(feature), float(threshold), left, right)
|
| 94 |
+
if best is None or best[0] >= parent_sse - 1e-10:
|
| 95 |
+
self.nodes[node_id] = {"value": leaf_value}
|
| 96 |
+
return node_id
|
| 97 |
+
_, feature, threshold, left, right = best
|
| 98 |
+
self.nodes[node_id] = {"feature": feature, "threshold": threshold,
|
| 99 |
+
"left": self._grow(x, y, left, depth + 1),
|
| 100 |
+
"right": self._grow(x, y, right, depth + 1)}
|
| 101 |
+
return node_id
|
| 102 |
+
|
| 103 |
+
def predict(self, x):
|
| 104 |
+
outputs = []
|
| 105 |
+
for row in np.asarray(x):
|
| 106 |
+
node = self.nodes[0]
|
| 107 |
+
while "value" not in node:
|
| 108 |
+
node = self.nodes[node["left"] if row[node["feature"]] <= node["threshold"] else node["right"]]
|
| 109 |
+
outputs.append(node["value"])
|
| 110 |
+
return np.asarray(outputs, dtype=np.float32)
|
| 111 |
+
|
| 112 |
+
def state_dict(self):
|
| 113 |
+
return {"config": vars(self.config), "nodes": self.nodes}
|
| 114 |
+
|
| 115 |
+
@classmethod
|
| 116 |
+
def from_state_dict(cls, state):
|
| 117 |
+
tree = cls(TreeConfig(**state["config"]))
|
| 118 |
+
tree.nodes = state["nodes"]
|
| 119 |
+
return tree
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class JointRandomForestRegressor:
|
| 123 |
+
"""Bootstrap ensemble retaining inseparable multi-output leaf predictions."""
|
| 124 |
+
|
| 125 |
+
def __init__(self, n_trees=2, seed=0, **tree_options):
|
| 126 |
+
self.n_trees, self.seed = int(n_trees), int(seed)
|
| 127 |
+
self.tree_config = TreeConfig(**tree_options)
|
| 128 |
+
self.trees = []
|
| 129 |
+
|
| 130 |
+
def fit(self, x, y):
|
| 131 |
+
x, y = np.asarray(x, np.float32), np.asarray(y, np.float32)
|
| 132 |
+
rng = np.random.default_rng(self.seed)
|
| 133 |
+
self.trees = []
|
| 134 |
+
for index in range(self.n_trees):
|
| 135 |
+
bootstrap = rng.integers(0, len(x), size=len(x))
|
| 136 |
+
tree = ExtraRandomRegressionTree(self.tree_config, self.seed + 1009 * (index + 1))
|
| 137 |
+
self.trees.append(tree.fit(x[bootstrap], y[bootstrap]))
|
| 138 |
+
return self
|
| 139 |
+
|
| 140 |
+
def predict(self, x):
|
| 141 |
+
if not self.trees:
|
| 142 |
+
raise RuntimeError("forest is not fitted")
|
| 143 |
+
return np.mean([tree.predict(x) for tree in self.trees], axis=0, dtype=np.float32)
|
| 144 |
+
|
| 145 |
+
def state_dict(self):
|
| 146 |
+
return {"n_trees": self.n_trees, "seed": self.seed,
|
| 147 |
+
"tree_config": vars(self.tree_config),
|
| 148 |
+
"trees": [tree.state_dict() for tree in self.trees]}
|
| 149 |
+
|
| 150 |
+
@classmethod
|
| 151 |
+
def from_state_dict(cls, state):
|
| 152 |
+
forest = cls(state["n_trees"], state["seed"], **state["tree_config"])
|
| 153 |
+
forest.trees = [ExtraRandomRegressionTree.from_state_dict(item) for item in state["trees"]]
|
| 154 |
+
return forest
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class StandardizedForest:
|
| 158 |
+
"""Block-standardized wrapper; one scalar mean/std is used per variable block."""
|
| 159 |
+
|
| 160 |
+
def __init__(self, forest, input_slices, output_slices, nonnegative_slice=None):
|
| 161 |
+
self.forest = forest
|
| 162 |
+
self.input_slices, self.output_slices = input_slices, output_slices
|
| 163 |
+
self.nonnegative_slice = nonnegative_slice
|
| 164 |
+
self.statistics = {}
|
| 165 |
+
|
| 166 |
+
@staticmethod
|
| 167 |
+
def _statistics(array, slices):
|
| 168 |
+
means, stds = np.zeros(array.shape[1], np.float32), np.ones(array.shape[1], np.float32)
|
| 169 |
+
for start, stop in slices:
|
| 170 |
+
mean = float(array[:, start:stop].mean())
|
| 171 |
+
std = max(float(array[:, start:stop].std()), 1e-6)
|
| 172 |
+
means[start:stop], stds[start:stop] = mean, std
|
| 173 |
+
return means, stds
|
| 174 |
+
|
| 175 |
+
def fit(self, x, y):
|
| 176 |
+
x = _check_array("inputs", x, self.input_slices[-1][1])
|
| 177 |
+
y = _check_array("targets", y, self.output_slices[-1][1])
|
| 178 |
+
x_mean, x_std = self._statistics(x, self.input_slices)
|
| 179 |
+
y_mean, y_std = self._statistics(y, self.output_slices)
|
| 180 |
+
self.statistics = {"input_mean": x_mean, "input_std": x_std,
|
| 181 |
+
"output_mean": y_mean, "output_std": y_std}
|
| 182 |
+
self.forest.fit((x - x_mean) / x_std, (y - y_mean) / y_std)
|
| 183 |
+
return self
|
| 184 |
+
|
| 185 |
+
def predict(self, x):
|
| 186 |
+
x = _check_array("inputs", x, len(self.statistics["input_mean"]))
|
| 187 |
+
prediction = self.forest.predict((x - self.statistics["input_mean"]) / self.statistics["input_std"])
|
| 188 |
+
prediction = prediction * self.statistics["output_std"] + self.statistics["output_mean"]
|
| 189 |
+
if self.nonnegative_slice is not None:
|
| 190 |
+
prediction[:, self.nonnegative_slice[0]:self.nonnegative_slice[1]] = np.maximum(
|
| 191 |
+
prediction[:, self.nonnegative_slice[0]:self.nonnegative_slice[1]], 0.0)
|
| 192 |
+
return prediction.astype(np.float32)
|
| 193 |
+
|
| 194 |
+
def state_dict(self):
|
| 195 |
+
return {"forest": self.forest.state_dict(), "input_slices": self.input_slices,
|
| 196 |
+
"output_slices": self.output_slices, "nonnegative_slice": self.nonnegative_slice,
|
| 197 |
+
"statistics": self.statistics}
|
| 198 |
+
|
| 199 |
+
@classmethod
|
| 200 |
+
def from_state_dict(cls, state):
|
| 201 |
+
model = cls(JointRandomForestRegressor.from_state_dict(state["forest"]),
|
| 202 |
+
state["input_slices"], state["output_slices"], state["nonnegative_slice"])
|
| 203 |
+
model.statistics = state["statistics"]
|
| 204 |
+
return model
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def build_pair(config, seed):
|
| 208 |
+
options = config["engineering"]
|
| 209 |
+
common = {"n_trees": options["trees"], "max_depth": options["max_depth"],
|
| 210 |
+
"min_samples_leaf": options["min_samples_leaf"],
|
| 211 |
+
"max_features": options["max_features"], "split_candidates": options["split_candidates"]}
|
| 212 |
+
tend = StandardizedForest(JointRandomForestRegressor(seed=seed, **common),
|
| 213 |
+
[(0, 48), (48, 96), (96, 144), (144, 145)],
|
| 214 |
+
[(0, 48), (48, 96), (96, 144)])
|
| 215 |
+
diff = StandardizedForest(JointRandomForestRegressor(seed=seed + 1, **common),
|
| 216 |
+
[(0, 15), (15, 30), (30, 45), (45, 60), (60, 61), (61, 62)],
|
| 217 |
+
[(0, 15), (15, 16), (16, 17)], nonnegative_slice=(0, 15))
|
| 218 |
+
return {"rf_tend": tend, "rf_diff": diff}
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def load_models(checkpoint):
|
| 222 |
+
if checkpoint.get("format_version") != FORMAT_VERSION or checkpoint.get("model_name") != MODEL_NAME:
|
| 223 |
+
raise ValueError("incompatible checkpoint model/format_version")
|
| 224 |
+
if not isinstance(checkpoint.get("model"), dict) or set(checkpoint["model"]) != set(SCALES):
|
| 225 |
+
raise ValueError("checkpoint model must contain all four scale forest states")
|
| 226 |
+
expected = {"rf_tend_input": 145, "rf_tend_output": 144,
|
| 227 |
+
"rf_diff_input": 62, "rf_diff_output": 17}
|
| 228 |
+
if checkpoint.get("model_config", {}).get("dimensions") != expected:
|
| 229 |
+
raise ValueError("checkpoint model_config dimensions are incompatible")
|
| 230 |
+
return {scale: {name: StandardizedForest.from_state_dict(state)
|
| 231 |
+
for name, state in pair.items()}
|
| 232 |
+
for scale, pair in checkpoint["model"].items()}
|
scripts/fake_data.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate one complete structured coarse-grid snapshot at every paper scale."""
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import yaml
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
sys.path.insert(0, str(ROOT))
|
| 12 |
+
from model.rf_climparam import (DIFF_INPUT_NAMES, DIFF_OUTPUT_NAMES,
|
| 13 |
+
FORMAT_VERSION, TEND_INPUT_NAMES,
|
| 14 |
+
TEND_OUTPUT_NAMES)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def make_columns(count, factor, seed, times=None, latitudes=None, longitudes=None):
|
| 18 |
+
rng = np.random.default_rng(seed)
|
| 19 |
+
if times is None:
|
| 20 |
+
times = np.arange(count, dtype=np.float32) * 3.0
|
| 21 |
+
if latitudes is None:
|
| 22 |
+
latitudes = rng.uniform(-90.0, 90.0, count).astype(np.float32)
|
| 23 |
+
if longitudes is None:
|
| 24 |
+
longitudes = rng.uniform(0.0, 360.0, count).astype(np.float32)
|
| 25 |
+
times, latitudes, longitudes = map(lambda x: np.asarray(x, np.float32), (times, latitudes, longitudes))
|
| 26 |
+
z = np.linspace(0.0, 1.0, 48, dtype=np.float32)[None, :]
|
| 27 |
+
lat = np.deg2rad(latitudes)[:, None]
|
| 28 |
+
lon = np.deg2rad(longitudes)[:, None]
|
| 29 |
+
phase = (2.0 * np.pi * times / (24.0 * 45.0))[:, None]
|
| 30 |
+
noise = 0.8 / np.sqrt(factor)
|
| 31 |
+
temperature = 300.0 - 78.0 * z - 13.0 * np.sin(lat) ** 2 + 1.5 * np.sin(lon + phase) * np.exp(-2 * z)
|
| 32 |
+
temperature += rng.normal(0, noise, (count, 48))
|
| 33 |
+
humidity = 0.018 * np.exp(-5.0 * z) * (0.35 + 0.65 * np.cos(lat) ** 2)
|
| 34 |
+
humidity *= 1.0 + 0.12 * np.sin(lon - phase)
|
| 35 |
+
humidity += rng.normal(0, noise * 2e-4, (count, 48))
|
| 36 |
+
humidity = np.maximum(humidity, 2e-6)
|
| 37 |
+
condensate = np.maximum(humidity - (0.009 * np.exp(-3.8 * z)), 0.0)
|
| 38 |
+
condensate += np.maximum(rng.normal(0, noise * 2e-5, (count, 48)), 0.0)
|
| 39 |
+
abs_y = np.abs(latitudes)[:, None] / 90.0
|
| 40 |
+
instability = np.maximum(temperature[:, :1] - temperature[:, 20:21] - 30.0, 0.0)
|
| 41 |
+
convective = (1.0 + 0.02 * instability) * humidity * np.exp(-1.5 * z)
|
| 42 |
+
h_tend = (-1.8e-5 * (temperature - temperature.mean(1, keepdims=True)) +
|
| 43 |
+
3e-4 * convective + rng.normal(0, noise * 2e-6, (count, 48)))
|
| 44 |
+
qt_tend = (-1.4e-4 * convective + 2e-6 * np.cos(lon + 3 * z) +
|
| 45 |
+
rng.normal(0, noise * 2e-7, (count, 48)))
|
| 46 |
+
qp_tend = np.maximum(8e-5 * convective + 5e-5 * condensate, 0.0)
|
| 47 |
+
tend_inputs = np.concatenate([temperature, humidity, condensate, abs_y], axis=1).astype(np.float32)
|
| 48 |
+
tend_targets = np.concatenate([h_tend, qt_tend, qp_tend], axis=1).astype(np.float32)
|
| 49 |
+
low_z = z[:, :15]
|
| 50 |
+
u = 7.0 * np.cos(lat) * (1.0 + low_z) + 2.0 * np.sin(lon + phase + low_z)
|
| 51 |
+
v = 3.0 * np.sin(2 * lat) * (1.0 - low_z) + 1.5 * np.cos(lon - phase + low_z)
|
| 52 |
+
v_nh = v * np.where(latitudes[:, None] < 0, -1.0, 1.0)
|
| 53 |
+
windsurf = np.sqrt(u[:, :1] ** 2 + v[:, :1] ** 2)
|
| 54 |
+
dbar = np.maximum((0.35 + 0.08 * windsurf) * np.exp(-3 * low_z) +
|
| 55 |
+
rng.normal(0, noise * 0.015, (count, 15)), 0.0)
|
| 56 |
+
h_flux = (12.0 + 0.8 * windsurf[:, 0] + 5.0 * np.cos(lat[:, 0]) ** 2).reshape(-1, 1)
|
| 57 |
+
q_flux = (2e-5 + 2e-6 * windsurf[:, 0] + 8e-6 * np.cos(lat[:, 0]) ** 2).reshape(-1, 1)
|
| 58 |
+
diff_inputs = np.concatenate([temperature[:, :15], humidity[:, :15], u, v_nh,
|
| 59 |
+
windsurf, abs_y], axis=1).astype(np.float32)
|
| 60 |
+
diff_targets = np.concatenate([dbar, h_flux, q_flux], axis=1).astype(np.float32)
|
| 61 |
+
return tend_inputs, tend_targets, diff_inputs, diff_targets, times, latitudes, longitudes
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def validate_flattening(rows, columns, ny, nx):
|
| 65 |
+
expected = np.arange(ny * nx)
|
| 66 |
+
restored = rows.astype(np.int64) * nx + columns.astype(np.int64)
|
| 67 |
+
if not np.array_equal(restored, expected):
|
| 68 |
+
raise ValueError(f"grid [{ny},{nx}] is not reversibly flattened in C order")
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def make_snapshot(ny, nx, factor, seed, time_hours=0.0):
|
| 72 |
+
rows = np.repeat(np.arange(ny, dtype=np.int32), nx)
|
| 73 |
+
columns = np.tile(np.arange(nx, dtype=np.int32), ny)
|
| 74 |
+
latitude = np.repeat(np.linspace(-90.0, 90.0, ny, dtype=np.float32), nx)
|
| 75 |
+
longitude = np.tile(np.linspace(0.0, 360.0, nx, endpoint=False, dtype=np.float32), ny)
|
| 76 |
+
time = np.full(ny * nx, time_hours, dtype=np.float32)
|
| 77 |
+
validate_flattening(rows, columns, ny, nx)
|
| 78 |
+
return make_columns(ny * nx, factor, seed, time, latitude, longitude), rows, columns
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def save(path, scale, arrays, grid_shape):
|
| 82 |
+
arrays, rows, columns = arrays
|
| 83 |
+
tend_x, tend_y, diff_x, diff_y, time, lat, lon = arrays
|
| 84 |
+
np.savez_compressed(path, format_version=np.array(FORMAT_VERSION), scale=np.array(scale),
|
| 85 |
+
grid_shape=np.asarray(grid_shape, dtype=np.int32),
|
| 86 |
+
snapshot_count=np.array(1, dtype=np.int32),
|
| 87 |
+
tend_inputs=tend_x, tend_targets=tend_y,
|
| 88 |
+
diff_inputs=diff_x, diff_targets=diff_y,
|
| 89 |
+
time_hours=time, latitude_deg=lat, longitude_deg=lon,
|
| 90 |
+
grid_row=rows, grid_column=columns,
|
| 91 |
+
tend_input_names=np.asarray(TEND_INPUT_NAMES),
|
| 92 |
+
tend_output_names=np.asarray(TEND_OUTPUT_NAMES),
|
| 93 |
+
diff_input_names=np.asarray(DIFF_INPUT_NAMES),
|
| 94 |
+
diff_output_names=np.asarray(DIFF_OUTPUT_NAMES))
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def main():
|
| 98 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 99 |
+
output = ROOT / config["data"]["root"]
|
| 100 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 101 |
+
snapshots, seed = int(config["data"]["snapshots_per_scale"]), int(config["seed"])
|
| 102 |
+
if snapshots != 1:
|
| 103 |
+
raise ValueError("compact synthetic protocol requires exactly one snapshot per scale")
|
| 104 |
+
for index, scale in enumerate(config["data"]["scales"]):
|
| 105 |
+
grid = config["data"]["coarse_grids"][scale]
|
| 106 |
+
factor, (ny, nx) = int(grid["factor"]), map(int, grid["grid"])
|
| 107 |
+
save(output / f"{scale}.npz", scale,
|
| 108 |
+
make_snapshot(ny, nx, factor, seed + index), (ny, nx))
|
| 109 |
+
ny, nx = map(int, config["evaluation"]["online_proxy_grid"])
|
| 110 |
+
save(output / "x32_online_proxy.npz", "x32",
|
| 111 |
+
make_snapshot(ny, nx, 32, seed + 100), (ny, nx))
|
| 112 |
+
print(f"generated=4 complete coarse snapshots online_native=1x{ny}x{nx}")
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
if __name__ == "__main__":
|
| 116 |
+
main()
|
scripts/inference.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run chunked inference over every point of all complete coarse grids."""
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import yaml
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 12 |
+
sys.path.insert(0, str(ROOT))
|
| 13 |
+
from model.rf_climparam import FORMAT_VERSION, load_models
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def load_checkpoint(path):
|
| 17 |
+
try:
|
| 18 |
+
return torch.load(path, map_location="cpu", weights_only=False)
|
| 19 |
+
except TypeError:
|
| 20 |
+
return torch.load(path, map_location="cpu")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def precipitation(targets, seconds):
|
| 24 |
+
# qT + qp tendencies are column-integrated with fixed positive layer mass weights.
|
| 25 |
+
moisture = targets[:, 48:96] + targets[:, 96:144]
|
| 26 |
+
weights = np.linspace(1.2, 0.2, 48, dtype=np.float32)
|
| 27 |
+
return np.maximum(-(moisture * weights).sum(1) * seconds, 0.0).astype(np.float32)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def predict_chunked(model, inputs, chunk_size):
|
| 31 |
+
prediction = np.empty((len(inputs), len(model.statistics["output_mean"])), dtype=np.float32)
|
| 32 |
+
for start in range(0, len(inputs), chunk_size):
|
| 33 |
+
stop = min(start + chunk_size, len(inputs))
|
| 34 |
+
prediction[start:stop] = model.predict(inputs[start:stop])
|
| 35 |
+
return prediction
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def infer_field(source, pair, seconds, chunk_size, prefix):
|
| 39 |
+
ny, nx = map(int, source["grid_shape"])
|
| 40 |
+
count = ny * nx
|
| 41 |
+
linear = source["grid_row"].astype(np.int64) * nx + source["grid_column"].astype(np.int64)
|
| 42 |
+
if len(source["tend_inputs"]) != count or not np.array_equal(linear, np.arange(count)):
|
| 43 |
+
raise ValueError(f"{prefix} is not one complete reversibly flattened grid")
|
| 44 |
+
tend_prediction = predict_chunked(pair["rf_tend"], source["tend_inputs"], chunk_size)
|
| 45 |
+
diff_prediction = predict_chunked(pair["rf_diff"], source["diff_inputs"], chunk_size)
|
| 46 |
+
values = {
|
| 47 |
+
f"{prefix}_grid_shape": np.asarray((ny, nx), dtype=np.int32),
|
| 48 |
+
f"{prefix}_grid_row": source["grid_row"], f"{prefix}_grid_column": source["grid_column"],
|
| 49 |
+
f"{prefix}_time_hours": source["time_hours"],
|
| 50 |
+
f"{prefix}_latitude_deg": source["latitude_deg"],
|
| 51 |
+
f"{prefix}_longitude_deg": source["longitude_deg"],
|
| 52 |
+
f"{prefix}_tend_targets": source["tend_targets"],
|
| 53 |
+
f"{prefix}_tend_predictions": tend_prediction,
|
| 54 |
+
f"{prefix}_diff_targets": source["diff_targets"],
|
| 55 |
+
f"{prefix}_diff_predictions": diff_prediction,
|
| 56 |
+
f"{prefix}_precipitation_target_3h": precipitation(source["tend_targets"], seconds),
|
| 57 |
+
f"{prefix}_precipitation_prediction_3h": precipitation(tend_prediction, seconds),
|
| 58 |
+
}
|
| 59 |
+
if not all(np.isfinite(value).all() for value in values.values()):
|
| 60 |
+
raise ValueError(f"non-finite prediction data for {prefix}")
|
| 61 |
+
return values
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def main():
|
| 65 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 66 |
+
checkpoint = load_checkpoint(ROOT / config["paths"]["checkpoint"])
|
| 67 |
+
models = load_models(checkpoint)
|
| 68 |
+
output = ROOT / config["paths"]["inference"]
|
| 69 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 70 |
+
seconds = int(config["evaluation"]["precipitation_seconds"])
|
| 71 |
+
chunk_size = int(config["evaluation"]["prediction_chunk_size"])
|
| 72 |
+
if chunk_size < 1:
|
| 73 |
+
raise ValueError("prediction_chunk_size must be positive")
|
| 74 |
+
scales = list(config["data"]["scales"])
|
| 75 |
+
packed = {}
|
| 76 |
+
variable_names = None
|
| 77 |
+
for scale in scales:
|
| 78 |
+
source = np.load(ROOT / config["data"]["root"] / f"{scale}.npz")
|
| 79 |
+
packed.update(infer_field(source, models[scale], seconds, chunk_size, scale))
|
| 80 |
+
if variable_names is None:
|
| 81 |
+
variable_names = {name: source[name] for name in ("tend_input_names", "tend_output_names",
|
| 82 |
+
"diff_input_names", "diff_output_names")}
|
| 83 |
+
source = np.load(ROOT / config["data"]["root"] / "x32_online_proxy.npz")
|
| 84 |
+
packed.update(infer_field(source, models["x32"], seconds, chunk_size, "online_x32_native"))
|
| 85 |
+
np.savez_compressed(output, format_version=np.array(FORMAT_VERSION), scales=np.asarray(scales),
|
| 86 |
+
online_scale=np.array("x32"), **variable_names, **packed)
|
| 87 |
+
print(f"predictions={output.relative_to(ROOT)} checkpoint_model={checkpoint['model_name']}")
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
if __name__ == "__main__":
|
| 91 |
+
main()
|
scripts/result.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate complete spatial predictions independently at every scale."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import matplotlib
|
| 7 |
+
matplotlib.use("Agg")
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import numpy as np
|
| 10 |
+
import yaml
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def scores(target, prediction):
|
| 17 |
+
residual = target - prediction
|
| 18 |
+
rmse = np.sqrt(np.mean(np.square(residual), axis=0))
|
| 19 |
+
denominator = np.sum(np.square(target - target.mean(axis=0)), axis=0)
|
| 20 |
+
r2 = 1.0 - np.sum(np.square(residual), axis=0) / np.maximum(denominator, 1e-20)
|
| 21 |
+
return r2, rmse
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def main():
|
| 25 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 26 |
+
data = np.load(ROOT / config["paths"]["inference"])
|
| 27 |
+
output = ROOT / config["paths"]["evaluation_dir"]
|
| 28 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 29 |
+
report = {"format_version": config["data"]["format_version"], "per_scale": {}}
|
| 30 |
+
plotted = []
|
| 31 |
+
scales = [str(scale) for scale in data["scales"]]
|
| 32 |
+
if scales != list(config["data"]["scales"]):
|
| 33 |
+
raise ValueError("prediction scales do not match configuration")
|
| 34 |
+
for scale_index, scale in enumerate(scales):
|
| 35 |
+
report["per_scale"][scale] = {"grid": data[f"{scale}_grid_shape"].tolist()}
|
| 36 |
+
for task, names in (("rf_tend", data["tend_output_names"]), ("rf_diff", data["diff_output_names"])):
|
| 37 |
+
target = data[f"{scale}_{task[3:]}_targets"]
|
| 38 |
+
prediction = data[f"{scale}_{task[3:]}_predictions"]
|
| 39 |
+
r2, rmse = scores(target, prediction)
|
| 40 |
+
if not np.isfinite(r2).all() or not np.isfinite(rmse).all():
|
| 41 |
+
raise FloatingPointError(f"non-finite offline metrics for {scale}/{task}")
|
| 42 |
+
report["per_scale"][scale][task] = {
|
| 43 |
+
"outputs": [{"name": str(name), "r2": float(r2[i]), "rmse": float(rmse[i])}
|
| 44 |
+
for i, name in enumerate(names)],
|
| 45 |
+
"mean_r2": float(r2.mean()), "mean_rmse": float(rmse.mean())}
|
| 46 |
+
plotted.append((scale, task, float(r2.mean()), float(rmse.mean())))
|
| 47 |
+
ny, nx = map(int, config["evaluation"]["online_proxy_grid"])
|
| 48 |
+
target = data["online_x32_native_precipitation_target_3h"].reshape(ny, nx).mean(axis=1)
|
| 49 |
+
prediction = data["online_x32_native_precipitation_prediction_3h"].reshape(ny, nx).mean(axis=1)
|
| 50 |
+
proxy_r2, proxy_rmse = scores(target[:, None], prediction[:, None])
|
| 51 |
+
report["online_engineering_proxy"] = {
|
| 52 |
+
"scale": "x32", "grid": [ny, nx], "time_steps": 1,
|
| 53 |
+
"metric": "zonal-mean 3h precipitation",
|
| 54 |
+
"r2": float(proxy_r2[0]), "rmse": float(proxy_rmse[0]),
|
| 55 |
+
"target_p99_9_3h": float(np.percentile(data["online_x32_native_precipitation_target_3h"], 99.9)),
|
| 56 |
+
"prediction_p99_9_3h": float(np.percentile(data["online_x32_native_precipitation_prediction_3h"], 99.9)),
|
| 57 |
+
"note": "Engineering proxy for online coupling, not the paper's 600-day SAM simulation."}
|
| 58 |
+
numeric = [scale[task]["mean_r2"] for scale in report["per_scale"].values()
|
| 59 |
+
for task in ("rf_tend", "rf_diff")]
|
| 60 |
+
numeric += [report["online_engineering_proxy"][key] for key in ("r2", "rmse", "target_p99_9_3h", "prediction_p99_9_3h")]
|
| 61 |
+
if not np.isfinite(numeric).all():
|
| 62 |
+
raise FloatingPointError("evaluation report contains non-finite values")
|
| 63 |
+
(output / "metrics.json").write_text(json.dumps(report, indent=2) + "\n")
|
| 64 |
+
figure, axes = plt.subplots(1, 2, figsize=(12, 4.5))
|
| 65 |
+
labels = [f"{scale}\n{task[3:]}" for scale, task, _, _ in plotted]
|
| 66 |
+
axes[0].bar(labels, [item[2] for item in plotted], color=["#275d6c", "#d98b3a"] * 4)
|
| 67 |
+
axes[0].axhline(0, color="black", linewidth=0.7); axes[0].set_ylabel("Mean output R2")
|
| 68 |
+
axes[0].tick_params(axis="x", labelsize=8); axes[0].set_title("Complete-grid column prediction")
|
| 69 |
+
latitude = np.linspace(-90, 90, ny)
|
| 70 |
+
axes[1].plot(latitude, target, label="target", color="#202020", linewidth=2)
|
| 71 |
+
axes[1].plot(latitude, prediction, label="RF prediction", color="#c44e52", linewidth=2)
|
| 72 |
+
axes[1].set(xlabel="Latitude (degrees)", ylabel="3 h precipitation (proxy units)",
|
| 73 |
+
title=f"x32 native zonal mean on {ny}x{nx} grid")
|
| 74 |
+
axes[1].legend(); figure.tight_layout(); figure.savefig(output / "comparison.png", dpi=150)
|
| 75 |
+
plt.close(figure)
|
| 76 |
+
print(f"evaluation={output.relative_to(ROOT)} online_proxy_r2={proxy_r2[0]:.3f}")
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
if __name__ == "__main__":
|
| 80 |
+
main()
|
scripts/train.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train four scale-specific pairs of joint multi-output random forests."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import random
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
import yaml
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 15 |
+
sys.path.insert(0, str(ROOT))
|
| 16 |
+
from model.rf_climparam import FORMAT_VERSION, MODEL_NAME, build_pair
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def load_scale(path, scale):
|
| 20 |
+
data = np.load(path)
|
| 21 |
+
expected = {"tend_inputs": 145, "tend_targets": 144, "diff_inputs": 62, "diff_targets": 17}
|
| 22 |
+
if str(data["format_version"]) != FORMAT_VERSION or str(data["scale"]) != scale:
|
| 23 |
+
raise ValueError(f"invalid metadata in {path}")
|
| 24 |
+
count = len(data["tend_inputs"])
|
| 25 |
+
ny, nx = map(int, data["grid_shape"])
|
| 26 |
+
if count != ny * nx or int(data["snapshot_count"]) != 1:
|
| 27 |
+
raise ValueError(f"{scale} must contain one complete [{ny},{nx}] snapshot")
|
| 28 |
+
linear = data["grid_row"].astype(np.int64) * nx + data["grid_column"].astype(np.int64)
|
| 29 |
+
if not np.array_equal(linear, np.arange(count)):
|
| 30 |
+
raise ValueError(f"{scale} grid cannot be reversibly flattened")
|
| 31 |
+
for name, width in expected.items():
|
| 32 |
+
value = data[name]
|
| 33 |
+
if value.shape != (count, width) or value.dtype != np.float32 or not np.isfinite(value).all():
|
| 34 |
+
raise ValueError(f"{scale}/{name} requires finite float32 [{count},{width}]")
|
| 35 |
+
if np.any(data["diff_targets"][:, :15] < 0):
|
| 36 |
+
raise ValueError("Dbar training targets must be nonnegative")
|
| 37 |
+
return data
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def sample_training_columns(data, columns_per_latitude, seed):
|
| 41 |
+
ny, nx = map(int, data["grid_shape"])
|
| 42 |
+
if not 1 <= columns_per_latitude <= nx:
|
| 43 |
+
raise ValueError("train_columns_per_latitude must be between 1 and grid width")
|
| 44 |
+
rng = np.random.default_rng(seed)
|
| 45 |
+
selected = []
|
| 46 |
+
for row in range(ny):
|
| 47 |
+
selected.extend(row * nx + rng.choice(nx, columns_per_latitude, replace=False))
|
| 48 |
+
return np.asarray(selected, dtype=np.int64)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def main():
|
| 52 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 53 |
+
seed = int(config["seed"])
|
| 54 |
+
random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
|
| 55 |
+
distributed = int(os.environ.get("WORLD_SIZE", "1")) > 1
|
| 56 |
+
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
| 57 |
+
if distributed:
|
| 58 |
+
torch.distributed.init_process_group("gloo")
|
| 59 |
+
rank = torch.distributed.get_rank() if distributed else 0
|
| 60 |
+
world = torch.distributed.get_world_size() if distributed else 1
|
| 61 |
+
scales = list(config["data"]["scales"])
|
| 62 |
+
assigned = [scale for i, scale in enumerate(scales) if i % world == rank]
|
| 63 |
+
local_states, local_metrics = {}, {}
|
| 64 |
+
for scale_index, scale in enumerate(assigned):
|
| 65 |
+
data = load_scale(ROOT / config["data"]["root"] / f"{scale}.npz", scale)
|
| 66 |
+
selected = sample_training_columns(
|
| 67 |
+
data, int(config["data"]["train_columns_per_latitude"]), seed + scales.index(scale))
|
| 68 |
+
pair = build_pair(config["model"], seed + scales.index(scale) * 100)
|
| 69 |
+
pair["rf_tend"].fit(data["tend_inputs"][selected], data["tend_targets"][selected])
|
| 70 |
+
pair["rf_diff"].fit(data["diff_inputs"][selected], data["diff_targets"][selected])
|
| 71 |
+
local_states[scale] = {name: model.state_dict() for name, model in pair.items()}
|
| 72 |
+
local_metrics[scale] = {"complete_grid_points": len(data["tend_inputs"]),
|
| 73 |
+
"train_columns": len(selected),
|
| 74 |
+
"columns_per_latitude": int(config["data"]["train_columns_per_latitude"])}
|
| 75 |
+
print(f"rank={rank} trained={scale} sampled_columns={len(selected)}")
|
| 76 |
+
if distributed:
|
| 77 |
+
gathered_states, gathered_metrics = [None] * world, [None] * world
|
| 78 |
+
torch.distributed.all_gather_object(gathered_states, local_states)
|
| 79 |
+
torch.distributed.all_gather_object(gathered_metrics, local_metrics)
|
| 80 |
+
states = {key: value for item in gathered_states for key, value in item.items()}
|
| 81 |
+
metrics = {key: value for item in gathered_metrics for key, value in item.items()}
|
| 82 |
+
else:
|
| 83 |
+
states, metrics = local_states, local_metrics
|
| 84 |
+
if rank == 0:
|
| 85 |
+
if set(states) != set(scales):
|
| 86 |
+
raise RuntimeError("not all scales were trained")
|
| 87 |
+
checkpoint = {
|
| 88 |
+
"model": states,
|
| 89 |
+
"model_name": MODEL_NAME,
|
| 90 |
+
"model_config": {"dimensions": {"rf_tend_input": 145, "rf_tend_output": 144,
|
| 91 |
+
"rf_diff_input": 62, "rf_diff_output": 17},
|
| 92 |
+
"engineering": config["model"]["engineering"],
|
| 93 |
+
"paper_model": config["paper_model"], "scales": scales},
|
| 94 |
+
"format_version": FORMAT_VERSION, "seed": seed}
|
| 95 |
+
path = ROOT / config["paths"]["checkpoint"]
|
| 96 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 97 |
+
torch.save(checkpoint, path)
|
| 98 |
+
metrics_path = ROOT / config["paths"]["training_metrics"]
|
| 99 |
+
metrics_path.parent.mkdir(parents=True, exist_ok=True)
|
| 100 |
+
metrics_path.write_text(json.dumps({"format_version": FORMAT_VERSION, "scales": metrics}, indent=2) + "\n")
|
| 101 |
+
print(f"checkpoint={path.relative_to(ROOT)} scales={len(states)}")
|
| 102 |
+
if distributed:
|
| 103 |
+
torch.distributed.barrier(); torch.distributed.destroy_process_group()
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
main()
|
weight/.gitkeep
ADDED
|
File without changes
|