Publish GlobalSurgeML engineering reproduction
Browse files- .gitattributes +1 -24
- README.md +139 -0
- README_zh.md +147 -0
- conf/config.yaml +55 -0
- config.json +72 -0
- configuration.json +12 -0
- model/globalsurgeml.py +141 -0
- scripts/fake_data.py +69 -0
- scripts/inference.py +53 -0
- scripts/result.py +78 -0
- scripts/train.py +130 -0
- weight/.gitkeep +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,12 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
| 1 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 2 |
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
| 3 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
| 4 |
*.model filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 5 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 6 |
*.npz filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 8 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 9 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 10 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
weight/* filter=lfs diff=lfs merge=lfs -text
|
|
|
README.md
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-4.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
tags:
|
| 6 |
+
- OneScience
|
| 7 |
+
- Earth Science
|
| 8 |
+
- Storm Surge Modeling
|
| 9 |
+
- Random Forest
|
| 10 |
+
frameworks: PyTorch
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
<p align="center">
|
| 14 |
+
<strong><span style="font-size: 30px;">GlobalSurgeML</span></strong>
|
| 15 |
+
</p>
|
| 16 |
+
|
| 17 |
+
# Model Introduction
|
| 18 |
+
|
| 19 |
+
GlobalSurgeML simulates daily maximum storm surge from wind fields, mean sea-level pressure, sea-surface temperature, and precipitation around tide gauges. It applies PCA to reduce multicollinearity in gridded predictors, then uses stepwise multiple linear regression or random forests to map station-level features to daily maximum non-tidal residuals in meters, providing a computationally efficient approach for long-period and large-scale storm-surge simulation.
|
| 20 |
+
|
| 21 |
+
Paper: Data-Driven Modeling of Global Storm Surges
|
| 22 |
+
https://doi.org/10.3389/fmars.2020.00260
|
| 23 |
+
|
| 24 |
+
# Model Description
|
| 25 |
+
|
| 26 |
+
The method was proposed by researchers from the University of Central Florida and Universidad de Cantabria. The paper trained and validated models using GESLA-2 tide gauges, CCMP, 20CRV2c, Microwave OI SST, GPCP, ERA-Interim, and GTSR. It supports daily maximum storm-surge regression at quasi-global tide gauges, extreme-event evaluation, and comparison with the GTSR hydrodynamic reanalysis.
|
| 27 |
+
|
| 28 |
+
# Use Cases
|
| 29 |
+
|
| 30 |
+
| Use Case | Description |
|
| 31 |
+
| :---: | :--- |
|
| 32 |
+
| Daily maximum surge simulation | Estimate the daily maximum non-tidal residual from PCA features of meteorological and oceanographic predictors around a station. |
|
| 33 |
+
| Lagged forcing | Use 6-hourly inputs and wind/pressure information up to 30 hours before surge occurrence. |
|
| 34 |
+
| Method comparison | Compare stepwise linear regression, random forests, remote-sensing inputs, and ERA-Interim inputs across six configurations. |
|
| 35 |
+
| ModelScope/OneCode execution | Validate training, inference, evaluation, visualization, and checkpoint workflows in ModelScope or OneCode. |
|
| 36 |
+
| Multi-GPU training | Launch distributed data-parallel optimization of the linear models with `torchrun`. |
|
| 37 |
+
|
| 38 |
+
# Usage Instructions
|
| 39 |
+
|
| 40 |
+
## 1.OneCode
|
| 41 |
+
|
| 42 |
+
Experience intelligent, one-click AI4S programming through the OneCode online environment:
|
| 43 |
+
|
| 44 |
+
[Try intelligent, one-click AI4S programming](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home)
|
| 45 |
+
|
| 46 |
+
## 2. Download and Installation
|
| 47 |
+
|
| 48 |
+
```bash
|
| 49 |
+
hf download OneScience-Group/GlobalSurgeML --local-dir ./GlobalSurgeML
|
| 50 |
+
cd GlobalSurgeML
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
### Environment Dependencies
|
| 54 |
+
|
| 55 |
+
**Hardware Requirements**
|
| 56 |
+
|
| 57 |
+
- A GPU or DCU is recommended.
|
| 58 |
+
- A CPU can be used for connectivity validation with the default small-sample configuration.
|
| 59 |
+
- DCU users must install DTK in advance. DTK 25.04.2 or later, or the OneScience-recommended version matching the current cluster, is recommended.
|
| 60 |
+
|
| 61 |
+
**DCU Environment**
|
| 62 |
+
|
| 63 |
+
```bash
|
| 64 |
+
# Activate DTK and Conda first
|
| 65 |
+
conda create -n onescience311 python=3.11 -y
|
| 66 |
+
conda activate onescience311
|
| 67 |
+
pip install onescience[earth-dcu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 68 |
+
```
|
| 69 |
+
|
| 70 |
+
**GPU Environment**
|
| 71 |
+
|
| 72 |
+
```bash
|
| 73 |
+
# Activate Conda first
|
| 74 |
+
conda create -n onescience311 python=3.11 -y libstdcxx-ng=12 libgcc-ng=12 gcc_linux-64=12 gxx_linux-64=12
|
| 75 |
+
conda activate onescience311
|
| 76 |
+
pip install onescience[earth-gpu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
### Training Data
|
| 80 |
+
|
| 81 |
+
This repository uses a small number of synthetic station-day records to validate the engineering workflow.
|
| 82 |
+
|
| 83 |
+
The synthetic data combines persistent weather states, a seasonal cycle, latitude effects, and nonlinear cyclone forcing to produce correlated features and surge targets rather than unrelated random noise. It preserves the approximate 50/300-dimensional configurations in paper Table 1. The paper also states that local raw grids may contain up to about 5,000 variables and generally retain 300 to 500 PCs at 90% explained variance, so no single fixed raw spatial tensor applies to every station. Synthetic results validate only the core method and engineering workflow; they do not represent the official data distribution or training scale.
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
python scripts/fake_data.py
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
### Training
|
| 90 |
+
|
| 91 |
+
```bash
|
| 92 |
+
python scripts/train.py
|
| 93 |
+
```
|
| 94 |
+
|
| 95 |
+
For multi-GPU training, use:
|
| 96 |
+
|
| 97 |
+
```bash
|
| 98 |
+
torchrun --nproc_per_node=8 --nnodes=1 --rdzv_id=1000 --rdzv_backend=c10d --max_restarts=0 --master_addr="localhost" --master_port=29500 scripts/train.py
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
The default engineering configuration preserves all four 50/300-dimensional inputs and the scalar output while reducing the sample count, maximum selected linear features, and random-forest depth. Formal experiments should rerun station-wise PCA, 10-fold cross-validation, and six-configuration majority-metric selection on complete real records.
|
| 102 |
+
|
| 103 |
+
```text
|
| 104 |
+
result/checkpoints/globalsurgeml.pt
|
| 105 |
+
result/training/metrics.json
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
### Trained Weights
|
| 109 |
+
|
| 110 |
+
No weights are bundled under `weight/`. The paper and its supplementary material do not provide a confirmed official model checkpoint. The generated checkpoint uses this independent PyTorch/scikit-learn engineering format and is not claimed to be compatible with external weights.
|
| 111 |
+
|
| 112 |
+
### Inference
|
| 113 |
+
|
| 114 |
+
```bash
|
| 115 |
+
python scripts/inference.py
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
Inference loads `globalsurgeml.pt`, restores all six configurations, and runs them on the four station-day PCA inputs. The complete numerical output includes six daily maximum surge predictions, observed targets, the GTSR baseline, timestamps, and station coordinates in `result/output/predictions.npz`.
|
| 119 |
+
|
| 120 |
+
### Evaluation and Visualization
|
| 121 |
+
|
| 122 |
+
```bash
|
| 123 |
+
python scripts/result.py
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
Evaluation follows the paper by computing Pearson correlation, RMSE, NSE, and relative RMSE, separately evaluating extreme surges above the observed 95th percentile and separating tropical from subtropical/extratropical samples by latitude. It saves `result/evaluation/metrics.json` and generates the surge-series and observed-versus-modeled plot `result/evaluation/comparison.png`. Synthetic-data results validate only the engineering workflow and do not represent paper performance on real test data.
|
| 127 |
+
|
| 128 |
+
# Official OneScience Information
|
| 129 |
+
|
| 130 |
+
| Platform | OneScience Main Repository | Skills Repository |
|
| 131 |
+
| --- | --- | --- |
|
| 132 |
+
| Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills |
|
| 133 |
+
| GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills |
|
| 134 |
+
|
| 135 |
+
# Citation and License
|
| 136 |
+
|
| 137 |
+
This repository is an independent engineering reproduction of the public GlobalSurgeML paper specifications.
|
| 138 |
+
|
| 139 |
+
Use of this repository's code, official model weights, and data remains subject to the licenses and terms of their respective projects.
|
README_zh.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: CC-BY-4.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
- zh
|
| 6 |
+
tags:
|
| 7 |
+
- OneScience
|
| 8 |
+
- 地球科学
|
| 9 |
+
- 风暴潮模拟
|
| 10 |
+
- 随机森林
|
| 11 |
+
frameworks: PyTorch
|
| 12 |
+
datasets:
|
| 13 |
+
- GESLA-2
|
| 14 |
+
- CCMP
|
| 15 |
+
- 20CRV2c
|
| 16 |
+
- GPCP
|
| 17 |
+
- ERA-Interim
|
| 18 |
+
- GTSR
|
| 19 |
+
---
|
| 20 |
+
|
| 21 |
+
<p align="center">
|
| 22 |
+
<strong><span style="font-size: 30px;">GlobalSurgeML</span></strong>
|
| 23 |
+
</p>
|
| 24 |
+
|
| 25 |
+
# 模型介绍
|
| 26 |
+
|
| 27 |
+
GlobalSurgeML 使用验潮站周围的风场、平均海平面气压、海表温度和降水信息模拟每日最大风暴增水。模型先以 PCA 消除网格预测变量的多重共线性,再使用逐步多元线性回归或随机森林建立站点级映射,输出单位为米的每日最大非潮汐残差,适合低成本的长时间尺度和大空间尺度风暴潮模拟。
|
| 28 |
+
|
| 29 |
+
论文:Data-Driven Modeling of Global Storm Surges
|
| 30 |
+
https://doi.org/10.3389/fmars.2020.00260
|
| 31 |
+
|
| 32 |
+
# 模型描述
|
| 33 |
+
|
| 34 |
+
该方法由 University of Central Florida 与 Universidad de Cantabria 的研究人员提出。论文使用 GESLA-2 验潮站、CCMP、20CRV2c、Microwave OI SST、GPCP、ERA-Interim 和 GTSR 数据训练与验证。模型适用于准全球验潮站的每日最大风暴增水回归、极端事件评估以及与 GTSR 水动力再分析比较。
|
| 35 |
+
|
| 36 |
+
# 适用场景
|
| 37 |
+
|
| 38 |
+
| 场景 | 说明 |
|
| 39 |
+
| :---: | :--- |
|
| 40 |
+
| 日最大增水模拟 | 从站点周边气象与海洋预测变量的 PCA 特征估计每日最大非潮汐残差。 |
|
| 41 |
+
| 滞后效应建模 | 使用 6 小时输入并考虑风暴潮发生前最多 30 小时的风速与气压信息。 |
|
| 42 |
+
| 方法对比 | 比较逐步线性回归、随机森林、遥感输入与 ERA-Interim 输入的六种配置。 |
|
| 43 |
+
| ModelScope/OneCode 运行 | 在 ModelScope 或 OneCode 环境中验证训练、推理、评估、可视化和 checkpoint 流程。 |
|
| 44 |
+
| 多卡训练 | 通过 `torchrun` 启动线性模型的分布式数据并行优化。 |
|
| 45 |
+
|
| 46 |
+
# 使用说明
|
| 47 |
+
|
| 48 |
+
## 1.OneCode
|
| 49 |
+
|
| 50 |
+
可通过 OneCode 在线环境体验智能化一键式 AI4S 编程:
|
| 51 |
+
|
| 52 |
+
[点击体验智能化一键式 AI4S 编程](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home)
|
| 53 |
+
|
| 54 |
+
## 2.下载安装
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
modelscope download --model OneScience/GlobalSurgeML --local_dir ./GlobalSurgeML
|
| 58 |
+
cd GlobalSurgeML
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
### 环境依赖
|
| 62 |
+
|
| 63 |
+
**硬件要求**
|
| 64 |
+
|
| 65 |
+
- 推荐使用 GPU 或 DCU 运行。
|
| 66 |
+
- CPU 可用于默认小样本配置的连通性验证。
|
| 67 |
+
- DCU 用户需预先安装 DTK,建议使用 DTK 25.04.2 以上版本或与当前集群匹配的 OneScience 推荐版本。
|
| 68 |
+
|
| 69 |
+
**DCU环境**
|
| 70 |
+
|
| 71 |
+
```bash
|
| 72 |
+
# 请首先激活DTK及CONDA
|
| 73 |
+
conda create -n onescience311 python=3.11 -y
|
| 74 |
+
conda activate onescience311
|
| 75 |
+
pip install onescience[earth-dcu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
**GPU环境**
|
| 79 |
+
|
| 80 |
+
```bash
|
| 81 |
+
# 请首先激活CONDA
|
| 82 |
+
conda create -n onescience311 python=3.11 -y libstdcxx-ng=12 libgcc-ng=12 gcc_linux-64=12 gxx_linux-64=12
|
| 83 |
+
conda activate onescience311
|
| 84 |
+
pip install onescience[earth-gpu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
### 训练数据介绍
|
| 88 |
+
|
| 89 |
+
本仓库使用少量虚拟站点日样本验证工程流程。
|
| 90 |
+
|
| 91 |
+
虚拟数据用具有持续性的天气状态、季节周期、纬度效应和非线性气旋强迫共同产生相关特征与增水目标,不是无关联随机噪声。它保持论文表 1 给出的约 50/300 维配置;论文同时指出原始局地网格最多约 5000 个变量,保留 90% 方差后通常为 300 至 500 个 PC,因此并不存在适用于所有站点的唯一固定原始空间张量尺寸。虚拟数据仅用于验证核心方法、训练、推理和评估流程,不代表官方数据分布与训练规模。
|
| 92 |
+
|
| 93 |
+
```bash
|
| 94 |
+
python scripts/fake_data.py
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
### 训练
|
| 98 |
+
|
| 99 |
+
```bash
|
| 100 |
+
python scripts/train.py
|
| 101 |
+
```
|
| 102 |
+
|
| 103 |
+
多卡训练可使用:
|
| 104 |
+
|
| 105 |
+
```bash
|
| 106 |
+
torchrun --nproc_per_node=8 --nnodes=1 --rdzv_id=1000 --rdzv_backend=c10d --max_restarts=0 --master_addr="localhost" --master_port=29500 scripts/train.py
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
默认工程配置保持四类输入的 50/300 维和单值输出不变,仅缩小样本数、线性回归最多保留的特征数和随机森林深度。正式实验应重新执行逐站点 PCA、10 折交叉验证和六配置多数指标选择,并使用完整真实记录。
|
| 110 |
+
|
| 111 |
+
```text
|
| 112 |
+
result/checkpoints/globalsurgeml.pt
|
| 113 |
+
result/training/metrics.json
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
### 训练权重
|
| 117 |
+
|
| 118 |
+
本仓库不在 `weight/` 中内置权重。论文及其补充材料未提供可确认的官方模型 checkpoint;当前 checkpoint 是本独立 PyTorch/scikit-learn 工程格式,不声明兼容任何外部权重。
|
| 119 |
+
|
| 120 |
+
### 推理
|
| 121 |
+
|
| 122 |
+
```bash
|
| 123 |
+
python scripts/inference.py
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
推理加载 `globalsurgeml.pt`,用四组站点日 PCA 输入恢复并运行六种模型配置。完整数值结果包含六组日最大增水预测、观测目标、GTSR 基线、时间和站点坐标,保存到 `result/output/predictions.npz`。
|
| 127 |
+
|
| 128 |
+
### 评估和可视化
|
| 129 |
+
|
| 130 |
+
```bash
|
| 131 |
+
python scripts/result.py
|
| 132 |
+
```
|
| 133 |
+
|
| 134 |
+
评估按论文协议计算 Pearson 相关系数、RMSE、NSE 和相对 RMSE,并对观测值第 95 百分位以上的极端增水单独评估���同时按纬度区分热带与副热带/温带样本。脚本保存 `result/evaluation/metrics.json`,并生成增水序列及观测-模拟散点图 `result/evaluation/comparison.png`。虚拟数据结果仅用于验证工程流程,不代表论文真实测试集性能。
|
| 135 |
+
|
| 136 |
+
# OneScience官方信息
|
| 137 |
+
|
| 138 |
+
| 平台 | OneScience 主仓库 | Skills 仓库 |
|
| 139 |
+
| --- | --- | --- |
|
| 140 |
+
| Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills |
|
| 141 |
+
| GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills |
|
| 142 |
+
|
| 143 |
+
# 引用与许可证
|
| 144 |
+
|
| 145 |
+
本仓库为 GlobalSurgeML 论文公开规格的独立工程复现版本。
|
| 146 |
+
|
| 147 |
+
本仓库代码、官方模型权重和数据的使用仍应以各自项目中的许可证及使用条款为准。
|
conf/config.yaml
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed: 42
|
| 2 |
+
data:
|
| 3 |
+
root: data
|
| 4 |
+
format_version: global_storm_surge_pca_station_daily_v1
|
| 5 |
+
train_samples: 60
|
| 6 |
+
test_samples: 24
|
| 7 |
+
target: daily_maximum_non_tidal_residual
|
| 8 |
+
target_unit: m
|
| 9 |
+
sample_interval_hours: 24
|
| 10 |
+
lag_interval_hours: 6
|
| 11 |
+
maximum_lag_hours: 30
|
| 12 |
+
rs_daily_features: 50
|
| 13 |
+
rs_lagged_features: 300
|
| 14 |
+
ar_daily_features: 50
|
| 15 |
+
ar_lagged_features: 300
|
| 16 |
+
feature_layout: NF
|
| 17 |
+
target_layout: N1
|
| 18 |
+
feature_groups:
|
| 19 |
+
rs: [u, u2, u3, v, v2, v3, slp, sst, precipitation]
|
| 20 |
+
ar: [u, u2, u3, v, v2, v3, slp]
|
| 21 |
+
model:
|
| 22 |
+
p_value_threshold: 0.05
|
| 23 |
+
maximum_selected_features: 20
|
| 24 |
+
random_forest_trees: 50
|
| 25 |
+
random_forest_max_depth: 4
|
| 26 |
+
random_forest_max_features: sqrt
|
| 27 |
+
bootstrap: true
|
| 28 |
+
paper_model:
|
| 29 |
+
p_value_threshold: 0.05
|
| 30 |
+
pca_explained_variance: 0.90
|
| 31 |
+
pca_feature_range: 300-500
|
| 32 |
+
daily_feature_count_approx: 50
|
| 33 |
+
lagged_feature_count_approx: 300
|
| 34 |
+
random_forest_trees: 50
|
| 35 |
+
random_forest_sensitivity_range: 1-200
|
| 36 |
+
random_forest_max_depth: not_reported
|
| 37 |
+
train:
|
| 38 |
+
epochs: 8
|
| 39 |
+
batch_size: 12
|
| 40 |
+
learning_rate: 0.03
|
| 41 |
+
optimizer: Adam
|
| 42 |
+
loss: mean_squared_error
|
| 43 |
+
weight_decay: 0.0
|
| 44 |
+
gradient_clip_norm: 10.0
|
| 45 |
+
num_workers: 0
|
| 46 |
+
runtime:
|
| 47 |
+
device: auto
|
| 48 |
+
paths:
|
| 49 |
+
checkpoint: result/checkpoints/globalsurgeml.pt
|
| 50 |
+
training_metrics: result/training/metrics.json
|
| 51 |
+
inference_dir: result/output
|
| 52 |
+
evaluation_dir: result/evaluation
|
| 53 |
+
evaluation:
|
| 54 |
+
extreme_percentile: 95.0
|
| 55 |
+
tropical_latitude_limit_degrees: 30.0
|
config.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_name": "GlobalSurgeML",
|
| 3 |
+
"model_type": "globalsurgeml",
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GlobalSurgeML"
|
| 6 |
+
],
|
| 7 |
+
"framework": "PyTorch",
|
| 8 |
+
"domain": "physical-oceanography",
|
| 9 |
+
"task": "daily-maximum-storm-surge-regression",
|
| 10 |
+
"implementation": {
|
| 11 |
+
"entry_point": "model/globalsurgeml.py",
|
| 12 |
+
"scope": "core-method and real-feature-dimension scaled end-to-end engineering reproduction",
|
| 13 |
+
"train_script": "scripts/train.py",
|
| 14 |
+
"inference_script": "scripts/inference.py",
|
| 15 |
+
"evaluation_script": "scripts/result.py",
|
| 16 |
+
"synthetic_data_script": "scripts/fake_data.py"
|
| 17 |
+
},
|
| 18 |
+
"architecture": {
|
| 19 |
+
"family": "station-wise PCA stepwise-linear and random-forest regression ensemble",
|
| 20 |
+
"configurations": [
|
| 21 |
+
"LR-RS",
|
| 22 |
+
"LR-RS-lag",
|
| 23 |
+
"RF-RS-lag",
|
| 24 |
+
"LR-AR",
|
| 25 |
+
"LR-AR-lag",
|
| 26 |
+
"RF-AR-lag"
|
| 27 |
+
],
|
| 28 |
+
"daily_feature_count": 50,
|
| 29 |
+
"lagged_feature_count": 300,
|
| 30 |
+
"p_value_threshold": 0.05,
|
| 31 |
+
"random_forest_trees": 50,
|
| 32 |
+
"engineering_random_forest_max_depth": 4,
|
| 33 |
+
"paper_pca_explained_variance": 0.9,
|
| 34 |
+
"paper_post_pca_feature_range": "300-500"
|
| 35 |
+
},
|
| 36 |
+
"data": {
|
| 37 |
+
"datasets": [
|
| 38 |
+
"GESLA-2",
|
| 39 |
+
"CCMP",
|
| 40 |
+
"20CRV2c",
|
| 41 |
+
"Microwave OI SST",
|
| 42 |
+
"GPCP",
|
| 43 |
+
"ERA-Interim",
|
| 44 |
+
"GTSR"
|
| 45 |
+
],
|
| 46 |
+
"protocol": "global_storm_surge_pca_station_daily_v1",
|
| 47 |
+
"format": "NPZ",
|
| 48 |
+
"input_layout": "NF",
|
| 49 |
+
"target_layout": "N1",
|
| 50 |
+
"input_shapes": {
|
| 51 |
+
"rs_daily": ["N", 50],
|
| 52 |
+
"rs_lagged": ["N", 300],
|
| 53 |
+
"ar_daily": ["N", 50],
|
| 54 |
+
"ar_lagged": ["N", 300]
|
| 55 |
+
},
|
| 56 |
+
"target": "daily maximum non-tidal residual",
|
| 57 |
+
"target_unit": "m",
|
| 58 |
+
"sample_interval_hours": 24,
|
| 59 |
+
"lag_interval_hours": 6,
|
| 60 |
+
"maximum_lag_hours": 30,
|
| 61 |
+
"rs_variables": ["u", "u2", "u3", "v", "v2", "v3", "slp", "sst", "precipitation"],
|
| 62 |
+
"ar_variables": ["u", "u2", "u3", "v", "v2", "v3", "slp"]
|
| 63 |
+
},
|
| 64 |
+
"configuration_sources": [
|
| 65 |
+
"conf/config.yaml",
|
| 66 |
+
"model/globalsurgeml.py",
|
| 67 |
+
"scripts/fake_data.py",
|
| 68 |
+
"scripts/train.py",
|
| 69 |
+
"scripts/inference.py",
|
| 70 |
+
"scripts/result.py"
|
| 71 |
+
]
|
| 72 |
+
}
|
configuration.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"framework": "PyTorch",
|
| 3 |
+
"task": "daily-maximum-storm-surge-regression",
|
| 4 |
+
"model": "GlobalSurgeML",
|
| 5 |
+
"input_format": "NF",
|
| 6 |
+
"protocol": "station-day PCA predictor regression with six paper model configurations",
|
| 7 |
+
"default_config": "conf/config.yaml",
|
| 8 |
+
"training": "scripts/train.py",
|
| 9 |
+
"inference": "scripts/inference.py",
|
| 10 |
+
"evaluation": "scripts/result.py",
|
| 11 |
+
"visualization": "scripts/result.py"
|
| 12 |
+
}
|
model/globalsurgeml.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Station-wise linear and random-forest storm-surge regression models."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from scipy import stats
|
| 10 |
+
from sklearn.ensemble import RandomForestRegressor
|
| 11 |
+
from torch import nn
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class StepwiseLinearSurge(nn.Module):
|
| 15 |
+
"""Linear surge equation after p-value based forward feature selection."""
|
| 16 |
+
|
| 17 |
+
def __init__(self, input_dim: int, maximum_selected_features: int = 20):
|
| 18 |
+
super().__init__()
|
| 19 |
+
self.input_dim = int(input_dim)
|
| 20 |
+
self.maximum_selected_features = int(maximum_selected_features)
|
| 21 |
+
self.register_buffer("selected", torch.full((maximum_selected_features,), -1, dtype=torch.long))
|
| 22 |
+
self.linear = nn.Linear(maximum_selected_features, 1)
|
| 23 |
+
|
| 24 |
+
def select_features(self, features: np.ndarray, targets: np.ndarray, threshold: float) -> None:
|
| 25 |
+
if features.ndim != 2 or features.shape[1] != self.input_dim:
|
| 26 |
+
raise ValueError(f"expected features [N,{self.input_dim}], got {features.shape}")
|
| 27 |
+
remaining = list(range(self.input_dim))
|
| 28 |
+
chosen: list[int] = []
|
| 29 |
+
while remaining and len(chosen) < self.maximum_selected_features:
|
| 30 |
+
best_feature, best_p = None, float("inf")
|
| 31 |
+
for feature in remaining:
|
| 32 |
+
columns = chosen + [feature]
|
| 33 |
+
design = np.column_stack([np.ones(len(features)), features[:, columns]])
|
| 34 |
+
beta = np.linalg.lstsq(design, targets, rcond=None)[0]
|
| 35 |
+
residual = targets - design @ beta
|
| 36 |
+
dof = max(len(targets) - design.shape[1], 1)
|
| 37 |
+
covariance = np.linalg.pinv(design.T @ design) * (residual @ residual / dof)
|
| 38 |
+
standard_error = math.sqrt(max(float(covariance[-1, -1]), 1e-12))
|
| 39 |
+
statistic = abs(float(beta[-1])) / standard_error
|
| 40 |
+
p_value = float(2.0 * stats.t.sf(statistic, dof))
|
| 41 |
+
if p_value < best_p:
|
| 42 |
+
best_feature, best_p = feature, p_value
|
| 43 |
+
if best_feature is None or best_p >= threshold:
|
| 44 |
+
break
|
| 45 |
+
chosen.append(best_feature)
|
| 46 |
+
remaining.remove(best_feature)
|
| 47 |
+
if not chosen:
|
| 48 |
+
correlations = np.abs(features.T @ (targets - targets.mean()))
|
| 49 |
+
chosen = [int(np.argmax(correlations))]
|
| 50 |
+
while len(chosen) < self.maximum_selected_features:
|
| 51 |
+
chosen.append(chosen[-1])
|
| 52 |
+
self.selected = torch.tensor(chosen, dtype=torch.long, device=self.selected.device)
|
| 53 |
+
|
| 54 |
+
def forward(self, features: torch.Tensor) -> torch.Tensor:
|
| 55 |
+
if features.ndim != 2 or features.shape[1] != self.input_dim:
|
| 56 |
+
raise ValueError(f"expected features [N,{self.input_dim}], got {tuple(features.shape)}")
|
| 57 |
+
if bool((self.selected < 0).any()):
|
| 58 |
+
raise RuntimeError("select_features must be called before forward")
|
| 59 |
+
return self.linear(features.index_select(1, self.selected))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class PackedRandomForestSurge(nn.Module):
|
| 63 |
+
"""Random-forest regressor packed into tensors for portable checkpoints."""
|
| 64 |
+
|
| 65 |
+
def __init__(self, input_dim: int, trees: int = 50, max_depth: int = 4,
|
| 66 |
+
max_features: str = "sqrt", bootstrap: bool = True, seed: int = 42):
|
| 67 |
+
super().__init__()
|
| 68 |
+
self.input_dim = int(input_dim)
|
| 69 |
+
self.trees = int(trees)
|
| 70 |
+
self.max_depth = int(max_depth)
|
| 71 |
+
self.max_features = max_features
|
| 72 |
+
self.bootstrap = bool(bootstrap)
|
| 73 |
+
self.seed = int(seed)
|
| 74 |
+
nodes = 2 ** (self.max_depth + 1) - 1
|
| 75 |
+
self.register_buffer("feature", torch.full((trees, nodes), -1, dtype=torch.long))
|
| 76 |
+
self.register_buffer("threshold", torch.zeros(trees, nodes))
|
| 77 |
+
self.register_buffer("left", torch.full((trees, nodes), -1, dtype=torch.long))
|
| 78 |
+
self.register_buffer("right", torch.full((trees, nodes), -1, dtype=torch.long))
|
| 79 |
+
self.register_buffer("value", torch.zeros(trees, nodes))
|
| 80 |
+
self.register_buffer("fitted", torch.tensor(False))
|
| 81 |
+
|
| 82 |
+
def fit(self, features: np.ndarray, targets: np.ndarray) -> None:
|
| 83 |
+
if features.ndim != 2 or features.shape[1] != self.input_dim:
|
| 84 |
+
raise ValueError(f"expected features [N,{self.input_dim}], got {features.shape}")
|
| 85 |
+
forest = RandomForestRegressor(
|
| 86 |
+
n_estimators=self.trees, max_depth=self.max_depth,
|
| 87 |
+
max_features=self.max_features, bootstrap=self.bootstrap,
|
| 88 |
+
random_state=self.seed, n_jobs=1,
|
| 89 |
+
).fit(features, targets)
|
| 90 |
+
for tree_index, estimator in enumerate(forest.estimators_):
|
| 91 |
+
tree = estimator.tree_
|
| 92 |
+
count = tree.node_count
|
| 93 |
+
self.feature[tree_index, :count] = torch.from_numpy(tree.feature.astype(np.int64))
|
| 94 |
+
self.threshold[tree_index, :count] = torch.from_numpy(tree.threshold.astype(np.float32))
|
| 95 |
+
self.left[tree_index, :count] = torch.from_numpy(tree.children_left.astype(np.int64))
|
| 96 |
+
self.right[tree_index, :count] = torch.from_numpy(tree.children_right.astype(np.int64))
|
| 97 |
+
self.value[tree_index, :count] = torch.from_numpy(tree.value[:, 0, 0].astype(np.float32))
|
| 98 |
+
self.fitted.fill_(True)
|
| 99 |
+
|
| 100 |
+
def forward(self, features: torch.Tensor) -> torch.Tensor:
|
| 101 |
+
if features.ndim != 2 or features.shape[1] != self.input_dim:
|
| 102 |
+
raise ValueError(f"expected features [N,{self.input_dim}], got {tuple(features.shape)}")
|
| 103 |
+
if not bool(self.fitted):
|
| 104 |
+
raise RuntimeError("fit must be called before forward")
|
| 105 |
+
outputs = []
|
| 106 |
+
sample_index = torch.arange(features.shape[0], device=features.device)
|
| 107 |
+
for tree_index in range(self.trees):
|
| 108 |
+
nodes = torch.zeros(features.shape[0], dtype=torch.long, device=features.device)
|
| 109 |
+
for _ in range(self.max_depth + 1):
|
| 110 |
+
node_features = self.feature[tree_index, nodes]
|
| 111 |
+
active = node_features >= 0
|
| 112 |
+
if not bool(active.any()):
|
| 113 |
+
break
|
| 114 |
+
safe_features = node_features.clamp_min(0)
|
| 115 |
+
values = features[sample_index, safe_features]
|
| 116 |
+
go_left = values <= self.threshold[tree_index, nodes]
|
| 117 |
+
children = torch.where(go_left, self.left[tree_index, nodes], self.right[tree_index, nodes])
|
| 118 |
+
nodes = torch.where(active, children, nodes)
|
| 119 |
+
outputs.append(self.value[tree_index, nodes])
|
| 120 |
+
return torch.stack(outputs, dim=0).mean(dim=0, keepdim=False).unsqueeze(-1)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class GlobalSurgeML(nn.Module):
|
| 124 |
+
"""Stable interface for one paper model configuration."""
|
| 125 |
+
|
| 126 |
+
def __init__(self, input_dim: int, method: str, config: dict, seed: int = 42):
|
| 127 |
+
super().__init__()
|
| 128 |
+
self.input_dim = int(input_dim)
|
| 129 |
+
self.method = method
|
| 130 |
+
if method == "linear":
|
| 131 |
+
self.regressor = StepwiseLinearSurge(input_dim, int(config["maximum_selected_features"]))
|
| 132 |
+
elif method == "random_forest":
|
| 133 |
+
self.regressor = PackedRandomForestSurge(
|
| 134 |
+
input_dim, int(config["random_forest_trees"]), int(config["random_forest_max_depth"]),
|
| 135 |
+
str(config["random_forest_max_features"]), bool(config["bootstrap"]), seed,
|
| 136 |
+
)
|
| 137 |
+
else:
|
| 138 |
+
raise ValueError("method must be 'linear' or 'random_forest'")
|
| 139 |
+
|
| 140 |
+
def forward(self, features: torch.Tensor) -> torch.Tensor:
|
| 141 |
+
return self.regressor(features)
|
scripts/fake_data.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate structured station-day PCA predictors with realistic surge drivers."""
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import yaml
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def correlated_features(rng, latent, count, phase):
|
| 14 |
+
weights = rng.normal(0, 0.35, (latent.shape[1], count))
|
| 15 |
+
weights[0, :12] += np.linspace(1.5, 0.4, 12)
|
| 16 |
+
weights[1, 12:24] += np.linspace(1.2, 0.3, 12)
|
| 17 |
+
features = latent @ weights + rng.normal(0, 0.12, (len(latent), count))
|
| 18 |
+
features += 0.08 * np.sin(np.arange(count)[None, :] * 0.13 + phase)
|
| 19 |
+
return features.astype(np.float32)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def make_split(path, count, config, seed, day_offset):
|
| 23 |
+
rng = np.random.default_rng(seed)
|
| 24 |
+
day = np.arange(count, dtype=np.float32) + day_offset
|
| 25 |
+
latitude = rng.uniform(-62, 62, count).astype(np.float32)
|
| 26 |
+
longitude = rng.uniform(-180, 180, count).astype(np.float32)
|
| 27 |
+
annual = np.sin(2 * np.pi * day / 365.25)
|
| 28 |
+
synoptic = np.zeros(count, np.float32)
|
| 29 |
+
for index in range(1, count):
|
| 30 |
+
synoptic[index] = 0.72 * synoptic[index - 1] + rng.normal(0, 0.7)
|
| 31 |
+
cyclone = np.maximum(synoptic - 0.55, 0) ** 2
|
| 32 |
+
tropical = (np.abs(latitude) < 30).astype(np.float32)
|
| 33 |
+
latent = np.column_stack([synoptic, cyclone, annual, np.cos(np.deg2rad(latitude)), tropical]).astype(np.float32)
|
| 34 |
+
data = config["data"]
|
| 35 |
+
rs_daily = correlated_features(rng, latent, int(data["rs_daily_features"]), 0.1)
|
| 36 |
+
rs_lagged = correlated_features(rng, latent, int(data["rs_lagged_features"]), 0.4)
|
| 37 |
+
ar_daily = correlated_features(rng, latent, int(data["ar_daily_features"]), 0.7)
|
| 38 |
+
ar_lagged = correlated_features(rng, latent, int(data["ar_lagged_features"]), 1.0)
|
| 39 |
+
surge = (0.035 * synoptic + 0.055 * cyclone + 0.012 * annual +
|
| 40 |
+
0.015 * rs_lagged[:, 0] - 0.010 * rs_lagged[:, 13])
|
| 41 |
+
surge += rng.normal(0, 0.012 + 0.010 * tropical, count)
|
| 42 |
+
gtsr = 0.72 * surge + rng.normal(0, 0.025, count)
|
| 43 |
+
timestamps = (np.datetime64("2000-01-01") + day.astype("timedelta64[D]")).astype("datetime64[s]").astype(np.int64)
|
| 44 |
+
np.savez_compressed(
|
| 45 |
+
path, format_version=np.asarray(data["format_version"]), data_source=np.asarray("structured_synthetic"),
|
| 46 |
+
rs_daily=rs_daily, rs_lagged=rs_lagged, ar_daily=ar_daily, ar_lagged=ar_lagged,
|
| 47 |
+
targets_m=surge.astype(np.float32)[:, None], gtsr_m=gtsr.astype(np.float32)[:, None],
|
| 48 |
+
latitude_degrees=latitude, longitude_degrees=longitude, timestamps_unix_s=timestamps,
|
| 49 |
+
feature_normalization=np.asarray("PCA scores centered and scaled per training feature"),
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def main():
|
| 54 |
+
parser = argparse.ArgumentParser()
|
| 55 |
+
parser.add_argument("--force", action="store_true")
|
| 56 |
+
args = parser.parse_args()
|
| 57 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 58 |
+
output = ROOT / config["data"]["root"]
|
| 59 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 60 |
+
for offset, (name, count, day_offset) in enumerate((("train.npz", config["data"]["train_samples"], 0),
|
| 61 |
+
("test.npz", config["data"]["test_samples"], 1000))):
|
| 62 |
+
path = output / name
|
| 63 |
+
if args.force or not path.exists():
|
| 64 |
+
make_split(path, int(count), config, int(config["seed"]) + offset, day_offset)
|
| 65 |
+
print(f"generated={path.relative_to(ROOT)} samples={count} daily=50 lagged=300 target=1")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
if __name__ == "__main__":
|
| 69 |
+
main()
|
scripts/inference.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run all six storm-surge regressors on station-day test records."""
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import yaml
|
| 9 |
+
from torch.utils.data import DataLoader
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 13 |
+
sys.path.insert(0, str(ROOT))
|
| 14 |
+
from model.globalsurgeml import GlobalSurgeML
|
| 15 |
+
from train import CONFIGURATIONS, SurgeDataset, device_from_config
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def main():
|
| 19 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 20 |
+
device = device_from_config(config)
|
| 21 |
+
checkpoint = torch.load(ROOT / config["paths"]["checkpoint"], map_location=device, weights_only=False)
|
| 22 |
+
if checkpoint["format_version"] != config["data"]["format_version"]:
|
| 23 |
+
raise ValueError("checkpoint and data format versions differ")
|
| 24 |
+
dataset = SurgeDataset(ROOT / config["data"]["root"] / "test.npz", config)
|
| 25 |
+
loader = DataLoader(dataset, batch_size=int(config["train"]["batch_size"]), shuffle=False)
|
| 26 |
+
predictions = {name: [] for name in CONFIGURATIONS}
|
| 27 |
+
models = {}
|
| 28 |
+
for index, (name, (feature_key, method)) in enumerate(CONFIGURATIONS.items()):
|
| 29 |
+
model = GlobalSurgeML(dataset.data[feature_key].shape[1], method, checkpoint["model_config"], int(config["seed"]) + index)
|
| 30 |
+
model.load_state_dict(checkpoint["model"][name])
|
| 31 |
+
models[name] = model.to(device).eval()
|
| 32 |
+
with torch.no_grad():
|
| 33 |
+
for batch in loader:
|
| 34 |
+
for name, (feature_key, _) in CONFIGURATIONS.items():
|
| 35 |
+
mean = torch.from_numpy(checkpoint["feature_means"][name]).to(device)
|
| 36 |
+
scale = torch.from_numpy(checkpoint["feature_scales"][name]).to(device)
|
| 37 |
+
predictions[name].append(models[name]((batch[feature_key].to(device) - mean) / scale).cpu().numpy())
|
| 38 |
+
payload = {f"predictions_{name}": np.concatenate(values) for name, values in predictions.items()}
|
| 39 |
+
for value in payload.values():
|
| 40 |
+
if not np.isfinite(value).all():
|
| 41 |
+
raise FloatingPointError("inference produced NaN or Inf")
|
| 42 |
+
source = dataset.data
|
| 43 |
+
output = ROOT / config["paths"]["inference_dir"] / "predictions.npz"
|
| 44 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
np.savez_compressed(output, **payload, targets_m=source["targets_m"], gtsr_m=source["gtsr_m"],
|
| 46 |
+
latitude_degrees=source["latitude_degrees"], longitude_degrees=source["longitude_degrees"],
|
| 47 |
+
timestamps_unix_s=source["timestamps_unix_s"], target_unit=np.asarray("m"),
|
| 48 |
+
format_version=np.asarray(config["data"]["format_version"]))
|
| 49 |
+
print(f"predictions={output.relative_to(ROOT)} shape={source['targets_m'].shape}")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
if __name__ == "__main__":
|
| 53 |
+
main()
|
scripts/result.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate Pearson correlation, RMSE, NSE and relative RMSE."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import matplotlib
|
| 7 |
+
matplotlib.use("Agg")
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import numpy as np
|
| 10 |
+
import yaml
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def metrics(prediction, target):
|
| 17 |
+
prediction, target = prediction.reshape(-1), target.reshape(-1)
|
| 18 |
+
error = prediction - target
|
| 19 |
+
target_centered = target - target.mean()
|
| 20 |
+
prediction_centered = prediction - prediction.mean()
|
| 21 |
+
denominator = np.sqrt(np.sum(target_centered ** 2) * np.sum(prediction_centered ** 2))
|
| 22 |
+
correlation = float(np.sum(target_centered * prediction_centered) / denominator) if denominator > 0 else 0.0
|
| 23 |
+
rmse = float(np.sqrt(np.mean(error ** 2)))
|
| 24 |
+
variance = float(np.sum(target_centered ** 2))
|
| 25 |
+
nse = float(1.0 - np.sum(error ** 2) / variance) if variance > 0 else 0.0
|
| 26 |
+
surge_range = float(np.ptp(target))
|
| 27 |
+
return {"pearson_correlation": correlation, "rmse_m": rmse, "nse": nse,
|
| 28 |
+
"relative_rmse_percent": float(100 * rmse / surge_range) if surge_range > 0 else 0.0}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def main():
|
| 32 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 33 |
+
data = np.load(ROOT / config["paths"]["inference_dir"] / "predictions.npz")
|
| 34 |
+
if str(data["format_version"]) != config["data"]["format_version"]:
|
| 35 |
+
raise ValueError("incompatible prediction format")
|
| 36 |
+
target = data["targets_m"]
|
| 37 |
+
names = [key.removeprefix("predictions_") for key in data.files if key.startswith("predictions_")]
|
| 38 |
+
by_model = {name: metrics(data[f"predictions_{name}"], target) for name in names}
|
| 39 |
+
by_model["GTSR_baseline"] = metrics(data["gtsr_m"], target)
|
| 40 |
+
threshold = float(np.percentile(target, float(config["evaluation"]["extreme_percentile"])))
|
| 41 |
+
extreme_mask = target[:, 0] > threshold
|
| 42 |
+
extreme = {name: metrics(data[f"predictions_{name}"][extreme_mask], target[extreme_mask]) for name in names}
|
| 43 |
+
best_name = max(names, key=lambda name: (by_model[name]["pearson_correlation"] + by_model[name]["nse"] -
|
| 44 |
+
by_model[name]["relative_rmse_percent"] / 100))
|
| 45 |
+
tropical = np.abs(data["latitude_degrees"]) < float(config["evaluation"]["tropical_latitude_limit_degrees"])
|
| 46 |
+
regions = {}
|
| 47 |
+
for region, mask in (("tropical", tropical), ("subtropical_extratropical", ~tropical)):
|
| 48 |
+
regions[region] = metrics(data[f"predictions_{best_name}"][mask], target[mask]) if mask.sum() >= 2 else None
|
| 49 |
+
report = {"samples": int(len(target)), "overall": by_model, "selected_model": best_name,
|
| 50 |
+
"extreme_above_observed_95th_percentile": extreme, "regional_selected_model": regions,
|
| 51 |
+
"evaluation_protocol": {"target": "daily maximum non-tidal residual", "unit": "m",
|
| 52 |
+
"extreme_threshold_m": threshold, "aggregation": "all station-day samples",
|
| 53 |
+
"missing_value_mask": "none in synthetic data"}}
|
| 54 |
+
numeric = [value for model_metrics in by_model.values() for value in model_metrics.values()]
|
| 55 |
+
if not np.isfinite(numeric).all():
|
| 56 |
+
raise FloatingPointError("evaluation contains non-finite metrics")
|
| 57 |
+
output = ROOT / config["paths"]["evaluation_dir"]
|
| 58 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 59 |
+
(output / "metrics.json").write_text(json.dumps(report, indent=2) + "\n")
|
| 60 |
+
prediction = data[f"predictions_{best_name}"][:, 0]
|
| 61 |
+
order = np.argsort(data["timestamps_unix_s"])
|
| 62 |
+
figure, axes = plt.subplots(1, 2, figsize=(11, 4.2))
|
| 63 |
+
axes[0].plot(target[order, 0] * 100, label="Observed", linewidth=1.5)
|
| 64 |
+
axes[0].plot(prediction[order] * 100, label=best_name, linewidth=1.2)
|
| 65 |
+
axes[0].set(xlabel="Station-day index", ylabel="Daily maximum surge (cm)", title="Synthetic surge sequence")
|
| 66 |
+
axes[0].legend()
|
| 67 |
+
axes[1].scatter(target[:, 0] * 100, prediction * 100, c=np.abs(data["latitude_degrees"]), cmap="viridis", s=28)
|
| 68 |
+
limits = [min(target.min(), prediction.min()) * 100, max(target.max(), prediction.max()) * 100]
|
| 69 |
+
axes[1].plot(limits, limits, "k--", linewidth=1)
|
| 70 |
+
axes[1].set(xlabel="Observed surge (cm)", ylabel="Modeled surge (cm)", title="Observed vs modeled")
|
| 71 |
+
figure.tight_layout()
|
| 72 |
+
figure.savefig(output / "comparison.png", dpi=150)
|
| 73 |
+
plt.close(figure)
|
| 74 |
+
print(f"evaluation={output.relative_to(ROOT)} selected={best_name}")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
if __name__ == "__main__":
|
| 78 |
+
main()
|
scripts/train.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fit the paper's six station-wise model configurations."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import sys
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
import yaml
|
| 11 |
+
from torch.nn.parallel import DistributedDataParallel
|
| 12 |
+
from torch.utils.data import DataLoader, Dataset, DistributedSampler
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 16 |
+
sys.path.insert(0, str(ROOT))
|
| 17 |
+
from model.globalsurgeml import GlobalSurgeML
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
CONFIGURATIONS = {
|
| 21 |
+
"LR-RS": ("rs_daily", "linear"),
|
| 22 |
+
"LR-RS-lag": ("rs_lagged", "linear"),
|
| 23 |
+
"RF-RS-lag": ("rs_lagged", "random_forest"),
|
| 24 |
+
"LR-AR": ("ar_daily", "linear"),
|
| 25 |
+
"LR-AR-lag": ("ar_lagged", "linear"),
|
| 26 |
+
"RF-AR-lag": ("ar_lagged", "random_forest"),
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class SurgeDataset(Dataset):
|
| 31 |
+
def __init__(self, path, config):
|
| 32 |
+
self.data = np.load(path)
|
| 33 |
+
expected = config["data"]
|
| 34 |
+
if str(self.data["format_version"]) != expected["format_version"]:
|
| 35 |
+
raise ValueError("incompatible storm-surge data format")
|
| 36 |
+
dimensions = {"rs_daily": expected["rs_daily_features"], "rs_lagged": expected["rs_lagged_features"],
|
| 37 |
+
"ar_daily": expected["ar_daily_features"], "ar_lagged": expected["ar_lagged_features"]}
|
| 38 |
+
for key, width in dimensions.items():
|
| 39 |
+
if self.data[key].shape[1:] != (int(width),):
|
| 40 |
+
raise ValueError(f"{key} must have shape [N,{width}]")
|
| 41 |
+
if self.data["targets_m"].shape[1:] != (1,):
|
| 42 |
+
raise ValueError("targets_m must have shape [N,1]")
|
| 43 |
+
|
| 44 |
+
def __len__(self):
|
| 45 |
+
return len(self.data["targets_m"])
|
| 46 |
+
|
| 47 |
+
def __getitem__(self, index):
|
| 48 |
+
return {key: torch.from_numpy(self.data[key][index]).float() for key in
|
| 49 |
+
("rs_daily", "rs_lagged", "ar_daily", "ar_lagged", "targets_m")}
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def device_from_config(config, rank=0):
|
| 53 |
+
requested = config["runtime"]["device"]
|
| 54 |
+
if requested == "auto":
|
| 55 |
+
return torch.device("cuda", rank) if torch.cuda.is_available() else torch.device("cpu")
|
| 56 |
+
return torch.device(requested)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def main():
|
| 60 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
|
| 61 |
+
seed = int(config["seed"])
|
| 62 |
+
np.random.seed(seed)
|
| 63 |
+
torch.manual_seed(seed)
|
| 64 |
+
distributed = int(os.environ.get("WORLD_SIZE", "1")) > 1
|
| 65 |
+
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
| 66 |
+
if distributed:
|
| 67 |
+
torch.distributed.init_process_group("nccl" if torch.cuda.is_available() else "gloo")
|
| 68 |
+
rank = torch.distributed.get_rank() if distributed else 0
|
| 69 |
+
device = device_from_config(config, local_rank)
|
| 70 |
+
if device.type == "cuda":
|
| 71 |
+
torch.cuda.set_device(device)
|
| 72 |
+
dataset = SurgeDataset(ROOT / config["data"]["root"] / "train.npz", config)
|
| 73 |
+
sampler = DistributedSampler(dataset, shuffle=True) if distributed else None
|
| 74 |
+
loader = DataLoader(dataset, batch_size=int(config["train"]["batch_size"]), sampler=sampler,
|
| 75 |
+
shuffle=sampler is None, num_workers=int(config["train"]["num_workers"]))
|
| 76 |
+
full = dataset.data
|
| 77 |
+
means, scales, states, history = {}, {}, {}, []
|
| 78 |
+
models = {}
|
| 79 |
+
for model_index, (name, (feature_key, method)) in enumerate(CONFIGURATIONS.items()):
|
| 80 |
+
array = full[feature_key].astype(np.float32)
|
| 81 |
+
means[name] = array.mean(0).astype(np.float32)
|
| 82 |
+
scales[name] = array.std(0).clip(1e-6).astype(np.float32)
|
| 83 |
+
standardized = (array - means[name]) / scales[name]
|
| 84 |
+
model = GlobalSurgeML(array.shape[1], method, config["model"], seed + model_index).to(device)
|
| 85 |
+
if method == "linear":
|
| 86 |
+
model.regressor.select_features(standardized, full["targets_m"][:, 0],
|
| 87 |
+
float(config["model"]["p_value_threshold"]))
|
| 88 |
+
wrapped = DistributedDataParallel(model, device_ids=[local_rank] if device.type == "cuda" else None) if distributed else model
|
| 89 |
+
optimizer = torch.optim.Adam(wrapped.parameters(), lr=float(config["train"]["learning_rate"]),
|
| 90 |
+
weight_decay=float(config["train"]["weight_decay"]))
|
| 91 |
+
for epoch in range(int(config["train"]["epochs"])):
|
| 92 |
+
if sampler:
|
| 93 |
+
sampler.set_epoch(epoch)
|
| 94 |
+
total, steps = 0.0, 0
|
| 95 |
+
for batch in loader:
|
| 96 |
+
features = (batch[feature_key].to(device) - torch.from_numpy(means[name]).to(device)) / torch.from_numpy(scales[name]).to(device)
|
| 97 |
+
target = batch["targets_m"].to(device)
|
| 98 |
+
prediction = wrapped(features)
|
| 99 |
+
loss = torch.nn.functional.mse_loss(prediction, target)
|
| 100 |
+
optimizer.zero_grad(set_to_none=True)
|
| 101 |
+
loss.backward()
|
| 102 |
+
torch.nn.utils.clip_grad_norm_(wrapped.parameters(), float(config["train"]["gradient_clip_norm"]))
|
| 103 |
+
optimizer.step()
|
| 104 |
+
total += float(loss.detach())
|
| 105 |
+
steps += 1
|
| 106 |
+
history.append({"model": name, "epoch": epoch + 1, "mse_m2": total / max(steps, 1)})
|
| 107 |
+
model = wrapped.module if distributed else wrapped
|
| 108 |
+
else:
|
| 109 |
+
model.regressor.fit(standardized, full["targets_m"][:, 0])
|
| 110 |
+
prediction = model(torch.from_numpy(standardized).to(device))
|
| 111 |
+
history.append({"model": name, "epoch": 1,
|
| 112 |
+
"mse_m2": float(torch.nn.functional.mse_loss(prediction.cpu(), torch.from_numpy(full["targets_m"])).item())})
|
| 113 |
+
models[name] = model
|
| 114 |
+
states[name] = model.state_dict()
|
| 115 |
+
if rank == 0:
|
| 116 |
+
checkpoint = ROOT / config["paths"]["checkpoint"]
|
| 117 |
+
metrics = ROOT / config["paths"]["training_metrics"]
|
| 118 |
+
checkpoint.parent.mkdir(parents=True, exist_ok=True)
|
| 119 |
+
metrics.parent.mkdir(parents=True, exist_ok=True)
|
| 120 |
+
torch.save({"model": states, "model_config": config["model"], "configurations": CONFIGURATIONS,
|
| 121 |
+
"feature_means": means, "feature_scales": scales,
|
| 122 |
+
"format_version": config["data"]["format_version"], "target_unit": "m"}, checkpoint)
|
| 123 |
+
metrics.write_text(json.dumps({"history": history}, indent=2) + "\n")
|
| 124 |
+
print(f"checkpoint={checkpoint.relative_to(ROOT)} models={len(states)}")
|
| 125 |
+
if distributed:
|
| 126 |
+
torch.distributed.destroy_process_group()
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
if __name__ == "__main__":
|
| 130 |
+
main()
|
weight/.gitkeep
ADDED
|
File without changes
|