Update StormScope Fast Space (package-probe)
Browse files- LICENSE +21 -0
- MODEL_CARD.md +83 -0
- NOTICE.md +23 -0
- README.md +240 -33
- app.py +48 -0
- pyproject.toml +43 -0
- requirements-live.txt +17 -0
- requirements.txt +2 -0
- rustwx_stormscope/__init__.py +9 -0
- rustwx_stormscope/core.py +549 -0
- rustwx_stormscope/datasources.py +30 -0
- rustwx_stormscope/fp8_linear.py +47 -0
- rustwx_stormscope/ops_render.py +674 -0
- rustwx_stormscope/optimize.py +57 -0
- rustwx_stormscope/output_validation.py +283 -0
- rustwx_stormscope/paths.py +50 -0
- rustwx_stormscope/run.py +98 -0
- rustwx_stormscope/rust_datasources.py +149 -0
- rustwx_stormscope/service.py +224 -0
- rustwx_stormscope/webmap_preview.py +407 -0
- rustwx_stormscope/wxstore_export.py +153 -0
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Fahrenheit Research
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
MODEL_CARD.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Model Card: rustwx-stormscope
|
| 2 |
+
|
| 3 |
+
`rustwx-stormscope` is an inference and operations package for NVIDIA
|
| 4 |
+
StormScope GOES/MRMS checkpoints through Earth2Studio. This repository ships
|
| 5 |
+
runtime glue, optimization code, data download adapters, rendering/export
|
| 6 |
+
tools, and operational service scripts. It does not train, modify, or
|
| 7 |
+
redistribute StormScope model weights.
|
| 8 |
+
|
| 9 |
+
## Intended Use
|
| 10 |
+
|
| 11 |
+
- Fast local StormScope nowcast inference on NVIDIA consumer Blackwell GPUs.
|
| 12 |
+
- Operational-style repeated runs that write inspectable `.npz`, PNG, GIF,
|
| 13 |
+
web-map overlay, and optional WXA products.
|
| 14 |
+
- Research and engineering evaluation of deterministic sampler reduction,
|
| 15 |
+
`torch.compile`, MXFP8, and public-data fetch paths.
|
| 16 |
+
|
| 17 |
+
## Out of Scope
|
| 18 |
+
|
| 19 |
+
- Warning issuance, automated public-alerting, or safety-critical decisions.
|
| 20 |
+
- CPU inference.
|
| 21 |
+
- Native Windows CUDA installs outside WSL2 or Docker.
|
| 22 |
+
- Guaranteed performance on pre-Blackwell or data-center Blackwell GPUs without
|
| 23 |
+
separate validation.
|
| 24 |
+
|
| 25 |
+
## Upstream Model
|
| 26 |
+
|
| 27 |
+
- Upstream model: `nvidia/stormscope-goes-mrms`.
|
| 28 |
+
- Upstream framework: NVIDIA Earth2Studio.
|
| 29 |
+
- Inputs: GOES ABI variables, MRMS composite reflectivity, and optional
|
| 30 |
+
conditioning variables depending on mode.
|
| 31 |
+
- Outputs: GOES/MRMS state variables on the StormScope/HRRR-style grid.
|
| 32 |
+
|
| 33 |
+
Weights are downloaded at runtime into the configured cache volume. Users are
|
| 34 |
+
responsible for reviewing and complying with the upstream model license.
|
| 35 |
+
|
| 36 |
+
## Fast Inference Settings
|
| 37 |
+
|
| 38 |
+
Recommended operational settings:
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
--num-steps 24 --mxfp8 --backend rust
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
Conservative settings:
|
| 45 |
+
|
| 46 |
+
```bash
|
| 47 |
+
--num-steps 32 --mxfp8 --backend rust
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
The package defaults to deterministic EDM sampling instead of the original
|
| 51 |
+
stochastic sampler. This makes outputs reproducible and makes reduced
|
| 52 |
+
step-count comparisons meaningful.
|
| 53 |
+
|
| 54 |
+
## Validation Summary
|
| 55 |
+
|
| 56 |
+
The optimization stack was validated with:
|
| 57 |
+
|
| 58 |
+
- numerical comparisons between baseline and optimized outputs;
|
| 59 |
+
- deterministic ODE convergence sweeps against a 100-step reference;
|
| 60 |
+
- observation checks against actual GOES valid-time data;
|
| 61 |
+
- output schema validation for completed operational cycles.
|
| 62 |
+
|
| 63 |
+
Known RTX 5090 measurements are summarized in `docs/benchmark_summary.md`.
|
| 64 |
+
External users should rerun the validation commands on their own hardware and
|
| 65 |
+
keep the generated `.npz` files as the audit trail.
|
| 66 |
+
|
| 67 |
+
## Limitations
|
| 68 |
+
|
| 69 |
+
- Weather-model quality is inherited from the upstream StormScope checkpoint
|
| 70 |
+
and the availability/latency of public input data.
|
| 71 |
+
- Reduced diffusion steps and MXFP8 are fast and empirically close to baseline,
|
| 72 |
+
but they are still inference-mode changes and should be validated for each
|
| 73 |
+
operational use case.
|
| 74 |
+
- First-run latency includes model-weight downloads, source-data downloads, and
|
| 75 |
+
`torch.compile` warmup.
|
| 76 |
+
- The Rust data backend currently decodes MRMS in Rust and GOES with h5netcdf
|
| 77 |
+
after Rust download.
|
| 78 |
+
|
| 79 |
+
## Output Contract
|
| 80 |
+
|
| 81 |
+
Primary science outputs are `.npz` frames plus cycle metadata. Rendered PNGs,
|
| 82 |
+
GIFs, web-map overlays, and WXA files are derivative products. See
|
| 83 |
+
`docs/output_schema.md` for the exact product contract.
|
NOTICE.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Notice
|
| 2 |
+
|
| 3 |
+
This repository packages an inference and operations workflow around NVIDIA
|
| 4 |
+
Earth2Studio StormScope and RustWX tooling.
|
| 5 |
+
|
| 6 |
+
Repository source code is released under the MIT License.
|
| 7 |
+
|
| 8 |
+
It does not redistribute StormScope model weights. Weights are downloaded at
|
| 9 |
+
runtime into the user-configured cache path according to the upstream access
|
| 10 |
+
mechanism and terms.
|
| 11 |
+
|
| 12 |
+
Upstream terms to preserve in release notes and user documentation:
|
| 13 |
+
|
| 14 |
+
- Earth2Studio is provided by NVIDIA under Apache License 2.0.
|
| 15 |
+
- StormScope GOES/MRMS weights are governed by NVIDIA Open Model License.
|
| 16 |
+
- RustWX is used as an integration dependency under its own MIT license.
|
| 17 |
+
|
| 18 |
+
Licensed by NVIDIA Corporation under the NVIDIA Open Model License.
|
| 19 |
+
|
| 20 |
+
This notice does not grant additional rights to upstream model weights,
|
| 21 |
+
datasets, NVIDIA trademarks, or third-party dependencies. Users are responsible
|
| 22 |
+
for reviewing and complying with the upstream model, data, and dependency
|
| 23 |
+
terms.
|
README.md
CHANGED
|
@@ -1,33 +1,240 @@
|
|
| 1 |
-
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# rustwx-stormscope
|
| 2 |
+
|
| 3 |
+
Fast NVIDIA Earth2Studio StormScope inference on consumer Blackwell GPUs
|
| 4 |
+
(RTX 5070 Ti / 5080 / 5090, `sm_120`), packaged for local WSL/Linux installs,
|
| 5 |
+
operational rendering, and web-map preview products. Docker remains available
|
| 6 |
+
as an optional path, but it is not required.
|
| 7 |
+
|
| 8 |
+
This combines:
|
| 9 |
+
|
| 10 |
+
1. Inference optimizations: deterministic EDM sampler, fewer sampler steps, `torch.compile`, and optional MXFP8 on DiT linears.
|
| 11 |
+
2. Operational service: continuous nowcast and GFS-conditioned nearcast cycles with status and retention.
|
| 12 |
+
3. Fast data path: parallel Rust download plus pure-Rust MRMS decode; GOES files are downloaded by Rust and decoded with h5netcdf.
|
| 13 |
+
4. Product layer: raw `.npz`, rustwx-render PNG/GIF dashboard, Leaflet web-map overlays, and optional native RustWX WXA export.
|
| 14 |
+
|
| 15 |
+
The default release path is the local WSL/Linux installer. It creates the conda
|
| 16 |
+
environment and builds the CUDA/Rust pieces needed for fast inference.
|
| 17 |
+
|
| 18 |
+
## Speedup
|
| 19 |
+
|
| 20 |
+
RTX 5090, nowcast 6 km, one forecast step = GOES + MRMS:
|
| 21 |
+
|
| 22 |
+
| config | per-step | speedup |
|
| 23 |
+
| --- | ---: | ---: |
|
| 24 |
+
| original `run_stormscope.py` | ~50.8 s | 1.0x |
|
| 25 |
+
| `--num-steps 32` deterministic + compile | ~11.6 s | 4.4x |
|
| 26 |
+
| `--num-steps 24 --mxfp8` | ~6.8 s steady-state | 7.5x |
|
| 27 |
+
|
| 28 |
+
The fast config was validated against real GOES observations: original 2.00% RMSE / 0.961 corr, MXFP8 det-24 about 2.07% / 0.958.
|
| 29 |
+
|
| 30 |
+
## Quick Start
|
| 31 |
+
|
| 32 |
+
```bash
|
| 33 |
+
git clone https://github.com/FahrenheitResearch/rustwx-stormscope.git
|
| 34 |
+
cd rustwx-stormscope
|
| 35 |
+
python scripts/doctor.py
|
| 36 |
+
bash scripts/install-local.sh
|
| 37 |
+
conda activate earth2
|
| 38 |
+
source ./env.cdrive.example.sh
|
| 39 |
+
./scripts/run-nowcast.sh
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
On Windows PowerShell with WSL2 Ubuntu:
|
| 43 |
+
|
| 44 |
+
```powershell
|
| 45 |
+
.\scripts\doctor.ps1
|
| 46 |
+
.\scripts\install-wsl.ps1
|
| 47 |
+
.\scripts\run-nowcast.ps1
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
The first run downloads model weights and source data into `cache/`, then writes
|
| 51 |
+
products under `volume/wxstores/stormscope/`.
|
| 52 |
+
|
| 53 |
+
## Local WSL/Linux
|
| 54 |
+
|
| 55 |
+
Direct one-shot after install:
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
rustwx-stormscope --mode nowcast --steps 6 --num-steps 24 --mxfp8 --outdir "$RUSTWX_STORMSCOPE_OUT"
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Continuous local operation:
|
| 62 |
+
|
| 63 |
+
```bash
|
| 64 |
+
rustwx-stormscope-service --outdir "$RUSTWX_STORMSCOPE_OUT" --num-steps 24 --mxfp8
|
| 65 |
+
rustwx-stormscope-render --outdir "$RUSTWX_STORMSCOPE_OUT" \
|
| 66 |
+
--site-dir "$RUSTWX_STORMSCOPE_SITE" --export-dir "$RUSTWX_STORMSCOPE_EXPORTS" --loop
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
## Optional Docker
|
| 70 |
+
|
| 71 |
+
Local image build:
|
| 72 |
+
|
| 73 |
+
```bash
|
| 74 |
+
docker build -t rustwx-stormscope:local -f docker/Dockerfile .
|
| 75 |
+
|
| 76 |
+
docker run --gpus all \
|
| 77 |
+
-v "$PWD/volume/wxstores:/wxstores" \
|
| 78 |
+
-v "$PWD/cache:/cache" \
|
| 79 |
+
rustwx-stormscope:local
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
One-shot:
|
| 83 |
+
|
| 84 |
+
```bash
|
| 85 |
+
docker run --gpus all \
|
| 86 |
+
-v "$PWD/volume/wxstores:/wxstores" \
|
| 87 |
+
-v "$PWD/cache:/cache" \
|
| 88 |
+
rustwx-stormscope:local \
|
| 89 |
+
rustwx-stormscope --mode nowcast --steps 6 --num-steps 24 --mxfp8 \
|
| 90 |
+
--outdir /wxstores/stormscope/out/operational
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
End products live in the mounted `volume/wxstores` directory. Weights, downloaded input data, and compiler caches live in the mounted `cache` directory.
|
| 94 |
+
|
| 95 |
+
`env.cdrive.example.sh` sets:
|
| 96 |
+
|
| 97 |
+
| variable | default |
|
| 98 |
+
| --- | --- |
|
| 99 |
+
| `RUSTWX_STORMSCOPE_WXSTORE_ROOT` | `$RUSTWX_STORMSCOPE_ROOT/volume/wxstores/stormscope` |
|
| 100 |
+
| `RUSTWX_STORMSCOPE_OUT` | `$RUSTWX_STORMSCOPE_WXSTORE_ROOT/out/operational` |
|
| 101 |
+
| `RUSTWX_STORMSCOPE_SITE` | `$RUSTWX_STORMSCOPE_WXSTORE_ROOT/site` |
|
| 102 |
+
| `RUSTWX_STORMSCOPE_EXPORTS` | `$RUSTWX_STORMSCOPE_WXSTORE_ROOT/exports` |
|
| 103 |
+
| `RUSTWX_STORMSCOPE_SPATIAL_ROOT` | `$RUSTWX_STORMSCOPE_WXSTORE_ROOT/spatial` |
|
| 104 |
+
| `RUSTWX_STORMSCOPE_WEBMAP` | `$RUSTWX_STORMSCOPE_WXSTORE_ROOT/webmap_preview` |
|
| 105 |
+
| `RUSTWX_STORMSCOPE_CACHE_DIR` | `$RUSTWX_STORMSCOPE_ROOT/cache` |
|
| 106 |
+
| `EARTH2STUDIO_CACHE` | `$RUSTWX_STORMSCOPE_CACHE_DIR/earth2studio` |
|
| 107 |
+
| `TORCHINDUCTOR_CACHE_DIR` | `$RUSTWX_STORMSCOPE_CACHE_DIR/torchinductor` |
|
| 108 |
+
| `TRITON_CACHE_DIR` | `$RUSTWX_STORMSCOPE_CACHE_DIR/triton` |
|
| 109 |
+
| `RUSTWX_STORMSCOPE_GOES_CACHE` | `$RUSTWX_STORMSCOPE_CACHE_DIR/goes_nc` |
|
| 110 |
+
| `RUSTWX_STORMSCOPE_MRMS_CACHE` | `$RUSTWX_STORMSCOPE_CACHE_DIR/mrms_grib2` |
|
| 111 |
+
|
| 112 |
+
## Usage
|
| 113 |
+
|
| 114 |
+
```bash
|
| 115 |
+
rustwx-stormscope --mode {nowcast,forecast} --steps N \
|
| 116 |
+
--num-steps 24 --mxfp8 --backend rust --outdir "$RUSTWX_STORMSCOPE_OUT"
|
| 117 |
+
|
| 118 |
+
rustwx-stormscope-service --outdir "$RUSTWX_STORMSCOPE_OUT" \
|
| 119 |
+
--interval 30 --keep-cycles 36 --num-steps 24 --mxfp8 --backend rust \
|
| 120 |
+
--nowcast-steps 12 --nearcast-steps 0
|
| 121 |
+
|
| 122 |
+
rustwx-stormscope-render --outdir "$RUSTWX_STORMSCOPE_OUT" \
|
| 123 |
+
--site-dir "$RUSTWX_STORMSCOPE_SITE" --export-dir "$RUSTWX_STORMSCOPE_EXPORTS" --loop
|
| 124 |
+
|
| 125 |
+
rustwx-stormscope-webmap-preview --outdir "$RUSTWX_STORMSCOPE_OUT" \
|
| 126 |
+
--output-dir "$RUSTWX_STORMSCOPE_WEBMAP" --products refc,ir --view central_plains
|
| 127 |
+
|
| 128 |
+
rustwx-stormscope-validate-output --outdir "$RUSTWX_STORMSCOPE_OUT" --json
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
On Drew's Windows/WSL workstation, `tools/start_operational.ps1` starts all three local processes and serves the site at `http://localhost:8787/`. It writes model frames to `volume/wxstores/stormscope/out/operational`, site assets to `volume/wxstores/stormscope/site`, and export GIFs/PNGs to `volume/wxstores/stormscope/exports`. It polls every minute with a 2-minute observation buffer, writes partial manifests as frames finish, and promotes only completed cycles to the site. `tools/stop_operational.ps1` stops those repo-local background processes.
|
| 132 |
+
|
| 133 |
+
StormScope calls the hourly GFS-conditioned model `forecast`; this repo labels that product `nearcast` in the service logs and website. Default operational products are nowcast `+10..+120 min` and nearcast `+1..+12 h`. GFS cycles initialize at `00/06/12/18Z`, and the forecast leads are hourly; installed Earth2Studio validates GFS_FX lead times through 384 hours, so 81 hourly frames are supported by the data source if you want a long product.
|
| 134 |
+
|
| 135 |
+
Useful flags:
|
| 136 |
+
|
| 137 |
+
- `--num-steps 24`: recommended fast mode.
|
| 138 |
+
- `--num-steps 32`: conservative mode.
|
| 139 |
+
- `--mxfp8`: Blackwell block-scaled FP8 for DiT linears.
|
| 140 |
+
- `--backend rust`: parallel fetch path; falls back to earth2studio if the extension is unavailable.
|
| 141 |
+
- `--backend earth2studio`: original Python data path.
|
| 142 |
+
- `--nowcast-steps`: 10-minute nowcast frame count.
|
| 143 |
+
- `--nearcast-steps`: hourly future horizon for the service; `0` disables nearcast. With GFS-rooted cycles, the service runs enough raw StormScope steps to cover that many hours from the current clock time.
|
| 144 |
+
- `RUSTWX_STORMSCOPE_GFS_BUFFER_HOURS`: GFS cycle publication buffer for operational nearcast selection, default `4`.
|
| 145 |
+
- `RUSTWX_STORMSCOPE_FORECAST_CONDITIONING=hrrr`: explicit experimental override back to HRRR conditioning.
|
| 146 |
+
- `--no-compile`, `--stochastic`, `--fp32`: diagnostics and fallback controls.
|
| 147 |
+
|
| 148 |
+
## Operational Web View
|
| 149 |
+
|
| 150 |
+
The web view is built from completed service manifests and does not touch the GPU. It renders two products per cycle:
|
| 151 |
+
|
| 152 |
+
- IR brightness temperature from `ch_ir_1035`
|
| 153 |
+
- Composite reflectivity from `refc`, masked below 5 dBZ
|
| 154 |
+
|
| 155 |
+
The renderer binary is `rust/stormscope_render/target/release/stormscope_render` and uses `rustwx-render` for projection, basemap linework, weather palettes, and PNG writing. `tools/ops_render.py` watches the output directory, calls the Rust renderer for each frame, assembles GIFs with Pillow, writes `site/index.html`, adds per-product `Export GIF` download links, and mirrors the latest GIFs/PNGs into the optional export folder.
|
| 156 |
+
|
| 157 |
+
## Web Map Preview
|
| 158 |
+
|
| 159 |
+
For browser maps, keep the `.npz` files as the raw StormScope output and generate web-friendly overlays from them. The preview command writes transparent PNG overlays, `manifest.json`, `manifest.js`, and a Leaflet `index.html`:
|
| 160 |
+
|
| 161 |
+
```bash
|
| 162 |
+
rustwx-stormscope-webmap-preview --meta "$RUSTWX_STORMSCOPE_OUT/stormscope_nowcast_6km_YYYYMMDD_HHMMz_meta.json" \
|
| 163 |
+
--outdir "$RUSTWX_STORMSCOPE_OUT" --output-dir "$RUSTWX_STORMSCOPE_WEBMAP" \
|
| 164 |
+
--products refc,ir --view central_plains
|
| 165 |
+
```
|
| 166 |
+
|
| 167 |
+
This is intentionally a fast preview/export shape, not a replacement for the science files. A production web map should keep `.npz` for archive/replay and publish PNG tile pyramids or cloud-optimized rasters from the same frames.
|
| 168 |
+
|
| 169 |
+
## Data Backend
|
| 170 |
+
|
| 171 |
+
The rust backend parallel-downloads GOES and MRMS. MRMS GRIB2 is decoded in Rust. GOES MCMIPC decode remains in Python via h5netcdf because rustwx's pure-Rust HDF5 reader does not yet handle these files' dense-attribute / v2-B-tree layout.
|
| 172 |
+
|
| 173 |
+
Expected fetch timing from the existing logs:
|
| 174 |
+
|
| 175 |
+
- cold rust fetch: about 24 s for GOES + MRMS
|
| 176 |
+
- cached rust fetch: about 5-8 s
|
| 177 |
+
- earth2studio fetch baseline: about 75 s
|
| 178 |
+
|
| 179 |
+
## Files
|
| 180 |
+
|
| 181 |
+
- `rustwx_stormscope/optimize.py`: sampler, compile, MXFP8 setup.
|
| 182 |
+
- `rustwx_stormscope/run.py`: one-shot CLI.
|
| 183 |
+
- `rustwx_stormscope/service.py`: operational loop.
|
| 184 |
+
- `rustwx_stormscope/rust_datasources.py`: GOES/MRMS rust-backed data adapter.
|
| 185 |
+
- `rust/stormscope_obs/`: pyo3 extension for parallel fetch and MRMS decode.
|
| 186 |
+
- `rust/stormscope_render/`: rustwx-render based PNG renderer for operational maps.
|
| 187 |
+
- `rust/stormscope_wxstore/`: optional native WXA exporter for RustWX store/replay.
|
| 188 |
+
- `tools/ops_render.py`: HTML/GIF site builder.
|
| 189 |
+
- `install.sh`: local conda installer for the full CUDA/Earth2Studio/NATTEN/torchao stack.
|
| 190 |
+
- `docker/Dockerfile`: optional full-stack container image.
|
| 191 |
+
- `spaces/stormscope-fast-demo/`: Gradio/Hugging Face Space tech demo.
|
| 192 |
+
Push profiles are `demo`, `package-probe`, and `live`; keep live mode disabled
|
| 193 |
+
until the hosted dependency stack completes a one-frame nowcast.
|
| 194 |
+
|
| 195 |
+
## Documentation
|
| 196 |
+
|
| 197 |
+
- `docs/supported_hardware.md`: GPU/driver/runtime expectations.
|
| 198 |
+
- `docs/local_install.md`: primary WSL/Linux install path for non-Docker users.
|
| 199 |
+
- `docs/huggingface_space.md`: Hugging Face Space tech-demo deployment notes.
|
| 200 |
+
- `docs/release_shape.md`: recommended public packaging shape and remaining proof gates.
|
| 201 |
+
- `docs/output_schema.md`: `.npz`, metadata, web-map, and WXA product contracts.
|
| 202 |
+
- `docs/reproducibility.md`: deterministic settings, validation strategy, benchmark hygiene.
|
| 203 |
+
- `docs/benchmark_summary.md`: current RTX 5090 speed and validation evidence.
|
| 204 |
+
- `docs/model_and_data.md`: upstream model/data notes and license boundary.
|
| 205 |
+
- `docs/troubleshooting.md`: common install/runtime issues.
|
| 206 |
+
- `docs/release_checklist.md`: gates before a GitHub or Docker release.
|
| 207 |
+
- `MODEL_CARD.md`: intended use, limits, upstream model boundary, and validation summary.
|
| 208 |
+
|
| 209 |
+
## Validation
|
| 210 |
+
|
| 211 |
+
Lightweight gates that do not require GPU inference:
|
| 212 |
+
|
| 213 |
+
```bash
|
| 214 |
+
python scripts/validate-release.py
|
| 215 |
+
```
|
| 216 |
+
|
| 217 |
+
With local RustWX vendored and a Rust toolchain:
|
| 218 |
+
|
| 219 |
+
```bash
|
| 220 |
+
python scripts/validate-release.py --rust
|
| 221 |
+
```
|
| 222 |
+
|
| 223 |
+
Docker image smoke test:
|
| 224 |
+
|
| 225 |
+
```bash
|
| 226 |
+
./scripts/docker-smoke.sh
|
| 227 |
+
```
|
| 228 |
+
|
| 229 |
+
Validate a completed StormScope cycle:
|
| 230 |
+
|
| 231 |
+
```bash
|
| 232 |
+
rustwx-stormscope-validate-output --outdir "$RUSTWX_STORMSCOPE_OUT" --json
|
| 233 |
+
```
|
| 234 |
+
|
| 235 |
+
## Notes
|
| 236 |
+
|
| 237 |
+
- `rust/vendor/rustwx` is an ignored external checkout used by the Rust extension and renderer. `install.sh` bootstraps the validated RustWX ref automatically; you can also run `bash scripts/bootstrap-rustwx.sh` or `.\scripts\bootstrap-rustwx.ps1` directly.
|
| 238 |
+
- `rust/stormscope_obs/target/`, `rust/stormscope_render/target/`, `cache/`, `volume/`, and old loose `site/`/`out/` folders are ignored by Git and Docker.
|
| 239 |
+
- This repository's source code is MIT licensed. StormScope weights are not
|
| 240 |
+
redistributed and remain governed by the upstream NVIDIA Open Model License.
|
app.py
CHANGED
|
@@ -246,6 +246,50 @@ def package_available() -> bool:
|
|
| 246 |
return importlib.util.find_spec("rustwx_stormscope") is not None
|
| 247 |
|
| 248 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
def live_status() -> str:
|
| 250 |
lines = [
|
| 251 |
"Live inference status",
|
|
@@ -426,6 +470,8 @@ def zero_gpu_probe() -> str:
|
|
| 426 |
lines.append("This probe reports the GPU Hugging Face assigns at launch time; hardware can vary.")
|
| 427 |
lines.append("This Space proves the public web-map product and hosted UX; live inference needs a separately validated HF dependency path.")
|
| 428 |
lines.append("")
|
|
|
|
|
|
|
| 429 |
lines.append(live_status())
|
| 430 |
return "\n".join(lines)
|
| 431 |
|
|
@@ -444,8 +490,10 @@ OpenStreetMap basemap. This is the hosted visualization shape for the
|
|
| 444 |
gr.Markdown("## Live Server Probe")
|
| 445 |
with gr.Row():
|
| 446 |
probe = gr.Button("Check ZeroGPU Runtime", variant="secondary")
|
|
|
|
| 447 |
probe_output = gr.Textbox(label="Runtime probe", lines=12)
|
| 448 |
probe.click(zero_gpu_probe, outputs=probe_output)
|
|
|
|
| 449 |
gr.Markdown("## Live Nowcast")
|
| 450 |
with gr.Row():
|
| 451 |
init_time = gr.Textbox(label="Init time", value="2026-05-30T22:40:00", scale=2)
|
|
|
|
| 246 |
return importlib.util.find_spec("rustwx_stormscope") is not None
|
| 247 |
|
| 248 |
|
| 249 |
+
def command_probe(cmd: list[str]) -> str:
|
| 250 |
+
try:
|
| 251 |
+
proc = subprocess.run(
|
| 252 |
+
cmd,
|
| 253 |
+
cwd=HERE,
|
| 254 |
+
text=True,
|
| 255 |
+
capture_output=True,
|
| 256 |
+
timeout=45,
|
| 257 |
+
)
|
| 258 |
+
except Exception as exc:
|
| 259 |
+
return "+ " + " ".join(cmd) + f"\nfailed: {type(exc).__name__}: {exc}"
|
| 260 |
+
|
| 261 |
+
stdout = proc.stdout.strip()
|
| 262 |
+
stderr = proc.stderr.strip()
|
| 263 |
+
lines = [
|
| 264 |
+
"+ " + " ".join(cmd),
|
| 265 |
+
f"exit code: {proc.returncode}",
|
| 266 |
+
]
|
| 267 |
+
if stdout:
|
| 268 |
+
lines.append(stdout[:1200])
|
| 269 |
+
if stderr:
|
| 270 |
+
lines.append(stderr[:1200])
|
| 271 |
+
return "\n".join(lines)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def package_probe() -> str:
|
| 275 |
+
lines = [
|
| 276 |
+
"StormScope package probe",
|
| 277 |
+
f"python: {platform.python_version()}",
|
| 278 |
+
f"rustwx_stormscope importable: {package_available()}",
|
| 279 |
+
]
|
| 280 |
+
if not package_available():
|
| 281 |
+
lines.append("")
|
| 282 |
+
lines.append("The Space payload does not include the release package source.")
|
| 283 |
+
lines.append("Push with `scripts/push-hf-space.py --profile package-probe` to test package import and CLI help.")
|
| 284 |
+
return "\n".join(lines)
|
| 285 |
+
|
| 286 |
+
lines.append("")
|
| 287 |
+
lines.append(command_probe([sys.executable, "-m", "rustwx_stormscope.run", "--help"]))
|
| 288 |
+
lines.append("")
|
| 289 |
+
lines.append(command_probe([sys.executable, "-m", "rustwx_stormscope.webmap_preview", "--help"]))
|
| 290 |
+
return "\n".join(lines)
|
| 291 |
+
|
| 292 |
+
|
| 293 |
def live_status() -> str:
|
| 294 |
lines = [
|
| 295 |
"Live inference status",
|
|
|
|
| 470 |
lines.append("This probe reports the GPU Hugging Face assigns at launch time; hardware can vary.")
|
| 471 |
lines.append("This Space proves the public web-map product and hosted UX; live inference needs a separately validated HF dependency path.")
|
| 472 |
lines.append("")
|
| 473 |
+
lines.append(package_probe())
|
| 474 |
+
lines.append("")
|
| 475 |
lines.append(live_status())
|
| 476 |
return "\n".join(lines)
|
| 477 |
|
|
|
|
| 490 |
gr.Markdown("## Live Server Probe")
|
| 491 |
with gr.Row():
|
| 492 |
probe = gr.Button("Check ZeroGPU Runtime", variant="secondary")
|
| 493 |
+
package_button = gr.Button("Check Package CLI", variant="secondary")
|
| 494 |
probe_output = gr.Textbox(label="Runtime probe", lines=12)
|
| 495 |
probe.click(zero_gpu_probe, outputs=probe_output)
|
| 496 |
+
package_button.click(package_probe, outputs=probe_output)
|
| 497 |
gr.Markdown("## Live Nowcast")
|
| 498 |
with gr.Row():
|
| 499 |
init_time = gr.Textbox(label="Init time", value="2026-05-30T22:40:00", scale=2)
|
pyproject.toml
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=77", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "rustwx-stormscope"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Fast StormScope (earth2studio) inference on consumer Blackwell (sm_120), with rustwx data/render tooling."
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.12"
|
| 11 |
+
authors = [{ name = "Fahrenheit Research" }]
|
| 12 |
+
license = "MIT"
|
| 13 |
+
license-files = ["LICENSE", "NOTICE.md"]
|
| 14 |
+
keywords = ["weather", "nowcasting", "diffusion", "stormscope", "earth2studio", "blackwell", "cuda"]
|
| 15 |
+
classifiers = [
|
| 16 |
+
"Development Status :: 3 - Alpha",
|
| 17 |
+
"Intended Audience :: Science/Research",
|
| 18 |
+
"Programming Language :: Python :: 3.12",
|
| 19 |
+
"Topic :: Scientific/Engineering :: Atmospheric Science",
|
| 20 |
+
]
|
| 21 |
+
# NOTE: heavy deps (torch cu128, earth2studio, physicsnemo, natten-from-source, earth2grid-from-git,
|
| 22 |
+
# torchao) are installed by install.sh / optional Docker because they need special indexes / source builds
|
| 23 |
+
# and cannot be resolved by a plain `pip install`. Keep this list to pure-python glue only.
|
| 24 |
+
dependencies = ["numpy", "Pillow"]
|
| 25 |
+
|
| 26 |
+
[project.optional-dependencies]
|
| 27 |
+
dev = ["pytest"]
|
| 28 |
+
|
| 29 |
+
[project.urls]
|
| 30 |
+
Homepage = "https://github.com/FahrenheitResearch/rustwx-stormscope"
|
| 31 |
+
Issues = "https://github.com/FahrenheitResearch/rustwx-stormscope/issues"
|
| 32 |
+
Source = "https://github.com/FahrenheitResearch/rustwx-stormscope"
|
| 33 |
+
|
| 34 |
+
[project.scripts]
|
| 35 |
+
rustwx-stormscope = "rustwx_stormscope.run:main"
|
| 36 |
+
rustwx-stormscope-service = "rustwx_stormscope.service:main"
|
| 37 |
+
rustwx-stormscope-render = "rustwx_stormscope.ops_render:main"
|
| 38 |
+
rustwx-stormscope-export-wxstore = "rustwx_stormscope.wxstore_export:main"
|
| 39 |
+
rustwx-stormscope-webmap-preview = "rustwx_stormscope.webmap_preview:main"
|
| 40 |
+
rustwx-stormscope-validate-output = "rustwx_stormscope.output_validation:main"
|
| 41 |
+
|
| 42 |
+
[tool.setuptools]
|
| 43 |
+
packages = ["rustwx_stormscope"]
|
requirements-live.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=6.0,<7
|
| 2 |
+
numpy
|
| 3 |
+
Pillow
|
| 4 |
+
h5netcdf
|
| 5 |
+
xarray
|
| 6 |
+
pytz
|
| 7 |
+
scipy>=1.15.2
|
| 8 |
+
nvtx>=0.2.11
|
| 9 |
+
earth2studio==0.12.1
|
| 10 |
+
nvidia-physicsnemo==1.3.0
|
| 11 |
+
git+https://github.com/NVlabs/earth2grid.git
|
| 12 |
+
torchao
|
| 13 |
+
|
| 14 |
+
# NATTEN is intentionally not enabled here by default.
|
| 15 |
+
# The local release path source-builds natten==0.21.6 for sm_120 with CUDA 12.8.
|
| 16 |
+
# Only turn on STORMSCOPE_ENABLE_LIVE after the Space image has a validated
|
| 17 |
+
# NATTEN/attention path for the assigned ZeroGPU runtime.
|
requirements.txt
CHANGED
|
@@ -1 +1,3 @@
|
|
| 1 |
gradio>=6.0,<7
|
|
|
|
|
|
|
|
|
| 1 |
gradio>=6.0,<7
|
| 2 |
+
numpy
|
| 3 |
+
Pillow
|
rustwx_stormscope/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""rustwx-stormscope: fastest StormScope inference on consumer Blackwell (sm_120).
|
| 2 |
+
|
| 3 |
+
- optimize.apply_opts: deterministic EDM sampler + reduced steps + torch.compile + MXFP8
|
| 4 |
+
- run: one-shot runner (drop-in for run_stormscope.py)
|
| 5 |
+
- service: continuous operational route (default container entrypoint)
|
| 6 |
+
- ops_render: completed-cycle renderer for wxstore products and the static site
|
| 7 |
+
- datasources: rustwx-backed GOES/MRMS adapters (fast fetch+decode)
|
| 8 |
+
"""
|
| 9 |
+
__version__ = "0.1.0"
|
rustwx_stormscope/core.py
ADDED
|
@@ -0,0 +1,549 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Run NVIDIA StormScope (regional nowcasting) on RTX 5090.
|
| 3 |
+
|
| 4 |
+
StormScope predicts CONUS satellite (GOES) and radar (MRMS) imagery
|
| 5 |
+
at 6km or 3km resolution with 10-min or 60-min timesteps.
|
| 6 |
+
|
| 7 |
+
Operational modes:
|
| 8 |
+
nowcast - 10-min steps, pure-obs (no GFS), uses latest GOES/MRMS
|
| 9 |
+
forecast - 60-min steps, GFS-conditioned, longer range
|
| 10 |
+
|
| 11 |
+
Usage:
|
| 12 |
+
# Operational: use latest observations
|
| 13 |
+
python run_stormscope.py --mode nowcast --steps 6
|
| 14 |
+
python run_stormscope.py --mode forecast --steps 6
|
| 15 |
+
|
| 16 |
+
# Historical: specify init time
|
| 17 |
+
python run_stormscope.py --mode nowcast --steps 6 --date 2023-12-05T12:00:00
|
| 18 |
+
python run_stormscope.py --mode nowcast --res 3km --steps 3 # 3km satellite only
|
| 19 |
+
|
| 20 |
+
Output:
|
| 21 |
+
Each step -> stormscope_{YYYYMMDD}_{HH}z_f{NN}.npz
|
| 22 |
+
Metadata JSON -> stormscope_{YYYYMMDD}_{HH}z_meta.json
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import argparse
|
| 26 |
+
import json
|
| 27 |
+
import os
|
| 28 |
+
import sys
|
| 29 |
+
import time
|
| 30 |
+
from datetime import datetime, timedelta, timezone
|
| 31 |
+
|
| 32 |
+
import numpy as np
|
| 33 |
+
import torch
|
| 34 |
+
|
| 35 |
+
# When True, GOES/MRMS inputs come from the rust fast path (rustwx_stormscope.rust_datasources) instead of
|
| 36 |
+
# earth2studio's fetch_data. Set by rustwx_stormscope.datasources.install_backend.
|
| 37 |
+
RUST_BACKEND = False
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _env(name: str, legacy_name: str | None = None, default: str = "") -> str:
|
| 41 |
+
if name in os.environ:
|
| 42 |
+
return os.environ[name]
|
| 43 |
+
if legacy_name and legacy_name in os.environ:
|
| 44 |
+
return os.environ[legacy_name]
|
| 45 |
+
return default
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# Map earth2studio variable names -> wxsection output field names
|
| 49 |
+
GOES_CHANNEL_MAP = {
|
| 50 |
+
"abi01c": "ch_blue_047", # GOES Ch1 blue 0.47um
|
| 51 |
+
"abi02c": "ch_red_064", # GOES Ch2 red 0.64um
|
| 52 |
+
"abi03c": "ch_veggie_086", # GOES Ch3 veggie 0.86um
|
| 53 |
+
"abi07c": "ch_swir_390", # GOES Ch7 shortwave IR 3.90um
|
| 54 |
+
"abi08c": "ch_wv_upper_619", # GOES Ch8 upper WV 6.19um
|
| 55 |
+
"abi09c": "ch_wv_mid_695", # GOES Ch9 mid WV 6.95um
|
| 56 |
+
"abi10c": "ch_wv_lower_734", # GOES Ch10 lower WV 7.34um
|
| 57 |
+
"abi13c": "ch_ir_1035", # GOES Ch13 clean IR 10.35um
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
# Channel metadata for wxsection colormaps
|
| 61 |
+
CHANNEL_META = {
|
| 62 |
+
"ch_blue_047": {"units": "reflectance", "range": [0.0, 1.0], "type": "visible"},
|
| 63 |
+
"ch_red_064": {"units": "reflectance", "range": [0.0, 1.0], "type": "visible"},
|
| 64 |
+
"ch_veggie_086": {"units": "reflectance", "range": [0.0, 1.0], "type": "visible"},
|
| 65 |
+
"ch_swir_390": {"units": "K", "range": [180.0, 310.0], "type": "ir"},
|
| 66 |
+
"ch_wv_upper_619": {"units": "K", "range": [190.0, 260.0], "type": "wv"},
|
| 67 |
+
"ch_wv_mid_695": {"units": "K", "range": [190.0, 270.0], "type": "wv"},
|
| 68 |
+
"ch_wv_lower_734": {"units": "K", "range": [190.0, 275.0], "type": "wv"},
|
| 69 |
+
"ch_ir_1035": {"units": "K", "range": [180.0, 310.0], "type": "ir"},
|
| 70 |
+
"refc": {"units": "dBZ", "range": [-20.0, 75.0], "type": "radar"},
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
# Model registry
|
| 74 |
+
# Nowcast 6km uses "6steps" sliding-window models (both GOES and MRMS need
|
| 75 |
+
# 6 past observations at 10-min intervals = 50 min of history).
|
| 76 |
+
# Nowcast 3km uses "eoe" single-step model (GOES only, no MRMS available).
|
| 77 |
+
# Forecast uses GFS-conditioned "eoe" models (single-step with GFS context).
|
| 78 |
+
MODEL_CONFIGS = {
|
| 79 |
+
("nowcast", "6km"): {
|
| 80 |
+
"goes": "6km_10min_natten_pure_obs_zenith_6steps",
|
| 81 |
+
"mrms": "6km_10min_natten_pure_obs_mrms_obs_6steps",
|
| 82 |
+
"step_minutes": 10,
|
| 83 |
+
"needs_gfs": False,
|
| 84 |
+
},
|
| 85 |
+
("nowcast", "3km"): {
|
| 86 |
+
"goes": "3km_10min_natten_pure_obs_cos_zenith_input_eoe",
|
| 87 |
+
"mrms": None, # No 3km MRMS model available
|
| 88 |
+
"step_minutes": 10,
|
| 89 |
+
"needs_gfs": False,
|
| 90 |
+
},
|
| 91 |
+
("forecast", "6km"): {
|
| 92 |
+
"goes": "6km_60min_natten_cos_zenith_input_eoe_v2",
|
| 93 |
+
"mrms": "6km_60min_natten_cos_zenith_input_mrms_eoe",
|
| 94 |
+
"step_minutes": 60,
|
| 95 |
+
"needs_gfs": True,
|
| 96 |
+
"conditioning": "gfs",
|
| 97 |
+
},
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
STORMSCOPE_VERSION = "0.12.1" # earth2studio version
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def forecast_conditioning_kind(cfg: dict | None = None) -> str:
|
| 104 |
+
"""Return the forecast conditioner kind.
|
| 105 |
+
|
| 106 |
+
Default is GFS, matching Earth2Studio's StormScopeGOES.load_model default.
|
| 107 |
+
RUSTWX_STORMSCOPE_FORECAST_CONDITIONING=hrrr is kept as an explicit experiment switch.
|
| 108 |
+
"""
|
| 109 |
+
default = (cfg or {}).get("conditioning", "gfs")
|
| 110 |
+
kind = _env("RUSTWX_STORMSCOPE_FORECAST_CONDITIONING", "SSFAST_FORECAST_CONDITIONING", default).strip().lower()
|
| 111 |
+
if kind not in {"gfs", "hrrr"}:
|
| 112 |
+
raise ValueError("RUSTWX_STORMSCOPE_FORECAST_CONDITIONING must be 'gfs' or 'hrrr'")
|
| 113 |
+
return kind
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def get_latest_init_time(step_minutes: int, needs_history: bool,
|
| 117 |
+
gfs_aligned: bool = False) -> str:
|
| 118 |
+
"""Get the best init time for operational runs using latest observations.
|
| 119 |
+
|
| 120 |
+
Rounds down to the nearest step_minutes boundary and subtracts a small
|
| 121 |
+
buffer to ensure GOES/MRMS data is available on NOAA S3.
|
| 122 |
+
|
| 123 |
+
If gfs_aligned=True, rounds to the nearest past 6-hour GFS cycle that
|
| 124 |
+
has had time to post on AWS (~4 hours after init).
|
| 125 |
+
"""
|
| 126 |
+
now = datetime.now(timezone.utc)
|
| 127 |
+
|
| 128 |
+
if gfs_aligned:
|
| 129 |
+
# GFS_FX runs at 00/06/12/18Z. The hourly leads are available from
|
| 130 |
+
# those cycle roots, but the cycle itself needs time to publish.
|
| 131 |
+
gfs_buffer_hours = int(_env("RUSTWX_STORMSCOPE_GFS_BUFFER_HOURS", "SSFAST_GFS_BUFFER_HOURS", "4"))
|
| 132 |
+
cutoff = now - timedelta(hours=gfs_buffer_hours)
|
| 133 |
+
cutoff = cutoff.replace(minute=0, second=0, microsecond=0)
|
| 134 |
+
cutoff = cutoff - timedelta(hours=cutoff.hour % 6)
|
| 135 |
+
return cutoff.strftime("%Y-%m-%dT%H:%M:%S")
|
| 136 |
+
|
| 137 |
+
# GOES CONUS scans every 5 min, MRMS every 2 min. Default to a modest
|
| 138 |
+
# 10-minute operational buffer; override with RUSTWX_STORMSCOPE_OBS_BUFFER_MIN for
|
| 139 |
+
# very fresh or very conservative cycling.
|
| 140 |
+
buffer_min = int(_env("RUSTWX_STORMSCOPE_OBS_BUFFER_MIN", "SSFAST_OBS_BUFFER_MIN", "10"))
|
| 141 |
+
init = now - timedelta(minutes=buffer_min)
|
| 142 |
+
# Round down to nearest step_minutes
|
| 143 |
+
init = init.replace(second=0, microsecond=0)
|
| 144 |
+
init = init - timedelta(minutes=init.minute % step_minutes)
|
| 145 |
+
return init.strftime("%Y-%m-%dT%H:%M:%S")
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def save_step(outdir, init_time_str, step_idx, step_minutes, mode, res,
|
| 149 |
+
goes_pred, goes_vars, mrms_pred, lats, lons, valid_mask,
|
| 150 |
+
mrms_valid_mask, dtype_name, goes_model_name, mrms_model_name):
|
| 151 |
+
"""Save one forecast step as a self-describing .npz file."""
|
| 152 |
+
init_dt = datetime.fromisoformat(init_time_str)
|
| 153 |
+
valid_dt = init_dt + timedelta(minutes=step_minutes * (step_idx + 1))
|
| 154 |
+
fhr = step_minutes * (step_idx + 1) / 60.0
|
| 155 |
+
|
| 156 |
+
cycle_str = init_dt.strftime("%Y%m%d_%H%Mz")
|
| 157 |
+
fname = f"stormscope_{mode}_{res}_{cycle_str}_f{step_idx + 1:02d}.npz"
|
| 158 |
+
|
| 159 |
+
fields = {}
|
| 160 |
+
|
| 161 |
+
# Extract individual GOES channels
|
| 162 |
+
for i, var_name in enumerate(goes_vars):
|
| 163 |
+
field = goes_pred[0, 0, 0, i].detach().cpu().to(torch.float32).numpy()
|
| 164 |
+
field = np.where(valid_mask[0, 0, 0, i].cpu().numpy(), field, np.nan)
|
| 165 |
+
out_name = GOES_CHANNEL_MAP.get(var_name, var_name)
|
| 166 |
+
fields[out_name] = field
|
| 167 |
+
|
| 168 |
+
# Extract MRMS reflectivity (if available)
|
| 169 |
+
if mrms_pred is not None:
|
| 170 |
+
refc = mrms_pred[0, 0, 0, 0].detach().cpu().to(torch.float32).numpy()
|
| 171 |
+
refc = np.where(mrms_valid_mask[0, 0, 0, 0].cpu().numpy(), refc, np.nan)
|
| 172 |
+
fields["refc"] = refc
|
| 173 |
+
|
| 174 |
+
# Grid metadata - convert lons from 0-360 to -180/+180 for wxsection
|
| 175 |
+
lons_fixed = lons.copy().astype(np.float32)
|
| 176 |
+
lons_fixed[lons_fixed > 180] -= 360
|
| 177 |
+
fields["lats"] = lats.astype(np.float32)
|
| 178 |
+
fields["lons"] = lons_fixed
|
| 179 |
+
|
| 180 |
+
# Timing metadata
|
| 181 |
+
fields["init_time"] = init_dt.isoformat()
|
| 182 |
+
fields["valid_time"] = valid_dt.isoformat()
|
| 183 |
+
fields["step_minutes"] = step_minutes
|
| 184 |
+
fields["forecast_hour"] = fhr
|
| 185 |
+
fields["mode"] = mode
|
| 186 |
+
fields["resolution"] = res
|
| 187 |
+
fields["dtype"] = dtype_name
|
| 188 |
+
fields["goes_model"] = goes_model_name
|
| 189 |
+
fields["mrms_model"] = mrms_model_name or "none"
|
| 190 |
+
fields["stormscope_version"] = STORMSCOPE_VERSION
|
| 191 |
+
|
| 192 |
+
outpath = os.path.join(outdir, fname)
|
| 193 |
+
np.savez_compressed(outpath, **fields)
|
| 194 |
+
|
| 195 |
+
# Print stats
|
| 196 |
+
label = f"T+{int(fhr)}h" if fhr == int(fhr) else f"T+{fhr:.1f}h"
|
| 197 |
+
print(f" Saved {fname} ({label}, valid {valid_dt.strftime('%Y-%m-%d %H:%M')}Z)")
|
| 198 |
+
for name, arr in fields.items():
|
| 199 |
+
if isinstance(arr, np.ndarray) and arr.ndim == 2:
|
| 200 |
+
valid = arr[~np.isnan(arr)]
|
| 201 |
+
if len(valid) > 0:
|
| 202 |
+
print(f" {name}: [{valid.min():.1f}, {valid.max():.1f}] mean={valid.mean():.1f}")
|
| 203 |
+
|
| 204 |
+
return outpath
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def save_cycle_metadata(outdir, init_time_str, mode, res, cfg, saved_files,
|
| 208 |
+
goes_model_name, mrms_model_name, grid_shape, dtype_name,
|
| 209 |
+
total_elapsed, complete=True):
|
| 210 |
+
"""Write a JSON manifest for the entire cycle (for unified_dashboard ingestion)."""
|
| 211 |
+
init_dt = datetime.fromisoformat(init_time_str)
|
| 212 |
+
cycle_str = init_dt.strftime("%Y%m%d_%H%Mz")
|
| 213 |
+
step_minutes = cfg["step_minutes"]
|
| 214 |
+
|
| 215 |
+
fields_available = list(GOES_CHANNEL_MAP.values())
|
| 216 |
+
if cfg.get("mrms"):
|
| 217 |
+
fields_available.append("refc")
|
| 218 |
+
|
| 219 |
+
meta = {
|
| 220 |
+
"model": "stormscope",
|
| 221 |
+
"cycle": cycle_str,
|
| 222 |
+
"init_time": init_dt.isoformat() + "Z",
|
| 223 |
+
"mode": mode,
|
| 224 |
+
"complete": complete,
|
| 225 |
+
"resolution": res,
|
| 226 |
+
"step_minutes": step_minutes,
|
| 227 |
+
"n_steps": len(saved_files),
|
| 228 |
+
"valid_times": [
|
| 229 |
+
(init_dt + timedelta(minutes=step_minutes * (i + 1))).isoformat() + "Z"
|
| 230 |
+
for i in range(len(saved_files))
|
| 231 |
+
],
|
| 232 |
+
"fields": fields_available,
|
| 233 |
+
"field_metadata": {k: v for k, v in CHANNEL_META.items() if k in fields_available},
|
| 234 |
+
"grid": {
|
| 235 |
+
"shape": list(grid_shape),
|
| 236 |
+
"projection": "hrrr_lcc",
|
| 237 |
+
},
|
| 238 |
+
"models": {
|
| 239 |
+
"goes": goes_model_name,
|
| 240 |
+
"mrms": mrms_model_name or None,
|
| 241 |
+
},
|
| 242 |
+
"inference": {
|
| 243 |
+
"dtype": dtype_name,
|
| 244 |
+
"device": torch.cuda.get_device_name() if torch.cuda.is_available() else "cpu",
|
| 245 |
+
"stormscope_version": STORMSCOPE_VERSION,
|
| 246 |
+
"conditioning_source": forecast_conditioning_kind(cfg) if cfg.get("needs_gfs") else None,
|
| 247 |
+
"total_seconds": round(total_elapsed, 1),
|
| 248 |
+
},
|
| 249 |
+
"files": [os.path.basename(f) for f in saved_files],
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
meta_path = os.path.join(outdir, f"stormscope_{mode}_{res}_{cycle_str}_meta.json")
|
| 253 |
+
with open(meta_path, "w") as f:
|
| 254 |
+
json.dump(meta, f, indent=2)
|
| 255 |
+
if complete:
|
| 256 |
+
print(f"\nMetadata: {os.path.basename(meta_path)}")
|
| 257 |
+
return meta_path
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def run_stormscope(date: str, nsteps: int, outdir: str, mode: str, res: str,
|
| 261 |
+
use_bf16: bool = True):
|
| 262 |
+
"""Run StormScope in the specified mode."""
|
| 263 |
+
from earth2studio.data import GFS_FX, HRRR_FX, GOES, MRMS, fetch_data
|
| 264 |
+
from earth2studio.models.px.stormscope import (
|
| 265 |
+
StormScopeBase,
|
| 266 |
+
StormScopeGOES,
|
| 267 |
+
StormScopeMRMS,
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
config_key = (mode, res)
|
| 271 |
+
if config_key not in MODEL_CONFIGS:
|
| 272 |
+
raise ValueError(f"No model config for mode={mode}, res={res}. "
|
| 273 |
+
f"Available: {list(MODEL_CONFIGS.keys())}")
|
| 274 |
+
|
| 275 |
+
cfg = MODEL_CONFIGS[config_key]
|
| 276 |
+
step_minutes = cfg["step_minutes"]
|
| 277 |
+
conditioning_kind = forecast_conditioning_kind(cfg) if cfg["needs_gfs"] else None
|
| 278 |
+
|
| 279 |
+
# Determine dtype
|
| 280 |
+
if use_bf16 and torch.cuda.is_bf16_supported():
|
| 281 |
+
compute_dtype = torch.bfloat16
|
| 282 |
+
dtype_name = "bfloat16"
|
| 283 |
+
else:
|
| 284 |
+
compute_dtype = torch.float32
|
| 285 |
+
dtype_name = "float32"
|
| 286 |
+
if use_bf16:
|
| 287 |
+
print("WARNING: bf16 requested but not supported, falling back to fp32")
|
| 288 |
+
|
| 289 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 290 |
+
os.makedirs(outdir, exist_ok=True)
|
| 291 |
+
|
| 292 |
+
print(f"Mode: {mode} ({res}, {step_minutes}-min steps)")
|
| 293 |
+
print(f"Precision: {dtype_name}")
|
| 294 |
+
if conditioning_kind:
|
| 295 |
+
print(f"Forecast conditioning: {conditioning_kind.upper()}_FX")
|
| 296 |
+
else:
|
| 297 |
+
print("Forecast conditioning: no (pure obs)")
|
| 298 |
+
print(f"MRMS model: {'yes' if cfg.get('mrms') else 'no'}")
|
| 299 |
+
|
| 300 |
+
run_start = time.time()
|
| 301 |
+
|
| 302 |
+
# --- Load models ---
|
| 303 |
+
print("\nLoading StormScope models...")
|
| 304 |
+
t0 = time.time()
|
| 305 |
+
package = StormScopeBase.load_default_package()
|
| 306 |
+
|
| 307 |
+
# GOES model - load to device first, cast to bf16 AFTER building interpolators
|
| 308 |
+
goes_model_name = cfg["goes"]
|
| 309 |
+
if cfg["needs_gfs"]:
|
| 310 |
+
if conditioning_kind == "gfs":
|
| 311 |
+
conditioning_source = GFS_FX()
|
| 312 |
+
elif conditioning_kind == "hrrr":
|
| 313 |
+
conditioning_source = HRRR_FX()
|
| 314 |
+
else:
|
| 315 |
+
raise ValueError(f"Unsupported forecast conditioning source: {conditioning_kind}")
|
| 316 |
+
model = StormScopeGOES.load_model(
|
| 317 |
+
package=package,
|
| 318 |
+
conditioning_data_source=conditioning_source,
|
| 319 |
+
model_name=goes_model_name,
|
| 320 |
+
)
|
| 321 |
+
else:
|
| 322 |
+
model = StormScopeGOES.load_model(
|
| 323 |
+
package=package,
|
| 324 |
+
model_name=goes_model_name,
|
| 325 |
+
)
|
| 326 |
+
model = model.to(device=device)
|
| 327 |
+
model.eval()
|
| 328 |
+
print(f" GOES model ({goes_model_name}) loaded in {time.time() - t0:.1f}s")
|
| 329 |
+
|
| 330 |
+
# Determine GOES satellite: GOES-16 retired Apr 7 2025, GOES-19 took over
|
| 331 |
+
init_dt = datetime.fromisoformat(date)
|
| 332 |
+
goes_cutover = datetime(2025, 4, 7)
|
| 333 |
+
goes_satellite = "goes19" if init_dt >= goes_cutover else "goes16"
|
| 334 |
+
scan_mode = "C"
|
| 335 |
+
print(f" Satellite: {goes_satellite.upper()} ({scan_mode})")
|
| 336 |
+
|
| 337 |
+
# MRMS model (optional)
|
| 338 |
+
model_mrms = None
|
| 339 |
+
mrms_model_name = cfg.get("mrms")
|
| 340 |
+
if mrms_model_name:
|
| 341 |
+
t0 = time.time()
|
| 342 |
+
model_mrms = StormScopeMRMS.load_model(
|
| 343 |
+
package=package,
|
| 344 |
+
conditioning_data_source=GOES(satellite=goes_satellite, scan_mode=scan_mode),
|
| 345 |
+
model_name=mrms_model_name,
|
| 346 |
+
)
|
| 347 |
+
model_mrms = model_mrms.to(device=device)
|
| 348 |
+
model_mrms.eval()
|
| 349 |
+
print(f" MRMS model ({mrms_model_name}) loaded in {time.time() - t0:.1f}s")
|
| 350 |
+
|
| 351 |
+
start_date = [np.datetime64(datetime.fromisoformat(date))]
|
| 352 |
+
print(f"\nInit time: {start_date[0]}")
|
| 353 |
+
|
| 354 |
+
goes_vars = list(model.input_coords()["variable"])
|
| 355 |
+
lat_out = model.latitudes.detach().cpu().to(torch.float32).numpy()
|
| 356 |
+
lon_out = model.longitudes.detach().cpu().to(torch.float32).numpy()
|
| 357 |
+
grid_shape = lat_out.shape
|
| 358 |
+
print(f"Output grid: {grid_shape}, lat [{lat_out.min():.1f}, {lat_out.max():.1f}], "
|
| 359 |
+
f"lon [{lon_out.min():.1f}, {lon_out.max():.1f}]")
|
| 360 |
+
|
| 361 |
+
goes = GOES(satellite=goes_satellite, scan_mode=scan_mode)
|
| 362 |
+
goes_lat, goes_lon = GOES.grid(satellite=goes_satellite, scan_mode=scan_mode)
|
| 363 |
+
|
| 364 |
+
# Build interpolators
|
| 365 |
+
print("Building interpolators...")
|
| 366 |
+
model.build_input_interpolator(goes_lat, goes_lon)
|
| 367 |
+
if cfg["needs_gfs"]:
|
| 368 |
+
if conditioning_kind == "gfs":
|
| 369 |
+
gfs_lon, gfs_lat = np.meshgrid(GFS_FX.GFS_LON, GFS_FX.GFS_LAT)
|
| 370 |
+
model.build_conditioning_interpolator(
|
| 371 |
+
gfs_lat.astype(np.float32),
|
| 372 |
+
gfs_lon.astype(np.float32),
|
| 373 |
+
)
|
| 374 |
+
elif conditioning_kind == "hrrr":
|
| 375 |
+
hrrr_lat, hrrr_lon = HRRR_FX.grid()
|
| 376 |
+
model.build_conditioning_interpolator(hrrr_lat, hrrr_lon)
|
| 377 |
+
|
| 378 |
+
in_coords = model.input_coords()
|
| 379 |
+
|
| 380 |
+
# --- Fetch initial data ---
|
| 381 |
+
print("Fetching GOES data...")
|
| 382 |
+
t0 = time.time()
|
| 383 |
+
if RUST_BACKEND:
|
| 384 |
+
from rustwx_stormscope import rust_datasources as _rds
|
| 385 |
+
x, x_coords = _rds.goes_input(goes_satellite, scan_mode, start_date, goes_vars,
|
| 386 |
+
in_coords["lead_time"], device, goes_lat.shape)
|
| 387 |
+
else:
|
| 388 |
+
x, x_coords = fetch_data(goes, time=start_date, variable=np.array(goes_vars),
|
| 389 |
+
lead_time=in_coords["lead_time"], device=device)
|
| 390 |
+
print(f" GOES data fetched in {time.time() - t0:.1f}s, shape={x.shape}")
|
| 391 |
+
|
| 392 |
+
# MRMS data (if model available)
|
| 393 |
+
x_mrms = None
|
| 394 |
+
x_coords_mrms = None
|
| 395 |
+
if model_mrms:
|
| 396 |
+
print("Fetching MRMS data...")
|
| 397 |
+
t0 = time.time()
|
| 398 |
+
mrms_in_coords = model_mrms.input_coords()
|
| 399 |
+
if RUST_BACKEND:
|
| 400 |
+
from rustwx_stormscope import rust_datasources as _rds
|
| 401 |
+
x_mrms, x_coords_mrms, _mlat, _mlon = _rds.mrms_input(
|
| 402 |
+
start_date, mrms_in_coords["lead_time"], device)
|
| 403 |
+
else:
|
| 404 |
+
mrms = MRMS()
|
| 405 |
+
x_mrms, x_coords_mrms = fetch_data(mrms, time=start_date, variable=np.array(["refc"]),
|
| 406 |
+
lead_time=mrms_in_coords["lead_time"], device=device)
|
| 407 |
+
print(f" MRMS data fetched in {time.time() - t0:.1f}s, shape={x_mrms.shape}")
|
| 408 |
+
|
| 409 |
+
model_mrms.build_input_interpolator(x_coords_mrms["lat"], x_coords_mrms["lon"])
|
| 410 |
+
model_mrms.build_conditioning_interpolator(goes_lat, goes_lon)
|
| 411 |
+
|
| 412 |
+
# --- Add batch dimension ---
|
| 413 |
+
batch_size = 1
|
| 414 |
+
if x.dim() == 5:
|
| 415 |
+
x = x.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1, 1)
|
| 416 |
+
x_coords["batch"] = np.arange(batch_size)
|
| 417 |
+
x_coords.move_to_end("batch", last=False)
|
| 418 |
+
if x_mrms is not None and x_mrms.dim() == 5:
|
| 419 |
+
x_mrms = x_mrms.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1, 1)
|
| 420 |
+
x_coords_mrms["batch"] = np.arange(batch_size)
|
| 421 |
+
x_coords_mrms.move_to_end("batch", last=False)
|
| 422 |
+
|
| 423 |
+
x = x.to(dtype=torch.float32)
|
| 424 |
+
if x_mrms is not None:
|
| 425 |
+
x_mrms = x_mrms.to(dtype=torch.float32)
|
| 426 |
+
|
| 427 |
+
# Use autocast for bf16 - PyTorch handles mixed precision automatically
|
| 428 |
+
use_autocast = (compute_dtype == torch.bfloat16)
|
| 429 |
+
if use_autocast:
|
| 430 |
+
print(f"Using torch.autocast(bf16) for inference")
|
| 431 |
+
|
| 432 |
+
# --- Run coupled forecast, saving each step ---
|
| 433 |
+
y, y_coords = x, x_coords
|
| 434 |
+
y_mrms, y_coords_mrms = x_mrms, x_coords_mrms
|
| 435 |
+
saved_files = []
|
| 436 |
+
|
| 437 |
+
total_minutes = nsteps * step_minutes
|
| 438 |
+
print(f"\nRunning {nsteps}-step {mode} ({total_minutes} min = {total_minutes/60:.1f}h)...")
|
| 439 |
+
for step_idx in range(nsteps):
|
| 440 |
+
t0 = time.time()
|
| 441 |
+
|
| 442 |
+
# GOES prediction
|
| 443 |
+
with torch.autocast("cuda", dtype=compute_dtype, enabled=use_autocast):
|
| 444 |
+
y_pred, y_pred_coords = model(y, y_coords)
|
| 445 |
+
|
| 446 |
+
# MRMS prediction conditioned on GOES (if available)
|
| 447 |
+
y_mrms_pred = None
|
| 448 |
+
y_coords_mrms_pred = None
|
| 449 |
+
if model_mrms and y_mrms is not None:
|
| 450 |
+
with torch.autocast("cuda", dtype=compute_dtype, enabled=use_autocast):
|
| 451 |
+
y_mrms_pred, y_coords_mrms_pred = model_mrms.call_with_conditioning(
|
| 452 |
+
y_mrms, y_coords_mrms, conditioning=y, conditioning_coords=y_coords
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
elapsed = time.time() - t0
|
| 456 |
+
print(f" Step {step_idx + 1}/{nsteps}: inference {elapsed:.1f}s")
|
| 457 |
+
|
| 458 |
+
# Save this step
|
| 459 |
+
mrms_valid_mask = None
|
| 460 |
+
if model_mrms and y_mrms_pred is not None:
|
| 461 |
+
mrms_valid_mask = model_mrms.valid_mask.expand_as(y_mrms_pred)
|
| 462 |
+
|
| 463 |
+
outpath = save_step(
|
| 464 |
+
outdir, date, step_idx, step_minutes, mode, res,
|
| 465 |
+
y_pred, goes_vars, y_mrms_pred,
|
| 466 |
+
lat_out, lon_out,
|
| 467 |
+
model.valid_mask.expand_as(y_pred),
|
| 468 |
+
mrms_valid_mask,
|
| 469 |
+
dtype_name, goes_model_name, mrms_model_name,
|
| 470 |
+
)
|
| 471 |
+
saved_files.append(outpath)
|
| 472 |
+
|
| 473 |
+
# Keep the operational renderer fed as soon as f01 exists. The service
|
| 474 |
+
# only treats complete=true manifests as completed cycles.
|
| 475 |
+
save_cycle_metadata(
|
| 476 |
+
outdir, date, mode, res, cfg, saved_files,
|
| 477 |
+
goes_model_name, mrms_model_name, grid_shape, dtype_name,
|
| 478 |
+
time.time() - run_start, complete=False,
|
| 479 |
+
)
|
| 480 |
+
|
| 481 |
+
# Update sliding window for next step
|
| 482 |
+
y_pred, y_pred_coords = model.next_input(y_pred, y_pred_coords, y, y_coords)
|
| 483 |
+
if model_mrms and y_mrms_pred is not None:
|
| 484 |
+
y_mrms_pred, y_coords_mrms_pred = model_mrms.next_input(
|
| 485 |
+
y_mrms_pred, y_coords_mrms_pred, y_mrms, y_coords_mrms
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
y = y_pred
|
| 489 |
+
y_coords = y_pred_coords
|
| 490 |
+
if y_mrms_pred is not None:
|
| 491 |
+
y_mrms = y_mrms_pred
|
| 492 |
+
y_coords_mrms = y_coords_mrms_pred
|
| 493 |
+
|
| 494 |
+
total_elapsed = time.time() - run_start
|
| 495 |
+
|
| 496 |
+
# Write cycle metadata JSON for unified_dashboard
|
| 497 |
+
meta_path = save_cycle_metadata(
|
| 498 |
+
outdir, date, mode, res, cfg, saved_files,
|
| 499 |
+
goes_model_name, mrms_model_name, grid_shape, dtype_name, total_elapsed,
|
| 500 |
+
complete=True,
|
| 501 |
+
)
|
| 502 |
+
|
| 503 |
+
print(f"\n=== StormScope {mode} complete ===")
|
| 504 |
+
print(f" {len(saved_files)} steps, {total_elapsed:.0f}s total")
|
| 505 |
+
print(f" Output: {outdir}/")
|
| 506 |
+
for f in saved_files:
|
| 507 |
+
print(f" {os.path.basename(f)}")
|
| 508 |
+
|
| 509 |
+
return saved_files
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
if __name__ == "__main__":
|
| 513 |
+
parser = argparse.ArgumentParser(description="Run NVIDIA StormScope")
|
| 514 |
+
parser.add_argument("--mode", type=str, default="nowcast",
|
| 515 |
+
choices=["nowcast", "forecast"],
|
| 516 |
+
help="nowcast=10min pure-obs, forecast=60min GFS-conditioned")
|
| 517 |
+
parser.add_argument("--res", type=str, default="6km",
|
| 518 |
+
choices=["6km", "3km"],
|
| 519 |
+
help="Grid resolution (3km only for nowcast mode)")
|
| 520 |
+
parser.add_argument("--steps", type=int, default=6,
|
| 521 |
+
help="Forecast steps (default: 6)")
|
| 522 |
+
parser.add_argument("--date", type=str, default=None,
|
| 523 |
+
help="Init datetime ISO format (default: latest available)")
|
| 524 |
+
parser.add_argument("--outdir", type=str, default="outputs/stormscope",
|
| 525 |
+
help="Output directory")
|
| 526 |
+
parser.add_argument("--fp32", action="store_true",
|
| 527 |
+
help="Force fp32 instead of bf16")
|
| 528 |
+
args = parser.parse_args()
|
| 529 |
+
|
| 530 |
+
print(f"StormScope on {torch.cuda.get_device_name()}")
|
| 531 |
+
print(f"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")
|
| 532 |
+
print()
|
| 533 |
+
|
| 534 |
+
# Resolve init time
|
| 535 |
+
cfg = MODEL_CONFIGS[(args.mode, args.res)]
|
| 536 |
+
if args.date is None:
|
| 537 |
+
needs_history = "6steps" in cfg["goes"]
|
| 538 |
+
date = get_latest_init_time(
|
| 539 |
+
cfg["step_minutes"],
|
| 540 |
+
needs_history,
|
| 541 |
+
gfs_aligned=cfg.get("needs_gfs", False)
|
| 542 |
+
and forecast_conditioning_kind(cfg) == "gfs",
|
| 543 |
+
)
|
| 544 |
+
print(f"Using latest init time: {date}Z")
|
| 545 |
+
else:
|
| 546 |
+
date = args.date
|
| 547 |
+
|
| 548 |
+
run_stormscope(date, args.steps, args.outdir, args.mode, args.res,
|
| 549 |
+
use_bf16=not args.fp32)
|
rustwx_stormscope/datasources.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""GOES/MRMS data backends for StormScope.
|
| 3 |
+
|
| 4 |
+
Two backends:
|
| 5 |
+
- 'earth2studio' (default): uses earth2studio's built-in GOES/MRMS data sources (Python S3 fetch).
|
| 6 |
+
- 'rust': uses the rustwx-backed `stormscope_obs` extension for parallel download + pure-Rust decode,
|
| 7 |
+
exposed through earth2studio-compatible RustGOES / RustMRMS adapters (see M2).
|
| 8 |
+
|
| 9 |
+
`install_backend(core, backend)` swaps the data-source classes that `core.run_stormscope` uses, so the
|
| 10 |
+
rest of the pipeline (interpolation, sampling, output) is unchanged.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def install_backend(core, backend: str):
|
| 15 |
+
"""Select the GOES/MRMS data backend on `core`. Falls back to earth2studio if rust is missing."""
|
| 16 |
+
if backend == "earth2studio":
|
| 17 |
+
core.RUST_BACKEND = False
|
| 18 |
+
return
|
| 19 |
+
if backend == "rust":
|
| 20 |
+
try:
|
| 21 |
+
import stormscope_obs # noqa: F401 (compiled rust extension)
|
| 22 |
+
import h5netcdf # noqa: F401 (GOES decode)
|
| 23 |
+
core.RUST_BACKEND = True
|
| 24 |
+
print("[rustwx_stormscope] rust data backend active (parallel download + MRMS rust decode)", flush=True)
|
| 25 |
+
except Exception as e:
|
| 26 |
+
core.RUST_BACKEND = False
|
| 27 |
+
print(f"[rustwx_stormscope] rust backend unavailable ({type(e).__name__}); "
|
| 28 |
+
f"falling back to earth2studio fetch", flush=True)
|
| 29 |
+
return
|
| 30 |
+
raise ValueError(f"unknown backend {backend!r}")
|
rustwx_stormscope/fp8_linear.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""FP8 (e4m3) dynamic-rowwise Linear for sm_120, drop-in for nn.Linear in the DiT blocks.
|
| 3 |
+
Static per-out-channel weight quant; dynamic per-row activation quant; torch._scaled_mm.
|
| 4 |
+
"""
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
|
| 8 |
+
FP8_MAX = 448.0
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class Fp8Linear(nn.Module):
|
| 12 |
+
def __init__(self, lin: nn.Linear):
|
| 13 |
+
super().__init__()
|
| 14 |
+
w = lin.weight.data
|
| 15 |
+
N, K = w.shape
|
| 16 |
+
sw = (w.float().abs().amax(dim=1, keepdim=True).clamp(min=1e-4) / FP8_MAX) # (N,1) fp32
|
| 17 |
+
self.register_buffer("wq", (w.float() / sw).to(torch.float8_e4m3fn)) # (N,K) fp8
|
| 18 |
+
self.register_buffer("sb", sw.t().contiguous()) # (1,N) fp32
|
| 19 |
+
self.bias = nn.Parameter(lin.bias.data.clone()) if lin.bias is not None else None
|
| 20 |
+
self.N, self.K = N, K
|
| 21 |
+
|
| 22 |
+
def forward(self, x):
|
| 23 |
+
lead = x.shape[:-1]
|
| 24 |
+
x2 = x.reshape(-1, self.K)
|
| 25 |
+
sx = (x2.float().abs().amax(dim=1, keepdim=True).clamp(min=1e-4) / FP8_MAX) # (M,1) fp32
|
| 26 |
+
xq = (x2.float() / sx).to(torch.float8_e4m3fn)
|
| 27 |
+
# rowwise _scaled_mm only supports bf16/fp16 output; compute in bf16 then cast back if needed
|
| 28 |
+
out = torch._scaled_mm(xq, self.wq.t(), scale_a=sx, scale_b=self.sb, out_dtype=torch.bfloat16)
|
| 29 |
+
if self.bias is not None:
|
| 30 |
+
out = out + self.bias.to(torch.bfloat16)
|
| 31 |
+
out = out.reshape(*lead, self.N)
|
| 32 |
+
return out.to(x.dtype) if x.dtype != torch.bfloat16 else out
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def swap_linears(module, name_substrings, verbose=False):
|
| 36 |
+
"""Replace nn.Linear whose qualified name contains any of name_substrings with Fp8Linear."""
|
| 37 |
+
n = 0
|
| 38 |
+
for parent_name, parent in module.named_modules():
|
| 39 |
+
for child_name, child in list(parent.named_children()):
|
| 40 |
+
if isinstance(child, nn.Linear):
|
| 41 |
+
qual = f"{parent_name}.{child_name}"
|
| 42 |
+
if any(s in qual for s in name_substrings):
|
| 43 |
+
setattr(parent, child_name, Fp8Linear(child).to(child.weight.device))
|
| 44 |
+
n += 1
|
| 45 |
+
if verbose:
|
| 46 |
+
print(f" fp8 swap: {qual} ({child.in_features}->{child.out_features})", flush=True)
|
| 47 |
+
return n
|
rustwx_stormscope/ops_render.py
ADDED
|
@@ -0,0 +1,674 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""Build the StormScope operational web view from cycle .npz outputs.
|
| 3 |
+
|
| 4 |
+
Rendering is delegated to rust/stormscope_render, which uses rustwx-render for
|
| 5 |
+
the basemap, projection, palettes, and PNG writing. This script only watches
|
| 6 |
+
cycle manifests, invokes the renderer, assembles GIFs, and writes the HTML shell.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
import shutil
|
| 14 |
+
import subprocess
|
| 15 |
+
import sys
|
| 16 |
+
import time
|
| 17 |
+
import traceback
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from datetime import datetime, timedelta, timezone
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
from PIL import Image
|
| 23 |
+
|
| 24 |
+
from .paths import default_exports_dir, default_outdir, default_site_dir, default_spatial_root, repo_root
|
| 25 |
+
from .wxstore_export import export_cycle as export_wxstore_cycle
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@dataclass(frozen=True)
|
| 29 |
+
class Product:
|
| 30 |
+
slug: str
|
| 31 |
+
renderer_key: str
|
| 32 |
+
field: str
|
| 33 |
+
title: str
|
| 34 |
+
duration_ms: int
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@dataclass(frozen=True)
|
| 38 |
+
class Region:
|
| 39 |
+
slug: str
|
| 40 |
+
title: str
|
| 41 |
+
west: float
|
| 42 |
+
east: float
|
| 43 |
+
south: float
|
| 44 |
+
north: float
|
| 45 |
+
pad_frac: float = 0.0
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
PRODUCTS = [
|
| 49 |
+
Product("ir", "ir", "ch_ir_1035", "IR Brightness Temperature", 650),
|
| 50 |
+
Product("refc", "refc", "refc", "Composite Reflectivity", 650),
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
REGIONS = {
|
| 54 |
+
"conus": Region("conus", "CONUS", -126.0, -66.0, 23.0, 50.0, 0.0),
|
| 55 |
+
"central_plains": Region("central_plains", "Central Plains", -106.5, -90.0, 31.5, 47.2, 0.0),
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def parse_args() -> argparse.Namespace:
|
| 60 |
+
root = repo_root()
|
| 61 |
+
renderer_name = "stormscope_render.exe" if os.name == "nt" else "stormscope_render"
|
| 62 |
+
wxstore_name = "stormscope_wxstore.exe" if os.name == "nt" else "stormscope_wxstore"
|
| 63 |
+
ap = argparse.ArgumentParser(description="Render StormScope operational site")
|
| 64 |
+
ap.add_argument("--outdir", default=str(default_outdir()))
|
| 65 |
+
ap.add_argument("--site-dir", default=str(default_site_dir()))
|
| 66 |
+
ap.add_argument("--renderer", default=str(root / "rust" / "stormscope_render" / "target" / "release" / renderer_name))
|
| 67 |
+
ap.add_argument("--wxstore-exporter", default=str(root / "rust" / "stormscope_wxstore" / "target" / "release" / wxstore_name))
|
| 68 |
+
ap.add_argument("--wxstore-spatial-root", default=str(default_spatial_root()))
|
| 69 |
+
ap.add_argument("--wxstore-fields", default="",
|
| 70 |
+
help="comma-separated WXA field allowlist; default exports all StormScope 2D fields")
|
| 71 |
+
ap.add_argument("--no-wxstore-export", action="store_true",
|
| 72 |
+
help="skip native rustwx WXA export")
|
| 73 |
+
ap.add_argument("--visual-dir", default=str(default_exports_dir()))
|
| 74 |
+
ap.add_argument("--export-dir", default="",
|
| 75 |
+
help="alias for --visual-dir; copies latest exported GIFs/PNGs there")
|
| 76 |
+
ap.add_argument("--regions", default="conus,central_plains",
|
| 77 |
+
help="comma-separated region slugs; available: conus,central_plains")
|
| 78 |
+
ap.add_argument("--loop", action="store_true")
|
| 79 |
+
ap.add_argument("--interval", type=int, default=60, help="seconds between render scans")
|
| 80 |
+
ap.add_argument("--width", type=int, default=1200)
|
| 81 |
+
ap.add_argument("--height", type=int, default=820)
|
| 82 |
+
ap.add_argument("--max-cycles-per-mode", type=int, default=2,
|
| 83 |
+
help="render only this many latest cycles per mode; 0 renders all")
|
| 84 |
+
ap.add_argument("--force", action="store_true")
|
| 85 |
+
args = ap.parse_args()
|
| 86 |
+
if args.export_dir and not args.visual_dir:
|
| 87 |
+
args.visual_dir = args.export_dir
|
| 88 |
+
args.region_list = parse_regions(args.regions)
|
| 89 |
+
return args
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def parse_regions(value: str) -> list[Region]:
|
| 93 |
+
regions = []
|
| 94 |
+
for raw in value.split(","):
|
| 95 |
+
slug = raw.strip()
|
| 96 |
+
if not slug:
|
| 97 |
+
continue
|
| 98 |
+
if slug not in REGIONS:
|
| 99 |
+
raise ValueError(f"unknown region {slug!r}; available: {', '.join(REGIONS)}")
|
| 100 |
+
regions.append(REGIONS[slug])
|
| 101 |
+
return regions or [REGIONS["conus"]]
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def read_json(path: Path) -> dict:
|
| 105 |
+
return json.loads(path.read_text())
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def display_mode(mode: str) -> str:
|
| 109 |
+
return "nearcast" if mode == "forecast" else mode
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def mode_title(mode: str) -> str:
|
| 113 |
+
return display_mode(mode).capitalize()
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def parse_dt(value: str) -> datetime:
|
| 117 |
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def dt_label(value: str) -> str:
|
| 121 |
+
return parse_dt(value).strftime("%Y-%m-%d %H:%MZ")
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def cycle_sort_key(meta: dict) -> datetime:
|
| 125 |
+
return parse_dt(meta["init_time"])
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def relpath(path: Path, root: Path) -> str:
|
| 129 |
+
return path.relative_to(root).as_posix()
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def should_update(target: Path, sources: list[Path], force: bool) -> bool:
|
| 133 |
+
if force or not target.exists():
|
| 134 |
+
return True
|
| 135 |
+
try:
|
| 136 |
+
if target.stat().st_size == 0:
|
| 137 |
+
return True
|
| 138 |
+
except OSError:
|
| 139 |
+
return True
|
| 140 |
+
target_mtime = target.stat().st_mtime
|
| 141 |
+
return any(src.exists() and src.stat().st_mtime > target_mtime for src in sources)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def frame_number(filename: str, index: int) -> str:
|
| 145 |
+
stem = Path(filename).stem
|
| 146 |
+
tail = stem.rsplit("_", 1)[-1]
|
| 147 |
+
if tail.startswith("f") and tail[1:].isdigit():
|
| 148 |
+
return tail.upper()
|
| 149 |
+
return f"F{index + 1:02d}"
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def visible_frame_indices(meta: dict) -> list[int]:
|
| 153 |
+
files = meta.get("files", [])
|
| 154 |
+
if display_mode(meta["mode"]) != "nearcast":
|
| 155 |
+
return list(range(len(files)))
|
| 156 |
+
|
| 157 |
+
valid_times = meta.get("valid_times", [])
|
| 158 |
+
cutoff = datetime.now(timezone.utc) - timedelta(minutes=15)
|
| 159 |
+
selected = [
|
| 160 |
+
i for i in range(len(files))
|
| 161 |
+
if i < len(valid_times) and parse_dt(valid_times[i]) >= cutoff
|
| 162 |
+
]
|
| 163 |
+
return selected or list(range(len(files)))
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def render_png(renderer: Path, npz: Path, output: Path, product: Product, region: Region, meta: dict,
|
| 167 |
+
index: int, width: int, height: int, force: bool) -> None:
|
| 168 |
+
if not should_update(output, [npz, renderer], force):
|
| 169 |
+
return
|
| 170 |
+
valid = meta["valid_times"][index] if index < len(meta.get("valid_times", [])) else ""
|
| 171 |
+
step_minutes = int(meta.get("step_minutes", 0))
|
| 172 |
+
lead_minutes = step_minutes * (index + 1)
|
| 173 |
+
if lead_minutes and lead_minutes % 60 == 0:
|
| 174 |
+
lead = f"+{lead_minutes // 60} h"
|
| 175 |
+
elif lead_minutes:
|
| 176 |
+
lead = f"+{lead_minutes} min"
|
| 177 |
+
else:
|
| 178 |
+
lead = frame_number(npz.name, index)
|
| 179 |
+
title = f"StormScope {mode_title(meta['mode'])} {region.title} {product.title}"
|
| 180 |
+
subtitle_left = f"Init {dt_label(meta['init_time'])} {frame_number(npz.name, index)} ({lead})"
|
| 181 |
+
if valid:
|
| 182 |
+
subtitle_left += f" valid {dt_label(valid)}"
|
| 183 |
+
subtitle_right = "rustwx-render / rustwx-stormscope"
|
| 184 |
+
|
| 185 |
+
cmd = [
|
| 186 |
+
str(renderer),
|
| 187 |
+
"--input", str(npz),
|
| 188 |
+
"--output", str(output),
|
| 189 |
+
"--product", product.renderer_key,
|
| 190 |
+
"--title", title,
|
| 191 |
+
"--subtitle-left", subtitle_left,
|
| 192 |
+
"--subtitle-right", subtitle_right,
|
| 193 |
+
"--width", str(width),
|
| 194 |
+
"--height", str(height),
|
| 195 |
+
f"--west={region.west}",
|
| 196 |
+
f"--east={region.east}",
|
| 197 |
+
f"--south={region.south}",
|
| 198 |
+
f"--north={region.north}",
|
| 199 |
+
f"--bounds-pad-frac={region.pad_frac}",
|
| 200 |
+
]
|
| 201 |
+
env = os.environ.copy()
|
| 202 |
+
vendor_assets = repo_root() / "rust" / "vendor" / "rustwx" / "assets"
|
| 203 |
+
if vendor_assets.exists():
|
| 204 |
+
env.setdefault("RUSTWX_ASSETS_DIR", str(vendor_assets))
|
| 205 |
+
subprocess.run(cmd, check=True, env=env)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def make_gif(frames: list[Path], gif_path: Path, duration_ms: int, force: bool) -> None:
|
| 209 |
+
if not frames:
|
| 210 |
+
return
|
| 211 |
+
if not should_update(gif_path, frames, force):
|
| 212 |
+
return
|
| 213 |
+
gif_path.parent.mkdir(parents=True, exist_ok=True)
|
| 214 |
+
images = []
|
| 215 |
+
for frame in frames:
|
| 216 |
+
try:
|
| 217 |
+
if not frame.exists() or frame.stat().st_size == 0:
|
| 218 |
+
frame.unlink(missing_ok=True)
|
| 219 |
+
continue
|
| 220 |
+
image = Image.open(frame)
|
| 221 |
+
image.load()
|
| 222 |
+
images.append(image.convert("RGB"))
|
| 223 |
+
image.close()
|
| 224 |
+
except Exception:
|
| 225 |
+
frame.unlink(missing_ok=True)
|
| 226 |
+
if not images:
|
| 227 |
+
return
|
| 228 |
+
try:
|
| 229 |
+
images[0].save(
|
| 230 |
+
gif_path,
|
| 231 |
+
save_all=True,
|
| 232 |
+
append_images=images[1:],
|
| 233 |
+
duration=duration_ms,
|
| 234 |
+
loop=0,
|
| 235 |
+
optimize=False,
|
| 236 |
+
)
|
| 237 |
+
finally:
|
| 238 |
+
for image in images:
|
| 239 |
+
image.close()
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def latest_meta_paths(meta_paths: list[Path], max_per_mode: int) -> list[Path]:
|
| 243 |
+
if max_per_mode <= 0:
|
| 244 |
+
return meta_paths
|
| 245 |
+
buckets: dict[str, list[tuple[datetime, Path]]] = {}
|
| 246 |
+
for path in meta_paths:
|
| 247 |
+
try:
|
| 248 |
+
meta = read_json(path)
|
| 249 |
+
key = display_mode(meta["mode"])
|
| 250 |
+
init = parse_dt(meta["init_time"])
|
| 251 |
+
except Exception:
|
| 252 |
+
continue
|
| 253 |
+
buckets.setdefault(key, []).append((init, path))
|
| 254 |
+
selected = []
|
| 255 |
+
for entries in buckets.values():
|
| 256 |
+
selected.extend(path for _, path in sorted(entries, reverse=True)[:max_per_mode])
|
| 257 |
+
return sorted(set(selected))
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def render_cycle(meta_path: Path, outdir: Path, site_dir: Path, renderer: Path,
|
| 261 |
+
regions: list[Region],
|
| 262 |
+
width: int, height: int, force: bool) -> dict:
|
| 263 |
+
meta = read_json(meta_path)
|
| 264 |
+
mode = display_mode(meta["mode"])
|
| 265 |
+
cycle = meta["cycle"]
|
| 266 |
+
products_out = []
|
| 267 |
+
fields = set(meta.get("fields", []))
|
| 268 |
+
|
| 269 |
+
for region in regions:
|
| 270 |
+
cycle_dir = site_dir / "assets" / region.slug / mode / cycle
|
| 271 |
+
for product in PRODUCTS:
|
| 272 |
+
if product.field not in fields:
|
| 273 |
+
continue
|
| 274 |
+
frame_paths = []
|
| 275 |
+
png_dir = cycle_dir / product.slug / "frames"
|
| 276 |
+
files = meta.get("files", [])
|
| 277 |
+
frame_indices = visible_frame_indices(meta)
|
| 278 |
+
for index in frame_indices:
|
| 279 |
+
filename = files[index]
|
| 280 |
+
npz = outdir / filename
|
| 281 |
+
if not npz.exists():
|
| 282 |
+
continue
|
| 283 |
+
png = png_dir / f"{Path(filename).stem}_{region.slug}_{product.slug}.png"
|
| 284 |
+
render_png(renderer, npz, png, product, region, meta, index, width, height, force)
|
| 285 |
+
frame_paths.append(png)
|
| 286 |
+
gif_path = cycle_dir / f"{product.slug}.gif"
|
| 287 |
+
make_gif(frame_paths, gif_path, product.duration_ms, force)
|
| 288 |
+
products_out.append({
|
| 289 |
+
"slug": product.slug,
|
| 290 |
+
"title": product.title,
|
| 291 |
+
"region_slug": region.slug,
|
| 292 |
+
"region_title": region.title,
|
| 293 |
+
"gif": gif_path,
|
| 294 |
+
"frames": frame_paths,
|
| 295 |
+
"n_frames": len(frame_paths),
|
| 296 |
+
})
|
| 297 |
+
|
| 298 |
+
return {
|
| 299 |
+
"mode": mode,
|
| 300 |
+
"source_mode": meta["mode"],
|
| 301 |
+
"cycle": cycle,
|
| 302 |
+
"init_time": meta["init_time"],
|
| 303 |
+
"valid_times": [
|
| 304 |
+
meta.get("valid_times", [])[i]
|
| 305 |
+
for i in visible_frame_indices(meta)
|
| 306 |
+
if i < len(meta.get("valid_times", []))
|
| 307 |
+
],
|
| 308 |
+
"step_minutes": meta.get("step_minutes"),
|
| 309 |
+
"resolution": meta.get("resolution"),
|
| 310 |
+
"products": products_out,
|
| 311 |
+
"meta_path": meta_path,
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def latest_by_mode(cycles: list[dict]) -> list[dict]:
|
| 316 |
+
latest = {}
|
| 317 |
+
for cycle in cycles:
|
| 318 |
+
key = cycle["mode"]
|
| 319 |
+
if key not in latest or parse_dt(cycle["init_time"]) > parse_dt(latest[key]["init_time"]):
|
| 320 |
+
latest[key] = cycle
|
| 321 |
+
return [latest[key] for key in ("nowcast", "nearcast") if key in latest]
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def write_site(site_dir: Path, outdir: Path, cycles: list[dict]) -> None:
|
| 325 |
+
site_dir.mkdir(parents=True, exist_ok=True)
|
| 326 |
+
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ")
|
| 327 |
+
service_status = {}
|
| 328 |
+
status_path = outdir / "status.json"
|
| 329 |
+
if status_path.exists():
|
| 330 |
+
try:
|
| 331 |
+
service_status = read_json(status_path)
|
| 332 |
+
except Exception:
|
| 333 |
+
service_status = {"state": "status unreadable"}
|
| 334 |
+
|
| 335 |
+
manifest_cycles = []
|
| 336 |
+
for cycle in cycles:
|
| 337 |
+
manifest_products = []
|
| 338 |
+
for product in cycle["products"]:
|
| 339 |
+
manifest_products.append({
|
| 340 |
+
"slug": product["slug"],
|
| 341 |
+
"title": product["title"],
|
| 342 |
+
"region_slug": product["region_slug"],
|
| 343 |
+
"region_title": product["region_title"],
|
| 344 |
+
"gif": relpath(product["gif"], site_dir) if product["gif"].exists() else None,
|
| 345 |
+
"frames": [relpath(frame, site_dir) for frame in product["frames"] if frame.exists()],
|
| 346 |
+
"n_frames": product["n_frames"],
|
| 347 |
+
})
|
| 348 |
+
manifest_cycles.append({**{k: cycle[k] for k in (
|
| 349 |
+
"mode", "source_mode", "cycle", "init_time", "valid_times", "step_minutes", "resolution"
|
| 350 |
+
)}, "products": manifest_products})
|
| 351 |
+
|
| 352 |
+
(site_dir / "manifest.json").write_text(json.dumps({
|
| 353 |
+
"updated": now,
|
| 354 |
+
"service_status": service_status,
|
| 355 |
+
"cycles": manifest_cycles,
|
| 356 |
+
}, indent=2))
|
| 357 |
+
|
| 358 |
+
cards = []
|
| 359 |
+
for cycle in latest_by_mode(cycles):
|
| 360 |
+
product_cards = []
|
| 361 |
+
for product in cycle["products"]:
|
| 362 |
+
if not product["gif"].exists():
|
| 363 |
+
continue
|
| 364 |
+
gif_rel = relpath(product["gif"], site_dir)
|
| 365 |
+
download_name = f"stormscope_{cycle['mode']}_{cycle['cycle']}_{product['region_slug']}_{product['slug']}.gif"
|
| 366 |
+
product_cards.append(f"""
|
| 367 |
+
<article class="product">
|
| 368 |
+
<div class="product-head">
|
| 369 |
+
<div>
|
| 370 |
+
<h3>{html_escape(product["title"])}</h3>
|
| 371 |
+
<span>{html_escape(product["region_title"])} | {product["n_frames"]} frames</span>
|
| 372 |
+
</div>
|
| 373 |
+
<a class="export" href="{gif_rel}" download="{html_escape(download_name)}">Export GIF</a>
|
| 374 |
+
</div>
|
| 375 |
+
<img src="{gif_rel}" alt="{html_escape(cycle["mode"])} {html_escape(product["region_title"])} {html_escape(product["title"])} animation">
|
| 376 |
+
</article>""")
|
| 377 |
+
valid_times = cycle.get("valid_times") or []
|
| 378 |
+
valid_span = ""
|
| 379 |
+
if valid_times:
|
| 380 |
+
valid_span = f"{dt_label(valid_times[0])} to {dt_label(valid_times[-1])}"
|
| 381 |
+
cards.append(f"""
|
| 382 |
+
<section class="cycle {cycle["mode"]}">
|
| 383 |
+
<div class="cycle-head">
|
| 384 |
+
<div>
|
| 385 |
+
<p class="kicker">{html_escape(cycle["mode"])}</p>
|
| 386 |
+
<h2>{html_escape(cycle["resolution"])} cycle {html_escape(cycle["cycle"])}</h2>
|
| 387 |
+
</div>
|
| 388 |
+
<dl>
|
| 389 |
+
<div><dt>Init</dt><dd>{dt_label(cycle["init_time"])}</dd></div>
|
| 390 |
+
<div><dt>Valid</dt><dd>{html_escape(valid_span)}</dd></div>
|
| 391 |
+
<div><dt>Step</dt><dd>{cycle["step_minutes"]} min</dd></div>
|
| 392 |
+
</dl>
|
| 393 |
+
</div>
|
| 394 |
+
<div class="products">
|
| 395 |
+
{"".join(product_cards) if product_cards else "<p>No rendered products yet.</p>"}
|
| 396 |
+
</div>
|
| 397 |
+
</section>""")
|
| 398 |
+
|
| 399 |
+
state = service_status.get("state", "unknown")
|
| 400 |
+
last_success = service_status.get("last_success", "")
|
| 401 |
+
next_run = service_status.get("next_run", "")
|
| 402 |
+
active_modes = {cycle["mode"] for cycle in latest_by_mode(cycles)}
|
| 403 |
+
if active_modes == {"nowcast"}:
|
| 404 |
+
page_subtitle = "Nowcast, refreshed from the latest completed cycle."
|
| 405 |
+
elif active_modes == {"nearcast"}:
|
| 406 |
+
page_subtitle = "GFS-conditioned nearcast, refreshed from the latest completed cycle."
|
| 407 |
+
else:
|
| 408 |
+
page_subtitle = "Nowcast and GFS-conditioned nearcast, refreshed from the latest completed cycle."
|
| 409 |
+
body = "".join(cards) if cards else """
|
| 410 |
+
<section class="empty">
|
| 411 |
+
<h2>Waiting for StormScope cycles</h2>
|
| 412 |
+
<p>The renderer is ready; the first products will appear after the service writes a cycle manifest.</p>
|
| 413 |
+
</section>"""
|
| 414 |
+
|
| 415 |
+
(site_dir / "index.html").write_text(f"""<!doctype html>
|
| 416 |
+
<html lang="en">
|
| 417 |
+
<head>
|
| 418 |
+
<meta charset="utf-8">
|
| 419 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 420 |
+
<meta http-equiv="refresh" content="60">
|
| 421 |
+
<title>StormScope Operational</title>
|
| 422 |
+
<style>
|
| 423 |
+
:root {{
|
| 424 |
+
color-scheme: light;
|
| 425 |
+
--bg: #f6f7f4;
|
| 426 |
+
--ink: #1d211f;
|
| 427 |
+
--muted: #657069;
|
| 428 |
+
--line: #d8ddd7;
|
| 429 |
+
--surface: #ffffff;
|
| 430 |
+
--green: #0f766e;
|
| 431 |
+
--orange: #b45309;
|
| 432 |
+
--red: #b91c1c;
|
| 433 |
+
}}
|
| 434 |
+
* {{ box-sizing: border-box; }}
|
| 435 |
+
body {{
|
| 436 |
+
margin: 0;
|
| 437 |
+
background: var(--bg);
|
| 438 |
+
color: var(--ink);
|
| 439 |
+
font: 15px/1.45 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
| 440 |
+
}}
|
| 441 |
+
header {{
|
| 442 |
+
padding: 18px 22px 12px;
|
| 443 |
+
border-bottom: 1px solid var(--line);
|
| 444 |
+
background: var(--surface);
|
| 445 |
+
display: flex;
|
| 446 |
+
align-items: end;
|
| 447 |
+
justify-content: space-between;
|
| 448 |
+
gap: 18px;
|
| 449 |
+
flex-wrap: wrap;
|
| 450 |
+
}}
|
| 451 |
+
h1, h2, h3, p {{ margin: 0; }}
|
| 452 |
+
h1 {{ font-size: clamp(24px, 3vw, 34px); letter-spacing: 0; }}
|
| 453 |
+
.status {{
|
| 454 |
+
display: grid;
|
| 455 |
+
grid-template-columns: repeat(3, max-content);
|
| 456 |
+
gap: 8px 14px;
|
| 457 |
+
color: var(--muted);
|
| 458 |
+
font-size: 13px;
|
| 459 |
+
}}
|
| 460 |
+
.status strong {{ color: var(--ink); }}
|
| 461 |
+
main {{ padding: 18px 22px 28px; display: grid; gap: 18px; }}
|
| 462 |
+
.cycle {{
|
| 463 |
+
background: var(--surface);
|
| 464 |
+
border: 1px solid var(--line);
|
| 465 |
+
border-top: 4px solid var(--green);
|
| 466 |
+
border-radius: 8px;
|
| 467 |
+
overflow: hidden;
|
| 468 |
+
}}
|
| 469 |
+
.cycle.nearcast {{ border-top-color: var(--orange); }}
|
| 470 |
+
.cycle-head {{
|
| 471 |
+
padding: 14px 16px;
|
| 472 |
+
display: flex;
|
| 473 |
+
align-items: start;
|
| 474 |
+
justify-content: space-between;
|
| 475 |
+
gap: 16px;
|
| 476 |
+
border-bottom: 1px solid var(--line);
|
| 477 |
+
}}
|
| 478 |
+
.kicker {{
|
| 479 |
+
text-transform: uppercase;
|
| 480 |
+
color: var(--muted);
|
| 481 |
+
font-size: 12px;
|
| 482 |
+
font-weight: 700;
|
| 483 |
+
}}
|
| 484 |
+
h2 {{ font-size: 20px; letter-spacing: 0; }}
|
| 485 |
+
dl {{
|
| 486 |
+
margin: 0;
|
| 487 |
+
display: grid;
|
| 488 |
+
grid-template-columns: repeat(3, max-content);
|
| 489 |
+
gap: 10px 16px;
|
| 490 |
+
font-size: 13px;
|
| 491 |
+
color: var(--muted);
|
| 492 |
+
}}
|
| 493 |
+
dt {{ font-weight: 700; color: var(--ink); }}
|
| 494 |
+
dd {{ margin: 2px 0 0; }}
|
| 495 |
+
.products {{
|
| 496 |
+
padding: 14px;
|
| 497 |
+
display: grid;
|
| 498 |
+
grid-template-columns: repeat(auto-fit, minmax(min(100%, 520px), 1fr));
|
| 499 |
+
gap: 14px;
|
| 500 |
+
}}
|
| 501 |
+
.product {{
|
| 502 |
+
border: 1px solid var(--line);
|
| 503 |
+
border-radius: 8px;
|
| 504 |
+
background: #fbfcfa;
|
| 505 |
+
overflow: hidden;
|
| 506 |
+
}}
|
| 507 |
+
.product-head {{
|
| 508 |
+
min-height: 46px;
|
| 509 |
+
padding: 10px 12px;
|
| 510 |
+
display: flex;
|
| 511 |
+
align-items: center;
|
| 512 |
+
justify-content: space-between;
|
| 513 |
+
gap: 10px;
|
| 514 |
+
border-bottom: 1px solid var(--line);
|
| 515 |
+
}}
|
| 516 |
+
h3 {{ font-size: 15px; letter-spacing: 0; }}
|
| 517 |
+
.product-head span {{ color: var(--muted); font-size: 12px; white-space: nowrap; }}
|
| 518 |
+
.export {{
|
| 519 |
+
display: inline-flex;
|
| 520 |
+
align-items: center;
|
| 521 |
+
justify-content: center;
|
| 522 |
+
min-height: 32px;
|
| 523 |
+
padding: 0 11px;
|
| 524 |
+
border: 1px solid var(--line);
|
| 525 |
+
border-radius: 6px;
|
| 526 |
+
color: var(--ink);
|
| 527 |
+
background: #fff;
|
| 528 |
+
font-size: 13px;
|
| 529 |
+
font-weight: 700;
|
| 530 |
+
text-decoration: none;
|
| 531 |
+
white-space: nowrap;
|
| 532 |
+
}}
|
| 533 |
+
.export:hover {{ border-color: var(--green); color: var(--green); }}
|
| 534 |
+
img {{ width: 100%; height: auto; display: block; background: #fff; }}
|
| 535 |
+
.empty {{
|
| 536 |
+
background: var(--surface);
|
| 537 |
+
border: 1px solid var(--line);
|
| 538 |
+
border-radius: 8px;
|
| 539 |
+
padding: 18px;
|
| 540 |
+
}}
|
| 541 |
+
@media (max-width: 800px) {{
|
| 542 |
+
header, main {{ padding-left: 12px; padding-right: 12px; }}
|
| 543 |
+
.status, dl {{ grid-template-columns: 1fr; }}
|
| 544 |
+
.cycle-head {{ display: grid; }}
|
| 545 |
+
}}
|
| 546 |
+
</style>
|
| 547 |
+
</head>
|
| 548 |
+
<body>
|
| 549 |
+
<header>
|
| 550 |
+
<div>
|
| 551 |
+
<h1>StormScope Operational</h1>
|
| 552 |
+
<p>{html_escape(page_subtitle)}</p>
|
| 553 |
+
</div>
|
| 554 |
+
<div class="status">
|
| 555 |
+
<span>Service <strong>{html_escape(str(state))}</strong></span>
|
| 556 |
+
<span>Updated <strong>{html_escape(now)}</strong></span>
|
| 557 |
+
<span>Next <strong>{html_escape(str(next_run or last_success or "pending"))}</strong></span>
|
| 558 |
+
</div>
|
| 559 |
+
</header>
|
| 560 |
+
<main>
|
| 561 |
+
{body}
|
| 562 |
+
</main>
|
| 563 |
+
</body>
|
| 564 |
+
</html>
|
| 565 |
+
""")
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
def html_escape(value: str) -> str:
|
| 569 |
+
return (
|
| 570 |
+
value.replace("&", "&")
|
| 571 |
+
.replace("<", "<")
|
| 572 |
+
.replace(">", ">")
|
| 573 |
+
.replace('"', """)
|
| 574 |
+
)
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def mirror_latest_visuals(site_dir: Path, cycles: list[dict], visual_dir: Path | None) -> None:
|
| 578 |
+
if not visual_dir:
|
| 579 |
+
return
|
| 580 |
+
visual_dir.mkdir(parents=True, exist_ok=True)
|
| 581 |
+
for cycle in latest_by_mode(cycles):
|
| 582 |
+
for product in cycle["products"]:
|
| 583 |
+
if product["gif"].exists():
|
| 584 |
+
shutil.copy2(
|
| 585 |
+
product["gif"],
|
| 586 |
+
visual_dir / f"latest_{cycle['mode']}_{product['region_slug']}_{product['slug']}.gif",
|
| 587 |
+
)
|
| 588 |
+
frames = [frame for frame in product["frames"] if frame.exists()]
|
| 589 |
+
if frames:
|
| 590 |
+
shutil.copy2(
|
| 591 |
+
frames[-1],
|
| 592 |
+
visual_dir / f"latest_{cycle['mode']}_{product['region_slug']}_{product['slug']}_last.png",
|
| 593 |
+
)
|
| 594 |
+
index = site_dir / "index.html"
|
| 595 |
+
if index.exists():
|
| 596 |
+
shutil.copy2(index, visual_dir / "index.html")
|
| 597 |
+
|
| 598 |
+
|
| 599 |
+
def export_wxstore_cycles(cycles: list[dict], outdir: Path, args: argparse.Namespace) -> None:
|
| 600 |
+
if args.no_wxstore_export:
|
| 601 |
+
return
|
| 602 |
+
spatial_root = Path(args.wxstore_spatial_root)
|
| 603 |
+
fields = args.wxstore_fields or None
|
| 604 |
+
for cycle in cycles:
|
| 605 |
+
meta_path = Path(cycle["meta_path"])
|
| 606 |
+
try:
|
| 607 |
+
result = export_wxstore_cycle(
|
| 608 |
+
meta_path,
|
| 609 |
+
outdir,
|
| 610 |
+
spatial_root,
|
| 611 |
+
binary=args.wxstore_exporter,
|
| 612 |
+
fields=fields,
|
| 613 |
+
force=args.force,
|
| 614 |
+
)
|
| 615 |
+
if result.get("skipped_current"):
|
| 616 |
+
continue
|
| 617 |
+
print(
|
| 618 |
+
f"[ops_render] WXA exported {result.get('model')} {result.get('cycle')} "
|
| 619 |
+
f"products={result.get('products', 'unknown')}",
|
| 620 |
+
flush=True,
|
| 621 |
+
)
|
| 622 |
+
except FileNotFoundError as exc:
|
| 623 |
+
print(f"[ops_render] WXA export unavailable: {exc}", flush=True)
|
| 624 |
+
return
|
| 625 |
+
except Exception as exc:
|
| 626 |
+
print(f"[ops_render] WXA export failed for {meta_path.name}: {exc}", flush=True)
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
def render_once(args: argparse.Namespace) -> list[dict]:
|
| 630 |
+
outdir = Path(args.outdir)
|
| 631 |
+
site_dir = Path(args.site_dir)
|
| 632 |
+
renderer = Path(args.renderer)
|
| 633 |
+
if not renderer.exists():
|
| 634 |
+
raise FileNotFoundError(f"renderer binary not found: {renderer}")
|
| 635 |
+
all_meta_paths = sorted(outdir.glob("stormscope_*_*_*_meta.json"))
|
| 636 |
+
complete_meta_paths = []
|
| 637 |
+
for path in all_meta_paths:
|
| 638 |
+
try:
|
| 639 |
+
if read_json(path).get("complete", True):
|
| 640 |
+
complete_meta_paths.append(path)
|
| 641 |
+
except Exception:
|
| 642 |
+
continue
|
| 643 |
+
meta_paths = latest_meta_paths(
|
| 644 |
+
complete_meta_paths,
|
| 645 |
+
args.max_cycles_per_mode,
|
| 646 |
+
)
|
| 647 |
+
cycles = [
|
| 648 |
+
render_cycle(path, outdir, site_dir, renderer, args.region_list, args.width, args.height, args.force)
|
| 649 |
+
for path in meta_paths
|
| 650 |
+
]
|
| 651 |
+
export_wxstore_cycles(cycles, outdir, args)
|
| 652 |
+
write_site(site_dir, outdir, cycles)
|
| 653 |
+
visual_dir = Path(args.visual_dir) if args.visual_dir else None
|
| 654 |
+
mirror_latest_visuals(site_dir, cycles, visual_dir)
|
| 655 |
+
return cycles
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
def main() -> int:
|
| 659 |
+
args = parse_args()
|
| 660 |
+
while True:
|
| 661 |
+
try:
|
| 662 |
+
cycles = render_once(args)
|
| 663 |
+
print(f"[ops_render] rendered/checked {len(cycles)} cycles at {datetime.now(timezone.utc).isoformat()}", flush=True)
|
| 664 |
+
except Exception:
|
| 665 |
+
traceback.print_exc()
|
| 666 |
+
if not args.loop:
|
| 667 |
+
return 1
|
| 668 |
+
if not args.loop:
|
| 669 |
+
return 0
|
| 670 |
+
time.sleep(max(5, args.interval))
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
if __name__ == "__main__":
|
| 674 |
+
raise SystemExit(main())
|
rustwx_stormscope/optimize.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""StormScope inference optimizations, validated on RTX 5090 (sm_120).
|
| 3 |
+
|
| 4 |
+
apply_opts(model, ...) mutates an earth2studio StormScope model in place:
|
| 5 |
+
- sampler: deterministic EDM (S_churn=0) + reduced num_steps [validated near-lossless vs
|
| 6 |
+
real obs: det-32 corr 0.9591 vs stoch-100 0.9610; det ODE error 0.088% @32 / 0.13% @24]
|
| 7 |
+
- torch.compile(default) on each diffusion expert network [-26%, 0.1% numerical]
|
| 8 |
+
- optional channels_last (marginal once compiled)
|
| 9 |
+
|
| 10 |
+
Defaults chosen as the validated-safe operating point. Set num_steps=24 for ~5.8x (still skillful),
|
| 11 |
+
or 18 for more speed with a small additional skill cost.
|
| 12 |
+
"""
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
# DiT linears safe to quantize: attention proj + MLP. NOTE the MLP is DiTBlock.linear (a timm Mlp)
|
| 16 |
+
# whose linears are `.layers.0` / `.layers.2` (NOT fc1/fc2). Keep adaLN modulation/embed/output bf16.
|
| 17 |
+
FP8_TARGETS = ["attention.qkv", "attention.proj", "linear.layers.0", "linear.layers.2"]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _mxfp8_quantize(net):
|
| 21 |
+
"""Cast net to bf16 (torchao MX needs bf16 weights) and MXFP8-quantize the DiT attn/MLP linears.
|
| 22 |
+
Gives ~1.29x on the network forward under torch.compile (fused quant), ~0.3% vs bf16. RTX 5090."""
|
| 23 |
+
import torch.nn as nn
|
| 24 |
+
from torchao.quantization import quantize_
|
| 25 |
+
from torchao.prototype.mx_formats import MXDynamicActivationMXWeightConfig
|
| 26 |
+
net = net.to(torch.bfloat16)
|
| 27 |
+
|
| 28 |
+
def filt(mod, fqn):
|
| 29 |
+
return isinstance(mod, nn.Linear) and any(s in fqn for s in FP8_TARGETS)
|
| 30 |
+
quantize_(net, MXDynamicActivationMXWeightConfig(), filter_fn=filt)
|
| 31 |
+
return net
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def apply_opts(model, num_steps=32, deterministic=True, compile_net=True,
|
| 35 |
+
channels_last=False, compile_mode="default", fp8=False, mxfp8=False, verbose=True):
|
| 36 |
+
if num_steps:
|
| 37 |
+
model.sampler_args["num_steps"] = int(num_steps)
|
| 38 |
+
if deterministic:
|
| 39 |
+
model.sampler_args["S_churn"] = 0.0
|
| 40 |
+
nets = model.stage_models
|
| 41 |
+
n_fp8 = 0
|
| 42 |
+
for i in range(len(nets)):
|
| 43 |
+
net = nets[i]
|
| 44 |
+
if mxfp8:
|
| 45 |
+
net = _mxfp8_quantize(net)
|
| 46 |
+
elif fp8:
|
| 47 |
+
from .fp8_linear import swap_linears
|
| 48 |
+
n_fp8 += swap_linears(net, FP8_TARGETS)
|
| 49 |
+
if channels_last:
|
| 50 |
+
net = net.to(memory_format=torch.channels_last)
|
| 51 |
+
if compile_net:
|
| 52 |
+
net = torch.compile(net, mode=compile_mode, dynamic=False)
|
| 53 |
+
nets[i] = net
|
| 54 |
+
if verbose:
|
| 55 |
+
print(f"[ss_optimize] sampler_args={model.sampler_args} compile={compile_net}"
|
| 56 |
+
f" mxfp8={mxfp8} channels_last={channels_last} fp8_linears={n_fp8}", flush=True)
|
| 57 |
+
return model
|
rustwx_stormscope/output_validation.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""Validate StormScope cycle metadata and .npz frame products."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
from dataclasses import asdict, dataclass, field
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
|
| 14 |
+
from .paths import default_outdir
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
EXPECTED_2D_FIELDS = [
|
| 18 |
+
"ch_blue_047",
|
| 19 |
+
"ch_red_064",
|
| 20 |
+
"ch_veggie_086",
|
| 21 |
+
"ch_swir_390",
|
| 22 |
+
"ch_wv_upper_619",
|
| 23 |
+
"ch_wv_mid_695",
|
| 24 |
+
"ch_wv_lower_734",
|
| 25 |
+
"ch_ir_1035",
|
| 26 |
+
"refc",
|
| 27 |
+
"lats",
|
| 28 |
+
"lons",
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
EXPECTED_PRODUCT_FIELDS = [field for field in EXPECTED_2D_FIELDS if field not in {"lats", "lons"}]
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass
|
| 35 |
+
class FieldSummary:
|
| 36 |
+
name: str
|
| 37 |
+
shape: list[int]
|
| 38 |
+
dtype: str
|
| 39 |
+
finite_count: int
|
| 40 |
+
min: float | None
|
| 41 |
+
max: float | None
|
| 42 |
+
mean: float | None
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class ValidationReport:
|
| 47 |
+
ok: bool
|
| 48 |
+
meta_path: str
|
| 49 |
+
outdir: str
|
| 50 |
+
cycle: str | None = None
|
| 51 |
+
mode: str | None = None
|
| 52 |
+
resolution: str | None = None
|
| 53 |
+
frame_count: int = 0
|
| 54 |
+
errors: list[str] = field(default_factory=list)
|
| 55 |
+
warnings: list[str] = field(default_factory=list)
|
| 56 |
+
fields: dict[str, FieldSummary] = field(default_factory=dict)
|
| 57 |
+
|
| 58 |
+
def add_error(self, message: str) -> None:
|
| 59 |
+
self.errors.append(message)
|
| 60 |
+
self.ok = False
|
| 61 |
+
|
| 62 |
+
def add_warning(self, message: str) -> None:
|
| 63 |
+
self.warnings.append(message)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def read_json(path: Path) -> dict[str, Any]:
|
| 67 |
+
return json.loads(path.read_text())
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def latest_meta(outdir: Path) -> Path:
|
| 71 |
+
candidates = sorted(outdir.glob("stormscope_*_*_*_meta.json"), key=lambda p: p.stat().st_mtime)
|
| 72 |
+
for path in reversed(candidates):
|
| 73 |
+
try:
|
| 74 |
+
meta = read_json(path)
|
| 75 |
+
except Exception:
|
| 76 |
+
continue
|
| 77 |
+
if meta.get("complete", True):
|
| 78 |
+
return path
|
| 79 |
+
raise FileNotFoundError(f"no completed StormScope cycle metadata found in {outdir}")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def scalar_string(data: np.lib.npyio.NpzFile, key: str) -> str | None:
|
| 83 |
+
if key not in data.files:
|
| 84 |
+
return None
|
| 85 |
+
value = data[key]
|
| 86 |
+
try:
|
| 87 |
+
return str(value.item())
|
| 88 |
+
except Exception:
|
| 89 |
+
return str(value)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def field_summary(name: str, values: np.ndarray) -> FieldSummary:
|
| 93 |
+
finite = np.isfinite(values)
|
| 94 |
+
finite_count = int(finite.sum())
|
| 95 |
+
if finite_count:
|
| 96 |
+
sample = values[finite].astype(np.float64)
|
| 97 |
+
return FieldSummary(
|
| 98 |
+
name=name,
|
| 99 |
+
shape=list(values.shape),
|
| 100 |
+
dtype=str(values.dtype),
|
| 101 |
+
finite_count=finite_count,
|
| 102 |
+
min=float(sample.min()),
|
| 103 |
+
max=float(sample.max()),
|
| 104 |
+
mean=float(sample.mean()),
|
| 105 |
+
)
|
| 106 |
+
return FieldSummary(
|
| 107 |
+
name=name,
|
| 108 |
+
shape=list(values.shape),
|
| 109 |
+
dtype=str(values.dtype),
|
| 110 |
+
finite_count=0,
|
| 111 |
+
min=None,
|
| 112 |
+
max=None,
|
| 113 |
+
mean=None,
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def expected_shape(meta: dict[str, Any]) -> tuple[int, int] | None:
|
| 118 |
+
shape = meta.get("grid", {}).get("shape")
|
| 119 |
+
if isinstance(shape, list) and len(shape) == 2:
|
| 120 |
+
return int(shape[0]), int(shape[1])
|
| 121 |
+
return None
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def validate_meta_contract(meta: dict[str, Any], report: ValidationReport, require_complete: bool) -> None:
|
| 125 |
+
for key in ["model", "cycle", "init_time", "mode", "resolution", "step_minutes", "files"]:
|
| 126 |
+
if key not in meta:
|
| 127 |
+
report.add_error(f"metadata missing required key {key!r}")
|
| 128 |
+
if require_complete and not meta.get("complete", False):
|
| 129 |
+
report.add_error("metadata complete flag is not true")
|
| 130 |
+
mode = meta.get("mode")
|
| 131 |
+
if mode not in {"nowcast", "forecast"}:
|
| 132 |
+
report.add_error(f"metadata mode must be nowcast or forecast, got {mode!r}")
|
| 133 |
+
if not isinstance(meta.get("files"), list) or not meta.get("files"):
|
| 134 |
+
report.add_error("metadata files must be a non-empty list")
|
| 135 |
+
valid_times = meta.get("valid_times")
|
| 136 |
+
if isinstance(valid_times, list) and isinstance(meta.get("files"), list) and len(valid_times) != len(meta["files"]):
|
| 137 |
+
report.add_error("metadata valid_times length does not match files length")
|
| 138 |
+
fields = meta.get("fields")
|
| 139 |
+
if isinstance(fields, list):
|
| 140 |
+
missing = [field for field in EXPECTED_PRODUCT_FIELDS if field not in fields]
|
| 141 |
+
if missing:
|
| 142 |
+
report.add_warning(f"metadata fields is missing expected products: {', '.join(missing)}")
|
| 143 |
+
else:
|
| 144 |
+
report.add_warning("metadata fields is absent or not a list")
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def validate_npz_frame(
|
| 148 |
+
npz_path: Path,
|
| 149 |
+
meta: dict[str, Any],
|
| 150 |
+
report: ValidationReport,
|
| 151 |
+
expected_grid_shape: tuple[int, int] | None,
|
| 152 |
+
strict_fields: bool,
|
| 153 |
+
first_frame: bool,
|
| 154 |
+
) -> None:
|
| 155 |
+
if not npz_path.exists():
|
| 156 |
+
report.add_error(f"missing frame file {npz_path}")
|
| 157 |
+
return
|
| 158 |
+
try:
|
| 159 |
+
with np.load(npz_path, allow_pickle=False) as data:
|
| 160 |
+
required = EXPECTED_2D_FIELDS if strict_fields else ["lats", "lons"]
|
| 161 |
+
for field_name in required:
|
| 162 |
+
if field_name not in data.files:
|
| 163 |
+
report.add_error(f"{npz_path.name} missing field {field_name!r}")
|
| 164 |
+
available_product_fields = [
|
| 165 |
+
field_name
|
| 166 |
+
for field_name in EXPECTED_2D_FIELDS
|
| 167 |
+
if field_name in data.files and getattr(data[field_name], "ndim", 0) == 2
|
| 168 |
+
]
|
| 169 |
+
for field_name in available_product_fields:
|
| 170 |
+
values = data[field_name]
|
| 171 |
+
if expected_grid_shape and tuple(values.shape) != expected_grid_shape:
|
| 172 |
+
report.add_error(
|
| 173 |
+
f"{npz_path.name}:{field_name} shape {values.shape} "
|
| 174 |
+
f"does not match metadata grid shape {expected_grid_shape}"
|
| 175 |
+
)
|
| 176 |
+
if values.dtype.kind not in {"f", "i", "u"}:
|
| 177 |
+
report.add_error(f"{npz_path.name}:{field_name} dtype {values.dtype} is not numeric")
|
| 178 |
+
if first_frame:
|
| 179 |
+
report.fields[field_name] = field_summary(field_name, values)
|
| 180 |
+
if "lats" in data.files and "lons" in data.files:
|
| 181 |
+
lats = data["lats"]
|
| 182 |
+
lons = data["lons"]
|
| 183 |
+
if lats.shape != lons.shape:
|
| 184 |
+
report.add_error(f"{npz_path.name}: lats/lons shape mismatch {lats.shape} vs {lons.shape}")
|
| 185 |
+
finite = np.isfinite(lats) & np.isfinite(lons)
|
| 186 |
+
if not np.any(finite):
|
| 187 |
+
report.add_error(f"{npz_path.name}: no finite lat/lon cells")
|
| 188 |
+
else:
|
| 189 |
+
lat_min = float(np.nanmin(lats[finite]))
|
| 190 |
+
lat_max = float(np.nanmax(lats[finite]))
|
| 191 |
+
lon_norm = np.where(lons[finite] > 180.0, lons[finite] - 360.0, lons[finite])
|
| 192 |
+
lon_min = float(np.nanmin(lon_norm))
|
| 193 |
+
lon_max = float(np.nanmax(lon_norm))
|
| 194 |
+
if not (-90.0 <= lat_min <= lat_max <= 90.0):
|
| 195 |
+
report.add_error(f"{npz_path.name}: latitude bounds out of range {lat_min}..{lat_max}")
|
| 196 |
+
if not (-180.0 <= lon_min <= lon_max <= 180.0):
|
| 197 |
+
report.add_error(f"{npz_path.name}: longitude bounds out of range {lon_min}..{lon_max}")
|
| 198 |
+
if meta.get("mode") and scalar_string(data, "mode") and scalar_string(data, "mode") != meta.get("mode"):
|
| 199 |
+
report.add_error(f"{npz_path.name}: scalar mode does not match metadata")
|
| 200 |
+
if meta.get("resolution") and scalar_string(data, "resolution") and scalar_string(data, "resolution") != meta.get("resolution"):
|
| 201 |
+
report.add_error(f"{npz_path.name}: scalar resolution does not match metadata")
|
| 202 |
+
except Exception as exc:
|
| 203 |
+
report.add_error(f"failed to read {npz_path}: {type(exc).__name__}: {exc}")
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def validate_cycle(
|
| 207 |
+
meta_path: Path,
|
| 208 |
+
outdir: Path | None = None,
|
| 209 |
+
*,
|
| 210 |
+
require_complete: bool = True,
|
| 211 |
+
strict_fields: bool = True,
|
| 212 |
+
) -> ValidationReport:
|
| 213 |
+
outdir = outdir or meta_path.parent
|
| 214 |
+
report = ValidationReport(ok=True, meta_path=str(meta_path), outdir=str(outdir))
|
| 215 |
+
try:
|
| 216 |
+
meta = read_json(meta_path)
|
| 217 |
+
except Exception as exc:
|
| 218 |
+
report.add_error(f"failed to read metadata: {type(exc).__name__}: {exc}")
|
| 219 |
+
return report
|
| 220 |
+
|
| 221 |
+
report.cycle = meta.get("cycle")
|
| 222 |
+
report.mode = meta.get("mode")
|
| 223 |
+
report.resolution = meta.get("resolution")
|
| 224 |
+
files = meta.get("files") if isinstance(meta.get("files"), list) else []
|
| 225 |
+
report.frame_count = len(files)
|
| 226 |
+
validate_meta_contract(meta, report, require_complete=require_complete)
|
| 227 |
+
|
| 228 |
+
grid_shape = expected_shape(meta)
|
| 229 |
+
if grid_shape is None:
|
| 230 |
+
report.add_warning("metadata grid.shape missing; frame shape checks will be weaker")
|
| 231 |
+
for idx, filename in enumerate(files):
|
| 232 |
+
validate_npz_frame(
|
| 233 |
+
outdir / filename,
|
| 234 |
+
meta,
|
| 235 |
+
report,
|
| 236 |
+
grid_shape,
|
| 237 |
+
strict_fields=strict_fields,
|
| 238 |
+
first_frame=idx == 0,
|
| 239 |
+
)
|
| 240 |
+
return report
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def parse_args() -> argparse.Namespace:
|
| 244 |
+
parser = argparse.ArgumentParser(description="Validate StormScope .npz cycle outputs")
|
| 245 |
+
parser.add_argument("--meta", default="", help="cycle metadata JSON; defaults to latest completed cycle")
|
| 246 |
+
parser.add_argument("--outdir", default=str(default_outdir()))
|
| 247 |
+
parser.add_argument("--allow-partial", action="store_true", help="do not require complete=true")
|
| 248 |
+
parser.add_argument("--relaxed-fields", action="store_true", help="only require lats/lons, not all StormScope fields")
|
| 249 |
+
parser.add_argument("--json", action="store_true", help="print JSON report")
|
| 250 |
+
return parser.parse_args()
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def main() -> int:
|
| 254 |
+
args = parse_args()
|
| 255 |
+
outdir = Path(args.outdir)
|
| 256 |
+
meta = Path(args.meta) if args.meta else latest_meta(outdir)
|
| 257 |
+
report = validate_cycle(
|
| 258 |
+
meta,
|
| 259 |
+
outdir,
|
| 260 |
+
require_complete=not args.allow_partial,
|
| 261 |
+
strict_fields=not args.relaxed_fields,
|
| 262 |
+
)
|
| 263 |
+
data = asdict(report)
|
| 264 |
+
if args.json:
|
| 265 |
+
print(json.dumps(data, indent=2))
|
| 266 |
+
else:
|
| 267 |
+
status = "OK" if report.ok else "FAIL"
|
| 268 |
+
print(f"{status} {report.mode or '?'} {report.cycle or meta.name} frames={report.frame_count}")
|
| 269 |
+
for error in report.errors:
|
| 270 |
+
print(f"ERROR {error}")
|
| 271 |
+
for warning in report.warnings:
|
| 272 |
+
print(f"WARN {warning}")
|
| 273 |
+
if report.ok:
|
| 274 |
+
for summary in report.fields.values():
|
| 275 |
+
print(
|
| 276 |
+
f"FIELD {summary.name:16s} shape={summary.shape} dtype={summary.dtype} "
|
| 277 |
+
f"finite={summary.finite_count} min={summary.min} max={summary.max}"
|
| 278 |
+
)
|
| 279 |
+
return 0 if report.ok else 1
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
if __name__ == "__main__":
|
| 283 |
+
raise SystemExit(main())
|
rustwx_stormscope/paths.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Default local paths for rustwx-stormscope products and caches."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def env_value(name: str, legacy_name: str | None = None, default: str | Path | None = None) -> str:
|
| 10 |
+
if name in os.environ:
|
| 11 |
+
return os.environ[name]
|
| 12 |
+
if legacy_name and legacy_name in os.environ:
|
| 13 |
+
return os.environ[legacy_name]
|
| 14 |
+
if default is None:
|
| 15 |
+
return ""
|
| 16 |
+
return str(default)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def repo_root() -> Path:
|
| 20 |
+
return Path(__file__).resolve().parents[1]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def wxstore_root() -> Path:
|
| 24 |
+
return Path(
|
| 25 |
+
env_value(
|
| 26 |
+
"RUSTWX_STORMSCOPE_WXSTORE_ROOT",
|
| 27 |
+
"SSFAST_WXSTORE_ROOT",
|
| 28 |
+
repo_root() / "volume" / "wxstores" / "stormscope",
|
| 29 |
+
)
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def default_outdir() -> Path:
|
| 34 |
+
return Path(env_value("RUSTWX_STORMSCOPE_OUT", "SSFAST_OUT", wxstore_root() / "out" / "operational"))
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def default_site_dir() -> Path:
|
| 38 |
+
return Path(env_value("RUSTWX_STORMSCOPE_SITE", "SSFAST_SITE", wxstore_root() / "site"))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def default_exports_dir() -> Path:
|
| 42 |
+
return Path(env_value("RUSTWX_STORMSCOPE_EXPORTS", "SSFAST_EXPORTS", wxstore_root() / "exports"))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def default_spatial_root() -> Path:
|
| 46 |
+
return Path(env_value("RUSTWX_STORMSCOPE_SPATIAL_ROOT", "SSFAST_SPATIAL_ROOT", wxstore_root() / "spatial"))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def default_webmap_dir() -> Path:
|
| 50 |
+
return Path(env_value("RUSTWX_STORMSCOPE_WEBMAP", "SSFAST_WEBMAP", wxstore_root() / "webmap_preview"))
|
rustwx_stormscope/run.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""One-shot optimized StormScope runner (drop-in for run_stormscope.py).
|
| 3 |
+
|
| 4 |
+
Wraps earth2studio's StormScope load_model to apply the validated optimizations
|
| 5 |
+
(deterministic EDM + reduced num_steps + torch.compile + MXFP8), selects the data
|
| 6 |
+
backend (earth2studio or rustwx), then runs the core forecast pipeline.
|
| 7 |
+
|
| 8 |
+
CLI:
|
| 9 |
+
python -m rustwx_stormscope.run --mode nowcast --steps 6 --num-steps 24 --mxfp8 \
|
| 10 |
+
--backend rust --outdir volume/wxstores/stormscope/out/operational \
|
| 11 |
+
[--date 2023-12-05T12:00:00]
|
| 12 |
+
"""
|
| 13 |
+
import argparse
|
| 14 |
+
|
| 15 |
+
from .paths import default_outdir
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _wrap_load(cls, cfg):
|
| 19 |
+
if getattr(cls, "_rustwx_stormscope_wrapped", False):
|
| 20 |
+
cls._rustwx_stormscope_cfg.update(cfg) # update config in place; don't stack wrappers
|
| 21 |
+
return
|
| 22 |
+
orig = cls.load_model.__func__
|
| 23 |
+
cls._rustwx_stormscope_cfg = dict(cfg)
|
| 24 |
+
|
| 25 |
+
def wrapped(klass, *a, **k):
|
| 26 |
+
from .optimize import apply_opts
|
| 27 |
+
|
| 28 |
+
model = orig(klass, *a, **k)
|
| 29 |
+
c = klass._rustwx_stormscope_cfg
|
| 30 |
+
apply_opts(model, num_steps=c["num_steps"], deterministic=c["deterministic"],
|
| 31 |
+
compile_net=c["compile"], mxfp8=c["mxfp8"], fp8=c["fp8"])
|
| 32 |
+
return model
|
| 33 |
+
cls.load_model = classmethod(wrapped)
|
| 34 |
+
cls._rustwx_stormscope_wrapped = True
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def run_cycle(mode, res, steps, date, outdir, cfg):
|
| 38 |
+
"""Run one forecast cycle with optimizations + selected data backend. Returns saved output paths."""
|
| 39 |
+
from . import core
|
| 40 |
+
from .datasources import install_backend
|
| 41 |
+
from earth2studio.models.px.stormscope import StormScopeGOES, StormScopeMRMS
|
| 42 |
+
|
| 43 |
+
_wrap_load(StormScopeGOES, cfg)
|
| 44 |
+
_wrap_load(StormScopeMRMS, cfg)
|
| 45 |
+
install_backend(core, cfg["backend"]) # swaps core's GOES/MRMS data sources if backend=='rust'
|
| 46 |
+
return core.run_stormscope(date, steps, outdir, mode, res, use_bf16=not cfg["fp32"])
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def build_cfg(a):
|
| 50 |
+
return {"num_steps": a.num_steps, "deterministic": not a.stochastic, "compile": not a.no_compile,
|
| 51 |
+
"mxfp8": a.mxfp8, "fp8": a.fp8, "fp32": a.fp32, "backend": a.backend}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def add_common_args(ap):
|
| 55 |
+
ap.add_argument("--mode", default="nowcast", choices=["nowcast", "forecast"])
|
| 56 |
+
ap.add_argument("--res", default="6km", choices=["6km", "3km"])
|
| 57 |
+
ap.add_argument("--steps", type=int, default=6)
|
| 58 |
+
ap.add_argument("--num-steps", type=int, default=24, help="EDM sampler steps (24 rec, 32 conservative)")
|
| 59 |
+
ap.add_argument("--stochastic", action="store_true", help="keep S_churn>0 (default deterministic)")
|
| 60 |
+
ap.add_argument("--no-compile", action="store_true")
|
| 61 |
+
ap.add_argument("--mxfp8", action="store_true", help="MXFP8 on DiT linears (Blackwell)")
|
| 62 |
+
ap.add_argument("--fp8", action="store_true", help="(deprecated) hand-rolled rowwise FP8 (slower)")
|
| 63 |
+
ap.add_argument("--fp32", action="store_true")
|
| 64 |
+
ap.add_argument("--backend", default="rust", choices=["earth2studio", "rust"],
|
| 65 |
+
help="GOES/MRMS data source: rust (fast parallel fetch, default; auto-falls back "
|
| 66 |
+
"to earth2studio if the extension isn't installed) or earth2studio")
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def main():
|
| 70 |
+
ap = argparse.ArgumentParser(description="Optimized StormScope (consumer Blackwell)")
|
| 71 |
+
add_common_args(ap)
|
| 72 |
+
ap.add_argument("--date", default=None)
|
| 73 |
+
ap.add_argument("--outdir", default=str(default_outdir()))
|
| 74 |
+
a = ap.parse_args()
|
| 75 |
+
|
| 76 |
+
from . import core
|
| 77 |
+
import torch
|
| 78 |
+
|
| 79 |
+
cfg = build_cfg(a)
|
| 80 |
+
|
| 81 |
+
c = core.MODEL_CONFIGS[(a.mode, a.res)]
|
| 82 |
+
if a.date is None:
|
| 83 |
+
date = core.get_latest_init_time(
|
| 84 |
+
c["step_minutes"],
|
| 85 |
+
"6steps" in c["goes"],
|
| 86 |
+
gfs_aligned=c.get("needs_gfs", False)
|
| 87 |
+
and core.forecast_conditioning_kind(c) == "gfs",
|
| 88 |
+
)
|
| 89 |
+
print(f"Using latest init time: {date}Z")
|
| 90 |
+
else:
|
| 91 |
+
date = a.date
|
| 92 |
+
print(f"StormScope-FAST on {torch.cuda.get_device_name()} | mode={a.mode} num_steps={a.num_steps} "
|
| 93 |
+
f"deterministic={not a.stochastic} compile={not a.no_compile} mxfp8={a.mxfp8} backend={a.backend}")
|
| 94 |
+
run_cycle(a.mode, a.res, a.steps, date, a.outdir, cfg)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
if __name__ == "__main__":
|
| 98 |
+
main()
|
rustwx_stormscope/rust_datasources.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""Fast GOES/MRMS inputs for StormScope, backed by the rust `stormscope_obs` extension.
|
| 3 |
+
|
| 4 |
+
- GOES: rust parallel-downloads the MCMIPC files; we decode the 8 StormScope channels with
|
| 5 |
+
h5netcdf (fast, proven). lat/lon come from earth2studio's GOES.grid() (same ABI fixed grid).
|
| 6 |
+
- MRMS: rust parallel-downloads + decodes the GRIB2 composite (pure rust).
|
| 7 |
+
|
| 8 |
+
`goes_input` / `mrms_input` return (tensor, coords) shaped exactly like earth2studio's fetch_data
|
| 9 |
+
output, so StormScope's interpolator/sampler behave identically. The download - the operational
|
| 10 |
+
bottleneck - is parallelized; the model runs unchanged.
|
| 11 |
+
"""
|
| 12 |
+
import os
|
| 13 |
+
import time
|
| 14 |
+
from collections import OrderedDict
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
import torch
|
| 19 |
+
|
| 20 |
+
_CMI = {"abi01c": "CMI_C01", "abi02c": "CMI_C02", "abi03c": "CMI_C03", "abi07c": "CMI_C07",
|
| 21 |
+
"abi08c": "CMI_C08", "abi09c": "CMI_C09", "abi10c": "CMI_C10", "abi13c": "CMI_C13"}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _env(name, legacy_name=None, default=None):
|
| 25 |
+
if name in os.environ:
|
| 26 |
+
return os.environ[name]
|
| 27 |
+
if legacy_name and legacy_name in os.environ:
|
| 28 |
+
return os.environ[legacy_name]
|
| 29 |
+
return default
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _cache_dir(kind):
|
| 33 |
+
root = _env("RUSTWX_STORMSCOPE_CACHE_DIR", "SSFAST_CACHE_DIR")
|
| 34 |
+
if root:
|
| 35 |
+
return os.path.join(root, kind)
|
| 36 |
+
return os.path.join(".", "cache", kind)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _iso_times(start_date, lead_times):
|
| 40 |
+
base = np.datetime64(start_date[0])
|
| 41 |
+
return [str((base + np.timedelta64(lt)).astype("datetime64[s]")).replace(" ", "T")
|
| 42 |
+
for lt in lead_times]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _goes_decode_cache_path(path, variables):
|
| 46 |
+
cache = Path(_env("RUSTWX_STORMSCOPE_GOES_DECODE_CACHE", "SSFAST_GOES_DECODE_CACHE", _cache_dir("goes_decoded")))
|
| 47 |
+
var_key = "-".join(str(v) for v in variables)
|
| 48 |
+
return cache / f"{Path(path).name}.{var_key}.f32.npy"
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _prune_decode_cache(cache_dir: Path):
|
| 52 |
+
max_files = int(_env("RUSTWX_STORMSCOPE_GOES_DECODE_CACHE_MAX_FILES", "SSFAST_GOES_DECODE_CACHE_MAX_FILES", "96"))
|
| 53 |
+
if max_files <= 0 or not cache_dir.exists():
|
| 54 |
+
return
|
| 55 |
+
now = time.time()
|
| 56 |
+
# Keep the common case cheap; pruning every call is fine because the cache is small.
|
| 57 |
+
files = sorted(
|
| 58 |
+
(p for p in cache_dir.glob("*.npy") if p.is_file()),
|
| 59 |
+
key=lambda p: p.stat().st_mtime,
|
| 60 |
+
reverse=True,
|
| 61 |
+
)
|
| 62 |
+
for p in files[max_files:]:
|
| 63 |
+
try:
|
| 64 |
+
p.unlink()
|
| 65 |
+
except OSError:
|
| 66 |
+
pass
|
| 67 |
+
max_age_hours = float(_env("RUSTWX_STORMSCOPE_GOES_DECODE_CACHE_MAX_AGE_HOURS", "SSFAST_GOES_DECODE_CACHE_MAX_AGE_HOURS", "18"))
|
| 68 |
+
if max_age_hours > 0:
|
| 69 |
+
cutoff = now - max_age_hours * 3600.0
|
| 70 |
+
for p in files[:max_files]:
|
| 71 |
+
try:
|
| 72 |
+
if p.stat().st_mtime < cutoff:
|
| 73 |
+
p.unlink()
|
| 74 |
+
except OSError:
|
| 75 |
+
pass
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _read_goes_frame(path, variables):
|
| 79 |
+
import xarray as xr
|
| 80 |
+
|
| 81 |
+
cache_path = _goes_decode_cache_path(path, variables)
|
| 82 |
+
try:
|
| 83 |
+
if cache_path.exists() and cache_path.stat().st_mtime >= Path(path).stat().st_mtime:
|
| 84 |
+
return np.load(cache_path)
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
|
| 88 |
+
ds = xr.open_dataset(path, engine="h5netcdf") # mask_and_scale applies scale/offset -> physical
|
| 89 |
+
try:
|
| 90 |
+
frame = np.stack(
|
| 91 |
+
[np.asarray(ds[_CMI[str(v)]].values, dtype=np.float32) for v in variables],
|
| 92 |
+
axis=0,
|
| 93 |
+
)
|
| 94 |
+
finally:
|
| 95 |
+
ds.close()
|
| 96 |
+
|
| 97 |
+
try:
|
| 98 |
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
| 99 |
+
tmp = cache_path.with_suffix(cache_path.suffix + ".tmp")
|
| 100 |
+
with open(tmp, "wb") as f:
|
| 101 |
+
np.save(f, frame)
|
| 102 |
+
os.replace(tmp, cache_path)
|
| 103 |
+
_prune_decode_cache(cache_path.parent)
|
| 104 |
+
except Exception:
|
| 105 |
+
pass
|
| 106 |
+
return frame
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def goes_input(satellite, scan_mode, start_date, variables, lead_times, device, hw):
|
| 110 |
+
"""Returns (x[1,L,C,H,W] float32 tensor, coords OrderedDict time/lead_time/variable/y/x)."""
|
| 111 |
+
import stormscope_obs
|
| 112 |
+
cache = _env("RUSTWX_STORMSCOPE_GOES_CACHE", "SSFAST_GOES_CACHE", _cache_dir("goes_nc"))
|
| 113 |
+
times = _iso_times(start_date, lead_times)
|
| 114 |
+
paths = stormscope_obs.download_goes_files(satellite, times, cache)
|
| 115 |
+
frames = [_read_goes_frame(p, variables) for p in paths] # [C, H, W]
|
| 116 |
+
data = np.stack(frames, axis=0)[None] # [1, L, C, H, W]
|
| 117 |
+
H, W = hw
|
| 118 |
+
coords = OrderedDict([
|
| 119 |
+
("time", np.asarray(start_date)),
|
| 120 |
+
("lead_time", np.asarray(lead_times)),
|
| 121 |
+
("variable", np.asarray(list(variables))),
|
| 122 |
+
("y", np.arange(H)),
|
| 123 |
+
("x", np.arange(W)),
|
| 124 |
+
])
|
| 125 |
+
return torch.from_numpy(np.ascontiguousarray(data)).to(device), coords
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def mrms_input(start_date, lead_times, device):
|
| 129 |
+
"""Returns (x[1,L,1,H,W] tensor, coords time/lead_time/variable/lat/lon, lat1d, lon1d)."""
|
| 130 |
+
import stormscope_obs
|
| 131 |
+
times = _iso_times(start_date, lead_times)
|
| 132 |
+
cache = _env("RUSTWX_STORMSCOPE_MRMS_CACHE", "SSFAST_MRMS_CACHE", _cache_dir("mrms_grib2"))
|
| 133 |
+
res = stormscope_obs.fetch_mrms_sequence(times, cache)
|
| 134 |
+
T, H, W = res["shape"]
|
| 135 |
+
data = np.asarray(res["data"], dtype=np.float32).reshape(T, 1, H, W)[None] # [1, L, 1, H, W]
|
| 136 |
+
# Keep MRMS missing/no-coverage flags finite (-999/-99), matching earth2studio.
|
| 137 |
+
# StormScopeMRMS.prep_input imputes values <= -20 dBZ to -10; NaNs poison the denoiser.
|
| 138 |
+
north, south, west, east = res["geo"] # row 0 = north, col 0 = west (MRMS N->S, W->E)
|
| 139 |
+
lats = np.linspace(north, south, H).astype(np.float32)
|
| 140 |
+
lons = np.linspace(west, east, W).astype(np.float32)
|
| 141 |
+
lons = np.where(lons < 0.0, lons + 360.0, lons).astype(np.float32)
|
| 142 |
+
coords = OrderedDict([
|
| 143 |
+
("time", np.asarray(start_date)),
|
| 144 |
+
("lead_time", np.asarray(lead_times)),
|
| 145 |
+
("variable", np.asarray(["refc"])),
|
| 146 |
+
("lat", lats),
|
| 147 |
+
("lon", lons),
|
| 148 |
+
])
|
| 149 |
+
return torch.from_numpy(np.ascontiguousarray(data)).to(device), coords, lats, lons
|
rustwx_stormscope/service.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""StormScope operational route - continuous nowcast/nearcast cycles.
|
| 3 |
+
|
| 4 |
+
Default container entrypoint. Each cycle runs the configured models with the validated
|
| 5 |
+
optimizations (det + reduced steps + compile + MXFP8) and the chosen data backend, enforces
|
| 6 |
+
retention, and writes a status/manifest for downstream ingestion.
|
| 7 |
+
|
| 8 |
+
rustwx-stormscope-service --interval 30 --keep-cycles 36 --num-steps 24 --mxfp8 --backend rust
|
| 9 |
+
"""
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
import time
|
| 14 |
+
import traceback
|
| 15 |
+
from datetime import datetime, timedelta, timezone
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
from . import run as runmod
|
| 19 |
+
from .paths import default_outdir
|
| 20 |
+
|
| 21 |
+
DEFAULT_RUN_CONFIGS = [
|
| 22 |
+
{"label": "nowcast", "mode": "nowcast", "res": "6km", "steps": 12},
|
| 23 |
+
# StormScope names this mode "forecast" internally. Operationally it is the
|
| 24 |
+
# GFS-conditioned hourly nearcast route, so surface that label downstream.
|
| 25 |
+
{"label": "nearcast", "mode": "forecast", "res": "6km", "steps": 12},
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def utc_stamp(dt=None):
|
| 30 |
+
return (dt or datetime.now(timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def build_run_configs(args):
|
| 34 |
+
configs = []
|
| 35 |
+
if args.nowcast_steps > 0:
|
| 36 |
+
configs.append({
|
| 37 |
+
**DEFAULT_RUN_CONFIGS[0],
|
| 38 |
+
"res": args.nowcast_res,
|
| 39 |
+
"steps": args.nowcast_steps,
|
| 40 |
+
})
|
| 41 |
+
if args.nearcast_steps > 0:
|
| 42 |
+
configs.append({
|
| 43 |
+
**DEFAULT_RUN_CONFIGS[1],
|
| 44 |
+
"res": args.nearcast_res,
|
| 45 |
+
"steps": args.nearcast_steps,
|
| 46 |
+
})
|
| 47 |
+
if not configs:
|
| 48 |
+
raise ValueError("At least one of --nowcast-steps or --nearcast-steps must be > 0")
|
| 49 |
+
return configs
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def model_cycle_key(mode, res):
|
| 53 |
+
from . import core
|
| 54 |
+
|
| 55 |
+
cfg = core.MODEL_CONFIGS[(mode, res)]
|
| 56 |
+
gfs_aligned = (
|
| 57 |
+
cfg.get("needs_gfs", False)
|
| 58 |
+
and core.forecast_conditioning_kind(cfg) == "gfs"
|
| 59 |
+
)
|
| 60 |
+
init = core.get_latest_init_time(cfg["step_minutes"], "6steps" in cfg["goes"],
|
| 61 |
+
gfs_aligned=gfs_aligned)
|
| 62 |
+
return init, datetime.fromisoformat(init).strftime("%Y%m%d_%H%Mz")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def completed_cycles(outdir, mode, res):
|
| 66 |
+
out = {}
|
| 67 |
+
for f in Path(outdir).glob(f"stormscope_{mode}_{res}_*_meta.json"):
|
| 68 |
+
try:
|
| 69 |
+
meta = json.loads(f.read_text())
|
| 70 |
+
except Exception:
|
| 71 |
+
continue
|
| 72 |
+
if meta.get("complete", True):
|
| 73 |
+
key = f.stem.replace(f"stormscope_{mode}_{res}_", "").replace("_meta", "")
|
| 74 |
+
out[key] = int(meta.get("n_steps", 0))
|
| 75 |
+
return out
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def required_steps_for_cycle(rc, init):
|
| 79 |
+
"""Interpret nearcast steps as future horizon when forecast cycles are GFS-rooted."""
|
| 80 |
+
from . import core
|
| 81 |
+
|
| 82 |
+
cfg = core.MODEL_CONFIGS[(rc["mode"], rc["res"])]
|
| 83 |
+
requested_steps = int(rc["steps"])
|
| 84 |
+
if requested_steps <= 0:
|
| 85 |
+
return 0
|
| 86 |
+
if not (
|
| 87 |
+
rc["mode"] == "forecast"
|
| 88 |
+
and cfg.get("needs_gfs", False)
|
| 89 |
+
and core.forecast_conditioning_kind(cfg) == "gfs"
|
| 90 |
+
):
|
| 91 |
+
return requested_steps
|
| 92 |
+
|
| 93 |
+
init_dt = datetime.fromisoformat(init).replace(tzinfo=timezone.utc)
|
| 94 |
+
step_seconds = cfg["step_minutes"] * 60
|
| 95 |
+
elapsed_steps = max(
|
| 96 |
+
0,
|
| 97 |
+
int((datetime.now(timezone.utc) - init_dt).total_seconds() // step_seconds),
|
| 98 |
+
)
|
| 99 |
+
max_steps = int(os.environ.get(
|
| 100 |
+
"RUSTWX_STORMSCOPE_MAX_NEARCAST_STEPS",
|
| 101 |
+
os.environ.get("SSFAST_MAX_NEARCAST_STEPS", "81"),
|
| 102 |
+
))
|
| 103 |
+
return min(max_steps, max(requested_steps, elapsed_steps + requested_steps))
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def evict_old(outdir, keep, run_configs):
|
| 107 |
+
for rc in run_configs:
|
| 108 |
+
metas = sorted(Path(outdir).glob(f"stormscope_{rc['mode']}_{rc['res']}_*_meta.json"))
|
| 109 |
+
for meta in metas[:max(0, len(metas) - keep)]:
|
| 110 |
+
prefix = meta.stem.replace("_meta", "")
|
| 111 |
+
for f in Path(outdir).glob(f"{prefix}_*"):
|
| 112 |
+
if f != meta:
|
| 113 |
+
f.unlink(missing_ok=True)
|
| 114 |
+
meta.unlink(missing_ok=True)
|
| 115 |
+
print(f" evicted {meta.name}", flush=True)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def write_status(outdir, status):
|
| 119 |
+
status["updated"] = utc_stamp()
|
| 120 |
+
with open(os.path.join(outdir, "status.json"), "w") as f:
|
| 121 |
+
json.dump(status, f, indent=2)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def main():
|
| 125 |
+
ap = argparse.ArgumentParser(description="StormScope operational service")
|
| 126 |
+
runmod.add_common_args(ap) # shares --num-steps/--mxfp8/--backend/etc.
|
| 127 |
+
ap.add_argument("--outdir", default=str(default_outdir()))
|
| 128 |
+
ap.add_argument("--interval", type=int, default=30, help="minutes between cycle checks")
|
| 129 |
+
ap.add_argument("--keep-cycles", type=int, default=36)
|
| 130 |
+
ap.add_argument("--max-failures", type=int, default=0,
|
| 131 |
+
help="exit after this many consecutive failures; 0 means keep retrying")
|
| 132 |
+
ap.add_argument("--nowcast-steps", type=int, default=12,
|
| 133 |
+
help="10-minute nowcast frames per cycle")
|
| 134 |
+
ap.add_argument("--nearcast-steps", type=int, default=12,
|
| 135 |
+
help="hourly GFS-conditioned nearcast horizon; 0 disables nearcast")
|
| 136 |
+
ap.add_argument("--nowcast-res", default="6km", choices=["6km", "3km"])
|
| 137 |
+
ap.add_argument("--nearcast-res", default="6km", choices=["6km"])
|
| 138 |
+
a = ap.parse_args()
|
| 139 |
+
|
| 140 |
+
from . import core
|
| 141 |
+
import torch
|
| 142 |
+
|
| 143 |
+
cfg = runmod.build_cfg(a)
|
| 144 |
+
run_configs = build_run_configs(a)
|
| 145 |
+
|
| 146 |
+
os.makedirs(a.outdir, exist_ok=True)
|
| 147 |
+
# set up optimizations + backend ONCE (idempotent wrap; model reloads per cycle, inductor-cached)
|
| 148 |
+
from earth2studio.models.px.stormscope import StormScopeGOES, StormScopeMRMS
|
| 149 |
+
runmod._wrap_load(StormScopeGOES, cfg)
|
| 150 |
+
runmod._wrap_load(StormScopeMRMS, cfg)
|
| 151 |
+
from .datasources import install_backend
|
| 152 |
+
install_backend(core, cfg["backend"])
|
| 153 |
+
|
| 154 |
+
model_list = ", ".join(f"{c['label']}({c['mode']}_{c['res']}, {c['steps']} steps)"
|
| 155 |
+
for c in run_configs)
|
| 156 |
+
print(f"=== StormScope Operational (fast) === GPU {torch.cuda.get_device_name()}", flush=True)
|
| 157 |
+
print(f" models=[{model_list}] interval={a.interval}m keep={a.keep_cycles} "
|
| 158 |
+
f"num_steps={a.num_steps} mxfp8={a.mxfp8} backend={a.backend}", flush=True)
|
| 159 |
+
|
| 160 |
+
status = {"state": "starting", "total_runs": 0, "consecutive_failures": 0}
|
| 161 |
+
write_status(a.outdir, status)
|
| 162 |
+
total, fails = 0, 0
|
| 163 |
+
|
| 164 |
+
while True:
|
| 165 |
+
try:
|
| 166 |
+
pending = []
|
| 167 |
+
for rc in run_configs:
|
| 168 |
+
init, key = model_cycle_key(rc["mode"], rc["res"])
|
| 169 |
+
steps = required_steps_for_cycle(rc, init)
|
| 170 |
+
done_steps = completed_cycles(a.outdir, rc["mode"], rc["res"]).get(key, 0)
|
| 171 |
+
if done_steps < steps:
|
| 172 |
+
pending.append({**rc, "steps": steps, "init": init, "key": key})
|
| 173 |
+
|
| 174 |
+
if not pending:
|
| 175 |
+
now = datetime.now(timezone.utc)
|
| 176 |
+
nxt = (now + timedelta(minutes=a.interval)).replace(second=0, microsecond=0)
|
| 177 |
+
nxt -= timedelta(minutes=nxt.minute % a.interval)
|
| 178 |
+
wait = max(10, (nxt - now).total_seconds())
|
| 179 |
+
status["state"] = "waiting"; status["next_run"] = utc_stamp(nxt)
|
| 180 |
+
write_status(a.outdir, status)
|
| 181 |
+
print(f"[{now:%H:%M:%S}Z] current; next check {wait:.0f}s", flush=True)
|
| 182 |
+
time.sleep(wait); continue
|
| 183 |
+
|
| 184 |
+
t0 = time.time()
|
| 185 |
+
for rc in pending:
|
| 186 |
+
label = f"{rc['label']}_{rc['res']}"
|
| 187 |
+
status["state"] = f"running_{label}"; write_status(a.outdir, status)
|
| 188 |
+
print(f"\n--- {label}: {rc['steps']} steps, init {rc['key']} ---", flush=True)
|
| 189 |
+
ts = time.time()
|
| 190 |
+
runmod.run_cycle(rc["mode"], rc["res"], rc["steps"], rc["init"], a.outdir, cfg)
|
| 191 |
+
print(f" [{label}] done in {time.time()-ts:.0f}s", flush=True)
|
| 192 |
+
torch.cuda.empty_cache()
|
| 193 |
+
total += 1; fails = 0
|
| 194 |
+
status.update(state="idle", total_runs=total, consecutive_failures=0,
|
| 195 |
+
last_success=utc_stamp(),
|
| 196 |
+
last_elapsed_seconds=round(time.time() - t0, 1))
|
| 197 |
+
status.pop("last_error", None)
|
| 198 |
+
write_status(a.outdir, status)
|
| 199 |
+
evict_old(a.outdir, a.keep_cycles, run_configs)
|
| 200 |
+
|
| 201 |
+
except KeyboardInterrupt:
|
| 202 |
+
status["state"] = "stopped"; write_status(a.outdir, status); break
|
| 203 |
+
except Exception as e:
|
| 204 |
+
msg = f"{type(e).__name__}: {e}"
|
| 205 |
+
if "no GOES file near" in msg or "no MRMS file near" in msg:
|
| 206 |
+
status.update(state="waiting_for_data",
|
| 207 |
+
last_error=msg,
|
| 208 |
+
next_run=utc_stamp(datetime.now(timezone.utc) + timedelta(seconds=30)))
|
| 209 |
+
write_status(a.outdir, status)
|
| 210 |
+
print(f"[WAIT_DATA] {msg}", flush=True)
|
| 211 |
+
time.sleep(30)
|
| 212 |
+
continue
|
| 213 |
+
fails += 1
|
| 214 |
+
status.update(state="error", consecutive_failures=fails,
|
| 215 |
+
last_error=msg)
|
| 216 |
+
write_status(a.outdir, status)
|
| 217 |
+
print(f"[ERROR] {type(e).__name__}: {e}", flush=True); traceback.print_exc()
|
| 218 |
+
if a.max_failures and fails >= a.max_failures:
|
| 219 |
+
print("too many failures; exiting", flush=True); raise
|
| 220 |
+
time.sleep(30)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
if __name__ == "__main__":
|
| 224 |
+
main()
|
rustwx_stormscope/webmap_preview.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""Make a lightweight Leaflet web-map preview from StormScope .npz outputs."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from datetime import datetime
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
from PIL import Image
|
| 14 |
+
|
| 15 |
+
from .paths import default_outdir, default_webmap_dir
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass(frozen=True)
|
| 19 |
+
class Product:
|
| 20 |
+
slug: str
|
| 21 |
+
field: str
|
| 22 |
+
title: str
|
| 23 |
+
default_opacity: float
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
PRODUCTS = {
|
| 27 |
+
"refc": Product("refc", "refc", "Composite Reflectivity", 0.82),
|
| 28 |
+
"ir": Product("ir", "ch_ir_1035", "IR Brightness Temperature", 0.64),
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
VIEWS = {
|
| 32 |
+
"data": None,
|
| 33 |
+
"conus": {"west": -126.0, "east": -66.0, "south": 23.0, "north": 50.0},
|
| 34 |
+
"central_plains": {"west": -106.5, "east": -90.0, "south": 31.5, "north": 47.2},
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
REFC_LEVELS = np.array([5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75], dtype=np.float32)
|
| 39 |
+
REFC_COLORS = np.array(
|
| 40 |
+
[
|
| 41 |
+
(236, 243, 252, 0),
|
| 42 |
+
(205, 222, 246, 125),
|
| 43 |
+
(160, 194, 232, 155),
|
| 44 |
+
(103, 157, 212, 185),
|
| 45 |
+
(32, 102, 172, 205),
|
| 46 |
+
(44, 162, 95, 215),
|
| 47 |
+
(166, 217, 106, 220),
|
| 48 |
+
(255, 240, 102, 225),
|
| 49 |
+
(253, 174, 55, 232),
|
| 50 |
+
(244, 109, 32, 238),
|
| 51 |
+
(215, 48, 31, 242),
|
| 52 |
+
(165, 0, 38, 246),
|
| 53 |
+
(122, 50, 148, 248),
|
| 54 |
+
(84, 39, 136, 250),
|
| 55 |
+
(68, 68, 68, 252),
|
| 56 |
+
],
|
| 57 |
+
dtype=np.uint8,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
IR_STOPS = np.array([190, 205, 220, 235, 250, 265, 280, 300], dtype=np.float32)
|
| 61 |
+
IR_COLORS = np.array(
|
| 62 |
+
[
|
| 63 |
+
(62, 20, 115, 220),
|
| 64 |
+
(44, 76, 160, 205),
|
| 65 |
+
(32, 143, 195, 190),
|
| 66 |
+
(99, 194, 174, 168),
|
| 67 |
+
(226, 237, 154, 135),
|
| 68 |
+
(250, 180, 92, 105),
|
| 69 |
+
(126, 126, 126, 70),
|
| 70 |
+
(255, 255, 255, 0),
|
| 71 |
+
],
|
| 72 |
+
dtype=np.float32,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def parse_args() -> argparse.Namespace:
|
| 77 |
+
ap = argparse.ArgumentParser(description="Build a web-map overlay preview from StormScope .npz")
|
| 78 |
+
ap.add_argument("--meta", default="", help="cycle metadata JSON; defaults to latest completed cycle")
|
| 79 |
+
ap.add_argument("--outdir", default=str(default_outdir()))
|
| 80 |
+
ap.add_argument("--output-dir", default=str(default_webmap_dir()))
|
| 81 |
+
ap.add_argument("--products", default="refc,ir", help="comma-separated: refc,ir")
|
| 82 |
+
ap.add_argument("--view", default="central_plains", choices=sorted(VIEWS), help="initial web-map view")
|
| 83 |
+
ap.add_argument("--max-frames", type=int, default=12)
|
| 84 |
+
ap.add_argument("--force", action="store_true")
|
| 85 |
+
return ap.parse_args()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def read_json(path: Path) -> dict:
|
| 89 |
+
return json.loads(path.read_text())
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def parse_dt(value: str) -> datetime:
|
| 93 |
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def latest_meta(outdir: Path) -> Path:
|
| 97 |
+
candidates: list[tuple[datetime, Path]] = []
|
| 98 |
+
for path in outdir.glob("stormscope_*_*_*_meta.json"):
|
| 99 |
+
try:
|
| 100 |
+
meta = read_json(path)
|
| 101 |
+
if meta.get("complete", True):
|
| 102 |
+
candidates.append((parse_dt(meta["init_time"]), path))
|
| 103 |
+
except Exception:
|
| 104 |
+
continue
|
| 105 |
+
if not candidates:
|
| 106 |
+
raise FileNotFoundError(f"no completed StormScope cycle metadata found in {outdir}")
|
| 107 |
+
return sorted(candidates)[-1][1]
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def normalize_lon(values: np.ndarray) -> np.ndarray:
|
| 111 |
+
return np.where(values > 180.0, values - 360.0, values)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def bounds_from_npz(npz_path: Path) -> dict:
|
| 115 |
+
with np.load(npz_path) as data:
|
| 116 |
+
lats = data["lats"].astype(np.float32)
|
| 117 |
+
lons = normalize_lon(data["lons"].astype(np.float32))
|
| 118 |
+
finite = np.isfinite(lats) & np.isfinite(lons)
|
| 119 |
+
return {
|
| 120 |
+
"west": float(np.nanmin(lons[finite])),
|
| 121 |
+
"east": float(np.nanmax(lons[finite])),
|
| 122 |
+
"south": float(np.nanmin(lats[finite])),
|
| 123 |
+
"north": float(np.nanmax(lats[finite])),
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def rgba_refc(values: np.ndarray) -> np.ndarray:
|
| 128 |
+
rgba = np.zeros((*values.shape, 4), dtype=np.uint8)
|
| 129 |
+
mask = np.isfinite(values) & (values >= REFC_LEVELS[0])
|
| 130 |
+
if not np.any(mask):
|
| 131 |
+
return rgba
|
| 132 |
+
indices = np.searchsorted(REFC_LEVELS, values[mask], side="right") - 1
|
| 133 |
+
indices = np.clip(indices, 0, len(REFC_COLORS) - 1)
|
| 134 |
+
rgba[mask] = REFC_COLORS[indices]
|
| 135 |
+
return rgba
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def rgba_ir(values: np.ndarray) -> np.ndarray:
|
| 139 |
+
rgba = np.zeros((*values.shape, 4), dtype=np.uint8)
|
| 140 |
+
mask = np.isfinite(values)
|
| 141 |
+
if not np.any(mask):
|
| 142 |
+
return rgba
|
| 143 |
+
clipped = np.clip(values[mask], IR_STOPS[0], IR_STOPS[-1])
|
| 144 |
+
channels = []
|
| 145 |
+
for channel in range(4):
|
| 146 |
+
channels.append(np.interp(clipped, IR_STOPS, IR_COLORS[:, channel]))
|
| 147 |
+
rgba[mask] = np.stack(channels, axis=1).clip(0, 255).astype(np.uint8)
|
| 148 |
+
return rgba
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def field_rgba(product: Product, values: np.ndarray) -> np.ndarray:
|
| 152 |
+
if product.slug == "refc":
|
| 153 |
+
return rgba_refc(values)
|
| 154 |
+
if product.slug == "ir":
|
| 155 |
+
return rgba_ir(values)
|
| 156 |
+
raise ValueError(f"unknown product {product.slug}")
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def frame_label(meta: dict, index: int) -> str:
|
| 160 |
+
step = int(meta.get("step_minutes") or 0)
|
| 161 |
+
if step:
|
| 162 |
+
lead = step * (index + 1)
|
| 163 |
+
if lead % 60 == 0:
|
| 164 |
+
return f"+{lead // 60}h"
|
| 165 |
+
return f"+{lead}m"
|
| 166 |
+
return f"f{index + 1:02d}"
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def render_product_frame(npz_path: Path, product: Product, output: Path, force: bool) -> dict:
|
| 170 |
+
if output.exists() and output.stat().st_size > 0 and not force and output.stat().st_mtime >= npz_path.stat().st_mtime:
|
| 171 |
+
return {"path": output}
|
| 172 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 173 |
+
with np.load(npz_path) as data:
|
| 174 |
+
values = data[product.field].astype(np.float32)
|
| 175 |
+
rgba = field_rgba(product, values)
|
| 176 |
+
Image.fromarray(rgba, "RGBA").save(output, optimize=False)
|
| 177 |
+
return {"path": output}
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def write_html(output_dir: Path) -> None:
|
| 181 |
+
(output_dir / "index.html").write_text(
|
| 182 |
+
"""<!doctype html>
|
| 183 |
+
<html lang="en">
|
| 184 |
+
<head>
|
| 185 |
+
<meta charset="utf-8">
|
| 186 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 187 |
+
<title>StormScope Web Map Preview</title>
|
| 188 |
+
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css">
|
| 189 |
+
<style>
|
| 190 |
+
* { box-sizing: border-box; }
|
| 191 |
+
html, body, #map { height: 100%; margin: 0; }
|
| 192 |
+
body { font: 14px/1.35 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #17201b; }
|
| 193 |
+
#map { background: #eef2ef; }
|
| 194 |
+
.panel {
|
| 195 |
+
position: absolute;
|
| 196 |
+
z-index: 700;
|
| 197 |
+
left: 12px;
|
| 198 |
+
right: 12px;
|
| 199 |
+
top: 12px;
|
| 200 |
+
display: flex;
|
| 201 |
+
align-items: center;
|
| 202 |
+
justify-content: space-between;
|
| 203 |
+
gap: 12px;
|
| 204 |
+
padding: 10px 12px;
|
| 205 |
+
background: rgba(255, 255, 255, .92);
|
| 206 |
+
border: 1px solid #d5ddd7;
|
| 207 |
+
border-radius: 8px;
|
| 208 |
+
box-shadow: 0 8px 28px rgba(20, 30, 25, .12);
|
| 209 |
+
}
|
| 210 |
+
h1 { margin: 0; font-size: 16px; letter-spacing: 0; }
|
| 211 |
+
.meta { color: #59655f; font-size: 12px; margin-top: 2px; }
|
| 212 |
+
.controls { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; justify-content: end; }
|
| 213 |
+
button, select {
|
| 214 |
+
min-height: 32px;
|
| 215 |
+
border: 1px solid #cbd5ce;
|
| 216 |
+
border-radius: 6px;
|
| 217 |
+
background: white;
|
| 218 |
+
color: #17201b;
|
| 219 |
+
font: inherit;
|
| 220 |
+
font-weight: 650;
|
| 221 |
+
padding: 0 10px;
|
| 222 |
+
}
|
| 223 |
+
input[type="range"] { width: min(42vw, 420px); }
|
| 224 |
+
#stamp { min-width: 92px; font-weight: 750; text-align: center; }
|
| 225 |
+
.leaflet-control-attribution { font-size: 10px; }
|
| 226 |
+
@media (max-width: 720px) {
|
| 227 |
+
.panel { display: grid; top: 8px; left: 8px; right: 8px; }
|
| 228 |
+
.controls { justify-content: start; }
|
| 229 |
+
input[type="range"] { width: 100%; }
|
| 230 |
+
}
|
| 231 |
+
</style>
|
| 232 |
+
</head>
|
| 233 |
+
<body>
|
| 234 |
+
<div id="map"></div>
|
| 235 |
+
<div class="panel">
|
| 236 |
+
<div>
|
| 237 |
+
<h1 id="title">StormScope</h1>
|
| 238 |
+
<div id="meta" class="meta"></div>
|
| 239 |
+
</div>
|
| 240 |
+
<div class="controls">
|
| 241 |
+
<select id="product"></select>
|
| 242 |
+
<button id="prev" type="button">Prev</button>
|
| 243 |
+
<input id="frame" type="range" min="0" max="0" value="0">
|
| 244 |
+
<button id="next" type="button">Next</button>
|
| 245 |
+
<button id="play" type="button">Play</button>
|
| 246 |
+
<span id="stamp"></span>
|
| 247 |
+
</div>
|
| 248 |
+
</div>
|
| 249 |
+
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>
|
| 250 |
+
<script src="manifest.js"></script>
|
| 251 |
+
<script>
|
| 252 |
+
const m = window.STORMSCOPE_WEBMAP;
|
| 253 |
+
const bounds = [[m.bounds.south, m.bounds.west], [m.bounds.north, m.bounds.east]];
|
| 254 |
+
const viewBounds = m.view_bounds ? [[m.view_bounds.south, m.view_bounds.west], [m.view_bounds.north, m.view_bounds.east]] : bounds;
|
| 255 |
+
const map = L.map("map", { preferCanvas: true, zoomControl: false });
|
| 256 |
+
L.tileLayer("https://tile.openstreetmap.org/{z}/{x}/{y}.png", {
|
| 257 |
+
maxZoom: 12,
|
| 258 |
+
attribution: "© OpenStreetMap contributors"
|
| 259 |
+
}).addTo(map);
|
| 260 |
+
L.control.zoom({ position: "bottomleft" }).addTo(map);
|
| 261 |
+
map.fitBounds(viewBounds, { padding: [18, 18] });
|
| 262 |
+
|
| 263 |
+
const productSelect = document.getElementById("product");
|
| 264 |
+
const slider = document.getElementById("frame");
|
| 265 |
+
const stamp = document.getElementById("stamp");
|
| 266 |
+
const playButton = document.getElementById("play");
|
| 267 |
+
document.getElementById("title").textContent = `${m.mode} ${m.cycle}`;
|
| 268 |
+
document.getElementById("meta").textContent = `${m.init_time} | ${m.resolution} | ${m.frame_count} frames`;
|
| 269 |
+
Object.entries(m.products).forEach(([slug, product]) => {
|
| 270 |
+
const option = document.createElement("option");
|
| 271 |
+
option.value = slug;
|
| 272 |
+
option.textContent = product.title;
|
| 273 |
+
productSelect.appendChild(option);
|
| 274 |
+
});
|
| 275 |
+
|
| 276 |
+
let product = productSelect.value;
|
| 277 |
+
let frame = 0;
|
| 278 |
+
let timer = null;
|
| 279 |
+
let overlay = null;
|
| 280 |
+
|
| 281 |
+
function currentProduct() { return m.products[product]; }
|
| 282 |
+
function currentFrame() { return currentProduct().frames[frame]; }
|
| 283 |
+
|
| 284 |
+
function updateOverlay() {
|
| 285 |
+
const p = currentProduct();
|
| 286 |
+
slider.max = String(p.frames.length - 1);
|
| 287 |
+
if (frame >= p.frames.length) frame = p.frames.length - 1;
|
| 288 |
+
const f = currentFrame();
|
| 289 |
+
slider.value = String(frame);
|
| 290 |
+
stamp.textContent = f.label;
|
| 291 |
+
if (!overlay) {
|
| 292 |
+
overlay = L.imageOverlay(f.url, bounds, { opacity: p.opacity, interactive: false }).addTo(map);
|
| 293 |
+
} else {
|
| 294 |
+
overlay.setUrl(f.url);
|
| 295 |
+
overlay.setOpacity(p.opacity);
|
| 296 |
+
}
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
productSelect.addEventListener("change", () => {
|
| 300 |
+
product = productSelect.value;
|
| 301 |
+
frame = Math.min(frame, currentProduct().frames.length - 1);
|
| 302 |
+
updateOverlay();
|
| 303 |
+
});
|
| 304 |
+
slider.addEventListener("input", () => { frame = Number(slider.value); updateOverlay(); });
|
| 305 |
+
document.getElementById("prev").addEventListener("click", () => {
|
| 306 |
+
frame = (frame + currentProduct().frames.length - 1) % currentProduct().frames.length;
|
| 307 |
+
updateOverlay();
|
| 308 |
+
});
|
| 309 |
+
document.getElementById("next").addEventListener("click", () => {
|
| 310 |
+
frame = (frame + 1) % currentProduct().frames.length;
|
| 311 |
+
updateOverlay();
|
| 312 |
+
});
|
| 313 |
+
playButton.addEventListener("click", () => {
|
| 314 |
+
if (timer) {
|
| 315 |
+
clearInterval(timer);
|
| 316 |
+
timer = null;
|
| 317 |
+
playButton.textContent = "Play";
|
| 318 |
+
} else {
|
| 319 |
+
timer = setInterval(() => {
|
| 320 |
+
frame = (frame + 1) % currentProduct().frames.length;
|
| 321 |
+
updateOverlay();
|
| 322 |
+
}, 650);
|
| 323 |
+
playButton.textContent = "Pause";
|
| 324 |
+
}
|
| 325 |
+
});
|
| 326 |
+
updateOverlay();
|
| 327 |
+
</script>
|
| 328 |
+
</body>
|
| 329 |
+
</html>
|
| 330 |
+
""",
|
| 331 |
+
encoding="utf-8",
|
| 332 |
+
)
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def build_preview(meta_path: Path, outdir: Path, output_dir: Path, product_slugs: list[str], view: str, max_frames: int, force: bool) -> dict:
|
| 336 |
+
meta = read_json(meta_path)
|
| 337 |
+
files = list(meta.get("files", []))
|
| 338 |
+
if max_frames > 0:
|
| 339 |
+
files = files[:max_frames]
|
| 340 |
+
if not files:
|
| 341 |
+
raise ValueError(f"metadata has no files: {meta_path}")
|
| 342 |
+
|
| 343 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 344 |
+
assets = output_dir / "assets" / meta.get("cycle", "cycle")
|
| 345 |
+
bounds = bounds_from_npz(outdir / files[0])
|
| 346 |
+
manifest = {
|
| 347 |
+
"schema": "rustwx-stormscope.webmap_preview.v1",
|
| 348 |
+
"source_meta": str(meta_path),
|
| 349 |
+
"cycle": meta.get("cycle"),
|
| 350 |
+
"mode": "nearcast" if meta.get("mode") == "forecast" else meta.get("mode"),
|
| 351 |
+
"resolution": meta.get("resolution"),
|
| 352 |
+
"init_time": meta.get("init_time"),
|
| 353 |
+
"bounds": bounds,
|
| 354 |
+
"view": view,
|
| 355 |
+
"view_bounds": VIEWS[view] or bounds,
|
| 356 |
+
"frame_count": len(files),
|
| 357 |
+
"products": {},
|
| 358 |
+
}
|
| 359 |
+
|
| 360 |
+
for slug in product_slugs:
|
| 361 |
+
if slug not in PRODUCTS:
|
| 362 |
+
raise ValueError(f"unknown product {slug!r}; available: {', '.join(PRODUCTS)}")
|
| 363 |
+
product = PRODUCTS[slug]
|
| 364 |
+
frames = []
|
| 365 |
+
for index, filename in enumerate(files):
|
| 366 |
+
npz_path = outdir / filename
|
| 367 |
+
png_name = f"{Path(filename).stem}_{product.slug}.png"
|
| 368 |
+
png_path = assets / product.slug / png_name
|
| 369 |
+
render_product_frame(npz_path, product, png_path, force)
|
| 370 |
+
valid_times = meta.get("valid_times") or []
|
| 371 |
+
frames.append(
|
| 372 |
+
{
|
| 373 |
+
"index": index,
|
| 374 |
+
"label": frame_label(meta, index),
|
| 375 |
+
"valid_time": valid_times[index] if index < len(valid_times) else None,
|
| 376 |
+
"url": png_path.relative_to(output_dir).as_posix(),
|
| 377 |
+
"source_npz": str(npz_path),
|
| 378 |
+
}
|
| 379 |
+
)
|
| 380 |
+
manifest["products"][slug] = {
|
| 381 |
+
"field": product.field,
|
| 382 |
+
"title": product.title,
|
| 383 |
+
"opacity": product.default_opacity,
|
| 384 |
+
"frames": frames,
|
| 385 |
+
}
|
| 386 |
+
|
| 387 |
+
(output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
| 388 |
+
(output_dir / "manifest.js").write_text(
|
| 389 |
+
"window.STORMSCOPE_WEBMAP = " + json.dumps(manifest, indent=2) + ";\n",
|
| 390 |
+
encoding="utf-8",
|
| 391 |
+
)
|
| 392 |
+
write_html(output_dir)
|
| 393 |
+
return manifest
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def main() -> int:
|
| 397 |
+
args = parse_args()
|
| 398 |
+
outdir = Path(args.outdir)
|
| 399 |
+
meta = Path(args.meta) if args.meta else latest_meta(outdir)
|
| 400 |
+
products = [slug.strip() for slug in args.products.split(",") if slug.strip()]
|
| 401 |
+
manifest = build_preview(meta, outdir, Path(args.output_dir), products, args.view, args.max_frames, args.force)
|
| 402 |
+
print(json.dumps({"ok": True, "index": str(Path(args.output_dir) / "index.html"), "cycle": manifest["cycle"]}, indent=2))
|
| 403 |
+
return 0
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
if __name__ == "__main__":
|
| 407 |
+
raise SystemExit(main())
|
rustwx_stormscope/wxstore_export.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""Export completed StormScope cycles into native rustwx WXA stores."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import shutil
|
| 10 |
+
import subprocess
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
from .paths import default_outdir, default_spatial_root, repo_root
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def parse_dt(value: str) -> str:
|
| 18 |
+
return value.replace("Z", "+00:00")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def display_mode(mode: str) -> str:
|
| 22 |
+
return "nearcast" if mode == "forecast" else mode
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def model_slug(meta: dict) -> str:
|
| 26 |
+
mode = display_mode(str(meta.get("mode", "stormscope")))
|
| 27 |
+
resolution = meta.get("resolution")
|
| 28 |
+
return f"stormscope_{mode}_{resolution}" if resolution else f"stormscope_{mode}"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def binary_name() -> str:
|
| 32 |
+
return "stormscope_wxstore.exe" if os.name == "nt" else "stormscope_wxstore"
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def resolve_binary(explicit: str | None = None) -> Path:
|
| 36 |
+
candidates = []
|
| 37 |
+
if explicit:
|
| 38 |
+
candidates.append(Path(explicit))
|
| 39 |
+
root = repo_root()
|
| 40 |
+
candidates.append(root / "rust" / "stormscope_wxstore" / "target" / "release" / binary_name())
|
| 41 |
+
found = shutil.which(binary_name())
|
| 42 |
+
if found:
|
| 43 |
+
candidates.append(Path(found))
|
| 44 |
+
for candidate in candidates:
|
| 45 |
+
if candidate.exists():
|
| 46 |
+
return candidate
|
| 47 |
+
raise FileNotFoundError(
|
| 48 |
+
"stormscope_wxstore binary not found; build with "
|
| 49 |
+
"`cd rust/stormscope_wxstore && cargo build --release`"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def read_json(path: Path) -> dict:
|
| 54 |
+
return json.loads(path.read_text())
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def latest_meta(outdir: Path) -> Path:
|
| 58 |
+
candidates = []
|
| 59 |
+
for path in outdir.glob("stormscope_*_*_*_meta.json"):
|
| 60 |
+
try:
|
| 61 |
+
meta = read_json(path)
|
| 62 |
+
except Exception:
|
| 63 |
+
continue
|
| 64 |
+
if not meta.get("complete", True):
|
| 65 |
+
continue
|
| 66 |
+
candidates.append((parse_dt(meta.get("init_time", "")), path))
|
| 67 |
+
if not candidates:
|
| 68 |
+
raise FileNotFoundError(f"no completed StormScope cycle metadata found in {outdir}")
|
| 69 |
+
return sorted(candidates)[-1][1]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def export_is_current(meta_path: Path, outdir: Path, spatial_root: Path) -> bool:
|
| 73 |
+
try:
|
| 74 |
+
meta = read_json(meta_path)
|
| 75 |
+
except Exception:
|
| 76 |
+
return False
|
| 77 |
+
sidecar = spatial_root / model_slug(meta) / meta["cycle"] / "stormscope-export.json"
|
| 78 |
+
if not sidecar.exists() or sidecar.stat().st_size == 0:
|
| 79 |
+
return False
|
| 80 |
+
target_mtime = sidecar.stat().st_mtime
|
| 81 |
+
sources = [meta_path] + [outdir / name for name in meta.get("files", [])]
|
| 82 |
+
return all(not src.exists() or src.stat().st_mtime <= target_mtime for src in sources)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def export_cycle(
|
| 86 |
+
meta_path: Path,
|
| 87 |
+
outdir: Path,
|
| 88 |
+
spatial_root: Path,
|
| 89 |
+
*,
|
| 90 |
+
binary: str | None = None,
|
| 91 |
+
fields: str | None = None,
|
| 92 |
+
force: bool = False,
|
| 93 |
+
) -> dict:
|
| 94 |
+
if not force and export_is_current(meta_path, outdir, spatial_root):
|
| 95 |
+
meta = read_json(meta_path)
|
| 96 |
+
return {
|
| 97 |
+
"ok": True,
|
| 98 |
+
"skipped_current": True,
|
| 99 |
+
"model": model_slug(meta),
|
| 100 |
+
"cycle": meta.get("cycle"),
|
| 101 |
+
"sidecar": str(spatial_root / model_slug(meta) / meta["cycle"] / "stormscope-export.json"),
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
exe = resolve_binary(binary)
|
| 105 |
+
spatial_root.mkdir(parents=True, exist_ok=True)
|
| 106 |
+
cmd = [
|
| 107 |
+
str(exe),
|
| 108 |
+
"export",
|
| 109 |
+
"--meta",
|
| 110 |
+
str(meta_path),
|
| 111 |
+
"--outdir",
|
| 112 |
+
str(outdir),
|
| 113 |
+
"--spatial-root",
|
| 114 |
+
str(spatial_root),
|
| 115 |
+
]
|
| 116 |
+
if fields:
|
| 117 |
+
cmd += ["--fields", fields]
|
| 118 |
+
proc = subprocess.run(cmd, check=True, text=True, capture_output=True)
|
| 119 |
+
try:
|
| 120 |
+
return json.loads(proc.stdout)
|
| 121 |
+
except json.JSONDecodeError:
|
| 122 |
+
return {"ok": True, "stdout": proc.stdout.strip()}
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def parse_args() -> argparse.Namespace:
|
| 126 |
+
ap = argparse.ArgumentParser(description="Export StormScope .npz cycles into rustwx WXA stores")
|
| 127 |
+
ap.add_argument("--meta", default="", help="cycle metadata JSON; defaults to latest completed cycle")
|
| 128 |
+
ap.add_argument("--outdir", default=str(default_outdir()))
|
| 129 |
+
ap.add_argument("--spatial-root", default=str(default_spatial_root()))
|
| 130 |
+
ap.add_argument("--binary", default="")
|
| 131 |
+
ap.add_argument("--fields", default="", help="comma-separated field allowlist")
|
| 132 |
+
ap.add_argument("--force", action="store_true")
|
| 133 |
+
return ap.parse_args()
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def main() -> int:
|
| 137 |
+
args = parse_args()
|
| 138 |
+
outdir = Path(args.outdir)
|
| 139 |
+
meta = Path(args.meta) if args.meta else latest_meta(outdir)
|
| 140 |
+
result = export_cycle(
|
| 141 |
+
meta,
|
| 142 |
+
outdir,
|
| 143 |
+
Path(args.spatial_root),
|
| 144 |
+
binary=args.binary or None,
|
| 145 |
+
fields=args.fields or None,
|
| 146 |
+
force=args.force,
|
| 147 |
+
)
|
| 148 |
+
print(json.dumps(result, indent=2))
|
| 149 |
+
return 0
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
if __name__ == "__main__":
|
| 153 |
+
raise SystemExit(main())
|