Add clean FireWx-FM ICLR technical-report release
#3
by yx21e - opened
This view is limited to 50 files because it contains too many changes. See the raw diff here.
- .gitattributes +5 -44
- .gitignore +0 -6
- CITATION.cff +10 -0
- LICENSE +23 -7
- README.md +68 -385
- REPRODUCIBILITY.md +102 -0
- artifacts/manifests/paper_outputs.sha256 +0 -15
- artifacts/manifests/paper_outputs.yml +0 -32
- artifacts/results/downstream_summary.md +26 -0
- artifacts/results/fireprone_contract_progression_summary.json +0 -0
- artifacts/results/pixel_context_tune3_summary.md +15 -0
- artifacts/results/selection_regret_all_backbones_20260504.csv +0 -25
- artifacts/results/selection_regret_all_backbones_20260504.json +0 -0
- artifacts/results/selection_regret_head_metrics.csv +0 -241
- artifacts/results/selection_regret_per_seed.csv +0 -121
- artifacts/results/selection_regret_rq2_figure_values.csv +0 -12
- artifacts/results/selection_regret_scope_sweep_20260505.csv +0 -45
- artifacts/results/selection_regret_scope_sweep_20260505.json +0 -0
- artifacts/results/selection_regret_summary.csv +0 -25
- artifacts/results/supporting_table4_event_repaired_compsci_20260902_summary_clean.md +36 -0
- artifacts/results/supporting_table4_smoke_heat_compsci_20260902_summary_clean.md +29 -0
- artifacts/results/supporting_table4_smoke_heat_fill_compsci_20260902_summary_clean.md +24 -0
- artifacts/results/tune4_repaired_eventfm_summary.md +32 -0
- artifacts/results/tune4_throttled_summary.md +35 -0
- assets/release_contents.svg +0 -33
- assets/wildfire_fm_model_card.svg +0 -48
- data_downloader/README.md +0 -133
- data_sources/DATA_SOURCES.md +48 -65
- docs/artifact_map.md +0 -33
- docs/huggingface_release_design.md +0 -33
- docs/hugh_handoff_status.md +0 -49
- docs/inference_contract.md +76 -0
- docs/model_card.md +39 -0
- docs/reproducibility_manifest.md +76 -0
- docs/serving_notes.md +48 -0
- docs/split_design.md +77 -0
- examples/final_prediction/README.md +17 -0
- examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png +0 -0
- examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif +0 -0
- assets/primary_rank_change_final.png → examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif +2 -2
- examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json +72 -0
- assets/selection_regret_final.png → examples/final_prediction/firewxfm_conus_20260706_t12_usgs_like_visual_rgba_1000m.tif +2 -2
- experiments/README.md +0 -25
- experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py +0 -335
- experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py +0 -656
- experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py +0 -333
- experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py +0 -350
- experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py +0 -344
- experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py +0 -353
- experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py +0 -306
.gitattributes
CHANGED
|
@@ -1,46 +1,7 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.
|
| 5 |
-
*.
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
assets/primary_rank_change_preview.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
-
assets/selection_regret_preview.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
-
assets/task_rank_map_preview.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
-
paper/manuscript_final.pdf filter=lfs diff=lfs merge=lfs -text
|
| 40 |
-
paper_outputs/figures/fig_selection_regret_scatter.pdf filter=lfs diff=lfs merge=lfs -text
|
| 41 |
-
paper_outputs/figures/overview_wildfire.pdf filter=lfs diff=lfs merge=lfs -text
|
| 42 |
-
assets/overview_final.png filter=lfs diff=lfs merge=lfs -text
|
| 43 |
-
assets/primary_rank_change_final.png filter=lfs diff=lfs merge=lfs -text
|
| 44 |
-
assets/selection_regret_final.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
-
assets/supporting_rank_map_final.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
-
paper/wildfire_fm_evaluation_contracts.pdf filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.tif filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.tiff filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.pdf filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 7 |
*.npz filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
DELETED
|
@@ -1,6 +0,0 @@
|
|
| 1 |
-
__pycache__/
|
| 2 |
-
*.py[cod]
|
| 3 |
-
.DS_Store
|
| 4 |
-
.ipynb_checkpoints/
|
| 5 |
-
*.log
|
| 6 |
-
tmp/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
CITATION.cff
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cff-version: 1.2.0
|
| 2 |
+
title: "FireWx-FM"
|
| 3 |
+
message: "If you use this repository, please cite the FireWx-FM technical report or model release."
|
| 4 |
+
type: software
|
| 5 |
+
authors:
|
| 6 |
+
- family-names: Yang
|
| 7 |
+
given-names: X.
|
| 8 |
+
date-released: 2026-07-02
|
| 9 |
+
repository-code: "https://github.com/yx21e/Wildfire-fm-evaluation-contracts"
|
| 10 |
+
abstract: "A wildfire-specialized gridded reference model for 12-hour active-fire occupancy prediction over the Lower 48 United States."
|
LICENSE
CHANGED
|
@@ -1,9 +1,25 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
-
|
| 4 |
-
Paper-output tables, figures, and bundled summary artifacts are released for
|
| 5 |
-
non-commercial research and review use with attribution.
|
| 6 |
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Code in this repository is released for research use under the MIT License.
|
| 2 |
|
| 3 |
+
Copyright (c) 2026 FireWx-FM contributors
|
|
|
|
|
|
|
| 4 |
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
| 6 |
+
the code files in this repository and associated documentation files to deal in
|
| 7 |
+
the code without restriction, including without limitation the rights to use,
|
| 8 |
+
copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
| 9 |
+
code, and to permit persons to whom the code is furnished to do so, subject to
|
| 10 |
+
the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the code.
|
| 14 |
+
|
| 15 |
+
THE CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
| 16 |
+
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
| 17 |
+
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
| 18 |
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
| 19 |
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
| 20 |
+
CONNECTION WITH THE CODE OR THE USE OR OTHER DEALINGS IN THE CODE.
|
| 21 |
+
|
| 22 |
+
Raw NOAA, NASA FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan
|
| 23 |
+
datasets are not redistributed by this
|
| 24 |
+
repository. Users must obtain source data from original providers and comply
|
| 25 |
+
with the applicable terms.
|
README.md
CHANGED
|
@@ -5,422 +5,105 @@ tags:
|
|
| 5 |
- geospatial
|
| 6 |
- weather
|
| 7 |
- earth-observation
|
| 8 |
-
- foundation-models
|
| 9 |
-
- evaluation
|
| 10 |
- pytorch
|
|
|
|
| 11 |
pipeline_tag: image-segmentation
|
| 12 |
library_name: pytorch
|
| 13 |
-
pretty_name:
|
| 14 |
---
|
| 15 |
|
| 16 |
-
|
| 17 |
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
-
|
| 23 |
|
| 24 |
-
|
| 25 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/resolve/main/paper/wildfire_fm_evaluation_contracts.pdf"><img alt="Paper PDF" src="https://img.shields.io/badge/Paper-compiled%20PDF-0F5C5F?style=flat-square"></a>
|
| 26 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/models/wildfire_fm/checkpoints"><img alt="Checkpoints" src="https://img.shields.io/badge/Checkpoints-5%20seeds-C66B2D?style=flat-square"></a>
|
| 27 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/models/wildfire_fm/modeling_unet.py"><img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-model%20code-EE4C2C?style=flat-square&logo=pytorch&logoColor=white"></a>
|
| 28 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/data_sources/DATA_SOURCES.md"><img alt="Data notes" src="https://img.shields.io/badge/Data-source%20notes-536065?style=flat-square"></a>
|
| 29 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/artifacts/results"><img alt="Results" src="https://img.shields.io/badge/Results-sanitized%20summaries-0F5C5F?style=flat-square"></a>
|
| 30 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/assets"><img alt="Figures" src="https://img.shields.io/badge/Figures-final%20previews-C66B2D?style=flat-square"></a>
|
| 31 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/data_sources/DATA_SOURCES.md"><img alt="Grid" src="https://img.shields.io/badge/Grid-5%20km%20EPSG%3A5070-0F5C5F?style=flat-square"></a>
|
| 32 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#quick-start"><img alt="Input" src="https://img.shields.io/badge/Input-16%20channels-C66B2D?style=flat-square"></a>
|
| 33 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#task-contract-snapshot"><img alt="Tasks" src="https://img.shields.io/badge/Contracts-6%20task%20views-536065?style=flat-square"></a>
|
| 34 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-MIT-0F5C5F?style=flat-square"></a>
|
| 35 |
-
</p>
|
| 36 |
-
|
| 37 |
-
<p>
|
| 38 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/paper/wildfire_fm_evaluation_contracts.pdf"><b>Read the paper PDF</b></a> ·
|
| 39 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/models/wildfire_fm/checkpoints"><b>Download checkpoints</b></a> ·
|
| 40 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/training"><b>Training/data loader</b></a> ·
|
| 41 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/spatial_serving"><b>Custom spatial serving</b></a> ·
|
| 42 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#quick-start"><b>Load the model</b></a> ·
|
| 43 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#task-contract-snapshot"><b>Inspect task-contract results</b></a>
|
| 44 |
-
</p>
|
| 45 |
-
|
| 46 |
-
<h3>5 California Seeded Checkpoints · 16-Channel Gridded Input · 12-Hour Occupancy Lead · CONUS Retraining Recipe</h3>
|
| 47 |
-
|
| 48 |
-
<p>
|
| 49 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#why-wildfire-fm">Why WildFIRE-FM</a> ·
|
| 50 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#release-navigation">Release Navigation</a> ·
|
| 51 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#visual-tour">Visual Tour</a> ·
|
| 52 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#quick-start">Quick Start</a> ·
|
| 53 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/training">Training Pipeline</a> ·
|
| 54 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/spatial_serving">Spatial Serving</a> ·
|
| 55 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/training/NATIONWIDE_RETRAINING.md">CONUS Retraining</a> ·
|
| 56 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/docs/hugh_handoff_status.md">Integration Handoff</a> ·
|
| 57 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#task-contract-snapshot">Task Snapshot</a> ·
|
| 58 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#data-sources">Data Sources</a> ·
|
| 59 |
-
<a href="https://huggingface.co/RAI-Lab/Wildfire-FM#how-to-cite">How to Cite</a>
|
| 60 |
-
</p>
|
| 61 |
-
|
| 62 |
-
</div>
|
| 63 |
-
|
| 64 |
-
---
|
| 65 |
-
|
| 66 |
-
<p align="center">
|
| 67 |
-
<img src="https://huggingface.co/RAI-Lab/Wildfire-FM/resolve/main/assets/wildfire_fm_model_card.svg" alt="WildFIRE-FM overview" width="95%">
|
| 68 |
-
</p>
|
| 69 |
-
|
| 70 |
-
## Release Navigation
|
| 71 |
-
|
| 72 |
-
<table>
|
| 73 |
-
<tr>
|
| 74 |
-
<td width="33%" valign="top">
|
| 75 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/paper/wildfire_fm_evaluation_contracts.pdf">Paper PDF</a></h3>
|
| 76 |
-
<p>Read the compiled manuscript that defines the fixed-contract wildfire transfer setting and reports the final results.</p>
|
| 77 |
-
</td>
|
| 78 |
-
<td width="33%" valign="top">
|
| 79 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/models/wildfire_fm/checkpoints">Model Checkpoints</a></h3>
|
| 80 |
-
<p>Access five seeded WildFIRE-FM weights with manifest metadata and SHA-256 hashes for release auditing.</p>
|
| 81 |
-
</td>
|
| 82 |
-
<td width="33%" valign="top">
|
| 83 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/models/wildfire_fm/modeling_unet.py">Model Code</a></h3>
|
| 84 |
-
<p>Inspect the compact U-Net implementation used to load the released wildfire occupancy backbone.</p>
|
| 85 |
-
</td>
|
| 86 |
-
</tr>
|
| 87 |
-
<tr>
|
| 88 |
-
<td width="33%" valign="top">
|
| 89 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/training">Training/Data Loader</a></h3>
|
| 90 |
-
<p>Read the original cache builder, tensor assembly, tile sampler, full-map loader, and training loop.</p>
|
| 91 |
-
</td>
|
| 92 |
-
<td width="33%" valign="top">
|
| 93 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/spatial_serving">Custom Spatial Serving</a></h3>
|
| 94 |
-
<p>Aggregate native grid probabilities to any user-defined polygon granularity.</p>
|
| 95 |
-
</td>
|
| 96 |
-
<td width="33%" valign="top">
|
| 97 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM#quick-start">Quick Start</a></h3>
|
| 98 |
-
<p>Clone the repository, instantiate the model, load a seeded checkpoint, and run the artifact check.</p>
|
| 99 |
-
</td>
|
| 100 |
-
</tr>
|
| 101 |
-
<tr>
|
| 102 |
-
<td width="33%" valign="top">
|
| 103 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM#task-contract-snapshot">Task Snapshot</a></h3>
|
| 104 |
-
<p>View the final-paper task-contract summary for occupancy, spread, retrieval, burned area, smoke, and heat.</p>
|
| 105 |
-
</td>
|
| 106 |
-
<td width="33%" valign="top">
|
| 107 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/tree/main/artifacts/results">Numeric Artifacts</a></h3>
|
| 108 |
-
<p>Use sanitized CSV and JSON summaries that back the public model-card tables and figure previews.</p>
|
| 109 |
-
</td>
|
| 110 |
-
<td width="33%" valign="top">
|
| 111 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM#visual-tour">Visual Tour</a></h3>
|
| 112 |
-
<p>Browse final-paper previews for matching-rule sensitivity, selection regret, and task-form ranking changes.</p>
|
| 113 |
-
</td>
|
| 114 |
-
</tr>
|
| 115 |
-
<tr>
|
| 116 |
-
<td width="33%" valign="top">
|
| 117 |
-
<h3><a href="https://huggingface.co/RAI-Lab/Wildfire-FM/blob/main/data_sources/DATA_SOURCES.md">Data Sources</a></h3>
|
| 118 |
-
<p>See the public provider links and roles for HRRR, FIRMS, LANDFIRE, WRC, LandScan, WFIGS, and MTBS.</p>
|
| 119 |
-
</td>
|
| 120 |
-
</tr>
|
| 121 |
-
</table>
|
| 122 |
-
|
| 123 |
-
---
|
| 124 |
-
|
| 125 |
-
## Why WildFIRE-FM
|
| 126 |
-
|
| 127 |
-
Wildfire Earth-FM transfer scores depend strongly on the contract used for comparison: task form, metric, matching rule, spatial scope, and head-selection criterion. The paper studies these choices by holding outputs or features fixed and then changing only the evaluation contract.
|
| 128 |
-
|
| 129 |
-
WildFIRE-FM is the in-region reference model used in those comparisons. It is trained for 12-hour gridded wildfire occupancy on a California 5 km grid, then evaluated under the same task-specific contracts as the transferred Earth-FM backbones.
|
| 130 |
-
|
| 131 |
-
The released weights are California regional checkpoints, not nationwide-trained weights. For nationwide use, the release includes a CONUS cache and training recipe under `training/NATIONWIDE_RETRAINING.md`.
|
| 132 |
-
|
| 133 |
-
This Hub repository is model-first. It provides the released weights, model-loading code, compact result summaries, paper previews, and source-data access notes needed to inspect the release boundary before running larger local reproductions.
|
| 134 |
-
|
| 135 |
-
---
|
| 136 |
-
|
| 137 |
-
## Key Features
|
| 138 |
-
|
| 139 |
-
- **Wildfire-specialized reference model:** Compact U-Net for 12-hour occupancy prediction on a projected California grid.
|
| 140 |
-
- **Five seeded checkpoints:** Seeds `1`, `7`, `42`, `99`, and `123` are released with manifest hashes.
|
| 141 |
-
- **Fixed-contract artifacts:** Compact summaries cover matching-rule, head-selection, and task-form comparisons from the final paper.
|
| 142 |
-
- **Source-data aware release:** Provider links and data roles are documented for each public resource used by the study.
|
| 143 |
-
- **Compiled paper PDF:** The readable manuscript is available directly from the Hub release.
|
| 144 |
-
- **Paper-aligned previews:** Figure assets summarize selection regret, supporting-task rank changes, and primary-task rank changes.
|
| 145 |
-
|
| 146 |
-
<table>
|
| 147 |
-
<tr>
|
| 148 |
-
<td width="50%" valign="top">
|
| 149 |
-
<h3>Reference model for wildfire transfer</h3>
|
| 150 |
-
<p>WildFIRE-FM combines regional weather, active-fire supervision, and static fuel, canopy, housing, and population context on the same grid used by the occupancy contract.</p>
|
| 151 |
-
</td>
|
| 152 |
-
<td width="50%" valign="top">
|
| 153 |
-
<h3>Contract-first evaluation artifacts</h3>
|
| 154 |
-
<p>The released summaries keep metric roles separate across decision, overlap, ranking, regression, and supporting-task checks.</p>
|
| 155 |
-
</td>
|
| 156 |
-
</tr>
|
| 157 |
-
<tr>
|
| 158 |
-
<td width="50%" valign="top">
|
| 159 |
-
<h3>Local-data reproducibility path</h3>
|
| 160 |
-
<p>Full raw-data reruns use provider-hosted sources and local feature caches. The Hub release documents those inputs and provides lightweight artifact checks.</p>
|
| 161 |
-
</td>
|
| 162 |
-
<td width="50%" valign="top">
|
| 163 |
-
<h3>Final-paper result previews</h3>
|
| 164 |
-
<p>The model card highlights the paper's fixed-output, fixed-feature, and task-form findings without requiring manuscript source files.</p>
|
| 165 |
-
</td>
|
| 166 |
-
</tr>
|
| 167 |
-
</table>
|
| 168 |
-
|
| 169 |
-
---
|
| 170 |
-
|
| 171 |
-
## Visual Tour
|
| 172 |
-
|
| 173 |
-
<p align="center">
|
| 174 |
-
<img src="https://huggingface.co/RAI-Lab/Wildfire-FM/resolve/main/assets/release_contents.svg" alt="Release contents" width="92%">
|
| 175 |
-
</p>
|
| 176 |
-
|
| 177 |
-
**Release contents.** The Hub repository is organized around weights, model code, source-data notes, compact artifacts, and final-paper visual previews.
|
| 178 |
-
|
| 179 |
-
<p align="center">
|
| 180 |
-
<img src="https://huggingface.co/RAI-Lab/Wildfire-FM/resolve/main/assets/selection_regret_final.png" alt="Head-selection regret" width="88%">
|
| 181 |
-
</p>
|
| 182 |
-
|
| 183 |
-
**Head-selection regret.** Choosing a lightweight head by a ranking metric can lose decision performance under the same frozen features.
|
| 184 |
-
|
| 185 |
-
<p align="center">
|
| 186 |
-
<img src="https://huggingface.co/RAI-Lab/Wildfire-FM/resolve/main/assets/supporting_rank_map_final.png" alt="Supporting task rank map" width="88%">
|
| 187 |
-
</p>
|
| 188 |
-
|
| 189 |
-
**Supporting-task rank map.** Backbone ordering changes across burned area, analog retrieval, smoke PM2.5, and extreme heat task contracts.
|
| 190 |
-
|
| 191 |
-
<p align="center">
|
| 192 |
-
<img src="https://huggingface.co/RAI-Lab/Wildfire-FM/resolve/main/assets/primary_rank_change_final.png" alt="Primary rank changes" width="88%">
|
| 193 |
-
</p>
|
| 194 |
-
|
| 195 |
-
**Primary-task rank changes.** Primary-task rankings change under fixed matching-rule and task-form choices.
|
| 196 |
-
|
| 197 |
-
---
|
| 198 |
-
|
| 199 |
-
## At A Glance
|
| 200 |
-
|
| 201 |
-
| Signal | Release value |
|
| 202 |
|---|---|
|
| 203 |
-
|
|
| 204 |
-
|
|
| 205 |
-
|
|
| 206 |
-
|
|
| 207 |
-
|
|
| 208 |
-
|
|
| 209 |
-
|
|
| 210 |
-
|
|
| 211 |
-
|
|
| 212 |
-
|
|
| 213 |
-
|
|
| 214 |
-
|
| 215 |
-
### Input Channel Contract
|
| 216 |
-
|
| 217 |
-
The pretrained checkpoints expect a fixed 16-channel tensor in `[channel, y, x]`
|
| 218 |
-
order. Channel order matters.
|
| 219 |
-
Inputs use native source units with no mean/std normalization, min/max scaling,
|
| 220 |
-
or unit conversion. Invalid or missing values are zero-filled, with validity
|
| 221 |
-
masks included as channels 10 and 11.
|
| 222 |
-
|
| 223 |
-
| Channels | Names | Source |
|
| 224 |
-
|---:|---|---|
|
| 225 |
-
| 0-9 | `t2m`, `d2m`, `u10`, `v10`, `cape`, `sp`, `blh`, `vis`, `prate`, `tp` | NOAA HRRR |
|
| 226 |
-
| 10 | `firewx_valid` | Cache-level dynamic input presence mask; 1.0 everywhere in this release |
|
| 227 |
-
| 11 | `static_valid` | Fraction of static layers valid after reprojection |
|
| 228 |
-
| 12-15 | `fuel_fbfm40`, `canopy_cover`, `housing_density`, `population` | LANDFIRE, WRC housing density, LandScan |
|
| 229 |
-
|
| 230 |
-
FIRMS provides the occupancy target, not an input channel. WFIGS and MTBS are
|
| 231 |
-
used for supporting event-level tasks and are not part of the 16-channel
|
| 232 |
-
pretrained occupancy input. Full channel-level details are in
|
| 233 |
-
`data_sources/DATA_SOURCES.md`; the same contract is available as
|
| 234 |
-
`models/wildfire_fm/input_channels.json`.
|
| 235 |
-
The `cape` channel is the HRRR surface instantaneous CAPE field selected with
|
| 236 |
-
`typeOfLevel=surface` and `stepType=instant`.
|
| 237 |
-
For validity channels, `firewx_valid` is 1.0 everywhere in the released
|
| 238 |
-
California regional cache and is not a per-variable HRRR missing-data mask;
|
| 239 |
-
`static_valid` is the fraction of the four static layers valid after
|
| 240 |
-
reprojection at each grid cell.
|
| 241 |
-
|
| 242 |
-
---
|
| 243 |
-
|
| 244 |
-
## Hugging Face Release
|
| 245 |
-
|
| 246 |
-
This Hub repository is a model release with paper-aligned reproducibility artifacts.
|
| 247 |
-
|
| 248 |
-
| Folder | Purpose | Example |
|
| 249 |
-
|---|---|---|
|
| 250 |
-
| `models/wildfire_fm/` | Model code, manifests, checkpoint metadata | `models/wildfire_fm/modeling_unet.py` |
|
| 251 |
-
| `models/wildfire_fm/input_channels.json` | Machine-readable 16-channel input contract | `models/wildfire_fm/input_channels.json` |
|
| 252 |
-
| `models/wildfire_fm/checkpoints/` | Five seeded PyTorch checkpoints | `seed_1/best_firms_prauc.pt` |
|
| 253 |
-
| `training/` | Original cache builder, data loader, tile sampler, and training loop | `train_cold_tiled_mainline.py` |
|
| 254 |
-
| `training/NATIONWIDE_RETRAINING.md` | CONUS retraining recipe and data coverage notes | `stage1_cache_conus_hrrr_us_5km_l12_template.json` |
|
| 255 |
-
| `spatial_serving/` | Grid-to-polygon serving adapter for custom spatial granularities | `grid_to_polygons.py` |
|
| 256 |
-
| `data_downloader/hrrr_downloader.py` | Direct NOAA HRRR public archive downloader | `hrrr_download_manifest.json` |
|
| 257 |
-
| `docs/hugh_handoff_status.md` | Integration answers for channel order, masks, normalization, HRRR, and serving boundary | handoff status |
|
| 258 |
-
| `paper/` | Compiled paper PDF | `wildfire_fm_evaluation_contracts.pdf` |
|
| 259 |
-
| `assets/` | Hub-page visuals and final-paper previews | `selection_regret_final.png` |
|
| 260 |
-
| `paper_outputs/figures/` | Selected final-paper figure PDFs | `fig_rank_heatmap1.pdf` |
|
| 261 |
-
| `artifacts/results/` | Sanitized compact numeric summaries | `selection_regret_scope_sweep_20260505.csv` |
|
| 262 |
-
| `data_sources/` | Public data-source roles and access notes | `DATA_SOURCES.md` |
|
| 263 |
-
| `experiments/` | Sanitized raw-rerun references and Slurm template | `slurm/submit_template.sbatch` |
|
| 264 |
-
|
| 265 |
-
Full local reruns require users to prepare the provider-hosted raw data and comparator assets described in `data_sources/`.
|
| 266 |
-
|
| 267 |
-
---
|
| 268 |
|
| 269 |
## Quick Start
|
| 270 |
|
| 271 |
-
Clone the Hub repository or download the files you need:
|
| 272 |
-
|
| 273 |
```bash
|
| 274 |
-
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
| 276 |
```
|
| 277 |
|
| 278 |
-
|
|
|
|
| 279 |
|
| 280 |
-
|
| 281 |
-
import torch
|
| 282 |
-
from models.wildfire_fm.modeling_unet import UNetSmallFlex
|
| 283 |
|
| 284 |
-
model
|
| 285 |
-
in_ch=16,
|
| 286 |
-
base=32,
|
| 287 |
-
dropout=0.1,
|
| 288 |
-
norm_type="group",
|
| 289 |
-
norm_groups=8,
|
| 290 |
-
use_aux_spatial_head=True,
|
| 291 |
-
)
|
| 292 |
-
checkpoint = torch.load(
|
| 293 |
-
"models/wildfire_fm/checkpoints/seed_1/best_firms_prauc.pt",
|
| 294 |
-
map_location="cpu",
|
| 295 |
-
)
|
| 296 |
-
state = checkpoint.get("model", checkpoint)
|
| 297 |
-
model.load_state_dict(state)
|
| 298 |
-
model.eval()
|
| 299 |
-
```
|
| 300 |
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
above and in `data_sources/DATA_SOURCES.md`.
|
| 309 |
-
|
| 310 |
-
---
|
| 311 |
-
|
| 312 |
-
## Task-Contract Snapshot
|
| 313 |
-
|
| 314 |
-
The card below reports the best final-paper mean for each displayed task contract, with the winning backbone named in the final column.
|
| 315 |
-
|
| 316 |
-
| Task contract | Best final-paper mean | Winner |
|
| 317 |
-
|---|---:|---|
|
| 318 |
-
| Occupancy union F1 | `60.1506 ± 7.5865` percent | ClimaX |
|
| 319 |
-
| Fire-spread spatial F1 | `80.9700 ± 2.0200` percent | WildFIRE-FM |
|
| 320 |
-
| Final burned-area log-RMSE | `1.1657 ± 0.0126`, lower is better | WildFIRE-FM |
|
| 321 |
-
| Analog retrieval nDCG@10 | `0.5099 ± 0.0336` | WildFIRE-FM |
|
| 322 |
-
| Smoke PM2.5 RMSE | `4.4403 ± 0.0488`, lower is better | AlphaEarth |
|
| 323 |
-
| Extreme-heat RMSE-C | `0.2179 ± 0.0043`, lower is better | WildFIRE-FM |
|
| 324 |
-
|
| 325 |
-
The release provides sanitized CSV/JSON summaries used to audit displayed values.
|
| 326 |
|
| 327 |
-
|
|
|
|
|
|
|
| 328 |
|
| 329 |
-
##
|
| 330 |
|
| 331 |
-
|
| 332 |
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
-
|
| 336 |
-
-
|
| 337 |
-
-
|
| 338 |
-
-
|
|
|
|
| 339 |
|
| 340 |
-
--
|
|
|
|
| 341 |
|
| 342 |
-
##
|
| 343 |
|
| 344 |
-
``
|
| 345 |
-
WildFIRE-FM/
|
| 346 |
-
README.md
|
| 347 |
-
LICENSE
|
| 348 |
-
requirements.txt
|
| 349 |
-
models/wildfire_fm/
|
| 350 |
-
modeling_unet.py
|
| 351 |
-
checkpoint_manifest.json
|
| 352 |
-
checkpoints/seed_*/best_firms_prauc.pt
|
| 353 |
-
training/
|
| 354 |
-
build_phase1_cache_regional_hrrr.py
|
| 355 |
-
train_cold_tiled_mainline.py
|
| 356 |
-
configs/
|
| 357 |
-
spatial_serving/
|
| 358 |
-
grid_to_polygons.py
|
| 359 |
-
data_downloader/
|
| 360 |
-
hrrr_downloader.py
|
| 361 |
-
paper/
|
| 362 |
-
wildfire_fm_evaluation_contracts.pdf
|
| 363 |
-
assets/
|
| 364 |
-
wildfire_fm_model_card.svg
|
| 365 |
-
release_contents.svg
|
| 366 |
-
selection_regret_final.png
|
| 367 |
-
supporting_rank_map_final.png
|
| 368 |
-
primary_rank_change_final.png
|
| 369 |
-
paper_outputs/figures/
|
| 370 |
-
matching.pdf
|
| 371 |
-
fig_task_contract_tiles.pdf
|
| 372 |
-
fig_selection_regret_scatter.pdf
|
| 373 |
-
fig_rank_heatmap1.pdf
|
| 374 |
-
artifacts/
|
| 375 |
-
manifests/
|
| 376 |
-
results/
|
| 377 |
-
data_sources/
|
| 378 |
-
DATA_SOURCES.md
|
| 379 |
-
experiments/
|
| 380 |
-
raw_reference/
|
| 381 |
-
slurm/
|
| 382 |
-
scripts/
|
| 383 |
-
audit_release.py
|
| 384 |
-
reproduce_paper_outputs.py
|
| 385 |
-
```
|
| 386 |
|
| 387 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
|
| 389 |
-
|
|
|
|
| 390 |
|
| 391 |
-
|
| 392 |
|
| 393 |
```bibtex
|
| 394 |
-
@misc{
|
| 395 |
-
title = {
|
| 396 |
-
author = {
|
| 397 |
year = {2026},
|
| 398 |
-
note = {
|
| 399 |
}
|
| 400 |
```
|
| 401 |
-
|
| 402 |
-
The citation will be updated with arXiv metadata after the preprint is public.
|
| 403 |
-
|
| 404 |
-
---
|
| 405 |
-
|
| 406 |
-
## News
|
| 407 |
-
|
| 408 |
-
- **2026-05-18** - The model card was reorganized with a larger navigation panel and final-paper visual previews.
|
| 409 |
-
- **2026-05-13** - The Hub release added the compiled paper PDF without checklist pages.
|
| 410 |
-
- **2026-05-13** - Five seeded WildFIRE-FM checkpoints were uploaded with SHA-256 metadata.
|
| 411 |
-
|
| 412 |
-
---
|
| 413 |
-
|
| 414 |
-
## Scope
|
| 415 |
-
|
| 416 |
-
WildFIRE-FM is scoped to the paper's California regional grid and fixed-contract wildfire transfer comparisons. Use in a new region or grid should start with local preprocessing, validation, and contract-specific evaluation. The release is designed for research inspection, artifact-level checking, and model-loading experiments rather than operational alerting.
|
| 417 |
-
|
| 418 |
-
---
|
| 419 |
-
|
| 420 |
-
## License
|
| 421 |
-
|
| 422 |
-
This release is provided under the MIT license. External data and comparator assets remain governed by their original providers and terms of use.
|
| 423 |
-
|
| 424 |
-
---
|
| 425 |
-
|
| 426 |
-
WildFIRE-FM is released to make wildfire Earth-FM transfer comparisons easier to inspect, reproduce at the artifact level, and evaluate under explicit contracts.
|
|
|
|
| 5 |
- geospatial
|
| 6 |
- weather
|
| 7 |
- earth-observation
|
|
|
|
|
|
|
| 8 |
- pytorch
|
| 9 |
+
- reproducibility
|
| 10 |
pipeline_tag: image-segmentation
|
| 11 |
library_name: pytorch
|
| 12 |
+
pretty_name: FireWx-FM
|
| 13 |
---
|
| 14 |
|
| 15 |
+
# FireWx-FM
|
| 16 |
|
| 17 |
+
FireWx-FM is a wildfire-focused gridded model for 12-hour active-fire occupancy
|
| 18 |
+
prediction over the Lower 48 United States. The released code maps a fixed
|
| 19 |
+
16-channel tensor on a 5 km EPSG:5070 grid to an active-fire occupancy
|
| 20 |
+
probability map.
|
| 21 |
|
| 22 |
+
This Hugging Face release is the clean ICLR/technical-report companion package:
|
| 23 |
+
it includes model code, checkpoints, inference utilities, training/cache
|
| 24 |
+
templates, public data-source notes, example outputs, the ICLR manuscript
|
| 25 |
+
source, and the technical report source.
|
| 26 |
|
| 27 |
+
## Contents
|
| 28 |
|
| 29 |
+
| Path | Purpose |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|---|---|
|
| 31 |
+
| `firewxfm/` | Model definition and tiled CONUS inference code. |
|
| 32 |
+
| `models/checkpoints/` | Released PyTorch checkpoints. |
|
| 33 |
+
| `models/metadata/` | Input-channel contract, normalization statistics, and run metadata. |
|
| 34 |
+
| `training/` | Cache-building, training, and metric utilities. |
|
| 35 |
+
| `training/configs/` | Path-free templates for CONUS and multi-year reruns. |
|
| 36 |
+
| `scripts/` | Data preparation, release checks, live-stack builder, seam audit, and smoke tests. |
|
| 37 |
+
| `examples/final_prediction/` | Small final probability/preview example for inspection. |
|
| 38 |
+
| `data_sources/` | Source-data provenance and access notes. |
|
| 39 |
+
| `paper/iclr_source/` | ICLR manuscript LaTeX source. |
|
| 40 |
+
| `technical_report/source/` | Technical report LaTeX source. |
|
| 41 |
+
| `artifacts/results/` | Compact public result summaries without internal filesystem paths. |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
## Quick Start
|
| 44 |
|
|
|
|
|
|
|
| 45 |
```bash
|
| 46 |
+
python3 -m venv .venv
|
| 47 |
+
source .venv/bin/activate
|
| 48 |
+
pip install -r requirements.txt
|
| 49 |
+
python3 scripts/check_release_files.py
|
| 50 |
+
python3 scripts/smoke_inference.py --device cpu
|
| 51 |
```
|
| 52 |
|
| 53 |
+
The smoke test uses a synthetic tensor. Full scientific reproduction requires
|
| 54 |
+
the public source datasets listed in `data_sources/DATA_SOURCES.md`.
|
| 55 |
|
| 56 |
+
## Model Input Contract
|
|
|
|
|
|
|
| 57 |
|
| 58 |
+
The model expects a channel-first tensor with shape `[16,H,W]`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
|
| 60 |
+
| Channels | Source | Role |
|
| 61 |
+
|---:|---|---|
|
| 62 |
+
| 0-9 | NOAA HRRR | Dynamic weather fields. |
|
| 63 |
+
| 10-11 | Input builder | Dynamic and static validity masks. |
|
| 64 |
+
| 12 | LANDFIRE | Fire-behavior fuel model. |
|
| 65 |
+
| 13 | LANDFIRE | Canopy cover. |
|
| 66 |
+
| 14-15 | WRC / LandScan | Housing density and population; zeroed in the no-exposure serving run. |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
+
See `models/metadata/input_channels.json` and
|
| 69 |
+
`models/metadata/input_normalization_stats.json` for the exact machine-readable
|
| 70 |
+
contract.
|
| 71 |
|
| 72 |
+
## Inference
|
| 73 |
|
| 74 |
+
Run inference from a prebuilt 16-channel stack:
|
| 75 |
|
| 76 |
+
```bash
|
| 77 |
+
python3 -m firewxfm.serve_conus \
|
| 78 |
+
--input-npy /path/to/input_stack_16chw.npy \
|
| 79 |
+
--checkpoint models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt \
|
| 80 |
+
--normalization-stats models/metadata/input_normalization_stats.json \
|
| 81 |
+
--output-npy outputs/firewxfm_probability.npy
|
| 82 |
+
```
|
| 83 |
|
| 84 |
+
For full-map serving, use overlap-window inference with phase averaging. The
|
| 85 |
+
default settings are documented in `docs/serving_notes.md`.
|
| 86 |
|
| 87 |
+
## Reproducibility
|
| 88 |
|
| 89 |
+
Use `REPRODUCIBILITY.md` for the exact verification path:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
+
- artifact presence check,
|
| 92 |
+
- checkpoint smoke inference,
|
| 93 |
+
- ICLR source compilation,
|
| 94 |
+
- technical report source compilation,
|
| 95 |
+
- real-data cache rebuild and retraining commands.
|
| 96 |
|
| 97 |
+
Raw source data are not redistributed. This package documents where each source
|
| 98 |
+
comes from and provides path-free templates for local reproduction.
|
| 99 |
|
| 100 |
+
## Citation
|
| 101 |
|
| 102 |
```bibtex
|
| 103 |
+
@misc{firewxfm2026,
|
| 104 |
+
title = {FireWx-FM: A Wildfire Foundation Model Technical Report},
|
| 105 |
+
author = {Xu, Yangshuang and Dai, Yuyang and Chang, Liling and Wang, Qi and Dong, Yushun},
|
| 106 |
year = {2026},
|
| 107 |
+
note = {FireWx-FM code, checkpoints, and reproducibility package}
|
| 108 |
}
|
| 109 |
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
REPRODUCIBILITY.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reproducibility
|
| 2 |
+
|
| 3 |
+
This release is designed to make the FireWx-FM technical report and ICLR
|
| 4 |
+
manuscript inspectable and reproducible from documented ingredients.
|
| 5 |
+
|
| 6 |
+
## What Is Included
|
| 7 |
+
|
| 8 |
+
- Model and inference code in `firewxfm/`.
|
| 9 |
+
- Training, cache-building, and metric utilities in `training/`.
|
| 10 |
+
- Public-data preparation and audit scripts in `scripts/`.
|
| 11 |
+
- Two released PyTorch checkpoints in `models/checkpoints/`.
|
| 12 |
+
- Input-channel metadata, normalization statistics, and run summaries in
|
| 13 |
+
`models/metadata/`.
|
| 14 |
+
- A small final prediction example in `examples/final_prediction/`.
|
| 15 |
+
- ICLR manuscript source in `paper/iclr_source/`.
|
| 16 |
+
- Technical report source in `technical_report/source/`.
|
| 17 |
+
- Source-data notes in `data_sources/DATA_SOURCES.md`.
|
| 18 |
+
|
| 19 |
+
## What Is Not Included
|
| 20 |
+
|
| 21 |
+
Raw HRRR, FIRMS, LANDFIRE, Wildfire Risk to Communities, LandScan, WFIGS, and
|
| 22 |
+
MTBS data are not redistributed. Users must download those data from the
|
| 23 |
+
original providers and follow their licenses.
|
| 24 |
+
|
| 25 |
+
## Quick Verification
|
| 26 |
+
|
| 27 |
+
Install dependencies:
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
python3 -m venv .venv
|
| 31 |
+
source .venv/bin/activate
|
| 32 |
+
pip install -r requirements.txt
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
Check that required artifacts are present:
|
| 36 |
+
|
| 37 |
+
```bash
|
| 38 |
+
python3 scripts/check_release_files.py
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
Verify checkpoint/model compatibility with a synthetic tensor:
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
python3 scripts/smoke_inference.py \
|
| 45 |
+
--checkpoint models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt \
|
| 46 |
+
--device cpu
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
Compile the technical report source:
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
cd technical_report/source
|
| 53 |
+
tectonic main.tex
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
Compile the ICLR manuscript source:
|
| 57 |
+
|
| 58 |
+
```bash
|
| 59 |
+
cd paper/iclr_source
|
| 60 |
+
tectonic main.tex
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
## Inference From A Real 16-Channel Stack
|
| 64 |
+
|
| 65 |
+
After constructing a `[16,H,W]` stack that follows
|
| 66 |
+
`models/metadata/input_channels.json`, run:
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
python3 -m firewxfm.serve_conus \
|
| 70 |
+
--input-npy /path/to/input_stack_16chw.npy \
|
| 71 |
+
--checkpoint models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt \
|
| 72 |
+
--normalization-stats models/metadata/input_normalization_stats.json \
|
| 73 |
+
--output-npy outputs/firewxfm_probability.npy
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
The output is an `[H,W]` probability map for 12-hour active-fire occupancy.
|
| 77 |
+
|
| 78 |
+
## Rebuilding Training Caches
|
| 79 |
+
|
| 80 |
+
Use the template configs under `training/configs/` after replacing placeholder
|
| 81 |
+
paths with local source-data paths:
|
| 82 |
+
|
| 83 |
+
```bash
|
| 84 |
+
python3 scripts/prepare_firms_target_hour_csvs.py \
|
| 85 |
+
--daily-dir /path/to/firms_daily_csv \
|
| 86 |
+
--output-dir /path/to/firms_target_hour_csv \
|
| 87 |
+
--config training/configs/stage1_cache_conus_multiyear_template.json
|
| 88 |
+
|
| 89 |
+
python3 scripts/audit_firewxfm_phase0.py \
|
| 90 |
+
--config training/configs/stage1_cache_conus_multiyear_template.json \
|
| 91 |
+
--output artifacts/manifests/phase0_data_split_audit.json
|
| 92 |
+
|
| 93 |
+
python3 training/build_phase1_cache_regional_hrrr.py \
|
| 94 |
+
--config training/configs/stage1_cache_conus_multiyear_template.json
|
| 95 |
+
|
| 96 |
+
python3 training/train_cold_tiled_mainline.py \
|
| 97 |
+
--config training/configs/train_firewxfm_conus_multiyear_template.json \
|
| 98 |
+
--run-name firewxfm_conus_multiyear_noexposure_seed42
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
Large cache building and training should be run on compute nodes, not login
|
| 102 |
+
nodes.
|
artifacts/manifests/paper_outputs.sha256
DELETED
|
@@ -1,15 +0,0 @@
|
|
| 1 |
-
6db4f4aff90da8709edff97e4782aee9b4c5e8feefec7c7431a4ec8787cfe57c assets/overview_final.png
|
| 2 |
-
ca11c75c03078a9be26421b527ab5a49f5fc43ce8e5edd7da14af120a247b67c assets/primary_rank_change_final.png
|
| 3 |
-
1f992d7245b55b8674c00753efbd0e25f4e3be658e736b353772624dea2b7a8f assets/release_contents.svg
|
| 4 |
-
5552fb6cca6a0a683592e724b4bd562f923cf99c04e2abdb846546b1d67aecc4 assets/selection_regret_final.png
|
| 5 |
-
34807e65ca71365a26a3b74cae70e6b40ae6f2151110e12c53e0efa9f8b726aa assets/supporting_rank_map_final.png
|
| 6 |
-
7adb6a949bce3203b6cb10a458e115c60ba0b3efdb10f2ca1a1fd508924bc2ac assets/wildfire_fm_model_card.svg
|
| 7 |
-
95596f05829639ecb3041721fe3c871caacc49f30c831d35f2c503b1073b498d paper/wildfire_fm_evaluation_contracts.pdf
|
| 8 |
-
b369d13e0419fa8272ccdc994b6642f3b141248a879c030218e387c583537eb2 paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf
|
| 9 |
-
e3110c70c3cf8ecb8671163a401a155920266e3f907f9c6baf08e27ec6e6c410 paper_outputs/figures/fig_primary_rank_change_map.pdf
|
| 10 |
-
4e5b791ba4d136f722bd75a61097203836819ce9411def1caac4cc1e6d881275 paper_outputs/figures/fig_rank_heatmap1.pdf
|
| 11 |
-
fabb8b55aac901199cc03773741a26685becffd074f52568c93bee517c2c42c0 paper_outputs/figures/fig_selection_regret_scatter.pdf
|
| 12 |
-
bc4d35ad9cb4c1f9ba8f31c7c340d9684c9dd2d55f5a2e60604a2b58b90cbe40 paper_outputs/figures/fig_task_contract_tiles.pdf
|
| 13 |
-
c382f5d69f25cc2f5db174601a33d0fd0928b44910a2a4b1c131954bd42113d9 paper_outputs/figures/fig_task_rank_map.pdf
|
| 14 |
-
015ab951b0af5c130e4894092a5dd0bb0fd62e710467163a9df8246d8cf369f4 paper_outputs/figures/matching.pdf
|
| 15 |
-
7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755 paper_outputs/figures/overview_wildfire.pdf
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/manifests/paper_outputs.yml
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
# Auto-generated public-output manifest for the Hugging Face release.
|
| 2 |
-
outputs:
|
| 3 |
-
- path: assets/overview_final.png
|
| 4 |
-
sha256: 6db4f4aff90da8709edff97e4782aee9b4c5e8feefec7c7431a4ec8787cfe57c
|
| 5 |
-
- path: assets/primary_rank_change_final.png
|
| 6 |
-
sha256: ca11c75c03078a9be26421b527ab5a49f5fc43ce8e5edd7da14af120a247b67c
|
| 7 |
-
- path: assets/release_contents.svg
|
| 8 |
-
sha256: 1f992d7245b55b8674c00753efbd0e25f4e3be658e736b353772624dea2b7a8f
|
| 9 |
-
- path: assets/selection_regret_final.png
|
| 10 |
-
sha256: 5552fb6cca6a0a683592e724b4bd562f923cf99c04e2abdb846546b1d67aecc4
|
| 11 |
-
- path: assets/supporting_rank_map_final.png
|
| 12 |
-
sha256: 34807e65ca71365a26a3b74cae70e6b40ae6f2151110e12c53e0efa9f8b726aa
|
| 13 |
-
- path: assets/wildfire_fm_model_card.svg
|
| 14 |
-
sha256: 7adb6a949bce3203b6cb10a458e115c60ba0b3efdb10f2ca1a1fd508924bc2ac
|
| 15 |
-
- path: paper/wildfire_fm_evaluation_contracts.pdf
|
| 16 |
-
sha256: 95596f05829639ecb3041721fe3c871caacc49f30c831d35f2c503b1073b498d
|
| 17 |
-
- path: paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf
|
| 18 |
-
sha256: b369d13e0419fa8272ccdc994b6642f3b141248a879c030218e387c583537eb2
|
| 19 |
-
- path: paper_outputs/figures/fig_primary_rank_change_map.pdf
|
| 20 |
-
sha256: e3110c70c3cf8ecb8671163a401a155920266e3f907f9c6baf08e27ec6e6c410
|
| 21 |
-
- path: paper_outputs/figures/fig_rank_heatmap1.pdf
|
| 22 |
-
sha256: 4e5b791ba4d136f722bd75a61097203836819ce9411def1caac4cc1e6d881275
|
| 23 |
-
- path: paper_outputs/figures/fig_selection_regret_scatter.pdf
|
| 24 |
-
sha256: fabb8b55aac901199cc03773741a26685becffd074f52568c93bee517c2c42c0
|
| 25 |
-
- path: paper_outputs/figures/fig_task_contract_tiles.pdf
|
| 26 |
-
sha256: bc4d35ad9cb4c1f9ba8f31c7c340d9684c9dd2d55f5a2e60604a2b58b90cbe40
|
| 27 |
-
- path: paper_outputs/figures/fig_task_rank_map.pdf
|
| 28 |
-
sha256: c382f5d69f25cc2f5db174601a33d0fd0928b44910a2a4b1c131954bd42113d9
|
| 29 |
-
- path: paper_outputs/figures/matching.pdf
|
| 30 |
-
sha256: 015ab951b0af5c130e4894092a5dd0bb0fd62e710467163a9df8246d8cf369f4
|
| 31 |
-
- path: paper_outputs/figures/overview_wildfire.pdf
|
| 32 |
-
sha256: 7dca6ae4a9b179693802f47d24dd66734c0f332b372a7976832a0d429333b755
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/downstream_summary.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Forced Mean/Std Gap-Fill Summary
|
| 2 |
+
|
| 3 |
+
## table2_headcontrol
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
## table3_occupancy
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
## table3_spread
|
| 10 |
+
|
| 11 |
+
- Prithvi-WxC: strict_f1 20.72 +/- 3.80 (n=5); spatial_f1 57.32 +/- 2.14 (n=5); ap 5.04 +/- 0.40 (n=5)
|
| 12 |
+
- Reference: strict_f1 20.32 +/- 1.15 (n=5); spatial_f1 68.58 +/- 1.98 (n=5); ap 30.44 +/- 1.81 (n=5)
|
| 13 |
+
|
| 14 |
+
## table4_final_area
|
| 15 |
+
|
| 16 |
+
- Reference: log_rmse 1.17 +/- 0.01 (n=5); log_mae 1.05 +/- 0.01 (n=5); log_spearman 0.59 +/- 0.03 (n=5)
|
| 17 |
+
|
| 18 |
+
## table4_analog
|
| 19 |
+
|
| 20 |
+
- reference_strict: ndcg_at_10 0.52 +/- 0.00 (n=5); log_rmse 1.18 +/- 0.00 (n=5); log_mae 0.97 +/- 0.00 (n=5)
|
| 21 |
+
|
| 22 |
+
## table4_smoke
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
## table4_heat
|
| 26 |
+
|
artifacts/results/fireprone_contract_progression_summary.json
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
artifacts/results/pixel_context_tune3_summary.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# FireWx-FM Downstream Tune2 Diagnostic
|
| 2 |
+
|
| 3 |
+
## Spread
|
| 4 |
+
### wave1
|
| 5 |
+
- prithvi_wxc: strict_f1 20.72 +/- 3.80 (n=5); spatial_s4_f1 57.32 +/- 2.14 (n=5); AP 5.04 +/- 0.40 (n=5); Brier 0.01 +/- 0.01 (n=5)
|
| 6 |
+
- reference: strict_f1 20.32 +/- 1.15 (n=5); spatial_s4_f1 68.58 +/- 1.98 (n=5); AP 30.44 +/- 1.81 (n=5); Brier 0.06 +/- 0.01 (n=5)
|
| 7 |
+
|
| 8 |
+
### tune2
|
| 9 |
+
- pending
|
| 10 |
+
|
| 11 |
+
## Occupancy
|
| 12 |
+
- pixel_context_tune3: strict_f1 0.69 +/- 0.12 (n=5); t0_s3_f1 14.14 +/- 1.37 (n=5); t3_s3_f1 38.51 +/- 2.34 (n=5); PR-AUC 1.01 +/- 0.31 (n=5)
|
| 13 |
+
|
| 14 |
+
## Head Control
|
| 15 |
+
- pending
|
artifacts/results/selection_regret_all_backbones_20260504.csv
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
model_tag,label,scope,n,seeds,exact_regret_mean,exact_regret_std,tolerated_regret_mean,tolerated_regret_std,union_regret_mean,union_regret_std
|
| 2 |
-
reference,Reference,global,5,1 7 42 99 123,0.0,0.0,0.08783024981138902,0.09670495645481135,0.08783024981138902,0.09670495645481135
|
| 3 |
-
reference,Reference,fire_prone,5,1 7 42 99 123,0.0,0.0,0.03402707057672223,0.032044658643147844,0.03402707057672223,0.032044658643147844
|
| 4 |
-
prithvi_wxc,Prithvi-WxC,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 5 |
-
prithvi_wxc,Prithvi-WxC,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 6 |
-
aurora,Aurora,global,5,1 7 42 99 123,0.00020004882767231798,0.00026703384456332115,0.09851983041506818,0.1298781980037557,0.09851983041506818,0.1298781980037557
|
| 7 |
-
aurora,Aurora,fire_prone,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.14391889430974364,0.32121904665016227,0.14391889430974364,0.32121904665016227
|
| 8 |
-
climax,ClimaX,global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0012959969982639485,0.0017746169760203706,0.0012959969982639485,0.0017746169760203706
|
| 9 |
-
climax,ClimaX,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 10 |
-
stormcast,StormCast,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 11 |
-
stormcast,StormCast,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 12 |
-
pangu_weather,Pangu-Weather,global,5,1 7 42 99 123,0.00013033979247265275,0.0002685372203690466,0.048806713097574374,0.10733308684741971,0.048806713097574374,0.10733308684741971
|
| 13 |
-
pangu_weather,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.027875386332505546,0.02348779386900393,0.43111948243387105,0.39355644251497235,0.43111948243387105,0.39355644251497235
|
| 14 |
-
dlwp,DLWP,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 15 |
-
dlwp,DLWP,fire_prone,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.043265915053601556,0.04332331365579739,0.043265915053601556,0.04332331365579739
|
| 16 |
-
fcn,FCN,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 17 |
-
fcn,FCN,fire_prone,5,1 7 42 99 123,5.960229415004348e-06,1.3327478133443526e-05,0.011679805987441694,0.019872372458657642,0.011679805987441694,0.019872372458657642
|
| 18 |
-
fengwu,FengWu,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 19 |
-
fengwu,FengWu,fire_prone,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.005222389249812243,0.0062394095558402415,0.005222389249812243,0.0062394095558402415
|
| 20 |
-
fuxi,FuXi,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 21 |
-
fuxi,FuXi,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0010839188523199318,0.0017288780545672386,0.0010839188523199318,0.0017288780545672386
|
| 22 |
-
pangu6,Pangu-Weather,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 23 |
-
pangu6,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0018491271881979853,0.0032630386057089294,0.0018491271881979853,0.0032630386057089294
|
| 24 |
-
alphaearth,AlphaEarth,global,5,1 7 42 99 123,0.0,0.0,0.1722171037486726,0.08849214830495522,0.1722171037486726,0.08849214830495522
|
| 25 |
-
alphaearth,AlphaEarth,fire_prone,5,1 7 42 99 123,0.0,0.0,0.038803552655092256,0.0594825313313219,0.038803552655092256,0.0594825313313219
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/selection_regret_all_backbones_20260504.json
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
artifacts/results/selection_regret_head_metrics.csv
DELETED
|
@@ -1,241 +0,0 @@
|
|
| 1 |
-
family,model_tag,scope,seed,selected_by,head_label,test_pr_auc,exact_f1,tolerated_f1,union_f1
|
| 2 |
-
AlphaEarth,alphaearth,fire_prone,1,PR-AUC,shallow spatial adapter,0.04390614036305261,0.09715762273901808,0.7863702028280513,0.7863702028280513
|
| 3 |
-
AlphaEarth,alphaearth,fire_prone,1,decision,shallow spatial adapter,,,,0.7863702028280513
|
| 4 |
-
AlphaEarth,alphaearth,fire_prone,7,PR-AUC,shallow spatial adapter,0.05955397140893137,0.12514898688915374,0.8294499693616161,0.8294499693616161
|
| 5 |
-
AlphaEarth,alphaearth,fire_prone,7,decision,shallow spatial adapter,,,,0.8294499693616161
|
| 6 |
-
AlphaEarth,alphaearth,fire_prone,42,PR-AUC,shallow spatial adapter,0.038083070948941686,0.08702469619756958,0.7112901458230849,0.7112901458230849
|
| 7 |
-
AlphaEarth,alphaearth,fire_prone,42,decision,pixel MLP head,,,,0.8461131676361712
|
| 8 |
-
AlphaEarth,alphaearth,fire_prone,99,PR-AUC,shallow spatial adapter,0.0458102699699856,0.10365251727541955,0.7758298037709835,0.7758298037709835
|
| 9 |
-
AlphaEarth,alphaearth,fire_prone,99,decision,pixel MLP head,,,,0.8350245452333586
|
| 10 |
-
AlphaEarth,alphaearth,fire_prone,123,PR-AUC,shallow spatial adapter,0.045809049876129763,0.10531544957774468,0.7789089693560928,0.7789089693560928
|
| 11 |
-
AlphaEarth,alphaearth,fire_prone,123,decision,shallow spatial adapter,,,,0.7789089693560928
|
| 12 |
-
AlphaEarth,alphaearth,global,1,PR-AUC,shallow spatial adapter,0.0006549130347299629,0.004193290734824281,0.40561891947698747,0.40561891947698747
|
| 13 |
-
AlphaEarth,alphaearth,global,1,decision,pixel MLP head,,,,0.6337627266658229
|
| 14 |
-
AlphaEarth,alphaearth,global,7,PR-AUC,shallow spatial adapter,0.001005722733868245,0.010460251046025104,0.6184842128568402,0.6184842128568402
|
| 15 |
-
AlphaEarth,alphaearth,global,7,decision,pixel MLP head,,,,0.6691395427484861
|
| 16 |
-
AlphaEarth,alphaearth,global,42,PR-AUC,shallow spatial adapter,0.0005634701573991865,0.004809747755451047,0.4087444681515033,0.4087444681515033
|
| 17 |
-
AlphaEarth,alphaearth,global,42,decision,pixel MLP head,,,,0.6812131506751973
|
| 18 |
-
AlphaEarth,alphaearth,global,99,PR-AUC,shallow spatial adapter,0.0006577120081349608,0.006780481898534931,0.3921547570095426,0.3921547570095426
|
| 19 |
-
AlphaEarth,alphaearth,global,99,decision,pixel MLP head,,,,0.5842714676652996
|
| 20 |
-
AlphaEarth,alphaearth,global,123,PR-AUC,shallow spatial adapter,0.0007047712457371991,0.006959088991986505,0.4427625907752311,0.4427625907752311
|
| 21 |
-
AlphaEarth,alphaearth,global,123,decision,pixel MLP head,,,,0.5604635792586619
|
| 22 |
-
Aurora,aurora,fire_prone,1,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 23 |
-
Aurora,aurora,fire_prone,1,decision,linear probe,,,,0.7185324707231184
|
| 24 |
-
Aurora,aurora,fire_prone,7,PR-AUC,linear probe,0.024802820904513342,0.0413500618483831,0.7184857293868923,0.7184857293868923
|
| 25 |
-
Aurora,aurora,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 26 |
-
Aurora,aurora,fire_prone,42,PR-AUC,linear probe,0.02613792907867929,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 27 |
-
Aurora,aurora,fire_prone,42,decision,linear probe,,,,0.7185324707231184
|
| 28 |
-
Aurora,aurora,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.0,0.0,0.0
|
| 29 |
-
Aurora,aurora,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 30 |
-
Aurora,aurora,fire_prone,123,PR-AUC,pixel MLP head,0.03014151567817997,0.04541038665445361,0.7175172112337449,0.7175172112337449
|
| 31 |
-
Aurora,aurora,fire_prone,123,decision,linear probe,,,,0.7185324707231184
|
| 32 |
-
Aurora,aurora,global,1,PR-AUC,linear probe,0.00024254792826221397,0.00048497822606044473,0.23755358049655212,0.23755358049655212
|
| 33 |
-
Aurora,aurora,global,1,decision,shallow spatial adapter,,,,0.240793572992212
|
| 34 |
-
Aurora,aurora,global,7,PR-AUC,linear probe,0.00027660331739269843,0.0,0.23734400297937533,0.23734400297937533
|
| 35 |
-
Aurora,aurora,global,7,decision,shallow spatial adapter,,,,0.240793572992212
|
| 36 |
-
Aurora,aurora,global,42,PR-AUC,linear probe,0.0002876372030063385,0.00048497822606044473,0.0,0.0
|
| 37 |
-
Aurora,aurora,global,42,decision,shallow spatial adapter,,,,0.240793572992212
|
| 38 |
-
Aurora,aurora,global,99,PR-AUC,linear probe,0.00024254792826221397,0.0,0.0,0.0
|
| 39 |
-
Aurora,aurora,global,99,decision,shallow spatial adapter,,,,0.240793572992212
|
| 40 |
-
Aurora,aurora,global,123,PR-AUC,pixel MLP head,0.00031683315961488916,0.0005311562430574265,0.23647112940979162,0.23647112940979162
|
| 41 |
-
Aurora,aurora,global,123,decision,shallow spatial adapter,,,,0.240793572992212
|
| 42 |
-
ClimaX,climax,fire_prone,1,PR-AUC,linear probe,0.02281272244151735,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 43 |
-
ClimaX,climax,fire_prone,1,decision,linear probe,,,,0.7185324707231184
|
| 44 |
-
ClimaX,climax,fire_prone,7,PR-AUC,linear probe,0.021317405351800135,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 45 |
-
ClimaX,climax,fire_prone,7,decision,linear probe,,,,0.7185324707231184
|
| 46 |
-
ClimaX,climax,fire_prone,42,PR-AUC,linear probe,0.021516770872035896,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 47 |
-
ClimaX,climax,fire_prone,42,decision,linear probe,,,,0.7185324707231184
|
| 48 |
-
ClimaX,climax,fire_prone,99,PR-AUC,shallow spatial adapter,0.02099123219536693,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 49 |
-
ClimaX,climax,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 50 |
-
ClimaX,climax,fire_prone,123,PR-AUC,shallow spatial adapter,0.024930358757410707,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 51 |
-
ClimaX,climax,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 52 |
-
ClimaX,climax,global,1,PR-AUC,linear probe,0.0002543464414550104,0.00048497822606044473,0.23755358049655212,0.23755358049655212
|
| 53 |
-
ClimaX,climax,global,1,decision,shallow spatial adapter,,,,0.240793572992212
|
| 54 |
-
ClimaX,climax,global,7,PR-AUC,pixel MLP head,0.00025423546642937565,0.00048497822606044473,0.23755358049655212,0.23755358049655212
|
| 55 |
-
ClimaX,climax,global,7,decision,shallow spatial adapter,,,,0.240793572992212
|
| 56 |
-
ClimaX,climax,global,42,PR-AUC,shallow spatial adapter,0.00023723426605001756,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 57 |
-
ClimaX,climax,global,42,decision,shallow spatial adapter,,,,0.240793572992212
|
| 58 |
-
ClimaX,climax,global,99,PR-AUC,shallow spatial adapter,0.0002340075376021003,0.00048696535779102983,0.2384111045733381,0.2384111045733381
|
| 59 |
-
ClimaX,climax,global,99,decision,shallow spatial adapter,,,,0.2384111045733381
|
| 60 |
-
ClimaX,climax,global,123,PR-AUC,shallow spatial adapter,0.00025952340213823634,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 61 |
-
ClimaX,climax,global,123,decision,shallow spatial adapter,,,,0.240793572992212
|
| 62 |
-
DLWP,dlwp,fire_prone,1,PR-AUC,linear probe,0.019747545663020845,0.043506471331489265,0.7280364139105968,0.7280364139105968
|
| 63 |
-
DLWP,dlwp,fire_prone,1,decision,linear probe,,,,0.7280364139105968
|
| 64 |
-
DLWP,dlwp,fire_prone,7,PR-AUC,pixel MLP head,0.018519739310339497,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 65 |
-
DLWP,dlwp,fire_prone,7,decision,shallow spatial adapter,,,,0.762536895087842
|
| 66 |
-
DLWP,dlwp,fire_prone,42,PR-AUC,pixel MLP head,0.020762153794205103,0.04637177602565815,0.6606900017471591,0.6606900017471591
|
| 67 |
-
DLWP,dlwp,fire_prone,42,decision,shallow spatial adapter,,,,0.7728400679088107
|
| 68 |
-
DLWP,dlwp,fire_prone,99,PR-AUC,linear probe,0.02136633936888583,0.04819843096725701,0.7187730423241409,0.7187730423241409
|
| 69 |
-
DLWP,dlwp,fire_prone,99,decision,shallow spatial adapter,,,,0.7653346239087763
|
| 70 |
-
DLWP,dlwp,fire_prone,123,PR-AUC,pixel MLP head,0.021118517500793188,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 71 |
-
DLWP,dlwp,fire_prone,123,decision,linear probe,,,,0.7321459738801157
|
| 72 |
-
DLWP,dlwp,global,1,PR-AUC,shallow spatial adapter,0.0006257446338466172,0.00487022180273714,0.38023285660836226,0.38023285660836226
|
| 73 |
-
DLWP,dlwp,global,1,decision,shallow spatial adapter,,,,0.38023285660836226
|
| 74 |
-
DLWP,dlwp,global,7,PR-AUC,shallow spatial adapter,0.0005264872646085452,0.0,0.3432315705541329,0.3432315705541329
|
| 75 |
-
DLWP,dlwp,global,7,decision,shallow spatial adapter,,,,0.3432315705541329
|
| 76 |
-
DLWP,dlwp,global,42,PR-AUC,shallow spatial adapter,0.0006203713852571992,0.0,0.3405125814370199,0.3405125814370199
|
| 77 |
-
DLWP,dlwp,global,42,decision,shallow spatial adapter,,,,0.3405125814370199
|
| 78 |
-
DLWP,dlwp,global,99,PR-AUC,shallow spatial adapter,0.0007477128447471452,0.0,0.3979559626836394,0.3979559626836394
|
| 79 |
-
DLWP,dlwp,global,99,decision,shallow spatial adapter,,,,0.3979559626836394
|
| 80 |
-
DLWP,dlwp,global,123,PR-AUC,shallow spatial adapter,0.0007129763973023342,0.0,0.3797689460796109,0.3797689460796109
|
| 81 |
-
DLWP,dlwp,global,123,decision,shallow spatial adapter,,,,0.3797689460796109
|
| 82 |
-
FCN,fcn,fire_prone,1,PR-AUC,shallow spatial adapter,0.01844011667219625,0.042068766252528166,0.7185324707231184,0.7185324707231184
|
| 83 |
-
FCN,fcn,fire_prone,1,decision,linear probe,,,,0.7182175622542595
|
| 84 |
-
FCN,fcn,fire_prone,7,PR-AUC,pixel MLP head,0.02050876208409485,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 85 |
-
FCN,fcn,fire_prone,7,decision,linear probe,,,,0.7644129739607127
|
| 86 |
-
FCN,fcn,fire_prone,42,PR-AUC,shallow spatial adapter,0.018030815062946615,0.0414596444738876,0.7197180735022655,0.7197180735022655
|
| 87 |
-
FCN,fcn,fire_prone,42,decision,shallow spatial adapter,,,,0.7197180735022655
|
| 88 |
-
FCN,fcn,fire_prone,99,PR-AUC,linear probe,0.029098665712304895,0.042822140550172624,0.726408418760773,0.726408418760773
|
| 89 |
-
FCN,fcn,fire_prone,99,decision,linear probe,,,,0.726408418760773
|
| 90 |
-
FCN,fcn,fire_prone,123,PR-AUC,pixel MLP head,0.019943278646881796,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 91 |
-
FCN,fcn,fire_prone,123,decision,linear probe,,,,0.7310509974227326
|
| 92 |
-
FCN,fcn,global,1,PR-AUC,shallow spatial adapter,0.00037256097806901117,0.0009319664492078285,0.31167484413093016,0.31167484413093016
|
| 93 |
-
FCN,fcn,global,1,decision,shallow spatial adapter,,,,0.31167484413093016
|
| 94 |
-
FCN,fcn,global,7,PR-AUC,shallow spatial adapter,0.0003268363416054406,0.001086071137659517,0.3051941376005135,0.3051941376005135
|
| 95 |
-
FCN,fcn,global,7,decision,shallow spatial adapter,,,,0.3051941376005135
|
| 96 |
-
FCN,fcn,global,42,PR-AUC,shallow spatial adapter,0.00041063897933390575,0.0007027406886858749,0.31987973649439366,0.31987973649439366
|
| 97 |
-
FCN,fcn,global,42,decision,pixel MLP head,,,,0.2870596305028149
|
| 98 |
-
FCN,fcn,global,99,PR-AUC,shallow spatial adapter,0.00038120453362995967,0.0018159806295399514,0.3054145960271247,0.3054145960271247
|
| 99 |
-
FCN,fcn,global,99,decision,shallow spatial adapter,,,,0.3054145960271247
|
| 100 |
-
FCN,fcn,global,123,PR-AUC,shallow spatial adapter,0.000387412312932838,0.0006535947712418301,0.3096850885545486,0.3096850885545486
|
| 101 |
-
FCN,fcn,global,123,decision,shallow spatial adapter,,,,0.3096850885545486
|
| 102 |
-
FengWu,fengwu,fire_prone,1,PR-AUC,shallow spatial adapter,0.022736128443885992,0.04452825597664091,0.7269980510116651,0.7269980510116651
|
| 103 |
-
FengWu,fengwu,fire_prone,1,decision,shallow spatial adapter,,,,0.7269980510116651
|
| 104 |
-
FengWu,fengwu,fire_prone,7,PR-AUC,pixel MLP head,0.016801706970978273,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 105 |
-
FengWu,fengwu,fire_prone,7,decision,shallow spatial adapter,,,,0.722901134194411
|
| 106 |
-
FengWu,fengwu,fire_prone,42,PR-AUC,shallow spatial adapter,0.021165185967854567,0.04574838388861263,0.7265705731122933,0.7265705731122933
|
| 107 |
-
FengWu,fengwu,fire_prone,42,decision,shallow spatial adapter,,,,0.7265705731122933
|
| 108 |
-
FengWu,fengwu,fire_prone,99,PR-AUC,linear probe,0.02199779031689959,0.0,0.7185324707231184,0.7185324707231184
|
| 109 |
-
FengWu,fengwu,fire_prone,99,decision,shallow spatial adapter,,,,0.7336829717908426
|
| 110 |
-
FengWu,fengwu,fire_prone,123,PR-AUC,pixel MLP head,0.020495144103834257,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 111 |
-
FengWu,fengwu,fire_prone,123,decision,shallow spatial adapter,,,,0.7251252524331628
|
| 112 |
-
FengWu,fengwu,global,1,PR-AUC,shallow spatial adapter,0.000398077435365184,0.0,0.31071390711162444,0.31071390711162444
|
| 113 |
-
FengWu,fengwu,global,1,decision,shallow spatial adapter,,,,0.31071390711162444
|
| 114 |
-
FengWu,fengwu,global,7,PR-AUC,shallow spatial adapter,0.00036963872610239895,0.0,0.30641904273669174,0.30641904273669174
|
| 115 |
-
FengWu,fengwu,global,7,decision,shallow spatial adapter,,,,0.30641904273669174
|
| 116 |
-
FengWu,fengwu,global,42,PR-AUC,shallow spatial adapter,0.00036987379624400454,0.0,0.31219058559732665,0.31219058559732665
|
| 117 |
-
FengWu,fengwu,global,42,decision,shallow spatial adapter,,,,0.31219058559732665
|
| 118 |
-
FengWu,fengwu,global,99,PR-AUC,shallow spatial adapter,0.00042782651526874734,0.0,0.3111214819309595,0.3111214819309595
|
| 119 |
-
FengWu,fengwu,global,99,decision,shallow spatial adapter,,,,0.3111214819309595
|
| 120 |
-
FengWu,fengwu,global,123,PR-AUC,shallow spatial adapter,0.0004116035724473925,0.0,0.3145618361221859,0.3145618361221859
|
| 121 |
-
FengWu,fengwu,global,123,decision,shallow spatial adapter,,,,0.3145618361221859
|
| 122 |
-
FuXi,fuxi,fire_prone,1,PR-AUC,shallow spatial adapter,0.01904942261850253,0.043196160341303,0.7246636456247585,0.7246636456247585
|
| 123 |
-
FuXi,fuxi,fire_prone,1,decision,linear probe,,,,0.7261195534617713
|
| 124 |
-
FuXi,fuxi,fire_prone,7,PR-AUC,shallow spatial adapter,0.018717347905238046,0.044572342126298965,0.7235687421646468,0.7235687421646468
|
| 125 |
-
FuXi,fuxi,fire_prone,7,decision,shallow spatial adapter,,,,0.7235687421646468
|
| 126 |
-
FuXi,fuxi,fire_prone,42,PR-AUC,shallow spatial adapter,0.020363596550511454,0.04603830266616599,0.7054843984273774,0.7054843984273774
|
| 127 |
-
FuXi,fuxi,fire_prone,42,decision,shallow spatial adapter,,,,0.7054843984273774
|
| 128 |
-
FuXi,fuxi,fire_prone,99,PR-AUC,pixel MLP head,0.02225455497934009,0.030284377692970574,0.7203102915557309,0.7203102915557309
|
| 129 |
-
FuXi,fuxi,fire_prone,99,decision,shallow spatial adapter,,,,0.7183426482806338
|
| 130 |
-
FuXi,fuxi,fire_prone,123,PR-AUC,pixel MLP head,0.021045021724179047,0.018543768748295608,0.7185324707231184,0.7185324707231184
|
| 131 |
-
FuXi,fuxi,fire_prone,123,decision,linear probe,,,,0.7224961571477053
|
| 132 |
-
FuXi,fuxi,global,1,PR-AUC,shallow spatial adapter,0.0003596048414560045,0.0006866311182961118,0.3091927111996169,0.3091927111996169
|
| 133 |
-
FuXi,fuxi,global,1,decision,linear probe,,,,0.21516044416153002
|
| 134 |
-
FuXi,fuxi,global,7,PR-AUC,shallow spatial adapter,0.00037118462783325537,0.0008450613619815959,0.3062654921252336,0.3062654921252336
|
| 135 |
-
FuXi,fuxi,global,7,decision,shallow spatial adapter,,,,0.3062654921252336
|
| 136 |
-
FuXi,fuxi,global,42,PR-AUC,shallow spatial adapter,0.0003830459807690152,0.0003188165529554294,0.316668570748256,0.316668570748256
|
| 137 |
-
FuXi,fuxi,global,42,decision,shallow spatial adapter,,,,0.316668570748256
|
| 138 |
-
FuXi,fuxi,global,99,PR-AUC,shallow spatial adapter,0.0003758107890486513,0.0007771032641515431,0.3108110703043797,0.3108110703043797
|
| 139 |
-
FuXi,fuxi,global,99,decision,shallow spatial adapter,,,,0.3108110703043797
|
| 140 |
-
FuXi,fuxi,global,123,PR-AUC,shallow spatial adapter,0.00039999784430104664,0.0009888839378897987,0.3096924811079745,0.3096924811079745
|
| 141 |
-
FuXi,fuxi,global,123,decision,shallow spatial adapter,,,,0.3096924811079745
|
| 142 |
-
Pangu-Weather,pangu_weather,fire_prone,1,PR-AUC,pixel MLP head,0.014813375233921266,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 143 |
-
Pangu-Weather,pangu_weather,fire_prone,1,decision,pixel MLP head,,,,0.7185324707231184
|
| 144 |
-
Pangu-Weather,pangu_weather,fire_prone,7,PR-AUC,pixel MLP head,0.029883397445312192,0.0,0.0,0.0
|
| 145 |
-
Pangu-Weather,pangu_weather,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 146 |
-
Pangu-Weather,pangu_weather,fire_prone,42,PR-AUC,pixel MLP head,0.02188826028256454,0.0,0.0,0.0
|
| 147 |
-
Pangu-Weather,pangu_weather,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 148 |
-
Pangu-Weather,pangu_weather,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.0,0.0,0.0
|
| 149 |
-
Pangu-Weather,pangu_weather,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 150 |
-
Pangu-Weather,pangu_weather,fire_prone,123,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 151 |
-
Pangu-Weather,pangu_weather,fire_prone,123,decision,linear probe,,,,0.7185324707231184
|
| 152 |
-
Pangu-Weather,pangu_weather,global,1,PR-AUC,linear probe,0.00024254792826221397,0.00048497822606044473,0.23755358049655212,0.23755358049655212
|
| 153 |
-
Pangu-Weather,pangu_weather,global,1,decision,shallow spatial adapter,,,,0.240793572992212
|
| 154 |
-
Pangu-Weather,pangu_weather,global,7,PR-AUC,shallow spatial adapter,0.00028045576992618193,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 155 |
-
Pangu-Weather,pangu_weather,global,7,decision,shallow spatial adapter,,,,0.240793572992212
|
| 156 |
-
Pangu-Weather,pangu_weather,global,42,PR-AUC,pixel MLP head,0.00021817709863716954,0.0,0.0,0.0
|
| 157 |
-
Pangu-Weather,pangu_weather,global,42,decision,shallow spatial adapter,,,,0.240793572992212
|
| 158 |
-
Pangu-Weather,pangu_weather,global,99,PR-AUC,shallow spatial adapter,0.0003408299850116487,0.0004887421693148924,0.23917716245227552,0.23917716245227552
|
| 159 |
-
Pangu-Weather,pangu_weather,global,99,decision,pixel MLP head,,,,0.229804399271568
|
| 160 |
-
Pangu-Weather,pangu_weather,global,123,PR-AUC,shallow spatial adapter,0.0003300754798792237,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 161 |
-
Pangu-Weather,pangu_weather,global,123,decision,shallow spatial adapter,,,,0.240793572992212
|
| 162 |
-
Pangu-Weather,pangu6,fire_prone,1,PR-AUC,shallow spatial adapter,0.02328829578760066,0.04418397717161179,0.7231388400090598,0.7231388400090598
|
| 163 |
-
Pangu-Weather,pangu6,fire_prone,1,decision,shallow spatial adapter,,,,0.7231388400090598
|
| 164 |
-
Pangu-Weather,pangu6,fire_prone,7,PR-AUC,pixel MLP head,0.018664183428008234,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 165 |
-
Pangu-Weather,pangu6,fire_prone,7,decision,shallow spatial adapter,,,,0.719461376864001
|
| 166 |
-
Pangu-Weather,pangu6,fire_prone,42,PR-AUC,shallow spatial adapter,0.021420904714594617,0.0430065972893146,0.7241279907754397,0.7241279907754397
|
| 167 |
-
Pangu-Weather,pangu6,fire_prone,42,decision,shallow spatial adapter,,,,0.7241279907754397
|
| 168 |
-
Pangu-Weather,pangu6,fire_prone,99,PR-AUC,linear probe,0.024740444457605402,0.0044004400440044,0.7183382629739177,0.7183382629739177
|
| 169 |
-
Pangu-Weather,pangu6,fire_prone,99,decision,shallow spatial adapter,,,,0.719015307962107
|
| 170 |
-
Pangu-Weather,pangu6,fire_prone,123,PR-AUC,pixel MLP head,0.02254610440161657,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 171 |
-
Pangu-Weather,pangu6,fire_prone,123,decision,linear probe,,,,0.7261721555350364
|
| 172 |
-
Pangu-Weather,pangu6,global,1,PR-AUC,shallow spatial adapter,0.00042881385578730365,0.0,0.32066974948800614,0.32066974948800614
|
| 173 |
-
Pangu-Weather,pangu6,global,1,decision,shallow spatial adapter,,,,0.32066974948800614
|
| 174 |
-
Pangu-Weather,pangu6,global,7,PR-AUC,shallow spatial adapter,0.00038395193539280824,0.0,0.31120670593952293,0.31120670593952293
|
| 175 |
-
Pangu-Weather,pangu6,global,7,decision,shallow spatial adapter,,,,0.31120670593952293
|
| 176 |
-
Pangu-Weather,pangu6,global,42,PR-AUC,shallow spatial adapter,0.00040651309469793043,0.0,0.32154249424354786,0.32154249424354786
|
| 177 |
-
Pangu-Weather,pangu6,global,42,decision,shallow spatial adapter,,,,0.32154249424354786
|
| 178 |
-
Pangu-Weather,pangu6,global,99,PR-AUC,shallow spatial adapter,0.0004450373086921994,0.0,0.3214547875801752,0.3214547875801752
|
| 179 |
-
Pangu-Weather,pangu6,global,99,decision,shallow spatial adapter,,,,0.3214547875801752
|
| 180 |
-
Pangu-Weather,pangu6,global,123,PR-AUC,shallow spatial adapter,0.00043738577276574255,0.0,0.31812983009681495,0.31812983009681495
|
| 181 |
-
Pangu-Weather,pangu6,global,123,decision,shallow spatial adapter,,,,0.31812983009681495
|
| 182 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,1,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 183 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,1,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 184 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,7,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 185 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 186 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,42,PR-AUC,pixel MLP head,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 187 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 188 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 189 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 190 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,123,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 191 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 192 |
-
Prithvi-WxC,prithvi_wxc,global,1,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 193 |
-
Prithvi-WxC,prithvi_wxc,global,1,decision,shallow spatial adapter,,,,0.240793572992212
|
| 194 |
-
Prithvi-WxC,prithvi_wxc,global,7,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 195 |
-
Prithvi-WxC,prithvi_wxc,global,7,decision,shallow spatial adapter,,,,0.240793572992212
|
| 196 |
-
Prithvi-WxC,prithvi_wxc,global,42,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 197 |
-
Prithvi-WxC,prithvi_wxc,global,42,decision,shallow spatial adapter,,,,0.240793572992212
|
| 198 |
-
Prithvi-WxC,prithvi_wxc,global,99,PR-AUC,shallow spatial adapter,0.0002454330184381399,0.00048497822606044473,0.23755358049655212,0.23755358049655212
|
| 199 |
-
Prithvi-WxC,prithvi_wxc,global,99,decision,shallow spatial adapter,,,,0.23755358049655212
|
| 200 |
-
Prithvi-WxC,prithvi_wxc,global,123,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 201 |
-
Prithvi-WxC,prithvi_wxc,global,123,decision,shallow spatial adapter,,,,0.240793572992212
|
| 202 |
-
Reference,reference,fire_prone,1,PR-AUC,shallow spatial adapter,0.10204224118176683,0.1611624834874505,0.799032457577039,0.799032457577039
|
| 203 |
-
Reference,reference,fire_prone,1,decision,linear probe,,,,0.8685988450931266
|
| 204 |
-
Reference,reference,fire_prone,7,PR-AUC,shallow spatial adapter,0.1323902067230726,0.22885572139303484,0.8027807546359551,0.8027807546359551
|
| 205 |
-
Reference,reference,fire_prone,7,decision,shallow spatial adapter,,,,0.8027807546359551
|
| 206 |
-
Reference,reference,fire_prone,42,PR-AUC,shallow spatial adapter,0.12048427320762313,0.19225806451612906,0.8358151221553631,0.8358151221553631
|
| 207 |
-
Reference,reference,fire_prone,42,decision,shallow spatial adapter,,,,0.8358151221553631
|
| 208 |
-
Reference,reference,fire_prone,99,PR-AUC,shallow spatial adapter,0.11947246238005743,0.19477124183006533,0.8534759193943048,0.8534759193943048
|
| 209 |
-
Reference,reference,fire_prone,99,decision,linear probe,,,,0.9039923296574045
|
| 210 |
-
Reference,reference,fire_prone,123,PR-AUC,shallow spatial adapter,0.11551882470432043,0.19165378670788252,0.8066689866810086,0.8066689866810086
|
| 211 |
-
Reference,reference,fire_prone,123,decision,pixel MLP head,,,,0.8567215417854325
|
| 212 |
-
Reference,reference,global,1,PR-AUC,shallow spatial adapter,0.002624341503354088,0.017134572294714646,0.6186566066408326,0.6186566066408326
|
| 213 |
-
Reference,reference,global,1,decision,linear probe,,,,0.7286109603326707
|
| 214 |
-
Reference,reference,global,7,PR-AUC,shallow spatial adapter,0.0030317345997110698,0.02287675150128682,0.5002193417313746,0.5002193417313746
|
| 215 |
-
Reference,reference,global,7,decision,shallow spatial adapter,,,,0.5002193417313746
|
| 216 |
-
Reference,reference,global,42,PR-AUC,shallow spatial adapter,0.0032729435045747413,0.025515210991167808,0.7239943741463363,0.7239943741463363
|
| 217 |
-
Reference,reference,global,42,decision,shallow spatial adapter,,,,0.7239943741463363
|
| 218 |
-
Reference,reference,global,99,PR-AUC,shallow spatial adapter,0.0031953098868323193,0.021876258220373104,0.6698005926442897,0.6698005926442897
|
| 219 |
-
Reference,reference,global,99,decision,linear probe,,,,0.7647466876509988
|
| 220 |
-
Reference,reference,global,123,PR-AUC,shallow spatial adapter,0.0025603887793133394,0.019320660641944532,0.5009707461135111,0.5009707461135111
|
| 221 |
-
Reference,reference,global,123,decision,pixel MLP head,,,,0.735221546471909
|
| 222 |
-
StormCast,stormcast,fire_prone,1,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 223 |
-
StormCast,stormcast,fire_prone,1,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 224 |
-
StormCast,stormcast,fire_prone,7,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 225 |
-
StormCast,stormcast,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 226 |
-
StormCast,stormcast,fire_prone,42,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 227 |
-
StormCast,stormcast,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 228 |
-
StormCast,stormcast,fire_prone,99,PR-AUC,pixel MLP head,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 229 |
-
StormCast,stormcast,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 230 |
-
StormCast,stormcast,fire_prone,123,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184
|
| 231 |
-
StormCast,stormcast,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184
|
| 232 |
-
StormCast,stormcast,global,1,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 233 |
-
StormCast,stormcast,global,1,decision,shallow spatial adapter,,,,0.240793572992212
|
| 234 |
-
StormCast,stormcast,global,7,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 235 |
-
StormCast,stormcast,global,7,decision,shallow spatial adapter,,,,0.240793572992212
|
| 236 |
-
StormCast,stormcast,global,42,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 237 |
-
StormCast,stormcast,global,42,decision,shallow spatial adapter,,,,0.240793572992212
|
| 238 |
-
StormCast,stormcast,global,99,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004905319945803844,0.2399251001974223,0.2399251001974223
|
| 239 |
-
StormCast,stormcast,global,99,decision,shallow spatial adapter,,,,0.2399251001974223
|
| 240 |
-
StormCast,stormcast,global,123,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212
|
| 241 |
-
StormCast,stormcast,global,123,decision,shallow spatial adapter,,,,0.240793572992212
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/selection_regret_per_seed.csv
DELETED
|
@@ -1,121 +0,0 @@
|
|
| 1 |
-
family,model_tag,scope,seed,ranking_head,decision_head,ranking_pr_auc,ranking_union_f1,decision_union_f1,regret,top1_agreement
|
| 2 |
-
AlphaEarth,alphaearth,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.04390614036305261,0.7863702028280513,0.7863702028280513,0.0,True
|
| 3 |
-
AlphaEarth,alphaearth,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.05955397140893137,0.8294499693616161,0.8294499693616161,0.0,True
|
| 4 |
-
AlphaEarth,alphaearth,fire_prone,42,shallow spatial adapter,pixel MLP head,0.038083070948941686,0.7112901458230849,0.8461131676361712,0.13482302181308625,False
|
| 5 |
-
AlphaEarth,alphaearth,fire_prone,99,shallow spatial adapter,pixel MLP head,0.0458102699699856,0.7758298037709835,0.8350245452333586,0.05919474146237502,False
|
| 6 |
-
AlphaEarth,alphaearth,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.045809049876129763,0.7789089693560928,0.7789089693560928,0.0,True
|
| 7 |
-
AlphaEarth,alphaearth,global,1,shallow spatial adapter,pixel MLP head,0.0006549130347299629,0.40561891947698747,0.6337627266658229,0.22814380718883542,False
|
| 8 |
-
AlphaEarth,alphaearth,global,7,shallow spatial adapter,pixel MLP head,0.001005722733868245,0.6184842128568402,0.6691395427484861,0.050655329891645895,False
|
| 9 |
-
AlphaEarth,alphaearth,global,42,shallow spatial adapter,pixel MLP head,0.0005634701573991865,0.4087444681515033,0.6812131506751973,0.272468682523694,False
|
| 10 |
-
AlphaEarth,alphaearth,global,99,shallow spatial adapter,pixel MLP head,0.0006577120081349608,0.3921547570095426,0.5842714676652996,0.19211671065575697,False
|
| 11 |
-
AlphaEarth,alphaearth,global,123,shallow spatial adapter,pixel MLP head,0.0007047712457371991,0.4427625907752311,0.5604635792586619,0.11770098848343075,False
|
| 12 |
-
Aurora,aurora,fire_prone,1,linear probe,linear probe,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True
|
| 13 |
-
Aurora,aurora,fire_prone,7,linear probe,shallow spatial adapter,0.024802820904513342,0.7184857293868923,0.7185324707231184,4.674133622617482e-05,False
|
| 14 |
-
Aurora,aurora,fire_prone,42,linear probe,linear probe,0.02613792907867929,0.7185324707231184,0.7185324707231184,0.0,True
|
| 15 |
-
Aurora,aurora,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.0,0.7185324707231184,0.7185324707231184,False
|
| 16 |
-
Aurora,aurora,fire_prone,123,pixel MLP head,linear probe,0.03014151567817997,0.7175172112337449,0.7185324707231184,0.0010152594893735323,False
|
| 17 |
-
Aurora,aurora,global,1,linear probe,shallow spatial adapter,0.00024254792826221397,0.23755358049655212,0.240793572992212,0.0032399924956598714,False
|
| 18 |
-
Aurora,aurora,global,7,linear probe,shallow spatial adapter,0.00027660331739269843,0.23734400297937533,0.240793572992212,0.0034495700128366613,False
|
| 19 |
-
Aurora,aurora,global,42,linear probe,shallow spatial adapter,0.0002876372030063385,0.0,0.240793572992212,0.240793572992212,False
|
| 20 |
-
Aurora,aurora,global,99,linear probe,shallow spatial adapter,0.00024254792826221397,0.0,0.240793572992212,0.240793572992212,False
|
| 21 |
-
Aurora,aurora,global,123,pixel MLP head,shallow spatial adapter,0.00031683315961488916,0.23647112940979162,0.240793572992212,0.004322443582420371,False
|
| 22 |
-
ClimaX,climax,fire_prone,1,linear probe,linear probe,0.02281272244151735,0.7185324707231184,0.7185324707231184,0.0,True
|
| 23 |
-
ClimaX,climax,fire_prone,7,linear probe,linear probe,0.021317405351800135,0.7185324707231184,0.7185324707231184,0.0,True
|
| 24 |
-
ClimaX,climax,fire_prone,42,linear probe,linear probe,0.021516770872035896,0.7185324707231184,0.7185324707231184,0.0,True
|
| 25 |
-
ClimaX,climax,fire_prone,99,shallow spatial adapter,shallow spatial adapter,0.02099123219536693,0.7185324707231184,0.7185324707231184,0.0,True
|
| 26 |
-
ClimaX,climax,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.024930358757410707,0.7185324707231184,0.7185324707231184,0.0,True
|
| 27 |
-
ClimaX,climax,global,1,linear probe,shallow spatial adapter,0.0002543464414550104,0.23755358049655212,0.240793572992212,0.0032399924956598714,False
|
| 28 |
-
ClimaX,climax,global,7,pixel MLP head,shallow spatial adapter,0.00025423546642937565,0.23755358049655212,0.240793572992212,0.0032399924956598714,False
|
| 29 |
-
ClimaX,climax,global,42,shallow spatial adapter,shallow spatial adapter,0.00023723426605001756,0.240793572992212,0.240793572992212,0.0,True
|
| 30 |
-
ClimaX,climax,global,99,shallow spatial adapter,shallow spatial adapter,0.0002340075376021003,0.2384111045733381,0.2384111045733381,0.0,True
|
| 31 |
-
ClimaX,climax,global,123,shallow spatial adapter,shallow spatial adapter,0.00025952340213823634,0.240793572992212,0.240793572992212,0.0,True
|
| 32 |
-
DLWP,dlwp,fire_prone,1,linear probe,linear probe,0.019747545663020845,0.7280364139105968,0.7280364139105968,0.0,True
|
| 33 |
-
DLWP,dlwp,fire_prone,7,pixel MLP head,shallow spatial adapter,0.018519739310339497,0.7185324707231184,0.762536895087842,0.044004424364723516,False
|
| 34 |
-
DLWP,dlwp,fire_prone,42,pixel MLP head,shallow spatial adapter,0.020762153794205103,0.6606900017471591,0.7728400679088107,0.11215006616165157,False
|
| 35 |
-
DLWP,dlwp,fire_prone,99,linear probe,shallow spatial adapter,0.02136633936888583,0.7187730423241409,0.7653346239087763,0.046561581584635414,False
|
| 36 |
-
DLWP,dlwp,fire_prone,123,pixel MLP head,linear probe,0.021118517500793188,0.7185324707231184,0.7321459738801157,0.013613503156997275,False
|
| 37 |
-
DLWP,dlwp,global,1,shallow spatial adapter,shallow spatial adapter,0.0006257446338466172,0.38023285660836226,0.38023285660836226,0.0,True
|
| 38 |
-
DLWP,dlwp,global,7,shallow spatial adapter,shallow spatial adapter,0.0005264872646085452,0.3432315705541329,0.3432315705541329,0.0,True
|
| 39 |
-
DLWP,dlwp,global,42,shallow spatial adapter,shallow spatial adapter,0.0006203713852571992,0.3405125814370199,0.3405125814370199,0.0,True
|
| 40 |
-
DLWP,dlwp,global,99,shallow spatial adapter,shallow spatial adapter,0.0007477128447471452,0.3979559626836394,0.3979559626836394,0.0,True
|
| 41 |
-
DLWP,dlwp,global,123,shallow spatial adapter,shallow spatial adapter,0.0007129763973023342,0.3797689460796109,0.3797689460796109,0.0,True
|
| 42 |
-
FCN,fcn,fire_prone,1,shallow spatial adapter,linear probe,0.01844011667219625,0.7185324707231184,0.7182175622542595,0.0,False
|
| 43 |
-
FCN,fcn,fire_prone,7,pixel MLP head,linear probe,0.02050876208409485,0.7185324707231184,0.7644129739607127,0.045880503237594294,False
|
| 44 |
-
FCN,fcn,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.018030815062946615,0.7197180735022655,0.7197180735022655,0.0,True
|
| 45 |
-
FCN,fcn,fire_prone,99,linear probe,linear probe,0.029098665712304895,0.726408418760773,0.726408418760773,0.0,True
|
| 46 |
-
FCN,fcn,fire_prone,123,pixel MLP head,linear probe,0.019943278646881796,0.7185324707231184,0.7310509974227326,0.012518526699614174,False
|
| 47 |
-
FCN,fcn,global,1,shallow spatial adapter,shallow spatial adapter,0.00037256097806901117,0.31167484413093016,0.31167484413093016,0.0,True
|
| 48 |
-
FCN,fcn,global,7,shallow spatial adapter,shallow spatial adapter,0.0003268363416054406,0.3051941376005135,0.3051941376005135,0.0,True
|
| 49 |
-
FCN,fcn,global,42,shallow spatial adapter,pixel MLP head,0.00041063897933390575,0.31987973649439366,0.2870596305028149,0.0,False
|
| 50 |
-
FCN,fcn,global,99,shallow spatial adapter,shallow spatial adapter,0.00038120453362995967,0.3054145960271247,0.3054145960271247,0.0,True
|
| 51 |
-
FCN,fcn,global,123,shallow spatial adapter,shallow spatial adapter,0.000387412312932838,0.3096850885545486,0.3096850885545486,0.0,True
|
| 52 |
-
FengWu,fengwu,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.022736128443885992,0.7269980510116651,0.7269980510116651,0.0,True
|
| 53 |
-
FengWu,fengwu,fire_prone,7,pixel MLP head,shallow spatial adapter,0.016801706970978273,0.7185324707231184,0.722901134194411,0.004368663471292611,False
|
| 54 |
-
FengWu,fengwu,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.021165185967854567,0.7265705731122933,0.7265705731122933,0.0,True
|
| 55 |
-
FengWu,fengwu,fire_prone,99,linear probe,shallow spatial adapter,0.02199779031689959,0.7185324707231184,0.7336829717908426,0.015150501067724198,False
|
| 56 |
-
FengWu,fengwu,fire_prone,123,pixel MLP head,shallow spatial adapter,0.020495144103834257,0.7185324707231184,0.7251252524331628,0.006592781710044404,False
|
| 57 |
-
FengWu,fengwu,global,1,shallow spatial adapter,shallow spatial adapter,0.000398077435365184,0.31071390711162444,0.31071390711162444,0.0,True
|
| 58 |
-
FengWu,fengwu,global,7,shallow spatial adapter,shallow spatial adapter,0.00036963872610239895,0.30641904273669174,0.30641904273669174,0.0,True
|
| 59 |
-
FengWu,fengwu,global,42,shallow spatial adapter,shallow spatial adapter,0.00036987379624400454,0.31219058559732665,0.31219058559732665,0.0,True
|
| 60 |
-
FengWu,fengwu,global,99,shallow spatial adapter,shallow spatial adapter,0.00042782651526874734,0.3111214819309595,0.3111214819309595,0.0,True
|
| 61 |
-
FengWu,fengwu,global,123,shallow spatial adapter,shallow spatial adapter,0.0004116035724473925,0.3145618361221859,0.3145618361221859,0.0,True
|
| 62 |
-
FuXi,fuxi,fire_prone,1,shallow spatial adapter,linear probe,0.01904942261850253,0.7246636456247585,0.7261195534617713,0.001455907837012771,False
|
| 63 |
-
FuXi,fuxi,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.018717347905238046,0.7235687421646468,0.7235687421646468,0.0,True
|
| 64 |
-
FuXi,fuxi,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.020363596550511454,0.7054843984273774,0.7054843984273774,0.0,True
|
| 65 |
-
FuXi,fuxi,fire_prone,99,pixel MLP head,shallow spatial adapter,0.02225455497934009,0.7203102915557309,0.7183426482806338,0.0,False
|
| 66 |
-
FuXi,fuxi,fire_prone,123,pixel MLP head,linear probe,0.021045021724179047,0.7185324707231184,0.7224961571477053,0.0039636864245868875,False
|
| 67 |
-
FuXi,fuxi,global,1,shallow spatial adapter,linear probe,0.0003596048414560045,0.3091927111996169,0.21516044416153002,0.0,False
|
| 68 |
-
FuXi,fuxi,global,7,shallow spatial adapter,shallow spatial adapter,0.00037118462783325537,0.3062654921252336,0.3062654921252336,0.0,True
|
| 69 |
-
FuXi,fuxi,global,42,shallow spatial adapter,shallow spatial adapter,0.0003830459807690152,0.316668570748256,0.316668570748256,0.0,True
|
| 70 |
-
FuXi,fuxi,global,99,shallow spatial adapter,shallow spatial adapter,0.0003758107890486513,0.3108110703043797,0.3108110703043797,0.0,True
|
| 71 |
-
FuXi,fuxi,global,123,shallow spatial adapter,shallow spatial adapter,0.00039999784430104664,0.3096924811079745,0.3096924811079745,0.0,True
|
| 72 |
-
Pangu-Weather,pangu_weather,fire_prone,1,pixel MLP head,pixel MLP head,0.014813375233921266,0.7185324707231184,0.7185324707231184,0.0,True
|
| 73 |
-
Pangu-Weather,pangu_weather,fire_prone,7,pixel MLP head,shallow spatial adapter,0.029883397445312192,0.0,0.7185324707231184,0.7185324707231184,False
|
| 74 |
-
Pangu-Weather,pangu_weather,fire_prone,42,pixel MLP head,shallow spatial adapter,0.02188826028256454,0.0,0.7185324707231184,0.7185324707231184,False
|
| 75 |
-
Pangu-Weather,pangu_weather,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.0,0.7185324707231184,0.7185324707231184,False
|
| 76 |
-
Pangu-Weather,pangu_weather,fire_prone,123,linear probe,linear probe,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True
|
| 77 |
-
Pangu-Weather,pangu_weather,global,1,linear probe,shallow spatial adapter,0.00024254792826221397,0.23755358049655212,0.240793572992212,0.0032399924956598714,False
|
| 78 |
-
Pangu-Weather,pangu_weather,global,7,shallow spatial adapter,shallow spatial adapter,0.00028045576992618193,0.240793572992212,0.240793572992212,0.0,True
|
| 79 |
-
Pangu-Weather,pangu_weather,global,42,pixel MLP head,shallow spatial adapter,0.00021817709863716954,0.0,0.240793572992212,0.240793572992212,False
|
| 80 |
-
Pangu-Weather,pangu_weather,global,99,shallow spatial adapter,pixel MLP head,0.0003408299850116487,0.23917716245227552,0.229804399271568,0.0,False
|
| 81 |
-
Pangu-Weather,pangu_weather,global,123,shallow spatial adapter,shallow spatial adapter,0.0003300754798792237,0.240793572992212,0.240793572992212,0.0,True
|
| 82 |
-
Pangu-Weather,pangu6,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.02328829578760066,0.7231388400090598,0.7231388400090598,0.0,True
|
| 83 |
-
Pangu-Weather,pangu6,fire_prone,7,pixel MLP head,shallow spatial adapter,0.018664183428008234,0.7185324707231184,0.719461376864001,0.0009289061408825905,False
|
| 84 |
-
Pangu-Weather,pangu6,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.021420904714594617,0.7241279907754397,0.7241279907754397,0.0,True
|
| 85 |
-
Pangu-Weather,pangu6,fire_prone,99,linear probe,shallow spatial adapter,0.024740444457605402,0.7183382629739177,0.719015307962107,0.0006770449881893237,False
|
| 86 |
-
Pangu-Weather,pangu6,fire_prone,123,pixel MLP head,linear probe,0.02254610440161657,0.7185324707231184,0.7261721555350364,0.007639684811918013,False
|
| 87 |
-
Pangu-Weather,pangu6,global,1,shallow spatial adapter,shallow spatial adapter,0.00042881385578730365,0.32066974948800614,0.32066974948800614,0.0,True
|
| 88 |
-
Pangu-Weather,pangu6,global,7,shallow spatial adapter,shallow spatial adapter,0.00038395193539280824,0.31120670593952293,0.31120670593952293,0.0,True
|
| 89 |
-
Pangu-Weather,pangu6,global,42,shallow spatial adapter,shallow spatial adapter,0.00040651309469793043,0.32154249424354786,0.32154249424354786,0.0,True
|
| 90 |
-
Pangu-Weather,pangu6,global,99,shallow spatial adapter,shallow spatial adapter,0.0004450373086921994,0.3214547875801752,0.3214547875801752,0.0,True
|
| 91 |
-
Pangu-Weather,pangu6,global,123,shallow spatial adapter,shallow spatial adapter,0.00043738577276574255,0.31812983009681495,0.31812983009681495,0.0,True
|
| 92 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True
|
| 93 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True
|
| 94 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,42,pixel MLP head,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 95 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 96 |
-
Prithvi-WxC,prithvi_wxc,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True
|
| 97 |
-
Prithvi-WxC,prithvi_wxc,global,1,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 98 |
-
Prithvi-WxC,prithvi_wxc,global,7,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 99 |
-
Prithvi-WxC,prithvi_wxc,global,42,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 100 |
-
Prithvi-WxC,prithvi_wxc,global,99,shallow spatial adapter,shallow spatial adapter,0.0002454330184381399,0.23755358049655212,0.23755358049655212,0.0,True
|
| 101 |
-
Prithvi-WxC,prithvi_wxc,global,123,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 102 |
-
Reference,reference,fire_prone,1,shallow spatial adapter,linear probe,0.10204224118176683,0.799032457577039,0.8685988450931266,0.06956638751608757,False
|
| 103 |
-
Reference,reference,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.1323902067230726,0.8027807546359551,0.8027807546359551,0.0,True
|
| 104 |
-
Reference,reference,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.12048427320762313,0.8358151221553631,0.8358151221553631,0.0,True
|
| 105 |
-
Reference,reference,fire_prone,99,shallow spatial adapter,linear probe,0.11947246238005743,0.8534759193943048,0.9039923296574045,0.05051641026309972,False
|
| 106 |
-
Reference,reference,fire_prone,123,shallow spatial adapter,pixel MLP head,0.11551882470432043,0.8066689866810086,0.8567215417854325,0.05005255510442386,False
|
| 107 |
-
Reference,reference,global,1,shallow spatial adapter,linear probe,0.002624341503354088,0.6186566066408326,0.7286109603326707,0.10995435369183815,False
|
| 108 |
-
Reference,reference,global,7,shallow spatial adapter,shallow spatial adapter,0.0030317345997110698,0.5002193417313746,0.5002193417313746,0.0,True
|
| 109 |
-
Reference,reference,global,42,shallow spatial adapter,shallow spatial adapter,0.0032729435045747413,0.7239943741463363,0.7239943741463363,0.0,True
|
| 110 |
-
Reference,reference,global,99,shallow spatial adapter,linear probe,0.0031953098868323193,0.6698005926442897,0.7647466876509988,0.09494609500670914,False
|
| 111 |
-
Reference,reference,global,123,shallow spatial adapter,pixel MLP head,0.0025603887793133394,0.5009707461135111,0.735221546471909,0.23425080035839785,False
|
| 112 |
-
StormCast,stormcast,fire_prone,1,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 113 |
-
StormCast,stormcast,fire_prone,7,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 114 |
-
StormCast,stormcast,fire_prone,42,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 115 |
-
StormCast,stormcast,fire_prone,99,pixel MLP head,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 116 |
-
StormCast,stormcast,fire_prone,123,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False
|
| 117 |
-
StormCast,stormcast,global,1,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 118 |
-
StormCast,stormcast,global,7,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 119 |
-
StormCast,stormcast,global,42,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
| 120 |
-
StormCast,stormcast,global,99,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.2399251001974223,0.2399251001974223,0.0,True
|
| 121 |
-
StormCast,stormcast,global,123,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/selection_regret_rq2_figure_values.csv
DELETED
|
@@ -1,12 +0,0 @@
|
|
| 1 |
-
feature_source,global_mean_pp,global_std_pp,top20_mean_pp,top20_std_pp
|
| 2 |
-
FireWx-FM ref.,7.3831,7.4536,2.9385,2.7513
|
| 3 |
-
Prithvi-WxC,0.0000,0.0000,0.0000,0.0000
|
| 4 |
-
Aurora,4.9455,10.6974,14.3706,32.1337
|
| 5 |
-
ClimaX,0.1296,0.1775,0.0000,0.0000
|
| 6 |
-
StormCast,0.0000,0.0000,0.0000,0.0000
|
| 7 |
-
DLWP,0.0000,0.0000,4.4634,4.3561
|
| 8 |
-
FCN,0.0000,0.0000,1.1680,1.9872
|
| 9 |
-
FengWu,0.0000,0.0000,0.5222,0.6239
|
| 10 |
-
FuXi,0.0000,0.0000,0.2833,0.3681
|
| 11 |
-
Pangu-Weather,0.0000,0.0000,0.1868,0.3255
|
| 12 |
-
AlphaEarth,17.2217,8.8492,3.8804,5.9483
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/selection_regret_scope_sweep_20260505.csv
DELETED
|
@@ -1,45 +0,0 @@
|
|
| 1 |
-
model_tag,label,scope,scope_label,n,seeds,exact_regret_mean,exact_regret_std,exact_regret_min,exact_regret_max,tolerated_regret_mean,tolerated_regret_std,tolerated_regret_min,tolerated_regret_max,union_regret_mean,union_regret_std,union_regret_min,union_regret_max
|
| 2 |
-
reference,FireWx-FM ref.,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.07383089600948442,0.07453636071372995,0.0,0.17497107865629713,0.07383089600948442,0.07453636071372995,0.0,0.17497107865629713
|
| 3 |
-
reference,FireWx-FM ref.,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.003663718115532055,0.006812231244812292,0.0,0.015676762201120686,0.003663718115532055,0.006812231244812292,0.0,0.015676762201120686
|
| 4 |
-
reference,FireWx-FM ref.,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.012275489592085752,0.012665162001740834,0.0,0.02670880922526031,0.012275489592085752,0.012665162001740834,0.0,0.02670880922526031
|
| 5 |
-
reference,FireWx-FM ref.,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.029384646387840017,0.02751315335001922,0.0,0.05675140203555318,0.029384646387840017,0.02751315335001922,0.0,0.05675140203555318
|
| 6 |
-
prithvi_wxc,Prithvi-WxC,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 7 |
-
prithvi_wxc,Prithvi-WxC,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 8 |
-
prithvi_wxc,Prithvi-WxC,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 9 |
-
prithvi_wxc,Prithvi-WxC,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 10 |
-
aurora,Aurora,global,\(\Omega=\)global,5,1 7 42 99 123,0.00010153879814819402,0.00021861477435572763,0.0,0.0004925501476206198,0.04945471159670635,0.10697394238964528,0.0,0.240793572992212,0.04945471159670635,0.10697394238964528,0.0,0.240793572992212
|
| 11 |
-
aurora,Aurora,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.023667154505540456,0.05292136630837888,0.0,0.11833577252770228,0.1542829840966487,0.34498724021162547,0.0,0.7714149204832434,0.1542829840966487,0.34498724021162547,0.0,0.7714149204832434
|
| 12 |
-
aurora,Aurora,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.014651279478173606,0.032761256870543834,0.0,0.07325639739086803,0.1399343063221699,0.31290262132065055,0.0,0.6996715316108496,0.1399343063221699,0.31290262132065055,0.0,0.6996715316108496
|
| 13 |
-
aurora,Aurora,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.0,0.04101254412979794,0.1437064941446237,0.32133748971555404,0.0,0.7185324707231184,0.1437064941446237,0.32133748971555404,0.0,0.7185324707231184
|
| 14 |
-
climax,ClimaX,global,\(\Omega=\)global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0,7.571921560175121e-06,0.0012959969982639485,0.0017746169760203706,0.0,0.0032399924956598714,0.0012959969982639485,0.0017746169760203706,0.0,0.0032399924956598714
|
| 15 |
-
climax,ClimaX,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 16 |
-
climax,ClimaX,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 17 |
-
climax,ClimaX,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 18 |
-
stormcast,StormCast,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 19 |
-
stormcast,StormCast,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 20 |
-
stormcast,StormCast,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 21 |
-
stormcast,StormCast,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 22 |
-
dlwp,DLWP,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 23 |
-
dlwp,DLWP,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0048037709215293075,0.006217185202152866,0.0,0.015203005078871956,0.016716228534155796,0.016079313546074458,0.0,0.03305057342744666,0.016716228534155796,0.016079313546074458,0.0,0.03305057342744666
|
| 24 |
-
dlwp,DLWP,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0017281632798742507,0.002514722758075371,0.0,0.005523780499856246,0.02846514801700826,0.026938012702643194,0.0,0.053927677500854476,0.02846514801700826,0.026938012702643194,0.0,0.053927677500854476
|
| 25 |
-
dlwp,DLWP,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.0,0.0023651634514294945,0.04463354681768479,0.04356064433532197,0.0,0.11215006616165157,0.04463354681768479,0.04356064433532197,0.0,0.11215006616165157
|
| 26 |
-
fcn,FCN,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 27 |
-
fcn,FCN,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0006342898232943345,0.0009899554165032742,0.0,0.002257520679520411,0.004509624980300697,0.010070611656609236,0.0,0.022524473456150496,0.004509624980300697,0.010070611656609236,0.0,0.022524473456150496
|
| 28 |
-
fcn,FCN,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.00021156854817603877,0.0004730816556225618,0.0,0.0010578427408801938,0.004199537050817615,0.009390450319657174,0.0,0.020997685254088072,0.004199537050817615,0.009390450319657174,0.0,0.020997685254088072
|
| 29 |
-
fcn,FCN,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,5.754560074337778e-06,1.2867587506825515e-05,0.0,2.877280037168889e-05,0.011679805987441694,0.019872372458657642,0.0,0.045880503237594294,0.011679805987441694,0.019872372458657642,0.0,0.045880503237594294
|
| 30 |
-
fengwu,FengWu,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 31 |
-
fengwu,FengWu,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0005029843170376968,0.0008109166521114917,0.0,0.0018628094907809783,0.008795951947678215,0.005532321338017505,0.0,0.01484136735033148,0.008795951947678215,0.005532321338017505,0.0,0.01484136735033148
|
| 32 |
-
fengwu,FengWu,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.000495228089292582,0.0007349190216431337,0.0,0.0016387212062008855,0.00402300475984525,0.005510851442075993,0.0,0.010273937098576491,0.00402300475984525,0.005510851442075993,0.0,0.010273937098576491
|
| 33 |
-
fengwu,FengWu,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.0,0.0027505832409660327,0.005222389249812243,0.0062394095558402415,0.0,0.015150501067724198,0.005222389249812243,0.0062394095558402415,0.0,0.015150501067724198
|
| 34 |
-
fuxi,FuXi,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 35 |
-
fuxi,FuXi,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.002973545331200933,0.0023946274991058026,0.0010927807990139538,0.007024214143542151,0.013545122545609134,0.02097023683418404,0.0,0.050156261654859424,0.013545122545609134,0.02097023683418404,0.0,0.050156261654859424
|
| 36 |
-
fuxi,FuXi,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.001383793743586542,0.0019248128430711165,0.0,0.003938013087198336,0.0016559834970027332,0.0037028916689159307,0.0,0.008279917485013666,0.0016559834970027332,0.0037028916689159307,0.0,0.008279917485013666
|
| 37 |
-
fuxi,FuXi,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.00283318355751887,0.0036808289681375247,0.0,0.008746323525994693,0.00283318355751887,0.0036808289681375247,0.0,0.008746323525994693
|
| 38 |
-
pangu6,Pangu-Weather,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
|
| 39 |
-
pangu6,Pangu-Weather,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.003154674487241463,0.002546125713211599,0.0,0.005711594157251587,0.007592888149777122,0.00897418737588444,0.0,0.019790633919317124,0.007592888149777122,0.00897418737588444,0.0,0.019790633919317124
|
| 40 |
-
pangu6,Pangu-Weather,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0017345627303725214,0.0019305189318827886,0.0,0.004535321555179647,0.003047840737004992,0.005053805614558161,0.0,0.011660780793438352,0.003047840737004992,0.005053805614558161,0.0,0.011660780793438352
|
| 41 |
-
pangu6,Pangu-Weather,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0,0.0027096086413018403,0.0018679847512695024,0.0032548337047755126,0.0,0.007639684811918013,0.0018679847512695024,0.0032548337047755126,0.0,0.007639684811918013
|
| 42 |
-
alphaearth,AlphaEarth,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.1722171037486726,0.08849214830495522,0.050655329891645895,0.272468682523694,0.1722171037486726,0.08849214830495522,0.050655329891645895,0.272468682523694
|
| 43 |
-
alphaearth,AlphaEarth,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.06384618090125256,0.04965276403138872,0.0,0.1365277562230962,0.06384618090125256,0.04965276403138872,0.0,0.1365277562230962
|
| 44 |
-
alphaearth,AlphaEarth,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.06573776411084173,0.06897015340160571,0.0,0.1615566566666954,0.06573776411084173,0.06897015340160571,0.0,0.1615566566666954
|
| 45 |
-
alphaearth,AlphaEarth,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.038803552655092256,0.0594825313313219,0.0,0.13482302181308625,0.038803552655092256,0.0594825313313219,0.0,0.13482302181308625
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/selection_regret_scope_sweep_20260505.json
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
artifacts/results/selection_regret_summary.csv
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
model_tag,label,scope,n,seeds,exact_regret_mean,exact_regret_std,tolerated_regret_mean,tolerated_regret_std,union_regret_mean,union_regret_std
|
| 2 |
-
reference,Reference,global,5,1 7 42 99 123,0.0,0.0,0.08783024981138902,0.09670495645481135,0.08783024981138902,0.09670495645481135
|
| 3 |
-
reference,Reference,fire_prone,5,1 7 42 99 123,0.0,0.0,0.03402707057672223,0.032044658643147844,0.03402707057672223,0.032044658643147844
|
| 4 |
-
prithvi_wxc,Prithvi-WxC,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 5 |
-
prithvi_wxc,Prithvi-WxC,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 6 |
-
aurora,Aurora,global,5,1 7 42 99 123,0.00020004882767231798,0.00026703384456332115,0.09851983041506818,0.1298781980037557,0.09851983041506818,0.1298781980037557
|
| 7 |
-
aurora,Aurora,fire_prone,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.14391889430974364,0.32121904665016227,0.14391889430974364,0.32121904665016227
|
| 8 |
-
climax,ClimaX,global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0012959969982639485,0.0017746169760203706,0.0012959969982639485,0.0017746169760203706
|
| 9 |
-
climax,ClimaX,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 10 |
-
stormcast,StormCast,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 11 |
-
stormcast,StormCast,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 12 |
-
pangu_weather,Pangu-Weather,global,5,1 7 42 99 123,0.00013033979247265275,0.0002685372203690466,0.048806713097574374,0.10733308684741971,0.048806713097574374,0.10733308684741971
|
| 13 |
-
pangu_weather,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.027875386332505546,0.02348779386900393,0.43111948243387105,0.39355644251497235,0.43111948243387105,0.39355644251497235
|
| 14 |
-
dlwp,DLWP,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 15 |
-
dlwp,DLWP,fire_prone,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.043265915053601556,0.04332331365579739,0.043265915053601556,0.04332331365579739
|
| 16 |
-
fcn,FCN,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 17 |
-
fcn,FCN,fire_prone,5,1 7 42 99 123,5.960229415004348e-06,1.3327478133443526e-05,0.011679805987441694,0.019872372458657642,0.011679805987441694,0.019872372458657642
|
| 18 |
-
fengwu,FengWu,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 19 |
-
fengwu,FengWu,fire_prone,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.005222389249812243,0.0062394095558402415,0.005222389249812243,0.0062394095558402415
|
| 20 |
-
fuxi,FuXi,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 21 |
-
fuxi,FuXi,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0010839188523199318,0.0017288780545672386,0.0010839188523199318,0.0017288780545672386
|
| 22 |
-
pangu6,Pangu-Weather,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0
|
| 23 |
-
pangu6,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0018491271881979853,0.0032630386057089294,0.0018491271881979853,0.0032630386057089294
|
| 24 |
-
alphaearth,AlphaEarth,global,5,1 7 42 99 123,0.0,0.0,0.1722171037486726,0.08849214830495522,0.1722171037486726,0.08849214830495522
|
| 25 |
-
alphaearth,AlphaEarth,fire_prone,5,1 7 42 99 123,0.0,0.0,0.038803552655092256,0.0594825313313219,0.038803552655092256,0.0594825313313219
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
artifacts/results/supporting_table4_event_repaired_compsci_20260902_summary_clean.md
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Forced Mean/Std Gap-Fill Summary
|
| 2 |
+
|
| 3 |
+
## table2_headcontrol
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
## table3_occupancy
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
## table3_spread
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## table4_final_area
|
| 13 |
+
|
| 14 |
+
- AlphaEarth: log_rmse 2.46 +/- 0.00 (n=5); log_mae 2.10 +/- 0.00 (n=5); log_spearman -0.30 +/- 0.00 (n=5)
|
| 15 |
+
- Aurora: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 16 |
+
- ClimaX: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 17 |
+
- Pangu24: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 18 |
+
- Prithvi-WxC: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 19 |
+
- Reference: log_rmse 1.17 +/- 0.01 (n=5); log_mae 1.05 +/- 0.01 (n=5); log_spearman 0.59 +/- 0.03 (n=5)
|
| 20 |
+
- StormCast: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 21 |
+
|
| 22 |
+
## table4_analog
|
| 23 |
+
|
| 24 |
+
- AlphaEarth: ndcg_at_10 0.50 +/- 0.00 (n=5); log_rmse 1.20 +/- 0.00 (n=5); log_mae 1.02 +/- 0.00 (n=5)
|
| 25 |
+
- Aurora: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 26 |
+
- ClimaX: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 27 |
+
- Pangu24: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 28 |
+
- Prithvi-WxC: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 29 |
+
- StormCast: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 30 |
+
- reference_strict: ndcg_at_10 0.52 +/- 0.00 (n=5); log_rmse 1.18 +/- 0.00 (n=5); log_mae 0.97 +/- 0.00 (n=5)
|
| 31 |
+
|
| 32 |
+
## table4_smoke
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
## table4_heat
|
| 36 |
+
|
artifacts/results/supporting_table4_smoke_heat_compsci_20260902_summary_clean.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Forced Mean/Std Gap-Fill Summary
|
| 2 |
+
|
| 3 |
+
## table2_headcontrol
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
## table3_occupancy
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
## table3_spread
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## table4_final_area
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
## table4_analog
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
## table4_smoke
|
| 19 |
+
|
| 20 |
+
- AlphaEarth: rmse 4.49 +/- 0.00 (n=5); mae 2.40 +/- 0.00 (n=5); pearson_r 0.63 +/- 0.00 (n=5)
|
| 21 |
+
- Aurora: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5)
|
| 22 |
+
- ClimaX: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.00 +/- 0.00 (n=5)
|
| 23 |
+
- Pangu24: rmse 5.95 +/- 0.00 (n=5); mae 3.75 +/- 0.00 (n=5); pearson_r 0.04 +/- 0.00 (n=5)
|
| 24 |
+
- Reference: rmse 4.47 +/- 0.01 (n=5); mae 2.41 +/- 0.00 (n=5); pearson_r 0.64 +/- 0.00 (n=5)
|
| 25 |
+
- StormCast: rmse 6.02 +/- 0.00 (n=5); mae 3.82 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5)
|
| 26 |
+
|
| 27 |
+
## table4_heat
|
| 28 |
+
|
| 29 |
+
- Reference: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5)
|
artifacts/results/supporting_table4_smoke_heat_fill_compsci_20260902_summary_clean.md
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Forced Mean/Std Gap-Fill Summary
|
| 2 |
+
|
| 3 |
+
## table2_headcontrol
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
## table3_occupancy
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
## table3_spread
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## table4_final_area
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
## table4_analog
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
## table4_smoke
|
| 19 |
+
|
| 20 |
+
- Prithvi-WxC: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5)
|
| 21 |
+
|
| 22 |
+
## table4_heat
|
| 23 |
+
|
| 24 |
+
- AlphaEarth: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5)
|
artifacts/results/tune4_repaired_eventfm_summary.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Forced Mean/Std Gap-Fill Summary
|
| 2 |
+
|
| 3 |
+
## table2_headcontrol
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
## table3_occupancy
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
## table3_spread
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## table4_final_area
|
| 13 |
+
|
| 14 |
+
- Aurora: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 15 |
+
- ClimaX: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 16 |
+
- Pangu24: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 17 |
+
- Prithvi-WxC: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 18 |
+
- StormCast: log_rmse 1.69 +/- 0.00 (n=5); log_mae 1.51 +/- 0.00 (n=5); log_spearman -0.12 +/- 0.00 (n=5)
|
| 19 |
+
|
| 20 |
+
## table4_analog
|
| 21 |
+
|
| 22 |
+
- Aurora: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 23 |
+
- ClimaX: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 24 |
+
- Pangu24: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 25 |
+
- Prithvi-WxC: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 26 |
+
- StormCast: ndcg_at_10 0.41 +/- 0.00 (n=5); log_rmse 1.41 +/- 0.00 (n=5); log_mae 1.32 +/- 0.00 (n=5)
|
| 27 |
+
|
| 28 |
+
## table4_smoke
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
## table4_heat
|
| 32 |
+
|
artifacts/results/tune4_throttled_summary.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Forced Mean/Std Gap-Fill Summary
|
| 2 |
+
|
| 3 |
+
## table2_headcontrol
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
## table3_occupancy
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
## table3_spread
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## table4_final_area
|
| 13 |
+
|
| 14 |
+
- AlphaEarth: log_rmse 2.46 +/- 0.00 (n=5); log_mae 2.10 +/- 0.00 (n=5); log_spearman -0.30 +/- 0.00 (n=5)
|
| 15 |
+
- Reference: log_rmse 1.17 +/- 0.01 (n=5); log_mae 1.05 +/- 0.01 (n=5); log_spearman 0.59 +/- 0.03 (n=5)
|
| 16 |
+
|
| 17 |
+
## table4_analog
|
| 18 |
+
|
| 19 |
+
- AlphaEarth: ndcg_at_10 0.50 +/- 0.00 (n=5); log_rmse 1.20 +/- 0.00 (n=5); log_mae 1.02 +/- 0.00 (n=5)
|
| 20 |
+
- reference_strict: ndcg_at_10 0.52 +/- 0.00 (n=5); log_rmse 1.18 +/- 0.00 (n=5); log_mae 0.97 +/- 0.00 (n=5)
|
| 21 |
+
|
| 22 |
+
## table4_smoke
|
| 23 |
+
|
| 24 |
+
- AlphaEarth: rmse 4.49 +/- 0.00 (n=5); mae 2.40 +/- 0.00 (n=5); pearson_r 0.63 +/- 0.00 (n=5)
|
| 25 |
+
- Aurora: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5)
|
| 26 |
+
- ClimaX: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.00 +/- 0.00 (n=5)
|
| 27 |
+
- Pangu24: rmse 5.95 +/- 0.00 (n=5); mae 3.75 +/- 0.00 (n=5); pearson_r 0.04 +/- 0.00 (n=5)
|
| 28 |
+
- Prithvi-WxC: rmse 5.93 +/- 0.00 (n=5); mae 3.73 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5)
|
| 29 |
+
- Reference: rmse 4.47 +/- 0.01 (n=5); mae 2.41 +/- 0.00 (n=5); pearson_r 0.64 +/- 0.00 (n=5)
|
| 30 |
+
- StormCast: rmse 6.02 +/- 0.00 (n=5); mae 3.82 +/- 0.00 (n=5); pearson_r 0.02 +/- 0.00 (n=5)
|
| 31 |
+
|
| 32 |
+
## table4_heat
|
| 33 |
+
|
| 34 |
+
- AlphaEarth: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5)
|
| 35 |
+
- Reference: rmse_c 0.22 +/- 0.00 (n=5); mae_c 0.18 +/- 0.00 (n=5); pearson_r 1.00 +/- 0.00 (n=5)
|
assets/release_contents.svg
DELETED
assets/wildfire_fm_model_card.svg
DELETED
data_downloader/README.md
DELETED
|
@@ -1,133 +0,0 @@
|
|
| 1 |
-
# Data Downloader
|
| 2 |
-
|
| 3 |
-
This directory contains a lightweight, client-facing downloader for data sources
|
| 4 |
-
used by FireWx-FM and planned general hazard-model work. It is intentionally a
|
| 5 |
-
raw-data downloader, not a model-ready dataloader. Dataset adapters and aligned
|
| 6 |
-
caches should be built after raw files are obtained under each provider's terms.
|
| 7 |
-
|
| 8 |
-
## Design
|
| 9 |
-
|
| 10 |
-
The downloader follows the older `Pyhazard_data_downloader` API pattern:
|
| 11 |
-
|
| 12 |
-
```python
|
| 13 |
-
from data_downloader import DownloadRequest, download_data, list_datasets
|
| 14 |
-
```
|
| 15 |
-
|
| 16 |
-
Each dataset declares:
|
| 17 |
-
|
| 18 |
-
- `access`: whether it is public direct download, ArcGIS REST, auth-required, or instructions-only.
|
| 19 |
-
- `time_mode`: how time selection works.
|
| 20 |
-
- `aoi_mode`: whether bounding-box selection is native, post-processing only, or unsupported.
|
| 21 |
-
|
| 22 |
-
This avoids pretending that every source supports the same query semantics.
|
| 23 |
-
|
| 24 |
-
## Quick Start
|
| 25 |
-
|
| 26 |
-
List datasets:
|
| 27 |
-
|
| 28 |
-
```bash
|
| 29 |
-
python3 -m data_downloader.cli --list
|
| 30 |
-
```
|
| 31 |
-
|
| 32 |
-
Dry-run a small client request:
|
| 33 |
-
|
| 34 |
-
```bash
|
| 35 |
-
python3 -m data_downloader.cli \
|
| 36 |
-
--datasets aqs_pm25 hms_smoke ibtracs \
|
| 37 |
-
--output-root ./downloads_demo \
|
| 38 |
-
--start-year 2024 \
|
| 39 |
-
--end-year 2024 \
|
| 40 |
-
--dry-run \
|
| 41 |
-
--max-files 3 \
|
| 42 |
-
--report ./downloads_demo/report.json
|
| 43 |
-
```
|
| 44 |
-
|
| 45 |
-
For datasets that use a bounding box, pass negative longitudes with the equals
|
| 46 |
-
form so the shell/argparse does not parse them as flags:
|
| 47 |
-
|
| 48 |
-
```bash
|
| 49 |
-
python3 -m data_downloader.cli \
|
| 50 |
-
--datasets firms_area \
|
| 51 |
-
--output-root ./downloads_firms \
|
| 52 |
-
--start-date 2024-06-01 \
|
| 53 |
-
--end-date 2024-06-02 \
|
| 54 |
-
--bbox=-125,32,-114,42 \
|
| 55 |
-
--dry-run
|
| 56 |
-
```
|
| 57 |
-
|
| 58 |
-
Python API:
|
| 59 |
-
|
| 60 |
-
```python
|
| 61 |
-
from data_downloader import DownloadRequest, download_data
|
| 62 |
-
|
| 63 |
-
request = DownloadRequest(
|
| 64 |
-
datasets=["aqs_pm25", "hms_smoke"],
|
| 65 |
-
output_root="./downloads",
|
| 66 |
-
years=(2024, 2025),
|
| 67 |
-
dry_run=False,
|
| 68 |
-
)
|
| 69 |
-
results = download_data(request)
|
| 70 |
-
```
|
| 71 |
-
|
| 72 |
-
Download HRRR files from the public NOAA AWS archive:
|
| 73 |
-
|
| 74 |
-
```bash
|
| 75 |
-
python3 data_downloader/hrrr_downloader.py \
|
| 76 |
-
--start-date 2024-06-01 \
|
| 77 |
-
--end-date 2024-06-02 \
|
| 78 |
-
--hours 00,06,12,18 \
|
| 79 |
-
--forecast-hours 00 \
|
| 80 |
-
--output-root ./downloads_hrrr \
|
| 81 |
-
--include-idx \
|
| 82 |
-
--dry-run
|
| 83 |
-
```
|
| 84 |
-
|
| 85 |
-
Remove `--dry-run` to download the files. The script writes
|
| 86 |
-
`hrrr_download_manifest.json` with every URL, output path, and download status.
|
| 87 |
-
The HRRR downloader retrieves native HRRR files; county, sub-county, or any
|
| 88 |
-
other custom spatial granularity should be produced later by the serving or
|
| 89 |
-
preprocessing layer using user-provided boundaries.
|
| 90 |
-
|
| 91 |
-
For a small real-download smoke test, download only the HRRR index sidecar:
|
| 92 |
-
|
| 93 |
-
```bash
|
| 94 |
-
python3 data_downloader/hrrr_downloader.py \
|
| 95 |
-
--start-date 2024-06-01 \
|
| 96 |
-
--end-date 2024-06-01 \
|
| 97 |
-
--hours 00 \
|
| 98 |
-
--forecast-hours 00 \
|
| 99 |
-
--idx-only \
|
| 100 |
-
--max-files 1 \
|
| 101 |
-
--output-root ./downloads_hrrr_idx_test
|
| 102 |
-
```
|
| 103 |
-
|
| 104 |
-
## Implemented Direct Downloads
|
| 105 |
-
|
| 106 |
-
These sources are implemented as direct public downloads:
|
| 107 |
-
|
| 108 |
-
- `aqs_pm25`: EPA AirData hourly PM2.5 files and site/monitor metadata.
|
| 109 |
-
- `hms_smoke`: NOAA HMS annual smoke-polygon and fire-point bundles.
|
| 110 |
-
- `ibtracs`: NOAA/NCEI IBTrACS v04r01 CSV tracks and documentation.
|
| 111 |
-
- `hurdat2`: NOAA/NHC HURDAT2 latest best-track text files and format docs.
|
| 112 |
-
- `usdm`: U.S. Drought Monitor weekly GIS shapefile archive.
|
| 113 |
-
- `hrrr_fireseason`: NOAA HRRR public S3 files for selected years, fire-season dates, hours, and product.
|
| 114 |
-
- `wfigs_current`: current WFIGS perimeter snapshot through ArcGIS REST.
|
| 115 |
-
- `landfire_static`: known LF2024 CONUS `FBFM40` and `CC` static products.
|
| 116 |
-
|
| 117 |
-
## Credential or Terms-Limited Sources
|
| 118 |
-
|
| 119 |
-
These are intentionally not silently downloaded:
|
| 120 |
-
|
| 121 |
-
- `firms_area`: requires a NASA FIRMS `MAP_KEY`; set `FIRMS_MAP_KEY` or pass it in `extra_options`.
|
| 122 |
-
- `merra2`: requires NASA Earthdata/GES DISC authenticated access.
|
| 123 |
-
- `landscan`: governed by ORNL/LandScan provider-specific access terms.
|
| 124 |
-
- `wrc_housing`: provider links can redirect/change; pass an explicit URL or use the provider portal.
|
| 125 |
-
- `mtbs_perimeters`: heavy ArcGIS chunk pulls are available in internal Slurm scripts, but this public client records instructions first.
|
| 126 |
-
|
| 127 |
-
## Notes for Clients
|
| 128 |
-
|
| 129 |
-
- Raw source files are not redistributed by this repository.
|
| 130 |
-
- The downloader writes `download_manifest.json` under each dataset output folder.
|
| 131 |
-
- Use `--dry-run` and `--max-files` before large requests.
|
| 132 |
-
- For large HRRR, GOES, MERRA-2, NWM, or full MTBS/RAVG pulls, run downloads on compute/storage infrastructure rather than a login node.
|
| 133 |
-
- `hrrr_downloader.py` is the direct HRRR client for custom date/hour/forecast-hour requests.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data_sources/DATA_SOURCES.md
CHANGED
|
@@ -1,72 +1,55 @@
|
|
| 1 |
# Data Sources
|
| 2 |
|
| 3 |
-
This
|
| 4 |
-
paper. The table below records each resource, its role in the experiments, and
|
| 5 |
-
public access entry points. Users should obtain each source under its own terms.
|
| 6 |
|
| 7 |
-
|
| 8 |
-
|---|---|---|
|
| 9 |
-
| NOAA High-Resolution Rapid Refresh (HRRR) | Dynamic weather fields for the California regional gridded occupancy inputs. | NOAA/NCEI HRRR product page: <https://www.ncei.noaa.gov/products/weather-climate-models/high-resolution-rapid-refresh>; AWS Open Data archive: <https://registry.opendata.aws/noaa-hrrr-pds/>. |
|
| 10 |
-
| NASA FIRMS active-fire detections | Active-fire detections used to derive gridded occupancy labels. | FIRMS download and API services: <https://firms.modaps.eosdis.nasa.gov/download/> and <https://firms.modaps.eosdis.nasa.gov/api/>. |
|
| 11 |
-
| LANDFIRE 40 Fire Behavior Fuel Models | Static fuel layer used in the FireWx-FM gridded input. | LANDFIRE data access portal: <https://landfire.gov/data>. |
|
| 12 |
-
| LANDFIRE Forest Canopy Cover | Static canopy layer used in the FireWx-FM gridded input. | LANDFIRE data access portal: <https://landfire.gov/data>. |
|
| 13 |
-
| Wildfire Risk to Communities housing-unit density | Static exposure layer used in the FireWx-FM gridded input. | Wildfire Risk to Communities data access: <https://wildfirerisk.org/download/>. |
|
| 14 |
-
| LandScan Global 2024 | Static population layer used in the FireWx-FM gridded input. | Oak Ridge National Laboratory LandScan access: <https://landscan.ornl.gov/>. |
|
| 15 |
-
| WFIGS incident/perimeter attributes | Event-level incident metadata for supporting burned-area and analog tasks. | NIFC Open Data portal for WFIGS layers: <https://data-nifc.opendata.arcgis.com/>. |
|
| 16 |
-
| MTBS burned area and burn severity | Event-scale burned-area and burn-severity records for supporting tasks. | MTBS data access and direct download pages: <https://www.mtbs.gov/> and <https://www.mtbs.gov/direct-download>. |
|
| 17 |
-
| Earth-FM/backbone sources | Frozen feature sources for transferred Earth-FM comparisons. | Original model providers and their terms. Examples include Hugging Face model cards, model-provider GitHub repositories, and provider-hosted model files. |
|
| 18 |
-
|
| 19 |
-
## Pretrained Input Channels
|
| 20 |
-
|
| 21 |
-
The released checkpoints expect a fixed 16-channel tensor in `[channel, y, x]`
|
| 22 |
-
order. Channel order matters. The pretrained 5 km, 12-hour-lead occupancy input
|
| 23 |
-
uses the following contract. A machine-readable copy is stored at
|
| 24 |
-
`models/wildfire_fm/input_channels.json`.
|
| 25 |
-
|
| 26 |
-
No mean/std normalization, min/max scaling, or unit conversion is applied before
|
| 27 |
-
inference. HRRR fields remain in native units after reprojection to the model
|
| 28 |
-
grid. Invalid or missing values are zero-filled, with validity masks included as
|
| 29 |
-
channels 10 and 11.
|
| 30 |
|
| 31 |
-
|
|
| 32 |
-
|---
|
| 33 |
-
|
|
| 34 |
-
|
|
| 35 |
-
|
|
| 36 |
-
|
|
| 37 |
-
|
|
| 38 |
-
|
|
| 39 |
-
|
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
|
| 46 |
-
|
|
| 47 |
-
|
|
| 48 |
-
|
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
`
|
| 62 |
-
|
| 63 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
## Notes
|
| 66 |
|
| 67 |
-
-
|
| 68 |
-
-
|
| 69 |
-
-
|
| 70 |
-
construct the intermediate grids/features described in the paper.
|
| 71 |
-
- Access mechanisms and licensing can change. The links above point to the
|
| 72 |
-
original providers.
|
|
|
|
| 1 |
# Data Sources
|
| 2 |
|
| 3 |
+
This release documents the source datasets needed for the FireWx-FM active-fire occupancy model. Raw source data are not redistributed. Users should obtain each resource from the original provider and follow the provider terms.
|
|
|
|
|
|
|
| 4 |
|
| 5 |
+
## Source Inventory
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
+
| Source | Role | Access |
|
| 8 |
+
|---|---|---|
|
| 9 |
+
| NOAA High-Resolution Rapid Refresh (HRRR) | Dynamic weather input channels on the 5 km grid. | NOAA/NCEI product page: <https://www.ncei.noaa.gov/products/weather-climate-models/high-resolution-rapid-refresh>; AWS Open Data archive: <https://registry.opendata.aws/noaa-hrrr-pds/>. |
|
| 10 |
+
| NASA FIRMS active-fire detections | Active-fire detections used to derive the occupancy target. | FIRMS download and API services: <https://firms.modaps.eosdis.nasa.gov/download/> and <https://firms.modaps.eosdis.nasa.gov/api/>. |
|
| 11 |
+
| LANDFIRE 40 Fire Behavior Fuel Models | Static fuel-model input channel. | LANDFIRE data portal: <https://landfire.gov/data>. |
|
| 12 |
+
| LANDFIRE canopy cover | Static canopy-cover input channel. | LANDFIRE data portal: <https://landfire.gov/data>. |
|
| 13 |
+
| Wildfire Risk to Communities housing-unit density | Static exposure input channel; zeroed in final no-exposure serving. | Wildfire Risk to Communities downloads: <https://wildfirerisk.org/download/>. |
|
| 14 |
+
| LandScan Global 2024 | Static population input channel; zeroed in final no-exposure serving. | ORNL LandScan access: <https://landscan.ornl.gov/>. |
|
| 15 |
+
| USGS Fire Danger Forecast | Used only for the calibrated example output, not as a training input channel. | USGS Fire Danger Forecast viewer and downloads: <https://firedanger.cr.usgs.gov/viewer/index.html>. |
|
| 16 |
+
|
| 17 |
+
## Input Channels
|
| 18 |
+
|
| 19 |
+
The released checkpoint expects a fixed 16-channel tensor in `[channel, y, x]` order.
|
| 20 |
+
|
| 21 |
+
| Channel | Name | Dataset/source | Level or selection | Units | Serving treatment |
|
| 22 |
+
|---:|---|---|---|---|---|
|
| 23 |
+
| 0 | `t2m` | NOAA HRRR | 2 m above ground | K | z-score normalized |
|
| 24 |
+
| 1 | `d2m` | NOAA HRRR | 2 m above ground | K | z-score normalized |
|
| 25 |
+
| 2 | `u10` | NOAA HRRR | 10 m above ground | m s-1 | z-score normalized |
|
| 26 |
+
| 3 | `v10` | NOAA HRRR | 10 m above ground | m s-1 | z-score normalized |
|
| 27 |
+
| 4 | `cape` | NOAA HRRR | 0-3000 m above-ground layer | J kg-1 | z-score normalized |
|
| 28 |
+
| 5 | `sp` | NOAA HRRR | surface | Pa | z-score normalized |
|
| 29 |
+
| 6 | `blh` | NOAA HRRR | boundary-layer diagnostic | m | z-score normalized |
|
| 30 |
+
| 7 | `vis` | NOAA HRRR | visibility diagnostic | m | z-score normalized |
|
| 31 |
+
| 8 | `prate` | NOAA HRRR | surface precipitation rate | kg m-2 s-1 | z-score normalized |
|
| 32 |
+
| 9 | `tp` | NOAA HRRR | accumulated precipitation | kg m-2 | z-score normalized |
|
| 33 |
+
| 10 | `dynamic_valid` | input builder | dynamic weather validity | 0-1 | passed through |
|
| 34 |
+
| 11 | `static_valid` | input builder | static-layer validity fraction | 0-1 | passed through |
|
| 35 |
+
| 12 | `fuel_fbfm40` | LANDFIRE | fire-behavior fuel model | category code | passed through |
|
| 36 |
+
| 13 | `canopy_cover` | LANDFIRE | canopy cover | percent | z-score normalized |
|
| 37 |
+
| 14 | `housing_density` | Wildfire Risk to Communities | static exposure | provider native | z-score normalized, then zeroed |
|
| 38 |
+
| 15 | `population` | LandScan Global 2024 | static exposure | persons | z-score normalized, then zeroed |
|
| 39 |
+
|
| 40 |
+
FIRMS detections define the occupancy target and are not input channels.
|
| 41 |
+
|
| 42 |
+
## Static Resampling
|
| 43 |
+
|
| 44 |
+
| Static layer | Resampling |
|
| 45 |
+
|---|---|
|
| 46 |
+
| LANDFIRE fire-behavior fuel model | nearest |
|
| 47 |
+
| LANDFIRE canopy cover | nearest |
|
| 48 |
+
| Wildfire Risk to Communities housing density | bilinear |
|
| 49 |
+
| LandScan population | bilinear |
|
| 50 |
|
| 51 |
## Notes
|
| 52 |
|
| 53 |
+
- Native grid: Lower-48 CONUS, EPSG:5070, 5 km.
|
| 54 |
+
- Prediction target: 12-hour active-fire occupancy probability.
|
| 55 |
+
- County and sub-county products are post-inference aggregations of the 5 km probability grid.
|
|
|
|
|
|
|
|
|
docs/artifact_map.md
DELETED
|
@@ -1,33 +0,0 @@
|
|
| 1 |
-
# Public Artifact Map
|
| 2 |
-
|
| 3 |
-
This map describes the public Hugging Face release boundary. The Hub release
|
| 4 |
-
publishes model weights, model-loading code, compact numeric summaries, selected
|
| 5 |
-
figure artifacts, source-data notes, and the compiled paper PDF. Manuscript
|
| 6 |
-
source remains in the authors' paper workspace.
|
| 7 |
-
|
| 8 |
-
## Included Public Artifacts
|
| 9 |
-
|
| 10 |
-
| Area | Release files | Notes |
|
| 11 |
-
|---|---|---|
|
| 12 |
-
| Model code | `models/wildfire_fm/modeling_unet.py` | Compact U-Net used by the released checkpoints. |
|
| 13 |
-
| Checkpoint metadata | `models/wildfire_fm/checkpoint_manifest.json` | Lists five seeded checkpoint paths, SHA-256 hashes, and byte sizes. |
|
| 14 |
-
| Paper PDF | `paper/wildfire_fm_evaluation_contracts.pdf` | Compiled from the final local manuscript source. |
|
| 15 |
-
| Figure previews | `assets/*.png`, `assets/*.svg` | Hub-page visuals and final-paper figure previews. |
|
| 16 |
-
| Figure PDFs | `paper_outputs/figures/*.pdf` | Selected final-paper figures retained for visual reproducibility. |
|
| 17 |
-
| Numeric summaries | `artifacts/results/*.csv`, `artifacts/results/*.json` | Sanitized compact summaries; local machine paths removed. |
|
| 18 |
-
| Data notes | `data_sources/DATA_SOURCES.md` | Source roles and access entry points for provider-hosted data. |
|
| 19 |
-
| Raw rerun references | `experiments/` | Sanitized scripts/templates requiring user-provided data and paths. |
|
| 20 |
-
|
| 21 |
-
## Manuscript Source Boundary
|
| 22 |
-
|
| 23 |
-
The public paper artifact is the compiled PDF. Manuscript TeX, generated table
|
| 24 |
-
source, TikZ source, and BibTeX are managed outside the Hub model release.
|
| 25 |
-
|
| 26 |
-
## Verification
|
| 27 |
-
|
| 28 |
-
```bash
|
| 29 |
-
python3 scripts/reproduce_paper_outputs.py
|
| 30 |
-
```
|
| 31 |
-
|
| 32 |
-
This command checks public artifact hashes and audits the release hygiene for
|
| 33 |
-
local paths and manuscript-source boundaries.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/huggingface_release_design.md
DELETED
|
@@ -1,33 +0,0 @@
|
|
| 1 |
-
# Hugging Face Release Design
|
| 2 |
-
|
| 3 |
-
This release follows a model-card-first pattern used by geospatial and weather
|
| 4 |
-
foundation-model releases on the Hub. The public page starts with model purpose,
|
| 5 |
-
released weights, input expectations, a quick loading example, data boundaries,
|
| 6 |
-
and a compact evaluation snapshot. Paper artifacts are included as supporting
|
| 7 |
-
reproducibility material rather than being the main organizing principle.
|
| 8 |
-
|
| 9 |
-
## Public Card Structure
|
| 10 |
-
|
| 11 |
-
- `README.md` is the public model card. It contains YAML metadata, intended use,
|
| 12 |
-
checkpoint locations, quick loading code, data-source boundaries, limitations,
|
| 13 |
-
and citation text.
|
| 14 |
-
- `assets/` contains lightweight visuals for the Hub page plus PNG previews of
|
| 15 |
-
final-paper figure previews.
|
| 16 |
-
- `models/wildfire_fm/` contains model code, manifests, and checkpoint metadata.
|
| 17 |
-
- `paper_outputs/` stores selected final-paper figure PDFs used for public
|
| 18 |
-
visual checking.
|
| 19 |
-
- `artifacts/results/` stores sanitized compact CSV/JSON summaries that can be public.
|
| 20 |
-
- `data_sources/` documents external data resources without redistributing them.
|
| 21 |
-
- `experiments/` contains raw-rerun reference scripts and Slurm templates.
|
| 22 |
-
|
| 23 |
-
## Data Boundary
|
| 24 |
-
|
| 25 |
-
The repository is a model release with reproducibility artifacts. Full raw-data
|
| 26 |
-
reruns require separately obtained source data, local feature caches, and
|
| 27 |
-
cluster-specific paths.
|
| 28 |
-
|
| 29 |
-
## Manuscript Boundary
|
| 30 |
-
|
| 31 |
-
The Hub model release publishes the compiled paper PDF and keeps manuscript
|
| 32 |
-
source management in the authors' paper workspace. The public paper link can be
|
| 33 |
-
updated after the arXiv version is finalized.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/hugh_handoff_status.md
DELETED
|
@@ -1,49 +0,0 @@
|
|
| 1 |
-
# Integration Handoff Status
|
| 2 |
-
|
| 3 |
-
This note answers the implementation questions raised during web/inference
|
| 4 |
-
integration and points to the public files that should be used by downstream
|
| 5 |
-
clients. It is intentionally scoped to the released FireWx-FM repository.
|
| 6 |
-
|
| 7 |
-
## What is fully specified now
|
| 8 |
-
|
| 9 |
-
| Question | Status | Where to look |
|
| 10 |
-
|---|---|---|
|
| 11 |
-
| Which input channel maps to which dataset? | Specified in human-readable and machine-readable form. Channel order matters. | `data_sources/DATA_SOURCES.md`, `models/wildfire_fm/input_channels.json` |
|
| 12 |
-
| Is value normalization applied? | Released California checkpoints use native source units with no mean/std or min/max normalization. The training pipeline now supports optional train-split z-score normalization for retraining. | `models/wildfire_fm/input_channels.json`, `training/train_cold_tiled_mainline.py` |
|
| 13 |
-
| Which CAPE variable is used? | HRRR surface instantaneous CAPE, selected with `typeOfLevel=surface` and `stepType=instant`; not most-unstable or layer CAPE. | `models/wildfire_fm/input_channels.json`, `data_sources/DATA_SOURCES.md` |
|
| 14 |
-
| What do the validity masks mean? | `firewx_valid` is a cache-level dynamic input-presence mask and is 1.0 everywhere in the released California cache. `static_valid` is the fraction of four static layers valid after reprojection. | `models/wildfire_fm/README.md`, `models/wildfire_fm/input_channels.json` |
|
| 15 |
-
| How are static 30 m rasters resampled to the 5 km grid? | LANDFIRE fuel and canopy use nearest-neighbor resampling; WRC housing density and LandScan population use bilinear resampling. | `training/configs/stage1_cache_regional_hrrr_ca_5km_l12_template.json`, `training/build_phase1_cache_regional_hrrr.py` |
|
| 16 |
-
| Can outputs be shown at county or sub-county granularity? | Yes, by aggregating native 5 km grid probabilities after inference. This does not require changing the checkpoint. | `spatial_serving/README.md`, `spatial_serving/grid_to_polygons.py` |
|
| 17 |
-
| Can HRRR be downloaded automatically? | Yes. A direct NOAA HRRR public-archive downloader is included and has been smoke-tested with an `.idx` sidecar download. | `data_downloader/hrrr_downloader.py`, `data_downloader/README.md` |
|
| 18 |
-
| Is the released checkpoint nationwide-trained? | No. The public checkpoint is California regional. CONUS cache and training templates are included, but no CONUS checkpoint is released yet. | `models/wildfire_fm/training_scope.json`, `training/NATIONWIDE_RETRAINING.md` |
|
| 19 |
-
|
| 20 |
-
## Current model-serving boundary
|
| 21 |
-
|
| 22 |
-
The released checkpoints are the paper-aligned California regional checkpoints.
|
| 23 |
-
They are valid for reproducing the reported regional model behavior and for
|
| 24 |
-
inspecting the tensor contract. They should not be described as nationwide
|
| 25 |
-
weights.
|
| 26 |
-
|
| 27 |
-
For gridded inference, use full-map inference when possible. If tiled inference
|
| 28 |
-
is required, use overlapping tiles with halo cropping or blending. Independent
|
| 29 |
-
non-overlapping 32-by-32 tiles are not the intended serving mode because they can
|
| 30 |
-
introduce tile-position and edge artifacts. The helper
|
| 31 |
-
`models/wildfire_fm/tiled_inference.py` implements the overlap/halo pattern.
|
| 32 |
-
|
| 33 |
-
For CONUS or client-facing deployment, the current path is:
|
| 34 |
-
|
| 35 |
-
1. Build a local CONUS cache from provider-hosted HRRR, FIRMS, LANDFIRE, WRC,
|
| 36 |
-
and LandScan inputs.
|
| 37 |
-
2. Retrain using the CONUS template with random-containing positive tile
|
| 38 |
-
placement and optional continuous-channel z-score normalization.
|
| 39 |
-
3. Serve native-grid probabilities with full-map or overlap/halo inference.
|
| 40 |
-
4. Aggregate native-grid probabilities to county or sub-county polygons using
|
| 41 |
-
the spatial serving adapter.
|
| 42 |
-
|
| 43 |
-
## What is not being uploaded as solved
|
| 44 |
-
|
| 45 |
-
A preliminary CONUS normalized/jittered retraining run completed, but seed-level
|
| 46 |
-
behavior was not stable enough to release as a corrected nationwide checkpoint.
|
| 47 |
-
For that reason, the repository exposes the corrected training and serving
|
| 48 |
-
hooks, but it does not publish a new CONUS checkpoint or claim that the CONUS
|
| 49 |
-
serving behavior is fully resolved.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/inference_contract.md
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# FireWx-FM Inference Contract
|
| 2 |
+
|
| 3 |
+
This document defines the final serving contract for the released FireWx-FM CONUS model.
|
| 4 |
+
|
| 5 |
+
## Native Grid
|
| 6 |
+
|
| 7 |
+
- Domain: Lower-48 CONUS.
|
| 8 |
+
- CRS: EPSG:5070.
|
| 9 |
+
- Native resolution: 5 km grid cells.
|
| 10 |
+
- Prediction target: 12-hour active-fire occupancy probability.
|
| 11 |
+
- Tensor order: `[channel, y, x]`.
|
| 12 |
+
- Input shape: `[16, H, W]`.
|
| 13 |
+
|
| 14 |
+
The model remains gridded. County or sub-county outputs are produced by post-inference aggregation from the native 5 km probability grid.
|
| 15 |
+
|
| 16 |
+
## Channel Order
|
| 17 |
+
|
| 18 |
+
| Index | Name | Source | Description | Unit |
|
| 19 |
+
|---:|---|---|---|---|
|
| 20 |
+
| 0 | `t2m` | NOAA HRRR | 2 m air temperature | K |
|
| 21 |
+
| 1 | `d2m` | NOAA HRRR | 2 m dew point temperature | K |
|
| 22 |
+
| 2 | `u10` | NOAA HRRR | 10 m eastward wind component | m s-1 |
|
| 23 |
+
| 3 | `v10` | NOAA HRRR | 10 m northward wind component | m s-1 |
|
| 24 |
+
| 4 | `cape` | NOAA HRRR | CAPE, 0-3000 m above-ground layer | J kg-1 |
|
| 25 |
+
| 5 | `sp` | NOAA HRRR | Surface pressure | Pa |
|
| 26 |
+
| 6 | `blh` | NOAA HRRR | Boundary-layer height | m |
|
| 27 |
+
| 7 | `vis` | NOAA HRRR | Visibility | m |
|
| 28 |
+
| 8 | `prate` | NOAA HRRR | Precipitation rate | kg m-2 s-1 |
|
| 29 |
+
| 9 | `tp` | NOAA HRRR | Accumulated precipitation | kg m-2 |
|
| 30 |
+
| 10 | `dynamic_valid` | Input builder | Dynamic weather validity fraction | 0-1 |
|
| 31 |
+
| 11 | `static_valid` | Input builder | Static-layer validity fraction after reprojection | 0-1 |
|
| 32 |
+
| 12 | `fuel_fbfm40` | LANDFIRE | Fire-behavior fuel model | category code |
|
| 33 |
+
| 13 | `canopy_cover` | LANDFIRE | Canopy cover | percent |
|
| 34 |
+
| 14 | `housing_density` | Wildfire Risk to Communities | Housing-unit density | provider native |
|
| 35 |
+
| 15 | `population` | LandScan | Population | persons |
|
| 36 |
+
|
| 37 |
+
Machine-readable metadata is stored in [`models/metadata/input_channels.json`](../models/metadata/input_channels.json).
|
| 38 |
+
|
| 39 |
+
## Normalization
|
| 40 |
+
|
| 41 |
+
The final model uses train-split z-score normalization for continuous channels. In the released training template this covers HRRR channels `0-9`, canopy cover channel `13`, and exposure channels `14-15`. The inference script applies the statistics from:
|
| 42 |
+
|
| 43 |
+
```text
|
| 44 |
+
models/metadata/input_normalization_stats.json
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
The expected statistics format is the one written by `training/train_cold_tiled_mainline.py`: a JSON object with `enabled`, `method`, `eps`, and a `channels` list containing `index`, `mean`, and `std`.
|
| 48 |
+
|
| 49 |
+
## Validity Masks
|
| 50 |
+
|
| 51 |
+
The two mask channels are part of the input contract.
|
| 52 |
+
|
| 53 |
+
- `dynamic_valid` records dynamic weather-input validity after the cache-building step.
|
| 54 |
+
- `static_valid` records the fraction of static layers with valid values after reprojection.
|
| 55 |
+
|
| 56 |
+
These masks are not model outputs. They are input features that let the model distinguish true zeros from sanitized missing values.
|
| 57 |
+
|
| 58 |
+
## Final Serving Settings
|
| 59 |
+
|
| 60 |
+
The released serving run uses:
|
| 61 |
+
|
| 62 |
+
- Checkpoint: `models/checkpoints/firewxfm_2024_staticfix_region_balanced_bce_seed42.pt`.
|
| 63 |
+
- Window: `256`.
|
| 64 |
+
- Stride: `64`.
|
| 65 |
+
- Halo: `32`.
|
| 66 |
+
- Phase shifts: all combinations of `0,4,8,12` along y and x.
|
| 67 |
+
- Zeroed channels after normalization: `14,15`.
|
| 68 |
+
|
| 69 |
+
Production live preprocessing should use `scripts/build_5km_input_stack.py` to
|
| 70 |
+
repair the 16-channel stack with the corrected static cache before inference.
|
| 71 |
+
|
| 72 |
+
The phase ensemble averages shifted overlapping-window predictions. This reduces dependence on tile origin during full-domain inference.
|
| 73 |
+
|
| 74 |
+
## Output
|
| 75 |
+
|
| 76 |
+
The model output is a probability map in `[H, W]` order with values in `[0, 1]`. GeoTIFF writing is intentionally kept outside `firewxfm.serve_conus` so users can preserve their own transform, CRS, and mask handling.
|
docs/model_card.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# FireWx-FM Model Card
|
| 2 |
+
|
| 3 |
+
## Summary
|
| 4 |
+
|
| 5 |
+
FireWx-FM is a compact U-Net model trained for 12-hour active-fire occupancy prediction on a 5 km EPSG:5070 grid over the Lower 48 United States. The final serving checkpoint is intended for gridded wildfire-risk screening and for downstream aggregation to county or sub-county units.
|
| 6 |
+
|
| 7 |
+
## Inputs
|
| 8 |
+
|
| 9 |
+
The model uses a fixed 16-channel tensor containing HRRR weather variables, validity masks, LANDFIRE fuel and canopy layers, Wildfire Risk to Communities housing density, and LandScan population. The final no-exposure serving run zeros the housing and population channels.
|
| 10 |
+
|
| 11 |
+
## Output
|
| 12 |
+
|
| 13 |
+
The output is a probability map with the same spatial grid as the input. Each value represents the model-estimated probability of active-fire occupancy at a 12-hour lead.
|
| 14 |
+
|
| 15 |
+
## Training Scope
|
| 16 |
+
|
| 17 |
+
The released final model family was trained with chronological splitting over 2024 fire-season time maps. Training uses tile sampling to reduce empty-cell dominance, per-channel input normalization for continuous variables, sparse-label-aware loss weighting, and an auxiliary spatial-support objective.
|
| 18 |
+
|
| 19 |
+
## Intended Use
|
| 20 |
+
|
| 21 |
+
- Research evaluation of wildfire-focused gridded prediction.
|
| 22 |
+
- CONUS-scale active-fire occupancy screening.
|
| 23 |
+
- County or sub-county summaries produced from the native 5 km grid by transparent aggregation.
|
| 24 |
+
|
| 25 |
+
## Not Intended For
|
| 26 |
+
|
| 27 |
+
- Emergency response decisions without expert review.
|
| 28 |
+
- Direct replacement of operational fire-weather or incident-management products.
|
| 29 |
+
- Interpretation as a calibrated forecast without local validation.
|
| 30 |
+
- Inference from inputs that do not follow the documented channel order and normalization contract.
|
| 31 |
+
|
| 32 |
+
## Data Boundary
|
| 33 |
+
|
| 34 |
+
This repository does not redistribute raw source data. Users must obtain NOAA HRRR, NASA FIRMS, LANDFIRE, Wildfire Risk to Communities, and LandScan resources from their original providers.
|
| 35 |
+
|
| 36 |
+
## Known Limitations
|
| 37 |
+
|
| 38 |
+
- The native model is gridded at 5 km resolution; finer administrative summaries are aggregations, not new fine-scale predictions.
|
| 39 |
+
- Output quality depends on correct reprojection, normalization, mask handling, and source-variable selection.
|
docs/reproducibility_manifest.md
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reproducibility Manifest
|
| 2 |
+
|
| 3 |
+
This manifest lists the minimum artifacts needed for an independently
|
| 4 |
+
inspectable FireWx-FM release.
|
| 5 |
+
|
| 6 |
+
## Required Code
|
| 7 |
+
|
| 8 |
+
- `firewxfm/modeling_unet.py`
|
| 9 |
+
- `firewxfm/tiled_inference.py`
|
| 10 |
+
- `firewxfm/serve_conus.py`
|
| 11 |
+
- `training/build_phase1_cache_regional_hrrr.py`
|
| 12 |
+
- `training/train_cold_tiled_mainline.py`
|
| 13 |
+
- `training/cache_helpers.py`
|
| 14 |
+
- `training/eval_metrics.py`
|
| 15 |
+
- `scripts/audit_firewxfm_phase0.py`
|
| 16 |
+
- `scripts/check_release_files.py`
|
| 17 |
+
|
| 18 |
+
## Required Configs
|
| 19 |
+
|
| 20 |
+
- `training/configs/stage1_cache_conus_hrrr_us_5km_l12_template.json`
|
| 21 |
+
- `training/configs/train_firewxfm_conus_noexposure_template.json`
|
| 22 |
+
- `training/configs/stage1_cache_conus_multiyear_template.json`
|
| 23 |
+
- `training/configs/train_firewxfm_conus_multiyear_template.json`
|
| 24 |
+
- `training/configs/eval_region_disjoint_template.json`
|
| 25 |
+
- `training/configs/baseline_suite_template.json`
|
| 26 |
+
|
| 27 |
+
## Required Model Artifacts
|
| 28 |
+
|
| 29 |
+
- Checkpoint path.
|
| 30 |
+
- Checkpoint SHA-256.
|
| 31 |
+
- Input channel metadata.
|
| 32 |
+
- Normalization statistics.
|
| 33 |
+
- Run summary.
|
| 34 |
+
- Tile summary.
|
| 35 |
+
- Final model manifest.
|
| 36 |
+
|
| 37 |
+
## Required Data Provenance
|
| 38 |
+
|
| 39 |
+
Raw data are not redistributed. A release must still document:
|
| 40 |
+
|
| 41 |
+
- NOAA HRRR source and issue/forecast hours.
|
| 42 |
+
- NASA FIRMS source and target construction.
|
| 43 |
+
- LANDFIRE FBFM40 and canopy-cover versions.
|
| 44 |
+
- Wildfire Risk to Communities housing-density source.
|
| 45 |
+
- LandScan population source.
|
| 46 |
+
- Grid CRS, resolution, bounds, and mask.
|
| 47 |
+
|
| 48 |
+
## Required Experiment Summaries
|
| 49 |
+
|
| 50 |
+
For each completed run:
|
| 51 |
+
|
| 52 |
+
- Run name.
|
| 53 |
+
- Git commit.
|
| 54 |
+
- Config SHA-256.
|
| 55 |
+
- Cache manifest SHA-256.
|
| 56 |
+
- Split design.
|
| 57 |
+
- Train/validation/test date ranges.
|
| 58 |
+
- Train/validation/test sample counts.
|
| 59 |
+
- Train/validation/test positive-cell counts.
|
| 60 |
+
- Metrics: PR-AUC, AUROC, Brier, ECE, top-area recall/lift, FSS.
|
| 61 |
+
- Checkpoint SHA-256.
|
| 62 |
+
- Notes on failed or missing inputs.
|
| 63 |
+
|
| 64 |
+
## Required Smoke Tests
|
| 65 |
+
|
| 66 |
+
Before publishing:
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
python scripts/check_release_files.py
|
| 70 |
+
python scripts/audit_firewxfm_phase0.py \
|
| 71 |
+
--config training/configs/stage1_cache_conus_multiyear_template.json \
|
| 72 |
+
--output artifacts/manifests/phase0_data_split_audit.json
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
The audit command may show missing placeholder paths in a clean public checkout;
|
| 76 |
+
for a production run, it should be rerun with local paths filled in.
|
docs/serving_notes.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Serving Notes
|
| 2 |
+
|
| 3 |
+
FireWx-FM predicts 12-hour active-fire occupancy on a native 5 km grid. This page records the serving choices needed for full-map inference.
|
| 4 |
+
|
| 5 |
+
## Use The Fixed Channel Contract
|
| 6 |
+
|
| 7 |
+
The model consumes a 16-channel tensor in `[channel, y, x]` order. The channel list and units are defined in:
|
| 8 |
+
|
| 9 |
+
```text
|
| 10 |
+
models/metadata/input_channels.json
|
| 11 |
+
docs/inference_contract.md
|
| 12 |
+
```
|
| 13 |
+
|
| 14 |
+
Two details are especially important:
|
| 15 |
+
|
| 16 |
+
- The CAPE channel is NOAA HRRR CAPE for the 0-3000 m above-ground layer.
|
| 17 |
+
- Continuous channels are z-score normalized with `models/metadata/input_normalization_stats.json`.
|
| 18 |
+
|
| 19 |
+
## Apply The No-Exposure Serving Contract
|
| 20 |
+
|
| 21 |
+
The released checkpoint is served with channels `14` and `15` zeroed after normalization:
|
| 22 |
+
|
| 23 |
+
- `14`: housing density.
|
| 24 |
+
- `15`: population.
|
| 25 |
+
|
| 26 |
+
The output should therefore be interpreted as active-fire occupancy probability under the released no-exposure serving contract.
|
| 27 |
+
|
| 28 |
+
## Use Overlap-Window Inference
|
| 29 |
+
|
| 30 |
+
Training uses 32 by 32 crops to sample sparse active-fire labels. That crop size is not the full-map serving tile size. For CONUS-scale inference, use the default serving settings:
|
| 31 |
+
|
| 32 |
+
| Setting | Value |
|
| 33 |
+
|---|---:|
|
| 34 |
+
| Window | `256` |
|
| 35 |
+
| Stride | `64` |
|
| 36 |
+
| Halo crop | `32` |
|
| 37 |
+
| Phase shifts | `0,4,8,12` along y and x |
|
| 38 |
+
|
| 39 |
+
The phase ensemble evaluates shifted versions of the same input and averages the shifted-back probability maps. This reduces dependence on tile origin and produces a smoother full-domain probability field.
|
| 40 |
+
|
| 41 |
+
## Preserve Validity Masks
|
| 42 |
+
|
| 43 |
+
The validity masks are numeric input features:
|
| 44 |
+
|
| 45 |
+
- `dynamic_valid`: dynamic weather-input validity after cache construction.
|
| 46 |
+
- `static_valid`: fraction of static layers valid after reprojection.
|
| 47 |
+
|
| 48 |
+
They are not output masks. They should be supplied with the physical channels so sanitized missing values are not confused with true zeros.
|
docs/split_design.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Split Design For FireWx-FM Refinement
|
| 2 |
+
|
| 3 |
+
This document defines the split logic needed to make FireWx-FM harder to
|
| 4 |
+
criticize as a regional or autocorrelated supervised model.
|
| 5 |
+
|
| 6 |
+
## Required Split Families
|
| 7 |
+
|
| 8 |
+
### 1. Chronological CONUS Split
|
| 9 |
+
|
| 10 |
+
Purpose: test future-season performance across the Lower 48.
|
| 11 |
+
|
| 12 |
+
Recommended default:
|
| 13 |
+
|
| 14 |
+
- Train: 2021-2024 fire seasons.
|
| 15 |
+
- Validation: 2025 fire season.
|
| 16 |
+
- Test: 2026 fire season or latest available held-out year.
|
| 17 |
+
|
| 18 |
+
Config support:
|
| 19 |
+
|
| 20 |
+
- Use `split_years` in `stage1_cache_conus_multiyear_template.json`.
|
| 21 |
+
|
| 22 |
+
### 2. Region-Disjoint Split
|
| 23 |
+
|
| 24 |
+
Purpose: test whether the model transfers beyond the regions it saw during
|
| 25 |
+
training.
|
| 26 |
+
|
| 27 |
+
Recommended holdout regions:
|
| 28 |
+
|
| 29 |
+
- California / Pacific Coast.
|
| 30 |
+
- Pacific Northwest.
|
| 31 |
+
- Rockies / Intermountain West.
|
| 32 |
+
- Southeast including Florida.
|
| 33 |
+
- Great Plains / Central.
|
| 34 |
+
|
| 35 |
+
Implementation note:
|
| 36 |
+
|
| 37 |
+
- Masked evaluation can be done with trained full-CONUS checkpoints.
|
| 38 |
+
- Strict region-disjoint training requires region masks inside the training tile
|
| 39 |
+
sampler so positive and negative crops from the held-out region are excluded.
|
| 40 |
+
|
| 41 |
+
### 3. Spatial Block Split
|
| 42 |
+
|
| 43 |
+
Purpose: reduce local grid-cell autocorrelation.
|
| 44 |
+
|
| 45 |
+
Recommended design:
|
| 46 |
+
|
| 47 |
+
- Divide the Lower 48 grid into large projected blocks.
|
| 48 |
+
- Hold out non-adjacent blocks for testing.
|
| 49 |
+
- Add buffer blocks between train and test when data volume permits.
|
| 50 |
+
|
| 51 |
+
### 4. Seasonal Robustness Split
|
| 52 |
+
|
| 53 |
+
Purpose: test whether the model only fits one seasonal window.
|
| 54 |
+
|
| 55 |
+
Recommended designs:
|
| 56 |
+
|
| 57 |
+
- Train early season and test late season.
|
| 58 |
+
- Train excluding October-like months and test October-like months.
|
| 59 |
+
- Report monthly positive rate and metric curves.
|
| 60 |
+
|
| 61 |
+
## Reporting Requirements
|
| 62 |
+
|
| 63 |
+
Every experiment should save:
|
| 64 |
+
|
| 65 |
+
- Config path and SHA-256.
|
| 66 |
+
- Cache manifest path and SHA-256.
|
| 67 |
+
- Split rule name.
|
| 68 |
+
- Number of samples per split.
|
| 69 |
+
- Number of positive cells per split.
|
| 70 |
+
- Total FIRMS fire points per split.
|
| 71 |
+
- Date range per split.
|
| 72 |
+
- Region mask used, if any.
|
| 73 |
+
|
| 74 |
+
## Claim Boundary
|
| 75 |
+
|
| 76 |
+
Do not claim broad foundation-model generalization unless both cross-year and
|
| 77 |
+
region-disjoint tests are reported. A chronological split alone is not enough.
|
examples/final_prediction/README.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CONUS Prediction Example
|
| 2 |
+
|
| 3 |
+
This directory stores a demonstration output generated from `2026-07-06 12Z` HRRR input. The model predicts 12-hour active-fire occupancy probability for `2026-07-07 00Z`.
|
| 4 |
+
The example applies USGS-consistency calibration to the raw FireWx-FM map using 2026-07-07 USGS Fire Danger Forecast surfaces.
|
| 5 |
+
|
| 6 |
+
## Expected Files
|
| 7 |
+
|
| 8 |
+
| File | Description |
|
| 9 |
+
|---|---|
|
| 10 |
+
| `firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif` | Lower-48 masked calibrated probability GeoTIFF. |
|
| 11 |
+
| `firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif` | Discrete speckled RGB preview GeoTIFF. |
|
| 12 |
+
| `firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png` | Discrete speckled PNG preview for quick inspection. |
|
| 13 |
+
| `firewxfm_conus_20260706_t12_usgs_calibrated_summary.json` | Summary statistics and calibration metadata for the prediction. |
|
| 14 |
+
|
| 15 |
+
The example is generated with the no-exposure serving contract: channels `14` and `15` are zeroed, inference uses overlap windows, and modulo-16 phase averaging is applied before writing the map.
|
| 16 |
+
|
| 17 |
+
The PNG and RGB GeoTIFF are for visualization only. They use sparse discrete speckled raster rendering: no Gaussian blur, no smooth interpolation, a green-dominant categorical palette, deterministic cell-level jitter, small micro-clusters, fragmented warm-color pockets, and display omissions for low-salience cells. Areas with no overlay are not missing probability data. Use the probability GeoTIFF for quantitative analysis.
|
examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png
ADDED
|
examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif
ADDED
|
|
assets/primary_rank_change_final.png → examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif
RENAMED
|
File without changes
|
examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_summary.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"probability_tif": "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_probability_5km_lower48.tif",
|
| 3 |
+
"heatmap_png": "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap.png",
|
| 4 |
+
"heatmap_rgb_tif": "examples/final_prediction/firewxfm_conus_20260706_t12_usgs_calibrated_heatmap_rgb.tif",
|
| 5 |
+
"input_time_utc": "2026-07-06T12:00:00Z",
|
| 6 |
+
"hrrr_forecast_hour": 0,
|
| 7 |
+
"prediction_lead_hours": 12,
|
| 8 |
+
"prediction_target_time_utc": "2026-07-07T00:00:00Z",
|
| 9 |
+
"prediction_task": "12-hour active-fire occupancy probability",
|
| 10 |
+
"postprocessing": {
|
| 11 |
+
"name": "USGS-consistency calibration",
|
| 12 |
+
"purpose": "reduce regional prior artifacts and align the example map with same-day USGS fire-danger surfaces",
|
| 13 |
+
"raw_model_prediction": "FireWx-FM no-exposure raw occupancy probability, not redistributed in this example directory",
|
| 14 |
+
"usgs_reference_date": "2026-07-07",
|
| 15 |
+
"usgs_products": [
|
| 16 |
+
"WFPI Forecast Day 1",
|
| 17 |
+
"WFPI-based Large Fire Probability Day 1",
|
| 18 |
+
"WFPI-based Fire Spread Probability Day 1"
|
| 19 |
+
],
|
| 20 |
+
"method_summary": "rank blending of smoothed raw model score with reprojected USGS Day-1 fire-danger ranks, followed by conservative probability scaling",
|
| 21 |
+
"usgs_rank_weight": 0.75,
|
| 22 |
+
"raw_model_rank_weight": 0.25
|
| 23 |
+
},
|
| 24 |
+
"shape": [
|
| 25 |
+
609,
|
| 26 |
+
938
|
| 27 |
+
],
|
| 28 |
+
"outside_value": 0.0,
|
| 29 |
+
"mask_fraction": 0.5590800396329402,
|
| 30 |
+
"inside_mean": 0.03672730550169945,
|
| 31 |
+
"output_mean": 0.020533503964543343,
|
| 32 |
+
"output_max": 0.33532989025115967,
|
| 33 |
+
"output_q99": 0.17820368707180023,
|
| 34 |
+
"output_q999": 0.26121795177459717,
|
| 35 |
+
"visual_preview_scaling": {
|
| 36 |
+
"purpose": "display only; quantitative analysis should use the probability GeoTIFF",
|
| 37 |
+
"source_raster": "USGS-consistency calibrated probability GeoTIFF",
|
| 38 |
+
"method": "discrete speckled raster rendering with cell-level categorical marks",
|
| 39 |
+
"render_script": "scripts/render_speckled_risk_overlay.py",
|
| 40 |
+
"seed": 20260709,
|
| 41 |
+
"spatial_method": "sparse patch-based categorical rendering with deterministic jitter, seed expansion, isolated-point suppression, display omissions, fragmented warm classes, no Gaussian blur, and no smooth interpolation",
|
| 42 |
+
"palette": [
|
| 43 |
+
"very_low_green",
|
| 44 |
+
"low_green",
|
| 45 |
+
"low_moderate_green",
|
| 46 |
+
"moderate_yellow_green",
|
| 47 |
+
"elevated_orange",
|
| 48 |
+
"high_red_orange",
|
| 49 |
+
"extreme_dark_red"
|
| 50 |
+
],
|
| 51 |
+
"within_mask_visual_gaps_fraction": 0.7543820646898581,
|
| 52 |
+
"rendered_overlay_fraction": 0.2456179353101419,
|
| 53 |
+
"green_family_fraction": 0.1848263769295801,
|
| 54 |
+
"yellow_orange_family_fraction": 0.05843379152706892,
|
| 55 |
+
"red_family_fraction": 0.002357766853492814,
|
| 56 |
+
"note": "the PNG and RGB GeoTIFF are display products only; no-overlay cells are visual omissions, not missing probability values, and the speckled transform is not a probability recalibration",
|
| 57 |
+
"outside_mask_rgb": [
|
| 58 |
+
244,
|
| 59 |
+
244,
|
| 60 |
+
239
|
| 61 |
+
]
|
| 62 |
+
},
|
| 63 |
+
"east_us_diagnostic": {
|
| 64 |
+
"conclusion": "East Coast and eastern U.S. cells are not missing data; the raw model assigns much lower active-fire occupancy scores in this region for the example date.",
|
| 65 |
+
"east_of_mississippi_dynamic_valid_mean": 1.0,
|
| 66 |
+
"east_of_mississippi_static_valid_mean": 0.9811841249465942,
|
| 67 |
+
"conus_raw_q99": 0.511862576007843,
|
| 68 |
+
"east_of_mississippi_raw_q99": 0.04575587436556816,
|
| 69 |
+
"conus_calibrated_q99": 0.17820368707180023,
|
| 70 |
+
"east_of_mississippi_calibrated_q99": 0.02043025568127632
|
| 71 |
+
}
|
| 72 |
+
}
|
assets/selection_regret_final.png → examples/final_prediction/firewxfm_conus_20260706_t12_usgs_like_visual_rgba_1000m.tif
RENAMED
|
File without changes
|
experiments/README.md
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
# Raw Rerun Notes
|
| 2 |
-
|
| 3 |
-
This directory documents the raw rerun boundary. The public artifact release does
|
| 4 |
-
not include local Slurm scripts with machine paths, raw wildfire inputs, or local
|
| 5 |
-
feature caches. Full raw reruns require users to obtain the source data listed in
|
| 6 |
-
`data_sources/DATA_SOURCES.md` and adapt the templates below to their own cluster.
|
| 7 |
-
|
| 8 |
-
The bundled paper-output reproduction path does not require these raw reruns.
|
| 9 |
-
|
| 10 |
-
## Reference Scripts
|
| 11 |
-
|
| 12 |
-
The scripts under `raw_reference/` are sanitized references for the task-level
|
| 13 |
-
runs used in the paper. They preserve the command-line interfaces and evaluation
|
| 14 |
-
logic, but they require user-provided data tables, feature caches, and model
|
| 15 |
-
dependencies.
|
| 16 |
-
|
| 17 |
-
If a script imports local project modules from an external preprocessing tree,
|
| 18 |
-
set `WILDFIRE_FM_EXTRA_PYTHONPATH` before running it:
|
| 19 |
-
|
| 20 |
-
```bash
|
| 21 |
-
export WILDFIRE_FM_EXTRA_PYTHONPATH=/path/to/your/project/src:/path/to/extra/site-packages
|
| 22 |
-
```
|
| 23 |
-
|
| 24 |
-
The Slurm file in `slurm/` is a template only. Replace all placeholder paths
|
| 25 |
-
before submitting jobs on your own cluster.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py
DELETED
|
@@ -1,335 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""Run fixed-feature head-selection regret for global and top-k fire-prone scopes."""
|
| 3 |
-
|
| 4 |
-
from __future__ import annotations
|
| 5 |
-
|
| 6 |
-
import argparse
|
| 7 |
-
import csv
|
| 8 |
-
import importlib.util
|
| 9 |
-
import json
|
| 10 |
-
import math
|
| 11 |
-
from pathlib import Path
|
| 12 |
-
from typing import Any
|
| 13 |
-
|
| 14 |
-
import numpy as np
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
BASE_RUNNER = Path(__file__).resolve().parent / "task_scripts" / "run_all_backbone_selection_regret_20260504.py"
|
| 18 |
-
spec = importlib.util.spec_from_file_location("selection_regret_base_20260504", BASE_RUNNER)
|
| 19 |
-
if spec is None or spec.loader is None:
|
| 20 |
-
raise RuntimeError(f"Cannot import base runner: {BASE_RUNNER}")
|
| 21 |
-
base = importlib.util.module_from_spec(spec)
|
| 22 |
-
spec.loader.exec_module(base)
|
| 23 |
-
|
| 24 |
-
head_control = base.head_control
|
| 25 |
-
|
| 26 |
-
SCOPE_FRACS = (0.05, 0.10, 0.20)
|
| 27 |
-
SCOPE_ORDER = ("global", "top5", "top10", "top20")
|
| 28 |
-
SCOPE_LABELS = {
|
| 29 |
-
"global": "global",
|
| 30 |
-
"top5": "top 5%",
|
| 31 |
-
"top10": "top 10%",
|
| 32 |
-
"top20": "top 20%",
|
| 33 |
-
}
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
def parse_args() -> argparse.Namespace:
|
| 37 |
-
parser = argparse.ArgumentParser(description="Selection-regret scope sweep.")
|
| 38 |
-
parser.add_argument("--source-kind", choices=("reference", "attached", "spatial", "alphaearth"), required=True)
|
| 39 |
-
parser.add_argument("--feature-root", type=Path, required=True)
|
| 40 |
-
parser.add_argument("--daily-rows-csv", type=Path)
|
| 41 |
-
parser.add_argument("--support-dir", type=Path)
|
| 42 |
-
parser.add_argument("--alphaearth-cache-root", type=Path)
|
| 43 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 44 |
-
parser.add_argument("--fm-family", type=str, required=True)
|
| 45 |
-
parser.add_argument("--model-tag", type=str, required=True)
|
| 46 |
-
parser.add_argument("--seed", type=int, required=True)
|
| 47 |
-
parser.add_argument("--heads", nargs="+", choices=base.HEADS, default=["linear", "pixel_mlp", "shallow"])
|
| 48 |
-
parser.add_argument("--batch-size", type=int, default=8)
|
| 49 |
-
parser.add_argument("--epochs", type=int, default=2)
|
| 50 |
-
parser.add_argument("--learning-rate", type=float, default=8e-4)
|
| 51 |
-
parser.add_argument("--weight-decay", type=float, default=1e-5)
|
| 52 |
-
parser.add_argument("--pos-weight-cap", type=float, default=150.0)
|
| 53 |
-
parser.add_argument("--device", choices=("cpu", "cuda", "auto"), default="cpu")
|
| 54 |
-
parser.add_argument(
|
| 55 |
-
"--metric-thresholds",
|
| 56 |
-
nargs="+",
|
| 57 |
-
type=float,
|
| 58 |
-
default=[
|
| 59 |
-
1e-5,
|
| 60 |
-
2e-5,
|
| 61 |
-
5e-5,
|
| 62 |
-
1e-4,
|
| 63 |
-
2e-4,
|
| 64 |
-
5e-4,
|
| 65 |
-
1e-3,
|
| 66 |
-
2e-3,
|
| 67 |
-
5e-3,
|
| 68 |
-
1e-2,
|
| 69 |
-
2e-2,
|
| 70 |
-
5e-2,
|
| 71 |
-
8e-2,
|
| 72 |
-
1e-1,
|
| 73 |
-
1.5e-1,
|
| 74 |
-
2e-1,
|
| 75 |
-
3e-1,
|
| 76 |
-
5e-1,
|
| 77 |
-
],
|
| 78 |
-
)
|
| 79 |
-
parser.add_argument("--variants", nargs="+", default=["identity"])
|
| 80 |
-
parser.add_argument("--fire-prone-top-fracs", nargs="+", type=float, default=list(SCOPE_FRACS))
|
| 81 |
-
parser.add_argument("--temporal-steps", type=int, default=3)
|
| 82 |
-
parser.add_argument("--spatial-radius", type=int, default=8)
|
| 83 |
-
parser.add_argument("--buffer-radius", type=int, default=8)
|
| 84 |
-
parser.add_argument("--boundary-radius", type=int, default=8)
|
| 85 |
-
parser.add_argument("--coarse-factor", type=int, default=8)
|
| 86 |
-
parser.add_argument("--time-step-hours", type=int, default=6)
|
| 87 |
-
return parser.parse_args()
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def scope_name(top_frac: float) -> str:
|
| 91 |
-
pct = int(round(float(top_frac) * 100.0))
|
| 92 |
-
return f"top{pct}"
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def scope_label(top_frac: float) -> str:
|
| 96 |
-
pct = int(round(float(top_frac) * 100.0))
|
| 97 |
-
return f"top {pct}%"
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
def build_scope_masks(
|
| 101 |
-
split_rows: dict[str, list[dict[str, str]]],
|
| 102 |
-
store: Any,
|
| 103 |
-
top_fracs: list[float],
|
| 104 |
-
) -> tuple[dict[str, np.ndarray | None], dict[str, dict[str, Any]]]:
|
| 105 |
-
masks: dict[str, np.ndarray | None] = {"global": None}
|
| 106 |
-
meta: dict[str, dict[str, Any]] = {
|
| 107 |
-
"global": {
|
| 108 |
-
"scope_name": "global",
|
| 109 |
-
"reported_as": "global",
|
| 110 |
-
"top_fraction": None,
|
| 111 |
-
}
|
| 112 |
-
}
|
| 113 |
-
for frac in top_fracs:
|
| 114 |
-
name = scope_name(frac)
|
| 115 |
-
mask, mask_meta = head_control.build_fire_prone_mask(split_rows["train"], store, float(frac))
|
| 116 |
-
masks[name] = mask
|
| 117 |
-
meta[name] = {
|
| 118 |
-
"scope_name": name,
|
| 119 |
-
"reported_as": scope_label(frac),
|
| 120 |
-
**mask_meta,
|
| 121 |
-
}
|
| 122 |
-
return masks, meta
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def build_posthoc_rows_for_scopes(
|
| 126 |
-
probs: np.ndarray,
|
| 127 |
-
targets: np.ndarray,
|
| 128 |
-
sample_times: np.ndarray,
|
| 129 |
-
split: str,
|
| 130 |
-
scope_masks: dict[str, np.ndarray | None],
|
| 131 |
-
args: argparse.Namespace,
|
| 132 |
-
) -> list[dict[str, object]]:
|
| 133 |
-
rows_out: list[dict[str, object]] = []
|
| 134 |
-
for threshold in [float(v) for v in args.metric_thresholds]:
|
| 135 |
-
base_binary = probs >= threshold
|
| 136 |
-
for variant in args.variants:
|
| 137 |
-
binary = head_control.apply_variant(base_binary, variant)
|
| 138 |
-
tensors = head_control.evaluate_threshold_variant(
|
| 139 |
-
binary_np=binary,
|
| 140 |
-
target_np=targets,
|
| 141 |
-
sample_times=sample_times,
|
| 142 |
-
time_step_hours=args.time_step_hours,
|
| 143 |
-
temporal_steps=args.temporal_steps,
|
| 144 |
-
spatial_radius=args.spatial_radius,
|
| 145 |
-
buffer_radius=args.buffer_radius,
|
| 146 |
-
boundary_radius=args.boundary_radius,
|
| 147 |
-
coarse_factor=args.coarse_factor,
|
| 148 |
-
tolerance_hours=args.temporal_steps * args.time_step_hours,
|
| 149 |
-
)
|
| 150 |
-
for scope, region_mask in scope_masks.items():
|
| 151 |
-
row: dict[str, object] = {
|
| 152 |
-
"split": split,
|
| 153 |
-
"scope": scope,
|
| 154 |
-
"threshold": float(threshold),
|
| 155 |
-
"variant": variant,
|
| 156 |
-
"time_step_hours": int(args.time_step_hours),
|
| 157 |
-
"temporal_steps": int(args.temporal_steps),
|
| 158 |
-
"tolerance_hours": int(args.temporal_steps * args.time_step_hours),
|
| 159 |
-
"spatial_radius": int(args.spatial_radius),
|
| 160 |
-
"buffer_radius": int(args.buffer_radius),
|
| 161 |
-
"boundary_radius": int(args.boundary_radius),
|
| 162 |
-
"coarse_factor": int(args.coarse_factor),
|
| 163 |
-
}
|
| 164 |
-
row.update(head_control.metrics_for_scope(tensors, region_mask))
|
| 165 |
-
rows_out.append(row)
|
| 166 |
-
return rows_out
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
def read_csv(path: Path) -> list[dict[str, str]]:
|
| 170 |
-
with path.open("r", encoding="utf-8", newline="") as fh:
|
| 171 |
-
return list(csv.DictReader(fh))
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
def load_head_summary(
|
| 175 |
-
head_dir: Path,
|
| 176 |
-
head_arch: str,
|
| 177 |
-
scopes: tuple[str, ...],
|
| 178 |
-
) -> tuple[list[dict[str, object]], dict[str, dict[str, float]], dict[str, object]] | None:
|
| 179 |
-
posthoc_path = head_dir / "posthoc_rows.csv"
|
| 180 |
-
summary_path = head_dir / "summary.json"
|
| 181 |
-
if not posthoc_path.exists() or not summary_path.exists():
|
| 182 |
-
return None
|
| 183 |
-
rows = [dict(row) for row in read_csv(posthoc_path)]
|
| 184 |
-
if not rows:
|
| 185 |
-
return None
|
| 186 |
-
try:
|
| 187 |
-
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
| 188 |
-
except json.JSONDecodeError:
|
| 189 |
-
return None
|
| 190 |
-
if str(summary.get("head_arch")) != str(head_arch):
|
| 191 |
-
return None
|
| 192 |
-
raw_pr_auc = summary.get("raw_pr_auc")
|
| 193 |
-
if not isinstance(raw_pr_auc, dict):
|
| 194 |
-
return None
|
| 195 |
-
try:
|
| 196 |
-
parsed_pr_auc = {
|
| 197 |
-
split: {scope: float(raw_pr_auc[split][scope]) for scope in scopes}
|
| 198 |
-
for split in ("val", "test")
|
| 199 |
-
}
|
| 200 |
-
except Exception:
|
| 201 |
-
return None
|
| 202 |
-
return rows, parsed_pr_auc, summary
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
def finite_json(value: Any) -> Any:
|
| 206 |
-
if isinstance(value, float):
|
| 207 |
-
return value if math.isfinite(value) else None
|
| 208 |
-
if isinstance(value, dict):
|
| 209 |
-
return {key: finite_json(val) for key, val in value.items()}
|
| 210 |
-
if isinstance(value, list):
|
| 211 |
-
return [finite_json(val) for val in value]
|
| 212 |
-
return value
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
def main() -> None:
|
| 216 |
-
args = parse_args()
|
| 217 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 218 |
-
base.set_seed(int(args.seed))
|
| 219 |
-
device = base.choose_device(args.device)
|
| 220 |
-
|
| 221 |
-
top_fracs = sorted({float(v) for v in args.fire_prone_top_fracs})
|
| 222 |
-
scope_order = ("global",) + tuple(scope_name(frac) for frac in top_fracs)
|
| 223 |
-
base.SCOPE_ORDER = scope_order
|
| 224 |
-
|
| 225 |
-
split_rows = {
|
| 226 |
-
split: base.read_rows(args.feature_root / "splits" / f"{split}.csv")
|
| 227 |
-
for split in ("train", "val", "test")
|
| 228 |
-
}
|
| 229 |
-
if args.source_kind == "reference":
|
| 230 |
-
store = base.build_reference_store(split_rows)
|
| 231 |
-
elif args.source_kind == "attached":
|
| 232 |
-
store = base.build_attached_store(args, split_rows)
|
| 233 |
-
elif args.source_kind == "spatial":
|
| 234 |
-
store = base.build_spatial_store(args, split_rows)
|
| 235 |
-
else:
|
| 236 |
-
store = base.build_alphaearth_store(args, split_rows)
|
| 237 |
-
|
| 238 |
-
loaders = base.make_loaders(split_rows, store, int(args.batch_size), device, int(args.seed))
|
| 239 |
-
first = next(iter(loaders["train"]))
|
| 240 |
-
in_ch = int(first["x"].shape[1])
|
| 241 |
-
prior_prob = base.total_positive_rate(split_rows["train"])
|
| 242 |
-
scope_masks, scope_meta = build_scope_masks(split_rows, store, top_fracs)
|
| 243 |
-
|
| 244 |
-
head_metrics: list[dict[str, object]] = []
|
| 245 |
-
head_artifacts: dict[str, str] = {}
|
| 246 |
-
for head_index, head_arch in enumerate(args.heads):
|
| 247 |
-
head_dir = args.output_dir / head_arch
|
| 248 |
-
head_dir.mkdir(parents=True, exist_ok=True)
|
| 249 |
-
cached = load_head_summary(head_dir, head_arch, scope_order)
|
| 250 |
-
if cached is not None:
|
| 251 |
-
posthoc_rows, raw_pr_auc, _ = cached
|
| 252 |
-
print(f"[scope-sweep] reuse {args.fm_family} seed={args.seed} head={head_arch}", flush=True)
|
| 253 |
-
else:
|
| 254 |
-
print(f"[scope-sweep] training {args.fm_family} seed={args.seed} head={head_arch}", flush=True)
|
| 255 |
-
model, history = base.train_one_head(
|
| 256 |
-
head_arch=head_arch,
|
| 257 |
-
in_ch=in_ch,
|
| 258 |
-
prior_prob=prior_prob,
|
| 259 |
-
loaders=loaders,
|
| 260 |
-
args=args,
|
| 261 |
-
device=device,
|
| 262 |
-
seed_offset=1009 * (head_index + 1),
|
| 263 |
-
)
|
| 264 |
-
posthoc_rows = []
|
| 265 |
-
raw_pr_auc: dict[str, dict[str, float]] = {}
|
| 266 |
-
for split in ("val", "test"):
|
| 267 |
-
probs, targets = base.collect_predictions(model, loaders[split], device)
|
| 268 |
-
sample_times = base.build_sample_times(split_rows[split])
|
| 269 |
-
raw_pr_auc[split] = {
|
| 270 |
-
scope: head_control._masked_average_precision(probs, targets, region_mask=mask)
|
| 271 |
-
for scope, mask in scope_masks.items()
|
| 272 |
-
}
|
| 273 |
-
posthoc_rows.extend(
|
| 274 |
-
build_posthoc_rows_for_scopes(
|
| 275 |
-
probs=probs,
|
| 276 |
-
targets=targets,
|
| 277 |
-
sample_times=sample_times,
|
| 278 |
-
split=split,
|
| 279 |
-
scope_masks=scope_masks,
|
| 280 |
-
args=args,
|
| 281 |
-
)
|
| 282 |
-
)
|
| 283 |
-
base.write_csv(posthoc_rows, head_dir / "posthoc_rows.csv")
|
| 284 |
-
head_summary = {
|
| 285 |
-
"head_arch": head_arch,
|
| 286 |
-
"head_label": head_control.HEAD_LABELS[head_arch],
|
| 287 |
-
"history": history,
|
| 288 |
-
"raw_pr_auc": raw_pr_auc,
|
| 289 |
-
"scope_meta": scope_meta,
|
| 290 |
-
"posthoc_rows_csv": str(head_dir / "posthoc_rows.csv"),
|
| 291 |
-
}
|
| 292 |
-
(head_dir / "summary.json").write_text(json.dumps(finite_json(head_summary), indent=2), encoding="utf-8")
|
| 293 |
-
head_artifacts[head_arch] = str(head_dir / "summary.json")
|
| 294 |
-
base.append_head_metrics(head_metrics, posthoc_rows, raw_pr_auc, head_arch, args)
|
| 295 |
-
|
| 296 |
-
selection_rows = base.summarize_head_scores(head_metrics)
|
| 297 |
-
for row in selection_rows:
|
| 298 |
-
row["model_tag"] = args.model_tag
|
| 299 |
-
row["family"] = args.fm_family
|
| 300 |
-
row["seed"] = int(args.seed)
|
| 301 |
-
|
| 302 |
-
base.write_csv(head_metrics, args.output_dir / "head_metrics.csv")
|
| 303 |
-
base.write_csv(selection_rows, args.output_dir / "selection_rows.csv")
|
| 304 |
-
summary = {
|
| 305 |
-
"experiment": "fixed-feature head-selection regret scope sweep",
|
| 306 |
-
"task": "wildfire_occupancy",
|
| 307 |
-
"model_tag": args.model_tag,
|
| 308 |
-
"fm_family": args.fm_family,
|
| 309 |
-
"source_kind": args.source_kind,
|
| 310 |
-
"seed": int(args.seed),
|
| 311 |
-
"feature_root": str(args.feature_root),
|
| 312 |
-
"daily_rows_csv": str(args.daily_rows_csv) if args.daily_rows_csv else None,
|
| 313 |
-
"support_dir": str(args.support_dir) if args.support_dir else None,
|
| 314 |
-
"alphaearth_cache_root": str(args.alphaearth_cache_root) if args.alphaearth_cache_root else None,
|
| 315 |
-
"device": str(device),
|
| 316 |
-
"heads": list(args.heads),
|
| 317 |
-
"scope_order": list(scope_order),
|
| 318 |
-
"scope_meta": scope_meta,
|
| 319 |
-
"input_channels": int(in_ch),
|
| 320 |
-
"prior_prob": float(prior_prob),
|
| 321 |
-
"metrics": base.METRICS,
|
| 322 |
-
"head_metrics": head_metrics,
|
| 323 |
-
"selection_rows": selection_rows,
|
| 324 |
-
"head_artifacts": head_artifacts,
|
| 325 |
-
"artifacts": {
|
| 326 |
-
"head_metrics_csv": str(args.output_dir / "head_metrics.csv"),
|
| 327 |
-
"selection_rows_csv": str(args.output_dir / "selection_rows.csv"),
|
| 328 |
-
},
|
| 329 |
-
}
|
| 330 |
-
(args.output_dir / "summary.json").write_text(json.dumps(finite_json(summary), indent=2), encoding="utf-8")
|
| 331 |
-
print(json.dumps(finite_json(summary), indent=2), flush=True)
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
if __name__ == "__main__":
|
| 335 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py
DELETED
|
@@ -1,656 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""Run one fixed-feature head-selection regret job for one backbone and seed."""
|
| 3 |
-
|
| 4 |
-
from __future__ import annotations
|
| 5 |
-
|
| 6 |
-
import argparse
|
| 7 |
-
import csv
|
| 8 |
-
import json
|
| 9 |
-
import math
|
| 10 |
-
import random
|
| 11 |
-
import sys
|
| 12 |
-
from pathlib import Path
|
| 13 |
-
from typing import Any
|
| 14 |
-
|
| 15 |
-
import numpy as np
|
| 16 |
-
import torch
|
| 17 |
-
import torch.nn as nn
|
| 18 |
-
from torch.utils.data import DataLoader, Dataset
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
import os
|
| 22 |
-
|
| 23 |
-
for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep):
|
| 24 |
-
if _p and _p not in sys.path:
|
| 25 |
-
sys.path.insert(0, _p)
|
| 26 |
-
|
| 27 |
-
import run_alphaearth_occupancy_benchmark as alpha_runner # noqa: E402
|
| 28 |
-
import run_attached_daily_occupancy_head_control as head_control # noqa: E402
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
HEADS = ("constant", "linear", "pixel_mlp", "shallow", "shallow_wide")
|
| 32 |
-
METRICS = {
|
| 33 |
-
"exact": "strict_f1",
|
| 34 |
-
"tolerated": "ts_f1",
|
| 35 |
-
"union": "comprehensive_union_f1",
|
| 36 |
-
}
|
| 37 |
-
SCOPE_ORDER = ("global", "fire_prone")
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def parse_args() -> argparse.Namespace:
|
| 41 |
-
parser = argparse.ArgumentParser(description="All-backbone fixed-feature head-selection regret.")
|
| 42 |
-
parser.add_argument("--source-kind", choices=("reference", "attached", "spatial", "alphaearth"), required=True)
|
| 43 |
-
parser.add_argument("--feature-root", type=Path, required=True)
|
| 44 |
-
parser.add_argument("--daily-rows-csv", type=Path)
|
| 45 |
-
parser.add_argument("--support-dir", type=Path)
|
| 46 |
-
parser.add_argument("--alphaearth-cache-root", type=Path)
|
| 47 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 48 |
-
parser.add_argument("--fm-family", type=str, required=True)
|
| 49 |
-
parser.add_argument("--model-tag", type=str, required=True)
|
| 50 |
-
parser.add_argument("--seed", type=int, required=True)
|
| 51 |
-
parser.add_argument("--heads", nargs="+", choices=HEADS, default=list(HEADS))
|
| 52 |
-
parser.add_argument("--batch-size", type=int, default=4)
|
| 53 |
-
parser.add_argument("--epochs", type=int, default=4)
|
| 54 |
-
parser.add_argument("--learning-rate", type=float, default=8e-4)
|
| 55 |
-
parser.add_argument("--weight-decay", type=float, default=1e-5)
|
| 56 |
-
parser.add_argument("--pos-weight-cap", type=float, default=150.0)
|
| 57 |
-
parser.add_argument("--device", choices=("cpu", "cuda", "auto"), default="auto")
|
| 58 |
-
parser.add_argument(
|
| 59 |
-
"--metric-thresholds",
|
| 60 |
-
nargs="+",
|
| 61 |
-
type=float,
|
| 62 |
-
default=[
|
| 63 |
-
1e-5,
|
| 64 |
-
2e-5,
|
| 65 |
-
5e-5,
|
| 66 |
-
1e-4,
|
| 67 |
-
2e-4,
|
| 68 |
-
5e-4,
|
| 69 |
-
1e-3,
|
| 70 |
-
2e-3,
|
| 71 |
-
5e-3,
|
| 72 |
-
1e-2,
|
| 73 |
-
2e-2,
|
| 74 |
-
5e-2,
|
| 75 |
-
8e-2,
|
| 76 |
-
1e-1,
|
| 77 |
-
1.5e-1,
|
| 78 |
-
2e-1,
|
| 79 |
-
3e-1,
|
| 80 |
-
5e-1,
|
| 81 |
-
],
|
| 82 |
-
)
|
| 83 |
-
parser.add_argument(
|
| 84 |
-
"--variants",
|
| 85 |
-
nargs="+",
|
| 86 |
-
default=["identity", "erode_r1", "close_r1"],
|
| 87 |
-
)
|
| 88 |
-
parser.add_argument("--fire-prone-top-frac", type=float, default=0.20)
|
| 89 |
-
parser.add_argument("--temporal-steps", type=int, default=3)
|
| 90 |
-
parser.add_argument("--spatial-radius", type=int, default=8)
|
| 91 |
-
parser.add_argument("--buffer-radius", type=int, default=8)
|
| 92 |
-
parser.add_argument("--boundary-radius", type=int, default=8)
|
| 93 |
-
parser.add_argument("--coarse-factor", type=int, default=8)
|
| 94 |
-
parser.add_argument("--time-step-hours", type=int, default=6)
|
| 95 |
-
return parser.parse_args()
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
def read_rows(path: Path) -> list[dict[str, str]]:
|
| 99 |
-
with path.open("r", encoding="utf-8", newline="") as fh:
|
| 100 |
-
return list(csv.DictReader(fh))
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
def choose_device(value: str) -> torch.device:
|
| 104 |
-
if value == "auto":
|
| 105 |
-
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 106 |
-
device = torch.device(value)
|
| 107 |
-
if device.type == "cuda" and not torch.cuda.is_available():
|
| 108 |
-
raise RuntimeError("CUDA requested but not available.")
|
| 109 |
-
return device
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
def set_seed(seed: int) -> None:
|
| 113 |
-
random.seed(seed)
|
| 114 |
-
np.random.seed(seed)
|
| 115 |
-
torch.manual_seed(seed)
|
| 116 |
-
torch.cuda.manual_seed_all(seed)
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
def total_positive_rate(rows: list[dict[str, str]]) -> float:
|
| 120 |
-
pos = float(sum(int(row["pos_cells"]) for row in rows))
|
| 121 |
-
arr = np.load(rows[0]["feature_path"], allow_pickle=True)
|
| 122 |
-
try:
|
| 123 |
-
total = float(len(rows) * np.squeeze(arr["y_occ"]).size)
|
| 124 |
-
finally:
|
| 125 |
-
arr.close()
|
| 126 |
-
return float(pos / total) if total > 0 else 0.0
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
class Normalizer:
|
| 130 |
-
def __init__(self, mean: np.ndarray, std: np.ndarray) -> None:
|
| 131 |
-
self.mean = mean.astype(np.float32)
|
| 132 |
-
self.std = np.maximum(std.astype(np.float32), 1e-6)
|
| 133 |
-
|
| 134 |
-
def apply(self, x: np.ndarray) -> np.ndarray:
|
| 135 |
-
return (x.astype(np.float32) - self.mean[:, None, None]) / self.std[:, None, None]
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
def compute_map_normalizer(rows: list[dict[str, str]], map_path_fn: Any) -> Normalizer:
|
| 139 |
-
sum_c: np.ndarray | None = None
|
| 140 |
-
sumsq_c: np.ndarray | None = None
|
| 141 |
-
count = 0
|
| 142 |
-
for row in rows:
|
| 143 |
-
arr = np.load(map_path_fn(row), allow_pickle=True)
|
| 144 |
-
try:
|
| 145 |
-
x = np.nan_to_num(arr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 146 |
-
finally:
|
| 147 |
-
arr.close()
|
| 148 |
-
flat = x.reshape(x.shape[0], -1)
|
| 149 |
-
if sum_c is None:
|
| 150 |
-
sum_c = np.zeros(x.shape[0], dtype=np.float64)
|
| 151 |
-
sumsq_c = np.zeros(x.shape[0], dtype=np.float64)
|
| 152 |
-
sum_c += flat.sum(axis=1)
|
| 153 |
-
sumsq_c += np.square(flat, dtype=np.float64).sum(axis=1)
|
| 154 |
-
count += flat.shape[1]
|
| 155 |
-
if sum_c is None or sumsq_c is None or count == 0:
|
| 156 |
-
raise RuntimeError("Cannot compute feature normalizer.")
|
| 157 |
-
mean = sum_c / float(count)
|
| 158 |
-
var = np.maximum(sumsq_c / float(count) - mean * mean, 1e-12)
|
| 159 |
-
return Normalizer(mean=mean.astype(np.float32), std=np.sqrt(var).astype(np.float32))
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
def load_support_manifest(path: Path) -> dict[str, dict[str, str]]:
|
| 163 |
-
rows = read_rows(path)
|
| 164 |
-
support: dict[str, dict[str, str]] = {}
|
| 165 |
-
for row in rows:
|
| 166 |
-
support_path = Path(row["support_path"])
|
| 167 |
-
if row.get("status") in {"generated", "existing"} and support_path.exists():
|
| 168 |
-
support[str(row["sample_id"])] = row
|
| 169 |
-
return support
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
class ReferenceFeatureStore:
|
| 173 |
-
def __init__(self, rows: list[dict[str, str]], normalizer: Normalizer) -> None:
|
| 174 |
-
self.rows_by_id = {str(row["sample_id"]): row for row in rows}
|
| 175 |
-
self.normalizer = normalizer
|
| 176 |
-
|
| 177 |
-
def get(self, sample_id: str) -> dict[str, np.ndarray | str]:
|
| 178 |
-
row = self.rows_by_id[str(sample_id)]
|
| 179 |
-
arr = np.load(row["feature_path"], allow_pickle=True)
|
| 180 |
-
try:
|
| 181 |
-
x = np.nan_to_num(arr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 182 |
-
y = np.nan_to_num(arr["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 183 |
-
finally:
|
| 184 |
-
arr.close()
|
| 185 |
-
return {"x": self.normalizer.apply(x), "y_occ": y, "target_timestamp": row["target_timestamp"]}
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
class SpatialSupportStore:
|
| 189 |
-
def __init__(
|
| 190 |
-
self,
|
| 191 |
-
rows: list[dict[str, str]],
|
| 192 |
-
support: dict[str, dict[str, str]],
|
| 193 |
-
normalizer: Normalizer,
|
| 194 |
-
) -> None:
|
| 195 |
-
self.rows_by_id = {str(row["sample_id"]): row for row in rows}
|
| 196 |
-
self.support = support
|
| 197 |
-
self.normalizer = normalizer
|
| 198 |
-
|
| 199 |
-
def get(self, sample_id: str) -> dict[str, np.ndarray | str]:
|
| 200 |
-
sid = str(sample_id)
|
| 201 |
-
row = self.rows_by_id[sid]
|
| 202 |
-
sarr = np.load(self.support[sid]["support_path"], allow_pickle=True)
|
| 203 |
-
farr = np.load(row["feature_path"], allow_pickle=True)
|
| 204 |
-
try:
|
| 205 |
-
x = np.nan_to_num(sarr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 206 |
-
y = np.nan_to_num(farr["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0)
|
| 207 |
-
finally:
|
| 208 |
-
sarr.close()
|
| 209 |
-
farr.close()
|
| 210 |
-
return {"x": self.normalizer.apply(x), "y_occ": y, "target_timestamp": row["target_timestamp"]}
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
class FullMapDataset(Dataset):
|
| 214 |
-
def __init__(self, rows: list[dict[str, str]], store: Any) -> None:
|
| 215 |
-
self.rows = rows
|
| 216 |
-
self.store = store
|
| 217 |
-
|
| 218 |
-
def __len__(self) -> int:
|
| 219 |
-
return len(self.rows)
|
| 220 |
-
|
| 221 |
-
def __getitem__(self, idx: int) -> dict[str, Any]:
|
| 222 |
-
row = self.rows[idx]
|
| 223 |
-
sample = self.store.get(str(row["sample_id"]))
|
| 224 |
-
return {
|
| 225 |
-
"x": torch.from_numpy(np.asarray(sample["x"], dtype=np.float32)),
|
| 226 |
-
"y": torch.from_numpy(np.asarray(sample["y_occ"], dtype=np.float32)),
|
| 227 |
-
"sample_id": str(row["sample_id"]),
|
| 228 |
-
"target_timestamp": str(sample["target_timestamp"]),
|
| 229 |
-
}
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
def make_loaders(
|
| 233 |
-
split_rows: dict[str, list[dict[str, str]]],
|
| 234 |
-
store: Any,
|
| 235 |
-
batch_size: int,
|
| 236 |
-
device: torch.device,
|
| 237 |
-
seed: int,
|
| 238 |
-
) -> dict[str, DataLoader]:
|
| 239 |
-
loaders: dict[str, DataLoader] = {}
|
| 240 |
-
for split, rows in split_rows.items():
|
| 241 |
-
kwargs: dict[str, Any] = {}
|
| 242 |
-
if split == "train":
|
| 243 |
-
kwargs["generator"] = torch.Generator().manual_seed(int(seed))
|
| 244 |
-
loaders[split] = DataLoader(
|
| 245 |
-
FullMapDataset(rows, store),
|
| 246 |
-
batch_size=int(batch_size),
|
| 247 |
-
shuffle=(split == "train"),
|
| 248 |
-
num_workers=0,
|
| 249 |
-
pin_memory=device.type == "cuda",
|
| 250 |
-
**kwargs,
|
| 251 |
-
)
|
| 252 |
-
return loaders
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
def build_attached_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> Any:
|
| 256 |
-
if args.daily_rows_csv is None:
|
| 257 |
-
raise ValueError("--daily-rows-csv is required for attached source.")
|
| 258 |
-
daily_lookup, ordered_times, ordered_features = head_control.build_daily_lookup(args.daily_rows_csv)
|
| 259 |
-
return head_control.FeatureStore(
|
| 260 |
-
split_rows["train"] + split_rows["val"] + split_rows["test"],
|
| 261 |
-
daily_lookup,
|
| 262 |
-
ordered_times,
|
| 263 |
-
ordered_features,
|
| 264 |
-
)
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
def build_alphaearth_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> Any:
|
| 268 |
-
if args.alphaearth_cache_root is None:
|
| 269 |
-
raise ValueError("--alphaearth-cache-root is required for AlphaEarth source.")
|
| 270 |
-
grid_cache = alpha_runner.GridCache(args.alphaearth_cache_root)
|
| 271 |
-
return alpha_runner.FeatureStore(split_rows["train"] + split_rows["val"] + split_rows["test"], grid_cache)
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
def build_spatial_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> SpatialSupportStore:
|
| 275 |
-
if args.support_dir is None:
|
| 276 |
-
raise ValueError("--support-dir is required for spatial source.")
|
| 277 |
-
support = load_support_manifest(args.support_dir / "support_manifest.csv")
|
| 278 |
-
missing = [
|
| 279 |
-
row["sample_id"]
|
| 280 |
-
for rows in split_rows.values()
|
| 281 |
-
for row in rows
|
| 282 |
-
if str(row["sample_id"]) not in support
|
| 283 |
-
]
|
| 284 |
-
if missing:
|
| 285 |
-
raise RuntimeError(f"Missing spatial support maps for {len(missing)} samples; first={missing[:5]}")
|
| 286 |
-
normalizer = compute_map_normalizer(split_rows["train"], lambda row: support[str(row["sample_id"])]["support_path"])
|
| 287 |
-
return SpatialSupportStore(split_rows["train"] + split_rows["val"] + split_rows["test"], support, normalizer)
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
def build_reference_store(split_rows: dict[str, list[dict[str, str]]]) -> ReferenceFeatureStore:
|
| 291 |
-
normalizer = compute_map_normalizer(split_rows["train"], lambda row: row["feature_path"])
|
| 292 |
-
return ReferenceFeatureStore(split_rows["train"] + split_rows["val"] + split_rows["test"], normalizer)
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
def build_head(head_arch: str, in_ch: int, prior_prob: float) -> nn.Module:
|
| 296 |
-
if head_arch == "constant":
|
| 297 |
-
return head_control.ConstantHead(prior_prob=prior_prob)
|
| 298 |
-
if head_arch == "linear":
|
| 299 |
-
return head_control.LinearHead(in_ch=in_ch, prior_prob=prior_prob)
|
| 300 |
-
if head_arch == "pixel_mlp":
|
| 301 |
-
return head_control.PixelMLPHead(in_ch=in_ch, hidden=16, dropout=0.05, prior_prob=prior_prob)
|
| 302 |
-
if head_arch == "shallow_wide":
|
| 303 |
-
return head_control.WildfireHead(
|
| 304 |
-
in_ch=in_ch,
|
| 305 |
-
hidden=64,
|
| 306 |
-
dropout=0.10,
|
| 307 |
-
norm_type="group",
|
| 308 |
-
norm_groups=8,
|
| 309 |
-
prior_prob=prior_prob,
|
| 310 |
-
)
|
| 311 |
-
return head_control.WildfireHead(
|
| 312 |
-
in_ch=in_ch,
|
| 313 |
-
hidden=32,
|
| 314 |
-
dropout=0.05,
|
| 315 |
-
norm_type="group",
|
| 316 |
-
norm_groups=8,
|
| 317 |
-
prior_prob=prior_prob,
|
| 318 |
-
)
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
@torch.no_grad()
|
| 322 |
-
def collect_predictions(model: nn.Module, loader: DataLoader, device: torch.device) -> tuple[np.ndarray, np.ndarray]:
|
| 323 |
-
model.eval()
|
| 324 |
-
probs: list[np.ndarray] = []
|
| 325 |
-
targets: list[np.ndarray] = []
|
| 326 |
-
for batch in loader:
|
| 327 |
-
x = batch["x"].to(device, non_blocking=True)
|
| 328 |
-
y = batch["y"].to(device, non_blocking=True)
|
| 329 |
-
logits = model(x)
|
| 330 |
-
probs.append(np.nan_to_num(torch.sigmoid(logits).detach().cpu().numpy()[:, 0], nan=0.0, posinf=1.0, neginf=0.0))
|
| 331 |
-
targets.append(np.nan_to_num(y.detach().cpu().numpy()[:, 0], nan=0.0, posinf=0.0, neginf=0.0))
|
| 332 |
-
return np.concatenate(probs, axis=0), np.concatenate(targets, axis=0)
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
def train_one_head(
|
| 336 |
-
head_arch: str,
|
| 337 |
-
in_ch: int,
|
| 338 |
-
prior_prob: float,
|
| 339 |
-
loaders: dict[str, DataLoader],
|
| 340 |
-
args: argparse.Namespace,
|
| 341 |
-
device: torch.device,
|
| 342 |
-
seed_offset: int,
|
| 343 |
-
) -> tuple[nn.Module, list[dict[str, float]]]:
|
| 344 |
-
set_seed(int(args.seed) + int(seed_offset))
|
| 345 |
-
model = build_head(head_arch, in_ch=in_ch, prior_prob=prior_prob).to(device)
|
| 346 |
-
optimizer = torch.optim.AdamW(model.parameters(), lr=float(args.learning_rate), weight_decay=float(args.weight_decay))
|
| 347 |
-
raw_weight = (1.0 - float(prior_prob)) / max(float(prior_prob), 1e-9)
|
| 348 |
-
pos_weight = float(min(float(args.pos_weight_cap), raw_weight))
|
| 349 |
-
criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([pos_weight], dtype=torch.float32, device=device))
|
| 350 |
-
history: list[dict[str, float]] = []
|
| 351 |
-
for epoch in range(1, int(args.epochs) + 1):
|
| 352 |
-
model.train()
|
| 353 |
-
losses: list[float] = []
|
| 354 |
-
for batch in loaders["train"]:
|
| 355 |
-
x = batch["x"].to(device, non_blocking=True)
|
| 356 |
-
y = batch["y"].to(device, non_blocking=True)
|
| 357 |
-
optimizer.zero_grad(set_to_none=True)
|
| 358 |
-
logits = model(x)
|
| 359 |
-
loss = criterion(logits, y)
|
| 360 |
-
if not torch.isfinite(loss):
|
| 361 |
-
raise RuntimeError(f"Non-finite loss for head={head_arch}")
|
| 362 |
-
loss.backward()
|
| 363 |
-
optimizer.step()
|
| 364 |
-
losses.append(float(loss.item()))
|
| 365 |
-
history.append({"epoch": epoch, "train_bce": float(np.mean(losses)), "pos_weight": pos_weight})
|
| 366 |
-
return model, history
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
def build_sample_times(rows: list[dict[str, str]]) -> np.ndarray:
|
| 370 |
-
return np.array([row["target_timestamp"] for row in rows], dtype="datetime64[h]")
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
def select_val_posthoc(
|
| 374 |
-
rows: list[dict[str, object]],
|
| 375 |
-
scope: str,
|
| 376 |
-
metric: str,
|
| 377 |
-
) -> dict[str, object]:
|
| 378 |
-
prefix = metric.rsplit("_", 1)[0]
|
| 379 |
-
precision_key = f"{prefix}_precision"
|
| 380 |
-
recall_key = f"{prefix}_recall"
|
| 381 |
-
selected = [row for row in rows if row["split"] == "val" and row["scope"] == scope]
|
| 382 |
-
if not selected:
|
| 383 |
-
raise RuntimeError(f"No validation rows for scope={scope}")
|
| 384 |
-
return max(
|
| 385 |
-
selected,
|
| 386 |
-
key=lambda row: (
|
| 387 |
-
float(row.get(metric, 0.0)),
|
| 388 |
-
float(row.get(precision_key, 0.0)),
|
| 389 |
-
float(row.get(recall_key, 0.0)),
|
| 390 |
-
-abs(float(row["threshold"]) - 0.5),
|
| 391 |
-
),
|
| 392 |
-
)
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
def matching_test_row(rows: list[dict[str, object]], scope: str, selected: dict[str, object]) -> dict[str, object]:
|
| 396 |
-
threshold = float(selected["threshold"])
|
| 397 |
-
variant = str(selected["variant"])
|
| 398 |
-
for row in rows:
|
| 399 |
-
if (
|
| 400 |
-
row["split"] == "test"
|
| 401 |
-
and row["scope"] == scope
|
| 402 |
-
and str(row["variant"]) == variant
|
| 403 |
-
and abs(float(row["threshold"]) - threshold) < 1e-12
|
| 404 |
-
):
|
| 405 |
-
return row
|
| 406 |
-
raise RuntimeError(f"No matching test row for scope={scope}, threshold={threshold}, variant={variant}")
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
def summarize_head_scores(
|
| 410 |
-
head_metrics: list[dict[str, object]],
|
| 411 |
-
) -> list[dict[str, object]]:
|
| 412 |
-
selection_rows: list[dict[str, object]] = []
|
| 413 |
-
for scope in SCOPE_ORDER:
|
| 414 |
-
candidates = [row for row in head_metrics if row["scope"] == scope]
|
| 415 |
-
if not candidates:
|
| 416 |
-
continue
|
| 417 |
-
ranking_selected = max(
|
| 418 |
-
candidates,
|
| 419 |
-
key=lambda row: (
|
| 420 |
-
float(row["val_pr_auc"]),
|
| 421 |
-
float(row["val_union_f1"]),
|
| 422 |
-
float(row["val_tolerated_f1"]),
|
| 423 |
-
float(row["val_exact_f1"]),
|
| 424 |
-
),
|
| 425 |
-
)
|
| 426 |
-
out: dict[str, object] = {
|
| 427 |
-
"scope": scope,
|
| 428 |
-
"ranking_selected_head": ranking_selected["head_label"],
|
| 429 |
-
"ranking_selected_head_arch": ranking_selected["head_arch"],
|
| 430 |
-
"ranking_selected_val_pr_auc": float(ranking_selected["val_pr_auc"]),
|
| 431 |
-
"ranking_selected_test_pr_auc": float(ranking_selected["test_pr_auc"]),
|
| 432 |
-
}
|
| 433 |
-
for short, column in (("exact", "exact_f1"), ("tolerated", "tolerated_f1"), ("union", "union_f1")):
|
| 434 |
-
val_column = f"val_{column}"
|
| 435 |
-
test_column = f"test_{column}"
|
| 436 |
-
decision_selected = max(
|
| 437 |
-
candidates,
|
| 438 |
-
key=lambda row: (
|
| 439 |
-
float(row[val_column]),
|
| 440 |
-
float(row["val_pr_auc"]),
|
| 441 |
-
str(row["head_arch"]),
|
| 442 |
-
),
|
| 443 |
-
)
|
| 444 |
-
val_gap = float(decision_selected[val_column]) - float(ranking_selected[val_column])
|
| 445 |
-
test_gap = float(decision_selected[test_column]) - float(ranking_selected[test_column])
|
| 446 |
-
out[f"{short}_val_ranking_score"] = float(ranking_selected[val_column])
|
| 447 |
-
out[f"{short}_val_decision_score"] = float(decision_selected[val_column])
|
| 448 |
-
out[f"{short}_val_gap"] = float(max(0.0, val_gap))
|
| 449 |
-
out[f"{short}_ranking_score"] = float(ranking_selected[test_column])
|
| 450 |
-
out[f"{short}_decision_score"] = float(decision_selected[test_column])
|
| 451 |
-
out[f"{short}_test_gap"] = float(test_gap)
|
| 452 |
-
out[f"{short}_regret"] = float(max(0.0, test_gap))
|
| 453 |
-
out[f"{short}_decision_head"] = decision_selected["head_label"]
|
| 454 |
-
out[f"{short}_decision_head_arch"] = decision_selected["head_arch"]
|
| 455 |
-
selection_rows.append(out)
|
| 456 |
-
return selection_rows
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
def write_csv(rows: list[dict[str, object]], path: Path) -> None:
|
| 460 |
-
path.parent.mkdir(parents=True, exist_ok=True)
|
| 461 |
-
fieldnames = sorted({key for row in rows for key in row})
|
| 462 |
-
with path.open("w", newline="", encoding="utf-8") as fh:
|
| 463 |
-
writer = csv.DictWriter(fh, fieldnames=fieldnames)
|
| 464 |
-
writer.writeheader()
|
| 465 |
-
writer.writerows(rows)
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
def load_head_summary(head_dir: Path, head_arch: str) -> tuple[list[dict[str, object]], dict[str, dict[str, float]], dict[str, object]] | None:
|
| 469 |
-
posthoc_path = head_dir / "posthoc_rows.csv"
|
| 470 |
-
summary_path = head_dir / "summary.json"
|
| 471 |
-
if not posthoc_path.exists() or not summary_path.exists():
|
| 472 |
-
return None
|
| 473 |
-
rows = [dict(row) for row in read_rows(posthoc_path)]
|
| 474 |
-
if not rows:
|
| 475 |
-
return None
|
| 476 |
-
try:
|
| 477 |
-
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
| 478 |
-
except json.JSONDecodeError:
|
| 479 |
-
return None
|
| 480 |
-
if str(summary.get("head_arch")) != str(head_arch):
|
| 481 |
-
return None
|
| 482 |
-
raw_pr_auc = summary.get("raw_pr_auc")
|
| 483 |
-
if not isinstance(raw_pr_auc, dict):
|
| 484 |
-
return None
|
| 485 |
-
try:
|
| 486 |
-
parsed_pr_auc = {
|
| 487 |
-
split: {
|
| 488 |
-
scope: float(raw_pr_auc[split][scope])
|
| 489 |
-
for scope in SCOPE_ORDER
|
| 490 |
-
}
|
| 491 |
-
for split in ("val", "test")
|
| 492 |
-
}
|
| 493 |
-
except Exception:
|
| 494 |
-
return None
|
| 495 |
-
return rows, parsed_pr_auc, summary
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
def append_head_metrics(
|
| 499 |
-
head_metrics: list[dict[str, object]],
|
| 500 |
-
posthoc_rows: list[dict[str, object]],
|
| 501 |
-
raw_pr_auc: dict[str, dict[str, float]],
|
| 502 |
-
head_arch: str,
|
| 503 |
-
args: argparse.Namespace,
|
| 504 |
-
) -> None:
|
| 505 |
-
for scope in SCOPE_ORDER:
|
| 506 |
-
metric_scores: dict[str, float] = {}
|
| 507 |
-
selected_thresholds: dict[str, float] = {}
|
| 508 |
-
selected_variants: dict[str, str] = {}
|
| 509 |
-
for short, metric in METRICS.items():
|
| 510 |
-
selected = select_val_posthoc(posthoc_rows, scope, metric)
|
| 511 |
-
test_row = matching_test_row(posthoc_rows, scope, selected)
|
| 512 |
-
metric_scores[f"val_{short}_f1"] = float(selected[metric])
|
| 513 |
-
metric_scores[f"test_{short}_f1"] = float(test_row[metric])
|
| 514 |
-
selected_thresholds[short] = float(selected["threshold"])
|
| 515 |
-
selected_variants[short] = str(selected["variant"])
|
| 516 |
-
head_metrics.append(
|
| 517 |
-
{
|
| 518 |
-
"model_tag": args.model_tag,
|
| 519 |
-
"family": args.fm_family,
|
| 520 |
-
"seed": int(args.seed),
|
| 521 |
-
"scope": scope,
|
| 522 |
-
"head_arch": head_arch,
|
| 523 |
-
"head_label": head_control.HEAD_LABELS[head_arch],
|
| 524 |
-
"val_pr_auc": float(raw_pr_auc["val"][scope]),
|
| 525 |
-
"test_pr_auc": float(raw_pr_auc["test"][scope]),
|
| 526 |
-
**metric_scores,
|
| 527 |
-
"selected_thresholds": selected_thresholds,
|
| 528 |
-
"selected_variants": selected_variants,
|
| 529 |
-
}
|
| 530 |
-
)
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
def main() -> None:
|
| 534 |
-
args = parse_args()
|
| 535 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 536 |
-
set_seed(int(args.seed))
|
| 537 |
-
device = choose_device(args.device)
|
| 538 |
-
|
| 539 |
-
split_rows = {
|
| 540 |
-
split: read_rows(args.feature_root / "splits" / f"{split}.csv")
|
| 541 |
-
for split in ("train", "val", "test")
|
| 542 |
-
}
|
| 543 |
-
if args.source_kind == "reference":
|
| 544 |
-
store = build_reference_store(split_rows)
|
| 545 |
-
elif args.source_kind == "attached":
|
| 546 |
-
store = build_attached_store(args, split_rows)
|
| 547 |
-
elif args.source_kind == "spatial":
|
| 548 |
-
store = build_spatial_store(args, split_rows)
|
| 549 |
-
else:
|
| 550 |
-
store = build_alphaearth_store(args, split_rows)
|
| 551 |
-
|
| 552 |
-
loaders = make_loaders(split_rows, store, int(args.batch_size), device, int(args.seed))
|
| 553 |
-
first = next(iter(loaders["train"]))
|
| 554 |
-
in_ch = int(first["x"].shape[1])
|
| 555 |
-
prior_prob = total_positive_rate(split_rows["train"])
|
| 556 |
-
fire_prone_mask, fire_prone_meta = head_control.build_fire_prone_mask(
|
| 557 |
-
split_rows["train"],
|
| 558 |
-
store,
|
| 559 |
-
float(args.fire_prone_top_frac),
|
| 560 |
-
)
|
| 561 |
-
|
| 562 |
-
head_metrics: list[dict[str, object]] = []
|
| 563 |
-
head_artifacts: dict[str, str] = {}
|
| 564 |
-
for head_index, head_arch in enumerate(args.heads):
|
| 565 |
-
head_dir = args.output_dir / head_arch
|
| 566 |
-
head_dir.mkdir(parents=True, exist_ok=True)
|
| 567 |
-
cached = load_head_summary(head_dir, head_arch)
|
| 568 |
-
if cached is not None:
|
| 569 |
-
posthoc_rows, raw_pr_auc, _ = cached
|
| 570 |
-
print(f"[selection-regret] reuse {args.fm_family} seed={args.seed} head={head_arch}", flush=True)
|
| 571 |
-
else:
|
| 572 |
-
print(f"[selection-regret] training {args.fm_family} seed={args.seed} head={head_arch}", flush=True)
|
| 573 |
-
model, history = train_one_head(
|
| 574 |
-
head_arch=head_arch,
|
| 575 |
-
in_ch=in_ch,
|
| 576 |
-
prior_prob=prior_prob,
|
| 577 |
-
loaders=loaders,
|
| 578 |
-
args=args,
|
| 579 |
-
device=device,
|
| 580 |
-
seed_offset=1009 * (head_index + 1),
|
| 581 |
-
)
|
| 582 |
-
posthoc_rows = []
|
| 583 |
-
raw_pr_auc = {}
|
| 584 |
-
for split in ("val", "test"):
|
| 585 |
-
probs, targets = collect_predictions(model, loaders[split], device)
|
| 586 |
-
sample_times = build_sample_times(split_rows[split])
|
| 587 |
-
raw_pr_auc[split] = {
|
| 588 |
-
"global": head_control._masked_average_precision(probs, targets, region_mask=None),
|
| 589 |
-
"fire_prone": head_control._masked_average_precision(probs, targets, region_mask=fire_prone_mask),
|
| 590 |
-
}
|
| 591 |
-
posthoc_rows.extend(
|
| 592 |
-
head_control.build_posthoc_rows(
|
| 593 |
-
probs=probs,
|
| 594 |
-
targets=targets,
|
| 595 |
-
sample_times=sample_times,
|
| 596 |
-
split=split,
|
| 597 |
-
fire_prone_mask=fire_prone_mask,
|
| 598 |
-
args=args,
|
| 599 |
-
)
|
| 600 |
-
)
|
| 601 |
-
|
| 602 |
-
write_csv(posthoc_rows, head_dir / "posthoc_rows.csv")
|
| 603 |
-
head_summary = {
|
| 604 |
-
"head_arch": head_arch,
|
| 605 |
-
"head_label": head_control.HEAD_LABELS[head_arch],
|
| 606 |
-
"history": history,
|
| 607 |
-
"raw_pr_auc": raw_pr_auc,
|
| 608 |
-
"posthoc_rows_csv": str(head_dir / "posthoc_rows.csv"),
|
| 609 |
-
}
|
| 610 |
-
(head_dir / "summary.json").write_text(json.dumps(head_summary, indent=2), encoding="utf-8")
|
| 611 |
-
head_artifacts[head_arch] = str(head_dir / "summary.json")
|
| 612 |
-
append_head_metrics(head_metrics, posthoc_rows, raw_pr_auc, head_arch, args)
|
| 613 |
-
|
| 614 |
-
selection_rows = summarize_head_scores(head_metrics)
|
| 615 |
-
for row in selection_rows:
|
| 616 |
-
row["model_tag"] = args.model_tag
|
| 617 |
-
row["family"] = args.fm_family
|
| 618 |
-
row["seed"] = int(args.seed)
|
| 619 |
-
|
| 620 |
-
write_csv(head_metrics, args.output_dir / "head_metrics.csv")
|
| 621 |
-
write_csv(selection_rows, args.output_dir / "selection_rows.csv")
|
| 622 |
-
summary = {
|
| 623 |
-
"experiment": "all-backbone fixed-feature head-selection regret",
|
| 624 |
-
"task": "wildfire_occupancy",
|
| 625 |
-
"model_tag": args.model_tag,
|
| 626 |
-
"fm_family": args.fm_family,
|
| 627 |
-
"source_kind": args.source_kind,
|
| 628 |
-
"seed": int(args.seed),
|
| 629 |
-
"feature_root": str(args.feature_root),
|
| 630 |
-
"daily_rows_csv": str(args.daily_rows_csv) if args.daily_rows_csv else None,
|
| 631 |
-
"support_dir": str(args.support_dir) if args.support_dir else None,
|
| 632 |
-
"alphaearth_cache_root": str(args.alphaearth_cache_root) if args.alphaearth_cache_root else None,
|
| 633 |
-
"device": str(device),
|
| 634 |
-
"heads": list(args.heads),
|
| 635 |
-
"input_channels": int(in_ch),
|
| 636 |
-
"prior_prob": float(prior_prob),
|
| 637 |
-
"fire_prone_scope": {
|
| 638 |
-
"scope_name": "fire_prone",
|
| 639 |
-
"reported_as": "top 20%",
|
| 640 |
-
**fire_prone_meta,
|
| 641 |
-
},
|
| 642 |
-
"metrics": METRICS,
|
| 643 |
-
"head_metrics": head_metrics,
|
| 644 |
-
"selection_rows": selection_rows,
|
| 645 |
-
"head_artifacts": head_artifacts,
|
| 646 |
-
"artifacts": {
|
| 647 |
-
"head_metrics_csv": str(args.output_dir / "head_metrics.csv"),
|
| 648 |
-
"selection_rows_csv": str(args.output_dir / "selection_rows.csv"),
|
| 649 |
-
},
|
| 650 |
-
}
|
| 651 |
-
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 652 |
-
print(json.dumps(summary, indent=2), flush=True)
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
if __name__ == "__main__":
|
| 656 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py
DELETED
|
@@ -1,333 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import argparse
|
| 5 |
-
import json
|
| 6 |
-
import sys
|
| 7 |
-
from pathlib import Path
|
| 8 |
-
from typing import Dict, List, Tuple
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
import os
|
| 12 |
-
|
| 13 |
-
for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep):
|
| 14 |
-
if _p and _p not in sys.path:
|
| 15 |
-
sys.path.insert(0, _p)
|
| 16 |
-
|
| 17 |
-
import numpy as np
|
| 18 |
-
import pandas as pd
|
| 19 |
-
from sklearn.compose import ColumnTransformer
|
| 20 |
-
from sklearn.impute import SimpleImputer
|
| 21 |
-
from sklearn.linear_model import ElasticNet, Ridge
|
| 22 |
-
from sklearn.metrics.pairwise import cosine_similarity
|
| 23 |
-
from sklearn.pipeline import Pipeline
|
| 24 |
-
from sklearn.preprocessing import OneHotEncoder, StandardScaler
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
DROP_COLUMNS = {
|
| 28 |
-
"Event_ID",
|
| 29 |
-
"Incid_Name",
|
| 30 |
-
"incident_name_norm",
|
| 31 |
-
"wfigs_name",
|
| 32 |
-
"Ig_Date",
|
| 33 |
-
"weather_date",
|
| 34 |
-
"BurnBndAc",
|
| 35 |
-
"target_log_burn_acres",
|
| 36 |
-
}
|
| 37 |
-
CATEGORICAL_COLUMNS = ["Incid_Type", "state_abbr", "county_name", "wfigs_match_type"]
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 41 |
-
ordered = df.sort_values("Ig_Date").reset_index(drop=True)
|
| 42 |
-
n = len(ordered)
|
| 43 |
-
train_end = max(int(round(n * 0.6)), 1)
|
| 44 |
-
val_end = max(int(round(n * 0.8)), train_end + 1)
|
| 45 |
-
val_end = min(val_end, n - 1) if n >= 3 else n
|
| 46 |
-
train = ordered.iloc[:train_end].copy()
|
| 47 |
-
val = ordered.iloc[train_end:val_end].copy()
|
| 48 |
-
test = ordered.iloc[val_end:].copy()
|
| 49 |
-
if len(val) == 0 and len(test) > 1:
|
| 50 |
-
val = test.iloc[:1].copy()
|
| 51 |
-
test = test.iloc[1:].copy()
|
| 52 |
-
return train, val, test
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def block_columns(df: pd.DataFrame, exclude: set[str]) -> Dict[str, List[str]]:
|
| 56 |
-
numeric = [
|
| 57 |
-
c
|
| 58 |
-
for c in df.columns
|
| 59 |
-
if c not in DROP_COLUMNS
|
| 60 |
-
and c not in CATEGORICAL_COLUMNS
|
| 61 |
-
and c not in exclude
|
| 62 |
-
and pd.api.types.is_numeric_dtype(df[c])
|
| 63 |
-
]
|
| 64 |
-
return {
|
| 65 |
-
"weather": [c for c in numeric if c.startswith("weather_")],
|
| 66 |
-
"geo_fire": [
|
| 67 |
-
c
|
| 68 |
-
for c in numeric
|
| 69 |
-
if c.startswith("firms_")
|
| 70 |
-
or c.startswith("landfire_")
|
| 71 |
-
or c in {"BurnBndLat", "BurnBndLon", "lat", "lon", "wfigs_acres", "wfigs_date_diff_days", "wfigs_dist_km", "is_conus_static"}
|
| 72 |
-
],
|
| 73 |
-
"categorical": [c for c in CATEGORICAL_COLUMNS if c in df.columns and c not in exclude],
|
| 74 |
-
}
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def make_block_matrix(train: pd.DataFrame, val: pd.DataFrame, test: pd.DataFrame, cols: List[str], categorical: bool) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 78 |
-
if not cols:
|
| 79 |
-
n_train, n_val, n_test = len(train), len(val), len(test)
|
| 80 |
-
return np.zeros((n_train, 0), dtype=np.float32), np.zeros((n_val, 0), dtype=np.float32), np.zeros((n_test, 0), dtype=np.float32)
|
| 81 |
-
if categorical:
|
| 82 |
-
transformer = ColumnTransformer(
|
| 83 |
-
[("cat", Pipeline([("impute", SimpleImputer(strategy="most_frequent")), ("onehot", OneHotEncoder(handle_unknown="ignore"))]), cols)],
|
| 84 |
-
remainder="drop",
|
| 85 |
-
)
|
| 86 |
-
else:
|
| 87 |
-
transformer = ColumnTransformer(
|
| 88 |
-
[("num", Pipeline([("impute", SimpleImputer(strategy="median")), ("scale", StandardScaler())]), cols)],
|
| 89 |
-
remainder="drop",
|
| 90 |
-
)
|
| 91 |
-
train_x = transformer.fit_transform(train[cols])
|
| 92 |
-
val_x = transformer.transform(val[cols])
|
| 93 |
-
test_x = transformer.transform(test[cols])
|
| 94 |
-
if hasattr(train_x, "toarray"):
|
| 95 |
-
train_x = train_x.toarray()
|
| 96 |
-
val_x = val_x.toarray()
|
| 97 |
-
test_x = test_x.toarray()
|
| 98 |
-
return train_x.astype(np.float32), val_x.astype(np.float32), test_x.astype(np.float32)
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
def graded_relevance(query_target: float, retrieved_targets: np.ndarray) -> np.ndarray:
|
| 102 |
-
delta = np.abs(np.asarray(retrieved_targets, dtype=np.float64) - float(query_target))
|
| 103 |
-
return np.select([delta <= 0.5, delta <= 1.0, delta <= 1.5], [3.0, 2.0, 1.0], default=0.0)
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
def dcg(relevance: np.ndarray) -> float:
|
| 107 |
-
rel = np.asarray(relevance, dtype=np.float64)
|
| 108 |
-
discounts = 1.0 / np.log2(np.arange(rel.size, dtype=np.float64) + 2.0)
|
| 109 |
-
return float(np.sum(rel * discounts))
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
def ndcg_at_k(relevance: np.ndarray, ideal_relevance: np.ndarray, k: int) -> float:
|
| 113 |
-
denom = dcg(ideal_relevance[:k])
|
| 114 |
-
return float(dcg(relevance[:k]) / denom) if denom > 0 else 0.0
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 118 |
-
return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2)))
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 122 |
-
value = pd.Series(y_true).corr(pd.Series(y_pred), method="spearman")
|
| 123 |
-
return float(value) if pd.notna(value) else 0.0
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
def target_weight_vectors(train_vec: np.ndarray, val_vec: np.ndarray, test_vec: np.ndarray, target: np.ndarray, power: float, floor: float) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 127 |
-
if train_vec.shape[1] == 0:
|
| 128 |
-
return train_vec, val_vec, test_vec
|
| 129 |
-
x = np.asarray(train_vec, dtype=np.float64)
|
| 130 |
-
y = np.asarray(target, dtype=np.float64)
|
| 131 |
-
y = y - y.mean()
|
| 132 |
-
x_centered = x - x.mean(axis=0, keepdims=True)
|
| 133 |
-
denom = np.clip(np.sqrt(np.sum(x_centered**2, axis=0)) * np.sqrt(np.sum(y**2)), 1e-12, None)
|
| 134 |
-
corr = np.abs(np.sum(x_centered * y[:, None], axis=0) / denom)
|
| 135 |
-
corr = np.nan_to_num(corr, nan=0.0, posinf=0.0, neginf=0.0)
|
| 136 |
-
if float(corr.max()) > 0:
|
| 137 |
-
corr = corr / float(corr.max())
|
| 138 |
-
weights = (floor + np.power(corr, power)).astype(np.float32)
|
| 139 |
-
return train_vec * weights, val_vec * weights, test_vec * weights
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
def score_vectors(query_vec: np.ndarray, library_vec: np.ndarray, query_df: pd.DataFrame, library_df: pd.DataFrame, k: int, mode: str) -> Dict[str, float]:
|
| 143 |
-
k_eff = min(k, library_vec.shape[0])
|
| 144 |
-
lib_norm = library_vec / np.clip(np.linalg.norm(library_vec, axis=1, keepdims=True), 1e-12, None)
|
| 145 |
-
query_norm = query_vec / np.clip(np.linalg.norm(query_vec, axis=1, keepdims=True), 1e-12, None)
|
| 146 |
-
sim_all = cosine_similarity(query_norm, lib_norm)
|
| 147 |
-
knn_idx = np.argsort(-sim_all, axis=1)[:, :k_eff]
|
| 148 |
-
knn_sim = np.take_along_axis(sim_all, knn_idx, axis=1)
|
| 149 |
-
target_lib = library_df["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 150 |
-
preds = []
|
| 151 |
-
ndcg5 = []
|
| 152 |
-
ndcg10 = []
|
| 153 |
-
hit1 = []
|
| 154 |
-
hit5 = []
|
| 155 |
-
hit10 = []
|
| 156 |
-
best_abs = []
|
| 157 |
-
for i in range(query_df.shape[0]):
|
| 158 |
-
idx = knn_idx[i]
|
| 159 |
-
sims = knn_sim[i]
|
| 160 |
-
top_targets = target_lib[idx]
|
| 161 |
-
true = float(query_df.iloc[i]["target_log_burn_acres"])
|
| 162 |
-
relevance = graded_relevance(true, top_targets)
|
| 163 |
-
ideal = np.sort(graded_relevance(true, target_lib))[::-1]
|
| 164 |
-
ndcg5.append(ndcg_at_k(relevance, ideal, 5))
|
| 165 |
-
ndcg10.append(ndcg_at_k(relevance, ideal, 10))
|
| 166 |
-
hit1.append(float(relevance[:1].max() >= 2.0))
|
| 167 |
-
hit5.append(float(relevance[: min(5, k_eff)].max() >= 2.0))
|
| 168 |
-
hit10.append(float(relevance[: min(10, k_eff)].max() >= 2.0))
|
| 169 |
-
best_abs.append(float(np.min(np.abs(top_targets - true))))
|
| 170 |
-
if mode == "weighted":
|
| 171 |
-
weights = np.maximum((sims + 1.0) / 2.0, 1e-6)
|
| 172 |
-
preds.append(float(np.sum(weights * top_targets) / np.sum(weights)))
|
| 173 |
-
else:
|
| 174 |
-
preds.append(float(np.mean(top_targets)))
|
| 175 |
-
pred = np.asarray(preds, dtype=np.float64)
|
| 176 |
-
true_log = query_df["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 177 |
-
return {
|
| 178 |
-
"count": int(len(query_df)),
|
| 179 |
-
"log_mae": float(np.mean(np.abs(true_log - pred))),
|
| 180 |
-
"log_rmse": rmse(true_log, pred),
|
| 181 |
-
"log_spearman": spearman_corr(true_log, pred),
|
| 182 |
-
"ndcg_at_5": float(np.mean(ndcg5)),
|
| 183 |
-
"ndcg_at_10": float(np.mean(ndcg10)),
|
| 184 |
-
"hit_at_1_tol1": float(np.mean(hit1)),
|
| 185 |
-
"hit_at_5_tol1": float(np.mean(hit5)),
|
| 186 |
-
"hit_at_10_tol1": float(np.mean(hit10)),
|
| 187 |
-
"mean_best_abs_log_delta_at_k": float(np.mean(best_abs)),
|
| 188 |
-
}
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
def append_supervised_scalar(
|
| 192 |
-
train_vec: np.ndarray,
|
| 193 |
-
val_vec: np.ndarray,
|
| 194 |
-
test_vec: np.ndarray,
|
| 195 |
-
train_df: pd.DataFrame,
|
| 196 |
-
model_name: str,
|
| 197 |
-
weight: float,
|
| 198 |
-
seed: int,
|
| 199 |
-
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 200 |
-
y = train_df["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 201 |
-
model = Ridge(alpha=1.0) if model_name == "ridge" else ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=seed, max_iter=10000)
|
| 202 |
-
model.fit(train_vec, y)
|
| 203 |
-
train_pred = model.predict(train_vec)
|
| 204 |
-
val_pred = model.predict(val_vec)
|
| 205 |
-
test_pred = model.predict(test_vec)
|
| 206 |
-
mean = float(np.mean(train_pred))
|
| 207 |
-
std = float(np.std(train_pred)) or 1.0
|
| 208 |
-
def _append(x: np.ndarray, pred: np.ndarray) -> np.ndarray:
|
| 209 |
-
scalar = ((pred - mean) / std).reshape(-1, 1).astype(np.float32) * float(weight)
|
| 210 |
-
return np.concatenate([x, scalar], axis=1)
|
| 211 |
-
return _append(train_vec, train_pred), _append(val_vec, val_pred), _append(test_vec, test_pred)
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
def main() -> None:
|
| 215 |
-
parser = argparse.ArgumentParser()
|
| 216 |
-
parser.add_argument("--event-table", type=Path, required=True)
|
| 217 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 218 |
-
parser.add_argument("--exclude-columns", nargs="*", default=[])
|
| 219 |
-
parser.add_argument("--seed", type=int, default=7)
|
| 220 |
-
args = parser.parse_args()
|
| 221 |
-
|
| 222 |
-
df = pd.read_csv(args.event_table)
|
| 223 |
-
df["Ig_Date"] = pd.to_datetime(df["Ig_Date"])
|
| 224 |
-
train_df, val_df, test_df = build_splits(df)
|
| 225 |
-
exclude = set(args.exclude_columns)
|
| 226 |
-
blocks = block_columns(df, exclude)
|
| 227 |
-
matrices = {
|
| 228 |
-
"weather": make_block_matrix(train_df, val_df, test_df, blocks["weather"], categorical=False),
|
| 229 |
-
"geo_fire": make_block_matrix(train_df, val_df, test_df, blocks["geo_fire"], categorical=False),
|
| 230 |
-
"categorical": make_block_matrix(train_df, val_df, test_df, blocks["categorical"], categorical=True),
|
| 231 |
-
}
|
| 232 |
-
|
| 233 |
-
candidate_rows: List[Dict[str, object]] = []
|
| 234 |
-
best = None
|
| 235 |
-
best_score = None
|
| 236 |
-
best_test = None
|
| 237 |
-
block_weight_grid = [
|
| 238 |
-
{"weather": 1.0, "geo_fire": 1.0, "categorical": 1.0},
|
| 239 |
-
{"weather": 0.5, "geo_fire": 1.5, "categorical": 1.0},
|
| 240 |
-
{"weather": 0.25, "geo_fire": 2.0, "categorical": 1.0},
|
| 241 |
-
{"weather": 1.5, "geo_fire": 1.0, "categorical": 0.5},
|
| 242 |
-
{"weather": 0.0, "geo_fire": 2.0, "categorical": 1.0},
|
| 243 |
-
{"weather": 2.0, "geo_fire": 0.5, "categorical": 0.5},
|
| 244 |
-
]
|
| 245 |
-
target_weight_settings = [(0.0, 1.0), (0.5, 0.25), (1.0, 0.25), (2.0, 0.10)]
|
| 246 |
-
scalar_settings = [("none", 0.0), ("ridge", 0.5), ("ridge", 1.0), ("ridge", 2.0), ("enet", 1.0), ("enet", 2.0)]
|
| 247 |
-
|
| 248 |
-
for bw in block_weight_grid:
|
| 249 |
-
base_train = np.concatenate([matrices[name][0] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1)
|
| 250 |
-
base_val = np.concatenate([matrices[name][1] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1)
|
| 251 |
-
base_test = np.concatenate([matrices[name][2] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1)
|
| 252 |
-
for power, floor in target_weight_settings:
|
| 253 |
-
tw_train, tw_val, tw_test = target_weight_vectors(
|
| 254 |
-
base_train,
|
| 255 |
-
base_val,
|
| 256 |
-
base_test,
|
| 257 |
-
train_df["target_log_burn_acres"].to_numpy(dtype=np.float64),
|
| 258 |
-
power=power,
|
| 259 |
-
floor=floor,
|
| 260 |
-
)
|
| 261 |
-
for scalar_model, scalar_weight in scalar_settings:
|
| 262 |
-
if scalar_model == "none":
|
| 263 |
-
train_vec, val_vec, test_vec = tw_train, tw_val, tw_test
|
| 264 |
-
else:
|
| 265 |
-
train_vec, val_vec, test_vec = append_supervised_scalar(
|
| 266 |
-
tw_train,
|
| 267 |
-
tw_val,
|
| 268 |
-
tw_test,
|
| 269 |
-
train_df,
|
| 270 |
-
scalar_model,
|
| 271 |
-
scalar_weight,
|
| 272 |
-
args.seed,
|
| 273 |
-
)
|
| 274 |
-
for k in [3, 5, 10, 15, 20]:
|
| 275 |
-
for mode in ["mean", "weighted"]:
|
| 276 |
-
val_metrics = score_vectors(val_vec, train_vec, val_df, train_df, k=k, mode=mode)
|
| 277 |
-
test_metrics = score_vectors(test_vec, train_vec, test_df, train_df, k=k, mode=mode)
|
| 278 |
-
row = {
|
| 279 |
-
"block_weights": bw,
|
| 280 |
-
"target_weight_power": power,
|
| 281 |
-
"target_weight_floor": floor,
|
| 282 |
-
"supervised_scalar": scalar_model,
|
| 283 |
-
"supervised_scalar_weight": scalar_weight,
|
| 284 |
-
"k": k,
|
| 285 |
-
"mode": mode,
|
| 286 |
-
"val_metrics": val_metrics,
|
| 287 |
-
"test_metrics": test_metrics,
|
| 288 |
-
}
|
| 289 |
-
candidate_rows.append(row)
|
| 290 |
-
score = float(val_metrics["ndcg_at_10"])
|
| 291 |
-
if best_score is None or score > best_score:
|
| 292 |
-
best_score = score
|
| 293 |
-
best = row
|
| 294 |
-
best_test = test_metrics
|
| 295 |
-
|
| 296 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 297 |
-
candidate_df = pd.DataFrame(
|
| 298 |
-
[
|
| 299 |
-
{
|
| 300 |
-
"val_ndcg_at_10": r["val_metrics"]["ndcg_at_10"],
|
| 301 |
-
"val_log_mae": r["val_metrics"]["log_mae"],
|
| 302 |
-
"test_ndcg_at_10": r["test_metrics"]["ndcg_at_10"],
|
| 303 |
-
"test_log_mae": r["test_metrics"]["log_mae"],
|
| 304 |
-
"k": r["k"],
|
| 305 |
-
"mode": r["mode"],
|
| 306 |
-
"target_weight_power": r["target_weight_power"],
|
| 307 |
-
"target_weight_floor": r["target_weight_floor"],
|
| 308 |
-
"supervised_scalar": r["supervised_scalar"],
|
| 309 |
-
"supervised_scalar_weight": r["supervised_scalar_weight"],
|
| 310 |
-
**{f"block_{k}": v for k, v in r["block_weights"].items()},
|
| 311 |
-
}
|
| 312 |
-
for r in candidate_rows
|
| 313 |
-
]
|
| 314 |
-
)
|
| 315 |
-
candidate_df.to_csv(args.output_dir / "candidate_grid.csv", index=False)
|
| 316 |
-
summary = {
|
| 317 |
-
"task_id": "wildfire_analog_retrieval_extended_hybrid_sweep",
|
| 318 |
-
"event_table": str(args.event_table),
|
| 319 |
-
"seed": int(args.seed),
|
| 320 |
-
"excluded_columns": sorted(exclude),
|
| 321 |
-
"split_sizes": {"train": len(train_df), "val": len(val_df), "test": len(test_df)},
|
| 322 |
-
"feature_blocks": blocks,
|
| 323 |
-
"selection_metric": "val_ndcg_at_10",
|
| 324 |
-
"selected_retrieval": best,
|
| 325 |
-
"test_metrics": best_test,
|
| 326 |
-
"candidate_count": len(candidate_rows),
|
| 327 |
-
}
|
| 328 |
-
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 329 |
-
print(json.dumps(summary, indent=2))
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
if __name__ == "__main__":
|
| 333 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py
DELETED
|
@@ -1,350 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import argparse
|
| 5 |
-
import json
|
| 6 |
-
import sys
|
| 7 |
-
from pathlib import Path
|
| 8 |
-
from typing import Dict, List, Tuple
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
import os
|
| 12 |
-
|
| 13 |
-
for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep):
|
| 14 |
-
if _p and _p not in sys.path:
|
| 15 |
-
sys.path.insert(0, _p)
|
| 16 |
-
|
| 17 |
-
import faiss
|
| 18 |
-
import hnswlib
|
| 19 |
-
import numpy as np
|
| 20 |
-
import pandas as pd
|
| 21 |
-
from sklearn.compose import ColumnTransformer
|
| 22 |
-
from sklearn.impute import SimpleImputer
|
| 23 |
-
from sklearn.metrics.pairwise import cosine_similarity
|
| 24 |
-
from sklearn.pipeline import Pipeline
|
| 25 |
-
from sklearn.preprocessing import OneHotEncoder, StandardScaler
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
DROP_COLUMNS = {
|
| 29 |
-
"Event_ID",
|
| 30 |
-
"Incid_Name",
|
| 31 |
-
"incident_name_norm",
|
| 32 |
-
"wfigs_name",
|
| 33 |
-
"Ig_Date",
|
| 34 |
-
"weather_date",
|
| 35 |
-
"BurnBndAc",
|
| 36 |
-
"target_log_burn_acres",
|
| 37 |
-
}
|
| 38 |
-
CATEGORICAL_COLUMNS = ["Incid_Type", "state_abbr", "county_name", "wfigs_match_type"]
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 42 |
-
return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2)))
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def mape(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 46 |
-
denom = np.clip(np.asarray(y_true, dtype=np.float64), 1e-6, None)
|
| 47 |
-
frac = np.abs(np.asarray(y_true, dtype=np.float64) - np.asarray(y_pred, dtype=np.float64)) / denom
|
| 48 |
-
return float(np.mean(frac))
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def r2_score_manual(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 52 |
-
y_true = np.asarray(y_true, dtype=np.float64)
|
| 53 |
-
y_pred = np.asarray(y_pred, dtype=np.float64)
|
| 54 |
-
ss_res = float(np.sum((y_true - y_pred) ** 2))
|
| 55 |
-
ss_tot = float(np.sum((y_true - y_true.mean()) ** 2))
|
| 56 |
-
return float(1.0 - ss_res / ss_tot) if ss_tot > 0 else 0.0
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 60 |
-
a = pd.Series(np.asarray(y_true, dtype=np.float64))
|
| 61 |
-
b = pd.Series(np.asarray(y_pred, dtype=np.float64))
|
| 62 |
-
value = a.corr(b, method="spearman")
|
| 63 |
-
return float(value) if pd.notna(value) else 0.0
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 67 |
-
ordered = df.sort_values("Ig_Date").reset_index(drop=True)
|
| 68 |
-
n = len(ordered)
|
| 69 |
-
train_end = max(int(round(n * 0.6)), 1)
|
| 70 |
-
val_end = max(int(round(n * 0.8)), train_end + 1)
|
| 71 |
-
val_end = min(val_end, n - 1) if n >= 3 else n
|
| 72 |
-
train = ordered.iloc[:train_end].copy()
|
| 73 |
-
val = ordered.iloc[train_end:val_end].copy()
|
| 74 |
-
test = ordered.iloc[val_end:].copy()
|
| 75 |
-
if len(val) == 0 and len(test) > 1:
|
| 76 |
-
val = test.iloc[:1].copy()
|
| 77 |
-
test = test.iloc[1:].copy()
|
| 78 |
-
return train, val, test
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def feature_columns(df: pd.DataFrame, feature_profile: str = "all") -> Tuple[List[str], List[str]]:
|
| 82 |
-
categorical = [c for c in CATEGORICAL_COLUMNS if c in df.columns]
|
| 83 |
-
numeric = []
|
| 84 |
-
for col in df.columns:
|
| 85 |
-
if col in DROP_COLUMNS or col in categorical:
|
| 86 |
-
continue
|
| 87 |
-
if pd.api.types.is_numeric_dtype(df[col]):
|
| 88 |
-
numeric.append(col)
|
| 89 |
-
if feature_profile == "weather_fm":
|
| 90 |
-
numeric = [c for c in numeric if c.startswith("weather_")]
|
| 91 |
-
categorical = []
|
| 92 |
-
return numeric, categorical
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def make_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer:
|
| 96 |
-
return ColumnTransformer(
|
| 97 |
-
transformers=[
|
| 98 |
-
(
|
| 99 |
-
"num",
|
| 100 |
-
Pipeline(
|
| 101 |
-
steps=[
|
| 102 |
-
("impute", SimpleImputer(strategy="median")),
|
| 103 |
-
("scale", StandardScaler()),
|
| 104 |
-
]
|
| 105 |
-
),
|
| 106 |
-
numeric_cols,
|
| 107 |
-
),
|
| 108 |
-
(
|
| 109 |
-
"cat",
|
| 110 |
-
Pipeline(
|
| 111 |
-
steps=[
|
| 112 |
-
("impute", SimpleImputer(strategy="most_frequent")),
|
| 113 |
-
("onehot", OneHotEncoder(handle_unknown="ignore")),
|
| 114 |
-
]
|
| 115 |
-
),
|
| 116 |
-
categorical_cols,
|
| 117 |
-
),
|
| 118 |
-
],
|
| 119 |
-
remainder="drop",
|
| 120 |
-
)
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
def to_dense_float32(x) -> np.ndarray:
|
| 124 |
-
if hasattr(x, "toarray"):
|
| 125 |
-
x = x.toarray()
|
| 126 |
-
return np.asarray(x, dtype=np.float32)
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
def weighted_prediction(sim: np.ndarray, targets: np.ndarray) -> float:
|
| 130 |
-
weights = np.maximum((np.asarray(sim, dtype=np.float64) + 1.0) / 2.0, 1e-6)
|
| 131 |
-
return float(np.sum(weights * targets) / np.sum(weights))
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
def graded_relevance(query_target: float, retrieved_targets: np.ndarray) -> np.ndarray:
|
| 135 |
-
delta = np.abs(np.asarray(retrieved_targets, dtype=np.float64) - float(query_target))
|
| 136 |
-
return np.select([delta <= 0.5, delta <= 1.0, delta <= 1.5], [3.0, 2.0, 1.0], default=0.0)
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
def dcg(relevance: np.ndarray) -> float:
|
| 140 |
-
rel = np.asarray(relevance, dtype=np.float64)
|
| 141 |
-
if rel.size == 0:
|
| 142 |
-
return 0.0
|
| 143 |
-
discounts = 1.0 / np.log2(np.arange(rel.size, dtype=np.float64) + 2.0)
|
| 144 |
-
return float(np.sum(rel * discounts))
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
def ndcg_at_k(relevance: np.ndarray, ideal_relevance: np.ndarray, k: int) -> float:
|
| 148 |
-
rel = np.asarray(relevance, dtype=np.float64)[:k]
|
| 149 |
-
ideal = np.asarray(ideal_relevance, dtype=np.float64)[:k]
|
| 150 |
-
denom = dcg(ideal)
|
| 151 |
-
return float(dcg(rel) / denom) if denom > 0 else 0.0
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
def score_backend(
|
| 155 |
-
name: str,
|
| 156 |
-
query_vec: np.ndarray,
|
| 157 |
-
library_vec: np.ndarray,
|
| 158 |
-
query_df: pd.DataFrame,
|
| 159 |
-
library_df: pd.DataFrame,
|
| 160 |
-
k: int,
|
| 161 |
-
mode: str,
|
| 162 |
-
) -> Tuple[Dict[str, float], pd.DataFrame]:
|
| 163 |
-
target_lib = library_df["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 164 |
-
rows = []
|
| 165 |
-
preds = []
|
| 166 |
-
ndcg5 = []
|
| 167 |
-
ndcg10 = []
|
| 168 |
-
hit1 = []
|
| 169 |
-
hit5 = []
|
| 170 |
-
hit10 = []
|
| 171 |
-
best_abs_delta = []
|
| 172 |
-
|
| 173 |
-
k_eff = min(int(k), int(library_vec.shape[0]))
|
| 174 |
-
if name == "cosine_exact":
|
| 175 |
-
sim_all = cosine_similarity(query_vec, library_vec)
|
| 176 |
-
knn_idx = np.argsort(-sim_all, axis=1)[:, :k_eff]
|
| 177 |
-
knn_sim = np.take_along_axis(sim_all, knn_idx, axis=1)
|
| 178 |
-
else:
|
| 179 |
-
library_norm = library_vec / np.clip(np.linalg.norm(library_vec, axis=1, keepdims=True), 1e-12, None)
|
| 180 |
-
query_norm = query_vec / np.clip(np.linalg.norm(query_vec, axis=1, keepdims=True), 1e-12, None)
|
| 181 |
-
if name == "faiss_flat_ip":
|
| 182 |
-
index = faiss.IndexFlatIP(library_norm.shape[1])
|
| 183 |
-
index.add(library_norm.astype(np.float32))
|
| 184 |
-
knn_sim, knn_idx = index.search(query_norm.astype(np.float32), k_eff)
|
| 185 |
-
elif name == "hnsw_cosine":
|
| 186 |
-
index = hnswlib.Index(space="cosine", dim=library_norm.shape[1])
|
| 187 |
-
index.init_index(max_elements=library_norm.shape[0], ef_construction=100, M=16)
|
| 188 |
-
index.add_items(library_norm.astype(np.float32), np.arange(library_norm.shape[0]))
|
| 189 |
-
index.set_ef(max(50, k_eff))
|
| 190 |
-
knn_idx, dist = index.knn_query(query_norm.astype(np.float32), k=k_eff)
|
| 191 |
-
knn_sim = 1.0 - dist
|
| 192 |
-
else:
|
| 193 |
-
raise ValueError(name)
|
| 194 |
-
|
| 195 |
-
for i in range(query_df.shape[0]):
|
| 196 |
-
order = knn_idx[i]
|
| 197 |
-
top_sim = knn_sim[i]
|
| 198 |
-
top_targets = target_lib[order]
|
| 199 |
-
query_target = float(query_df.iloc[i]["target_log_burn_acres"])
|
| 200 |
-
relevance = graded_relevance(query_target, top_targets)
|
| 201 |
-
ideal_relevance = np.sort(graded_relevance(query_target, target_lib))[::-1]
|
| 202 |
-
abs_delta = np.abs(top_targets - float(query_df.iloc[i]["target_log_burn_acres"]))
|
| 203 |
-
ndcg5.append(ndcg_at_k(relevance, ideal_relevance, 5))
|
| 204 |
-
ndcg10.append(ndcg_at_k(relevance, ideal_relevance, 10))
|
| 205 |
-
hit1.append(float(relevance[:1].max() >= 2.0))
|
| 206 |
-
hit5.append(float(relevance[: min(5, k_eff)].max() >= 2.0))
|
| 207 |
-
hit10.append(float(relevance[: min(10, k_eff)].max() >= 2.0))
|
| 208 |
-
best_abs_delta.append(float(abs_delta.min()))
|
| 209 |
-
pred = float(np.mean(top_targets)) if mode == "mean" else weighted_prediction(top_sim, top_targets)
|
| 210 |
-
preds.append(pred)
|
| 211 |
-
rows.append(
|
| 212 |
-
{
|
| 213 |
-
"query_event_id": query_df.iloc[i]["Event_ID"],
|
| 214 |
-
"true_log_burn_acres": float(query_df.iloc[i]["target_log_burn_acres"]),
|
| 215 |
-
"pred_log_burn_acres": pred,
|
| 216 |
-
"backend": name,
|
| 217 |
-
"k": k,
|
| 218 |
-
"effective_k": k_eff,
|
| 219 |
-
"mode": mode,
|
| 220 |
-
"top_relevance": relevance.tolist(),
|
| 221 |
-
"best_abs_log_delta": float(abs_delta.min()),
|
| 222 |
-
}
|
| 223 |
-
)
|
| 224 |
-
|
| 225 |
-
pred_arr = np.asarray(preds, dtype=np.float64)
|
| 226 |
-
true_log = query_df["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 227 |
-
true_acres = query_df["BurnBndAc"].to_numpy(dtype=np.float64)
|
| 228 |
-
pred_acres = np.exp(pred_arr)
|
| 229 |
-
metrics = {
|
| 230 |
-
"count": int(len(query_df)),
|
| 231 |
-
"log_mae": float(np.mean(np.abs(true_log - pred_arr))),
|
| 232 |
-
"log_rmse": rmse(true_log, pred_arr),
|
| 233 |
-
"log_r2": r2_score_manual(true_log, pred_arr),
|
| 234 |
-
"log_spearman": spearman_corr(true_log, pred_arr),
|
| 235 |
-
"log_median_ae": float(np.median(np.abs(true_log - pred_arr))),
|
| 236 |
-
"acres_mae": float(np.mean(np.abs(true_acres - pred_acres))),
|
| 237 |
-
"acres_rmse": rmse(true_acres, pred_acres),
|
| 238 |
-
"acres_median_ae": float(np.median(np.abs(true_acres - pred_acres))),
|
| 239 |
-
"acres_mape": mape(true_acres, pred_acres),
|
| 240 |
-
"ndcg_at_5": float(np.mean(ndcg5)) if ndcg5 else 0.0,
|
| 241 |
-
"ndcg_at_10": float(np.mean(ndcg10)) if ndcg10 else 0.0,
|
| 242 |
-
"hit_at_1_tol1": float(np.mean(hit1)) if hit1 else 0.0,
|
| 243 |
-
"hit_at_5_tol1": float(np.mean(hit5)) if hit5 else 0.0,
|
| 244 |
-
"hit_at_10_tol1": float(np.mean(hit10)) if hit10 else 0.0,
|
| 245 |
-
"mean_best_abs_log_delta_at_k": float(np.mean(best_abs_delta)) if best_abs_delta else 0.0,
|
| 246 |
-
}
|
| 247 |
-
return metrics, pd.DataFrame(rows)
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
def target_weight_vectors(train_vec: np.ndarray, val_vec: np.ndarray, test_vec: np.ndarray, target: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 251 |
-
x = np.asarray(train_vec, dtype=np.float64)
|
| 252 |
-
y = np.asarray(target, dtype=np.float64)
|
| 253 |
-
y = y - y.mean()
|
| 254 |
-
x_centered = x - x.mean(axis=0, keepdims=True)
|
| 255 |
-
denom = np.clip(np.sqrt(np.sum(x_centered**2, axis=0)) * np.sqrt(np.sum(y**2)), 1e-12, None)
|
| 256 |
-
corr = np.abs(np.sum(x_centered * y[:, None], axis=0) / denom)
|
| 257 |
-
corr = np.nan_to_num(corr, nan=0.0, posinf=0.0, neginf=0.0)
|
| 258 |
-
if float(corr.max()) > 0:
|
| 259 |
-
corr = corr / float(corr.max())
|
| 260 |
-
weights = (0.25 + corr).astype(np.float32)
|
| 261 |
-
return train_vec * weights, val_vec * weights, test_vec * weights
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
def main() -> None:
|
| 265 |
-
parser = argparse.ArgumentParser()
|
| 266 |
-
parser.add_argument("--event-table", type=Path, required=True)
|
| 267 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 268 |
-
parser.add_argument("--selection-metric", choices=("log_mae", "ndcg_at_10"), default="ndcg_at_10")
|
| 269 |
-
parser.add_argument("--feature-profile", choices=("all", "weather_fm"), default="all")
|
| 270 |
-
parser.add_argument("--fm-family", type=str, default="")
|
| 271 |
-
parser.add_argument("--seed", type=int, default=7)
|
| 272 |
-
args = parser.parse_args()
|
| 273 |
-
|
| 274 |
-
df = pd.read_csv(args.event_table)
|
| 275 |
-
df["Ig_Date"] = pd.to_datetime(df["Ig_Date"])
|
| 276 |
-
train_df, val_df, test_df = build_splits(df)
|
| 277 |
-
numeric_cols, categorical_cols = feature_columns(df, feature_profile=args.feature_profile)
|
| 278 |
-
if not numeric_cols and not categorical_cols:
|
| 279 |
-
raise SystemExit(f"No usable features found for profile={args.feature_profile}")
|
| 280 |
-
x_cols = numeric_cols + categorical_cols
|
| 281 |
-
pre = make_preprocessor(numeric_cols, categorical_cols)
|
| 282 |
-
train_vec = to_dense_float32(pre.fit_transform(train_df[x_cols]))
|
| 283 |
-
val_vec = to_dense_float32(pre.transform(val_df[x_cols]))
|
| 284 |
-
test_vec = to_dense_float32(pre.transform(test_df[x_cols]))
|
| 285 |
-
weighted_train_vec, weighted_val_vec, weighted_test_vec = target_weight_vectors(
|
| 286 |
-
train_vec,
|
| 287 |
-
val_vec,
|
| 288 |
-
test_vec,
|
| 289 |
-
train_df["target_log_burn_acres"].to_numpy(dtype=np.float64),
|
| 290 |
-
)
|
| 291 |
-
vector_variants = {
|
| 292 |
-
"standard": (train_vec, val_vec, test_vec),
|
| 293 |
-
"target_weighted": (weighted_train_vec, weighted_val_vec, weighted_test_vec),
|
| 294 |
-
}
|
| 295 |
-
|
| 296 |
-
candidate_validation: List[Dict[str, object]] = []
|
| 297 |
-
best = None
|
| 298 |
-
best_score = None
|
| 299 |
-
best_val_rows = None
|
| 300 |
-
best_test_rows = None
|
| 301 |
-
for variant, (lib_vec, v_vec, _) in vector_variants.items():
|
| 302 |
-
for backend in ["cosine_exact", "faiss_flat_ip", "hnsw_cosine"]:
|
| 303 |
-
for k in [1, 3, 5, 10]:
|
| 304 |
-
for mode in ["mean", "weighted"]:
|
| 305 |
-
val_metrics, val_rows = score_backend(backend, v_vec, lib_vec, val_df, train_df, k, mode)
|
| 306 |
-
candidate_validation.append({"variant": variant, "backend": backend, "k": k, "mode": mode, "val_metrics": val_metrics})
|
| 307 |
-
score = float(val_metrics[args.selection_metric])
|
| 308 |
-
better = score > best_score if args.selection_metric == "ndcg_at_10" and best_score is not None else score < best_score if best_score is not None else True
|
| 309 |
-
if better:
|
| 310 |
-
best_score = score
|
| 311 |
-
best = {"variant": variant, "backend": backend, "k": k, "mode": mode}
|
| 312 |
-
best_val_rows = val_rows
|
| 313 |
-
|
| 314 |
-
assert best is not None
|
| 315 |
-
best_train_vec, _, best_test_vec = vector_variants[str(best["variant"])]
|
| 316 |
-
test_metrics, test_rows = score_backend(best["backend"], best_test_vec, best_train_vec, test_df, train_df, int(best["k"]), str(best["mode"]))
|
| 317 |
-
best_test_rows = test_rows
|
| 318 |
-
|
| 319 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 320 |
-
if best_val_rows is not None:
|
| 321 |
-
best_val_rows.to_csv(args.output_dir / "val_retrieval_examples.csv", index=False)
|
| 322 |
-
if best_test_rows is not None:
|
| 323 |
-
best_test_rows.to_csv(args.output_dir / "test_retrieval_examples.csv", index=False)
|
| 324 |
-
|
| 325 |
-
summary = {
|
| 326 |
-
"task_id": "wildfire_analog_retrieval_taskmodels",
|
| 327 |
-
"task_form": "event_level_retrieval_with_induced_outcome_error",
|
| 328 |
-
"event_table": str(args.event_table),
|
| 329 |
-
"output_dir": str(args.output_dir),
|
| 330 |
-
"feature_profile": args.feature_profile,
|
| 331 |
-
"seed": int(args.seed),
|
| 332 |
-
"split_sizes": {
|
| 333 |
-
"train": int(len(train_df)),
|
| 334 |
-
"val": int(len(val_df)),
|
| 335 |
-
"test": int(len(test_df)),
|
| 336 |
-
},
|
| 337 |
-
"feature_columns": {"numeric": numeric_cols, "categorical": categorical_cols},
|
| 338 |
-
"candidate_validation": candidate_validation,
|
| 339 |
-
"selected_retrieval": best,
|
| 340 |
-
"selection_metric": args.selection_metric,
|
| 341 |
-
"test_metrics": test_metrics,
|
| 342 |
-
"model_family": "popular_open_source_retrieval_backends_with_train_only_target_weighting",
|
| 343 |
-
"fm_family": (args.fm_family or "weather_fm_derived_features") if args.feature_profile == "weather_fm" else None,
|
| 344 |
-
}
|
| 345 |
-
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 346 |
-
print(json.dumps(summary, indent=2))
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
if __name__ == "__main__":
|
| 350 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py
DELETED
|
@@ -1,344 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import argparse
|
| 5 |
-
import json
|
| 6 |
-
import math
|
| 7 |
-
import re
|
| 8 |
-
import sys
|
| 9 |
-
from pathlib import Path
|
| 10 |
-
from typing import Dict, Iterable, List, Optional, Tuple
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
import os
|
| 14 |
-
|
| 15 |
-
for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep):
|
| 16 |
-
if _p and _p not in sys.path:
|
| 17 |
-
sys.path.insert(0, _p)
|
| 18 |
-
|
| 19 |
-
import numpy as np
|
| 20 |
-
import pandas as pd
|
| 21 |
-
from catboost import CatBoostRegressor
|
| 22 |
-
from lightgbm import LGBMRegressor
|
| 23 |
-
from netCDF4 import Dataset
|
| 24 |
-
from sklearn.linear_model import ElasticNet, Ridge
|
| 25 |
-
from sklearn.metrics import mean_absolute_error, mean_squared_error
|
| 26 |
-
from xgboost import XGBRegressor
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
PRED_RE = re.compile(r"pred_(\d{8})_(\d{2})\.nc$")
|
| 30 |
-
WEATHER_VARS = ["T2M", "QV2M", "TQV", "U10M", "V10M", "TS"]
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def parse_args() -> argparse.Namespace:
|
| 34 |
-
parser = argparse.ArgumentParser()
|
| 35 |
-
parser.add_argument("--pred-root", type=Path, action="append", required=True)
|
| 36 |
-
parser.add_argument("--merra-root", type=Path, required=True)
|
| 37 |
-
parser.add_argument("--alphaearth-year-csv", type=Path, required=True)
|
| 38 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 39 |
-
parser.add_argument("--model-family", choices=("full", "lite"), default="full")
|
| 40 |
-
parser.add_argument("--alphaearth-prefix", type=str, default="alphaearth_")
|
| 41 |
-
parser.add_argument("--lat-min", type=float, default=24.0)
|
| 42 |
-
parser.add_argument("--lat-max", type=float, default=50.0)
|
| 43 |
-
parser.add_argument("--lon-min", type=float, default=-125.0)
|
| 44 |
-
parser.add_argument("--lon-max", type=float, default=-66.0)
|
| 45 |
-
parser.add_argument("--seed", type=int, default=7)
|
| 46 |
-
return parser.parse_args()
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
def choose_split(path: Path) -> Optional[str]:
|
| 50 |
-
name = path.name
|
| 51 |
-
if name == "output2022":
|
| 52 |
-
return "train"
|
| 53 |
-
if name == "output2024":
|
| 54 |
-
return "val"
|
| 55 |
-
if name == "output2025":
|
| 56 |
-
return "test"
|
| 57 |
-
return None
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def parse_pred_timestamp(path: Path) -> Tuple[str, int]:
|
| 61 |
-
match = PRED_RE.match(path.name)
|
| 62 |
-
if not match:
|
| 63 |
-
raise ValueError(f"Unexpected prediction filename: {path}")
|
| 64 |
-
return match.group(1), int(match.group(2))
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
def nearest_indices(src: np.ndarray, dst: np.ndarray) -> np.ndarray:
|
| 68 |
-
idx = np.searchsorted(dst, src)
|
| 69 |
-
idx = np.clip(idx, 0, len(dst) - 1)
|
| 70 |
-
prev_idx = np.clip(idx - 1, 0, len(dst) - 1)
|
| 71 |
-
choose_prev = np.abs(dst[prev_idx] - src) <= np.abs(dst[idx] - src)
|
| 72 |
-
return np.where(choose_prev, prev_idx, idx).astype(np.int64)
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
def build_grid_alignment(sample_pred: Path, sample_merra: Path, lat_min: float, lat_max: float, lon_min: float, lon_max: float) -> Dict[str, np.ndarray]:
|
| 76 |
-
with Dataset(sample_pred) as pred_ds, Dataset(sample_merra) as merra_ds:
|
| 77 |
-
pred_lat = np.asarray(pred_ds.variables["lat"][:], dtype=np.float64)
|
| 78 |
-
pred_lon = np.asarray(pred_ds.variables["lon"][:], dtype=np.float64)
|
| 79 |
-
merra_lat = np.asarray(merra_ds.variables["lat"][:], dtype=np.float64)
|
| 80 |
-
merra_lon = np.asarray(merra_ds.variables["lon"][:], dtype=np.float64)
|
| 81 |
-
|
| 82 |
-
lat_mask = (pred_lat >= lat_min) & (pred_lat <= lat_max)
|
| 83 |
-
lon_mask = (pred_lon >= lon_min) & (pred_lon <= lon_max)
|
| 84 |
-
pred_lat_idx = np.flatnonzero(lat_mask)
|
| 85 |
-
pred_lon_idx = np.flatnonzero(lon_mask)
|
| 86 |
-
pred_lat_sel = pred_lat[pred_lat_idx]
|
| 87 |
-
pred_lon_sel = pred_lon[pred_lon_idx]
|
| 88 |
-
merra_lat_idx = nearest_indices(pred_lat_sel, merra_lat)
|
| 89 |
-
merra_lon_idx = nearest_indices(pred_lon_sel, merra_lon)
|
| 90 |
-
|
| 91 |
-
return {
|
| 92 |
-
"pred_lat_idx": pred_lat_idx,
|
| 93 |
-
"pred_lon_idx": pred_lon_idx,
|
| 94 |
-
"merra_lat_idx": merra_lat_idx,
|
| 95 |
-
"merra_lon_idx": merra_lon_idx,
|
| 96 |
-
}
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def feature_stats(arr: np.ndarray) -> Dict[str, float]:
|
| 100 |
-
return {"mean": float(np.mean(arr)), "max": float(np.max(arr)), "std": float(np.std(arr))}
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
def build_rows(pred_roots: Iterable[Path], merra_root: Path, alignment: Dict[str, np.ndarray]) -> pd.DataFrame:
|
| 104 |
-
rows: List[Dict[str, float]] = []
|
| 105 |
-
for root in pred_roots:
|
| 106 |
-
split = choose_split(root)
|
| 107 |
-
if split is None:
|
| 108 |
-
continue
|
| 109 |
-
for path in sorted(root.glob("pred_*.nc")):
|
| 110 |
-
day, hour = parse_pred_timestamp(path)
|
| 111 |
-
if hour % 3 != 0:
|
| 112 |
-
continue
|
| 113 |
-
merra_path = merra_root / f"MERRA2_sfc_{day}.nc"
|
| 114 |
-
if not merra_path.exists():
|
| 115 |
-
continue
|
| 116 |
-
time_index = hour // 3
|
| 117 |
-
with Dataset(path) as pred_ds, Dataset(merra_path) as merra_ds:
|
| 118 |
-
date = pd.Timestamp(day)
|
| 119 |
-
record: Dict[str, float] = {"split": split, "hour": float(hour), "year": float(date.year), "date": day}
|
| 120 |
-
record["doy"] = float(date.dayofyear)
|
| 121 |
-
record["month"] = float(date.month)
|
| 122 |
-
for var in WEATHER_VARS:
|
| 123 |
-
pred_arr = np.asarray(
|
| 124 |
-
pred_ds.variables[var][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]],
|
| 125 |
-
dtype=np.float64,
|
| 126 |
-
)
|
| 127 |
-
stats = feature_stats(pred_arr)
|
| 128 |
-
record[f"pred_{var.lower()}_mean"] = stats["mean"]
|
| 129 |
-
record[f"pred_{var.lower()}_max"] = stats["max"]
|
| 130 |
-
record[f"pred_{var.lower()}_std"] = stats["std"]
|
| 131 |
-
record["pred_wind_mean"] = float(
|
| 132 |
-
np.mean(
|
| 133 |
-
np.sqrt(
|
| 134 |
-
np.square(pred_ds.variables["U10M"][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]])
|
| 135 |
-
+ np.square(pred_ds.variables["V10M"][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]])
|
| 136 |
-
)
|
| 137 |
-
)
|
| 138 |
-
)
|
| 139 |
-
truth_t2m = np.asarray(merra_ds.variables["T2M"][time_index], dtype=np.float64)[
|
| 140 |
-
np.ix_(alignment["merra_lat_idx"], alignment["merra_lon_idx"])
|
| 141 |
-
]
|
| 142 |
-
truth_ts = np.asarray(merra_ds.variables["TS"][time_index], dtype=np.float64)[
|
| 143 |
-
np.ix_(alignment["merra_lat_idx"], alignment["merra_lon_idx"])
|
| 144 |
-
]
|
| 145 |
-
record["target_t2m_mean_c"] = float(np.mean(truth_t2m) - 273.15)
|
| 146 |
-
record["target_t2m_max_c"] = float(np.max(truth_t2m) - 273.15)
|
| 147 |
-
record["target_ts_mean_c"] = float(np.mean(truth_ts) - 273.15)
|
| 148 |
-
rows.append(record)
|
| 149 |
-
|
| 150 |
-
if not rows:
|
| 151 |
-
raise SystemExit("No extreme-heat rows were built from the provided roots.")
|
| 152 |
-
df = pd.DataFrame(rows)
|
| 153 |
-
angle_day = 2.0 * np.pi * df["doy"].to_numpy(dtype=np.float64) / 366.0
|
| 154 |
-
angle_hour = 2.0 * np.pi * df["hour"].to_numpy(dtype=np.float64) / 24.0
|
| 155 |
-
df["doy_sin"] = np.sin(angle_day)
|
| 156 |
-
df["doy_cos"] = np.cos(angle_day)
|
| 157 |
-
df["hour_sin"] = np.sin(angle_hour)
|
| 158 |
-
df["hour_cos"] = np.cos(angle_hour)
|
| 159 |
-
return df
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
def drop_nonfinite_rows(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame:
|
| 163 |
-
mask = np.ones(len(df), dtype=bool)
|
| 164 |
-
for col in columns:
|
| 165 |
-
mask &= np.isfinite(pd.to_numeric(df[col], errors="coerce").to_numpy(dtype=np.float64))
|
| 166 |
-
return df.loc[mask].reset_index(drop=True)
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 170 |
-
return float(math.sqrt(mean_squared_error(y_true, y_pred)))
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 174 |
-
a = np.asarray(y_true, dtype=np.float64)
|
| 175 |
-
b = np.asarray(y_pred, dtype=np.float64)
|
| 176 |
-
if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]):
|
| 177 |
-
return 0.0
|
| 178 |
-
value = float(np.corrcoef(a, b)[0, 1])
|
| 179 |
-
return value if np.isfinite(value) else 0.0
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]:
|
| 183 |
-
truth = np.asarray(y_true >= threshold)
|
| 184 |
-
pred = np.asarray(y_pred >= threshold)
|
| 185 |
-
tp = int(np.logical_and(pred, truth).sum())
|
| 186 |
-
fp = int(np.logical_and(pred, ~truth).sum())
|
| 187 |
-
fn = int(np.logical_and(~pred, truth).sum())
|
| 188 |
-
precision = float(tp / (tp + fp)) if (tp + fp) else 0.0
|
| 189 |
-
recall = float(tp / (tp + fn)) if (tp + fn) else 0.0
|
| 190 |
-
f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0
|
| 191 |
-
return {"precision": precision, "recall": recall, "f1": f1}
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
def evaluate(y_true: np.ndarray, y_pred: np.ndarray) -> Dict[str, float]:
|
| 195 |
-
return {
|
| 196 |
-
"count": int(y_true.shape[0]),
|
| 197 |
-
"rmse_c": rmse(y_true, y_pred),
|
| 198 |
-
"mae_c": float(mean_absolute_error(y_true, y_pred)),
|
| 199 |
-
"pearson_r": pearson_corr(y_true, y_pred),
|
| 200 |
-
}
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
def main() -> None:
|
| 204 |
-
args = parse_args()
|
| 205 |
-
pred_files = [path for root in args.pred_root for path in root.glob("pred_*.nc")]
|
| 206 |
-
if not pred_files:
|
| 207 |
-
raise SystemExit("No prediction files found.")
|
| 208 |
-
sample_pred = sorted(pred_files)[0]
|
| 209 |
-
sample_day, _ = parse_pred_timestamp(sample_pred)
|
| 210 |
-
sample_merra = args.merra_root / f"MERRA2_sfc_{sample_day}.nc"
|
| 211 |
-
if not sample_merra.exists():
|
| 212 |
-
raise SystemExit(f"Sample MERRA file missing: {sample_merra}")
|
| 213 |
-
|
| 214 |
-
alignment = build_grid_alignment(
|
| 215 |
-
sample_pred,
|
| 216 |
-
sample_merra,
|
| 217 |
-
lat_min=args.lat_min,
|
| 218 |
-
lat_max=args.lat_max,
|
| 219 |
-
lon_min=args.lon_min,
|
| 220 |
-
lon_max=args.lon_max,
|
| 221 |
-
)
|
| 222 |
-
df = build_rows(args.pred_root, args.merra_root, alignment)
|
| 223 |
-
|
| 224 |
-
alpha = pd.read_csv(args.alphaearth_year_csv)
|
| 225 |
-
alpha["source_year_key"] = pd.to_numeric(alpha["alphaearth_source_year"], errors="coerce").astype("Int64")
|
| 226 |
-
df["source_year_key"] = pd.to_numeric(df["year"], errors="coerce").clip(lower=2017, upper=2024).astype("Int64")
|
| 227 |
-
df = df.merge(alpha.drop(columns=[c for c in ["requested_year"] if c in alpha.columns]), on="source_year_key", how="left")
|
| 228 |
-
|
| 229 |
-
feature_cols = [c for c in df.columns if c.startswith("pred_") or c in {"month", "doy_sin", "doy_cos", "hour_sin", "hour_cos"}]
|
| 230 |
-
feature_cols.extend(sorted([c for c in df.columns if c.startswith(args.alphaearth_prefix)]))
|
| 231 |
-
finite_cols = feature_cols + ["target_t2m_mean_c"]
|
| 232 |
-
df = drop_nonfinite_rows(df, finite_cols)
|
| 233 |
-
if df.empty:
|
| 234 |
-
raise SystemExit("Extreme-heat AlphaEarth suite has no finite rows after filtering.")
|
| 235 |
-
|
| 236 |
-
train = df[df["split"] == "train"].copy()
|
| 237 |
-
val = df[df["split"] == "val"].copy()
|
| 238 |
-
test = df[df["split"] == "test"].copy()
|
| 239 |
-
if len(train) == 0 or len(val) == 0 or len(test) == 0:
|
| 240 |
-
raise SystemExit("Extreme-heat AlphaEarth suite is missing one of train/val/test.")
|
| 241 |
-
|
| 242 |
-
x_train = train[feature_cols].to_numpy(dtype=np.float64)
|
| 243 |
-
x_val = val[feature_cols].to_numpy(dtype=np.float64)
|
| 244 |
-
x_test = test[feature_cols].to_numpy(dtype=np.float64)
|
| 245 |
-
y_train = train["target_t2m_mean_c"].to_numpy(dtype=np.float64)
|
| 246 |
-
y_val = val["target_t2m_mean_c"].to_numpy(dtype=np.float64)
|
| 247 |
-
y_test = test["target_t2m_mean_c"].to_numpy(dtype=np.float64)
|
| 248 |
-
|
| 249 |
-
candidates: Dict[str, object] = {
|
| 250 |
-
"ridge": Ridge(alpha=1.0, random_state=args.seed),
|
| 251 |
-
"enet": ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000),
|
| 252 |
-
}
|
| 253 |
-
if args.model_family == "full":
|
| 254 |
-
candidates.update(
|
| 255 |
-
{
|
| 256 |
-
"xgboost": XGBRegressor(
|
| 257 |
-
n_estimators=300,
|
| 258 |
-
max_depth=6,
|
| 259 |
-
learning_rate=0.05,
|
| 260 |
-
subsample=0.8,
|
| 261 |
-
colsample_bytree=0.8,
|
| 262 |
-
objective="reg:squarederror",
|
| 263 |
-
tree_method="hist",
|
| 264 |
-
random_state=args.seed,
|
| 265 |
-
n_jobs=8,
|
| 266 |
-
),
|
| 267 |
-
"lightgbm": LGBMRegressor(
|
| 268 |
-
n_estimators=300,
|
| 269 |
-
learning_rate=0.05,
|
| 270 |
-
num_leaves=63,
|
| 271 |
-
subsample=0.8,
|
| 272 |
-
colsample_bytree=0.8,
|
| 273 |
-
random_state=args.seed,
|
| 274 |
-
n_jobs=8,
|
| 275 |
-
verbose=-1,
|
| 276 |
-
),
|
| 277 |
-
"catboost": CatBoostRegressor(
|
| 278 |
-
iterations=400,
|
| 279 |
-
depth=8,
|
| 280 |
-
learning_rate=0.05,
|
| 281 |
-
loss_function="RMSE",
|
| 282 |
-
eval_metric="RMSE",
|
| 283 |
-
random_seed=args.seed,
|
| 284 |
-
verbose=False,
|
| 285 |
-
),
|
| 286 |
-
}
|
| 287 |
-
)
|
| 288 |
-
|
| 289 |
-
candidate_rows = []
|
| 290 |
-
best_name = None
|
| 291 |
-
best_model = None
|
| 292 |
-
best_rmse = None
|
| 293 |
-
for name, model in candidates.items():
|
| 294 |
-
model.fit(x_train, y_train)
|
| 295 |
-
val_pred = np.asarray(model.predict(x_val), dtype=np.float64)
|
| 296 |
-
val_metrics = evaluate(y_val, val_pred)
|
| 297 |
-
candidate_rows.append({"model": name, "validation": val_metrics})
|
| 298 |
-
if best_rmse is None or val_metrics["rmse_c"] < best_rmse:
|
| 299 |
-
best_name = name
|
| 300 |
-
best_model = model
|
| 301 |
-
best_rmse = val_metrics["rmse_c"]
|
| 302 |
-
|
| 303 |
-
assert best_model is not None and best_name is not None
|
| 304 |
-
val_pred = np.asarray(best_model.predict(x_val), dtype=np.float64)
|
| 305 |
-
test_pred = np.asarray(best_model.predict(x_test), dtype=np.float64)
|
| 306 |
-
|
| 307 |
-
thresholds = [27.0, 30.0, 33.0]
|
| 308 |
-
val_events = [{"threshold_c": t, **prf(y_val, val_pred, t)} for t in thresholds]
|
| 309 |
-
val_events = sorted(val_events, key=lambda row: (-row["f1"], -row["recall"], -row["precision"], row["threshold_c"]))
|
| 310 |
-
selected_event = val_events[0]
|
| 311 |
-
test_event = {"threshold_c": selected_event["threshold_c"], **prf(y_test, test_pred, selected_event["threshold_c"])}
|
| 312 |
-
|
| 313 |
-
summary = {
|
| 314 |
-
"task_id": "extreme_heat_alphaearth",
|
| 315 |
-
"core_line": "extreme_heat",
|
| 316 |
-
"task_form": "continuous_temperature_forecast_with_secondary_exceedance_view",
|
| 317 |
-
"seed": int(args.seed),
|
| 318 |
-
"pred_roots": [str(path) for path in args.pred_root],
|
| 319 |
-
"merra_root": str(args.merra_root),
|
| 320 |
-
"alphaearth_year_csv": str(args.alphaearth_year_csv),
|
| 321 |
-
"model_family": args.model_family,
|
| 322 |
-
"feature_columns": feature_cols,
|
| 323 |
-
"alphaearth_feature_count": int(sum(c.startswith(args.alphaearth_prefix) for c in feature_cols)),
|
| 324 |
-
"candidate_validation": candidate_rows,
|
| 325 |
-
"selected_model": best_name,
|
| 326 |
-
"validation_metrics": evaluate(y_val, val_pred),
|
| 327 |
-
"test_metrics": evaluate(y_test, test_pred),
|
| 328 |
-
"selected_event_candidate": selected_event,
|
| 329 |
-
"selected_event_candidate_test": test_event,
|
| 330 |
-
"selection_rule": "same heat benchmark; choose regressor by validation RMSE and choose exceedance threshold by validation F1",
|
| 331 |
-
"tmt_policy": {
|
| 332 |
-
"task": "extreme_heat",
|
| 333 |
-
"metric": "continuous RMSE/MAE with thresholded exceedance as a secondary event policy",
|
| 334 |
-
"tolerance": "none for continuous headline; event threshold only for operational view"
|
| 335 |
-
},
|
| 336 |
-
}
|
| 337 |
-
|
| 338 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 339 |
-
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 340 |
-
print(json.dumps(summary, indent=2))
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
if __name__ == "__main__":
|
| 344 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py
DELETED
|
@@ -1,353 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import argparse
|
| 5 |
-
import json
|
| 6 |
-
import sys
|
| 7 |
-
from pathlib import Path
|
| 8 |
-
from typing import Dict, List, Tuple
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
import os
|
| 12 |
-
|
| 13 |
-
for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep):
|
| 14 |
-
if _p and _p not in sys.path:
|
| 15 |
-
sys.path.insert(0, _p)
|
| 16 |
-
|
| 17 |
-
import numpy as np
|
| 18 |
-
import pandas as pd
|
| 19 |
-
from catboost import CatBoostRegressor
|
| 20 |
-
from lightgbm import LGBMRegressor
|
| 21 |
-
from sklearn.compose import ColumnTransformer
|
| 22 |
-
from sklearn.impute import SimpleImputer
|
| 23 |
-
from sklearn.linear_model import ElasticNet
|
| 24 |
-
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
|
| 25 |
-
from sklearn.pipeline import Pipeline
|
| 26 |
-
from sklearn.preprocessing import OneHotEncoder, StandardScaler
|
| 27 |
-
from xgboost import XGBRegressor
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
DROP_COLUMNS = {
|
| 31 |
-
"Event_ID",
|
| 32 |
-
"Incid_Name",
|
| 33 |
-
"incident_name_norm",
|
| 34 |
-
"wfigs_name",
|
| 35 |
-
"Ig_Date",
|
| 36 |
-
"weather_date",
|
| 37 |
-
"BurnBndAc",
|
| 38 |
-
"target_log_burn_acres",
|
| 39 |
-
}
|
| 40 |
-
|
| 41 |
-
CATEGORICAL_COLUMNS = [
|
| 42 |
-
"Incid_Type",
|
| 43 |
-
"state_abbr",
|
| 44 |
-
"county_name",
|
| 45 |
-
"wfigs_match_type",
|
| 46 |
-
]
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 50 |
-
return float(np.sqrt(mean_squared_error(y_true, y_pred)))
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def mape(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 54 |
-
denom = np.clip(np.asarray(y_true, dtype=np.float64), 1e-6, None)
|
| 55 |
-
frac = np.abs(np.asarray(y_true, dtype=np.float64) - np.asarray(y_pred, dtype=np.float64)) / denom
|
| 56 |
-
return float(np.mean(frac))
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 60 |
-
a = pd.Series(np.asarray(y_true, dtype=np.float64))
|
| 61 |
-
b = pd.Series(np.asarray(y_pred, dtype=np.float64))
|
| 62 |
-
value = a.corr(b, method="spearman")
|
| 63 |
-
return float(value) if pd.notna(value) else 0.0
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 67 |
-
ordered = df.sort_values("Ig_Date").reset_index(drop=True)
|
| 68 |
-
n = len(ordered)
|
| 69 |
-
train_end = max(int(round(n * 0.6)), 1)
|
| 70 |
-
val_end = max(int(round(n * 0.8)), train_end + 1)
|
| 71 |
-
val_end = min(val_end, n - 1) if n >= 3 else n
|
| 72 |
-
train = ordered.iloc[:train_end].copy()
|
| 73 |
-
val = ordered.iloc[train_end:val_end].copy()
|
| 74 |
-
test = ordered.iloc[val_end:].copy()
|
| 75 |
-
if len(val) == 0 and len(test) > 1:
|
| 76 |
-
val = test.iloc[:1].copy()
|
| 77 |
-
test = test.iloc[1:].copy()
|
| 78 |
-
return train, val, test
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def feature_columns(df: pd.DataFrame, feature_profile: str = "all") -> Tuple[List[str], List[str]]:
|
| 82 |
-
categorical = [c for c in CATEGORICAL_COLUMNS if c in df.columns]
|
| 83 |
-
numeric = []
|
| 84 |
-
for col in df.columns:
|
| 85 |
-
if col in DROP_COLUMNS or col in categorical:
|
| 86 |
-
continue
|
| 87 |
-
if pd.api.types.is_numeric_dtype(df[col]):
|
| 88 |
-
numeric.append(col)
|
| 89 |
-
if feature_profile == "weather_fm":
|
| 90 |
-
numeric = [c for c in numeric if c.startswith("weather_")]
|
| 91 |
-
categorical = []
|
| 92 |
-
return numeric, categorical
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def make_sparse_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer:
|
| 96 |
-
return ColumnTransformer(
|
| 97 |
-
transformers=[
|
| 98 |
-
(
|
| 99 |
-
"num",
|
| 100 |
-
Pipeline(
|
| 101 |
-
steps=[
|
| 102 |
-
("impute", SimpleImputer(strategy="median")),
|
| 103 |
-
("scale", StandardScaler()),
|
| 104 |
-
]
|
| 105 |
-
),
|
| 106 |
-
numeric_cols,
|
| 107 |
-
),
|
| 108 |
-
(
|
| 109 |
-
"cat",
|
| 110 |
-
Pipeline(
|
| 111 |
-
steps=[
|
| 112 |
-
("impute", SimpleImputer(strategy="most_frequent")),
|
| 113 |
-
("onehot", OneHotEncoder(handle_unknown="ignore")),
|
| 114 |
-
]
|
| 115 |
-
),
|
| 116 |
-
categorical_cols,
|
| 117 |
-
),
|
| 118 |
-
],
|
| 119 |
-
remainder="drop",
|
| 120 |
-
)
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
def prepare_catboost_frames(
|
| 124 |
-
train_df: pd.DataFrame,
|
| 125 |
-
val_df: pd.DataFrame,
|
| 126 |
-
test_df: pd.DataFrame,
|
| 127 |
-
numeric_cols: List[str],
|
| 128 |
-
categorical_cols: List[str],
|
| 129 |
-
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 130 |
-
medians = {c: float(train_df[c].median()) for c in numeric_cols}
|
| 131 |
-
modes = {
|
| 132 |
-
c: str(train_df[c].mode(dropna=True).iloc[0]) if not train_df[c].mode(dropna=True).empty else "missing"
|
| 133 |
-
for c in categorical_cols
|
| 134 |
-
}
|
| 135 |
-
|
| 136 |
-
def _prep(frame: pd.DataFrame) -> pd.DataFrame:
|
| 137 |
-
out = frame[numeric_cols + categorical_cols].copy()
|
| 138 |
-
for col in numeric_cols:
|
| 139 |
-
out[col] = pd.to_numeric(out[col], errors="coerce").fillna(medians[col])
|
| 140 |
-
for col in categorical_cols:
|
| 141 |
-
out[col] = out[col].astype("string").fillna(modes[col]).astype(str)
|
| 142 |
-
return out
|
| 143 |
-
|
| 144 |
-
return _prep(train_df), _prep(val_df), _prep(test_df)
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
def evaluate_split(frame: pd.DataFrame, pred_log: np.ndarray) -> Dict[str, float]:
|
| 148 |
-
true_log = frame["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 149 |
-
true_acres = frame["BurnBndAc"].to_numpy(dtype=np.float64)
|
| 150 |
-
pred_log = np.asarray(pred_log, dtype=np.float64)
|
| 151 |
-
pred_acres = np.exp(pred_log)
|
| 152 |
-
return {
|
| 153 |
-
"count": int(len(frame)),
|
| 154 |
-
"log_mae": float(mean_absolute_error(true_log, pred_log)),
|
| 155 |
-
"log_rmse": rmse(true_log, pred_log),
|
| 156 |
-
"log_r2": float(r2_score(true_log, pred_log)) if len(frame) > 1 else 0.0,
|
| 157 |
-
"log_spearman": spearman_corr(true_log, pred_log),
|
| 158 |
-
"log_median_ae": float(np.median(np.abs(true_log - pred_log))),
|
| 159 |
-
"acres_mae": float(mean_absolute_error(true_acres, pred_acres)),
|
| 160 |
-
"acres_rmse": rmse(true_acres, pred_acres),
|
| 161 |
-
"acres_median_ae": float(np.median(np.abs(true_acres - pred_acres))),
|
| 162 |
-
"acres_mape": mape(true_acres, pred_acres),
|
| 163 |
-
}
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
def main() -> None:
|
| 167 |
-
parser = argparse.ArgumentParser()
|
| 168 |
-
parser.add_argument("--event-table", type=Path, required=True)
|
| 169 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 170 |
-
parser.add_argument("--feature-profile", choices=("all", "weather_fm"), default="all")
|
| 171 |
-
parser.add_argument("--model-family", choices=("full", "lite"), default="full")
|
| 172 |
-
parser.add_argument("--fm-family", type=str, default="")
|
| 173 |
-
parser.add_argument("--seed", type=int, default=7)
|
| 174 |
-
args = parser.parse_args()
|
| 175 |
-
|
| 176 |
-
df = pd.read_csv(args.event_table)
|
| 177 |
-
df["Ig_Date"] = pd.to_datetime(df["Ig_Date"])
|
| 178 |
-
train_df, val_df, test_df = build_splits(df)
|
| 179 |
-
numeric_cols, categorical_cols = feature_columns(df, feature_profile=args.feature_profile)
|
| 180 |
-
if not numeric_cols and not categorical_cols:
|
| 181 |
-
raise SystemExit(f"No usable features found for profile={args.feature_profile}")
|
| 182 |
-
x_cols = numeric_cols + categorical_cols
|
| 183 |
-
|
| 184 |
-
pre = make_sparse_preprocessor(numeric_cols, categorical_cols)
|
| 185 |
-
x_train = pre.fit_transform(train_df[x_cols])
|
| 186 |
-
x_val = pre.transform(val_df[x_cols])
|
| 187 |
-
x_test = pre.transform(test_df[x_cols])
|
| 188 |
-
y_train = train_df["target_log_burn_acres"].to_numpy(dtype=np.float64)
|
| 189 |
-
|
| 190 |
-
cat_train, cat_val, cat_test = prepare_catboost_frames(train_df, val_df, test_df, numeric_cols, categorical_cols)
|
| 191 |
-
cat_feature_idx = list(range(len(numeric_cols), len(numeric_cols) + len(categorical_cols)))
|
| 192 |
-
|
| 193 |
-
candidates: List[Tuple[str, object, str]] = [
|
| 194 |
-
(
|
| 195 |
-
"enet",
|
| 196 |
-
ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000),
|
| 197 |
-
"sparse",
|
| 198 |
-
),
|
| 199 |
-
]
|
| 200 |
-
if args.model_family == "full":
|
| 201 |
-
candidates.extend(
|
| 202 |
-
[
|
| 203 |
-
(
|
| 204 |
-
"xgboost",
|
| 205 |
-
XGBRegressor(
|
| 206 |
-
n_estimators=400,
|
| 207 |
-
max_depth=6,
|
| 208 |
-
learning_rate=0.05,
|
| 209 |
-
subsample=0.8,
|
| 210 |
-
colsample_bytree=0.8,
|
| 211 |
-
reg_lambda=1.0,
|
| 212 |
-
objective="reg:squarederror",
|
| 213 |
-
tree_method="hist",
|
| 214 |
-
random_state=args.seed,
|
| 215 |
-
n_jobs=8,
|
| 216 |
-
),
|
| 217 |
-
"sparse",
|
| 218 |
-
),
|
| 219 |
-
(
|
| 220 |
-
"lightgbm",
|
| 221 |
-
LGBMRegressor(
|
| 222 |
-
n_estimators=400,
|
| 223 |
-
learning_rate=0.05,
|
| 224 |
-
num_leaves=63,
|
| 225 |
-
subsample=0.8,
|
| 226 |
-
colsample_bytree=0.8,
|
| 227 |
-
reg_lambda=1.0,
|
| 228 |
-
random_state=args.seed,
|
| 229 |
-
n_jobs=8,
|
| 230 |
-
verbose=-1,
|
| 231 |
-
),
|
| 232 |
-
"sparse",
|
| 233 |
-
),
|
| 234 |
-
(
|
| 235 |
-
"catboost",
|
| 236 |
-
CatBoostRegressor(
|
| 237 |
-
iterations=500,
|
| 238 |
-
depth=8,
|
| 239 |
-
learning_rate=0.05,
|
| 240 |
-
loss_function="RMSE",
|
| 241 |
-
eval_metric="RMSE",
|
| 242 |
-
random_seed=args.seed,
|
| 243 |
-
verbose=False,
|
| 244 |
-
),
|
| 245 |
-
"cat",
|
| 246 |
-
),
|
| 247 |
-
]
|
| 248 |
-
)
|
| 249 |
-
|
| 250 |
-
candidate_validation: List[Dict[str, object]] = []
|
| 251 |
-
best_name = None
|
| 252 |
-
best_kind = None
|
| 253 |
-
best_model = None
|
| 254 |
-
best_score = None
|
| 255 |
-
|
| 256 |
-
for name, model, kind in candidates:
|
| 257 |
-
if kind == "sparse":
|
| 258 |
-
model.fit(x_train, y_train)
|
| 259 |
-
val_pred = model.predict(x_val)
|
| 260 |
-
else:
|
| 261 |
-
model.fit(cat_train, y_train, cat_features=cat_feature_idx, eval_set=(cat_val, val_df["target_log_burn_acres"]), use_best_model=False)
|
| 262 |
-
val_pred = model.predict(cat_val)
|
| 263 |
-
val_metrics = evaluate_split(val_df, val_pred)
|
| 264 |
-
candidate_validation.append({"model_name": name, "val_metrics": val_metrics})
|
| 265 |
-
score = float(val_metrics["log_mae"])
|
| 266 |
-
if best_score is None or score < best_score:
|
| 267 |
-
best_score = score
|
| 268 |
-
best_name = name
|
| 269 |
-
best_kind = kind
|
| 270 |
-
best_model = model
|
| 271 |
-
|
| 272 |
-
assert best_model is not None and best_name is not None and best_kind is not None
|
| 273 |
-
|
| 274 |
-
combined_train = pd.concat([train_df, val_df], ignore_index=True)
|
| 275 |
-
if best_kind == "sparse":
|
| 276 |
-
x_combined = pre.fit_transform(combined_train[x_cols])
|
| 277 |
-
x_train_final = pre.transform(train_df[x_cols])
|
| 278 |
-
x_val_final = pre.transform(val_df[x_cols])
|
| 279 |
-
x_test_final = pre.transform(test_df[x_cols])
|
| 280 |
-
best_model.fit(x_combined, combined_train["target_log_burn_acres"].to_numpy(dtype=np.float64))
|
| 281 |
-
train_pred = best_model.predict(x_train_final)
|
| 282 |
-
val_pred = best_model.predict(x_val_final)
|
| 283 |
-
test_pred = best_model.predict(x_test_final)
|
| 284 |
-
else:
|
| 285 |
-
cat_combined, cat_train_final, cat_test_final = prepare_catboost_frames(
|
| 286 |
-
combined_train, train_df, test_df, numeric_cols, categorical_cols
|
| 287 |
-
)
|
| 288 |
-
cat_val_final = prepare_catboost_frames(val_df, val_df, val_df, numeric_cols, categorical_cols)[0]
|
| 289 |
-
best_model.fit(
|
| 290 |
-
cat_combined,
|
| 291 |
-
combined_train["target_log_burn_acres"].to_numpy(dtype=np.float64),
|
| 292 |
-
cat_features=cat_feature_idx,
|
| 293 |
-
use_best_model=False,
|
| 294 |
-
)
|
| 295 |
-
train_pred = best_model.predict(cat_train_final)
|
| 296 |
-
val_pred = best_model.predict(cat_val_final)
|
| 297 |
-
test_pred = best_model.predict(cat_test_final)
|
| 298 |
-
|
| 299 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 300 |
-
pred_df = pd.concat(
|
| 301 |
-
[
|
| 302 |
-
train_df.assign(split="train", pred_log_burn_acres=train_pred, pred_burn_acres=np.exp(train_pred)),
|
| 303 |
-
val_df.assign(split="val", pred_log_burn_acres=val_pred, pred_burn_acres=np.exp(val_pred)),
|
| 304 |
-
test_df.assign(split="test", pred_log_burn_acres=test_pred, pred_burn_acres=np.exp(test_pred)),
|
| 305 |
-
],
|
| 306 |
-
axis=0,
|
| 307 |
-
ignore_index=True,
|
| 308 |
-
)
|
| 309 |
-
pred_path = args.output_dir / "predictions.csv"
|
| 310 |
-
pred_df.to_csv(pred_path, index=False)
|
| 311 |
-
|
| 312 |
-
summary = {
|
| 313 |
-
"task_id": "wildfire_final_area_scalar_taskmodels",
|
| 314 |
-
"task_form": "event_level_regression",
|
| 315 |
-
"event_table": str(args.event_table),
|
| 316 |
-
"output_dir": str(args.output_dir),
|
| 317 |
-
"feature_profile": args.feature_profile,
|
| 318 |
-
"seed": int(args.seed),
|
| 319 |
-
"benchmark_protocol": "fm_lite_protocol" if args.feature_profile == "weather_fm" and args.model_family == "lite" else "standard_protocol",
|
| 320 |
-
"split_sizes": {
|
| 321 |
-
"train": int(len(train_df)),
|
| 322 |
-
"val": int(len(val_df)),
|
| 323 |
-
"test": int(len(test_df)),
|
| 324 |
-
},
|
| 325 |
-
"feature_columns": {
|
| 326 |
-
"numeric": numeric_cols,
|
| 327 |
-
"categorical": categorical_cols,
|
| 328 |
-
},
|
| 329 |
-
"candidate_validation": candidate_validation,
|
| 330 |
-
"selected_model": best_name,
|
| 331 |
-
"train_metrics": evaluate_split(train_df, train_pred),
|
| 332 |
-
"val_metrics": evaluate_split(val_df, val_pred),
|
| 333 |
-
"test_metrics": evaluate_split(test_df, test_pred),
|
| 334 |
-
"headline_metrics": {
|
| 335 |
-
"log_mae": float(evaluate_split(test_df, test_pred)["log_mae"]),
|
| 336 |
-
"log_rmse": float(evaluate_split(test_df, test_pred)["log_rmse"]),
|
| 337 |
-
"log_spearman": float(evaluate_split(test_df, test_pred)["log_spearman"]),
|
| 338 |
-
},
|
| 339 |
-
"predictions_path": str(pred_path),
|
| 340 |
-
"model_family": "lightweight_linear_task_heads" if args.model_family == "lite" else "popular_open_source_task_models",
|
| 341 |
-
"fm_family": (args.fm_family or "weather_fm_derived_features") if args.feature_profile == "weather_fm" else None,
|
| 342 |
-
"tmt_policy": {
|
| 343 |
-
"task": "final_burned_area",
|
| 344 |
-
"metric": "log-area regression error with rank agreement",
|
| 345 |
-
"tolerance": "secondary magnitude-band interpretation only",
|
| 346 |
-
},
|
| 347 |
-
}
|
| 348 |
-
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 349 |
-
print(json.dumps(summary, indent=2))
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
if __name__ == "__main__":
|
| 353 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py
DELETED
|
@@ -1,306 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import argparse
|
| 5 |
-
import json
|
| 6 |
-
import math
|
| 7 |
-
import sys
|
| 8 |
-
from pathlib import Path
|
| 9 |
-
from typing import Dict, List, Tuple
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
import os
|
| 13 |
-
|
| 14 |
-
for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep):
|
| 15 |
-
if _p and _p not in sys.path:
|
| 16 |
-
sys.path.insert(0, _p)
|
| 17 |
-
|
| 18 |
-
import numpy as np
|
| 19 |
-
import pandas as pd
|
| 20 |
-
from catboost import CatBoostRegressor
|
| 21 |
-
from lightgbm import LGBMRegressor
|
| 22 |
-
from sklearn.metrics import mean_absolute_error, mean_squared_error
|
| 23 |
-
from xgboost import XGBRegressor
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
def parse_args() -> argparse.Namespace:
|
| 27 |
-
parser = argparse.ArgumentParser()
|
| 28 |
-
parser.add_argument("--aqs-daily", type=Path, required=True)
|
| 29 |
-
parser.add_argument("--output-dir", type=Path, required=True)
|
| 30 |
-
parser.add_argument("--exceedance-threshold", type=float, default=35.0)
|
| 31 |
-
parser.add_argument("--alphaearth-prefix", type=str, default="alphaearth_")
|
| 32 |
-
parser.add_argument("--seed", type=int, default=7)
|
| 33 |
-
return parser.parse_args()
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
def assign_split(ts: pd.Timestamp) -> str:
|
| 37 |
-
year = int(ts.year)
|
| 38 |
-
if year <= 2023:
|
| 39 |
-
return "train"
|
| 40 |
-
if year == 2024:
|
| 41 |
-
return "val"
|
| 42 |
-
if year == 2025:
|
| 43 |
-
return "test"
|
| 44 |
-
return "other"
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 48 |
-
return float(math.sqrt(mean_squared_error(y_true, y_pred)))
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float:
|
| 52 |
-
a = np.asarray(y_true, dtype=np.float64)
|
| 53 |
-
b = np.asarray(y_pred, dtype=np.float64)
|
| 54 |
-
if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]):
|
| 55 |
-
return 0.0
|
| 56 |
-
value = float(np.corrcoef(a, b)[0, 1])
|
| 57 |
-
return value if np.isfinite(value) else 0.0
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]:
|
| 61 |
-
truth = np.asarray(y_true >= threshold)
|
| 62 |
-
pred = np.asarray(y_pred >= threshold)
|
| 63 |
-
tp = int(np.logical_and(pred, truth).sum())
|
| 64 |
-
fp = int(np.logical_and(pred, ~truth).sum())
|
| 65 |
-
fn = int(np.logical_and(~pred, truth).sum())
|
| 66 |
-
precision = float(tp / (tp + fp)) if (tp + fp) else 0.0
|
| 67 |
-
recall = float(tp / (tp + fn)) if (tp + fn) else 0.0
|
| 68 |
-
f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0
|
| 69 |
-
return {"precision": precision, "recall": recall, "f1": f1}
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
def evaluate_frame(frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]:
|
| 73 |
-
y_true = frame["pm25_mean"].to_numpy(dtype=np.float64)
|
| 74 |
-
y_pred = frame[pred_col].to_numpy(dtype=np.float64)
|
| 75 |
-
event = prf(y_true, y_pred, threshold)
|
| 76 |
-
bias = np.asarray(y_pred - y_true, dtype=np.float64)
|
| 77 |
-
denom = float(np.sum(y_true))
|
| 78 |
-
return {
|
| 79 |
-
"count": int(len(frame)),
|
| 80 |
-
"rmse": rmse(y_true, y_pred),
|
| 81 |
-
"mae": float(mean_absolute_error(y_true, y_pred)),
|
| 82 |
-
"mean_bias": float(np.mean(bias)),
|
| 83 |
-
"normalized_mean_bias": float(np.sum(bias) / denom) if abs(denom) > 1e-12 else 0.0,
|
| 84 |
-
"pearson_r": pearson_corr(y_true, y_pred),
|
| 85 |
-
"event_precision": event["precision"],
|
| 86 |
-
"event_recall": event["recall"],
|
| 87 |
-
"event_f1": event["f1"],
|
| 88 |
-
}
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def tune_event_shift(val_frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]:
|
| 92 |
-
best = None
|
| 93 |
-
for delta in np.linspace(-5.0, 15.0, 161):
|
| 94 |
-
shifted = val_frame.copy()
|
| 95 |
-
shifted["_shifted_pred"] = shifted[pred_col] + float(delta)
|
| 96 |
-
metrics = evaluate_frame(shifted, "_shifted_pred", threshold)
|
| 97 |
-
score = (metrics["event_f1"], metrics["event_recall"], -abs(float(delta)))
|
| 98 |
-
if best is None or score > best["score"]:
|
| 99 |
-
best = {"delta": float(delta), "metrics": metrics, "score": score}
|
| 100 |
-
assert best is not None
|
| 101 |
-
return {"delta": best["delta"], "val_event_calibrated_metrics": best["metrics"]}
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
def build_features(df: pd.DataFrame) -> pd.DataFrame:
|
| 105 |
-
df = df.sort_values(["site_key", "date"]).reset_index(drop=True).copy()
|
| 106 |
-
df["doy"] = df["date"].dt.dayofyear.astype(np.int32)
|
| 107 |
-
df["month"] = df["date"].dt.month.astype(np.int32)
|
| 108 |
-
df["doy_sin"] = np.sin(2.0 * np.pi * df["doy"] / 366.0)
|
| 109 |
-
df["doy_cos"] = np.cos(2.0 * np.pi * df["doy"] / 366.0)
|
| 110 |
-
grp = df.groupby("site_key", sort=False)
|
| 111 |
-
for lag in [1, 2, 3, 7]:
|
| 112 |
-
df[f"lag{lag}_pm25"] = grp["pm25_mean"].shift(lag)
|
| 113 |
-
df["roll3_prev"] = grp["pm25_mean"].rolling(3, min_periods=1).mean().reset_index(level=0, drop=True).shift(1)
|
| 114 |
-
df["roll7_prev"] = grp["pm25_mean"].rolling(7, min_periods=1).mean().reset_index(level=0, drop=True).shift(1)
|
| 115 |
-
return df
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
def prepare_frames(df: pd.DataFrame, alphaearth_prefix: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, List[str]]:
|
| 119 |
-
df = build_features(df)
|
| 120 |
-
train = df[df["split"] == "train"].copy()
|
| 121 |
-
val = df[df["split"] == "val"].copy()
|
| 122 |
-
test = df[df["split"] == "test"].copy()
|
| 123 |
-
|
| 124 |
-
site_mean = train.groupby("site_key")["pm25_mean"].mean()
|
| 125 |
-
global_mean = float(train["pm25_mean"].mean())
|
| 126 |
-
for frame in [train, val, test]:
|
| 127 |
-
frame["site_climo"] = frame["site_key"].map(site_mean).fillna(global_mean)
|
| 128 |
-
for frame in [train, val, test]:
|
| 129 |
-
for col in ["lag1_pm25", "lag2_pm25", "lag3_pm25", "lag7_pm25", "roll3_prev", "roll7_prev"]:
|
| 130 |
-
frame[col] = pd.to_numeric(frame[col], errors="coerce").fillna(frame["site_climo"])
|
| 131 |
-
|
| 132 |
-
feature_cols = [
|
| 133 |
-
"latitude",
|
| 134 |
-
"longitude",
|
| 135 |
-
"obs_count",
|
| 136 |
-
"site_climo",
|
| 137 |
-
"lag1_pm25",
|
| 138 |
-
"lag2_pm25",
|
| 139 |
-
"lag3_pm25",
|
| 140 |
-
"lag7_pm25",
|
| 141 |
-
"roll3_prev",
|
| 142 |
-
"roll7_prev",
|
| 143 |
-
"doy_sin",
|
| 144 |
-
"doy_cos",
|
| 145 |
-
"month",
|
| 146 |
-
]
|
| 147 |
-
alpha_cols = [
|
| 148 |
-
c for c in df.columns
|
| 149 |
-
if c.startswith(alphaearth_prefix) and pd.api.types.is_numeric_dtype(df[c])
|
| 150 |
-
]
|
| 151 |
-
feature_cols.extend(sorted(alpha_cols))
|
| 152 |
-
medians = train[feature_cols].median(numeric_only=True).fillna(0.0)
|
| 153 |
-
for frame in [train, val, test]:
|
| 154 |
-
frame.loc[:, feature_cols] = frame[feature_cols].fillna(medians)
|
| 155 |
-
frame.loc[:, feature_cols] = frame[feature_cols].fillna(0.0)
|
| 156 |
-
return train, val, test, feature_cols
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
def main() -> None:
|
| 160 |
-
args = parse_args()
|
| 161 |
-
df = pd.read_csv(args.aqs_daily, compression="infer", low_memory=False)
|
| 162 |
-
df["date"] = pd.to_datetime(df["date_gmt"], errors="coerce")
|
| 163 |
-
df["pm25_mean"] = pd.to_numeric(df["pm25_mean"], errors="coerce")
|
| 164 |
-
df["pm25_max"] = pd.to_numeric(df["pm25_max"], errors="coerce")
|
| 165 |
-
df["obs_count"] = pd.to_numeric(df["obs_count"], errors="coerce")
|
| 166 |
-
df = df.dropna(subset=["date", "site_key", "pm25_mean"]).copy()
|
| 167 |
-
df = (
|
| 168 |
-
df.groupby(["date", "site_key"], as_index=False)
|
| 169 |
-
.agg(
|
| 170 |
-
latitude=("Latitude", "first"),
|
| 171 |
-
longitude=("Longitude", "first"),
|
| 172 |
-
pm25_mean=("pm25_mean", "mean"),
|
| 173 |
-
pm25_max=("pm25_max", "max"),
|
| 174 |
-
obs_count=("obs_count", "sum"),
|
| 175 |
-
**{c: (c, "first") for c in df.columns if c.startswith(args.alphaearth_prefix)},
|
| 176 |
-
)
|
| 177 |
-
.sort_values(["site_key", "date"])
|
| 178 |
-
.reset_index(drop=True)
|
| 179 |
-
)
|
| 180 |
-
df["split"] = df["date"].map(assign_split)
|
| 181 |
-
df = df[df["split"].isin(["train", "val", "test"])].copy()
|
| 182 |
-
|
| 183 |
-
train, val, test, feature_cols = prepare_frames(df, alphaearth_prefix=args.alphaearth_prefix)
|
| 184 |
-
y_train = train["pm25_mean"].to_numpy(dtype=np.float64)
|
| 185 |
-
|
| 186 |
-
candidates = {
|
| 187 |
-
"xgboost": XGBRegressor(
|
| 188 |
-
n_estimators=300,
|
| 189 |
-
max_depth=8,
|
| 190 |
-
learning_rate=0.05,
|
| 191 |
-
subsample=0.8,
|
| 192 |
-
colsample_bytree=0.8,
|
| 193 |
-
objective="reg:squarederror",
|
| 194 |
-
tree_method="hist",
|
| 195 |
-
random_state=args.seed,
|
| 196 |
-
n_jobs=8,
|
| 197 |
-
),
|
| 198 |
-
"lightgbm": LGBMRegressor(
|
| 199 |
-
n_estimators=300,
|
| 200 |
-
learning_rate=0.05,
|
| 201 |
-
num_leaves=127,
|
| 202 |
-
subsample=0.8,
|
| 203 |
-
colsample_bytree=0.8,
|
| 204 |
-
random_state=args.seed,
|
| 205 |
-
n_jobs=8,
|
| 206 |
-
verbose=-1,
|
| 207 |
-
),
|
| 208 |
-
"catboost": CatBoostRegressor(
|
| 209 |
-
iterations=400,
|
| 210 |
-
depth=8,
|
| 211 |
-
learning_rate=0.05,
|
| 212 |
-
loss_function="RMSE",
|
| 213 |
-
eval_metric="RMSE",
|
| 214 |
-
random_seed=args.seed,
|
| 215 |
-
verbose=False,
|
| 216 |
-
),
|
| 217 |
-
}
|
| 218 |
-
|
| 219 |
-
candidate_validation: List[Dict[str, object]] = []
|
| 220 |
-
best_name = None
|
| 221 |
-
best_model = None
|
| 222 |
-
best_score = None
|
| 223 |
-
for name, model in candidates.items():
|
| 224 |
-
if name == "catboost":
|
| 225 |
-
model.fit(train[feature_cols], y_train, use_best_model=False)
|
| 226 |
-
val_pred = model.predict(val[feature_cols])
|
| 227 |
-
else:
|
| 228 |
-
model.fit(train[feature_cols].to_numpy(dtype=np.float32), y_train)
|
| 229 |
-
val_pred = model.predict(val[feature_cols].to_numpy(dtype=np.float32))
|
| 230 |
-
val_frame = val.copy()
|
| 231 |
-
val_frame["pred"] = val_pred
|
| 232 |
-
metrics = evaluate_frame(val_frame, "pred", args.exceedance_threshold)
|
| 233 |
-
candidate_validation.append({"candidate": name, "val_metrics": metrics})
|
| 234 |
-
score = float(metrics["rmse"])
|
| 235 |
-
if best_score is None or score < best_score:
|
| 236 |
-
best_score = score
|
| 237 |
-
best_name = name
|
| 238 |
-
best_model = model
|
| 239 |
-
|
| 240 |
-
assert best_name is not None and best_model is not None
|
| 241 |
-
combined = pd.concat([train, val], ignore_index=True)
|
| 242 |
-
if best_name == "catboost":
|
| 243 |
-
best_model.fit(combined[feature_cols], combined["pm25_mean"].to_numpy(dtype=np.float64), use_best_model=False)
|
| 244 |
-
train_pred = best_model.predict(train[feature_cols])
|
| 245 |
-
val_pred = best_model.predict(val[feature_cols])
|
| 246 |
-
test_pred = best_model.predict(test[feature_cols])
|
| 247 |
-
else:
|
| 248 |
-
best_model.fit(combined[feature_cols].to_numpy(dtype=np.float32), combined["pm25_mean"].to_numpy(dtype=np.float64))
|
| 249 |
-
train_pred = best_model.predict(train[feature_cols].to_numpy(dtype=np.float32))
|
| 250 |
-
val_pred = best_model.predict(val[feature_cols].to_numpy(dtype=np.float32))
|
| 251 |
-
test_pred = best_model.predict(test[feature_cols].to_numpy(dtype=np.float32))
|
| 252 |
-
|
| 253 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 254 |
-
pred_df = pd.concat(
|
| 255 |
-
[
|
| 256 |
-
train.assign(pred_pm25=train_pred),
|
| 257 |
-
val.assign(pred_pm25=val_pred),
|
| 258 |
-
test.assign(pred_pm25=test_pred),
|
| 259 |
-
],
|
| 260 |
-
ignore_index=True,
|
| 261 |
-
)
|
| 262 |
-
pred_path = args.output_dir / "predictions.csv.gz"
|
| 263 |
-
pred_df.to_csv(pred_path, index=False, compression="gzip")
|
| 264 |
-
|
| 265 |
-
train_eval = train.assign(pred=train_pred)
|
| 266 |
-
val_eval = val.assign(pred=val_pred)
|
| 267 |
-
test_eval = test.assign(pred=test_pred)
|
| 268 |
-
event_shift = tune_event_shift(val_eval, "pred", args.exceedance_threshold)
|
| 269 |
-
delta = event_shift["delta"]
|
| 270 |
-
train_event_eval = train_eval.assign(pred_event_calibrated=train_eval["pred"] + delta)
|
| 271 |
-
val_event_eval = val_eval.assign(pred_event_calibrated=val_eval["pred"] + delta)
|
| 272 |
-
test_event_eval = test_eval.assign(pred_event_calibrated=test_eval["pred"] + delta)
|
| 273 |
-
|
| 274 |
-
summary = {
|
| 275 |
-
"task_id": "smoke_pm25_alphaearth",
|
| 276 |
-
"task_form": "station_daily_regression",
|
| 277 |
-
"aqs_daily": str(args.aqs_daily),
|
| 278 |
-
"output_dir": str(args.output_dir),
|
| 279 |
-
"seed": int(args.seed),
|
| 280 |
-
"feature_columns": feature_cols,
|
| 281 |
-
"alphaearth_feature_count": int(sum(c.startswith(args.alphaearth_prefix) for c in feature_cols)),
|
| 282 |
-
"split_sizes": {"train": int(len(train)), "val": int(len(val)), "test": int(len(test))},
|
| 283 |
-
"candidate_validation": candidate_validation,
|
| 284 |
-
"selected_model": best_name,
|
| 285 |
-
"train_metrics": evaluate_frame(train_eval, "pred", args.exceedance_threshold),
|
| 286 |
-
"val_metrics": evaluate_frame(val_eval, "pred", args.exceedance_threshold),
|
| 287 |
-
"test_metrics": evaluate_frame(test_eval, "pred", args.exceedance_threshold),
|
| 288 |
-
"event_calibration": {
|
| 289 |
-
"delta": float(delta),
|
| 290 |
-
"val_metrics": event_shift["val_event_calibrated_metrics"],
|
| 291 |
-
"test_metrics": evaluate_frame(test_event_eval, "pred_event_calibrated", args.exceedance_threshold),
|
| 292 |
-
},
|
| 293 |
-
"predictions_path": str(pred_path),
|
| 294 |
-
"selection_rule": "same smoke benchmark; choose task-specific regressor by validation RMSE, then calibrate exceedance on validation only",
|
| 295 |
-
"tmt_policy": {
|
| 296 |
-
"task": "smoke_pm25",
|
| 297 |
-
"metric": "continuous RMSE/MAE with thresholded exceedance PRF",
|
| 298 |
-
"tolerance": "secondary event policy only",
|
| 299 |
-
},
|
| 300 |
-
}
|
| 301 |
-
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 302 |
-
print(json.dumps(summary, indent=2))
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
if __name__ == "__main__":
|
| 306 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|