Time Series Forecasting
PyTorch
English
pluvium
weather
weather-forecasting
climate
meteorology
earth-science
precipitation
rainfall
extreme-rainfall
nowcasting
probabilistic-forecasting
tropical-cyclones
hurricane
multi-model-ensemble
post-processing
online-learning
geospatial
caribbean
naturev1
ihelix
graphcast
aifs
Download tests/test_program.py from CorxLabs/Pluvium-500: direct link, hf CLI and curl.
- Browser
- Download file 8.77 kB
-
https://huggingface.co/CorxLabs/Pluvium-500/resolve/main/tests/test_program.py
- Command line
-
hf download hf://CorxLabs/Pluvium-500/tests/test_program.py
-
curl -L -o test_program.py https://huggingface.co/CorxLabs/Pluvium-500/resolve/main/tests/test_program.py
8.77 kB
| # Copyright 2026 Nathan. Apache-2.0. | |
| """The training program's settings: the years it trains on and holds out follow what the reanalysis archive really | |
| holds, and the GPU memory settings are read from the environment and checked. Needs neither a network nor a GPU.""" | |
| import numpy as np | |
| import pytest | |
| from pluvium import program | |
| from pluvium.program import Program, ProgramConfig, choose_years | |
| from pluvium.sources import era5 | |
| from pluvium.util import hour_of | |
| class Store: | |
| """Stands in for the hourly ERA5 store: only its time axis is asked for.""" | |
| def __init__(self, first: str, last: str) -> None: | |
| self.first, self.last = hour_of(first), hour_of(last) | |
| def hours(self) -> np.ndarray: | |
| return np.arange(self.first, self.last + 1, dtype=np.int64) | |
| def test_whole_years_are_read_from_the_archive_itself(): | |
| # the real store on 15 November 2026: named 1959-2022, and ending on 31 December 2021 | |
| assert era5.hourly_years(Store("2019-01-01T00", "2021-12-31T23")) == (2019, 2021) | |
| assert era5.hourly_years(Store("2019-01-01T00", "2022-06-30T23")) == (2019, 2021) # a year cut short is not whole | |
| assert era5.hourly_years(Store("2019-03-01T00", "2022-12-31T23")) == (2020, 2022) # nor one that starts late | |
| class Unreachable: | |
| def hours(self): | |
| raise OSError("no network") | |
| assert era5.hourly_years(Unreachable()) == (1959, 2021) # what the code remembers | |
| assert era5.HOURLY_LAST == hour_of("2021-12-31T23") | |
| def test_years_asked_for_are_fitted_to_the_archive(tmp_path, monkeypatch): | |
| monkeypatch.setattr(era5, "hourly_years", lambda store=None: (1959, 2021)) | |
| def years(**settings): | |
| made = Program(ProgramConfig(root=str(tmp_path / str(len(settings))), hub_repo=None, **settings)) | |
| made.check_years() | |
| return made.years, made.cfg.holdout_year | |
| assert ProgramConfig().holdout_year == 2021 and choose_years(24) == (2015, 2016, 2017, 2018, 2019, 2020) | |
| assert years() == ((2015, 2016, 2017, 2018, 2019, 2020), 2021) | |
| # 2022 was the default held-out year until 0.4.3: the archive does not have it, and the run stopped while | |
| # staging it, after every training year had been downloaded | |
| assert years(holdout_year=2022) == ((2015, 2016, 2017, 2018, 2019, 2020), 2021) | |
| assert years(train_years=(2019, 2020, 2021, 2022), holdout_year=2022) == ((2019, 2020), 2021) | |
| assert years(train_years=(2018, 2019), holdout_year=2020) == ((2018, 2019), 2020) # a valid choice is kept | |
| with pytest.raises(ValueError, match="no training year is left"): | |
| years(train_years=(2021,)) | |
| monkeypatch.setattr(era5, "hourly_years", lambda store=None: (1959, 2023)) # the archive grows | |
| assert years(holdout_year=2023, train_years=(2021, 2022)) == ((2021, 2022), 2023) | |
| def test_memory_settings_come_from_the_environment_and_are_checked(tmp_path, monkeypatch): | |
| for name in ("GPU_HOURS", "PLUVIUM_ROOT", "HF_REPO", "HF_TOKEN", "PLUVIUM_SIZE", "PLUVIUM_COUNTRIES", "PLUVIUM_BOX", | |
| "PLUVIUM_VRAM", "PLUVIUM_VRAM_SHARE", "PLUVIUM_BATCH"): | |
| monkeypatch.delenv(name, raising=False) | |
| cfg = ProgramConfig.from_env(root=str(tmp_path), hub_repo=None) | |
| assert (cfg.vram, cfg.vram_share, cfg.batch_size) == ("fill", 0.95, 0) # fill the card by default | |
| monkeypatch.setenv("PLUVIUM_VRAM_SHARE", "0.9") | |
| monkeypatch.setenv("PLUVIUM_VRAM", "fastest") | |
| monkeypatch.setenv("PLUVIUM_BATCH", "6") | |
| cfg = ProgramConfig.from_env(root=str(tmp_path), hub_repo=None) | |
| cfg.hub_repo = None | |
| assert (cfg.vram, cfg.vram_share, cfg.batch_size) == ("fastest", 0.9, 6) | |
| train = Program(cfg).train_cfg(1.0) | |
| assert (train.memory, train.memory_share, train.batch_size) == ("fastest", 0.9, 6) | |
| for wrong in ({"vram": "all"}, {"vram_share": 1.0}, {"vram_share": 0.1}): | |
| with pytest.raises(ValueError): | |
| Program(ProgramConfig(root=str(tmp_path), hub_repo=None, **wrong)).train_cfg(1.0) | |
| source = open(program.__file__).read() # run as a module, it starts | |
| assert callable(program.main) and source.rstrip().endswith('if __name__ == "__main__":\n main()') | |
| def test_staged_data_go_up_in_a_few_commits_and_only_what_the_hub_lacks(tmp_path): | |
| """The Hub takes at most 128 commits an hour in a repository. One commit per staged file (until 0.5.2) used that | |
| up with the data alone, and most of them were refused with HTTP 429.""" | |
| import hashlib | |
| from types import SimpleNamespace | |
| from pluvium.program import FILES_PER_COMMIT, mirror_files | |
| files = [] | |
| for k in range(2 * FILES_PER_COMMIT + 5): | |
| path = tmp_path / f"f{k}.npy" | |
| path.write_bytes(bytes([k % 256]) * 100) | |
| files.append((path, f"data/f{k}.npy")) | |
| same = files[0][0].read_bytes() | |
| old = tmp_path / "old.npy" | |
| old.write_bytes(b"older content") | |
| class Api: | |
| def __init__(self, fail=False): | |
| self.commits, self.fail = [], fail | |
| def get_paths_info(self, repo, paths, repo_type="model"): | |
| if self.fail: | |
| raise RuntimeError("429 Too Many Requests") | |
| return [SimpleNamespace(path="data/f0.npy", lfs={"sha256": hashlib.sha256(same).hexdigest()}), # held | |
| SimpleNamespace(path="data/f1.npy", lfs={"sha256": hashlib.sha256(b"older").hexdigest()})] # stale | |
| def create_commit(self, repo, repo_type, commit_message, operations): | |
| self.commits.append([op.path_in_repo for op in operations]) | |
| mirror = SimpleNamespace(can_push=True, repo_id="org/repo", token="t") | |
| api = Api() | |
| assert mirror_files(mirror, files + [(tmp_path / "missing.npy", "data/missing.npy")], "data", api=api) == len(files) - 1 | |
| assert [len(c) for c in api.commits] == [FILES_PER_COMMIT, FILES_PER_COMMIT, 4] # three commits, not 125 | |
| pushed = [p for c in api.commits for p in c] | |
| assert "data/f0.npy" not in pushed and "data/f1.npy" in pushed and "data/missing.npy" not in pushed | |
| assert mirror_files(mirror, files, "data", api=Api(fail=True)) == 0 # refused: said, not raised | |
| assert mirror_files(SimpleNamespace(can_push=False), files, "data", api=Api()) == 0 # no token: nothing to do | |
| def test_a_year_with_only_one_of_its_two_rain_files_is_staged_again(tmp_path, monkeypatch): | |
| """The Hub once held ``targets_2016.npy`` without ``mesh_2016.npy`` (the uploads of an earlier run were refused | |
| half way). The stager asked only the targets file, took 2016 as done, and a training run on a 96 GB card stopped | |
| when it first needed the file that was never made.""" | |
| import pluvium.sources.cmorph as C | |
| from pluvium.util import year_hours | |
| class Points: | |
| def __init__(self, n): | |
| self.n, self.extra = n, {"row": np.zeros(n, int), "col": np.zeros(n, int)} | |
| def __len__(self): | |
| return self.n | |
| targets, mesh, fetched = Points(3), Points(5), [] | |
| for name in ("cmorph_window", "mesh_operator", "default_half_widths", "_init"): | |
| monkeypatch.setattr(C, name, lambda *a, **k: None) | |
| monkeypatch.setattr(C, "_one_hour", lambda hour: (fetched.append(hour) or hour, np.full(3, 7, np.uint16), np.full(5, 9, np.uint16))) | |
| first, _ = year_hours(2016) | |
| period = (first, first + 48) | |
| folder = tmp_path / "cmorph" | |
| C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False) | |
| assert len(fetched) == 48 and (folder / "targets_2016.npy").exists() and (folder / "mesh_2016.npy").exists() | |
| fetched.clear() | |
| C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False) | |
| assert fetched == [] # a whole year is not fetched twice | |
| (folder / "mesh_2016.npy").unlink() # half the year, as it came from the Hub | |
| cache = C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False) | |
| assert len(fetched) == 48 and (folder / "mesh_2016.npy").exists() | |
| assert cache.mesh(np.arange(first, first + 3)).shape == (3, 5) and np.isfinite(cache.mesh(np.arange(first, first + 3))).all() | |
| def test_the_background_fetch_does_not_keep_a_finished_run_alive(tmp_path): | |
| import multiprocessing | |
| import time | |
| prog = Program(ProgramConfig(root=str(tmp_path), hub_repo=None)) | |
| prog.stop_prefetch() # nothing running: nothing to do | |
| prog._prefetch = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(120,)) | |
| prog._prefetch.start() | |
| began = time.time() | |
| prog.stop_prefetch() | |
| assert not prog._prefetch.is_alive() and time.time() - began < 30 | |