File size: 8,769 Bytes
1f77abb
 
527d56f
1f77abb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
527d56f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27b6636
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
# Copyright 2026 Nathan. Apache-2.0.
"""The training program's settings: the years it trains on and holds out follow what the reanalysis archive really
holds, and the GPU memory settings are read from the environment and checked. Needs neither a network nor a GPU."""

import numpy as np
import pytest

from pluvium import program
from pluvium.program import Program, ProgramConfig, choose_years
from pluvium.sources import era5
from pluvium.util import hour_of


class Store:
    """Stands in for the hourly ERA5 store: only its time axis is asked for."""

    def __init__(self, first: str, last: str) -> None:
        self.first, self.last = hour_of(first), hour_of(last)

    def hours(self) -> np.ndarray:
        return np.arange(self.first, self.last + 1, dtype=np.int64)


def test_whole_years_are_read_from_the_archive_itself():
    # the real store on 15 November 2026: named 1959-2022, and ending on 31 December 2021
    assert era5.hourly_years(Store("2019-01-01T00", "2021-12-31T23")) == (2019, 2021)
    assert era5.hourly_years(Store("2019-01-01T00", "2022-06-30T23")) == (2019, 2021)      # a year cut short is not whole
    assert era5.hourly_years(Store("2019-03-01T00", "2022-12-31T23")) == (2020, 2022)      # nor one that starts late

    class Unreachable:
        def hours(self):
            raise OSError("no network")

    assert era5.hourly_years(Unreachable()) == (1959, 2021)                              # what the code remembers
    assert era5.HOURLY_LAST == hour_of("2021-12-31T23")


def test_years_asked_for_are_fitted_to_the_archive(tmp_path, monkeypatch):
    monkeypatch.setattr(era5, "hourly_years", lambda store=None: (1959, 2021))

    def years(**settings):
        made = Program(ProgramConfig(root=str(tmp_path / str(len(settings))), hub_repo=None, **settings))
        made.check_years()
        return made.years, made.cfg.holdout_year

    assert ProgramConfig().holdout_year == 2021 and choose_years(24) == (2015, 2016, 2017, 2018, 2019, 2020)
    assert years() == ((2015, 2016, 2017, 2018, 2019, 2020), 2021)
    # 2022 was the default held-out year until 0.4.3: the archive does not have it, and the run stopped while
    # staging it, after every training year had been downloaded
    assert years(holdout_year=2022) == ((2015, 2016, 2017, 2018, 2019, 2020), 2021)
    assert years(train_years=(2019, 2020, 2021, 2022), holdout_year=2022) == ((2019, 2020), 2021)
    assert years(train_years=(2018, 2019), holdout_year=2020) == ((2018, 2019), 2020)        # a valid choice is kept
    with pytest.raises(ValueError, match="no training year is left"):
        years(train_years=(2021,))
    monkeypatch.setattr(era5, "hourly_years", lambda store=None: (1959, 2023))               # the archive grows
    assert years(holdout_year=2023, train_years=(2021, 2022)) == ((2021, 2022), 2023)


def test_memory_settings_come_from_the_environment_and_are_checked(tmp_path, monkeypatch):
    for name in ("GPU_HOURS", "PLUVIUM_ROOT", "HF_REPO", "HF_TOKEN", "PLUVIUM_SIZE", "PLUVIUM_COUNTRIES", "PLUVIUM_BOX",
                 "PLUVIUM_VRAM", "PLUVIUM_VRAM_SHARE", "PLUVIUM_BATCH"):
        monkeypatch.delenv(name, raising=False)
    cfg = ProgramConfig.from_env(root=str(tmp_path), hub_repo=None)
    assert (cfg.vram, cfg.vram_share, cfg.batch_size) == ("fill", 0.95, 0)                   # fill the card by default
    monkeypatch.setenv("PLUVIUM_VRAM_SHARE", "0.9")
    monkeypatch.setenv("PLUVIUM_VRAM", "fastest")
    monkeypatch.setenv("PLUVIUM_BATCH", "6")
    cfg = ProgramConfig.from_env(root=str(tmp_path), hub_repo=None)
    cfg.hub_repo = None
    assert (cfg.vram, cfg.vram_share, cfg.batch_size) == ("fastest", 0.9, 6)
    train = Program(cfg).train_cfg(1.0)
    assert (train.memory, train.memory_share, train.batch_size) == ("fastest", 0.9, 6)
    for wrong in ({"vram": "all"}, {"vram_share": 1.0}, {"vram_share": 0.1}):
        with pytest.raises(ValueError):
            Program(ProgramConfig(root=str(tmp_path), hub_repo=None, **wrong)).train_cfg(1.0)
    source = open(program.__file__).read()                                                  # run as a module, it starts
    assert callable(program.main) and source.rstrip().endswith('if __name__ == "__main__":\n    main()')


def test_staged_data_go_up_in_a_few_commits_and_only_what_the_hub_lacks(tmp_path):
    """The Hub takes at most 128 commits an hour in a repository. One commit per staged file (until 0.5.2) used that
    up with the data alone, and most of them were refused with HTTP 429."""
    import hashlib
    from types import SimpleNamespace

    from pluvium.program import FILES_PER_COMMIT, mirror_files

    files = []
    for k in range(2 * FILES_PER_COMMIT + 5):
        path = tmp_path / f"f{k}.npy"
        path.write_bytes(bytes([k % 256]) * 100)
        files.append((path, f"data/f{k}.npy"))
    same = files[0][0].read_bytes()
    old = tmp_path / "old.npy"
    old.write_bytes(b"older content")

    class Api:
        def __init__(self, fail=False):
            self.commits, self.fail = [], fail

        def get_paths_info(self, repo, paths, repo_type="model"):
            if self.fail:
                raise RuntimeError("429 Too Many Requests")
            return [SimpleNamespace(path="data/f0.npy", lfs={"sha256": hashlib.sha256(same).hexdigest()}),   # held
                    SimpleNamespace(path="data/f1.npy", lfs={"sha256": hashlib.sha256(b"older").hexdigest()})]  # stale

        def create_commit(self, repo, repo_type, commit_message, operations):
            self.commits.append([op.path_in_repo for op in operations])

    mirror = SimpleNamespace(can_push=True, repo_id="org/repo", token="t")
    api = Api()
    assert mirror_files(mirror, files + [(tmp_path / "missing.npy", "data/missing.npy")], "data", api=api) == len(files) - 1
    assert [len(c) for c in api.commits] == [FILES_PER_COMMIT, FILES_PER_COMMIT, 4]        # three commits, not 125
    pushed = [p for c in api.commits for p in c]
    assert "data/f0.npy" not in pushed and "data/f1.npy" in pushed and "data/missing.npy" not in pushed
    assert mirror_files(mirror, files, "data", api=Api(fail=True)) == 0                    # refused: said, not raised
    assert mirror_files(SimpleNamespace(can_push=False), files, "data", api=Api()) == 0    # no token: nothing to do


def test_a_year_with_only_one_of_its_two_rain_files_is_staged_again(tmp_path, monkeypatch):
    """The Hub once held ``targets_2016.npy`` without ``mesh_2016.npy`` (the uploads of an earlier run were refused
    half way). The stager asked only the targets file, took 2016 as done, and a training run on a 96 GB card stopped
    when it first needed the file that was never made."""
    import pluvium.sources.cmorph as C
    from pluvium.util import year_hours

    class Points:
        def __init__(self, n):
            self.n, self.extra = n, {"row": np.zeros(n, int), "col": np.zeros(n, int)}

        def __len__(self):
            return self.n

    targets, mesh, fetched = Points(3), Points(5), []
    for name in ("cmorph_window", "mesh_operator", "default_half_widths", "_init"):
        monkeypatch.setattr(C, name, lambda *a, **k: None)
    monkeypatch.setattr(C, "_one_hour", lambda hour: (fetched.append(hour) or hour, np.full(3, 7, np.uint16), np.full(5, 9, np.uint16)))
    first, _ = year_hours(2016)
    period = (first, first + 48)
    folder = tmp_path / "cmorph"
    C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False)
    assert len(fetched) == 48 and (folder / "targets_2016.npy").exists() and (folder / "mesh_2016.npy").exists()
    fetched.clear()
    C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False)
    assert fetched == []                                                      # a whole year is not fetched twice
    (folder / "mesh_2016.npy").unlink()                                       # half the year, as it came from the Hub
    cache = C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False)
    assert len(fetched) == 48 and (folder / "mesh_2016.npy").exists()
    assert cache.mesh(np.arange(first, first + 3)).shape == (3, 5) and np.isfinite(cache.mesh(np.arange(first, first + 3))).all()


def test_the_background_fetch_does_not_keep_a_finished_run_alive(tmp_path):
    import multiprocessing
    import time

    prog = Program(ProgramConfig(root=str(tmp_path), hub_repo=None))
    prog.stop_prefetch()                                                      # nothing running: nothing to do
    prog._prefetch = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(120,))
    prog._prefetch.start()
    began = time.time()
    prog.stop_prefetch()
    assert not prog._prefetch.is_alive() and time.time() - began < 30