File size: 16,668 Bytes
f3c3632
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
# scripts/eval_multistep.py
import os, sys, argparse, json
from pathlib import Path
from datetime import timedelta

import torch
import numpy as np
import pandas as pd
import xarray as xr
from tqdm import tqdm
from omegaconf import OmegaConf
from hydra.utils import instantiate

sys.path.append(str(Path(__file__).parent.parent))

from evaluation.deterministic_metrics import headline_wrmse
from dataloaders.preprocessors import reverse_default_preprocessor


# -------------------- helpers --------------------
def _time_from_sample(sample):
    t = sample.get("time")
    if isinstance(t, (list, tuple)):
        t = t[0]
    return t

def _prev_offset(ds):
    # Era5Forecast.__len__ 보정과 동일 규칙
    return (ds.lead_time_hours // ds.timedelta) if getattr(ds, "load_prev", False) else 0

def _time_map_fast(ds):
    """__getitem__ 호출 없이 id2pt에서 바로 map 구성 (매우 빠름)."""
    off = _prev_offset(ds)
    m = {}
    for i in range(len(ds)):
        _, _, ts = ds.id2pt[i + off]  # (file_id, line_id, np.datetime64)
        t = pd.to_datetime(ts).strftime('%Y-%m-%d-%H-%M')
        m[t] = i
    return m

def _latlon_from_hw(H, W):
    lat_step = 180.0 / H
    lon_step = 360.0 / W
    lats = np.arange(-90.0, 90.0, lat_step)   # H개
    lons = np.arange(0.0, 360.0, lon_step)    # W개
    return lats, lons

def _area_weights(H, W, device, dtype):
    lats, _ = _latlon_from_hw(H, W)
    w = np.cos(np.deg2rad(lats))
    w = w / w.mean()
    w = torch.as_tensor(w, device=device, dtype=dtype)[:, None].repeat(1, W)  # [H,W]
    return w

def _to_dt(tstr: str):
    return pd.to_datetime(tstr, utc=True, format="%Y-%m-%d-%H-%M").tz_localize(None)

def _fmt_dt(dt) -> str:
    return dt.strftime("%Y-%m-%d-%H-%M")

def _pair_hr_lr_at_time(t_str, hr_norm_ds, lr_norm_ds, t2i_hr, t2i_lr):
    """현재 t의 HR 정규배치 + LR(now, prev)의 surface 4ch를 붙여서 반환.
       prev LR가 없으면 None (스킵)."""
    if t_str not in t2i_hr or t_str not in t2i_lr:
        return None
    ih, il = t2i_hr[t_str], t2i_lr[t_str]
    a = hr_norm_ds[ih]   # HR normalized
    b = lr_norm_ds[il]   # LR normalized

    # prev LR 시각
    lead = int(getattr(hr_norm_ds, "lead_time_hours", 24))
    prev_t = _fmt_dt(_to_dt(t_str) - pd.Timedelta(hours=lead))
    if prev_t not in t2i_lr:
        return None
    b_prev = lr_norm_ds[t2i_lr[prev_t]]

    if "state_surface" not in b or "state_surface" not in b_prev:
        return None

    out = dict(a)
    out["state_surface_lr"]      = b["state_surface"]
    out["prev_state_surface_lr"] = b_prev["state_surface"]

    # time 표준화
    t = _time_from_sample(a)
    out["time"] = [t] if isinstance(t, str) else a["time"]
    return out

def _merge_next_pair_from_pred(hr_norm_ds, lr_norm_ds, t2i_lr, pred, hr_norm_batch):
    """HR는 pred 기반으로 unroll(normalize_next_batch), LR는 실측 now/prev로 갱신.
       두 LR 타임이 없으면 None."""
    hr_next = hr_norm_ds.normalize_next_batch(pred, hr_norm_batch)
    next_t = hr_next["time"][0] if isinstance(hr_next["time"], (list, tuple)) else hr_next["time"]

    # next 시점의 now/prev LR 확보
    lead = int(getattr(hr_norm_ds, "lead_time_hours", 24))
    prev_t = _fmt_dt(_to_dt(next_t) - pd.Timedelta(hours=lead))
    if next_t not in t2i_lr or prev_t not in t2i_lr:
        return None

    b_now  = lr_norm_ds[t2i_lr[next_t]]
    b_prev = lr_norm_ds[t2i_lr[prev_t]]
    if "state_surface" not in b_now or "state_surface" not in b_prev:
        return None

    out = dict(hr_next)
    out["state_surface_lr"]      = b_now["state_surface"]
    out["prev_state_surface_lr"] = b_prev["state_surface"]
    return out

def _build_model(cfg):
    """
    다양한 cfg 형태를 지원해서 backbone/module을 생성.
    - 형태1: cfg.module.module + cfg.module.backbone
    - 형태2: cfg.module(_target_ 포함) + cfg.backbone
    - 형태3: cfg.module(_target_) 내부에서 backbone까지 생성
    """
    if "module" not in cfg:
        raise RuntimeError("cfg.module 노드가 없습니다. --cfg-path를 확인하세요.")

    node = cfg.module
    # 형태1
    if (isinstance(node, dict) or ("_target_" not in node)) and ("module" in node and "backbone" in node):
        backbone = instantiate(node.backbone)
        pl_module = instantiate(node.module, backbone=backbone)
        return pl_module
    # 형태2
    if "_target_" in node and "backbone" in cfg:
        backbone = instantiate(cfg.backbone)
        pl_module = instantiate(node, backbone=backbone)
        return pl_module
    # 형태3
    if "_target_" in node:
        pl_module = instantiate(node)
        return pl_module

    raise RuntimeError("알 수 없는 cfg 구조입니다. cfg.module을 확인하세요.")


# -------------------- main --------------------
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--ckpt-path", type=str, required=True)
    ap.add_argument("--cfg-path", type=str, required=True)
    ap.add_argument("--dataset-cfg-path", type=str, required=True)
    ap.add_argument("--multistep", type=int, default=10)
    ap.add_argument("--force", action="store_true")
    ap.add_argument("--debug", action="store_true")
    args = ap.parse_args()

    torch.set_grad_enabled(False)
    device = "cuda:0" if torch.cuda.is_available() else "cpu"
    write_frequency = 10

    # ---------- load cfgs ----------
    cfg    = OmegaConf.load(args.cfg_path)
    ds_cfg = OmegaConf.load(args.dataset_cfg_path)

    # ---------- build model & load ckpt ----------
    pl_module = _build_model(cfg)
    state = torch.load(args.ckpt_path, map_location="cpu", weights_only=False)
    state_dict = state.get("state_dict", state)
    pl_module.load_state_dict(state_dict, strict=False)
    pl_module = pl_module.to(device).eval()
    if hasattr(pl_module, "use_prev"):
        pl_module.use_prev = True  # prev LR/HR 경로 활성화

    # ---------- resolve dataset config ----------
    # 허용:
    #  A) {dataloader: {dataset_hr: ..., dataset_lr: ...}}
    #  B) {dataset_hr: ..., dataset_lr: ...}
    #  C) {dataset: ...}  (HR/LR 동일 사용)
    if "dataloader" in ds_cfg:
        dl_cfg = ds_cfg.dataloader
    else:
        dl_cfg = ds_cfg

    if "dataset_hr" in dl_cfg and "dataset_lr" in dl_cfg:
        ds_hr_node = dl_cfg.dataset_hr
        ds_lr_node = dl_cfg.dataset_lr
    elif "dataset" in dl_cfg:
        ds_hr_node = dl_cfg.dataset
        ds_lr_node = dl_cfg.dataset
    else:
        raise RuntimeError(
            "dataset config 형태가 다릅니다. 다음 중 하나여야 합니다:\n"
            "  - dataloader.dataset_hr & dataloader.dataset_lr\n"
            "  - dataset_hr & dataset_lr\n"
            "  - dataset"
        )

    # ---------- datasets (정규화/원시) ----------
    # 정규화된 HR/LR (모델 입력용)
    hr_norm_ds = instantiate(ds_hr_node, domain="test", z0012=True)
    lr_norm_ds = instantiate(ds_lr_node, domain="test", z0012=True)

    # 원시 HR (denorm GT 저장/메트릭 계산용)
    hr_raw_ds  = instantiate(
        ds_hr_node,
        domain="test", z0012=True,
        multistep=args.multistep,
        input_norm_scheme=None,
        output_norm_scheme=None,
    )

    # 시간 매핑(빠른 버전)
    t2i_hr = _time_map_fast(hr_norm_ds)
    t2i_lr = _time_map_fast(lr_norm_ds)

    raw_loader = torch.utils.data.DataLoader(hr_raw_ds, batch_size=1, num_workers=4, shuffle=False)

    outs = dict(traj_surface=[], traj_level=[], timestamps=[])
    err_log = []

    lvl_names = hr_norm_ds.variables["state_level"]   # ex) ['Z50',...,'Q1000']
    srf_names = hr_norm_ds.variables["state_surface"] # ['U10','V10','T2m','SP']

    # ---- per-var/time accumulators for WRMSE(JSON) ----
    want_surf = ["U10", "V10", "T2m", "SP"]
    want_lvl  = {"Z500":("Z",500), "U850":("U",850), "V850":("V",850), "T850":("T",850), "Q700":("Q",700)}

    pres_list = hr_norm_ds.pressure_levels  # [50,...,1000]
    pres2idx  = {int(p): i for i, p in enumerate(pres_list)}
    var2idx   = {"Z":0, "U":1, "V":2, "T":3, "Q":4}

    json_sums, json_den = {}, {}
    def accumulate_json(key, err2_map, wmap):
        json_sums[key] = json_sums.get(key, 0.0) + float((err2_map * wmap).sum().item())
        json_den[key]  = json_den.get(key,  0.0) + float(wmap.sum().item())

    first_debug_printed = False

    ckpt_stem = Path(args.ckpt_path).stem
    save_dir  = Path("evalstore") / ckpt_stem
    save_dir.mkdir(parents=True, exist_ok=True)
    zarr_path = save_dir / f"test2020-multistep{args.multistep}.zarr"
    if zarr_path.exists() and args.force:
        import shutil; shutil.rmtree(zarr_path)

    # ---------- per-time rollout ----------
    debug_steps = {1, 2, 10}   # 1-based
    printed_steps = set()
    first_valid_seen = False

    for i, raw in tqdm(enumerate(raw_loader), total=len(raw_loader)):
        t0 = _time_from_sample(raw)
        pair = _pair_hr_lr_at_time(t0, hr_norm_ds, lr_norm_ds, t2i_hr, t2i_lr)
        if pair is None:
            continue  # prev LR가 없는 시작 시점은 스킵
        
        is_first_valid = False
        if not first_valid_seen:
            is_first_valid = True
            first_valid_seen = True

        hr_batch = {k: v for k, v in pair.items() if not k.endswith("_lr")}

        traj_s, traj_l = [], []
        for k in range(args.multistep):
            pair_k = {kk: (vv.to(device) if hasattr(vv, "to") else vv) for kk, vv in pair.items()}
            pred   = pl_module.forward(pair_k)

            # 디바이스 혼합 방지: denorm 계산은 CPU에서
            denorm_pred = hr_norm_ds.denormalize({k2: (v2.detach().cpu() if torch.is_tensor(v2) else v2)
                                                  for k2, v2 in pred.items()},
                                                 {k2: (v2.cpu() if torch.is_tensor(v2) else v2)
                                                  for k2, v2 in hr_batch.items()})

            traj_s.append(denorm_pred["next_state_surface"].cpu())
            traj_l.append(denorm_pred["next_state_level"].cpu())

            # ---- per-var/time JSON 누적 ----
            s_pred = denorm_pred["next_state_surface"].squeeze(-3)  # [B,4,H,W]
            l_pred = denorm_pred["next_state_level"]                # [B,5,L,H,W]
            if k == 0:
                s_gt = raw["next_state_surface"].squeeze(-3)        # [B,4,H,W]
                l_gt = raw["next_state_level"]                      # [B,5,L,H,W]
            else:
                s_gt = raw["future_state_surface"][:, k-1].squeeze(-3)
                l_gt = raw["future_state_level"][:, k-1]

            B, _, H, W = s_pred.shape
            wmap = _area_weights(H, W, device=s_pred.device, dtype=s_pred.dtype)  # [H,W]

            # surface vars
            for vi, name in enumerate(want_surf):
                err2 = (s_pred[:, vi] - s_gt[:, vi]).pow(2).mean(0)  # [H,W]
                key  = f"lr2hr_H{H}xW{W}/t{k+1}/{name}"
                accumulate_json(key, err2, wmap)

            # level vars at target pressures
            for jname, (vtag, plevel) in want_lvl.items():
                vc = var2idx[vtag]
                li = pres2idx[int(plevel)]
                err2 = (l_pred[:, vc, li] - l_gt[:, vc, li]).pow(2).mean(0)
                key  = f"lr2hr_H{H}xW{W}/t{k+1}/{jname}"
                accumulate_json(key, err2, wmap)

            # ---- 첫 유효 스텝에서 디버그 프린트 (항상 1회) ----
            if is_first_valid and ((k+1) in debug_steps) and ((k+1) not in printed_steps):
                print(f"\n[DEBUG — first valid rollout @ t={k+1}]")
                for name in ["state_surface_lr","prev_state_surface_lr","state_constant","state_level"]:
                    if name in pair_k and hasattr(pair_k[name], "shape"):
                        print(f"  batch['{name}']:", tuple(pair_k[name].shape))
                print("  pred.next_state_surface:", tuple(denorm_pred["next_state_surface"].shape))
                print("  pred.next_state_level:  ", tuple(denorm_pred["next_state_level"].shape))
                # 간단 WRMSE 샘플
                def wrmse_scalar(a, b, w):
                    e2 = (a - b).pow(2).mean(0)  # [H,W]
                    return float(torch.sqrt((e2 * w).sum() / w.sum()).item())
                u10_rmse = wrmse_scalar(s_pred[:,0], s_gt[:,0], wmap)
                z500_rmse= wrmse_scalar(l_pred[:,0,pres2idx[500]], l_gt[:,0,pres2idx[500]], wmap)
                print(f"  U10@t{k+1} WRMSE: {u10_rmse:.4f}")
                print(f"  Z500@t{k+1} WRMSE: {z500_rmse:.4f}\n")
                printed_steps.add(k+1)

            # 다음 스텝용 페어 갱신 (HR unroll + 실측 LR now/prev)
            merged = _merge_next_pair_from_pred(hr_norm_ds, lr_norm_ds, t2i_lr, pred, hr_batch)
            if merged is None:
                break
            pair     = merged
            hr_batch = {k2: v2 for k2, v2 in pair.items() if not k2.endswith("_lr")}

        if not traj_s:
            continue  # 안전장치

        traj_s = torch.stack(traj_s, dim=1)  # [1,T,4,1,H,W] 또는 [1,T,4,H,W]
        traj_l = torch.stack(traj_l, dim=1)  # [1,T,5,L,H,W]

        outs["timestamps"].append(t0)
        outs["traj_surface"].append(traj_s)
        outs["traj_level"].append(traj_l)

        # headline metrics
        gt_traj = dict(
            traj_surface=torch.cat([raw["next_state_surface"][:, None], raw["future_state_surface"]], dim=1),
            traj_level  =torch.cat([raw["next_state_level"][:, None],   raw["future_state_level"]],   dim=1),
        )
        err_log.append(headline_wrmse(dict(traj_surface=traj_s, traj_level=traj_l), gt_traj, prefix="traj"))

        # 주기적 저장
        if (i % write_frequency == 0) or (i == len(raw_loader) - 1):
            level_outs = torch.cat(outs["traj_level"]).cpu()
            level_outs = reverse_default_preprocessor(level_outs).numpy()

            surface_outs = torch.cat(outs["traj_surface"]).cpu()
            surface_outs = reverse_default_preprocessor(surface_outs).numpy()

            Hs, Ws = surface_outs.shape[-3], surface_outs.shape[-2]
            lats, lons = _latlon_from_hw(Hs, Ws)
            T = surface_outs.shape[1]

            xds = xr.Dataset(
                data_vars={
                    **{lvl_names[ii]: (["time", "prediction_timedelta", "level", "latitude", "longitude"],
                                       level_outs[:, :, ii])
                       for ii in range(len(lvl_names))},
                    **{srf_names[ii]: (["time", "prediction_timedelta", "latitude", "longitude"],
                                       surface_outs[:, :, ii, 0])
                       for ii in range(len(srf_names))},
                },
                coords=dict(
                    time=pd.to_datetime(outs["timestamps"], utc=True, format="%Y-%m-%d-%H-%M").tz_localize(None),
                    prediction_timedelta=[timedelta(k+1) for k in range(T)],
                    latitude=lats,
                    longitude=lons,
                    level=[50,100,150,200,250,300,400,500,600,700,850,925,1000],
                ),
            )

            xds.to_zarr(
                zarr_path,
                append_dim="time" if i else None,
                encoding=dict(time=dict(units="hours since 2000-01-01")) if not i else None,
            )

            # running errors
            if err_log:
                avg_err = {k: torch.cat([e[k] for e in err_log], dim=0).mean(0)
                           for k in err_log[0].keys()}
                print("[running headline_wrmse]",
                      {k: (v.item() if torch.numel(v)==1 else v.mean().item())
                       for k, v in avg_err.items()})

            outs = dict(traj_surface=[], traj_level=[], timestamps=[])

        if args.debug:
            break

    # 최종 headline 저장
    if err_log:
        avg_err = {k: torch.cat([e[k] for e in err_log], dim=0).mean(0)
                   for k in err_log[0].keys()}
        torch.save(avg_err, save_dir / f"allmetrics-multistep{args.multistep}.pt")
        print("[final headline_wrmse]",
              {k: (v.item() if torch.numel(v)==1 else v.mean().item())
               for k, v in avg_err.items()})

    # ---- JSON 출력 (per-var/time WRMSE) ----
    json_out = {}
    for key in sorted(json_sums.keys()):
        rmse = np.sqrt(json_sums[key] / max(json_den[key], 1e-12))
        json_out[key] = float(rmse)
    json_path = save_dir / f"pervar_rmse_t{args.multistep}.json"
    with open(json_path, "w") as f:
        json.dump(json_out, f, indent=2)
    print(f"[saved JSON] {json_path}")


if __name__ == "__main__":
    main()