Download scripts/eval_multistep.py from bidulki-99/ArchesWeather-reimpl: direct link, hf CLI and curl.
- Browser
- Download file 16.7 kB
-
https://huggingface.co/bidulki-99/ArchesWeather-reimpl/resolve/main/scripts/eval_multistep.py
- Command line
-
hf download hf://bidulki-99/ArchesWeather-reimpl/scripts/eval_multistep.py
-
curl -L -o eval_multistep.py https://huggingface.co/bidulki-99/ArchesWeather-reimpl/resolve/main/scripts/eval_multistep.py
16.7 kB
| # scripts/eval_multistep.py | |
| import os, sys, argparse, json | |
| from pathlib import Path | |
| from datetime import timedelta | |
| import torch | |
| import numpy as np | |
| import pandas as pd | |
| import xarray as xr | |
| from tqdm import tqdm | |
| from omegaconf import OmegaConf | |
| from hydra.utils import instantiate | |
| sys.path.append(str(Path(__file__).parent.parent)) | |
| from evaluation.deterministic_metrics import headline_wrmse | |
| from dataloaders.preprocessors import reverse_default_preprocessor | |
| # -------------------- helpers -------------------- | |
| def _time_from_sample(sample): | |
| t = sample.get("time") | |
| if isinstance(t, (list, tuple)): | |
| t = t[0] | |
| return t | |
| def _prev_offset(ds): | |
| # Era5Forecast.__len__ 보정과 동일 규칙 | |
| return (ds.lead_time_hours // ds.timedelta) if getattr(ds, "load_prev", False) else 0 | |
| def _time_map_fast(ds): | |
| """__getitem__ 호출 없이 id2pt에서 바로 map 구성 (매우 빠름).""" | |
| off = _prev_offset(ds) | |
| m = {} | |
| for i in range(len(ds)): | |
| _, _, ts = ds.id2pt[i + off] # (file_id, line_id, np.datetime64) | |
| t = pd.to_datetime(ts).strftime('%Y-%m-%d-%H-%M') | |
| m[t] = i | |
| return m | |
| def _latlon_from_hw(H, W): | |
| lat_step = 180.0 / H | |
| lon_step = 360.0 / W | |
| lats = np.arange(-90.0, 90.0, lat_step) # H개 | |
| lons = np.arange(0.0, 360.0, lon_step) # W개 | |
| return lats, lons | |
| def _area_weights(H, W, device, dtype): | |
| lats, _ = _latlon_from_hw(H, W) | |
| w = np.cos(np.deg2rad(lats)) | |
| w = w / w.mean() | |
| w = torch.as_tensor(w, device=device, dtype=dtype)[:, None].repeat(1, W) # [H,W] | |
| return w | |
| def _to_dt(tstr: str): | |
| return pd.to_datetime(tstr, utc=True, format="%Y-%m-%d-%H-%M").tz_localize(None) | |
| def _fmt_dt(dt) -> str: | |
| return dt.strftime("%Y-%m-%d-%H-%M") | |
| def _pair_hr_lr_at_time(t_str, hr_norm_ds, lr_norm_ds, t2i_hr, t2i_lr): | |
| """현재 t의 HR 정규배치 + LR(now, prev)의 surface 4ch를 붙여서 반환. | |
| prev LR가 없으면 None (스킵).""" | |
| if t_str not in t2i_hr or t_str not in t2i_lr: | |
| return None | |
| ih, il = t2i_hr[t_str], t2i_lr[t_str] | |
| a = hr_norm_ds[ih] # HR normalized | |
| b = lr_norm_ds[il] # LR normalized | |
| # prev LR 시각 | |
| lead = int(getattr(hr_norm_ds, "lead_time_hours", 24)) | |
| prev_t = _fmt_dt(_to_dt(t_str) - pd.Timedelta(hours=lead)) | |
| if prev_t not in t2i_lr: | |
| return None | |
| b_prev = lr_norm_ds[t2i_lr[prev_t]] | |
| if "state_surface" not in b or "state_surface" not in b_prev: | |
| return None | |
| out = dict(a) | |
| out["state_surface_lr"] = b["state_surface"] | |
| out["prev_state_surface_lr"] = b_prev["state_surface"] | |
| # time 표준화 | |
| t = _time_from_sample(a) | |
| out["time"] = [t] if isinstance(t, str) else a["time"] | |
| return out | |
| def _merge_next_pair_from_pred(hr_norm_ds, lr_norm_ds, t2i_lr, pred, hr_norm_batch): | |
| """HR는 pred 기반으로 unroll(normalize_next_batch), LR는 실측 now/prev로 갱신. | |
| 두 LR 타임이 없으면 None.""" | |
| hr_next = hr_norm_ds.normalize_next_batch(pred, hr_norm_batch) | |
| next_t = hr_next["time"][0] if isinstance(hr_next["time"], (list, tuple)) else hr_next["time"] | |
| # next 시점의 now/prev LR 확보 | |
| lead = int(getattr(hr_norm_ds, "lead_time_hours", 24)) | |
| prev_t = _fmt_dt(_to_dt(next_t) - pd.Timedelta(hours=lead)) | |
| if next_t not in t2i_lr or prev_t not in t2i_lr: | |
| return None | |
| b_now = lr_norm_ds[t2i_lr[next_t]] | |
| b_prev = lr_norm_ds[t2i_lr[prev_t]] | |
| if "state_surface" not in b_now or "state_surface" not in b_prev: | |
| return None | |
| out = dict(hr_next) | |
| out["state_surface_lr"] = b_now["state_surface"] | |
| out["prev_state_surface_lr"] = b_prev["state_surface"] | |
| return out | |
| def _build_model(cfg): | |
| """ | |
| 다양한 cfg 형태를 지원해서 backbone/module을 생성. | |
| - 형태1: cfg.module.module + cfg.module.backbone | |
| - 형태2: cfg.module(_target_ 포함) + cfg.backbone | |
| - 형태3: cfg.module(_target_) 내부에서 backbone까지 생성 | |
| """ | |
| if "module" not in cfg: | |
| raise RuntimeError("cfg.module 노드가 없습니다. --cfg-path를 확인하세요.") | |
| node = cfg.module | |
| # 형태1 | |
| if (isinstance(node, dict) or ("_target_" not in node)) and ("module" in node and "backbone" in node): | |
| backbone = instantiate(node.backbone) | |
| pl_module = instantiate(node.module, backbone=backbone) | |
| return pl_module | |
| # 형태2 | |
| if "_target_" in node and "backbone" in cfg: | |
| backbone = instantiate(cfg.backbone) | |
| pl_module = instantiate(node, backbone=backbone) | |
| return pl_module | |
| # 형태3 | |
| if "_target_" in node: | |
| pl_module = instantiate(node) | |
| return pl_module | |
| raise RuntimeError("알 수 없는 cfg 구조입니다. cfg.module을 확인하세요.") | |
| # -------------------- main -------------------- | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--ckpt-path", type=str, required=True) | |
| ap.add_argument("--cfg-path", type=str, required=True) | |
| ap.add_argument("--dataset-cfg-path", type=str, required=True) | |
| ap.add_argument("--multistep", type=int, default=10) | |
| ap.add_argument("--force", action="store_true") | |
| ap.add_argument("--debug", action="store_true") | |
| args = ap.parse_args() | |
| torch.set_grad_enabled(False) | |
| device = "cuda:0" if torch.cuda.is_available() else "cpu" | |
| write_frequency = 10 | |
| # ---------- load cfgs ---------- | |
| cfg = OmegaConf.load(args.cfg_path) | |
| ds_cfg = OmegaConf.load(args.dataset_cfg_path) | |
| # ---------- build model & load ckpt ---------- | |
| pl_module = _build_model(cfg) | |
| state = torch.load(args.ckpt_path, map_location="cpu", weights_only=False) | |
| state_dict = state.get("state_dict", state) | |
| pl_module.load_state_dict(state_dict, strict=False) | |
| pl_module = pl_module.to(device).eval() | |
| if hasattr(pl_module, "use_prev"): | |
| pl_module.use_prev = True # prev LR/HR 경로 활성화 | |
| # ---------- resolve dataset config ---------- | |
| # 허용: | |
| # A) {dataloader: {dataset_hr: ..., dataset_lr: ...}} | |
| # B) {dataset_hr: ..., dataset_lr: ...} | |
| # C) {dataset: ...} (HR/LR 동일 사용) | |
| if "dataloader" in ds_cfg: | |
| dl_cfg = ds_cfg.dataloader | |
| else: | |
| dl_cfg = ds_cfg | |
| if "dataset_hr" in dl_cfg and "dataset_lr" in dl_cfg: | |
| ds_hr_node = dl_cfg.dataset_hr | |
| ds_lr_node = dl_cfg.dataset_lr | |
| elif "dataset" in dl_cfg: | |
| ds_hr_node = dl_cfg.dataset | |
| ds_lr_node = dl_cfg.dataset | |
| else: | |
| raise RuntimeError( | |
| "dataset config 형태가 다릅니다. 다음 중 하나여야 합니다:\n" | |
| " - dataloader.dataset_hr & dataloader.dataset_lr\n" | |
| " - dataset_hr & dataset_lr\n" | |
| " - dataset" | |
| ) | |
| # ---------- datasets (정규화/원시) ---------- | |
| # 정규화된 HR/LR (모델 입력용) | |
| hr_norm_ds = instantiate(ds_hr_node, domain="test", z0012=True) | |
| lr_norm_ds = instantiate(ds_lr_node, domain="test", z0012=True) | |
| # 원시 HR (denorm GT 저장/메트릭 계산용) | |
| hr_raw_ds = instantiate( | |
| ds_hr_node, | |
| domain="test", z0012=True, | |
| multistep=args.multistep, | |
| input_norm_scheme=None, | |
| output_norm_scheme=None, | |
| ) | |
| # 시간 매핑(빠른 버전) | |
| t2i_hr = _time_map_fast(hr_norm_ds) | |
| t2i_lr = _time_map_fast(lr_norm_ds) | |
| raw_loader = torch.utils.data.DataLoader(hr_raw_ds, batch_size=1, num_workers=4, shuffle=False) | |
| outs = dict(traj_surface=[], traj_level=[], timestamps=[]) | |
| err_log = [] | |
| lvl_names = hr_norm_ds.variables["state_level"] # ex) ['Z50',...,'Q1000'] | |
| srf_names = hr_norm_ds.variables["state_surface"] # ['U10','V10','T2m','SP'] | |
| # ---- per-var/time accumulators for WRMSE(JSON) ---- | |
| want_surf = ["U10", "V10", "T2m", "SP"] | |
| want_lvl = {"Z500":("Z",500), "U850":("U",850), "V850":("V",850), "T850":("T",850), "Q700":("Q",700)} | |
| pres_list = hr_norm_ds.pressure_levels # [50,...,1000] | |
| pres2idx = {int(p): i for i, p in enumerate(pres_list)} | |
| var2idx = {"Z":0, "U":1, "V":2, "T":3, "Q":4} | |
| json_sums, json_den = {}, {} | |
| def accumulate_json(key, err2_map, wmap): | |
| json_sums[key] = json_sums.get(key, 0.0) + float((err2_map * wmap).sum().item()) | |
| json_den[key] = json_den.get(key, 0.0) + float(wmap.sum().item()) | |
| first_debug_printed = False | |
| ckpt_stem = Path(args.ckpt_path).stem | |
| save_dir = Path("evalstore") / ckpt_stem | |
| save_dir.mkdir(parents=True, exist_ok=True) | |
| zarr_path = save_dir / f"test2020-multistep{args.multistep}.zarr" | |
| if zarr_path.exists() and args.force: | |
| import shutil; shutil.rmtree(zarr_path) | |
| # ---------- per-time rollout ---------- | |
| debug_steps = {1, 2, 10} # 1-based | |
| printed_steps = set() | |
| first_valid_seen = False | |
| for i, raw in tqdm(enumerate(raw_loader), total=len(raw_loader)): | |
| t0 = _time_from_sample(raw) | |
| pair = _pair_hr_lr_at_time(t0, hr_norm_ds, lr_norm_ds, t2i_hr, t2i_lr) | |
| if pair is None: | |
| continue # prev LR가 없는 시작 시점은 스킵 | |
| is_first_valid = False | |
| if not first_valid_seen: | |
| is_first_valid = True | |
| first_valid_seen = True | |
| hr_batch = {k: v for k, v in pair.items() if not k.endswith("_lr")} | |
| traj_s, traj_l = [], [] | |
| for k in range(args.multistep): | |
| pair_k = {kk: (vv.to(device) if hasattr(vv, "to") else vv) for kk, vv in pair.items()} | |
| pred = pl_module.forward(pair_k) | |
| # 디바이스 혼합 방지: denorm 계산은 CPU에서 | |
| denorm_pred = hr_norm_ds.denormalize({k2: (v2.detach().cpu() if torch.is_tensor(v2) else v2) | |
| for k2, v2 in pred.items()}, | |
| {k2: (v2.cpu() if torch.is_tensor(v2) else v2) | |
| for k2, v2 in hr_batch.items()}) | |
| traj_s.append(denorm_pred["next_state_surface"].cpu()) | |
| traj_l.append(denorm_pred["next_state_level"].cpu()) | |
| # ---- per-var/time JSON 누적 ---- | |
| s_pred = denorm_pred["next_state_surface"].squeeze(-3) # [B,4,H,W] | |
| l_pred = denorm_pred["next_state_level"] # [B,5,L,H,W] | |
| if k == 0: | |
| s_gt = raw["next_state_surface"].squeeze(-3) # [B,4,H,W] | |
| l_gt = raw["next_state_level"] # [B,5,L,H,W] | |
| else: | |
| s_gt = raw["future_state_surface"][:, k-1].squeeze(-3) | |
| l_gt = raw["future_state_level"][:, k-1] | |
| B, _, H, W = s_pred.shape | |
| wmap = _area_weights(H, W, device=s_pred.device, dtype=s_pred.dtype) # [H,W] | |
| # surface vars | |
| for vi, name in enumerate(want_surf): | |
| err2 = (s_pred[:, vi] - s_gt[:, vi]).pow(2).mean(0) # [H,W] | |
| key = f"lr2hr_H{H}xW{W}/t{k+1}/{name}" | |
| accumulate_json(key, err2, wmap) | |
| # level vars at target pressures | |
| for jname, (vtag, plevel) in want_lvl.items(): | |
| vc = var2idx[vtag] | |
| li = pres2idx[int(plevel)] | |
| err2 = (l_pred[:, vc, li] - l_gt[:, vc, li]).pow(2).mean(0) | |
| key = f"lr2hr_H{H}xW{W}/t{k+1}/{jname}" | |
| accumulate_json(key, err2, wmap) | |
| # ---- 첫 유효 스텝에서 디버그 프린트 (항상 1회) ---- | |
| if is_first_valid and ((k+1) in debug_steps) and ((k+1) not in printed_steps): | |
| print(f"\n[DEBUG — first valid rollout @ t={k+1}]") | |
| for name in ["state_surface_lr","prev_state_surface_lr","state_constant","state_level"]: | |
| if name in pair_k and hasattr(pair_k[name], "shape"): | |
| print(f" batch['{name}']:", tuple(pair_k[name].shape)) | |
| print(" pred.next_state_surface:", tuple(denorm_pred["next_state_surface"].shape)) | |
| print(" pred.next_state_level: ", tuple(denorm_pred["next_state_level"].shape)) | |
| # 간단 WRMSE 샘플 | |
| def wrmse_scalar(a, b, w): | |
| e2 = (a - b).pow(2).mean(0) # [H,W] | |
| return float(torch.sqrt((e2 * w).sum() / w.sum()).item()) | |
| u10_rmse = wrmse_scalar(s_pred[:,0], s_gt[:,0], wmap) | |
| z500_rmse= wrmse_scalar(l_pred[:,0,pres2idx[500]], l_gt[:,0,pres2idx[500]], wmap) | |
| print(f" U10@t{k+1} WRMSE: {u10_rmse:.4f}") | |
| print(f" Z500@t{k+1} WRMSE: {z500_rmse:.4f}\n") | |
| printed_steps.add(k+1) | |
| # 다음 스텝용 페어 갱신 (HR unroll + 실측 LR now/prev) | |
| merged = _merge_next_pair_from_pred(hr_norm_ds, lr_norm_ds, t2i_lr, pred, hr_batch) | |
| if merged is None: | |
| break | |
| pair = merged | |
| hr_batch = {k2: v2 for k2, v2 in pair.items() if not k2.endswith("_lr")} | |
| if not traj_s: | |
| continue # 안전장치 | |
| traj_s = torch.stack(traj_s, dim=1) # [1,T,4,1,H,W] 또는 [1,T,4,H,W] | |
| traj_l = torch.stack(traj_l, dim=1) # [1,T,5,L,H,W] | |
| outs["timestamps"].append(t0) | |
| outs["traj_surface"].append(traj_s) | |
| outs["traj_level"].append(traj_l) | |
| # headline metrics | |
| gt_traj = dict( | |
| traj_surface=torch.cat([raw["next_state_surface"][:, None], raw["future_state_surface"]], dim=1), | |
| traj_level =torch.cat([raw["next_state_level"][:, None], raw["future_state_level"]], dim=1), | |
| ) | |
| err_log.append(headline_wrmse(dict(traj_surface=traj_s, traj_level=traj_l), gt_traj, prefix="traj")) | |
| # 주기적 저장 | |
| if (i % write_frequency == 0) or (i == len(raw_loader) - 1): | |
| level_outs = torch.cat(outs["traj_level"]).cpu() | |
| level_outs = reverse_default_preprocessor(level_outs).numpy() | |
| surface_outs = torch.cat(outs["traj_surface"]).cpu() | |
| surface_outs = reverse_default_preprocessor(surface_outs).numpy() | |
| Hs, Ws = surface_outs.shape[-3], surface_outs.shape[-2] | |
| lats, lons = _latlon_from_hw(Hs, Ws) | |
| T = surface_outs.shape[1] | |
| xds = xr.Dataset( | |
| data_vars={ | |
| **{lvl_names[ii]: (["time", "prediction_timedelta", "level", "latitude", "longitude"], | |
| level_outs[:, :, ii]) | |
| for ii in range(len(lvl_names))}, | |
| **{srf_names[ii]: (["time", "prediction_timedelta", "latitude", "longitude"], | |
| surface_outs[:, :, ii, 0]) | |
| for ii in range(len(srf_names))}, | |
| }, | |
| coords=dict( | |
| time=pd.to_datetime(outs["timestamps"], utc=True, format="%Y-%m-%d-%H-%M").tz_localize(None), | |
| prediction_timedelta=[timedelta(k+1) for k in range(T)], | |
| latitude=lats, | |
| longitude=lons, | |
| level=[50,100,150,200,250,300,400,500,600,700,850,925,1000], | |
| ), | |
| ) | |
| xds.to_zarr( | |
| zarr_path, | |
| append_dim="time" if i else None, | |
| encoding=dict(time=dict(units="hours since 2000-01-01")) if not i else None, | |
| ) | |
| # running errors | |
| if err_log: | |
| avg_err = {k: torch.cat([e[k] for e in err_log], dim=0).mean(0) | |
| for k in err_log[0].keys()} | |
| print("[running headline_wrmse]", | |
| {k: (v.item() if torch.numel(v)==1 else v.mean().item()) | |
| for k, v in avg_err.items()}) | |
| outs = dict(traj_surface=[], traj_level=[], timestamps=[]) | |
| if args.debug: | |
| break | |
| # 최종 headline 저장 | |
| if err_log: | |
| avg_err = {k: torch.cat([e[k] for e in err_log], dim=0).mean(0) | |
| for k in err_log[0].keys()} | |
| torch.save(avg_err, save_dir / f"allmetrics-multistep{args.multistep}.pt") | |
| print("[final headline_wrmse]", | |
| {k: (v.item() if torch.numel(v)==1 else v.mean().item()) | |
| for k, v in avg_err.items()}) | |
| # ---- JSON 출력 (per-var/time WRMSE) ---- | |
| json_out = {} | |
| for key in sorted(json_sums.keys()): | |
| rmse = np.sqrt(json_sums[key] / max(json_den[key], 1e-12)) | |
| json_out[key] = float(rmse) | |
| json_path = save_dir / f"pervar_rmse_t{args.multistep}.json" | |
| with open(json_path, "w") as f: | |
| json.dump(json_out, f, indent=2) | |
| print(f"[saved JSON] {json_path}") | |
| if __name__ == "__main__": | |
| main() | |