File size: 12,111 Bytes
6848dc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | """Sun*+ ライブラリ: 全国版 特徴量生成 (Sun* V2 の特徴量定義を完全に含み, 全国向け特徴を追加)。
出力の特徴列は先頭が V2 と同名同順 (V2 モデルを同じ特徴行列で評価可能)、その後ろに Sun*+ 追加特徴が続く。"""
from pathlib import Path
import numpy as np, pandas as pd, xarray as xr
from scipy.ndimage import uniform_filter
HOLD_V2 = ["0658", "0664", "0670"]
REGIME_B_START = "2024-10-01"
VARS6 = ["temp_c", "wind_u_ms", "wind_v_ms", "humidity_pct", "wind_speed_ms", "precip_mm"]
TERRAIN_VARS = ["elevation_m", "slope_deg", "aspect_sin", "aspect_cos", "tpi_500m", "tpi_2km", "tpi_10km", "relief_1km", "relief_5km",
"coast_dist_km", "land_mask", "lc_tree_frac", "lc_crop_frac", "lc_built_frac", "lc_water_frac", "lc_grass_frac", "lc_shrub_frac"]
ERA_VARS = ["2m_temperature_c", "2m_dewpoint_temperature_c", "10m_u_component_of_wind", "10m_v_component_of_wind",
"100m_u_component_of_wind", "100m_v_component_of_wind", "surface_pressure_hpa", "total_precipitation",
"surface_solar_radiation_downwards", "total_cloud_cover", "low_cloud_cover", "medium_cloud_cover", "high_cloud_cover",
"total_column_water_vapour", "convective_available_potential_energy", "mean_sea_level_pressure_hpa",
"skin_temperature_c", "instantaneous_10m_wind_gust", "total_sky_direct_solar_radiation_at_surface"]
REGION = {**{p: 0 for p in range(11, 25)}, **{p: 1 for p in range(31, 37)}, **{p: 2 for p in range(40, 47)}, 48: 3, 49: 3, 54: 3,
55: 4, 56: 4, 57: 4, 50: 5, 51: 5, 52: 5, 53: 5, **{p: 6 for p in range(60, 66)}, **{p: 7 for p in (66, 67, 68, 69, 81)},
**{p: 8 for p in range(71, 75)}, **{p: 9 for p in range(82, 89)}, 91: 10}
def rh_from(t_c, td_c):
a, b = 17.625, 243.04
return np.clip(100 * np.exp(a * td_c / (b + td_c)) / np.exp(a * t_c / (b + t_c)), 0, 100)
# ---------------------------------------------------------------- stations
def load_stations(csv_path):
"""data/stations_nationwide.csv (地点メタ + 地形特徴) を読む"""
st = pd.read_csv(csv_path, dtype={"block_no": str})
st["region"] = st["region"].astype(int)
return st
# ---------------------------------------------------------------- ERA5 (Japan grid)
class Grid:
"""ERA5 0.25° 格子の静的場。Grid(static_nc) もしくは Grid.from_arrays(lat, lon, orog_m, lsm, sdor, hv, lv)"""
def __init__(self, static_nc=None):
if static_nc is None: return
st = xr.open_dataset(static_nc).load()
self.lat_c = st.latitude.values.astype(np.float64); self.lon_c = st.longitude.values.astype(np.float64)
self.orog = (st["geopotential_at_surface"].values / 9.80665).astype(np.float32)
self.lsm = st["land_sea_mask"].values.astype(np.float32); self.sdor = st["standard_deviation_of_orography"].values.astype(np.float32)
self.hv = st["high_vegetation_cover"].values.astype(np.float32); self.lv = st["low_vegetation_cover"].values.astype(np.float32)
@classmethod
def from_arrays(cls, lat, lon, orog_m, lsm, sdor, hv, lv):
g = cls(); g.lat_c = np.asarray(lat, np.float64); g.lon_c = np.asarray(lon, np.float64)
g.orog, g.lsm, g.sdor, g.hv, g.lv = [np.asarray(x, np.float32) for x in (orog_m, lsm, sdor, hv, lv)]
return g
def points(self, lat, lon, elev):
lat = np.asarray(lat, np.float64); lon = np.asarray(lon, np.float64)
fy = (self.lat_c[0] - lat) / 0.25; fx = (lon - self.lon_c[0]) / 0.25
iy0 = np.clip(np.floor(fy).astype(int), 1, len(self.lat_c) - 3); ix0 = np.clip(np.floor(fx).astype(int), 1, len(self.lon_c) - 3)
dy = np.clip(fy - iy0, 0, 1); dx = np.clip(fx - ix0, 0, 1)
return dict(iy0=iy0, ix0=ix0, dy=dy, dx=dx, lat=lat, lon=lon, elev=np.asarray(elev, np.float64))
# ---------------------------------------------------------------- feature helpers (same as V2)
def bil(f, p):
iy, ix, dy, dx = p["iy0"], p["ix0"], p["dy"], p["dx"]
g = lambda y, x: f[..., y, x]
return (g(iy, ix) * (1 - dy) * (1 - dx) + g(iy, ix + 1) * (1 - dy) * dx + g(iy + 1, ix) * dy * (1 - dx) + g(iy + 1, ix + 1) * dy * dx)
def shift(a, k):
out = np.full_like(a, np.nan)
if k > 0: out[:-k] = a[k:]
elif k < 0: out[-k:] = a[:k]
else: out[:] = a
return out
def roll_mean(a, lo, hi):
acc = np.zeros_like(a, dtype=np.float32); n = np.zeros_like(a, dtype=np.float32)
for k in range(lo, hi + 1):
s = shift(a, k); m = np.isfinite(s); acc += np.where(m, s, 0); n += m
return acc / np.maximum(n, 1)
def solar_elev(times, lat, lon):
doy = times.dayofyear.values.astype(np.float64)[:, None]; hr = (times.hour.values + 0.0)[:, None]
g = 2 * np.pi / 365.0 * (doy - 1 + (hr - 12) / 24)
decl = 0.006918 - 0.399912 * np.cos(g) + 0.070257 * np.sin(g) - 0.006758 * np.cos(2 * g) + 0.000907 * np.sin(2 * g)
eqt = 229.18 * (0.000075 + 0.001868 * np.cos(g) - 0.032077 * np.sin(g) - 0.014615 * np.cos(2 * g) - 0.040849 * np.sin(2 * g))
tst = hr * 60 + eqt + 4 * lon[None]; ha = np.deg2rad(tst / 4 - 180)
la = np.deg2rad(lat[None])
sin_e = np.sin(la) * np.sin(decl) + np.cos(la) * np.cos(decl) * np.cos(ha)
return np.rad2deg(np.arcsin(np.clip(sin_e, -1, 1)))
class StaticSet:
"""per-station static arrays derived from the station table + ERA5 static grid"""
def __init__(self, st, grid):
self.st = st; self.grid = grid
self.p = grid.points(st.lat.values, st.lon.values, st.elevation_m.values)
self.terr = {v: st["t_" + v].values.astype(np.float32) for v in TERRAIN_VARS}
self.r10 = st.t_elev_mean_r10.values.astype(np.float32); self.r40 = st.t_elev_mean_r40.values.astype(np.float32)
def build_features(times, E, ss, sl=None):
"""times (T,), E (T,H,W) dict covering times, ss StaticSet. returns F (T,S,nf), names, base dict (T,S).
Contexts use +-24 h shifts: caller must pad by >=24 h and crop."""
p = ss.p; g = ss.grid; S = len(p["lat"]); T = len(times)
names, cols = [], []
def add(n, a):
names.append(n); cols.append(np.asarray(a, np.float32).reshape(T, S) if np.ndim(a) == 2 else np.broadcast_to(np.asarray(a, np.float32)[None], (T, S)))
I = {k: bil(v, p) for k, v in E.items()}
t2, d2, u10, v10 = I["2m_temperature_c"], I["2m_dewpoint_temperature_c"], I["10m_u_component_of_wind"], I["10m_v_component_of_wind"]
u100, v100 = I["100m_u_component_of_wind"], I["100m_v_component_of_wind"]
tp = np.maximum(I["total_precipitation"], 0); ssrd = I["surface_solar_radiation_downwards"]
rh = rh_from(t2, d2); spd10 = np.hypot(u10, v10); spd100 = np.hypot(u100, v100)
elev = p["elev"]; orog_i = bil(g.orog, p); dz = (elev - orog_i).astype(np.float32)
base = dict(t2m=t2, u10=u10, v10=v10, rh=rh, tp=tp, spd10=spd10, d2m=d2)
for n, a in [("t2m", t2), ("d2m", d2), ("u10", u10), ("v10", v10), ("u100", u100), ("v100", v100), ("spd10", spd10), ("spd100", spd100),
("rh", rh), ("dd", t2 - d2), ("sp", I["surface_pressure_hpa"]), ("msl", I["mean_sea_level_pressure_hpa"]), ("tp", tp), ("ssrd", ssrd / 3600.0),
("fdir", I["total_sky_direct_solar_radiation_at_surface"] / 3600.0), ("tcc", I["total_cloud_cover"]), ("lcc", I["low_cloud_cover"]),
("mcc", I["medium_cloud_cover"]), ("hcc", I["high_cloud_cover"]), ("tcwv", I["total_column_water_vapour"]),
("cape", np.sqrt(np.maximum(I["convective_available_potential_energy"], 0))), ("skt", I["skin_temperature_c"]), ("skt_t2", I["skin_temperature_c"] - t2),
("gust", I["instantaneous_10m_wind_gust"]),
("shear", spd100 - spd10), ("t2m_lapse", t2 - 0.0065 * dz[None])]:
add(n, a)
add("wdir_s", u10 / np.maximum(spd10, 1e-3)); add("wdir_c", v10 / np.maximum(spd10, 1e-3))
for n, a in [("t2m", t2), ("skt", I["skin_temperature_c"]), ("u10", u10), ("v10", v10), ("spd10", spd10), ("tcc", I["total_cloud_cover"]), ("rh", rh), ("ssrd", ssrd / 3600.0), ("tp", tp)]:
ks = (-3, -1, 1, 3) if n not in ("t2m", "rh") else (-6, -3, -1, 1, 3, 6)
for k in ks:
s = shift(a, k); add(f"{n}_d{k:+d}", s - a if n not in ("tp", "ssrd", "tcc") else s)
add("t2m_mean24", roll_mean(t2, -12, 12) - t2); add("t2m_prev24", roll_mean(t2, -24, -1) - t2)
tpa = tp
add("tp_sum_p3", sum(shift(tpa, -k) for k in (1, 2, 3))); add("tp_sum_f3", sum(shift(tpa, k) for k in (1, 2, 3))); add("tp_sum_p12", roll_mean(tpa, -12, -1) * 12)
add("tcc_mean6", roll_mean(I["total_cloud_cover"], -3, 3)); add("ssrd_mean6", roll_mean(ssrd / 3600.0, -3, 3))
tm = np.stack([shift(t2, k) for k in range(-12, 13)], 0); add("t2m_max24", np.nanmax(tm, 0) - t2); add("t2m_min24", np.nanmin(tm, 0) - t2); del tm
iy, ix = p["iy0"], p["ix0"]
def nb(f, fn):
patch = np.stack([f[:, iy + a, ix + b] for a in (0, 1) for b in (0, 1)], 0); return fn(patch)
tpf = np.maximum(E["total_precipitation"], 0)
add("tp_nbmax", nb(tpf, lambda a: a.max(0))); add("tp_nbmin", nb(tpf, lambda a: a.min(0)))
wide = lambda f, fn: fn(np.stack([f[:, np.clip(iy + a, 0, f.shape[1] - 1), np.clip(ix + b, 0, f.shape[2] - 1)] for a in (-1, 0, 1, 2) for b in (-1, 0, 1, 2)], 0))
add("tp_wmean", wide(tpf, lambda a: a.mean(0))); add("tp_wmax", wide(tpf, lambda a: a.max(0)))
add("cape_wmax", wide(np.sqrt(np.maximum(E["convective_available_potential_energy"], 0)), lambda a: a.max(0)))
t2f = E["2m_temperature_c"]; add("t2m_nbspread", nb(t2f, lambda a: a.max(0) - a.min(0)))
add("t2m_gy", bil(np.gradient(t2f, axis=1), p)); add("t2m_gx", bil(np.gradient(t2f, axis=2), p))
add("tcc_wmean", wide(E["total_cloud_cover"], lambda a: a.mean(0)))
add("elev", elev); add("dz", dz); add("orog_era", orog_i); add("lsm", bil(g.lsm, p)); add("sdor", bil(g.sdor, p)); add("hv", bil(g.hv, p)); add("lv", bil(g.lv, p))
for v in TERRAIN_VARS: add("terr_" + v, ss.terr[v])
add("elev_mean_r10", ss.r10); add("elev_mean_r40", ss.r40)
hh = times.hour.values[:, None] * np.ones((1, S)); doy = times.dayofyear.values[:, None] * np.ones((1, S))
add("hr_s", np.sin(2 * np.pi * hh / 24)); add("hr_c", np.cos(2 * np.pi * hh / 24)); add("doy_s", np.sin(2 * np.pi * doy / 365.25)); add("doy_c", np.cos(2 * np.pi * doy / 365.25))
add("hr_utc", hh)
sel = solar_elev(times, p["lat"], p["lon"]); add("sun_elev", sel)
asp_s = ss.terr["aspect_sin"][None]; asp_c = ss.terr["aspect_cos"][None]; sl_ = ss.terr["slope_deg"][None]
add("down_slope_flow", (u10 * asp_s + v10 * asp_c) * sl_)
add("flow_x_tpi2k", spd10 * ss.terr["tpi_2km"][None]); add("flow_x_tpi10k", spd10 * ss.terr["tpi_10km"][None])
add("night_x_tpi2k", np.maximum(-sel, 0) * ss.terr["tpi_2km"][None]); add("night_x_dz", np.maximum(-sel, 0) / 90 * dz[None])
n_v2 = len(names)
# ------------- Sun*+ 追加特徴 (全国版) -------------
st = ss.st
for n, col in [("lat", "lat"), ("lon", "lon"), ("region", "region"), ("t_elev_max_r10", "t_elev_max_r10"), ("t_elev_min_r40", "t_elev_min_r40"),
("t_sea_frac_20km", "t_sea_frac_20km"), ("t_sea_frac_50km", "t_sea_frac_50km"), ("t_sea_frac_150km", "t_sea_frac_150km")]:
add("x_" + n, st[col].values)
add("x_dem_minus_stn", (st.t_dem_pt_m.values - st.elevation_m.values))
# 地形 x 気象 の追加相互作用 (沿岸・斜面)
cd = ss.terr["coast_dist_km"][None]
add("x_coast_x_sun", np.clip(cd, -20, 20) * np.maximum(sel, 0) / 90) # 日射と海岸距離 (海風)
add("x_coast_x_spd", np.exp(-np.abs(cd) / 10) * spd10) # 海岸付近の風
add("x_t2m_dem", t2 - 0.0065 * (st.t_elevation_m.values - orog_i)[None]) # DEM標高での気温補正
add("x_valley_night", np.maximum(-sel, 0) / 90 * ss.terr["tpi_10km"][None])
add("x_cold_pool", np.maximum(-sel, 0) / 90 * (st.t_elev_min_r40.values[None]) * (1.0 / (1 + spd10)))
add("x_abs_lat_x_doy", np.abs(st.lat.values[None] - 36) * np.cos(2 * np.pi * (doy - 15) / 365.25))
F = np.stack(cols, 2).astype(np.float32)
return F, names, base, n_v2
|