File size: 9,097 Bytes
c4be3a9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | """X21 validation of an F-generator checkpoint on the 40 X20 pilot plants (seed 7171, disjoint from the X21 training plants).
Per plant: K filters sampled with c = 0 (ask for the plant's best), phi = sat_1. A candidate counts iff F is stable with the
dataset's margin (f_ok, 0.01) and the loop is certified at the pilot's s1 = 0.5 sigma_p (regional circle criterion, rstar);
certified candidates are costed at x0 = s1 D (siso_x19 cost_batch). Best-of-b (b = 1, 4, 16) is compared with the F-only
per-plant optimum (X20b), the joint (F, phi) reference and the tuned linear baselines (X20 J_lin).
Run: PYTHONPATH=. python -m nonlinear.experiments.x21_eval <ckpt.pth> [guidance ...] (guidance 1 = conditional only)
<ckpt.pth> may also be retrieval, retrieval_cl or bl:<arm> (X22 baselines, x21_baselines.py).
Env: X21_EVAL_WORKERS (default 16), X21_K (samples per plant, default 16), X21_TREPAIR=1 (null-space F repair). Appends one block per (ckpt, guidance) to results/icml_r3/X21/val_report.txt (X21_REPORT overrides).
X21_TEST=1: the fresh test set instead (stage testset, seed 8282; J_fonly = best label, J_lin = min(static gain, tuned
LQG), no joint reference) -> results/icml_r3/X21/test/ and test_report.txt.
"""
import os, sys, time, types, glob
import numpy as np
import torch
import concurrent.futures as cf
from diffusers import DDPMScheduler
from utils.models.ConditionalDiffusion_mlp import ReverseDiffusionModel
from utils.samplers import sample_diffusion
from nonlinear.experiments import siso_x20_unstable as X
from nonlinear.training.train_x21 import islog, slog, cl_decode, cl_encode, sylvester
K, SEED, OUT = int(os.environ.get("X21_K", 16)), 4242, "results/icml_r3/X21/val"
TEST = os.environ.get("X21_TEST") == "1"
if TEST:
OUT = "results/icml_r3/X21/test_eval"
def refs():
if TEST:
Z = [z for z in map(np.load, sorted(glob.glob(f"{X.DATA_DIR}/test/plant_*.npz"))) if not bool(z["dropped"]) and int(z["M"])] # M = 0: no F-only reference
J_lqg = np.array([np.where(z["lqg_cert"], z["lqg_J"], np.inf).min() for z in Z])
return dict(i=np.array([int(z["i"]) for z in Z]), stratum=np.array([str(z["stratum"]) for z in Z]),
g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=np.array([float(z["s"]) for z in Z]),
J_ref=np.full(len(Z), np.nan), J_lin=np.minimum([z["sg_J"].min() for z in Z], J_lqg), J_lqg=J_lqg,
J_fonly=np.array([z["J"].min() if int(z["M"]) else np.inf for z in Z]))
T = X.table()
Z = [np.load(f"{X.OUT}/pilot/plant_{i:02d}.npz") for i in T["i"]]
J_fonly = np.array([float(np.load(f"{X.OUT}/ablate/plant_{i:02d}.npz")["fonly_J"]) for i in T["i"]])
return dict(i=T["i"], g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=T["s1"],
J_ref=T["J_ref_s0"], J_lin=T["J_lin_s0"], J_fonly=J_fonly)
def sample(path, G, guidance, dev):
ck = torch.load(path, map_location=dev, weights_only=False)
cfg = types.SimpleNamespace(**ck["cfg"], device=dev)
m = ReverseDiffusionModel(cfg).to(dev)
m.load_state_dict(ck["ema"]); m.eval()
st = ck["stats"]
cond = np.concatenate([(G - st["g_mu"]) / st["g_sd"], np.zeros((len(G), 1))], 1)
cond = torch.tensor(np.repeat(cond, K, 0), dtype=torch.float32, device=dev)
if ck.get("loss") == "reg": # regression twin: candidate 1 deterministic, the rest MC-dropout draws
z, t0 = torch.zeros(len(cond), cfg.output_dim, device=dev), torch.zeros(len(cond), 1, device=dev, dtype=torch.long)
with torch.no_grad():
x = m(cond, z, t0)
m.train(); torch.manual_seed(SEED)
mc = m(cond, z, t0); m.eval()
x = torch.where((torch.arange(len(cond), device=dev) % K == 0)[:, None], x, mc)
else:
ns = DDPMScheduler(num_train_timesteps=cfg.num_timesteps, beta_schedule="squaredcos_cap_v2")
gen = torch.Generator(device=dev).manual_seed(SEED)
x = sample_diffusion(m, cond, (len(cond), cfg.output_dim), ns, device=dev, use_cfg=guidance != 1, guidance_scale=guidance, generator=gen)
y = (x.cpu().numpy() * st["F_sd"] + st["F_mu"]).reshape(len(G), K, -1)
if str(st.get("rep", "coef")) == "cl":
return np.array([[np.concatenate(cl_decode(G[p], yk)) for yk in y[p]] for p in range(len(G))]), int(ck["step"])
return islog(y), int(ck["step"])
def retrieve(G, cl=False):
"""Look-up baseline: the labels of the nearest training plant (standardized g, Euclidean), cheapest first, topped up
from the next nearest plants to K."""
T = np.load("results/icml_r3/X21/train.npz")
mu, sd = T["g"].mean(0), np.where(T["g"].std(0) < 1e-8, 1.0, T["g"].std(0))
out = []
for g in G:
order = np.argsort((((T["g"] - g) / sd) ** 2).sum(1))
rows = []
for p in order:
k = np.argsort(T["J"][p])[: int(T["M"][p])]
rows += [np.concatenate(cl_decode(g, cl_encode(T["g"][p], T["num"][p, j], T["den"][p, j])) if cl else
[T["num"][p, j], T["den"][p, j]]) for j in k]
if len(rows) >= K:
break
out.append(np.stack(rows[:K]))
return np.stack(out), 0
TREPAIR = os.environ.get("X21_TREPAIR", "0") == "1"
T_GRID = np.concatenate([-np.logspace(-3, 5, 200)[::-1], [0.0], np.logspace(-3, 5, 200)])
def t_repair(g, f):
"""Null-space repair, no verifier call: theta = theta_0 + t v keeps the linearized closed-loop polynomial (v spans the
null space of the plant's Sylvester map); if F is unstable, move t to the grid point nearest the current one (in slog)
that makes F stable with the 0.01 margin. Returns (f, repaired)."""
if X.f_ok(f[:5], f[5:], 0.01):
return f, False
v = sylvester(g)[2]
t0 = v @ f
th0 = f - t0 * v
ok = [t for t in T_GRID if X.f_ok((th0 + t * v)[:5], (th0 + t * v)[5:], 0.01)]
if not ok:
return f, False
t = min(ok, key=lambda t: abs(slog(t) - slog(t0)))
return th0 + t * v, True
def score(args):
"""One plant: (certified [K], J [K], inf where not certified)."""
g, F, s, D = args
torch.set_num_threads(1)
cert, loops = np.zeros(len(F), bool), {}
for k, f in enumerate(F):
if TREPAIR:
f = t_repair(g, f)[0]
num, den = f[:5], f[5:]
if not X.f_ok(num, den, 0.01):
continue
r = X.realize(g, X.f5(num, den))
if r is not None and X.rstar(*r[:3], X.PHI_SAT, D, s=s) >= s:
cert[k], loops[k] = True, r
J = np.full(len(F), np.inf)
if loops:
ks = sorted(loops)
J[ks] = X.cost_batch([(*loops[k], X.PHI_SAT) for k in ks], [s * D] * len(ks))[0]
return cert, J
def main(path, *guid):
dev = "cuda" if torch.cuda.is_available() else "cpu"
R = refs()
os.makedirs(OUT, exist_ok=True)
L = []
for w in [float(x) for x in guid] or [1.0]:
t0 = time.time()
if path.startswith("bl:"): # X22 baselines (x21_baselines.py)
from nonlinear.experiments.x21_baselines import candidates
F, step = candidates(path[3:], R["g"]), 0
else:
F, step = retrieve(R["g"], cl=path == "retrieval_cl") if path.startswith("retrieval") else sample(path, R["g"], w, dev)
ts = time.time() - t0
with cf.ProcessPoolExecutor(int(os.environ.get("X21_EVAL_WORKERS", 16))) as ex:
res = list(ex.map(score, [(R["g"][p], F[p], R["s"][p], R["D"][p]) for p in range(len(F))]))
cert, J = np.stack([r[0] for r in res]), np.stack([r[1] for r in res])
tag = path.replace(":", "_") if path.startswith(("retrieval", "bl:")) else f"{os.path.basename(os.path.dirname(path))}_{os.path.basename(path)[:-4]}_w{w:g}"
tag += ("_tr" if TREPAIR else "") + (f"_K{K}" if K != 16 else "")
np.savez_compressed(f"{OUT}/{tag}.npz", F=F, cert=cert, J=J, **{f"ref_{k}": v for k, v in R.items()})
L.append(f"== {path} (step {step}){' + t-repair' if TREPAIR else ''}, guidance {w:g}, K = {K}, {len(F)} plants; sampling {ts:.0f} s, scoring {time.time() - t0 - ts:.0f} s")
L.append(f" certified per candidate {cert.mean() * 100:.1f}%; F unstable or not certified otherwise")
for b in [b for b in (1, 4, 16, 64) if b <= K]:
Jb = J[:, :b].min(1)
d = np.where(np.isfinite(Jb), (Jb - R["J_fonly"]) / R["J_fonly"], 1e3)
med, lo, hi = X.boot_ci(d, np.median)
L.append(f" b={b:2d}: plants with a certified candidate {np.isfinite(Jb).mean() * 100:5.1f}% | regret vs F-only optimum median "
f"{med * 100:7.2f}% [{lo * 100:.2f}, {hi * 100:.2f}] | beats tuned linear on {np.mean(Jb < R['J_lin']) * 100:5.1f}% | "
f"median (J_lin - J_b)/J_b {np.median(np.where(np.isfinite(Jb), (R['J_lin'] - Jb) / Jb, -1)) * 100:6.1f}%")
open(os.environ.get("X21_REPORT", f"results/icml_r3/X21/{'test' if TEST else 'val'}_report.txt"), "a").write("\n".join(L) + "\n")
print("\n".join(L), flush=True)
if __name__ == "__main__":
main(*sys.argv[1:])
|