"""X21 validation of an F-generator checkpoint on the 40 X20 pilot plants (seed 7171, disjoint from the X21 training plants). Per plant: K filters sampled with c = 0 (ask for the plant's best), phi = sat_1. A candidate counts iff F is stable with the dataset's margin (f_ok, 0.01) and the loop is certified at the pilot's s1 = 0.5 sigma_p (regional circle criterion, rstar); certified candidates are costed at x0 = s1 D (siso_x19 cost_batch). Best-of-b (b = 1, 4, 16) is compared with the F-only per-plant optimum (X20b), the joint (F, phi) reference and the tuned linear baselines (X20 J_lin). Run: PYTHONPATH=. python -m nonlinear.experiments.x21_eval [guidance ...] (guidance 1 = conditional only) may also be retrieval, retrieval_cl or bl: (X22 baselines, x21_baselines.py). Env: X21_EVAL_WORKERS (default 16), X21_K (samples per plant, default 16), X21_TREPAIR=1 (null-space F repair). Appends one block per (ckpt, guidance) to results/icml_r3/X21/val_report.txt (X21_REPORT overrides). X21_TEST=1: the fresh test set instead (stage testset, seed 8282; J_fonly = best label, J_lin = min(static gain, tuned LQG), no joint reference) -> results/icml_r3/X21/test/ and test_report.txt. """ import os, sys, time, types, glob import numpy as np import torch import concurrent.futures as cf from diffusers import DDPMScheduler from utils.models.ConditionalDiffusion_mlp import ReverseDiffusionModel from utils.samplers import sample_diffusion from nonlinear.experiments import siso_x20_unstable as X from nonlinear.training.train_x21 import islog, slog, cl_decode, cl_encode, sylvester K, SEED, OUT = int(os.environ.get("X21_K", 16)), 4242, "results/icml_r3/X21/val" TEST = os.environ.get("X21_TEST") == "1" if TEST: OUT = "results/icml_r3/X21/test_eval" def refs(): if TEST: Z = [z for z in map(np.load, sorted(glob.glob(f"{X.DATA_DIR}/test/plant_*.npz"))) if not bool(z["dropped"]) and int(z["M"])] # M = 0: no F-only reference J_lqg = np.array([np.where(z["lqg_cert"], z["lqg_J"], np.inf).min() for z in Z]) return dict(i=np.array([int(z["i"]) for z in Z]), stratum=np.array([str(z["stratum"]) for z in Z]), g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=np.array([float(z["s"]) for z in Z]), J_ref=np.full(len(Z), np.nan), J_lin=np.minimum([z["sg_J"].min() for z in Z], J_lqg), J_lqg=J_lqg, J_fonly=np.array([z["J"].min() if int(z["M"]) else np.inf for z in Z])) T = X.table() Z = [np.load(f"{X.OUT}/pilot/plant_{i:02d}.npz") for i in T["i"]] J_fonly = np.array([float(np.load(f"{X.OUT}/ablate/plant_{i:02d}.npz")["fonly_J"]) for i in T["i"]]) return dict(i=T["i"], g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=T["s1"], J_ref=T["J_ref_s0"], J_lin=T["J_lin_s0"], J_fonly=J_fonly) def sample(path, G, guidance, dev): ck = torch.load(path, map_location=dev, weights_only=False) cfg = types.SimpleNamespace(**ck["cfg"], device=dev) m = ReverseDiffusionModel(cfg).to(dev) m.load_state_dict(ck["ema"]); m.eval() st = ck["stats"] cond = np.concatenate([(G - st["g_mu"]) / st["g_sd"], np.zeros((len(G), 1))], 1) cond = torch.tensor(np.repeat(cond, K, 0), dtype=torch.float32, device=dev) if ck.get("loss") == "reg": # regression twin: candidate 1 deterministic, the rest MC-dropout draws z, t0 = torch.zeros(len(cond), cfg.output_dim, device=dev), torch.zeros(len(cond), 1, device=dev, dtype=torch.long) with torch.no_grad(): x = m(cond, z, t0) m.train(); torch.manual_seed(SEED) mc = m(cond, z, t0); m.eval() x = torch.where((torch.arange(len(cond), device=dev) % K == 0)[:, None], x, mc) else: ns = DDPMScheduler(num_train_timesteps=cfg.num_timesteps, beta_schedule="squaredcos_cap_v2") gen = torch.Generator(device=dev).manual_seed(SEED) x = sample_diffusion(m, cond, (len(cond), cfg.output_dim), ns, device=dev, use_cfg=guidance != 1, guidance_scale=guidance, generator=gen) y = (x.cpu().numpy() * st["F_sd"] + st["F_mu"]).reshape(len(G), K, -1) if str(st.get("rep", "coef")) == "cl": return np.array([[np.concatenate(cl_decode(G[p], yk)) for yk in y[p]] for p in range(len(G))]), int(ck["step"]) return islog(y), int(ck["step"]) def retrieve(G, cl=False): """Look-up baseline: the labels of the nearest training plant (standardized g, Euclidean), cheapest first, topped up from the next nearest plants to K.""" T = np.load("results/icml_r3/X21/train.npz") mu, sd = T["g"].mean(0), np.where(T["g"].std(0) < 1e-8, 1.0, T["g"].std(0)) out = [] for g in G: order = np.argsort((((T["g"] - g) / sd) ** 2).sum(1)) rows = [] for p in order: k = np.argsort(T["J"][p])[: int(T["M"][p])] rows += [np.concatenate(cl_decode(g, cl_encode(T["g"][p], T["num"][p, j], T["den"][p, j])) if cl else [T["num"][p, j], T["den"][p, j]]) for j in k] if len(rows) >= K: break out.append(np.stack(rows[:K])) return np.stack(out), 0 TREPAIR = os.environ.get("X21_TREPAIR", "0") == "1" T_GRID = np.concatenate([-np.logspace(-3, 5, 200)[::-1], [0.0], np.logspace(-3, 5, 200)]) def t_repair(g, f): """Null-space repair, no verifier call: theta = theta_0 + t v keeps the linearized closed-loop polynomial (v spans the null space of the plant's Sylvester map); if F is unstable, move t to the grid point nearest the current one (in slog) that makes F stable with the 0.01 margin. Returns (f, repaired).""" if X.f_ok(f[:5], f[5:], 0.01): return f, False v = sylvester(g)[2] t0 = v @ f th0 = f - t0 * v ok = [t for t in T_GRID if X.f_ok((th0 + t * v)[:5], (th0 + t * v)[5:], 0.01)] if not ok: return f, False t = min(ok, key=lambda t: abs(slog(t) - slog(t0))) return th0 + t * v, True def score(args): """One plant: (certified [K], J [K], inf where not certified).""" g, F, s, D = args torch.set_num_threads(1) cert, loops = np.zeros(len(F), bool), {} for k, f in enumerate(F): if TREPAIR: f = t_repair(g, f)[0] num, den = f[:5], f[5:] if not X.f_ok(num, den, 0.01): continue r = X.realize(g, X.f5(num, den)) if r is not None and X.rstar(*r[:3], X.PHI_SAT, D, s=s) >= s: cert[k], loops[k] = True, r J = np.full(len(F), np.inf) if loops: ks = sorted(loops) J[ks] = X.cost_batch([(*loops[k], X.PHI_SAT) for k in ks], [s * D] * len(ks))[0] return cert, J def main(path, *guid): dev = "cuda" if torch.cuda.is_available() else "cpu" R = refs() os.makedirs(OUT, exist_ok=True) L = [] for w in [float(x) for x in guid] or [1.0]: t0 = time.time() if path.startswith("bl:"): # X22 baselines (x21_baselines.py) from nonlinear.experiments.x21_baselines import candidates F, step = candidates(path[3:], R["g"]), 0 else: F, step = retrieve(R["g"], cl=path == "retrieval_cl") if path.startswith("retrieval") else sample(path, R["g"], w, dev) ts = time.time() - t0 with cf.ProcessPoolExecutor(int(os.environ.get("X21_EVAL_WORKERS", 16))) as ex: res = list(ex.map(score, [(R["g"][p], F[p], R["s"][p], R["D"][p]) for p in range(len(F))])) cert, J = np.stack([r[0] for r in res]), np.stack([r[1] for r in res]) tag = path.replace(":", "_") if path.startswith(("retrieval", "bl:")) else f"{os.path.basename(os.path.dirname(path))}_{os.path.basename(path)[:-4]}_w{w:g}" tag += ("_tr" if TREPAIR else "") + (f"_K{K}" if K != 16 else "") np.savez_compressed(f"{OUT}/{tag}.npz", F=F, cert=cert, J=J, **{f"ref_{k}": v for k, v in R.items()}) L.append(f"== {path} (step {step}){' + t-repair' if TREPAIR else ''}, guidance {w:g}, K = {K}, {len(F)} plants; sampling {ts:.0f} s, scoring {time.time() - t0 - ts:.0f} s") L.append(f" certified per candidate {cert.mean() * 100:.1f}%; F unstable or not certified otherwise") for b in [b for b in (1, 4, 16, 64) if b <= K]: Jb = J[:, :b].min(1) d = np.where(np.isfinite(Jb), (Jb - R["J_fonly"]) / R["J_fonly"], 1e3) med, lo, hi = X.boot_ci(d, np.median) L.append(f" b={b:2d}: plants with a certified candidate {np.isfinite(Jb).mean() * 100:5.1f}% | regret vs F-only optimum median " f"{med * 100:7.2f}% [{lo * 100:.2f}, {hi * 100:.2f}] | beats tuned linear on {np.mean(Jb < R['J_lin']) * 100:5.1f}% | " f"median (J_lin - J_b)/J_b {np.median(np.where(np.isfinite(Jb), (R['J_lin'] - Jb) / Jb, -1)) * 100:6.1f}%") open(os.environ.get("X21_REPORT", f"results/icml_r3/X21/{'test' if TEST else 'val'}_report.txt"), "a").write("\n".join(L) + "\n") print("\n".join(L), flush=True) if __name__ == "__main__": main(*sys.argv[1:])