Download andromeda_variants/nonlinear/experiments/x21_eval.py from matCercola18/DiffusionControl: direct link, hf CLI and curl.
- Browser
- Download file 9.1 kB
-
https://huggingface.co/matCercola18/DiffusionControl/resolve/main/andromeda_variants/nonlinear/experiments/x21_eval.py
- Command line
-
hf download hf://matCercola18/DiffusionControl/andromeda_variants/nonlinear/experiments/x21_eval.py
-
curl -L -o x21_eval.py https://huggingface.co/matCercola18/DiffusionControl/resolve/main/andromeda_variants/nonlinear/experiments/x21_eval.py
9.1 kB
| """X21 validation of an F-generator checkpoint on the 40 X20 pilot plants (seed 7171, disjoint from the X21 training plants). | |
| Per plant: K filters sampled with c = 0 (ask for the plant's best), phi = sat_1. A candidate counts iff F is stable with the | |
| dataset's margin (f_ok, 0.01) and the loop is certified at the pilot's s1 = 0.5 sigma_p (regional circle criterion, rstar); | |
| certified candidates are costed at x0 = s1 D (siso_x19 cost_batch). Best-of-b (b = 1, 4, 16) is compared with the F-only | |
| per-plant optimum (X20b), the joint (F, phi) reference and the tuned linear baselines (X20 J_lin). | |
| Run: PYTHONPATH=. python -m nonlinear.experiments.x21_eval <ckpt.pth> [guidance ...] (guidance 1 = conditional only) | |
| <ckpt.pth> may also be retrieval, retrieval_cl or bl:<arm> (X22 baselines, x21_baselines.py). | |
| Env: X21_EVAL_WORKERS (default 16), X21_K (samples per plant, default 16), X21_TREPAIR=1 (null-space F repair). Appends one block per (ckpt, guidance) to results/icml_r3/X21/val_report.txt (X21_REPORT overrides). | |
| X21_TEST=1: the fresh test set instead (stage testset, seed 8282; J_fonly = best label, J_lin = min(static gain, tuned | |
| LQG), no joint reference) -> results/icml_r3/X21/test/ and test_report.txt. | |
| """ | |
| import os, sys, time, types, glob | |
| import numpy as np | |
| import torch | |
| import concurrent.futures as cf | |
| from diffusers import DDPMScheduler | |
| from utils.models.ConditionalDiffusion_mlp import ReverseDiffusionModel | |
| from utils.samplers import sample_diffusion | |
| from nonlinear.experiments import siso_x20_unstable as X | |
| from nonlinear.training.train_x21 import islog, slog, cl_decode, cl_encode, sylvester | |
| K, SEED, OUT = int(os.environ.get("X21_K", 16)), 4242, "results/icml_r3/X21/val" | |
| TEST = os.environ.get("X21_TEST") == "1" | |
| if TEST: | |
| OUT = "results/icml_r3/X21/test_eval" | |
| def refs(): | |
| if TEST: | |
| Z = [z for z in map(np.load, sorted(glob.glob(f"{X.DATA_DIR}/test/plant_*.npz"))) if not bool(z["dropped"]) and int(z["M"])] # M = 0: no F-only reference | |
| J_lqg = np.array([np.where(z["lqg_cert"], z["lqg_J"], np.inf).min() for z in Z]) | |
| return dict(i=np.array([int(z["i"]) for z in Z]), stratum=np.array([str(z["stratum"]) for z in Z]), | |
| g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=np.array([float(z["s"]) for z in Z]), | |
| J_ref=np.full(len(Z), np.nan), J_lin=np.minimum([z["sg_J"].min() for z in Z], J_lqg), J_lqg=J_lqg, | |
| J_fonly=np.array([z["J"].min() if int(z["M"]) else np.inf for z in Z])) | |
| T = X.table() | |
| Z = [np.load(f"{X.OUT}/pilot/plant_{i:02d}.npz") for i in T["i"]] | |
| J_fonly = np.array([float(np.load(f"{X.OUT}/ablate/plant_{i:02d}.npz")["fonly_J"]) for i in T["i"]]) | |
| return dict(i=T["i"], g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=T["s1"], | |
| J_ref=T["J_ref_s0"], J_lin=T["J_lin_s0"], J_fonly=J_fonly) | |
| def sample(path, G, guidance, dev): | |
| ck = torch.load(path, map_location=dev, weights_only=False) | |
| cfg = types.SimpleNamespace(**ck["cfg"], device=dev) | |
| m = ReverseDiffusionModel(cfg).to(dev) | |
| m.load_state_dict(ck["ema"]); m.eval() | |
| st = ck["stats"] | |
| cond = np.concatenate([(G - st["g_mu"]) / st["g_sd"], np.zeros((len(G), 1))], 1) | |
| cond = torch.tensor(np.repeat(cond, K, 0), dtype=torch.float32, device=dev) | |
| if ck.get("loss") == "reg": # regression twin: candidate 1 deterministic, the rest MC-dropout draws | |
| z, t0 = torch.zeros(len(cond), cfg.output_dim, device=dev), torch.zeros(len(cond), 1, device=dev, dtype=torch.long) | |
| with torch.no_grad(): | |
| x = m(cond, z, t0) | |
| m.train(); torch.manual_seed(SEED) | |
| mc = m(cond, z, t0); m.eval() | |
| x = torch.where((torch.arange(len(cond), device=dev) % K == 0)[:, None], x, mc) | |
| else: | |
| ns = DDPMScheduler(num_train_timesteps=cfg.num_timesteps, beta_schedule="squaredcos_cap_v2") | |
| gen = torch.Generator(device=dev).manual_seed(SEED) | |
| x = sample_diffusion(m, cond, (len(cond), cfg.output_dim), ns, device=dev, use_cfg=guidance != 1, guidance_scale=guidance, generator=gen) | |
| y = (x.cpu().numpy() * st["F_sd"] + st["F_mu"]).reshape(len(G), K, -1) | |
| if str(st.get("rep", "coef")) == "cl": | |
| return np.array([[np.concatenate(cl_decode(G[p], yk)) for yk in y[p]] for p in range(len(G))]), int(ck["step"]) | |
| return islog(y), int(ck["step"]) | |
| def retrieve(G, cl=False): | |
| """Look-up baseline: the labels of the nearest training plant (standardized g, Euclidean), cheapest first, topped up | |
| from the next nearest plants to K.""" | |
| T = np.load("results/icml_r3/X21/train.npz") | |
| mu, sd = T["g"].mean(0), np.where(T["g"].std(0) < 1e-8, 1.0, T["g"].std(0)) | |
| out = [] | |
| for g in G: | |
| order = np.argsort((((T["g"] - g) / sd) ** 2).sum(1)) | |
| rows = [] | |
| for p in order: | |
| k = np.argsort(T["J"][p])[: int(T["M"][p])] | |
| rows += [np.concatenate(cl_decode(g, cl_encode(T["g"][p], T["num"][p, j], T["den"][p, j])) if cl else | |
| [T["num"][p, j], T["den"][p, j]]) for j in k] | |
| if len(rows) >= K: | |
| break | |
| out.append(np.stack(rows[:K])) | |
| return np.stack(out), 0 | |
| TREPAIR = os.environ.get("X21_TREPAIR", "0") == "1" | |
| T_GRID = np.concatenate([-np.logspace(-3, 5, 200)[::-1], [0.0], np.logspace(-3, 5, 200)]) | |
| def t_repair(g, f): | |
| """Null-space repair, no verifier call: theta = theta_0 + t v keeps the linearized closed-loop polynomial (v spans the | |
| null space of the plant's Sylvester map); if F is unstable, move t to the grid point nearest the current one (in slog) | |
| that makes F stable with the 0.01 margin. Returns (f, repaired).""" | |
| if X.f_ok(f[:5], f[5:], 0.01): | |
| return f, False | |
| v = sylvester(g)[2] | |
| t0 = v @ f | |
| th0 = f - t0 * v | |
| ok = [t for t in T_GRID if X.f_ok((th0 + t * v)[:5], (th0 + t * v)[5:], 0.01)] | |
| if not ok: | |
| return f, False | |
| t = min(ok, key=lambda t: abs(slog(t) - slog(t0))) | |
| return th0 + t * v, True | |
| def score(args): | |
| """One plant: (certified [K], J [K], inf where not certified).""" | |
| g, F, s, D = args | |
| torch.set_num_threads(1) | |
| cert, loops = np.zeros(len(F), bool), {} | |
| for k, f in enumerate(F): | |
| if TREPAIR: | |
| f = t_repair(g, f)[0] | |
| num, den = f[:5], f[5:] | |
| if not X.f_ok(num, den, 0.01): | |
| continue | |
| r = X.realize(g, X.f5(num, den)) | |
| if r is not None and X.rstar(*r[:3], X.PHI_SAT, D, s=s) >= s: | |
| cert[k], loops[k] = True, r | |
| J = np.full(len(F), np.inf) | |
| if loops: | |
| ks = sorted(loops) | |
| J[ks] = X.cost_batch([(*loops[k], X.PHI_SAT) for k in ks], [s * D] * len(ks))[0] | |
| return cert, J | |
| def main(path, *guid): | |
| dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| R = refs() | |
| os.makedirs(OUT, exist_ok=True) | |
| L = [] | |
| for w in [float(x) for x in guid] or [1.0]: | |
| t0 = time.time() | |
| if path.startswith("bl:"): # X22 baselines (x21_baselines.py) | |
| from nonlinear.experiments.x21_baselines import candidates | |
| F, step = candidates(path[3:], R["g"]), 0 | |
| else: | |
| F, step = retrieve(R["g"], cl=path == "retrieval_cl") if path.startswith("retrieval") else sample(path, R["g"], w, dev) | |
| ts = time.time() - t0 | |
| with cf.ProcessPoolExecutor(int(os.environ.get("X21_EVAL_WORKERS", 16))) as ex: | |
| res = list(ex.map(score, [(R["g"][p], F[p], R["s"][p], R["D"][p]) for p in range(len(F))])) | |
| cert, J = np.stack([r[0] for r in res]), np.stack([r[1] for r in res]) | |
| tag = path.replace(":", "_") if path.startswith(("retrieval", "bl:")) else f"{os.path.basename(os.path.dirname(path))}_{os.path.basename(path)[:-4]}_w{w:g}" | |
| tag += ("_tr" if TREPAIR else "") + (f"_K{K}" if K != 16 else "") | |
| np.savez_compressed(f"{OUT}/{tag}.npz", F=F, cert=cert, J=J, **{f"ref_{k}": v for k, v in R.items()}) | |
| L.append(f"== {path} (step {step}){' + t-repair' if TREPAIR else ''}, guidance {w:g}, K = {K}, {len(F)} plants; sampling {ts:.0f} s, scoring {time.time() - t0 - ts:.0f} s") | |
| L.append(f" certified per candidate {cert.mean() * 100:.1f}%; F unstable or not certified otherwise") | |
| for b in [b for b in (1, 4, 16, 64) if b <= K]: | |
| Jb = J[:, :b].min(1) | |
| d = np.where(np.isfinite(Jb), (Jb - R["J_fonly"]) / R["J_fonly"], 1e3) | |
| med, lo, hi = X.boot_ci(d, np.median) | |
| L.append(f" b={b:2d}: plants with a certified candidate {np.isfinite(Jb).mean() * 100:5.1f}% | regret vs F-only optimum median " | |
| f"{med * 100:7.2f}% [{lo * 100:.2f}, {hi * 100:.2f}] | beats tuned linear on {np.mean(Jb < R['J_lin']) * 100:5.1f}% | " | |
| f"median (J_lin - J_b)/J_b {np.median(np.where(np.isfinite(Jb), (R['J_lin'] - Jb) / Jb, -1)) * 100:6.1f}%") | |
| open(os.environ.get("X21_REPORT", f"results/icml_r3/X21/{'test' if TEST else 'val'}_report.txt"), "a").write("\n".join(L) + "\n") | |
| print("\n".join(L), flush=True) | |
| if __name__ == "__main__": | |
| main(*sys.argv[1:]) | |