matCercola18's picture
Backup of files only on andromeda (models, results, banks, logs) (1/11)
c4be3a9 verified
Raw History Blame Contribute Delete
9.1 kB
"""X21 validation of an F-generator checkpoint on the 40 X20 pilot plants (seed 7171, disjoint from the X21 training plants).
Per plant: K filters sampled with c = 0 (ask for the plant's best), phi = sat_1. A candidate counts iff F is stable with the
dataset's margin (f_ok, 0.01) and the loop is certified at the pilot's s1 = 0.5 sigma_p (regional circle criterion, rstar);
certified candidates are costed at x0 = s1 D (siso_x19 cost_batch). Best-of-b (b = 1, 4, 16) is compared with the F-only
per-plant optimum (X20b), the joint (F, phi) reference and the tuned linear baselines (X20 J_lin).
Run: PYTHONPATH=. python -m nonlinear.experiments.x21_eval <ckpt.pth> [guidance ...] (guidance 1 = conditional only)
<ckpt.pth> may also be retrieval, retrieval_cl or bl:<arm> (X22 baselines, x21_baselines.py).
Env: X21_EVAL_WORKERS (default 16), X21_K (samples per plant, default 16), X21_TREPAIR=1 (null-space F repair). Appends one block per (ckpt, guidance) to results/icml_r3/X21/val_report.txt (X21_REPORT overrides).
X21_TEST=1: the fresh test set instead (stage testset, seed 8282; J_fonly = best label, J_lin = min(static gain, tuned
LQG), no joint reference) -> results/icml_r3/X21/test/ and test_report.txt.
"""
import os, sys, time, types, glob
import numpy as np
import torch
import concurrent.futures as cf
from diffusers import DDPMScheduler
from utils.models.ConditionalDiffusion_mlp import ReverseDiffusionModel
from utils.samplers import sample_diffusion
from nonlinear.experiments import siso_x20_unstable as X
from nonlinear.training.train_x21 import islog, slog, cl_decode, cl_encode, sylvester
K, SEED, OUT = int(os.environ.get("X21_K", 16)), 4242, "results/icml_r3/X21/val"
TEST = os.environ.get("X21_TEST") == "1"
if TEST:
OUT = "results/icml_r3/X21/test_eval"
def refs():
if TEST:
Z = [z for z in map(np.load, sorted(glob.glob(f"{X.DATA_DIR}/test/plant_*.npz"))) if not bool(z["dropped"]) and int(z["M"])] # M = 0: no F-only reference
J_lqg = np.array([np.where(z["lqg_cert"], z["lqg_J"], np.inf).min() for z in Z])
return dict(i=np.array([int(z["i"]) for z in Z]), stratum=np.array([str(z["stratum"]) for z in Z]),
g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=np.array([float(z["s"]) for z in Z]),
J_ref=np.full(len(Z), np.nan), J_lin=np.minimum([z["sg_J"].min() for z in Z], J_lqg), J_lqg=J_lqg,
J_fonly=np.array([z["J"].min() if int(z["M"]) else np.inf for z in Z]))
T = X.table()
Z = [np.load(f"{X.OUT}/pilot/plant_{i:02d}.npz") for i in T["i"]]
J_fonly = np.array([float(np.load(f"{X.OUT}/ablate/plant_{i:02d}.npz")["fonly_J"]) for i in T["i"]])
return dict(i=T["i"], g=np.stack([z["g"] for z in Z]), D=np.stack([z["D"] for z in Z]), s=T["s1"],
J_ref=T["J_ref_s0"], J_lin=T["J_lin_s0"], J_fonly=J_fonly)
def sample(path, G, guidance, dev):
ck = torch.load(path, map_location=dev, weights_only=False)
cfg = types.SimpleNamespace(**ck["cfg"], device=dev)
m = ReverseDiffusionModel(cfg).to(dev)
m.load_state_dict(ck["ema"]); m.eval()
st = ck["stats"]
cond = np.concatenate([(G - st["g_mu"]) / st["g_sd"], np.zeros((len(G), 1))], 1)
cond = torch.tensor(np.repeat(cond, K, 0), dtype=torch.float32, device=dev)
if ck.get("loss") == "reg": # regression twin: candidate 1 deterministic, the rest MC-dropout draws
z, t0 = torch.zeros(len(cond), cfg.output_dim, device=dev), torch.zeros(len(cond), 1, device=dev, dtype=torch.long)
with torch.no_grad():
x = m(cond, z, t0)
m.train(); torch.manual_seed(SEED)
mc = m(cond, z, t0); m.eval()
x = torch.where((torch.arange(len(cond), device=dev) % K == 0)[:, None], x, mc)
else:
ns = DDPMScheduler(num_train_timesteps=cfg.num_timesteps, beta_schedule="squaredcos_cap_v2")
gen = torch.Generator(device=dev).manual_seed(SEED)
x = sample_diffusion(m, cond, (len(cond), cfg.output_dim), ns, device=dev, use_cfg=guidance != 1, guidance_scale=guidance, generator=gen)
y = (x.cpu().numpy() * st["F_sd"] + st["F_mu"]).reshape(len(G), K, -1)
if str(st.get("rep", "coef")) == "cl":
return np.array([[np.concatenate(cl_decode(G[p], yk)) for yk in y[p]] for p in range(len(G))]), int(ck["step"])
return islog(y), int(ck["step"])
def retrieve(G, cl=False):
"""Look-up baseline: the labels of the nearest training plant (standardized g, Euclidean), cheapest first, topped up
from the next nearest plants to K."""
T = np.load("results/icml_r3/X21/train.npz")
mu, sd = T["g"].mean(0), np.where(T["g"].std(0) < 1e-8, 1.0, T["g"].std(0))
out = []
for g in G:
order = np.argsort((((T["g"] - g) / sd) ** 2).sum(1))
rows = []
for p in order:
k = np.argsort(T["J"][p])[: int(T["M"][p])]
rows += [np.concatenate(cl_decode(g, cl_encode(T["g"][p], T["num"][p, j], T["den"][p, j])) if cl else
[T["num"][p, j], T["den"][p, j]]) for j in k]
if len(rows) >= K:
break
out.append(np.stack(rows[:K]))
return np.stack(out), 0
TREPAIR = os.environ.get("X21_TREPAIR", "0") == "1"
T_GRID = np.concatenate([-np.logspace(-3, 5, 200)[::-1], [0.0], np.logspace(-3, 5, 200)])
def t_repair(g, f):
"""Null-space repair, no verifier call: theta = theta_0 + t v keeps the linearized closed-loop polynomial (v spans the
null space of the plant's Sylvester map); if F is unstable, move t to the grid point nearest the current one (in slog)
that makes F stable with the 0.01 margin. Returns (f, repaired)."""
if X.f_ok(f[:5], f[5:], 0.01):
return f, False
v = sylvester(g)[2]
t0 = v @ f
th0 = f - t0 * v
ok = [t for t in T_GRID if X.f_ok((th0 + t * v)[:5], (th0 + t * v)[5:], 0.01)]
if not ok:
return f, False
t = min(ok, key=lambda t: abs(slog(t) - slog(t0)))
return th0 + t * v, True
def score(args):
"""One plant: (certified [K], J [K], inf where not certified)."""
g, F, s, D = args
torch.set_num_threads(1)
cert, loops = np.zeros(len(F), bool), {}
for k, f in enumerate(F):
if TREPAIR:
f = t_repair(g, f)[0]
num, den = f[:5], f[5:]
if not X.f_ok(num, den, 0.01):
continue
r = X.realize(g, X.f5(num, den))
if r is not None and X.rstar(*r[:3], X.PHI_SAT, D, s=s) >= s:
cert[k], loops[k] = True, r
J = np.full(len(F), np.inf)
if loops:
ks = sorted(loops)
J[ks] = X.cost_batch([(*loops[k], X.PHI_SAT) for k in ks], [s * D] * len(ks))[0]
return cert, J
def main(path, *guid):
dev = "cuda" if torch.cuda.is_available() else "cpu"
R = refs()
os.makedirs(OUT, exist_ok=True)
L = []
for w in [float(x) for x in guid] or [1.0]:
t0 = time.time()
if path.startswith("bl:"): # X22 baselines (x21_baselines.py)
from nonlinear.experiments.x21_baselines import candidates
F, step = candidates(path[3:], R["g"]), 0
else:
F, step = retrieve(R["g"], cl=path == "retrieval_cl") if path.startswith("retrieval") else sample(path, R["g"], w, dev)
ts = time.time() - t0
with cf.ProcessPoolExecutor(int(os.environ.get("X21_EVAL_WORKERS", 16))) as ex:
res = list(ex.map(score, [(R["g"][p], F[p], R["s"][p], R["D"][p]) for p in range(len(F))]))
cert, J = np.stack([r[0] for r in res]), np.stack([r[1] for r in res])
tag = path.replace(":", "_") if path.startswith(("retrieval", "bl:")) else f"{os.path.basename(os.path.dirname(path))}_{os.path.basename(path)[:-4]}_w{w:g}"
tag += ("_tr" if TREPAIR else "") + (f"_K{K}" if K != 16 else "")
np.savez_compressed(f"{OUT}/{tag}.npz", F=F, cert=cert, J=J, **{f"ref_{k}": v for k, v in R.items()})
L.append(f"== {path} (step {step}){' + t-repair' if TREPAIR else ''}, guidance {w:g}, K = {K}, {len(F)} plants; sampling {ts:.0f} s, scoring {time.time() - t0 - ts:.0f} s")
L.append(f" certified per candidate {cert.mean() * 100:.1f}%; F unstable or not certified otherwise")
for b in [b for b in (1, 4, 16, 64) if b <= K]:
Jb = J[:, :b].min(1)
d = np.where(np.isfinite(Jb), (Jb - R["J_fonly"]) / R["J_fonly"], 1e3)
med, lo, hi = X.boot_ci(d, np.median)
L.append(f" b={b:2d}: plants with a certified candidate {np.isfinite(Jb).mean() * 100:5.1f}% | regret vs F-only optimum median "
f"{med * 100:7.2f}% [{lo * 100:.2f}, {hi * 100:.2f}] | beats tuned linear on {np.mean(Jb < R['J_lin']) * 100:5.1f}% | "
f"median (J_lin - J_b)/J_b {np.median(np.where(np.isfinite(Jb), (R['J_lin'] - Jb) / Jb, -1)) * 100:6.1f}%")
open(os.environ.get("X21_REPORT", f"results/icml_r3/X21/{'test' if TEST else 'val'}_report.txt"), "a").write("\n".join(L) + "\n")
print("\n".join(L), flush=True)
if __name__ == "__main__":
main(*sys.argv[1:])