"""Table 7: cFID and csFID of the Euler models beside their reference points. python reproduce/appendix/sfid_probe.py --system euler --samples direct_si=DIR --samples edm=DIR --samples cascaded_si=DIR \\ --out DIR [--n 10000] [--shifts 1,2,4,8,16,32,64] [--sections floor,models,climatology,symmetry] [--device cuda] Rows (each a mean +- SEM over the ten test conditions, both metrics from one Inception forward pass): floor FID between two disjoint halves of the truth ensemble of a condition models the Table 2 models (their archived or regenerated test samples) against the truth ensemble climatology a no-skill reference: truth members pooled from the other nine conditions symmetry the truth ensemble against itself after an exact periodic shift, rotation or reflection Writes OUT/sfid_probe.csv and OUT/sfid_probe.json; the truth features are cached under OUT/cache/. """ from __future__ import annotations import argparse import sys from datetime import datetime, timezone from pathlib import Path import numpy as np REPO = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO)) from reproduce.appendix.common import (SystemInputs, add_system_args, both_features, extractors, flip, rot90, # noqa: E402 shifted, write_json) from bifurcation.fid import backend as BE # noqa: E402 from bifurcation.fid.evaluate import bank_members # noqa: E402 from bifurcation.models.spectral import K_C, mode_int_wavenumbers # noqa: E402 def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) add_system_args(ap) ap.add_argument("--n", type=int, default=10000, help="members per ensemble") ap.add_argument("--shifts", default="1,2,4,8,16,32,64") ap.add_argument("--sections", default="floor,models,climatology,symmetry") ap.add_argument("--no-cache", action="store_true", help="recompute the truth features") a = ap.parse_args(argv) import pandas as pd inp = SystemInputs(a) outdir = a.out.resolve(); (outdir / "cache").mkdir(parents=True, exist_ok=True) ext, dev, layer_rec = extractors(inp.fid_cfg, a.device) batch = int(inp.fid_cfg.get("feature_batch", 200)) adapter, gids, t_idx = inp.adapter, inp.gids, inp.t_idx kx, ky = mode_int_wavenumbers(K_C) n_disp = int(adapter.params.get("n_disp", 128)) def feats(states_raw): imgs, _ = adapter.to_images(np.asarray(states_raw, np.float64)) return both_features(imgs, ext, dev, batch) rows = [] def record(probe, detail, per_gid_c, per_gid_s): c, s = np.array(per_gid_c, float), np.array(per_gid_s, float) rows.append({"probe": probe, "detail": detail, "n_ic": len(c), "cfid": c.mean(), "cfid_sem": c.std(ddof=1) / np.sqrt(len(c)), "sfid": s.mean(), "sfid_sem": s.std(ddof=1) / np.sqrt(len(s)), "sfid_over_cfid": s.mean() / c.mean() if c.mean() else float("nan"), "per_gid_cfid": {int(g): float(v) for g, v in zip(gids, c)}, "per_gid_sfid": {int(g): float(v) for g, v in zip(gids, s)}}) print(f" {probe:24s} {detail:14s} cFID {c.mean():8.3f} +- {c.std(ddof=1)/np.sqrt(len(c)):5.3f}" f" csFID {s.mean():9.3f} +- {s.std(ddof=1)/np.sqrt(len(s)):6.3f}", flush=True) sections = {x.strip() for x in a.sections.split(",") if x.strip()} print(f"truth features ({len(gids)} conditions x {a.n} members)") FC, FS = {}, {} for g in gids: p = outdir / "cache" / f"truth_gid{g:04d}_t{t_idx}_n{a.n}.npz" if not a.no_cache and p.is_file(): with np.load(p) as z: FC[g], FS[g] = np.asarray(z["cfid"], np.float32), np.asarray(z["sfid"], np.float32) else: FC[g], FS[g] = feats(bank_members(inp.bank, g, t_idx, a.n)) np.savez_compressed(p, cfid=FC[g], sfid=FS[g]) print(f" pool3 feature {FC[gids[0]].shape}, spatial feature {FS[gids[0]].shape}\n") if "floor" in sections: print("FLOOR") h = a.n // 2 record("floor (truth vs truth)", f"{h}+{h}", [BE.fid_from_features(FC[g][:h], FC[g][h:]) for g in gids], [BE.fid_from_features(FS[g][:h], FS[g][h:]) for g in gids]) if "models" in sections: print("\nMODELS") for title, method in (("Cascaded SI", "cascaded_si"), ("Direct SI", "direct_si"), ("EDM", "edm")): cc, ss = [], [] for g in gids: fc, fs = feats(inp.generated(method, g, a.n)) cc.append(BE.fid_from_features(fc, FC[g])); ss.append(BE.fid_from_features(fs, FS[g])) record("model", title, cc, ss) if "climatology" in sections: print("\nNO-SKILL REFERENCE") rng = np.random.default_rng(0) cc, ss = [], [] for g in gids: others = [x for x in gids if x != g] per = a.n // len(others) idx = [rng.permutation(len(FC[o]))[: per + (1 if i < a.n - per * len(others) else 0)] for i, o in enumerate(others)] pc = np.concatenate([FC[o][ix] for o, ix in zip(others, idx)]) ps = np.concatenate([FS[o][ix] for o, ix in zip(others, idx)]) cc.append(BE.fid_from_features(pc, FC[g])); ss.append(BE.fid_from_features(ps, FS[g])) record("climatology (LOO)", f"{a.n} members", cc, ss) if "symmetry" in sections: print("\nSYMMETRY / LAYOUT FLOOR (truth vs exactly transformed truth)") transforms = [(f"shift {d} px", (lambda v, d=d: shifted(v, kx, ky, d, d, n_disp))) for d in [int(x) for x in a.shifts.split(",")]] transforms += [("rot 90", lambda v: rot90(v, kx, ky, 1)), ("rot 180", lambda v: rot90(v, kx, ky, 2)), ("flip x", lambda v: flip(v, kx, ky, 0))] for name, fn in transforms: cc, ss = [], [] for g in gids: fc, fs = feats(fn(bank_members(inp.bank, g, t_idx, a.n))) cc.append(BE.fid_from_features(fc, FC[g])); ss.append(BE.fid_from_features(fs, FS[g])) record("symmetry floor", name, cc, ss) pd.DataFrame([{k: v for k, v in r.items() if not k.startswith("per_gid")} for r in rows]).to_csv(outdir / "sfid_probe.csv", index=False) write_json(outdir / "sfid_probe.json", {"at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "system": a.system, "n": a.n, "gids": gids, "t_idx": t_idx, "layer": layer_rec, "adapter": adapter.params, "pins": BE.pins(), "rows": rows}) print(f"\nwrote {outdir / 'sfid_probe.csv'}\n {outdir / 'sfid_probe.json'}") return 0 if __name__ == "__main__": raise SystemExit(main())