| """Table 7: cFID and csFID of the Euler models beside their reference points. |
| |
| python reproduce/appendix/sfid_probe.py --system euler --samples direct_si=DIR --samples edm=DIR --samples cascaded_si=DIR \\ |
| --out DIR [--n 10000] [--shifts 1,2,4,8,16,32,64] [--sections floor,models,climatology,symmetry] [--device cuda] |
| |
| Rows (each a mean +- SEM over the ten test conditions, both metrics from one Inception forward pass): |
| floor FID between two disjoint halves of the truth ensemble of a condition |
| models the Table 2 models (their archived or regenerated test samples) against the truth ensemble |
| climatology a no-skill reference: truth members pooled from the other nine conditions |
| symmetry the truth ensemble against itself after an exact periodic shift, rotation or reflection |
| Writes OUT/sfid_probe.csv and OUT/sfid_probe.json; the truth features are cached under OUT/cache/. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import sys |
| from datetime import datetime, timezone |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
| REPO = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(REPO)) |
|
|
| from reproduce.appendix.common import (SystemInputs, add_system_args, both_features, extractors, flip, rot90, |
| shifted, write_json) |
| from bifurcation.fid import backend as BE |
| from bifurcation.fid.evaluate import bank_members |
| from bifurcation.models.spectral import K_C, mode_int_wavenumbers |
|
|
|
|
| def main(argv=None) -> int: |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| add_system_args(ap) |
| ap.add_argument("--n", type=int, default=10000, help="members per ensemble") |
| ap.add_argument("--shifts", default="1,2,4,8,16,32,64") |
| ap.add_argument("--sections", default="floor,models,climatology,symmetry") |
| ap.add_argument("--no-cache", action="store_true", help="recompute the truth features") |
| a = ap.parse_args(argv) |
| import pandas as pd |
|
|
| inp = SystemInputs(a) |
| outdir = a.out.resolve(); (outdir / "cache").mkdir(parents=True, exist_ok=True) |
| ext, dev, layer_rec = extractors(inp.fid_cfg, a.device) |
| batch = int(inp.fid_cfg.get("feature_batch", 200)) |
| adapter, gids, t_idx = inp.adapter, inp.gids, inp.t_idx |
| kx, ky = mode_int_wavenumbers(K_C) |
| n_disp = int(adapter.params.get("n_disp", 128)) |
|
|
| def feats(states_raw): |
| imgs, _ = adapter.to_images(np.asarray(states_raw, np.float64)) |
| return both_features(imgs, ext, dev, batch) |
|
|
| rows = [] |
|
|
| def record(probe, detail, per_gid_c, per_gid_s): |
| c, s = np.array(per_gid_c, float), np.array(per_gid_s, float) |
| rows.append({"probe": probe, "detail": detail, "n_ic": len(c), |
| "cfid": c.mean(), "cfid_sem": c.std(ddof=1) / np.sqrt(len(c)), |
| "sfid": s.mean(), "sfid_sem": s.std(ddof=1) / np.sqrt(len(s)), |
| "sfid_over_cfid": s.mean() / c.mean() if c.mean() else float("nan"), |
| "per_gid_cfid": {int(g): float(v) for g, v in zip(gids, c)}, |
| "per_gid_sfid": {int(g): float(v) for g, v in zip(gids, s)}}) |
| print(f" {probe:24s} {detail:14s} cFID {c.mean():8.3f} +- {c.std(ddof=1)/np.sqrt(len(c)):5.3f}" |
| f" csFID {s.mean():9.3f} +- {s.std(ddof=1)/np.sqrt(len(s)):6.3f}", flush=True) |
|
|
| sections = {x.strip() for x in a.sections.split(",") if x.strip()} |
| print(f"truth features ({len(gids)} conditions x {a.n} members)") |
| FC, FS = {}, {} |
| for g in gids: |
| p = outdir / "cache" / f"truth_gid{g:04d}_t{t_idx}_n{a.n}.npz" |
| if not a.no_cache and p.is_file(): |
| with np.load(p) as z: |
| FC[g], FS[g] = np.asarray(z["cfid"], np.float32), np.asarray(z["sfid"], np.float32) |
| else: |
| FC[g], FS[g] = feats(bank_members(inp.bank, g, t_idx, a.n)) |
| np.savez_compressed(p, cfid=FC[g], sfid=FS[g]) |
| print(f" pool3 feature {FC[gids[0]].shape}, spatial feature {FS[gids[0]].shape}\n") |
|
|
| if "floor" in sections: |
| print("FLOOR") |
| h = a.n // 2 |
| record("floor (truth vs truth)", f"{h}+{h}", |
| [BE.fid_from_features(FC[g][:h], FC[g][h:]) for g in gids], |
| [BE.fid_from_features(FS[g][:h], FS[g][h:]) for g in gids]) |
| if "models" in sections: |
| print("\nMODELS") |
| for title, method in (("Cascaded SI", "cascaded_si"), ("Direct SI", "direct_si"), ("EDM", "edm")): |
| cc, ss = [], [] |
| for g in gids: |
| fc, fs = feats(inp.generated(method, g, a.n)) |
| cc.append(BE.fid_from_features(fc, FC[g])); ss.append(BE.fid_from_features(fs, FS[g])) |
| record("model", title, cc, ss) |
| if "climatology" in sections: |
| print("\nNO-SKILL REFERENCE") |
| rng = np.random.default_rng(0) |
| cc, ss = [], [] |
| for g in gids: |
| others = [x for x in gids if x != g] |
| per = a.n // len(others) |
| idx = [rng.permutation(len(FC[o]))[: per + (1 if i < a.n - per * len(others) else 0)] |
| for i, o in enumerate(others)] |
| pc = np.concatenate([FC[o][ix] for o, ix in zip(others, idx)]) |
| ps = np.concatenate([FS[o][ix] for o, ix in zip(others, idx)]) |
| cc.append(BE.fid_from_features(pc, FC[g])); ss.append(BE.fid_from_features(ps, FS[g])) |
| record("climatology (LOO)", f"{a.n} members", cc, ss) |
| if "symmetry" in sections: |
| print("\nSYMMETRY / LAYOUT FLOOR (truth vs exactly transformed truth)") |
| transforms = [(f"shift {d} px", (lambda v, d=d: shifted(v, kx, ky, d, d, n_disp))) |
| for d in [int(x) for x in a.shifts.split(",")]] |
| transforms += [("rot 90", lambda v: rot90(v, kx, ky, 1)), ("rot 180", lambda v: rot90(v, kx, ky, 2)), |
| ("flip x", lambda v: flip(v, kx, ky, 0))] |
| for name, fn in transforms: |
| cc, ss = [], [] |
| for g in gids: |
| fc, fs = feats(fn(bank_members(inp.bank, g, t_idx, a.n))) |
| cc.append(BE.fid_from_features(fc, FC[g])); ss.append(BE.fid_from_features(fs, FS[g])) |
| record("symmetry floor", name, cc, ss) |
|
|
| pd.DataFrame([{k: v for k, v in r.items() if not k.startswith("per_gid")} for r in rows]).to_csv(outdir / "sfid_probe.csv", index=False) |
| write_json(outdir / "sfid_probe.json", {"at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "system": a.system, |
| "n": a.n, "gids": gids, "t_idx": t_idx, "layer": layer_rec, "adapter": adapter.params, |
| "pins": BE.pins(), "rows": rows}) |
| print(f"\nwrote {outdir / 'sfid_probe.csv'}\n {outdir / 'sfid_probe.json'}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
