File size: 6,886 Bytes
41dd7e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """Table 7: cFID and csFID of the Euler models beside their reference points.
python reproduce/appendix/sfid_probe.py --system euler --samples direct_si=DIR --samples edm=DIR --samples cascaded_si=DIR \\
--out DIR [--n 10000] [--shifts 1,2,4,8,16,32,64] [--sections floor,models,climatology,symmetry] [--device cuda]
Rows (each a mean +- SEM over the ten test conditions, both metrics from one Inception forward pass):
floor FID between two disjoint halves of the truth ensemble of a condition
models the Table 2 models (their archived or regenerated test samples) against the truth ensemble
climatology a no-skill reference: truth members pooled from the other nine conditions
symmetry the truth ensemble against itself after an exact periodic shift, rotation or reflection
Writes OUT/sfid_probe.csv and OUT/sfid_probe.json; the truth features are cached under OUT/cache/.
"""
from __future__ import annotations
import argparse
import sys
from datetime import datetime, timezone
from pathlib import Path
import numpy as np
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO))
from reproduce.appendix.common import (SystemInputs, add_system_args, both_features, extractors, flip, rot90, # noqa: E402
shifted, write_json)
from bifurcation.fid import backend as BE # noqa: E402
from bifurcation.fid.evaluate import bank_members # noqa: E402
from bifurcation.models.spectral import K_C, mode_int_wavenumbers # noqa: E402
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
add_system_args(ap)
ap.add_argument("--n", type=int, default=10000, help="members per ensemble")
ap.add_argument("--shifts", default="1,2,4,8,16,32,64")
ap.add_argument("--sections", default="floor,models,climatology,symmetry")
ap.add_argument("--no-cache", action="store_true", help="recompute the truth features")
a = ap.parse_args(argv)
import pandas as pd
inp = SystemInputs(a)
outdir = a.out.resolve(); (outdir / "cache").mkdir(parents=True, exist_ok=True)
ext, dev, layer_rec = extractors(inp.fid_cfg, a.device)
batch = int(inp.fid_cfg.get("feature_batch", 200))
adapter, gids, t_idx = inp.adapter, inp.gids, inp.t_idx
kx, ky = mode_int_wavenumbers(K_C)
n_disp = int(adapter.params.get("n_disp", 128))
def feats(states_raw):
imgs, _ = adapter.to_images(np.asarray(states_raw, np.float64))
return both_features(imgs, ext, dev, batch)
rows = []
def record(probe, detail, per_gid_c, per_gid_s):
c, s = np.array(per_gid_c, float), np.array(per_gid_s, float)
rows.append({"probe": probe, "detail": detail, "n_ic": len(c),
"cfid": c.mean(), "cfid_sem": c.std(ddof=1) / np.sqrt(len(c)),
"sfid": s.mean(), "sfid_sem": s.std(ddof=1) / np.sqrt(len(s)),
"sfid_over_cfid": s.mean() / c.mean() if c.mean() else float("nan"),
"per_gid_cfid": {int(g): float(v) for g, v in zip(gids, c)},
"per_gid_sfid": {int(g): float(v) for g, v in zip(gids, s)}})
print(f" {probe:24s} {detail:14s} cFID {c.mean():8.3f} +- {c.std(ddof=1)/np.sqrt(len(c)):5.3f}"
f" csFID {s.mean():9.3f} +- {s.std(ddof=1)/np.sqrt(len(s)):6.3f}", flush=True)
sections = {x.strip() for x in a.sections.split(",") if x.strip()}
print(f"truth features ({len(gids)} conditions x {a.n} members)")
FC, FS = {}, {}
for g in gids:
p = outdir / "cache" / f"truth_gid{g:04d}_t{t_idx}_n{a.n}.npz"
if not a.no_cache and p.is_file():
with np.load(p) as z:
FC[g], FS[g] = np.asarray(z["cfid"], np.float32), np.asarray(z["sfid"], np.float32)
else:
FC[g], FS[g] = feats(bank_members(inp.bank, g, t_idx, a.n))
np.savez_compressed(p, cfid=FC[g], sfid=FS[g])
print(f" pool3 feature {FC[gids[0]].shape}, spatial feature {FS[gids[0]].shape}\n")
if "floor" in sections:
print("FLOOR")
h = a.n // 2
record("floor (truth vs truth)", f"{h}+{h}",
[BE.fid_from_features(FC[g][:h], FC[g][h:]) for g in gids],
[BE.fid_from_features(FS[g][:h], FS[g][h:]) for g in gids])
if "models" in sections:
print("\nMODELS")
for title, method in (("Cascaded SI", "cascaded_si"), ("Direct SI", "direct_si"), ("EDM", "edm")):
cc, ss = [], []
for g in gids:
fc, fs = feats(inp.generated(method, g, a.n))
cc.append(BE.fid_from_features(fc, FC[g])); ss.append(BE.fid_from_features(fs, FS[g]))
record("model", title, cc, ss)
if "climatology" in sections:
print("\nNO-SKILL REFERENCE")
rng = np.random.default_rng(0)
cc, ss = [], []
for g in gids:
others = [x for x in gids if x != g]
per = a.n // len(others)
idx = [rng.permutation(len(FC[o]))[: per + (1 if i < a.n - per * len(others) else 0)]
for i, o in enumerate(others)]
pc = np.concatenate([FC[o][ix] for o, ix in zip(others, idx)])
ps = np.concatenate([FS[o][ix] for o, ix in zip(others, idx)])
cc.append(BE.fid_from_features(pc, FC[g])); ss.append(BE.fid_from_features(ps, FS[g]))
record("climatology (LOO)", f"{a.n} members", cc, ss)
if "symmetry" in sections:
print("\nSYMMETRY / LAYOUT FLOOR (truth vs exactly transformed truth)")
transforms = [(f"shift {d} px", (lambda v, d=d: shifted(v, kx, ky, d, d, n_disp)))
for d in [int(x) for x in a.shifts.split(",")]]
transforms += [("rot 90", lambda v: rot90(v, kx, ky, 1)), ("rot 180", lambda v: rot90(v, kx, ky, 2)),
("flip x", lambda v: flip(v, kx, ky, 0))]
for name, fn in transforms:
cc, ss = [], []
for g in gids:
fc, fs = feats(fn(bank_members(inp.bank, g, t_idx, a.n)))
cc.append(BE.fid_from_features(fc, FC[g])); ss.append(BE.fid_from_features(fs, FS[g]))
record("symmetry floor", name, cc, ss)
pd.DataFrame([{k: v for k, v in r.items() if not k.startswith("per_gid")} for r in rows]).to_csv(outdir / "sfid_probe.csv", index=False)
write_json(outdir / "sfid_probe.json", {"at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "system": a.system,
"n": a.n, "gids": gids, "t_idx": t_idx, "layer": layer_rec, "adapter": adapter.params,
"pins": BE.pins(), "rows": rows})
print(f"\nwrote {outdir / 'sfid_probe.csv'}\n {outdir / 'sfid_probe.json'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|