FuseReg / fuselift /code /src /compute_fuselift_stats.py
Hongyang-Du's picture
Add fuselift/: h0 DiT-XL (RAEv2 K=7, 80 epochs), drop-in code and README
0f2a234 verified
Raw History Blame Contribute Delete
3.52 kB
"""Per-position statistics of the lifted decoder state for stage1.fuselift.RAEFuseLift.
mean / var : per-(channel, position) moments of the h_k patch tokens, shape [C, H, W]
mean_cls : dataset mean of the decoder CLS token at depth k, shape [1, 1, C]
torchrun --standalone --nproc_per_node=1 src/compute_fuselift_stats.py \
--config configs/stage2/training/fuselift-h0-raev2k7-ditxl.yaml \
--num-samples 50000 --batch 64 --out checkpoints/fuselift/h0_stats_raev2k7.pt
"""
import argparse
import dataclasses
from pathlib import Path
import torch
from omegaconf import OmegaConf
from configs.stage2 import Stage2Config
from data import prepare_imagenet_dataloader
from utils.dist_utils import setup_distributed
from utils.model_utils import instantiate_from_config
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--config", required=True)
ap.add_argument("--num-samples", type=int, default=50000)
ap.add_argument("--batch", type=int, default=64)
ap.add_argument("--out", required=True)
ap.add_argument("opts", nargs="*", help="Config overrides")
args = ap.parse_args()
if args.num_samples < 1 or args.batch < 1:
ap.error("--num-samples and --batch must be positive")
rank, world_size, device = setup_distributed()
if world_size != 1:
raise ValueError("Statistics are computed on one rank; use torchrun --nproc_per_node=1")
config: Stage2Config = OmegaConf.to_object(
OmegaConf.merge(OmegaConf.structured(Stage2Config), OmegaConf.load(args.config), OmegaConf.from_dotlist(args.opts)))
config.stage_1.params["lift_stats_path"] = None # raw lifted state
torch.manual_seed(config.training.global_seed)
rae = instantiate_from_config(config.stage_1).to(device).eval()
loader = prepare_imagenet_dataloader(
config=dataclasses.asdict(config.dataset),
image_size=config.training.image_size,
batch_size=args.batch,
num_workers=config.training.num_workers,
rank=rank, world_size=world_size,
transform=None,
)
s = ss = s_cls = None
seen = 0
for images, _ in loader:
images = images[:args.num_samples - seen].to(device)
h = rae.lift(images).double() # [B, N + 1, C]
if s is None:
s = torch.zeros_like(h[0, 1:])
ss = torch.zeros_like(h[0, 1:])
s_cls = torch.zeros_like(h[0, 0])
s += h[:, 1:].sum(0); ss += (h[:, 1:] ** 2).sum(0); s_cls += h[:, 0].sum(0)
seen += h.shape[0]
if seen % (args.batch * 20) == 0:
print(f" {seen}/{args.num_samples}", flush=True)
if seen >= args.num_samples:
break
if seen != args.num_samples:
raise ValueError(f"Requested {args.num_samples} images, but loader supplied {seen}")
n, c = s.shape
side = int(n ** 0.5)
mean = s / seen
var = (ss / seen - mean * mean).clamp_min(0)
out = {
"mean": mean.view(side, side, c).permute(2, 0, 1).contiguous().float().cpu(),
"var": var.view(side, side, c).permute(2, 0, 1).contiguous().float().cpu(),
"mean_cls": (s_cls / seen).view(1, 1, c).float().cpu(),
"num_samples": seen,
}
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
torch.save(out, args.out)
print(f"saved -> {args.out} | mean {tuple(out['mean'].shape)} var range "
f"[{out['var'].min():.4f}, {out['var'].max():.4f}] ({seen} images)")
if __name__ == "__main__":
main()