Download fuselift/code/src/compute_fuselift_stats.py from Hongyang-Du/FuseReg: direct link, hf CLI and curl.
- Browser
- Download file 3.52 kB
-
https://huggingface.co/Hongyang-Du/FuseReg/resolve/main/fuselift/code/src/compute_fuselift_stats.py
- Command line
-
hf download hf://Hongyang-Du/FuseReg/fuselift/code/src/compute_fuselift_stats.py
-
curl -L -o compute_fuselift_stats.py https://huggingface.co/Hongyang-Du/FuseReg/resolve/main/fuselift/code/src/compute_fuselift_stats.py
3.52 kB
| """Per-position statistics of the lifted decoder state for stage1.fuselift.RAEFuseLift. | |
| mean / var : per-(channel, position) moments of the h_k patch tokens, shape [C, H, W] | |
| mean_cls : dataset mean of the decoder CLS token at depth k, shape [1, 1, C] | |
| torchrun --standalone --nproc_per_node=1 src/compute_fuselift_stats.py \ | |
| --config configs/stage2/training/fuselift-h0-raev2k7-ditxl.yaml \ | |
| --num-samples 50000 --batch 64 --out checkpoints/fuselift/h0_stats_raev2k7.pt | |
| """ | |
| import argparse | |
| import dataclasses | |
| from pathlib import Path | |
| import torch | |
| from omegaconf import OmegaConf | |
| from configs.stage2 import Stage2Config | |
| from data import prepare_imagenet_dataloader | |
| from utils.dist_utils import setup_distributed | |
| from utils.model_utils import instantiate_from_config | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--config", required=True) | |
| ap.add_argument("--num-samples", type=int, default=50000) | |
| ap.add_argument("--batch", type=int, default=64) | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("opts", nargs="*", help="Config overrides") | |
| args = ap.parse_args() | |
| if args.num_samples < 1 or args.batch < 1: | |
| ap.error("--num-samples and --batch must be positive") | |
| rank, world_size, device = setup_distributed() | |
| if world_size != 1: | |
| raise ValueError("Statistics are computed on one rank; use torchrun --nproc_per_node=1") | |
| config: Stage2Config = OmegaConf.to_object( | |
| OmegaConf.merge(OmegaConf.structured(Stage2Config), OmegaConf.load(args.config), OmegaConf.from_dotlist(args.opts))) | |
| config.stage_1.params["lift_stats_path"] = None # raw lifted state | |
| torch.manual_seed(config.training.global_seed) | |
| rae = instantiate_from_config(config.stage_1).to(device).eval() | |
| loader = prepare_imagenet_dataloader( | |
| config=dataclasses.asdict(config.dataset), | |
| image_size=config.training.image_size, | |
| batch_size=args.batch, | |
| num_workers=config.training.num_workers, | |
| rank=rank, world_size=world_size, | |
| transform=None, | |
| ) | |
| s = ss = s_cls = None | |
| seen = 0 | |
| for images, _ in loader: | |
| images = images[:args.num_samples - seen].to(device) | |
| h = rae.lift(images).double() # [B, N + 1, C] | |
| if s is None: | |
| s = torch.zeros_like(h[0, 1:]) | |
| ss = torch.zeros_like(h[0, 1:]) | |
| s_cls = torch.zeros_like(h[0, 0]) | |
| s += h[:, 1:].sum(0); ss += (h[:, 1:] ** 2).sum(0); s_cls += h[:, 0].sum(0) | |
| seen += h.shape[0] | |
| if seen % (args.batch * 20) == 0: | |
| print(f" {seen}/{args.num_samples}", flush=True) | |
| if seen >= args.num_samples: | |
| break | |
| if seen != args.num_samples: | |
| raise ValueError(f"Requested {args.num_samples} images, but loader supplied {seen}") | |
| n, c = s.shape | |
| side = int(n ** 0.5) | |
| mean = s / seen | |
| var = (ss / seen - mean * mean).clamp_min(0) | |
| out = { | |
| "mean": mean.view(side, side, c).permute(2, 0, 1).contiguous().float().cpu(), | |
| "var": var.view(side, side, c).permute(2, 0, 1).contiguous().float().cpu(), | |
| "mean_cls": (s_cls / seen).view(1, 1, c).float().cpu(), | |
| "num_samples": seen, | |
| } | |
| Path(args.out).parent.mkdir(parents=True, exist_ok=True) | |
| torch.save(out, args.out) | |
| print(f"saved -> {args.out} | mean {tuple(out['mean'].shape)} var range " | |
| f"[{out['var'].min():.4f}, {out['var'].max():.4f}] ({seen} images)") | |
| if __name__ == "__main__": | |
| main() | |