File size: 3,518 Bytes
0f2a234
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
"""Per-position statistics of the lifted decoder state for stage1.fuselift.RAEFuseLift.

  mean / var : per-(channel, position) moments of the h_k patch tokens, shape [C, H, W]
  mean_cls   : dataset mean of the decoder CLS token at depth k, shape [1, 1, C]

  torchrun --standalone --nproc_per_node=1 src/compute_fuselift_stats.py \
      --config configs/stage2/training/fuselift-h0-raev2k7-ditxl.yaml \
      --num-samples 50000 --batch 64 --out checkpoints/fuselift/h0_stats_raev2k7.pt
"""
import argparse
import dataclasses
from pathlib import Path

import torch
from omegaconf import OmegaConf

from configs.stage2 import Stage2Config
from data import prepare_imagenet_dataloader
from utils.dist_utils import setup_distributed
from utils.model_utils import instantiate_from_config


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--config", required=True)
    ap.add_argument("--num-samples", type=int, default=50000)
    ap.add_argument("--batch", type=int, default=64)
    ap.add_argument("--out", required=True)
    ap.add_argument("opts", nargs="*", help="Config overrides")
    args = ap.parse_args()

    if args.num_samples < 1 or args.batch < 1:
        ap.error("--num-samples and --batch must be positive")
    rank, world_size, device = setup_distributed()
    if world_size != 1:
        raise ValueError("Statistics are computed on one rank; use torchrun --nproc_per_node=1")
    config: Stage2Config = OmegaConf.to_object(
        OmegaConf.merge(OmegaConf.structured(Stage2Config), OmegaConf.load(args.config), OmegaConf.from_dotlist(args.opts)))
    config.stage_1.params["lift_stats_path"] = None      # raw lifted state

    torch.manual_seed(config.training.global_seed)
    rae = instantiate_from_config(config.stage_1).to(device).eval()

    loader = prepare_imagenet_dataloader(
        config=dataclasses.asdict(config.dataset),
        image_size=config.training.image_size,
        batch_size=args.batch,
        num_workers=config.training.num_workers,
        rank=rank, world_size=world_size,
        transform=None,
    )

    s = ss = s_cls = None
    seen = 0
    for images, _ in loader:
        images = images[:args.num_samples - seen].to(device)
        h = rae.lift(images).double()                     # [B, N + 1, C]
        if s is None:
            s = torch.zeros_like(h[0, 1:])
            ss = torch.zeros_like(h[0, 1:])
            s_cls = torch.zeros_like(h[0, 0])
        s += h[:, 1:].sum(0); ss += (h[:, 1:] ** 2).sum(0); s_cls += h[:, 0].sum(0)
        seen += h.shape[0]
        if seen % (args.batch * 20) == 0:
            print(f"  {seen}/{args.num_samples}", flush=True)
        if seen >= args.num_samples:
            break

    if seen != args.num_samples:
        raise ValueError(f"Requested {args.num_samples} images, but loader supplied {seen}")

    n, c = s.shape
    side = int(n ** 0.5)
    mean = s / seen
    var = (ss / seen - mean * mean).clamp_min(0)
    out = {
        "mean": mean.view(side, side, c).permute(2, 0, 1).contiguous().float().cpu(),
        "var": var.view(side, side, c).permute(2, 0, 1).contiguous().float().cpu(),
        "mean_cls": (s_cls / seen).view(1, 1, c).float().cpu(),
        "num_samples": seen,
    }
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    torch.save(out, args.out)
    print(f"saved -> {args.out}  | mean {tuple(out['mean'].shape)} var range "
          f"[{out['var'].min():.4f}, {out['var'].max():.4f}] ({seen} images)")


if __name__ == "__main__":
    main()