File size: 9,571 Bytes
679eaab
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
#!/usr/bin/env python3
"""Download and load official Sapiens2 + S2HAND replacement tensors.

Returns 308-channel heatmaps for normalized RGB tensors. This loader preserves
the final complete-decoder routing; it does not require a research checkout.
"""
from __future__ import annotations

import argparse
import copy
import hashlib
import importlib.util
import json
from pathlib import Path

import torch
from torch import nn
from safetensors.torch import load_file
from huggingface_hub import hf_hub_download

REPO = 'mage0001/S2HAND'
WEIGHT_REVISION = '4e4fa442ec0c3ae4d6b2e05c71eec8229976ed98'
BASES = {
    '0.4b': '73572e121602e8a6eb40eb841732e88f386d250b',
    '0.8b': '76f06b9854d04bb1e59e0624de39b1e8921b12aa',
    '1b': 'f5fed8b97b99698d5eea1d14ff0855d0b4c3f000',
    '5b': 'ada1f29aa1fd454ca28665c700923a0101b6b24f',
}
NAMES = ('S2HAND-MT-5B', 'S2HAND-Pose-5B', 'S2HAND-Pose-1B', 'S2HAND-Pose-0.8B', 'S2HAND-Pose-0.4B')
# Filled with the exact bundled official standalone source fingerprint.
BACKBONE_SHA256 = '7046d482da440cbedf56d9c7fd5596fa0fb35747bb7dc9d34d3c5a638d8c51db'


def verify(path, expected):
    h = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for block in iter(lambda: stream.read(8 * 1024 * 1024), b''):
            h.update(block)
    if h.hexdigest() != expected:
        raise ValueError(f'SHA256 mismatch: {path}')


def prepare(model_name='S2HAND-MT-5B', *, cache_dir=None, base_path=None, delta_path=None, manifest_path=None, local_files_only=False):
    """Download pinned weights once, or verify supplied local files offline."""
    if model_name not in NAMES:
        raise ValueError(f'Choose one of {NAMES}')
    scale = model_name.rsplit('-', 1)[1].lower()
    options = dict(cache_dir=cache_dir, local_files_only=local_files_only)
    manifest_path = manifest_path or hf_hub_download(REPO, f'models/{model_name}/manifest.json', revision=WEIGHT_REVISION, **options)
    manifest = json.loads(Path(manifest_path).read_text())
    if manifest['name'] != model_name:
        raise ValueError('Manifest names a different model')
    base_path = base_path or hf_hub_download(f'facebook/sapiens2-pose-{scale}', manifest['official_base']['filename'], revision=BASES[scale], **options)
    delta_path = delta_path or hf_hub_download(REPO, manifest['file'], revision=WEIGHT_REVISION, **options)
    print(f'Verifying {model_name} base and delta SHA256...', flush=True)
    verify(base_path, manifest['official_base']['sha256'])
    verify(delta_path, manifest['sha256'])
    return Path(base_path), Path(delta_path), manifest


def backbone_class(cache_dir=None, local_files_only=False):
    source = Path(__file__).with_name('sapiens2_backbone.py')
    if not source.is_file():
        source = Path(hf_hub_download(REPO, 'sapiens2_backbone.py', cache_dir=cache_dir, local_files_only=local_files_only))
    verify(source, BACKBONE_SHA256)
    spec = importlib.util.spec_from_file_location('_s2hand_official_backbone', source)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module.Sapiens2


class RoutedPoseHead(nn.Module):
    def __init__(self, channels, deconv_channels=(1024,768), decoder_channels=(512,512,256)):
        super().__init__()
        first, second = deconv_channels
        self.deconv_layers = nn.Sequential(
            nn.ConvTranspose2d(channels, first, 4, stride=2, padding=1, bias=False),
            nn.InstanceNorm2d(first), nn.SiLU(inplace=True),
            nn.ConvTranspose2d(first, second, 4, stride=2, padding=1, bias=False),
            nn.InstanceNorm2d(second), nn.SiLU(inplace=True),
        )
        layers = []
        cin = second
        for cout in decoder_channels:
            layers += [nn.Conv2d(cin, cout, 1), nn.InstanceNorm2d(cout), nn.SiLU(inplace=True)]
            cin = cout
        self.conv_layers = nn.Sequential(*layers)
        self.conv_pose = nn.Conv2d(cin, 308, 1)
        self.public_decoder = None
        self.public_projection = None
        protected = torch.ones(308, dtype=torch.bool, device='cpu')
        protected[21:63] = False
        protected[[41, 62]] = True
        self.register_buffer('protected_mask', protected, persistent=False)

    def freeze_public_route(self):
        self.public_decoder = copy.deepcopy(self.conv_layers).requires_grad_(False)
        self.public_projection = copy.deepcopy(self.conv_pose).requires_grad_(False)

    def forward(self, features):
        features = self.deconv_layers(features)
        adapted = self.conv_pose(self.conv_layers(features))
        public = self.public_projection(self.public_decoder(features))
        return torch.where(self.protected_mask[None, :, None, None], public, adapted)


class S2Hand(nn.Module):
    def __init__(self, backbone, head):
        super().__init__()
        self.backbone = backbone
        self.decode_head = head

    def forward(self, normalized_rgb):
        """B×3×1024×768 RGB, ImageNet normalized -> B×308×256×192 heatmaps."""
        return self.decode_head(self.backbone(normalized_rgb)[0])


def load_model(model_name='S2HAND-MT-5B', *, device='cpu', cache_dir=None,
               base_path=None, delta_path=None, manifest_path=None, local_files_only=False):
    """Load full base, freeze public decoder, then replace trained parameters.

    Uses meta initialization and mmap-backed safetensors to avoid allocating a
    second randomly initialized 5B model. Keep model parameters in FP32; use
    autocast for optional BF16 inference, as in the research pipeline.
    """
    base, delta, manifest = prepare(model_name, cache_dir=cache_dir, base_path=base_path,
        delta_path=delta_path, manifest_path=manifest_path, local_files_only=local_files_only)
    scale = model_name.rsplit('-', 1)[1].lower()
    cls = backbone_class(cache_dir, local_files_only)
    widths = {'0.4b':1024, '0.8b':1280, '1b':1536, '5b':2432}
    state = load_file(str(base), device='cpu')
    # Decoder widths differ by scale; obtain the exact architecture from base tensors.
    deconv_channels = tuple(state[f'decode_head.deconv_layers.{i}.weight'].shape[1] for i in (0,3))
    decoder_channels = tuple(state[f'decode_head.conv_layers.{i}.weight'].shape[0] for i in (0,3,6))
    with torch.device('meta'):
        backbone = cls(arch=f'sapiens2_{scale}', img_size=(1024, 768), patch_size=16,
                       out_type='featmap', pos_embed_rope_dtype='bf16')
        model = S2Hand(backbone, RoutedPoseHead(widths[scale], deconv_channels, decoder_channels))
    model.load_state_dict(state, strict=True, assign=True)
    del state
    # Original runtime uses BF16 RoPE periods; 5B deltas explicitly override it.
    model.backbone.rope_embed.periods = model.backbone.rope_embed.periods.to(torch.bfloat16)
    model.decode_head.freeze_public_route()
    replacements = load_file(str(delta), device='cpu')
    if set(replacements) != set(manifest['tensor_names']):
        raise ValueError('Delta tensor names differ from manifest')
    current = model.state_dict()
    for name, value in replacements.items():
        if name not in current or value.shape != current[name].shape:
            raise ValueError(f'Invalid replacement tensor: {name}')
    # Copy into the runtime tensor dtypes, especially the BF16 RoPE buffer,
    # exactly as the research loader does. assign=True would change its dtype.
    model.load_state_dict(replacements, strict=False)
    del replacements, current
    model.requires_grad_(False).eval()
    model.model_name = model_name
    return model.to(device)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--model', choices=NAMES, default=NAMES[0])
    parser.add_argument('--cache-dir')
    parser.add_argument('--base-path')
    parser.add_argument('--delta-path')
    parser.add_argument('--manifest-path')
    parser.add_argument('--offline', action='store_true')
    parser.add_argument('--download-only', action='store_true')
    parser.add_argument('--device', default='cpu')
    parser.add_argument('--input-tensor', help='Safetensors file containing ImageNet-normalized RGB tensor named images')
    parser.add_argument('--output', default='s2hand_heatmaps.safetensors')
    parser.add_argument('--bf16', action='store_true', help='Use BF16 autocast during inference; keep FP32 parameters')
    args = parser.parse_args()
    options = dict(cache_dir=args.cache_dir, base_path=args.base_path, delta_path=args.delta_path,
                   manifest_path=args.manifest_path, local_files_only=args.offline)
    if args.download_only:
        base, delta, _ = prepare(args.model, **options)
        print(f'Base: {base}\nDelta: {delta}\nReady for load_model().')
        return
    model = load_model(args.model, device=args.device, **options)
    print(f'Loaded {args.model}: full base + delta + protected decoder route.', flush=True)
    if args.input_tensor:
        from safetensors.torch import save_file
        images = load_file(args.input_tensor)['images']
        if images.ndim != 4 or tuple(images.shape[1:]) != (3,1024,768):
            raise ValueError('Expected images shape B×3×1024×768; RGB with ImageNet normalization')
        with torch.inference_mode(), torch.autocast(torch.device(args.device).type, dtype=torch.bfloat16, enabled=args.bf16):
            heatmaps = model(images.to(device=args.device, dtype=torch.float32))
        save_file({'heatmaps':heatmaps.float().cpu().contiguous()},args.output)
        print(f'Saved heatmaps: {args.output}')


if __name__ == '__main__':
    main()