#!/usr/bin/env python3 """Download and load official Sapiens2 + S2HAND replacement tensors. Returns 308-channel heatmaps for normalized RGB tensors. This loader preserves the final complete-decoder routing; it does not require a research checkout. """ from __future__ import annotations import argparse import copy import hashlib import importlib.util import json from pathlib import Path import torch from torch import nn from safetensors.torch import load_file from huggingface_hub import hf_hub_download REPO = 'mage0001/S2HAND' WEIGHT_REVISION = '4e4fa442ec0c3ae4d6b2e05c71eec8229976ed98' BASES = { '0.4b': '73572e121602e8a6eb40eb841732e88f386d250b', '0.8b': '76f06b9854d04bb1e59e0624de39b1e8921b12aa', '1b': 'f5fed8b97b99698d5eea1d14ff0855d0b4c3f000', '5b': 'ada1f29aa1fd454ca28665c700923a0101b6b24f', } NAMES = ('S2HAND-MT-5B', 'S2HAND-Pose-5B', 'S2HAND-Pose-1B', 'S2HAND-Pose-0.8B', 'S2HAND-Pose-0.4B') # Filled with the exact bundled official standalone source fingerprint. BACKBONE_SHA256 = '7046d482da440cbedf56d9c7fd5596fa0fb35747bb7dc9d34d3c5a638d8c51db' def verify(path, expected): h = hashlib.sha256() with Path(path).open('rb') as stream: for block in iter(lambda: stream.read(8 * 1024 * 1024), b''): h.update(block) if h.hexdigest() != expected: raise ValueError(f'SHA256 mismatch: {path}') def prepare(model_name='S2HAND-MT-5B', *, cache_dir=None, base_path=None, delta_path=None, manifest_path=None, local_files_only=False): """Download pinned weights once, or verify supplied local files offline.""" if model_name not in NAMES: raise ValueError(f'Choose one of {NAMES}') scale = model_name.rsplit('-', 1)[1].lower() options = dict(cache_dir=cache_dir, local_files_only=local_files_only) manifest_path = manifest_path or hf_hub_download(REPO, f'models/{model_name}/manifest.json', revision=WEIGHT_REVISION, **options) manifest = json.loads(Path(manifest_path).read_text()) if manifest['name'] != model_name: raise ValueError('Manifest names a different model') base_path = base_path or hf_hub_download(f'facebook/sapiens2-pose-{scale}', manifest['official_base']['filename'], revision=BASES[scale], **options) delta_path = delta_path or hf_hub_download(REPO, manifest['file'], revision=WEIGHT_REVISION, **options) print(f'Verifying {model_name} base and delta SHA256...', flush=True) verify(base_path, manifest['official_base']['sha256']) verify(delta_path, manifest['sha256']) return Path(base_path), Path(delta_path), manifest def backbone_class(cache_dir=None, local_files_only=False): source = Path(__file__).with_name('sapiens2_backbone.py') if not source.is_file(): source = Path(hf_hub_download(REPO, 'sapiens2_backbone.py', cache_dir=cache_dir, local_files_only=local_files_only)) verify(source, BACKBONE_SHA256) spec = importlib.util.spec_from_file_location('_s2hand_official_backbone', source) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module.Sapiens2 class RoutedPoseHead(nn.Module): def __init__(self, channels, deconv_channels=(1024,768), decoder_channels=(512,512,256)): super().__init__() first, second = deconv_channels self.deconv_layers = nn.Sequential( nn.ConvTranspose2d(channels, first, 4, stride=2, padding=1, bias=False), nn.InstanceNorm2d(first), nn.SiLU(inplace=True), nn.ConvTranspose2d(first, second, 4, stride=2, padding=1, bias=False), nn.InstanceNorm2d(second), nn.SiLU(inplace=True), ) layers = [] cin = second for cout in decoder_channels: layers += [nn.Conv2d(cin, cout, 1), nn.InstanceNorm2d(cout), nn.SiLU(inplace=True)] cin = cout self.conv_layers = nn.Sequential(*layers) self.conv_pose = nn.Conv2d(cin, 308, 1) self.public_decoder = None self.public_projection = None protected = torch.ones(308, dtype=torch.bool, device='cpu') protected[21:63] = False protected[[41, 62]] = True self.register_buffer('protected_mask', protected, persistent=False) def freeze_public_route(self): self.public_decoder = copy.deepcopy(self.conv_layers).requires_grad_(False) self.public_projection = copy.deepcopy(self.conv_pose).requires_grad_(False) def forward(self, features): features = self.deconv_layers(features) adapted = self.conv_pose(self.conv_layers(features)) public = self.public_projection(self.public_decoder(features)) return torch.where(self.protected_mask[None, :, None, None], public, adapted) class S2Hand(nn.Module): def __init__(self, backbone, head): super().__init__() self.backbone = backbone self.decode_head = head def forward(self, normalized_rgb): """B×3×1024×768 RGB, ImageNet normalized -> B×308×256×192 heatmaps.""" return self.decode_head(self.backbone(normalized_rgb)[0]) def load_model(model_name='S2HAND-MT-5B', *, device='cpu', cache_dir=None, base_path=None, delta_path=None, manifest_path=None, local_files_only=False): """Load full base, freeze public decoder, then replace trained parameters. Uses meta initialization and mmap-backed safetensors to avoid allocating a second randomly initialized 5B model. Keep model parameters in FP32; use autocast for optional BF16 inference, as in the research pipeline. """ base, delta, manifest = prepare(model_name, cache_dir=cache_dir, base_path=base_path, delta_path=delta_path, manifest_path=manifest_path, local_files_only=local_files_only) scale = model_name.rsplit('-', 1)[1].lower() cls = backbone_class(cache_dir, local_files_only) widths = {'0.4b':1024, '0.8b':1280, '1b':1536, '5b':2432} state = load_file(str(base), device='cpu') # Decoder widths differ by scale; obtain the exact architecture from base tensors. deconv_channels = tuple(state[f'decode_head.deconv_layers.{i}.weight'].shape[1] for i in (0,3)) decoder_channels = tuple(state[f'decode_head.conv_layers.{i}.weight'].shape[0] for i in (0,3,6)) with torch.device('meta'): backbone = cls(arch=f'sapiens2_{scale}', img_size=(1024, 768), patch_size=16, out_type='featmap', pos_embed_rope_dtype='bf16') model = S2Hand(backbone, RoutedPoseHead(widths[scale], deconv_channels, decoder_channels)) model.load_state_dict(state, strict=True, assign=True) del state # Original runtime uses BF16 RoPE periods; 5B deltas explicitly override it. model.backbone.rope_embed.periods = model.backbone.rope_embed.periods.to(torch.bfloat16) model.decode_head.freeze_public_route() replacements = load_file(str(delta), device='cpu') if set(replacements) != set(manifest['tensor_names']): raise ValueError('Delta tensor names differ from manifest') current = model.state_dict() for name, value in replacements.items(): if name not in current or value.shape != current[name].shape: raise ValueError(f'Invalid replacement tensor: {name}') # Copy into the runtime tensor dtypes, especially the BF16 RoPE buffer, # exactly as the research loader does. assign=True would change its dtype. model.load_state_dict(replacements, strict=False) del replacements, current model.requires_grad_(False).eval() model.model_name = model_name return model.to(device) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--model', choices=NAMES, default=NAMES[0]) parser.add_argument('--cache-dir') parser.add_argument('--base-path') parser.add_argument('--delta-path') parser.add_argument('--manifest-path') parser.add_argument('--offline', action='store_true') parser.add_argument('--download-only', action='store_true') parser.add_argument('--device', default='cpu') parser.add_argument('--input-tensor', help='Safetensors file containing ImageNet-normalized RGB tensor named images') parser.add_argument('--output', default='s2hand_heatmaps.safetensors') parser.add_argument('--bf16', action='store_true', help='Use BF16 autocast during inference; keep FP32 parameters') args = parser.parse_args() options = dict(cache_dir=args.cache_dir, base_path=args.base_path, delta_path=args.delta_path, manifest_path=args.manifest_path, local_files_only=args.offline) if args.download_only: base, delta, _ = prepare(args.model, **options) print(f'Base: {base}\nDelta: {delta}\nReady for load_model().') return model = load_model(args.model, device=args.device, **options) print(f'Loaded {args.model}: full base + delta + protected decoder route.', flush=True) if args.input_tensor: from safetensors.torch import save_file images = load_file(args.input_tensor)['images'] if images.ndim != 4 or tuple(images.shape[1:]) != (3,1024,768): raise ValueError('Expected images shape B×3×1024×768; RGB with ImageNet normalization') with torch.inference_mode(), torch.autocast(torch.device(args.device).type, dtype=torch.bfloat16, enabled=args.bf16): heatmaps = model(images.to(device=args.device, dtype=torch.float32)) save_file({'heatmaps':heatmaps.float().cpu().contiguous()},args.output) print(f'Saved heatmaps: {args.output}') if __name__ == '__main__': main()