Download modeling_indepth.py from DexagonAI/indepth-10M: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/DexagonAI/indepth-10M/resolve/main/modeling_indepth.py
- Command line
-
hf download hf://DexagonAI/indepth-10M/modeling_indepth.py
-
curl -L -o modeling_indepth.py https://huggingface.co/DexagonAI/indepth-10M/resolve/main/modeling_indepth.py
11.6 kB
| """InDepth model definition — self-contained PyTorch implementation. | |
| A tiny rectified-flow DiT for 16x16 RGBA sprites, conditioned on captions through a | |
| word + hashed char-trigram subword text encoder trained jointly from scratch. | |
| Weights: model.safetensors (EMA), config.json, vocab.json. | |
| """ | |
| import json | |
| import math | |
| from pathlib import Path | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class IndepthConfig: | |
| def __init__(self, image_size=16, in_channels=4, patch_size=2, dim=256, depth=8, heads=4, | |
| cond_dim=128, vocab_size=8192, text_dim=128, text_layers=2, text_heads=4, | |
| max_len=16, subword_buckets=4096, **kwargs): | |
| self.image_size = image_size | |
| self.in_channels = in_channels | |
| self.patch_size = patch_size | |
| self.dim = dim | |
| self.depth = depth | |
| self.heads = heads | |
| self.cond_dim = cond_dim | |
| self.vocab_size = vocab_size | |
| self.text_dim = text_dim | |
| self.text_layers = text_layers | |
| self.text_heads = text_heads | |
| self.max_len = max_len | |
| self.subword_buckets = subword_buckets | |
| if dim % heads or dim % 4 or text_dim % text_heads: | |
| raise ValueError("dims must divide head counts") | |
| def _time_embedding(values, dim): | |
| half = dim // 2 | |
| freqs = torch.exp(-math.log(10000) * torch.arange(half, dtype=torch.float32, device=values.device) / half) | |
| args = values[:, None].float() * freqs[None] | |
| return torch.cat([torch.sin(args), torch.cos(args)], dim=-1) | |
| def _grid_embedding(height, width, dim, device): | |
| half = dim // 4 | |
| y = torch.arange(height, device=device).float().unsqueeze(1).expand(height, width).reshape(-1) | |
| x = torch.arange(width, device=device).float().unsqueeze(0).expand(height, width).reshape(-1) | |
| freqs = torch.exp(-math.log(10000) * torch.arange(half, device=device).float() / half) | |
| args_x = x[:, None] * freqs[None] | |
| args_y = y[:, None] * freqs[None] | |
| return torch.cat([torch.sin(args_x), torch.cos(args_x), torch.sin(args_y), torch.cos(args_y)], dim=-1) | |
| class IndepthDiTBlock(nn.Module): | |
| """adaLN-Zero transformer block over 2x2 patch tokens (matches the trained weights).""" | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(cfg.dim, elementwise_affine=False) | |
| self.qkv = nn.Linear(cfg.dim, cfg.dim * 3) | |
| self.proj = nn.Linear(cfg.dim, cfg.dim) | |
| self.norm2 = nn.LayerNorm(cfg.dim, elementwise_affine=False) | |
| self.fc1 = nn.Linear(cfg.dim, cfg.dim * 4) | |
| self.fc2 = nn.Linear(cfg.dim * 4, cfg.dim) | |
| self.modulation = nn.Sequential(nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.dim * 6)) | |
| nn.init.zeros_(self.modulation[-1].weight) | |
| nn.init.zeros_(self.modulation[-1].bias) | |
| def forward(self, tokens, condition): | |
| cfg = self.config_ref | |
| batch, count, dim = tokens.shape | |
| mod = self.modulation(condition) | |
| shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1) | |
| normed = self.norm1(tokens) * (1.0 + scale_msa[:, None, :]) + shift_msa[:, None, :] | |
| qkv = self.qkv(normed).reshape(batch, count, 3, cfg.heads, dim // cfg.heads).permute(2, 0, 3, 1, 4) | |
| attended = F.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2]) | |
| attended = attended.transpose(1, 2).reshape(batch, count, dim) | |
| tokens = tokens + gate_msa[:, None, :] * self.proj(attended) | |
| normed = self.norm2(tokens) * (1.0 + scale_mlp[:, None, :]) + shift_mlp[:, None, :] | |
| hidden = F.gelu(self.fc1(normed), approximate="tanh") | |
| return tokens + gate_mlp[:, None, :] * self.fc2(hidden) | |
| class IndepthTextBlock(nn.Module): | |
| def __init__(self, width, heads): | |
| super().__init__() | |
| self.heads = heads | |
| self.head_dim = width // heads | |
| self.norm1 = nn.LayerNorm(width) | |
| self.qkv = nn.Linear(width, width * 3) | |
| self.proj = nn.Linear(width, width) | |
| self.norm2 = nn.LayerNorm(width) | |
| self.fc1 = nn.Linear(width, width * 4) | |
| self.fc2 = nn.Linear(width * 4, width) | |
| def forward(self, tokens, pad_mask): | |
| batch, count, width = tokens.shape | |
| normalized = self.norm1(tokens) | |
| qkv = self.qkv(normalized).reshape(batch, count, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4) | |
| additive = torch.zeros((batch, 1, 1, count), device=tokens.device, dtype=qkv[0].dtype) | |
| additive.masked_fill_(pad_mask[:, None, None, :], float("-inf")) | |
| attended = F.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2], attn_mask=additive) | |
| attended = attended.transpose(1, 2).reshape(batch, count, width) | |
| tokens = tokens + self.proj(attended) | |
| hidden = F.gelu(self.fc1(self.norm2(tokens)), approximate="tanh") | |
| return tokens + self.fc2(hidden) | |
| class IndepthTextEncoder(nn.Module): | |
| """Word embeddings + hashed char-trigram subword features, 2-layer transformer, masked mean.""" | |
| SUBWORD_SLOTS = 16 | |
| def __init__(self, config): | |
| super().__init__() | |
| self.subword_buckets = int(config.subword_buckets or 0) | |
| self.token_embedding = nn.Embedding(config.vocab_size, config.text_dim) | |
| self.position_embedding = nn.Parameter(torch.zeros(1, config.max_len, config.text_dim)) | |
| nn.init.normal_(self.position_embedding, std=0.02) | |
| self.subword_embedding = None | |
| if self.subword_buckets: | |
| self.subword_embedding = nn.Embedding(self.subword_buckets, config.text_dim, padding_idx=0) | |
| self.blocks = nn.ModuleList([IndepthTextBlock(config.text_dim, config.text_heads) for _ in range(config.text_layers)]) | |
| self.final_norm = nn.LayerNorm(config.text_dim) | |
| self.out = nn.Linear(config.text_dim, config.cond_dim) | |
| def forward(self, tokens, subwords=None): | |
| mask = tokens != 0 | |
| hidden = self.token_embedding(tokens) + self.position_embedding[:, : tokens.shape[1]] | |
| if self.subword_embedding is not None: | |
| if subwords is None: | |
| subwords = torch.zeros((*tokens.shape, self.SUBWORD_SLOTS), dtype=tokens.dtype, device=tokens.device) | |
| counts = (subwords != 0).sum(dim=-1, keepdim=True).clamp(min=1).to(hidden.dtype) | |
| hidden = hidden + self.subword_embedding(subwords).sum(dim=2) / counts | |
| for block in self.blocks: | |
| hidden = block(hidden, ~mask) | |
| weights = mask.to(hidden.dtype) | |
| mean = (hidden * weights[:, :, None]).sum(dim=1) / weights.sum(dim=1, keepdim=True).clamp(min=1.0) | |
| return self.out(self.final_norm(mean)) | |
| class IndepthModel(nn.Module): | |
| """Rectified-flow DiT over 16x16 RGBA (4 x 16x16 in [-1, 1], alpha in {-1, 1}).""" | |
| def __init__(self, config=None): | |
| super().__init__() | |
| self.config = config or IndepthConfig() | |
| cfg = self.config | |
| self.grid_size = cfg.image_size // cfg.patch_size | |
| self.patch_embed = nn.Conv2d(cfg.in_channels, cfg.dim, kernel_size=cfg.patch_size, stride=cfg.patch_size) | |
| self.register_buffer("position", _grid_embedding(self.grid_size, self.grid_size, cfg.dim, torch.device("cpu")).unsqueeze(0), persistent=False) | |
| self.time_mlp = nn.Sequential(nn.Linear(cfg.cond_dim, cfg.cond_dim), nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.cond_dim)) | |
| self.step_mlp = nn.Sequential(nn.Linear(cfg.cond_dim, cfg.cond_dim), nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.cond_dim)) | |
| self.text_encoder = IndepthTextEncoder(cfg) | |
| self.text_proj = nn.Linear(cfg.cond_dim, cfg.cond_dim) | |
| self.blocks = nn.ModuleList() | |
| for _ in range(cfg.depth): | |
| block = IndepthDiTBlock(cfg) | |
| block.config_ref = cfg | |
| self.blocks.append(block) | |
| self.final_modulation = nn.Sequential(nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.dim * 2)) | |
| self.final_norm = nn.LayerNorm(cfg.dim, elementwise_affine=False) | |
| self.output = nn.Linear(cfg.dim, cfg.patch_size * cfg.patch_size * cfg.in_channels) | |
| def forward(self, image, time, step_size, tokens, subwords=None): | |
| cfg = self.config | |
| condition = self.time_mlp(_time_embedding(time, cfg.cond_dim)) | |
| condition = condition + self.step_mlp(_time_embedding(step_size, cfg.cond_dim)) | |
| condition = condition + self.text_proj(self.text_encoder(tokens, subwords)) | |
| tokens_image = self.patch_embed(image).flatten(2).transpose(1, 2) | |
| tokens_image = tokens_image + self.position.to(dtype=tokens_image.dtype) | |
| for block in self.blocks: | |
| tokens_image = block(tokens_image, condition) | |
| shift, scale = self.final_modulation(condition).chunk(2, dim=-1) | |
| tokens_image = self.final_norm(tokens_image) * (1.0 + scale[:, None, :]) + shift[:, None, :] | |
| patches = self.output(tokens_image) | |
| batch = image.shape[0] | |
| patches = patches.reshape(batch, self.grid_size, self.grid_size, cfg.in_channels, cfg.patch_size, cfg.patch_size) | |
| return patches.permute(0, 3, 1, 4, 2, 5).reshape(batch, cfg.in_channels, cfg.image_size, cfg.image_size) | |
| def load_indepth(model_dir): | |
| """Load the model + tokenizer from a local directory or the Hugging Face hub.""" | |
| import numpy as np | |
| from safetensors.torch import load_file | |
| directory = Path(model_dir) | |
| if not directory.exists(): | |
| from huggingface_hub import snapshot_download | |
| directory = Path(snapshot_download("DexagonAI/indepth-10M")) | |
| config = IndepthConfig(**json.loads((directory / "config.json").read_text())) | |
| model = IndepthModel(config) | |
| try: | |
| from safetensors.torch import load_file | |
| sd = load_file(str(directory / "model.safetensors")) | |
| except ImportError: | |
| from safetensors.numpy import load_file | |
| sd = {k: torch.from_numpy(v) for k, v in load_file(str(directory / "model.safetensors")).items()} | |
| model.load_state_dict(sd) | |
| return model.eval(), config, directory | |
| def generate(model, config, token_rows, subword_rows, seed=13372000, steps=4, guidance=1.5, device="cpu", generator=None): | |
| """Rectified-flow sampling with classifier-free guidance. Returns (B, 4, 16, 16) in [-1, 1].""" | |
| token_rows = token_rows.to(device) | |
| subword_rows = subword_rows.to(device) if subword_rows is not None else None | |
| batch = token_rows.shape[0] | |
| gen = generator or torch.Generator(device=device).manual_seed(seed) | |
| image = torch.randn((batch, config.in_channels, config.image_size, config.image_size), generator=gen, device=device) | |
| null_rows = torch.zeros_like(token_rows) | |
| null_sub = torch.zeros_like(subword_rows) if subword_rows is not None else None | |
| for index in range(steps): | |
| t = torch.full((batch,), index / steps, device=device) | |
| d = torch.full((batch,), 1.0 / steps, device=device) | |
| if guidance != 1.0: | |
| v_uncond = model(image, t, d, null_rows, null_sub).float() | |
| v_cond = model(image, t, d, token_rows, subword_rows).float() | |
| velocity = v_uncond + guidance * (v_cond - v_uncond) | |
| else: | |
| velocity = model(image, t, d, token_rows, subword_rows).float() | |
| image = image + (1.0 / steps) * velocity | |
| image = image.clamp(-1, 1) | |
| image[:, 3] = torch.where(image[:, 3] >= 0, 1.0, -1.0) | |
| image[:, :3] = torch.where(image[:, 3:4] >= 0, image[:, :3], 0.0) | |
| return image | |
| def decode_to_rgba(image): | |
| """(B, 4, 16, 16) in [-1, 1] -> (B, 16, 16, 4) uint8.""" | |
| arr = ((image.cpu().permute(0, 2, 3, 1).numpy() + 1.0) * 127.5).round().clip(0, 255).astype("uint8") | |
| return arr | |