Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/moss-image-depth-control: direct link, hf CLI and curl.
- Browser
- Download file 16.7 kB
-
https://huggingface.co/spaces/hugging-apps/moss-image-depth-control/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/moss-image-depth-control/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/moss-image-depth-control/resolve/main/app.py
16.7 kB
| import spaces # must be imported before torch / CUDA | |
| import math | |
| import random | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from diffusers import DiffusionPipeline | |
| from huggingface_hub import hf_hub_download | |
| from PIL import Image | |
| from safetensors.torch import load_file | |
| from transformers import AutoImageProcessor, AutoModelForDepthEstimation | |
| BASE_MODEL = "OpenMOSS-Team/MOSS-Image-SFT" | |
| LORA_REPO = "thedeoxen/MOSS-Image-depth-control-lora" | |
| LORA_FILE = "moss_image_sft_depth_control_lora.safetensors" | |
| DEPTH_MODEL = "depth-anything/Depth-Anything-V2-Large-hf" | |
| QWEN_PROCESSOR = "Qwen/Qwen3.5-9B" # MOSS ships no image processor; same as ai-toolkit | |
| DEFAULT_NEGATIVE = "nsfw, worst quality, bad anatomy, watermark, jpeg artifact" | |
| TARGET_PIXELS = 1024 * 1024 | |
| VLM_PIXELS = 384 * 384 | |
| MAX_SEED = 2**31 - 1 | |
| MODE_DEPTH = "Depth map (near = white)" | |
| MODE_PHOTO = "Photo → extract depth" | |
| device = "cuda" | |
| dtype = torch.bfloat16 | |
| # ---------------------------------------------------------------- models | |
| pipe = DiffusionPipeline.from_pretrained( | |
| BASE_MODEL, | |
| custom_pipeline="pipeline_moss_image", | |
| trust_remote_code=True, | |
| torch_dtype={"default": torch.bfloat16, "vae": torch.float32}, | |
| ) | |
| pipe.scheduler = type(pipe.scheduler).from_config(pipe.scheduler.config) | |
| pipe.to(device) | |
| transformer, text_encoder, tokenizer, vae = pipe.transformer, pipe.text_encoder, pipe.tokenizer, pipe.vae | |
| image_processor = AutoImageProcessor.from_pretrained(QWEN_PROCESSOR) | |
| depth_processor = AutoImageProcessor.from_pretrained(DEPTH_MODEL) | |
| depth_model = AutoModelForDepthEstimation.from_pretrained(DEPTH_MODEL).to(device).eval() | |
| # ---------------------------------------------------------------- LoRA (ai-toolkit / ComfyUI keys, alpha == rank) | |
| class LoRALinear(torch.nn.Module): | |
| def __init__(self, base, down, up): | |
| super().__init__() | |
| self.base = base | |
| self.down = torch.nn.Parameter(down, requires_grad=False) | |
| self.up = torch.nn.Parameter(up, requires_grad=False) | |
| self.scale = 1.0 | |
| def weight(self): | |
| return self.base.weight | |
| def forward(self, x): | |
| out = self.base(x) | |
| if self.scale: | |
| out = out + F.linear(F.linear(x.to(self.down.dtype), self.down), self.up) * self.scale | |
| return out | |
| lora_sd = load_file(hf_hub_download(LORA_REPO, LORA_FILE)) | |
| lora_modules = [] | |
| for key in [k for k in lora_sd if k.endswith(".lora_A.weight")]: | |
| path = key[len("diffusion_model."):-len(".lora_A.weight")] # layers.N.proj.q | |
| parts = path.split(".") | |
| layer = transformer.layers[int(parts[1])] | |
| name = parts[3] | |
| wrapped = LoRALinear( | |
| layer.proj[name], | |
| lora_sd[key].to(device, dtype), | |
| lora_sd[key.replace("lora_A", "lora_B")].to(device, dtype), | |
| ) | |
| layer.proj[name] = wrapped | |
| lora_modules.append(wrapped) | |
| del lora_sd | |
| print(f"Applied LoRA to {len(lora_modules)} linear layers") | |
| def set_lora_scale(scale): | |
| for m in lora_modules: | |
| m.scale = float(scale) | |
| # ---------------------------------------------------------------- DiT with reference latents | |
| # Mirrors ai-toolkit's MOSS-Image edit training with kv_cache on (ComfyUI "ai-toolkit" edit format): | |
| # clean reference latents after the text, t=0 modulation, RoPE index r=1, attending only to | |
| # themselves, so their per-block K/V are computed once and reused every step and in both CFG branches. | |
| HEADS = transformer.config.num_heads | |
| def _rotate(x, rope): | |
| sin, cos = rope.chunk(2, -1) | |
| sin = torch.cat((sin, sin), -1).unsqueeze(-2) | |
| cos = torch.cat((cos, cos), -1).unsqueeze(-2) | |
| a, b = x.chunk(2, -1) | |
| return x * cos + torch.cat((-b, a), -1) * sin | |
| def _rope(coords): | |
| bands = transformer.rope.bands | |
| if bands.dtype != torch.float32: | |
| bands = transformer.rope._default_bands(coords.device) | |
| phase = coords.unsqueeze(-1) * bands | |
| return torch.cat((phase.sin(), phase.cos()), -2).flatten(-2).unsqueeze(0) | |
| def _grid_coords(index, frame, h, w): | |
| hh, ww = torch.meshgrid(torch.arange(h, device=device, dtype=torch.float32), | |
| torch.arange(w, device=device, dtype=torch.float32), indexing="ij") | |
| c = torch.zeros(h * w, 4, device=device) | |
| c[:, 0] = index | |
| c[:, 1] = frame | |
| c[:, 2] = (hh - (h - 1) / 2).flatten() | |
| c[:, 3] = (ww - (w - 1) / 2).flatten() | |
| return c | |
| def _time_mod(t): | |
| half = transformer.config.freq_dim // 2 | |
| times = torch.as_tensor(t, device=device, dtype=torch.float32).reshape(-1) | |
| freq = 10000 ** (-torch.arange(half, device=device, dtype=torch.float64) / half) | |
| phase = times.double()[:, None] * freq | |
| emb = torch.cat((phase.cos(), phase.sin()), -1).float() | |
| time = transformer.time_embedding(emb.to(dtype)).float() | |
| mod = transformer.shared_block_modulation(time.to(dtype)).view(1, 6, -1) | |
| head = transformer.video_head_modulation(time.to(dtype)).view(1, 2, -1) | |
| return mod, head | |
| def _block(blk, x, rope, mod, ref_kv=None, capture=None): | |
| shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = mod.unbind(1) | |
| y = (blk.norm1(x) * (1 + scale_a[:, None]) + shift_a[:, None]).to(dtype) | |
| shape = (*y.shape[:2], HEADS, -1) | |
| q = _rotate(blk.q_norm(blk.proj["q"](y).view(shape)), rope).transpose(1, 2) | |
| k = _rotate(blk.k_norm(blk.proj["k"](y).view(shape)), rope).transpose(1, 2) | |
| v = blk.proj["v"](y).view(shape).transpose(1, 2) | |
| q, k = q.to(v.dtype), k.to(v.dtype) | |
| if capture is not None: | |
| capture.append((k, v)) | |
| if ref_kv is not None: | |
| k = torch.cat((k, ref_kv[0]), 2) | |
| v = torch.cat((v, ref_kv[1]), 2) | |
| attn = F.scaled_dot_product_attention(q, k, v).transpose(1, 2).flatten(2) | |
| x = x + blk.proj["o"](attn) * gate_a[:, None] | |
| y = (blk.norm2(x) * (1 + scale_f[:, None]) + shift_f[:, None]).to(dtype) | |
| hidden, gate = blk.proj["ffn_up_gate"](y).chunk(2, -1) | |
| return x + blk.proj["ffn_down"](_situ(hidden, gate)) * gate_f[:, None] | |
| def _situ(hidden, gate): | |
| return (4.0 * torch.tanh(gate / 4.0) * torch.sigmoid(gate)) * (25.0 * torch.tanh(hidden / 25.0)) | |
| def reference_kv(ref_latent): | |
| """Per-block post-RoPE K/V of the isolated reference tokens (t = 0). | |
| Coordinates are shifted so the image and references sit at frame 0 and the text at | |
| [-L, 0); RoPE is relative, so this equals ai-toolkit's layout for any prompt length.""" | |
| _, _, h, w = ref_latent.shape | |
| x = transformer.video_embedder(ref_latent.flatten(2).transpose(1, 2).to(dtype)) | |
| rope = _rope(_grid_coords(1, 0, h, w)).to(dtype) | |
| mod, _ = _time_mod(0.0) | |
| kv = [] | |
| for blk in transformer.layers: | |
| cap = [] | |
| x = _block(blk, x, rope, mod, capture=cap) | |
| kv.append(cap[0]) | |
| return kv | |
| def dit(latents, t, context, ref_kv): | |
| _, _, h, w = latents.shape | |
| n_img, n_txt = h * w, context.shape[1] | |
| text_coords = torch.zeros(n_txt, 4, device=device) | |
| text_coords[:, 1] = torch.arange(n_txt, device=device, dtype=torch.float32) - n_txt | |
| rope = _rope(torch.cat((_grid_coords(0, 0, h, w), text_coords))).to(dtype) | |
| mod, head = _time_mod(t) | |
| video = transformer.video_embedder(latents.flatten(2).transpose(1, 2).to(dtype)) | |
| text = transformer.text_embedder(context, rope[:, n_img:]) | |
| x = torch.cat((video, text), 1) | |
| for blk, kv in zip(transformer.layers, ref_kv): | |
| x = _block(blk, x, rope, mod, ref_kv=kv) | |
| x = transformer.final_norm_video(x[:, :n_img]) | |
| x = x * (1 + head[:, 1, None]) + head[:, 0, None] | |
| x = transformer.final_linear_video(x.to(dtype)) | |
| return x.transpose(1, 2).reshape(1, -1, h, w) | |
| # ---------------------------------------------------------------- text + VAE | |
| def encode_prompt(text, ref_image): | |
| """Qwen3.5 final states of 'Picture 1: <vision>' + prompt (the depth map rides in the prompt too).""" | |
| img = torch.from_numpy(np.asarray(ref_image, dtype=np.float32) / 255.0).permute(2, 0, 1) | |
| h, w = img.shape[1:] | |
| scale = min(1.0, math.sqrt(VLM_PIXELS / (h * w))) | |
| nh, nw = max(round(h * scale), 32), max(round(w * scale), 32) | |
| if (nh, nw) != (h, w): | |
| img = F.interpolate(img[None], size=(nh, nw), mode="bicubic", antialias=True)[0].clamp(0, 1) | |
| pixels = image_processor(images=[img], return_tensors="pt", do_rescale=False) | |
| grid = pixels["image_grid_thw"] | |
| n = int(grid.prod(-1).item() // image_processor.merge_size**2) | |
| content = f"Picture 1: <|vision_start|>{'<|image_pad|>' * n}<|vision_end|>" + text | |
| chat = tokenizer.apply_chat_template([{"role": "user", "content": content}], tokenize=False, | |
| add_generation_prompt=True, enable_thinking=True) | |
| input_ids = tokenizer(chat, return_tensors="pt")["input_ids"].to(device) | |
| kwargs = dict(input_ids=input_ids, pixel_values=pixels["pixel_values"].to(device, dtype), | |
| image_grid_thw=grid.to(device), use_cache=False) | |
| mm = (input_ids == text_encoder.config.image_token_id).int() | |
| try: | |
| out = text_encoder(**kwargs, mm_token_type_ids=mm) | |
| except TypeError: | |
| out = text_encoder(**kwargs) | |
| return out.last_hidden_state.to(dtype) | |
| def _bn(z): | |
| bn = vae.bn | |
| return bn.running_mean.to(z).view(1, -1, 1, 1), (bn.running_var + vae.config.batch_norm_eps).sqrt().to(z).view(1, -1, 1, 1) | |
| def vae_encode(image): | |
| x = torch.from_numpy(np.asarray(image, dtype=np.float32) / 127.5 - 1.0).permute(2, 0, 1)[None] | |
| z = vae.encode(x.to(device, vae.dtype)).latent_dist.mode() | |
| b, c, hh, ww = z.shape | |
| z = z.reshape(b, c, hh // 2, 2, ww // 2, 2).permute(0, 1, 3, 5, 2, 4).reshape(b, c * 4, hh // 2, ww // 2) | |
| mean, std = _bn(z) | |
| return (z - mean) / std | |
| def vae_decode(latents): | |
| z = latents.to(vae.dtype) | |
| mean, std = _bn(z) | |
| z = z * std + mean | |
| b, c, h, w = z.shape | |
| z = z.reshape(b, c // 4, 2, 2, h, w).permute(0, 1, 4, 2, 5, 3).reshape(b, c // 4, 2 * h, 2 * w) | |
| img = vae.decode(z, return_dict=False)[0].clamp(-1, 1) | |
| img = ((img[0].permute(1, 2, 0).float().cpu().numpy() + 1) * 127.5).round().astype(np.uint8) | |
| return Image.fromarray(img) | |
| # ---------------------------------------------------------------- depth | |
| def target_size(w, h): | |
| s = math.sqrt(TARGET_PIXELS / (w * h)) | |
| return max(16, round(w * s / 16) * 16), max(16, round(h * s / 16) * 16) | |
| def estimate_depth(image): | |
| inputs = depth_processor(images=image, return_tensors="pt").to(device) | |
| pred = depth_model(**inputs).predicted_depth | |
| pred = F.interpolate(pred[:, None].float(), size=(image.height, image.width), mode="bicubic", align_corners=False)[0, 0] | |
| pred = (pred - pred.min()) / (pred.max() - pred.min() + 1e-8) # disparity: near = white | |
| arr = (pred.clamp(0, 1) * 255).round().byte().cpu().numpy() | |
| return Image.fromarray(arr).convert("RGB") | |
| # ---------------------------------------------------------------- inference | |
| def gpu_duration(image, prompt, input_type=MODE_DEPTH, negative_prompt=DEFAULT_NEGATIVE, steps=30, | |
| guidance_scale=4.0, *args, **kwargs): | |
| # measured ~45 s for 30 steps with CFG on ZeroGPU; ~15% margin | |
| per_step = 1.5 if guidance_scale != 1.0 else 0.8 | |
| return int(10 + per_step * int(steps)) | |
| def generate(image, prompt, input_type=MODE_DEPTH, negative_prompt=DEFAULT_NEGATIVE, steps=30, | |
| guidance_scale=4.0, lora_scale=1.0, seed=42, randomize_seed=True, | |
| progress=gr.Progress(track_tqdm=True)): | |
| if image is None: | |
| raise gr.Error("Please upload a depth map or a photo first.") | |
| if not prompt or not prompt.strip(): | |
| raise gr.Error("Please describe the image you want.") | |
| if randomize_seed: | |
| seed = random.randint(0, MAX_SEED) | |
| seed = int(seed) | |
| image = image.convert("RGB") | |
| w, h = target_size(*image.size) | |
| if input_type == MODE_PHOTO: | |
| depth = estimate_depth(image).resize((w, h), Image.BILINEAR) | |
| else: | |
| depth = image.convert("L").convert("RGB").resize((w, h), Image.BILINEAR) | |
| set_lora_scale(lora_scale) | |
| ref_kv = reference_kv(vae_encode(depth)) | |
| cond = encode_prompt(prompt.strip(), depth) | |
| do_cfg = guidance_scale != 1.0 | |
| uncond = encode_prompt((negative_prompt or "").strip(), depth) if do_cfg else None | |
| generator = torch.Generator("cpu").manual_seed(seed) | |
| latents = torch.randn((1, transformer.config.latent_channels, h // 16, w // 16), generator=generator, | |
| dtype=torch.float32).to(device) | |
| scheduler = pipe.scheduler | |
| scheduler.set_timesteps(int(steps), device=device) | |
| for t in progress.tqdm(scheduler.timesteps, desc="Denoising"): | |
| pred = dit(latents, t, cond, ref_kv).float() | |
| if do_cfg: | |
| pred_u = dit(latents, t, uncond, ref_kv).float() | |
| pred = pred_u + guidance_scale * (pred - pred_u) | |
| latents = scheduler.step(pred, t, latents, return_dict=False)[0] | |
| result = vae_decode(latents) | |
| return (depth, result), seed | |
| # ---------------------------------------------------------------- UI | |
| EXAMPLES = [ | |
| ["examples/stag_depth.png", "A majestic stag made of polished brass gears and copper plates, steam drifting from its chest, standing in a misty pine forest beside an old factory. Overcast sky, cinematic photograph.", MODE_DEPTH], | |
| ["examples/figure_depth.png", "Dramatic low angle, face and torso huge. A chunky voxel person built from primary-color bricks, one brick hand reaching toward the camera, the other arm thrown out. White gallery, soft studio light, sharp detail.", MODE_DEPTH], | |
| ["examples/ramen_depth.png", "A tiny wooden ramen stall on a snowy mountain ridge, a cook in a dark blue jacket stirring a steaming pot, a customer in a fur coat sitting on a stool, woven baskets, snow-capped peaks behind.", MODE_DEPTH], | |
| ["examples/espresso_depth.png", "A hand pouring a thin ribbon of espresso from a small cup into a porcelain cup on a dark wooden table, torn sourdough bread, a glass of orange juice, morning sunlight through a window.", MODE_DEPTH], | |
| ["examples/subway_depth.png", "An astronaut in an orange suit stepping through the open hatch of a sleek space station corridor, another astronaut visible through a porthole, glossy black floor with reflections, cinematic lighting.", MODE_DEPTH], | |
| ["examples/horse.jpg", "A horse sculpted from swirling blue and white porcelain, gold leaf details on its mane, standing in a sunlit marble museum hall.", MODE_PHOTO], | |
| ] | |
| with gr.Blocks() as demo: | |
| gr.Markdown( | |
| "# MOSS-Image Depth Control\n" | |
| "Upload a **depth map** (near = white) or a **photo** to auto-extract one with Depth Anything V2, " | |
| "then describe the scene. The [depth-control LoRA](https://huggingface.co/thedeoxen/MOSS-Image-depth-control-lora) " | |
| "keeps the camera depth and layout; the prompt sets subject, materials and style " | |
| "on [MOSS-Image-SFT](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT)." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| image = gr.Image(label="Depth map or photo", type="pil", height=420) | |
| input_type = gr.Radio([MODE_DEPTH, MODE_PHOTO], value=MODE_DEPTH, label="Input is a") | |
| prompt = gr.Textbox(label="Prompt", lines=3, placeholder="Describe the picture to generate…") | |
| run = gr.Button("Generate", variant="primary") | |
| with gr.Accordion("Advanced settings", open=False): | |
| negative_prompt = gr.Textbox(label="Negative prompt", value=DEFAULT_NEGATIVE) | |
| steps = gr.Slider(10, 50, value=30, step=1, label="Steps") | |
| guidance_scale = gr.Slider(1.0, 8.0, value=4.0, step=0.1, label="CFG scale") | |
| lora_scale = gr.Slider(0.0, 1.5, value=1.0, step=0.05, label="LoRA strength", | |
| info="1.0 is the trained value; ~0.7 loosens the control") | |
| with gr.Row(): | |
| seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") | |
| randomize_seed = gr.Checkbox(value=True, label="Randomize seed") | |
| with gr.Column(): | |
| output = gr.ImageSlider(label="Depth control ↔ result", type="pil", height=560) | |
| used_seed = gr.Number(label="Seed used", precision=0) | |
| inputs = [image, prompt, input_type, negative_prompt, steps, guidance_scale, lora_scale, seed, randomize_seed] | |
| run.click(generate, inputs, [output, used_seed]) | |
| prompt.submit(generate, inputs, [output, used_seed]) | |
| gr.Examples(examples=EXAMPLES, inputs=[image, prompt, input_type], outputs=[output, used_seed], | |
| fn=generate, cache_examples=True, cache_mode="lazy") | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus()) | |