import spaces # must be imported before torch / CUDA import math import random import gradio as gr import numpy as np import torch import torch.nn.functional as F from diffusers import DiffusionPipeline from huggingface_hub import hf_hub_download from PIL import Image from safetensors.torch import load_file from transformers import AutoImageProcessor, AutoModelForDepthEstimation BASE_MODEL = "OpenMOSS-Team/MOSS-Image-SFT" LORA_REPO = "thedeoxen/MOSS-Image-depth-control-lora" LORA_FILE = "moss_image_sft_depth_control_lora.safetensors" DEPTH_MODEL = "depth-anything/Depth-Anything-V2-Large-hf" QWEN_PROCESSOR = "Qwen/Qwen3.5-9B" # MOSS ships no image processor; same as ai-toolkit DEFAULT_NEGATIVE = "nsfw, worst quality, bad anatomy, watermark, jpeg artifact" TARGET_PIXELS = 1024 * 1024 VLM_PIXELS = 384 * 384 MAX_SEED = 2**31 - 1 MODE_DEPTH = "Depth map (near = white)" MODE_PHOTO = "Photo → extract depth" device = "cuda" dtype = torch.bfloat16 # ---------------------------------------------------------------- models pipe = DiffusionPipeline.from_pretrained( BASE_MODEL, custom_pipeline="pipeline_moss_image", trust_remote_code=True, torch_dtype={"default": torch.bfloat16, "vae": torch.float32}, ) pipe.scheduler = type(pipe.scheduler).from_config(pipe.scheduler.config) pipe.to(device) transformer, text_encoder, tokenizer, vae = pipe.transformer, pipe.text_encoder, pipe.tokenizer, pipe.vae image_processor = AutoImageProcessor.from_pretrained(QWEN_PROCESSOR) depth_processor = AutoImageProcessor.from_pretrained(DEPTH_MODEL) depth_model = AutoModelForDepthEstimation.from_pretrained(DEPTH_MODEL).to(device).eval() # ---------------------------------------------------------------- LoRA (ai-toolkit / ComfyUI keys, alpha == rank) class LoRALinear(torch.nn.Module): def __init__(self, base, down, up): super().__init__() self.base = base self.down = torch.nn.Parameter(down, requires_grad=False) self.up = torch.nn.Parameter(up, requires_grad=False) self.scale = 1.0 @property def weight(self): return self.base.weight def forward(self, x): out = self.base(x) if self.scale: out = out + F.linear(F.linear(x.to(self.down.dtype), self.down), self.up) * self.scale return out lora_sd = load_file(hf_hub_download(LORA_REPO, LORA_FILE)) lora_modules = [] for key in [k for k in lora_sd if k.endswith(".lora_A.weight")]: path = key[len("diffusion_model."):-len(".lora_A.weight")] # layers.N.proj.q parts = path.split(".") layer = transformer.layers[int(parts[1])] name = parts[3] wrapped = LoRALinear( layer.proj[name], lora_sd[key].to(device, dtype), lora_sd[key.replace("lora_A", "lora_B")].to(device, dtype), ) layer.proj[name] = wrapped lora_modules.append(wrapped) del lora_sd print(f"Applied LoRA to {len(lora_modules)} linear layers") def set_lora_scale(scale): for m in lora_modules: m.scale = float(scale) # ---------------------------------------------------------------- DiT with reference latents # Mirrors ai-toolkit's MOSS-Image edit training with kv_cache on (ComfyUI "ai-toolkit" edit format): # clean reference latents after the text, t=0 modulation, RoPE index r=1, attending only to # themselves, so their per-block K/V are computed once and reused every step and in both CFG branches. HEADS = transformer.config.num_heads def _rotate(x, rope): sin, cos = rope.chunk(2, -1) sin = torch.cat((sin, sin), -1).unsqueeze(-2) cos = torch.cat((cos, cos), -1).unsqueeze(-2) a, b = x.chunk(2, -1) return x * cos + torch.cat((-b, a), -1) * sin def _rope(coords): bands = transformer.rope.bands if bands.dtype != torch.float32: bands = transformer.rope._default_bands(coords.device) phase = coords.unsqueeze(-1) * bands return torch.cat((phase.sin(), phase.cos()), -2).flatten(-2).unsqueeze(0) def _grid_coords(index, frame, h, w): hh, ww = torch.meshgrid(torch.arange(h, device=device, dtype=torch.float32), torch.arange(w, device=device, dtype=torch.float32), indexing="ij") c = torch.zeros(h * w, 4, device=device) c[:, 0] = index c[:, 1] = frame c[:, 2] = (hh - (h - 1) / 2).flatten() c[:, 3] = (ww - (w - 1) / 2).flatten() return c def _time_mod(t): half = transformer.config.freq_dim // 2 times = torch.as_tensor(t, device=device, dtype=torch.float32).reshape(-1) freq = 10000 ** (-torch.arange(half, device=device, dtype=torch.float64) / half) phase = times.double()[:, None] * freq emb = torch.cat((phase.cos(), phase.sin()), -1).float() time = transformer.time_embedding(emb.to(dtype)).float() mod = transformer.shared_block_modulation(time.to(dtype)).view(1, 6, -1) head = transformer.video_head_modulation(time.to(dtype)).view(1, 2, -1) return mod, head def _block(blk, x, rope, mod, ref_kv=None, capture=None): shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = mod.unbind(1) y = (blk.norm1(x) * (1 + scale_a[:, None]) + shift_a[:, None]).to(dtype) shape = (*y.shape[:2], HEADS, -1) q = _rotate(blk.q_norm(blk.proj["q"](y).view(shape)), rope).transpose(1, 2) k = _rotate(blk.k_norm(blk.proj["k"](y).view(shape)), rope).transpose(1, 2) v = blk.proj["v"](y).view(shape).transpose(1, 2) q, k = q.to(v.dtype), k.to(v.dtype) if capture is not None: capture.append((k, v)) if ref_kv is not None: k = torch.cat((k, ref_kv[0]), 2) v = torch.cat((v, ref_kv[1]), 2) attn = F.scaled_dot_product_attention(q, k, v).transpose(1, 2).flatten(2) x = x + blk.proj["o"](attn) * gate_a[:, None] y = (blk.norm2(x) * (1 + scale_f[:, None]) + shift_f[:, None]).to(dtype) hidden, gate = blk.proj["ffn_up_gate"](y).chunk(2, -1) return x + blk.proj["ffn_down"](_situ(hidden, gate)) * gate_f[:, None] def _situ(hidden, gate): return (4.0 * torch.tanh(gate / 4.0) * torch.sigmoid(gate)) * (25.0 * torch.tanh(hidden / 25.0)) @torch.no_grad() def reference_kv(ref_latent): """Per-block post-RoPE K/V of the isolated reference tokens (t = 0). Coordinates are shifted so the image and references sit at frame 0 and the text at [-L, 0); RoPE is relative, so this equals ai-toolkit's layout for any prompt length.""" _, _, h, w = ref_latent.shape x = transformer.video_embedder(ref_latent.flatten(2).transpose(1, 2).to(dtype)) rope = _rope(_grid_coords(1, 0, h, w)).to(dtype) mod, _ = _time_mod(0.0) kv = [] for blk in transformer.layers: cap = [] x = _block(blk, x, rope, mod, capture=cap) kv.append(cap[0]) return kv @torch.no_grad() def dit(latents, t, context, ref_kv): _, _, h, w = latents.shape n_img, n_txt = h * w, context.shape[1] text_coords = torch.zeros(n_txt, 4, device=device) text_coords[:, 1] = torch.arange(n_txt, device=device, dtype=torch.float32) - n_txt rope = _rope(torch.cat((_grid_coords(0, 0, h, w), text_coords))).to(dtype) mod, head = _time_mod(t) video = transformer.video_embedder(latents.flatten(2).transpose(1, 2).to(dtype)) text = transformer.text_embedder(context, rope[:, n_img:]) x = torch.cat((video, text), 1) for blk, kv in zip(transformer.layers, ref_kv): x = _block(blk, x, rope, mod, ref_kv=kv) x = transformer.final_norm_video(x[:, :n_img]) x = x * (1 + head[:, 1, None]) + head[:, 0, None] x = transformer.final_linear_video(x.to(dtype)) return x.transpose(1, 2).reshape(1, -1, h, w) # ---------------------------------------------------------------- text + VAE @torch.no_grad() def encode_prompt(text, ref_image): """Qwen3.5 final states of 'Picture 1: ' + prompt (the depth map rides in the prompt too).""" img = torch.from_numpy(np.asarray(ref_image, dtype=np.float32) / 255.0).permute(2, 0, 1) h, w = img.shape[1:] scale = min(1.0, math.sqrt(VLM_PIXELS / (h * w))) nh, nw = max(round(h * scale), 32), max(round(w * scale), 32) if (nh, nw) != (h, w): img = F.interpolate(img[None], size=(nh, nw), mode="bicubic", antialias=True)[0].clamp(0, 1) pixels = image_processor(images=[img], return_tensors="pt", do_rescale=False) grid = pixels["image_grid_thw"] n = int(grid.prod(-1).item() // image_processor.merge_size**2) content = f"Picture 1: <|vision_start|>{'<|image_pad|>' * n}<|vision_end|>" + text chat = tokenizer.apply_chat_template([{"role": "user", "content": content}], tokenize=False, add_generation_prompt=True, enable_thinking=True) input_ids = tokenizer(chat, return_tensors="pt")["input_ids"].to(device) kwargs = dict(input_ids=input_ids, pixel_values=pixels["pixel_values"].to(device, dtype), image_grid_thw=grid.to(device), use_cache=False) mm = (input_ids == text_encoder.config.image_token_id).int() try: out = text_encoder(**kwargs, mm_token_type_ids=mm) except TypeError: out = text_encoder(**kwargs) return out.last_hidden_state.to(dtype) def _bn(z): bn = vae.bn return bn.running_mean.to(z).view(1, -1, 1, 1), (bn.running_var + vae.config.batch_norm_eps).sqrt().to(z).view(1, -1, 1, 1) @torch.no_grad() def vae_encode(image): x = torch.from_numpy(np.asarray(image, dtype=np.float32) / 127.5 - 1.0).permute(2, 0, 1)[None] z = vae.encode(x.to(device, vae.dtype)).latent_dist.mode() b, c, hh, ww = z.shape z = z.reshape(b, c, hh // 2, 2, ww // 2, 2).permute(0, 1, 3, 5, 2, 4).reshape(b, c * 4, hh // 2, ww // 2) mean, std = _bn(z) return (z - mean) / std @torch.no_grad() def vae_decode(latents): z = latents.to(vae.dtype) mean, std = _bn(z) z = z * std + mean b, c, h, w = z.shape z = z.reshape(b, c // 4, 2, 2, h, w).permute(0, 1, 4, 2, 5, 3).reshape(b, c // 4, 2 * h, 2 * w) img = vae.decode(z, return_dict=False)[0].clamp(-1, 1) img = ((img[0].permute(1, 2, 0).float().cpu().numpy() + 1) * 127.5).round().astype(np.uint8) return Image.fromarray(img) # ---------------------------------------------------------------- depth def target_size(w, h): s = math.sqrt(TARGET_PIXELS / (w * h)) return max(16, round(w * s / 16) * 16), max(16, round(h * s / 16) * 16) @torch.no_grad() def estimate_depth(image): inputs = depth_processor(images=image, return_tensors="pt").to(device) pred = depth_model(**inputs).predicted_depth pred = F.interpolate(pred[:, None].float(), size=(image.height, image.width), mode="bicubic", align_corners=False)[0, 0] pred = (pred - pred.min()) / (pred.max() - pred.min() + 1e-8) # disparity: near = white arr = (pred.clamp(0, 1) * 255).round().byte().cpu().numpy() return Image.fromarray(arr).convert("RGB") # ---------------------------------------------------------------- inference def gpu_duration(image, prompt, input_type=MODE_DEPTH, negative_prompt=DEFAULT_NEGATIVE, steps=30, guidance_scale=4.0, *args, **kwargs): # measured ~45 s for 30 steps with CFG on ZeroGPU; ~15% margin per_step = 1.5 if guidance_scale != 1.0 else 0.8 return int(10 + per_step * int(steps)) @spaces.GPU(duration=gpu_duration) def generate(image, prompt, input_type=MODE_DEPTH, negative_prompt=DEFAULT_NEGATIVE, steps=30, guidance_scale=4.0, lora_scale=1.0, seed=42, randomize_seed=True, progress=gr.Progress(track_tqdm=True)): if image is None: raise gr.Error("Please upload a depth map or a photo first.") if not prompt or not prompt.strip(): raise gr.Error("Please describe the image you want.") if randomize_seed: seed = random.randint(0, MAX_SEED) seed = int(seed) image = image.convert("RGB") w, h = target_size(*image.size) if input_type == MODE_PHOTO: depth = estimate_depth(image).resize((w, h), Image.BILINEAR) else: depth = image.convert("L").convert("RGB").resize((w, h), Image.BILINEAR) set_lora_scale(lora_scale) ref_kv = reference_kv(vae_encode(depth)) cond = encode_prompt(prompt.strip(), depth) do_cfg = guidance_scale != 1.0 uncond = encode_prompt((negative_prompt or "").strip(), depth) if do_cfg else None generator = torch.Generator("cpu").manual_seed(seed) latents = torch.randn((1, transformer.config.latent_channels, h // 16, w // 16), generator=generator, dtype=torch.float32).to(device) scheduler = pipe.scheduler scheduler.set_timesteps(int(steps), device=device) for t in progress.tqdm(scheduler.timesteps, desc="Denoising"): pred = dit(latents, t, cond, ref_kv).float() if do_cfg: pred_u = dit(latents, t, uncond, ref_kv).float() pred = pred_u + guidance_scale * (pred - pred_u) latents = scheduler.step(pred, t, latents, return_dict=False)[0] result = vae_decode(latents) return (depth, result), seed # ---------------------------------------------------------------- UI EXAMPLES = [ ["examples/stag_depth.png", "A majestic stag made of polished brass gears and copper plates, steam drifting from its chest, standing in a misty pine forest beside an old factory. Overcast sky, cinematic photograph.", MODE_DEPTH], ["examples/figure_depth.png", "Dramatic low angle, face and torso huge. A chunky voxel person built from primary-color bricks, one brick hand reaching toward the camera, the other arm thrown out. White gallery, soft studio light, sharp detail.", MODE_DEPTH], ["examples/ramen_depth.png", "A tiny wooden ramen stall on a snowy mountain ridge, a cook in a dark blue jacket stirring a steaming pot, a customer in a fur coat sitting on a stool, woven baskets, snow-capped peaks behind.", MODE_DEPTH], ["examples/espresso_depth.png", "A hand pouring a thin ribbon of espresso from a small cup into a porcelain cup on a dark wooden table, torn sourdough bread, a glass of orange juice, morning sunlight through a window.", MODE_DEPTH], ["examples/subway_depth.png", "An astronaut in an orange suit stepping through the open hatch of a sleek space station corridor, another astronaut visible through a porthole, glossy black floor with reflections, cinematic lighting.", MODE_DEPTH], ["examples/horse.jpg", "A horse sculpted from swirling blue and white porcelain, gold leaf details on its mane, standing in a sunlit marble museum hall.", MODE_PHOTO], ] with gr.Blocks() as demo: gr.Markdown( "# MOSS-Image Depth Control\n" "Upload a **depth map** (near = white) or a **photo** to auto-extract one with Depth Anything V2, " "then describe the scene. The [depth-control LoRA](https://huggingface.co/thedeoxen/MOSS-Image-depth-control-lora) " "keeps the camera depth and layout; the prompt sets subject, materials and style " "on [MOSS-Image-SFT](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT)." ) with gr.Row(): with gr.Column(): image = gr.Image(label="Depth map or photo", type="pil", height=420) input_type = gr.Radio([MODE_DEPTH, MODE_PHOTO], value=MODE_DEPTH, label="Input is a") prompt = gr.Textbox(label="Prompt", lines=3, placeholder="Describe the picture to generate…") run = gr.Button("Generate", variant="primary") with gr.Accordion("Advanced settings", open=False): negative_prompt = gr.Textbox(label="Negative prompt", value=DEFAULT_NEGATIVE) steps = gr.Slider(10, 50, value=30, step=1, label="Steps") guidance_scale = gr.Slider(1.0, 8.0, value=4.0, step=0.1, label="CFG scale") lora_scale = gr.Slider(0.0, 1.5, value=1.0, step=0.05, label="LoRA strength", info="1.0 is the trained value; ~0.7 loosens the control") with gr.Row(): seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") randomize_seed = gr.Checkbox(value=True, label="Randomize seed") with gr.Column(): output = gr.ImageSlider(label="Depth control ↔ result", type="pil", height=560) used_seed = gr.Number(label="Seed used", precision=0) inputs = [image, prompt, input_type, negative_prompt, steps, guidance_scale, lora_scale, seed, randomize_seed] run.click(generate, inputs, [output, used_seed]) prompt.submit(generate, inputs, [output, used_seed]) gr.Examples(examples=EXAMPLES, inputs=[image, prompt, input_type], outputs=[output, used_seed], fn=generate, cache_examples=True, cache_mode="lazy") if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus())