multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
103f089 verified
Raw History Blame Contribute Delete
16.7 kB
import spaces # must be imported before torch / CUDA
import math
import random
import gradio as gr
import numpy as np
import torch
import torch.nn.functional as F
from diffusers import DiffusionPipeline
from huggingface_hub import hf_hub_download
from PIL import Image
from safetensors.torch import load_file
from transformers import AutoImageProcessor, AutoModelForDepthEstimation
BASE_MODEL = "OpenMOSS-Team/MOSS-Image-SFT"
LORA_REPO = "thedeoxen/MOSS-Image-depth-control-lora"
LORA_FILE = "moss_image_sft_depth_control_lora.safetensors"
DEPTH_MODEL = "depth-anything/Depth-Anything-V2-Large-hf"
QWEN_PROCESSOR = "Qwen/Qwen3.5-9B" # MOSS ships no image processor; same as ai-toolkit
DEFAULT_NEGATIVE = "nsfw, worst quality, bad anatomy, watermark, jpeg artifact"
TARGET_PIXELS = 1024 * 1024
VLM_PIXELS = 384 * 384
MAX_SEED = 2**31 - 1
MODE_DEPTH = "Depth map (near = white)"
MODE_PHOTO = "Photo → extract depth"
device = "cuda"
dtype = torch.bfloat16
# ---------------------------------------------------------------- models
pipe = DiffusionPipeline.from_pretrained(
BASE_MODEL,
custom_pipeline="pipeline_moss_image",
trust_remote_code=True,
torch_dtype={"default": torch.bfloat16, "vae": torch.float32},
)
pipe.scheduler = type(pipe.scheduler).from_config(pipe.scheduler.config)
pipe.to(device)
transformer, text_encoder, tokenizer, vae = pipe.transformer, pipe.text_encoder, pipe.tokenizer, pipe.vae
image_processor = AutoImageProcessor.from_pretrained(QWEN_PROCESSOR)
depth_processor = AutoImageProcessor.from_pretrained(DEPTH_MODEL)
depth_model = AutoModelForDepthEstimation.from_pretrained(DEPTH_MODEL).to(device).eval()
# ---------------------------------------------------------------- LoRA (ai-toolkit / ComfyUI keys, alpha == rank)
class LoRALinear(torch.nn.Module):
def __init__(self, base, down, up):
super().__init__()
self.base = base
self.down = torch.nn.Parameter(down, requires_grad=False)
self.up = torch.nn.Parameter(up, requires_grad=False)
self.scale = 1.0
@property
def weight(self):
return self.base.weight
def forward(self, x):
out = self.base(x)
if self.scale:
out = out + F.linear(F.linear(x.to(self.down.dtype), self.down), self.up) * self.scale
return out
lora_sd = load_file(hf_hub_download(LORA_REPO, LORA_FILE))
lora_modules = []
for key in [k for k in lora_sd if k.endswith(".lora_A.weight")]:
path = key[len("diffusion_model."):-len(".lora_A.weight")] # layers.N.proj.q
parts = path.split(".")
layer = transformer.layers[int(parts[1])]
name = parts[3]
wrapped = LoRALinear(
layer.proj[name],
lora_sd[key].to(device, dtype),
lora_sd[key.replace("lora_A", "lora_B")].to(device, dtype),
)
layer.proj[name] = wrapped
lora_modules.append(wrapped)
del lora_sd
print(f"Applied LoRA to {len(lora_modules)} linear layers")
def set_lora_scale(scale):
for m in lora_modules:
m.scale = float(scale)
# ---------------------------------------------------------------- DiT with reference latents
# Mirrors ai-toolkit's MOSS-Image edit training with kv_cache on (ComfyUI "ai-toolkit" edit format):
# clean reference latents after the text, t=0 modulation, RoPE index r=1, attending only to
# themselves, so their per-block K/V are computed once and reused every step and in both CFG branches.
HEADS = transformer.config.num_heads
def _rotate(x, rope):
sin, cos = rope.chunk(2, -1)
sin = torch.cat((sin, sin), -1).unsqueeze(-2)
cos = torch.cat((cos, cos), -1).unsqueeze(-2)
a, b = x.chunk(2, -1)
return x * cos + torch.cat((-b, a), -1) * sin
def _rope(coords):
bands = transformer.rope.bands
if bands.dtype != torch.float32:
bands = transformer.rope._default_bands(coords.device)
phase = coords.unsqueeze(-1) * bands
return torch.cat((phase.sin(), phase.cos()), -2).flatten(-2).unsqueeze(0)
def _grid_coords(index, frame, h, w):
hh, ww = torch.meshgrid(torch.arange(h, device=device, dtype=torch.float32),
torch.arange(w, device=device, dtype=torch.float32), indexing="ij")
c = torch.zeros(h * w, 4, device=device)
c[:, 0] = index
c[:, 1] = frame
c[:, 2] = (hh - (h - 1) / 2).flatten()
c[:, 3] = (ww - (w - 1) / 2).flatten()
return c
def _time_mod(t):
half = transformer.config.freq_dim // 2
times = torch.as_tensor(t, device=device, dtype=torch.float32).reshape(-1)
freq = 10000 ** (-torch.arange(half, device=device, dtype=torch.float64) / half)
phase = times.double()[:, None] * freq
emb = torch.cat((phase.cos(), phase.sin()), -1).float()
time = transformer.time_embedding(emb.to(dtype)).float()
mod = transformer.shared_block_modulation(time.to(dtype)).view(1, 6, -1)
head = transformer.video_head_modulation(time.to(dtype)).view(1, 2, -1)
return mod, head
def _block(blk, x, rope, mod, ref_kv=None, capture=None):
shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = mod.unbind(1)
y = (blk.norm1(x) * (1 + scale_a[:, None]) + shift_a[:, None]).to(dtype)
shape = (*y.shape[:2], HEADS, -1)
q = _rotate(blk.q_norm(blk.proj["q"](y).view(shape)), rope).transpose(1, 2)
k = _rotate(blk.k_norm(blk.proj["k"](y).view(shape)), rope).transpose(1, 2)
v = blk.proj["v"](y).view(shape).transpose(1, 2)
q, k = q.to(v.dtype), k.to(v.dtype)
if capture is not None:
capture.append((k, v))
if ref_kv is not None:
k = torch.cat((k, ref_kv[0]), 2)
v = torch.cat((v, ref_kv[1]), 2)
attn = F.scaled_dot_product_attention(q, k, v).transpose(1, 2).flatten(2)
x = x + blk.proj["o"](attn) * gate_a[:, None]
y = (blk.norm2(x) * (1 + scale_f[:, None]) + shift_f[:, None]).to(dtype)
hidden, gate = blk.proj["ffn_up_gate"](y).chunk(2, -1)
return x + blk.proj["ffn_down"](_situ(hidden, gate)) * gate_f[:, None]
def _situ(hidden, gate):
return (4.0 * torch.tanh(gate / 4.0) * torch.sigmoid(gate)) * (25.0 * torch.tanh(hidden / 25.0))
@torch.no_grad()
def reference_kv(ref_latent):
"""Per-block post-RoPE K/V of the isolated reference tokens (t = 0).
Coordinates are shifted so the image and references sit at frame 0 and the text at
[-L, 0); RoPE is relative, so this equals ai-toolkit's layout for any prompt length."""
_, _, h, w = ref_latent.shape
x = transformer.video_embedder(ref_latent.flatten(2).transpose(1, 2).to(dtype))
rope = _rope(_grid_coords(1, 0, h, w)).to(dtype)
mod, _ = _time_mod(0.0)
kv = []
for blk in transformer.layers:
cap = []
x = _block(blk, x, rope, mod, capture=cap)
kv.append(cap[0])
return kv
@torch.no_grad()
def dit(latents, t, context, ref_kv):
_, _, h, w = latents.shape
n_img, n_txt = h * w, context.shape[1]
text_coords = torch.zeros(n_txt, 4, device=device)
text_coords[:, 1] = torch.arange(n_txt, device=device, dtype=torch.float32) - n_txt
rope = _rope(torch.cat((_grid_coords(0, 0, h, w), text_coords))).to(dtype)
mod, head = _time_mod(t)
video = transformer.video_embedder(latents.flatten(2).transpose(1, 2).to(dtype))
text = transformer.text_embedder(context, rope[:, n_img:])
x = torch.cat((video, text), 1)
for blk, kv in zip(transformer.layers, ref_kv):
x = _block(blk, x, rope, mod, ref_kv=kv)
x = transformer.final_norm_video(x[:, :n_img])
x = x * (1 + head[:, 1, None]) + head[:, 0, None]
x = transformer.final_linear_video(x.to(dtype))
return x.transpose(1, 2).reshape(1, -1, h, w)
# ---------------------------------------------------------------- text + VAE
@torch.no_grad()
def encode_prompt(text, ref_image):
"""Qwen3.5 final states of 'Picture 1: <vision>' + prompt (the depth map rides in the prompt too)."""
img = torch.from_numpy(np.asarray(ref_image, dtype=np.float32) / 255.0).permute(2, 0, 1)
h, w = img.shape[1:]
scale = min(1.0, math.sqrt(VLM_PIXELS / (h * w)))
nh, nw = max(round(h * scale), 32), max(round(w * scale), 32)
if (nh, nw) != (h, w):
img = F.interpolate(img[None], size=(nh, nw), mode="bicubic", antialias=True)[0].clamp(0, 1)
pixels = image_processor(images=[img], return_tensors="pt", do_rescale=False)
grid = pixels["image_grid_thw"]
n = int(grid.prod(-1).item() // image_processor.merge_size**2)
content = f"Picture 1: <|vision_start|>{'<|image_pad|>' * n}<|vision_end|>" + text
chat = tokenizer.apply_chat_template([{"role": "user", "content": content}], tokenize=False,
add_generation_prompt=True, enable_thinking=True)
input_ids = tokenizer(chat, return_tensors="pt")["input_ids"].to(device)
kwargs = dict(input_ids=input_ids, pixel_values=pixels["pixel_values"].to(device, dtype),
image_grid_thw=grid.to(device), use_cache=False)
mm = (input_ids == text_encoder.config.image_token_id).int()
try:
out = text_encoder(**kwargs, mm_token_type_ids=mm)
except TypeError:
out = text_encoder(**kwargs)
return out.last_hidden_state.to(dtype)
def _bn(z):
bn = vae.bn
return bn.running_mean.to(z).view(1, -1, 1, 1), (bn.running_var + vae.config.batch_norm_eps).sqrt().to(z).view(1, -1, 1, 1)
@torch.no_grad()
def vae_encode(image):
x = torch.from_numpy(np.asarray(image, dtype=np.float32) / 127.5 - 1.0).permute(2, 0, 1)[None]
z = vae.encode(x.to(device, vae.dtype)).latent_dist.mode()
b, c, hh, ww = z.shape
z = z.reshape(b, c, hh // 2, 2, ww // 2, 2).permute(0, 1, 3, 5, 2, 4).reshape(b, c * 4, hh // 2, ww // 2)
mean, std = _bn(z)
return (z - mean) / std
@torch.no_grad()
def vae_decode(latents):
z = latents.to(vae.dtype)
mean, std = _bn(z)
z = z * std + mean
b, c, h, w = z.shape
z = z.reshape(b, c // 4, 2, 2, h, w).permute(0, 1, 4, 2, 5, 3).reshape(b, c // 4, 2 * h, 2 * w)
img = vae.decode(z, return_dict=False)[0].clamp(-1, 1)
img = ((img[0].permute(1, 2, 0).float().cpu().numpy() + 1) * 127.5).round().astype(np.uint8)
return Image.fromarray(img)
# ---------------------------------------------------------------- depth
def target_size(w, h):
s = math.sqrt(TARGET_PIXELS / (w * h))
return max(16, round(w * s / 16) * 16), max(16, round(h * s / 16) * 16)
@torch.no_grad()
def estimate_depth(image):
inputs = depth_processor(images=image, return_tensors="pt").to(device)
pred = depth_model(**inputs).predicted_depth
pred = F.interpolate(pred[:, None].float(), size=(image.height, image.width), mode="bicubic", align_corners=False)[0, 0]
pred = (pred - pred.min()) / (pred.max() - pred.min() + 1e-8) # disparity: near = white
arr = (pred.clamp(0, 1) * 255).round().byte().cpu().numpy()
return Image.fromarray(arr).convert("RGB")
# ---------------------------------------------------------------- inference
def gpu_duration(image, prompt, input_type=MODE_DEPTH, negative_prompt=DEFAULT_NEGATIVE, steps=30,
guidance_scale=4.0, *args, **kwargs):
# measured ~45 s for 30 steps with CFG on ZeroGPU; ~15% margin
per_step = 1.5 if guidance_scale != 1.0 else 0.8
return int(10 + per_step * int(steps))
@spaces.GPU(duration=gpu_duration)
def generate(image, prompt, input_type=MODE_DEPTH, negative_prompt=DEFAULT_NEGATIVE, steps=30,
guidance_scale=4.0, lora_scale=1.0, seed=42, randomize_seed=True,
progress=gr.Progress(track_tqdm=True)):
if image is None:
raise gr.Error("Please upload a depth map or a photo first.")
if not prompt or not prompt.strip():
raise gr.Error("Please describe the image you want.")
if randomize_seed:
seed = random.randint(0, MAX_SEED)
seed = int(seed)
image = image.convert("RGB")
w, h = target_size(*image.size)
if input_type == MODE_PHOTO:
depth = estimate_depth(image).resize((w, h), Image.BILINEAR)
else:
depth = image.convert("L").convert("RGB").resize((w, h), Image.BILINEAR)
set_lora_scale(lora_scale)
ref_kv = reference_kv(vae_encode(depth))
cond = encode_prompt(prompt.strip(), depth)
do_cfg = guidance_scale != 1.0
uncond = encode_prompt((negative_prompt or "").strip(), depth) if do_cfg else None
generator = torch.Generator("cpu").manual_seed(seed)
latents = torch.randn((1, transformer.config.latent_channels, h // 16, w // 16), generator=generator,
dtype=torch.float32).to(device)
scheduler = pipe.scheduler
scheduler.set_timesteps(int(steps), device=device)
for t in progress.tqdm(scheduler.timesteps, desc="Denoising"):
pred = dit(latents, t, cond, ref_kv).float()
if do_cfg:
pred_u = dit(latents, t, uncond, ref_kv).float()
pred = pred_u + guidance_scale * (pred - pred_u)
latents = scheduler.step(pred, t, latents, return_dict=False)[0]
result = vae_decode(latents)
return (depth, result), seed
# ---------------------------------------------------------------- UI
EXAMPLES = [
["examples/stag_depth.png", "A majestic stag made of polished brass gears and copper plates, steam drifting from its chest, standing in a misty pine forest beside an old factory. Overcast sky, cinematic photograph.", MODE_DEPTH],
["examples/figure_depth.png", "Dramatic low angle, face and torso huge. A chunky voxel person built from primary-color bricks, one brick hand reaching toward the camera, the other arm thrown out. White gallery, soft studio light, sharp detail.", MODE_DEPTH],
["examples/ramen_depth.png", "A tiny wooden ramen stall on a snowy mountain ridge, a cook in a dark blue jacket stirring a steaming pot, a customer in a fur coat sitting on a stool, woven baskets, snow-capped peaks behind.", MODE_DEPTH],
["examples/espresso_depth.png", "A hand pouring a thin ribbon of espresso from a small cup into a porcelain cup on a dark wooden table, torn sourdough bread, a glass of orange juice, morning sunlight through a window.", MODE_DEPTH],
["examples/subway_depth.png", "An astronaut in an orange suit stepping through the open hatch of a sleek space station corridor, another astronaut visible through a porthole, glossy black floor with reflections, cinematic lighting.", MODE_DEPTH],
["examples/horse.jpg", "A horse sculpted from swirling blue and white porcelain, gold leaf details on its mane, standing in a sunlit marble museum hall.", MODE_PHOTO],
]
with gr.Blocks() as demo:
gr.Markdown(
"# MOSS-Image Depth Control\n"
"Upload a **depth map** (near = white) or a **photo** to auto-extract one with Depth Anything V2, "
"then describe the scene. The [depth-control LoRA](https://huggingface.co/thedeoxen/MOSS-Image-depth-control-lora) "
"keeps the camera depth and layout; the prompt sets subject, materials and style "
"on [MOSS-Image-SFT](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT)."
)
with gr.Row():
with gr.Column():
image = gr.Image(label="Depth map or photo", type="pil", height=420)
input_type = gr.Radio([MODE_DEPTH, MODE_PHOTO], value=MODE_DEPTH, label="Input is a")
prompt = gr.Textbox(label="Prompt", lines=3, placeholder="Describe the picture to generate…")
run = gr.Button("Generate", variant="primary")
with gr.Accordion("Advanced settings", open=False):
negative_prompt = gr.Textbox(label="Negative prompt", value=DEFAULT_NEGATIVE)
steps = gr.Slider(10, 50, value=30, step=1, label="Steps")
guidance_scale = gr.Slider(1.0, 8.0, value=4.0, step=0.1, label="CFG scale")
lora_scale = gr.Slider(0.0, 1.5, value=1.0, step=0.05, label="LoRA strength",
info="1.0 is the trained value; ~0.7 loosens the control")
with gr.Row():
seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed")
randomize_seed = gr.Checkbox(value=True, label="Randomize seed")
with gr.Column():
output = gr.ImageSlider(label="Depth control ↔ result", type="pil", height=560)
used_seed = gr.Number(label="Seed used", precision=0)
inputs = [image, prompt, input_type, negative_prompt, steps, guidance_scale, lora_scale, seed, randomize_seed]
run.click(generate, inputs, [output, used_seed])
prompt.submit(generate, inputs, [output, used_seed])
gr.Examples(examples=EXAMPLES, inputs=[image, prompt, input_type], outputs=[output, used_seed],
fn=generate, cache_examples=True, cache_mode="lazy")
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus())