multimodalart's picture
multimodalart HF Staff
Default empty extra instruction
c76687e verified
Raw History Blame Contribute Delete
12.6 kB
"""QMA Camera Angles β€” Qwen-Image-2.1 edit LoRA that re-renders a photo from a new camera position.
LoRA: gribbliest/qwen-image-2.1-qma-camera-angles :: qwen-image-2.1-qma-camera-angles.safetensors
Base: Qwen/Qwen-Image-2.1 (QwenImage21Pipeline, diffusers)
Recipe from the LoRA card: prompt `<qma> {azimuth} {elevation} {distance}`, LoRA strength 1.0,
30 steps, CFG 3, euler sampler / simple scheduler (= diffusers' FlowMatchEulerDiscreteScheduler),
source image wired as the reference image, ~1 MP output (768 for quick previews).
The checkpoint is in ai-toolkit/ComfyUI layout (``diffusion_model.`` prefix, fused
``img_mlp.gate_up`` projection, no alpha β†’ scale 1). It is fused straight into the transformer's
Linear weights, splitting gate_up row-wise into diffusers' [gate_layer; proj] exactly as ComfyUI
does. Fusing keeps the module tree identical to the one the ZeroGPU AOTI artifacts in
``hugging-apps/qwen-image-2-1-aoti`` were exported against (eager fallback if unavailable).
"""
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 β€” must precede torch / any CUDA-touching import
import math # noqa: E402
import random # noqa: E402
import re # noqa: E402
import time # noqa: E402
import gradio as gr # noqa: E402
import torch # noqa: E402
from diffusers import QwenImage21Pipeline # noqa: E402
from huggingface_hub import hf_hub_download # noqa: E402
from PIL import Image # noqa: E402
from safetensors.torch import load_file # noqa: E402
from qwen21_aoti import aoti_load_pipeline # noqa: E402
BASE_REPO = "Qwen/Qwen-Image-2.1"
LORA_REPO = "gribbliest/qwen-image-2.1-qma-camera-angles"
LORA_FILE = "qwen-image-2.1-qma-camera-angles.safetensors"
AOTI_REPO = "hugging-apps/qwen-image-2-1-aoti"
TRIGGER = "<qma>"
MAX_SEED = 2**31 - 1
DEFAULT_STEPS = 30
DEFAULT_CFG = 3.0
DEFAULT_RES = 1024
AZIMUTHS = [
"front view",
"front-right quarter view",
"right side view",
"back-right quarter view",
"back view",
"back-left quarter view",
"left side view",
"front-left quarter view",
]
ELEVATIONS = ["low-angle shot", "eye-level shot", "elevated shot", "high-angle shot"]
DISTANCES = ["close-up", "medium shot", "wide shot"]
# --------------------------------------------------------------------------------------
# Model
# --------------------------------------------------------------------------------------
print(f"[load] {BASE_REPO} …", flush=True)
pipe = QwenImage21Pipeline.from_pretrained(BASE_REPO, torch_dtype=torch.bfloat16)
def fuse_lora(transformer: torch.nn.Module, path: str, scale: float = 1.0) -> int:
raw = load_file(path)
pairs: dict[str, dict[str, torch.Tensor]] = {}
alphas: dict[str, float] = {}
for key, tensor in raw.items():
m = re.fullmatch(r"(?:diffusion_model\.)?(.+)\.lora_(A|B)\.weight", key)
if m:
pairs.setdefault(m.group(1), {})[m.group(2)] = tensor.float()
continue
m = re.fullmatch(r"(?:diffusion_model\.)?(.+)\.alpha", key)
if m:
alphas[m.group(1)] = float(tensor.reshape(-1)[0].item())
continue
raise RuntimeError(f"unexpected LoRA key layout: {key!r}")
fused = 0
for name, ab in sorted(pairs.items()):
rank = ab["A"].shape[0]
delta = (ab["B"] @ ab["A"]).mul_(scale * alphas.get(name, float(rank)) / rank)
if name.endswith(".img_mlp.gate_up"):
prefix = name[: -len(".gate_up")]
gate, up = delta.chunk(2, dim=0)
targets = [(f"{prefix}.gate_layer", gate), (f"{prefix}.proj", up)]
else:
targets = [(name, delta)]
for target, d in targets:
weight = transformer.get_submodule(target).weight
if d.shape != weight.shape:
raise RuntimeError(f"{target}: delta {tuple(d.shape)} != weight {tuple(weight.shape)}")
weight.data.add_(d.to(weight.dtype))
fused += 1
return fused
_t0 = time.perf_counter()
_fused = fuse_lora(pipe.transformer, hf_hub_download(LORA_REPO, LORA_FILE), 1.0)
print(f"[lora] fused {_fused} modules in {time.perf_counter() - _t0:.1f}s", flush=True)
if _fused != 32 * 7:
raise RuntimeError(f"expected {32 * 7} fused modules, got {_fused}")
# Vision patch-embed Conv3d -> matmul (bitwise-equal, avoids a slow bf16 cuDNN path).
_patch_embed = pipe.text_encoder.model.visual.patch_embed
def _patch_embed_forward(hidden_states):
proj = _patch_embed.proj
weight = proj.weight.reshape(proj.weight.shape[0], -1)
return hidden_states.to(weight.dtype).flatten(1) @ weight.T + proj.bias
_patch_embed.forward = _patch_embed_forward
pipe.to("cuda")
AOTI_LOADED: list[str] = []
if os.environ.get("QWEN21_AOTI", "1") != "0":
try:
AOTI_LOADED, _cfg = aoti_load_pipeline(pipe, AOTI_REPO, token=os.environ.get("HF_TOKEN"))
print(f"[load] aoti kernels: {AOTI_LOADED} (torch {_cfg['torch']})", flush=True)
except Exception as exc: # noqa: BLE001
print(f"[load] aoti unavailable, running eager: {exc!r}", flush=True)
print("[load] ready", flush=True)
# --------------------------------------------------------------------------------------
# Inference
# --------------------------------------------------------------------------------------
def build_prompt(azimuth: str, elevation: str, distance: str, extra: str = "") -> str:
prompt = f"{TRIGGER} {azimuth} {elevation} {distance}"
extra = (extra or "").strip()
return f"{prompt}, {extra}" if extra else prompt
def _duration(image=None, azimuth=None, elevation=None, distance=None, extra="", seed=0,
randomize_seed=True, steps=DEFAULT_STEPS, guidance=DEFAULT_CFG, resolution=DEFAULT_RES, *a, **k):
try:
steps, res, cfg = int(steps), int(resolution), float(guidance)
except (TypeError, ValueError):
steps, res, cfg = DEFAULT_STEPS, DEFAULT_RES, DEFAULT_CFG
# measured on ZeroGPU with AOTI, 30 steps, CFG 3: 1024px -> 28.4 s, 768px -> 17.7 s
per_step = 0.4 * (res / 1024) ** 2.0 * (2.0 if cfg > 1 else 1.0)
return int(math.ceil((6 + steps * per_step) * 1.15))
@spaces.GPU(duration=_duration)
def generate(
image: Image.Image,
azimuth: str = "front-right quarter view",
elevation: str = "eye-level shot",
distance: str = "medium shot",
extra: str = "",
seed: int = 0,
randomize_seed: bool = True,
steps: int = DEFAULT_STEPS,
guidance: float = DEFAULT_CFG,
resolution: int = DEFAULT_RES,
progress=gr.Progress(track_tqdm=True),
):
"""Re-render an image from a new camera position with the QMA camera-angles LoRA.
Args:
image: source image.
azimuth: where the camera orbits to, relative to the original camera (e.g. "right side view").
elevation: camera height ("low-angle shot", "eye-level shot", "elevated shot", "high-angle shot").
distance: framing ("close-up", "medium shot", "wide shot").
extra: optional extra edit instruction appended after the camera prompt.
seed: random seed.
randomize_seed: pick a random seed instead of `seed`.
steps: denoising steps (card: 30).
guidance: true CFG scale (card: 3; 1 disables CFG and is ~2x faster).
resolution: output side length budget in px (1024 β‰ˆ 1 MP; 768 for quick previews).
Returns:
The re-rendered image, the seed used and the prompt sent to the model.
"""
if image is None:
raise gr.Error("Please upload a source image first.")
if azimuth not in AZIMUTHS or elevation not in ELEVATIONS or distance not in DISTANCES:
raise gr.Error("Invalid camera position.")
if randomize_seed:
seed = random.randint(0, MAX_SEED)
seed = int(seed)
prompt = build_prompt(azimuth, elevation, distance, extra)
kwargs = dict(
prompt=prompt,
image=image.convert("RGB"),
num_inference_steps=int(steps),
output_resolution=int(resolution),
generator=torch.Generator("cpu").manual_seed(seed),
)
if float(guidance) > 1.0:
kwargs["negative_prompt"] = " "
kwargs["true_cfg_scale"] = float(guidance)
t0 = time.perf_counter()
out = pipe(**kwargs).images[0]
print(f"[gen] {prompt!r} steps={steps} cfg={guidance} res={resolution} seed={seed} "
f"aoti={bool(AOTI_LOADED)} -> {out.size} in {time.perf_counter() - t0:.1f}s", flush=True)
return out, seed, prompt
# --------------------------------------------------------------------------------------
# UI
# --------------------------------------------------------------------------------------
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
"""
with gr.Blocks() as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
f"""
# πŸŽ₯ QMA Camera Angles β€” Qwen-Image-2.1
Move the camera, keep everything else. Upload an image, pick where the camera goes β€” orbit around the
subject, raise or lower it, push in or pull back β€” and the
[QMA Camera Angles LoRA](https://huggingface.co/{LORA_REPO}) on
[Qwen-Image-2.1](https://huggingface.co/{BASE_REPO}) renders the same scene from that viewpoint.
Directions are relative to the **original camera**: "right side view" swings the camera 90Β° to its own right.
"""
)
with gr.Row():
with gr.Column(scale=1):
image = gr.Image(label="Source image", type="pil", height=380)
azimuth = gr.Radio(AZIMUTHS, value="front-right quarter view", label="🧭 Azimuth β€” orbit")
elevation = gr.Radio(ELEVATIONS, value="eye-level shot", label="↕️ Elevation β€” camera height")
distance = gr.Radio(DISTANCES, value="medium shot", label="πŸ” Distance β€” framing")
run_btn = gr.Button("🎬 Move camera", variant="primary", size="lg")
with gr.Column(scale=1):
result = gr.Image(label="New view", height=520, format="png")
prompt_preview = gr.Textbox(
label="Prompt sent to the model",
value=build_prompt("front-right quarter view", "eye-level shot", "medium shot"),
interactive=False,
)
with gr.Accordion("βš™οΈ Advanced settings", open=False):
extra = gr.Textbox(
label="Extra edit instruction (optional)",
value="",
placeholder="e.g. at sunset β€” appended after the camera prompt",
)
resolution = gr.Radio([768, 1024], value=DEFAULT_RES, label="Output resolution",
info="1024 β‰ˆ 1 MP (best detail) Β· 768 for quick previews")
steps = gr.Slider(10, 50, value=DEFAULT_STEPS, step=1, label="Steps")
guidance = gr.Slider(1.0, 6.0, value=DEFAULT_CFG, step=0.5, label="CFG",
info="Card recommends 3 Β· 1 disables CFG (~2x faster)")
seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
randomize_seed = gr.Checkbox(value=True, label="Randomize seed")
gr.Markdown("Tips: quarter views, elevated shots and distance changes are the most reliable. "
"Anything the source doesn't show (e.g. a character's back) is imagined.")
gr.Examples(
examples=[
["examples/sniper.jpg", "back-right quarter view", "eye-level shot", "medium shot"],
["examples/neon_girl.jpg", "front-left quarter view", "elevated shot", "medium shot"],
["examples/earth.jpg", "front-right quarter view", "low-angle shot", "wide shot"],
["examples/sniper.jpg", "front view", "high-angle shot", "close-up"],
],
inputs=[image, azimuth, elevation, distance],
outputs=[result, seed, prompt_preview],
fn=generate,
cache_examples=True,
cache_mode="lazy",
)
for c in (azimuth, elevation, distance, extra):
c.change(build_prompt, [azimuth, elevation, distance, extra], prompt_preview,
api_visibility="private", queue=False)
run_btn.click(
generate,
inputs=[image, azimuth, elevation, distance, extra, seed, randomize_seed, steps, guidance, resolution],
outputs=[result, seed, prompt_preview],
api_name="generate",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)