Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/qma-camera-angles: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/spaces/hugging-apps/qma-camera-angles/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/qma-camera-angles/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/qma-camera-angles/resolve/main/app.py
12.6 kB
| """QMA Camera Angles β Qwen-Image-2.1 edit LoRA that re-renders a photo from a new camera position. | |
| LoRA: gribbliest/qwen-image-2.1-qma-camera-angles :: qwen-image-2.1-qma-camera-angles.safetensors | |
| Base: Qwen/Qwen-Image-2.1 (QwenImage21Pipeline, diffusers) | |
| Recipe from the LoRA card: prompt `<qma> {azimuth} {elevation} {distance}`, LoRA strength 1.0, | |
| 30 steps, CFG 3, euler sampler / simple scheduler (= diffusers' FlowMatchEulerDiscreteScheduler), | |
| source image wired as the reference image, ~1 MP output (768 for quick previews). | |
| The checkpoint is in ai-toolkit/ComfyUI layout (``diffusion_model.`` prefix, fused | |
| ``img_mlp.gate_up`` projection, no alpha β scale 1). It is fused straight into the transformer's | |
| Linear weights, splitting gate_up row-wise into diffusers' [gate_layer; proj] exactly as ComfyUI | |
| does. Fusing keeps the module tree identical to the one the ZeroGPU AOTI artifacts in | |
| ``hugging-apps/qwen-image-2-1-aoti`` were exported against (eager fallback if unavailable). | |
| """ | |
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 β must precede torch / any CUDA-touching import | |
| import math # noqa: E402 | |
| import random # noqa: E402 | |
| import re # noqa: E402 | |
| import time # noqa: E402 | |
| import gradio as gr # noqa: E402 | |
| import torch # noqa: E402 | |
| from diffusers import QwenImage21Pipeline # noqa: E402 | |
| from huggingface_hub import hf_hub_download # noqa: E402 | |
| from PIL import Image # noqa: E402 | |
| from safetensors.torch import load_file # noqa: E402 | |
| from qwen21_aoti import aoti_load_pipeline # noqa: E402 | |
| BASE_REPO = "Qwen/Qwen-Image-2.1" | |
| LORA_REPO = "gribbliest/qwen-image-2.1-qma-camera-angles" | |
| LORA_FILE = "qwen-image-2.1-qma-camera-angles.safetensors" | |
| AOTI_REPO = "hugging-apps/qwen-image-2-1-aoti" | |
| TRIGGER = "<qma>" | |
| MAX_SEED = 2**31 - 1 | |
| DEFAULT_STEPS = 30 | |
| DEFAULT_CFG = 3.0 | |
| DEFAULT_RES = 1024 | |
| AZIMUTHS = [ | |
| "front view", | |
| "front-right quarter view", | |
| "right side view", | |
| "back-right quarter view", | |
| "back view", | |
| "back-left quarter view", | |
| "left side view", | |
| "front-left quarter view", | |
| ] | |
| ELEVATIONS = ["low-angle shot", "eye-level shot", "elevated shot", "high-angle shot"] | |
| DISTANCES = ["close-up", "medium shot", "wide shot"] | |
| # -------------------------------------------------------------------------------------- | |
| # Model | |
| # -------------------------------------------------------------------------------------- | |
| print(f"[load] {BASE_REPO} β¦", flush=True) | |
| pipe = QwenImage21Pipeline.from_pretrained(BASE_REPO, torch_dtype=torch.bfloat16) | |
| def fuse_lora(transformer: torch.nn.Module, path: str, scale: float = 1.0) -> int: | |
| raw = load_file(path) | |
| pairs: dict[str, dict[str, torch.Tensor]] = {} | |
| alphas: dict[str, float] = {} | |
| for key, tensor in raw.items(): | |
| m = re.fullmatch(r"(?:diffusion_model\.)?(.+)\.lora_(A|B)\.weight", key) | |
| if m: | |
| pairs.setdefault(m.group(1), {})[m.group(2)] = tensor.float() | |
| continue | |
| m = re.fullmatch(r"(?:diffusion_model\.)?(.+)\.alpha", key) | |
| if m: | |
| alphas[m.group(1)] = float(tensor.reshape(-1)[0].item()) | |
| continue | |
| raise RuntimeError(f"unexpected LoRA key layout: {key!r}") | |
| fused = 0 | |
| for name, ab in sorted(pairs.items()): | |
| rank = ab["A"].shape[0] | |
| delta = (ab["B"] @ ab["A"]).mul_(scale * alphas.get(name, float(rank)) / rank) | |
| if name.endswith(".img_mlp.gate_up"): | |
| prefix = name[: -len(".gate_up")] | |
| gate, up = delta.chunk(2, dim=0) | |
| targets = [(f"{prefix}.gate_layer", gate), (f"{prefix}.proj", up)] | |
| else: | |
| targets = [(name, delta)] | |
| for target, d in targets: | |
| weight = transformer.get_submodule(target).weight | |
| if d.shape != weight.shape: | |
| raise RuntimeError(f"{target}: delta {tuple(d.shape)} != weight {tuple(weight.shape)}") | |
| weight.data.add_(d.to(weight.dtype)) | |
| fused += 1 | |
| return fused | |
| _t0 = time.perf_counter() | |
| _fused = fuse_lora(pipe.transformer, hf_hub_download(LORA_REPO, LORA_FILE), 1.0) | |
| print(f"[lora] fused {_fused} modules in {time.perf_counter() - _t0:.1f}s", flush=True) | |
| if _fused != 32 * 7: | |
| raise RuntimeError(f"expected {32 * 7} fused modules, got {_fused}") | |
| # Vision patch-embed Conv3d -> matmul (bitwise-equal, avoids a slow bf16 cuDNN path). | |
| _patch_embed = pipe.text_encoder.model.visual.patch_embed | |
| def _patch_embed_forward(hidden_states): | |
| proj = _patch_embed.proj | |
| weight = proj.weight.reshape(proj.weight.shape[0], -1) | |
| return hidden_states.to(weight.dtype).flatten(1) @ weight.T + proj.bias | |
| _patch_embed.forward = _patch_embed_forward | |
| pipe.to("cuda") | |
| AOTI_LOADED: list[str] = [] | |
| if os.environ.get("QWEN21_AOTI", "1") != "0": | |
| try: | |
| AOTI_LOADED, _cfg = aoti_load_pipeline(pipe, AOTI_REPO, token=os.environ.get("HF_TOKEN")) | |
| print(f"[load] aoti kernels: {AOTI_LOADED} (torch {_cfg['torch']})", flush=True) | |
| except Exception as exc: # noqa: BLE001 | |
| print(f"[load] aoti unavailable, running eager: {exc!r}", flush=True) | |
| print("[load] ready", flush=True) | |
| # -------------------------------------------------------------------------------------- | |
| # Inference | |
| # -------------------------------------------------------------------------------------- | |
| def build_prompt(azimuth: str, elevation: str, distance: str, extra: str = "") -> str: | |
| prompt = f"{TRIGGER} {azimuth} {elevation} {distance}" | |
| extra = (extra or "").strip() | |
| return f"{prompt}, {extra}" if extra else prompt | |
| def _duration(image=None, azimuth=None, elevation=None, distance=None, extra="", seed=0, | |
| randomize_seed=True, steps=DEFAULT_STEPS, guidance=DEFAULT_CFG, resolution=DEFAULT_RES, *a, **k): | |
| try: | |
| steps, res, cfg = int(steps), int(resolution), float(guidance) | |
| except (TypeError, ValueError): | |
| steps, res, cfg = DEFAULT_STEPS, DEFAULT_RES, DEFAULT_CFG | |
| # measured on ZeroGPU with AOTI, 30 steps, CFG 3: 1024px -> 28.4 s, 768px -> 17.7 s | |
| per_step = 0.4 * (res / 1024) ** 2.0 * (2.0 if cfg > 1 else 1.0) | |
| return int(math.ceil((6 + steps * per_step) * 1.15)) | |
| def generate( | |
| image: Image.Image, | |
| azimuth: str = "front-right quarter view", | |
| elevation: str = "eye-level shot", | |
| distance: str = "medium shot", | |
| extra: str = "", | |
| seed: int = 0, | |
| randomize_seed: bool = True, | |
| steps: int = DEFAULT_STEPS, | |
| guidance: float = DEFAULT_CFG, | |
| resolution: int = DEFAULT_RES, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Re-render an image from a new camera position with the QMA camera-angles LoRA. | |
| Args: | |
| image: source image. | |
| azimuth: where the camera orbits to, relative to the original camera (e.g. "right side view"). | |
| elevation: camera height ("low-angle shot", "eye-level shot", "elevated shot", "high-angle shot"). | |
| distance: framing ("close-up", "medium shot", "wide shot"). | |
| extra: optional extra edit instruction appended after the camera prompt. | |
| seed: random seed. | |
| randomize_seed: pick a random seed instead of `seed`. | |
| steps: denoising steps (card: 30). | |
| guidance: true CFG scale (card: 3; 1 disables CFG and is ~2x faster). | |
| resolution: output side length budget in px (1024 β 1 MP; 768 for quick previews). | |
| Returns: | |
| The re-rendered image, the seed used and the prompt sent to the model. | |
| """ | |
| if image is None: | |
| raise gr.Error("Please upload a source image first.") | |
| if azimuth not in AZIMUTHS or elevation not in ELEVATIONS or distance not in DISTANCES: | |
| raise gr.Error("Invalid camera position.") | |
| if randomize_seed: | |
| seed = random.randint(0, MAX_SEED) | |
| seed = int(seed) | |
| prompt = build_prompt(azimuth, elevation, distance, extra) | |
| kwargs = dict( | |
| prompt=prompt, | |
| image=image.convert("RGB"), | |
| num_inference_steps=int(steps), | |
| output_resolution=int(resolution), | |
| generator=torch.Generator("cpu").manual_seed(seed), | |
| ) | |
| if float(guidance) > 1.0: | |
| kwargs["negative_prompt"] = " " | |
| kwargs["true_cfg_scale"] = float(guidance) | |
| t0 = time.perf_counter() | |
| out = pipe(**kwargs).images[0] | |
| print(f"[gen] {prompt!r} steps={steps} cfg={guidance} res={resolution} seed={seed} " | |
| f"aoti={bool(AOTI_LOADED)} -> {out.size} in {time.perf_counter() - t0:.1f}s", flush=True) | |
| return out, seed, prompt | |
| # -------------------------------------------------------------------------------------- | |
| # UI | |
| # -------------------------------------------------------------------------------------- | |
| CSS = """ | |
| #col-container { max-width: 1200px; margin: 0 auto; } | |
| """ | |
| with gr.Blocks() as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.Markdown( | |
| f""" | |
| # π₯ QMA Camera Angles β Qwen-Image-2.1 | |
| Move the camera, keep everything else. Upload an image, pick where the camera goes β orbit around the | |
| subject, raise or lower it, push in or pull back β and the | |
| [QMA Camera Angles LoRA](https://huggingface.co/{LORA_REPO}) on | |
| [Qwen-Image-2.1](https://huggingface.co/{BASE_REPO}) renders the same scene from that viewpoint. | |
| Directions are relative to the **original camera**: "right side view" swings the camera 90Β° to its own right. | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| image = gr.Image(label="Source image", type="pil", height=380) | |
| azimuth = gr.Radio(AZIMUTHS, value="front-right quarter view", label="π§ Azimuth β orbit") | |
| elevation = gr.Radio(ELEVATIONS, value="eye-level shot", label="βοΈ Elevation β camera height") | |
| distance = gr.Radio(DISTANCES, value="medium shot", label="π Distance β framing") | |
| run_btn = gr.Button("π¬ Move camera", variant="primary", size="lg") | |
| with gr.Column(scale=1): | |
| result = gr.Image(label="New view", height=520, format="png") | |
| prompt_preview = gr.Textbox( | |
| label="Prompt sent to the model", | |
| value=build_prompt("front-right quarter view", "eye-level shot", "medium shot"), | |
| interactive=False, | |
| ) | |
| with gr.Accordion("βοΈ Advanced settings", open=False): | |
| extra = gr.Textbox( | |
| label="Extra edit instruction (optional)", | |
| value="", | |
| placeholder="e.g. at sunset β appended after the camera prompt", | |
| ) | |
| resolution = gr.Radio([768, 1024], value=DEFAULT_RES, label="Output resolution", | |
| info="1024 β 1 MP (best detail) Β· 768 for quick previews") | |
| steps = gr.Slider(10, 50, value=DEFAULT_STEPS, step=1, label="Steps") | |
| guidance = gr.Slider(1.0, 6.0, value=DEFAULT_CFG, step=0.5, label="CFG", | |
| info="Card recommends 3 Β· 1 disables CFG (~2x faster)") | |
| seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed") | |
| randomize_seed = gr.Checkbox(value=True, label="Randomize seed") | |
| gr.Markdown("Tips: quarter views, elevated shots and distance changes are the most reliable. " | |
| "Anything the source doesn't show (e.g. a character's back) is imagined.") | |
| gr.Examples( | |
| examples=[ | |
| ["examples/sniper.jpg", "back-right quarter view", "eye-level shot", "medium shot"], | |
| ["examples/neon_girl.jpg", "front-left quarter view", "elevated shot", "medium shot"], | |
| ["examples/earth.jpg", "front-right quarter view", "low-angle shot", "wide shot"], | |
| ["examples/sniper.jpg", "front view", "high-angle shot", "close-up"], | |
| ], | |
| inputs=[image, azimuth, elevation, distance], | |
| outputs=[result, seed, prompt_preview], | |
| fn=generate, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| for c in (azimuth, elevation, distance, extra): | |
| c.change(build_prompt, [azimuth, elevation, distance, extra], prompt_preview, | |
| api_visibility="private", queue=False) | |
| run_btn.click( | |
| generate, | |
| inputs=[image, azimuth, elevation, distance, extra, seed, randomize_seed, steps, guidance, resolution], | |
| outputs=[result, seed, prompt_preview], | |
| api_name="generate", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |