"""Clip tool.""" from __future__ import annotations import os # Before torch. See the sibling LMS Space: the guide VAE encode tripped the ZeroGPU allocator's expandable-segments # path (`NVML_SUCCESS == r INTERNAL ASSERT FAILED`) with memory free; plain `cudaMalloc` never queries NVML. _ALLOC_CONF_INHERITED = os.environ.get("PYTORCH_CUDA_ALLOC_CONF") os.environ["PYTORCH_CUDA_ALLOC_CONF"] = os.environ.get("CLIP_ALLOC_CONF", "expandable_segments:False") import faulthandler import hashlib import math import tempfile import time import traceback import uuid from functools import cache import spaces import gradio as gr import torch import h3_video # Model, adapters and helper Spaces come from the Space's secrets. MODEL_REPO = os.environ["CLIP_BASE"] CONDITIONER_SPACE = os.environ["CLIP_HELPER"] # The helper encodes in ~15 s warm; past this it is not coming back. CONDITIONER_TIMEOUT_SEC = 150 # Longest legit scripted pass (HD, 10 s) is ~10 min; a call past this is stuck, not slow. CALL_WATCHDOG_SEC = 900 RMBG_SPACE = os.environ["CLIP_CUTOUT"] GPU_SIZE = os.environ.get("CLIP_GPU_SIZE", "xlarge") ATTENTION = os.environ.get("CLIP_ATTENTION", "_native_cudnn") # The turbo adapter first, then the task adapter, both folded into the base weights before the GPU. LORAS = [ ( os.environ["CLIP_TURBO_REPO"], os.environ["CLIP_TURBO_FILE"], float(os.environ.get("CLIP_TURBO_STRENGTH", "1.0")), "turbo", ), ( os.environ["CLIP_LORA_REPO"], os.environ["CLIP_LORA_FILE"], float(os.environ.get("CLIP_LORA_STRENGTH", "1.0")), "task", ), ] # The `body_swap:` prompt. `{details}` is the reference # description, `{target}` names which guide subject to replace when more than one person appears. PROMPT_MAIN = ( "body_swap: subject_definitions: is the {details} shown in , whose identity, face, " "head, hair or fur, skin, body shape, proportions and clothing define the character in the target video. The " "guide video provides the scene and the performance: its background, environment, lighting, camera, framing, " "body pose, hand and arm movement, head rotation, gaze direction, lip movement and facial expression, frame by " "frame; only the main subject of the guide video is replaced: its face, hair, body and clothing become " ". summary: [video generation + reference] Replace the guide subject with , keeping the " "scene, camera and the complete motion and performance of the guide video. detailed_description: [Shot 1] " " takes the exact place of the guide subject and moves with the same timing, body pose, gestures, " "head motion, gaze, speech and facial expressions, while keeping the complete look from : head, " "face, skin, hair or fur, body shape and clothing stay stable in every frame, with no trace of the original " "subject's face, hair or clothing. The background, lighting, colors and camera of the guide video are " "preserved. Prevent identity drift, clothing changes, texture crawling and flicker. The result is a real, " "photographic video." ) PROMPT_TARGETED = ( "body_swap: subject_definitions: is the {details} shown in , whose identity, face, " "head, hair or fur, skin, body shape, proportions and clothing define the character in the target video. The " "guide video provides the scene and the performance: its background, environment, lighting, camera, framing, " "body pose, hand and arm movement, head rotation, gaze direction, lip movement and facial expression, frame by " "frame; only {target} is replaced: their face, hair, body and clothing become . summary: [video " "generation + reference] Replace {target} in the guide video with , keeping the scene, camera and " "the complete motion and performance of the guide video. detailed_description: [Shot 1] takes the " "exact place of {target} and moves with the same timing, body pose, gestures, head motion, gaze, speech and " "facial expressions, while keeping the complete look from : head, face, skin, hair or fur, body " "shape and clothing stay stable in every frame, with no trace of the original subject's face, hair or " "clothing. The background, lighting, colors and camera of the guide video are preserved. Prevent identity " "drift, clothing changes, texture crawling and flicker. The result is a real, photographic video." ) DEFAULT_DETAILS = "character" FPS = h3_video.FPS DEFAULT_STEPS = 8 DEFAULT_SEED = 46 # `RandomNoise(46, "fixed")` in the workflow. # `AIToolkitMiniMaxH3RefVideo.max_length` — a cap on the 24 fps frame count, snapped down to 17n + 5 afterwards. CLIP_LENGTHS = { "0.9 s · 22 frames": 22, "1.6 s · 39 frames": 39, "2.3 s · 56 frames": 56, "3.0 s · 73 frames": 73, "3.8 s · 90 frames": 90, "5.2 s · 124 frames": 124, "7.3 s · 175 frames": 175, "10.1 s · 243 frames": 243, } DEFAULT_CLIP = "2.3 s · 56 frames" # Working pixel area of the guide / generation, on the clip's own aspect ratio. DETAIL_LEVELS = { "0.26 MP · fastest (672x384 at 16:9)": 0.258, "0.42 MP · balanced (864x480 at 16:9)": 0.42, "0.59 MP · sharp (1024x576 at 16:9)": 0.59, "0.92 MP · 720p (1280x720 at 16:9)": 0.92, } DEFAULT_DETAIL = "0.42 MP · balanced (864x480 at 16:9)" # The reference image is encoded at its own resolution; the workflow sizes it to ~0.3 MP. The conditioner also # validates a canvas/frame count it never uses for an image-only `ref2va` request (the `` embedding does # not depend on either) — these just have to be inside its accepted range, so they are fixed rather than tied to the # target, which can be shorter than the conditioner's own 2 s duration floor. REFERENCE_MEGAPIXELS = float(os.environ.get("CLIP_REF_MP", "0.3")) CONDITIONER_CANVAS = "960x544 · 16:9 fast" CONDITIONER_FRAMES = 124 OUTPUT_DIR = os.path.join(tempfile.gettempdir(), "h3-charswap-out") os.makedirs(OUTPUT_DIR, exist_ok=True) PIPE = None LOAD_ERROR: str | None = None LOAD_NOTES: list[str] = [] _PROMPT_CACHE: dict[str, tuple] = {} _RMBG_CACHE: dict[str, str] = {} # ── Model loading ─────────────────────────────────────────────────────────── def load_models() -> str | None: """Load the denoising half — `transformer_ref` + video VAE + schedulers — and fold both LoRAs in.""" global PIPE, LOAD_ERROR if PIPE is not None or LOAD_ERROR is not None: return LOAD_ERROR started = time.time() try: from diffusers import ComponentsManager from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline from huggingface_hub import hf_hub_download from h3_charswap_blocks import MiniMaxH3CharswapGeneratorBlocks from h3_lora import fold_comfy_lora MiniMaxH3ModularPipeline.min_duration = property(lambda self: 0.2) blocks = MiniMaxH3CharswapGeneratorBlocks() print(f"[gen] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True) pipe = blocks.init_pipeline(MODEL_REPO, components_manager=ComponentsManager(), collection="h3") pipe.load_components(dtype=torch.bfloat16, trust_remote_code=True) for name in ("transformer_ref", "vae", "scheduler", "audio_scheduler"): if getattr(pipe, name, None) is None: raise RuntimeError(f"Component `{name}` did not load; see the warning above for why.") pipe.vae.set_attention_backend("native") pipe.transformer_ref.set_attention_backend(ATTENTION) for repo, filename, strength, label in LORAS: if strength <= 0: continue path = hf_hub_download(repo, filename) status = fold_comfy_lora(pipe.transformer_ref, path, strength=strength, label=label) LOAD_NOTES.append(status) print(f"[lora] {status}", flush=True) pipe.transformer_ref.to("cuda") PIPE = pipe print(f"[gen] ready in {time.time() - started:.0f}s", flush=True) except Exception as error: traceback.print_exc() LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`" return LOAD_ERROR # ── Remote conditioning + background removal ────────────────────────────────── def _caller_ip_token() -> str | None: from gradio.context import LocalContext request = LocalContext.request.get() return request.headers.get("x-ip-token") if request is not None else None @cache def _anonymous_client(space: str): from gradio_client import Client return Client(space) def _client(space: str, ip_token: str | None): """A client for a helper Space, billed to the visitor when their `x-ip-token` is available.""" if not ip_token: return _anonymous_client(space) from gradio_client import Client return Client(space, headers={"x-ip-token": ip_token}) def _file_digest(path: str) -> str: with open(path, "rb") as handle: return hashlib.sha256(handle.read()).hexdigest() def remove_background(image_path: str, ip_token: str | None) -> str: """Cut the reference subject out onto neutral gray via the RMBG service; fall back to the raw image.""" digest = _file_digest(image_path) if digest in _RMBG_CACHE and os.path.exists(_RMBG_CACHE[digest]): return _RMBG_CACHE[digest] from gradio_client import handle_file try: result = _client(RMBG_SPACE, ip_token).predict( image_path=handle_file(image_path), background="gray", api_name="/rmbg" ) except Exception as error: print(f"[rmbg] keeping the raw reference, service failed: {error}", flush=True) return image_path _RMBG_CACHE[digest] = result return result def _read_conditioning(path: str): from safetensors import safe_open with safe_open(path, framework="pt") as handle: return handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags") def encode_prompt(prompt: str, image_path: str): """`prompt_embeds` + `text_token_tags` for a `body_swap` caption with the reference image as ``. The reference image reaches Qwen3-VL as an ordered `ref2va` image reference, so the embedding carries its `` vision block. Cached per `(prompt, reference image)`; the conditioner's `num_frames` is a fixed in-range value because an image-only embedding does not depend on it. """ key = f"{prompt}\n{_file_digest(image_path)}" if key in _PROMPT_CACHE: return _PROMPT_CACHE[key] from gradio_client import handle_file ip_token = _caller_ip_token() def call(): # Bounded: an unanswered helper call would hold this Space's one slot forever. return _client(CONDITIONER_SPACE, ip_token).submit( prompt=prompt, media=[handle_file(image_path)], kinds="image", canvas=CONDITIONER_CANVAS, num_frames=CONDITIONER_FRAMES, rewrite_prompt=False, api_name="/encode_ref2va", ).result(timeout=CONDITIONER_TIMEOUT_SEC) try: path, _plan = call() except Exception as first: print(f"[conditioner] retrying with a fresh client after: {first}", flush=True) _anonymous_client.cache_clear() try: path, _plan = call() except Exception as second: raise gr.Error( "The Qwen3-VL conditioner Space could not encode the reference " f"({type(second).__name__}: {second})." ) from second embeds = _read_conditioning(path) _PROMPT_CACHE[key] = embeds return embeds # ── GPU budget ────────────────────────────────────────────────────────────── # Seconds per step vs packed row count: linear matmuls, quadratic full self-attention. Calibrated on this pool. _DUR_B, _DUR_C = 1.1745e-4, 3.8396e-9 _VAE_PER_REFERENCE_PIXELS, _REFERENCE_PIXELS = 26.0, 960 * 544 * 124 _PLACEMENT_ALLOWANCE = 15 _SAFETY = 1.2 def _reference_latent_patches(megapixels: float) -> int: """Patch rows a reference image sized to `megapixels` adds (one latent frame, /32 patch grid).""" side = math.sqrt(megapixels * 1e6) return max(1, round(side / 32) ** 2) def estimate_duration(height: int, width: int, num_frames: int, steps: int) -> int: """Seconds of GPU time one swap pass needs: reference + guide conditioning rows and the target included.""" latent_frames = (num_frames - 5) // 17 * 5 + 2 patches = (height // 32) * (width // 32) # Guide doubles the video rows (one conditioning row per target row); the reference adds one small frame. rows = 2 * latent_frames * patches + _reference_latent_patches(REFERENCE_MEGAPIXELS) denoise = steps * (_DUR_B * rows + _DUR_C * rows**2) vae = _VAE_PER_REFERENCE_PIXELS * (height * width * num_frames) / _REFERENCE_PIXELS return max(60, int((denoise + vae) * _SAFETY) + _PLACEMENT_ALLOWANCE) def _gpu_duration(ref_pixels, pixels, prompt_embeds, tags, height, width, num_frames, steps, seed, *a, **k): # The pipeline runs steps + 1 denoise passes (see `_swap`): declare what actually runs. return estimate_duration(int(height), int(width), int(num_frames), int(steps) + 1) # ── Inference ─────────────────────────────────────────────────────────────── @torch.no_grad() def encode_condition(vae, pixels, pixel_mean, pixel_std, encode_seed): """`diffusers...minimax_h3.encoders.encode_vae_condition`, cast to the VAE's own dtype (see the LMS Space). A single-frame `(1, 3, 1, H, W)` input (the reference image) runs the spatial encoder alone and returns one latent frame; a clip runs the temporal chunking. `no_grad` keeps the float32 encoder's activations from being retained for a backward pass that never comes. """ latents_mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1) latents_std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1) pixel_mean = torch.tensor(pixel_mean, device=pixels.device).view(1, -1, 1, 1, 1) pixel_std = torch.tensor(pixel_std, device=pixels.device).view(1, -1, 1, 1, 1) pixels = (pixels.to(torch.float32).div(255.0) - pixel_mean) / pixel_std posterior = vae.encode(pixels.to(vae.dtype), return_dict=False)[0] latents = posterior.sample(generator=torch.Generator().manual_seed(encode_seed)) latents = latents.to(torch.float16).float().cpu() return (latents - latents_mean) / latents_std @spaces.GPU(duration=_gpu_duration, size=GPU_SIZE) def _swap(ref_pixels, pixels, prompt_embeds, text_token_tags, height, width, num_frames, steps, seed): """Encode the reference image and the guide clip, run the joint denoising loop, decode the video.""" PIPE.vae.to("cuda") free, total = torch.cuda.mem_get_info() print( f"[gpu] {free / 2**30:.1f} GiB free of {total / 2**30:.1f} GiB · alloc conf " f"{os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!r}", flush=True, ) started = time.time() reference = encode_condition(PIPE.vae, ref_pixels.to("cuda"), PIPE.pixel_mean, PIPE.pixel_std, PIPE.keyframe_encode_seed) del ref_pixels pixels = pixels.to("cuda") try: guide = encode_condition(PIPE.vae, pixels, PIPE.pixel_mean, PIPE.pixel_std, PIPE.keyframe_encode_seed) except RuntimeError as error: # The failed attempt's traceback pins every intermediate activation alive; drop it before retrying on tiles. error.__traceback__ = None torch.cuda.empty_cache() print(f"[gpu] guide encode failed ({type(error).__name__}); retrying on 128px tiles.", flush=True) PIPE.vae.enable_tiling(tile_sample_min_height=128, tile_sample_min_width=128) guide = encode_condition(PIPE.vae, pixels, PIPE.pixel_mean, PIPE.pixel_std, PIPE.keyframe_encode_seed) del pixels torch.cuda.empty_cache() encoded_in = time.time() - started print( f"[gpu] encoded reference {tuple(reference.shape)} + guide {tuple(guide.shape)} in {encoded_in:.1f}s", flush=True, ) started = time.time() state = PIPE( prompt_embeds=prompt_embeds.to("cuda"), text_token_tags=text_token_tags, # Packed order: reference image(s) first, then the aligned guide — the layout reserves them in this order. condition_latents=[reference, guide], num_reference_images=1, height=int(height), width=int(width), num_frames=int(num_frames), num_inference_steps=int(steps) + 1, generator=torch.Generator("cpu").manual_seed(int(seed)), output_type="np", ) return state.get("videos")[0], encoded_in, time.time() - started def swap( video: str | None, reference: str | None, details: str = "", target: str = "", remove_bg: bool = True, clip_length: str = DEFAULT_CLIP, detail: str = DEFAULT_DETAIL, steps: int = DEFAULT_STEPS, seed: int = DEFAULT_SEED, progress=gr.Progress(track_tqdm=True), ): """Swap the main subject of a guide video for the character in a reference image. Args: video: The guide clip — the scene and the performance to keep. Any resolution / frame rate. reference: The character image (``) whose identity, body and clothing replace the subject. details: A short description of the reference character, e.g. "an elderly man with a grey beard". target: Which guide subject to replace when more than one person appears, e.g. "the woman on the left". remove_bg: Cut the reference onto neutral gray first (recommended). clip_length: How much of the guide to process, as a 17n + 5 frame count at 24 fps. detail: Working pixel area; the whole pass runs here, guide and result alike. steps: Euler steps. The shipped workflow uses 8. seed: Noise seed. Returns: The swapped clip, with the guide's own soundtrack, and a one-line report of what ran. """ if LOAD_ERROR: raise gr.Error(LOAD_ERROR.replace("**", "")) if not video: raise gr.Error("Upload a guide video — the scene and performance to keep.") if not reference: raise gr.Error("Upload a reference character image — the identity to swap in.") max_length = CLIP_LENGTHS.get(clip_length, CLIP_LENGTHS[DEFAULT_CLIP]) megapixels = DETAIL_LEVELS.get(detail, DETAIL_LEVELS[DEFAULT_DETAIL]) progress(0.02, desc="Planning the guide clip ...") plan = h3_video.plan_clip(video, megapixels, max_length) height, width, num_frames = plan["height"], plan["width"], plan["num_frames"] ip_token = _caller_ip_token() if remove_bg: progress(0.08, desc="Removing the reference background ...") reference = remove_background(reference, ip_token) details = (details or "").strip() or DEFAULT_DETAILS target = (target or "").strip() prompt = (PROMPT_TARGETED if target else PROMPT_MAIN).format(details=details, target=target) progress(0.14, desc="Encoding the reference and caption ...") prompt_embeds, text_token_tags = encode_prompt(prompt, reference) progress(0.2, desc="Sizing the reference image ...") ref_pixels = h3_video.load_image(reference, REFERENCE_MEGAPIXELS) progress(0.26, desc=f"Re-timing {plan['source_frames']} frames to 24 fps ...") pixels = h3_video.load_guide(video, plan) progress(0.34, desc=f"Swapping {num_frames} frames at {width}x{height} in {int(steps)} steps ...") frames, encoded_in, denoised_in = _swap( ref_pixels, pixels, prompt_embeds, text_token_tags, height, width, num_frames, int(steps), int(seed) ) progress(0.94, desc="Encoding the result ...") stem = os.path.join(OUTPUT_DIR, uuid.uuid4().hex) silent = h3_video.write_video(frames, f"{stem}.mp4") out = h3_video.mux_source_audio(silent, video, num_frames, f"{stem}-audio.mp4") report = ( f"**{width}x{height} · {num_frames} frames ({num_frames / FPS:.2f} s @ {FPS} fps)** — " f"guide {plan['source_width']}x{plan['source_height']} @{plan['source_fps']:.4g} fps · " f"encode {encoded_in:.1f} s · {int(steps)} steps in {denoised_in:.1f} s · seed {int(seed)}" + ("" if out is not silent else " · no soundtrack in the guide") ) return out, report def swap_api(video, reference, details: str, prompt_override: str, max_frames: float, megapixels: float, steps: float, seed: float, ref_megapixels: float): """Scripted pass: numeric knobs instead of the UI's preset labels; an empty prompt_override uses the body_swap template.""" if LOAD_ERROR: raise gr.Error(LOAD_ERROR.replace("**", "")) call = uuid.uuid4().hex[:6] t0 = time.monotonic() def mark(stage: str) -> None: print(f"[call {call}] {stage} +{time.monotonic() - t0:.1f}s", flush=True) # A call that never returns holds this Space's one slot and every later call queues behind it # (06.11). If one runs past the longest legit pass, every thread's stack goes to the log. faulthandler.dump_traceback_later(CALL_WATCHDOG_SEC, exit=False) try: mark(f"start frames<={int(max_frames)} mp={float(megapixels)} steps={int(steps)}") plan = h3_video.plan_clip(video, float(megapixels), int(max_frames)) height, width, num_frames = plan["height"], plan["width"], plan["num_frames"] details = (details or "").strip() or DEFAULT_DETAILS prompt = (prompt_override or "").strip() or PROMPT_MAIN.format(details=details, target="") prompt_embeds, text_token_tags = encode_prompt(prompt, reference) mark("conditioned") ref_pixels = h3_video.load_image(reference, float(ref_megapixels) or REFERENCE_MEGAPIXELS) pixels = h3_video.load_guide(video, plan) mark(f"gpu wait {width}x{height} {num_frames}f") frames, encoded_in, denoised_in = _swap( ref_pixels, pixels, prompt_embeds, text_token_tags, height, width, num_frames, int(steps), int(seed) ) mark("gpu done") stem = os.path.join(OUTPUT_DIR, uuid.uuid4().hex) out = h3_video.write_video(frames, f"{stem}.mp4") mark("written") report = f"{width}x{height} {num_frames}f encode {encoded_in:.1f}s denoise {denoised_in:.1f}s steps {int(steps)}" return out, report finally: faulthandler.cancel_dump_traceback_later() # ── UI ────────────────────────────────────────────────────────────────────── CSS = """ .contain { max-width: 1200px !important; margin: auto; } #col-out video { max-height: 560px; } """ # ── Anchor-frame selection (CPU, no GPU) ───────────────────────────────────── # The reference H3 propagates must show the body it will reveal; frame 0 can hide it (a curled or # turned subject), the one failure class. mediapipe Pose scores body-landmark visibility across the # clip and picks the most visible, frontal frame. CPU-only — never touches the ZeroGPU budget. _POSE = None def _pose(): global _POSE if _POSE is None: import mediapipe as mp _POSE = mp.solutions.pose.Pose(static_image_mode=True, model_complexity=1) return _POSE def _frame_score(lm) -> float: def v(i): return lm[i].visibility LS, RS, LH, RH, NOSE, LE, RE = 11, 12, 23, 24, 0, 2, 5 torso = (v(LS) + v(RS) + v(LH) + v(RH)) / 4 # all four corners visible = unoccluded torso face = (v(NOSE) + v(LE) + v(RE)) / 3 # facing the camera ys = [lm[i].y for i in (LS, RS, LH, RH) if lm[i].visibility > 0.3] extent = (max(ys) - min(ys)) if len(ys) >= 2 else 0.0 # extended body, not curled return torso * 0.6 + face * 0.2 + min(extent, 0.5) * 0.4 def pick_anchor_frame(video, n: int = 12): """Return the path to the guide frame with the most visible, frontal torso. Falls back to the first sampled frame when pose finds nothing (a stylised clip, heavy occlusion throughout).""" import av from PIL import Image path = video["video"] if isinstance(video, dict) else video total = h3_video.probe(path)[0] idxs = sorted({min(round(i * (total - 1) / (n - 1)), total - 1) for i in range(max(2, n))}) want, last = set(idxs), idxs[-1] frames = {} with av.open(path) as c: c.streams.video[0].thread_type = "AUTO" for i, fr in enumerate(c.decode(video=0)): if i in want: frames[i] = fr.to_ndarray(format="rgb24") if i >= last: break if not frames: raise gr.Error("could not decode the guide video") pose = _pose() best_idx, best_s = next(iter(frames)), -1.0 for i in idxs: arr = frames.get(i) if arr is None: continue res = pose.process(arr) if not res.pose_landmarks: continue s = _frame_score(res.pose_landmarks.landmark) if s > best_s: best_s, best_idx = s, i out = os.path.join(OUTPUT_DIR, f"anchor_{uuid.uuid4().hex}.png") Image.fromarray(frames[best_idx]).save(out) print(f"[pick] anchor frame {best_idx}/{total} score {best_s:.3f}", flush=True) return out with gr.Blocks(title="clip") as demo: gr.Markdown("# clip") with gr.Row(): with gr.Column(): video = gr.Video(label="Guide video (scene + performance)", height=300) reference = gr.Image(label="Reference character (Picture 1)", type="filepath", height=240) run = gr.Button("Swap character", variant="primary") details = gr.Textbox( label="Reference description", placeholder="an elderly man with a grey beard and a dark jacket", lines=2, info="Describe the reference character. Clear descriptions help the swap hold identity.", ) with gr.Accordion("Advanced", open=False): target = gr.Textbox( label="Which subject to replace (optional)", placeholder="the woman on the left", info="Only needed when several people appear; otherwise the most prominent subject is swapped.", ) remove_bg = gr.Checkbox( label="Remove reference background", value=True, info="Cuts the subject onto neutral gray first.", ) with gr.Row(): clip_length = gr.Dropdown( label="Clip length", choices=list(CLIP_LENGTHS), value=DEFAULT_CLIP, info="Trimmed from the start. Longer costs quadratically more.", ) detail = gr.Dropdown( label="Detail level", choices=list(DETAIL_LEVELS), value=DEFAULT_DETAIL, info="Working pixel area, on the guide's own aspect ratio.", ) steps = gr.Slider( label="Steps", minimum=4, maximum=16, step=1, value=DEFAULT_STEPS, info="Euler steps. The shipped workflow uses 8 on top of a few-step turbo LoRA.", ) seed = gr.Slider(label="Seed", minimum=0, maximum=2**31 - 1, step=1, value=DEFAULT_SEED) with gr.Column(elem_id="col-out"): result = gr.Video(label="Swapped", height=300, autoplay=True) report = gr.Markdown() gr.Examples( examples=[ [ "examples/guide_man_glasses.mp4", "examples/ref_vangogh.jpg", "a man with red hair and a red beard, a blue coat and a pale blue background, in a Van Gogh self-portrait", "", True, DEFAULT_CLIP, DEFAULT_DETAIL, DEFAULT_STEPS, DEFAULT_SEED, ], [ "examples/guide_woman_laughing.mp4", "examples/ref_vermeer.jpg", "a young woman with a blue and gold headscarf, a pearl earring and a white collar, Vermeer painting", "", True, DEFAULT_CLIP, DEFAULT_DETAIL, DEFAULT_STEPS, DEFAULT_SEED, ], [ "examples/guide_podcaster.mp4", "examples/ref_monalisa.jpg", "a woman with long dark hair and a faint smile, in a Renaissance oil portrait", "", True, DEFAULT_CLIP, DEFAULT_DETAIL, DEFAULT_STEPS, DEFAULT_SEED, ], ], inputs=[video, reference, details, target, remove_bg, clip_length, detail, steps, seed], outputs=[result, report], fn=swap, cache_examples=True, cache_mode="lazy", ) run.click( swap, inputs=[video, reference, details, target, remove_bg, clip_length, detail, steps, seed], outputs=[result, report], api_name="swap", ) with gr.Row(visible=False): api_prompt = gr.Textbox() api_frames = gr.Number(value=124) api_mp = gr.Number(value=0.59) api_steps = gr.Number(value=8) api_seed = gr.Number(value=46) api_ref_mp = gr.Number(value=0.3) api_btn = gr.Button() api_btn.click( swap_api, inputs=[video, reference, details, api_prompt, api_frames, api_mp, api_steps, api_seed, api_ref_mp], outputs=[result, report], api_name="swap_api", ) with gr.Row(visible=False): pick_video = gr.Video() pick_out = gr.Image(type="filepath") pick_btn = gr.Button() pick_btn.click(pick_anchor_frame, inputs=[pick_video], outputs=[pick_out], api_name="pick_frame") if load_models(): print(LOAD_ERROR, flush=True) if __name__ == "__main__": demo.launch( theme=gr.themes.Citrus(), css=CSS, show_error=True, allowed_paths=[OUTPUT_DIR], )