Download app.py from rustholm/clip-lab: direct link, hf CLI and curl.
- Browser
- Download file 30.6 kB
-
https://huggingface.co/spaces/rustholm/clip-lab/resolve/main/app.py
- Command line
-
hf download hf://spaces/rustholm/clip-lab/app.py
-
curl -L -o app.py https://huggingface.co/spaces/rustholm/clip-lab/resolve/main/app.py
30.6 kB
| """Clip tool.""" | |
| from __future__ import annotations | |
| import os | |
| # Before torch. See the sibling LMS Space: the guide VAE encode tripped the ZeroGPU allocator's expandable-segments | |
| # path (`NVML_SUCCESS == r INTERNAL ASSERT FAILED`) with memory free; plain `cudaMalloc` never queries NVML. | |
| _ALLOC_CONF_INHERITED = os.environ.get("PYTORCH_CUDA_ALLOC_CONF") | |
| os.environ["PYTORCH_CUDA_ALLOC_CONF"] = os.environ.get("CLIP_ALLOC_CONF", "expandable_segments:False") | |
| import faulthandler | |
| import hashlib | |
| import math | |
| import tempfile | |
| import time | |
| import traceback | |
| import uuid | |
| from functools import cache | |
| import spaces | |
| import gradio as gr | |
| import torch | |
| import h3_video | |
| # Model, adapters and helper Spaces come from the Space's secrets. | |
| MODEL_REPO = os.environ["CLIP_BASE"] | |
| CONDITIONER_SPACE = os.environ["CLIP_HELPER"] | |
| # The helper encodes in ~15 s warm; past this it is not coming back. | |
| CONDITIONER_TIMEOUT_SEC = 150 | |
| # Longest legit scripted pass (HD, 10 s) is ~10 min; a call past this is stuck, not slow. | |
| CALL_WATCHDOG_SEC = 900 | |
| RMBG_SPACE = os.environ["CLIP_CUTOUT"] | |
| GPU_SIZE = os.environ.get("CLIP_GPU_SIZE", "xlarge") | |
| ATTENTION = os.environ.get("CLIP_ATTENTION", "_native_cudnn") | |
| # The turbo adapter first, then the task adapter, both folded into the base weights before the GPU. | |
| LORAS = [ | |
| ( | |
| os.environ["CLIP_TURBO_REPO"], | |
| os.environ["CLIP_TURBO_FILE"], | |
| float(os.environ.get("CLIP_TURBO_STRENGTH", "1.0")), | |
| "turbo", | |
| ), | |
| ( | |
| os.environ["CLIP_LORA_REPO"], | |
| os.environ["CLIP_LORA_FILE"], | |
| float(os.environ.get("CLIP_LORA_STRENGTH", "1.0")), | |
| "task", | |
| ), | |
| ] | |
| # The `body_swap:` prompt. `{details}` is the reference | |
| # description, `{target}` names which guide subject to replace when more than one person appears. | |
| PROMPT_MAIN = ( | |
| "body_swap: subject_definitions: <Subject 1> is the {details} shown in <Picture 1>, whose identity, face, " | |
| "head, hair or fur, skin, body shape, proportions and clothing define the character in the target video. The " | |
| "guide video provides the scene and the performance: its background, environment, lighting, camera, framing, " | |
| "body pose, hand and arm movement, head rotation, gaze direction, lip movement and facial expression, frame by " | |
| "frame; only the main subject of the guide video is replaced: its face, hair, body and clothing become " | |
| "<Subject 1>. summary: [video generation + reference] Replace the guide subject with <Subject 1>, keeping the " | |
| "scene, camera and the complete motion and performance of the guide video. detailed_description: [Shot 1] " | |
| "<Subject 1> takes the exact place of the guide subject and moves with the same timing, body pose, gestures, " | |
| "head motion, gaze, speech and facial expressions, while keeping the complete look from <Picture 1>: head, " | |
| "face, skin, hair or fur, body shape and clothing stay stable in every frame, with no trace of the original " | |
| "subject's face, hair or clothing. The background, lighting, colors and camera of the guide video are " | |
| "preserved. Prevent identity drift, clothing changes, texture crawling and flicker. The result is a real, " | |
| "photographic video." | |
| ) | |
| PROMPT_TARGETED = ( | |
| "body_swap: subject_definitions: <Subject 1> is the {details} shown in <Picture 1>, whose identity, face, " | |
| "head, hair or fur, skin, body shape, proportions and clothing define the character in the target video. The " | |
| "guide video provides the scene and the performance: its background, environment, lighting, camera, framing, " | |
| "body pose, hand and arm movement, head rotation, gaze direction, lip movement and facial expression, frame by " | |
| "frame; only {target} is replaced: their face, hair, body and clothing become <Subject 1>. summary: [video " | |
| "generation + reference] Replace {target} in the guide video with <Subject 1>, keeping the scene, camera and " | |
| "the complete motion and performance of the guide video. detailed_description: [Shot 1] <Subject 1> takes the " | |
| "exact place of {target} and moves with the same timing, body pose, gestures, head motion, gaze, speech and " | |
| "facial expressions, while keeping the complete look from <Picture 1>: head, face, skin, hair or fur, body " | |
| "shape and clothing stay stable in every frame, with no trace of the original subject's face, hair or " | |
| "clothing. The background, lighting, colors and camera of the guide video are preserved. Prevent identity " | |
| "drift, clothing changes, texture crawling and flicker. The result is a real, photographic video." | |
| ) | |
| DEFAULT_DETAILS = "character" | |
| FPS = h3_video.FPS | |
| DEFAULT_STEPS = 8 | |
| DEFAULT_SEED = 46 # `RandomNoise(46, "fixed")` in the workflow. | |
| # `AIToolkitMiniMaxH3RefVideo.max_length` β a cap on the 24 fps frame count, snapped down to 17n + 5 afterwards. | |
| CLIP_LENGTHS = { | |
| "0.9 s Β· 22 frames": 22, | |
| "1.6 s Β· 39 frames": 39, | |
| "2.3 s Β· 56 frames": 56, | |
| "3.0 s Β· 73 frames": 73, | |
| "3.8 s Β· 90 frames": 90, | |
| "5.2 s Β· 124 frames": 124, | |
| "7.3 s Β· 175 frames": 175, | |
| "10.1 s Β· 243 frames": 243, | |
| } | |
| DEFAULT_CLIP = "2.3 s Β· 56 frames" | |
| # Working pixel area of the guide / generation, on the clip's own aspect ratio. | |
| DETAIL_LEVELS = { | |
| "0.26 MP Β· fastest (672x384 at 16:9)": 0.258, | |
| "0.42 MP Β· balanced (864x480 at 16:9)": 0.42, | |
| "0.59 MP Β· sharp (1024x576 at 16:9)": 0.59, | |
| "0.92 MP Β· 720p (1280x720 at 16:9)": 0.92, | |
| } | |
| DEFAULT_DETAIL = "0.42 MP Β· balanced (864x480 at 16:9)" | |
| # The reference image is encoded at its own resolution; the workflow sizes it to ~0.3 MP. The conditioner also | |
| # validates a canvas/frame count it never uses for an image-only `ref2va` request (the `<Picture 1>` embedding does | |
| # not depend on either) β these just have to be inside its accepted range, so they are fixed rather than tied to the | |
| # target, which can be shorter than the conditioner's own 2 s duration floor. | |
| REFERENCE_MEGAPIXELS = float(os.environ.get("CLIP_REF_MP", "0.3")) | |
| CONDITIONER_CANVAS = "960x544 Β· 16:9 fast" | |
| CONDITIONER_FRAMES = 124 | |
| OUTPUT_DIR = os.path.join(tempfile.gettempdir(), "h3-charswap-out") | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| PIPE = None | |
| LOAD_ERROR: str | None = None | |
| LOAD_NOTES: list[str] = [] | |
| _PROMPT_CACHE: dict[str, tuple] = {} | |
| _RMBG_CACHE: dict[str, str] = {} | |
| # ββ Model loading βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def load_models() -> str | None: | |
| """Load the denoising half β `transformer_ref` + video VAE + schedulers β and fold both LoRAs in.""" | |
| global PIPE, LOAD_ERROR | |
| if PIPE is not None or LOAD_ERROR is not None: | |
| return LOAD_ERROR | |
| started = time.time() | |
| try: | |
| from diffusers import ComponentsManager | |
| from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline | |
| from huggingface_hub import hf_hub_download | |
| from h3_charswap_blocks import MiniMaxH3CharswapGeneratorBlocks | |
| from h3_lora import fold_comfy_lora | |
| MiniMaxH3ModularPipeline.min_duration = property(lambda self: 0.2) | |
| blocks = MiniMaxH3CharswapGeneratorBlocks() | |
| print(f"[gen] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True) | |
| pipe = blocks.init_pipeline(MODEL_REPO, components_manager=ComponentsManager(), collection="h3") | |
| pipe.load_components(dtype=torch.bfloat16, trust_remote_code=True) | |
| for name in ("transformer_ref", "vae", "scheduler", "audio_scheduler"): | |
| if getattr(pipe, name, None) is None: | |
| raise RuntimeError(f"Component `{name}` did not load; see the warning above for why.") | |
| pipe.vae.set_attention_backend("native") | |
| pipe.transformer_ref.set_attention_backend(ATTENTION) | |
| for repo, filename, strength, label in LORAS: | |
| if strength <= 0: | |
| continue | |
| path = hf_hub_download(repo, filename) | |
| status = fold_comfy_lora(pipe.transformer_ref, path, strength=strength, label=label) | |
| LOAD_NOTES.append(status) | |
| print(f"[lora] {status}", flush=True) | |
| pipe.transformer_ref.to("cuda") | |
| PIPE = pipe | |
| print(f"[gen] ready in {time.time() - started:.0f}s", flush=True) | |
| except Exception as error: | |
| traceback.print_exc() | |
| LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`" | |
| return LOAD_ERROR | |
| # ββ Remote conditioning + background removal ββββββββββββββββββββββββββββββββββ | |
| def _caller_ip_token() -> str | None: | |
| from gradio.context import LocalContext | |
| request = LocalContext.request.get() | |
| return request.headers.get("x-ip-token") if request is not None else None | |
| def _anonymous_client(space: str): | |
| from gradio_client import Client | |
| return Client(space) | |
| def _client(space: str, ip_token: str | None): | |
| """A client for a helper Space, billed to the visitor when their `x-ip-token` is available.""" | |
| if not ip_token: | |
| return _anonymous_client(space) | |
| from gradio_client import Client | |
| return Client(space, headers={"x-ip-token": ip_token}) | |
| def _file_digest(path: str) -> str: | |
| with open(path, "rb") as handle: | |
| return hashlib.sha256(handle.read()).hexdigest() | |
| def remove_background(image_path: str, ip_token: str | None) -> str: | |
| """Cut the reference subject out onto neutral gray via the RMBG service; fall back to the raw image.""" | |
| digest = _file_digest(image_path) | |
| if digest in _RMBG_CACHE and os.path.exists(_RMBG_CACHE[digest]): | |
| return _RMBG_CACHE[digest] | |
| from gradio_client import handle_file | |
| try: | |
| result = _client(RMBG_SPACE, ip_token).predict( | |
| image_path=handle_file(image_path), background="gray", api_name="/rmbg" | |
| ) | |
| except Exception as error: | |
| print(f"[rmbg] keeping the raw reference, service failed: {error}", flush=True) | |
| return image_path | |
| _RMBG_CACHE[digest] = result | |
| return result | |
| def _read_conditioning(path: str): | |
| from safetensors import safe_open | |
| with safe_open(path, framework="pt") as handle: | |
| return handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags") | |
| def encode_prompt(prompt: str, image_path: str): | |
| """`prompt_embeds` + `text_token_tags` for a `body_swap` caption with the reference image as `<Picture 1>`. | |
| The reference image reaches Qwen3-VL as an ordered `ref2va` image reference, so the embedding carries its | |
| `<Picture 1>` vision block. Cached per `(prompt, reference image)`; the conditioner's `num_frames` is a fixed | |
| in-range value because an image-only embedding does not depend on it. | |
| """ | |
| key = f"{prompt}\n{_file_digest(image_path)}" | |
| if key in _PROMPT_CACHE: | |
| return _PROMPT_CACHE[key] | |
| from gradio_client import handle_file | |
| ip_token = _caller_ip_token() | |
| def call(): | |
| # Bounded: an unanswered helper call would hold this Space's one slot forever. | |
| return _client(CONDITIONER_SPACE, ip_token).submit( | |
| prompt=prompt, | |
| media=[handle_file(image_path)], | |
| kinds="image", | |
| canvas=CONDITIONER_CANVAS, | |
| num_frames=CONDITIONER_FRAMES, | |
| rewrite_prompt=False, | |
| api_name="/encode_ref2va", | |
| ).result(timeout=CONDITIONER_TIMEOUT_SEC) | |
| try: | |
| path, _plan = call() | |
| except Exception as first: | |
| print(f"[conditioner] retrying with a fresh client after: {first}", flush=True) | |
| _anonymous_client.cache_clear() | |
| try: | |
| path, _plan = call() | |
| except Exception as second: | |
| raise gr.Error( | |
| "The Qwen3-VL conditioner Space could not encode the reference " | |
| f"({type(second).__name__}: {second})." | |
| ) from second | |
| embeds = _read_conditioning(path) | |
| _PROMPT_CACHE[key] = embeds | |
| return embeds | |
| # ββ GPU budget ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Seconds per step vs packed row count: linear matmuls, quadratic full self-attention. Calibrated on this pool. | |
| _DUR_B, _DUR_C = 1.1745e-4, 3.8396e-9 | |
| _VAE_PER_REFERENCE_PIXELS, _REFERENCE_PIXELS = 26.0, 960 * 544 * 124 | |
| _PLACEMENT_ALLOWANCE = 15 | |
| _SAFETY = 1.2 | |
| def _reference_latent_patches(megapixels: float) -> int: | |
| """Patch rows a reference image sized to `megapixels` adds (one latent frame, /32 patch grid).""" | |
| side = math.sqrt(megapixels * 1e6) | |
| return max(1, round(side / 32) ** 2) | |
| def estimate_duration(height: int, width: int, num_frames: int, steps: int) -> int: | |
| """Seconds of GPU time one swap pass needs: reference + guide conditioning rows and the target included.""" | |
| latent_frames = (num_frames - 5) // 17 * 5 + 2 | |
| patches = (height // 32) * (width // 32) | |
| # Guide doubles the video rows (one conditioning row per target row); the reference adds one small frame. | |
| rows = 2 * latent_frames * patches + _reference_latent_patches(REFERENCE_MEGAPIXELS) | |
| denoise = steps * (_DUR_B * rows + _DUR_C * rows**2) | |
| vae = _VAE_PER_REFERENCE_PIXELS * (height * width * num_frames) / _REFERENCE_PIXELS | |
| return max(60, int((denoise + vae) * _SAFETY) + _PLACEMENT_ALLOWANCE) | |
| def _gpu_duration(ref_pixels, pixels, prompt_embeds, tags, height, width, num_frames, steps, seed, *a, **k): | |
| # The pipeline runs steps + 1 denoise passes (see `_swap`): declare what actually runs. | |
| return estimate_duration(int(height), int(width), int(num_frames), int(steps) + 1) | |
| # ββ Inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def encode_condition(vae, pixels, pixel_mean, pixel_std, encode_seed): | |
| """`diffusers...minimax_h3.encoders.encode_vae_condition`, cast to the VAE's own dtype (see the LMS Space). | |
| A single-frame `(1, 3, 1, H, W)` input (the reference image) runs the spatial encoder alone and returns one | |
| latent frame; a clip runs the temporal chunking. `no_grad` keeps the float32 encoder's activations from being | |
| retained for a backward pass that never comes. | |
| """ | |
| latents_mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1) | |
| latents_std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1) | |
| pixel_mean = torch.tensor(pixel_mean, device=pixels.device).view(1, -1, 1, 1, 1) | |
| pixel_std = torch.tensor(pixel_std, device=pixels.device).view(1, -1, 1, 1, 1) | |
| pixels = (pixels.to(torch.float32).div(255.0) - pixel_mean) / pixel_std | |
| posterior = vae.encode(pixels.to(vae.dtype), return_dict=False)[0] | |
| latents = posterior.sample(generator=torch.Generator().manual_seed(encode_seed)) | |
| latents = latents.to(torch.float16).float().cpu() | |
| return (latents - latents_mean) / latents_std | |
| def _swap(ref_pixels, pixels, prompt_embeds, text_token_tags, height, width, num_frames, steps, seed): | |
| """Encode the reference image and the guide clip, run the joint denoising loop, decode the video.""" | |
| PIPE.vae.to("cuda") | |
| free, total = torch.cuda.mem_get_info() | |
| print( | |
| f"[gpu] {free / 2**30:.1f} GiB free of {total / 2**30:.1f} GiB Β· alloc conf " | |
| f"{os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!r}", | |
| flush=True, | |
| ) | |
| started = time.time() | |
| reference = encode_condition(PIPE.vae, ref_pixels.to("cuda"), PIPE.pixel_mean, PIPE.pixel_std, PIPE.keyframe_encode_seed) | |
| del ref_pixels | |
| pixels = pixels.to("cuda") | |
| try: | |
| guide = encode_condition(PIPE.vae, pixels, PIPE.pixel_mean, PIPE.pixel_std, PIPE.keyframe_encode_seed) | |
| except RuntimeError as error: | |
| # The failed attempt's traceback pins every intermediate activation alive; drop it before retrying on tiles. | |
| error.__traceback__ = None | |
| torch.cuda.empty_cache() | |
| print(f"[gpu] guide encode failed ({type(error).__name__}); retrying on 128px tiles.", flush=True) | |
| PIPE.vae.enable_tiling(tile_sample_min_height=128, tile_sample_min_width=128) | |
| guide = encode_condition(PIPE.vae, pixels, PIPE.pixel_mean, PIPE.pixel_std, PIPE.keyframe_encode_seed) | |
| del pixels | |
| torch.cuda.empty_cache() | |
| encoded_in = time.time() - started | |
| print( | |
| f"[gpu] encoded reference {tuple(reference.shape)} + guide {tuple(guide.shape)} in {encoded_in:.1f}s", | |
| flush=True, | |
| ) | |
| started = time.time() | |
| state = PIPE( | |
| prompt_embeds=prompt_embeds.to("cuda"), | |
| text_token_tags=text_token_tags, | |
| # Packed order: reference image(s) first, then the aligned guide β the layout reserves them in this order. | |
| condition_latents=[reference, guide], | |
| num_reference_images=1, | |
| height=int(height), | |
| width=int(width), | |
| num_frames=int(num_frames), | |
| num_inference_steps=int(steps) + 1, | |
| generator=torch.Generator("cpu").manual_seed(int(seed)), | |
| output_type="np", | |
| ) | |
| return state.get("videos")[0], encoded_in, time.time() - started | |
| def swap( | |
| video: str | None, | |
| reference: str | None, | |
| details: str = "", | |
| target: str = "", | |
| remove_bg: bool = True, | |
| clip_length: str = DEFAULT_CLIP, | |
| detail: str = DEFAULT_DETAIL, | |
| steps: int = DEFAULT_STEPS, | |
| seed: int = DEFAULT_SEED, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Swap the main subject of a guide video for the character in a reference image. | |
| Args: | |
| video: The guide clip β the scene and the performance to keep. Any resolution / frame rate. | |
| reference: The character image (`<Picture 1>`) whose identity, body and clothing replace the subject. | |
| details: A short description of the reference character, e.g. "an elderly man with a grey beard". | |
| target: Which guide subject to replace when more than one person appears, e.g. "the woman on the left". | |
| remove_bg: Cut the reference onto neutral gray first (recommended). | |
| clip_length: How much of the guide to process, as a 17n + 5 frame count at 24 fps. | |
| detail: Working pixel area; the whole pass runs here, guide and result alike. | |
| steps: Euler steps. The shipped workflow uses 8. | |
| seed: Noise seed. | |
| Returns: | |
| The swapped clip, with the guide's own soundtrack, and a one-line report of what ran. | |
| """ | |
| if LOAD_ERROR: | |
| raise gr.Error(LOAD_ERROR.replace("**", "")) | |
| if not video: | |
| raise gr.Error("Upload a guide video β the scene and performance to keep.") | |
| if not reference: | |
| raise gr.Error("Upload a reference character image β the identity to swap in.") | |
| max_length = CLIP_LENGTHS.get(clip_length, CLIP_LENGTHS[DEFAULT_CLIP]) | |
| megapixels = DETAIL_LEVELS.get(detail, DETAIL_LEVELS[DEFAULT_DETAIL]) | |
| progress(0.02, desc="Planning the guide clip ...") | |
| plan = h3_video.plan_clip(video, megapixels, max_length) | |
| height, width, num_frames = plan["height"], plan["width"], plan["num_frames"] | |
| ip_token = _caller_ip_token() | |
| if remove_bg: | |
| progress(0.08, desc="Removing the reference background ...") | |
| reference = remove_background(reference, ip_token) | |
| details = (details or "").strip() or DEFAULT_DETAILS | |
| target = (target or "").strip() | |
| prompt = (PROMPT_TARGETED if target else PROMPT_MAIN).format(details=details, target=target) | |
| progress(0.14, desc="Encoding the reference and caption ...") | |
| prompt_embeds, text_token_tags = encode_prompt(prompt, reference) | |
| progress(0.2, desc="Sizing the reference image ...") | |
| ref_pixels = h3_video.load_image(reference, REFERENCE_MEGAPIXELS) | |
| progress(0.26, desc=f"Re-timing {plan['source_frames']} frames to 24 fps ...") | |
| pixels = h3_video.load_guide(video, plan) | |
| progress(0.34, desc=f"Swapping {num_frames} frames at {width}x{height} in {int(steps)} steps ...") | |
| frames, encoded_in, denoised_in = _swap( | |
| ref_pixels, pixels, prompt_embeds, text_token_tags, height, width, num_frames, int(steps), int(seed) | |
| ) | |
| progress(0.94, desc="Encoding the result ...") | |
| stem = os.path.join(OUTPUT_DIR, uuid.uuid4().hex) | |
| silent = h3_video.write_video(frames, f"{stem}.mp4") | |
| out = h3_video.mux_source_audio(silent, video, num_frames, f"{stem}-audio.mp4") | |
| report = ( | |
| f"**{width}x{height} Β· {num_frames} frames ({num_frames / FPS:.2f} s @ {FPS} fps)** β " | |
| f"guide {plan['source_width']}x{plan['source_height']} @{plan['source_fps']:.4g} fps Β· " | |
| f"encode {encoded_in:.1f} s Β· {int(steps)} steps in {denoised_in:.1f} s Β· seed {int(seed)}" | |
| + ("" if out is not silent else " Β· no soundtrack in the guide") | |
| ) | |
| return out, report | |
| def swap_api(video, reference, details: str, prompt_override: str, max_frames: float, megapixels: float, | |
| steps: float, seed: float, ref_megapixels: float): | |
| """Scripted pass: numeric knobs instead of the UI's preset labels; an empty prompt_override uses the body_swap template.""" | |
| if LOAD_ERROR: | |
| raise gr.Error(LOAD_ERROR.replace("**", "")) | |
| call = uuid.uuid4().hex[:6] | |
| t0 = time.monotonic() | |
| def mark(stage: str) -> None: | |
| print(f"[call {call}] {stage} +{time.monotonic() - t0:.1f}s", flush=True) | |
| # A call that never returns holds this Space's one slot and every later call queues behind it | |
| # (06.11). If one runs past the longest legit pass, every thread's stack goes to the log. | |
| faulthandler.dump_traceback_later(CALL_WATCHDOG_SEC, exit=False) | |
| try: | |
| mark(f"start frames<={int(max_frames)} mp={float(megapixels)} steps={int(steps)}") | |
| plan = h3_video.plan_clip(video, float(megapixels), int(max_frames)) | |
| height, width, num_frames = plan["height"], plan["width"], plan["num_frames"] | |
| details = (details or "").strip() or DEFAULT_DETAILS | |
| prompt = (prompt_override or "").strip() or PROMPT_MAIN.format(details=details, target="") | |
| prompt_embeds, text_token_tags = encode_prompt(prompt, reference) | |
| mark("conditioned") | |
| ref_pixels = h3_video.load_image(reference, float(ref_megapixels) or REFERENCE_MEGAPIXELS) | |
| pixels = h3_video.load_guide(video, plan) | |
| mark(f"gpu wait {width}x{height} {num_frames}f") | |
| frames, encoded_in, denoised_in = _swap( | |
| ref_pixels, pixels, prompt_embeds, text_token_tags, height, width, num_frames, int(steps), int(seed) | |
| ) | |
| mark("gpu done") | |
| stem = os.path.join(OUTPUT_DIR, uuid.uuid4().hex) | |
| out = h3_video.write_video(frames, f"{stem}.mp4") | |
| mark("written") | |
| report = f"{width}x{height} {num_frames}f encode {encoded_in:.1f}s denoise {denoised_in:.1f}s steps {int(steps)}" | |
| return out, report | |
| finally: | |
| faulthandler.cancel_dump_traceback_later() | |
| # ββ UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| CSS = """ | |
| .contain { max-width: 1200px !important; margin: auto; } | |
| #col-out video { max-height: 560px; } | |
| """ | |
| # ββ Anchor-frame selection (CPU, no GPU) βββββββββββββββββββββββββββββββββββββ | |
| # The reference H3 propagates must show the body it will reveal; frame 0 can hide it (a curled or | |
| # turned subject), the one failure class. mediapipe Pose scores body-landmark visibility across the | |
| # clip and picks the most visible, frontal frame. CPU-only β never touches the ZeroGPU budget. | |
| _POSE = None | |
| def _pose(): | |
| global _POSE | |
| if _POSE is None: | |
| import mediapipe as mp | |
| _POSE = mp.solutions.pose.Pose(static_image_mode=True, model_complexity=1) | |
| return _POSE | |
| def _frame_score(lm) -> float: | |
| def v(i): | |
| return lm[i].visibility | |
| LS, RS, LH, RH, NOSE, LE, RE = 11, 12, 23, 24, 0, 2, 5 | |
| torso = (v(LS) + v(RS) + v(LH) + v(RH)) / 4 # all four corners visible = unoccluded torso | |
| face = (v(NOSE) + v(LE) + v(RE)) / 3 # facing the camera | |
| ys = [lm[i].y for i in (LS, RS, LH, RH) if lm[i].visibility > 0.3] | |
| extent = (max(ys) - min(ys)) if len(ys) >= 2 else 0.0 # extended body, not curled | |
| return torso * 0.6 + face * 0.2 + min(extent, 0.5) * 0.4 | |
| def pick_anchor_frame(video, n: int = 12): | |
| """Return the path to the guide frame with the most visible, frontal torso. Falls back to the | |
| first sampled frame when pose finds nothing (a stylised clip, heavy occlusion throughout).""" | |
| import av | |
| from PIL import Image | |
| path = video["video"] if isinstance(video, dict) else video | |
| total = h3_video.probe(path)[0] | |
| idxs = sorted({min(round(i * (total - 1) / (n - 1)), total - 1) for i in range(max(2, n))}) | |
| want, last = set(idxs), idxs[-1] | |
| frames = {} | |
| with av.open(path) as c: | |
| c.streams.video[0].thread_type = "AUTO" | |
| for i, fr in enumerate(c.decode(video=0)): | |
| if i in want: | |
| frames[i] = fr.to_ndarray(format="rgb24") | |
| if i >= last: | |
| break | |
| if not frames: | |
| raise gr.Error("could not decode the guide video") | |
| pose = _pose() | |
| best_idx, best_s = next(iter(frames)), -1.0 | |
| for i in idxs: | |
| arr = frames.get(i) | |
| if arr is None: | |
| continue | |
| res = pose.process(arr) | |
| if not res.pose_landmarks: | |
| continue | |
| s = _frame_score(res.pose_landmarks.landmark) | |
| if s > best_s: | |
| best_s, best_idx = s, i | |
| out = os.path.join(OUTPUT_DIR, f"anchor_{uuid.uuid4().hex}.png") | |
| Image.fromarray(frames[best_idx]).save(out) | |
| print(f"[pick] anchor frame {best_idx}/{total} score {best_s:.3f}", flush=True) | |
| return out | |
| with gr.Blocks(title="clip") as demo: | |
| gr.Markdown("# clip") | |
| with gr.Row(): | |
| with gr.Column(): | |
| video = gr.Video(label="Guide video (scene + performance)", height=300) | |
| reference = gr.Image(label="Reference character (Picture 1)", type="filepath", height=240) | |
| run = gr.Button("Swap character", variant="primary") | |
| details = gr.Textbox( | |
| label="Reference description", | |
| placeholder="an elderly man with a grey beard and a dark jacket", | |
| lines=2, | |
| info="Describe the reference character. Clear descriptions help the swap hold identity.", | |
| ) | |
| with gr.Accordion("Advanced", open=False): | |
| target = gr.Textbox( | |
| label="Which subject to replace (optional)", | |
| placeholder="the woman on the left", | |
| info="Only needed when several people appear; otherwise the most prominent subject is swapped.", | |
| ) | |
| remove_bg = gr.Checkbox( | |
| label="Remove reference background", value=True, | |
| info="Cuts the subject onto neutral gray first.", | |
| ) | |
| with gr.Row(): | |
| clip_length = gr.Dropdown( | |
| label="Clip length", choices=list(CLIP_LENGTHS), value=DEFAULT_CLIP, | |
| info="Trimmed from the start. Longer costs quadratically more.", | |
| ) | |
| detail = gr.Dropdown( | |
| label="Detail level", choices=list(DETAIL_LEVELS), value=DEFAULT_DETAIL, | |
| info="Working pixel area, on the guide's own aspect ratio.", | |
| ) | |
| steps = gr.Slider( | |
| label="Steps", minimum=4, maximum=16, step=1, value=DEFAULT_STEPS, | |
| info="Euler steps. The shipped workflow uses 8 on top of a few-step turbo LoRA.", | |
| ) | |
| seed = gr.Slider(label="Seed", minimum=0, maximum=2**31 - 1, step=1, value=DEFAULT_SEED) | |
| with gr.Column(elem_id="col-out"): | |
| result = gr.Video(label="Swapped", height=300, autoplay=True) | |
| report = gr.Markdown() | |
| gr.Examples( | |
| examples=[ | |
| [ | |
| "examples/guide_man_glasses.mp4", | |
| "examples/ref_vangogh.jpg", | |
| "a man with red hair and a red beard, a blue coat and a pale blue background, in a Van Gogh self-portrait", | |
| "", True, DEFAULT_CLIP, DEFAULT_DETAIL, DEFAULT_STEPS, DEFAULT_SEED, | |
| ], | |
| [ | |
| "examples/guide_woman_laughing.mp4", | |
| "examples/ref_vermeer.jpg", | |
| "a young woman with a blue and gold headscarf, a pearl earring and a white collar, Vermeer painting", | |
| "", True, DEFAULT_CLIP, DEFAULT_DETAIL, DEFAULT_STEPS, DEFAULT_SEED, | |
| ], | |
| [ | |
| "examples/guide_podcaster.mp4", | |
| "examples/ref_monalisa.jpg", | |
| "a woman with long dark hair and a faint smile, in a Renaissance oil portrait", | |
| "", True, DEFAULT_CLIP, DEFAULT_DETAIL, DEFAULT_STEPS, DEFAULT_SEED, | |
| ], | |
| ], | |
| inputs=[video, reference, details, target, remove_bg, clip_length, detail, steps, seed], | |
| outputs=[result, report], | |
| fn=swap, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| run.click( | |
| swap, | |
| inputs=[video, reference, details, target, remove_bg, clip_length, detail, steps, seed], | |
| outputs=[result, report], | |
| api_name="swap", | |
| ) | |
| with gr.Row(visible=False): | |
| api_prompt = gr.Textbox() | |
| api_frames = gr.Number(value=124) | |
| api_mp = gr.Number(value=0.59) | |
| api_steps = gr.Number(value=8) | |
| api_seed = gr.Number(value=46) | |
| api_ref_mp = gr.Number(value=0.3) | |
| api_btn = gr.Button() | |
| api_btn.click( | |
| swap_api, | |
| inputs=[video, reference, details, api_prompt, api_frames, api_mp, api_steps, api_seed, api_ref_mp], | |
| outputs=[result, report], | |
| api_name="swap_api", | |
| ) | |
| with gr.Row(visible=False): | |
| pick_video = gr.Video() | |
| pick_out = gr.Image(type="filepath") | |
| pick_btn = gr.Button() | |
| pick_btn.click(pick_anchor_frame, inputs=[pick_video], outputs=[pick_out], api_name="pick_frame") | |
| if load_models(): | |
| print(LOAD_ERROR, flush=True) | |
| if __name__ == "__main__": | |
| demo.launch( | |
| theme=gr.themes.Citrus(), | |
| css=CSS, | |
| show_error=True, | |
| allowed_paths=[OUTPUT_DIR], | |
| ) | |