"""MiniMax-H3-Image prompt encoder service for Ultronprime/photo-person-swap-h3. Adapted from CozyByte/minimax-h3-image-encoder so user photos stay inside this account's own Spaces. Turns a prompt (plus up to 8 reference images) into the two tensors the MiniMax-H3-Image diffusion transformer reads: `prompt_embeds` of shape `(1, tokens, 5120)` — the 50th hidden state of the 33B Qwen3-VL conditioner — and `text_token_tags`, the modality tag of every row. The presentation is built exactly as the diffusers `MiniMaxH3ImageEditTextEncoderStep` builds it: a `Picture i: ` label and a vision block per reference, then the prompt verbatim. This runs on its own ZeroGPU slice so the studio Space can keep the 66 GB transformer resident on its own. See https://huggingface.co/MiniMaxAI/MiniMax-H3-Image. """ # Before anything that could initialize CUDA. import spaces import os import tempfile import time import gradio as gr import torch from PIL import Image, ImageOps from safetensors.torch import save_file from transformers import AutoTokenizer, Qwen3VLForConditionalGeneration, Qwen3VLProcessor REPO = os.environ.get("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3-Image") MAX_IMAGES = 8 MULTIPLE = 32 VISION_SHORT_EDGE = 1024 MAX_SEQUENCE_LENGTH = 4096 TEXT_ENCODER_LAYER = 50 TEXT_TAG, IMAGE_TAG = 1, 0 tokenizer = AutoTokenizer.from_pretrained(REPO, subfolder="tokenizer") processor = Qwen3VLProcessor.from_pretrained(REPO, subfolder="processor") text_encoder = ( Qwen3VLForConditionalGeneration.from_pretrained( REPO, subfolder="text_encoder", dtype=torch.bfloat16 ) .eval() ) # Only hidden state 50 (the output of decoder layer 50) is read, so layers 51-64 and the LM head never matter: # dropping them leaves every number identical and saves ~13 GB the GPU must take on, and ~20% of the compute. LAYERS = text_encoder.model.language_model.layers del LAYERS[TEXT_ENCODER_LAYER:] text_encoder.lm_head = torch.nn.Identity() text_encoder.to("cuda") CAPTURED = {} def _keep_output(module, args, output): CAPTURED["hidden"] = output[0] if isinstance(output, (tuple, list)) else output LAYERS[TEXT_ENCODER_LAYER - 1].register_forward_hook(_keep_output) def image_paths(images): """Gradio hands back a list of paths, sometimes wrapped in one-element lists.""" return [item[0] if isinstance(item, (list, tuple)) else item for item in images or []] def bound_short_edge(width: int, height: int, min_short_edge: int, max_short_edge: int, multiple: int): short_edge = min(width, height) if min_short_edge <= short_edge <= max_short_edge: return width, height scale = (min_short_edge if short_edge < min_short_edge else max_short_edge) / short_edge return ( max(multiple, round(width * scale / multiple) * multiple), max(multiple, round(height * scale / multiple) * multiple), ) def presentation(prompt: str, paths: list[str]): """The token ids and per-row modality tags of an H3-Image presentation.""" images = [] for path in paths[:MAX_IMAGES]: image = ImageOps.exif_transpose(Image.open(path)).convert("RGB") size = bound_short_edge(*image.size, VISION_SHORT_EDGE, VISION_SHORT_EDGE, MULTIPLE) if image.size != size: image = image.resize(size, Image.LANCZOS) images.append(image) token_ids: list[int] = [] token_tags: list[int] = [] vision = {"pixel_values": None, "image_grid_thw": None} if images: vision = processor.image_processor(images=images, return_tensors="pt") merge_size = processor.image_processor.merge_size**2 for index, grid in enumerate(vision["image_grid_thw"]): # The label carries no angle brackets; the prompt refers to it as ``. label_ids = tokenizer(f"Picture {index + 1}: ", add_special_tokens=False)["input_ids"] label_ids = label_ids[:MAX_SEQUENCE_LENGTH] vision_ids = ( [tokenizer.convert_tokens_to_ids("<|vision_start|>")] + [tokenizer.convert_tokens_to_ids("<|image_pad|>")] * (int(grid.prod()) // merge_size) + [tokenizer.convert_tokens_to_ids("<|vision_end|>")] ) token_ids += label_ids + vision_ids token_tags += [TEXT_TAG] * len(label_ids) + [IMAGE_TAG] * len(vision_ids) prompt_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"][:MAX_SEQUENCE_LENGTH] token_ids += prompt_ids token_tags += [TEXT_TAG] * len(prompt_ids) return token_ids, token_tags, vision def embed(token_ids: list[int], token_tags: list[int], vision: dict) -> tuple[torch.Tensor, torch.Tensor]: """One forward through the conditioner, keeping hidden state 50 and the row tags.""" input_ids = torch.tensor([token_ids], dtype=torch.long, device="cuda") mm_token_type_ids = torch.tensor( processor.create_mm_token_type_ids([token_ids]), dtype=torch.long, device="cuda" ) kwargs = {} if vision.get("pixel_values") is not None: kwargs["pixel_values"] = vision["pixel_values"].to("cuda", text_encoder.dtype) kwargs["image_grid_thw"] = vision["image_grid_thw"].to("cuda") with torch.no_grad(): outputs = text_encoder.model( input_ids=input_ids, attention_mask=torch.ones_like(input_ids), mm_token_type_ids=mm_token_type_ids, use_cache=False, **kwargs, ) del outputs embeds = CAPTURED.pop("hidden").to(torch.bfloat16).cpu().contiguous() tags = torch.tensor(token_tags, dtype=torch.long).contiguous() return embeds, tags def _duration(prompt, images): # One vision block per reference is ~1k extra rows for the 33B conditioner; a whole run with 5 pictures # measured 12-17 s end to end, so this keeps generous headroom while asking ZeroGPU for less. return min(120, 20 + 6 * len(image_paths(images))) @spaces.GPU(duration=_duration, size="xlarge") def encode_gpu(prompt: str, images: list[str] | None = None): """Encode a prompt and up to 8 reference images into `prompt_embeds` + `text_token_tags`. Reference images are ordered: the prompt refers to them as ``, ``, ... Returns a safetensors file holding both tensors. """ if not prompt or not prompt.strip(): raise gr.Error("Enter a prompt.") paths = image_paths(images) if len(paths) > MAX_IMAGES: raise gr.Error(f"At most {MAX_IMAGES} reference images.") start = time.time() print(f"[timing] GPU attached {start:.1f}", flush=True) token_ids, token_tags, vision = presentation(prompt.strip(), paths) embeds, tags = embed(token_ids, token_tags, vision) path = tempfile.NamedTemporaryFile(suffix=".safetensors", delete=False).name save_file({"prompt_embeds": embeds, "text_token_tags": tags}, path) print(f"encoded {len(token_ids)} tokens ({len(paths)} refs) in {time.time() - start:.1f}s") return path, f"{len(token_ids)} tokens · {len(paths)} reference image(s) · {time.time() - start:.1f}s" def encode(prompt: str, images: list[str] | None = None): """Encode a prompt and up to 8 reference images into `prompt_embeds` + `text_token_tags` (safetensors file).""" arrived = time.time() print(f"[timing] request arrived {arrived:.1f} (uploads done)", flush=True) result = encode_gpu(prompt, images) print(f"[timing] GPU call returned after {time.time() - arrived:.1f}s", flush=True) return result with gr.Blocks(title="H3-Image encoder") as demo: gr.Markdown("**H3-Image prompt encoder**: backend for Ultronprime/photo-person-swap-h3. Nothing is stored.") prompt = gr.Textbox(label="Prompt", lines=4) files = gr.File(label="Reference images, in order", file_count="multiple", file_types=["image"], type="filepath") run = gr.Button("Encode") wire = gr.File(label="prompt_embeds + text_token_tags") info = gr.Markdown() run.click(encode, [prompt, files], [wire, info], api_name="encode") demo.queue(default_concurrency_limit=2).launch()