Spaces:
Running on Zero
Running on Zero
Download app.py from Ultronprime/h3-image-encoder: direct link, hf CLI and curl.
- Browser
- Download file 8.08 kB
-
https://huggingface.co/spaces/Ultronprime/h3-image-encoder/resolve/main/app.py
- Command line
-
hf download hf://spaces/Ultronprime/h3-image-encoder/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Ultronprime/h3-image-encoder/resolve/main/app.py
8.08 kB
| """MiniMax-H3-Image prompt encoder service for Ultronprime/photo-person-swap-h3. | |
| Adapted from CozyByte/minimax-h3-image-encoder so user photos stay inside this account's own Spaces. | |
| Turns a prompt (plus up to 8 reference images) into the two tensors the MiniMax-H3-Image | |
| diffusion transformer reads: `prompt_embeds` of shape `(1, tokens, 5120)` — the 50th hidden | |
| state of the 33B Qwen3-VL conditioner — and `text_token_tags`, the modality tag of every row. | |
| The presentation is built exactly as the diffusers `MiniMaxH3ImageEditTextEncoderStep` builds | |
| it: a `Picture i: ` label and a vision block per reference, then the prompt verbatim. | |
| This runs on its own ZeroGPU slice so the studio Space can keep the 66 GB transformer resident | |
| on its own. See https://huggingface.co/MiniMaxAI/MiniMax-H3-Image. | |
| """ | |
| # Before anything that could initialize CUDA. | |
| import spaces | |
| import os | |
| import tempfile | |
| import time | |
| import gradio as gr | |
| import torch | |
| from PIL import Image, ImageOps | |
| from safetensors.torch import save_file | |
| from transformers import AutoTokenizer, Qwen3VLForConditionalGeneration, Qwen3VLProcessor | |
| REPO = os.environ.get("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3-Image") | |
| MAX_IMAGES = 8 | |
| MULTIPLE = 32 | |
| VISION_SHORT_EDGE = 1024 | |
| MAX_SEQUENCE_LENGTH = 4096 | |
| TEXT_ENCODER_LAYER = 50 | |
| TEXT_TAG, IMAGE_TAG = 1, 0 | |
| tokenizer = AutoTokenizer.from_pretrained(REPO, subfolder="tokenizer") | |
| processor = Qwen3VLProcessor.from_pretrained(REPO, subfolder="processor") | |
| text_encoder = ( | |
| Qwen3VLForConditionalGeneration.from_pretrained( | |
| REPO, subfolder="text_encoder", dtype=torch.bfloat16 | |
| ) | |
| .eval() | |
| ) | |
| # Only hidden state 50 (the output of decoder layer 50) is read, so layers 51-64 and the LM head never matter: | |
| # dropping them leaves every number identical and saves ~13 GB the GPU must take on, and ~20% of the compute. | |
| LAYERS = text_encoder.model.language_model.layers | |
| del LAYERS[TEXT_ENCODER_LAYER:] | |
| text_encoder.lm_head = torch.nn.Identity() | |
| text_encoder.to("cuda") | |
| CAPTURED = {} | |
| def _keep_output(module, args, output): | |
| CAPTURED["hidden"] = output[0] if isinstance(output, (tuple, list)) else output | |
| LAYERS[TEXT_ENCODER_LAYER - 1].register_forward_hook(_keep_output) | |
| def image_paths(images): | |
| """Gradio hands back a list of paths, sometimes wrapped in one-element lists.""" | |
| return [item[0] if isinstance(item, (list, tuple)) else item for item in images or []] | |
| def bound_short_edge(width: int, height: int, min_short_edge: int, max_short_edge: int, multiple: int): | |
| short_edge = min(width, height) | |
| if min_short_edge <= short_edge <= max_short_edge: | |
| return width, height | |
| scale = (min_short_edge if short_edge < min_short_edge else max_short_edge) / short_edge | |
| return ( | |
| max(multiple, round(width * scale / multiple) * multiple), | |
| max(multiple, round(height * scale / multiple) * multiple), | |
| ) | |
| def presentation(prompt: str, paths: list[str]): | |
| """The token ids and per-row modality tags of an H3-Image presentation.""" | |
| images = [] | |
| for path in paths[:MAX_IMAGES]: | |
| image = ImageOps.exif_transpose(Image.open(path)).convert("RGB") | |
| size = bound_short_edge(*image.size, VISION_SHORT_EDGE, VISION_SHORT_EDGE, MULTIPLE) | |
| if image.size != size: | |
| image = image.resize(size, Image.LANCZOS) | |
| images.append(image) | |
| token_ids: list[int] = [] | |
| token_tags: list[int] = [] | |
| vision = {"pixel_values": None, "image_grid_thw": None} | |
| if images: | |
| vision = processor.image_processor(images=images, return_tensors="pt") | |
| merge_size = processor.image_processor.merge_size**2 | |
| for index, grid in enumerate(vision["image_grid_thw"]): | |
| # The label carries no angle brackets; the prompt refers to it as `<Picture i>`. | |
| label_ids = tokenizer(f"Picture {index + 1}: ", add_special_tokens=False)["input_ids"] | |
| label_ids = label_ids[:MAX_SEQUENCE_LENGTH] | |
| vision_ids = ( | |
| [tokenizer.convert_tokens_to_ids("<|vision_start|>")] | |
| + [tokenizer.convert_tokens_to_ids("<|image_pad|>")] * (int(grid.prod()) // merge_size) | |
| + [tokenizer.convert_tokens_to_ids("<|vision_end|>")] | |
| ) | |
| token_ids += label_ids + vision_ids | |
| token_tags += [TEXT_TAG] * len(label_ids) + [IMAGE_TAG] * len(vision_ids) | |
| prompt_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"][:MAX_SEQUENCE_LENGTH] | |
| token_ids += prompt_ids | |
| token_tags += [TEXT_TAG] * len(prompt_ids) | |
| return token_ids, token_tags, vision | |
| def embed(token_ids: list[int], token_tags: list[int], vision: dict) -> tuple[torch.Tensor, torch.Tensor]: | |
| """One forward through the conditioner, keeping hidden state 50 and the row tags.""" | |
| input_ids = torch.tensor([token_ids], dtype=torch.long, device="cuda") | |
| mm_token_type_ids = torch.tensor( | |
| processor.create_mm_token_type_ids([token_ids]), dtype=torch.long, device="cuda" | |
| ) | |
| kwargs = {} | |
| if vision.get("pixel_values") is not None: | |
| kwargs["pixel_values"] = vision["pixel_values"].to("cuda", text_encoder.dtype) | |
| kwargs["image_grid_thw"] = vision["image_grid_thw"].to("cuda") | |
| with torch.no_grad(): | |
| outputs = text_encoder.model( | |
| input_ids=input_ids, | |
| attention_mask=torch.ones_like(input_ids), | |
| mm_token_type_ids=mm_token_type_ids, | |
| use_cache=False, | |
| **kwargs, | |
| ) | |
| del outputs | |
| embeds = CAPTURED.pop("hidden").to(torch.bfloat16).cpu().contiguous() | |
| tags = torch.tensor(token_tags, dtype=torch.long).contiguous() | |
| return embeds, tags | |
| def _duration(prompt, images): | |
| # One vision block per reference is ~1k extra rows for the 33B conditioner; a whole run with 5 pictures | |
| # measured 12-17 s end to end, so this keeps generous headroom while asking ZeroGPU for less. | |
| return min(120, 20 + 6 * len(image_paths(images))) | |
| def encode_gpu(prompt: str, images: list[str] | None = None): | |
| """Encode a prompt and up to 8 reference images into `prompt_embeds` + `text_token_tags`. | |
| Reference images are ordered: the prompt refers to them as `<Picture 1>`, `<Picture 2>`, ... | |
| Returns a safetensors file holding both tensors. | |
| """ | |
| if not prompt or not prompt.strip(): | |
| raise gr.Error("Enter a prompt.") | |
| paths = image_paths(images) | |
| if len(paths) > MAX_IMAGES: | |
| raise gr.Error(f"At most {MAX_IMAGES} reference images.") | |
| start = time.time() | |
| print(f"[timing] GPU attached {start:.1f}", flush=True) | |
| token_ids, token_tags, vision = presentation(prompt.strip(), paths) | |
| embeds, tags = embed(token_ids, token_tags, vision) | |
| path = tempfile.NamedTemporaryFile(suffix=".safetensors", delete=False).name | |
| save_file({"prompt_embeds": embeds, "text_token_tags": tags}, path) | |
| print(f"encoded {len(token_ids)} tokens ({len(paths)} refs) in {time.time() - start:.1f}s") | |
| return path, f"{len(token_ids)} tokens · {len(paths)} reference image(s) · {time.time() - start:.1f}s" | |
| def encode(prompt: str, images: list[str] | None = None): | |
| """Encode a prompt and up to 8 reference images into `prompt_embeds` + `text_token_tags` (safetensors file).""" | |
| arrived = time.time() | |
| print(f"[timing] request arrived {arrived:.1f} (uploads done)", flush=True) | |
| result = encode_gpu(prompt, images) | |
| print(f"[timing] GPU call returned after {time.time() - arrived:.1f}s", flush=True) | |
| return result | |
| with gr.Blocks(title="H3-Image encoder") as demo: | |
| gr.Markdown("**H3-Image prompt encoder**: backend for Ultronprime/photo-person-swap-h3. Nothing is stored.") | |
| prompt = gr.Textbox(label="Prompt", lines=4) | |
| files = gr.File(label="Reference images, in <Picture N> order", file_count="multiple", file_types=["image"], | |
| type="filepath") | |
| run = gr.Button("Encode") | |
| wire = gr.File(label="prompt_embeds + text_token_tags") | |
| info = gr.Markdown() | |
| run.click(encode, [prompt, files], [wire, info], api_name="encode") | |
| demo.queue(default_concurrency_limit=2).launch() | |