Ultronprime's picture
Drop decoder layers 51-64 and LM head (only hidden state 50 is read); timing logs
0cc3fca verified
Raw History Blame Contribute Delete
8.08 kB
"""MiniMax-H3-Image prompt encoder service for Ultronprime/photo-person-swap-h3.
Adapted from CozyByte/minimax-h3-image-encoder so user photos stay inside this account's own Spaces.
Turns a prompt (plus up to 8 reference images) into the two tensors the MiniMax-H3-Image
diffusion transformer reads: `prompt_embeds` of shape `(1, tokens, 5120)` — the 50th hidden
state of the 33B Qwen3-VL conditioner — and `text_token_tags`, the modality tag of every row.
The presentation is built exactly as the diffusers `MiniMaxH3ImageEditTextEncoderStep` builds
it: a `Picture i: ` label and a vision block per reference, then the prompt verbatim.
This runs on its own ZeroGPU slice so the studio Space can keep the 66 GB transformer resident
on its own. See https://huggingface.co/MiniMaxAI/MiniMax-H3-Image.
"""
# Before anything that could initialize CUDA.
import spaces
import os
import tempfile
import time
import gradio as gr
import torch
from PIL import Image, ImageOps
from safetensors.torch import save_file
from transformers import AutoTokenizer, Qwen3VLForConditionalGeneration, Qwen3VLProcessor
REPO = os.environ.get("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3-Image")
MAX_IMAGES = 8
MULTIPLE = 32
VISION_SHORT_EDGE = 1024
MAX_SEQUENCE_LENGTH = 4096
TEXT_ENCODER_LAYER = 50
TEXT_TAG, IMAGE_TAG = 1, 0
tokenizer = AutoTokenizer.from_pretrained(REPO, subfolder="tokenizer")
processor = Qwen3VLProcessor.from_pretrained(REPO, subfolder="processor")
text_encoder = (
Qwen3VLForConditionalGeneration.from_pretrained(
REPO, subfolder="text_encoder", dtype=torch.bfloat16
)
.eval()
)
# Only hidden state 50 (the output of decoder layer 50) is read, so layers 51-64 and the LM head never matter:
# dropping them leaves every number identical and saves ~13 GB the GPU must take on, and ~20% of the compute.
LAYERS = text_encoder.model.language_model.layers
del LAYERS[TEXT_ENCODER_LAYER:]
text_encoder.lm_head = torch.nn.Identity()
text_encoder.to("cuda")
CAPTURED = {}
def _keep_output(module, args, output):
CAPTURED["hidden"] = output[0] if isinstance(output, (tuple, list)) else output
LAYERS[TEXT_ENCODER_LAYER - 1].register_forward_hook(_keep_output)
def image_paths(images):
"""Gradio hands back a list of paths, sometimes wrapped in one-element lists."""
return [item[0] if isinstance(item, (list, tuple)) else item for item in images or []]
def bound_short_edge(width: int, height: int, min_short_edge: int, max_short_edge: int, multiple: int):
short_edge = min(width, height)
if min_short_edge <= short_edge <= max_short_edge:
return width, height
scale = (min_short_edge if short_edge < min_short_edge else max_short_edge) / short_edge
return (
max(multiple, round(width * scale / multiple) * multiple),
max(multiple, round(height * scale / multiple) * multiple),
)
def presentation(prompt: str, paths: list[str]):
"""The token ids and per-row modality tags of an H3-Image presentation."""
images = []
for path in paths[:MAX_IMAGES]:
image = ImageOps.exif_transpose(Image.open(path)).convert("RGB")
size = bound_short_edge(*image.size, VISION_SHORT_EDGE, VISION_SHORT_EDGE, MULTIPLE)
if image.size != size:
image = image.resize(size, Image.LANCZOS)
images.append(image)
token_ids: list[int] = []
token_tags: list[int] = []
vision = {"pixel_values": None, "image_grid_thw": None}
if images:
vision = processor.image_processor(images=images, return_tensors="pt")
merge_size = processor.image_processor.merge_size**2
for index, grid in enumerate(vision["image_grid_thw"]):
# The label carries no angle brackets; the prompt refers to it as `<Picture i>`.
label_ids = tokenizer(f"Picture {index + 1}: ", add_special_tokens=False)["input_ids"]
label_ids = label_ids[:MAX_SEQUENCE_LENGTH]
vision_ids = (
[tokenizer.convert_tokens_to_ids("<|vision_start|>")]
+ [tokenizer.convert_tokens_to_ids("<|image_pad|>")] * (int(grid.prod()) // merge_size)
+ [tokenizer.convert_tokens_to_ids("<|vision_end|>")]
)
token_ids += label_ids + vision_ids
token_tags += [TEXT_TAG] * len(label_ids) + [IMAGE_TAG] * len(vision_ids)
prompt_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"][:MAX_SEQUENCE_LENGTH]
token_ids += prompt_ids
token_tags += [TEXT_TAG] * len(prompt_ids)
return token_ids, token_tags, vision
def embed(token_ids: list[int], token_tags: list[int], vision: dict) -> tuple[torch.Tensor, torch.Tensor]:
"""One forward through the conditioner, keeping hidden state 50 and the row tags."""
input_ids = torch.tensor([token_ids], dtype=torch.long, device="cuda")
mm_token_type_ids = torch.tensor(
processor.create_mm_token_type_ids([token_ids]), dtype=torch.long, device="cuda"
)
kwargs = {}
if vision.get("pixel_values") is not None:
kwargs["pixel_values"] = vision["pixel_values"].to("cuda", text_encoder.dtype)
kwargs["image_grid_thw"] = vision["image_grid_thw"].to("cuda")
with torch.no_grad():
outputs = text_encoder.model(
input_ids=input_ids,
attention_mask=torch.ones_like(input_ids),
mm_token_type_ids=mm_token_type_ids,
use_cache=False,
**kwargs,
)
del outputs
embeds = CAPTURED.pop("hidden").to(torch.bfloat16).cpu().contiguous()
tags = torch.tensor(token_tags, dtype=torch.long).contiguous()
return embeds, tags
def _duration(prompt, images):
# One vision block per reference is ~1k extra rows for the 33B conditioner; a whole run with 5 pictures
# measured 12-17 s end to end, so this keeps generous headroom while asking ZeroGPU for less.
return min(120, 20 + 6 * len(image_paths(images)))
@spaces.GPU(duration=_duration, size="xlarge")
def encode_gpu(prompt: str, images: list[str] | None = None):
"""Encode a prompt and up to 8 reference images into `prompt_embeds` + `text_token_tags`.
Reference images are ordered: the prompt refers to them as `<Picture 1>`, `<Picture 2>`, ...
Returns a safetensors file holding both tensors.
"""
if not prompt or not prompt.strip():
raise gr.Error("Enter a prompt.")
paths = image_paths(images)
if len(paths) > MAX_IMAGES:
raise gr.Error(f"At most {MAX_IMAGES} reference images.")
start = time.time()
print(f"[timing] GPU attached {start:.1f}", flush=True)
token_ids, token_tags, vision = presentation(prompt.strip(), paths)
embeds, tags = embed(token_ids, token_tags, vision)
path = tempfile.NamedTemporaryFile(suffix=".safetensors", delete=False).name
save_file({"prompt_embeds": embeds, "text_token_tags": tags}, path)
print(f"encoded {len(token_ids)} tokens ({len(paths)} refs) in {time.time() - start:.1f}s")
return path, f"{len(token_ids)} tokens · {len(paths)} reference image(s) · {time.time() - start:.1f}s"
def encode(prompt: str, images: list[str] | None = None):
"""Encode a prompt and up to 8 reference images into `prompt_embeds` + `text_token_tags` (safetensors file)."""
arrived = time.time()
print(f"[timing] request arrived {arrived:.1f} (uploads done)", flush=True)
result = encode_gpu(prompt, images)
print(f"[timing] GPU call returned after {time.time() - arrived:.1f}s", flush=True)
return result
with gr.Blocks(title="H3-Image encoder") as demo:
gr.Markdown("**H3-Image prompt encoder**: backend for Ultronprime/photo-person-swap-h3. Nothing is stored.")
prompt = gr.Textbox(label="Prompt", lines=4)
files = gr.File(label="Reference images, in <Picture N> order", file_count="multiple", file_types=["image"],
type="filepath")
run = gr.Button("Encode")
wire = gr.File(label="prompt_embeds + text_token_tags")
info = gr.Markdown()
run.click(encode, [prompt, files], [wire, info], api_name="encode")
demo.queue(default_concurrency_limit=2).launch()