Image-Text-to-Text
Transformers
Safetensors
English
Chinese
pixel_dit_qwen3_vl
text-generation
multimodal
text-to-image
image-to-image
image-editing
custom_code
qwen3-vl
conversational
Instructions to use ernie-research/HiUni with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ernie-research/HiUni with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="ernie-research/HiUni", trust_remote_code=True) messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("ernie-research/HiUni", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use ernie-research/HiUni with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "ernie-research/HiUni" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "ernie-research/HiUni", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/ernie-research/HiUni
- SGLang
How to use ernie-research/HiUni with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "ernie-research/HiUni" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "ernie-research/HiUni", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "ernie-research/HiUni" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "ernie-research/HiUni", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use ernie-research/HiUni with Docker Model Runner:
docker model run hf.co/ernie-research/HiUni
File size: 8,907 Bytes
bc25b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """Pixel layout, attention masks and flow matching used by HF image generation."""
import math
import random
import torch
INSTRUCTIONS = [
"\u8bf7\u6839\u636e\u4ee5\u4e0b\u63cf\u8ff0\u751f\u6210\u4e00\u5f20\u56fe\u7247\uff1a",
"\u6839\u636e\u4e0b\u9762\u7684\u63cf\u8ff0\u751f\u6210\u56fe\u7247\uff1a",
"\u751f\u6210\u56fe\u7247\uff1a", "\u8bf7\u753b\u4e00\u5f20\u56fe\uff1a",
"\u5e2e\u6211\u751f\u6210\u4e00\u5f20\u56fe\u7247\uff1a",
"\u6839\u636e\u63cf\u8ff0\u521b\u5efa\u56fe\u50cf\uff1a",
"\u8bf7\u6839\u636e\u6587\u5b57\u63cf\u8ff0\u7ed8\u5236\u56fe\u7247\uff1a",
"\u6309\u7167\u4ee5\u4e0b\u5185\u5bb9\u751f\u6210\u56fe\u7247\uff1a",
"\u751f\u6210\u4e00\u5f20\u7b26\u5408\u63cf\u8ff0\u7684\u56fe\u7247\uff1a",
"\u8bf7\u4e3a\u4ee5\u4e0b\u5185\u5bb9\u521b\u4f5c\u4e00\u5f20\u56fe\u7247\uff1a",
"\u6839\u636e\u63d0\u793a\u751f\u6210\u56fe\u50cf\uff1a", "\u56fe\u7247\u751f\u6210\uff1a",
"Generate an image based on the following description: ", "Create an image: ",
"Please generate an image: ", "Draw an image based on: ", "Create a picture: ",
"Generate: ", "Please draw: ", "Create an image from the description: ",
"Generate a picture: ", "Please create an image based on: ",
]
def merge_patches(raw, height, width):
"""Pack raster patches as [patch_channel, block_row, block_column]."""
dim = raw.shape[-1]
return raw.reshape(height, 2, width, 2, dim).permute(0, 2, 4, 1, 3).reshape(-1, dim * 4)
def unmerge_patches(merged, height, width):
"""Invert the checkpoint's 2x2 pixel packing."""
dim = merged.shape[-1] // 4
return merged.reshape(height, width, dim, 2, 2).permute(0, 3, 1, 4, 2).reshape(-1, dim)
def vision_patches(raw, height, width, patch_size=16):
"""Convert raster RGB patches to Qwen's block-major, duplicated-frame layout."""
patches = raw.reshape(height, 2, width, 2, 3, patch_size, patch_size)
patches = patches.permute(0, 2, 1, 3, 4, 5, 6)
return patches.unsqueeze(5).expand(-1, -1, -1, -1, -1, 2, -1, -1).reshape(
4 * height * width, 6 * patch_size * patch_size
)
def build_sequence(tokenizer, config, prompt, height, width, instruction="", chat=True, route=False):
"""Build the training chat template without retokenizing instruction boundaries."""
def encode(text):
"""Encode text without injecting tokenizer-level special tokens."""
return tokenizer.encode(text, add_special_tokens=False)
image = [config.vision_start_token_id, config.time_token_id]
image += [config.image_token_id] * (height * width)
image += [config.vision_end_token_id, 151645]
if not chat:
return torch.tensor([image], dtype=torch.long)
prefix = [151644] + encode("user") + encode("\n")
prefix += encode(instruction) + encode(prompt) + [151645] + encode("\n")
prefix += [151644] + encode("assistant") + encode("\n")
if route:
if encode("<|image_gate|>") != [config.image_gate_token_id]:
raise ValueError("Tokenizer does not contain the configured image gate token")
prefix.append(config.image_gate_token_id)
return torch.tensor([prefix + image], dtype=torch.long)
def sequence_layout(ids, config, height, width, dtype):
"""Return M-RoPE positions and causal attention with bidirectional image tokens."""
image_mask = ids.eq(config.image_token_id)
indices = image_mask[0].nonzero(as_tuple=True)[0]
if indices.numel() != height * width:
raise ValueError("Prompt must not contain image special tokens")
start, end = int(indices[0]), int(indices[-1]) + 1
if end - start != height * width or ids.eq(config.time_token_id).sum() != 1:
raise ValueError("Expected one contiguous image block and one time token")
length = ids.shape[1]
positions = torch.arange(length, device=ids.device).view(1, 1, -1).repeat(3, 1, 1)
positions[0, 0, start:end] = start
positions[1, 0, start:end] = start + torch.arange(height, device=ids.device).repeat_interleave(width)
positions[2, 0, start:end] = start + torch.arange(width, device=ids.device).repeat(height)
positions[:, 0, end:] = start + max(height, width) + torch.arange(length - end, device=ids.device)
allowed = torch.ones(length, length, dtype=torch.bool, device=ids.device).tril()
allowed[start:end, start:end] = True
# The Megatron startend mask has LTS=L-1, masking the final EOS row.
# Its output is unused by the image head; keep that row valid for SDPA.
mask = torch.zeros(length, length, device=ids.device, dtype=dtype)
mask.masked_fill_(~allowed, torch.finfo(dtype).min)
return positions, mask[None, None], image_mask
def flow_sigmas(steps, shift, device):
"""Reproduce FlowMatchScheduler(extra_one_step=True), including its sigma floor."""
if steps <= 0 or not math.isfinite(shift) or shift <= 0:
raise ValueError("steps and shift must be positive")
sigma = torch.linspace(1.0, 0.003 / 1.002, steps + 1, device=device)[:-1]
sigma = shift * sigma / (1 + (shift - 1) * sigma)
return torch.cat((sigma, sigma.new_zeros(1)))
def flow_step(prediction, latents, sigma, next_sigma, predict_x0):
"""Match Pixel-DiT Euler sampling, which disables the training loss sigma clamp."""
velocity = (latents - prediction) / sigma if predict_x0 else prediction
return latents + velocity * (next_sigma - sigma)
@torch.inference_mode()
def generate_image(model, tokenizer, prompt, height=1024, width=1024, num_steps=50,
guidance_scale=4.0, shift=4.0, seed=42, negative_prompt="",
cfg_keep_chat_template=False, instruction=None, predict_x0=None,
drop_dit_text=False, dynamic_noise_scale=None, dynamic_noise_ref_tokens=None,
dynamic_noise_base_scale=None):
"""Generate one PIL image; dimensions are pixels and seeds are local to the call."""
from PIL import Image
if height <= 0 or width <= 0 or height % 32 or width % 32:
raise ValueError("height and width must be positive multiples of 32 pixels")
if not math.isfinite(guidance_scale) or guidance_scale < 1:
raise ValueError("guidance_scale must be finite and at least 1")
model.eval()
config = model.config
device, dtype = model.device, model.dtype
h, w = height // 32, width // 32
prefix = random.Random(seed).choice(INSTRUCTIONS) if instruction is None else instruction
cond = build_sequence(tokenizer, config, prompt, h, w, prefix, route=config.predict_gate_mode).to(device)
sequences = [cond]
if guidance_scale > 1:
sequences.append(build_sequence(tokenizer, config, negative_prompt, h, w,
prefix if cfg_keep_chat_template else "",
chat=cfg_keep_chat_template).to(device))
layouts = [sequence_layout(ids, config, h, w, dtype) for ids in sequences]
max_length = config.text_config.max_position_embeddings
if max(ids.shape[1] for ids in sequences) > max_length:
raise ValueError(f"Prompt and image exceed max_position_embeddings={max_length}")
generator = torch.Generator(device=device).manual_seed(seed)
latents = torch.randn(4 * h * w, 768, generator=generator, device=device, dtype=torch.float32)
dynamic_noise_scale = config.use_dynamic_noise_scale if dynamic_noise_scale is None else dynamic_noise_scale
noise_ref = config.dynamic_noise_ref_tokens if dynamic_noise_ref_tokens is None else dynamic_noise_ref_tokens
noise_base = config.dynamic_noise_base_scale if dynamic_noise_base_scale is None else dynamic_noise_base_scale
if noise_ref <= 0 or not math.isfinite(noise_base) or noise_base <= 0:
raise ValueError("Dynamic noise reference tokens and base scale must be positive and finite")
if dynamic_noise_scale:
latents *= math.sqrt(4 * h * w / noise_ref) * noise_base
sigmas = flow_sigmas(num_steps, shift, device)
predict_x0 = config.predict_x0 if predict_x0 is None else predict_x0
for sigma, next_sigma in zip(sigmas[:-1], sigmas[1:]):
predictions = [model.predict_image(ids, latents, sigma.reshape(1) * 1000,
h, w, *layout, drop_dit_text=drop_dit_text)
for ids, layout in zip(sequences, layouts)]
prediction = predictions[0]
if len(predictions) == 2:
prediction = predictions[1] + guidance_scale * (prediction - predictions[1])
latents = flow_step(prediction, latents, sigma, next_sigma, predict_x0)
if not torch.isfinite(latents).all():
raise RuntimeError("Non-finite pixels produced during denoising")
pixels = latents.reshape(2 * h, 2 * w, 3, 16, 16).permute(0, 3, 1, 4, 2)
pixels = pixels.reshape(height, width, 3).add(1).div(2).clamp(0, 1)
return Image.fromarray(pixels.mul(255).add(0.5).to(torch.uint8).cpu().numpy())
|