Spaces:
Running on Zero
Running on Zero
File size: 8,907 Bytes
e72458c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """Pixel layout, attention masks and flow matching used by HF image generation."""
import math
import random
import torch
INSTRUCTIONS = [
"\u8bf7\u6839\u636e\u4ee5\u4e0b\u63cf\u8ff0\u751f\u6210\u4e00\u5f20\u56fe\u7247\uff1a",
"\u6839\u636e\u4e0b\u9762\u7684\u63cf\u8ff0\u751f\u6210\u56fe\u7247\uff1a",
"\u751f\u6210\u56fe\u7247\uff1a", "\u8bf7\u753b\u4e00\u5f20\u56fe\uff1a",
"\u5e2e\u6211\u751f\u6210\u4e00\u5f20\u56fe\u7247\uff1a",
"\u6839\u636e\u63cf\u8ff0\u521b\u5efa\u56fe\u50cf\uff1a",
"\u8bf7\u6839\u636e\u6587\u5b57\u63cf\u8ff0\u7ed8\u5236\u56fe\u7247\uff1a",
"\u6309\u7167\u4ee5\u4e0b\u5185\u5bb9\u751f\u6210\u56fe\u7247\uff1a",
"\u751f\u6210\u4e00\u5f20\u7b26\u5408\u63cf\u8ff0\u7684\u56fe\u7247\uff1a",
"\u8bf7\u4e3a\u4ee5\u4e0b\u5185\u5bb9\u521b\u4f5c\u4e00\u5f20\u56fe\u7247\uff1a",
"\u6839\u636e\u63d0\u793a\u751f\u6210\u56fe\u50cf\uff1a", "\u56fe\u7247\u751f\u6210\uff1a",
"Generate an image based on the following description: ", "Create an image: ",
"Please generate an image: ", "Draw an image based on: ", "Create a picture: ",
"Generate: ", "Please draw: ", "Create an image from the description: ",
"Generate a picture: ", "Please create an image based on: ",
]
def merge_patches(raw, height, width):
"""Pack raster patches as [patch_channel, block_row, block_column]."""
dim = raw.shape[-1]
return raw.reshape(height, 2, width, 2, dim).permute(0, 2, 4, 1, 3).reshape(-1, dim * 4)
def unmerge_patches(merged, height, width):
"""Invert the checkpoint's 2x2 pixel packing."""
dim = merged.shape[-1] // 4
return merged.reshape(height, width, dim, 2, 2).permute(0, 3, 1, 4, 2).reshape(-1, dim)
def vision_patches(raw, height, width, patch_size=16):
"""Convert raster RGB patches to Qwen's block-major, duplicated-frame layout."""
patches = raw.reshape(height, 2, width, 2, 3, patch_size, patch_size)
patches = patches.permute(0, 2, 1, 3, 4, 5, 6)
return patches.unsqueeze(5).expand(-1, -1, -1, -1, -1, 2, -1, -1).reshape(
4 * height * width, 6 * patch_size * patch_size
)
def build_sequence(tokenizer, config, prompt, height, width, instruction="", chat=True, route=False):
"""Build the training chat template without retokenizing instruction boundaries."""
def encode(text):
"""Encode text without injecting tokenizer-level special tokens."""
return tokenizer.encode(text, add_special_tokens=False)
image = [config.vision_start_token_id, config.time_token_id]
image += [config.image_token_id] * (height * width)
image += [config.vision_end_token_id, 151645]
if not chat:
return torch.tensor([image], dtype=torch.long)
prefix = [151644] + encode("user") + encode("\n")
prefix += encode(instruction) + encode(prompt) + [151645] + encode("\n")
prefix += [151644] + encode("assistant") + encode("\n")
if route:
if encode("<|image_gate|>") != [config.image_gate_token_id]:
raise ValueError("Tokenizer does not contain the configured image gate token")
prefix.append(config.image_gate_token_id)
return torch.tensor([prefix + image], dtype=torch.long)
def sequence_layout(ids, config, height, width, dtype):
"""Return M-RoPE positions and causal attention with bidirectional image tokens."""
image_mask = ids.eq(config.image_token_id)
indices = image_mask[0].nonzero(as_tuple=True)[0]
if indices.numel() != height * width:
raise ValueError("Prompt must not contain image special tokens")
start, end = int(indices[0]), int(indices[-1]) + 1
if end - start != height * width or ids.eq(config.time_token_id).sum() != 1:
raise ValueError("Expected one contiguous image block and one time token")
length = ids.shape[1]
positions = torch.arange(length, device=ids.device).view(1, 1, -1).repeat(3, 1, 1)
positions[0, 0, start:end] = start
positions[1, 0, start:end] = start + torch.arange(height, device=ids.device).repeat_interleave(width)
positions[2, 0, start:end] = start + torch.arange(width, device=ids.device).repeat(height)
positions[:, 0, end:] = start + max(height, width) + torch.arange(length - end, device=ids.device)
allowed = torch.ones(length, length, dtype=torch.bool, device=ids.device).tril()
allowed[start:end, start:end] = True
# The Megatron startend mask has LTS=L-1, masking the final EOS row.
# Its output is unused by the image head; keep that row valid for SDPA.
mask = torch.zeros(length, length, device=ids.device, dtype=dtype)
mask.masked_fill_(~allowed, torch.finfo(dtype).min)
return positions, mask[None, None], image_mask
def flow_sigmas(steps, shift, device):
"""Reproduce FlowMatchScheduler(extra_one_step=True), including its sigma floor."""
if steps <= 0 or not math.isfinite(shift) or shift <= 0:
raise ValueError("steps and shift must be positive")
sigma = torch.linspace(1.0, 0.003 / 1.002, steps + 1, device=device)[:-1]
sigma = shift * sigma / (1 + (shift - 1) * sigma)
return torch.cat((sigma, sigma.new_zeros(1)))
def flow_step(prediction, latents, sigma, next_sigma, predict_x0):
"""Match Pixel-DiT Euler sampling, which disables the training loss sigma clamp."""
velocity = (latents - prediction) / sigma if predict_x0 else prediction
return latents + velocity * (next_sigma - sigma)
@torch.inference_mode()
def generate_image(model, tokenizer, prompt, height=1024, width=1024, num_steps=50,
guidance_scale=4.0, shift=4.0, seed=42, negative_prompt="",
cfg_keep_chat_template=False, instruction=None, predict_x0=None,
drop_dit_text=False, dynamic_noise_scale=None, dynamic_noise_ref_tokens=None,
dynamic_noise_base_scale=None):
"""Generate one PIL image; dimensions are pixels and seeds are local to the call."""
from PIL import Image
if height <= 0 or width <= 0 or height % 32 or width % 32:
raise ValueError("height and width must be positive multiples of 32 pixels")
if not math.isfinite(guidance_scale) or guidance_scale < 1:
raise ValueError("guidance_scale must be finite and at least 1")
model.eval()
config = model.config
device, dtype = model.device, model.dtype
h, w = height // 32, width // 32
prefix = random.Random(seed).choice(INSTRUCTIONS) if instruction is None else instruction
cond = build_sequence(tokenizer, config, prompt, h, w, prefix, route=config.predict_gate_mode).to(device)
sequences = [cond]
if guidance_scale > 1:
sequences.append(build_sequence(tokenizer, config, negative_prompt, h, w,
prefix if cfg_keep_chat_template else "",
chat=cfg_keep_chat_template).to(device))
layouts = [sequence_layout(ids, config, h, w, dtype) for ids in sequences]
max_length = config.text_config.max_position_embeddings
if max(ids.shape[1] for ids in sequences) > max_length:
raise ValueError(f"Prompt and image exceed max_position_embeddings={max_length}")
generator = torch.Generator(device=device).manual_seed(seed)
latents = torch.randn(4 * h * w, 768, generator=generator, device=device, dtype=torch.float32)
dynamic_noise_scale = config.use_dynamic_noise_scale if dynamic_noise_scale is None else dynamic_noise_scale
noise_ref = config.dynamic_noise_ref_tokens if dynamic_noise_ref_tokens is None else dynamic_noise_ref_tokens
noise_base = config.dynamic_noise_base_scale if dynamic_noise_base_scale is None else dynamic_noise_base_scale
if noise_ref <= 0 or not math.isfinite(noise_base) or noise_base <= 0:
raise ValueError("Dynamic noise reference tokens and base scale must be positive and finite")
if dynamic_noise_scale:
latents *= math.sqrt(4 * h * w / noise_ref) * noise_base
sigmas = flow_sigmas(num_steps, shift, device)
predict_x0 = config.predict_x0 if predict_x0 is None else predict_x0
for sigma, next_sigma in zip(sigmas[:-1], sigmas[1:]):
predictions = [model.predict_image(ids, latents, sigma.reshape(1) * 1000,
h, w, *layout, drop_dit_text=drop_dit_text)
for ids, layout in zip(sequences, layouts)]
prediction = predictions[0]
if len(predictions) == 2:
prediction = predictions[1] + guidance_scale * (prediction - predictions[1])
latents = flow_step(prediction, latents, sigma, next_sigma, predict_x0)
if not torch.isfinite(latents).all():
raise RuntimeError("Non-finite pixels produced during denoising")
pixels = latents.reshape(2 * h, 2 * w, 3, 16, 16).permute(0, 3, 1, 4, 2)
pixels = pixels.reshape(height, width, 3).add(1).div(2).clamp(0, 1)
return Image.fromarray(pixels.mul(255).add(0.5).to(torch.uint8).cpu().numpy())
|