invoker-render / qwen_encoder.py
multimodalart's picture
multimodalart HF Staff
Keep loaded dtypes
485f23c verified
Raw History Blame Contribute Delete
6.08 kB
"""MiniMax-H3's Qwen3-VL condition encoder (layer-50 hidden state) for Invoker-Render.
Port of CodeWorldModel's `FrozenCWMEncoders.condition` + `qwen_inputs` (static-image mode):
the first image is presented as `<Picture 1>: <vision tokens>` followed by the prompt, and the
DiT receives the hidden state after decoder layer 50 with per-row tags (1 text / 0 vision).
Weights come from `MiniMaxAI/MiniMax-H3` (`Ref2VA/text_encoder`). They are streamed shard by
shard; the 14 unused top layers and the LM head are never loaded, and the decoder-layer
linears are stored FP8 (weight-only) so the encoder can visit the GPU next to the DiT.
"""
from __future__ import annotations
import gc
import json
import os
import torch
import torch.nn as nn
from invoker_dit import FP8Linear, _set_module
HIDDEN_LAYER = 50
_QUANT = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj",
"mlp.gate_proj", "mlp.up_proj", "mlp.down_proj")
def load_qwen(repo_id: str, subfolder: str = "Ref2VA", token=None, log=print):
from accelerate import init_empty_weights
from accelerate.utils import set_module_tensor_to_device
from huggingface_hub import hf_hub_download
from safetensors import safe_open
from transformers import AutoConfig, AutoProcessor, Qwen3VLForConditionalGeneration
te = f"{subfolder}/text_encoder"
config_path = hf_hub_download(repo_id, f"{te}/config.json", token=token)
config = AutoConfig.from_pretrained(os.path.dirname(config_path))
config.text_config.num_hidden_layers = HIDDEN_LAYER
with init_empty_weights():
model = Qwen3VLForConditionalGeneration(config)
model.lm_head = nn.Identity() # never used: the encoder reads a hidden layer
index = json.load(open(hf_hub_download(repo_id, f"{te}/model.safetensors.index.json", token=token)))
params = {n for n, _ in model.named_parameters()}
def wanted(key):
if key.startswith("model.language_model.layers."):
return int(key.split(".")[3]) < HIDDEN_LAYER
return key in params
shards = sorted({s for k, s in index["weight_map"].items() if wanted(k)})
cache_dir = os.environ.get("INVOKER_SHARD_DIR", "/tmp/invoker-shards")
for i, shard in enumerate(shards):
path = hf_hub_download(repo_id, f"{te}/{shard}", token=token, local_dir=cache_dir)
with safe_open(path, framework="pt") as f:
for key in f.keys():
if not wanted(key):
continue
tensor = f.get_tensor(key)
module_name = key.rsplit(".", 1)[0]
if key.startswith("model.language_model.layers.") and module_name.endswith(_QUANT):
_set_module(model, module_name, FP8Linear(tensor))
else:
set_module_tensor_to_device(model, key, "cpu", value=tensor.to(torch.bfloat16), dtype=torch.bfloat16)
del tensor
os.remove(path)
gc.collect()
log(f"[qwen] shard {i + 1}/{len(shards)} {shard} done")
meta = [n for n, p in model.named_parameters() if p.is_meta]
if meta:
raise RuntimeError(f"unloaded Qwen tensors: {meta[:8]}")
model.eval().requires_grad_(False)
proc_path = hf_hub_download(repo_id, f"{subfolder}/processor/preprocessor_config.json", token=token)
for name in ("tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt", "chat_template.json",
"video_preprocessor_config.json"):
hf_hub_download(repo_id, f"{subfolder}/processor/{name}", token=token)
processor = AutoProcessor.from_pretrained(os.path.dirname(proc_path))
return model, processor
def qwen_inputs(processor, prompt, images=()):
"""`qwen_inputs` from cwm_encoders.py, restricted to the static-image presentation (no videos)."""
tokenizer = processor.tokenizer
ids, tags = [], []
merge = int(processor.image_processor.merge_size) ** 2
def text(value):
rows = tokenizer(value, add_special_tokens=False)["input_ids"]
ids.extend(rows)
tags.extend([1] * len(rows))
def vision(pad, count):
rows = [tokenizer.convert_tokens_to_ids("<|vision_start|>")]
rows += [tokenizer.convert_tokens_to_ids(pad)] * count
rows += [tokenizer.convert_tokens_to_ids("<|vision_end|>")]
ids.extend(rows)
tags.extend([0] * len(rows))
extra = {}
if len(images):
pictures = processor.image_processor(images=list(images), return_tensors="pt")
extra.update({k: pictures[k] for k in ("pixel_values", "image_grid_thw")})
for i, g in enumerate(pictures["image_grid_thw"]):
text(f"<Picture {i + 1}>: ")
vision("<|image_pad|>", int(g.prod()) // merge)
text(prompt)
input_ids = torch.tensor([ids], dtype=torch.long)
token_types = torch.zeros_like(input_ids, dtype=torch.int32)
token_types[input_ids == tokenizer.convert_tokens_to_ids("<|image_pad|>")] = 1
token_types[input_ids == tokenizer.convert_tokens_to_ids("<|video_pad|>")] = 2
return dict(input_ids=input_ids, attention_mask=torch.ones_like(input_ids), mm_token_type_ids=token_types,
**extra), torch.tensor(tags, dtype=torch.long)
@torch.no_grad()
def encode_condition(model, processor, prompt, image, device="cuda"):
"""-> prompt_embeds [L, 5120] bf16 (layer-50 output), token_tags [L] long."""
kwargs, tags = qwen_inputs(processor, prompt, images=[image])
kwargs = {k: v.to(device, dtype=torch.bfloat16 if v.is_floating_point() else v.dtype) for k, v in kwargs.items()}
layers = model.model.language_model.layers
captured = []
handle = layers[HIDDEN_LAYER - 1].register_forward_hook(
lambda module, args, output: captured.append(output[0] if isinstance(output, tuple) else output)
)
try:
model.model(**kwargs, use_cache=False)
finally:
handle.remove()
hidden = captured[0][0]
if len(hidden) != len(tags):
raise ValueError("Qwen rows and presentation tags disagree")
return hidden.to(torch.bfloat16), tags