"""MiniMax-H3's Qwen3-VL condition encoder (layer-50 hidden state) for Invoker-Render. Port of CodeWorldModel's `FrozenCWMEncoders.condition` + `qwen_inputs` (static-image mode): the first image is presented as `: ` followed by the prompt, and the DiT receives the hidden state after decoder layer 50 with per-row tags (1 text / 0 vision). Weights come from `MiniMaxAI/MiniMax-H3` (`Ref2VA/text_encoder`). They are streamed shard by shard; the 14 unused top layers and the LM head are never loaded, and the decoder-layer linears are stored FP8 (weight-only) so the encoder can visit the GPU next to the DiT. """ from __future__ import annotations import gc import json import os import torch import torch.nn as nn from invoker_dit import FP8Linear, _set_module HIDDEN_LAYER = 50 _QUANT = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj") def load_qwen(repo_id: str, subfolder: str = "Ref2VA", token=None, log=print): from accelerate import init_empty_weights from accelerate.utils import set_module_tensor_to_device from huggingface_hub import hf_hub_download from safetensors import safe_open from transformers import AutoConfig, AutoProcessor, Qwen3VLForConditionalGeneration te = f"{subfolder}/text_encoder" config_path = hf_hub_download(repo_id, f"{te}/config.json", token=token) config = AutoConfig.from_pretrained(os.path.dirname(config_path)) config.text_config.num_hidden_layers = HIDDEN_LAYER with init_empty_weights(): model = Qwen3VLForConditionalGeneration(config) model.lm_head = nn.Identity() # never used: the encoder reads a hidden layer index = json.load(open(hf_hub_download(repo_id, f"{te}/model.safetensors.index.json", token=token))) params = {n for n, _ in model.named_parameters()} def wanted(key): if key.startswith("model.language_model.layers."): return int(key.split(".")[3]) < HIDDEN_LAYER return key in params shards = sorted({s for k, s in index["weight_map"].items() if wanted(k)}) cache_dir = os.environ.get("INVOKER_SHARD_DIR", "/tmp/invoker-shards") for i, shard in enumerate(shards): path = hf_hub_download(repo_id, f"{te}/{shard}", token=token, local_dir=cache_dir) with safe_open(path, framework="pt") as f: for key in f.keys(): if not wanted(key): continue tensor = f.get_tensor(key) module_name = key.rsplit(".", 1)[0] if key.startswith("model.language_model.layers.") and module_name.endswith(_QUANT): _set_module(model, module_name, FP8Linear(tensor)) else: set_module_tensor_to_device(model, key, "cpu", value=tensor.to(torch.bfloat16), dtype=torch.bfloat16) del tensor os.remove(path) gc.collect() log(f"[qwen] shard {i + 1}/{len(shards)} {shard} done") meta = [n for n, p in model.named_parameters() if p.is_meta] if meta: raise RuntimeError(f"unloaded Qwen tensors: {meta[:8]}") model.eval().requires_grad_(False) proc_path = hf_hub_download(repo_id, f"{subfolder}/processor/preprocessor_config.json", token=token) for name in ("tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt", "chat_template.json", "video_preprocessor_config.json"): hf_hub_download(repo_id, f"{subfolder}/processor/{name}", token=token) processor = AutoProcessor.from_pretrained(os.path.dirname(proc_path)) return model, processor def qwen_inputs(processor, prompt, images=()): """`qwen_inputs` from cwm_encoders.py, restricted to the static-image presentation (no videos).""" tokenizer = processor.tokenizer ids, tags = [], [] merge = int(processor.image_processor.merge_size) ** 2 def text(value): rows = tokenizer(value, add_special_tokens=False)["input_ids"] ids.extend(rows) tags.extend([1] * len(rows)) def vision(pad, count): rows = [tokenizer.convert_tokens_to_ids("<|vision_start|>")] rows += [tokenizer.convert_tokens_to_ids(pad)] * count rows += [tokenizer.convert_tokens_to_ids("<|vision_end|>")] ids.extend(rows) tags.extend([0] * len(rows)) extra = {} if len(images): pictures = processor.image_processor(images=list(images), return_tensors="pt") extra.update({k: pictures[k] for k in ("pixel_values", "image_grid_thw")}) for i, g in enumerate(pictures["image_grid_thw"]): text(f": ") vision("<|image_pad|>", int(g.prod()) // merge) text(prompt) input_ids = torch.tensor([ids], dtype=torch.long) token_types = torch.zeros_like(input_ids, dtype=torch.int32) token_types[input_ids == tokenizer.convert_tokens_to_ids("<|image_pad|>")] = 1 token_types[input_ids == tokenizer.convert_tokens_to_ids("<|video_pad|>")] = 2 return dict(input_ids=input_ids, attention_mask=torch.ones_like(input_ids), mm_token_type_ids=token_types, **extra), torch.tensor(tags, dtype=torch.long) @torch.no_grad() def encode_condition(model, processor, prompt, image, device="cuda"): """-> prompt_embeds [L, 5120] bf16 (layer-50 output), token_tags [L] long.""" kwargs, tags = qwen_inputs(processor, prompt, images=[image]) kwargs = {k: v.to(device, dtype=torch.bfloat16 if v.is_floating_point() else v.dtype) for k, v in kwargs.items()} layers = model.model.language_model.layers captured = [] handle = layers[HIDDEN_LAYER - 1].register_forward_hook( lambda module, args, output: captured.append(output[0] if isinstance(output, tuple) else output) ) try: model.model(**kwargs, use_cache=False) finally: handle.remove() hidden = captured[0][0] if len(hidden) != len(tags): raise ValueError("Qwen rows and presentation tags disagree") return hidden.to(torch.bfloat16), tags