Spaces:
Running on Zero
Running on Zero
Download qwen_encoder.py from hugging-apps/invoker-render: direct link, hf CLI and curl.
- Browser
- Download file 6.08 kB
-
https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/qwen_encoder.py
- Command line
-
hf download hf://spaces/hugging-apps/invoker-render/qwen_encoder.py
-
curl -L -o qwen_encoder.py https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/qwen_encoder.py
6.08 kB
| """MiniMax-H3's Qwen3-VL condition encoder (layer-50 hidden state) for Invoker-Render. | |
| Port of CodeWorldModel's `FrozenCWMEncoders.condition` + `qwen_inputs` (static-image mode): | |
| the first image is presented as `<Picture 1>: <vision tokens>` followed by the prompt, and the | |
| DiT receives the hidden state after decoder layer 50 with per-row tags (1 text / 0 vision). | |
| Weights come from `MiniMaxAI/MiniMax-H3` (`Ref2VA/text_encoder`). They are streamed shard by | |
| shard; the 14 unused top layers and the LM head are never loaded, and the decoder-layer | |
| linears are stored FP8 (weight-only) so the encoder can visit the GPU next to the DiT. | |
| """ | |
| from __future__ import annotations | |
| import gc | |
| import json | |
| import os | |
| import torch | |
| import torch.nn as nn | |
| from invoker_dit import FP8Linear, _set_module | |
| HIDDEN_LAYER = 50 | |
| _QUANT = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", | |
| "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj") | |
| def load_qwen(repo_id: str, subfolder: str = "Ref2VA", token=None, log=print): | |
| from accelerate import init_empty_weights | |
| from accelerate.utils import set_module_tensor_to_device | |
| from huggingface_hub import hf_hub_download | |
| from safetensors import safe_open | |
| from transformers import AutoConfig, AutoProcessor, Qwen3VLForConditionalGeneration | |
| te = f"{subfolder}/text_encoder" | |
| config_path = hf_hub_download(repo_id, f"{te}/config.json", token=token) | |
| config = AutoConfig.from_pretrained(os.path.dirname(config_path)) | |
| config.text_config.num_hidden_layers = HIDDEN_LAYER | |
| with init_empty_weights(): | |
| model = Qwen3VLForConditionalGeneration(config) | |
| model.lm_head = nn.Identity() # never used: the encoder reads a hidden layer | |
| index = json.load(open(hf_hub_download(repo_id, f"{te}/model.safetensors.index.json", token=token))) | |
| params = {n for n, _ in model.named_parameters()} | |
| def wanted(key): | |
| if key.startswith("model.language_model.layers."): | |
| return int(key.split(".")[3]) < HIDDEN_LAYER | |
| return key in params | |
| shards = sorted({s for k, s in index["weight_map"].items() if wanted(k)}) | |
| cache_dir = os.environ.get("INVOKER_SHARD_DIR", "/tmp/invoker-shards") | |
| for i, shard in enumerate(shards): | |
| path = hf_hub_download(repo_id, f"{te}/{shard}", token=token, local_dir=cache_dir) | |
| with safe_open(path, framework="pt") as f: | |
| for key in f.keys(): | |
| if not wanted(key): | |
| continue | |
| tensor = f.get_tensor(key) | |
| module_name = key.rsplit(".", 1)[0] | |
| if key.startswith("model.language_model.layers.") and module_name.endswith(_QUANT): | |
| _set_module(model, module_name, FP8Linear(tensor)) | |
| else: | |
| set_module_tensor_to_device(model, key, "cpu", value=tensor.to(torch.bfloat16), dtype=torch.bfloat16) | |
| del tensor | |
| os.remove(path) | |
| gc.collect() | |
| log(f"[qwen] shard {i + 1}/{len(shards)} {shard} done") | |
| meta = [n for n, p in model.named_parameters() if p.is_meta] | |
| if meta: | |
| raise RuntimeError(f"unloaded Qwen tensors: {meta[:8]}") | |
| model.eval().requires_grad_(False) | |
| proc_path = hf_hub_download(repo_id, f"{subfolder}/processor/preprocessor_config.json", token=token) | |
| for name in ("tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt", "chat_template.json", | |
| "video_preprocessor_config.json"): | |
| hf_hub_download(repo_id, f"{subfolder}/processor/{name}", token=token) | |
| processor = AutoProcessor.from_pretrained(os.path.dirname(proc_path)) | |
| return model, processor | |
| def qwen_inputs(processor, prompt, images=()): | |
| """`qwen_inputs` from cwm_encoders.py, restricted to the static-image presentation (no videos).""" | |
| tokenizer = processor.tokenizer | |
| ids, tags = [], [] | |
| merge = int(processor.image_processor.merge_size) ** 2 | |
| def text(value): | |
| rows = tokenizer(value, add_special_tokens=False)["input_ids"] | |
| ids.extend(rows) | |
| tags.extend([1] * len(rows)) | |
| def vision(pad, count): | |
| rows = [tokenizer.convert_tokens_to_ids("<|vision_start|>")] | |
| rows += [tokenizer.convert_tokens_to_ids(pad)] * count | |
| rows += [tokenizer.convert_tokens_to_ids("<|vision_end|>")] | |
| ids.extend(rows) | |
| tags.extend([0] * len(rows)) | |
| extra = {} | |
| if len(images): | |
| pictures = processor.image_processor(images=list(images), return_tensors="pt") | |
| extra.update({k: pictures[k] for k in ("pixel_values", "image_grid_thw")}) | |
| for i, g in enumerate(pictures["image_grid_thw"]): | |
| text(f"<Picture {i + 1}>: ") | |
| vision("<|image_pad|>", int(g.prod()) // merge) | |
| text(prompt) | |
| input_ids = torch.tensor([ids], dtype=torch.long) | |
| token_types = torch.zeros_like(input_ids, dtype=torch.int32) | |
| token_types[input_ids == tokenizer.convert_tokens_to_ids("<|image_pad|>")] = 1 | |
| token_types[input_ids == tokenizer.convert_tokens_to_ids("<|video_pad|>")] = 2 | |
| return dict(input_ids=input_ids, attention_mask=torch.ones_like(input_ids), mm_token_type_ids=token_types, | |
| **extra), torch.tensor(tags, dtype=torch.long) | |
| def encode_condition(model, processor, prompt, image, device="cuda"): | |
| """-> prompt_embeds [L, 5120] bf16 (layer-50 output), token_tags [L] long.""" | |
| kwargs, tags = qwen_inputs(processor, prompt, images=[image]) | |
| kwargs = {k: v.to(device, dtype=torch.bfloat16 if v.is_floating_point() else v.dtype) for k, v in kwargs.items()} | |
| layers = model.model.language_model.layers | |
| captured = [] | |
| handle = layers[HIDDEN_LAYER - 1].register_forward_hook( | |
| lambda module, args, output: captured.append(output[0] if isinstance(output, tuple) else output) | |
| ) | |
| try: | |
| model.model(**kwargs, use_cache=False) | |
| finally: | |
| handle.remove() | |
| hidden = captured[0][0] | |
| if len(hidden) != len(tags): | |
| raise ValueError("Qwen rows and presentation tags disagree") | |
| return hidden.to(torch.bfloat16), tags | |