Download frozen/trace2task/gui_owl_parallel4075_model.py from dongjintao/GUI-Swift-4B: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/dongjintao/GUI-Swift-4B/resolve/main/frozen/trace2task/gui_owl_parallel4075_model.py
- Command line
-
hf download hf://dongjintao/GUI-Swift-4B/frozen/trace2task/gui_owl_parallel4075_model.py
-
curl -L -o gui_owl_parallel4075_model.py https://huggingface.co/dongjintao/GUI-Swift-4B/resolve/main/frozen/trace2task/gui_owl_parallel4075_model.py
14.4 kB
| """Pure-parallel Standard40/Compact75 SFT with official PEFT LoRA. | |
| The model consumes a label-free, fixed all-mask table. Table embeddings are | |
| zeroed, so neither targets nor the tokenizer's mask embedding enter the single | |
| language forward. As in ordinary next-token pretraining, table item ``j`` is | |
| predicted from hidden state ``j - 1`` through the original tied vocabulary | |
| head. Prefix queries are causal and cannot attend to the table; table queries | |
| are bidirectional and may attend to the prefix and the whole empty table. | |
| """ | |
| from __future__ import annotations | |
| import functools | |
| import hashlib | |
| import json | |
| from enum import Enum | |
| from pathlib import Path | |
| import torch | |
| from torch import nn | |
| from .gui_owl_native_sft_model import _codec_contract, parallel_attention_mask | |
| from .gui_owl_standard40_model import LORA_PROJECTIONS, OriginalPatchEmbedding | |
| def _language(base): | |
| return base.model.language_model | |
| def configure_parallel_peft(base, *, rank=16, alpha=32, dropout=0.05): | |
| """Freeze ``base`` and install language-only PEFT LoRA adapters.""" | |
| from peft import LoraConfig, get_peft_model | |
| if type(rank) is not int or rank <= 0 or alpha <= 0 or not 0 <= dropout < 1: | |
| raise ValueError("Invalid LoRA rank, alpha, or dropout") | |
| base.requires_grad_(False) | |
| embedding, head = base.get_input_embeddings(), base.lm_head | |
| if head.weight is not embedding.weight: | |
| raise ValueError("Expected the original tied vocabulary head") | |
| mergers = [base.model.visual.merger, *base.model.visual.deepstack_merger_list] | |
| if len(mergers) != 4: | |
| raise ValueError(f"Expected exactly four frozen visual mergers, found {len(mergers)}") | |
| language = _language(base) | |
| inventory = [ | |
| name for name, module in language.named_modules() | |
| if isinstance(module, nn.Linear) and name.rsplit(".", 1)[-1] in LORA_PROJECTIONS | |
| ] | |
| expected = len(language.layers) * len(LORA_PROJECTIONS) | |
| if len(inventory) != expected: | |
| raise ValueError(f"Expected {expected} language projections, found {len(inventory)}") | |
| config = LoraConfig( | |
| r=rank, | |
| lora_alpha=alpha, | |
| lora_dropout=dropout, | |
| bias="none", | |
| target_modules=list(LORA_PROJECTIONS), | |
| task_type=None, | |
| ) | |
| # PEFT's default autocast_adapter_dtype keeps BF16 base weights frozen while | |
| # promoting trainable adapter weights to FP32. | |
| wrapped = get_peft_model(language, config, autocast_adapter_dtype=True) | |
| base.model.language_model = wrapped | |
| trainable = {name: p for name, p in base.named_parameters() if p.requires_grad} | |
| if not trainable or any("lora_" not in name for name in trainable): | |
| raise RuntimeError("PEFT introduced a trainable parameter outside LoRA") | |
| if any(p.dtype != torch.float32 for p in trainable.values()): | |
| raise RuntimeError("All LoRA trainables must remain FP32") | |
| if (head is not base.lm_head or head.weight is not base.get_input_embeddings().weight | |
| or head.weight.requires_grad or any(p.requires_grad for p in base.model.visual.parameters())): | |
| raise RuntimeError("Vision, mergers, embedding, and tied head must remain frozen") | |
| return { | |
| "total": sum(p.numel() for p in base.parameters()), | |
| "trainable": sum(p.numel() for p in trainable.values()), | |
| "frozen": sum(p.numel() for p in base.parameters() if not p.requires_grad), | |
| "lora_modules": len(inventory), "lora_rank": rank, "lora_alpha": alpha, | |
| "lora_dropout": dropout, "visual_trainable": 0, | |
| "visual_merger_modules": len(mergers), | |
| "new_output_head_parameters": 0, "original_head_frozen": True, | |
| "peft": True, | |
| } | |
| def prepare_parallel_inputs(cpu_inputs, base, device, codec): | |
| """Validate an exact label-free all-mask batch and compute official MRoPE.""" | |
| required = {"input_ids", "attention_mask", "pixel_values", "image_grid_thw", | |
| "slot_positions", "prefix_lengths", "lengths"} | |
| if set(cpu_inputs) != required: | |
| raise ValueError("Unexpected fields in label-free parallel model inputs") | |
| if any(value.device.type != "cpu" for value in cpu_inputs.values()): | |
| raise ValueError("Input validation and MRoPE indexing must occur on CPU") | |
| template, relative = _codec_contract(codec) | |
| ids, valid = cpu_inputs["input_ids"], cpu_inputs["attention_mask"] | |
| lengths, prefixes, slots = (cpu_inputs[k] for k in | |
| ("lengths", "prefix_lengths", "slot_positions")) | |
| if (ids.ndim != 2 or valid.shape != ids.shape or lengths.shape != (ids.shape[0],) | |
| or prefixes.shape != lengths.shape | |
| or slots.shape != (ids.shape[0], codec.output_slots)): | |
| raise ValueError("Invalid parallel input shape") | |
| expected_valid = torch.arange(ids.shape[1])[None] < lengths[:, None] | |
| if not torch.equal(valid.bool(), expected_valid): | |
| raise ValueError("Attention validity must exactly match sequence lengths") | |
| for row in range(ids.shape[0]): | |
| prefix, length = int(prefixes[row]), int(lengths[row]) | |
| expected_slots = torch.tensor([prefix + p for p in relative], dtype=slots.dtype) | |
| if (prefix <= 0 or length - prefix != len(template) | |
| or not torch.equal(slots[row].cpu(), expected_slots) | |
| or tuple(ids[row, prefix:length].tolist()) != template): | |
| raise ValueError("Input must end in the exact codec all-mask table") | |
| position_ids, _ = base.model.get_rope_index( | |
| ids, image_grid_thw=cpu_inputs["image_grid_thw"], attention_mask=valid) | |
| result = {name: value.to(device, non_blocking=True) for name, value in cpu_inputs.items() | |
| if name != "lengths"} | |
| result["position_ids"] = position_ids.to(device, non_blocking=True) | |
| return result | |
| class GuiOwlParallel4075Model(nn.Module): | |
| """One visual and one language forward for either fixed-table codec.""" | |
| main_input_name = "input_ids" | |
| accepts_loss_kwargs = False | |
| def __init__(self, base, codec): | |
| super().__init__() | |
| _codec_contract(codec) | |
| if codec.vocab_size > base.config.text_config.vocab_size: | |
| raise ValueError("Codec vocabulary exceeds the original vocabulary head") | |
| self.base, self.codec = base, codec | |
| # Trainer/Accelerate inspect these standard PreTrainedModel attributes; | |
| # keeping them as the original objects also prevents config drift. | |
| self.config = base.config | |
| self.generation_config = getattr(base, "generation_config", None) | |
| def train(self, mode=True): | |
| super().train(mode) | |
| self.base.model.visual.eval() | |
| return self | |
| def forward(self, *, input_ids, attention_mask, pixel_values, image_grid_thw, | |
| slot_positions, prefix_lengths, position_ids): | |
| template, relative = _codec_contract(self.codec) | |
| if (slot_positions.shape != (input_ids.shape[0], self.codec.output_slots) | |
| or prefix_lengths.shape != (input_ids.shape[0],)): | |
| raise ValueError("Invalid parallel positions") | |
| for row in range(input_ids.shape[0]): | |
| prefix = int(prefix_lengths[row]) | |
| length = int(attention_mask[row].bool().sum()) | |
| expected = torch.tensor([prefix + p for p in relative], | |
| dtype=slot_positions.dtype, device=slot_positions.device) | |
| if (length - prefix != len(template) or not torch.equal(slot_positions[row], expected) | |
| or tuple(input_ids[row, prefix:length].tolist()) != template): | |
| raise ValueError("Forward requires the exact all-mask table") | |
| embeddings = self.base.get_input_embeddings()(input_ids) | |
| image_embeddings, deepstack = self.base.model.get_image_features( | |
| pixel_values, image_grid_thw) | |
| image_embeddings = torch.cat(image_embeddings, 0).to(embeddings.dtype) | |
| visual_positions = input_ids == self.base.config.image_token_id | |
| embeddings = embeddings.masked_scatter(visual_positions[..., None], image_embeddings) | |
| rows = torch.arange(input_ids.shape[0], device=input_ids.device)[:, None] | |
| if not torch.all(input_ids[rows, slot_positions] == self.codec.mask_id): | |
| raise ValueError("Every output slot must contain the mask token") | |
| embeddings = embeddings.clone() | |
| embeddings[rows, slot_positions] = 0 | |
| outputs = _language(self.base)( | |
| inputs_embeds=embeddings, position_ids=position_ids, | |
| attention_mask=parallel_attention_mask(attention_mask, prefix_lengths), | |
| use_cache=False, visual_pos_masks=visual_positions, | |
| deepstack_visual_embeds=deepstack, | |
| ) | |
| return self.base.lm_head(outputs.last_hidden_state[rows, slot_positions - 1]) | |
| def _json_safe(value): | |
| if isinstance(value, Enum): | |
| return value.value | |
| if isinstance(value, Path): | |
| return str(value) | |
| if isinstance(value, dict): | |
| return {str(k): _json_safe(v) for k, v in value.items()} | |
| if isinstance(value, (set, frozenset)): | |
| converted = [_json_safe(v) for v in value] | |
| # PEFT represents target_modules as a set. Sorting by canonical JSON, | |
| # rather than Python's hash-dependent iteration order, makes fresh | |
| # multi-process adapter identity checks reproducible across ranks. | |
| return sorted(converted, key=lambda item: json.dumps( | |
| item, sort_keys=True, separators=(",", ":"))) | |
| if isinstance(value, (list, tuple)): | |
| return [_json_safe(v) for v in value] | |
| return value | |
| def adapter_config_json(model): | |
| """Return a canonical, JSON-safe record of the active PEFT configuration.""" | |
| configs = _language(model.base).peft_config | |
| payload = {name: _json_safe(config.to_dict()) for name, config in configs.items()} | |
| return json.dumps(payload, sort_keys=True, separators=(",", ":")) | |
| def adapter_state(model): | |
| from peft import get_peft_model_state_dict | |
| return {name: value.detach().cpu().clone() for name, value in | |
| get_peft_model_state_dict(_language(model.base)).items()} | |
| def adapter_digest(model): | |
| digest = hashlib.sha256(adapter_config_json(model).encode()) | |
| digest.update(adapter_tensor_digest(model).encode()) | |
| return digest.hexdigest() | |
| def adapter_tensor_digest(model): | |
| """Digest only adapter tensors, separating initialization from config identity.""" | |
| digest = hashlib.sha256() | |
| for name, value in sorted(adapter_state(model).items()): | |
| digest.update(name.encode()) | |
| digest.update(value.contiguous().numpy().tobytes()) | |
| return digest.hexdigest() | |
| def load_adapter_state(model, state): | |
| from peft import set_peft_model_state_dict | |
| expected = adapter_state(model) | |
| if state.keys() != expected.keys(): | |
| raise ValueError("Checkpoint PEFT adapter inventory differs") | |
| for name, value in state.items(): | |
| if value.shape != expected[name].shape or value.dtype != expected[name].dtype: | |
| raise ValueError("Checkpoint PEFT adapter shape/dtype differs") | |
| result = set_peft_model_state_dict(_language(model.base), state) | |
| # strict=False necessarily reports frozen base keys as missing because the | |
| # supplied state is intentionally adapter-only. Our inventory/shape check | |
| # above is the strict adapter check. | |
| if getattr(result, "unexpected_keys", ()): | |
| raise ValueError(f"PEFT adapter did not restore exactly: {result}") | |
| def build_parallel4075_model(model_path, device, *, codec, checkpoint_stride=2, | |
| lora_rank=16, lora_alpha=32, lora_dropout=0.05): | |
| """Load the pinned local GUI-Owl base and build a fresh pure-parallel model.""" | |
| from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( | |
| CheckpointImpl, | |
| apply_activation_checkpointing, | |
| checkpoint_wrapper, | |
| ) | |
| from transformers import Qwen3VLForConditionalGeneration | |
| if checkpoint_stride not in (0, 1, 2): | |
| raise ValueError("Activation checkpoint stride must be 0, 1, or 2") | |
| base, loading = Qwen3VLForConditionalGeneration.from_pretrained( | |
| model_path, local_files_only=True, use_safetensors=True, dtype=torch.bfloat16, | |
| attn_implementation="sdpa", output_loading_info=True) | |
| if any(loading.get(key) for key in | |
| ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")): | |
| raise ValueError(f"Original GUI-Owl weights did not load exactly: {loading}") | |
| if (base.config.model_type != "qwen3_vl" or len(base.model.language_model.layers) != 36 | |
| or sum(p.numel() for p in base.parameters()) != 4437815808): | |
| raise ValueError("Expected the pinned original GUI-Owl-1.5-4B architecture") | |
| patch = base.model.visual.patch_embed | |
| original = tuple(patch.parameters()) | |
| base.model.visual.patch_embed = OriginalPatchEmbedding(patch) | |
| if any(a is not b for a, b in zip(original, base.model.visual.patch_embed.parameters(), strict=True)): | |
| raise RuntimeError("Original patch projection parameters changed") | |
| counts = configure_parallel_peft(base, rank=lora_rank, alpha=lora_alpha, | |
| dropout=lora_dropout) | |
| base.to(device) | |
| # PEFT exposes the original language module as base_model.model. | |
| layers = _language(base).base_model.model.layers | |
| selected = {id(layer) for index, layer in enumerate(layers) | |
| if checkpoint_stride and index % checkpoint_stride == 0} | |
| if selected: | |
| apply_activation_checkpointing( | |
| _language(base), checkpoint_wrapper_fn=functools.partial( | |
| checkpoint_wrapper, checkpoint_impl=CheckpointImpl.NO_REENTRANT), | |
| check_fn=lambda module: id(module) in selected) | |
| counts.update(checkpoint_stride=checkpoint_stride, checkpointed_layers=len(selected), | |
| weight_storage_dtype="bfloat16-frozen-float32-peft", | |
| attention_pattern="causal-prefix-bidirectional-empty-table", | |
| prediction_alignment="j-1", output_slots=codec.output_slots) | |
| return GuiOwlParallel4075Model(base, codec).train(), counts | |
| __all__ = ["GuiOwlParallel4075Model", "adapter_config_json", "adapter_digest", | |
| "adapter_state", "adapter_tensor_digest", "build_parallel4075_model", | |
| "configure_parallel_peft", "load_adapter_state", "prepare_parallel_inputs"] | |