GUI-Swift-4B / frozen /trace2task /gui_owl_parallel4075_model.py
dongjintao's picture
Release GUI-Swift-4B: Desktop131 SFT step3368
46f5c24 verified
Raw History Blame Contribute Delete
14.4 kB
"""Pure-parallel Standard40/Compact75 SFT with official PEFT LoRA.
The model consumes a label-free, fixed all-mask table. Table embeddings are
zeroed, so neither targets nor the tokenizer's mask embedding enter the single
language forward. As in ordinary next-token pretraining, table item ``j`` is
predicted from hidden state ``j - 1`` through the original tied vocabulary
head. Prefix queries are causal and cannot attend to the table; table queries
are bidirectional and may attend to the prefix and the whole empty table.
"""
from __future__ import annotations
import functools
import hashlib
import json
from enum import Enum
from pathlib import Path
import torch
from torch import nn
from .gui_owl_native_sft_model import _codec_contract, parallel_attention_mask
from .gui_owl_standard40_model import LORA_PROJECTIONS, OriginalPatchEmbedding
def _language(base):
return base.model.language_model
def configure_parallel_peft(base, *, rank=16, alpha=32, dropout=0.05):
"""Freeze ``base`` and install language-only PEFT LoRA adapters."""
from peft import LoraConfig, get_peft_model
if type(rank) is not int or rank <= 0 or alpha <= 0 or not 0 <= dropout < 1:
raise ValueError("Invalid LoRA rank, alpha, or dropout")
base.requires_grad_(False)
embedding, head = base.get_input_embeddings(), base.lm_head
if head.weight is not embedding.weight:
raise ValueError("Expected the original tied vocabulary head")
mergers = [base.model.visual.merger, *base.model.visual.deepstack_merger_list]
if len(mergers) != 4:
raise ValueError(f"Expected exactly four frozen visual mergers, found {len(mergers)}")
language = _language(base)
inventory = [
name for name, module in language.named_modules()
if isinstance(module, nn.Linear) and name.rsplit(".", 1)[-1] in LORA_PROJECTIONS
]
expected = len(language.layers) * len(LORA_PROJECTIONS)
if len(inventory) != expected:
raise ValueError(f"Expected {expected} language projections, found {len(inventory)}")
config = LoraConfig(
r=rank,
lora_alpha=alpha,
lora_dropout=dropout,
bias="none",
target_modules=list(LORA_PROJECTIONS),
task_type=None,
)
# PEFT's default autocast_adapter_dtype keeps BF16 base weights frozen while
# promoting trainable adapter weights to FP32.
wrapped = get_peft_model(language, config, autocast_adapter_dtype=True)
base.model.language_model = wrapped
trainable = {name: p for name, p in base.named_parameters() if p.requires_grad}
if not trainable or any("lora_" not in name for name in trainable):
raise RuntimeError("PEFT introduced a trainable parameter outside LoRA")
if any(p.dtype != torch.float32 for p in trainable.values()):
raise RuntimeError("All LoRA trainables must remain FP32")
if (head is not base.lm_head or head.weight is not base.get_input_embeddings().weight
or head.weight.requires_grad or any(p.requires_grad for p in base.model.visual.parameters())):
raise RuntimeError("Vision, mergers, embedding, and tied head must remain frozen")
return {
"total": sum(p.numel() for p in base.parameters()),
"trainable": sum(p.numel() for p in trainable.values()),
"frozen": sum(p.numel() for p in base.parameters() if not p.requires_grad),
"lora_modules": len(inventory), "lora_rank": rank, "lora_alpha": alpha,
"lora_dropout": dropout, "visual_trainable": 0,
"visual_merger_modules": len(mergers),
"new_output_head_parameters": 0, "original_head_frozen": True,
"peft": True,
}
def prepare_parallel_inputs(cpu_inputs, base, device, codec):
"""Validate an exact label-free all-mask batch and compute official MRoPE."""
required = {"input_ids", "attention_mask", "pixel_values", "image_grid_thw",
"slot_positions", "prefix_lengths", "lengths"}
if set(cpu_inputs) != required:
raise ValueError("Unexpected fields in label-free parallel model inputs")
if any(value.device.type != "cpu" for value in cpu_inputs.values()):
raise ValueError("Input validation and MRoPE indexing must occur on CPU")
template, relative = _codec_contract(codec)
ids, valid = cpu_inputs["input_ids"], cpu_inputs["attention_mask"]
lengths, prefixes, slots = (cpu_inputs[k] for k in
("lengths", "prefix_lengths", "slot_positions"))
if (ids.ndim != 2 or valid.shape != ids.shape or lengths.shape != (ids.shape[0],)
or prefixes.shape != lengths.shape
or slots.shape != (ids.shape[0], codec.output_slots)):
raise ValueError("Invalid parallel input shape")
expected_valid = torch.arange(ids.shape[1])[None] < lengths[:, None]
if not torch.equal(valid.bool(), expected_valid):
raise ValueError("Attention validity must exactly match sequence lengths")
for row in range(ids.shape[0]):
prefix, length = int(prefixes[row]), int(lengths[row])
expected_slots = torch.tensor([prefix + p for p in relative], dtype=slots.dtype)
if (prefix <= 0 or length - prefix != len(template)
or not torch.equal(slots[row].cpu(), expected_slots)
or tuple(ids[row, prefix:length].tolist()) != template):
raise ValueError("Input must end in the exact codec all-mask table")
position_ids, _ = base.model.get_rope_index(
ids, image_grid_thw=cpu_inputs["image_grid_thw"], attention_mask=valid)
result = {name: value.to(device, non_blocking=True) for name, value in cpu_inputs.items()
if name != "lengths"}
result["position_ids"] = position_ids.to(device, non_blocking=True)
return result
class GuiOwlParallel4075Model(nn.Module):
"""One visual and one language forward for either fixed-table codec."""
main_input_name = "input_ids"
accepts_loss_kwargs = False
def __init__(self, base, codec):
super().__init__()
_codec_contract(codec)
if codec.vocab_size > base.config.text_config.vocab_size:
raise ValueError("Codec vocabulary exceeds the original vocabulary head")
self.base, self.codec = base, codec
# Trainer/Accelerate inspect these standard PreTrainedModel attributes;
# keeping them as the original objects also prevents config drift.
self.config = base.config
self.generation_config = getattr(base, "generation_config", None)
def train(self, mode=True):
super().train(mode)
self.base.model.visual.eval()
return self
def forward(self, *, input_ids, attention_mask, pixel_values, image_grid_thw,
slot_positions, prefix_lengths, position_ids):
template, relative = _codec_contract(self.codec)
if (slot_positions.shape != (input_ids.shape[0], self.codec.output_slots)
or prefix_lengths.shape != (input_ids.shape[0],)):
raise ValueError("Invalid parallel positions")
for row in range(input_ids.shape[0]):
prefix = int(prefix_lengths[row])
length = int(attention_mask[row].bool().sum())
expected = torch.tensor([prefix + p for p in relative],
dtype=slot_positions.dtype, device=slot_positions.device)
if (length - prefix != len(template) or not torch.equal(slot_positions[row], expected)
or tuple(input_ids[row, prefix:length].tolist()) != template):
raise ValueError("Forward requires the exact all-mask table")
embeddings = self.base.get_input_embeddings()(input_ids)
image_embeddings, deepstack = self.base.model.get_image_features(
pixel_values, image_grid_thw)
image_embeddings = torch.cat(image_embeddings, 0).to(embeddings.dtype)
visual_positions = input_ids == self.base.config.image_token_id
embeddings = embeddings.masked_scatter(visual_positions[..., None], image_embeddings)
rows = torch.arange(input_ids.shape[0], device=input_ids.device)[:, None]
if not torch.all(input_ids[rows, slot_positions] == self.codec.mask_id):
raise ValueError("Every output slot must contain the mask token")
embeddings = embeddings.clone()
embeddings[rows, slot_positions] = 0
outputs = _language(self.base)(
inputs_embeds=embeddings, position_ids=position_ids,
attention_mask=parallel_attention_mask(attention_mask, prefix_lengths),
use_cache=False, visual_pos_masks=visual_positions,
deepstack_visual_embeds=deepstack,
)
return self.base.lm_head(outputs.last_hidden_state[rows, slot_positions - 1])
def _json_safe(value):
if isinstance(value, Enum):
return value.value
if isinstance(value, Path):
return str(value)
if isinstance(value, dict):
return {str(k): _json_safe(v) for k, v in value.items()}
if isinstance(value, (set, frozenset)):
converted = [_json_safe(v) for v in value]
# PEFT represents target_modules as a set. Sorting by canonical JSON,
# rather than Python's hash-dependent iteration order, makes fresh
# multi-process adapter identity checks reproducible across ranks.
return sorted(converted, key=lambda item: json.dumps(
item, sort_keys=True, separators=(",", ":")))
if isinstance(value, (list, tuple)):
return [_json_safe(v) for v in value]
return value
def adapter_config_json(model):
"""Return a canonical, JSON-safe record of the active PEFT configuration."""
configs = _language(model.base).peft_config
payload = {name: _json_safe(config.to_dict()) for name, config in configs.items()}
return json.dumps(payload, sort_keys=True, separators=(",", ":"))
def adapter_state(model):
from peft import get_peft_model_state_dict
return {name: value.detach().cpu().clone() for name, value in
get_peft_model_state_dict(_language(model.base)).items()}
def adapter_digest(model):
digest = hashlib.sha256(adapter_config_json(model).encode())
digest.update(adapter_tensor_digest(model).encode())
return digest.hexdigest()
def adapter_tensor_digest(model):
"""Digest only adapter tensors, separating initialization from config identity."""
digest = hashlib.sha256()
for name, value in sorted(adapter_state(model).items()):
digest.update(name.encode())
digest.update(value.contiguous().numpy().tobytes())
return digest.hexdigest()
def load_adapter_state(model, state):
from peft import set_peft_model_state_dict
expected = adapter_state(model)
if state.keys() != expected.keys():
raise ValueError("Checkpoint PEFT adapter inventory differs")
for name, value in state.items():
if value.shape != expected[name].shape or value.dtype != expected[name].dtype:
raise ValueError("Checkpoint PEFT adapter shape/dtype differs")
result = set_peft_model_state_dict(_language(model.base), state)
# strict=False necessarily reports frozen base keys as missing because the
# supplied state is intentionally adapter-only. Our inventory/shape check
# above is the strict adapter check.
if getattr(result, "unexpected_keys", ()):
raise ValueError(f"PEFT adapter did not restore exactly: {result}")
def build_parallel4075_model(model_path, device, *, codec, checkpoint_stride=2,
lora_rank=16, lora_alpha=32, lora_dropout=0.05):
"""Load the pinned local GUI-Owl base and build a fresh pure-parallel model."""
from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (
CheckpointImpl,
apply_activation_checkpointing,
checkpoint_wrapper,
)
from transformers import Qwen3VLForConditionalGeneration
if checkpoint_stride not in (0, 1, 2):
raise ValueError("Activation checkpoint stride must be 0, 1, or 2")
base, loading = Qwen3VLForConditionalGeneration.from_pretrained(
model_path, local_files_only=True, use_safetensors=True, dtype=torch.bfloat16,
attn_implementation="sdpa", output_loading_info=True)
if any(loading.get(key) for key in
("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")):
raise ValueError(f"Original GUI-Owl weights did not load exactly: {loading}")
if (base.config.model_type != "qwen3_vl" or len(base.model.language_model.layers) != 36
or sum(p.numel() for p in base.parameters()) != 4437815808):
raise ValueError("Expected the pinned original GUI-Owl-1.5-4B architecture")
patch = base.model.visual.patch_embed
original = tuple(patch.parameters())
base.model.visual.patch_embed = OriginalPatchEmbedding(patch)
if any(a is not b for a, b in zip(original, base.model.visual.patch_embed.parameters(), strict=True)):
raise RuntimeError("Original patch projection parameters changed")
counts = configure_parallel_peft(base, rank=lora_rank, alpha=lora_alpha,
dropout=lora_dropout)
base.to(device)
# PEFT exposes the original language module as base_model.model.
layers = _language(base).base_model.model.layers
selected = {id(layer) for index, layer in enumerate(layers)
if checkpoint_stride and index % checkpoint_stride == 0}
if selected:
apply_activation_checkpointing(
_language(base), checkpoint_wrapper_fn=functools.partial(
checkpoint_wrapper, checkpoint_impl=CheckpointImpl.NO_REENTRANT),
check_fn=lambda module: id(module) in selected)
counts.update(checkpoint_stride=checkpoint_stride, checkpointed_layers=len(selected),
weight_storage_dtype="bfloat16-frozen-float32-peft",
attention_pattern="causal-prefix-bidirectional-empty-table",
prediction_alignment="j-1", output_slots=codec.output_slots)
return GuiOwlParallel4075Model(base, codec).train(), counts
__all__ = ["GuiOwlParallel4075Model", "adapter_config_json", "adapter_digest",
"adapter_state", "adapter_tensor_digest", "build_parallel4075_model",
"configure_parallel_peft", "load_adapter_state", "prepare_parallel_inputs"]