"""Pure-parallel Standard40/Compact75 SFT with official PEFT LoRA. The model consumes a label-free, fixed all-mask table. Table embeddings are zeroed, so neither targets nor the tokenizer's mask embedding enter the single language forward. As in ordinary next-token pretraining, table item ``j`` is predicted from hidden state ``j - 1`` through the original tied vocabulary head. Prefix queries are causal and cannot attend to the table; table queries are bidirectional and may attend to the prefix and the whole empty table. """ from __future__ import annotations import functools import hashlib import json from enum import Enum from pathlib import Path import torch from torch import nn from .gui_owl_native_sft_model import _codec_contract, parallel_attention_mask from .gui_owl_standard40_model import LORA_PROJECTIONS, OriginalPatchEmbedding def _language(base): return base.model.language_model def configure_parallel_peft(base, *, rank=16, alpha=32, dropout=0.05): """Freeze ``base`` and install language-only PEFT LoRA adapters.""" from peft import LoraConfig, get_peft_model if type(rank) is not int or rank <= 0 or alpha <= 0 or not 0 <= dropout < 1: raise ValueError("Invalid LoRA rank, alpha, or dropout") base.requires_grad_(False) embedding, head = base.get_input_embeddings(), base.lm_head if head.weight is not embedding.weight: raise ValueError("Expected the original tied vocabulary head") mergers = [base.model.visual.merger, *base.model.visual.deepstack_merger_list] if len(mergers) != 4: raise ValueError(f"Expected exactly four frozen visual mergers, found {len(mergers)}") language = _language(base) inventory = [ name for name, module in language.named_modules() if isinstance(module, nn.Linear) and name.rsplit(".", 1)[-1] in LORA_PROJECTIONS ] expected = len(language.layers) * len(LORA_PROJECTIONS) if len(inventory) != expected: raise ValueError(f"Expected {expected} language projections, found {len(inventory)}") config = LoraConfig( r=rank, lora_alpha=alpha, lora_dropout=dropout, bias="none", target_modules=list(LORA_PROJECTIONS), task_type=None, ) # PEFT's default autocast_adapter_dtype keeps BF16 base weights frozen while # promoting trainable adapter weights to FP32. wrapped = get_peft_model(language, config, autocast_adapter_dtype=True) base.model.language_model = wrapped trainable = {name: p for name, p in base.named_parameters() if p.requires_grad} if not trainable or any("lora_" not in name for name in trainable): raise RuntimeError("PEFT introduced a trainable parameter outside LoRA") if any(p.dtype != torch.float32 for p in trainable.values()): raise RuntimeError("All LoRA trainables must remain FP32") if (head is not base.lm_head or head.weight is not base.get_input_embeddings().weight or head.weight.requires_grad or any(p.requires_grad for p in base.model.visual.parameters())): raise RuntimeError("Vision, mergers, embedding, and tied head must remain frozen") return { "total": sum(p.numel() for p in base.parameters()), "trainable": sum(p.numel() for p in trainable.values()), "frozen": sum(p.numel() for p in base.parameters() if not p.requires_grad), "lora_modules": len(inventory), "lora_rank": rank, "lora_alpha": alpha, "lora_dropout": dropout, "visual_trainable": 0, "visual_merger_modules": len(mergers), "new_output_head_parameters": 0, "original_head_frozen": True, "peft": True, } def prepare_parallel_inputs(cpu_inputs, base, device, codec): """Validate an exact label-free all-mask batch and compute official MRoPE.""" required = {"input_ids", "attention_mask", "pixel_values", "image_grid_thw", "slot_positions", "prefix_lengths", "lengths"} if set(cpu_inputs) != required: raise ValueError("Unexpected fields in label-free parallel model inputs") if any(value.device.type != "cpu" for value in cpu_inputs.values()): raise ValueError("Input validation and MRoPE indexing must occur on CPU") template, relative = _codec_contract(codec) ids, valid = cpu_inputs["input_ids"], cpu_inputs["attention_mask"] lengths, prefixes, slots = (cpu_inputs[k] for k in ("lengths", "prefix_lengths", "slot_positions")) if (ids.ndim != 2 or valid.shape != ids.shape or lengths.shape != (ids.shape[0],) or prefixes.shape != lengths.shape or slots.shape != (ids.shape[0], codec.output_slots)): raise ValueError("Invalid parallel input shape") expected_valid = torch.arange(ids.shape[1])[None] < lengths[:, None] if not torch.equal(valid.bool(), expected_valid): raise ValueError("Attention validity must exactly match sequence lengths") for row in range(ids.shape[0]): prefix, length = int(prefixes[row]), int(lengths[row]) expected_slots = torch.tensor([prefix + p for p in relative], dtype=slots.dtype) if (prefix <= 0 or length - prefix != len(template) or not torch.equal(slots[row].cpu(), expected_slots) or tuple(ids[row, prefix:length].tolist()) != template): raise ValueError("Input must end in the exact codec all-mask table") position_ids, _ = base.model.get_rope_index( ids, image_grid_thw=cpu_inputs["image_grid_thw"], attention_mask=valid) result = {name: value.to(device, non_blocking=True) for name, value in cpu_inputs.items() if name != "lengths"} result["position_ids"] = position_ids.to(device, non_blocking=True) return result class GuiOwlParallel4075Model(nn.Module): """One visual and one language forward for either fixed-table codec.""" main_input_name = "input_ids" accepts_loss_kwargs = False def __init__(self, base, codec): super().__init__() _codec_contract(codec) if codec.vocab_size > base.config.text_config.vocab_size: raise ValueError("Codec vocabulary exceeds the original vocabulary head") self.base, self.codec = base, codec # Trainer/Accelerate inspect these standard PreTrainedModel attributes; # keeping them as the original objects also prevents config drift. self.config = base.config self.generation_config = getattr(base, "generation_config", None) def train(self, mode=True): super().train(mode) self.base.model.visual.eval() return self def forward(self, *, input_ids, attention_mask, pixel_values, image_grid_thw, slot_positions, prefix_lengths, position_ids): template, relative = _codec_contract(self.codec) if (slot_positions.shape != (input_ids.shape[0], self.codec.output_slots) or prefix_lengths.shape != (input_ids.shape[0],)): raise ValueError("Invalid parallel positions") for row in range(input_ids.shape[0]): prefix = int(prefix_lengths[row]) length = int(attention_mask[row].bool().sum()) expected = torch.tensor([prefix + p for p in relative], dtype=slot_positions.dtype, device=slot_positions.device) if (length - prefix != len(template) or not torch.equal(slot_positions[row], expected) or tuple(input_ids[row, prefix:length].tolist()) != template): raise ValueError("Forward requires the exact all-mask table") embeddings = self.base.get_input_embeddings()(input_ids) image_embeddings, deepstack = self.base.model.get_image_features( pixel_values, image_grid_thw) image_embeddings = torch.cat(image_embeddings, 0).to(embeddings.dtype) visual_positions = input_ids == self.base.config.image_token_id embeddings = embeddings.masked_scatter(visual_positions[..., None], image_embeddings) rows = torch.arange(input_ids.shape[0], device=input_ids.device)[:, None] if not torch.all(input_ids[rows, slot_positions] == self.codec.mask_id): raise ValueError("Every output slot must contain the mask token") embeddings = embeddings.clone() embeddings[rows, slot_positions] = 0 outputs = _language(self.base)( inputs_embeds=embeddings, position_ids=position_ids, attention_mask=parallel_attention_mask(attention_mask, prefix_lengths), use_cache=False, visual_pos_masks=visual_positions, deepstack_visual_embeds=deepstack, ) return self.base.lm_head(outputs.last_hidden_state[rows, slot_positions - 1]) def _json_safe(value): if isinstance(value, Enum): return value.value if isinstance(value, Path): return str(value) if isinstance(value, dict): return {str(k): _json_safe(v) for k, v in value.items()} if isinstance(value, (set, frozenset)): converted = [_json_safe(v) for v in value] # PEFT represents target_modules as a set. Sorting by canonical JSON, # rather than Python's hash-dependent iteration order, makes fresh # multi-process adapter identity checks reproducible across ranks. return sorted(converted, key=lambda item: json.dumps( item, sort_keys=True, separators=(",", ":"))) if isinstance(value, (list, tuple)): return [_json_safe(v) for v in value] return value def adapter_config_json(model): """Return a canonical, JSON-safe record of the active PEFT configuration.""" configs = _language(model.base).peft_config payload = {name: _json_safe(config.to_dict()) for name, config in configs.items()} return json.dumps(payload, sort_keys=True, separators=(",", ":")) def adapter_state(model): from peft import get_peft_model_state_dict return {name: value.detach().cpu().clone() for name, value in get_peft_model_state_dict(_language(model.base)).items()} def adapter_digest(model): digest = hashlib.sha256(adapter_config_json(model).encode()) digest.update(adapter_tensor_digest(model).encode()) return digest.hexdigest() def adapter_tensor_digest(model): """Digest only adapter tensors, separating initialization from config identity.""" digest = hashlib.sha256() for name, value in sorted(adapter_state(model).items()): digest.update(name.encode()) digest.update(value.contiguous().numpy().tobytes()) return digest.hexdigest() def load_adapter_state(model, state): from peft import set_peft_model_state_dict expected = adapter_state(model) if state.keys() != expected.keys(): raise ValueError("Checkpoint PEFT adapter inventory differs") for name, value in state.items(): if value.shape != expected[name].shape or value.dtype != expected[name].dtype: raise ValueError("Checkpoint PEFT adapter shape/dtype differs") result = set_peft_model_state_dict(_language(model.base), state) # strict=False necessarily reports frozen base keys as missing because the # supplied state is intentionally adapter-only. Our inventory/shape check # above is the strict adapter check. if getattr(result, "unexpected_keys", ()): raise ValueError(f"PEFT adapter did not restore exactly: {result}") def build_parallel4075_model(model_path, device, *, codec, checkpoint_stride=2, lora_rank=16, lora_alpha=32, lora_dropout=0.05): """Load the pinned local GUI-Owl base and build a fresh pure-parallel model.""" from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( CheckpointImpl, apply_activation_checkpointing, checkpoint_wrapper, ) from transformers import Qwen3VLForConditionalGeneration if checkpoint_stride not in (0, 1, 2): raise ValueError("Activation checkpoint stride must be 0, 1, or 2") base, loading = Qwen3VLForConditionalGeneration.from_pretrained( model_path, local_files_only=True, use_safetensors=True, dtype=torch.bfloat16, attn_implementation="sdpa", output_loading_info=True) if any(loading.get(key) for key in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")): raise ValueError(f"Original GUI-Owl weights did not load exactly: {loading}") if (base.config.model_type != "qwen3_vl" or len(base.model.language_model.layers) != 36 or sum(p.numel() for p in base.parameters()) != 4437815808): raise ValueError("Expected the pinned original GUI-Owl-1.5-4B architecture") patch = base.model.visual.patch_embed original = tuple(patch.parameters()) base.model.visual.patch_embed = OriginalPatchEmbedding(patch) if any(a is not b for a, b in zip(original, base.model.visual.patch_embed.parameters(), strict=True)): raise RuntimeError("Original patch projection parameters changed") counts = configure_parallel_peft(base, rank=lora_rank, alpha=lora_alpha, dropout=lora_dropout) base.to(device) # PEFT exposes the original language module as base_model.model. layers = _language(base).base_model.model.layers selected = {id(layer) for index, layer in enumerate(layers) if checkpoint_stride and index % checkpoint_stride == 0} if selected: apply_activation_checkpointing( _language(base), checkpoint_wrapper_fn=functools.partial( checkpoint_wrapper, checkpoint_impl=CheckpointImpl.NO_REENTRANT), check_fn=lambda module: id(module) in selected) counts.update(checkpoint_stride=checkpoint_stride, checkpointed_layers=len(selected), weight_storage_dtype="bfloat16-frozen-float32-peft", attention_pattern="causal-prefix-bidirectional-empty-table", prediction_alignment="j-1", output_slots=codec.output_slots) return GuiOwlParallel4075Model(base, codec).train(), counts __all__ = ["GuiOwlParallel4075Model", "adapter_config_json", "adapter_digest", "adapter_state", "adapter_tensor_digest", "build_parallel4075_model", "configure_parallel_peft", "load_adapter_state", "prepare_parallel_inputs"]