Download src/psi/models/interaction_arch/shared_av.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/shared_av.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/models/interaction_arch/shared_av.py
-
curl -L -o shared_av.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/shared_av.py
12.7 kB
| """Process-local single-AV Thinker experiment; never modifies formal sources. | |
| Install ONLY after trainer.prepare/device/dtype placement, on the unwrapped | |
| model. Existing QA LoRA Parameter objects (and their optimizer state) survive. | |
| AV tokens always use body W; only the private text continuation uses W + LoRA. | |
| This is deliberate token-routed adaptation, NOT reuse of W+LoRA AV caches. | |
| close() restores the global dispatcher and drops pending graphs, but does not | |
| restore the discarded independent QA base. Dispose of the experimental model | |
| after closing. This prevents retaining a hidden second copy of base weights. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import replace | |
| import weakref | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from transformers.cache_utils import DynamicCache | |
| from psi.models import causal_av | |
| from psi.models.swa_qa_branch import BranchLoRALinear | |
| class ReferencedLinear(nn.Module): | |
| """Unregistered, live reference to a body's Linear; no base Parameter copy.""" | |
| def __init__(self, source: nn.Linear, *, detach: bool): | |
| super().__init__() | |
| if not isinstance(source, nn.Linear): | |
| raise TypeError("Shared AV requires ordinary body Linear projections") | |
| self._source_ref = weakref.ref(source) | |
| self.detach = bool(detach) | |
| self.in_features = source.in_features | |
| self.out_features = source.out_features | |
| def source(self): | |
| source = self._source_ref() | |
| if source is None: | |
| raise RuntimeError("Shared AV body projection no longer exists") | |
| return source | |
| def weight(self): | |
| value = self.source.weight | |
| return value.detach() if self.detach else value | |
| def bias(self): | |
| value = self.source.bias | |
| return value.detach() if self.detach and value is not None else value | |
| def forward(self, x): | |
| return F.linear(x, self.weight, self.bias) | |
| class ReferencedRMSNorm(nn.Module): | |
| """Qwen RMSNorm with the same live-reference and gradient ownership rule.""" | |
| def __init__(self, source: nn.Module, *, detach: bool): | |
| super().__init__() | |
| if not hasattr(source, "variance_epsilon") or not hasattr(source, "weight"): | |
| raise TypeError("Shared AV expects Qwen RMSNorm") | |
| self._source_ref = weakref.ref(source) | |
| self.detach = bool(detach) | |
| def source(self): | |
| source = self._source_ref() | |
| if source is None: | |
| raise RuntimeError("Shared AV body norm no longer exists") | |
| return source | |
| def weight(self): | |
| value = self.source.weight | |
| return value.detach() if self.detach else value | |
| def forward(self, x): | |
| dtype = x.dtype | |
| x = x.float() | |
| x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.source.variance_epsilon) | |
| return self.weight * x.to(dtype) | |
| def clone_av_snapshot(cache, *, detach: bool): | |
| """Clone BOTH container and storage; QA rebind/compact/write cannot mutate AV.""" | |
| def copy(value): | |
| if value is None: | |
| return None | |
| return (value.detach() if detach else value).clone() | |
| past = cache.past_key_values | |
| if past is not None: | |
| past = DynamicCache(ddp_cache_data=[ | |
| (copy(layer.keys), copy(layer.values)) for layer in past.layers | |
| ]) | |
| return replace(cache, past_key_values=past, | |
| attention_mask=copy(cache.attention_mask), | |
| token_birth_windows=copy(cache.token_birth_windows), qa_cache=None) | |
| def _adapter_parameters(qa): | |
| return {id(p): p for module in qa.modules() if isinstance(module, BranchLoRALinear) | |
| for part in (module.down, module.up) for p in part.parameters()} | |
| def audit_optimizer(model, optimizer): | |
| """The QA update clock/clipping group must contain adapters and nothing else.""" | |
| adapters = _adapter_parameters(model.qa_model) | |
| all_ids, qa_ids = [], [] | |
| for group in optimizer.param_groups: | |
| ids = [id(p) for p in group["params"]] | |
| all_ids.extend(ids) | |
| if group.get("group_name") == "swa_qa": | |
| qa_ids.extend(ids) | |
| if len(all_ids) != len(set(all_ids)): | |
| raise RuntimeError("Optimizer contains duplicate shared Parameter objects") | |
| if set(qa_ids) != set(adapters): | |
| raise RuntimeError("swa_qa optimizer group must contain exactly existing QA LoRA parameters") | |
| known = {id(p) for p in model.parameters()} | |
| if not set(all_ids) <= known: | |
| raise RuntimeError("Optimizer retains discarded independent QA base parameters") | |
| class SharedAVRuntime: | |
| def __init__(self, model, *, joint=False, optimizer=None, prepared=False): | |
| if not prepared: | |
| raise RuntimeError("Install shared AV only AFTER trainer.prepare; pass prepared=True") | |
| if getattr(causal_av, "_shared_av_runtime", None) is not None: | |
| raise RuntimeError("A shared AV dispatcher is already installed in this process") | |
| body = model.vlm_model | |
| qa = model.qa_model | |
| if qa is None: | |
| raise ValueError("Shared AV experiment requires the existing online QA branch") | |
| reader = qa.vlm_model | |
| if body is reader: | |
| raise ValueError("QA text shell and body must be distinct modules") | |
| body_layers, qa_layers = body.model.layers, reader.model.layers | |
| if len(body_layers) != len(qa_layers) or len(body_layers) < 8: | |
| raise ValueError("Shared AV needs matching Qwen models with at least eight layers") | |
| split = len(body_layers) - 8 | |
| # Validate completely before releasing/replacing any independent weights. | |
| projections = (("self_attn", ("q_proj", "k_proj", "v_proj", "o_proj")), | |
| ("mlp", ("gate_proj", "up_proj", "down_proj"))) | |
| for index in range(split, len(body_layers)): | |
| for owner, names in projections: | |
| for name in names: | |
| dst = getattr(getattr(qa_layers[index], owner), name) | |
| src = getattr(getattr(body_layers[index], owner), name) | |
| if not isinstance(dst, BranchLoRALinear) or not isinstance(src, nn.Linear): | |
| raise TypeError("Expected body Linear and QA BranchLoRALinear tail projections") | |
| if dst.down.weight.shape[1] != src.in_features or dst.up.weight.shape[0] != src.out_features: | |
| raise ValueError("Body and QA projection shapes differ") | |
| if dst.down.weight.device != src.weight.device: | |
| raise RuntimeError("Install after both branches have reached their final device") | |
| for name in ("input_layernorm", "post_attention_layernorm"): | |
| if not hasattr(getattr(body_layers[index], name), "variance_epsilon"): | |
| raise TypeError("Expected body Qwen RMSNorm") | |
| frozen = list(body_layers[:split]) + [body.model.embed_tokens, body.lm_head] | |
| if any(p.requires_grad for module in frozen for p in module.parameters()): | |
| raise RuntimeError("Shared AV requires frozen body prefix, embedding and lm_head") | |
| if not hasattr(body.model.norm, "variance_epsilon"): | |
| raise TypeError("Expected body final Qwen RMSNorm") | |
| if joint and not any(p.requires_grad for layer in body_layers[-8:] for p in layer.parameters()): | |
| raise RuntimeError("joint=True requires the body's last eight layers to be trainable") | |
| if optimizer is not None: | |
| audit_optimizer(model, optimizer) | |
| self.joint = bool(joint) | |
| self._body_ref = weakref.ref(body) | |
| self._reader_ref = weakref.ref(reader) | |
| self.pending = None | |
| self.body_av_forwards = 0 | |
| self.qa_av_reads = 0 | |
| self.closed = False | |
| self._original_encode = causal_av.encode_av | |
| adapter_ids = set(_adapter_parameters(qa)) | |
| for index in range(split): | |
| qa_layers[index] = body_layers[index] | |
| for index in range(split, len(body_layers)): | |
| for owner, names in projections: | |
| for name in names: | |
| dst = getattr(getattr(qa_layers[index], owner), name) | |
| src = getattr(getattr(body_layers[index], owner), name) | |
| dst.base = ReferencedLinear(src, detach=not joint) | |
| for name in ("input_layernorm", "post_attention_layernorm"): | |
| setattr(qa_layers[index], name, | |
| ReferencedRMSNorm(getattr(body_layers[index], name), detach=not joint)) | |
| for name in ("embed_tokens", "rotary_emb"): | |
| setattr(reader.model, name, getattr(body.model, name)) | |
| # The formal body can train final norm. Never freeze it or directly | |
| # alias its trainable weight into QA: isolation must apply here too. | |
| reader.model.norm = ReferencedRMSNorm(body.model.norm, detach=not joint) | |
| reader.lm_head = body.lm_head | |
| if hasattr(reader, "audio_tower"): | |
| reader.audio_tower = None | |
| if hasattr(reader, "visual"): | |
| reader.visual = None | |
| # Config is not a parameter; synchronize special ids and text geometry. | |
| reader.config = body.config | |
| shared_ids = {id(p) for p in body.parameters()} | |
| reader_ids = {id(p) for p in reader.parameters()} | |
| if reader_ids - shared_ids != adapter_ids: | |
| raise RuntimeError("QA retains unrecognized independent non-LoRA parameters") | |
| if optimizer is not None: | |
| audit_optimizer(model, optimizer) | |
| self._dispatch = self.encode_av | |
| causal_av.encode_av = self._dispatch | |
| causal_av._shared_av_runtime = self | |
| def stats(self): | |
| return {"body_av_forwards": self.body_av_forwards, "qa_av_reads": self.qa_av_reads} | |
| def encode_av(self, qwen, cache, packets, active, **kwargs): | |
| if self.closed: | |
| raise RuntimeError("Shared AV experiment has been closed") | |
| body, reader = self._body_ref(), self._reader_ref() | |
| prepared = kwargs.get("prepared") | |
| if qwen is body: | |
| if self.pending is not None: | |
| self.pending = None | |
| raise RuntimeError("Previous body AV write was not paired with a QA read") | |
| if prepared is None: | |
| raise RuntimeError("Shared AV pairing requires an explicit PreparedAVBatch") | |
| if self.joint and cache.past_key_values is not None: | |
| if any(t.requires_grad for layer in cache.past_key_values.layers | |
| for t in (layer.keys, layer.values)): | |
| raise RuntimeError("Detach body streaming history between windows; joint only covers current AV") | |
| result = self._original_encode(qwen, cache, packets, active, **kwargs) | |
| self.body_av_forwards += 1 | |
| self.pending = (prepared, packets, result) | |
| return result | |
| if qwen is reader: | |
| pending, self.pending = self.pending, None | |
| if pending is None: | |
| raise RuntimeError("QA AV encode without an immediately paired body write") | |
| expected, expected_packets, result = pending | |
| if prepared is not expected or packets is not expected_packets: | |
| raise RuntimeError("QA AV read must use the identical prepared body packet batch") | |
| if cache.source_ids != result[-1].source_ids: | |
| raise RuntimeError("QA/body source ids disagree in shared AV pairing") | |
| snapshot = clone_av_snapshot(result[-1], detach=not self.joint) | |
| hidden = result[0] if self.joint else result[0].detach() | |
| self.qa_av_reads += 1 | |
| return hidden, result[1], result[2], result[3], snapshot | |
| if self.pending is not None: | |
| self.pending = None | |
| raise RuntimeError("Another AV model interrupted shared AV body/QA pairing") | |
| return self._original_encode(qwen, cache, packets, active, **kwargs) | |
| def close(self): | |
| self.pending = None | |
| if self.closed: | |
| return | |
| if causal_av.encode_av is not self._dispatch: | |
| raise RuntimeError("Cannot restore shared AV dispatcher: another patch replaced it") | |
| causal_av.encode_av = self._original_encode | |
| if getattr(causal_av, "_shared_av_runtime", None) is self: | |
| del causal_av._shared_av_runtime | |
| self.closed = True | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, *_): | |
| self.close() | |
| def install_shared_av(model, *, joint=False, optimizer=None, prepared=False): | |
| return SharedAVRuntime(model, joint=joint, optimizer=optimizer, prepared=prepared) | |