"""Experimental, scoped frozen-prefix replay for real HF Qwen2.5-Omni. Nothing is installed on import. The two text models keep independent parameters, tails and mutable DynamicCache containers. Only explicitly paired AV forwards can replay. A changed mask, position, input, frozen weight, or historical prefix KV disables replay; matching a current ledger is deliberately insufficient. Example:: replay = SharedPrefixReplay(body.model, qa.model, prefix_layers=20) with replay.pair() as packet: with packet.body(): body_result = encode_av(body, ...) with packet.qa(): qa_result = encode_av(qa, ...) replay.close() This prototype targets the installed Transformers DynamicLayer implementation, whose update concatenates rather than mutating existing key/value tensors. It fails closed for sliding/static/offloaded caches and stochastic prefixes. No unchecked "trust the ledger" or "assume equal history" mode is provided. """ from collections import Counter from contextlib import contextmanager from dataclasses import dataclass, field from functools import wraps import inspect import json import torch from transformers.cache_utils import DynamicCache, DynamicLayer from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import ( Qwen2_5OmniThinkerTextModel, ) def _same(left, right): if isinstance(left, torch.Tensor) or isinstance(right, torch.Tensor): return (isinstance(left, torch.Tensor) and isinstance(right, torch.Tensor) and left.dtype == right.dtype and left.device == right.device and left.shape == right.shape and torch.equal(left, right)) if type(left) is not type(right): return False if isinstance(left, dict): return left.keys() == right.keys() and all(_same(left[k], right[k]) for k in left) if isinstance(left, (list, tuple)): return len(left) == len(right) and all(_same(a, b) for a, b in zip(left, right)) return left == right def _snapshot(value): if isinstance(value, torch.Tensor): return value.detach().clone() if isinstance(value, dict): return {key: _snapshot(item) for key, item in value.items()} if isinstance(value, (tuple, list)): return type(value)(_snapshot(item) for item in value) return value def _stamp(tensor): # Retained tensor references prevent allocator address reuse. A detached # alias is valid because it shares both storage and the version counter. try: version = tensor._version except RuntimeError: return None # Inference-mode tensors have no mutation counter. return (tensor.data_ptr(), version, tuple(tensor.shape), tuple(tensor.stride()), tensor.storage_offset(), tensor.dtype, tensor.device) def _prefix_cache(cache, count): if cache is None: return () if type(cache) is not DynamicCache or getattr(cache, "offloading", False): raise ValueError("unsupported_cache") result = [] for index in range(count): if index >= len(cache.layers): result.append(None) continue layer = cache.layers[index] if type(layer) is not DynamicLayer: raise ValueError("unsupported_cache") if not layer.is_initialized or layer.get_seq_length() == 0: result.append(None) else: result.append((layer.keys, layer.values)) return tuple(result) if any(item is not None for item in result) else () def _kv_stamps(history): return tuple(None if layer is None else tuple(_stamp(t) for t in layer) for layer in history) def _has_versions(stamps): return all(layer is None or all(item is not None for item in layer) for layer in stamps) def _history_requires_grad(history): return any(t.requires_grad for layer in history if layer is not None for t in layer) def _autocast_state(): return tuple((device, torch.is_autocast_enabled(device), torch.get_autocast_dtype(device)) for device in ("cpu", "cuda")) @dataclass class _Packet: owner: object role: str | None = None calls: Counter = field(default_factory=Counter) body_args: dict | None = None body_history: tuple = () body_stamps: tuple = () body_cache: object = None body_layers: dict = field(default_factory=dict) can_record: bool = False can_replay: bool = False reason: str | None = None autocast: tuple = () history_cloned: bool = False @contextmanager def _scope(self, role): if self.owner._packet is not self or self.role is not None: raise RuntimeError("AV role scopes must be non-nested inside their pair") self.role = role try: yield finally: self.role = None def body(self): return self._scope("body") def qa(self): return self._scope("qa") class SharedPrefixReplay: """Install reversible forward wrappers; replay only inside AV role scopes. ``history_mode='exact'`` compares every pre-forward prefix KV value. The ``'lineage'`` mode skips that bandwidth only when both histories still have the exact retained storage/version stamps of the last proven-equal pair. Compaction, refill, row resets, external writes or divergent histories lose that certificate and trigger the same full comparison. A successful exact comparison can safely establish a new certificate, even after divergence. Weights must be exactly equal and frozen at construction. Subsequent changes disable replay (reconstruct this object to revalidate weights). Inputs requiring gradients are not replayed, preserving upstream gradients. """ def __init__(self, body, qa, *, prefix_layers, history_mode="exact", alias_prefix=False): if not isinstance(body, Qwen2_5OmniThinkerTextModel) or not isinstance(qa, Qwen2_5OmniThinkerTextModel): raise TypeError("Expected two real Qwen2_5OmniThinkerTextModel instances") if body is qa or any(a is b for a, b in zip(body.layers, qa.layers)): raise ValueError("Distinct model/layer modules are required; parameter aliases are allowed") if not 0 < prefix_layers < min(len(body.layers), len(qa.layers)): raise ValueError("Prefix must leave at least one independent tail layer") if history_mode not in {"exact", "lineage"}: raise ValueError("history_mode must be exact or lineage") self.body, self.qa = body, qa self.prefix_layers = int(prefix_layers) self.history_mode = history_mode self.stats = Counter() self._packet = None self._certificate = None self._originals = [] self._closed = False self._validate_prefix() self.aliased_parameter_bytes = 0 self.aliased_buffer_bytes = 0 if alias_prefix: self._alias_prefix() self.stats["aliased_parameter_bytes"] = self.aliased_parameter_bytes self.stats["aliased_buffer_bytes"] = self.aliased_buffer_bytes self._signatures = {id(m): inspect.signature(m.forward) for m in (body, qa)} self._baseline = {id(m): self._weight_signature(m) for m in (body, qa)} for role, model in (("body", body), ("qa", qa)): self._patch_model(model, role) for index in range(self.prefix_layers): self._patch_layer(model.layers[index], role, index) def _parts(self, model): # Only the original frozen vocabulary belongs to the shared prefix. # Three private body state rows are never part of a prepared AV packet. embedding = model.embed_tokens if hasattr(embedding, "storage_weight"): embedding = embedding.base return [("embed_tokens", embedding), ("rotary_emb", model.rotary_emb)] + [ (f"layers.{i}", model.layers[i]) for i in range(self.prefix_layers)] def _tensors(self, model): return {f"{name}.{key}": value for name, module in self._parts(model) for key, value in list(module.named_parameters()) + list(module.named_buffers())} def _config(self, model): data = model.config.to_dict() # HF deliberately omits this private selector from config.to_dict(). # Eager, SDPA and FlashAttention need not have identical numerics. data["__attn_implementation"] = model.config._attn_implementation return json.dumps(data, sort_keys=True, default=str) def _weight_signature(self, model): return (self._config(model), tuple((name, id(t), _stamp(t), t.requires_grad) for name, t in self._tensors(model).items())) def _validate_prefix(self): left, right = self._tensors(self.body), self._tensors(self.qa) if any("lora_" in name.lower() for name in (*left.keys(), *right.keys())): raise ValueError("LoRA/adapters in the purported frozen prefix are forbidden") for model in (self.body, self.qa): if any(p.requires_grad for _, part in self._parts(model) for p in part.parameters()): raise ValueError("All embedding, rotary and prefix parameters must be frozen") if getattr(model, "has_sliding_layers", False): raise ValueError("Prototype supports full-attention DynamicCache only") if any(getattr(module, "lora_A", None) is not None or "lora" in type(module).__name__.lower() for _, part in self._parts(model) for module in part.modules()): raise ValueError("LoRA/adapters in the prefix are forbidden, even when frozen") if self._config(self.body) != self._config(self.qa) or left.keys() != right.keys(): raise ValueError("Prefix model configurations or tensor structures differ") if any(_stamp(tensor) is None for tensor in (*left.values(), *right.values())): raise ValueError("Initialize prefix weights outside inference_mode so weight mutations can be guarded") if not all(_same(left[key], right[key]) for key in left): raise ValueError("Frozen prefix weights/buffers are not exactly equal") def _alias_prefix(self): """Release QA's equal frozen storage but preserve its module identities. This is intentionally one-way: close() removes forward wrappers, not aliases. Reconstituting a duplicate frozen backbone would defeat the memory experiment. No old Parameter/tensor references are retained. """ for (_, body_part), (_, qa_part) in zip(self._parts(self.body), self._parts(self.qa)): for name, source in body_part.named_parameters(): path, _, leaf = name.rpartition(".") destination = qa_part.get_submodule(path) if path else qa_part previous = destination._parameters[leaf] if source is not previous: self.aliased_parameter_bytes += previous.numel() * previous.element_size() destination._parameters[leaf] = source for name, source in body_part.named_buffers(): path, _, leaf = name.rpartition(".") destination = qa_part.get_submodule(path) if path else qa_part previous = destination._buffers[leaf] if source is not previous: self.aliased_buffer_bytes += previous.numel() * previous.element_size() destination._buffers[leaf] = source # HF rotary also retains this non-registered reference for reset. if hasattr(self.body.rotary_emb, "original_inv_freq"): self.qa.rotary_emb.original_inv_freq = self.body.rotary_emb.original_inv_freq def _guard_model(self, model, arguments): if self._weight_signature(model) != self._baseline[id(model)]: return "prefix_weights_changed" if model.gradient_checkpointing and model.training: return "gradient_checkpointing" if model.training and model.config.attention_dropout: return "stochastic_prefix" for _, part in self._parts(model): for module in part.modules(): if isinstance(module, torch.nn.Dropout) and module.training and module.p: return "stochastic_prefix" if module.training and getattr(module, "attention_dropout", 0): return "stochastic_prefix" if arguments.get("use_cache", model.config.use_cache) is not True: return "cache_disabled" if arguments.get("output_attentions", model.config.output_attentions): return "output_attentions" if arguments.get("output_hidden_states", model.config.output_hidden_states): return "output_hidden_states" embeds = arguments.get("inputs_embeds") if not isinstance(embeds, torch.Tensor) or embeds.requires_grad: return "input_not_frozen_embeds" if arguments.get("input_ids") is not None: return "input_ids_not_av" # encode_av explicitly supplies these; reject ambiguous defaults. if not all(isinstance(arguments.get(key), torch.Tensor) for key in ("attention_mask", "position_ids", "cache_position")): return "implicit_av_geometry" if arguments.get("kwargs"): return "extra_forward_kwargs" return None @contextmanager def pair(self): if self._closed or self._packet is not None: raise RuntimeError("Replay is closed or another AV pair is active") packet = self._packet = _Packet(self) try: yield packet finally: self._packet = None # Releasing the pair also releases intermediate prefix activations. packet.body_layers.clear() def _arguments(self, model, args, kwargs): bound = self._signatures[id(model)].bind(*args, **kwargs) return dict(bound.arguments) def _input_arguments(self, arguments): return {key: value for key, value in arguments.items() if key not in { "past_key_values", "return_dict", "use_cache", "output_attentions", "output_hidden_states"}} def _patch_model(self, model, role): original = model.forward @wraps(original) def forward(*args, **kwargs): packet = self._packet if packet is None or packet.role != role: self.stats[f"{role}_bypass_forwards"] += 1 return original(*args, **kwargs) packet.calls[role] += 1 if packet.calls[role] != 1: raise RuntimeError("Exactly one model forward is permitted in each AV role scope") arguments = self._arguments(model, args, kwargs) reason = self._guard_model(model, arguments) try: history = _prefix_cache(arguments.get("past_key_values"), self.prefix_layers) except ValueError as error: history, reason = (), str(error) if _history_requires_grad(history): reason = reason or "prefix_history_requires_grad" if role == "body": packet.autocast = _autocast_state() packet.reason = reason packet.can_record = reason is None packet.body_args = _snapshot(self._input_arguments(arguments)) if packet.can_record else None packet.body_stamps = _kv_stamps(history) packet.history_cloned = not _has_versions(packet.body_stamps) # Inference-mode KVs have no version counters. Isolate the # pre-forward values, then compare numerically; never issue a # lineage certificate for unversioned storage. packet.body_history = _snapshot(history) if packet.history_cloned else history result = original(*args, **kwargs) if packet.can_record: packet.body_cache = result.past_key_values if hasattr(result, "past_key_values") else result[1] return result reason = reason or packet.reason if packet.calls["body"] != 1 or not packet.can_record: reason = reason or "body_not_recorded" if reason is None and not _same(packet.body_args, self._input_arguments(arguments)): reason = "current_input_mismatch" if reason is None and packet.autocast != _autocast_state(): reason = "autocast_mismatch" if reason is None and arguments.get("past_key_values") is packet.body_cache: reason = "shared_mutable_cache" if reason is None: if (not packet.history_cloned and packet.body_stamps != _kv_stamps(packet.body_history)): reason = "body_history_mutated" else: stamps = _kv_stamps(history) certificate = self._certificate if (self.history_mode == "lineage" and certificate is not None and certificate[2] == packet.body_stamps and certificate[3] == stamps and _has_versions(stamps) and not packet.history_cloned): self.stats["lineage_history_hits"] += 1 else: self.stats["exact_history_checks"] += 1 if not _same(packet.body_history, history): reason = "past_prefix_kv_mismatch" packet.reason = reason packet.can_replay = reason is None and len(packet.body_layers) == self.prefix_layers if not packet.can_replay: reason = reason or "incomplete_body_record" self.stats[f"fallback:{reason}"] += 1 self._certificate = None result = original(*args, **kwargs) if packet.can_replay: self.stats["replayed_pairs"] += 1 qa_cache = result.past_key_values if hasattr(result, "past_key_values") else result[1] body_history = _prefix_cache(packet.body_cache, self.prefix_layers) qa_history = _prefix_cache(qa_cache, self.prefix_layers) self._certificate = (body_history, qa_history, _kv_stamps(body_history), _kv_stamps(qa_history)) return result self._originals.append((model, original)) model.forward = forward def _patch_layer(self, layer, role, index): original = layer.forward @wraps(original) def forward(*args, **kwargs): packet = self._packet scoped = packet is not None and packet.role == role if scoped and role == "qa" and packet.can_replay: hidden, new_keys, new_values = packet.body_layers[index] cache = kwargs["past_key_values"] cache.update(new_keys, new_values, index, {"cache_position": kwargs["cache_position"]}) self.stats["qa_prefix_layers_skipped"] += 1 return (hidden,) result = original(*args, **kwargs) self.stats[f"{role}_prefix_layers_executed"] += 1 if scoped and role == "body" and packet.can_record: hidden = result[0] if hidden.requires_grad: packet.can_record = False packet.reason = "prefix_output_requires_grad" else: cache = kwargs["past_key_values"] length = hidden.shape[1] cached = cache.layers[index] # DynamicLayer.update creates a fresh independent allocation # in QA. Slice references are safe: no shared mutable layer or # cache container, and no full-history device-to-device copy. packet.body_layers[index] = (hidden, cached.keys[..., -length:, :], cached.values[..., -length:, :]) return result self._originals.append((layer, original)) layer.forward = forward def av_dispatch(self, encode_av, body_qwen, qa_qwen): """Wrap encode_av inside an outer pair, keeping side decode out of scope. Temporarily patch causal_av.encode_av with this dispatcher and wrap the existing _encode_qwen_stream_input with ``with replay.pair():``. No formal implementation file needs to be edited. Unrecognized models or calls outside the outer pair run the original function unchanged. """ @wraps(encode_av) def dispatch(qwen, *args, **kwargs): packet = self._packet if packet is None or qwen is not body_qwen and qwen is not qa_qwen: return encode_av(qwen, *args, **kwargs) scope = packet.body if qwen is body_qwen else packet.qa with scope(): return encode_av(qwen, *args, **kwargs) return dispatch def close(self): if self._packet is not None: raise RuntimeError("Cannot close replay inside an active pair") if not self._closed: for module, original in reversed(self._originals): module.forward = original self._originals.clear() self._certificate = None self._closed = True def __enter__(self): return self def __exit__(self, *_): self.close() class _SwaInstallation: """Runtime-only installation; close restores the original AV entrypoints.""" def __init__(self, model, replay, causal_av): self.model, self.replay, self.causal_av = model, replay, causal_av self.original_encode = causal_av.encode_av self.original_stream = model._encode_qwen_stream_input self.dispatch = replay.av_dispatch(self.original_encode, model.vlm_model, model.qa_model.vlm_model) @wraps(self.original_stream) def stream(*args, **kwargs): with replay.pair(): return self.original_stream(*args, **kwargs) self.stream = stream self.closed = False causal_av.encode_av = self.dispatch model._encode_qwen_stream_input = stream @property def stats(self): return self.replay.stats def close(self): if not self.closed: if self.causal_av.encode_av is self.dispatch: self.causal_av.encode_av = self.original_encode if self.model._encode_qwen_stream_input is self.stream: self.model._encode_qwen_stream_input = self.original_stream self.replay.close() self.closed = True def __enter__(self): return self def __exit__(self, *_): self.close() def install_on_swa(model, *, prefix_layers=None, history_mode="lineage", alias_prefix=True): """Install on an isolated SWA instance without editing formal source files. Install after loading weights, device/dtype placement, and wrapping tail FP32-master projections, but BEFORE optimizer/DDP construction. Frozen parameter aliases persist after close; tail modules, heads and all mutable cache containers remain separate. The actual 36-layer model uses prefix 28. This experimental dispatcher is single-threaded, one installation at a time. """ from psi.models import causal_av if model.qa_model is None: raise ValueError("SWA must have an independent QA branch") body, qa = model.vlm_model.model, model.qa_model.vlm_model.model count = len(body.layers) - 8 if prefix_layers is None else prefix_layers replay = SharedPrefixReplay(body, qa, prefix_layers=count, history_mode=history_mode, alias_prefix=alias_prefix) return _SwaInstallation(model, replay, causal_av)