Download src/psi/models/interaction_arch/shared_prefix.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 24 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/shared_prefix.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/models/interaction_arch/shared_prefix.py
-
curl -L -o shared_prefix.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/shared_prefix.py
24 kB
| """Experimental, scoped frozen-prefix replay for real HF Qwen2.5-Omni. | |
| Nothing is installed on import. The two text models keep independent parameters, | |
| tails and mutable DynamicCache containers. Only explicitly paired AV forwards | |
| can replay. A changed mask, position, input, frozen weight, or historical prefix | |
| KV disables replay; matching a current ledger is deliberately insufficient. | |
| Example:: | |
| replay = SharedPrefixReplay(body.model, qa.model, prefix_layers=20) | |
| with replay.pair() as packet: | |
| with packet.body(): | |
| body_result = encode_av(body, ...) | |
| with packet.qa(): | |
| qa_result = encode_av(qa, ...) | |
| replay.close() | |
| This prototype targets the installed Transformers DynamicLayer implementation, | |
| whose update concatenates rather than mutating existing key/value tensors. It | |
| fails closed for sliding/static/offloaded caches and stochastic prefixes. No | |
| unchecked "trust the ledger" or "assume equal history" mode is provided. | |
| """ | |
| from collections import Counter | |
| from contextlib import contextmanager | |
| from dataclasses import dataclass, field | |
| from functools import wraps | |
| import inspect | |
| import json | |
| import torch | |
| from transformers.cache_utils import DynamicCache, DynamicLayer | |
| from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import ( | |
| Qwen2_5OmniThinkerTextModel, | |
| ) | |
| def _same(left, right): | |
| if isinstance(left, torch.Tensor) or isinstance(right, torch.Tensor): | |
| return (isinstance(left, torch.Tensor) and isinstance(right, torch.Tensor) | |
| and left.dtype == right.dtype and left.device == right.device | |
| and left.shape == right.shape and torch.equal(left, right)) | |
| if type(left) is not type(right): | |
| return False | |
| if isinstance(left, dict): | |
| return left.keys() == right.keys() and all(_same(left[k], right[k]) for k in left) | |
| if isinstance(left, (list, tuple)): | |
| return len(left) == len(right) and all(_same(a, b) for a, b in zip(left, right)) | |
| return left == right | |
| def _snapshot(value): | |
| if isinstance(value, torch.Tensor): | |
| return value.detach().clone() | |
| if isinstance(value, dict): | |
| return {key: _snapshot(item) for key, item in value.items()} | |
| if isinstance(value, (tuple, list)): | |
| return type(value)(_snapshot(item) for item in value) | |
| return value | |
| def _stamp(tensor): | |
| # Retained tensor references prevent allocator address reuse. A detached | |
| # alias is valid because it shares both storage and the version counter. | |
| try: | |
| version = tensor._version | |
| except RuntimeError: | |
| return None # Inference-mode tensors have no mutation counter. | |
| return (tensor.data_ptr(), version, tuple(tensor.shape), tuple(tensor.stride()), | |
| tensor.storage_offset(), tensor.dtype, tensor.device) | |
| def _prefix_cache(cache, count): | |
| if cache is None: | |
| return () | |
| if type(cache) is not DynamicCache or getattr(cache, "offloading", False): | |
| raise ValueError("unsupported_cache") | |
| result = [] | |
| for index in range(count): | |
| if index >= len(cache.layers): | |
| result.append(None) | |
| continue | |
| layer = cache.layers[index] | |
| if type(layer) is not DynamicLayer: | |
| raise ValueError("unsupported_cache") | |
| if not layer.is_initialized or layer.get_seq_length() == 0: | |
| result.append(None) | |
| else: | |
| result.append((layer.keys, layer.values)) | |
| return tuple(result) if any(item is not None for item in result) else () | |
| def _kv_stamps(history): | |
| return tuple(None if layer is None else tuple(_stamp(t) for t in layer) | |
| for layer in history) | |
| def _has_versions(stamps): | |
| return all(layer is None or all(item is not None for item in layer) for layer in stamps) | |
| def _history_requires_grad(history): | |
| return any(t.requires_grad for layer in history if layer is not None for t in layer) | |
| def _autocast_state(): | |
| return tuple((device, torch.is_autocast_enabled(device), torch.get_autocast_dtype(device)) | |
| for device in ("cpu", "cuda")) | |
| class _Packet: | |
| owner: object | |
| role: str | None = None | |
| calls: Counter = field(default_factory=Counter) | |
| body_args: dict | None = None | |
| body_history: tuple = () | |
| body_stamps: tuple = () | |
| body_cache: object = None | |
| body_layers: dict = field(default_factory=dict) | |
| can_record: bool = False | |
| can_replay: bool = False | |
| reason: str | None = None | |
| autocast: tuple = () | |
| history_cloned: bool = False | |
| def _scope(self, role): | |
| if self.owner._packet is not self or self.role is not None: | |
| raise RuntimeError("AV role scopes must be non-nested inside their pair") | |
| self.role = role | |
| try: | |
| yield | |
| finally: | |
| self.role = None | |
| def body(self): | |
| return self._scope("body") | |
| def qa(self): | |
| return self._scope("qa") | |
| class SharedPrefixReplay: | |
| """Install reversible forward wrappers; replay only inside AV role scopes. | |
| ``history_mode='exact'`` compares every pre-forward prefix KV value. The | |
| ``'lineage'`` mode skips that bandwidth only when both histories still have | |
| the exact retained storage/version stamps of the last proven-equal pair. | |
| Compaction, refill, row resets, external writes or divergent histories lose | |
| that certificate and trigger the same full comparison. A successful exact | |
| comparison can safely establish a new certificate, even after divergence. | |
| Weights must be exactly equal and frozen at construction. Subsequent | |
| changes disable replay (reconstruct this object to revalidate weights). | |
| Inputs requiring gradients are not replayed, preserving upstream gradients. | |
| """ | |
| def __init__(self, body, qa, *, prefix_layers, history_mode="exact", alias_prefix=False): | |
| if not isinstance(body, Qwen2_5OmniThinkerTextModel) or not isinstance(qa, Qwen2_5OmniThinkerTextModel): | |
| raise TypeError("Expected two real Qwen2_5OmniThinkerTextModel instances") | |
| if body is qa or any(a is b for a, b in zip(body.layers, qa.layers)): | |
| raise ValueError("Distinct model/layer modules are required; parameter aliases are allowed") | |
| if not 0 < prefix_layers < min(len(body.layers), len(qa.layers)): | |
| raise ValueError("Prefix must leave at least one independent tail layer") | |
| if history_mode not in {"exact", "lineage"}: | |
| raise ValueError("history_mode must be exact or lineage") | |
| self.body, self.qa = body, qa | |
| self.prefix_layers = int(prefix_layers) | |
| self.history_mode = history_mode | |
| self.stats = Counter() | |
| self._packet = None | |
| self._certificate = None | |
| self._originals = [] | |
| self._closed = False | |
| self._validate_prefix() | |
| self.aliased_parameter_bytes = 0 | |
| self.aliased_buffer_bytes = 0 | |
| if alias_prefix: | |
| self._alias_prefix() | |
| self.stats["aliased_parameter_bytes"] = self.aliased_parameter_bytes | |
| self.stats["aliased_buffer_bytes"] = self.aliased_buffer_bytes | |
| self._signatures = {id(m): inspect.signature(m.forward) for m in (body, qa)} | |
| self._baseline = {id(m): self._weight_signature(m) for m in (body, qa)} | |
| for role, model in (("body", body), ("qa", qa)): | |
| self._patch_model(model, role) | |
| for index in range(self.prefix_layers): | |
| self._patch_layer(model.layers[index], role, index) | |
| def _parts(self, model): | |
| # Only the original frozen vocabulary belongs to the shared prefix. | |
| # Three private body state rows are never part of a prepared AV packet. | |
| embedding = model.embed_tokens | |
| if hasattr(embedding, "storage_weight"): | |
| embedding = embedding.base | |
| return [("embed_tokens", embedding), ("rotary_emb", model.rotary_emb)] + [ | |
| (f"layers.{i}", model.layers[i]) for i in range(self.prefix_layers)] | |
| def _tensors(self, model): | |
| return {f"{name}.{key}": value for name, module in self._parts(model) | |
| for key, value in list(module.named_parameters()) + list(module.named_buffers())} | |
| def _config(self, model): | |
| data = model.config.to_dict() | |
| # HF deliberately omits this private selector from config.to_dict(). | |
| # Eager, SDPA and FlashAttention need not have identical numerics. | |
| data["__attn_implementation"] = model.config._attn_implementation | |
| return json.dumps(data, sort_keys=True, default=str) | |
| def _weight_signature(self, model): | |
| return (self._config(model), tuple((name, id(t), _stamp(t), t.requires_grad) | |
| for name, t in self._tensors(model).items())) | |
| def _validate_prefix(self): | |
| left, right = self._tensors(self.body), self._tensors(self.qa) | |
| if any("lora_" in name.lower() for name in (*left.keys(), *right.keys())): | |
| raise ValueError("LoRA/adapters in the purported frozen prefix are forbidden") | |
| for model in (self.body, self.qa): | |
| if any(p.requires_grad for _, part in self._parts(model) for p in part.parameters()): | |
| raise ValueError("All embedding, rotary and prefix parameters must be frozen") | |
| if getattr(model, "has_sliding_layers", False): | |
| raise ValueError("Prototype supports full-attention DynamicCache only") | |
| if any(getattr(module, "lora_A", None) is not None or "lora" in type(module).__name__.lower() | |
| for _, part in self._parts(model) for module in part.modules()): | |
| raise ValueError("LoRA/adapters in the prefix are forbidden, even when frozen") | |
| if self._config(self.body) != self._config(self.qa) or left.keys() != right.keys(): | |
| raise ValueError("Prefix model configurations or tensor structures differ") | |
| if any(_stamp(tensor) is None for tensor in (*left.values(), *right.values())): | |
| raise ValueError("Initialize prefix weights outside inference_mode so weight mutations can be guarded") | |
| if not all(_same(left[key], right[key]) for key in left): | |
| raise ValueError("Frozen prefix weights/buffers are not exactly equal") | |
| def _alias_prefix(self): | |
| """Release QA's equal frozen storage but preserve its module identities. | |
| This is intentionally one-way: close() removes forward wrappers, not | |
| aliases. Reconstituting a duplicate frozen backbone would defeat the | |
| memory experiment. No old Parameter/tensor references are retained. | |
| """ | |
| for (_, body_part), (_, qa_part) in zip(self._parts(self.body), self._parts(self.qa)): | |
| for name, source in body_part.named_parameters(): | |
| path, _, leaf = name.rpartition(".") | |
| destination = qa_part.get_submodule(path) if path else qa_part | |
| previous = destination._parameters[leaf] | |
| if source is not previous: | |
| self.aliased_parameter_bytes += previous.numel() * previous.element_size() | |
| destination._parameters[leaf] = source | |
| for name, source in body_part.named_buffers(): | |
| path, _, leaf = name.rpartition(".") | |
| destination = qa_part.get_submodule(path) if path else qa_part | |
| previous = destination._buffers[leaf] | |
| if source is not previous: | |
| self.aliased_buffer_bytes += previous.numel() * previous.element_size() | |
| destination._buffers[leaf] = source | |
| # HF rotary also retains this non-registered reference for reset. | |
| if hasattr(self.body.rotary_emb, "original_inv_freq"): | |
| self.qa.rotary_emb.original_inv_freq = self.body.rotary_emb.original_inv_freq | |
| def _guard_model(self, model, arguments): | |
| if self._weight_signature(model) != self._baseline[id(model)]: | |
| return "prefix_weights_changed" | |
| if model.gradient_checkpointing and model.training: | |
| return "gradient_checkpointing" | |
| if model.training and model.config.attention_dropout: | |
| return "stochastic_prefix" | |
| for _, part in self._parts(model): | |
| for module in part.modules(): | |
| if isinstance(module, torch.nn.Dropout) and module.training and module.p: | |
| return "stochastic_prefix" | |
| if module.training and getattr(module, "attention_dropout", 0): | |
| return "stochastic_prefix" | |
| if arguments.get("use_cache", model.config.use_cache) is not True: | |
| return "cache_disabled" | |
| if arguments.get("output_attentions", model.config.output_attentions): | |
| return "output_attentions" | |
| if arguments.get("output_hidden_states", model.config.output_hidden_states): | |
| return "output_hidden_states" | |
| embeds = arguments.get("inputs_embeds") | |
| if not isinstance(embeds, torch.Tensor) or embeds.requires_grad: | |
| return "input_not_frozen_embeds" | |
| if arguments.get("input_ids") is not None: | |
| return "input_ids_not_av" | |
| # encode_av explicitly supplies these; reject ambiguous defaults. | |
| if not all(isinstance(arguments.get(key), torch.Tensor) | |
| for key in ("attention_mask", "position_ids", "cache_position")): | |
| return "implicit_av_geometry" | |
| if arguments.get("kwargs"): | |
| return "extra_forward_kwargs" | |
| return None | |
| def pair(self): | |
| if self._closed or self._packet is not None: | |
| raise RuntimeError("Replay is closed or another AV pair is active") | |
| packet = self._packet = _Packet(self) | |
| try: | |
| yield packet | |
| finally: | |
| self._packet = None | |
| # Releasing the pair also releases intermediate prefix activations. | |
| packet.body_layers.clear() | |
| def _arguments(self, model, args, kwargs): | |
| bound = self._signatures[id(model)].bind(*args, **kwargs) | |
| return dict(bound.arguments) | |
| def _input_arguments(self, arguments): | |
| return {key: value for key, value in arguments.items() if key not in { | |
| "past_key_values", "return_dict", "use_cache", "output_attentions", "output_hidden_states"}} | |
| def _patch_model(self, model, role): | |
| original = model.forward | |
| def forward(*args, **kwargs): | |
| packet = self._packet | |
| if packet is None or packet.role != role: | |
| self.stats[f"{role}_bypass_forwards"] += 1 | |
| return original(*args, **kwargs) | |
| packet.calls[role] += 1 | |
| if packet.calls[role] != 1: | |
| raise RuntimeError("Exactly one model forward is permitted in each AV role scope") | |
| arguments = self._arguments(model, args, kwargs) | |
| reason = self._guard_model(model, arguments) | |
| try: | |
| history = _prefix_cache(arguments.get("past_key_values"), self.prefix_layers) | |
| except ValueError as error: | |
| history, reason = (), str(error) | |
| if _history_requires_grad(history): | |
| reason = reason or "prefix_history_requires_grad" | |
| if role == "body": | |
| packet.autocast = _autocast_state() | |
| packet.reason = reason | |
| packet.can_record = reason is None | |
| packet.body_args = _snapshot(self._input_arguments(arguments)) if packet.can_record else None | |
| packet.body_stamps = _kv_stamps(history) | |
| packet.history_cloned = not _has_versions(packet.body_stamps) | |
| # Inference-mode KVs have no version counters. Isolate the | |
| # pre-forward values, then compare numerically; never issue a | |
| # lineage certificate for unversioned storage. | |
| packet.body_history = _snapshot(history) if packet.history_cloned else history | |
| result = original(*args, **kwargs) | |
| if packet.can_record: | |
| packet.body_cache = result.past_key_values if hasattr(result, "past_key_values") else result[1] | |
| return result | |
| reason = reason or packet.reason | |
| if packet.calls["body"] != 1 or not packet.can_record: | |
| reason = reason or "body_not_recorded" | |
| if reason is None and not _same(packet.body_args, self._input_arguments(arguments)): | |
| reason = "current_input_mismatch" | |
| if reason is None and packet.autocast != _autocast_state(): | |
| reason = "autocast_mismatch" | |
| if reason is None and arguments.get("past_key_values") is packet.body_cache: | |
| reason = "shared_mutable_cache" | |
| if reason is None: | |
| if (not packet.history_cloned | |
| and packet.body_stamps != _kv_stamps(packet.body_history)): | |
| reason = "body_history_mutated" | |
| else: | |
| stamps = _kv_stamps(history) | |
| certificate = self._certificate | |
| if (self.history_mode == "lineage" and certificate is not None | |
| and certificate[2] == packet.body_stamps and certificate[3] == stamps | |
| and _has_versions(stamps) and not packet.history_cloned): | |
| self.stats["lineage_history_hits"] += 1 | |
| else: | |
| self.stats["exact_history_checks"] += 1 | |
| if not _same(packet.body_history, history): | |
| reason = "past_prefix_kv_mismatch" | |
| packet.reason = reason | |
| packet.can_replay = reason is None and len(packet.body_layers) == self.prefix_layers | |
| if not packet.can_replay: | |
| reason = reason or "incomplete_body_record" | |
| self.stats[f"fallback:{reason}"] += 1 | |
| self._certificate = None | |
| result = original(*args, **kwargs) | |
| if packet.can_replay: | |
| self.stats["replayed_pairs"] += 1 | |
| qa_cache = result.past_key_values if hasattr(result, "past_key_values") else result[1] | |
| body_history = _prefix_cache(packet.body_cache, self.prefix_layers) | |
| qa_history = _prefix_cache(qa_cache, self.prefix_layers) | |
| self._certificate = (body_history, qa_history, _kv_stamps(body_history), _kv_stamps(qa_history)) | |
| return result | |
| self._originals.append((model, original)) | |
| model.forward = forward | |
| def _patch_layer(self, layer, role, index): | |
| original = layer.forward | |
| def forward(*args, **kwargs): | |
| packet = self._packet | |
| scoped = packet is not None and packet.role == role | |
| if scoped and role == "qa" and packet.can_replay: | |
| hidden, new_keys, new_values = packet.body_layers[index] | |
| cache = kwargs["past_key_values"] | |
| cache.update(new_keys, new_values, index, {"cache_position": kwargs["cache_position"]}) | |
| self.stats["qa_prefix_layers_skipped"] += 1 | |
| return (hidden,) | |
| result = original(*args, **kwargs) | |
| self.stats[f"{role}_prefix_layers_executed"] += 1 | |
| if scoped and role == "body" and packet.can_record: | |
| hidden = result[0] | |
| if hidden.requires_grad: | |
| packet.can_record = False | |
| packet.reason = "prefix_output_requires_grad" | |
| else: | |
| cache = kwargs["past_key_values"] | |
| length = hidden.shape[1] | |
| cached = cache.layers[index] | |
| # DynamicLayer.update creates a fresh independent allocation | |
| # in QA. Slice references are safe: no shared mutable layer or | |
| # cache container, and no full-history device-to-device copy. | |
| packet.body_layers[index] = (hidden, cached.keys[..., -length:, :], | |
| cached.values[..., -length:, :]) | |
| return result | |
| self._originals.append((layer, original)) | |
| layer.forward = forward | |
| def av_dispatch(self, encode_av, body_qwen, qa_qwen): | |
| """Wrap encode_av inside an outer pair, keeping side decode out of scope. | |
| Temporarily patch causal_av.encode_av with this dispatcher and wrap the | |
| existing _encode_qwen_stream_input with ``with replay.pair():``. No | |
| formal implementation file needs to be edited. Unrecognized models or | |
| calls outside the outer pair run the original function unchanged. | |
| """ | |
| def dispatch(qwen, *args, **kwargs): | |
| packet = self._packet | |
| if packet is None or qwen is not body_qwen and qwen is not qa_qwen: | |
| return encode_av(qwen, *args, **kwargs) | |
| scope = packet.body if qwen is body_qwen else packet.qa | |
| with scope(): | |
| return encode_av(qwen, *args, **kwargs) | |
| return dispatch | |
| def close(self): | |
| if self._packet is not None: | |
| raise RuntimeError("Cannot close replay inside an active pair") | |
| if not self._closed: | |
| for module, original in reversed(self._originals): | |
| module.forward = original | |
| self._originals.clear() | |
| self._certificate = None | |
| self._closed = True | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, *_): | |
| self.close() | |
| class _SwaInstallation: | |
| """Runtime-only installation; close restores the original AV entrypoints.""" | |
| def __init__(self, model, replay, causal_av): | |
| self.model, self.replay, self.causal_av = model, replay, causal_av | |
| self.original_encode = causal_av.encode_av | |
| self.original_stream = model._encode_qwen_stream_input | |
| self.dispatch = replay.av_dispatch(self.original_encode, model.vlm_model, | |
| model.qa_model.vlm_model) | |
| def stream(*args, **kwargs): | |
| with replay.pair(): | |
| return self.original_stream(*args, **kwargs) | |
| self.stream = stream | |
| self.closed = False | |
| causal_av.encode_av = self.dispatch | |
| model._encode_qwen_stream_input = stream | |
| def stats(self): | |
| return self.replay.stats | |
| def close(self): | |
| if not self.closed: | |
| if self.causal_av.encode_av is self.dispatch: | |
| self.causal_av.encode_av = self.original_encode | |
| if self.model._encode_qwen_stream_input is self.stream: | |
| self.model._encode_qwen_stream_input = self.original_stream | |
| self.replay.close() | |
| self.closed = True | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, *_): | |
| self.close() | |
| def install_on_swa(model, *, prefix_layers=None, history_mode="lineage", alias_prefix=True): | |
| """Install on an isolated SWA instance without editing formal source files. | |
| Install after loading weights, device/dtype placement, and wrapping tail | |
| FP32-master projections, but BEFORE optimizer/DDP construction. Frozen | |
| parameter aliases persist after close; tail modules, heads and all mutable | |
| cache containers remain separate. The actual 36-layer model uses prefix 28. | |
| This experimental dispatcher is single-threaded, one installation at a time. | |
| """ | |
| from psi.models import causal_av | |
| if model.qa_model is None: | |
| raise ValueError("SWA must have an independent QA branch") | |
| body, qa = model.vlm_model.model, model.qa_model.vlm_model.model | |
| count = len(body.layers) - 8 if prefix_layers is None else prefix_layers | |
| replay = SharedPrefixReplay(body, qa, prefix_layers=count, | |
| history_mode=history_mode, alias_prefix=alias_prefix) | |
| return _SwaInstallation(model, replay, causal_av) | |