from __future__ import annotations import json import math import os from dataclasses import dataclass from pathlib import Path from typing import Any, Optional, Sequence import torch import torch.nn as nn import torch.nn.functional as F from diffusers.utils.outputs import BaseOutput from torch.utils.checkpoint import checkpoint from psi.models.psi0 import Psi0Model @dataclass(frozen=True) class StreamingKVCache: """Explicit cache whose identity is permanently bound to one source batch. Every real audio window is appended. A false per-row update mask denotes only a padded row after that source has ended; it is not a semantic phase switch. The optional valid mask keeps those padded rows out of attention. """ source_ids: tuple[str, ...] keys: tuple[Optional[torch.Tensor], ...] values: tuple[Optional[torch.Tensor], ...] # Boolean positions that belong to each source. Batched sources may finish # at different times, so their valid cache lengths need not be equal even # though K/V tensors share one padded time dimension. valid_mask: Optional[torch.Tensor] = None @classmethod def empty(cls, source_ids: list[str] | tuple[str, ...], num_layers: int) -> "StreamingKVCache": ids = tuple(str(value) for value in source_ids) if not ids or any(not value for value in ids): raise ValueError("A streaming KV cache requires non-empty source ids") return cls(ids, (None,) * int(num_layers), (None,) * int(num_layers), None) def detach(self) -> "StreamingKVCache": return StreamingKVCache( self.source_ids, tuple(None if value is None else value.detach() for value in self.keys), tuple(None if value is None else value.detach() for value in self.values), None if self.valid_mask is None else self.valid_mask.detach(), ) @property def sequence_lengths(self) -> tuple[int, ...]: if self.valid_mask is None: first = self.keys[0] if self.keys else None length = 0 if first is None else int(first.shape[-2]) return (length,) * len(self.source_ids) return tuple(int(value) for value in self.valid_mask.sum(dim=-1).tolist()) @property def sequence_length(self) -> int: lengths = self.sequence_lengths if len(set(lengths)) > 1: raise RuntimeError( "Batched streaming cache has different source lengths; " "use sequence_lengths instead" ) return lengths[0] if lengths else 0 @dataclass class QwenStreamingKVCache: """Recyclable source-slot Qwen2.5-Omni DynamicCache used by HFDv2. ``token_count`` is the physical dense cache width shared by a ragged source batch. ``logical_window_counts`` and ``logical_token_counts`` exclude trainer padding rows. A slot can be rebound to a new source: its old attention row is invalidated before the new source starts at logical position zero, so no old instruction can leak across the boundary. Dead physical columns are compacted only when worthwhile or required by the context limit. """ source_ids: tuple[str, ...] past_key_values: Any = None attention_mask: Optional[torch.Tensor] = None token_count: int = 0 logical_window_counts: tuple[int, ...] = () logical_token_counts: tuple[int, ...] = () slot_rebind_count: int = 0 compaction_count: int = 0 reclaimed_token_count: int = 0 # Per-row audio birth index; -1 is the fixed system prompt, -2 padding. token_birth_windows: Optional[torch.Tensor] = None # Per-row absolute multimodal RoPE position. Audio-only runs leave zeros. av_next_positions: tuple[int, ...] = () # Independent QA branch; never reuse body K/V under different weights. qa_cache: Optional["QwenStreamingKVCache"] = None def expire_audio_windows(self, keep_windows: int) -> None: if keep_windows <= 0 or self.token_birth_windows is None: return if self.attention_mask is None or self.token_birth_windows.shape != self.attention_mask.shape: raise RuntimeError("Rolling KV birth ledger and attention mask disagree") cutoff = torch.tensor(self.logical_window_counts, device=self.attention_mask.device)[:, None] - keep_windows + 1 birth = self.token_birth_windows self.attention_mask = self.attention_mask * ((birth == -1) | (birth >= cutoff)).to(self.attention_mask.dtype) @classmethod def empty( cls, source_ids: list[str] | tuple[str, ...], ) -> "QwenStreamingKVCache": ids = tuple(str(value) for value in source_ids) if not ids or any(not value for value in ids): raise ValueError("A Qwen streaming cache requires source ids") return cls( source_ids=ids, logical_window_counts=(0,) * len(ids), logical_token_counts=(0,) * len(ids), av_next_positions=(0,) * len(ids), ) def resolved_logical_token_counts(self) -> tuple[int, ...]: if len(self.logical_token_counts) == len(self.source_ids): return tuple(int(value) for value in self.logical_token_counts) if self.attention_mask is not None: return tuple( int(value) for value in self.attention_mask.sum(dim=-1).tolist() ) if int(self.token_count) > 0: return (int(self.token_count),) * len(self.source_ids) return (0,) * len(self.source_ids) def rebind_rows( self, source_ids: Sequence[str], reset_rows: Sequence[bool] | torch.Tensor, ) -> "QwenStreamingKVCache": """Reset selected logical rows while retaining the dense allocation.""" next_ids = tuple(str(value) for value in source_ids) if len(next_ids) != len(self.source_ids) or any( not value for value in next_ids ): raise ValueError( "Rebound Qwen source ids must preserve the non-empty slot batch" ) if isinstance(reset_rows, torch.Tensor): reset_values = tuple( bool(value) for value in reset_rows.detach() .to(device="cpu", dtype=torch.bool) .reshape(-1) .tolist() ) else: reset_values = tuple(bool(value) for value in reset_rows) if len(reset_values) != len(next_ids): raise ValueError( "Qwen slot reset mask must match the source batch: " f"mask={len(reset_values)} batch={len(next_ids)}" ) for row, reset in enumerate(reset_values): if not reset and next_ids[row] != self.source_ids[row]: raise RuntimeError( "A Qwen slot source id may change only with an explicit " f"reset: row={row} old={self.source_ids[row]!r} " f"new={next_ids[row]!r}" ) if not any(reset_values): return self if self.attention_mask is None and int(self.token_count) > 0: first_layer = None layers = getattr(self.past_key_values, "layers", None) if layers: first_layer = layers[0] first_key = ( None if first_layer is None else getattr(first_layer, "keys", None) ) device = ( first_key.device if isinstance(first_key, torch.Tensor) else torch.device("cpu") ) self.attention_mask = torch.ones( (len(next_ids), int(self.token_count)), device=device, dtype=torch.long, ) elif self.attention_mask is not None: self.attention_mask = self.attention_mask.clone() if self.attention_mask is not None: row_mask = torch.tensor( reset_values, device=self.attention_mask.device, dtype=torch.bool, ) self.attention_mask[row_mask] = 0 if self.token_birth_windows is not None: self.token_birth_windows = self.token_birth_windows.clone() self.token_birth_windows[row_mask] = -2 window_counts = list(self.logical_window_counts) token_counts = list(self.resolved_logical_token_counts()) av_positions = list(self.av_next_positions or (0,) * len(next_ids)) if len(window_counts) != len(next_ids): window_counts = [0] * len(next_ids) for row, reset in enumerate(reset_values): if reset: window_counts[row] = 0 token_counts[row] = 0 av_positions[row] = 0 self.source_ids = next_ids self.logical_window_counts = tuple(window_counts) self.logical_token_counts = tuple(token_counts) self.av_next_positions = tuple(av_positions) self.slot_rebind_count += sum(reset_values) if self.qa_cache is not None: self.qa_cache.rebind_rows(next_ids, reset_values) return self def compact_invalid_tokens(self) -> int: """Pack every row's visible KV leftward and drop dead columns.""" if self.attention_mask is None or int(self.token_count) <= 0: return 0 mask = self.attention_mask.to(dtype=torch.bool) if mask.ndim != 2 or int(mask.shape[-1]) != int(self.token_count): raise RuntimeError( "Qwen cache attention mask cannot be compacted: " f"mask={tuple(mask.shape)} physical={self.token_count}" ) lengths = mask.sum(dim=-1).to(dtype=torch.long) compact_width = int(lengths.max().item()) if lengths.numel() else 0 reclaimed = int(self.token_count) - compact_width if reclaimed <= 0: return 0 layers = getattr(self.past_key_values, "layers", None) if layers is None: raise TypeError( "Qwen slot compaction requires a Transformers cache with layers" ) if compact_width > 0: width = int(self.token_count) positions = torch.arange( width, device=mask.device, dtype=torch.long, ).unsqueeze(0).expand(mask.shape[0], -1) priority = torch.where(mask, positions, positions + width) gather_indices = torch.topk( priority, k=compact_width, dim=-1, largest=False, sorted=True, ).indices compact_mask = ( torch.arange( compact_width, device=mask.device, dtype=torch.long, )[None, :] < lengths[:, None] ).to(dtype=self.attention_mask.dtype) else: gather_indices = None compact_mask = self.attention_mask[:, :0] for layer in layers: if not bool(getattr(layer, "is_initialized", False)): continue keys = getattr(layer, "keys", None) values = getattr(layer, "values", None) if not isinstance(keys, torch.Tensor) or not isinstance( values, torch.Tensor ): raise RuntimeError("Initialized Qwen cache layer has no K/V") if compact_width == 0: layer.keys = keys[..., :0, :] layer.values = values[..., :0, :] continue layer_indices = gather_indices.to(device=keys.device)[ :, None, :, None ].expand( keys.shape[0], keys.shape[1], compact_width, keys.shape[-1], ) layer.keys = torch.gather(keys, dim=-2, index=layer_indices) layer.values = torch.gather( values, dim=-2, index=layer_indices, ) self.attention_mask = compact_mask if self.token_birth_windows is not None: self.token_birth_windows = ( self.token_birth_windows[:, :0] if gather_indices is None else self.token_birth_windows.gather(1, gather_indices).masked_fill(~compact_mask.bool(), -2) ) self.token_count = compact_width self.compaction_count += 1 self.reclaimed_token_count += reclaimed return reclaimed def compact_if_worthwhile( self, *, current_tokens: int, auxiliary_tokens: int, max_context_tokens: int, minimum_garbage_tokens: int = 256, garbage_ratio: float = 0.25, ) -> int: logical = self.resolved_logical_token_counts() max_logical = (int(self.attention_mask.sum(-1).max()) if self.attention_mask is not None else max(logical, default=0)) garbage = max(int(self.token_count) - int(max_logical), 0) context_pressure = ( int(max_context_tokens) > 0 and int(self.token_count) + int(current_tokens) + int(auxiliary_tokens) > int(max_context_tokens) ) wasteful = ( garbage >= int(minimum_garbage_tokens) and garbage >= float(garbage_ratio) * max(int(self.token_count), 1) ) if context_pressure or wasteful: return self.compact_invalid_tokens() return 0 @property def sequence_lengths(self) -> tuple[int, ...]: return self.logical_window_counts @property def sequence_length(self) -> int: if len(set(self.logical_window_counts)) > 1: raise RuntimeError( "Batched Qwen cache has different logical source lengths; " "use sequence_lengths" ) return ( self.logical_window_counts[0] if self.logical_window_counts else 0 ) def detach(self) -> "QwenStreamingKVCache": from psi.data.seed.qwen_streaming_audio import ( detach_transformers_cache_, ) detach_transformers_cache_(self.past_key_values) if self.attention_mask is not None: self.attention_mask = self.attention_mask.detach() if self.qa_cache is not None: self.qa_cache.detach() return self SemanticKVCache = StreamingKVCache | QwenStreamingKVCache @dataclass class FullDuplexPsi0Output(BaseOutput): action: torch.Tensor # Optional defaults are required by Accelerate's output fp32 wrapper, # which reconstructs Mapping subclasses from one dictionary argument. control_logits: Optional[torch.Tensor] = None kv_cache: Optional[SemanticKVCache] = None memory_token: Optional[torch.Tensor] = None control_probabilities: Optional[torch.Tensor] = None control_condition_weights: Optional[torch.Tensor] = None control_entropy: Optional[torch.Tensor] = None motion_duration_log1p_seconds: Optional[torch.Tensor] = None motion_duration_logits: Optional[torch.Tensor] = None latest_instruction_loss: Optional[torch.Tensor] = None latest_instruction_token_correct: Optional[torch.Tensor] = None latest_instruction_token_count: Optional[torch.Tensor] = None instruction_memory_loss: Optional[torch.Tensor] = None instruction_memory_token_correct: Optional[torch.Tensor] = None instruction_memory_token_count: Optional[torch.Tensor] = None swa_reply_loss: Optional[torch.Tensor] = None swa_qa_loss_sum: Optional[torch.Tensor] = None swa_qa_token_correct: Optional[torch.Tensor] = None swa_qa_token_count: Optional[torch.Tensor] = None control_token_anchor_loss: Optional[torch.Tensor] = None @dataclass(frozen=True) class FullDuplexStreamCondition: """One immutable streaming condition shared by every flow solver step.""" vlm_hidden_states: torch.Tensor attention_mask: torch.Tensor control_logits: torch.Tensor control_probabilities: torch.Tensor control_condition_weights: torch.Tensor control_entropy: torch.Tensor control_condition_token: torch.Tensor control_condition_mode: str kv_cache: SemanticKVCache memory_token: torch.Tensor motion_duration_log1p_seconds: Optional[torch.Tensor] = None motion_duration_logits: Optional[torch.Tensor] = None latest_instruction_loss: Optional[torch.Tensor] = None latest_instruction_token_correct: Optional[torch.Tensor] = None latest_instruction_token_count: Optional[torch.Tensor] = None instruction_memory_loss: Optional[torch.Tensor] = None instruction_memory_token_correct: Optional[torch.Tensor] = None instruction_memory_token_count: Optional[torch.Tensor] = None class StreamingCausalMemoryLayer(nn.Module): def __init__(self, hidden_dim: int, num_heads: int, dropout: float) -> None: super().__init__() if hidden_dim % num_heads != 0: raise ValueError( f"streaming_memory_dim={hidden_dim} must be divisible by heads={num_heads}" ) self.hidden_dim = int(hidden_dim) self.num_heads = int(num_heads) self.head_dim = self.hidden_dim // self.num_heads self.norm1 = nn.LayerNorm(self.hidden_dim) self.q_proj = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False) self.k_proj = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False) self.v_proj = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False) self.out_proj = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False) self.norm2 = nn.LayerNorm(self.hidden_dim) self.ff = nn.Sequential( nn.Linear(self.hidden_dim, self.hidden_dim * 4), nn.GELU(), nn.Dropout(float(dropout)), nn.Linear(self.hidden_dim * 4, self.hidden_dim), ) self.dropout = float(dropout) def _split_heads(self, value: torch.Tensor) -> torch.Tensor: batch, length, _ = value.shape return value.view(batch, length, self.num_heads, self.head_dim).transpose(1, 2) def forward( self, token: torch.Tensor, past_key: Optional[torch.Tensor], past_value: Optional[torch.Tensor], *, append_cache: bool, valid_mask: Optional[torch.Tensor], max_cache_tokens: int, retention_indices: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: normalised = self.norm1(token) query = self._split_heads(self.q_proj(normalised)) current_key = self._split_heads(self.k_proj(normalised)) current_value = self._split_heads(self.v_proj(normalised)) if append_cache: key = current_key if past_key is None else torch.cat([past_key, current_key], dim=-2) value = current_value if past_value is None else torch.cat([past_value, current_value], dim=-2) if retention_indices is not None: if ( retention_indices.ndim != 2 or retention_indices.shape[0] != key.shape[0] ): raise ValueError( "Per-source cache retention indices must be (B,T): " f"indices={tuple(retention_indices.shape)} key={tuple(key.shape)}" ) gather_index = retention_indices[:, None, :, None].expand( key.shape[0], key.shape[1], retention_indices.shape[1], key.shape[-1], ) key = torch.gather(key, dim=-2, index=gather_index) value = torch.gather(value, dim=-2, index=gather_index) elif max_cache_tokens > 0 and key.shape[-2] > max_cache_tokens: key = key[..., -max_cache_tokens:, :] value = value[..., -max_cache_tokens:, :] next_key, next_value = key, value else: if past_key is None or past_value is None: raise RuntimeError( "A non-updating padding row cannot query an empty streaming KV cache" ) key, value = past_key, past_value next_key, next_value = past_key, past_value attention_mask = None if valid_mask is not None: if valid_mask.ndim != 2 or valid_mask.shape != (key.shape[0], key.shape[-2]): raise ValueError( "Streaming cache mask must match batched K/V time dimensions: " f"mask={tuple(valid_mask.shape)} key={tuple(key.shape)}" ) attention_mask = valid_mask[:, None, None, :] attended = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=self.dropout if self.training else 0.0, is_causal=False, ) attended = attended.transpose(1, 2).reshape(token.shape[0], token.shape[1], self.hidden_dim) token = token + self.out_proj(attended) token = token + self.ff(self.norm2(token)) return token, next_key, next_value class StreamingCausalMemory(nn.Module): def __init__( self, *, input_dim: int, hidden_dim: int, num_layers: int, num_heads: int, dropout: float, max_cache_tokens: int, ) -> None: super().__init__() self.num_layers = int(num_layers) self.max_cache_tokens = int(max_cache_tokens) self.input_proj = nn.Linear(int(input_dim), int(hidden_dim)) # Keep the historical parameter name for checkpoint compatibility. # It is now used only for padded batch rows whose losses are masked and # whose token is excluded from the per-source semantic cache. self.motion_query = nn.Parameter(torch.empty(1, 1, int(hidden_dim))) nn.init.normal_(self.motion_query, std=0.02) self.layers = nn.ModuleList( [ StreamingCausalMemoryLayer(hidden_dim, num_heads, dropout) for _ in range(self.num_layers) ] ) self.final_norm = nn.LayerNorm(int(hidden_dim)) def begin_source(self, source_ids: list[str] | tuple[str, ...]) -> StreamingKVCache: return StreamingKVCache.empty(source_ids, self.num_layers) @staticmethod def _normalise_update_mask( update_cache: bool | torch.Tensor | Sequence[bool], *, batch_size: int, device: torch.device, ) -> tuple[tuple[bool, ...], torch.Tensor]: if isinstance(update_cache, bool): values = (bool(update_cache),) * int(batch_size) elif isinstance(update_cache, torch.Tensor): flat = update_cache.detach().to(device="cpu", dtype=torch.bool).reshape(-1) values = tuple(bool(value) for value in flat.tolist()) else: values = tuple(bool(value) for value in update_cache) if len(values) != int(batch_size): raise ValueError( "Per-source cache update mask must match batch size: " f"mask={len(values)} batch={batch_size}" ) return values, torch.tensor(values, device=device, dtype=torch.bool) @staticmethod def _per_source_retention( valid_mask: torch.Tensor, *, max_cache_tokens: int, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: """Keep the newest valid tokens independently for every source row. Batched sources can finish at different times. A simple shared ``[..., -K:, :]`` slice would make a finished row lose one real token whenever another row appends a new token. This compaction preserves ``min(real_windows, K)`` tokens per source while retaining dense K/V tensors and a matching validity mask. """ limit = int(max_cache_tokens) if limit <= 0 or valid_mask.shape[-1] <= limit: return valid_mask, None width = int(valid_mask.shape[-1]) positions = torch.arange( width, device=valid_mask.device, dtype=torch.long, ).unsqueeze(0).expand(valid_mask.shape[0], -1) # A valid position is retained iff at most K valid positions occur at # or after it. This selects the newest K real observations per row. rank_from_right = ( valid_mask.to(dtype=torch.long) .flip(dims=(-1,)) .cumsum(dim=-1) .flip(dims=(-1,)) ) keep = valid_mask & (rank_from_right <= limit) # All kept positions outrank placeholders. Flipping top-k's descending # result puts placeholders first and retained observations in temporal # order, i.e. valid tokens are right-aligned under the returned mask. priority = torch.where(keep, positions + width, positions) retention_indices = torch.topk( priority, k=limit, dim=-1, largest=True, sorted=True, ).indices.flip(dims=(-1,)) compact_mask = torch.gather(keep, dim=-1, index=retention_indices) return compact_mask, retention_indices def forward( self, pooled_clip: torch.Tensor, cache: StreamingKVCache, *, source_ids: list[str] | tuple[str, ...], update_cache: bool | torch.Tensor | Sequence[bool], ) -> tuple[torch.Tensor, StreamingKVCache, torch.Tensor]: expected_ids = tuple(str(value) for value in source_ids) if expected_ids != cache.source_ids: raise RuntimeError( "Refusing to reuse streaming KV across sources: " f"cache={cache.source_ids}, request={expected_ids}" ) if pooled_clip.shape[0] != len(expected_ids): raise ValueError( f"Batch/source id mismatch: batch={pooled_clip.shape[0]} ids={len(expected_ids)}" ) update_values, update_mask = self._normalise_update_mask( update_cache, batch_size=pooled_clip.shape[0], device=pooled_clip.device, ) append_cache = any(update_values) previous_mask = cache.valid_mask first_key = cache.keys[0] if cache.keys else None previous_width = 0 if first_key is None else int(first_key.shape[-2]) retention_indices: Optional[torch.Tensor] = None if append_cache: if previous_mask is None and all(update_values): # The dense single-source/common-phase path stays mask-free, so # inference and batch=1 retain their original SDPA kernel. next_valid_mask = None else: if previous_mask is None: previous_mask = torch.ones( (pooled_clip.shape[0], previous_width), device=pooled_clip.device, dtype=torch.bool, ) next_valid_mask = torch.cat( [previous_mask.to(device=pooled_clip.device), update_mask[:, None]], dim=-1, ) if next_valid_mask is not None: next_valid_mask, retention_indices = self._per_source_retention( next_valid_mask, max_cache_tokens=self.max_cache_tokens, ) else: if first_key is None: raise RuntimeError( "A non-updating padding row cannot query an empty streaming KV cache" ) next_valid_mask = ( None if previous_mask is None else previous_mask.to(device=pooled_clip.device) ) input_token = self.input_proj(pooled_clip).unsqueeze(1) motion_token = self.motion_query.expand(pooled_clip.shape[0], -1, -1) token = torch.where( update_mask[:, None, None], input_token, motion_token, ) next_keys: list[Optional[torch.Tensor]] = [] next_values: list[Optional[torch.Tensor]] = [] for layer_index, layer in enumerate(self.layers): token, key, value = layer( token, cache.keys[layer_index], cache.values[layer_index], append_cache=append_cache, valid_mask=next_valid_mask, max_cache_tokens=self.max_cache_tokens, retention_indices=retention_indices, ) next_keys.append(key) next_values.append(value) next_cache = StreamingKVCache( cache.source_ids, tuple(next_keys), tuple(next_values), next_valid_mask, ) return self.final_norm(token), next_cache, input_token class FullDuplexPsi0Model(Psi0Model): """Psi0 action head with source-scoped continuous-audio memory and control.""" _CONTROL_CONDITION_MODES = frozenset( {"soft", "hard", "straight_through"} ) def __init__(self, model_cfg, vlm_model: Optional[nn.Module]): super().__init__(model_cfg=model_cfg, vlm_model=vlm_model) input_dim = int(model_cfg.view_feature_dim) memory_dim = int(model_cfg.streaming_memory_dim) self._staged_model_cfg = model_cfg self.streaming_qwen_enabled = bool( getattr(model_cfg, "streaming_qwen_enabled", False) ) self.state_aware_serial_control = bool( getattr(model_cfg, "state_aware_serial_control", False) ) if self.streaming_qwen_enabled: if self.vlm_model is None: raise ValueError( "HFDv2 streaming_qwen_enabled requires a loaded " "Qwen2.5-Omni thinker" ) self.streaming_memory = None self.memory_to_vlm = None if self.state_aware_serial_control: self.control_state_encoder = nn.Sequential( nn.LayerNorm(int(model_cfg.odim)), nn.Linear(int(model_cfg.odim), memory_dim), nn.GELU(), ) self.control_rtc_encoder = nn.Sequential( nn.LayerNorm(int(model_cfg.action_dim)), nn.Linear(int(model_cfg.action_dim), memory_dim), nn.GELU(), ) self.control_progress_encoder = nn.Sequential( nn.Linear(5, memory_dim), nn.GELU(), ) # IDs 0..2 are , , ; ID 3 is the # source-start sentinel where no previous decision exists. self.previous_control_embedding = nn.Embedding(4, memory_dim) self.control_head = nn.Sequential( nn.LayerNorm(input_dim + memory_dim * 4 + 1), nn.Linear(input_dim + memory_dim * 4 + 1, memory_dim), nn.GELU(), nn.Linear(memory_dim, 3), ) else: self.control_state_encoder = None self.control_rtc_encoder = None self.control_progress_encoder = None self.previous_control_embedding = None self.control_head = nn.Sequential( nn.LayerNorm(input_dim), nn.Linear(input_dim, memory_dim), nn.GELU(), nn.Linear(memory_dim, 3), ) else: self.streaming_memory = StreamingCausalMemory( input_dim=input_dim, hidden_dim=memory_dim, num_layers=int(model_cfg.streaming_memory_layers), num_heads=int(model_cfg.streaming_memory_heads), dropout=float(model_cfg.streaming_memory_dropout), max_cache_tokens=int(model_cfg.streaming_max_cache_tokens), ) self.memory_to_vlm = nn.Linear(memory_dim, input_dim) self.control_head = nn.Sequential( nn.LayerNorm(memory_dim * 2), nn.Linear(memory_dim * 2, memory_dim), nn.GELU(), nn.Linear(memory_dim, 3), ) # HFDv3's serial decision is fused after either semantic-memory path. if getattr(model_cfg, "control_head_type", "mlp") == "token_linear": if not self.streaming_qwen_enabled or self.state_aware_serial_control: raise ValueError("token_linear requires semantic-only native Qwen") # Three vocabulary-like output rows, not a second unused MLP. self.control_head = nn.Linear(input_dim, 3, bias=False) nn.init.normal_(self.control_head.weight, std=0.02) self.training_stage = 0 self.instruction_memory_mode = getattr(model_cfg, "instruction_memory_mode", "legacy") self.qwen_memory_window_seconds = float(getattr(model_cfg, "qwen_memory_window_seconds", 0)) # Production training and inference both consume the model's own hard # prediction. Action loss therefore trains this embedding and the # action model, while control CE alone trains ``control_head``. self.control_condition_embedding = nn.Embedding(3, input_dim) nn.init.normal_(self.control_condition_embedding.weight, std=0.02) self.control_condition_train_mode = self._validate_control_condition_mode( str(getattr(model_cfg, "control_condition_train_mode", "hard")) ) self.control_condition_inference_mode = self._validate_control_condition_mode( str(getattr(model_cfg, "control_condition_inference_mode", "hard")) ) if self.control_condition_inference_mode == "straight_through": raise ValueError( "Inference must use hard or soft control conditioning" ) self.control_condition_temperature = float( getattr(model_cfg, "control_condition_temperature", 1.0) ) if not self.control_condition_temperature > 0.0: raise ValueError("control_condition_temperature must be positive") self.predict_motion_duration = bool( getattr(model_cfg, "predict_motion_duration", False) ) self.motion_duration_default_seconds = float( getattr(model_cfg, "motion_duration_default_seconds", 8.0) ) self.motion_duration_distribution_bins = int( getattr(model_cfg, "motion_duration_distribution_bins", 1) ) self.motion_duration_min_seconds = float( getattr(model_cfg, "motion_duration_min_seconds", 0.5) ) self.motion_duration_max_seconds = float( getattr(model_cfg, "motion_duration_max_seconds", 180.0) ) if self.predict_motion_duration: if not self.streaming_qwen_enabled: raise ValueError( "Motion-duration prediction requires Qwen native streaming" ) self.motion_duration_head = nn.Sequential( nn.LayerNorm(input_dim), nn.Linear(input_dim, memory_dim), nn.GELU(), nn.Linear( memory_dim, self.motion_duration_distribution_bins, ), ) nn.init.zeros_(self.motion_duration_head[-1].weight) if self.motion_duration_distribution_bins == 1: nn.init.constant_( self.motion_duration_head[-1].bias, math.log1p(self.motion_duration_default_seconds), ) else: centers = torch.linspace( math.log1p(self.motion_duration_min_seconds), math.log1p(self.motion_duration_max_seconds), self.motion_duration_distribution_bins, dtype=self.motion_duration_head[-1].bias.dtype, ) default = math.log1p(self.motion_duration_default_seconds) prior_logits = -0.5 * ((centers - default) / 0.35).square() with torch.no_grad(): self.motion_duration_head[-1].bias.copy_(prior_logits) else: self.motion_duration_head = None self.instruction_recency_slots = int( getattr(model_cfg, "instruction_recency_slots", 0) ) if self.instruction_recency_slots not in {0, 3}: raise ValueError( "instruction_recency_slots must be 0 or 3 " "(latest/previous/first)" ) if self.instruction_recency_slots: if not self.streaming_qwen_enabled: raise ValueError( "Instruction recency markers require Qwen native streaming" ) self.instruction_recency_embedding = nn.Embedding( self.instruction_recency_slots, input_dim, ) # Match the successful frozen-KV A/B initialization: the marker # begins neutral, while the complete natural-language query still # provides the retrieval task. Each slot can then learn a distinct # relative-turn selector. nn.init.zeros_(self.instruction_recency_embedding.weight) else: self.instruction_recency_embedding = None if self.instruction_memory_mode == "relative_window": from psi.models.relative_instruction_memory import RelativeLagEmbedding self.instruction_recency_embedding = RelativeLagEmbedding(input_dim) self.full_source_gradient_checkpointing = False @classmethod def _validate_control_condition_mode(cls, mode: str) -> str: resolved = str(mode).strip().lower() if resolved not in cls._CONTROL_CONDITION_MODES: raise ValueError( "control condition mode must be one of " f"{sorted(cls._CONTROL_CONDITION_MODES)}, got {mode!r}" ) return resolved def set_control_condition_inference_mode(self, mode: str) -> None: """Select hard or soft predicted-control conditioning at inference.""" resolved = self._validate_control_condition_mode(mode) if resolved == "straight_through": raise ValueError( "Inference must use hard or soft control conditioning" ) self.control_condition_inference_mode = resolved def motion_duration_log1p_centers( self, *, device: torch.device | None = None, dtype: torch.dtype = torch.float32, ) -> torch.Tensor: """Return the fixed ordered centers of the duration distribution.""" bins = int(getattr(self, "motion_duration_distribution_bins", 1)) if bins <= 1: raise RuntimeError("Scalar duration regression has no bin centers") return torch.linspace( math.log1p(float(self.motion_duration_min_seconds)), math.log1p(float(self.motion_duration_max_seconds)), bins, device=device, dtype=dtype, ) def motion_duration_quantile_log1p( self, logits: torch.Tensor, quantile: float, ) -> torch.Tensor: """Decode a categorical duration distribution without averaging modes.""" if logits.ndim < 1 or logits.shape[-1] != int( self.motion_duration_distribution_bins ): raise ValueError( "Duration logits do not match configured bins: " f"shape={tuple(logits.shape)} " f"bins={self.motion_duration_distribution_bins}" ) q = float(quantile) if not 0.0 <= q <= 1.0: raise ValueError(f"Duration quantile must be in [0,1], got {q}") cumulative = logits.float().softmax(dim=-1).cumsum(dim=-1) indices = (cumulative < q).sum(dim=-1).clamp_max( logits.shape[-1] - 1 ) centers = self.motion_duration_log1p_centers( device=logits.device, dtype=torch.float32, ) return centers[indices] def _active_control_condition_mode( self, override: Optional[str] = None, ) -> str: if override is not None: return self._validate_control_condition_mode(override) return ( self.control_condition_train_mode if self.training else self.control_condition_inference_mode ) def _build_control_condition( self, control_logits: torch.Tensor, *, mode: Optional[str] = None, ) -> tuple[ torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, str, ]: """Convert the model's own control prediction into an action token.""" resolved_mode = self._active_control_condition_mode(mode) soft_probabilities = torch.softmax( control_logits.float() / self.control_condition_temperature, dim=-1, ) entropy = -( soft_probabilities * soft_probabilities.clamp_min( torch.finfo(torch.float32).tiny ).log() ).sum(dim=-1) if resolved_mode == "soft": action_probabilities = soft_probabilities else: hard_probabilities = F.one_hot( soft_probabilities.argmax(dim=-1), num_classes=self.control_condition_embedding.num_embeddings, ).to(dtype=soft_probabilities.dtype) if resolved_mode == "straight_through": action_probabilities = ( hard_probabilities + soft_probabilities - soft_probabilities.detach() ) else: action_probabilities = hard_probabilities control_token = torch.matmul( action_probabilities, self.control_condition_embedding.weight.float(), ).unsqueeze(1) return ( soft_probabilities, action_probabilities, entropy, control_token, resolved_mode, ) def train(self, mode: bool = True): """Keep frozen Qwen modules deterministic while tuning only the tail. The shipped thinker/audio configuration has zero attention, activation, and encoder-layer dropout. We still enforce module modes explicitly so a future checkpoint/config change cannot silently create train/infer drift in the frozen audio tower or lower decoder layers. """ super().train(mode) if self.streaming_qwen_enabled and self.vlm_model is not None: self.vlm_model.eval() if mode: layers = self.vlm_model.model.layers trainable_last = int( getattr( self, "_streaming_qwen_trainable_last_layers", 0, ) or 0 ) if trainable_last > 0: for layer in layers[-trainable_last:]: layer.train(True) if bool( getattr( self, "_streaming_qwen_train_final_norm", False, ) ): self.vlm_model.model.norm.train(True) return self def configure_streaming_qwen_train_mode( self, *, trainable_last_layers: int, train_final_norm: bool, ) -> None: self._streaming_qwen_trainable_last_layers = int( trainable_last_layers ) self._streaming_qwen_train_final_norm = bool(train_final_norm) self.train(self.training) def enable_full_source_gradient_checkpointing(self) -> None: """Recompute the action path without cutting the recurrent KV graph.""" self.full_source_gradient_checkpointing = True def begin_source( self, source_ids: list[str] | tuple[str, ...], ) -> SemanticKVCache: if self.streaming_qwen_enabled: return QwenStreamingKVCache.empty(source_ids) if self.streaming_memory is None: raise RuntimeError("Legacy streaming memory is not initialized") return self.streaming_memory.begin_source(source_ids) @classmethod def from_streaming_checkpoint( cls, run_dir: str | os.PathLike[str], ckpt_step: str | int, launch_config, device: str | torch.device, ) -> "FullDuplexPsi0Model": """Load base Qwen once, then overlay the fused trainable delta.""" from diffusers.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, ) from safetensors.torch import load_file from transformers import ( Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration, ) from psi.data.seed.qwen_streaming_audio import ( hfdv2_train_infer_contract, ) run_path = Path(run_dir).expanduser().resolve() checkpoint_name = str(ckpt_step) if checkpoint_name in {"best", "best_ckpt"}: checkpoint_dir = run_path / "checkpoints" / "best_ckpt" elif checkpoint_name.startswith("ckpt_"): checkpoint_dir = run_path / "checkpoints" / checkpoint_name else: checkpoint_dir = run_path / "checkpoints" / f"ckpt_{checkpoint_name}" if not checkpoint_dir.is_dir(): fallback = run_path / checkpoint_dir.name if fallback.is_dir(): checkpoint_dir = fallback single_file = checkpoint_dir / "model.safetensors" manifest_file = checkpoint_dir / "hfd_fused_delta_manifest.json" if not single_file.is_file() or not manifest_file.is_file(): raise FileNotFoundError( "Fused checkpoint requires model.safetensors and " f"hfd_fused_delta_manifest.json under {checkpoint_dir}" ) state_dict = load_file(str(single_file), device="cpu") manifest = json.loads(manifest_file.read_text(encoding="utf-8")) if manifest.get("format") != "hfd_qwen_serial_delta_v1": raise RuntimeError( "Unsupported fused checkpoint format: " f"{manifest.get('format')}" ) configured_backbone = str(launch_config.model.model_name_or_path) override = os.environ.get("PSI_PRETRAIN_BACKBONE_PATH", "").strip() backbone = Path(override or configured_backbone).expanduser() if backbone.is_absolute() and not (backbone / "config.json").is_file(): raise FileNotFoundError( f"HFDv2 base Qwen is incomplete: {backbone}/config.json" ) from psi.models.relative_instruction_memory import architecture_contract active_contract = hfdv2_train_infer_contract( architecture_options=architecture_contract(launch_config.model), full_duplex_semantics_version=( launch_config.train.full_duplex_semantics_version ), backbone=backbone, continuous_audio_root=launch_config.data.continuous_audio_root, background_assignment_path=( launch_config.data.background_assignment_path ), qwen_trainable_last_layers=( launch_config.model.streaming_qwen_trainable_last_layers ), qwen_train_final_norm=( launch_config.model.streaming_qwen_train_final_norm ), qwen_kv_gradient_horizon_windows=( launch_config.train.qwen_kv_gradient_horizon_windows ), dense_window_accumulation_steps=int( getattr( launch_config.train, "dense_window_accumulation_steps", 1, ) ), train_microbatch_per_gpu=( launch_config.train.train_batch_size ), source_accumulation_groups=( launch_config.train.source_accumulation_groups ), audio_sampling_rate=launch_config.data.audio_sampling_rate, audio_window_seconds=launch_config.data.audio_window_seconds, audio_stride_frames=( launch_config.data.audio_sample_step_frames ), action_sample_step_frames=( launch_config.data.action_sample_step_frames ), motion_fps=launch_config.data.motion_fps, action_chunk_frames=launch_config.model.action_chunk_size, action_execute_frames=( launch_config.model.action_exec_horizon ), control_condition_train_mode=( launch_config.model.control_condition_train_mode ), control_condition_inference_mode=( launch_config.model.control_condition_inference_mode ), control_condition_temperature=( launch_config.model.control_condition_temperature ), joint_velocity_loss_weight=( launch_config.train.joint_velocity_loss_weight ), joint_acceleration_loss_weight=( launch_config.train.joint_acceleration_loss_weight ), root_continuity_loss_weight=( launch_config.train.root_continuity_loss_weight ), stationary_root_height_loss_weight=( launch_config.train.stationary_root_height_loss_weight ), session_min_turns=getattr( launch_config.data, "session_min_turns", 1, ), session_max_turns=getattr( launch_config.data, "session_max_turns", 1, ), session_voice_variants=getattr( launch_config.data, "session_voice_variants", 0, ), interruption_sessions=getattr( launch_config.data, "interruption_sessions", False, ), interruption_ratio=getattr( launch_config.data, "interruption_ratio", 0.0, ), interruption_min_start_seconds=getattr( launch_config.data, "interruption_min_start_seconds", 2.0, ), interruption_max_start_seconds=getattr( launch_config.data, "interruption_max_start_seconds", 6.0, ), interruption_min_remaining_seconds=getattr( launch_config.data, "interruption_min_remaining_seconds", 1.5, ), initial_idle_max_steps=getattr( launch_config.data, "initial_idle_max_steps", 0, ), stationary_blend_frames=getattr( launch_config.data, "stationary_blend_frames", 20, ), turn_transition_frames=getattr( launch_config.data, "turn_transition_frames", 20, ), causal_listen_timing=getattr( launch_config.data, "causal_listen_timing", False, ), max_speech_onset_steps=getattr( launch_config.data, "max_speech_onset_steps", 0, ), listen_reaction_steps=getattr( launch_config.data, "listen_reaction_steps", 0, ), motion_duration_prediction=bool( getattr(launch_config.model, "predict_motion_duration", False) ), motion_duration_default_seconds=float( getattr( launch_config.model, "motion_duration_default_seconds", 8.0, ) ), motion_duration_distribution_bins=int( getattr( launch_config.model, "motion_duration_distribution_bins", 1, ) ), motion_duration_min_seconds=float( getattr( launch_config.model, "motion_duration_min_seconds", 0.5, ) ), motion_duration_max_seconds=float( getattr( launch_config.model, "motion_duration_max_seconds", 180.0, ) ), motion_duration_loss_weight=float( getattr( launch_config.train, "motion_duration_loss_weight", 0.0, ) ), latest_instruction_loss_weight=float( getattr( launch_config.train, "latest_instruction_loss_weight", 0.0, ) ), latest_instruction_metadata_path=getattr( launch_config.data, "metadata_jsonl", None, ), instruction_recency_slots=int( getattr( launch_config.model, "instruction_recency_slots", 0, ) ), instruction_memory_loss_weight=float( getattr( launch_config.train, "instruction_memory_loss_weight", 0.0, ) ), max_stream_steps=int( getattr(launch_config.data, "max_stream_steps", 1923) ), state_aware_serial_control=bool( getattr( launch_config.model, "state_aware_serial_control", False, ) ), independent_task_pools=bool( getattr(launch_config.data, "independent_task_pools", False) ), task_session_variants={ "turn-taking": int( getattr( launch_config.data, "turn_taking_session_variants", 0, ) ), "interruption": int( getattr( launch_config.data, "command_interruption_session_variants", 0, ) ), "emergency-stop": int( getattr( launch_config.data, "emergency_stop_session_variants", 0, ) ), "safe-hold": int( getattr( launch_config.data, "safe_hold_session_variants", 0, ) ), "backchannel": int( getattr( launch_config.data, "backchannel_session_variants", 0, ) ), "hesitation": int( getattr( launch_config.data, "hesitation_session_variants", 0, ) ), "pause": int( getattr( launch_config.data, "pause_session_variants", 0, ) ), }, interruption_task_session_ratio=float( getattr( launch_config.data, "interruption_task_session_ratio", 0.0, ) ), control_task_session_ratio=float( getattr( launch_config.data, "control_task_session_ratio", 0.0, ) ), backchannel_task_session_ratio=float( getattr( launch_config.data, "backchannel_task_session_ratio", 0.0, ) ), hesitation_task_window_ratio=float( getattr( launch_config.data, "hesitation_task_window_ratio", 0.0, ) ), pause_task_window_ratio=float( getattr( launch_config.data, "pause_task_window_ratio", 0.0, ) ), control_followup_ratio=float( getattr(launch_config.data, "control_followup_ratio", 0.0) ), control_post_idle_min_steps=int( getattr( launch_config.data, "control_post_idle_min_steps", 1, ) ), control_post_idle_max_steps=int( getattr( launch_config.data, "control_post_idle_max_steps", 0, ) ), control_post_idle_sampling=str( getattr( launch_config.data, "control_post_idle_sampling", "legacy_uniform", ) ), control_min_start_seconds=float( getattr( launch_config.data, "control_min_start_seconds", 2.0, ) ), control_max_start_seconds=float( getattr( launch_config.data, "control_max_start_seconds", 6.0, ) ), control_min_remaining_seconds=float( getattr( launch_config.data, "control_min_remaining_seconds", 0.4, ) ), control_activity_aware_timing=bool( getattr( launch_config.data, "control_activity_aware_timing", False, ) ), backchannel_command_manifest=getattr( launch_config.data, "backchannel_command_manifest", None, ), hesitation_pause_manifest=getattr( launch_config.data, "hesitation_pause_manifest", None, ), new_task_max_rows_per_instruction_train=int( getattr( launch_config.data, "new_task_max_rows_per_instruction_train", 32, ) ), new_task_max_rows_per_instruction_val=int( getattr( launch_config.data, "new_task_max_rows_per_instruction_val", 8, ) ), hesitation_filled_ratio=float( getattr( launch_config.data, "hesitation_filled_ratio", 0.60, ) ), self_repair_implicit_ratio=float( getattr( launch_config.data, "self_repair_implicit_ratio", 0.0, ) ), backchannel_min_start_seconds=float( getattr( launch_config.data, "backchannel_min_start_seconds", 1.2, ) ), backchannel_max_start_seconds=float( getattr( launch_config.data, "backchannel_max_start_seconds", 6.0, ) ), backchannel_min_remaining_seconds=float( getattr( launch_config.data, "backchannel_min_remaining_seconds", 1.2, ) ), filter_known_tts_translation_mismatches=bool( getattr( launch_config.data, "filter_known_tts_translation_mismatches", False, ) ), canonical_motion_manifest=getattr( launch_config.data, "canonical_motion_manifest", "", ), control_command_manifest=getattr( launch_config.data, "control_command_manifest", None, ), safe_hold_stability_window_frames=int( getattr( launch_config.data, "safe_hold_stability_window_frames", 20, ) ), safe_stationary_anchors=bool( getattr(launch_config.data, "safe_stationary_anchors", False) ), stationary_idle_residual_retargeting=bool( getattr( launch_config.data, "stationary_idle_residual_retargeting", False, ) ), dynamic_task_entry_blend=bool( getattr(launch_config.data, "dynamic_task_entry_blend", False) ), stationary_listen_hold=bool(getattr(launch_config.data, "stationary_listen_hold", False)), long_session_drop_ratio=float( getattr(launch_config.data, "long_session_drop_ratio", 0.0) ), stop_enter_loss_weight=float( getattr(launch_config.train, "stop_enter_loss_weight", 0.0) ), stop_stay_loss_weight=float( getattr(launch_config.train, "stop_stay_loss_weight", 0.0) ), stop_release_loss_weight=float( getattr(launch_config.train, "stop_release_loss_weight", 0.0) ), ) if manifest.get("contract") != active_contract: raise RuntimeError( "Fused inference contract mismatch: Qwen, prompt, raw data, " "trainable tail, KV horizon, effective batch, RTC geometry, " "or serial-control policy differs from training" ) force_float32 = os.environ.get("PSI_INFER_FLOAT32", "0") == "1" dtype = torch.float32 if force_float32 else torch.bfloat16 attention = ( "sdpa" if force_float32 or not str(device).startswith("cuda") else "flash_attention_2" ) qwen = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( str(backbone), attn_implementation=attention, dtype=dtype, ) qwen.config.use_cache = True if hasattr(qwen, "gradient_checkpointing_disable"): qwen.gradient_checkpointing_disable() model = cls(model_cfg=launch_config.model, vlm_model=qwen) layers = model.vlm_model.model.layers last_layers = int( launch_config.model.streaming_qwen_trainable_last_layers ) train_final_norm = bool( launch_config.model.streaming_qwen_train_final_norm ) qa = getattr(model, "qa_model", None) qa_delta = set() if qa is None else {"qa_model." + k for k in qa.delta_keys} expected_delta = { name for name in model.state_dict() if name in qa_delta or (not name.startswith("qa_model.") and (not name.startswith("vlm_model.") or any( name.startswith(f"vlm_model.model.layers.{index}.") for index in range(len(layers) - last_layers, len(layers)) ) or ( train_final_norm and name.startswith("vlm_model.model.norm.") ))) } if set(state_dict) != expected_delta: raise RuntimeError( "Fused delta/model mismatch: " f"missing={sorted(expected_delta.difference(state_dict))} " f"unexpected={sorted(set(state_dict).difference(expected_delta))}" ) if getattr(launch_config.model, "qwen_trainable_master_fp32", False): for layer in layers[-last_layers:]: layer.float() if train_final_norm: model.vlm_model.model.norm.float() _, unexpected = model.load_state_dict(state_dict, strict=False) if unexpected: raise RuntimeError(f"Unexpected fused delta keys: {unexpected}") model.vlm_processor = Qwen2_5OmniProcessor.from_pretrained( str(backbone) ) if str(launch_config.model.noise_scheduler) != "flow": raise ValueError( "Full-duplex streaming inference currently requires the flow scheduler" ) model.noise_scheduler = FlowMatchEulerDiscreteScheduler( num_train_timesteps=int(launch_config.model.train_diffusion_steps) ) model.device = torch.device(device) # Inference-only loader: eval() does not disable parameter gradients. # Live AV validates that its online audio frontend is frozen, exactly # as the training initializer does. Full-state training resume uses # the trainer/accelerator path, not this inference loader. model.to(model.device).eval().requires_grad_(False) return model def forward( self, *, source_ids: list[str] | tuple[str, ...], kv_cache: SemanticKVCache, update_semantic_cache: bool | torch.Tensor | Sequence[bool], vlm_hidden_states: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], action_samples: torch.Tensor, known_action_prefix: Optional[torch.Tensor], states: torch.Tensor, timestep: torch.Tensor, progress: Optional[torch.Tensor] = None, future_progress: Optional[torch.Tensor] = None, control_progress: Optional[torch.Tensor] = None, previous_control_ids: Optional[torch.Tensor] = None, control_rtc_prefix: Optional[torch.Tensor] = None, control_rtc_mask: Optional[torch.Tensor] = None, stream_input_ids: Optional[torch.Tensor] = None, stream_input_features: Optional[torch.Tensor] = None, stream_feature_attention_mask: Optional[torch.Tensor] = None, latest_instruction_input_ids: Optional[torch.Tensor] = None, latest_instruction_attention_mask: Optional[torch.Tensor] = None, latest_instruction_labels: Optional[torch.Tensor] = None, latest_instruction_recency_ids: Optional[torch.Tensor] = None, instruction_memory_input_ids: Optional[torch.Tensor] = None, instruction_memory_attention_mask: Optional[torch.Tensor] = None, instruction_memory_labels: Optional[torch.Tensor] = None, instruction_memory_recency_ids: Optional[torch.Tensor] = None, reused_condition_hidden_states: Optional[torch.Tensor] = None, reused_condition_attention_mask: Optional[torch.Tensor] = None, reused_control_logits: Optional[torch.Tensor] = None, reused_control_probabilities: Optional[torch.Tensor] = None, reused_control_condition_weights: Optional[torch.Tensor] = None, reused_control_entropy: Optional[torch.Tensor] = None, reused_memory_token: Optional[torch.Tensor] = None, reused_motion_duration_log1p_seconds: Optional[torch.Tensor] = None, reused_motion_duration_logits: Optional[torch.Tensor] = None, action_active_mask: Optional[torch.Tensor] = None, ) -> FullDuplexPsi0Output: # Keep the streaming arguments on ``forward`` so DDP/Accelerate sees # every parameter use. Cache construction itself remains explicit. return self.forward_stream( source_ids=source_ids, kv_cache=kv_cache, update_semantic_cache=update_semantic_cache, vlm_hidden_states=vlm_hidden_states, attention_mask=attention_mask, action_samples=action_samples, known_action_prefix=known_action_prefix, states=states, timestep=timestep, progress=progress, future_progress=future_progress, control_progress=control_progress, previous_control_ids=previous_control_ids, control_rtc_prefix=control_rtc_prefix, control_rtc_mask=control_rtc_mask, stream_input_ids=stream_input_ids, stream_input_features=stream_input_features, stream_feature_attention_mask=stream_feature_attention_mask, latest_instruction_input_ids=latest_instruction_input_ids, latest_instruction_attention_mask=( latest_instruction_attention_mask ), latest_instruction_labels=latest_instruction_labels, latest_instruction_recency_ids=latest_instruction_recency_ids, instruction_memory_input_ids=instruction_memory_input_ids, instruction_memory_attention_mask=( instruction_memory_attention_mask ), instruction_memory_labels=instruction_memory_labels, instruction_memory_recency_ids=instruction_memory_recency_ids, reused_condition_hidden_states=reused_condition_hidden_states, reused_condition_attention_mask=reused_condition_attention_mask, reused_control_logits=reused_control_logits, reused_control_probabilities=reused_control_probabilities, reused_control_condition_weights=reused_control_condition_weights, reused_control_entropy=reused_control_entropy, reused_memory_token=reused_memory_token, reused_motion_duration_log1p_seconds=( reused_motion_duration_log1p_seconds ), reused_motion_duration_logits=reused_motion_duration_logits, action_active_mask=action_active_mask, ) @staticmethod def _masked_pool(hidden_states: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor: mask = attention_mask.to(device=hidden_states.device, dtype=hidden_states.dtype).unsqueeze(-1) denominator = mask.sum(dim=1).clamp_min(1.0) return (hidden_states * mask).sum(dim=1) / denominator def _predict_control_logits( self, semantic_summary: torch.Tensor, *, states: Optional[torch.Tensor], control_progress: Optional[torch.Tensor], previous_control_ids: Optional[torch.Tensor], control_rtc_prefix: Optional[torch.Tensor], control_rtc_mask: Optional[torch.Tensor], ) -> torch.Tensor: """Predict the serial token from deployment-causal information only.""" if not bool(getattr(self, "state_aware_serial_control", False)): return self.control_head(semantic_summary) required = { "states": states, "control_progress": control_progress, "previous_control_ids": previous_control_ids, "control_rtc_prefix": control_rtc_prefix, "control_rtc_mask": control_rtc_mask, } missing = [name for name, value in required.items() if value is None] if missing: raise ValueError( "State-aware serial control is missing causal inputs: " f"{missing}" ) assert states is not None assert control_progress is not None assert previous_control_ids is not None assert control_rtc_prefix is not None assert control_rtc_mask is not None batch_size = int(semantic_summary.shape[0]) state_rows = states[:, -1] if states.ndim == 3 else states if state_rows.ndim != 2 or state_rows.shape[0] != batch_size: raise ValueError( "Control state must have shape (B,D) or (B,1,D), got " f"{tuple(states.shape)}" ) if control_progress.shape != (batch_size, 5): raise ValueError( "Causal control progress must have shape (B,5), got " f"{tuple(control_progress.shape)}" ) previous_ids = previous_control_ids.to( device=semantic_summary.device, dtype=torch.long, ).reshape(-1) if previous_ids.shape != (batch_size,) or bool( ((previous_ids < 0) | (previous_ids > 3)).any().item() ): raise ValueError( "Previous control IDs must be B values in [0,3], got " f"{previous_ids.tolist()}" ) if ( control_rtc_prefix.ndim != 3 or control_rtc_prefix.shape[0] != batch_size or control_rtc_prefix.shape[-1] != self.action_dim ): raise ValueError( "Control RTC prefix must have shape (B,P,action_dim), got " f"{tuple(control_rtc_prefix.shape)}" ) expected_mask = control_rtc_prefix.shape[:2] if tuple(control_rtc_mask.shape) != tuple(expected_mask): raise ValueError( "Control RTC mask must match prefix frames: " f"prefix={tuple(control_rtc_prefix.shape)} " f"mask={tuple(control_rtc_mask.shape)}" ) rtc_mask = control_rtc_mask.to( device=semantic_summary.device, dtype=semantic_summary.dtype, ) if not torch.all((rtc_mask == 0) | (rtc_mask == 1)): raise ValueError("Control RTC mask must be binary") assert self.control_state_encoder is not None assert self.control_rtc_encoder is not None assert self.control_progress_encoder is not None assert self.previous_control_embedding is not None state_context = self.control_state_encoder( state_rows.to(device=semantic_summary.device) ) rtc_rows = self.control_rtc_encoder( control_rtc_prefix.to(device=semantic_summary.device) ) rtc_count = rtc_mask.sum(dim=1, keepdim=True) rtc_context = ( rtc_rows * rtc_mask.unsqueeze(-1) ).sum(dim=1) / rtc_count.clamp_min(1.0) rtc_context = torch.where( rtc_count > 0, rtc_context, torch.zeros_like(rtc_context), ) progress_context = self.control_progress_encoder( control_progress.to(device=semantic_summary.device) ) previous_context = self.previous_control_embedding(previous_ids) rtc_fraction = rtc_count / float(max(int(rtc_mask.shape[1]), 1)) control_input = torch.cat( [ semantic_summary, state_context, rtc_context, progress_context, previous_context, rtc_fraction, ], dim=-1, ) return self.control_head(control_input) def forward_stream( self, *, source_ids: list[str] | tuple[str, ...], kv_cache: SemanticKVCache, update_semantic_cache: bool | torch.Tensor | Sequence[bool], vlm_hidden_states: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], action_samples: torch.Tensor, known_action_prefix: Optional[torch.Tensor], states: torch.Tensor, timestep: torch.Tensor, progress: Optional[torch.Tensor] = None, future_progress: Optional[torch.Tensor] = None, control_progress: Optional[torch.Tensor] = None, previous_control_ids: Optional[torch.Tensor] = None, control_rtc_prefix: Optional[torch.Tensor] = None, control_rtc_mask: Optional[torch.Tensor] = None, stream_input_ids: Optional[torch.Tensor] = None, stream_input_features: Optional[torch.Tensor] = None, stream_feature_attention_mask: Optional[torch.Tensor] = None, latest_instruction_input_ids: Optional[torch.Tensor] = None, latest_instruction_attention_mask: Optional[torch.Tensor] = None, latest_instruction_labels: Optional[torch.Tensor] = None, latest_instruction_recency_ids: Optional[torch.Tensor] = None, instruction_memory_input_ids: Optional[torch.Tensor] = None, instruction_memory_attention_mask: Optional[torch.Tensor] = None, instruction_memory_labels: Optional[torch.Tensor] = None, instruction_memory_recency_ids: Optional[torch.Tensor] = None, reused_condition_hidden_states: Optional[torch.Tensor] = None, reused_condition_attention_mask: Optional[torch.Tensor] = None, reused_control_logits: Optional[torch.Tensor] = None, reused_control_probabilities: Optional[torch.Tensor] = None, reused_control_condition_weights: Optional[torch.Tensor] = None, reused_control_entropy: Optional[torch.Tensor] = None, reused_memory_token: Optional[torch.Tensor] = None, reused_motion_duration_log1p_seconds: Optional[torch.Tensor] = None, reused_motion_duration_logits: Optional[torch.Tensor] = None, action_active_mask: Optional[torch.Tensor] = None, ) -> FullDuplexPsi0Output: reuse_values = ( reused_condition_hidden_states, reused_condition_attention_mask, reused_control_logits, reused_control_probabilities, reused_control_condition_weights, reused_control_entropy, reused_memory_token, ) reuse_condition = any(value is not None for value in reuse_values) if reuse_condition: if not all(value is not None for value in reuse_values): raise ValueError( "Reused semantic condition is incomplete" ) update_values = self._normalise_update_values( update_semantic_cache, batch_size=int(action_samples.shape[0]), ) if any(update_values): raise ValueError( "A reused Qwen condition cannot also update semantic KV" ) if tuple(str(value) for value in source_ids) != kv_cache.source_ids: raise RuntimeError( "Refusing to reuse a semantic condition across source " f"boundary: cache={kv_cache.source_ids} " f"request={tuple(str(value) for value in source_ids)}" ) condition_hidden = reused_condition_hidden_states condition_mask = reused_condition_attention_mask control_logits = reused_control_logits control_probabilities = reused_control_probabilities control_condition_weights = reused_control_condition_weights control_entropy = reused_control_entropy memory_token = reused_memory_token motion_duration_log1p_seconds = ( reused_motion_duration_log1p_seconds ) motion_duration_logits = reused_motion_duration_logits if getattr(self, "training_stage", 0) == 2 and self.training: # Replay stores raw frozen Qwen features, never a frozen copy # of the trainable control embedding or duration prediction. control_probabilities, control_condition_weights, control_entropy, token, _ = self._build_control_condition(control_logits) condition_hidden = torch.cat((condition_hidden[:, :-1], token.to(condition_hidden.dtype)), dim=1) if self.motion_duration_head is not None: motion_duration_logits = self.motion_duration_head(memory_token[:, 0]) motion_duration_log1p_seconds = self.motion_duration_quantile_log1p(motion_duration_logits, 0.5) latest_instruction_loss = None latest_instruction_token_correct = None latest_instruction_token_count = None instruction_memory_loss = None instruction_memory_token_correct = None instruction_memory_token_count = None if ( bool(getattr(self, "predict_motion_duration", False)) and motion_duration_log1p_seconds is None ): raise ValueError( "Reused v16 condition has no motion-duration prediction" ) if ( int(getattr(self, "motion_duration_distribution_bins", 1)) > 1 and motion_duration_logits is None ): raise ValueError( "Reused duration-distribution condition has no logits" ) next_cache = kv_cache else: condition = self.encode_stream_input( source_ids=source_ids, kv_cache=kv_cache, update_semantic_cache=update_semantic_cache, vlm_hidden_states=vlm_hidden_states, attention_mask=attention_mask, stream_input_ids=stream_input_ids, stream_input_features=stream_input_features, stream_feature_attention_mask=stream_feature_attention_mask, latest_instruction_input_ids=latest_instruction_input_ids, latest_instruction_attention_mask=( latest_instruction_attention_mask ), latest_instruction_labels=latest_instruction_labels, latest_instruction_recency_ids=( latest_instruction_recency_ids ), instruction_memory_input_ids=instruction_memory_input_ids, instruction_memory_attention_mask=( instruction_memory_attention_mask ), instruction_memory_labels=instruction_memory_labels, instruction_memory_recency_ids=( instruction_memory_recency_ids ), states=states, control_progress=control_progress, previous_control_ids=previous_control_ids, control_rtc_prefix=control_rtc_prefix, control_rtc_mask=control_rtc_mask, ) condition_hidden = condition.vlm_hidden_states condition_mask = condition.attention_mask control_logits = condition.control_logits control_probabilities = condition.control_probabilities control_condition_weights = condition.control_condition_weights control_entropy = condition.control_entropy memory_token = condition.memory_token motion_duration_log1p_seconds = ( condition.motion_duration_log1p_seconds ) motion_duration_logits = condition.motion_duration_logits latest_instruction_loss = condition.latest_instruction_loss latest_instruction_token_correct = ( condition.latest_instruction_token_correct ) latest_instruction_token_count = ( condition.latest_instruction_token_count ) instruction_memory_loss = condition.instruction_memory_loss instruction_memory_token_correct = ( condition.instruction_memory_token_correct ) instruction_memory_token_count = ( condition.instruction_memory_token_count ) next_cache = condition.kv_cache if condition_hidden is None or condition_mask is None: raise RuntimeError("Full-duplex action condition is missing") # Keep this off the public BaseOutput: Accelerate converts returned # BF16 tensors to FP32, which would duplicate this reusable condition # and double its storage. The trainer detaches this private reference # immediately after backward before the next optimizer step. self._latest_reusable_condition = { "condition_hidden_states": condition_hidden, "condition_attention_mask": condition_mask, "control_logits": control_logits, "control_probabilities": control_probabilities, "control_condition_weights": control_condition_weights, "control_entropy": control_entropy, "memory_token": memory_token, } if motion_duration_log1p_seconds is not None: self._latest_reusable_condition[ "motion_duration_log1p_seconds" ] = motion_duration_log1p_seconds if motion_duration_logits is not None: self._latest_reusable_condition[ "motion_duration_logits" ] = motion_duration_logits action_args = ( condition_hidden, condition_mask, action_samples, known_action_prefix, states, timestep, progress, future_progress, ) compact_action_args = action_args active_indices: Optional[torch.Tensor] = None if action_active_mask is not None: active_mask = action_active_mask.to( device=action_samples.device, dtype=torch.bool, ).reshape(-1) batch_size = int(action_samples.shape[0]) if active_mask.numel() != batch_size: raise ValueError( "Action active-row mask must match the streaming batch: " f"mask={active_mask.numel()} batch={batch_size}" ) if not bool(active_mask.any().item()): raise ValueError( "At least one action row must remain active so every DDP " "rank traverses the trainable action graph" ) if not bool(active_mask.all().item()): active_indices = active_mask.nonzero( as_tuple=False ).flatten() compact_action_args = tuple( None if value is None else value.index_select(0, active_indices) for value in action_args ) if getattr(self, "training_stage", 0) == 1: action = torch.zeros_like(action_samples) + control_logits.sum().to(action_samples.dtype) * 0.0 active_indices = None elif ( self.training and torch.is_grad_enabled() and self.full_source_gradient_checkpointing ): # Only the 500M-parameter action path is recomputed in backward. # The streaming-memory tensors remain ordinary graph-connected # inputs, so a late action loss can still reach early KV writes. action = checkpoint( self._forward_action_tensors, *compact_action_args, use_reentrant=False, preserve_rng_state=True, ) else: action = self._forward_action_tensors(*compact_action_args) if active_indices is not None: # Qwen's native DynamicCache stays bound to the complete source # batch, but the 500M action path need not evaluate rows whose # source has already ended. Scatter zeros back only to preserve # the public full-batch shape; every inactive loss weight is zero. # Allocate from the action output rather than action_samples: # autocast can produce BF16 actions from FP32 flow samples, while # index_copy requires destination and source dtypes to match. action = torch.index_copy( action.new_zeros( (int(action_samples.shape[0]), *action.shape[1:]) ), 0, active_indices, action, ) if ( self.training and getattr(self, "instruction_recency_embedding", None) is not None ): # Historical text targets are sparse and do not necessarily land # on every DDP rank in the same optimizer step. Keep the marker # parameter in every rank's graph with an exactly-zero numerical # dependency; ranks that own a real target still contribute the # ordinary non-zero marker gradient through the side readout. action = action + ( sum(p.sum() for p in self.instruction_recency_embedding.parameters()).to( dtype=action.dtype ) * 0.0 ) control_token_anchor_loss = None semantic_anchor = getattr(self, "control_token_semantic_anchor", None) if semantic_anchor is not None: state_rows = self.control_condition_embedding.weight.float() control_token_anchor_loss = 1.0 - F.cosine_similarity( state_rows, semantic_anchor.float().to(state_rows.device), dim=-1, ).mean() return FullDuplexPsi0Output( control_token_anchor_loss=control_token_anchor_loss, action=action, control_logits=control_logits, kv_cache=next_cache, memory_token=memory_token, control_probabilities=control_probabilities, control_condition_weights=control_condition_weights, control_entropy=control_entropy, motion_duration_log1p_seconds=motion_duration_log1p_seconds, motion_duration_logits=motion_duration_logits, latest_instruction_loss=latest_instruction_loss, latest_instruction_token_correct=( latest_instruction_token_correct ), latest_instruction_token_count=latest_instruction_token_count, instruction_memory_loss=instruction_memory_loss, instruction_memory_token_correct=( instruction_memory_token_correct ), instruction_memory_token_count=instruction_memory_token_count, ) def encode_stream_input( self, *, source_ids: list[str] | tuple[str, ...], kv_cache: SemanticKVCache, update_semantic_cache: bool | torch.Tensor | Sequence[bool], vlm_hidden_states: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], stream_input_ids: Optional[torch.Tensor] = None, stream_input_features: Optional[torch.Tensor] = None, stream_feature_attention_mask: Optional[torch.Tensor] = None, latest_instruction_input_ids: Optional[torch.Tensor] = None, latest_instruction_attention_mask: Optional[torch.Tensor] = None, latest_instruction_labels: Optional[torch.Tensor] = None, latest_instruction_recency_ids: Optional[torch.Tensor] = None, instruction_memory_input_ids: Optional[torch.Tensor] = None, instruction_memory_attention_mask: Optional[torch.Tensor] = None, instruction_memory_labels: Optional[torch.Tensor] = None, instruction_memory_recency_ids: Optional[torch.Tensor] = None, control_condition_mode: Optional[str] = None, states: Optional[torch.Tensor] = None, control_progress: Optional[torch.Tensor] = None, previous_control_ids: Optional[torch.Tensor] = None, control_rtc_prefix: Optional[torch.Tensor] = None, control_rtc_mask: Optional[torch.Tensor] = None, ) -> FullDuplexStreamCondition: """Consume one 0.6 s input exactly once and return a reusable condition.""" if self.streaming_qwen_enabled: if not isinstance(kv_cache, QwenStreamingKVCache): raise TypeError( "HFDv2 expected QwenStreamingKVCache, got " f"{type(kv_cache).__name__}" ) if ( stream_input_ids is None or stream_input_features is None or stream_feature_attention_mask is None or attention_mask is None ): raise ValueError( "HFDv2 online streaming requires input ids, compact audio " "features, feature mask and current token mask" ) return self._encode_qwen_stream_input( source_ids=source_ids, cache=kv_cache, update_semantic_cache=update_semantic_cache, input_ids=stream_input_ids, current_attention_mask=attention_mask, input_features=stream_input_features, feature_attention_mask=stream_feature_attention_mask, latest_instruction_input_ids=( latest_instruction_input_ids ), latest_instruction_attention_mask=( latest_instruction_attention_mask ), latest_instruction_labels=latest_instruction_labels, latest_instruction_recency_ids=( latest_instruction_recency_ids ), instruction_memory_input_ids=instruction_memory_input_ids, instruction_memory_attention_mask=( instruction_memory_attention_mask ), instruction_memory_labels=instruction_memory_labels, instruction_memory_recency_ids=( instruction_memory_recency_ids ), control_condition_mode=control_condition_mode, states=states, control_progress=control_progress, previous_control_ids=previous_control_ids, control_rtc_prefix=control_rtc_prefix, control_rtc_mask=control_rtc_mask, ) if not isinstance(kv_cache, StreamingKVCache): raise TypeError( "Legacy HFD expected StreamingKVCache, got " f"{type(kv_cache).__name__}" ) if vlm_hidden_states is None or attention_mask is None: raise ValueError( "Cached-hidden HFD requires vlm_hidden_states and attention_mask" ) if vlm_hidden_states.ndim != 3: raise ValueError( f"vlm_hidden_states must be (B,S,D), got {tuple(vlm_hidden_states.shape)}" ) if attention_mask.shape != vlm_hidden_states.shape[:2]: raise ValueError( f"attention mask {tuple(attention_mask.shape)} does not match hidden states " f"{tuple(vlm_hidden_states.shape)}" ) pooled_clip = self._masked_pool(vlm_hidden_states, attention_mask) if self.streaming_memory is None or self.memory_to_vlm is None: raise RuntimeError("Legacy streaming memory is not initialized") memory_token, next_cache, current_token = self.streaming_memory( pooled_clip, kv_cache, source_ids=source_ids, update_cache=update_semantic_cache, ) memory_condition = self.memory_to_vlm(memory_token) conditioned_hidden = torch.cat( [vlm_hidden_states, memory_condition.to(vlm_hidden_states.dtype)], dim=1 ) conditioned_mask = torch.cat( [ attention_mask, torch.ones( (attention_mask.shape[0], 1), dtype=attention_mask.dtype, device=attention_mask.device, ), ], dim=1, ) control_logits = self.control_head( torch.cat([memory_token, current_token], dim=-1).squeeze(1) ) ( control_probabilities, control_condition_weights, control_entropy, control_condition_token, resolved_condition_mode, ) = self._build_control_condition( control_logits, mode=control_condition_mode, ) conditioned_hidden = torch.cat( [ conditioned_hidden, control_condition_token.to(vlm_hidden_states.dtype), ], dim=1, ) conditioned_mask = torch.cat( [ conditioned_mask, torch.ones( (attention_mask.shape[0], 1), dtype=attention_mask.dtype, device=attention_mask.device, ), ], dim=1, ) return FullDuplexStreamCondition( vlm_hidden_states=conditioned_hidden, attention_mask=conditioned_mask, control_logits=control_logits, control_probabilities=control_probabilities, control_condition_weights=control_condition_weights, control_entropy=control_entropy, control_condition_token=control_condition_token, control_condition_mode=resolved_condition_mode, kv_cache=next_cache, memory_token=memory_token, motion_duration_log1p_seconds=None, ) @staticmethod def _normalise_update_values( update_semantic_cache: bool | torch.Tensor | Sequence[bool], *, batch_size: int, ) -> tuple[bool, ...]: if isinstance(update_semantic_cache, bool): values = (bool(update_semantic_cache),) * int(batch_size) elif isinstance(update_semantic_cache, torch.Tensor): values = tuple( bool(value) for value in update_semantic_cache.detach() .to(device="cpu", dtype=torch.bool) .reshape(-1) .tolist() ) else: values = tuple(bool(value) for value in update_semantic_cache) if len(values) != int(batch_size): raise ValueError( "Qwen cache update mask must match batch: " f"mask={len(values)} batch={batch_size}" ) return values def _split_mixed_source_prefix( self, *, input_ids: torch.Tensor, attention_mask: torch.Tensor, logical_token_counts: Sequence[int], update_values: Sequence[bool], ) -> Optional[ tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] ]: """Factor one shared prompt out of a mixed refill Qwen batch. A recycled source begins with the same fixed system/user prefix as every other source, while continuing rows append only the audio token suffix. Sending their right-padded union through Qwen would make all continuing rows pay for the long prompt. Return one shared prefix, audio-only rows and the rows that need the prefix. Initial batches and ordinary inference deliberately retain the historical single forward. """ if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape: raise ValueError("Mixed-prefix Qwen inputs must share shape (B,T)") counts = tuple(int(value) for value in logical_token_counts) updates = tuple(bool(value) for value in update_values) if len(counts) != input_ids.shape[0] or len(updates) != input_ids.shape[0]: raise ValueError("Mixed-prefix metadata must match the Qwen batch") first_rows = [ row for row, (active, count) in enumerate(zip(updates, counts)) if active and count == 0 ] continuing_rows = [ row for row, (active, count) in enumerate(zip(updates, counts)) if active and count > 0 ] if not first_rows or not continuing_rows: return None text_config = getattr( self.vlm_model.config, "text_config", self.vlm_model.config, ) if self.training and float( getattr(text_config, "attention_dropout", 0.0) ) != 0.0: # Sharing one stochastic prefix would correlate dropout masks that # were independent in the historical dense batch. return None audio_start_id = getattr( self.vlm_model.config, "audio_start_token_id", None, ) if audio_start_id is None: return None audio_start_id = int(audio_start_id) audio_end_id = getattr( self.vlm_model.config, "audio_end_token_id", None, ) if audio_end_id is not None: audio_end_id = int(audio_end_id) valid_lengths = attention_mask.to(dtype=torch.long).sum(dim=-1).tolist() prefix_ids: Optional[torch.Tensor] = None suffix_rows: dict[int, torch.Tensor] = {} for row, (active, valid_length) in enumerate( zip(updates, valid_lengths) ): valid_length = int(valid_length) if active: if valid_length <= 0 or not bool( attention_mask[row, :valid_length].bool().all().item() ) or bool( attention_mask[row, valid_length:].bool().any().item() ): raise RuntimeError( "Qwen source-refill rows must be left-packed before " f"prompt factoring: row={row} valid={valid_length}" ) valid_ids = input_ids[row, :valid_length] elif audio_end_id is not None: # Inactive rows still carry shape-compatible audio features. # Recover their placeholder suffix through audio_eos so prompt # factoring cannot make ids/features disagree on a short tail. end_matches = input_ids[row].eq(audio_end_id).nonzero( as_tuple=False ).flatten() if int(end_matches.numel()) != 1: continue valid_ids = input_ids[row, : int(end_matches[0].item()) + 1] else: continue audio_starts = valid_ids.eq(audio_start_id).nonzero( as_tuple=False ).flatten() if int(audio_starts.numel()) != 1: raise RuntimeError( "Every active Qwen source window must contain exactly one " f"audio start token: row={row} matches={audio_starts.numel()}" ) audio_offset = int(audio_starts[0].item()) if active and counts[row] == 0: if audio_offset <= 0: raise RuntimeError( "A recycled source must restart with the fixed prompt" ) candidate = valid_ids[:audio_offset] if prefix_ids is None: prefix_ids = candidate elif not torch.equal(candidate, prefix_ids): raise RuntimeError( "Recycled sources disagree on the shared Qwen prompt" ) elif active and audio_offset != 0: raise RuntimeError( "A continuing source unexpectedly repeated the Qwen prompt" ) suffix_rows[row] = valid_ids[audio_offset:] if prefix_ids is None or int(prefix_ids.numel()) <= 0: return None suffix_template = next(iter(suffix_rows.values())) max_suffix = max(int(value.numel()) for value in suffix_rows.values()) suffix_ids = torch.zeros( (input_ids.shape[0], max_suffix), device=input_ids.device, dtype=input_ids.dtype, ) suffix_mask = torch.zeros_like(suffix_ids, dtype=torch.long) for row in range(input_ids.shape[0]): suffix = suffix_rows.get(row, suffix_template) length = int(suffix.numel()) suffix_ids[row, :length] = suffix if updates[row]: suffix_mask[row, :length] = 1 first_indices = torch.tensor( first_rows, device=input_ids.device, dtype=torch.long, ) return prefix_ids, suffix_ids, suffix_mask, first_indices def _install_shared_qwen_prefix( self, *, cache: QwenStreamingKVCache, prefix_ids: torch.Tensor, first_indices: torch.Tensor, ) -> torch.Tensor: """Install one freshly computed prompt KV into selected reset rows.""" if cache.past_key_values is None or cache.attention_mask is None: raise RuntimeError( "Mixed source refill requires an initialized dense Qwen cache" ) prefix_ids = prefix_ids.reshape(1, -1) prefix_tokens = int(prefix_ids.shape[1]) if prefix_tokens <= 0 or int(cache.token_count) < prefix_tokens: raise RuntimeError( "A recycled Qwen slot has insufficient physical storage for " f"the shared prompt: cache={cache.token_count} prompt={prefix_tokens}" ) prefix_mask = torch.ones_like(prefix_ids, dtype=torch.long) prefix_positions = torch.arange( prefix_tokens, device=prefix_ids.device, dtype=torch.long, ) prefix_output = self.vlm_model.model( attention_mask=prefix_mask, position_ids=prefix_positions.view(1, 1, -1).expand(3, 1, -1), past_key_values=None, inputs_embeds=self.vlm_model.get_input_embeddings()(prefix_ids), use_cache=True, output_attentions=False, output_hidden_states=False, return_dict=True, cache_position=prefix_positions, ) prefix_layers = getattr(prefix_output.past_key_values, "layers", None) cache_layers = getattr(cache.past_key_values, "layers", None) if prefix_layers is None or cache_layers is None or len(prefix_layers) != len( cache_layers ): raise RuntimeError("Qwen shared-prefix cache layers do not align") replacement_count = int(first_indices.numel()) if replacement_count <= 0: raise ValueError("Shared Qwen prefix has no replacement rows") for cache_layer, prefix_layer in zip(cache_layers, prefix_layers): cache_keys = getattr(cache_layer, "keys", None) cache_values = getattr(cache_layer, "values", None) prefix_keys = getattr(prefix_layer, "keys", None) prefix_values = getattr(prefix_layer, "values", None) if not all( isinstance(value, torch.Tensor) for value in ( cache_keys, cache_values, prefix_keys, prefix_values, ) ): raise RuntimeError("Initialized Qwen prefix/cache layer has no K/V") if ( tuple(cache_keys.shape[1:2] + cache_keys.shape[3:]) != tuple(prefix_keys.shape[1:2] + prefix_keys.shape[3:]) or int(prefix_keys.shape[0]) != 1 or int(prefix_keys.shape[-2]) != prefix_tokens ): raise RuntimeError( "Qwen shared-prefix K/V geometry differs from the source cache" ) selected_keys = cache_keys.index_select(0, first_indices) selected_values = cache_values.index_select(0, first_indices) replacement_keys = torch.cat( [ prefix_keys.expand(replacement_count, -1, -1, -1), selected_keys[..., prefix_tokens:, :], ], dim=-2, ) replacement_values = torch.cat( [ prefix_values.expand(replacement_count, -1, -1, -1), selected_values[..., prefix_tokens:, :], ], dim=-2, ) cache_layer.keys = torch.index_copy( cache_keys, 0, first_indices, replacement_keys, ) cache_layer.values = torch.index_copy( cache_values, 0, first_indices, replacement_values, ) cache.attention_mask = cache.attention_mask.clone() cache.attention_mask[first_indices] = 0 cache.attention_mask[first_indices, :prefix_tokens] = 1 if cache.token_birth_windows is not None: cache.token_birth_windows = cache.token_birth_windows.clone() cache.token_birth_windows[first_indices] = -2 cache.token_birth_windows[first_indices, :prefix_tokens] = -1 logical_counts = list(cache.resolved_logical_token_counts()) for row in first_indices.detach().cpu().tolist(): logical_counts[int(row)] = prefix_tokens cache.logical_token_counts = tuple(logical_counts) return prefix_output.last_hidden_state def _latest_instruction_teacher_forcing( self, *, past_key_values: Any, past_attention_mask: torch.Tensor, past_token_count: int, past_position_counts: Sequence[int], input_ids: torch.Tensor, attention_mask: torch.Tensor, labels: torch.Tensor, recency_slot_ids: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Decode the exact latest TTS text from a read-only Qwen-KV branch. Only rows containing supervised answer tokens are selected. Their KV is copied into a short-lived side cache, so teacher forcing can append a text query without changing the production audio cache consumed by the next stream window. The Qwen LM head remains frozen; gradients update the already-trainable final decoder layers and final norm. """ if not ( input_ids.ndim == attention_mask.ndim == labels.ndim == 2 and input_ids.shape == attention_mask.shape == labels.shape ): raise ValueError( "Latest-instruction tensors must share shape (B,T): " f"ids={tuple(input_ids.shape)} " f"mask={tuple(attention_mask.shape)} " f"labels={tuple(labels.shape)}" ) if input_ids.shape[0] != past_attention_mask.shape[0]: raise ValueError( "Latest-instruction batch does not match Qwen KV attention: " f"text={input_ids.shape[0]} past={past_attention_mask.shape[0]}" ) if len(tuple(past_position_counts)) != int(input_ids.shape[0]): raise ValueError( "Latest-instruction logical positions must match the Qwen " f"batch: positions={len(tuple(past_position_counts))} " f"batch={input_ids.shape[0]}" ) marker_enabled = getattr( self, "instruction_recency_embedding", None, ) is not None if marker_enabled: if recency_slot_ids is None or recency_slot_ids.shape != ( input_ids.shape[0], ): raise ValueError( "Recency-aware instruction readout requires one slot id " f"per Qwen row: slots={None if recency_slot_ids is None else tuple(recency_slot_ids.shape)} " f"batch={input_ids.shape[0]}" ) relative = getattr(self, "instruction_memory_mode", "legacy") == "relative_window" if relative and bool((recency_slot_ids < 1).any()): raise ValueError("Relative instruction number must be >=1") if not relative and bool( ( (recency_slot_ids < 0) | (recency_slot_ids >= self.instruction_recency_slots) ).any().item() ): raise ValueError( "Instruction recency slot ids must be within " f"[0,{self.instruction_recency_slots})" ) if not torch.all((attention_mask == 0) | (attention_mask == 1)): raise ValueError("Latest-instruction attention mask must be binary") if bool(((labels != -100) & (attention_mask == 0)).any().item()): raise ValueError( "Latest-instruction labels cannot supervise padded tokens" ) target_rows = (labels != -100).any(dim=-1) batch_size = int(input_ids.shape[0]) zero = torch.zeros( batch_size, device=input_ids.device, dtype=torch.float32, ) if not bool(target_rows.any().item()): return zero, zero.clone(), zero.clone() row_indices = target_rows.nonzero(as_tuple=False).flatten() cache_layers = getattr(past_key_values, "layers", None) if cache_layers is None: raise TypeError( "Latest-instruction supervision requires a Transformers " "DynamicCache with explicit layers" ) cache_data = [] for layer in cache_layers: keys = getattr(layer, "keys", None) values = getattr(layer, "values", None) if keys is None or values is None: raise RuntimeError( "Latest-instruction side cache encountered an uninitialized " "Qwen layer" ) cache_data.append( ( keys.index_select(0, row_indices), values.index_select(0, row_indices), ) ) try: side_cache = past_key_values.__class__(ddp_cache_data=cache_data) except TypeError: # Transformers 4.57 DynamicCache accepts the keyword above. Keep a # positional fallback for lightweight compatible test doubles. side_cache = past_key_values.__class__(cache_data) selected_ids = input_ids.index_select(0, row_indices) selected_mask = attention_mask.index_select(0, row_indices).long() selected_labels = labels.index_select(0, row_indices).long() selected_recency_slots = ( recency_slot_ids.index_select(0, row_indices).long() if marker_enabled and recency_slot_ids is not None else None ) selected_past_mask = past_attention_mask.index_select( 0, row_indices, ).long() full_attention_mask = torch.cat( [selected_past_mask, selected_mask], dim=-1, ) current_tokens = int(selected_ids.shape[1]) + int(marker_enabled) if marker_enabled: marker_mask = torch.ones( (len(row_indices), 1), device=selected_mask.device, dtype=selected_mask.dtype, ) selected_mask = torch.cat([marker_mask, selected_mask], dim=-1) selected_labels = torch.cat( [ torch.full( (len(row_indices), 1), -100, device=selected_labels.device, dtype=selected_labels.dtype, ), selected_labels, ], dim=-1, ) full_attention_mask = torch.cat( [selected_past_mask, selected_mask], dim=-1, ) text_config = getattr( self.vlm_model.config, "text_config", self.vlm_model.config, ) max_context_tokens = int( getattr(text_config, "max_position_embeddings", 0) ) if ( max_context_tokens > 0 and int(past_token_count) + current_tokens > max_context_tokens ): raise RuntimeError( "Latest-instruction side readout exceeds the Qwen context: " f"past={past_token_count} text={current_tokens} " f"limit={max_context_tokens}" ) cache_position = torch.arange( int(past_token_count), int(past_token_count) + current_tokens, device=input_ids.device, dtype=torch.long, ) selected_position_counts = torch.tensor( tuple(int(value) for value in past_position_counts), device=input_ids.device, dtype=torch.long, ).index_select(0, row_indices) logical_positions = selected_position_counts[:, None] + torch.arange( current_tokens, device=input_ids.device, dtype=torch.long, )[None, :] position_ids = logical_positions.unsqueeze(0).expand(3, -1, -1) inputs_embeds = self.vlm_model.get_input_embeddings()(selected_ids) if marker_enabled: if selected_recency_slots is None: raise AssertionError("Missing selected instruction recency slots") marker = self.instruction_recency_embedding( selected_recency_slots ).unsqueeze(1) inputs_embeds = torch.cat( [marker.to(dtype=inputs_embeds.dtype), inputs_embeds], dim=1, ) outputs = self.vlm_model.model( attention_mask=full_attention_mask, position_ids=position_ids, past_key_values=side_cache, inputs_embeds=inputs_embeds, use_cache=True, output_attentions=False, output_hidden_states=False, return_dict=True, cache_position=cache_position, ) hidden = outputs.last_hidden_state shifted_labels = selected_labels[:, 1:] supervised = shifted_labels.ne(-100) prediction_hidden = hidden[:, :-1][supervised] flat_labels = shifted_labels[supervised] if prediction_hidden.numel() == 0: raise RuntimeError( "Latest-instruction target rows contain no predictable token" ) lm_head = getattr(self.vlm_model, "lm_head", None) if lm_head is None: raise RuntimeError("Qwen thinker has no language-model head") def token_statistics( hidden_chunk: torch.Tensor, label_chunk: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: logits = lm_head(hidden_chunk).float() return ( F.cross_entropy(logits, label_chunk, reduction="none"), logits.argmax(dim=-1).eq(label_chunk).float(), ) # The frozen Qwen vocabulary head is large. Bound its temporary # logits to 64 target tokens and recompute them during backward instead # of materializing B*text_length*vocab activations at once. loss_chunks: list[torch.Tensor] = [] correct_chunks: list[torch.Tensor] = [] for start in range(0, int(flat_labels.numel()), 64): hidden_chunk = prediction_hidden[start : start + 64] label_chunk = flat_labels[start : start + 64] if torch.is_grad_enabled() and hidden_chunk.requires_grad: chunk_loss, chunk_correct = checkpoint( token_statistics, hidden_chunk, label_chunk, use_reentrant=False, ) else: chunk_loss, chunk_correct = token_statistics( hidden_chunk, label_chunk, ) loss_chunks.append(chunk_loss) correct_chunks.append(chunk_correct) token_loss = torch.cat(loss_chunks, dim=0) token_correct = torch.cat(correct_chunks, dim=0) selected_token_rows = supervised.nonzero(as_tuple=False)[:, 0] selected_loss_sum = torch.zeros( len(row_indices), device=input_ids.device, dtype=torch.float32, ).scatter_add(0, selected_token_rows, token_loss) selected_correct_sum = torch.zeros_like(selected_loss_sum).scatter_add( 0, selected_token_rows, token_correct, ) selected_count = torch.zeros_like(selected_loss_sum).scatter_add( 0, selected_token_rows, torch.ones_like(token_loss), ) # Keep token sums and counts separate so an uneven DDP batch computes # the standard global token-level language-model cross entropy. per_sample_loss = zero.scatter(0, row_indices, selected_loss_sum) per_sample_correct = zero.scatter( 0, row_indices, selected_correct_sum, ) per_sample_count = zero.scatter(0, row_indices, selected_count) return per_sample_loss, per_sample_correct, per_sample_count def _encode_qwen_stream_input( self, *, source_ids: list[str] | tuple[str, ...], cache: QwenStreamingKVCache, update_semantic_cache: bool | torch.Tensor | Sequence[bool], input_ids: torch.Tensor, current_attention_mask: torch.Tensor, input_features: torch.Tensor, feature_attention_mask: torch.Tensor, latest_instruction_input_ids: Optional[torch.Tensor] = None, latest_instruction_attention_mask: Optional[torch.Tensor] = None, latest_instruction_labels: Optional[torch.Tensor] = None, latest_instruction_recency_ids: Optional[torch.Tensor] = None, instruction_memory_input_ids: Optional[torch.Tensor] = None, instruction_memory_attention_mask: Optional[torch.Tensor] = None, instruction_memory_labels: Optional[torch.Tensor] = None, instruction_memory_recency_ids: Optional[torch.Tensor] = None, control_condition_mode: Optional[str] = None, states: Optional[torch.Tensor] = None, control_progress: Optional[torch.Tensor] = None, previous_control_ids: Optional[torch.Tensor] = None, control_rtc_prefix: Optional[torch.Tensor] = None, control_rtc_mask: Optional[torch.Tensor] = None, ) -> FullDuplexStreamCondition: """Append one native Qwen audio segment to the source DynamicCache.""" expected_ids = tuple(str(value) for value in source_ids) if expected_ids != cache.source_ids: raise RuntimeError( "Refusing to reuse Qwen KV across sources: " f"cache={cache.source_ids} request={expected_ids}" ) if input_ids.ndim != 2: raise ValueError( f"stream_input_ids must be (B,S), got {tuple(input_ids.shape)}" ) batch_size, current_tokens = input_ids.shape if batch_size != len(expected_ids): raise ValueError( f"Qwen batch/source mismatch: batch={batch_size} ids={len(expected_ids)}" ) if current_attention_mask.shape != input_ids.shape: raise ValueError( "Current Qwen attention mask must match input ids: " f"mask={tuple(current_attention_mask.shape)} " f"ids={tuple(input_ids.shape)}" ) update_values = self._normalise_update_values( update_semantic_cache, batch_size=batch_size, ) if not any(update_values): raise RuntimeError( "An all-padding rank must skip the Qwen forward instead of " "calling it with no real source windows" ) if not torch.all( (current_attention_mask == 0) | (current_attention_mask == 1) ): raise ValueError("Current Qwen attention mask must be binary") if any( bool(active) and int(current_attention_mask[row].sum().item()) <= 0 for row, active in enumerate(update_values) ): raise ValueError("Every active Qwen stream row needs a real token") logical_token_counts = cache.resolved_logical_token_counts() mixed_prefix = self._split_mixed_source_prefix( input_ids=input_ids, attention_mask=current_attention_mask, logical_token_counts=logical_token_counts, update_values=update_values, ) shared_prefix_hidden: Optional[torch.Tensor] = None shared_prefix_rows: Optional[torch.Tensor] = None shared_prefix_tokens = 0 if mixed_prefix is not None: ( shared_prefix_ids, input_ids, current_attention_mask, shared_prefix_rows, ) = mixed_prefix shared_prefix_tokens = int(shared_prefix_ids.numel()) current_tokens = int(input_ids.shape[1]) # DynamicCache is physically dense but logically source-local. Ended # rows and right padding stay invisible under this mask; a recycled # slot additionally clears its complete historical mask before the new # source begins at logical position zero. A mixed refill reuses those # now-invalid physical columns for the shared fixed prompt, then only # appends the short audio suffix for the complete batch. physical_current_mask = current_attention_mask.to( dtype=torch.long, device=input_ids.device, ) text_config = getattr( self.vlm_model.config, "text_config", self.vlm_model.config, ) max_context_tokens = int( getattr(text_config, "max_position_embeddings", 0) ) auxiliary_tokens = max( ( int(value.shape[1]) for value in ( latest_instruction_input_ids, instruction_memory_input_ids, ) if value is not None ), default=0, ) keep_windows = int(round(getattr(self, "qwen_memory_window_seconds", 0) / 0.4)) cache.expire_audio_windows(keep_windows) if auxiliary_tokens and getattr( self, "instruction_recency_embedding", None, ) is not None: auxiliary_tokens += 1 cache.compact_if_worthwhile( current_tokens=int(current_tokens), auxiliary_tokens=auxiliary_tokens, max_context_tokens=max_context_tokens, ) if mixed_prefix is not None: shared_prefix_hidden = self._install_shared_qwen_prefix( cache=cache, prefix_ids=shared_prefix_ids, first_indices=shared_prefix_rows, ) logical_token_counts = cache.resolved_logical_token_counts() full_attention_mask = ( physical_current_mask if cache.attention_mask is None else torch.cat( [ cache.attention_mask.to( device=input_ids.device, dtype=torch.long, ), physical_current_mask, ], dim=-1, ) ) if int(full_attention_mask.shape[-1]) != ( int(cache.token_count) + int(current_tokens) ): raise RuntimeError( "Qwen cache/token mask length mismatch: " f"past={cache.token_count} current={current_tokens} " f"mask={full_attention_mask.shape[-1]}" ) if ( max_context_tokens > 0 and int(full_attention_mask.shape[-1]) > max_context_tokens ): raise RuntimeError( "HFDv2 source exceeds the Qwen native context length: " f"tokens={full_attention_mask.shape[-1]} " f"limit={max_context_tokens}. Split the source at a real " "source boundary; silently truncating KV would change the " "full-duplex semantics." ) projected_logical_tokens = tuple( previous + int(current) for previous, current in zip( logical_token_counts, physical_current_mask.sum(dim=-1).tolist(), ) ) if ( max_context_tokens > 0 and keep_windows <= 0 and max(projected_logical_tokens, default=0) + auxiliary_tokens > max_context_tokens ): raise RuntimeError( "HFDv2 source exceeds the Qwen logical context length: " f"tokens={max(projected_logical_tokens, default=0)} " f"auxiliary={auxiliary_tokens} limit={max_context_tokens}. " "Split the source at a real source boundary." ) inputs_embeds = self.vlm_model.get_input_embeddings()(input_ids) audio_dtype = next(self.vlm_model.audio_tower.parameters()).dtype audio_features = self.vlm_model.get_audio_features( input_features.to(dtype=audio_dtype), feature_attention_mask=feature_attention_mask, audio_feature_lengths=None, ).to(device=inputs_embeds.device, dtype=inputs_embeds.dtype) _, _, audio_mask = self.vlm_model.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, ) if int(audio_mask.sum().item()) != int(audio_features.numel()): # audio_mask is expanded over D, so compare element counts. raise RuntimeError( "Qwen audio placeholders do not match audio tower output: " f"mask_values={audio_mask.sum().item()} " f"feature_values={audio_features.numel()}" ) inputs_embeds = inputs_embeds.masked_scatter( audio_mask, audio_features, ) cache_position = torch.arange( int(cache.token_count), int(cache.token_count) + int(current_tokens), device=input_ids.device, dtype=torch.long, ) # ``cache_position`` addresses the physical dense allocation, whereas # rotary position ids are source-local. This separation lets a new # source reuse one row without inheriting either the old semantics or # its absolute phase. logical_positions = torch.tensor( logical_token_counts, device=input_ids.device, dtype=torch.long, )[:, None] + torch.arange( int(current_tokens), device=input_ids.device, dtype=torch.long, )[None, :] position_ids = logical_positions.unsqueeze(0).expand(3, -1, -1) outputs = self.vlm_model.model( attention_mask=full_attention_mask, position_ids=position_ids, past_key_values=cache.past_key_values, inputs_embeds=inputs_embeds, use_cache=True, output_attentions=False, output_hidden_states=False, return_dict=True, cache_position=cache_position, ) hidden = outputs.last_hidden_state if hidden.shape[:2] != input_ids.shape: raise RuntimeError( "Qwen streaming hidden shape mismatch: " f"hidden={tuple(hidden.shape)} ids={tuple(input_ids.shape)}" ) condition_hidden = hidden condition_attention_mask = physical_current_mask if mixed_prefix is not None: if shared_prefix_hidden is None or shared_prefix_rows is None: raise RuntimeError("Qwen shared-prefix state is incomplete") condition_tokens = shared_prefix_tokens + int(current_tokens) condition_hidden = hidden.new_zeros( (batch_size, condition_tokens, hidden.shape[-1]) ) condition_attention_mask = physical_current_mask.new_zeros( (batch_size, condition_tokens) ) # Continuing rows retain the historical audio-first/right-padding # layout. Recycled rows receive the equivalent prompt+audio # layout reconstructed from the one shared prompt forward. condition_hidden[:, :current_tokens] = hidden condition_attention_mask[:, :current_tokens] = ( physical_current_mask ) replacement_hidden = torch.cat( [ shared_prefix_hidden.expand( int(shared_prefix_rows.numel()), -1, -1 ), hidden.index_select(0, shared_prefix_rows), ], dim=1, ) replacement_mask = torch.cat( [ torch.ones( ( int(shared_prefix_rows.numel()), shared_prefix_tokens, ), dtype=physical_current_mask.dtype, device=physical_current_mask.device, ), physical_current_mask.index_select( 0, shared_prefix_rows ), ], dim=1, ) condition_hidden = torch.index_copy( condition_hidden, 0, shared_prefix_rows, replacement_hidden, ) condition_attention_mask = torch.index_copy( condition_attention_mask, 0, shared_prefix_rows, replacement_mask, ) logical_counts = tuple( previous + int(active) for previous, active in zip( cache.logical_window_counts, update_values, ) ) next_logical_token_counts = projected_logical_tokens next_birth = None if keep_windows > 0: current_birth = torch.tensor(cache.logical_window_counts, device=input_ids.device)[:, None].expand_as(input_ids).clone() audio_start = int(self.vlm_model.config.audio_start_token_id) for row, count in enumerate(logical_token_counts): if count == 0 and update_values[row]: first_audio = input_ids[row].eq(audio_start).nonzero().flatten() if not first_audio.numel(): raise RuntimeError("First Qwen window has no audio-start token") current_birth[row, :int(first_audio[0])] = -1 current_birth.masked_fill_(~physical_current_mask.bool(), -2) if cache.token_count and cache.token_birth_windows is None: raise RuntimeError("Cannot silently enable a rolling ledger on an existing legacy KV") next_birth = current_birth if cache.token_birth_windows is None else torch.cat((cache.token_birth_windows, current_birth), -1) next_cache = QwenStreamingKVCache( source_ids=cache.source_ids, past_key_values=outputs.past_key_values, attention_mask=full_attention_mask, token_count=int(cache.token_count) + int(current_tokens), logical_window_counts=logical_counts, logical_token_counts=next_logical_token_counts, slot_rebind_count=int(cache.slot_rebind_count), compaction_count=int(cache.compaction_count), reclaimed_token_count=int(cache.reclaimed_token_count), token_birth_windows=next_birth, ) latest_instruction_tensors = ( latest_instruction_input_ids, latest_instruction_attention_mask, latest_instruction_labels, ) if any(value is not None for value in latest_instruction_tensors) and not all( value is not None for value in latest_instruction_tensors ): raise ValueError( "Latest-instruction input ids, mask and labels must be provided " "together" ) if all(value is not None for value in latest_instruction_tensors): ( latest_instruction_loss, latest_instruction_token_correct, latest_instruction_token_count, ) = self._latest_instruction_teacher_forcing( past_key_values=outputs.past_key_values, past_attention_mask=full_attention_mask, past_token_count=int(next_cache.token_count), past_position_counts=next_logical_token_counts, input_ids=latest_instruction_input_ids, attention_mask=latest_instruction_attention_mask, labels=latest_instruction_labels, recency_slot_ids=latest_instruction_recency_ids, ) else: latest_instruction_loss = None latest_instruction_token_correct = None latest_instruction_token_count = None instruction_memory_tensors = ( instruction_memory_input_ids, instruction_memory_attention_mask, instruction_memory_labels, ) if any(value is not None for value in instruction_memory_tensors) and not all( value is not None for value in instruction_memory_tensors ): raise ValueError( "Historical-instruction input ids, mask and labels must be " "provided together" ) if all(value is not None for value in instruction_memory_tensors): ( instruction_memory_loss, instruction_memory_token_correct, instruction_memory_token_count, ) = self._latest_instruction_teacher_forcing( past_key_values=outputs.past_key_values, past_attention_mask=full_attention_mask, past_token_count=int(next_cache.token_count), past_position_counts=next_logical_token_counts, input_ids=instruction_memory_input_ids, attention_mask=instruction_memory_attention_mask, labels=instruction_memory_labels, recency_slot_ids=instruction_memory_recency_ids, ) else: instruction_memory_loss = None instruction_memory_token_correct = None instruction_memory_token_count = None # The final audio-eos token is history-conditioned by Qwen causal # attention and provides one compact decision representation. The # action head still receives every current token, not this summary only. last_valid = ( physical_current_mask.sum(dim=-1).clamp_min(1) - 1 ).to(dtype=torch.long) summary = hidden.gather( dim=1, index=last_valid[:, None, None].expand( -1, 1, hidden.shape[-1], ), ) control_logits = self._predict_control_logits( summary[:, 0], states=states, control_progress=control_progress, previous_control_ids=previous_control_ids, control_rtc_prefix=control_rtc_prefix, control_rtc_mask=control_rtc_mask, ) duration_head_output = ( self.motion_duration_head(summary[:, 0]) if getattr(self, "motion_duration_head", None) is not None else None ) if duration_head_output is None: motion_duration_log1p_seconds = None motion_duration_logits = None elif int(getattr(self, "motion_duration_distribution_bins", 1)) == 1: motion_duration_log1p_seconds = duration_head_output.squeeze(-1) motion_duration_logits = None else: motion_duration_logits = duration_head_output motion_duration_log1p_seconds = self.motion_duration_quantile_log1p( motion_duration_logits, 0.5, ) ( control_probabilities, control_condition_weights, control_entropy, control_condition_token, resolved_condition_mode, ) = self._build_control_condition( control_logits, mode=control_condition_mode, ) conditioned_hidden = torch.cat( [ condition_hidden, control_condition_token.to(condition_hidden.dtype), ], dim=1, ) conditioned_mask = torch.cat( [ condition_attention_mask, torch.ones( (batch_size, 1), dtype=condition_attention_mask.dtype, device=condition_attention_mask.device, ), ], dim=1, ) return FullDuplexStreamCondition( vlm_hidden_states=conditioned_hidden, attention_mask=conditioned_mask, control_logits=control_logits, control_probabilities=control_probabilities, control_condition_weights=control_condition_weights, control_entropy=control_entropy, control_condition_token=control_condition_token, control_condition_mode=resolved_condition_mode, kv_cache=next_cache, memory_token=summary, motion_duration_log1p_seconds=motion_duration_log1p_seconds, motion_duration_logits=motion_duration_logits, latest_instruction_loss=latest_instruction_loss, latest_instruction_token_correct=( latest_instruction_token_correct ), latest_instruction_token_count=latest_instruction_token_count, instruction_memory_loss=instruction_memory_loss, instruction_memory_token_correct=( instruction_memory_token_correct ), instruction_memory_token_count=instruction_memory_token_count, ) def forward_action_conditioned( self, *, condition: FullDuplexStreamCondition, action_samples: torch.Tensor, known_action_prefix: torch.Tensor, states: torch.Tensor, timestep: torch.Tensor, progress: Optional[torch.Tensor] = None, future_progress: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Run only the action path; this never reads or mutates the KV cache.""" return self._forward_action_tensors( condition.vlm_hidden_states, condition.attention_mask, action_samples, known_action_prefix, states, timestep, progress, future_progress, ) def _forward_action_tensors( self, vlm_hidden_states: torch.Tensor, attention_mask: torch.Tensor, action_samples: torch.Tensor, known_action_prefix: torch.Tensor, states: torch.Tensor, timestep: torch.Tensor, progress: Optional[torch.Tensor], future_progress: Optional[torch.Tensor], ) -> torch.Tensor: """Tensor-only action call suitable for non-reentrant checkpointing.""" action_output = super().forward( input_ids=torch.ones_like(attention_mask, dtype=torch.long), attention_mask=attention_mask, pixel_values=None, image_grid_thw=None, vlm_hidden_states=vlm_hidden_states, action_samples=action_samples, known_action_prefix=known_action_prefix, states=states, timestep=timestep, progress=progress, future_progress=future_progress, return_dict=True, ) return action_output.action