"""One causal AV packet writer shared by the body and private QA branches.""" from dataclasses import dataclass import torch from torch.nn.utils.rnn import pad_sequence from psi.models.full_duplex_psi0 import QwenStreamingKVCache @dataclass(frozen=True) class PreparedAVBatch: embeds: torch.Tensor ids: torch.Tensor positions: torch.Tensor mask: torch.Tensor active: tuple[bool, ...] lengths: tuple[int, ...] advances: tuple[int, ...] vision_starts: tuple[int, ...] prepended_state: tuple[bool, ...] = () def _prepend_pending_control_tokens(qwen, prepared): """Fuse the previous packet's predicted state with the next AV packet. The token is placed immediately before the packet and therefore has the same causal meaning as a standalone append followed by the AV forward. A1 reuses this immutable prepared batch in both branches, so the predicted interaction state is shared context rather than a second Qwen launch. """ pending = getattr(qwen, "_hfd_pending_control_token_ids", None) if pending is None: return prepared pending = tuple(None if value is None else int(value) for value in pending) batch = len(prepared.lengths) if len(pending) != batch: raise RuntimeError("Pending control-token rows do not match the AV batch") flags = tuple(value is not None for value in pending) if not any(flags): return PreparedAVBatch( prepared.embeds, prepared.ids, prepared.positions, prepared.mask, prepared.active, prepared.lengths, prepared.advances, prepared.vision_starts, flags, ) if any(flag and not active for flag, active in zip(flags, prepared.active)): raise RuntimeError("An inactive AV row cannot consume a pending state token") device = prepared.embeds.device embedding = qwen.get_input_embeddings() ids_rows, embed_rows, position_rows = [], [], [] with torch.no_grad(): for row, (length, token_id) in enumerate(zip(prepared.lengths, pending)): length = int(length) ids = prepared.ids[row, :length] embeds = prepared.embeds[row, :length] positions = prepared.positions[:, row, :length] if token_id is not None: state_id = torch.tensor([token_id], device=device, dtype=ids.dtype) state_embed = embedding(state_id).detach().to(embeds) ids = torch.cat((state_id, ids), 0) embeds = torch.cat((state_embed, embeds), 0) positions = torch.cat( (torch.zeros((3, 1), device=device, dtype=positions.dtype), positions + 1), -1, ) ids_rows.append(ids) embed_rows.append(embeds) position_rows.append(positions.T) ids = pad_sequence(ids_rows, batch_first=True) embeds = pad_sequence(embed_rows, batch_first=True) positions = pad_sequence(position_rows, batch_first=True).permute(2, 0, 1) lengths = tuple(length + int(flag) for length, flag in zip(prepared.lengths, flags)) mask = ((torch.arange(max(lengths), device=device)[None] < torch.tensor(lengths, device=device)[:, None]) & torch.tensor(prepared.active, device=device)[:, None]).long() return PreparedAVBatch( embeds, ids, positions, mask, prepared.active, lengths, tuple(value + int(flag) for value, flag in zip(prepared.advances, flags)), tuple(value + int(flag) if value >= 0 else value for value, flag in zip(prepared.vision_starts, flags)), flags, ) def prepare_av_batch(qwen, packets, active): """Collate stored frontend packets before copying to CUDA, once per batch. The body and QA read the same immutable inputs but keep separate KV and autograd graphs. Python ledgers never read one CUDA scalar per source. """ live = [bool(p.get("live_audio", False)) for p in packets] if any(live): if not all(live): raise ValueError("Cannot mix legacy frozen packets with live microphone packets") return prepare_live_av_batch(qwen, packets, active) from psi.models.interaction_arch.control_tokens import storage_weight weight = storage_weight(qwen.model.embed_tokens) count = len(packets) if isinstance(active, torch.Tensor): active = active.detach().cpu().reshape(-1).tolist() if isinstance(active, (bool, int)): active = [bool(active)] * count active = tuple(bool(x) for x in active) if len(active) != count or not count: raise ValueError("AV active mask must have one entry per packet") lengths = tuple(len(p["ids"]) for p in packets) if min(lengths) < 1: raise ValueError("AV packets must not be empty") ids_cpu = [p["ids"].detach().to(device="cpu", dtype=torch.long) for p in packets] vision_starts = [] for ids in ids_cpu: starts = (ids == qwen.config.vision_start_token_id).nonzero() vision_starts.append(int(starts[0, 0]) if starts.numel() else -1) ids = pad_sequence(ids_cpu, batch_first=True) embeds = pad_sequence([p["embeds"].detach().to(device="cpu", dtype=weight.dtype) for p in packets], batch_first=True) positions = pad_sequence([p["positions"].detach().to(device="cpu").T for p in packets], batch_first=True).permute(2, 0, 1) mask = ((torch.arange(max(lengths))[None] < torch.tensor(lengths)[:, None]) & torch.tensor(active)[:, None]).long() def copy(tensor): if weight.device.type == "cuda": tensor = tensor.pin_memory() return tensor.to(weight.device, non_blocking=True) return _prepend_pending_control_tokens(qwen, PreparedAVBatch( copy(embeds), copy(ids), copy(positions), copy(mask), active, lengths, tuple(int(p["next_position"]) for p in packets), tuple(vision_starts))) @torch.no_grad() def prepare_live_av_batch(qwen, packets, active): """Batch the frozen audio tower once; reuse independent image features. The merged microphone is encoded online, so no old command/QA audio can leak from a prerecorded source. Body and private QA consume the identical resulting embeddings. Only shape/position templates are memoized. """ from psi.models.interaction_arch.control_tokens import storage_weight weight = storage_weight(qwen.model.embed_tokens) if any(p.requires_grad for p in qwen.audio_tower.parameters()): raise RuntimeError("Live overlay input path requires a frozen audio tower") if isinstance(active, torch.Tensor): active = active.detach().cpu().reshape(-1).tolist() if isinstance(active, (bool, int)): active = [bool(active)] * len(packets) active = tuple(bool(x) for x in active) if not packets or len(active) != len(packets): raise ValueError("One active mask is required per live packet") from psi.data.seed.native_visual import resolve_visual visuals = [resolve_visual(qwen, p, a) for p, a in zip(packets, active)] prefix = getattr(qwen, "_hfd_av_prefix_ids", None) if prefix is None: raise RuntimeError("Live AV system prompt has not been bound to the tokenizer") templates = getattr(qwen, "_hfd_live_av_templates", None) if templates is None: templates = qwen._hfd_live_av_templates = {} features = torch.stack([p["input_features"].float() for p in packets]) feature_masks = torch.stack([p["feature_attention_mask"].long() for p in packets]) # Accelerate may already have placed the packets on CUDA. Shape/rope # templates are built on CPU; transfer only this tiny length vector once, # never the actual microphone features or once-per-row CUDA scalars. feature_lengths = feature_masks.sum(-1).detach().cpu() _, audio_lengths = qwen.audio_tower._get_feat_extract_output_lengths(feature_lengths) ids_list, positions_list, starts, advances = [], [], [], [] conf = qwen.config for row, packet in enumerate(packets): visual = visuals[row] nvideo = len(visual) if visual is not None else 0 count = int(packet["camera_count"]) if nvideo else 0 audio_len = int(audio_lengths[row]) key = (bool(packet["first_window"]), count, audio_len, int(feature_lengths[row])) if key not in templates: tokens = list(prefix) if packet["first_window"] else [] start = len(tokens) if nvideo: tokens += [conf.vision_start_token_id] + [conf.video_token_id] * nvideo + [conf.vision_end_token_id] tokens += [conf.audio_start_token_id] + [conf.audio_token_id] * audio_len + [conf.audio_end_token_id] ids = torch.tensor(tokens, dtype=torch.long) grid = torch.tensor([[(count + 1) // 2, 16, 16]]) if nvideo else None if nvideo and nvideo != int(grid.prod()) // 4: raise ValueError("Live visual features differ from the audited 224px camera geometry") positions, _ = qwen.get_rope_index( ids[None], video_grid_thw=grid, attention_mask=torch.ones_like(ids[None]), audio_seqlens=feature_lengths[row:row+1], second_per_grids=torch.tensor([.25]) if nvideo else None, ) templates[key] = (ids, positions[:, 0], start, int(positions.max()) + 1) ids, positions, start, advance = templates[key] ids_list.append(ids); positions_list.append(positions.T) starts.append(start); advances.append(advance) lengths = tuple(map(len, ids_list)) ids = pad_sequence(ids_list, batch_first=True).to(weight.device) positions = pad_sequence(positions_list, batch_first=True).permute(2, 0, 1).to(weight.device) mask = ((torch.arange(max(lengths))[None] < torch.tensor(lengths)[:, None]) & torch.tensor(active)[:, None]).long().to(weight.device) embeds = qwen.get_input_embeddings()(ids) audio = qwen.get_audio_features(features.to(device=weight.device, dtype=weight.dtype), feature_masks.to(weight.device)) if int(audio_lengths.sum()) != int((ids == conf.audio_token_id).sum()): raise RuntimeError("Live microphone token geometry mismatch") embeds[ids == conf.audio_token_id] = audio.to(embeds) for row, packet in enumerate(packets): visual = visuals[row] if visual is not None: embeds[row, ids[row] == conf.video_token_id] = visual.to(embeds) return _prepend_pending_control_tokens(qwen, PreparedAVBatch( embeds, ids, positions, mask, active, lengths, tuple(advances), tuple(starts))) def fit_complete_av_windows(cache, *, current_tokens, auxiliary_tokens, context_limit): """Keep the newest complete packets when the token budget beats the time cap. A 96-second camera history can exceed the model's token budget before its time-based expiration. Preserve the system prefix and absolute RoPE positions; never split an audio/video packet or reset another source row. """ budget = int(context_limit) - int(current_tokens) - int(auxiliary_tokens) if budget < 0: raise RuntimeError("A single AV packet plus readout exceeds the context budget") if cache.token_count <= budget: return birth, mask = cache.token_birth_windows, cache.attention_mask if birth is None or mask is None or birth.shape != mask.shape: raise RuntimeError("AV context pressure requires a valid packet birth ledger") valid = mask.bool() protected = valid & (birth == -1) temporal = valid & (birth >= 0) available = budget - protected.sum(-1) if (available < 0).any(): raise RuntimeError("AV system prefix plus current packet exceeds the context budget") # Reverse token count selects a suffix; if it cuts a packet, remove that # entire oldest packet rather than retain an incomplete multimodal input. suffix_count = temporal.flip(-1).long().cumsum(-1).flip(-1) candidates = temporal & (suffix_count <= available[:, None]) sentinel = torch.iinfo(birth.dtype).max oldest = torch.where(candidates, birth, sentinel).amin(-1) boundary = temporal & (birth == oldest[:, None]) partial = (boundary & candidates).sum(-1) < boundary.sum(-1) retained = candidates & (~boundary | ~partial[:, None]) cache.attention_mask = (protected | retained).to(mask.dtype) cache.compact_invalid_tokens() if cache.token_count > budget: raise RuntimeError("Whole-packet AV eviction did not satisfy context budget") def encode_av(qwen, cache, packets, active, *, memory_seconds, context_limit, auxiliary=128, prepared=None): batch = len(cache.source_ids) if len(packets) != batch: raise RuntimeError("AV packet/cache batch mismatch") from psi.models.interaction_arch.control_tokens import storage_weight device = storage_weight(qwen.model.embed_tokens).device prepared = prepared if prepared is not None else prepare_av_batch(qwen, packets, active) if len(prepared.active) != batch: raise RuntimeError("Prepared AV input/cache batch mismatch") active = prepared.active width = prepared.ids.shape[1] cache.expire_audio_windows(int(round(memory_seconds / .4))) cache.compact_if_worthwhile(current_tokens=width, auxiliary_tokens=auxiliary, max_context_tokens=context_limit) fit_complete_av_windows(cache, current_tokens=width, auxiliary_tokens=auxiliary, context_limit=context_limit) if cache.token_count + width + auxiliary > context_limit: raise RuntimeError("SWA causal AV context exceeds its safe bound") if cache.past_key_values is not None and cache.past_key_values.get_seq_length() != cache.token_count: raise RuntimeError("AV KV length differs from its ledger") embeds, mask, ids = prepared.embeds, prepared.mask, prepared.ids offsets = cache.av_next_positions or (0,) * batch next_offsets = tuple(offset + advance * flag for offset, advance, flag in zip(offsets, prepared.advances, active)) # Match old padding positions exactly, including inactive rows. real_tokens = torch.arange(width, device=device)[None] < torch.tensor(prepared.lengths, device=device)[:, None] positions = prepared.positions + torch.tensor(offsets, device=device)[None, :, None] * real_tokens[None] full_mask = mask if cache.attention_mask is None else torch.cat((cache.attention_mask, mask), -1) result = qwen.model(inputs_embeds=embeds, attention_mask=full_mask, position_ids=positions, past_key_values=cache.past_key_values, cache_position=torch.arange(cache.token_count, cache.token_count + width, device=device), use_cache=True, return_dict=True) next_cache = QwenStreamingKVCache( source_ids=cache.source_ids, past_key_values=result.past_key_values, attention_mask=full_mask, token_count=cache.token_count + width, logical_window_counts=tuple(c + int(v) for c, v in zip(cache.logical_window_counts, active)), logical_token_counts=tuple(c + n * flag for c, n, flag in zip(cache.resolved_logical_token_counts(), prepared.lengths, active)), slot_rebind_count=cache.slot_rebind_count, compaction_count=cache.compaction_count, reclaimed_token_count=cache.reclaimed_token_count, av_next_positions=tuple(next_offsets), ) birth = torch.tensor(cache.logical_window_counts, device=device)[:, None].expand(batch, width).clone() for row in range(batch): if prepared.prepended_state and prepared.prepended_state[row]: if cache.logical_window_counts[row] <= 0: raise RuntimeError("A source-start packet cannot consume stale state") birth[row, 0] = cache.logical_window_counts[row] - 1 if cache.logical_window_counts[row] == 0: start = prepared.vision_starts[row] if start >= 0: birth[row, :start] = -1 birth.masked_fill_(mask == 0, -2) next_cache.token_birth_windows = birth if cache.token_birth_windows is None else torch.cat((cache.token_birth_windows, birth), -1) return result.last_hidden_state, ids, embeds, mask, next_cache