"""Causal camera/audio extension used by formal speaking-while-acting runs.""" from __future__ import annotations import torch from torch import nn from torch.nn import functional as F from torch.utils.checkpoint import checkpoint from transformers.cache_utils import DynamicCache from psi.models.full_duplex_psi0 import ( FullDuplexPsi0Model, FullDuplexStreamCondition, QwenStreamingKVCache, ) REPLY_QUERY = ( "<|im_end|>\n<|im_start|>user\nAnswer the latest spoken visual question " "with a short phrase. Use the camera view when it was asked.<|im_end|>\n" "<|im_start|>assistant\n" ) def swa_auxiliary_losses(output, action_loss, *, required: bool): """Fail closed if mixed precision/DDP dropped any SWA supervision.""" values = {} for name in ("swa_reply_loss", "swa_qa_loss_sum", "swa_qa_token_correct", "swa_qa_token_count"): value = getattr(output, name, None) if value is None: if required: raise RuntimeError(f"Missing {name}: SWA auxiliary output was lost; refusing unsupervised SWA training") value = torch.zeros_like(action_loss) if name == "swa_reply_loss" else action_loss.sum() * 0.0 values[name] = value return values def _clone_cache(past, rows: torch.Tensor) -> DynamicCache: return DynamicCache(ddp_cache_data=[ (layer.keys.index_select(0, rows), layer.values.index_select(0, rows)) for layer in past.layers ]) class FullDuplexSwaModel(FullDuplexPsi0Model): """One HFD model that accepts either raw-audio or causal AV packets.""" def __init__(self, model_cfg, vlm_model): super().__init__(model_cfg, vlm_model) hidden = int(getattr(getattr(vlm_model.config, "text_config", vlm_model.config), "hidden_size", 2048)) self.reply_head = nn.Linear(hidden, 2) nn.init.normal_(self.reply_head.weight, std=0.01) nn.init.zeros_(self.reply_head.bias) self.swa_packets = None self.swa_compute_auxiliary = False self.reply_logits = None self.keep_tape = False self.tape = [] self.av_context_limit = min( 24576, int(getattr(getattr(vlm_model.config, "text_config", vlm_model.config), "max_position_embeddings", 32768)), ) self.qa_model = None if getattr(model_cfg, "swa_qa_mode", "shared_legacy") == "independent_online_lora": from psi.models.swa_qa_branch import OnlineQaBranch self.qa_model = OnlineQaBranch(model_cfg) # Keep the readiness/readout objective from changing the body policy. # The private QA branch has its own Qwen/KV and already isolates answer # token gradients. ``shared_legacy`` remains unchanged for old # compatibility experiments. self.swa_reply_detach_body = bool( getattr(model_cfg, "swa_reply_detach_body", True) and self.qa_model is not None ) def _reply_summary_for_loss(self, summary: torch.Tensor) -> torch.Tensor: """Read the body summary without giving SWA readiness loss body grads.""" return summary.detach() if self.swa_reply_detach_body else summary def forward(self, *args, **kwargs): output = super().forward(*args, **kwargs) zero = self.reply_head.weight.sum() * 0.0 # BaseOutput is a Mapping. New attributes alone are NOT dictionary # entries and disappear when Accelerate recursively converts outputs # to fp32. Item assignment preserves both the mapping and attributes. output["swa_reply_loss"] = zero.expand(len(kwargs["source_ids"])) output["swa_qa_loss_sum"] = zero output["swa_qa_token_correct"] = zero.detach() output["swa_qa_token_count"] = zero.detach() if self.swa_packets is not None and self.swa_compute_auxiliary: targets = getattr(self, "swa_reply_targets", None) answers = getattr(self, "swa_answer_targets", None) if targets is None or answers is None or self.reply_logits is None: raise RuntimeError("SWA forward is missing reply/answer targets") output["swa_reply_loss"] = F.cross_entropy( self.reply_logits.float(), targets, weight=torch.tensor( [1.0, float(getattr(self, "swa_reply_positive_weight", 8.0))], device=self.reply_logits.device, ), reduction="none", ) qa_loss, correct, count = self.qa_loss(output.kv_cache, answers) output["swa_qa_loss_sum"] = qa_loss output["swa_qa_token_correct"] = correct output["swa_qa_token_count"] = count return output def _encode_qwen_stream_input(self, **kw): if self.swa_packets is None: self.reply_logits = None return super()._encode_qwen_stream_input(**kw) cache = kw["cache"] source_ids = tuple(kw["source_ids"]) if source_ids != cache.source_ids: raise RuntimeError("SWA packets and Qwen cache rows disagree") packets = self.swa_packets if len(packets) != len(source_ids): raise RuntimeError("SWA packet batch does not match source rows") device = self.reply_head.weight.device batch = len(source_ids) auxiliary = max([128] + [ int(kw[name].shape[1]) + 1 for name in ("latest_instruction_input_ids", "instruction_memory_input_ids") if kw.get(name) is not None ]) from psi.models.causal_av import encode_av, prepare_av_batch if packets[0].get("live_audio", False) and not hasattr(self.vlm_model, "_hfd_av_prefix_ids"): from psi.data.seed.live_multimodal_overlay import AV_SYSTEM self.vlm_model._hfd_av_prefix_ids = tuple(self.vlm_processor.tokenizer.encode( '<|im_start|>system\n' + AV_SYSTEM + '<|im_end|>\n<|im_start|>user\n', add_special_tokens=False)) prepared = prepare_av_batch(self.vlm_model, packets, kw["update_semantic_cache"]) hidden, input_ids, embeds, current_mask, next_cache = encode_av( self.vlm_model, cache, packets, kw["update_semantic_cache"], memory_seconds=self.qwen_memory_window_seconds, context_limit=self.av_context_limit, auxiliary=auxiliary, prepared=prepared, ) full_mask, next_offsets = next_cache.attention_mask, next_cache.av_next_positions # Independent online writes, including background. Previous packets # keep numerical KV; only the current answer-boundary packet retains # a graph, matching the verified online-LoRA experiment. if self.qa_model is not None: private = cache.qa_cache or QwenStreamingKVCache.empty(source_ids) self.qa_model.vlm_processor = self.vlm_processor qa_grad = torch.is_grad_enabled() and self.training and any( getattr(self, "swa_answer_targets", ()) or ()) with torch.set_grad_enabled(qa_grad): next_cache.qa_cache = self.qa_model.encode(private, packets, kw["update_semantic_cache"], prepared=prepared) end = (current_mask.sum(-1) - 1).clamp_min(0) summary = hidden[torch.arange(batch, device=device), end][:, None, :] control = self._predict_control_logits(summary[:, 0], states=None, control_progress=None, previous_control_ids=None, control_rtc_prefix=None, control_rtc_mask=None) probs, weights, entropy, token, mode = self._build_control_condition(control, mode=kw.get("control_condition_mode")) duration = self.motion_duration_head(summary[:, 0]) self.reply_logits = self.reply_head( self._reply_summary_for_loss(summary[:, 0]) ) if self.keep_tape: if batch != 1: raise RuntimeError("Native Talker tape is inference-only and requires batch size one") self.tape.append((input_ids.detach(), embeds.detach(), hidden.detach())) aux = {} for prefix in ("latest_instruction", "instruction_memory"): if kw.get(prefix + "_input_ids") is not None: value = self._latest_instruction_teacher_forcing( past_key_values=next_cache.past_key_values, past_attention_mask=full_mask, past_token_count=next_cache.token_count, past_position_counts=next_offsets, input_ids=kw[prefix + "_input_ids"], attention_mask=kw[prefix + "_attention_mask"], labels=kw[prefix + "_labels"], recency_slot_ids=kw[prefix + "_recency_ids"], ) aux.update({prefix + "_loss": value[0], prefix + "_token_correct": value[1], prefix + "_token_count": value[2]}) return FullDuplexStreamCondition( vlm_hidden_states=torch.cat((hidden, token.to(hidden)), 1), attention_mask=torch.cat((current_mask, torch.ones(batch, 1, device=device, dtype=torch.long)), 1), control_logits=control, control_probabilities=probs, control_condition_weights=weights, control_entropy=entropy, control_condition_token=token, control_condition_mode=mode, kv_cache=next_cache, memory_token=summary, motion_duration_logits=duration, motion_duration_log1p_seconds=self.motion_duration_quantile_log1p(duration, 0.5), **aux, ) def qa_loss(self, cache, targets): private_model = getattr(self, "qa_model", None) if private_model is not None: if cache.qa_cache is None: raise RuntimeError("Independent QA has no online AV history") return private_model.qa_loss(cache.qa_cache, targets) device = self.reply_head.weight.device rows = torch.tensor([i for i, target in enumerate(targets) if target], device=device, dtype=torch.long) zero = self.reply_head.weight.sum() * 0 if rows.numel() == 0: return zero, zero.detach(), zero.detach() tokenizer = self.vlm_processor.tokenizer prefix = tokenizer.encode(REPLY_QUERY, add_special_tokens=False) full = [prefix + tokenizer.encode(targets[i], add_special_tokens=False) + [self.vlm_model.config.eos_token_id] for i in rows.tolist()] length = max(map(len, full)) ids = torch.zeros(len(full), length, device=device, dtype=torch.long) mask = torch.zeros_like(ids) labels = torch.full_like(ids, -100) for row, sequence in enumerate(full): ids[row, :len(sequence)] = torch.tensor(sequence, device=device) mask[row, :len(sequence)] = 1 labels[row, len(prefix):len(sequence)] = ids[row, len(prefix):len(sequence)] side = _clone_cache(cache.past_key_values, rows) offsets = torch.tensor(cache.av_next_positions, device=device)[rows] result = self.vlm_model.model( inputs_embeds=self.vlm_model.get_input_embeddings()(ids), attention_mask=torch.cat((cache.attention_mask[rows], mask), 1), position_ids=(offsets[:, None] + torch.arange(length, device=device)[None, :])[None].expand(3, -1, -1), past_key_values=side, cache_position=torch.arange(cache.token_count, cache.token_count + length, device=device), use_cache=True, return_dict=True, ) labels = labels[:, 1:] valid = labels != -100 states = result.last_hidden_state[:, :-1][valid] target = labels[valid] losses, correct = [], [] def token_loss(x, y): logits = self.vlm_model.lm_head(x).float() return F.cross_entropy(logits, y, reduction="sum"), (logits.argmax(-1) == y).sum().float() for start in range(0, len(target), 32): value = checkpoint(token_loss, states[start:start + 32], target[start:start + 32], use_reentrant=False) if torch.is_grad_enabled() else token_loss(states[start:start + 32], target[start:start + 32]) losses.append(value[0]); correct.append(value[1]) return torch.stack(losses).sum(), torch.stack(correct).sum(), torch.tensor(len(target), device=device, dtype=torch.float32) @torch.inference_mode() def free_answer(self, cache, row: int = 0, max_tokens: int = 48): private_model = getattr(self, "qa_model", None) if private_model is not None: if cache.qa_cache is None: raise RuntimeError("Independent QA has no online AV history") return private_model.free_answer(cache.qa_cache, row, max_tokens) device = self.reply_head.weight.device tokenizer = self.vlm_processor.tokenizer rows = torch.tensor([row], device=device) side = _clone_cache(cache.past_key_values, rows) ids = torch.tensor([tokenizer.encode(REPLY_QUERY, add_special_tokens=False)], device=device) offset = cache.av_next_positions[row] mask = torch.cat((cache.attention_mask[row:row + 1], torch.ones_like(ids)), 1) embeds = self.vlm_model.get_input_embeddings()(ids) result = self.vlm_model.model( inputs_embeds=embeds, attention_mask=mask, position_ids=(torch.arange(ids.shape[1], device=device) + offset)[None, None].expand(3, 1, -1), past_key_values=side, cache_position=torch.arange(cache.token_count, cache.token_count + ids.shape[1], device=device), use_cache=True, return_dict=True, ) offset += ids.shape[1] branch_tape = [(ids, embeds, result.last_hidden_state)] generated = [] for _ in range(max_tokens): token = self.vlm_model.lm_head(result.last_hidden_state[:, -1:]).argmax(-1) generated.append(int(token.item())) if generated[-1] == self.vlm_model.config.eos_token_id: break embed = self.vlm_model.get_input_embeddings()(token) width = side.get_seq_length() mask = torch.cat((mask, torch.ones_like(token)), 1) result = self.vlm_model.model( inputs_embeds=embed, attention_mask=mask, position_ids=torch.full((3, 1, 1), offset, device=device, dtype=torch.long), past_key_values=side, cache_position=torch.tensor([width], device=device), use_cache=True, return_dict=True, ) branch_tape.append((token, embed, result.last_hidden_state)) offset += 1 return { "text": tokenizer.decode(generated, skip_special_tokens=True), "ids": generated, "eos": bool(generated and generated[-1] == self.vlm_model.config.eos_token_id), "branch_tape": branch_tape, }