code-HFD / src /psi /models /full_duplex_swa.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
14.9 kB
"""Causal camera/audio extension used by formal speaking-while-acting runs."""
from __future__ import annotations
import torch
from torch import nn
from torch.nn import functional as F
from torch.utils.checkpoint import checkpoint
from transformers.cache_utils import DynamicCache
from psi.models.full_duplex_psi0 import (
FullDuplexPsi0Model,
FullDuplexStreamCondition,
QwenStreamingKVCache,
)
REPLY_QUERY = (
"<|im_end|>\n<|im_start|>user\nAnswer the latest spoken visual question "
"with a short phrase. Use the camera view when it was asked.<|im_end|>\n"
"<|im_start|>assistant\n"
)
def swa_auxiliary_losses(output, action_loss, *, required: bool):
"""Fail closed if mixed precision/DDP dropped any SWA supervision."""
values = {}
for name in ("swa_reply_loss", "swa_qa_loss_sum", "swa_qa_token_correct", "swa_qa_token_count"):
value = getattr(output, name, None)
if value is None:
if required:
raise RuntimeError(f"Missing {name}: SWA auxiliary output was lost; refusing unsupervised SWA training")
value = torch.zeros_like(action_loss) if name == "swa_reply_loss" else action_loss.sum() * 0.0
values[name] = value
return values
def _clone_cache(past, rows: torch.Tensor) -> DynamicCache:
return DynamicCache(ddp_cache_data=[
(layer.keys.index_select(0, rows), layer.values.index_select(0, rows))
for layer in past.layers
])
class FullDuplexSwaModel(FullDuplexPsi0Model):
"""One HFD model that accepts either raw-audio or causal AV packets."""
def __init__(self, model_cfg, vlm_model):
super().__init__(model_cfg, vlm_model)
hidden = int(getattr(getattr(vlm_model.config, "text_config", vlm_model.config), "hidden_size", 2048))
self.reply_head = nn.Linear(hidden, 2)
nn.init.normal_(self.reply_head.weight, std=0.01)
nn.init.zeros_(self.reply_head.bias)
self.swa_packets = None
self.swa_compute_auxiliary = False
self.reply_logits = None
self.keep_tape = False
self.tape = []
self.av_context_limit = min(
24576,
int(getattr(getattr(vlm_model.config, "text_config", vlm_model.config), "max_position_embeddings", 32768)),
)
self.qa_model = None
if getattr(model_cfg, "swa_qa_mode", "shared_legacy") == "independent_online_lora":
from psi.models.swa_qa_branch import OnlineQaBranch
self.qa_model = OnlineQaBranch(model_cfg)
# Keep the readiness/readout objective from changing the body policy.
# The private QA branch has its own Qwen/KV and already isolates answer
# token gradients. ``shared_legacy`` remains unchanged for old
# compatibility experiments.
self.swa_reply_detach_body = bool(
getattr(model_cfg, "swa_reply_detach_body", True)
and self.qa_model is not None
)
def _reply_summary_for_loss(self, summary: torch.Tensor) -> torch.Tensor:
"""Read the body summary without giving SWA readiness loss body grads."""
return summary.detach() if self.swa_reply_detach_body else summary
def forward(self, *args, **kwargs):
output = super().forward(*args, **kwargs)
zero = self.reply_head.weight.sum() * 0.0
# BaseOutput is a Mapping. New attributes alone are NOT dictionary
# entries and disappear when Accelerate recursively converts outputs
# to fp32. Item assignment preserves both the mapping and attributes.
output["swa_reply_loss"] = zero.expand(len(kwargs["source_ids"]))
output["swa_qa_loss_sum"] = zero
output["swa_qa_token_correct"] = zero.detach()
output["swa_qa_token_count"] = zero.detach()
if self.swa_packets is not None and self.swa_compute_auxiliary:
targets = getattr(self, "swa_reply_targets", None)
answers = getattr(self, "swa_answer_targets", None)
if targets is None or answers is None or self.reply_logits is None:
raise RuntimeError("SWA forward is missing reply/answer targets")
output["swa_reply_loss"] = F.cross_entropy(
self.reply_logits.float(), targets,
weight=torch.tensor(
[1.0, float(getattr(self, "swa_reply_positive_weight", 8.0))],
device=self.reply_logits.device,
),
reduction="none",
)
qa_loss, correct, count = self.qa_loss(output.kv_cache, answers)
output["swa_qa_loss_sum"] = qa_loss
output["swa_qa_token_correct"] = correct
output["swa_qa_token_count"] = count
return output
def _encode_qwen_stream_input(self, **kw):
if self.swa_packets is None:
self.reply_logits = None
return super()._encode_qwen_stream_input(**kw)
cache = kw["cache"]
source_ids = tuple(kw["source_ids"])
if source_ids != cache.source_ids:
raise RuntimeError("SWA packets and Qwen cache rows disagree")
packets = self.swa_packets
if len(packets) != len(source_ids):
raise RuntimeError("SWA packet batch does not match source rows")
device = self.reply_head.weight.device
batch = len(source_ids)
auxiliary = max([128] + [
int(kw[name].shape[1]) + 1
for name in ("latest_instruction_input_ids", "instruction_memory_input_ids")
if kw.get(name) is not None
])
from psi.models.causal_av import encode_av, prepare_av_batch
if packets[0].get("live_audio", False) and not hasattr(self.vlm_model, "_hfd_av_prefix_ids"):
from psi.data.seed.live_multimodal_overlay import AV_SYSTEM
self.vlm_model._hfd_av_prefix_ids = tuple(self.vlm_processor.tokenizer.encode(
'<|im_start|>system\n' + AV_SYSTEM + '<|im_end|>\n<|im_start|>user\n',
add_special_tokens=False))
prepared = prepare_av_batch(self.vlm_model, packets, kw["update_semantic_cache"])
hidden, input_ids, embeds, current_mask, next_cache = encode_av(
self.vlm_model, cache, packets, kw["update_semantic_cache"],
memory_seconds=self.qwen_memory_window_seconds,
context_limit=self.av_context_limit, auxiliary=auxiliary, prepared=prepared,
)
full_mask, next_offsets = next_cache.attention_mask, next_cache.av_next_positions
# Independent online writes, including background. Previous packets
# keep numerical KV; only the current answer-boundary packet retains
# a graph, matching the verified online-LoRA experiment.
if self.qa_model is not None:
private = cache.qa_cache or QwenStreamingKVCache.empty(source_ids)
self.qa_model.vlm_processor = self.vlm_processor
qa_grad = torch.is_grad_enabled() and self.training and any(
getattr(self, "swa_answer_targets", ()) or ())
with torch.set_grad_enabled(qa_grad):
next_cache.qa_cache = self.qa_model.encode(private, packets, kw["update_semantic_cache"], prepared=prepared)
end = (current_mask.sum(-1) - 1).clamp_min(0)
summary = hidden[torch.arange(batch, device=device), end][:, None, :]
control = self._predict_control_logits(summary[:, 0], states=None, control_progress=None,
previous_control_ids=None, control_rtc_prefix=None, control_rtc_mask=None)
probs, weights, entropy, token, mode = self._build_control_condition(control, mode=kw.get("control_condition_mode"))
duration = self.motion_duration_head(summary[:, 0])
self.reply_logits = self.reply_head(
self._reply_summary_for_loss(summary[:, 0])
)
if self.keep_tape:
if batch != 1:
raise RuntimeError("Native Talker tape is inference-only and requires batch size one")
self.tape.append((input_ids.detach(), embeds.detach(), hidden.detach()))
aux = {}
for prefix in ("latest_instruction", "instruction_memory"):
if kw.get(prefix + "_input_ids") is not None:
value = self._latest_instruction_teacher_forcing(
past_key_values=next_cache.past_key_values,
past_attention_mask=full_mask,
past_token_count=next_cache.token_count,
past_position_counts=next_offsets,
input_ids=kw[prefix + "_input_ids"],
attention_mask=kw[prefix + "_attention_mask"],
labels=kw[prefix + "_labels"],
recency_slot_ids=kw[prefix + "_recency_ids"],
)
aux.update({prefix + "_loss": value[0], prefix + "_token_correct": value[1], prefix + "_token_count": value[2]})
return FullDuplexStreamCondition(
vlm_hidden_states=torch.cat((hidden, token.to(hidden)), 1),
attention_mask=torch.cat((current_mask, torch.ones(batch, 1, device=device, dtype=torch.long)), 1),
control_logits=control, control_probabilities=probs,
control_condition_weights=weights, control_entropy=entropy,
control_condition_token=token, control_condition_mode=mode,
kv_cache=next_cache, memory_token=summary,
motion_duration_logits=duration,
motion_duration_log1p_seconds=self.motion_duration_quantile_log1p(duration, 0.5),
**aux,
)
def qa_loss(self, cache, targets):
private_model = getattr(self, "qa_model", None)
if private_model is not None:
if cache.qa_cache is None:
raise RuntimeError("Independent QA has no online AV history")
return private_model.qa_loss(cache.qa_cache, targets)
device = self.reply_head.weight.device
rows = torch.tensor([i for i, target in enumerate(targets) if target], device=device, dtype=torch.long)
zero = self.reply_head.weight.sum() * 0
if rows.numel() == 0:
return zero, zero.detach(), zero.detach()
tokenizer = self.vlm_processor.tokenizer
prefix = tokenizer.encode(REPLY_QUERY, add_special_tokens=False)
full = [prefix + tokenizer.encode(targets[i], add_special_tokens=False) + [self.vlm_model.config.eos_token_id] for i in rows.tolist()]
length = max(map(len, full))
ids = torch.zeros(len(full), length, device=device, dtype=torch.long)
mask = torch.zeros_like(ids)
labels = torch.full_like(ids, -100)
for row, sequence in enumerate(full):
ids[row, :len(sequence)] = torch.tensor(sequence, device=device)
mask[row, :len(sequence)] = 1
labels[row, len(prefix):len(sequence)] = ids[row, len(prefix):len(sequence)]
side = _clone_cache(cache.past_key_values, rows)
offsets = torch.tensor(cache.av_next_positions, device=device)[rows]
result = self.vlm_model.model(
inputs_embeds=self.vlm_model.get_input_embeddings()(ids),
attention_mask=torch.cat((cache.attention_mask[rows], mask), 1),
position_ids=(offsets[:, None] + torch.arange(length, device=device)[None, :])[None].expand(3, -1, -1),
past_key_values=side,
cache_position=torch.arange(cache.token_count, cache.token_count + length, device=device),
use_cache=True, return_dict=True,
)
labels = labels[:, 1:]
valid = labels != -100
states = result.last_hidden_state[:, :-1][valid]
target = labels[valid]
losses, correct = [], []
def token_loss(x, y):
logits = self.vlm_model.lm_head(x).float()
return F.cross_entropy(logits, y, reduction="sum"), (logits.argmax(-1) == y).sum().float()
for start in range(0, len(target), 32):
value = checkpoint(token_loss, states[start:start + 32], target[start:start + 32], use_reentrant=False) if torch.is_grad_enabled() else token_loss(states[start:start + 32], target[start:start + 32])
losses.append(value[0]); correct.append(value[1])
return torch.stack(losses).sum(), torch.stack(correct).sum(), torch.tensor(len(target), device=device, dtype=torch.float32)
@torch.inference_mode()
def free_answer(self, cache, row: int = 0, max_tokens: int = 48):
private_model = getattr(self, "qa_model", None)
if private_model is not None:
if cache.qa_cache is None:
raise RuntimeError("Independent QA has no online AV history")
return private_model.free_answer(cache.qa_cache, row, max_tokens)
device = self.reply_head.weight.device
tokenizer = self.vlm_processor.tokenizer
rows = torch.tensor([row], device=device)
side = _clone_cache(cache.past_key_values, rows)
ids = torch.tensor([tokenizer.encode(REPLY_QUERY, add_special_tokens=False)], device=device)
offset = cache.av_next_positions[row]
mask = torch.cat((cache.attention_mask[row:row + 1], torch.ones_like(ids)), 1)
embeds = self.vlm_model.get_input_embeddings()(ids)
result = self.vlm_model.model(
inputs_embeds=embeds, attention_mask=mask,
position_ids=(torch.arange(ids.shape[1], device=device) + offset)[None, None].expand(3, 1, -1),
past_key_values=side,
cache_position=torch.arange(cache.token_count, cache.token_count + ids.shape[1], device=device),
use_cache=True, return_dict=True,
)
offset += ids.shape[1]
branch_tape = [(ids, embeds, result.last_hidden_state)]
generated = []
for _ in range(max_tokens):
token = self.vlm_model.lm_head(result.last_hidden_state[:, -1:]).argmax(-1)
generated.append(int(token.item()))
if generated[-1] == self.vlm_model.config.eos_token_id:
break
embed = self.vlm_model.get_input_embeddings()(token)
width = side.get_seq_length()
mask = torch.cat((mask, torch.ones_like(token)), 1)
result = self.vlm_model.model(
inputs_embeds=embed, attention_mask=mask,
position_ids=torch.full((3, 1, 1), offset, device=device, dtype=torch.long),
past_key_values=side, cache_position=torch.tensor([width], device=device),
use_cache=True, return_dict=True,
)
branch_tape.append((token, embed, result.last_hidden_state))
offset += 1
return {
"text": tokenizer.decode(generated, skip_special_tokens=True),
"ids": generated,
"eos": bool(generated and generated[-1] == self.vlm_model.config.eos_token_id),
"branch_tape": branch_tape,
}