Download src/psi/models/causal_av.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/causal_av.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/models/causal_av.py
-
curl -L -o causal_av.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/causal_av.py
16.4 kB
| """One causal AV packet writer shared by the body and private QA branches.""" | |
| from dataclasses import dataclass | |
| import torch | |
| from torch.nn.utils.rnn import pad_sequence | |
| from psi.models.full_duplex_psi0 import QwenStreamingKVCache | |
| class PreparedAVBatch: | |
| embeds: torch.Tensor | |
| ids: torch.Tensor | |
| positions: torch.Tensor | |
| mask: torch.Tensor | |
| active: tuple[bool, ...] | |
| lengths: tuple[int, ...] | |
| advances: tuple[int, ...] | |
| vision_starts: tuple[int, ...] | |
| prepended_state: tuple[bool, ...] = () | |
| def _prepend_pending_control_tokens(qwen, prepared): | |
| """Fuse the previous packet's predicted state with the next AV packet. | |
| The token is placed immediately before the packet and therefore has the | |
| same causal meaning as a standalone append followed by the AV forward. | |
| A1 reuses this immutable prepared batch in both branches, so the predicted | |
| interaction state is shared context rather than a second Qwen launch. | |
| """ | |
| pending = getattr(qwen, "_hfd_pending_control_token_ids", None) | |
| if pending is None: | |
| return prepared | |
| pending = tuple(None if value is None else int(value) for value in pending) | |
| batch = len(prepared.lengths) | |
| if len(pending) != batch: | |
| raise RuntimeError("Pending control-token rows do not match the AV batch") | |
| flags = tuple(value is not None for value in pending) | |
| if not any(flags): | |
| return PreparedAVBatch( | |
| prepared.embeds, prepared.ids, prepared.positions, prepared.mask, | |
| prepared.active, prepared.lengths, prepared.advances, | |
| prepared.vision_starts, flags, | |
| ) | |
| if any(flag and not active for flag, active in zip(flags, prepared.active)): | |
| raise RuntimeError("An inactive AV row cannot consume a pending state token") | |
| device = prepared.embeds.device | |
| embedding = qwen.get_input_embeddings() | |
| ids_rows, embed_rows, position_rows = [], [], [] | |
| with torch.no_grad(): | |
| for row, (length, token_id) in enumerate(zip(prepared.lengths, pending)): | |
| length = int(length) | |
| ids = prepared.ids[row, :length] | |
| embeds = prepared.embeds[row, :length] | |
| positions = prepared.positions[:, row, :length] | |
| if token_id is not None: | |
| state_id = torch.tensor([token_id], device=device, dtype=ids.dtype) | |
| state_embed = embedding(state_id).detach().to(embeds) | |
| ids = torch.cat((state_id, ids), 0) | |
| embeds = torch.cat((state_embed, embeds), 0) | |
| positions = torch.cat( | |
| (torch.zeros((3, 1), device=device, dtype=positions.dtype), positions + 1), | |
| -1, | |
| ) | |
| ids_rows.append(ids) | |
| embed_rows.append(embeds) | |
| position_rows.append(positions.T) | |
| ids = pad_sequence(ids_rows, batch_first=True) | |
| embeds = pad_sequence(embed_rows, batch_first=True) | |
| positions = pad_sequence(position_rows, batch_first=True).permute(2, 0, 1) | |
| lengths = tuple(length + int(flag) for length, flag in zip(prepared.lengths, flags)) | |
| mask = ((torch.arange(max(lengths), device=device)[None] | |
| < torch.tensor(lengths, device=device)[:, None]) | |
| & torch.tensor(prepared.active, device=device)[:, None]).long() | |
| return PreparedAVBatch( | |
| embeds, ids, positions, mask, prepared.active, lengths, | |
| tuple(value + int(flag) for value, flag in zip(prepared.advances, flags)), | |
| tuple(value + int(flag) if value >= 0 else value | |
| for value, flag in zip(prepared.vision_starts, flags)), | |
| flags, | |
| ) | |
| def prepare_av_batch(qwen, packets, active): | |
| """Collate stored frontend packets before copying to CUDA, once per batch. | |
| The body and QA read the same immutable inputs but keep separate KV and | |
| autograd graphs. Python ledgers never read one CUDA scalar per source. | |
| """ | |
| live = [bool(p.get("live_audio", False)) for p in packets] | |
| if any(live): | |
| if not all(live): | |
| raise ValueError("Cannot mix legacy frozen packets with live microphone packets") | |
| return prepare_live_av_batch(qwen, packets, active) | |
| from psi.models.interaction_arch.control_tokens import storage_weight | |
| weight = storage_weight(qwen.model.embed_tokens) | |
| count = len(packets) | |
| if isinstance(active, torch.Tensor): | |
| active = active.detach().cpu().reshape(-1).tolist() | |
| if isinstance(active, (bool, int)): | |
| active = [bool(active)] * count | |
| active = tuple(bool(x) for x in active) | |
| if len(active) != count or not count: | |
| raise ValueError("AV active mask must have one entry per packet") | |
| lengths = tuple(len(p["ids"]) for p in packets) | |
| if min(lengths) < 1: | |
| raise ValueError("AV packets must not be empty") | |
| ids_cpu = [p["ids"].detach().to(device="cpu", dtype=torch.long) for p in packets] | |
| vision_starts = [] | |
| for ids in ids_cpu: | |
| starts = (ids == qwen.config.vision_start_token_id).nonzero() | |
| vision_starts.append(int(starts[0, 0]) if starts.numel() else -1) | |
| ids = pad_sequence(ids_cpu, batch_first=True) | |
| embeds = pad_sequence([p["embeds"].detach().to(device="cpu", dtype=weight.dtype) | |
| for p in packets], batch_first=True) | |
| positions = pad_sequence([p["positions"].detach().to(device="cpu").T | |
| for p in packets], batch_first=True).permute(2, 0, 1) | |
| mask = ((torch.arange(max(lengths))[None] < torch.tensor(lengths)[:, None]) | |
| & torch.tensor(active)[:, None]).long() | |
| def copy(tensor): | |
| if weight.device.type == "cuda": | |
| tensor = tensor.pin_memory() | |
| return tensor.to(weight.device, non_blocking=True) | |
| return _prepend_pending_control_tokens(qwen, PreparedAVBatch( | |
| copy(embeds), copy(ids), copy(positions), copy(mask), active, lengths, | |
| tuple(int(p["next_position"]) for p in packets), tuple(vision_starts))) | |
| def prepare_live_av_batch(qwen, packets, active): | |
| """Batch the frozen audio tower once; reuse independent image features. | |
| The merged microphone is encoded online, so no old command/QA audio can | |
| leak from a prerecorded source. Body and private QA consume the identical | |
| resulting embeddings. Only shape/position templates are memoized. | |
| """ | |
| from psi.models.interaction_arch.control_tokens import storage_weight | |
| weight = storage_weight(qwen.model.embed_tokens) | |
| if any(p.requires_grad for p in qwen.audio_tower.parameters()): | |
| raise RuntimeError("Live overlay input path requires a frozen audio tower") | |
| if isinstance(active, torch.Tensor): | |
| active = active.detach().cpu().reshape(-1).tolist() | |
| if isinstance(active, (bool, int)): | |
| active = [bool(active)] * len(packets) | |
| active = tuple(bool(x) for x in active) | |
| if not packets or len(active) != len(packets): | |
| raise ValueError("One active mask is required per live packet") | |
| from psi.data.seed.native_visual import resolve_visual | |
| visuals = [resolve_visual(qwen, p, a) for p, a in zip(packets, active)] | |
| prefix = getattr(qwen, "_hfd_av_prefix_ids", None) | |
| if prefix is None: | |
| raise RuntimeError("Live AV system prompt has not been bound to the tokenizer") | |
| templates = getattr(qwen, "_hfd_live_av_templates", None) | |
| if templates is None: | |
| templates = qwen._hfd_live_av_templates = {} | |
| features = torch.stack([p["input_features"].float() for p in packets]) | |
| feature_masks = torch.stack([p["feature_attention_mask"].long() for p in packets]) | |
| # Accelerate may already have placed the packets on CUDA. Shape/rope | |
| # templates are built on CPU; transfer only this tiny length vector once, | |
| # never the actual microphone features or once-per-row CUDA scalars. | |
| feature_lengths = feature_masks.sum(-1).detach().cpu() | |
| _, audio_lengths = qwen.audio_tower._get_feat_extract_output_lengths(feature_lengths) | |
| ids_list, positions_list, starts, advances = [], [], [], [] | |
| conf = qwen.config | |
| for row, packet in enumerate(packets): | |
| visual = visuals[row] | |
| nvideo = len(visual) if visual is not None else 0 | |
| count = int(packet["camera_count"]) if nvideo else 0 | |
| audio_len = int(audio_lengths[row]) | |
| key = (bool(packet["first_window"]), count, audio_len, int(feature_lengths[row])) | |
| if key not in templates: | |
| tokens = list(prefix) if packet["first_window"] else [] | |
| start = len(tokens) | |
| if nvideo: | |
| tokens += [conf.vision_start_token_id] + [conf.video_token_id] * nvideo + [conf.vision_end_token_id] | |
| tokens += [conf.audio_start_token_id] + [conf.audio_token_id] * audio_len + [conf.audio_end_token_id] | |
| ids = torch.tensor(tokens, dtype=torch.long) | |
| grid = torch.tensor([[(count + 1) // 2, 16, 16]]) if nvideo else None | |
| if nvideo and nvideo != int(grid.prod()) // 4: | |
| raise ValueError("Live visual features differ from the audited 224px camera geometry") | |
| positions, _ = qwen.get_rope_index( | |
| ids[None], video_grid_thw=grid, attention_mask=torch.ones_like(ids[None]), | |
| audio_seqlens=feature_lengths[row:row+1], | |
| second_per_grids=torch.tensor([.25]) if nvideo else None, | |
| ) | |
| templates[key] = (ids, positions[:, 0], start, int(positions.max()) + 1) | |
| ids, positions, start, advance = templates[key] | |
| ids_list.append(ids); positions_list.append(positions.T) | |
| starts.append(start); advances.append(advance) | |
| lengths = tuple(map(len, ids_list)) | |
| ids = pad_sequence(ids_list, batch_first=True).to(weight.device) | |
| positions = pad_sequence(positions_list, batch_first=True).permute(2, 0, 1).to(weight.device) | |
| mask = ((torch.arange(max(lengths))[None] < torch.tensor(lengths)[:, None]) | |
| & torch.tensor(active)[:, None]).long().to(weight.device) | |
| embeds = qwen.get_input_embeddings()(ids) | |
| audio = qwen.get_audio_features(features.to(device=weight.device, dtype=weight.dtype), | |
| feature_masks.to(weight.device)) | |
| if int(audio_lengths.sum()) != int((ids == conf.audio_token_id).sum()): | |
| raise RuntimeError("Live microphone token geometry mismatch") | |
| embeds[ids == conf.audio_token_id] = audio.to(embeds) | |
| for row, packet in enumerate(packets): | |
| visual = visuals[row] | |
| if visual is not None: | |
| embeds[row, ids[row] == conf.video_token_id] = visual.to(embeds) | |
| return _prepend_pending_control_tokens(qwen, PreparedAVBatch( | |
| embeds, ids, positions, mask, active, lengths, | |
| tuple(advances), tuple(starts))) | |
| def fit_complete_av_windows(cache, *, current_tokens, auxiliary_tokens, context_limit): | |
| """Keep the newest complete packets when the token budget beats the time cap. | |
| A 96-second camera history can exceed the model's token budget before | |
| its time-based expiration. Preserve the system prefix and absolute RoPE | |
| positions; never split an audio/video packet or reset another source row. | |
| """ | |
| budget = int(context_limit) - int(current_tokens) - int(auxiliary_tokens) | |
| if budget < 0: | |
| raise RuntimeError("A single AV packet plus readout exceeds the context budget") | |
| if cache.token_count <= budget: | |
| return | |
| birth, mask = cache.token_birth_windows, cache.attention_mask | |
| if birth is None or mask is None or birth.shape != mask.shape: | |
| raise RuntimeError("AV context pressure requires a valid packet birth ledger") | |
| valid = mask.bool() | |
| protected = valid & (birth == -1) | |
| temporal = valid & (birth >= 0) | |
| available = budget - protected.sum(-1) | |
| if (available < 0).any(): | |
| raise RuntimeError("AV system prefix plus current packet exceeds the context budget") | |
| # Reverse token count selects a suffix; if it cuts a packet, remove that | |
| # entire oldest packet rather than retain an incomplete multimodal input. | |
| suffix_count = temporal.flip(-1).long().cumsum(-1).flip(-1) | |
| candidates = temporal & (suffix_count <= available[:, None]) | |
| sentinel = torch.iinfo(birth.dtype).max | |
| oldest = torch.where(candidates, birth, sentinel).amin(-1) | |
| boundary = temporal & (birth == oldest[:, None]) | |
| partial = (boundary & candidates).sum(-1) < boundary.sum(-1) | |
| retained = candidates & (~boundary | ~partial[:, None]) | |
| cache.attention_mask = (protected | retained).to(mask.dtype) | |
| cache.compact_invalid_tokens() | |
| if cache.token_count > budget: | |
| raise RuntimeError("Whole-packet AV eviction did not satisfy context budget") | |
| def encode_av(qwen, cache, packets, active, *, memory_seconds, context_limit, | |
| auxiliary=128, prepared=None): | |
| batch = len(cache.source_ids) | |
| if len(packets) != batch: | |
| raise RuntimeError("AV packet/cache batch mismatch") | |
| from psi.models.interaction_arch.control_tokens import storage_weight | |
| device = storage_weight(qwen.model.embed_tokens).device | |
| prepared = prepared if prepared is not None else prepare_av_batch(qwen, packets, active) | |
| if len(prepared.active) != batch: | |
| raise RuntimeError("Prepared AV input/cache batch mismatch") | |
| active = prepared.active | |
| width = prepared.ids.shape[1] | |
| cache.expire_audio_windows(int(round(memory_seconds / .4))) | |
| cache.compact_if_worthwhile(current_tokens=width, auxiliary_tokens=auxiliary, | |
| max_context_tokens=context_limit) | |
| fit_complete_av_windows(cache, current_tokens=width, auxiliary_tokens=auxiliary, | |
| context_limit=context_limit) | |
| if cache.token_count + width + auxiliary > context_limit: | |
| raise RuntimeError("SWA causal AV context exceeds its safe bound") | |
| if cache.past_key_values is not None and cache.past_key_values.get_seq_length() != cache.token_count: | |
| raise RuntimeError("AV KV length differs from its ledger") | |
| embeds, mask, ids = prepared.embeds, prepared.mask, prepared.ids | |
| offsets = cache.av_next_positions or (0,) * batch | |
| next_offsets = tuple(offset + advance * flag for offset, advance, flag in | |
| zip(offsets, prepared.advances, active)) | |
| # Match old padding positions exactly, including inactive rows. | |
| real_tokens = torch.arange(width, device=device)[None] < torch.tensor(prepared.lengths, device=device)[:, None] | |
| positions = prepared.positions + torch.tensor(offsets, device=device)[None, :, None] * real_tokens[None] | |
| full_mask = mask if cache.attention_mask is None else torch.cat((cache.attention_mask, mask), -1) | |
| result = qwen.model(inputs_embeds=embeds, attention_mask=full_mask, position_ids=positions, | |
| past_key_values=cache.past_key_values, | |
| cache_position=torch.arange(cache.token_count, cache.token_count + width, device=device), | |
| use_cache=True, return_dict=True) | |
| next_cache = QwenStreamingKVCache( | |
| source_ids=cache.source_ids, past_key_values=result.past_key_values, | |
| attention_mask=full_mask, token_count=cache.token_count + width, | |
| logical_window_counts=tuple(c + int(v) for c, v in zip(cache.logical_window_counts, active)), | |
| logical_token_counts=tuple(c + n * flag for c, n, flag in | |
| zip(cache.resolved_logical_token_counts(), prepared.lengths, active)), | |
| slot_rebind_count=cache.slot_rebind_count, compaction_count=cache.compaction_count, | |
| reclaimed_token_count=cache.reclaimed_token_count, av_next_positions=tuple(next_offsets), | |
| ) | |
| birth = torch.tensor(cache.logical_window_counts, device=device)[:, None].expand(batch, width).clone() | |
| for row in range(batch): | |
| if prepared.prepended_state and prepared.prepended_state[row]: | |
| if cache.logical_window_counts[row] <= 0: | |
| raise RuntimeError("A source-start packet cannot consume stale state") | |
| birth[row, 0] = cache.logical_window_counts[row] - 1 | |
| if cache.logical_window_counts[row] == 0: | |
| start = prepared.vision_starts[row] | |
| if start >= 0: | |
| birth[row, :start] = -1 | |
| birth.masked_fill_(mask == 0, -2) | |
| next_cache.token_birth_windows = birth if cache.token_birth_windows is None else torch.cat((cache.token_birth_windows, birth), -1) | |
| return result.last_hidden_state, ids, embeds, mask, next_cache | |