from types import SimpleNamespace import pytest import torch from torch import nn from transformers.cache_utils import DynamicCache from psi.models.causal_av import prepare_av_batch, encode_av, fit_complete_av_windows from psi.models.full_duplex_psi0 import QwenStreamingKVCache class FakeQwen(nn.Module): def __init__(self): super().__init__() self.model = FakeCore() self.config = SimpleNamespace(vision_start_token_id=9) class FakeCore(nn.Module): def __init__(self): super().__init__() self.embed_tokens = nn.Embedding(10, 4) self.calls = [] def forward(self, **kw): self.calls.append(kw) x = kw["inputs_embeds"] past = kw["past_key_values"] or DynamicCache() past.update(x[:, None], x[:, None], 0) return SimpleNamespace(last_hidden_state=x + self.embed_tokens.weight[0], past_key_values=past) def packets(): return [dict(ids=torch.tensor(ids), embeds=torch.arange(len(ids)*4).reshape(-1,4).float(), positions=torch.arange(len(ids)).expand(3,-1), next_position=len(ids)+2) for ids in ([1,2,9,4], [1,9], [1,2,9])] def test_packed_av_matches_original_input_offsets_and_inactive_ledger(): qwen = FakeQwen() rows = packets() prepared = prepare_av_batch(qwen, rows, [True, False, True]) cache = QwenStreamingKVCache.empty(["a","b","c"]) cache.av_next_positions = (8,10,20) result = encode_av(qwen, cache, rows, [True,False,True], memory_seconds=96, context_limit=512, prepared=prepared) call = qwen.model.calls[-1] for row, packet in enumerate(rows): size = len(packet["ids"]) torch.testing.assert_close(call["inputs_embeds"][row,:size], packet["embeds"]) torch.testing.assert_close(call["position_ids"][:,row,:size], packet["positions"] + [8,10,20][row]) assert call["inputs_embeds"][row,size:].count_nonzero() == 0 assert call["position_ids"][:,row,size:].count_nonzero() == 0 assert result[-1].logical_window_counts == (1,0,1) assert result[-1].logical_token_counts == (4,0,3) assert result[-1].av_next_positions == (14,10,25) assert result[-1].token_birth_windows.tolist() == [[-1,-1,0,0],[-2,-2,-2,-2],[-1,-1,0,-2]] # Reusing immutable inputs cannot share private model KVs/gradients. private = FakeQwen() next_private = encode_av(private, QwenStreamingKVCache.empty(["a","b","c"]), rows, [True,False,True], memory_seconds=96, context_limit=512, prepared=prepared) next_private[0].sum().backward() assert qwen.model.embed_tokens.weight.grad is None assert private.model.embed_tokens.weight.grad is not None assert next_private[-1].past_key_values is not result[-1].past_key_values def test_scalar_activity_and_bad_shapes(): qwen = FakeQwen() assert prepare_av_batch(qwen, packets(), True).active == (True,True,True) with pytest.raises(ValueError, match="one entry"): prepare_av_batch(qwen, packets(), [True]) def test_token_bound_keeps_whole_packets_prefix_positions_and_each_source(): cache = QwenStreamingKVCache.empty(["a", "b"]) cache.logical_window_counts = (3, 3) cache.av_next_positions = (123, 456) cache.token_count = 10 cache.attention_mask = torch.tensor([[1]*10, [1]*8+[0]*2]) cache.token_birth_windows = torch.tensor([ [-1,-1,0,0,0,1,1,1,2,2], [-1,0,0,1,1,1,2,2,-2,-2], ]) keys = torch.arange(20).reshape(2,1,10,1).float() cache.past_key_values = DynamicCache() cache.past_key_values.update(keys, keys + 100, 0) fit_complete_av_windows(cache, current_tokens=2, auxiliary_tokens=1, context_limit=9) # Six old tokens fit, but row a's 3-token packet 1 must not be split. assert cache.token_count == 6 assert cache.attention_mask.tolist() == [[1,1,1,1,0,0], [1]*6] assert cache.token_birth_windows.tolist() == [[-1,-1,2,2,-2,-2], [-1,1,1,1,2,2]] assert cache.past_key_values.layers[0].keys[0,0,:4,0].tolist() == [0,1,8,9] assert cache.past_key_values.layers[0].keys[1,0,:,0].tolist() == [10,13,14,15,16,17] assert cache.logical_window_counts == (3,3) assert cache.av_next_positions == (123,456) cache.rebind_rows(["new","b"], [True,False]) assert cache.av_next_positions == (0,456) assert cache.attention_mask[0].sum() == 0 assert cache.attention_mask[1].sum() == 6 def test_av_stream_exceeds_time_vs_token_budget_without_crashing(): qwen = FakeQwen().eval() cache = QwenStreamingKVCache.empty(["a"]) rows = [packets()[0]] with torch.inference_mode(): for i in range(30): result = encode_av(qwen, cache, rows, True, memory_seconds=96, context_limit=24, auxiliary=4) cache = result[-1] cache.detach() assert cache.token_count <= 20 assert cache.logical_window_counts == (i+1,) assert cache.av_next_positions == ((i+1)*6,) assert cache.reclaimed_token_count > 0 def test_av_fitting_is_noop_without_pressure_and_rejects_impossible_prefix(): cache = QwenStreamingKVCache.empty(["a"]) fit_complete_av_windows(cache, current_tokens=4, auxiliary_tokens=4, context_limit=8) assert cache.past_key_values is None cache.token_count=3 cache.attention_mask=torch.ones(1,3,dtype=torch.long) cache.token_birth_windows=torch.full((1,3),-1) with pytest.raises(RuntimeError,match="system prefix"): fit_complete_av_windows(cache,current_tokens=4,auxiliary_tokens=4,context_limit=10) class LiveQwen(FakeQwen): def __init__(self): super().__init__() self.config = SimpleNamespace(vision_start_token_id=9, video_token_id=8, vision_end_token_id=7, audio_start_token_id=6, audio_token_id=5, audio_end_token_id=4) self._hfd_av_prefix_ids = (1, 2) self.audio_tower = nn.Linear(1,1).requires_grad_(False) self.audio_tower._get_feat_extract_output_lengths = lambda n: (n, (n-1)//2//2+1) def get_input_embeddings(self): return self.model.embed_tokens def get_rope_index(self, ids, **kw): assert ids.device.type == kw['audio_seqlens'].device.type == 'cpu' return torch.arange(ids.shape[-1])[None,None].expand(3,1,-1), None def get_audio_features(self, features, masks): counts = ((masks.sum(-1)-1)//2//2+1).tolist() return torch.cat([features[i].mean().expand(n,4) for i,n in enumerate(counts)]) def live_packet(value, *, visual=True, first=True): return dict(live_audio=True, first_window=first, camera_count=5, input_features=torch.full((128,63),float(value)), feature_attention_mask=torch.ones(63), visual_features=torch.full((192,4),13.) if visual else None) @pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA placement regression') def test_live_packets_already_placed_by_accelerate(): model = LiveQwen().cuda() packet = {k:(v.cuda() if isinstance(v,torch.Tensor) else v) for k,v in live_packet(3).items()} prepared = prepare_av_batch(model,[packet],True) assert prepared.embeds.device.type == 'cuda' assert prepared.positions.device.type == 'cuda' actual = prepared.embeds[0,prepared.ids[0]==5] torch.testing.assert_close(actual,torch.full_like(actual,3.)) def test_live_audio_visual_and_refill_share_one_batch_without_cross_source_state(): model = LiveQwen() a,b = live_packet(3),live_packet(7,visual=False) prepared = prepare_av_batch(model,[a,b],[True,True]) for row, value in enumerate((3.,7.)): assert torch.all(prepared.embeds[row,prepared.ids[row]==5] == value) assert torch.all(prepared.embeds[0,prepared.ids[0]==8] == 13.) assert (prepared.ids[1]==8).sum() == 0 result = encode_av(model,QwenStreamingKVCache.empty(['a','b']),[a,b],True, memory_seconds=96,context_limit=1024,prepared=prepared) cache=result[-1];cache.detach() b_offset=cache.av_next_positions[1] cache.rebind_rows(['c','b'],[True,False]) assert cache.av_next_positions==(0,b_offset) assert cache.attention_mask[0].sum()==0 and cache.attention_mask[1].sum()>0 next_packets=[live_packet(11,visual=False),live_packet(19,first=False)] next_p=prepare_av_batch(model,next_packets,True) result=encode_av(model,cache,next_packets,True,memory_seconds=96,context_limit=1024,prepared=next_p) assert result[-1].logical_window_counts == (1,2) assert torch.all(next_p.embeds[0,next_p.ids[0]==5]==11) assert model.model.calls[-1]['position_ids'][0,0,0]==0 assert model.model.calls[-1]['position_ids'][0,1,0]==b_offset assert not next_p.embeds.requires_grad