from types import SimpleNamespace import numpy as np import pytest import torch from PIL import Image from psi.data.seed.native_visual import request_frames, resolve_visual from psi.data.seed.live_multimodal_overlay import attach_live_packets, camera_count def test_native_image_uses_same_pad_repeat(tmp_path): from PIL import ImageOps path = tmp_path/'image.png' Image.fromarray(np.arange(32*48*3, dtype=np.uint8).reshape(32,48,3)).save(path) ref = np.asarray(ImageOps.pad(Image.open(path).convert('RGB'), (224,224), color=(0,0,0))) for count in (3,4,5): value = request_frames(dict(path=str(path),kind='image',count=count,index=0,duration=None)) assert np.array_equal(value,np.repeat(ref[None],count,axis=0)) def test_raw_packets_do_not_open_feature_file(tmp_path): image=tmp_path/'input.jpg' Image.new('RGB',(224,224)).save(image) source = dict(multimodal_overlay=dict(mode='visual-only',visual_input_mode='raw', camera_kind='static',image_path=str(image)), steps=[dict(update_semantic_cache=True,semantic_window_index=i, stream_input_features=torch.ones(2),stream_feature_attention_mask=torch.ones(2)) for i in range(3)]) attach_live_packets(source) for i,p in enumerate(source['swa_packets']): assert p['visual_features'] is None assert p['native_visual']['count']==camera_count(i) assert p['native_visual']['index']==i def test_native_inactive_does_not_decode_or_encode(): qwen=SimpleNamespace(model=SimpleNamespace(embed_tokens=torch.nn.Embedding(5,2048))) p=dict(native_visual=dict(count=5,path='/not/read'),visual_features=None) assert resolve_visual(qwen,p,False).shape==(192,2048) assert not resolve_visual(qwen,p,False).any() p['visual_features']=torch.ones(2,3) with pytest.raises(ValueError,match='not both'):resolve_visual(qwen,p) def test_raw_short_video_fails_before_missing_cache_access(): source=dict(multimodal_overlay=dict(mode='visual-only',visual_input_mode='raw', camera_kind='dynamic',camera_duration_seconds=2.,session_required_seconds=3.),steps=[]) with pytest.raises(ValueError,match='cover the complete session'):attach_live_packets(source) def test_native_video_phase_and_end(monkeypatch,tmp_path): path=tmp_path/'video.mp4';path.touch() from psi.data.seed import native_visual frames=np.zeros((8,224,224,3),np.uint8) monkeypatch.setattr(native_visual,'_rgb',lambda *a:(frames,[[0,1,2,3,4],[5,6,7]],None)) req=dict(path=str(path),kind='video',index=1,count=3,duration=1.) assert request_frames(req).shape[0]==3 with pytest.raises(ValueError,match='phase'):request_frames({**req,'count':4}) with pytest.raises(ValueError,match='ended'):request_frames({**req,'index':2}) def test_both_modes_prepare_identical_qwen_inputs(monkeypatch): from test_causal_av_packing import LiveQwen, live_packet from psi.models.causal_av import prepare_av_batch from psi.data.seed import native_visual qwen=LiveQwen() cached=live_packet(3) raw={**cached,'visual_features':None,'native_visual':{'count':5}} monkeypatch.setattr(native_visual,'request_frames',lambda r:np.zeros((5,224,224,3),np.uint8)) monkeypatch.setattr(native_visual,'encode_frames',lambda *a:cached['visual_features']) qwen._hfd_native_visual_processor=object() left=prepare_av_batch(qwen,[cached],True) right=prepare_av_batch(qwen,[raw],True) for name in ('embeds','ids','positions','mask'): assert torch.equal(getattr(left,name),getattr(right,name)) assert left.advances==right.advances assert raw['visual_features'] is None # no retained CUDA tensors in source packets def test_native_mode_rejects_invalid_config(): from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig with pytest.raises(ValueError):FullDuplexAudioCommandDataConfig(multimodal_visual_input_mode='auto') def test_raw_mode_is_exposed_by_launcher(): from pathlib import Path from psi.hfd_training_launcher import resolve root=Path(__file__).resolve().parents[1] options,_=resolve(root,{'HFD_VISUAL_INPUT_MODE':'raw'}) assert options['data.multimodal-visual-input-mode']=='raw' def test_switching_rollout_mode_does_not_mutate_original(tmp_path): from psi.data.seed.native_visual import source_visual_mode Image.new('RGB',(224,224)).save(tmp_path/'i.jpg') old={'multimodal_overlay':dict(mode='visual-only',camera_kind='static',image_path=str(tmp_path/'i.jpg')), 'steps':[dict(update_semantic_cache=True,semantic_window_index=0, stream_input_features=torch.ones(2),stream_feature_attention_mask=torch.ones(2))], 'swa_packets':[{'old':True}]} new=source_visual_mode(old,'raw') assert old['swa_packets']==[{'old':True}] assert 'visual_input_mode' not in old['multimodal_overlay'] assert new['swa_packets'][0]['native_visual']['kind']=='image'