File size: 4,984 Bytes
aa02c28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | from types import SimpleNamespace
import numpy as np
import pytest
import torch
from PIL import Image
from psi.data.seed.native_visual import request_frames, resolve_visual
from psi.data.seed.live_multimodal_overlay import attach_live_packets, camera_count
def test_native_image_uses_same_pad_repeat(tmp_path):
from PIL import ImageOps
path = tmp_path/'image.png'
Image.fromarray(np.arange(32*48*3, dtype=np.uint8).reshape(32,48,3)).save(path)
ref = np.asarray(ImageOps.pad(Image.open(path).convert('RGB'), (224,224), color=(0,0,0)))
for count in (3,4,5):
value = request_frames(dict(path=str(path),kind='image',count=count,index=0,duration=None))
assert np.array_equal(value,np.repeat(ref[None],count,axis=0))
def test_raw_packets_do_not_open_feature_file(tmp_path):
image=tmp_path/'input.jpg'
Image.new('RGB',(224,224)).save(image)
source = dict(multimodal_overlay=dict(mode='visual-only',visual_input_mode='raw',
camera_kind='static',image_path=str(image)),
steps=[dict(update_semantic_cache=True,semantic_window_index=i,
stream_input_features=torch.ones(2),stream_feature_attention_mask=torch.ones(2)) for i in range(3)])
attach_live_packets(source)
for i,p in enumerate(source['swa_packets']):
assert p['visual_features'] is None
assert p['native_visual']['count']==camera_count(i)
assert p['native_visual']['index']==i
def test_native_inactive_does_not_decode_or_encode():
qwen=SimpleNamespace(model=SimpleNamespace(embed_tokens=torch.nn.Embedding(5,2048)))
p=dict(native_visual=dict(count=5,path='/not/read'),visual_features=None)
assert resolve_visual(qwen,p,False).shape==(192,2048)
assert not resolve_visual(qwen,p,False).any()
p['visual_features']=torch.ones(2,3)
with pytest.raises(ValueError,match='not both'):resolve_visual(qwen,p)
def test_raw_short_video_fails_before_missing_cache_access():
source=dict(multimodal_overlay=dict(mode='visual-only',visual_input_mode='raw',
camera_kind='dynamic',camera_duration_seconds=2.,session_required_seconds=3.),steps=[])
with pytest.raises(ValueError,match='cover the complete session'):attach_live_packets(source)
def test_native_video_phase_and_end(monkeypatch,tmp_path):
path=tmp_path/'video.mp4';path.touch()
from psi.data.seed import native_visual
frames=np.zeros((8,224,224,3),np.uint8)
monkeypatch.setattr(native_visual,'_rgb',lambda *a:(frames,[[0,1,2,3,4],[5,6,7]],None))
req=dict(path=str(path),kind='video',index=1,count=3,duration=1.)
assert request_frames(req).shape[0]==3
with pytest.raises(ValueError,match='phase'):request_frames({**req,'count':4})
with pytest.raises(ValueError,match='ended'):request_frames({**req,'index':2})
def test_both_modes_prepare_identical_qwen_inputs(monkeypatch):
from test_causal_av_packing import LiveQwen, live_packet
from psi.models.causal_av import prepare_av_batch
from psi.data.seed import native_visual
qwen=LiveQwen()
cached=live_packet(3)
raw={**cached,'visual_features':None,'native_visual':{'count':5}}
monkeypatch.setattr(native_visual,'request_frames',lambda r:np.zeros((5,224,224,3),np.uint8))
monkeypatch.setattr(native_visual,'encode_frames',lambda *a:cached['visual_features'])
qwen._hfd_native_visual_processor=object()
left=prepare_av_batch(qwen,[cached],True)
right=prepare_av_batch(qwen,[raw],True)
for name in ('embeds','ids','positions','mask'):
assert torch.equal(getattr(left,name),getattr(right,name))
assert left.advances==right.advances
assert raw['visual_features'] is None # no retained CUDA tensors in source packets
def test_native_mode_rejects_invalid_config():
from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig
with pytest.raises(ValueError):FullDuplexAudioCommandDataConfig(multimodal_visual_input_mode='auto')
def test_raw_mode_is_exposed_by_launcher():
from pathlib import Path
from psi.hfd_training_launcher import resolve
root=Path(__file__).resolve().parents[1]
options,_=resolve(root,{'HFD_VISUAL_INPUT_MODE':'raw'})
assert options['data.multimodal-visual-input-mode']=='raw'
def test_switching_rollout_mode_does_not_mutate_original(tmp_path):
from psi.data.seed.native_visual import source_visual_mode
Image.new('RGB',(224,224)).save(tmp_path/'i.jpg')
old={'multimodal_overlay':dict(mode='visual-only',camera_kind='static',image_path=str(tmp_path/'i.jpg')),
'steps':[dict(update_semantic_cache=True,semantic_window_index=0,
stream_input_features=torch.ones(2),stream_feature_attention_mask=torch.ones(2))],
'swa_packets':[{'old':True}]}
new=source_visual_mode(old,'raw')
assert old['swa_packets']==[{'old':True}]
assert 'visual_input_mode' not in old['multimodal_overlay']
assert new['swa_packets'][0]['native_visual']['kind']=='image'
|