File size: 4,984 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from PIL import Image

from psi.data.seed.native_visual import request_frames, resolve_visual
from psi.data.seed.live_multimodal_overlay import attach_live_packets, camera_count


def test_native_image_uses_same_pad_repeat(tmp_path):
    from PIL import ImageOps
    path = tmp_path/'image.png'
    Image.fromarray(np.arange(32*48*3, dtype=np.uint8).reshape(32,48,3)).save(path)
    ref = np.asarray(ImageOps.pad(Image.open(path).convert('RGB'), (224,224), color=(0,0,0)))
    for count in (3,4,5):
        value = request_frames(dict(path=str(path),kind='image',count=count,index=0,duration=None))
        assert np.array_equal(value,np.repeat(ref[None],count,axis=0))


def test_raw_packets_do_not_open_feature_file(tmp_path):
    image=tmp_path/'input.jpg'
    Image.new('RGB',(224,224)).save(image)
    source = dict(multimodal_overlay=dict(mode='visual-only',visual_input_mode='raw',
        camera_kind='static',image_path=str(image)),
        steps=[dict(update_semantic_cache=True,semantic_window_index=i,
                    stream_input_features=torch.ones(2),stream_feature_attention_mask=torch.ones(2)) for i in range(3)])
    attach_live_packets(source)
    for i,p in enumerate(source['swa_packets']):
        assert p['visual_features'] is None
        assert p['native_visual']['count']==camera_count(i)
        assert p['native_visual']['index']==i


def test_native_inactive_does_not_decode_or_encode():
    qwen=SimpleNamespace(model=SimpleNamespace(embed_tokens=torch.nn.Embedding(5,2048)))
    p=dict(native_visual=dict(count=5,path='/not/read'),visual_features=None)
    assert resolve_visual(qwen,p,False).shape==(192,2048)
    assert not resolve_visual(qwen,p,False).any()
    p['visual_features']=torch.ones(2,3)
    with pytest.raises(ValueError,match='not both'):resolve_visual(qwen,p)


def test_raw_short_video_fails_before_missing_cache_access():
    source=dict(multimodal_overlay=dict(mode='visual-only',visual_input_mode='raw',
        camera_kind='dynamic',camera_duration_seconds=2.,session_required_seconds=3.),steps=[])
    with pytest.raises(ValueError,match='cover the complete session'):attach_live_packets(source)


def test_native_video_phase_and_end(monkeypatch,tmp_path):
    path=tmp_path/'video.mp4';path.touch()
    from psi.data.seed import native_visual
    frames=np.zeros((8,224,224,3),np.uint8)
    monkeypatch.setattr(native_visual,'_rgb',lambda *a:(frames,[[0,1,2,3,4],[5,6,7]],None))
    req=dict(path=str(path),kind='video',index=1,count=3,duration=1.)
    assert request_frames(req).shape[0]==3
    with pytest.raises(ValueError,match='phase'):request_frames({**req,'count':4})
    with pytest.raises(ValueError,match='ended'):request_frames({**req,'index':2})


def test_both_modes_prepare_identical_qwen_inputs(monkeypatch):
    from test_causal_av_packing import LiveQwen, live_packet
    from psi.models.causal_av import prepare_av_batch
    from psi.data.seed import native_visual
    qwen=LiveQwen()
    cached=live_packet(3)
    raw={**cached,'visual_features':None,'native_visual':{'count':5}}
    monkeypatch.setattr(native_visual,'request_frames',lambda r:np.zeros((5,224,224,3),np.uint8))
    monkeypatch.setattr(native_visual,'encode_frames',lambda *a:cached['visual_features'])
    qwen._hfd_native_visual_processor=object()
    left=prepare_av_batch(qwen,[cached],True)
    right=prepare_av_batch(qwen,[raw],True)
    for name in ('embeds','ids','positions','mask'):
        assert torch.equal(getattr(left,name),getattr(right,name))
    assert left.advances==right.advances
    assert raw['visual_features'] is None  # no retained CUDA tensors in source packets


def test_native_mode_rejects_invalid_config():
    from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig
    with pytest.raises(ValueError):FullDuplexAudioCommandDataConfig(multimodal_visual_input_mode='auto')


def test_raw_mode_is_exposed_by_launcher():
    from pathlib import Path
    from psi.hfd_training_launcher import resolve
    root=Path(__file__).resolve().parents[1]
    options,_=resolve(root,{'HFD_VISUAL_INPUT_MODE':'raw'})
    assert options['data.multimodal-visual-input-mode']=='raw'


def test_switching_rollout_mode_does_not_mutate_original(tmp_path):
    from psi.data.seed.native_visual import source_visual_mode
    Image.new('RGB',(224,224)).save(tmp_path/'i.jpg')
    old={'multimodal_overlay':dict(mode='visual-only',camera_kind='static',image_path=str(tmp_path/'i.jpg')),
         'steps':[dict(update_semantic_cache=True,semantic_window_index=0,
                       stream_input_features=torch.ones(2),stream_feature_attention_mask=torch.ones(2))],
         'swa_packets':[{'old':True}]}
    new=source_visual_mode(old,'raw')
    assert old['swa_packets']==[{'old':True}]
    assert 'visual_input_mode' not in old['multimodal_overlay']
    assert new['swa_packets'][0]['native_visual']['kind']=='image'