File size: 5,287 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""Frozen native camera frontend; same B1 geometry as the published caches.

Raw requests carry paths, not future features/KV. Decoded RGB is a bounded
CPU cache only; every requested packet is encoded online, outside autograd.
"""
from functools import lru_cache
from pathlib import Path

import numpy as np
import torch

from psi.data.seed.video_overlay import camera_plan, decode_video


def visual_request(relation, index, count):
    dynamic = relation.get("camera_kind") == "dynamic"
    path = relation["video_path" if dynamic else "image_path"]
    if not path:
        raise ValueError("Native visual input needs an original image/video")
    return dict(path=str(path), kind="video" if dynamic else "image",
                index=int(index), count=int(count),
                duration=relation.get("camera_duration_seconds"))


def source_visual_mode(source, mode):
    """Switch an existing live rollout fixture without mutating saved tensors."""
    if mode not in ('cached', 'raw'):
        raise ValueError('visual_input_mode must be cached or raw')
    relation = source.get('multimodal_overlay')
    if relation is None:
        if mode == 'raw':
            raise ValueError('Legacy preassembled SWA has no native media contract')
        return source
    if relation.get('visual_input_mode','cached') == mode:
        return source
    from psi.data.seed.live_multimodal_overlay import attach_live_packets
    return attach_live_packets({**source, 'multimodal_overlay': {**relation,'visual_input_mode':mode}})


@lru_cache(maxsize=8)
def _rgb(path, kind, duration, size, mtime_ns):
    if kind == "video":
        ticks, groups = camera_plan(float(duration))
        frames, pts = decode_video(path, ticks, end_seconds=float(duration))
        return frames, groups, pts
    if kind != "image":
        raise ValueError("Unknown native camera kind")
    from PIL import Image, ImageOps
    with Image.open(path) as image:
        frame = np.asarray(ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0)))
    return frame[None], None, None


def request_frames(request):
    # CPU workers may have decoded the packet already. NumPy uint8 stays on
    # the CPU through Accelerate; do not ship an entire video to CUDA.
    if 'frames' in request:
        frames = request['frames']
        if not isinstance(frames, np.ndarray) or frames.dtype != np.uint8 or frames.shape != (int(request['count']),224,224,3):
            raise ValueError('Malformed native RGB packet')
        return frames
    stat = Path(request["path"]).stat()
    frames, groups, _ = _rgb(request["path"], request["kind"], request.get("duration"),
                             stat.st_size, stat.st_mtime_ns)
    count = int(request["count"])
    if count not in (3, 4, 5):
        raise ValueError("Native camera requires the audited 8 Hz / 0.4s geometry")
    if groups is None:
        return np.repeat(frames, count, axis=0)
    index = int(request["index"])
    if index < 0 or index >= len(groups):
        raise ValueError("Video ended before session; frozen EOF tails are forbidden")
    clip = frames[groups[index]]
    if len(clip) != count:
        raise ValueError("Native camera packet phase mismatch")
    return clip


@torch.no_grad()
def encode_frames(qwen, processor, frames):
    """Do not pack B>1: historical BF16 packed vision failed exact parity."""
    visual = qwen.visual
    if any(p.requires_grad for p in visual.parameters()):
        raise ValueError("Cached/native equivalence requires a frozen visual encoder")
    visual.eval()
    weight = next(visual.parameters())
    if weight.dtype != torch.bfloat16:
        raise ValueError("Published visual caches require BF16 visual weights")
    clip = torch.from_numpy(np.array(frames, copy=True)).permute(0, 3, 1, 2)
    inputs = processor.video_processor(videos=[clip], do_sample_frames=False,
        size={"shortest_edge": 224**2, "longest_edge": 224**2}, return_tensors="pt")
    grid = inputs["video_grid_thw"]
    if grid.tolist() != [[(len(frames) + 1) // 2, 16, 16]]:
        raise ValueError("Native visual preprocessing differs from published cache geometry")
    with torch.autocast(device_type=weight.device.type, enabled=False):
        return qwen.get_video_features(
            inputs["pixel_values_videos"].to(device=weight.device, dtype=weight.dtype),
            grid.to(weight.device))


def resolve_visual(qwen, packet, active=True):
    request = packet.get("native_visual")
    cached = packet.get("visual_features")
    if request is None:
        return cached
    if cached is not None:
        raise ValueError("Choose native OR cached visual input, not both")
    if not active:
        from psi.models.interaction_arch.control_tokens import storage_weight
        weight = storage_weight(qwen.model.embed_tokens)
        return weight.new_zeros(((int(request["count"]) + 1) // 2 * 64, weight.shape[-1]))
    processor = getattr(qwen, "_hfd_native_visual_processor", None)
    if processor is None:
        from transformers import Qwen2_5OmniProcessor
        processor = Qwen2_5OmniProcessor.from_pretrained(qwen.config._name_or_path, local_files_only=True)
        qwen._hfd_native_visual_processor = processor
    return encode_frames(qwen, processor, request_frames(request))