File size: 5,287 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | """Frozen native camera frontend; same B1 geometry as the published caches.
Raw requests carry paths, not future features/KV. Decoded RGB is a bounded
CPU cache only; every requested packet is encoded online, outside autograd.
"""
from functools import lru_cache
from pathlib import Path
import numpy as np
import torch
from psi.data.seed.video_overlay import camera_plan, decode_video
def visual_request(relation, index, count):
dynamic = relation.get("camera_kind") == "dynamic"
path = relation["video_path" if dynamic else "image_path"]
if not path:
raise ValueError("Native visual input needs an original image/video")
return dict(path=str(path), kind="video" if dynamic else "image",
index=int(index), count=int(count),
duration=relation.get("camera_duration_seconds"))
def source_visual_mode(source, mode):
"""Switch an existing live rollout fixture without mutating saved tensors."""
if mode not in ('cached', 'raw'):
raise ValueError('visual_input_mode must be cached or raw')
relation = source.get('multimodal_overlay')
if relation is None:
if mode == 'raw':
raise ValueError('Legacy preassembled SWA has no native media contract')
return source
if relation.get('visual_input_mode','cached') == mode:
return source
from psi.data.seed.live_multimodal_overlay import attach_live_packets
return attach_live_packets({**source, 'multimodal_overlay': {**relation,'visual_input_mode':mode}})
@lru_cache(maxsize=8)
def _rgb(path, kind, duration, size, mtime_ns):
if kind == "video":
ticks, groups = camera_plan(float(duration))
frames, pts = decode_video(path, ticks, end_seconds=float(duration))
return frames, groups, pts
if kind != "image":
raise ValueError("Unknown native camera kind")
from PIL import Image, ImageOps
with Image.open(path) as image:
frame = np.asarray(ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0)))
return frame[None], None, None
def request_frames(request):
# CPU workers may have decoded the packet already. NumPy uint8 stays on
# the CPU through Accelerate; do not ship an entire video to CUDA.
if 'frames' in request:
frames = request['frames']
if not isinstance(frames, np.ndarray) or frames.dtype != np.uint8 or frames.shape != (int(request['count']),224,224,3):
raise ValueError('Malformed native RGB packet')
return frames
stat = Path(request["path"]).stat()
frames, groups, _ = _rgb(request["path"], request["kind"], request.get("duration"),
stat.st_size, stat.st_mtime_ns)
count = int(request["count"])
if count not in (3, 4, 5):
raise ValueError("Native camera requires the audited 8 Hz / 0.4s geometry")
if groups is None:
return np.repeat(frames, count, axis=0)
index = int(request["index"])
if index < 0 or index >= len(groups):
raise ValueError("Video ended before session; frozen EOF tails are forbidden")
clip = frames[groups[index]]
if len(clip) != count:
raise ValueError("Native camera packet phase mismatch")
return clip
@torch.no_grad()
def encode_frames(qwen, processor, frames):
"""Do not pack B>1: historical BF16 packed vision failed exact parity."""
visual = qwen.visual
if any(p.requires_grad for p in visual.parameters()):
raise ValueError("Cached/native equivalence requires a frozen visual encoder")
visual.eval()
weight = next(visual.parameters())
if weight.dtype != torch.bfloat16:
raise ValueError("Published visual caches require BF16 visual weights")
clip = torch.from_numpy(np.array(frames, copy=True)).permute(0, 3, 1, 2)
inputs = processor.video_processor(videos=[clip], do_sample_frames=False,
size={"shortest_edge": 224**2, "longest_edge": 224**2}, return_tensors="pt")
grid = inputs["video_grid_thw"]
if grid.tolist() != [[(len(frames) + 1) // 2, 16, 16]]:
raise ValueError("Native visual preprocessing differs from published cache geometry")
with torch.autocast(device_type=weight.device.type, enabled=False):
return qwen.get_video_features(
inputs["pixel_values_videos"].to(device=weight.device, dtype=weight.dtype),
grid.to(weight.device))
def resolve_visual(qwen, packet, active=True):
request = packet.get("native_visual")
cached = packet.get("visual_features")
if request is None:
return cached
if cached is not None:
raise ValueError("Choose native OR cached visual input, not both")
if not active:
from psi.models.interaction_arch.control_tokens import storage_weight
weight = storage_weight(qwen.model.embed_tokens)
return weight.new_zeros(((int(request["count"]) + 1) // 2 * 64, weight.shape[-1]))
processor = getattr(qwen, "_hfd_native_visual_processor", None)
if processor is None:
from transformers import Qwen2_5OmniProcessor
processor = Qwen2_5OmniProcessor.from_pretrained(qwen.config._name_or_path, local_files_only=True)
qwen._hfd_native_visual_processor = processor
return encode_frames(qwen, processor, request_frames(request))
|