"""Frozen native camera frontend; same B1 geometry as the published caches. Raw requests carry paths, not future features/KV. Decoded RGB is a bounded CPU cache only; every requested packet is encoded online, outside autograd. """ from functools import lru_cache from pathlib import Path import numpy as np import torch from psi.data.seed.video_overlay import camera_plan, decode_video def visual_request(relation, index, count): dynamic = relation.get("camera_kind") == "dynamic" path = relation["video_path" if dynamic else "image_path"] if not path: raise ValueError("Native visual input needs an original image/video") return dict(path=str(path), kind="video" if dynamic else "image", index=int(index), count=int(count), duration=relation.get("camera_duration_seconds")) def source_visual_mode(source, mode): """Switch an existing live rollout fixture without mutating saved tensors.""" if mode not in ('cached', 'raw'): raise ValueError('visual_input_mode must be cached or raw') relation = source.get('multimodal_overlay') if relation is None: if mode == 'raw': raise ValueError('Legacy preassembled SWA has no native media contract') return source if relation.get('visual_input_mode','cached') == mode: return source from psi.data.seed.live_multimodal_overlay import attach_live_packets return attach_live_packets({**source, 'multimodal_overlay': {**relation,'visual_input_mode':mode}}) @lru_cache(maxsize=8) def _rgb(path, kind, duration, size, mtime_ns): if kind == "video": ticks, groups = camera_plan(float(duration)) frames, pts = decode_video(path, ticks, end_seconds=float(duration)) return frames, groups, pts if kind != "image": raise ValueError("Unknown native camera kind") from PIL import Image, ImageOps with Image.open(path) as image: frame = np.asarray(ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0))) return frame[None], None, None def request_frames(request): # CPU workers may have decoded the packet already. NumPy uint8 stays on # the CPU through Accelerate; do not ship an entire video to CUDA. if 'frames' in request: frames = request['frames'] if not isinstance(frames, np.ndarray) or frames.dtype != np.uint8 or frames.shape != (int(request['count']),224,224,3): raise ValueError('Malformed native RGB packet') return frames stat = Path(request["path"]).stat() frames, groups, _ = _rgb(request["path"], request["kind"], request.get("duration"), stat.st_size, stat.st_mtime_ns) count = int(request["count"]) if count not in (3, 4, 5): raise ValueError("Native camera requires the audited 8 Hz / 0.4s geometry") if groups is None: return np.repeat(frames, count, axis=0) index = int(request["index"]) if index < 0 or index >= len(groups): raise ValueError("Video ended before session; frozen EOF tails are forbidden") clip = frames[groups[index]] if len(clip) != count: raise ValueError("Native camera packet phase mismatch") return clip @torch.no_grad() def encode_frames(qwen, processor, frames): """Do not pack B>1: historical BF16 packed vision failed exact parity.""" visual = qwen.visual if any(p.requires_grad for p in visual.parameters()): raise ValueError("Cached/native equivalence requires a frozen visual encoder") visual.eval() weight = next(visual.parameters()) if weight.dtype != torch.bfloat16: raise ValueError("Published visual caches require BF16 visual weights") clip = torch.from_numpy(np.array(frames, copy=True)).permute(0, 3, 1, 2) inputs = processor.video_processor(videos=[clip], do_sample_frames=False, size={"shortest_edge": 224**2, "longest_edge": 224**2}, return_tensors="pt") grid = inputs["video_grid_thw"] if grid.tolist() != [[(len(frames) + 1) // 2, 16, 16]]: raise ValueError("Native visual preprocessing differs from published cache geometry") with torch.autocast(device_type=weight.device.type, enabled=False): return qwen.get_video_features( inputs["pixel_values_videos"].to(device=weight.device, dtype=weight.dtype), grid.to(weight.device)) def resolve_visual(qwen, packet, active=True): request = packet.get("native_visual") cached = packet.get("visual_features") if request is None: return cached if cached is not None: raise ValueError("Choose native OR cached visual input, not both") if not active: from psi.models.interaction_arch.control_tokens import storage_weight weight = storage_weight(qwen.model.embed_tokens) return weight.new_zeros(((int(request["count"]) + 1) // 2 * 64, weight.shape[-1])) processor = getattr(qwen, "_hfd_native_visual_processor", None) if processor is None: from transformers import Qwen2_5OmniProcessor processor = Qwen2_5OmniProcessor.from_pretrained(qwen.config._name_or_path, local_files_only=True) qwen._hfd_native_visual_processor = processor return encode_frames(qwen, processor, request_frames(request))