Download src/psi/data/seed/native_visual.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.29 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/native_visual.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/native_visual.py
-
curl -L -o native_visual.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/native_visual.py
5.29 kB
| """Frozen native camera frontend; same B1 geometry as the published caches. | |
| Raw requests carry paths, not future features/KV. Decoded RGB is a bounded | |
| CPU cache only; every requested packet is encoded online, outside autograd. | |
| """ | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from psi.data.seed.video_overlay import camera_plan, decode_video | |
| def visual_request(relation, index, count): | |
| dynamic = relation.get("camera_kind") == "dynamic" | |
| path = relation["video_path" if dynamic else "image_path"] | |
| if not path: | |
| raise ValueError("Native visual input needs an original image/video") | |
| return dict(path=str(path), kind="video" if dynamic else "image", | |
| index=int(index), count=int(count), | |
| duration=relation.get("camera_duration_seconds")) | |
| def source_visual_mode(source, mode): | |
| """Switch an existing live rollout fixture without mutating saved tensors.""" | |
| if mode not in ('cached', 'raw'): | |
| raise ValueError('visual_input_mode must be cached or raw') | |
| relation = source.get('multimodal_overlay') | |
| if relation is None: | |
| if mode == 'raw': | |
| raise ValueError('Legacy preassembled SWA has no native media contract') | |
| return source | |
| if relation.get('visual_input_mode','cached') == mode: | |
| return source | |
| from psi.data.seed.live_multimodal_overlay import attach_live_packets | |
| return attach_live_packets({**source, 'multimodal_overlay': {**relation,'visual_input_mode':mode}}) | |
| def _rgb(path, kind, duration, size, mtime_ns): | |
| if kind == "video": | |
| ticks, groups = camera_plan(float(duration)) | |
| frames, pts = decode_video(path, ticks, end_seconds=float(duration)) | |
| return frames, groups, pts | |
| if kind != "image": | |
| raise ValueError("Unknown native camera kind") | |
| from PIL import Image, ImageOps | |
| with Image.open(path) as image: | |
| frame = np.asarray(ImageOps.pad(image.convert("RGB"), (224, 224), color=(0, 0, 0))) | |
| return frame[None], None, None | |
| def request_frames(request): | |
| # CPU workers may have decoded the packet already. NumPy uint8 stays on | |
| # the CPU through Accelerate; do not ship an entire video to CUDA. | |
| if 'frames' in request: | |
| frames = request['frames'] | |
| if not isinstance(frames, np.ndarray) or frames.dtype != np.uint8 or frames.shape != (int(request['count']),224,224,3): | |
| raise ValueError('Malformed native RGB packet') | |
| return frames | |
| stat = Path(request["path"]).stat() | |
| frames, groups, _ = _rgb(request["path"], request["kind"], request.get("duration"), | |
| stat.st_size, stat.st_mtime_ns) | |
| count = int(request["count"]) | |
| if count not in (3, 4, 5): | |
| raise ValueError("Native camera requires the audited 8 Hz / 0.4s geometry") | |
| if groups is None: | |
| return np.repeat(frames, count, axis=0) | |
| index = int(request["index"]) | |
| if index < 0 or index >= len(groups): | |
| raise ValueError("Video ended before session; frozen EOF tails are forbidden") | |
| clip = frames[groups[index]] | |
| if len(clip) != count: | |
| raise ValueError("Native camera packet phase mismatch") | |
| return clip | |
| def encode_frames(qwen, processor, frames): | |
| """Do not pack B>1: historical BF16 packed vision failed exact parity.""" | |
| visual = qwen.visual | |
| if any(p.requires_grad for p in visual.parameters()): | |
| raise ValueError("Cached/native equivalence requires a frozen visual encoder") | |
| visual.eval() | |
| weight = next(visual.parameters()) | |
| if weight.dtype != torch.bfloat16: | |
| raise ValueError("Published visual caches require BF16 visual weights") | |
| clip = torch.from_numpy(np.array(frames, copy=True)).permute(0, 3, 1, 2) | |
| inputs = processor.video_processor(videos=[clip], do_sample_frames=False, | |
| size={"shortest_edge": 224**2, "longest_edge": 224**2}, return_tensors="pt") | |
| grid = inputs["video_grid_thw"] | |
| if grid.tolist() != [[(len(frames) + 1) // 2, 16, 16]]: | |
| raise ValueError("Native visual preprocessing differs from published cache geometry") | |
| with torch.autocast(device_type=weight.device.type, enabled=False): | |
| return qwen.get_video_features( | |
| inputs["pixel_values_videos"].to(device=weight.device, dtype=weight.dtype), | |
| grid.to(weight.device)) | |
| def resolve_visual(qwen, packet, active=True): | |
| request = packet.get("native_visual") | |
| cached = packet.get("visual_features") | |
| if request is None: | |
| return cached | |
| if cached is not None: | |
| raise ValueError("Choose native OR cached visual input, not both") | |
| if not active: | |
| from psi.models.interaction_arch.control_tokens import storage_weight | |
| weight = storage_weight(qwen.model.embed_tokens) | |
| return weight.new_zeros(((int(request["count"]) + 1) // 2 * 64, weight.shape[-1])) | |
| processor = getattr(qwen, "_hfd_native_visual_processor", None) | |
| if processor is None: | |
| from transformers import Qwen2_5OmniProcessor | |
| processor = Qwen2_5OmniProcessor.from_pretrained(qwen.config._name_or_path, local_files_only=True) | |
| qwen._hfd_native_visual_processor = processor | |
| return encode_frames(qwen, processor, request_frames(request)) | |