Spaces:
Running on Zero
Running on Zero
| """PyAV-based video reader for qwen-vl-utils. | |
| The Space runtime ships torchvision >= 0.28, which removed | |
| `torchvision.io.read_video` — the only decoder qwen-vl-utils 0.0.14 can call | |
| without extra system libraries (torchcodec needs system FFmpeg shared libs; | |
| decord has no cp310 wheels). PyAV wheels bundle their own FFmpeg, so this | |
| reader reproduces qwen-vl-utils' official torchvision backend exactly: | |
| * frame-count/average-fps metadata from the container stream, | |
| * `smart_nframes` for the sampled-frame count (fps 2.0, min 4 / max 20, | |
| floor to factor 2 — identical because we reuse their own function), | |
| * `torch.linspace(0, total - 1, nframes).round().long()` index selection, | |
| * `sample_fps = nframes / total * video_fps`, | |
| * `video_metadata` dict with fps / frames_indices / total_num_frames. | |
| `fetch_video` then handles smart_resize/BICUBIC/antialias exactly as before — | |
| only the decode step is swapped, via `VIDEO_READER_BACKENDS["torchvision"]`. | |
| """ | |
| from __future__ import annotations | |
| import time | |
| from typing import Any, Dict, Tuple | |
| import av | |
| import torch | |
| def read_video_pyav(ele: Dict[str, Any]) -> Tuple[torch.Tensor, Dict[str, Any], float]: | |
| """Decode a video with PyAV, returning (TCHW uint8 tensor, metadata, sample_fps). | |
| Mirrors `qwen_vl_utils.vision_process._read_video_torchvision`. | |
| """ | |
| from qwen_vl_utils.vision_process import smart_nframes | |
| video_path = ele["video"] | |
| started = time.time() | |
| with av.open(video_path) as container: | |
| stream = container.streams.video[0] | |
| total_frames = stream.frames | |
| video_fps = float(stream.average_rate) if stream.average_rate else 0.0 | |
| if not total_frames: # some containers don't report a frame count | |
| total_frames = sum(1 for _ in container.decode(video=0)) | |
| container.seek(0) | |
| nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps) | |
| idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist() | |
| sample_fps = nframes / max(total_frames, 1e-6) * video_fps | |
| wanted = set(idx) | |
| frames = [] | |
| for position, frame in enumerate(container.decode(video=0)): | |
| if position > idx[-1]: | |
| break | |
| if position in wanted: | |
| # to_ndarray(format="rgb24") -> HWC uint8, matching read_video | |
| array = frame.to_ndarray(format="rgb24") | |
| frames.append(torch.from_numpy(array).permute(2, 0, 1)) | |
| video = torch.stack(frames).contiguous() | |
| if video.shape[0] != nframes: # truncated file: pad by repeating the last frame | |
| pad = torch.stack([video[-1]] * (nframes - video.shape[0])) | |
| video = torch.cat([video, pad]) | |
| metadata = { | |
| "fps": video_fps, | |
| "frames_indices": idx, | |
| "total_num_frames": total_frames, | |
| "video_backend": "pyav", | |
| } | |
| print(f"pyav: decoded {video.shape[0]} frames of {total_frames} in {time.time() - started:.2f}s", flush=True) | |
| return video, metadata, sample_fps |