Spaces:
Sleeping
Sleeping
Download iris/pipeline/visual.py from NextNeural/IRIS: direct link, hf CLI and curl.
- Browser
- Download file 3.5 kB
-
https://huggingface.co/spaces/NextNeural/IRIS/resolve/main/iris/pipeline/visual.py
- Command line
-
hf download hf://spaces/NextNeural/IRIS/iris/pipeline/visual.py
-
curl -L -o visual.py https://huggingface.co/spaces/NextNeural/IRIS/resolve/main/iris/pipeline/visual.py
3.5 kB
| from __future__ import annotations | |
| import subprocess | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from PIL import Image | |
| from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device | |
| from iris.pipeline.ffmpeg import resolve_ffmpeg_executable | |
| from iris.pipeline.search import SearchMatch, merge_nearby_matches | |
| class SampledFrame: | |
| path: Path | |
| timestamp: float | |
| def sample_video_frames(video_path: Path, frames_dir: Path, interval_seconds: float) -> list[SampledFrame]: | |
| frames_dir.mkdir(parents=True, exist_ok=True) | |
| existing = sorted(frames_dir.glob("frame_*.jpg")) | |
| interval_seconds = max(0.25, interval_seconds) | |
| if not existing: | |
| command = [ | |
| resolve_ffmpeg_executable(), | |
| "-y", | |
| "-i", | |
| str(video_path), | |
| "-vf", | |
| f"fps=1/{interval_seconds}", | |
| str(frames_dir / "frame_%06d.jpg"), | |
| ] | |
| try: | |
| subprocess.run(command, check=True, capture_output=True, text=True) | |
| except OSError as exc: | |
| raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc | |
| except subprocess.CalledProcessError as exc: | |
| detail = exc.stderr.strip() or exc.stdout.strip() or str(exc) | |
| raise RuntimeError(f"ffmpeg failed to sample frames: {detail}") from exc | |
| existing = sorted(frames_dir.glob("frame_*.jpg")) | |
| return [ | |
| SampledFrame(path=path, timestamp=index * interval_seconds) | |
| for index, path in enumerate(existing) | |
| ] | |
| class VisualSearcher: | |
| def __init__(self, settings: Settings) -> None: | |
| self.settings = settings | |
| self._classifier = None | |
| def search(self, query: str, frames: list[SampledFrame], limit: int) -> list[SearchMatch]: | |
| if not frames: | |
| return [] | |
| classifier = self._get_classifier() | |
| matches: list[SearchMatch] = [] | |
| candidate_labels = [query, f"not {query}", "unrelated scene"] | |
| for frame in frames: | |
| with Image.open(frame.path) as image: | |
| result = classifier(image.convert("RGB"), candidate_labels=candidate_labels) | |
| if not result: | |
| continue | |
| prompt_result = next((item for item in result if item.get("label") == query), result[0]) | |
| score = float(prompt_result["score"]) | |
| if score < self.settings.min_visual_score: | |
| continue | |
| matches.append( | |
| SearchMatch( | |
| start=frame.timestamp, | |
| end=frame.timestamp + self.settings.frame_sample_interval_seconds, | |
| text=f"Visual match for '{query}'", | |
| score=round(score, 4), | |
| source="visual", | |
| ) | |
| ) | |
| ranked = sorted(matches, key=lambda item: (-item.score, item.start))[: max(limit * 3, limit)] | |
| return merge_nearby_matches(ranked, max_gap_seconds=self.settings.frame_sample_interval_seconds)[:limit] | |
| def _get_classifier(self): | |
| if self._classifier is None: | |
| from transformers import pipeline | |
| self._classifier = pipeline( | |
| task="zero-shot-image-classification", | |
| model=self.settings.visual_model, | |
| device=transformers_pipeline_device(self.settings.device), | |
| dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype), | |
| ) | |
| return self._classifier | |