from __future__ import annotations import subprocess from dataclasses import dataclass from pathlib import Path from PIL import Image from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device from iris.pipeline.ffmpeg import resolve_ffmpeg_executable from iris.pipeline.search import SearchMatch, merge_nearby_matches @dataclass(frozen=True) class SampledFrame: path: Path timestamp: float def sample_video_frames(video_path: Path, frames_dir: Path, interval_seconds: float) -> list[SampledFrame]: frames_dir.mkdir(parents=True, exist_ok=True) existing = sorted(frames_dir.glob("frame_*.jpg")) interval_seconds = max(0.25, interval_seconds) if not existing: command = [ resolve_ffmpeg_executable(), "-y", "-i", str(video_path), "-vf", f"fps=1/{interval_seconds}", str(frames_dir / "frame_%06d.jpg"), ] try: subprocess.run(command, check=True, capture_output=True, text=True) except OSError as exc: raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc except subprocess.CalledProcessError as exc: detail = exc.stderr.strip() or exc.stdout.strip() or str(exc) raise RuntimeError(f"ffmpeg failed to sample frames: {detail}") from exc existing = sorted(frames_dir.glob("frame_*.jpg")) return [ SampledFrame(path=path, timestamp=index * interval_seconds) for index, path in enumerate(existing) ] class VisualSearcher: def __init__(self, settings: Settings) -> None: self.settings = settings self._classifier = None def search(self, query: str, frames: list[SampledFrame], limit: int) -> list[SearchMatch]: if not frames: return [] classifier = self._get_classifier() matches: list[SearchMatch] = [] candidate_labels = [query, f"not {query}", "unrelated scene"] for frame in frames: with Image.open(frame.path) as image: result = classifier(image.convert("RGB"), candidate_labels=candidate_labels) if not result: continue prompt_result = next((item for item in result if item.get("label") == query), result[0]) score = float(prompt_result["score"]) if score < self.settings.min_visual_score: continue matches.append( SearchMatch( start=frame.timestamp, end=frame.timestamp + self.settings.frame_sample_interval_seconds, text=f"Visual match for '{query}'", score=round(score, 4), source="visual", ) ) ranked = sorted(matches, key=lambda item: (-item.score, item.start))[: max(limit * 3, limit)] return merge_nearby_matches(ranked, max_gap_seconds=self.settings.frame_sample_interval_seconds)[:limit] def _get_classifier(self): if self._classifier is None: from transformers import pipeline self._classifier = pipeline( task="zero-shot-image-classification", model=self.settings.visual_model, device=transformers_pipeline_device(self.settings.device), dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype), ) return self._classifier