IRIS / iris /pipeline /visual.py
CompleteKris's picture
Bundle IRIS core for Space build
b2300e2
Raw History Blame Contribute Delete
3.5 kB
from __future__ import annotations
import subprocess
from dataclasses import dataclass
from pathlib import Path
from PIL import Image
from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device
from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
from iris.pipeline.search import SearchMatch, merge_nearby_matches
@dataclass(frozen=True)
class SampledFrame:
path: Path
timestamp: float
def sample_video_frames(video_path: Path, frames_dir: Path, interval_seconds: float) -> list[SampledFrame]:
frames_dir.mkdir(parents=True, exist_ok=True)
existing = sorted(frames_dir.glob("frame_*.jpg"))
interval_seconds = max(0.25, interval_seconds)
if not existing:
command = [
resolve_ffmpeg_executable(),
"-y",
"-i",
str(video_path),
"-vf",
f"fps=1/{interval_seconds}",
str(frames_dir / "frame_%06d.jpg"),
]
try:
subprocess.run(command, check=True, capture_output=True, text=True)
except OSError as exc:
raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
except subprocess.CalledProcessError as exc:
detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
raise RuntimeError(f"ffmpeg failed to sample frames: {detail}") from exc
existing = sorted(frames_dir.glob("frame_*.jpg"))
return [
SampledFrame(path=path, timestamp=index * interval_seconds)
for index, path in enumerate(existing)
]
class VisualSearcher:
def __init__(self, settings: Settings) -> None:
self.settings = settings
self._classifier = None
def search(self, query: str, frames: list[SampledFrame], limit: int) -> list[SearchMatch]:
if not frames:
return []
classifier = self._get_classifier()
matches: list[SearchMatch] = []
candidate_labels = [query, f"not {query}", "unrelated scene"]
for frame in frames:
with Image.open(frame.path) as image:
result = classifier(image.convert("RGB"), candidate_labels=candidate_labels)
if not result:
continue
prompt_result = next((item for item in result if item.get("label") == query), result[0])
score = float(prompt_result["score"])
if score < self.settings.min_visual_score:
continue
matches.append(
SearchMatch(
start=frame.timestamp,
end=frame.timestamp + self.settings.frame_sample_interval_seconds,
text=f"Visual match for '{query}'",
score=round(score, 4),
source="visual",
)
)
ranked = sorted(matches, key=lambda item: (-item.score, item.start))[: max(limit * 3, limit)]
return merge_nearby_matches(ranked, max_gap_seconds=self.settings.frame_sample_interval_seconds)[:limit]
def _get_classifier(self):
if self._classifier is None:
from transformers import pipeline
self._classifier = pipeline(
task="zero-shot-image-classification",
model=self.settings.visual_model,
device=transformers_pipeline_device(self.settings.device),
dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype),
)
return self._classifier