CompleteKris commited on
Commit
b2300e2
·
1 Parent(s): 9fca884

Bundle IRIS core for Space build

Browse files
README.md CHANGED
@@ -14,9 +14,9 @@ short_description: Intelligent search for spoken moments inside video.
14
 
15
  # IRIS
16
 
17
- IRIS is a Gradio demo for searching videos by spoken content. It references
18
- [`NextNeural/IRIS-core`](https://huggingface.co/NextNeural/IRIS-core) for the
19
- processing pipeline, transcription, embeddings, and search behavior.
20
 
21
  Upload a video, enter a query, and IRIS returns matching timestamp windows.
22
  This Space defaults to CPU-friendly settings:
 
14
 
15
  # IRIS
16
 
17
+ IRIS is a Gradio demo for searching videos by spoken content. The Space mirrors
18
+ the `iris/` package from `../IRIS-core` so Hugging Face can build it without
19
+ cloning a private dependency during installation.
20
 
21
  Upload a video, enter a query, and IRIS returns matching timestamp windows.
22
  This Space defaults to CPU-friendly settings:
iris/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """IRIS video search package."""
2
+
3
+ __all__ = ["__version__"]
4
+
5
+ __version__ = "0.1.0"
6
+
iris/api/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ """FastAPI application package."""
2
+
iris/api/dependencies.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from fastapi import Request
4
+
5
+ from iris.config import Settings
6
+ from iris.pipeline.orchestrator import VideoProcessor
7
+ from iris.storage.database import Database
8
+ from iris.storage.files import FileStore
9
+
10
+
11
+ def get_settings(request: Request) -> Settings:
12
+ return request.app.state.settings
13
+
14
+
15
+ def get_database(request: Request) -> Database:
16
+ return request.app.state.database
17
+
18
+
19
+ def get_file_store(request: Request) -> FileStore:
20
+ return request.app.state.file_store
21
+
22
+
23
+ def get_video_processor(request: Request) -> VideoProcessor:
24
+ return request.app.state.video_processor
25
+
iris/api/main.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from fastapi import FastAPI
4
+
5
+ from iris.api.routes import router
6
+ from iris.config import Settings
7
+ from iris.pipeline.orchestrator import VideoProcessor
8
+ from iris.storage.database import Database
9
+ from iris.storage.files import FileStore
10
+
11
+
12
+ def create_app(settings: Settings | None = None) -> FastAPI:
13
+ app_settings = settings or Settings.from_env()
14
+ app_settings.ensure_directories()
15
+
16
+ database = Database(app_settings.database_path)
17
+ database.init()
18
+ file_store = FileStore(app_settings)
19
+ processor = VideoProcessor(app_settings, database, file_store)
20
+
21
+ app = FastAPI(title="IRIS", version="0.1.0")
22
+ app.state.settings = app_settings
23
+ app.state.database = database
24
+ app.state.file_store = file_store
25
+ app.state.video_processor = processor
26
+ app.include_router(router)
27
+ return app
28
+
29
+
30
+ app = create_app()
31
+
iris/api/routes.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, UploadFile, status
6
+
7
+ from iris.api.dependencies import get_database, get_file_store, get_settings, get_video_processor
8
+ from iris.config import Settings, device_info
9
+ from iris.models.schemas import (
10
+ SearchRequest,
11
+ SearchResponse,
12
+ SearchResultResponse,
13
+ TranscriptChunkResponse,
14
+ TranscriptResponse,
15
+ UploadVideoResponse,
16
+ VideoStatusResponse,
17
+ )
18
+ from iris.pipeline.chunk import window_transcript_chunks
19
+ from iris.pipeline.embed import LocalEmbedder
20
+ from iris.pipeline.search import detail_window_settings, format_timestamp, rank_speech_chunks
21
+ from iris.pipeline.visual import VisualSearcher, sample_video_frames
22
+ from iris.storage.database import Database
23
+ from iris.storage.files import FileStore
24
+ from iris.pipeline.orchestrator import VideoProcessor
25
+
26
+ router = APIRouter()
27
+
28
+
29
+ @router.get("/health")
30
+ def health(settings: Settings = Depends(get_settings)) -> dict:
31
+ return {"status": "ok", "device": device_info(settings.device, settings.torch_dtype)}
32
+
33
+
34
+ @router.post("/videos", response_model=UploadVideoResponse, status_code=status.HTTP_202_ACCEPTED)
35
+ def upload_video(
36
+ file: UploadFile,
37
+ background_tasks: BackgroundTasks,
38
+ database: Database = Depends(get_database),
39
+ file_store: FileStore = Depends(get_file_store),
40
+ processor: VideoProcessor = Depends(get_video_processor),
41
+ ) -> UploadVideoResponse:
42
+ try:
43
+ video_id, path = file_store.save_upload(file)
44
+ except ValueError as exc:
45
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
46
+
47
+ database.create_video(video_id, file.filename or path.name, path)
48
+ if getattr(processor, "process_video", None) and processor is not None:
49
+ background_tasks.add_task(processor.process_video, video_id)
50
+ return UploadVideoResponse(video_id=video_id, status="queued")
51
+
52
+
53
+ @router.post("/videos/{video_id}/process", response_model=UploadVideoResponse, status_code=status.HTTP_202_ACCEPTED)
54
+ def process_video_again(
55
+ video_id: str,
56
+ background_tasks: BackgroundTasks,
57
+ database: Database = Depends(get_database),
58
+ processor: VideoProcessor = Depends(get_video_processor),
59
+ ) -> UploadVideoResponse:
60
+ _get_video_or_404(database, video_id)
61
+ database.update_video_status(video_id, "queued")
62
+ background_tasks.add_task(processor.process_video, video_id)
63
+ return UploadVideoResponse(video_id=video_id, status="queued")
64
+
65
+
66
+ @router.get("/videos/{video_id}/status", response_model=VideoStatusResponse)
67
+ def get_video_status(
68
+ video_id: str,
69
+ database: Database = Depends(get_database),
70
+ ) -> VideoStatusResponse:
71
+ record = _get_video_or_404(database, video_id)
72
+ return VideoStatusResponse(video_id=video_id, status=record["status"], error=record["error"])
73
+
74
+
75
+ @router.get("/videos/{video_id}/transcript", response_model=TranscriptResponse)
76
+ def get_transcript(
77
+ video_id: str,
78
+ database: Database = Depends(get_database),
79
+ ) -> TranscriptResponse:
80
+ _get_video_or_404(database, video_id)
81
+ chunks = database.list_transcript_chunks(video_id)
82
+ return TranscriptResponse(
83
+ video_id=video_id,
84
+ chunks=[TranscriptChunkResponse(start=chunk.start, end=chunk.end, text=chunk.text) for chunk in chunks],
85
+ )
86
+
87
+
88
+ @router.post("/videos/{video_id}/search", response_model=SearchResponse)
89
+ def search_video(
90
+ video_id: str,
91
+ request: SearchRequest,
92
+ database: Database = Depends(get_database),
93
+ file_store: FileStore = Depends(get_file_store),
94
+ settings: Settings = Depends(get_settings),
95
+ ) -> SearchResponse:
96
+ record = _get_video_or_404(database, video_id)
97
+ if record["status"] != "completed":
98
+ raise HTTPException(
99
+ status_code=status.HTTP_409_CONFLICT,
100
+ detail=f"Video is not ready for search. Current status: {record['status']}.",
101
+ )
102
+
103
+ atomic_chunks = database.list_transcript_chunks(video_id)
104
+ default_window, default_overlap, default_merge_gap = detail_window_settings(request.detail)
105
+ window_seconds = request.window_seconds or default_window
106
+ overlap_seconds = min(default_overlap, max(0.0, window_seconds - 0.1))
107
+ merge_gap_seconds = request.merge_gap_seconds if request.merge_gap_seconds is not None else default_merge_gap
108
+ chunks = window_transcript_chunks(
109
+ atomic_chunks,
110
+ window_seconds=window_seconds,
111
+ overlap_seconds=overlap_seconds,
112
+ )
113
+
114
+ embedder = LocalEmbedder(settings)
115
+ chunk_embeddings = embedder.encode([chunk.text for chunk in chunks])
116
+ query_embedding = embedder.encode([request.query])[0]
117
+ speech_results = rank_speech_chunks(
118
+ query=request.query,
119
+ chunks=chunks,
120
+ chunk_embeddings=chunk_embeddings,
121
+ query_embedding=query_embedding,
122
+ settings=settings,
123
+ limit=request.limit,
124
+ merge_gap_seconds=merge_gap_seconds,
125
+ )
126
+
127
+ results = list(speech_results)
128
+ if request.include_visual:
129
+ if not settings.visual_search_enabled:
130
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Visual search is disabled.")
131
+ frames = sample_video_frames(
132
+ Path(record["video_path"]),
133
+ file_store.frames_dir(video_id),
134
+ settings.frame_sample_interval_seconds,
135
+ )
136
+ visual_results = VisualSearcher(settings).search(request.query, frames, request.limit)
137
+ results.extend(visual_results)
138
+
139
+ database.record_search(video_id, request.query, request.include_visual)
140
+ ranked = sorted(results, key=lambda item: (-item.score, item.start))[: request.limit]
141
+ return SearchResponse(
142
+ video_id=video_id,
143
+ query=request.query,
144
+ results=[
145
+ SearchResultResponse(
146
+ start=item.start,
147
+ end=item.end,
148
+ start_time=format_timestamp(item.start),
149
+ end_time=format_timestamp(item.end),
150
+ timestamp=f"{format_timestamp(item.start)}-{format_timestamp(item.end)}",
151
+ text=item.text,
152
+ score=item.score,
153
+ source=item.source,
154
+ )
155
+ for item in ranked
156
+ ],
157
+ )
158
+
159
+
160
+ def _get_video_or_404(database: Database, video_id: str) -> dict:
161
+ record = database.get_video(video_id)
162
+ if record is None:
163
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Video not found.")
164
+ return record
iris/config.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+
7
+
8
+ def _bool_env(name: str, default: bool) -> bool:
9
+ value = os.getenv(name)
10
+ if value is None:
11
+ return default
12
+ return value.strip().lower() in {"1", "true", "yes", "on"}
13
+
14
+
15
+ def _float_env(name: str, default: float) -> float:
16
+ value = os.getenv(name)
17
+ return default if value is None else float(value)
18
+
19
+
20
+ def _int_env(name: str, default: int) -> int:
21
+ value = os.getenv(name)
22
+ return default if value is None else int(value)
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class Settings:
27
+ data_dir: Path
28
+ database_path: Path
29
+ asr_model: str
30
+ asr_timestamps: str
31
+ embedding_model: str
32
+ visual_model: str
33
+ device: str
34
+ torch_dtype: str
35
+ chunk_length_seconds: float
36
+ chunk_overlap_seconds: float
37
+ frame_sample_interval_seconds: float
38
+ max_audio_seconds: float | None
39
+ visual_search_enabled: bool
40
+ keyword_boost: float
41
+ semantic_weight: float
42
+ keyword_weight: float
43
+ min_speech_score: float
44
+ min_visual_score: float
45
+
46
+ @classmethod
47
+ def from_env(cls) -> "Settings":
48
+ data_dir = Path(os.getenv("IRIS_DATA_DIR", ".iris-data")).resolve()
49
+ return cls(
50
+ data_dir=data_dir,
51
+ database_path=Path(os.getenv("IRIS_DATABASE_PATH", data_dir / "iris.sqlite3")).resolve(),
52
+ asr_model=os.getenv("IRIS_ASR_MODEL", "openai/whisper-large-v3-turbo"),
53
+ asr_timestamps=os.getenv("IRIS_ASR_TIMESTAMPS", "word"),
54
+ embedding_model=os.getenv("IRIS_EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2"),
55
+ visual_model=os.getenv("IRIS_VISUAL_MODEL", "openai/clip-vit-large-patch14"),
56
+ device=os.getenv("IRIS_DEVICE", "auto"),
57
+ torch_dtype=os.getenv("IRIS_TORCH_DTYPE", "auto"),
58
+ chunk_length_seconds=_float_env("IRIS_CHUNK_LENGTH_SECONDS", 24.0),
59
+ chunk_overlap_seconds=_float_env("IRIS_CHUNK_OVERLAP_SECONDS", 4.0),
60
+ frame_sample_interval_seconds=_float_env("IRIS_FRAME_SAMPLE_INTERVAL_SECONDS", 2.0),
61
+ max_audio_seconds=_optional_float_env("IRIS_MAX_AUDIO_SECONDS"),
62
+ visual_search_enabled=_bool_env("IRIS_VISUAL_SEARCH_ENABLED", True),
63
+ keyword_boost=_float_env("IRIS_KEYWORD_BOOST", 0.15),
64
+ semantic_weight=_float_env("IRIS_SEMANTIC_WEIGHT", 0.78),
65
+ keyword_weight=_float_env("IRIS_KEYWORD_WEIGHT", 0.22),
66
+ min_speech_score=_float_env("IRIS_MIN_SPEECH_SCORE", 0.55),
67
+ min_visual_score=_float_env("IRIS_MIN_VISUAL_SCORE", 0.45),
68
+ )
69
+
70
+ def ensure_directories(self) -> None:
71
+ self.data_dir.mkdir(parents=True, exist_ok=True)
72
+ self.database_path.parent.mkdir(parents=True, exist_ok=True)
73
+
74
+
75
+ def resolve_device(device: str) -> str:
76
+ if device != "auto":
77
+ return device
78
+ try:
79
+ import torch
80
+
81
+ return "cuda" if torch.cuda.is_available() else "cpu"
82
+ except Exception:
83
+ return "cpu"
84
+
85
+
86
+ def transformers_pipeline_device(device: str) -> int:
87
+ resolved = resolve_device(device)
88
+ return 0 if resolved == "cuda" else -1
89
+
90
+
91
+ def resolve_torch_dtype(device: str, dtype: str):
92
+ normalized = dtype.strip().lower()
93
+ resolved_device = resolve_device(device)
94
+ if normalized == "auto":
95
+ normalized = "float16" if resolved_device == "cuda" else "float32"
96
+
97
+ try:
98
+ import torch
99
+
100
+ if normalized in {"float16", "fp16", "half"}:
101
+ return torch.float16
102
+ if normalized in {"bfloat16", "bf16"}:
103
+ return torch.bfloat16
104
+ if normalized in {"float32", "fp32", "full"}:
105
+ return torch.float32
106
+ except Exception:
107
+ pass
108
+ return "auto"
109
+
110
+
111
+ def device_info(device: str, dtype: str) -> dict:
112
+ resolved = resolve_device(device)
113
+ info = {
114
+ "requested_device": device,
115
+ "resolved_device": resolved,
116
+ "requested_dtype": dtype,
117
+ "cuda_available": False,
118
+ "cuda_version": None,
119
+ "gpu_count": 0,
120
+ "gpu_name": None,
121
+ }
122
+ try:
123
+ import torch
124
+
125
+ info.update(
126
+ {
127
+ "torch_version": torch.__version__,
128
+ "cuda_available": torch.cuda.is_available(),
129
+ "cuda_version": torch.version.cuda,
130
+ "gpu_count": torch.cuda.device_count(),
131
+ "gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
132
+ }
133
+ )
134
+ except Exception as exc:
135
+ info["torch_error"] = str(exc)
136
+ return info
137
+
138
+
139
+ def _optional_float_env(name: str) -> float | None:
140
+ value = os.getenv(name)
141
+ if value is None or value.strip() == "":
142
+ return None
143
+ return float(value)
iris/models/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ """Pydantic API models."""
2
+
iris/models/schemas.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Literal
4
+
5
+ from pydantic import BaseModel, Field
6
+
7
+
8
+ VideoStatus = Literal["queued", "processing", "completed", "failed"]
9
+ ResultSource = Literal["speech", "visual"]
10
+ SearchDetail = Literal["fine", "medium", "coarse"]
11
+
12
+
13
+ class UploadVideoResponse(BaseModel):
14
+ video_id: str
15
+ status: VideoStatus
16
+
17
+
18
+ class VideoStatusResponse(BaseModel):
19
+ video_id: str
20
+ status: VideoStatus
21
+ error: str | None = None
22
+
23
+
24
+ class TranscriptChunkResponse(BaseModel):
25
+ start: float
26
+ end: float
27
+ text: str
28
+
29
+
30
+ class TranscriptResponse(BaseModel):
31
+ video_id: str
32
+ chunks: list[TranscriptChunkResponse]
33
+
34
+
35
+ class SearchRequest(BaseModel):
36
+ query: str = Field(min_length=1)
37
+ include_visual: bool = False
38
+ limit: int = Field(default=10, ge=1, le=100)
39
+ detail: SearchDetail = "fine"
40
+ window_seconds: float | None = Field(default=None, ge=2.0, le=120.0)
41
+ merge_gap_seconds: float | None = Field(default=None, ge=0.0, le=30.0)
42
+
43
+
44
+ class SearchResultResponse(BaseModel):
45
+ start: float
46
+ end: float
47
+ start_time: str
48
+ end_time: str
49
+ timestamp: str
50
+ text: str
51
+ score: float
52
+ source: ResultSource
53
+
54
+
55
+ class SearchResponse(BaseModel):
56
+ video_id: str
57
+ query: str
58
+ results: list[SearchResultResponse]
iris/pipeline/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ """IRIS processing pipeline."""
2
+
iris/pipeline/audio.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import subprocess
4
+ from pathlib import Path
5
+
6
+ from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
7
+
8
+
9
+ def extract_audio(video_path: Path, audio_path: Path, max_duration_seconds: float | None = None) -> Path:
10
+ audio_path.parent.mkdir(parents=True, exist_ok=True)
11
+ command = [
12
+ resolve_ffmpeg_executable(),
13
+ "-y",
14
+ "-i",
15
+ str(video_path),
16
+ "-vn",
17
+ "-ac",
18
+ "1",
19
+ "-ar",
20
+ "16000",
21
+ ]
22
+ if max_duration_seconds is not None:
23
+ command.extend(["-t", str(max_duration_seconds)])
24
+ command.append(str(audio_path))
25
+ try:
26
+ subprocess.run(command, check=True, capture_output=True, text=True)
27
+ except OSError as exc:
28
+ raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
29
+ except subprocess.CalledProcessError as exc:
30
+ detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
31
+ raise RuntimeError(f"ffmpeg failed to extract audio: {detail}") from exc
32
+ return audio_path
iris/pipeline/chunk.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Iterable
5
+
6
+
7
+ @dataclass(frozen=True)
8
+ class TranscriptChunk:
9
+ start: float
10
+ end: float
11
+ text: str
12
+
13
+
14
+ def normalize_asr_chunks(raw_output: dict) -> list[TranscriptChunk]:
15
+ raw_chunks = raw_output.get("chunks") or []
16
+ normalized: list[TranscriptChunk] = []
17
+ for item in raw_chunks:
18
+ timestamp = item.get("timestamp")
19
+ text = (item.get("text") or "").strip()
20
+ if not text or not timestamp or len(timestamp) != 2:
21
+ continue
22
+ start, end = timestamp
23
+ if start is None or end is None:
24
+ continue
25
+ start_f = max(0.0, float(start))
26
+ end_f = max(start_f, float(end))
27
+ normalized.append(TranscriptChunk(start=start_f, end=end_f, text=text))
28
+
29
+ if normalized:
30
+ return normalized
31
+
32
+ text = (raw_output.get("text") or "").strip()
33
+ return [TranscriptChunk(start=0.0, end=0.0, text=text)] if text else []
34
+
35
+
36
+ def window_transcript_chunks(
37
+ chunks: Iterable[TranscriptChunk],
38
+ window_seconds: float,
39
+ overlap_seconds: float,
40
+ ) -> list[TranscriptChunk]:
41
+ source = sorted(chunks, key=lambda chunk: (chunk.start, chunk.end))
42
+ if not source:
43
+ return []
44
+
45
+ window_seconds = max(1.0, window_seconds)
46
+ overlap_seconds = max(0.0, min(overlap_seconds, window_seconds - 0.1))
47
+ windows: list[TranscriptChunk] = []
48
+
49
+ current: list[TranscriptChunk] = []
50
+ current_start = source[0].start
51
+
52
+ for chunk in source:
53
+ if current and chunk.end - current_start > window_seconds:
54
+ windows.append(_merge_text_window(current))
55
+ keep_after = max(current[-1].end - overlap_seconds, current_start)
56
+ current = [item for item in current if item.end >= keep_after]
57
+ current_start = current[0].start if current else chunk.start
58
+ current.append(chunk)
59
+
60
+ if current:
61
+ windows.append(_merge_text_window(current))
62
+
63
+ return _dedupe_windows(windows)
64
+
65
+
66
+ def _merge_text_window(chunks: list[TranscriptChunk]) -> TranscriptChunk:
67
+ text = " ".join(chunk.text.strip() for chunk in chunks if chunk.text.strip())
68
+ return TranscriptChunk(start=chunks[0].start, end=chunks[-1].end, text=text)
69
+
70
+
71
+ def _dedupe_windows(windows: list[TranscriptChunk]) -> list[TranscriptChunk]:
72
+ seen: set[tuple[float, float, str]] = set()
73
+ deduped: list[TranscriptChunk] = []
74
+ for window in windows:
75
+ key = (round(window.start, 3), round(window.end, 3), window.text)
76
+ if key not in seen:
77
+ seen.add(key)
78
+ deduped.append(window)
79
+ return deduped
80
+
iris/pipeline/embed.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ import numpy as np
6
+
7
+ from iris.config import Settings, resolve_device
8
+
9
+
10
+ class LocalEmbedder:
11
+ def __init__(self, settings: Settings) -> None:
12
+ self.settings = settings
13
+ self._model = None
14
+
15
+ def encode(self, texts: list[str]) -> np.ndarray:
16
+ if not texts:
17
+ return np.empty((0, 0), dtype=np.float32)
18
+ model = self._get_model()
19
+ vectors = model.encode(
20
+ texts,
21
+ convert_to_numpy=True,
22
+ normalize_embeddings=True,
23
+ show_progress_bar=False,
24
+ )
25
+ return np.asarray(vectors, dtype=np.float32)
26
+
27
+ def _get_model(self):
28
+ if self._model is None:
29
+ from sentence_transformers import SentenceTransformer
30
+
31
+ self._model = SentenceTransformer(
32
+ self.settings.embedding_model,
33
+ device=resolve_device(self.settings.device),
34
+ )
35
+ return self._model
36
+
37
+
38
+ def save_embeddings(path: Path, embeddings: np.ndarray) -> None:
39
+ path.parent.mkdir(parents=True, exist_ok=True)
40
+ np.save(path, embeddings)
41
+
42
+
43
+ def load_embeddings(path: Path) -> np.ndarray:
44
+ return np.load(path)
45
+
iris/pipeline/ffmpeg.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import shutil
4
+
5
+
6
+ def resolve_ffmpeg_executable() -> str:
7
+ executable = shutil.which("ffmpeg")
8
+ if executable:
9
+ return executable
10
+
11
+ try:
12
+ import imageio_ffmpeg
13
+
14
+ return imageio_ffmpeg.get_ffmpeg_exe()
15
+ except Exception as exc:
16
+ raise RuntimeError(
17
+ "ffmpeg is required for video processing. Install ffmpeg on PATH or install "
18
+ "the Python package 'imageio-ffmpeg'."
19
+ ) from exc
20
+
iris/pipeline/orchestrator.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ from iris.config import Settings
7
+ from iris.pipeline.audio import extract_audio
8
+ from iris.pipeline.chunk import normalize_asr_chunks
9
+ from iris.pipeline.embed import LocalEmbedder, save_embeddings
10
+ from iris.pipeline.transcribe import WhisperTranscriber
11
+ from iris.storage.database import Database
12
+ from iris.storage.files import FileStore
13
+
14
+
15
+ class VideoProcessor:
16
+ def __init__(
17
+ self,
18
+ settings: Settings,
19
+ database: Database,
20
+ file_store: FileStore,
21
+ transcriber: WhisperTranscriber | None = None,
22
+ embedder: LocalEmbedder | None = None,
23
+ ) -> None:
24
+ self.settings = settings
25
+ self.database = database
26
+ self.file_store = file_store
27
+ self.transcriber = transcriber or WhisperTranscriber(settings)
28
+ self.embedder = embedder or LocalEmbedder(settings)
29
+
30
+ def process_video(self, video_id: str) -> None:
31
+ record = self.database.get_video(video_id)
32
+ if record is None:
33
+ return
34
+
35
+ try:
36
+ self.database.update_video_status(video_id, "processing")
37
+ video_path = Path(record["video_path"])
38
+ audio_path = extract_audio(
39
+ video_path,
40
+ self.file_store.audio_path(video_id),
41
+ max_duration_seconds=self.settings.max_audio_seconds,
42
+ )
43
+ self.database.set_artifact_paths(video_id, audio_path=audio_path)
44
+
45
+ raw_transcript = self.transcriber.transcribe(audio_path)
46
+ transcript_chunks = normalize_asr_chunks(raw_transcript)
47
+ transcript_path = self.file_store.transcript_path(video_id)
48
+ transcript_path.parent.mkdir(parents=True, exist_ok=True)
49
+ transcript_path.write_text(
50
+ json.dumps([chunk.__dict__ for chunk in transcript_chunks], indent=2),
51
+ encoding="utf-8",
52
+ )
53
+ self.database.replace_transcript_chunks(video_id, transcript_chunks)
54
+
55
+ embeddings = self.embedder.encode([chunk.text for chunk in transcript_chunks])
56
+ embeddings_path = self.file_store.embeddings_path(video_id)
57
+ save_embeddings(embeddings_path, embeddings)
58
+ self.database.set_artifact_paths(
59
+ video_id,
60
+ transcript_path=transcript_path,
61
+ embeddings_path=embeddings_path,
62
+ )
63
+ self.database.update_video_status(video_id, "completed")
64
+ except Exception as exc:
65
+ self.database.update_video_status(video_id, "failed", error=str(exc))
iris/pipeline/search.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from dataclasses import dataclass
5
+
6
+ import numpy as np
7
+
8
+ from iris.config import Settings
9
+ from iris.pipeline.chunk import TranscriptChunk
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class SearchMatch:
14
+ start: float
15
+ end: float
16
+ text: str
17
+ score: float
18
+ source: str = "speech"
19
+
20
+
21
+ def keyword_score(query: str, text: str) -> float:
22
+ query_norm = _normalize_text(query)
23
+ text_norm = _normalize_text(text)
24
+ if not query_norm or not text_norm:
25
+ return 0.0
26
+ if query_norm in text_norm:
27
+ return 1.0
28
+
29
+ query_terms = set(query_norm.split())
30
+ text_terms = set(text_norm.split())
31
+ if not query_terms:
32
+ return 0.0
33
+ overlap = len(query_terms & text_terms) / len(query_terms)
34
+ return min(1.0, overlap)
35
+
36
+
37
+ def rank_speech_chunks(
38
+ query: str,
39
+ chunks: list[TranscriptChunk],
40
+ chunk_embeddings: np.ndarray,
41
+ query_embedding: np.ndarray,
42
+ settings: Settings,
43
+ limit: int,
44
+ merge_gap_seconds: float | None = 3.0,
45
+ ) -> list[SearchMatch]:
46
+ if not chunks:
47
+ return []
48
+ if chunk_embeddings.ndim != 2 or len(chunk_embeddings) != len(chunks):
49
+ raise ValueError("Chunk embeddings must be a 2D array with one vector per transcript chunk.")
50
+
51
+ query_vector = np.asarray(query_embedding, dtype=np.float32).reshape(-1)
52
+ if query_vector.size == 0:
53
+ semantic_scores = np.zeros(len(chunks), dtype=np.float32)
54
+ else:
55
+ semantic_scores = chunk_embeddings @ query_vector
56
+ semantic_scores = np.clip((semantic_scores + 1.0) / 2.0, 0.0, 1.0)
57
+
58
+ matches: list[SearchMatch] = []
59
+ for index, chunk in enumerate(chunks):
60
+ lexical = keyword_score(query, chunk.text)
61
+ semantic = float(semantic_scores[index])
62
+ score = settings.semantic_weight * semantic + settings.keyword_weight * lexical
63
+ if lexical > 0:
64
+ score = min(1.0, score + settings.keyword_boost)
65
+ if score >= settings.min_speech_score:
66
+ matches.append(
67
+ SearchMatch(
68
+ start=chunk.start,
69
+ end=chunk.end,
70
+ text=chunk.text,
71
+ score=round(float(score), 4),
72
+ )
73
+ )
74
+
75
+ ranked = sorted(matches, key=lambda item: (-item.score, item.start))
76
+ shortlisted = ranked[: max(limit * 3, limit)]
77
+ if merge_gap_seconds is None:
78
+ return shortlisted[:limit]
79
+ return merge_nearby_matches(shortlisted, max_gap_seconds=merge_gap_seconds)[:limit]
80
+
81
+
82
+ def detail_window_settings(detail: str) -> tuple[float, float, float | None]:
83
+ if detail == "coarse":
84
+ return 45.0, 8.0, 8.0
85
+ if detail == "medium":
86
+ return 18.0, 3.0, 2.0
87
+ return 6.0, 0.0, None
88
+
89
+
90
+ def format_timestamp(seconds: float) -> str:
91
+ total = max(0, int(round(seconds)))
92
+ hours, remainder = divmod(total, 3600)
93
+ minutes, secs = divmod(remainder, 60)
94
+ if hours:
95
+ return f"{hours}:{minutes:02d}:{secs:02d}"
96
+ return f"{minutes}:{secs:02d}"
97
+
98
+
99
+ def merge_nearby_matches(matches: list[SearchMatch], max_gap_seconds: float) -> list[SearchMatch]:
100
+ if not matches:
101
+ return []
102
+ ordered = sorted(matches, key=lambda item: (item.start, item.end))
103
+ merged: list[SearchMatch] = [ordered[0]]
104
+
105
+ for match in ordered[1:]:
106
+ previous = merged[-1]
107
+ if match.source == previous.source and match.start <= previous.end + max_gap_seconds:
108
+ merged[-1] = SearchMatch(
109
+ start=min(previous.start, match.start),
110
+ end=max(previous.end, match.end),
111
+ text=_join_unique_text(previous.text, match.text),
112
+ score=max(previous.score, match.score),
113
+ source=previous.source,
114
+ )
115
+ else:
116
+ merged.append(match)
117
+
118
+ return sorted(merged, key=lambda item: (-item.score, item.start))
119
+
120
+
121
+ def _normalize_text(value: str) -> str:
122
+ return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9]+", " ", value.lower())).strip()
123
+
124
+
125
+ def _join_unique_text(left: str, right: str) -> str:
126
+ if right in left:
127
+ return left
128
+ if left in right:
129
+ return right
130
+ return f"{left} {right}".strip()
iris/pipeline/transcribe.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device
6
+
7
+
8
+ class WhisperTranscriber:
9
+ def __init__(self, settings: Settings) -> None:
10
+ self.settings = settings
11
+ self._pipeline = None
12
+
13
+ def transcribe(self, audio_path: Path) -> dict:
14
+ transcriber = self._get_pipeline()
15
+ return transcriber(
16
+ str(audio_path),
17
+ return_timestamps=self._timestamp_mode(),
18
+ chunk_length_s=30,
19
+ )
20
+
21
+ def _get_pipeline(self):
22
+ if self._pipeline is None:
23
+ from transformers import pipeline
24
+
25
+ self._pipeline = pipeline(
26
+ task="automatic-speech-recognition",
27
+ model=self.settings.asr_model,
28
+ device=transformers_pipeline_device(self.settings.device),
29
+ dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype),
30
+ )
31
+ return self._pipeline
32
+
33
+ def _timestamp_mode(self):
34
+ mode = self.settings.asr_timestamps.strip().lower()
35
+ if mode in {"word", "char"}:
36
+ return mode
37
+ if mode in {"segment", "segments", "true"}:
38
+ return True
39
+ return True
iris/pipeline/video.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+
6
+ SUPPORTED_VIDEO_EXTENSIONS = {".mp4", ".mov", ".mkv", ".webm"}
7
+
8
+
9
+ def validate_video_filename(filename: str) -> None:
10
+ suffix = Path(filename).suffix.lower()
11
+ if suffix not in SUPPORTED_VIDEO_EXTENSIONS:
12
+ supported = ", ".join(sorted(SUPPORTED_VIDEO_EXTENSIONS))
13
+ raise ValueError(f"Unsupported video extension '{suffix}'. Supported extensions: {supported}.")
14
+
iris/pipeline/visual.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import subprocess
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+
7
+ from PIL import Image
8
+
9
+ from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device
10
+ from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
11
+ from iris.pipeline.search import SearchMatch, merge_nearby_matches
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class SampledFrame:
16
+ path: Path
17
+ timestamp: float
18
+
19
+
20
+ def sample_video_frames(video_path: Path, frames_dir: Path, interval_seconds: float) -> list[SampledFrame]:
21
+ frames_dir.mkdir(parents=True, exist_ok=True)
22
+ existing = sorted(frames_dir.glob("frame_*.jpg"))
23
+ interval_seconds = max(0.25, interval_seconds)
24
+ if not existing:
25
+ command = [
26
+ resolve_ffmpeg_executable(),
27
+ "-y",
28
+ "-i",
29
+ str(video_path),
30
+ "-vf",
31
+ f"fps=1/{interval_seconds}",
32
+ str(frames_dir / "frame_%06d.jpg"),
33
+ ]
34
+ try:
35
+ subprocess.run(command, check=True, capture_output=True, text=True)
36
+ except OSError as exc:
37
+ raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
38
+ except subprocess.CalledProcessError as exc:
39
+ detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
40
+ raise RuntimeError(f"ffmpeg failed to sample frames: {detail}") from exc
41
+ existing = sorted(frames_dir.glob("frame_*.jpg"))
42
+
43
+ return [
44
+ SampledFrame(path=path, timestamp=index * interval_seconds)
45
+ for index, path in enumerate(existing)
46
+ ]
47
+
48
+
49
+ class VisualSearcher:
50
+ def __init__(self, settings: Settings) -> None:
51
+ self.settings = settings
52
+ self._classifier = None
53
+
54
+ def search(self, query: str, frames: list[SampledFrame], limit: int) -> list[SearchMatch]:
55
+ if not frames:
56
+ return []
57
+ classifier = self._get_classifier()
58
+ matches: list[SearchMatch] = []
59
+ candidate_labels = [query, f"not {query}", "unrelated scene"]
60
+ for frame in frames:
61
+ with Image.open(frame.path) as image:
62
+ result = classifier(image.convert("RGB"), candidate_labels=candidate_labels)
63
+ if not result:
64
+ continue
65
+ prompt_result = next((item for item in result if item.get("label") == query), result[0])
66
+ score = float(prompt_result["score"])
67
+ if score < self.settings.min_visual_score:
68
+ continue
69
+ matches.append(
70
+ SearchMatch(
71
+ start=frame.timestamp,
72
+ end=frame.timestamp + self.settings.frame_sample_interval_seconds,
73
+ text=f"Visual match for '{query}'",
74
+ score=round(score, 4),
75
+ source="visual",
76
+ )
77
+ )
78
+
79
+ ranked = sorted(matches, key=lambda item: (-item.score, item.start))[: max(limit * 3, limit)]
80
+ return merge_nearby_matches(ranked, max_gap_seconds=self.settings.frame_sample_interval_seconds)[:limit]
81
+
82
+ def _get_classifier(self):
83
+ if self._classifier is None:
84
+ from transformers import pipeline
85
+
86
+ self._classifier = pipeline(
87
+ task="zero-shot-image-classification",
88
+ model=self.settings.visual_model,
89
+ device=transformers_pipeline_device(self.settings.device),
90
+ dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype),
91
+ )
92
+ return self._classifier
iris/storage/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ """Storage helpers for IRIS."""
2
+
iris/storage/database.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import sqlite3
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ from iris.models.schemas import VideoStatus
8
+ from iris.pipeline.chunk import TranscriptChunk
9
+
10
+
11
+ class Database:
12
+ def __init__(self, path: Path) -> None:
13
+ self.path = path
14
+
15
+ def init(self) -> None:
16
+ self.path.parent.mkdir(parents=True, exist_ok=True)
17
+ with self._connect() as connection:
18
+ connection.executescript(
19
+ """
20
+ PRAGMA journal_mode=WAL;
21
+
22
+ CREATE TABLE IF NOT EXISTS videos (
23
+ id TEXT PRIMARY KEY,
24
+ original_filename TEXT NOT NULL,
25
+ video_path TEXT NOT NULL,
26
+ audio_path TEXT,
27
+ transcript_path TEXT,
28
+ embeddings_path TEXT,
29
+ status TEXT NOT NULL,
30
+ error TEXT,
31
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
32
+ updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
33
+ );
34
+
35
+ CREATE TABLE IF NOT EXISTS transcript_chunks (
36
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
37
+ video_id TEXT NOT NULL,
38
+ start REAL NOT NULL,
39
+ end REAL NOT NULL,
40
+ text TEXT NOT NULL,
41
+ FOREIGN KEY(video_id) REFERENCES videos(id) ON DELETE CASCADE
42
+ );
43
+
44
+ CREATE TABLE IF NOT EXISTS searches (
45
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
46
+ video_id TEXT NOT NULL,
47
+ query TEXT NOT NULL,
48
+ include_visual INTEGER NOT NULL,
49
+ created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
50
+ FOREIGN KEY(video_id) REFERENCES videos(id) ON DELETE CASCADE
51
+ );
52
+ """
53
+ )
54
+
55
+ def create_video(self, video_id: str, original_filename: str, video_path: Path) -> dict[str, Any]:
56
+ with self._connect() as connection:
57
+ connection.execute(
58
+ """
59
+ INSERT INTO videos (id, original_filename, video_path, status)
60
+ VALUES (?, ?, ?, 'queued')
61
+ """,
62
+ (video_id, original_filename, str(video_path)),
63
+ )
64
+ return self.get_video(video_id) or {}
65
+
66
+ def get_video(self, video_id: str) -> dict[str, Any] | None:
67
+ with self._connect() as connection:
68
+ row = connection.execute("SELECT * FROM videos WHERE id = ?", (video_id,)).fetchone()
69
+ return dict(row) if row else None
70
+
71
+ def update_video_status(self, video_id: str, status: VideoStatus, error: str | None = None) -> None:
72
+ with self._connect() as connection:
73
+ connection.execute(
74
+ """
75
+ UPDATE videos
76
+ SET status = ?, error = ?, updated_at = CURRENT_TIMESTAMP
77
+ WHERE id = ?
78
+ """,
79
+ (status, error, video_id),
80
+ )
81
+
82
+ def set_artifact_paths(
83
+ self,
84
+ video_id: str,
85
+ audio_path: Path | None = None,
86
+ transcript_path: Path | None = None,
87
+ embeddings_path: Path | None = None,
88
+ ) -> None:
89
+ with self._connect() as connection:
90
+ connection.execute(
91
+ """
92
+ UPDATE videos
93
+ SET audio_path = COALESCE(?, audio_path),
94
+ transcript_path = COALESCE(?, transcript_path),
95
+ embeddings_path = COALESCE(?, embeddings_path),
96
+ updated_at = CURRENT_TIMESTAMP
97
+ WHERE id = ?
98
+ """,
99
+ (
100
+ str(audio_path) if audio_path else None,
101
+ str(transcript_path) if transcript_path else None,
102
+ str(embeddings_path) if embeddings_path else None,
103
+ video_id,
104
+ ),
105
+ )
106
+
107
+ def replace_transcript_chunks(self, video_id: str, chunks: list[TranscriptChunk]) -> None:
108
+ with self._connect() as connection:
109
+ connection.execute("DELETE FROM transcript_chunks WHERE video_id = ?", (video_id,))
110
+ connection.executemany(
111
+ """
112
+ INSERT INTO transcript_chunks (video_id, start, end, text)
113
+ VALUES (?, ?, ?, ?)
114
+ """,
115
+ [(video_id, chunk.start, chunk.end, chunk.text) for chunk in chunks],
116
+ )
117
+
118
+ def list_transcript_chunks(self, video_id: str) -> list[TranscriptChunk]:
119
+ with self._connect() as connection:
120
+ rows = connection.execute(
121
+ """
122
+ SELECT start, end, text
123
+ FROM transcript_chunks
124
+ WHERE video_id = ?
125
+ ORDER BY start, end, id
126
+ """,
127
+ (video_id,),
128
+ ).fetchall()
129
+ return [
130
+ TranscriptChunk(start=float(row["start"]), end=float(row["end"]), text=str(row["text"]))
131
+ for row in rows
132
+ ]
133
+
134
+ def record_search(self, video_id: str, query: str, include_visual: bool) -> None:
135
+ with self._connect() as connection:
136
+ connection.execute(
137
+ """
138
+ INSERT INTO searches (video_id, query, include_visual)
139
+ VALUES (?, ?, ?)
140
+ """,
141
+ (video_id, query, int(include_visual)),
142
+ )
143
+
144
+ def _connect(self) -> sqlite3.Connection:
145
+ connection = sqlite3.connect(self.path)
146
+ connection.row_factory = sqlite3.Row
147
+ connection.execute("PRAGMA foreign_keys=ON;")
148
+ return connection
149
+
iris/storage/files.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import shutil
4
+ from pathlib import Path
5
+ from uuid import uuid4
6
+
7
+ from fastapi import UploadFile
8
+
9
+ from iris.config import Settings
10
+ from iris.pipeline.video import validate_video_filename
11
+
12
+
13
+ class FileStore:
14
+ def __init__(self, settings: Settings) -> None:
15
+ self.settings = settings
16
+
17
+ def video_dir(self, video_id: str) -> Path:
18
+ return self.settings.data_dir / "videos" / video_id
19
+
20
+ def video_path(self, video_id: str, filename: str) -> Path:
21
+ return self.video_dir(video_id) / "source" / filename
22
+
23
+ def audio_path(self, video_id: str) -> Path:
24
+ return self.video_dir(video_id) / "audio" / "audio.wav"
25
+
26
+ def transcript_path(self, video_id: str) -> Path:
27
+ return self.video_dir(video_id) / "transcript" / "transcript.json"
28
+
29
+ def embeddings_path(self, video_id: str) -> Path:
30
+ return self.video_dir(video_id) / "embeddings" / "speech.npy"
31
+
32
+ def frames_dir(self, video_id: str) -> Path:
33
+ return self.video_dir(video_id) / "frames"
34
+
35
+ def save_upload(self, upload: UploadFile) -> tuple[str, Path]:
36
+ filename = Path(upload.filename or "video").name
37
+ validate_video_filename(filename)
38
+ video_id = str(uuid4())
39
+ destination = self.video_path(video_id, filename)
40
+ destination.parent.mkdir(parents=True, exist_ok=True)
41
+ with destination.open("wb") as output:
42
+ shutil.copyfileobj(upload.file, output)
43
+ return video_id, destination
44
+
requirements.txt CHANGED
@@ -1,2 +1,8 @@
1
- gradio
2
- iris-core @ git+https://huggingface.co/NextNeural/IRIS-core
 
 
 
 
 
 
 
1
+ fastapi>=0.115.0
2
+ imageio-ffmpeg>=0.5.0
3
+ numpy>=1.26.0
4
+ pillow>=10.0.0
5
+ python-multipart>=0.0.9
6
+ sentence-transformers>=3.0.0
7
+ torch>=2.2.0
8
+ transformers>=4.45.0