Spaces:
Sleeping
Sleeping
Commit ·
b2300e2
1
Parent(s): 9fca884
Bundle IRIS core for Space build
Browse files- README.md +3 -3
- iris/__init__.py +6 -0
- iris/api/__init__.py +2 -0
- iris/api/dependencies.py +25 -0
- iris/api/main.py +31 -0
- iris/api/routes.py +164 -0
- iris/config.py +143 -0
- iris/models/__init__.py +2 -0
- iris/models/schemas.py +58 -0
- iris/pipeline/__init__.py +2 -0
- iris/pipeline/audio.py +32 -0
- iris/pipeline/chunk.py +80 -0
- iris/pipeline/embed.py +45 -0
- iris/pipeline/ffmpeg.py +20 -0
- iris/pipeline/orchestrator.py +65 -0
- iris/pipeline/search.py +130 -0
- iris/pipeline/transcribe.py +39 -0
- iris/pipeline/video.py +14 -0
- iris/pipeline/visual.py +92 -0
- iris/storage/__init__.py +2 -0
- iris/storage/database.py +149 -0
- iris/storage/files.py +44 -0
- requirements.txt +8 -2
README.md
CHANGED
|
@@ -14,9 +14,9 @@ short_description: Intelligent search for spoken moments inside video.
|
|
| 14 |
|
| 15 |
# IRIS
|
| 16 |
|
| 17 |
-
IRIS is a Gradio demo for searching videos by spoken content.
|
| 18 |
-
|
| 19 |
-
|
| 20 |
|
| 21 |
Upload a video, enter a query, and IRIS returns matching timestamp windows.
|
| 22 |
This Space defaults to CPU-friendly settings:
|
|
|
|
| 14 |
|
| 15 |
# IRIS
|
| 16 |
|
| 17 |
+
IRIS is a Gradio demo for searching videos by spoken content. The Space mirrors
|
| 18 |
+
the `iris/` package from `../IRIS-core` so Hugging Face can build it without
|
| 19 |
+
cloning a private dependency during installation.
|
| 20 |
|
| 21 |
Upload a video, enter a query, and IRIS returns matching timestamp windows.
|
| 22 |
This Space defaults to CPU-friendly settings:
|
iris/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""IRIS video search package."""
|
| 2 |
+
|
| 3 |
+
__all__ = ["__version__"]
|
| 4 |
+
|
| 5 |
+
__version__ = "0.1.0"
|
| 6 |
+
|
iris/api/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastAPI application package."""
|
| 2 |
+
|
iris/api/dependencies.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from fastapi import Request
|
| 4 |
+
|
| 5 |
+
from iris.config import Settings
|
| 6 |
+
from iris.pipeline.orchestrator import VideoProcessor
|
| 7 |
+
from iris.storage.database import Database
|
| 8 |
+
from iris.storage.files import FileStore
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def get_settings(request: Request) -> Settings:
|
| 12 |
+
return request.app.state.settings
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def get_database(request: Request) -> Database:
|
| 16 |
+
return request.app.state.database
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def get_file_store(request: Request) -> FileStore:
|
| 20 |
+
return request.app.state.file_store
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def get_video_processor(request: Request) -> VideoProcessor:
|
| 24 |
+
return request.app.state.video_processor
|
| 25 |
+
|
iris/api/main.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from fastapi import FastAPI
|
| 4 |
+
|
| 5 |
+
from iris.api.routes import router
|
| 6 |
+
from iris.config import Settings
|
| 7 |
+
from iris.pipeline.orchestrator import VideoProcessor
|
| 8 |
+
from iris.storage.database import Database
|
| 9 |
+
from iris.storage.files import FileStore
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def create_app(settings: Settings | None = None) -> FastAPI:
|
| 13 |
+
app_settings = settings or Settings.from_env()
|
| 14 |
+
app_settings.ensure_directories()
|
| 15 |
+
|
| 16 |
+
database = Database(app_settings.database_path)
|
| 17 |
+
database.init()
|
| 18 |
+
file_store = FileStore(app_settings)
|
| 19 |
+
processor = VideoProcessor(app_settings, database, file_store)
|
| 20 |
+
|
| 21 |
+
app = FastAPI(title="IRIS", version="0.1.0")
|
| 22 |
+
app.state.settings = app_settings
|
| 23 |
+
app.state.database = database
|
| 24 |
+
app.state.file_store = file_store
|
| 25 |
+
app.state.video_processor = processor
|
| 26 |
+
app.include_router(router)
|
| 27 |
+
return app
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
app = create_app()
|
| 31 |
+
|
iris/api/routes.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, UploadFile, status
|
| 6 |
+
|
| 7 |
+
from iris.api.dependencies import get_database, get_file_store, get_settings, get_video_processor
|
| 8 |
+
from iris.config import Settings, device_info
|
| 9 |
+
from iris.models.schemas import (
|
| 10 |
+
SearchRequest,
|
| 11 |
+
SearchResponse,
|
| 12 |
+
SearchResultResponse,
|
| 13 |
+
TranscriptChunkResponse,
|
| 14 |
+
TranscriptResponse,
|
| 15 |
+
UploadVideoResponse,
|
| 16 |
+
VideoStatusResponse,
|
| 17 |
+
)
|
| 18 |
+
from iris.pipeline.chunk import window_transcript_chunks
|
| 19 |
+
from iris.pipeline.embed import LocalEmbedder
|
| 20 |
+
from iris.pipeline.search import detail_window_settings, format_timestamp, rank_speech_chunks
|
| 21 |
+
from iris.pipeline.visual import VisualSearcher, sample_video_frames
|
| 22 |
+
from iris.storage.database import Database
|
| 23 |
+
from iris.storage.files import FileStore
|
| 24 |
+
from iris.pipeline.orchestrator import VideoProcessor
|
| 25 |
+
|
| 26 |
+
router = APIRouter()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@router.get("/health")
|
| 30 |
+
def health(settings: Settings = Depends(get_settings)) -> dict:
|
| 31 |
+
return {"status": "ok", "device": device_info(settings.device, settings.torch_dtype)}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@router.post("/videos", response_model=UploadVideoResponse, status_code=status.HTTP_202_ACCEPTED)
|
| 35 |
+
def upload_video(
|
| 36 |
+
file: UploadFile,
|
| 37 |
+
background_tasks: BackgroundTasks,
|
| 38 |
+
database: Database = Depends(get_database),
|
| 39 |
+
file_store: FileStore = Depends(get_file_store),
|
| 40 |
+
processor: VideoProcessor = Depends(get_video_processor),
|
| 41 |
+
) -> UploadVideoResponse:
|
| 42 |
+
try:
|
| 43 |
+
video_id, path = file_store.save_upload(file)
|
| 44 |
+
except ValueError as exc:
|
| 45 |
+
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
| 46 |
+
|
| 47 |
+
database.create_video(video_id, file.filename or path.name, path)
|
| 48 |
+
if getattr(processor, "process_video", None) and processor is not None:
|
| 49 |
+
background_tasks.add_task(processor.process_video, video_id)
|
| 50 |
+
return UploadVideoResponse(video_id=video_id, status="queued")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@router.post("/videos/{video_id}/process", response_model=UploadVideoResponse, status_code=status.HTTP_202_ACCEPTED)
|
| 54 |
+
def process_video_again(
|
| 55 |
+
video_id: str,
|
| 56 |
+
background_tasks: BackgroundTasks,
|
| 57 |
+
database: Database = Depends(get_database),
|
| 58 |
+
processor: VideoProcessor = Depends(get_video_processor),
|
| 59 |
+
) -> UploadVideoResponse:
|
| 60 |
+
_get_video_or_404(database, video_id)
|
| 61 |
+
database.update_video_status(video_id, "queued")
|
| 62 |
+
background_tasks.add_task(processor.process_video, video_id)
|
| 63 |
+
return UploadVideoResponse(video_id=video_id, status="queued")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
@router.get("/videos/{video_id}/status", response_model=VideoStatusResponse)
|
| 67 |
+
def get_video_status(
|
| 68 |
+
video_id: str,
|
| 69 |
+
database: Database = Depends(get_database),
|
| 70 |
+
) -> VideoStatusResponse:
|
| 71 |
+
record = _get_video_or_404(database, video_id)
|
| 72 |
+
return VideoStatusResponse(video_id=video_id, status=record["status"], error=record["error"])
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@router.get("/videos/{video_id}/transcript", response_model=TranscriptResponse)
|
| 76 |
+
def get_transcript(
|
| 77 |
+
video_id: str,
|
| 78 |
+
database: Database = Depends(get_database),
|
| 79 |
+
) -> TranscriptResponse:
|
| 80 |
+
_get_video_or_404(database, video_id)
|
| 81 |
+
chunks = database.list_transcript_chunks(video_id)
|
| 82 |
+
return TranscriptResponse(
|
| 83 |
+
video_id=video_id,
|
| 84 |
+
chunks=[TranscriptChunkResponse(start=chunk.start, end=chunk.end, text=chunk.text) for chunk in chunks],
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@router.post("/videos/{video_id}/search", response_model=SearchResponse)
|
| 89 |
+
def search_video(
|
| 90 |
+
video_id: str,
|
| 91 |
+
request: SearchRequest,
|
| 92 |
+
database: Database = Depends(get_database),
|
| 93 |
+
file_store: FileStore = Depends(get_file_store),
|
| 94 |
+
settings: Settings = Depends(get_settings),
|
| 95 |
+
) -> SearchResponse:
|
| 96 |
+
record = _get_video_or_404(database, video_id)
|
| 97 |
+
if record["status"] != "completed":
|
| 98 |
+
raise HTTPException(
|
| 99 |
+
status_code=status.HTTP_409_CONFLICT,
|
| 100 |
+
detail=f"Video is not ready for search. Current status: {record['status']}.",
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
atomic_chunks = database.list_transcript_chunks(video_id)
|
| 104 |
+
default_window, default_overlap, default_merge_gap = detail_window_settings(request.detail)
|
| 105 |
+
window_seconds = request.window_seconds or default_window
|
| 106 |
+
overlap_seconds = min(default_overlap, max(0.0, window_seconds - 0.1))
|
| 107 |
+
merge_gap_seconds = request.merge_gap_seconds if request.merge_gap_seconds is not None else default_merge_gap
|
| 108 |
+
chunks = window_transcript_chunks(
|
| 109 |
+
atomic_chunks,
|
| 110 |
+
window_seconds=window_seconds,
|
| 111 |
+
overlap_seconds=overlap_seconds,
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
embedder = LocalEmbedder(settings)
|
| 115 |
+
chunk_embeddings = embedder.encode([chunk.text for chunk in chunks])
|
| 116 |
+
query_embedding = embedder.encode([request.query])[0]
|
| 117 |
+
speech_results = rank_speech_chunks(
|
| 118 |
+
query=request.query,
|
| 119 |
+
chunks=chunks,
|
| 120 |
+
chunk_embeddings=chunk_embeddings,
|
| 121 |
+
query_embedding=query_embedding,
|
| 122 |
+
settings=settings,
|
| 123 |
+
limit=request.limit,
|
| 124 |
+
merge_gap_seconds=merge_gap_seconds,
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
results = list(speech_results)
|
| 128 |
+
if request.include_visual:
|
| 129 |
+
if not settings.visual_search_enabled:
|
| 130 |
+
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Visual search is disabled.")
|
| 131 |
+
frames = sample_video_frames(
|
| 132 |
+
Path(record["video_path"]),
|
| 133 |
+
file_store.frames_dir(video_id),
|
| 134 |
+
settings.frame_sample_interval_seconds,
|
| 135 |
+
)
|
| 136 |
+
visual_results = VisualSearcher(settings).search(request.query, frames, request.limit)
|
| 137 |
+
results.extend(visual_results)
|
| 138 |
+
|
| 139 |
+
database.record_search(video_id, request.query, request.include_visual)
|
| 140 |
+
ranked = sorted(results, key=lambda item: (-item.score, item.start))[: request.limit]
|
| 141 |
+
return SearchResponse(
|
| 142 |
+
video_id=video_id,
|
| 143 |
+
query=request.query,
|
| 144 |
+
results=[
|
| 145 |
+
SearchResultResponse(
|
| 146 |
+
start=item.start,
|
| 147 |
+
end=item.end,
|
| 148 |
+
start_time=format_timestamp(item.start),
|
| 149 |
+
end_time=format_timestamp(item.end),
|
| 150 |
+
timestamp=f"{format_timestamp(item.start)}-{format_timestamp(item.end)}",
|
| 151 |
+
text=item.text,
|
| 152 |
+
score=item.score,
|
| 153 |
+
source=item.source,
|
| 154 |
+
)
|
| 155 |
+
for item in ranked
|
| 156 |
+
],
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _get_video_or_404(database: Database, video_id: str) -> dict:
|
| 161 |
+
record = database.get_video(video_id)
|
| 162 |
+
if record is None:
|
| 163 |
+
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Video not found.")
|
| 164 |
+
return record
|
iris/config.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _bool_env(name: str, default: bool) -> bool:
|
| 9 |
+
value = os.getenv(name)
|
| 10 |
+
if value is None:
|
| 11 |
+
return default
|
| 12 |
+
return value.strip().lower() in {"1", "true", "yes", "on"}
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _float_env(name: str, default: float) -> float:
|
| 16 |
+
value = os.getenv(name)
|
| 17 |
+
return default if value is None else float(value)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _int_env(name: str, default: int) -> int:
|
| 21 |
+
value = os.getenv(name)
|
| 22 |
+
return default if value is None else int(value)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@dataclass(frozen=True)
|
| 26 |
+
class Settings:
|
| 27 |
+
data_dir: Path
|
| 28 |
+
database_path: Path
|
| 29 |
+
asr_model: str
|
| 30 |
+
asr_timestamps: str
|
| 31 |
+
embedding_model: str
|
| 32 |
+
visual_model: str
|
| 33 |
+
device: str
|
| 34 |
+
torch_dtype: str
|
| 35 |
+
chunk_length_seconds: float
|
| 36 |
+
chunk_overlap_seconds: float
|
| 37 |
+
frame_sample_interval_seconds: float
|
| 38 |
+
max_audio_seconds: float | None
|
| 39 |
+
visual_search_enabled: bool
|
| 40 |
+
keyword_boost: float
|
| 41 |
+
semantic_weight: float
|
| 42 |
+
keyword_weight: float
|
| 43 |
+
min_speech_score: float
|
| 44 |
+
min_visual_score: float
|
| 45 |
+
|
| 46 |
+
@classmethod
|
| 47 |
+
def from_env(cls) -> "Settings":
|
| 48 |
+
data_dir = Path(os.getenv("IRIS_DATA_DIR", ".iris-data")).resolve()
|
| 49 |
+
return cls(
|
| 50 |
+
data_dir=data_dir,
|
| 51 |
+
database_path=Path(os.getenv("IRIS_DATABASE_PATH", data_dir / "iris.sqlite3")).resolve(),
|
| 52 |
+
asr_model=os.getenv("IRIS_ASR_MODEL", "openai/whisper-large-v3-turbo"),
|
| 53 |
+
asr_timestamps=os.getenv("IRIS_ASR_TIMESTAMPS", "word"),
|
| 54 |
+
embedding_model=os.getenv("IRIS_EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2"),
|
| 55 |
+
visual_model=os.getenv("IRIS_VISUAL_MODEL", "openai/clip-vit-large-patch14"),
|
| 56 |
+
device=os.getenv("IRIS_DEVICE", "auto"),
|
| 57 |
+
torch_dtype=os.getenv("IRIS_TORCH_DTYPE", "auto"),
|
| 58 |
+
chunk_length_seconds=_float_env("IRIS_CHUNK_LENGTH_SECONDS", 24.0),
|
| 59 |
+
chunk_overlap_seconds=_float_env("IRIS_CHUNK_OVERLAP_SECONDS", 4.0),
|
| 60 |
+
frame_sample_interval_seconds=_float_env("IRIS_FRAME_SAMPLE_INTERVAL_SECONDS", 2.0),
|
| 61 |
+
max_audio_seconds=_optional_float_env("IRIS_MAX_AUDIO_SECONDS"),
|
| 62 |
+
visual_search_enabled=_bool_env("IRIS_VISUAL_SEARCH_ENABLED", True),
|
| 63 |
+
keyword_boost=_float_env("IRIS_KEYWORD_BOOST", 0.15),
|
| 64 |
+
semantic_weight=_float_env("IRIS_SEMANTIC_WEIGHT", 0.78),
|
| 65 |
+
keyword_weight=_float_env("IRIS_KEYWORD_WEIGHT", 0.22),
|
| 66 |
+
min_speech_score=_float_env("IRIS_MIN_SPEECH_SCORE", 0.55),
|
| 67 |
+
min_visual_score=_float_env("IRIS_MIN_VISUAL_SCORE", 0.45),
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
def ensure_directories(self) -> None:
|
| 71 |
+
self.data_dir.mkdir(parents=True, exist_ok=True)
|
| 72 |
+
self.database_path.parent.mkdir(parents=True, exist_ok=True)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def resolve_device(device: str) -> str:
|
| 76 |
+
if device != "auto":
|
| 77 |
+
return device
|
| 78 |
+
try:
|
| 79 |
+
import torch
|
| 80 |
+
|
| 81 |
+
return "cuda" if torch.cuda.is_available() else "cpu"
|
| 82 |
+
except Exception:
|
| 83 |
+
return "cpu"
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def transformers_pipeline_device(device: str) -> int:
|
| 87 |
+
resolved = resolve_device(device)
|
| 88 |
+
return 0 if resolved == "cuda" else -1
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def resolve_torch_dtype(device: str, dtype: str):
|
| 92 |
+
normalized = dtype.strip().lower()
|
| 93 |
+
resolved_device = resolve_device(device)
|
| 94 |
+
if normalized == "auto":
|
| 95 |
+
normalized = "float16" if resolved_device == "cuda" else "float32"
|
| 96 |
+
|
| 97 |
+
try:
|
| 98 |
+
import torch
|
| 99 |
+
|
| 100 |
+
if normalized in {"float16", "fp16", "half"}:
|
| 101 |
+
return torch.float16
|
| 102 |
+
if normalized in {"bfloat16", "bf16"}:
|
| 103 |
+
return torch.bfloat16
|
| 104 |
+
if normalized in {"float32", "fp32", "full"}:
|
| 105 |
+
return torch.float32
|
| 106 |
+
except Exception:
|
| 107 |
+
pass
|
| 108 |
+
return "auto"
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def device_info(device: str, dtype: str) -> dict:
|
| 112 |
+
resolved = resolve_device(device)
|
| 113 |
+
info = {
|
| 114 |
+
"requested_device": device,
|
| 115 |
+
"resolved_device": resolved,
|
| 116 |
+
"requested_dtype": dtype,
|
| 117 |
+
"cuda_available": False,
|
| 118 |
+
"cuda_version": None,
|
| 119 |
+
"gpu_count": 0,
|
| 120 |
+
"gpu_name": None,
|
| 121 |
+
}
|
| 122 |
+
try:
|
| 123 |
+
import torch
|
| 124 |
+
|
| 125 |
+
info.update(
|
| 126 |
+
{
|
| 127 |
+
"torch_version": torch.__version__,
|
| 128 |
+
"cuda_available": torch.cuda.is_available(),
|
| 129 |
+
"cuda_version": torch.version.cuda,
|
| 130 |
+
"gpu_count": torch.cuda.device_count(),
|
| 131 |
+
"gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
|
| 132 |
+
}
|
| 133 |
+
)
|
| 134 |
+
except Exception as exc:
|
| 135 |
+
info["torch_error"] = str(exc)
|
| 136 |
+
return info
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _optional_float_env(name: str) -> float | None:
|
| 140 |
+
value = os.getenv(name)
|
| 141 |
+
if value is None or value.strip() == "":
|
| 142 |
+
return None
|
| 143 |
+
return float(value)
|
iris/models/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pydantic API models."""
|
| 2 |
+
|
iris/models/schemas.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Literal
|
| 4 |
+
|
| 5 |
+
from pydantic import BaseModel, Field
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
VideoStatus = Literal["queued", "processing", "completed", "failed"]
|
| 9 |
+
ResultSource = Literal["speech", "visual"]
|
| 10 |
+
SearchDetail = Literal["fine", "medium", "coarse"]
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class UploadVideoResponse(BaseModel):
|
| 14 |
+
video_id: str
|
| 15 |
+
status: VideoStatus
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class VideoStatusResponse(BaseModel):
|
| 19 |
+
video_id: str
|
| 20 |
+
status: VideoStatus
|
| 21 |
+
error: str | None = None
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class TranscriptChunkResponse(BaseModel):
|
| 25 |
+
start: float
|
| 26 |
+
end: float
|
| 27 |
+
text: str
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class TranscriptResponse(BaseModel):
|
| 31 |
+
video_id: str
|
| 32 |
+
chunks: list[TranscriptChunkResponse]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class SearchRequest(BaseModel):
|
| 36 |
+
query: str = Field(min_length=1)
|
| 37 |
+
include_visual: bool = False
|
| 38 |
+
limit: int = Field(default=10, ge=1, le=100)
|
| 39 |
+
detail: SearchDetail = "fine"
|
| 40 |
+
window_seconds: float | None = Field(default=None, ge=2.0, le=120.0)
|
| 41 |
+
merge_gap_seconds: float | None = Field(default=None, ge=0.0, le=30.0)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class SearchResultResponse(BaseModel):
|
| 45 |
+
start: float
|
| 46 |
+
end: float
|
| 47 |
+
start_time: str
|
| 48 |
+
end_time: str
|
| 49 |
+
timestamp: str
|
| 50 |
+
text: str
|
| 51 |
+
score: float
|
| 52 |
+
source: ResultSource
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class SearchResponse(BaseModel):
|
| 56 |
+
video_id: str
|
| 57 |
+
query: str
|
| 58 |
+
results: list[SearchResultResponse]
|
iris/pipeline/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""IRIS processing pipeline."""
|
| 2 |
+
|
iris/pipeline/audio.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import subprocess
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def extract_audio(video_path: Path, audio_path: Path, max_duration_seconds: float | None = None) -> Path:
|
| 10 |
+
audio_path.parent.mkdir(parents=True, exist_ok=True)
|
| 11 |
+
command = [
|
| 12 |
+
resolve_ffmpeg_executable(),
|
| 13 |
+
"-y",
|
| 14 |
+
"-i",
|
| 15 |
+
str(video_path),
|
| 16 |
+
"-vn",
|
| 17 |
+
"-ac",
|
| 18 |
+
"1",
|
| 19 |
+
"-ar",
|
| 20 |
+
"16000",
|
| 21 |
+
]
|
| 22 |
+
if max_duration_seconds is not None:
|
| 23 |
+
command.extend(["-t", str(max_duration_seconds)])
|
| 24 |
+
command.append(str(audio_path))
|
| 25 |
+
try:
|
| 26 |
+
subprocess.run(command, check=True, capture_output=True, text=True)
|
| 27 |
+
except OSError as exc:
|
| 28 |
+
raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
|
| 29 |
+
except subprocess.CalledProcessError as exc:
|
| 30 |
+
detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
|
| 31 |
+
raise RuntimeError(f"ffmpeg failed to extract audio: {detail}") from exc
|
| 32 |
+
return audio_path
|
iris/pipeline/chunk.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Iterable
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(frozen=True)
|
| 8 |
+
class TranscriptChunk:
|
| 9 |
+
start: float
|
| 10 |
+
end: float
|
| 11 |
+
text: str
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def normalize_asr_chunks(raw_output: dict) -> list[TranscriptChunk]:
|
| 15 |
+
raw_chunks = raw_output.get("chunks") or []
|
| 16 |
+
normalized: list[TranscriptChunk] = []
|
| 17 |
+
for item in raw_chunks:
|
| 18 |
+
timestamp = item.get("timestamp")
|
| 19 |
+
text = (item.get("text") or "").strip()
|
| 20 |
+
if not text or not timestamp or len(timestamp) != 2:
|
| 21 |
+
continue
|
| 22 |
+
start, end = timestamp
|
| 23 |
+
if start is None or end is None:
|
| 24 |
+
continue
|
| 25 |
+
start_f = max(0.0, float(start))
|
| 26 |
+
end_f = max(start_f, float(end))
|
| 27 |
+
normalized.append(TranscriptChunk(start=start_f, end=end_f, text=text))
|
| 28 |
+
|
| 29 |
+
if normalized:
|
| 30 |
+
return normalized
|
| 31 |
+
|
| 32 |
+
text = (raw_output.get("text") or "").strip()
|
| 33 |
+
return [TranscriptChunk(start=0.0, end=0.0, text=text)] if text else []
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def window_transcript_chunks(
|
| 37 |
+
chunks: Iterable[TranscriptChunk],
|
| 38 |
+
window_seconds: float,
|
| 39 |
+
overlap_seconds: float,
|
| 40 |
+
) -> list[TranscriptChunk]:
|
| 41 |
+
source = sorted(chunks, key=lambda chunk: (chunk.start, chunk.end))
|
| 42 |
+
if not source:
|
| 43 |
+
return []
|
| 44 |
+
|
| 45 |
+
window_seconds = max(1.0, window_seconds)
|
| 46 |
+
overlap_seconds = max(0.0, min(overlap_seconds, window_seconds - 0.1))
|
| 47 |
+
windows: list[TranscriptChunk] = []
|
| 48 |
+
|
| 49 |
+
current: list[TranscriptChunk] = []
|
| 50 |
+
current_start = source[0].start
|
| 51 |
+
|
| 52 |
+
for chunk in source:
|
| 53 |
+
if current and chunk.end - current_start > window_seconds:
|
| 54 |
+
windows.append(_merge_text_window(current))
|
| 55 |
+
keep_after = max(current[-1].end - overlap_seconds, current_start)
|
| 56 |
+
current = [item for item in current if item.end >= keep_after]
|
| 57 |
+
current_start = current[0].start if current else chunk.start
|
| 58 |
+
current.append(chunk)
|
| 59 |
+
|
| 60 |
+
if current:
|
| 61 |
+
windows.append(_merge_text_window(current))
|
| 62 |
+
|
| 63 |
+
return _dedupe_windows(windows)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _merge_text_window(chunks: list[TranscriptChunk]) -> TranscriptChunk:
|
| 67 |
+
text = " ".join(chunk.text.strip() for chunk in chunks if chunk.text.strip())
|
| 68 |
+
return TranscriptChunk(start=chunks[0].start, end=chunks[-1].end, text=text)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _dedupe_windows(windows: list[TranscriptChunk]) -> list[TranscriptChunk]:
|
| 72 |
+
seen: set[tuple[float, float, str]] = set()
|
| 73 |
+
deduped: list[TranscriptChunk] = []
|
| 74 |
+
for window in windows:
|
| 75 |
+
key = (round(window.start, 3), round(window.end, 3), window.text)
|
| 76 |
+
if key not in seen:
|
| 77 |
+
seen.add(key)
|
| 78 |
+
deduped.append(window)
|
| 79 |
+
return deduped
|
| 80 |
+
|
iris/pipeline/embed.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
from iris.config import Settings, resolve_device
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class LocalEmbedder:
|
| 11 |
+
def __init__(self, settings: Settings) -> None:
|
| 12 |
+
self.settings = settings
|
| 13 |
+
self._model = None
|
| 14 |
+
|
| 15 |
+
def encode(self, texts: list[str]) -> np.ndarray:
|
| 16 |
+
if not texts:
|
| 17 |
+
return np.empty((0, 0), dtype=np.float32)
|
| 18 |
+
model = self._get_model()
|
| 19 |
+
vectors = model.encode(
|
| 20 |
+
texts,
|
| 21 |
+
convert_to_numpy=True,
|
| 22 |
+
normalize_embeddings=True,
|
| 23 |
+
show_progress_bar=False,
|
| 24 |
+
)
|
| 25 |
+
return np.asarray(vectors, dtype=np.float32)
|
| 26 |
+
|
| 27 |
+
def _get_model(self):
|
| 28 |
+
if self._model is None:
|
| 29 |
+
from sentence_transformers import SentenceTransformer
|
| 30 |
+
|
| 31 |
+
self._model = SentenceTransformer(
|
| 32 |
+
self.settings.embedding_model,
|
| 33 |
+
device=resolve_device(self.settings.device),
|
| 34 |
+
)
|
| 35 |
+
return self._model
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def save_embeddings(path: Path, embeddings: np.ndarray) -> None:
|
| 39 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 40 |
+
np.save(path, embeddings)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def load_embeddings(path: Path) -> np.ndarray:
|
| 44 |
+
return np.load(path)
|
| 45 |
+
|
iris/pipeline/ffmpeg.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import shutil
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def resolve_ffmpeg_executable() -> str:
|
| 7 |
+
executable = shutil.which("ffmpeg")
|
| 8 |
+
if executable:
|
| 9 |
+
return executable
|
| 10 |
+
|
| 11 |
+
try:
|
| 12 |
+
import imageio_ffmpeg
|
| 13 |
+
|
| 14 |
+
return imageio_ffmpeg.get_ffmpeg_exe()
|
| 15 |
+
except Exception as exc:
|
| 16 |
+
raise RuntimeError(
|
| 17 |
+
"ffmpeg is required for video processing. Install ffmpeg on PATH or install "
|
| 18 |
+
"the Python package 'imageio-ffmpeg'."
|
| 19 |
+
) from exc
|
| 20 |
+
|
iris/pipeline/orchestrator.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from iris.config import Settings
|
| 7 |
+
from iris.pipeline.audio import extract_audio
|
| 8 |
+
from iris.pipeline.chunk import normalize_asr_chunks
|
| 9 |
+
from iris.pipeline.embed import LocalEmbedder, save_embeddings
|
| 10 |
+
from iris.pipeline.transcribe import WhisperTranscriber
|
| 11 |
+
from iris.storage.database import Database
|
| 12 |
+
from iris.storage.files import FileStore
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class VideoProcessor:
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
settings: Settings,
|
| 19 |
+
database: Database,
|
| 20 |
+
file_store: FileStore,
|
| 21 |
+
transcriber: WhisperTranscriber | None = None,
|
| 22 |
+
embedder: LocalEmbedder | None = None,
|
| 23 |
+
) -> None:
|
| 24 |
+
self.settings = settings
|
| 25 |
+
self.database = database
|
| 26 |
+
self.file_store = file_store
|
| 27 |
+
self.transcriber = transcriber or WhisperTranscriber(settings)
|
| 28 |
+
self.embedder = embedder or LocalEmbedder(settings)
|
| 29 |
+
|
| 30 |
+
def process_video(self, video_id: str) -> None:
|
| 31 |
+
record = self.database.get_video(video_id)
|
| 32 |
+
if record is None:
|
| 33 |
+
return
|
| 34 |
+
|
| 35 |
+
try:
|
| 36 |
+
self.database.update_video_status(video_id, "processing")
|
| 37 |
+
video_path = Path(record["video_path"])
|
| 38 |
+
audio_path = extract_audio(
|
| 39 |
+
video_path,
|
| 40 |
+
self.file_store.audio_path(video_id),
|
| 41 |
+
max_duration_seconds=self.settings.max_audio_seconds,
|
| 42 |
+
)
|
| 43 |
+
self.database.set_artifact_paths(video_id, audio_path=audio_path)
|
| 44 |
+
|
| 45 |
+
raw_transcript = self.transcriber.transcribe(audio_path)
|
| 46 |
+
transcript_chunks = normalize_asr_chunks(raw_transcript)
|
| 47 |
+
transcript_path = self.file_store.transcript_path(video_id)
|
| 48 |
+
transcript_path.parent.mkdir(parents=True, exist_ok=True)
|
| 49 |
+
transcript_path.write_text(
|
| 50 |
+
json.dumps([chunk.__dict__ for chunk in transcript_chunks], indent=2),
|
| 51 |
+
encoding="utf-8",
|
| 52 |
+
)
|
| 53 |
+
self.database.replace_transcript_chunks(video_id, transcript_chunks)
|
| 54 |
+
|
| 55 |
+
embeddings = self.embedder.encode([chunk.text for chunk in transcript_chunks])
|
| 56 |
+
embeddings_path = self.file_store.embeddings_path(video_id)
|
| 57 |
+
save_embeddings(embeddings_path, embeddings)
|
| 58 |
+
self.database.set_artifact_paths(
|
| 59 |
+
video_id,
|
| 60 |
+
transcript_path=transcript_path,
|
| 61 |
+
embeddings_path=embeddings_path,
|
| 62 |
+
)
|
| 63 |
+
self.database.update_video_status(video_id, "completed")
|
| 64 |
+
except Exception as exc:
|
| 65 |
+
self.database.update_video_status(video_id, "failed", error=str(exc))
|
iris/pipeline/search.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
from iris.config import Settings
|
| 9 |
+
from iris.pipeline.chunk import TranscriptChunk
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass(frozen=True)
|
| 13 |
+
class SearchMatch:
|
| 14 |
+
start: float
|
| 15 |
+
end: float
|
| 16 |
+
text: str
|
| 17 |
+
score: float
|
| 18 |
+
source: str = "speech"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def keyword_score(query: str, text: str) -> float:
|
| 22 |
+
query_norm = _normalize_text(query)
|
| 23 |
+
text_norm = _normalize_text(text)
|
| 24 |
+
if not query_norm or not text_norm:
|
| 25 |
+
return 0.0
|
| 26 |
+
if query_norm in text_norm:
|
| 27 |
+
return 1.0
|
| 28 |
+
|
| 29 |
+
query_terms = set(query_norm.split())
|
| 30 |
+
text_terms = set(text_norm.split())
|
| 31 |
+
if not query_terms:
|
| 32 |
+
return 0.0
|
| 33 |
+
overlap = len(query_terms & text_terms) / len(query_terms)
|
| 34 |
+
return min(1.0, overlap)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def rank_speech_chunks(
|
| 38 |
+
query: str,
|
| 39 |
+
chunks: list[TranscriptChunk],
|
| 40 |
+
chunk_embeddings: np.ndarray,
|
| 41 |
+
query_embedding: np.ndarray,
|
| 42 |
+
settings: Settings,
|
| 43 |
+
limit: int,
|
| 44 |
+
merge_gap_seconds: float | None = 3.0,
|
| 45 |
+
) -> list[SearchMatch]:
|
| 46 |
+
if not chunks:
|
| 47 |
+
return []
|
| 48 |
+
if chunk_embeddings.ndim != 2 or len(chunk_embeddings) != len(chunks):
|
| 49 |
+
raise ValueError("Chunk embeddings must be a 2D array with one vector per transcript chunk.")
|
| 50 |
+
|
| 51 |
+
query_vector = np.asarray(query_embedding, dtype=np.float32).reshape(-1)
|
| 52 |
+
if query_vector.size == 0:
|
| 53 |
+
semantic_scores = np.zeros(len(chunks), dtype=np.float32)
|
| 54 |
+
else:
|
| 55 |
+
semantic_scores = chunk_embeddings @ query_vector
|
| 56 |
+
semantic_scores = np.clip((semantic_scores + 1.0) / 2.0, 0.0, 1.0)
|
| 57 |
+
|
| 58 |
+
matches: list[SearchMatch] = []
|
| 59 |
+
for index, chunk in enumerate(chunks):
|
| 60 |
+
lexical = keyword_score(query, chunk.text)
|
| 61 |
+
semantic = float(semantic_scores[index])
|
| 62 |
+
score = settings.semantic_weight * semantic + settings.keyword_weight * lexical
|
| 63 |
+
if lexical > 0:
|
| 64 |
+
score = min(1.0, score + settings.keyword_boost)
|
| 65 |
+
if score >= settings.min_speech_score:
|
| 66 |
+
matches.append(
|
| 67 |
+
SearchMatch(
|
| 68 |
+
start=chunk.start,
|
| 69 |
+
end=chunk.end,
|
| 70 |
+
text=chunk.text,
|
| 71 |
+
score=round(float(score), 4),
|
| 72 |
+
)
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
ranked = sorted(matches, key=lambda item: (-item.score, item.start))
|
| 76 |
+
shortlisted = ranked[: max(limit * 3, limit)]
|
| 77 |
+
if merge_gap_seconds is None:
|
| 78 |
+
return shortlisted[:limit]
|
| 79 |
+
return merge_nearby_matches(shortlisted, max_gap_seconds=merge_gap_seconds)[:limit]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def detail_window_settings(detail: str) -> tuple[float, float, float | None]:
|
| 83 |
+
if detail == "coarse":
|
| 84 |
+
return 45.0, 8.0, 8.0
|
| 85 |
+
if detail == "medium":
|
| 86 |
+
return 18.0, 3.0, 2.0
|
| 87 |
+
return 6.0, 0.0, None
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def format_timestamp(seconds: float) -> str:
|
| 91 |
+
total = max(0, int(round(seconds)))
|
| 92 |
+
hours, remainder = divmod(total, 3600)
|
| 93 |
+
minutes, secs = divmod(remainder, 60)
|
| 94 |
+
if hours:
|
| 95 |
+
return f"{hours}:{minutes:02d}:{secs:02d}"
|
| 96 |
+
return f"{minutes}:{secs:02d}"
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def merge_nearby_matches(matches: list[SearchMatch], max_gap_seconds: float) -> list[SearchMatch]:
|
| 100 |
+
if not matches:
|
| 101 |
+
return []
|
| 102 |
+
ordered = sorted(matches, key=lambda item: (item.start, item.end))
|
| 103 |
+
merged: list[SearchMatch] = [ordered[0]]
|
| 104 |
+
|
| 105 |
+
for match in ordered[1:]:
|
| 106 |
+
previous = merged[-1]
|
| 107 |
+
if match.source == previous.source and match.start <= previous.end + max_gap_seconds:
|
| 108 |
+
merged[-1] = SearchMatch(
|
| 109 |
+
start=min(previous.start, match.start),
|
| 110 |
+
end=max(previous.end, match.end),
|
| 111 |
+
text=_join_unique_text(previous.text, match.text),
|
| 112 |
+
score=max(previous.score, match.score),
|
| 113 |
+
source=previous.source,
|
| 114 |
+
)
|
| 115 |
+
else:
|
| 116 |
+
merged.append(match)
|
| 117 |
+
|
| 118 |
+
return sorted(merged, key=lambda item: (-item.score, item.start))
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _normalize_text(value: str) -> str:
|
| 122 |
+
return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9]+", " ", value.lower())).strip()
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _join_unique_text(left: str, right: str) -> str:
|
| 126 |
+
if right in left:
|
| 127 |
+
return left
|
| 128 |
+
if left in right:
|
| 129 |
+
return right
|
| 130 |
+
return f"{left} {right}".strip()
|
iris/pipeline/transcribe.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class WhisperTranscriber:
|
| 9 |
+
def __init__(self, settings: Settings) -> None:
|
| 10 |
+
self.settings = settings
|
| 11 |
+
self._pipeline = None
|
| 12 |
+
|
| 13 |
+
def transcribe(self, audio_path: Path) -> dict:
|
| 14 |
+
transcriber = self._get_pipeline()
|
| 15 |
+
return transcriber(
|
| 16 |
+
str(audio_path),
|
| 17 |
+
return_timestamps=self._timestamp_mode(),
|
| 18 |
+
chunk_length_s=30,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
def _get_pipeline(self):
|
| 22 |
+
if self._pipeline is None:
|
| 23 |
+
from transformers import pipeline
|
| 24 |
+
|
| 25 |
+
self._pipeline = pipeline(
|
| 26 |
+
task="automatic-speech-recognition",
|
| 27 |
+
model=self.settings.asr_model,
|
| 28 |
+
device=transformers_pipeline_device(self.settings.device),
|
| 29 |
+
dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype),
|
| 30 |
+
)
|
| 31 |
+
return self._pipeline
|
| 32 |
+
|
| 33 |
+
def _timestamp_mode(self):
|
| 34 |
+
mode = self.settings.asr_timestamps.strip().lower()
|
| 35 |
+
if mode in {"word", "char"}:
|
| 36 |
+
return mode
|
| 37 |
+
if mode in {"segment", "segments", "true"}:
|
| 38 |
+
return True
|
| 39 |
+
return True
|
iris/pipeline/video.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
SUPPORTED_VIDEO_EXTENSIONS = {".mp4", ".mov", ".mkv", ".webm"}
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def validate_video_filename(filename: str) -> None:
|
| 10 |
+
suffix = Path(filename).suffix.lower()
|
| 11 |
+
if suffix not in SUPPORTED_VIDEO_EXTENSIONS:
|
| 12 |
+
supported = ", ".join(sorted(SUPPORTED_VIDEO_EXTENSIONS))
|
| 13 |
+
raise ValueError(f"Unsupported video extension '{suffix}'. Supported extensions: {supported}.")
|
| 14 |
+
|
iris/pipeline/visual.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import subprocess
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
from PIL import Image
|
| 8 |
+
|
| 9 |
+
from iris.config import Settings, resolve_torch_dtype, transformers_pipeline_device
|
| 10 |
+
from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
|
| 11 |
+
from iris.pipeline.search import SearchMatch, merge_nearby_matches
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass(frozen=True)
|
| 15 |
+
class SampledFrame:
|
| 16 |
+
path: Path
|
| 17 |
+
timestamp: float
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def sample_video_frames(video_path: Path, frames_dir: Path, interval_seconds: float) -> list[SampledFrame]:
|
| 21 |
+
frames_dir.mkdir(parents=True, exist_ok=True)
|
| 22 |
+
existing = sorted(frames_dir.glob("frame_*.jpg"))
|
| 23 |
+
interval_seconds = max(0.25, interval_seconds)
|
| 24 |
+
if not existing:
|
| 25 |
+
command = [
|
| 26 |
+
resolve_ffmpeg_executable(),
|
| 27 |
+
"-y",
|
| 28 |
+
"-i",
|
| 29 |
+
str(video_path),
|
| 30 |
+
"-vf",
|
| 31 |
+
f"fps=1/{interval_seconds}",
|
| 32 |
+
str(frames_dir / "frame_%06d.jpg"),
|
| 33 |
+
]
|
| 34 |
+
try:
|
| 35 |
+
subprocess.run(command, check=True, capture_output=True, text=True)
|
| 36 |
+
except OSError as exc:
|
| 37 |
+
raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
|
| 38 |
+
except subprocess.CalledProcessError as exc:
|
| 39 |
+
detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
|
| 40 |
+
raise RuntimeError(f"ffmpeg failed to sample frames: {detail}") from exc
|
| 41 |
+
existing = sorted(frames_dir.glob("frame_*.jpg"))
|
| 42 |
+
|
| 43 |
+
return [
|
| 44 |
+
SampledFrame(path=path, timestamp=index * interval_seconds)
|
| 45 |
+
for index, path in enumerate(existing)
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class VisualSearcher:
|
| 50 |
+
def __init__(self, settings: Settings) -> None:
|
| 51 |
+
self.settings = settings
|
| 52 |
+
self._classifier = None
|
| 53 |
+
|
| 54 |
+
def search(self, query: str, frames: list[SampledFrame], limit: int) -> list[SearchMatch]:
|
| 55 |
+
if not frames:
|
| 56 |
+
return []
|
| 57 |
+
classifier = self._get_classifier()
|
| 58 |
+
matches: list[SearchMatch] = []
|
| 59 |
+
candidate_labels = [query, f"not {query}", "unrelated scene"]
|
| 60 |
+
for frame in frames:
|
| 61 |
+
with Image.open(frame.path) as image:
|
| 62 |
+
result = classifier(image.convert("RGB"), candidate_labels=candidate_labels)
|
| 63 |
+
if not result:
|
| 64 |
+
continue
|
| 65 |
+
prompt_result = next((item for item in result if item.get("label") == query), result[0])
|
| 66 |
+
score = float(prompt_result["score"])
|
| 67 |
+
if score < self.settings.min_visual_score:
|
| 68 |
+
continue
|
| 69 |
+
matches.append(
|
| 70 |
+
SearchMatch(
|
| 71 |
+
start=frame.timestamp,
|
| 72 |
+
end=frame.timestamp + self.settings.frame_sample_interval_seconds,
|
| 73 |
+
text=f"Visual match for '{query}'",
|
| 74 |
+
score=round(score, 4),
|
| 75 |
+
source="visual",
|
| 76 |
+
)
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
ranked = sorted(matches, key=lambda item: (-item.score, item.start))[: max(limit * 3, limit)]
|
| 80 |
+
return merge_nearby_matches(ranked, max_gap_seconds=self.settings.frame_sample_interval_seconds)[:limit]
|
| 81 |
+
|
| 82 |
+
def _get_classifier(self):
|
| 83 |
+
if self._classifier is None:
|
| 84 |
+
from transformers import pipeline
|
| 85 |
+
|
| 86 |
+
self._classifier = pipeline(
|
| 87 |
+
task="zero-shot-image-classification",
|
| 88 |
+
model=self.settings.visual_model,
|
| 89 |
+
device=transformers_pipeline_device(self.settings.device),
|
| 90 |
+
dtype=resolve_torch_dtype(self.settings.device, self.settings.torch_dtype),
|
| 91 |
+
)
|
| 92 |
+
return self._classifier
|
iris/storage/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Storage helpers for IRIS."""
|
| 2 |
+
|
iris/storage/database.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import sqlite3
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from iris.models.schemas import VideoStatus
|
| 8 |
+
from iris.pipeline.chunk import TranscriptChunk
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class Database:
|
| 12 |
+
def __init__(self, path: Path) -> None:
|
| 13 |
+
self.path = path
|
| 14 |
+
|
| 15 |
+
def init(self) -> None:
|
| 16 |
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
| 17 |
+
with self._connect() as connection:
|
| 18 |
+
connection.executescript(
|
| 19 |
+
"""
|
| 20 |
+
PRAGMA journal_mode=WAL;
|
| 21 |
+
|
| 22 |
+
CREATE TABLE IF NOT EXISTS videos (
|
| 23 |
+
id TEXT PRIMARY KEY,
|
| 24 |
+
original_filename TEXT NOT NULL,
|
| 25 |
+
video_path TEXT NOT NULL,
|
| 26 |
+
audio_path TEXT,
|
| 27 |
+
transcript_path TEXT,
|
| 28 |
+
embeddings_path TEXT,
|
| 29 |
+
status TEXT NOT NULL,
|
| 30 |
+
error TEXT,
|
| 31 |
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
| 32 |
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
| 33 |
+
);
|
| 34 |
+
|
| 35 |
+
CREATE TABLE IF NOT EXISTS transcript_chunks (
|
| 36 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 37 |
+
video_id TEXT NOT NULL,
|
| 38 |
+
start REAL NOT NULL,
|
| 39 |
+
end REAL NOT NULL,
|
| 40 |
+
text TEXT NOT NULL,
|
| 41 |
+
FOREIGN KEY(video_id) REFERENCES videos(id) ON DELETE CASCADE
|
| 42 |
+
);
|
| 43 |
+
|
| 44 |
+
CREATE TABLE IF NOT EXISTS searches (
|
| 45 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 46 |
+
video_id TEXT NOT NULL,
|
| 47 |
+
query TEXT NOT NULL,
|
| 48 |
+
include_visual INTEGER NOT NULL,
|
| 49 |
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
| 50 |
+
FOREIGN KEY(video_id) REFERENCES videos(id) ON DELETE CASCADE
|
| 51 |
+
);
|
| 52 |
+
"""
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
def create_video(self, video_id: str, original_filename: str, video_path: Path) -> dict[str, Any]:
|
| 56 |
+
with self._connect() as connection:
|
| 57 |
+
connection.execute(
|
| 58 |
+
"""
|
| 59 |
+
INSERT INTO videos (id, original_filename, video_path, status)
|
| 60 |
+
VALUES (?, ?, ?, 'queued')
|
| 61 |
+
""",
|
| 62 |
+
(video_id, original_filename, str(video_path)),
|
| 63 |
+
)
|
| 64 |
+
return self.get_video(video_id) or {}
|
| 65 |
+
|
| 66 |
+
def get_video(self, video_id: str) -> dict[str, Any] | None:
|
| 67 |
+
with self._connect() as connection:
|
| 68 |
+
row = connection.execute("SELECT * FROM videos WHERE id = ?", (video_id,)).fetchone()
|
| 69 |
+
return dict(row) if row else None
|
| 70 |
+
|
| 71 |
+
def update_video_status(self, video_id: str, status: VideoStatus, error: str | None = None) -> None:
|
| 72 |
+
with self._connect() as connection:
|
| 73 |
+
connection.execute(
|
| 74 |
+
"""
|
| 75 |
+
UPDATE videos
|
| 76 |
+
SET status = ?, error = ?, updated_at = CURRENT_TIMESTAMP
|
| 77 |
+
WHERE id = ?
|
| 78 |
+
""",
|
| 79 |
+
(status, error, video_id),
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
def set_artifact_paths(
|
| 83 |
+
self,
|
| 84 |
+
video_id: str,
|
| 85 |
+
audio_path: Path | None = None,
|
| 86 |
+
transcript_path: Path | None = None,
|
| 87 |
+
embeddings_path: Path | None = None,
|
| 88 |
+
) -> None:
|
| 89 |
+
with self._connect() as connection:
|
| 90 |
+
connection.execute(
|
| 91 |
+
"""
|
| 92 |
+
UPDATE videos
|
| 93 |
+
SET audio_path = COALESCE(?, audio_path),
|
| 94 |
+
transcript_path = COALESCE(?, transcript_path),
|
| 95 |
+
embeddings_path = COALESCE(?, embeddings_path),
|
| 96 |
+
updated_at = CURRENT_TIMESTAMP
|
| 97 |
+
WHERE id = ?
|
| 98 |
+
""",
|
| 99 |
+
(
|
| 100 |
+
str(audio_path) if audio_path else None,
|
| 101 |
+
str(transcript_path) if transcript_path else None,
|
| 102 |
+
str(embeddings_path) if embeddings_path else None,
|
| 103 |
+
video_id,
|
| 104 |
+
),
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def replace_transcript_chunks(self, video_id: str, chunks: list[TranscriptChunk]) -> None:
|
| 108 |
+
with self._connect() as connection:
|
| 109 |
+
connection.execute("DELETE FROM transcript_chunks WHERE video_id = ?", (video_id,))
|
| 110 |
+
connection.executemany(
|
| 111 |
+
"""
|
| 112 |
+
INSERT INTO transcript_chunks (video_id, start, end, text)
|
| 113 |
+
VALUES (?, ?, ?, ?)
|
| 114 |
+
""",
|
| 115 |
+
[(video_id, chunk.start, chunk.end, chunk.text) for chunk in chunks],
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
def list_transcript_chunks(self, video_id: str) -> list[TranscriptChunk]:
|
| 119 |
+
with self._connect() as connection:
|
| 120 |
+
rows = connection.execute(
|
| 121 |
+
"""
|
| 122 |
+
SELECT start, end, text
|
| 123 |
+
FROM transcript_chunks
|
| 124 |
+
WHERE video_id = ?
|
| 125 |
+
ORDER BY start, end, id
|
| 126 |
+
""",
|
| 127 |
+
(video_id,),
|
| 128 |
+
).fetchall()
|
| 129 |
+
return [
|
| 130 |
+
TranscriptChunk(start=float(row["start"]), end=float(row["end"]), text=str(row["text"]))
|
| 131 |
+
for row in rows
|
| 132 |
+
]
|
| 133 |
+
|
| 134 |
+
def record_search(self, video_id: str, query: str, include_visual: bool) -> None:
|
| 135 |
+
with self._connect() as connection:
|
| 136 |
+
connection.execute(
|
| 137 |
+
"""
|
| 138 |
+
INSERT INTO searches (video_id, query, include_visual)
|
| 139 |
+
VALUES (?, ?, ?)
|
| 140 |
+
""",
|
| 141 |
+
(video_id, query, int(include_visual)),
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
def _connect(self) -> sqlite3.Connection:
|
| 145 |
+
connection = sqlite3.connect(self.path)
|
| 146 |
+
connection.row_factory = sqlite3.Row
|
| 147 |
+
connection.execute("PRAGMA foreign_keys=ON;")
|
| 148 |
+
return connection
|
| 149 |
+
|
iris/storage/files.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import shutil
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from uuid import uuid4
|
| 6 |
+
|
| 7 |
+
from fastapi import UploadFile
|
| 8 |
+
|
| 9 |
+
from iris.config import Settings
|
| 10 |
+
from iris.pipeline.video import validate_video_filename
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class FileStore:
|
| 14 |
+
def __init__(self, settings: Settings) -> None:
|
| 15 |
+
self.settings = settings
|
| 16 |
+
|
| 17 |
+
def video_dir(self, video_id: str) -> Path:
|
| 18 |
+
return self.settings.data_dir / "videos" / video_id
|
| 19 |
+
|
| 20 |
+
def video_path(self, video_id: str, filename: str) -> Path:
|
| 21 |
+
return self.video_dir(video_id) / "source" / filename
|
| 22 |
+
|
| 23 |
+
def audio_path(self, video_id: str) -> Path:
|
| 24 |
+
return self.video_dir(video_id) / "audio" / "audio.wav"
|
| 25 |
+
|
| 26 |
+
def transcript_path(self, video_id: str) -> Path:
|
| 27 |
+
return self.video_dir(video_id) / "transcript" / "transcript.json"
|
| 28 |
+
|
| 29 |
+
def embeddings_path(self, video_id: str) -> Path:
|
| 30 |
+
return self.video_dir(video_id) / "embeddings" / "speech.npy"
|
| 31 |
+
|
| 32 |
+
def frames_dir(self, video_id: str) -> Path:
|
| 33 |
+
return self.video_dir(video_id) / "frames"
|
| 34 |
+
|
| 35 |
+
def save_upload(self, upload: UploadFile) -> tuple[str, Path]:
|
| 36 |
+
filename = Path(upload.filename or "video").name
|
| 37 |
+
validate_video_filename(filename)
|
| 38 |
+
video_id = str(uuid4())
|
| 39 |
+
destination = self.video_path(video_id, filename)
|
| 40 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 41 |
+
with destination.open("wb") as output:
|
| 42 |
+
shutil.copyfileobj(upload.file, output)
|
| 43 |
+
return video_id, destination
|
| 44 |
+
|
requirements.txt
CHANGED
|
@@ -1,2 +1,8 @@
|
|
| 1 |
-
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.115.0
|
| 2 |
+
imageio-ffmpeg>=0.5.0
|
| 3 |
+
numpy>=1.26.0
|
| 4 |
+
pillow>=10.0.0
|
| 5 |
+
python-multipart>=0.0.9
|
| 6 |
+
sentence-transformers>=3.0.0
|
| 7 |
+
torch>=2.2.0
|
| 8 |
+
transformers>=4.45.0
|