Update app.py

app.py

@@ -62,6 +62,26 @@ def to_wav(src: str, dst: str) -> bool:
     return r.returncode == 0 and Path(dst).exists()
 
 
+def video_has_audio(path: str) -> bool:
+    r = subprocess.run(
+        ["ffprobe", "-v", "error", "-select_streams", "a",
+         "-show_entries", "stream=codec_type", "-of", "csv=p=0", path],
+        capture_output=True, text=True,
+    )
+    return bool(r.stdout.strip())
+
+
+def collect_video_paths(messages: list[dict]) -> list[str]:
+    paths = []
+    for m in messages:
+        content = m.get("content")
+        if isinstance(content, list):
+            for part in content:
+                if part.get("type") == "video":
+                    paths.append(part["video"])
+    return paths
+
+
 def decode_b64(data: str) -> bytes:
     try:
         raw = base64.b64decode(data, validate=True)
@@ -156,21 +176,31 @@ def convert_messages(messages: list[dict], workdir: str):
 
 
 def run_model(messages: list[dict], has_video: bool, max_new_tokens: int) -> str:
-    kwargs = {}
-    if has_video:
-        kwargs = {
-            "load_audio_from_video": True,
-            "processor_kwargs": {"use_audio_in_video": True},
-        }
-
-    inputs = processor.apply_chat_template(
-        messages,
-        add_generation_prompt=True,
-        tokenize=True,
-        return_dict=True,
-        return_tensors="pt",
-        **kwargs,
-    ).to(DEVICE)
+    videos = collect_video_paths(messages) if has_video else []
+    # Only ask for the soundtrack when every video actually has one.
+    use_audio = bool(videos) and all(video_has_audio(v) for v in videos)
+    audio_kwargs = (
+        {"load_audio_from_video": True, "processor_kwargs": {"use_audio_in_video": True}}
+        if use_audio else {}
+    )
+
+    def build_inputs(kwargs: dict):
+        return processor.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+            **kwargs,
+        ).to(DEVICE)
+
+    try:
+        inputs = build_inputs(audio_kwargs)
+    except Exception:
+        if not videos:
+            raise
+        log.exception("Video preprocessing with audio failed; retrying with frames only")
+        inputs = build_inputs({})
 
     with model_lock, torch.no_grad():
         out = model.generate(