Spaces:
Running on T4
Running on T4
Fix WhisperX (keyword call, auto language, prewarm), fp16 video extractor, fp16 text model via accelerate
Browse files- app.py +138 -4
- requirements.txt +1 -0
app.py
CHANGED
|
@@ -18,6 +18,16 @@ from urllib.parse import urlparse
|
|
| 18 |
# Disable hf_transfer — uvx sandboxes don't have it
|
| 19 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
import gradio as gr
|
| 22 |
import numpy as np
|
| 23 |
import pandas as pd
|
|
@@ -29,6 +39,120 @@ if hf_token:
|
|
| 29 |
os.environ["HUGGING_FACE_HUB_TOKEN"] = hf_token
|
| 30 |
print("HF token configured")
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
FUNCTIONAL_REGIONS = {
|
| 33 |
"visual_processing": {
|
| 34 |
"description": "Visual cortex",
|
|
@@ -103,7 +227,11 @@ def load_model():
|
|
| 103 |
from nilearn.datasets import fetch_atlas_surf_destrieux
|
| 104 |
|
| 105 |
print("Loading TRIBE v2 model...")
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
|
| 108 |
print("Loading Destrieux atlas...")
|
| 109 |
atlas = fetch_atlas_surf_destrieux()
|
|
@@ -507,10 +635,15 @@ def _analyze_video_locked(video_url: str, filename: str = "video.mp4", ad_contex
|
|
| 507 |
# Build events — try full pipeline (WhisperX + audio + video), fallback to audio-only
|
| 508 |
try:
|
| 509 |
print("Building events with full pipeline (WhisperX)...")
|
| 510 |
-
|
| 511 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 512 |
except Exception as whisperx_err:
|
| 513 |
-
print(f"WhisperX
|
|
|
|
| 514 |
df = build_events_without_whisperx(video_path)
|
| 515 |
print(f"Events: {len(df)} rows (audio-only fallback)")
|
| 516 |
|
|
@@ -549,6 +682,7 @@ def _analyze_video_locked(video_url: str, filename: str = "video.mp4", ad_contex
|
|
| 549 |
elapsed = time.time() - start_time
|
| 550 |
|
| 551 |
report = {
|
|
|
|
| 552 |
"filename": filename,
|
| 553 |
"duration_seconds": valid_len + hrf_offset,
|
| 554 |
"analyzed_seconds": valid_len,
|
|
|
|
| 18 |
# Disable hf_transfer — uvx sandboxes don't have it
|
| 19 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
|
| 20 |
|
| 21 |
+
# Persistent storage, when the Space has it: the WhisperX environment (uvx),
|
| 22 |
+
# the whisper large-v3 weights and every HF model then survive a restart
|
| 23 |
+
# instead of being downloaded again on each cold start.
|
| 24 |
+
if os.path.isdir("/data") and os.access("/data", os.W_OK):
|
| 25 |
+
os.environ.setdefault("UV_CACHE_DIR", "/data/uv-cache")
|
| 26 |
+
os.environ.setdefault("HF_HOME", "/data/hf-home")
|
| 27 |
+
for _d in (os.environ["UV_CACHE_DIR"], os.environ["HF_HOME"]):
|
| 28 |
+
os.makedirs(_d, exist_ok=True)
|
| 29 |
+
print("Persistent /data found: uv and HF caches live there")
|
| 30 |
+
|
| 31 |
import gradio as gr
|
| 32 |
import numpy as np
|
| 33 |
import pandas as pd
|
|
|
|
| 39 |
os.environ["HUGGING_FACE_HUB_TOKEN"] = hf_token
|
| 40 |
print("HF token configured")
|
| 41 |
|
| 42 |
+
import torch
|
| 43 |
+
|
| 44 |
+
CUDA = torch.cuda.is_available()
|
| 45 |
+
print(f"torch {torch.__version__} | cuda={CUDA}" + (f" | {torch.cuda.get_device_name(0)} | {torch.cuda.get_device_properties(0).total_memory / 2**30:.1f} GiB" if CUDA else ""))
|
| 46 |
+
if CUDA:
|
| 47 |
+
torch.backends.cudnn.benchmark = True
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _patch_video_extractor_fp16() -> None:
|
| 51 |
+
"""Run the VJEPA2 (ViT-Giant, 64 frames per clip) forward under fp16 autocast.
|
| 52 |
+
|
| 53 |
+
The extractor already sits on the GPU, but in fp32 a T4 does ~8 TFLOPS;
|
| 54 |
+
its tensor cores do ~65 in fp16. Hidden states are cast back to fp32 before
|
| 55 |
+
neuralset aggregates them, so nothing downstream changes dtype.
|
| 56 |
+
"""
|
| 57 |
+
if not CUDA:
|
| 58 |
+
return
|
| 59 |
+
try:
|
| 60 |
+
from neuralset.extractors import image as ns_image
|
| 61 |
+
except Exception as e: # pragma: no cover
|
| 62 |
+
print(f"[perf] could not import neuralset.extractors.image: {e}")
|
| 63 |
+
return
|
| 64 |
+
original = ns_image.HuggingFaceImage._get_hidden_states
|
| 65 |
+
|
| 66 |
+
def _fp16_hidden_states(self, images):
|
| 67 |
+
if str(self.device).startswith("cuda"):
|
| 68 |
+
with torch.autocast("cuda", dtype=torch.float16):
|
| 69 |
+
states = original(self, images)
|
| 70 |
+
return [st.float() for st in states]
|
| 71 |
+
return original(self, images)
|
| 72 |
+
|
| 73 |
+
ns_image.HuggingFaceImage._get_hidden_states = _fp16_hidden_states
|
| 74 |
+
print("[perf] video/image extractor runs under fp16 autocast")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _patch_whisperx_language() -> None:
|
| 78 |
+
"""Let WhisperX detect the language instead of the library's hard-coded English.
|
| 79 |
+
|
| 80 |
+
Ads are short and speech starts early, so Whisper's detection on the first
|
| 81 |
+
30 s is reliable here; forcing English turns Italian or Hebrew copy into
|
| 82 |
+
garbage words, and garbage words become garbage text features.
|
| 83 |
+
Set TRIBE_WHISPER_LANGUAGE=english (or another supported name) to force one.
|
| 84 |
+
"""
|
| 85 |
+
try:
|
| 86 |
+
from tribev2 import eventstransforms as et
|
| 87 |
+
except Exception as e: # pragma: no cover
|
| 88 |
+
print(f"[whisperx] could not import tribev2.eventstransforms: {e}")
|
| 89 |
+
return
|
| 90 |
+
forced = os.environ.get("TRIBE_WHISPER_LANGUAGE", "auto").strip().lower()
|
| 91 |
+
original = et.ExtractWordsFromAudio._get_transcript_from_audio
|
| 92 |
+
|
| 93 |
+
def _transcribe(wav_filename, language):
|
| 94 |
+
import json as _json
|
| 95 |
+
import subprocess as _sp
|
| 96 |
+
import tempfile as _tf
|
| 97 |
+
|
| 98 |
+
if forced != "auto":
|
| 99 |
+
return original(wav_filename, forced)
|
| 100 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 101 |
+
with _tf.TemporaryDirectory() as output_dir:
|
| 102 |
+
cmd = [
|
| 103 |
+
"uvx", "whisperx", str(wav_filename),
|
| 104 |
+
"--model", "large-v3",
|
| 105 |
+
"--device", device,
|
| 106 |
+
"--compute_type", "float16" if device == "cuda" else "int8",
|
| 107 |
+
"--batch_size", "16",
|
| 108 |
+
"--output_dir", output_dir,
|
| 109 |
+
"--output_format", "json",
|
| 110 |
+
]
|
| 111 |
+
env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"}
|
| 112 |
+
t0 = time.time()
|
| 113 |
+
result = _sp.run(cmd, capture_output=True, text=True, env=env)
|
| 114 |
+
if result.returncode != 0:
|
| 115 |
+
raise RuntimeError(f"whisperx failed:\n{result.stderr[-3000:]}")
|
| 116 |
+
json_path = Path(output_dir) / f"{Path(wav_filename).stem}.json"
|
| 117 |
+
transcript = _json.loads(json_path.read_text())
|
| 118 |
+
detected = transcript.get("language", "?")
|
| 119 |
+
n_words = sum(len(seg.get("words", [])) for seg in transcript.get("segments", []))
|
| 120 |
+
print(f"[whisperx] language={detected} words={n_words} in {time.time() - t0:.0f}s")
|
| 121 |
+
words = []
|
| 122 |
+
for i, segment in enumerate(transcript.get("segments", [])):
|
| 123 |
+
sentence = segment.get("text", "").replace('"', "")
|
| 124 |
+
for word in segment.get("words", []):
|
| 125 |
+
if "start" not in word or "end" not in word:
|
| 126 |
+
continue
|
| 127 |
+
words.append({
|
| 128 |
+
"text": word["word"].replace('"', ""),
|
| 129 |
+
"start": word["start"],
|
| 130 |
+
"duration": word["end"] - word["start"],
|
| 131 |
+
"sequence_id": i,
|
| 132 |
+
"sentence": sentence,
|
| 133 |
+
})
|
| 134 |
+
return pd.DataFrame(words)
|
| 135 |
+
|
| 136 |
+
et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(_transcribe)
|
| 137 |
+
print(f"[whisperx] language mode: {forced}")
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _prewarm_whisperx() -> None:
|
| 141 |
+
"""Build the uvx whisperx environment once at startup so the first job does not pay for it."""
|
| 142 |
+
def run():
|
| 143 |
+
try:
|
| 144 |
+
t0 = time.time()
|
| 145 |
+
r = subprocess.run(["uvx", "whisperx", "--help"], capture_output=True, text=True, timeout=1800)
|
| 146 |
+
print(f"[whisperx] prewarm {'ok' if r.returncode == 0 else 'FAILED'} in {time.time() - t0:.0f}s" + ("" if r.returncode == 0 else f": {r.stderr[-800:]}"))
|
| 147 |
+
except Exception as e:
|
| 148 |
+
print(f"[whisperx] prewarm error: {e}")
|
| 149 |
+
threading.Thread(target=run, daemon=True, name="whisperx-prewarm").start()
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
_patch_video_extractor_fp16()
|
| 153 |
+
_patch_whisperx_language()
|
| 154 |
+
_prewarm_whisperx()
|
| 155 |
+
|
| 156 |
FUNCTIONAL_REGIONS = {
|
| 157 |
"visual_processing": {
|
| 158 |
"description": "Visual cortex",
|
|
|
|
| 227 |
from nilearn.datasets import fetch_atlas_surf_destrieux
|
| 228 |
|
| 229 |
print("Loading TRIBE v2 model...")
|
| 230 |
+
# Text features come from Llama-3.2-3B. In fp32 that is ~12.8 GiB and cannot
|
| 231 |
+
# share a 16 GiB T4 with VJEPA2 (~4.4 GiB); "accelerate" loads it in fp16
|
| 232 |
+
# with device_map=auto (~6.4 GiB, spilling to CPU rather than OOM-ing).
|
| 233 |
+
config_update = {"data.text_feature.device": "accelerate"} if CUDA else None
|
| 234 |
+
model = TribeModel.from_pretrained("facebook/tribev2", cache_folder="./cache", config_update=config_update)
|
| 235 |
|
| 236 |
print("Loading Destrieux atlas...")
|
| 237 |
atlas = fetch_atlas_surf_destrieux()
|
|
|
|
| 635 |
# Build events — try full pipeline (WhisperX + audio + video), fallback to audio-only
|
| 636 |
try:
|
| 637 |
print("Building events with full pipeline (WhisperX)...")
|
| 638 |
+
# Keyword argument: the library accepts exactly one of text/audio/video.
|
| 639 |
+
# Passing three positionals set all three and it refused, which is why
|
| 640 |
+
# every analysis before 2026-09-24 silently fell back to audio-only.
|
| 641 |
+
df = model.get_events_dataframe(video_path=video_path)
|
| 642 |
+
n_words = int((df["type"] == "Word").sum()) if "type" in df else 0
|
| 643 |
+
print(f"Events: {len(df)} rows, {n_words} words (full pipeline with transcription)")
|
| 644 |
except Exception as whisperx_err:
|
| 645 |
+
print(f"WhisperX pipeline failed, falling back to audio-only: {type(whisperx_err).__name__}: {whisperx_err}")
|
| 646 |
+
traceback.print_exc()
|
| 647 |
df = build_events_without_whisperx(video_path)
|
| 648 |
print(f"Events: {len(df)} rows (audio-only fallback)")
|
| 649 |
|
|
|
|
| 682 |
elapsed = time.time() - start_time
|
| 683 |
|
| 684 |
report = {
|
| 685 |
+
"pipeline": "full" if ("type" in df and (df["type"] == "Word").any()) else "audio_only",
|
| 686 |
"filename": filename,
|
| 687 |
"duration_seconds": valid_len + hrf_offset,
|
| 688 |
"analyzed_seconds": valid_len,
|
requirements.txt
CHANGED
|
@@ -6,3 +6,4 @@ gradio
|
|
| 6 |
requests
|
| 7 |
uv
|
| 8 |
hf_transfer
|
|
|
|
|
|
| 6 |
requests
|
| 7 |
uv
|
| 8 |
hf_transfer
|
| 9 |
+
accelerate
|