MoburstAI commited on
Commit
6e48ea5
·
verified ·
1 Parent(s): a8ae6d3

Fix WhisperX (keyword call, auto language, prewarm), fp16 video extractor, fp16 text model via accelerate

Browse files
Files changed (2) hide show
  1. app.py +138 -4
  2. requirements.txt +1 -0
app.py CHANGED
@@ -18,6 +18,16 @@ from urllib.parse import urlparse
18
  # Disable hf_transfer — uvx sandboxes don't have it
19
  os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
20
 
 
 
 
 
 
 
 
 
 
 
21
  import gradio as gr
22
  import numpy as np
23
  import pandas as pd
@@ -29,6 +39,120 @@ if hf_token:
29
  os.environ["HUGGING_FACE_HUB_TOKEN"] = hf_token
30
  print("HF token configured")
31
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  FUNCTIONAL_REGIONS = {
33
  "visual_processing": {
34
  "description": "Visual cortex",
@@ -103,7 +227,11 @@ def load_model():
103
  from nilearn.datasets import fetch_atlas_surf_destrieux
104
 
105
  print("Loading TRIBE v2 model...")
106
- model = TribeModel.from_pretrained("facebook/tribev2", cache_folder="./cache")
 
 
 
 
107
 
108
  print("Loading Destrieux atlas...")
109
  atlas = fetch_atlas_surf_destrieux()
@@ -507,10 +635,15 @@ def _analyze_video_locked(video_url: str, filename: str = "video.mp4", ad_contex
507
  # Build events — try full pipeline (WhisperX + audio + video), fallback to audio-only
508
  try:
509
  print("Building events with full pipeline (WhisperX)...")
510
- df = model.get_events_dataframe(video_path, "default", "default")
511
- print(f"Events: {len(df)} rows (full pipeline with transcription)")
 
 
 
 
512
  except Exception as whisperx_err:
513
- print(f"WhisperX failed ({whisperx_err}), falling back to audio-only...")
 
514
  df = build_events_without_whisperx(video_path)
515
  print(f"Events: {len(df)} rows (audio-only fallback)")
516
 
@@ -549,6 +682,7 @@ def _analyze_video_locked(video_url: str, filename: str = "video.mp4", ad_contex
549
  elapsed = time.time() - start_time
550
 
551
  report = {
 
552
  "filename": filename,
553
  "duration_seconds": valid_len + hrf_offset,
554
  "analyzed_seconds": valid_len,
 
18
  # Disable hf_transfer — uvx sandboxes don't have it
19
  os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
20
 
21
+ # Persistent storage, when the Space has it: the WhisperX environment (uvx),
22
+ # the whisper large-v3 weights and every HF model then survive a restart
23
+ # instead of being downloaded again on each cold start.
24
+ if os.path.isdir("/data") and os.access("/data", os.W_OK):
25
+ os.environ.setdefault("UV_CACHE_DIR", "/data/uv-cache")
26
+ os.environ.setdefault("HF_HOME", "/data/hf-home")
27
+ for _d in (os.environ["UV_CACHE_DIR"], os.environ["HF_HOME"]):
28
+ os.makedirs(_d, exist_ok=True)
29
+ print("Persistent /data found: uv and HF caches live there")
30
+
31
  import gradio as gr
32
  import numpy as np
33
  import pandas as pd
 
39
  os.environ["HUGGING_FACE_HUB_TOKEN"] = hf_token
40
  print("HF token configured")
41
 
42
+ import torch
43
+
44
+ CUDA = torch.cuda.is_available()
45
+ print(f"torch {torch.__version__} | cuda={CUDA}" + (f" | {torch.cuda.get_device_name(0)} | {torch.cuda.get_device_properties(0).total_memory / 2**30:.1f} GiB" if CUDA else ""))
46
+ if CUDA:
47
+ torch.backends.cudnn.benchmark = True
48
+
49
+
50
+ def _patch_video_extractor_fp16() -> None:
51
+ """Run the VJEPA2 (ViT-Giant, 64 frames per clip) forward under fp16 autocast.
52
+
53
+ The extractor already sits on the GPU, but in fp32 a T4 does ~8 TFLOPS;
54
+ its tensor cores do ~65 in fp16. Hidden states are cast back to fp32 before
55
+ neuralset aggregates them, so nothing downstream changes dtype.
56
+ """
57
+ if not CUDA:
58
+ return
59
+ try:
60
+ from neuralset.extractors import image as ns_image
61
+ except Exception as e: # pragma: no cover
62
+ print(f"[perf] could not import neuralset.extractors.image: {e}")
63
+ return
64
+ original = ns_image.HuggingFaceImage._get_hidden_states
65
+
66
+ def _fp16_hidden_states(self, images):
67
+ if str(self.device).startswith("cuda"):
68
+ with torch.autocast("cuda", dtype=torch.float16):
69
+ states = original(self, images)
70
+ return [st.float() for st in states]
71
+ return original(self, images)
72
+
73
+ ns_image.HuggingFaceImage._get_hidden_states = _fp16_hidden_states
74
+ print("[perf] video/image extractor runs under fp16 autocast")
75
+
76
+
77
+ def _patch_whisperx_language() -> None:
78
+ """Let WhisperX detect the language instead of the library's hard-coded English.
79
+
80
+ Ads are short and speech starts early, so Whisper's detection on the first
81
+ 30 s is reliable here; forcing English turns Italian or Hebrew copy into
82
+ garbage words, and garbage words become garbage text features.
83
+ Set TRIBE_WHISPER_LANGUAGE=english (or another supported name) to force one.
84
+ """
85
+ try:
86
+ from tribev2 import eventstransforms as et
87
+ except Exception as e: # pragma: no cover
88
+ print(f"[whisperx] could not import tribev2.eventstransforms: {e}")
89
+ return
90
+ forced = os.environ.get("TRIBE_WHISPER_LANGUAGE", "auto").strip().lower()
91
+ original = et.ExtractWordsFromAudio._get_transcript_from_audio
92
+
93
+ def _transcribe(wav_filename, language):
94
+ import json as _json
95
+ import subprocess as _sp
96
+ import tempfile as _tf
97
+
98
+ if forced != "auto":
99
+ return original(wav_filename, forced)
100
+ device = "cuda" if torch.cuda.is_available() else "cpu"
101
+ with _tf.TemporaryDirectory() as output_dir:
102
+ cmd = [
103
+ "uvx", "whisperx", str(wav_filename),
104
+ "--model", "large-v3",
105
+ "--device", device,
106
+ "--compute_type", "float16" if device == "cuda" else "int8",
107
+ "--batch_size", "16",
108
+ "--output_dir", output_dir,
109
+ "--output_format", "json",
110
+ ]
111
+ env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"}
112
+ t0 = time.time()
113
+ result = _sp.run(cmd, capture_output=True, text=True, env=env)
114
+ if result.returncode != 0:
115
+ raise RuntimeError(f"whisperx failed:\n{result.stderr[-3000:]}")
116
+ json_path = Path(output_dir) / f"{Path(wav_filename).stem}.json"
117
+ transcript = _json.loads(json_path.read_text())
118
+ detected = transcript.get("language", "?")
119
+ n_words = sum(len(seg.get("words", [])) for seg in transcript.get("segments", []))
120
+ print(f"[whisperx] language={detected} words={n_words} in {time.time() - t0:.0f}s")
121
+ words = []
122
+ for i, segment in enumerate(transcript.get("segments", [])):
123
+ sentence = segment.get("text", "").replace('"', "")
124
+ for word in segment.get("words", []):
125
+ if "start" not in word or "end" not in word:
126
+ continue
127
+ words.append({
128
+ "text": word["word"].replace('"', ""),
129
+ "start": word["start"],
130
+ "duration": word["end"] - word["start"],
131
+ "sequence_id": i,
132
+ "sentence": sentence,
133
+ })
134
+ return pd.DataFrame(words)
135
+
136
+ et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(_transcribe)
137
+ print(f"[whisperx] language mode: {forced}")
138
+
139
+
140
+ def _prewarm_whisperx() -> None:
141
+ """Build the uvx whisperx environment once at startup so the first job does not pay for it."""
142
+ def run():
143
+ try:
144
+ t0 = time.time()
145
+ r = subprocess.run(["uvx", "whisperx", "--help"], capture_output=True, text=True, timeout=1800)
146
+ print(f"[whisperx] prewarm {'ok' if r.returncode == 0 else 'FAILED'} in {time.time() - t0:.0f}s" + ("" if r.returncode == 0 else f": {r.stderr[-800:]}"))
147
+ except Exception as e:
148
+ print(f"[whisperx] prewarm error: {e}")
149
+ threading.Thread(target=run, daemon=True, name="whisperx-prewarm").start()
150
+
151
+
152
+ _patch_video_extractor_fp16()
153
+ _patch_whisperx_language()
154
+ _prewarm_whisperx()
155
+
156
  FUNCTIONAL_REGIONS = {
157
  "visual_processing": {
158
  "description": "Visual cortex",
 
227
  from nilearn.datasets import fetch_atlas_surf_destrieux
228
 
229
  print("Loading TRIBE v2 model...")
230
+ # Text features come from Llama-3.2-3B. In fp32 that is ~12.8 GiB and cannot
231
+ # share a 16 GiB T4 with VJEPA2 (~4.4 GiB); "accelerate" loads it in fp16
232
+ # with device_map=auto (~6.4 GiB, spilling to CPU rather than OOM-ing).
233
+ config_update = {"data.text_feature.device": "accelerate"} if CUDA else None
234
+ model = TribeModel.from_pretrained("facebook/tribev2", cache_folder="./cache", config_update=config_update)
235
 
236
  print("Loading Destrieux atlas...")
237
  atlas = fetch_atlas_surf_destrieux()
 
635
  # Build events — try full pipeline (WhisperX + audio + video), fallback to audio-only
636
  try:
637
  print("Building events with full pipeline (WhisperX)...")
638
+ # Keyword argument: the library accepts exactly one of text/audio/video.
639
+ # Passing three positionals set all three and it refused, which is why
640
+ # every analysis before 2026-09-24 silently fell back to audio-only.
641
+ df = model.get_events_dataframe(video_path=video_path)
642
+ n_words = int((df["type"] == "Word").sum()) if "type" in df else 0
643
+ print(f"Events: {len(df)} rows, {n_words} words (full pipeline with transcription)")
644
  except Exception as whisperx_err:
645
+ print(f"WhisperX pipeline failed, falling back to audio-only: {type(whisperx_err).__name__}: {whisperx_err}")
646
+ traceback.print_exc()
647
  df = build_events_without_whisperx(video_path)
648
  print(f"Events: {len(df)} rows (audio-only fallback)")
649
 
 
682
  elapsed = time.time() - start_time
683
 
684
  report = {
685
+ "pipeline": "full" if ("type" in df and (df["type"] == "Word").any()) else "audio_only",
686
  "filename": filename,
687
  "duration_seconds": valid_len + hrf_offset,
688
  "analyzed_seconds": valid_len,
requirements.txt CHANGED
@@ -6,3 +6,4 @@ gradio
6
  requests
7
  uv
8
  hf_transfer
 
 
6
  requests
7
  uv
8
  hf_transfer
9
+ accelerate