Malina Claude Opus 4.8 commited on
Commit
d0ff022
Β·
1 Parent(s): 51e7ae9

Fix diarization hub kwarg, yt-dlp cookies, stronger output formatting

Browse files

- diarize: shim huggingface_hub.hf_hub_download/snapshot_download to translate the
removed use_auth_token kwarg β†’ token, so pyannote 3.3.2 loads on hub 1.x
(fixes "Diarization unavailable: ... unexpected keyword argument 'use_auth_token'").
- fetch: support YouTube/Facebook cookies (YT_DLP_COOKIES[_B64] secret) + a real
browser UA, the actual fix for "Sign in to confirm you're not a bot / age" on
cloud IPs; still best-effort.
- synthesis/chat: enforce a strong "CRITICAL OUTPUT FORMAT" (headings, bullets,
numbered steps, tables for comparisons; no walls of text) for scannable answers.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (4) hide show
  1. chat.py +5 -4
  2. diarize.py +21 -0
  3. fetch.py +40 -0
  4. synthesis.py +7 -7
chat.py CHANGED
@@ -132,10 +132,11 @@ def _answer_from_context(messages: list, transcript: str):
132
  f"Ground everything in the conversation and the stored transcript below; do not invent "
133
  f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
134
  f"If the answer needs visual detail you don't have, say so and suggest re-analyzing. "
135
- f"FORMAT: match the answer's shape to the question (Markdown) β€” a short question gets a "
136
- f"sentence or two; a long or multi-part answer uses short paragraphs, '- ' bullet lists "
137
- f"for sets of items/steps, and **bold** for key terms. Never reply with one big block of "
138
- f"text, and never over-format a small answer."
 
139
  )
140
  user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
141
  if last.get("quote"):
 
132
  f"Ground everything in the conversation and the stored transcript below; do not invent "
133
  f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
134
  f"If the answer needs visual detail you don't have, say so and suggest re-analyzing. "
135
+ f"CRITICAL OUTPUT FORMAT β€” your response must be visually structured; assume the user is "
136
+ f"skimming, not reading word-for-word, so optimize for scannability. Use (whichever fit) "
137
+ f"Markdown headings, bullet lists, numbered steps, and tables when comparing items. "
138
+ f"Forbidden: large blocks of uninterrupted text, or any answer longer than ~5 lines "
139
+ f"without formatting. A trivial one-line answer may stay plain β€” don't pad it."
140
  )
141
  user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
142
  if last.get("quote"):
diarize.py CHANGED
@@ -59,6 +59,26 @@ def _shim_torchaudio(torchaudio) -> None:
59
  _shimmed = True
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def _get_pipeline():
63
  global _pipeline, _failed
64
  if _failed:
@@ -68,6 +88,7 @@ def _get_pipeline():
68
  import torch
69
  import torchaudio
70
  _shim_torchaudio(torchaudio) # paper over torchaudio 2.11's removed I/O API
 
71
  from pyannote.audio import Pipeline
72
  # huggingface-hub renamed use_auth_token β†’ token; support both.
73
  try:
 
59
  _shimmed = True
60
 
61
 
62
+ def _shim_hf_hub() -> None:
63
+ """pyannote 3.3.2 internally calls hf_hub_download(use_auth_token=...) β€” a kwarg
64
+ that huggingface-hub 1.x removed (it's now `token`). Translate it so the gated
65
+ model still loads. Must run BEFORE `from pyannote.audio import Pipeline` so
66
+ pyannote binds the patched function."""
67
+ import huggingface_hub as _hf
68
+ for _name in ("hf_hub_download", "snapshot_download"):
69
+ _orig = getattr(_hf, _name, None)
70
+ if _orig is None or getattr(_orig, "_nina_shim", False):
71
+ continue
72
+ def _make(orig):
73
+ def _wrapped(*a, **k):
74
+ if "use_auth_token" in k:
75
+ k.setdefault("token", k.pop("use_auth_token"))
76
+ return orig(*a, **k)
77
+ _wrapped._nina_shim = True
78
+ return _wrapped
79
+ setattr(_hf, _name, _make(_orig))
80
+
81
+
82
  def _get_pipeline():
83
  global _pipeline, _failed
84
  if _failed:
 
88
  import torch
89
  import torchaudio
90
  _shim_torchaudio(torchaudio) # paper over torchaudio 2.11's removed I/O API
91
+ _shim_hf_hub() # paper over huggingface-hub's use_auth_token removal
92
  from pyannote.audio import Pipeline
93
  # huggingface-hub renamed use_auth_token β†’ token; support both.
94
  try:
fetch.py CHANGED
@@ -7,6 +7,7 @@
7
  # website always offers file upload as the fallback. We pull a low-res stream (NINA
8
  # only needs audio + sampled frames) and cap the duration.
9
 
 
10
  import os
11
  import tempfile
12
 
@@ -15,6 +16,41 @@ from auth import verify_usage_token
15
  MAX_FETCH_SECONDS = 45 * 60 # reject videos longer than 45 min
16
  # Prefer a small ≀480p stream (+ best audio); NINA downscales/ samples anyway.
17
  YT_DLP_FORMAT = "bv*[height<=480]+ba/b[height<=480]/best"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
 
20
  def fetch_video(url: str, token: str = ""):
@@ -43,7 +79,11 @@ def fetch_video(url: str, token: str = ""):
43
  "socket_timeout": 30,
44
  "retries": 2,
45
  "concurrent_fragment_downloads": 4,
 
46
  }
 
 
 
47
 
48
  try:
49
  with yt_dlp.YoutubeDL(opts) as ydl:
 
7
  # website always offers file upload as the fallback. We pull a low-res stream (NINA
8
  # only needs audio + sampled frames) and cap the duration.
9
 
10
+ import base64
11
  import os
12
  import tempfile
13
 
 
16
  MAX_FETCH_SECONDS = 45 * 60 # reject videos longer than 45 min
17
  # Prefer a small ≀480p stream (+ best audio); NINA downscales/ samples anyway.
18
  YT_DLP_FORMAT = "bv*[height<=480]+ba/b[height<=480]/best"
19
+ # A real browser UA helps a little against bot checks; cookies are the actual fix.
20
+ _UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
21
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
22
+
23
+ _cookie_path = None
24
+
25
+
26
+ def _cookiefile():
27
+ """YouTube/Facebook block datacenter IPs ('confirm you're not a bot' / age gate).
28
+ Set a Space secret with an exported Netscape cookies.txt to authenticate yt-dlp:
29
+ β€’ YT_DLP_COOKIES_B64 β€” base64 of the cookies.txt (preferred; survives newlines), or
30
+ β€’ YT_DLP_COOKIES β€” the raw cookies.txt contents.
31
+ Returns a path to a temp cookie file, or None when no cookies are configured."""
32
+ global _cookie_path
33
+ if _cookie_path and os.path.exists(_cookie_path):
34
+ return _cookie_path
35
+ raw = None
36
+ b64 = os.getenv("YT_DLP_COOKIES_B64")
37
+ if b64:
38
+ try:
39
+ raw = base64.b64decode(b64).decode("utf-8", "replace")
40
+ except Exception:
41
+ raw = None
42
+ if raw is None:
43
+ raw = os.getenv("YT_DLP_COOKIES")
44
+ if not raw or not raw.strip():
45
+ return None
46
+ try:
47
+ p = os.path.join(tempfile.gettempdir(), "nina_yt_cookies.txt")
48
+ with open(p, "w", encoding="utf-8") as f:
49
+ f.write(raw)
50
+ _cookie_path = p
51
+ return p
52
+ except Exception:
53
+ return None
54
 
55
 
56
  def fetch_video(url: str, token: str = ""):
 
79
  "socket_timeout": 30,
80
  "retries": 2,
81
  "concurrent_fragment_downloads": 4,
82
+ "http_headers": {"User-Agent": _UA},
83
  }
84
+ cookies = _cookiefile()
85
+ if cookies:
86
+ opts["cookiefile"] = cookies
87
 
88
  try:
89
  with yt_dlp.YoutubeDL(opts) as ydl:
synthesis.py CHANGED
@@ -124,13 +124,13 @@ def _detect_language(text: str) -> str:
124
  # which the chat renders) instead of always emitting one prose block. Shared by the
125
  # text and vision synthesis prompts.
126
  _FORMAT_GUIDE = (
127
- "FORMAT β€” Match the shape of the answer to the question, using Markdown:\n"
128
- "- A simple or short question β†’ one or two short sentences. No headings or lists.\n"
129
- "- A longer or multi-part answer β†’ do NOT write one big block of text. Break it into "
130
- "short paragraphs (1–3 sentences each), use a '- ' bulleted list for any set of items, "
131
- "steps or examples, and **bold** the key terms. Add a short '## heading' only when there "
132
- "are genuinely distinct sections.\n"
133
- "- Never over-format a small answer or add empty headings; keep it natural and scannable.\n"
134
  "Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]."
135
  )
136
 
 
124
  # which the chat renders) instead of always emitting one prose block. Shared by the
125
  # text and vision synthesis prompts.
126
  _FORMAT_GUIDE = (
127
+ "CRITICAL OUTPUT FORMAT β€” your response must be visually structured. Assume the user "
128
+ "is skimming, not reading word-for-word; optimize for scannability.\n"
129
+ "Required (use whichever fit the answer): Markdown headings, bullet lists, numbered "
130
+ "steps, and tables (Markdown `| col | col |`) when comparing items.\n"
131
+ "Forbidden: large blocks of uninterrupted text; any answer longer than ~5 lines without "
132
+ "formatting.\n"
133
+ "(A trivial one-line factual answer may stay plain β€” don't pad it into a table.)\n"
134
  "Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]."
135
  )
136