pkheria commited on
Commit
33c0d0b
·
1 Parent(s): c9b7fb8

Fix youtube ingestion

Browse files
README.md CHANGED
@@ -92,6 +92,23 @@ HF_TOKEN=<token-if-needed-for-gated-model-downloads>
92
 
93
  Use a hosted Qdrant instance for Hugging Face Spaces. `localhost:6333` only works for local development.
94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  ## Setup
96
 
97
  ```bash
 
92
 
93
  Use a hosted Qdrant instance for Hugging Face Spaces. `localhost:6333` only works for local development.
94
 
95
+ ## Hosted YouTube Transcript Note
96
+
97
+ YouTube transcript fetching can fail on hosted platforms such as Hugging Face Spaces because YouTube may block transcript requests from datacenter IP addresses.
98
+
99
+ If a YouTube URL fails on the hosted app:
100
+
101
+ 1. Open the video on YouTube.
102
+ 2. Copy the transcript manually.
103
+ 3. Paste it into the **YouTube transcript fallback** box.
104
+ 4. Keep the YouTube URL in the URL field and run ingestion again.
105
+
106
+ This bypasses hosted transcript fetching while still storing the content as a YouTube source in Qdrant.
107
+
108
+ ## Qdrant Collection Name
109
+
110
+ The Ingest and Retrieve tabs each have their own collection-name field. Set both to the same Qdrant collection when you want to search what you just ingested. The fields are intentionally not auto-synced because auto-sync can cause continuous refreshes in hosted Gradio Spaces.
111
+
112
  ## Setup
113
 
114
  ```bash
app/extractors/youtube.py CHANGED
@@ -1,6 +1,7 @@
1
  from urllib.parse import parse_qs, urlparse
2
 
3
  from youtube_transcript_api import YouTubeTranscriptApi
 
4
 
5
  from app.core.models import Document, SourceType
6
 
@@ -21,14 +22,24 @@ def _extract_video_id(url: str) -> str:
21
  def extract_youtube(url: str) -> Document:
22
  video_id = _extract_video_id(url)
23
  api = YouTubeTranscriptApi()
24
- if hasattr(api, "fetch"):
25
- transcript = api.fetch(video_id)
26
- transcript_items = transcript.to_raw_data()
27
- else:
28
- transcript_items = YouTubeTranscriptApi.get_transcript(video_id)
 
 
 
 
 
 
 
29
 
30
  if not transcript_items:
31
- raise ValueError("No transcript was available for this YouTube video.")
 
 
 
32
 
33
  lines = []
34
  for item in transcript_items:
 
1
  from urllib.parse import parse_qs, urlparse
2
 
3
  from youtube_transcript_api import YouTubeTranscriptApi
4
+ from youtube_transcript_api._errors import YouTubeTranscriptApiException
5
 
6
  from app.core.models import Document, SourceType
7
 
 
22
  def extract_youtube(url: str) -> Document:
23
  video_id = _extract_video_id(url)
24
  api = YouTubeTranscriptApi()
25
+ try:
26
+ if hasattr(api, "fetch"):
27
+ transcript = api.fetch(video_id)
28
+ transcript_items = transcript.to_raw_data()
29
+ else:
30
+ transcript_items = YouTubeTranscriptApi.get_transcript(video_id)
31
+ except YouTubeTranscriptApiException as exc:
32
+ raise ValueError(
33
+ "Could not fetch the YouTube transcript. On hosted environments such as "
34
+ "Hugging Face Spaces, YouTube often blocks transcript requests from datacenter IPs. "
35
+ "Paste the transcript into the YouTube transcript fallback box and retry."
36
+ ) from exc
37
 
38
  if not transcript_items:
39
+ raise ValueError(
40
+ "No transcript was available for this YouTube video. Paste the transcript into "
41
+ "the YouTube transcript fallback box and retry."
42
+ )
43
 
44
  lines = []
45
  for item in transcript_items:
app/services/ingestion.py CHANGED
@@ -15,13 +15,25 @@ from app.utils.source_detection import detect_source
15
  EXPORT_DIR = Path("data/exports")
16
 
17
 
18
- def extract_document(url: str | None = None, pdf_path: str | None = None) -> Document:
 
 
 
 
19
  source_type = detect_source(url, pdf_path)
20
  if source_type == SourceType.PDF:
21
  return extract_pdf(str(pdf_path))
22
  if source_type == SourceType.ARXIV:
23
  return extract_arxiv(str(url))
24
  if source_type == SourceType.YOUTUBE:
 
 
 
 
 
 
 
 
25
  return extract_youtube(str(url))
26
  raise ValueError(f"Unsupported source type: {source_type}")
27
 
@@ -57,8 +69,9 @@ def ingest_source(
57
  chunk_size: int | None = None,
58
  chunk_overlap: int | None = None,
59
  collection_name: str | None = None,
 
60
  ) -> IngestionResult:
61
- document = extract_document(url=url, pdf_path=pdf_path)
62
  chunks = chunk_document(
63
  document,
64
  chunk_size=chunk_size or settings.CHUNK_SIZE,
 
15
  EXPORT_DIR = Path("data/exports")
16
 
17
 
18
+ def extract_document(
19
+ url: str | None = None,
20
+ pdf_path: str | None = None,
21
+ manual_transcript: str | None = None,
22
+ ) -> Document:
23
  source_type = detect_source(url, pdf_path)
24
  if source_type == SourceType.PDF:
25
  return extract_pdf(str(pdf_path))
26
  if source_type == SourceType.ARXIV:
27
  return extract_arxiv(str(url))
28
  if source_type == SourceType.YOUTUBE:
29
+ if manual_transcript and manual_transcript.strip():
30
+ return Document(
31
+ source_type=SourceType.YOUTUBE,
32
+ title="YouTube Transcript",
33
+ text=manual_transcript.strip(),
34
+ source=str(url),
35
+ metadata={"transcript_source": "manual"},
36
+ )
37
  return extract_youtube(str(url))
38
  raise ValueError(f"Unsupported source type: {source_type}")
39
 
 
69
  chunk_size: int | None = None,
70
  chunk_overlap: int | None = None,
71
  collection_name: str | None = None,
72
+ manual_transcript: str | None = None,
73
  ) -> IngestionResult:
74
+ document = extract_document(url=url, pdf_path=pdf_path, manual_transcript=manual_transcript)
75
  chunks = chunk_document(
76
  document,
77
  chunk_size=chunk_size or settings.CHUNK_SIZE,
app/ui/gradio_app.py CHANGED
@@ -29,7 +29,14 @@ def _format_metadata(metadata: dict) -> str:
29
 
30
 
31
  @gpu()
32
- def _ingest(url: str, pdf_file: str | None, chunk_size: int, chunk_overlap: int, collection_name: str):
 
 
 
 
 
 
 
33
  logger.info(
34
  "Ingest requested url=%s pdf_file=%s chunk_size=%s chunk_overlap=%s collection=%s",
35
  url,
@@ -47,6 +54,7 @@ def _ingest(url: str, pdf_file: str | None, chunk_size: int, chunk_overlap: int,
47
  chunk_size=chunk_size,
48
  chunk_overlap=chunk_overlap,
49
  collection_name=collection_name,
 
50
  )
51
  document = result.document
52
  status = (
@@ -198,6 +206,11 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
198
  file_types=[".pdf"],
199
  type="filepath",
200
  )
 
 
 
 
 
201
  with gr.Row():
202
  chunk_size = gr.Slider(
203
  400,
@@ -259,7 +272,14 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
259
 
260
  ingest_btn.click(
261
  fn=_ingest,
262
- inputs=[source_url, pdf_file, chunk_size, chunk_overlap, collection_name_ingest],
 
 
 
 
 
 
 
263
  outputs=[
264
  status,
265
  title,
@@ -315,14 +335,6 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
315
  outputs=[answer_output, reasoning_output, search_results],
316
  )
317
 
318
- # Sync collection names across tabs
319
- collection_name_ingest.change(
320
- fn=lambda x: x, inputs=[collection_name_ingest], outputs=[collection_name_retrieve]
321
- )
322
- collection_name_retrieve.change(
323
- fn=lambda x: x, inputs=[collection_name_retrieve], outputs=[collection_name_ingest]
324
- )
325
-
326
  return demo
327
 
328
 
 
29
 
30
 
31
  @gpu()
32
+ def _ingest(
33
+ url: str,
34
+ pdf_file: str | None,
35
+ manual_transcript: str,
36
+ chunk_size: int,
37
+ chunk_overlap: int,
38
+ collection_name: str,
39
+ ):
40
  logger.info(
41
  "Ingest requested url=%s pdf_file=%s chunk_size=%s chunk_overlap=%s collection=%s",
42
  url,
 
54
  chunk_size=chunk_size,
55
  chunk_overlap=chunk_overlap,
56
  collection_name=collection_name,
57
+ manual_transcript=manual_transcript,
58
  )
59
  document = result.document
60
  status = (
 
206
  file_types=[".pdf"],
207
  type="filepath",
208
  )
209
+ manual_transcript = gr.Textbox(
210
+ label="YouTube transcript fallback",
211
+ placeholder="If hosted YouTube transcript extraction is blocked, paste the transcript here and keep the YouTube URL above.",
212
+ lines=5,
213
+ )
214
  with gr.Row():
215
  chunk_size = gr.Slider(
216
  400,
 
272
 
273
  ingest_btn.click(
274
  fn=_ingest,
275
+ inputs=[
276
+ source_url,
277
+ pdf_file,
278
+ manual_transcript,
279
+ chunk_size,
280
+ chunk_overlap,
281
+ collection_name_ingest,
282
+ ],
283
  outputs=[
284
  status,
285
  title,
 
335
  outputs=[answer_output, reasoning_output, search_results],
336
  )
337
 
 
 
 
 
 
 
 
 
338
  return demo
339
 
340