Fix youtube ingestion
Browse files- README.md +17 -0
- app/extractors/youtube.py +17 -6
- app/services/ingestion.py +15 -2
- app/ui/gradio_app.py +22 -10
README.md
CHANGED
|
@@ -92,6 +92,23 @@ HF_TOKEN=<token-if-needed-for-gated-model-downloads>
|
|
| 92 |
|
| 93 |
Use a hosted Qdrant instance for Hugging Face Spaces. `localhost:6333` only works for local development.
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
## Setup
|
| 96 |
|
| 97 |
```bash
|
|
|
|
| 92 |
|
| 93 |
Use a hosted Qdrant instance for Hugging Face Spaces. `localhost:6333` only works for local development.
|
| 94 |
|
| 95 |
+
## Hosted YouTube Transcript Note
|
| 96 |
+
|
| 97 |
+
YouTube transcript fetching can fail on hosted platforms such as Hugging Face Spaces because YouTube may block transcript requests from datacenter IP addresses.
|
| 98 |
+
|
| 99 |
+
If a YouTube URL fails on the hosted app:
|
| 100 |
+
|
| 101 |
+
1. Open the video on YouTube.
|
| 102 |
+
2. Copy the transcript manually.
|
| 103 |
+
3. Paste it into the **YouTube transcript fallback** box.
|
| 104 |
+
4. Keep the YouTube URL in the URL field and run ingestion again.
|
| 105 |
+
|
| 106 |
+
This bypasses hosted transcript fetching while still storing the content as a YouTube source in Qdrant.
|
| 107 |
+
|
| 108 |
+
## Qdrant Collection Name
|
| 109 |
+
|
| 110 |
+
The Ingest and Retrieve tabs each have their own collection-name field. Set both to the same Qdrant collection when you want to search what you just ingested. The fields are intentionally not auto-synced because auto-sync can cause continuous refreshes in hosted Gradio Spaces.
|
| 111 |
+
|
| 112 |
## Setup
|
| 113 |
|
| 114 |
```bash
|
app/extractors/youtube.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from urllib.parse import parse_qs, urlparse
|
| 2 |
|
| 3 |
from youtube_transcript_api import YouTubeTranscriptApi
|
|
|
|
| 4 |
|
| 5 |
from app.core.models import Document, SourceType
|
| 6 |
|
|
@@ -21,14 +22,24 @@ def _extract_video_id(url: str) -> str:
|
|
| 21 |
def extract_youtube(url: str) -> Document:
|
| 22 |
video_id = _extract_video_id(url)
|
| 23 |
api = YouTubeTranscriptApi()
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
if not transcript_items:
|
| 31 |
-
raise ValueError(
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
lines = []
|
| 34 |
for item in transcript_items:
|
|
|
|
| 1 |
from urllib.parse import parse_qs, urlparse
|
| 2 |
|
| 3 |
from youtube_transcript_api import YouTubeTranscriptApi
|
| 4 |
+
from youtube_transcript_api._errors import YouTubeTranscriptApiException
|
| 5 |
|
| 6 |
from app.core.models import Document, SourceType
|
| 7 |
|
|
|
|
| 22 |
def extract_youtube(url: str) -> Document:
|
| 23 |
video_id = _extract_video_id(url)
|
| 24 |
api = YouTubeTranscriptApi()
|
| 25 |
+
try:
|
| 26 |
+
if hasattr(api, "fetch"):
|
| 27 |
+
transcript = api.fetch(video_id)
|
| 28 |
+
transcript_items = transcript.to_raw_data()
|
| 29 |
+
else:
|
| 30 |
+
transcript_items = YouTubeTranscriptApi.get_transcript(video_id)
|
| 31 |
+
except YouTubeTranscriptApiException as exc:
|
| 32 |
+
raise ValueError(
|
| 33 |
+
"Could not fetch the YouTube transcript. On hosted environments such as "
|
| 34 |
+
"Hugging Face Spaces, YouTube often blocks transcript requests from datacenter IPs. "
|
| 35 |
+
"Paste the transcript into the YouTube transcript fallback box and retry."
|
| 36 |
+
) from exc
|
| 37 |
|
| 38 |
if not transcript_items:
|
| 39 |
+
raise ValueError(
|
| 40 |
+
"No transcript was available for this YouTube video. Paste the transcript into "
|
| 41 |
+
"the YouTube transcript fallback box and retry."
|
| 42 |
+
)
|
| 43 |
|
| 44 |
lines = []
|
| 45 |
for item in transcript_items:
|
app/services/ingestion.py
CHANGED
|
@@ -15,13 +15,25 @@ from app.utils.source_detection import detect_source
|
|
| 15 |
EXPORT_DIR = Path("data/exports")
|
| 16 |
|
| 17 |
|
| 18 |
-
def extract_document(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
source_type = detect_source(url, pdf_path)
|
| 20 |
if source_type == SourceType.PDF:
|
| 21 |
return extract_pdf(str(pdf_path))
|
| 22 |
if source_type == SourceType.ARXIV:
|
| 23 |
return extract_arxiv(str(url))
|
| 24 |
if source_type == SourceType.YOUTUBE:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
return extract_youtube(str(url))
|
| 26 |
raise ValueError(f"Unsupported source type: {source_type}")
|
| 27 |
|
|
@@ -57,8 +69,9 @@ def ingest_source(
|
|
| 57 |
chunk_size: int | None = None,
|
| 58 |
chunk_overlap: int | None = None,
|
| 59 |
collection_name: str | None = None,
|
|
|
|
| 60 |
) -> IngestionResult:
|
| 61 |
-
document = extract_document(url=url, pdf_path=pdf_path)
|
| 62 |
chunks = chunk_document(
|
| 63 |
document,
|
| 64 |
chunk_size=chunk_size or settings.CHUNK_SIZE,
|
|
|
|
| 15 |
EXPORT_DIR = Path("data/exports")
|
| 16 |
|
| 17 |
|
| 18 |
+
def extract_document(
|
| 19 |
+
url: str | None = None,
|
| 20 |
+
pdf_path: str | None = None,
|
| 21 |
+
manual_transcript: str | None = None,
|
| 22 |
+
) -> Document:
|
| 23 |
source_type = detect_source(url, pdf_path)
|
| 24 |
if source_type == SourceType.PDF:
|
| 25 |
return extract_pdf(str(pdf_path))
|
| 26 |
if source_type == SourceType.ARXIV:
|
| 27 |
return extract_arxiv(str(url))
|
| 28 |
if source_type == SourceType.YOUTUBE:
|
| 29 |
+
if manual_transcript and manual_transcript.strip():
|
| 30 |
+
return Document(
|
| 31 |
+
source_type=SourceType.YOUTUBE,
|
| 32 |
+
title="YouTube Transcript",
|
| 33 |
+
text=manual_transcript.strip(),
|
| 34 |
+
source=str(url),
|
| 35 |
+
metadata={"transcript_source": "manual"},
|
| 36 |
+
)
|
| 37 |
return extract_youtube(str(url))
|
| 38 |
raise ValueError(f"Unsupported source type: {source_type}")
|
| 39 |
|
|
|
|
| 69 |
chunk_size: int | None = None,
|
| 70 |
chunk_overlap: int | None = None,
|
| 71 |
collection_name: str | None = None,
|
| 72 |
+
manual_transcript: str | None = None,
|
| 73 |
) -> IngestionResult:
|
| 74 |
+
document = extract_document(url=url, pdf_path=pdf_path, manual_transcript=manual_transcript)
|
| 75 |
chunks = chunk_document(
|
| 76 |
document,
|
| 77 |
chunk_size=chunk_size or settings.CHUNK_SIZE,
|
app/ui/gradio_app.py
CHANGED
|
@@ -29,7 +29,14 @@ def _format_metadata(metadata: dict) -> str:
|
|
| 29 |
|
| 30 |
|
| 31 |
@gpu()
|
| 32 |
-
def _ingest(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
logger.info(
|
| 34 |
"Ingest requested url=%s pdf_file=%s chunk_size=%s chunk_overlap=%s collection=%s",
|
| 35 |
url,
|
|
@@ -47,6 +54,7 @@ def _ingest(url: str, pdf_file: str | None, chunk_size: int, chunk_overlap: int,
|
|
| 47 |
chunk_size=chunk_size,
|
| 48 |
chunk_overlap=chunk_overlap,
|
| 49 |
collection_name=collection_name,
|
|
|
|
| 50 |
)
|
| 51 |
document = result.document
|
| 52 |
status = (
|
|
@@ -198,6 +206,11 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
|
|
| 198 |
file_types=[".pdf"],
|
| 199 |
type="filepath",
|
| 200 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
with gr.Row():
|
| 202 |
chunk_size = gr.Slider(
|
| 203 |
400,
|
|
@@ -259,7 +272,14 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
|
|
| 259 |
|
| 260 |
ingest_btn.click(
|
| 261 |
fn=_ingest,
|
| 262 |
-
inputs=[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 263 |
outputs=[
|
| 264 |
status,
|
| 265 |
title,
|
|
@@ -315,14 +335,6 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
|
|
| 315 |
outputs=[answer_output, reasoning_output, search_results],
|
| 316 |
)
|
| 317 |
|
| 318 |
-
# Sync collection names across tabs
|
| 319 |
-
collection_name_ingest.change(
|
| 320 |
-
fn=lambda x: x, inputs=[collection_name_ingest], outputs=[collection_name_retrieve]
|
| 321 |
-
)
|
| 322 |
-
collection_name_retrieve.change(
|
| 323 |
-
fn=lambda x: x, inputs=[collection_name_retrieve], outputs=[collection_name_ingest]
|
| 324 |
-
)
|
| 325 |
-
|
| 326 |
return demo
|
| 327 |
|
| 328 |
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
@gpu()
|
| 32 |
+
def _ingest(
|
| 33 |
+
url: str,
|
| 34 |
+
pdf_file: str | None,
|
| 35 |
+
manual_transcript: str,
|
| 36 |
+
chunk_size: int,
|
| 37 |
+
chunk_overlap: int,
|
| 38 |
+
collection_name: str,
|
| 39 |
+
):
|
| 40 |
logger.info(
|
| 41 |
"Ingest requested url=%s pdf_file=%s chunk_size=%s chunk_overlap=%s collection=%s",
|
| 42 |
url,
|
|
|
|
| 54 |
chunk_size=chunk_size,
|
| 55 |
chunk_overlap=chunk_overlap,
|
| 56 |
collection_name=collection_name,
|
| 57 |
+
manual_transcript=manual_transcript,
|
| 58 |
)
|
| 59 |
document = result.document
|
| 60 |
status = (
|
|
|
|
| 206 |
file_types=[".pdf"],
|
| 207 |
type="filepath",
|
| 208 |
)
|
| 209 |
+
manual_transcript = gr.Textbox(
|
| 210 |
+
label="YouTube transcript fallback",
|
| 211 |
+
placeholder="If hosted YouTube transcript extraction is blocked, paste the transcript here and keep the YouTube URL above.",
|
| 212 |
+
lines=5,
|
| 213 |
+
)
|
| 214 |
with gr.Row():
|
| 215 |
chunk_size = gr.Slider(
|
| 216 |
400,
|
|
|
|
| 272 |
|
| 273 |
ingest_btn.click(
|
| 274 |
fn=_ingest,
|
| 275 |
+
inputs=[
|
| 276 |
+
source_url,
|
| 277 |
+
pdf_file,
|
| 278 |
+
manual_transcript,
|
| 279 |
+
chunk_size,
|
| 280 |
+
chunk_overlap,
|
| 281 |
+
collection_name_ingest,
|
| 282 |
+
],
|
| 283 |
outputs=[
|
| 284 |
status,
|
| 285 |
title,
|
|
|
|
| 335 |
outputs=[answer_output, reasoning_output, search_results],
|
| 336 |
)
|
| 337 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
return demo
|
| 339 |
|
| 340 |
|