Update app.py
Browse files
app.py
CHANGED
|
@@ -31,6 +31,7 @@ from youtube_transcript_api import YouTubeTranscriptApi
|
|
| 31 |
from langchain_community.document_loaders import YoutubeLoader
|
| 32 |
|
| 33 |
import re
|
|
|
|
| 34 |
|
| 35 |
# start application
|
| 36 |
app = FastAPI()
|
|
@@ -592,10 +593,36 @@ async def update_fb_transaction(transaction_ref: firestore.DocumentReference, no
|
|
| 592 |
return {"status": f"Error: {str(e)}"}
|
| 593 |
|
| 594 |
async def get_youtube_transcript_url(video_id: str) -> str:
|
|
|
|
| 595 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
headers = {
|
| 597 |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
| 598 |
-
'Accept-Language': 'ko,en;q=0.9'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 599 |
}
|
| 600 |
|
| 601 |
url = f"https://www.youtube.com/watch?v={video_id}"
|
|
@@ -649,15 +676,10 @@ async def get_youtube_transcript_url(video_id: str) -> str:
|
|
| 649 |
return urls[lang]
|
| 650 |
else:
|
| 651 |
logging.warning("No transcript URLs found in the page content")
|
| 652 |
-
|
| 653 |
-
sample_size = 200
|
| 654 |
-
logging.debug(f"Page content sample: {page_content[:sample_size]}...")
|
| 655 |
-
return None
|
| 656 |
-
|
| 657 |
except Exception as e:
|
| 658 |
-
logging.error(f"Error
|
| 659 |
-
|
| 660 |
-
return None
|
| 661 |
|
| 662 |
async def process_youtube_video(video_id: str, video_title: str = "") -> dict:
|
| 663 |
try:
|
|
|
|
| 31 |
from langchain_community.document_loaders import YoutubeLoader
|
| 32 |
|
| 33 |
import re
|
| 34 |
+
import asyncio
|
| 35 |
|
| 36 |
# start application
|
| 37 |
app = FastAPI()
|
|
|
|
| 593 |
return {"status": f"Error: {str(e)}"}
|
| 594 |
|
| 595 |
async def get_youtube_transcript_url(video_id: str) -> str:
|
| 596 |
+
# First try with youtube_transcript_api
|
| 597 |
try:
|
| 598 |
+
transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
|
| 599 |
+
|
| 600 |
+
# Try to get Korean transcript first, then English, then any available
|
| 601 |
+
try:
|
| 602 |
+
transcript = transcript_list.find_transcript(['ko'])
|
| 603 |
+
except Exception:
|
| 604 |
+
try:
|
| 605 |
+
transcript = transcript_list.find_transcript(['en'])
|
| 606 |
+
except Exception:
|
| 607 |
+
transcript = transcript_list.find_manually_created_transcript()
|
| 608 |
+
|
| 609 |
+
# Get the transcript URL
|
| 610 |
+
transcript_data = transcript.fetch()
|
| 611 |
+
if transcript_data:
|
| 612 |
+
logging.info(f"Found transcript using youtube_transcript_api")
|
| 613 |
+
return "Found" # We found a transcript
|
| 614 |
+
|
| 615 |
+
except Exception as e:
|
| 616 |
+
logging.warning(f"youtube_transcript_api failed: {str(e)}")
|
| 617 |
+
# Fall back to original method
|
| 618 |
headers = {
|
| 619 |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
| 620 |
+
'Accept-Language': 'ko,en;q=0.9',
|
| 621 |
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
| 622 |
+
'Accept-Encoding': 'gzip, deflate, br',
|
| 623 |
+
'Connection': 'keep-alive',
|
| 624 |
+
'DNT': '1',
|
| 625 |
+
'Upgrade-Insecure-Requests': '1'
|
| 626 |
}
|
| 627 |
|
| 628 |
url = f"https://www.youtube.com/watch?v={video_id}"
|
|
|
|
| 676 |
return urls[lang]
|
| 677 |
else:
|
| 678 |
logging.warning("No transcript URLs found in the page content")
|
| 679 |
+
return "No transcript URLs found"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
except Exception as e:
|
| 681 |
+
logging.error(f"Error in get_youtube_transcript_url: {str(e)}")
|
| 682 |
+
return f"Error: {str(e)}"
|
|
|
|
| 683 |
|
| 684 |
async def process_youtube_video(video_id: str, video_title: str = "") -> dict:
|
| 685 |
try:
|