shayekh commited on
Commit
cff65d0
Β·
verified Β·
1 Parent(s): f3a1cdf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +137 -20
app.py CHANGED
@@ -46,6 +46,8 @@ import base64
46
  import soundfile as sf
47
  import torch
48
  import os
 
 
49
 
50
  from supertonic import TTS
51
  from transformers import AutoProcessor, AutoModelForImageTextToText
@@ -82,6 +84,85 @@ def extract_pdf_content(pdf_path, max_pages=2):
82
  images.append(img)
83
  return text, images
84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  def extract_website_content(url, max_images=2):
86
  """Extract text and images from a website URL."""
87
  import requests
@@ -452,7 +533,7 @@ def hash_file(filepath):
452
  return hashlib.md5(f.read(1024*1024)).hexdigest()
453
 
454
  @spaces.GPU(duration=120)
455
- def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang, max_text_char, repetition_penalty_val, last_source_hash, last_korean_words, progress=gr.Progress()):
456
  global tts, voice_style
457
 
458
  # Clean language choices from "Family - Language" to just "Language"
@@ -464,16 +545,20 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
464
  os.makedirs("log", exist_ok=True)
465
 
466
  is_url = bool(url_input and url_input.strip())
467
- if pdf_file is None and not is_url:
468
- yield "<p>Please upload a PDF or enter a URL.</p>", None, None, "", "", []
 
469
  return
470
 
471
- if is_url:
 
 
472
  current_source_hash = hashlib.md5(url_input.strip().encode()).hexdigest()
473
  else:
474
  current_source_hash = hash_file(pdf_file.name)
475
 
476
  vocab_list = []
 
477
 
478
  # if last_source_hash == current_source_hash and last_korean_words:
479
  # # Just run text-to-text LLM
@@ -486,7 +571,23 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
486
  # else:
487
 
488
  try:
489
- if is_url:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
490
  progress(0, desc="Fetching Website...")
491
  content_text, images = extract_website_content(url_input.strip())
492
  else:
@@ -494,10 +595,12 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
494
  content_text, images = extract_pdf_content(pdf_file.name)
495
 
496
  if not content_text.strip() and not images:
497
- yield "<p>No content found.</p>", current_source_hash, None, "", "", []
498
  return
499
  except Exception as e:
500
- yield f"<p>Error reading content: {e}</p>", None, None, "", "", []
 
 
501
  return
502
 
503
  vocab_list = []
@@ -508,13 +611,13 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
508
  stream_text = stream_t
509
  if v_list is not None:
510
  vocab_list = v_list
511
- yield "", current_source_hash, None, stream_text, content_text, images
512
 
513
  if vocab_list:
514
  break
515
 
516
  if not vocab_list:
517
- yield "<p>Failed to extract or translate vocabulary after 3 attempts.</p>", current_source_hash, None, stream_text, content_text, images
518
  return
519
 
520
  progress(0.6, desc="Generating TTS audio...")
@@ -840,7 +943,7 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
840
  safe_srcdoc = html.escape(iframe_html)
841
 
842
  # Return the iframe containing the whole SPA
843
- yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images
844
 
845
  LANGUAGE_DATA = """Indo-European English, French, Portuguese, German, Romanian, Swedish, Danish, Bulgarian, Russian, Czech, Greek, Ukrainian, Spanish, Dutch, Slovak, Croatian, Polish, Lithuanian, Norwegian BokmΓ₯l, Norwegian Nynorsk, Persian, Slovenian, Gujarati, Latvian, Italian, Occitan, Nepali, Marathi, Belarusian, Serbian, Luxembourgish, Venetian, Assamese, Welsh, Silesian, Asturian, Chhattisgarhi, Awadhi, Maithili, Bhojpuri, Sindhi, Irish, Faroese, Hindi, Punjabi, Bengali, Oriya, Tajik, Eastern Yiddish, Lombard, Ligurian, Sicilian, Friulian, Sardinian, Galician, Catalan, Icelandic, Tosk Albanian, Limburgish, Dari, Afrikaans, Macedonian, Sinhala, Urdu, Magahi, Bosnian, Armenian, Latgalian, Scottish Gaelic, Central Kurdish, Northern Kurdish, Southern Pashto, Sanskrit, Dhundari, Marwari, Ahirani, Bagheli, Bagri, Bundeli, Braj, Kumaoni, Kashmiri
846
  Sino-Tibetan Chinese (Simplified), Chinese (Traditional), Cantonese, Burmese, Standard Tibetan, Meitei
@@ -894,10 +997,10 @@ def process_pdf_force(partial_text, pdf_file, url_input, translit_lang, translit
894
  content_text, images = extract_pdf_content(pdf_file.name)
895
 
896
  if not content_text.strip() and not images:
897
- yield "<p>No content found.</p>", current_source_hash, None, partial_text, "", []
898
  return
899
  except Exception as e:
900
- yield f"<p>Error reading content: {e}</p>", None, None, partial_text, "", []
901
  return
902
 
903
  vocab_list = []
@@ -908,10 +1011,10 @@ def process_pdf_force(partial_text, pdf_file, url_input, translit_lang, translit
908
  stream_text = stream_t
909
  if v_list is not None:
910
  vocab_list = v_list
911
- yield "", current_source_hash, None, stream_text, content_text, images
912
 
913
  if not vocab_list:
914
- yield "<p>Failed to parse forced JSON.</p>", current_source_hash, None, stream_text, content_text, images
915
  return
916
 
917
  progress(0.6, desc="Generating TTS audio...")
@@ -949,7 +1052,7 @@ def process_pdf_force(partial_text, pdf_file, url_input, translit_lang, translit
949
  )
950
 
951
  safe_srcdoc = html_output.replace('"', '&quot;')
952
- yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images
953
 
954
 
955
  def create_demo():
@@ -1059,8 +1162,8 @@ def create_demo():
1059
  """
1060
 
1061
  with gr.Blocks(title="LocalDuo", theme=custom_theme, css=css) as demo:
1062
- gr.Markdown("# πŸ‡°πŸ‡·βœ¨ LocalDuo - Learn Korean from PDFs & Websites")
1063
- gr.Markdown("Enter a website URL 🌐 or upload a Korean book PDF πŸ“„. The app uses a **Vision-Language Model (VLM)** 🧠 to extract vocabulary from text and images, and a **Text-to-Speech (TTS)** engine πŸ—£οΈ to generate pronunciation audio.")
1064
 
1065
  with gr.Row():
1066
  with gr.Column(scale=1):
@@ -1071,6 +1174,11 @@ def create_demo():
1071
  url_input = gr.Textbox(label="Enter a Website URL 🌐",
1072
  placeholder=r"e.g. # https://www.bbc.com/korean/articles/cn0p7rkvxdgo",
1073
  value=r"https://www.bbc.com/korean/articles/cn0p7rkvxdgo")
 
 
 
 
 
1074
 
1075
  pdf_input = gr.File(label="Or Upload Book PDF πŸ“š", file_types=[".pdf"], value=example_pdf)
1076
 
@@ -1101,6 +1209,7 @@ def create_demo():
1101
  with gr.Accordion("πŸ“„ Extracted Source Content", open=True):
1102
  extracted_text_box = gr.Textbox(label="Extracted Text", lines=10, max_lines=15, interactive=False)
1103
  extracted_images_gallery = gr.Gallery(label="Extracted Images", columns=4, height="auto", object_fit="contain")
 
1104
 
1105
  last_source_state = gr.State(None)
1106
  last_korean_words_state = gr.State(None)
@@ -1112,8 +1221,8 @@ def create_demo():
1112
 
1113
  generate_event = submit_btn.click(
1114
  fn=process_pdf,
1115
- inputs=[pdf_input, url_input, translit_lang, translit_format, target_lang, max_text_char_input, repetition_penalty_input, last_source_state, last_korean_words_state],
1116
- outputs=[output_html, last_source_state, last_korean_words_state, stream_box, extracted_text_box, extracted_images_gallery]
1117
  )
1118
 
1119
  stop_thinking_btn.click(fn=set_stop_thinking, inputs=None, outputs=stop_thinking_btn, queue=False)
@@ -1136,7 +1245,7 @@ def create_demo():
1136
 
1137
 
1138
  if __name__ == "__main__":
1139
- global model, processor, tts, voice_style
1140
 
1141
 
1142
  # model_id = "Qwen/Qwen3.5-9B"
@@ -1315,6 +1424,14 @@ if __name__ == "__main__":
1315
  trust_remote_code=True
1316
  )
1317
 
 
 
 
 
 
 
 
 
1318
  print("Loading Supertonic TTS...")
1319
  tts = TTS(model="supertonic-3")
1320
  try:
 
46
  import soundfile as sf
47
  import torch
48
  import os
49
+ import tempfile
50
+ import re as re_module
51
 
52
  from supertonic import TTS
53
  from transformers import AutoProcessor, AutoModelForImageTextToText
 
84
  images.append(img)
85
  return text, images
86
 
87
+ def is_youtube_url(url):
88
+ """Check if a URL is a YouTube link."""
89
+ if not url:
90
+ return False
91
+ youtube_patterns = [
92
+ r'(https?://)?(www\.)?youtube\.com/watch',
93
+ r'(https?://)?(www\.)?youtube\.com/shorts/',
94
+ r'(https?://)?youtu\.be/',
95
+ r'(https?://)?(www\.)?youtube\.com/embed/',
96
+ r'(https?://)?m\.youtube\.com/',
97
+ ]
98
+ for pattern in youtube_patterns:
99
+ if re_module.search(pattern, url):
100
+ return True
101
+ return False
102
+
103
+ def extract_youtube_audio(url, max_duration_sec=300):
104
+ """Extract audio from YouTube video (first max_duration_sec seconds).
105
+ Returns path to the downloaded audio file.
106
+ """
107
+ import yt_dlp
108
+ from yt_dlp.utils import download_range_func
109
+
110
+ os.makedirs("log", exist_ok=True)
111
+ output_path = os.path.join("log", "yt_audio")
112
+
113
+ ydl_opts = {
114
+ 'format': 'bestaudio/best',
115
+ 'postprocessors': [{
116
+ 'key': 'FFmpegExtractAudio',
117
+ 'preferredcodec': 'wav',
118
+ 'preferredquality': '192',
119
+ }],
120
+ 'download_ranges': download_range_func(None, [(0, max_duration_sec)]),
121
+ 'force_keyframes_at_cuts': True,
122
+ 'outtmpl': output_path + '.%(ext)s',
123
+ 'quiet': True,
124
+ 'no_warnings': True,
125
+ }
126
+
127
+ with yt_dlp.YoutubeDL(ydl_opts) as ydl:
128
+ info = ydl.extract_info(url, download=True)
129
+ title = info.get('title', 'YouTube Video')
130
+
131
+ # The output file will be output_path.wav after postprocessing
132
+ wav_path = output_path + '.wav'
133
+ if not os.path.exists(wav_path):
134
+ # Sometimes the extension might differ, search for it
135
+ for ext in ['wav', 'mp3', 'opus', 'webm', 'm4a']:
136
+ candidate = output_path + '.' + ext
137
+ if os.path.exists(candidate):
138
+ wav_path = candidate
139
+ break
140
+
141
+ if not os.path.exists(wav_path):
142
+ raise FileNotFoundError(f"Could not find downloaded audio file at {output_path}.*")
143
+
144
+ return wav_path, title
145
+
146
+ @spaces.GPU(duration=120)
147
+ def transcribe_audio_with_asr(audio_path):
148
+ """Transcribe audio file using Cohere ASR model via transformers."""
149
+ global asr_model, asr_processor
150
+ from transformers.audio_utils import load_audio
151
+
152
+ audio = load_audio(audio_path, sampling_rate=16000)
153
+
154
+ inputs = asr_processor(audio, language="ko", sampling_rate=16000, return_tensors="pt")
155
+ inputs = inputs.to(asr_model.device, dtype=asr_model.dtype)
156
+
157
+ outputs = asr_model.generate(**inputs, max_new_tokens=2048)
158
+ texts = asr_processor.decode(outputs, skip_special_tokens=True)
159
+ # text = texts[0] if isinstance(texts, list) else texts
160
+ # join texts
161
+ text = "\n".join(texts) if isinstance(texts, list) else texts
162
+
163
+
164
+ return text.strip()
165
+
166
  def extract_website_content(url, max_images=2):
167
  """Extract text and images from a website URL."""
168
  import requests
 
533
  return hashlib.md5(f.read(1024*1024)).hexdigest()
534
 
535
  @spaces.GPU(duration=120)
536
+ def process_pdf(pdf_file, url_input, yt_url_input, translit_lang, translit_format, target_lang, max_text_char, repetition_penalty_val, last_source_hash, last_korean_words, progress=gr.Progress()):
537
  global tts, voice_style
538
 
539
  # Clean language choices from "Family - Language" to just "Language"
 
545
  os.makedirs("log", exist_ok=True)
546
 
547
  is_url = bool(url_input and url_input.strip())
548
+ is_youtube = bool(yt_url_input and yt_url_input.strip() and is_youtube_url(yt_url_input.strip()))
549
+ if pdf_file is None and not is_url and not is_youtube:
550
+ yield "<p>Please upload a PDF, enter a URL, or provide a YouTube link.</p>", None, None, "", "", [], None
551
  return
552
 
553
+ if is_youtube:
554
+ current_source_hash = hashlib.md5(yt_url_input.strip().encode()).hexdigest()
555
+ elif is_url:
556
  current_source_hash = hashlib.md5(url_input.strip().encode()).hexdigest()
557
  else:
558
  current_source_hash = hash_file(pdf_file.name)
559
 
560
  vocab_list = []
561
+ extracted_audio_path = None
562
 
563
  # if last_source_hash == current_source_hash and last_korean_words:
564
  # # Just run text-to-text LLM
 
571
  # else:
572
 
573
  try:
574
+ if is_youtube:
575
+ progress(0, desc="Downloading YouTube audio (first 5 min)...")
576
+ audio_path, yt_title = extract_youtube_audio(yt_url_input.strip(), max_duration_sec=300)
577
+
578
+ progress(0.1, desc=f"Transcribing audio with Cohere ASR...")
579
+ content_text = transcribe_audio_with_asr(audio_path)
580
+ images = []
581
+ extracted_audio_path = audio_path
582
+
583
+ if not content_text.strip():
584
+ yield "<p>Could not transcribe any text from the YouTube video.</p>", current_source_hash, None, "", "", [], extracted_audio_path
585
+ return
586
+
587
+ # Log the transcription
588
+ with open("log/debug_yt_transcription.txt", "w", encoding="utf-8") as f:
589
+ f.write(f"Title: {yt_title}\n\n{content_text}")
590
+ elif is_url:
591
  progress(0, desc="Fetching Website...")
592
  content_text, images = extract_website_content(url_input.strip())
593
  else:
 
595
  content_text, images = extract_pdf_content(pdf_file.name)
596
 
597
  if not content_text.strip() and not images:
598
+ yield "<p>No content found.</p>", current_source_hash, None, "", "", [], extracted_audio_path
599
  return
600
  except Exception as e:
601
+ import traceback
602
+ traceback.print_exc()
603
+ yield f"<p>Error reading content: {e}</p>", None, None, "", "", [], None
604
  return
605
 
606
  vocab_list = []
 
611
  stream_text = stream_t
612
  if v_list is not None:
613
  vocab_list = v_list
614
+ yield "", current_source_hash, None, stream_text, content_text, images, extracted_audio_path
615
 
616
  if vocab_list:
617
  break
618
 
619
  if not vocab_list:
620
+ yield "<p>Failed to extract or translate vocabulary after 3 attempts.</p>", current_source_hash, None, stream_text, content_text, images, extracted_audio_path
621
  return
622
 
623
  progress(0.6, desc="Generating TTS audio...")
 
943
  safe_srcdoc = html.escape(iframe_html)
944
 
945
  # Return the iframe containing the whole SPA
946
+ yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images, extracted_audio_path
947
 
948
  LANGUAGE_DATA = """Indo-European English, French, Portuguese, German, Romanian, Swedish, Danish, Bulgarian, Russian, Czech, Greek, Ukrainian, Spanish, Dutch, Slovak, Croatian, Polish, Lithuanian, Norwegian BokmΓ₯l, Norwegian Nynorsk, Persian, Slovenian, Gujarati, Latvian, Italian, Occitan, Nepali, Marathi, Belarusian, Serbian, Luxembourgish, Venetian, Assamese, Welsh, Silesian, Asturian, Chhattisgarhi, Awadhi, Maithili, Bhojpuri, Sindhi, Irish, Faroese, Hindi, Punjabi, Bengali, Oriya, Tajik, Eastern Yiddish, Lombard, Ligurian, Sicilian, Friulian, Sardinian, Galician, Catalan, Icelandic, Tosk Albanian, Limburgish, Dari, Afrikaans, Macedonian, Sinhala, Urdu, Magahi, Bosnian, Armenian, Latgalian, Scottish Gaelic, Central Kurdish, Northern Kurdish, Southern Pashto, Sanskrit, Dhundari, Marwari, Ahirani, Bagheli, Bagri, Bundeli, Braj, Kumaoni, Kashmiri
949
  Sino-Tibetan Chinese (Simplified), Chinese (Traditional), Cantonese, Burmese, Standard Tibetan, Meitei
 
997
  content_text, images = extract_pdf_content(pdf_file.name)
998
 
999
  if not content_text.strip() and not images:
1000
+ yield "<p>No content found.</p>", current_source_hash, None, partial_text, "", [], None
1001
  return
1002
  except Exception as e:
1003
+ yield f"<p>Error reading content: {e}</p>", None, None, partial_text, "", [], None
1004
  return
1005
 
1006
  vocab_list = []
 
1011
  stream_text = stream_t
1012
  if v_list is not None:
1013
  vocab_list = v_list
1014
+ yield "", current_source_hash, None, stream_text, content_text, images, None
1015
 
1016
  if not vocab_list:
1017
+ yield "<p>Failed to parse forced JSON.</p>", current_source_hash, None, stream_text, content_text, images, None
1018
  return
1019
 
1020
  progress(0.6, desc="Generating TTS audio...")
 
1052
  )
1053
 
1054
  safe_srcdoc = html_output.replace('"', '&quot;')
1055
+ yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images, None
1056
 
1057
 
1058
  def create_demo():
 
1162
  """
1163
 
1164
  with gr.Blocks(title="LocalDuo", theme=custom_theme, css=css) as demo:
1165
+ gr.Markdown("# πŸ‡°πŸ‡·βœ¨ LocalDuo - Learn Korean from PDFs, Websites & YouTube")
1166
+ gr.Markdown("Enter a website URL 🌐, a YouTube link 🎬, or upload a Korean book PDF πŸ“„. The app uses a **Vision-Language Model (VLM)** 🧠 to extract vocabulary from text and images, **ASR** 🎀 to transcribe YouTube audio, and a **Text-to-Speech (TTS)** engine πŸ—£οΈ to generate pronunciation audio.")
1167
 
1168
  with gr.Row():
1169
  with gr.Column(scale=1):
 
1174
  url_input = gr.Textbox(label="Enter a Website URL 🌐",
1175
  placeholder=r"e.g. # https://www.bbc.com/korean/articles/cn0p7rkvxdgo",
1176
  value=r"https://www.bbc.com/korean/articles/cn0p7rkvxdgo")
1177
+
1178
+ yt_url_input = gr.Textbox(label="Or Enter a YouTube Link 🎬",
1179
+ placeholder=r"e.g. https://www.youtube.com/watch?v=...",
1180
+ value="https://www.youtube.com/watch?v=9Nj7l73PBWE",
1181
+ info="Audio from the first 5 minutes will be transcribed using Cohere ASR")
1182
 
1183
  pdf_input = gr.File(label="Or Upload Book PDF πŸ“š", file_types=[".pdf"], value=example_pdf)
1184
 
 
1209
  with gr.Accordion("πŸ“„ Extracted Source Content", open=True):
1210
  extracted_text_box = gr.Textbox(label="Extracted Text", lines=10, max_lines=15, interactive=False)
1211
  extracted_images_gallery = gr.Gallery(label="Extracted Images", columns=4, height="auto", object_fit="contain")
1212
+ extracted_audio_player = gr.Audio(label="Extracted Audio (YouTube)", type="filepath", interactive=False)
1213
 
1214
  last_source_state = gr.State(None)
1215
  last_korean_words_state = gr.State(None)
 
1221
 
1222
  generate_event = submit_btn.click(
1223
  fn=process_pdf,
1224
+ inputs=[pdf_input, url_input, yt_url_input, translit_lang, translit_format, target_lang, max_text_char_input, repetition_penalty_input, last_source_state, last_korean_words_state],
1225
+ outputs=[output_html, last_source_state, last_korean_words_state, stream_box, extracted_text_box, extracted_images_gallery, extracted_audio_player]
1226
  )
1227
 
1228
  stop_thinking_btn.click(fn=set_stop_thinking, inputs=None, outputs=stop_thinking_btn, queue=False)
 
1245
 
1246
 
1247
  if __name__ == "__main__":
1248
+ global model, processor, tts, voice_style, asr_model, asr_processor
1249
 
1250
 
1251
  # model_id = "Qwen/Qwen3.5-9B"
 
1424
  trust_remote_code=True
1425
  )
1426
 
1427
+ print("Loading Cohere ASR model...")
1428
+ from transformers import CohereAsrForConditionalGeneration
1429
+ asr_processor = AutoProcessor.from_pretrained("CohereLabs/cohere-transcribe-03-2026")
1430
+ asr_model = CohereAsrForConditionalGeneration.from_pretrained(
1431
+ "CohereLabs/cohere-transcribe-03-2026",
1432
+ device_map="cpu",
1433
+ )
1434
+
1435
  print("Loading Supertonic TTS...")
1436
  tts = TTS(model="supertonic-3")
1437
  try: