Update app.py
Browse files
app.py
CHANGED
|
@@ -46,6 +46,8 @@ import base64
|
|
| 46 |
import soundfile as sf
|
| 47 |
import torch
|
| 48 |
import os
|
|
|
|
|
|
|
| 49 |
|
| 50 |
from supertonic import TTS
|
| 51 |
from transformers import AutoProcessor, AutoModelForImageTextToText
|
|
@@ -82,6 +84,85 @@ def extract_pdf_content(pdf_path, max_pages=2):
|
|
| 82 |
images.append(img)
|
| 83 |
return text, images
|
| 84 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
def extract_website_content(url, max_images=2):
|
| 86 |
"""Extract text and images from a website URL."""
|
| 87 |
import requests
|
|
@@ -452,7 +533,7 @@ def hash_file(filepath):
|
|
| 452 |
return hashlib.md5(f.read(1024*1024)).hexdigest()
|
| 453 |
|
| 454 |
@spaces.GPU(duration=120)
|
| 455 |
-
def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang, max_text_char, repetition_penalty_val, last_source_hash, last_korean_words, progress=gr.Progress()):
|
| 456 |
global tts, voice_style
|
| 457 |
|
| 458 |
# Clean language choices from "Family - Language" to just "Language"
|
|
@@ -464,16 +545,20 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
|
|
| 464 |
os.makedirs("log", exist_ok=True)
|
| 465 |
|
| 466 |
is_url = bool(url_input and url_input.strip())
|
| 467 |
-
|
| 468 |
-
|
|
|
|
| 469 |
return
|
| 470 |
|
| 471 |
-
if
|
|
|
|
|
|
|
| 472 |
current_source_hash = hashlib.md5(url_input.strip().encode()).hexdigest()
|
| 473 |
else:
|
| 474 |
current_source_hash = hash_file(pdf_file.name)
|
| 475 |
|
| 476 |
vocab_list = []
|
|
|
|
| 477 |
|
| 478 |
# if last_source_hash == current_source_hash and last_korean_words:
|
| 479 |
# # Just run text-to-text LLM
|
|
@@ -486,7 +571,23 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
|
|
| 486 |
# else:
|
| 487 |
|
| 488 |
try:
|
| 489 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
progress(0, desc="Fetching Website...")
|
| 491 |
content_text, images = extract_website_content(url_input.strip())
|
| 492 |
else:
|
|
@@ -494,10 +595,12 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
|
|
| 494 |
content_text, images = extract_pdf_content(pdf_file.name)
|
| 495 |
|
| 496 |
if not content_text.strip() and not images:
|
| 497 |
-
yield "<p>No content found.</p>", current_source_hash, None, "", "", []
|
| 498 |
return
|
| 499 |
except Exception as e:
|
| 500 |
-
|
|
|
|
|
|
|
| 501 |
return
|
| 502 |
|
| 503 |
vocab_list = []
|
|
@@ -508,13 +611,13 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
|
|
| 508 |
stream_text = stream_t
|
| 509 |
if v_list is not None:
|
| 510 |
vocab_list = v_list
|
| 511 |
-
yield "", current_source_hash, None, stream_text, content_text, images
|
| 512 |
|
| 513 |
if vocab_list:
|
| 514 |
break
|
| 515 |
|
| 516 |
if not vocab_list:
|
| 517 |
-
yield "<p>Failed to extract or translate vocabulary after 3 attempts.</p>", current_source_hash, None, stream_text, content_text, images
|
| 518 |
return
|
| 519 |
|
| 520 |
progress(0.6, desc="Generating TTS audio...")
|
|
@@ -840,7 +943,7 @@ def process_pdf(pdf_file, url_input, translit_lang, translit_format, target_lang
|
|
| 840 |
safe_srcdoc = html.escape(iframe_html)
|
| 841 |
|
| 842 |
# Return the iframe containing the whole SPA
|
| 843 |
-
yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images
|
| 844 |
|
| 845 |
LANGUAGE_DATA = """Indo-European English, French, Portuguese, German, Romanian, Swedish, Danish, Bulgarian, Russian, Czech, Greek, Ukrainian, Spanish, Dutch, Slovak, Croatian, Polish, Lithuanian, Norwegian BokmΓ₯l, Norwegian Nynorsk, Persian, Slovenian, Gujarati, Latvian, Italian, Occitan, Nepali, Marathi, Belarusian, Serbian, Luxembourgish, Venetian, Assamese, Welsh, Silesian, Asturian, Chhattisgarhi, Awadhi, Maithili, Bhojpuri, Sindhi, Irish, Faroese, Hindi, Punjabi, Bengali, Oriya, Tajik, Eastern Yiddish, Lombard, Ligurian, Sicilian, Friulian, Sardinian, Galician, Catalan, Icelandic, Tosk Albanian, Limburgish, Dari, Afrikaans, Macedonian, Sinhala, Urdu, Magahi, Bosnian, Armenian, Latgalian, Scottish Gaelic, Central Kurdish, Northern Kurdish, Southern Pashto, Sanskrit, Dhundari, Marwari, Ahirani, Bagheli, Bagri, Bundeli, Braj, Kumaoni, Kashmiri
|
| 846 |
Sino-Tibetan Chinese (Simplified), Chinese (Traditional), Cantonese, Burmese, Standard Tibetan, Meitei
|
|
@@ -894,10 +997,10 @@ def process_pdf_force(partial_text, pdf_file, url_input, translit_lang, translit
|
|
| 894 |
content_text, images = extract_pdf_content(pdf_file.name)
|
| 895 |
|
| 896 |
if not content_text.strip() and not images:
|
| 897 |
-
yield "<p>No content found.</p>", current_source_hash, None, partial_text, "", []
|
| 898 |
return
|
| 899 |
except Exception as e:
|
| 900 |
-
yield f"<p>Error reading content: {e}</p>", None, None, partial_text, "", []
|
| 901 |
return
|
| 902 |
|
| 903 |
vocab_list = []
|
|
@@ -908,10 +1011,10 @@ def process_pdf_force(partial_text, pdf_file, url_input, translit_lang, translit
|
|
| 908 |
stream_text = stream_t
|
| 909 |
if v_list is not None:
|
| 910 |
vocab_list = v_list
|
| 911 |
-
yield "", current_source_hash, None, stream_text, content_text, images
|
| 912 |
|
| 913 |
if not vocab_list:
|
| 914 |
-
yield "<p>Failed to parse forced JSON.</p>", current_source_hash, None, stream_text, content_text, images
|
| 915 |
return
|
| 916 |
|
| 917 |
progress(0.6, desc="Generating TTS audio...")
|
|
@@ -949,7 +1052,7 @@ def process_pdf_force(partial_text, pdf_file, url_input, translit_lang, translit
|
|
| 949 |
)
|
| 950 |
|
| 951 |
safe_srcdoc = html_output.replace('"', '"')
|
| 952 |
-
yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images
|
| 953 |
|
| 954 |
|
| 955 |
def create_demo():
|
|
@@ -1059,8 +1162,8 @@ def create_demo():
|
|
| 1059 |
"""
|
| 1060 |
|
| 1061 |
with gr.Blocks(title="LocalDuo", theme=custom_theme, css=css) as demo:
|
| 1062 |
-
gr.Markdown("# π°π·β¨ LocalDuo - Learn Korean from PDFs &
|
| 1063 |
-
gr.Markdown("Enter a website URL π or upload a Korean book PDF π. The app uses a **Vision-Language Model (VLM)** π§ to extract vocabulary from text and images, and a **Text-to-Speech (TTS)** engine π£οΈ to generate pronunciation audio.")
|
| 1064 |
|
| 1065 |
with gr.Row():
|
| 1066 |
with gr.Column(scale=1):
|
|
@@ -1071,6 +1174,11 @@ def create_demo():
|
|
| 1071 |
url_input = gr.Textbox(label="Enter a Website URL π",
|
| 1072 |
placeholder=r"e.g. # https://www.bbc.com/korean/articles/cn0p7rkvxdgo",
|
| 1073 |
value=r"https://www.bbc.com/korean/articles/cn0p7rkvxdgo")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1074 |
|
| 1075 |
pdf_input = gr.File(label="Or Upload Book PDF π", file_types=[".pdf"], value=example_pdf)
|
| 1076 |
|
|
@@ -1101,6 +1209,7 @@ def create_demo():
|
|
| 1101 |
with gr.Accordion("π Extracted Source Content", open=True):
|
| 1102 |
extracted_text_box = gr.Textbox(label="Extracted Text", lines=10, max_lines=15, interactive=False)
|
| 1103 |
extracted_images_gallery = gr.Gallery(label="Extracted Images", columns=4, height="auto", object_fit="contain")
|
|
|
|
| 1104 |
|
| 1105 |
last_source_state = gr.State(None)
|
| 1106 |
last_korean_words_state = gr.State(None)
|
|
@@ -1112,8 +1221,8 @@ def create_demo():
|
|
| 1112 |
|
| 1113 |
generate_event = submit_btn.click(
|
| 1114 |
fn=process_pdf,
|
| 1115 |
-
inputs=[pdf_input, url_input, translit_lang, translit_format, target_lang, max_text_char_input, repetition_penalty_input, last_source_state, last_korean_words_state],
|
| 1116 |
-
outputs=[output_html, last_source_state, last_korean_words_state, stream_box, extracted_text_box, extracted_images_gallery]
|
| 1117 |
)
|
| 1118 |
|
| 1119 |
stop_thinking_btn.click(fn=set_stop_thinking, inputs=None, outputs=stop_thinking_btn, queue=False)
|
|
@@ -1136,7 +1245,7 @@ def create_demo():
|
|
| 1136 |
|
| 1137 |
|
| 1138 |
if __name__ == "__main__":
|
| 1139 |
-
global model, processor, tts, voice_style
|
| 1140 |
|
| 1141 |
|
| 1142 |
# model_id = "Qwen/Qwen3.5-9B"
|
|
@@ -1315,6 +1424,14 @@ if __name__ == "__main__":
|
|
| 1315 |
trust_remote_code=True
|
| 1316 |
)
|
| 1317 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1318 |
print("Loading Supertonic TTS...")
|
| 1319 |
tts = TTS(model="supertonic-3")
|
| 1320 |
try:
|
|
|
|
| 46 |
import soundfile as sf
|
| 47 |
import torch
|
| 48 |
import os
|
| 49 |
+
import tempfile
|
| 50 |
+
import re as re_module
|
| 51 |
|
| 52 |
from supertonic import TTS
|
| 53 |
from transformers import AutoProcessor, AutoModelForImageTextToText
|
|
|
|
| 84 |
images.append(img)
|
| 85 |
return text, images
|
| 86 |
|
| 87 |
+
def is_youtube_url(url):
|
| 88 |
+
"""Check if a URL is a YouTube link."""
|
| 89 |
+
if not url:
|
| 90 |
+
return False
|
| 91 |
+
youtube_patterns = [
|
| 92 |
+
r'(https?://)?(www\.)?youtube\.com/watch',
|
| 93 |
+
r'(https?://)?(www\.)?youtube\.com/shorts/',
|
| 94 |
+
r'(https?://)?youtu\.be/',
|
| 95 |
+
r'(https?://)?(www\.)?youtube\.com/embed/',
|
| 96 |
+
r'(https?://)?m\.youtube\.com/',
|
| 97 |
+
]
|
| 98 |
+
for pattern in youtube_patterns:
|
| 99 |
+
if re_module.search(pattern, url):
|
| 100 |
+
return True
|
| 101 |
+
return False
|
| 102 |
+
|
| 103 |
+
def extract_youtube_audio(url, max_duration_sec=300):
|
| 104 |
+
"""Extract audio from YouTube video (first max_duration_sec seconds).
|
| 105 |
+
Returns path to the downloaded audio file.
|
| 106 |
+
"""
|
| 107 |
+
import yt_dlp
|
| 108 |
+
from yt_dlp.utils import download_range_func
|
| 109 |
+
|
| 110 |
+
os.makedirs("log", exist_ok=True)
|
| 111 |
+
output_path = os.path.join("log", "yt_audio")
|
| 112 |
+
|
| 113 |
+
ydl_opts = {
|
| 114 |
+
'format': 'bestaudio/best',
|
| 115 |
+
'postprocessors': [{
|
| 116 |
+
'key': 'FFmpegExtractAudio',
|
| 117 |
+
'preferredcodec': 'wav',
|
| 118 |
+
'preferredquality': '192',
|
| 119 |
+
}],
|
| 120 |
+
'download_ranges': download_range_func(None, [(0, max_duration_sec)]),
|
| 121 |
+
'force_keyframes_at_cuts': True,
|
| 122 |
+
'outtmpl': output_path + '.%(ext)s',
|
| 123 |
+
'quiet': True,
|
| 124 |
+
'no_warnings': True,
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
| 128 |
+
info = ydl.extract_info(url, download=True)
|
| 129 |
+
title = info.get('title', 'YouTube Video')
|
| 130 |
+
|
| 131 |
+
# The output file will be output_path.wav after postprocessing
|
| 132 |
+
wav_path = output_path + '.wav'
|
| 133 |
+
if not os.path.exists(wav_path):
|
| 134 |
+
# Sometimes the extension might differ, search for it
|
| 135 |
+
for ext in ['wav', 'mp3', 'opus', 'webm', 'm4a']:
|
| 136 |
+
candidate = output_path + '.' + ext
|
| 137 |
+
if os.path.exists(candidate):
|
| 138 |
+
wav_path = candidate
|
| 139 |
+
break
|
| 140 |
+
|
| 141 |
+
if not os.path.exists(wav_path):
|
| 142 |
+
raise FileNotFoundError(f"Could not find downloaded audio file at {output_path}.*")
|
| 143 |
+
|
| 144 |
+
return wav_path, title
|
| 145 |
+
|
| 146 |
+
@spaces.GPU(duration=120)
|
| 147 |
+
def transcribe_audio_with_asr(audio_path):
|
| 148 |
+
"""Transcribe audio file using Cohere ASR model via transformers."""
|
| 149 |
+
global asr_model, asr_processor
|
| 150 |
+
from transformers.audio_utils import load_audio
|
| 151 |
+
|
| 152 |
+
audio = load_audio(audio_path, sampling_rate=16000)
|
| 153 |
+
|
| 154 |
+
inputs = asr_processor(audio, language="ko", sampling_rate=16000, return_tensors="pt")
|
| 155 |
+
inputs = inputs.to(asr_model.device, dtype=asr_model.dtype)
|
| 156 |
+
|
| 157 |
+
outputs = asr_model.generate(**inputs, max_new_tokens=2048)
|
| 158 |
+
texts = asr_processor.decode(outputs, skip_special_tokens=True)
|
| 159 |
+
# text = texts[0] if isinstance(texts, list) else texts
|
| 160 |
+
# join texts
|
| 161 |
+
text = "\n".join(texts) if isinstance(texts, list) else texts
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
return text.strip()
|
| 165 |
+
|
| 166 |
def extract_website_content(url, max_images=2):
|
| 167 |
"""Extract text and images from a website URL."""
|
| 168 |
import requests
|
|
|
|
| 533 |
return hashlib.md5(f.read(1024*1024)).hexdigest()
|
| 534 |
|
| 535 |
@spaces.GPU(duration=120)
|
| 536 |
+
def process_pdf(pdf_file, url_input, yt_url_input, translit_lang, translit_format, target_lang, max_text_char, repetition_penalty_val, last_source_hash, last_korean_words, progress=gr.Progress()):
|
| 537 |
global tts, voice_style
|
| 538 |
|
| 539 |
# Clean language choices from "Family - Language" to just "Language"
|
|
|
|
| 545 |
os.makedirs("log", exist_ok=True)
|
| 546 |
|
| 547 |
is_url = bool(url_input and url_input.strip())
|
| 548 |
+
is_youtube = bool(yt_url_input and yt_url_input.strip() and is_youtube_url(yt_url_input.strip()))
|
| 549 |
+
if pdf_file is None and not is_url and not is_youtube:
|
| 550 |
+
yield "<p>Please upload a PDF, enter a URL, or provide a YouTube link.</p>", None, None, "", "", [], None
|
| 551 |
return
|
| 552 |
|
| 553 |
+
if is_youtube:
|
| 554 |
+
current_source_hash = hashlib.md5(yt_url_input.strip().encode()).hexdigest()
|
| 555 |
+
elif is_url:
|
| 556 |
current_source_hash = hashlib.md5(url_input.strip().encode()).hexdigest()
|
| 557 |
else:
|
| 558 |
current_source_hash = hash_file(pdf_file.name)
|
| 559 |
|
| 560 |
vocab_list = []
|
| 561 |
+
extracted_audio_path = None
|
| 562 |
|
| 563 |
# if last_source_hash == current_source_hash and last_korean_words:
|
| 564 |
# # Just run text-to-text LLM
|
|
|
|
| 571 |
# else:
|
| 572 |
|
| 573 |
try:
|
| 574 |
+
if is_youtube:
|
| 575 |
+
progress(0, desc="Downloading YouTube audio (first 5 min)...")
|
| 576 |
+
audio_path, yt_title = extract_youtube_audio(yt_url_input.strip(), max_duration_sec=300)
|
| 577 |
+
|
| 578 |
+
progress(0.1, desc=f"Transcribing audio with Cohere ASR...")
|
| 579 |
+
content_text = transcribe_audio_with_asr(audio_path)
|
| 580 |
+
images = []
|
| 581 |
+
extracted_audio_path = audio_path
|
| 582 |
+
|
| 583 |
+
if not content_text.strip():
|
| 584 |
+
yield "<p>Could not transcribe any text from the YouTube video.</p>", current_source_hash, None, "", "", [], extracted_audio_path
|
| 585 |
+
return
|
| 586 |
+
|
| 587 |
+
# Log the transcription
|
| 588 |
+
with open("log/debug_yt_transcription.txt", "w", encoding="utf-8") as f:
|
| 589 |
+
f.write(f"Title: {yt_title}\n\n{content_text}")
|
| 590 |
+
elif is_url:
|
| 591 |
progress(0, desc="Fetching Website...")
|
| 592 |
content_text, images = extract_website_content(url_input.strip())
|
| 593 |
else:
|
|
|
|
| 595 |
content_text, images = extract_pdf_content(pdf_file.name)
|
| 596 |
|
| 597 |
if not content_text.strip() and not images:
|
| 598 |
+
yield "<p>No content found.</p>", current_source_hash, None, "", "", [], extracted_audio_path
|
| 599 |
return
|
| 600 |
except Exception as e:
|
| 601 |
+
import traceback
|
| 602 |
+
traceback.print_exc()
|
| 603 |
+
yield f"<p>Error reading content: {e}</p>", None, None, "", "", [], None
|
| 604 |
return
|
| 605 |
|
| 606 |
vocab_list = []
|
|
|
|
| 611 |
stream_text = stream_t
|
| 612 |
if v_list is not None:
|
| 613 |
vocab_list = v_list
|
| 614 |
+
yield "", current_source_hash, None, stream_text, content_text, images, extracted_audio_path
|
| 615 |
|
| 616 |
if vocab_list:
|
| 617 |
break
|
| 618 |
|
| 619 |
if not vocab_list:
|
| 620 |
+
yield "<p>Failed to extract or translate vocabulary after 3 attempts.</p>", current_source_hash, None, stream_text, content_text, images, extracted_audio_path
|
| 621 |
return
|
| 622 |
|
| 623 |
progress(0.6, desc="Generating TTS audio...")
|
|
|
|
| 943 |
safe_srcdoc = html.escape(iframe_html)
|
| 944 |
|
| 945 |
# Return the iframe containing the whole SPA
|
| 946 |
+
yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images, extracted_audio_path
|
| 947 |
|
| 948 |
LANGUAGE_DATA = """Indo-European English, French, Portuguese, German, Romanian, Swedish, Danish, Bulgarian, Russian, Czech, Greek, Ukrainian, Spanish, Dutch, Slovak, Croatian, Polish, Lithuanian, Norwegian BokmΓ₯l, Norwegian Nynorsk, Persian, Slovenian, Gujarati, Latvian, Italian, Occitan, Nepali, Marathi, Belarusian, Serbian, Luxembourgish, Venetian, Assamese, Welsh, Silesian, Asturian, Chhattisgarhi, Awadhi, Maithili, Bhojpuri, Sindhi, Irish, Faroese, Hindi, Punjabi, Bengali, Oriya, Tajik, Eastern Yiddish, Lombard, Ligurian, Sicilian, Friulian, Sardinian, Galician, Catalan, Icelandic, Tosk Albanian, Limburgish, Dari, Afrikaans, Macedonian, Sinhala, Urdu, Magahi, Bosnian, Armenian, Latgalian, Scottish Gaelic, Central Kurdish, Northern Kurdish, Southern Pashto, Sanskrit, Dhundari, Marwari, Ahirani, Bagheli, Bagri, Bundeli, Braj, Kumaoni, Kashmiri
|
| 949 |
Sino-Tibetan Chinese (Simplified), Chinese (Traditional), Cantonese, Burmese, Standard Tibetan, Meitei
|
|
|
|
| 997 |
content_text, images = extract_pdf_content(pdf_file.name)
|
| 998 |
|
| 999 |
if not content_text.strip() and not images:
|
| 1000 |
+
yield "<p>No content found.</p>", current_source_hash, None, partial_text, "", [], None
|
| 1001 |
return
|
| 1002 |
except Exception as e:
|
| 1003 |
+
yield f"<p>Error reading content: {e}</p>", None, None, partial_text, "", [], None
|
| 1004 |
return
|
| 1005 |
|
| 1006 |
vocab_list = []
|
|
|
|
| 1011 |
stream_text = stream_t
|
| 1012 |
if v_list is not None:
|
| 1013 |
vocab_list = v_list
|
| 1014 |
+
yield "", current_source_hash, None, stream_text, content_text, images, None
|
| 1015 |
|
| 1016 |
if not vocab_list:
|
| 1017 |
+
yield "<p>Failed to parse forced JSON.</p>", current_source_hash, None, stream_text, content_text, images, None
|
| 1018 |
return
|
| 1019 |
|
| 1020 |
progress(0.6, desc="Generating TTS audio...")
|
|
|
|
| 1052 |
)
|
| 1053 |
|
| 1054 |
safe_srcdoc = html_output.replace('"', '"')
|
| 1055 |
+
yield f'<iframe srcdoc="{safe_srcdoc}" style="width: 100%; height: 650px; border: none; overflow-y: auto;"></iframe>', current_source_hash, vocab_list, stream_text, content_text, images, None
|
| 1056 |
|
| 1057 |
|
| 1058 |
def create_demo():
|
|
|
|
| 1162 |
"""
|
| 1163 |
|
| 1164 |
with gr.Blocks(title="LocalDuo", theme=custom_theme, css=css) as demo:
|
| 1165 |
+
gr.Markdown("# π°π·β¨ LocalDuo - Learn Korean from PDFs, Websites & YouTube")
|
| 1166 |
+
gr.Markdown("Enter a website URL π, a YouTube link π¬, or upload a Korean book PDF π. The app uses a **Vision-Language Model (VLM)** π§ to extract vocabulary from text and images, **ASR** π€ to transcribe YouTube audio, and a **Text-to-Speech (TTS)** engine π£οΈ to generate pronunciation audio.")
|
| 1167 |
|
| 1168 |
with gr.Row():
|
| 1169 |
with gr.Column(scale=1):
|
|
|
|
| 1174 |
url_input = gr.Textbox(label="Enter a Website URL π",
|
| 1175 |
placeholder=r"e.g. # https://www.bbc.com/korean/articles/cn0p7rkvxdgo",
|
| 1176 |
value=r"https://www.bbc.com/korean/articles/cn0p7rkvxdgo")
|
| 1177 |
+
|
| 1178 |
+
yt_url_input = gr.Textbox(label="Or Enter a YouTube Link π¬",
|
| 1179 |
+
placeholder=r"e.g. https://www.youtube.com/watch?v=...",
|
| 1180 |
+
value="https://www.youtube.com/watch?v=9Nj7l73PBWE",
|
| 1181 |
+
info="Audio from the first 5 minutes will be transcribed using Cohere ASR")
|
| 1182 |
|
| 1183 |
pdf_input = gr.File(label="Or Upload Book PDF π", file_types=[".pdf"], value=example_pdf)
|
| 1184 |
|
|
|
|
| 1209 |
with gr.Accordion("π Extracted Source Content", open=True):
|
| 1210 |
extracted_text_box = gr.Textbox(label="Extracted Text", lines=10, max_lines=15, interactive=False)
|
| 1211 |
extracted_images_gallery = gr.Gallery(label="Extracted Images", columns=4, height="auto", object_fit="contain")
|
| 1212 |
+
extracted_audio_player = gr.Audio(label="Extracted Audio (YouTube)", type="filepath", interactive=False)
|
| 1213 |
|
| 1214 |
last_source_state = gr.State(None)
|
| 1215 |
last_korean_words_state = gr.State(None)
|
|
|
|
| 1221 |
|
| 1222 |
generate_event = submit_btn.click(
|
| 1223 |
fn=process_pdf,
|
| 1224 |
+
inputs=[pdf_input, url_input, yt_url_input, translit_lang, translit_format, target_lang, max_text_char_input, repetition_penalty_input, last_source_state, last_korean_words_state],
|
| 1225 |
+
outputs=[output_html, last_source_state, last_korean_words_state, stream_box, extracted_text_box, extracted_images_gallery, extracted_audio_player]
|
| 1226 |
)
|
| 1227 |
|
| 1228 |
stop_thinking_btn.click(fn=set_stop_thinking, inputs=None, outputs=stop_thinking_btn, queue=False)
|
|
|
|
| 1245 |
|
| 1246 |
|
| 1247 |
if __name__ == "__main__":
|
| 1248 |
+
global model, processor, tts, voice_style, asr_model, asr_processor
|
| 1249 |
|
| 1250 |
|
| 1251 |
# model_id = "Qwen/Qwen3.5-9B"
|
|
|
|
| 1424 |
trust_remote_code=True
|
| 1425 |
)
|
| 1426 |
|
| 1427 |
+
print("Loading Cohere ASR model...")
|
| 1428 |
+
from transformers import CohereAsrForConditionalGeneration
|
| 1429 |
+
asr_processor = AutoProcessor.from_pretrained("CohereLabs/cohere-transcribe-03-2026")
|
| 1430 |
+
asr_model = CohereAsrForConditionalGeneration.from_pretrained(
|
| 1431 |
+
"CohereLabs/cohere-transcribe-03-2026",
|
| 1432 |
+
device_map="cpu",
|
| 1433 |
+
)
|
| 1434 |
+
|
| 1435 |
print("Loading Supertonic TTS...")
|
| 1436 |
tts = TTS(model="supertonic-3")
|
| 1437 |
try:
|