Spaces:
Paused
Paused
Commit Β·
c6ae51e
1
Parent(s): 0e40ecf
perf: preload all models at startup, improve Q&A context and auto-submit
Browse files- Preload Whisper-small, Qwen2.5-3B-Instruct, and TTS at startup
- Use .change() event for auto-submit (fires after audio value committed)
- Pass full story context (or top-5 relevant paragraphs) to Q&A
- Fix total_mem -> total_memory attribute error
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
- app.py +23 -6
- inference.py +21 -7
app.py
CHANGED
|
@@ -387,11 +387,20 @@ else:
|
|
| 387 |
_default_voice_name = None
|
| 388 |
print("[MomsVoice] No saved voice profile β using stock voice as default.")
|
| 389 |
|
| 390 |
-
# Preload
|
| 391 |
print("[MomsVoice] Preloading Qwen3-TTS 0.6B CustomVoice model...")
|
| 392 |
from voice_clone import get_custom_voice_model
|
| 393 |
get_custom_voice_model()
|
| 394 |
-
print("[MomsVoice]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 395 |
|
| 396 |
|
| 397 |
# Gradio Application Core setup
|
|
@@ -1022,10 +1031,18 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1022 |
outputs=[answer_display, answer_audio]
|
| 1023 |
)
|
| 1024 |
|
| 1025 |
-
# Auto-submit when voice recording
|
| 1026 |
-
|
| 1027 |
-
|
| 1028 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1029 |
outputs=[answer_display, answer_audio]
|
| 1030 |
)
|
| 1031 |
|
|
|
|
| 387 |
_default_voice_name = None
|
| 388 |
print("[MomsVoice] No saved voice profile β using stock voice as default.")
|
| 389 |
|
| 390 |
+
# Preload models at startup for fast response
|
| 391 |
print("[MomsVoice] Preloading Qwen3-TTS 0.6B CustomVoice model...")
|
| 392 |
from voice_clone import get_custom_voice_model
|
| 393 |
get_custom_voice_model()
|
| 394 |
+
print("[MomsVoice] TTS model ready.")
|
| 395 |
+
|
| 396 |
+
print("[MomsVoice] Preloading Whisper-small ASR model...")
|
| 397 |
+
from inference import get_asr_pipeline, get_qa_model
|
| 398 |
+
get_asr_pipeline()
|
| 399 |
+
print("[MomsVoice] ASR model ready.")
|
| 400 |
+
|
| 401 |
+
print("[MomsVoice] Preloading Qwen2.5-3B-Instruct Q&A model...")
|
| 402 |
+
get_qa_model()
|
| 403 |
+
print("[MomsVoice] Q&A model ready. All models preloaded.")
|
| 404 |
|
| 405 |
|
| 406 |
# Gradio Application Core setup
|
|
|
|
| 1031 |
outputs=[answer_display, answer_audio]
|
| 1032 |
)
|
| 1033 |
|
| 1034 |
+
# Auto-submit when voice recording completes
|
| 1035 |
+
# Use .change() instead of .stop_recording() β .change() fires after audio value is committed
|
| 1036 |
+
def auto_submit_on_recording(audio_path, question_txt, paragraphs, slider_val, profile_id):
|
| 1037 |
+
"""Auto-submit only when a new recording is available (not on clear)."""
|
| 1038 |
+
if not audio_path:
|
| 1039 |
+
# Audio was cleared, don't submit
|
| 1040 |
+
return gr.update(), gr.update()
|
| 1041 |
+
return handle_question_submit(question_txt, audio_path, paragraphs, slider_val, profile_id)
|
| 1042 |
+
|
| 1043 |
+
question_audio.change(
|
| 1044 |
+
auto_submit_on_recording,
|
| 1045 |
+
inputs=[question_audio, question_text, paragraphs_state, timeline_slider, voice_profile_state],
|
| 1046 |
outputs=[answer_display, answer_audio]
|
| 1047 |
)
|
| 1048 |
|
inference.py
CHANGED
|
@@ -70,7 +70,7 @@ def get_qa_model():
|
|
| 70 |
# Check available VRAM β if less than 3GB free, use CPU
|
| 71 |
use_gpu = False
|
| 72 |
if torch.cuda.is_available():
|
| 73 |
-
free_vram = torch.cuda.get_device_properties(0).
|
| 74 |
free_vram_gb = free_vram / (1024**3)
|
| 75 |
logger.info("Free VRAM: %.1f GB", free_vram_gb)
|
| 76 |
if free_vram_gb >= 3.0:
|
|
@@ -98,11 +98,19 @@ def get_qa_model():
|
|
| 98 |
|
| 99 |
|
| 100 |
def _get_relevant_context(paragraphs: list[str], current_idx: int, question: str) -> str:
|
| 101 |
-
"""
|
| 102 |
if not paragraphs:
|
| 103 |
return ""
|
| 104 |
|
| 105 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
question_words = set(question.lower().split())
|
| 107 |
|
| 108 |
scored = []
|
|
@@ -110,13 +118,19 @@ def _get_relevant_context(paragraphs: list[str], current_idx: int, question: str
|
|
| 110 |
para_words = set(para.lower().split())
|
| 111 |
overlap = len(question_words & para_words)
|
| 112 |
# Boost paragraphs near current position
|
| 113 |
-
proximity_bonus = max(0,
|
| 114 |
scored.append((overlap + proximity_bonus, i, para))
|
| 115 |
|
| 116 |
scored.sort(key=lambda x: x[0], reverse=True)
|
| 117 |
-
# Take top
|
| 118 |
-
top_paras = [
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
|
| 122 |
def answer_story_question(
|
|
|
|
| 70 |
# Check available VRAM β if less than 3GB free, use CPU
|
| 71 |
use_gpu = False
|
| 72 |
if torch.cuda.is_available():
|
| 73 |
+
free_vram = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated(0)
|
| 74 |
free_vram_gb = free_vram / (1024**3)
|
| 75 |
logger.info("Free VRAM: %.1f GB", free_vram_gb)
|
| 76 |
if free_vram_gb >= 3.0:
|
|
|
|
| 98 |
|
| 99 |
|
| 100 |
def _get_relevant_context(paragraphs: list[str], current_idx: int, question: str) -> str:
|
| 101 |
+
"""Return full story with emphasis on current section for context."""
|
| 102 |
if not paragraphs:
|
| 103 |
return ""
|
| 104 |
|
| 105 |
+
# Build context: full story (truncated if too long) with current paragraph highlighted
|
| 106 |
+
total_text = "\n\n".join(paragraphs)
|
| 107 |
+
|
| 108 |
+
# If story is short enough (< 2000 chars), use it all
|
| 109 |
+
if len(total_text) <= 2000:
|
| 110 |
+
current_marker = f"\n\n[Currently reading]: {paragraphs[current_idx]}" if current_idx < len(paragraphs) else ""
|
| 111 |
+
return total_text + current_marker
|
| 112 |
+
|
| 113 |
+
# For longer stories: use top relevant paragraphs + surrounding context
|
| 114 |
question_words = set(question.lower().split())
|
| 115 |
|
| 116 |
scored = []
|
|
|
|
| 118 |
para_words = set(para.lower().split())
|
| 119 |
overlap = len(question_words & para_words)
|
| 120 |
# Boost paragraphs near current position
|
| 121 |
+
proximity_bonus = max(0, 5 - abs(i - current_idx))
|
| 122 |
scored.append((overlap + proximity_bonus, i, para))
|
| 123 |
|
| 124 |
scored.sort(key=lambda x: x[0], reverse=True)
|
| 125 |
+
# Take top 5 most relevant paragraphs
|
| 126 |
+
top_paras = sorted(scored[:5], key=lambda x: x[1]) # sort by position
|
| 127 |
+
context = "\n\n".join(s[2] for s in top_paras)
|
| 128 |
+
|
| 129 |
+
# Add current paragraph marker
|
| 130 |
+
if current_idx < len(paragraphs):
|
| 131 |
+
context += f"\n\n[Currently reading]: {paragraphs[current_idx]}"
|
| 132 |
+
|
| 133 |
+
return context
|
| 134 |
|
| 135 |
|
| 136 |
def answer_story_question(
|