minhahwang Copilot commited on
Commit
c6ae51e
Β·
1 Parent(s): 0e40ecf

perf: preload all models at startup, improve Q&A context and auto-submit

Browse files

- Preload Whisper-small, Qwen2.5-3B-Instruct, and TTS at startup
- Use .change() event for auto-submit (fires after audio value committed)
- Pass full story context (or top-5 relevant paragraphs) to Q&A
- Fix total_mem -> total_memory attribute error

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Files changed (2) hide show
  1. app.py +23 -6
  2. inference.py +21 -7
app.py CHANGED
@@ -387,11 +387,20 @@ else:
387
  _default_voice_name = None
388
  print("[MomsVoice] No saved voice profile β€” using stock voice as default.")
389
 
390
- # Preload the 0.6B CustomVoice model at startup for fast first-play response
391
  print("[MomsVoice] Preloading Qwen3-TTS 0.6B CustomVoice model...")
392
  from voice_clone import get_custom_voice_model
393
  get_custom_voice_model()
394
- print("[MomsVoice] Model ready.")
 
 
 
 
 
 
 
 
 
395
 
396
 
397
  # Gradio Application Core setup
@@ -1022,10 +1031,18 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
1022
  outputs=[answer_display, answer_audio]
1023
  )
1024
 
1025
- # Auto-submit when voice recording stops (end-of-question detection)
1026
- question_audio.stop_recording(
1027
- handle_question_submit,
1028
- inputs=[question_text, question_audio, paragraphs_state, timeline_slider, voice_profile_state],
 
 
 
 
 
 
 
 
1029
  outputs=[answer_display, answer_audio]
1030
  )
1031
 
 
387
  _default_voice_name = None
388
  print("[MomsVoice] No saved voice profile β€” using stock voice as default.")
389
 
390
+ # Preload models at startup for fast response
391
  print("[MomsVoice] Preloading Qwen3-TTS 0.6B CustomVoice model...")
392
  from voice_clone import get_custom_voice_model
393
  get_custom_voice_model()
394
+ print("[MomsVoice] TTS model ready.")
395
+
396
+ print("[MomsVoice] Preloading Whisper-small ASR model...")
397
+ from inference import get_asr_pipeline, get_qa_model
398
+ get_asr_pipeline()
399
+ print("[MomsVoice] ASR model ready.")
400
+
401
+ print("[MomsVoice] Preloading Qwen2.5-3B-Instruct Q&A model...")
402
+ get_qa_model()
403
+ print("[MomsVoice] Q&A model ready. All models preloaded.")
404
 
405
 
406
  # Gradio Application Core setup
 
1031
  outputs=[answer_display, answer_audio]
1032
  )
1033
 
1034
+ # Auto-submit when voice recording completes
1035
+ # Use .change() instead of .stop_recording() β€” .change() fires after audio value is committed
1036
+ def auto_submit_on_recording(audio_path, question_txt, paragraphs, slider_val, profile_id):
1037
+ """Auto-submit only when a new recording is available (not on clear)."""
1038
+ if not audio_path:
1039
+ # Audio was cleared, don't submit
1040
+ return gr.update(), gr.update()
1041
+ return handle_question_submit(question_txt, audio_path, paragraphs, slider_val, profile_id)
1042
+
1043
+ question_audio.change(
1044
+ auto_submit_on_recording,
1045
+ inputs=[question_audio, question_text, paragraphs_state, timeline_slider, voice_profile_state],
1046
  outputs=[answer_display, answer_audio]
1047
  )
1048
 
inference.py CHANGED
@@ -70,7 +70,7 @@ def get_qa_model():
70
  # Check available VRAM β€” if less than 3GB free, use CPU
71
  use_gpu = False
72
  if torch.cuda.is_available():
73
- free_vram = torch.cuda.get_device_properties(0).total_mem - torch.cuda.memory_allocated(0)
74
  free_vram_gb = free_vram / (1024**3)
75
  logger.info("Free VRAM: %.1f GB", free_vram_gb)
76
  if free_vram_gb >= 3.0:
@@ -98,11 +98,19 @@ def get_qa_model():
98
 
99
 
100
  def _get_relevant_context(paragraphs: list[str], current_idx: int, question: str) -> str:
101
- """Simple TF-IDF-like retrieval: return the most relevant paragraphs as context."""
102
  if not paragraphs:
103
  return ""
104
 
105
- # Use paragraphs around the current position + simple keyword overlap
 
 
 
 
 
 
 
 
106
  question_words = set(question.lower().split())
107
 
108
  scored = []
@@ -110,13 +118,19 @@ def _get_relevant_context(paragraphs: list[str], current_idx: int, question: str
110
  para_words = set(para.lower().split())
111
  overlap = len(question_words & para_words)
112
  # Boost paragraphs near current position
113
- proximity_bonus = max(0, 3 - abs(i - current_idx))
114
  scored.append((overlap + proximity_bonus, i, para))
115
 
116
  scored.sort(key=lambda x: x[0], reverse=True)
117
- # Take top 3 most relevant paragraphs
118
- top_paras = [s[2] for s in scored[:3]]
119
- return "\n\n".join(top_paras)
 
 
 
 
 
 
120
 
121
 
122
  def answer_story_question(
 
70
  # Check available VRAM β€” if less than 3GB free, use CPU
71
  use_gpu = False
72
  if torch.cuda.is_available():
73
+ free_vram = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated(0)
74
  free_vram_gb = free_vram / (1024**3)
75
  logger.info("Free VRAM: %.1f GB", free_vram_gb)
76
  if free_vram_gb >= 3.0:
 
98
 
99
 
100
  def _get_relevant_context(paragraphs: list[str], current_idx: int, question: str) -> str:
101
+ """Return full story with emphasis on current section for context."""
102
  if not paragraphs:
103
  return ""
104
 
105
+ # Build context: full story (truncated if too long) with current paragraph highlighted
106
+ total_text = "\n\n".join(paragraphs)
107
+
108
+ # If story is short enough (< 2000 chars), use it all
109
+ if len(total_text) <= 2000:
110
+ current_marker = f"\n\n[Currently reading]: {paragraphs[current_idx]}" if current_idx < len(paragraphs) else ""
111
+ return total_text + current_marker
112
+
113
+ # For longer stories: use top relevant paragraphs + surrounding context
114
  question_words = set(question.lower().split())
115
 
116
  scored = []
 
118
  para_words = set(para.lower().split())
119
  overlap = len(question_words & para_words)
120
  # Boost paragraphs near current position
121
+ proximity_bonus = max(0, 5 - abs(i - current_idx))
122
  scored.append((overlap + proximity_bonus, i, para))
123
 
124
  scored.sort(key=lambda x: x[0], reverse=True)
125
+ # Take top 5 most relevant paragraphs
126
+ top_paras = sorted(scored[:5], key=lambda x: x[1]) # sort by position
127
+ context = "\n\n".join(s[2] for s in top_paras)
128
+
129
+ # Add current paragraph marker
130
+ if current_idx < len(paragraphs):
131
+ context += f"\n\n[Currently reading]: {paragraphs[current_idx]}"
132
+
133
+ return context
134
 
135
 
136
  def answer_story_question(