Spaces:
Paused
Paused
Commit ·
d422be7
1
Parent(s): c5bd25e
rename: app.py -> app_cascade.py, app_lfm.py -> app.py
Browse filesLFM variant is now the default app.py for HF Space compatibility.
Original 3-model pipeline preserved as app_cascade.py.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
- app.py +35 -43
- app_lfm.py → app_cascade.py +43 -33
- test_lfm_answer.wav +3 -0
- test_lfm_answer2.wav +3 -0
app.py
CHANGED
|
@@ -10,6 +10,7 @@ from pathlib import Path
|
|
| 10 |
|
| 11 |
from tts import split_into_chunks, generate_audio_stream
|
| 12 |
from inference import transcribe_audio, answer_story_question
|
|
|
|
| 13 |
from voice_clone import load_default_profile, list_saved_profiles
|
| 14 |
|
| 15 |
# Create directories for sample audio files
|
|
@@ -139,6 +140,8 @@ with open(os.path.join(os.path.dirname(__file__), "static", "style.css"), encodi
|
|
| 139 |
css_code = _f.read()
|
| 140 |
|
| 141 |
def generate_dashboard_html(books_list):
|
|
|
|
|
|
|
| 142 |
featured = books_list[0]
|
| 143 |
others = books_list[1:]
|
| 144 |
featured_js_title = featured['title'].replace("'", "\\'").replace('"', '\\"')
|
|
@@ -1016,7 +1019,7 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1016 |
}"""
|
| 1017 |
)
|
| 1018 |
|
| 1019 |
-
# 9. Submit question —
|
| 1020 |
def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val, profile_id):
|
| 1021 |
q_txt = (question_txt or "").strip()
|
| 1022 |
has_audio = question_audio_path is not None and question_audio_path != ""
|
|
@@ -1029,56 +1032,45 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1029 |
"""
|
| 1030 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1031 |
|
| 1032 |
-
#
|
| 1033 |
-
if
|
| 1034 |
-
|
| 1035 |
-
|
| 1036 |
-
|
| 1037 |
-
|
| 1038 |
-
|
| 1039 |
-
|
| 1040 |
-
except Exception as e:
|
| 1041 |
-
q_text = f"(transcription failed: {e})"
|
| 1042 |
-
|
| 1043 |
-
# Step 10: Generate grounded answer from story context using Qwen
|
| 1044 |
-
current_idx = int(slider_val) if slider_val else 0
|
| 1045 |
try:
|
| 1046 |
-
answer_text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1047 |
if not answer_text:
|
| 1048 |
answer_text = "Hmm, I'm not sure about that! Let's keep listening to find out."
|
| 1049 |
except Exception as e:
|
| 1050 |
answer_text = f"Oops, I couldn't think of an answer right now. Let's keep reading! ({type(e).__name__})"
|
| 1051 |
-
|
| 1052 |
-
|
|
|
|
| 1053 |
|
| 1054 |
-
#
|
| 1055 |
-
|
| 1056 |
-
|
| 1057 |
-
|
| 1058 |
-
|
| 1059 |
-
|
| 1060 |
-
|
| 1061 |
-
|
| 1062 |
-
|
| 1063 |
-
|
| 1064 |
-
sample_rate = 16000
|
| 1065 |
-
for sr, wav, idx, total, err in _gen_stream(chunks, voice_profile_id=profile_id):
|
| 1066 |
-
if wav is not None:
|
| 1067 |
-
audio_segments.append(wav)
|
| 1068 |
-
sample_rate = sr
|
| 1069 |
-
if audio_segments:
|
| 1070 |
-
full_audio = np.concatenate(audio_segments)
|
| 1071 |
-
answer_audio_path = f"sample_sounds/qa_answer_{uuid.uuid4().hex[:8]}.wav"
|
| 1072 |
-
sf.write(answer_audio_path, full_audio, sample_rate)
|
| 1073 |
-
except Exception:
|
| 1074 |
-
# Fall back to no audio if TTS fails
|
| 1075 |
-
pass
|
| 1076 |
|
| 1077 |
safe_answer = html.escape(answer_text)
|
| 1078 |
-
safe_question = html.escape(
|
| 1079 |
answer_html = f"""
|
| 1080 |
<div style="margin-top: 12px; padding: 16px; background: rgba(240,253,244,0.1); border: 1px solid rgba(187,247,208,0.3); border-radius: 14px;">
|
| 1081 |
-
<div style="font-size: 10px; font-weight: 700; text-transform: uppercase; color: #4ade80; margin-bottom: 6px;">Answer
|
| 1082 |
<div style="font-family: 'Playfair Display', Georgia, serif; font-style: italic; color: #FAF7F2; font-size: 13px; line-height: 1.6;">
|
| 1083 |
“{safe_answer}”
|
| 1084 |
</div>
|
|
@@ -1087,8 +1079,8 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1087 |
</div>
|
| 1088 |
</div>
|
| 1089 |
"""
|
| 1090 |
-
if
|
| 1091 |
-
return gr.HTML(value=answer_html, visible=True), gr.Audio(value=
|
| 1092 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1093 |
|
| 1094 |
submit_question_btn.click(
|
|
|
|
| 10 |
|
| 11 |
from tts import split_into_chunks, generate_audio_stream
|
| 12 |
from inference import transcribe_audio, answer_story_question
|
| 13 |
+
from inference_lfm import answer_question_audio, SAMPLE_RATE as LFM_SR
|
| 14 |
from voice_clone import load_default_profile, list_saved_profiles
|
| 15 |
|
| 16 |
# Create directories for sample audio files
|
|
|
|
| 140 |
css_code = _f.read()
|
| 141 |
|
| 142 |
def generate_dashboard_html(books_list):
|
| 143 |
+
if not books_list:
|
| 144 |
+
return "<div style='padding:40px;text-align:center;color:#FAF7F2;'>No stories found. Add .txt files to the stories/ folder.</div>"
|
| 145 |
featured = books_list[0]
|
| 146 |
others = books_list[1:]
|
| 147 |
featured_js_title = featured['title'].replace("'", "\\'").replace('"', '\\"')
|
|
|
|
| 1019 |
}"""
|
| 1020 |
)
|
| 1021 |
|
| 1022 |
+
# 9. Submit question — LFM2.5-Audio end-to-end (single model: audio-in → audio+text-out)
|
| 1023 |
def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val, profile_id):
|
| 1024 |
q_txt = (question_txt or "").strip()
|
| 1025 |
has_audio = question_audio_path is not None and question_audio_path != ""
|
|
|
|
| 1032 |
"""
|
| 1033 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1034 |
|
| 1035 |
+
# Build story context from paragraphs
|
| 1036 |
+
story_context = "\n\n".join(paragraphs) if paragraphs else ""
|
| 1037 |
+
|
| 1038 |
+
# Use LFM2.5-Audio end-to-end: accepts audio/text → produces text+audio answer
|
| 1039 |
+
import uuid
|
| 1040 |
+
import soundfile as sf
|
| 1041 |
+
import logging as _log
|
| 1042 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1043 |
try:
|
| 1044 |
+
answer_text, waveform, sr = answer_question_audio(
|
| 1045 |
+
question_audio_path=question_audio_path if has_audio else None,
|
| 1046 |
+
question_text=q_txt if q_txt and not has_audio else None,
|
| 1047 |
+
story_context=story_context,
|
| 1048 |
+
max_new_tokens=200,
|
| 1049 |
+
)
|
| 1050 |
if not answer_text:
|
| 1051 |
answer_text = "Hmm, I'm not sure about that! Let's keep listening to find out."
|
| 1052 |
except Exception as e:
|
| 1053 |
answer_text = f"Oops, I couldn't think of an answer right now. Let's keep reading! ({type(e).__name__})"
|
| 1054 |
+
waveform = None
|
| 1055 |
+
sr = LFM_SR
|
| 1056 |
+
_log.getLogger(__name__).exception("LFM Q&A failed: %s", e)
|
| 1057 |
|
| 1058 |
+
# Save audio if LFM produced it
|
| 1059 |
+
answer_audio_path_out = None
|
| 1060 |
+
if waveform is not None:
|
| 1061 |
+
answer_audio_path_out = f"sample_sounds/qa_answer_{uuid.uuid4().hex[:8]}.wav"
|
| 1062 |
+
sf.write(answer_audio_path_out, waveform, sr)
|
| 1063 |
+
|
| 1064 |
+
# Display question text (transcribe if needed for display only)
|
| 1065 |
+
display_question = q_txt
|
| 1066 |
+
if not display_question and has_audio:
|
| 1067 |
+
display_question = "(audio question)"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1068 |
|
| 1069 |
safe_answer = html.escape(answer_text)
|
| 1070 |
+
safe_question = html.escape(display_question)
|
| 1071 |
answer_html = f"""
|
| 1072 |
<div style="margin-top: 12px; padding: 16px; background: rgba(240,253,244,0.1); border: 1px solid rgba(187,247,208,0.3); border-radius: 14px;">
|
| 1073 |
+
<div style="font-size: 10px; font-weight: 700; text-transform: uppercase; color: #4ade80; margin-bottom: 6px;">Answer (LFM2.5-Audio)</div>
|
| 1074 |
<div style="font-family: 'Playfair Display', Georgia, serif; font-style: italic; color: #FAF7F2; font-size: 13px; line-height: 1.6;">
|
| 1075 |
“{safe_answer}”
|
| 1076 |
</div>
|
|
|
|
| 1079 |
</div>
|
| 1080 |
</div>
|
| 1081 |
"""
|
| 1082 |
+
if answer_audio_path_out:
|
| 1083 |
+
return gr.HTML(value=answer_html, visible=True), gr.Audio(value=answer_audio_path_out, visible=True)
|
| 1084 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1085 |
|
| 1086 |
submit_question_btn.click(
|
app_lfm.py → app_cascade.py
RENAMED
|
@@ -10,7 +10,6 @@ from pathlib import Path
|
|
| 10 |
|
| 11 |
from tts import split_into_chunks, generate_audio_stream
|
| 12 |
from inference import transcribe_audio, answer_story_question
|
| 13 |
-
from inference_lfm import answer_question_audio, SAMPLE_RATE as LFM_SR
|
| 14 |
from voice_clone import load_default_profile, list_saved_profiles
|
| 15 |
|
| 16 |
# Create directories for sample audio files
|
|
@@ -1017,7 +1016,7 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1017 |
}"""
|
| 1018 |
)
|
| 1019 |
|
| 1020 |
-
# 9. Submit question —
|
| 1021 |
def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val, profile_id):
|
| 1022 |
q_txt = (question_txt or "").strip()
|
| 1023 |
has_audio = question_audio_path is not None and question_audio_path != ""
|
|
@@ -1030,45 +1029,56 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1030 |
"""
|
| 1031 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1032 |
|
| 1033 |
-
#
|
| 1034 |
-
|
| 1035 |
-
|
| 1036 |
-
|
| 1037 |
-
|
| 1038 |
-
|
| 1039 |
-
|
| 1040 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1041 |
try:
|
| 1042 |
-
answer_text
|
| 1043 |
-
question_audio_path=question_audio_path if has_audio else None,
|
| 1044 |
-
question_text=q_txt if q_txt and not has_audio else None,
|
| 1045 |
-
story_context=story_context,
|
| 1046 |
-
max_new_tokens=200,
|
| 1047 |
-
)
|
| 1048 |
if not answer_text:
|
| 1049 |
answer_text = "Hmm, I'm not sure about that! Let's keep listening to find out."
|
| 1050 |
except Exception as e:
|
| 1051 |
answer_text = f"Oops, I couldn't think of an answer right now. Let's keep reading! ({type(e).__name__})"
|
| 1052 |
-
|
| 1053 |
-
|
| 1054 |
-
_log.getLogger(__name__).exception("LFM Q&A failed: %s", e)
|
| 1055 |
-
|
| 1056 |
-
# Save audio if LFM produced it
|
| 1057 |
-
answer_audio_path_out = None
|
| 1058 |
-
if waveform is not None:
|
| 1059 |
-
answer_audio_path_out = f"sample_sounds/qa_answer_{uuid.uuid4().hex[:8]}.wav"
|
| 1060 |
-
sf.write(answer_audio_path_out, waveform, sr)
|
| 1061 |
|
| 1062 |
-
#
|
| 1063 |
-
|
| 1064 |
-
|
| 1065 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1066 |
|
| 1067 |
safe_answer = html.escape(answer_text)
|
| 1068 |
-
safe_question = html.escape(
|
| 1069 |
answer_html = f"""
|
| 1070 |
<div style="margin-top: 12px; padding: 16px; background: rgba(240,253,244,0.1); border: 1px solid rgba(187,247,208,0.3); border-radius: 14px;">
|
| 1071 |
-
<div style="font-size: 10px; font-weight: 700; text-transform: uppercase; color: #4ade80; margin-bottom: 6px;">Answer
|
| 1072 |
<div style="font-family: 'Playfair Display', Georgia, serif; font-style: italic; color: #FAF7F2; font-size: 13px; line-height: 1.6;">
|
| 1073 |
“{safe_answer}”
|
| 1074 |
</div>
|
|
@@ -1077,8 +1087,8 @@ with gr.Blocks(title="MomsVoice", css=css_code) as demo:
|
|
| 1077 |
</div>
|
| 1078 |
</div>
|
| 1079 |
"""
|
| 1080 |
-
if
|
| 1081 |
-
return gr.HTML(value=answer_html, visible=True), gr.Audio(value=
|
| 1082 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1083 |
|
| 1084 |
submit_question_btn.click(
|
|
|
|
| 10 |
|
| 11 |
from tts import split_into_chunks, generate_audio_stream
|
| 12 |
from inference import transcribe_audio, answer_story_question
|
|
|
|
| 13 |
from voice_clone import load_default_profile, list_saved_profiles
|
| 14 |
|
| 15 |
# Create directories for sample audio files
|
|
|
|
| 1016 |
}"""
|
| 1017 |
)
|
| 1018 |
|
| 1019 |
+
# 9. Submit question — always auto-submits when recording stops
|
| 1020 |
def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val, profile_id):
|
| 1021 |
q_txt = (question_txt or "").strip()
|
| 1022 |
has_audio = question_audio_path is not None and question_audio_path != ""
|
|
|
|
| 1029 |
"""
|
| 1030 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1031 |
|
| 1032 |
+
# Step 9: Transcribe audio if text is empty (on-demand ASR)
|
| 1033 |
+
if q_txt:
|
| 1034 |
+
q_text = q_txt
|
| 1035 |
+
else:
|
| 1036 |
+
try:
|
| 1037 |
+
q_text = transcribe_audio(question_audio_path)
|
| 1038 |
+
if not q_text:
|
| 1039 |
+
q_text = "(could not understand audio)"
|
| 1040 |
+
except Exception as e:
|
| 1041 |
+
q_text = f"(transcription failed: {e})"
|
| 1042 |
+
|
| 1043 |
+
# Step 10: Generate grounded answer from story context using Qwen
|
| 1044 |
+
current_idx = int(slider_val) if slider_val else 0
|
| 1045 |
try:
|
| 1046 |
+
answer_text = answer_story_question(q_text, paragraphs, current_idx)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1047 |
if not answer_text:
|
| 1048 |
answer_text = "Hmm, I'm not sure about that! Let's keep listening to find out."
|
| 1049 |
except Exception as e:
|
| 1050 |
answer_text = f"Oops, I couldn't think of an answer right now. Let's keep reading! ({type(e).__name__})"
|
| 1051 |
+
import logging
|
| 1052 |
+
logging.getLogger(__name__).exception("Q&A failed: %s", e)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1053 |
|
| 1054 |
+
# Synthesize answer in cloned voice via TTS
|
| 1055 |
+
answer_audio_path = None
|
| 1056 |
+
try:
|
| 1057 |
+
from tts import split_into_chunks as _split, generate_audio_stream as _gen_stream
|
| 1058 |
+
import soundfile as sf
|
| 1059 |
+
import numpy as np
|
| 1060 |
+
import uuid
|
| 1061 |
+
|
| 1062 |
+
chunks = _split(answer_text)
|
| 1063 |
+
audio_segments = []
|
| 1064 |
+
sample_rate = 16000
|
| 1065 |
+
for sr, wav, idx, total, err in _gen_stream(chunks, voice_profile_id=profile_id):
|
| 1066 |
+
if wav is not None:
|
| 1067 |
+
audio_segments.append(wav)
|
| 1068 |
+
sample_rate = sr
|
| 1069 |
+
if audio_segments:
|
| 1070 |
+
full_audio = np.concatenate(audio_segments)
|
| 1071 |
+
answer_audio_path = f"sample_sounds/qa_answer_{uuid.uuid4().hex[:8]}.wav"
|
| 1072 |
+
sf.write(answer_audio_path, full_audio, sample_rate)
|
| 1073 |
+
except Exception:
|
| 1074 |
+
# Fall back to no audio if TTS fails
|
| 1075 |
+
pass
|
| 1076 |
|
| 1077 |
safe_answer = html.escape(answer_text)
|
| 1078 |
+
safe_question = html.escape(q_text)
|
| 1079 |
answer_html = f"""
|
| 1080 |
<div style="margin-top: 12px; padding: 16px; background: rgba(240,253,244,0.1); border: 1px solid rgba(187,247,208,0.3); border-radius: 14px;">
|
| 1081 |
+
<div style="font-size: 10px; font-weight: 700; text-transform: uppercase; color: #4ade80; margin-bottom: 6px;">Answer in Narrator's Voice</div>
|
| 1082 |
<div style="font-family: 'Playfair Display', Georgia, serif; font-style: italic; color: #FAF7F2; font-size: 13px; line-height: 1.6;">
|
| 1083 |
“{safe_answer}”
|
| 1084 |
</div>
|
|
|
|
| 1087 |
</div>
|
| 1088 |
</div>
|
| 1089 |
"""
|
| 1090 |
+
if answer_audio_path:
|
| 1091 |
+
return gr.HTML(value=answer_html, visible=True), gr.Audio(value=answer_audio_path, visible=True)
|
| 1092 |
return gr.HTML(value=answer_html, visible=True), gr.Audio(visible=False)
|
| 1093 |
|
| 1094 |
submit_question_btn.click(
|
test_lfm_answer.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0bfeb07501ccf11c32d6b1d2614b8edabad10ac5f05fbf48b81563a66060432a
|
| 3 |
+
size 188204
|
test_lfm_answer2.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f889b069437f2cf0d8466ee43174d8d5f0f169f337c4206bb9f53ca05e928358
|
| 3 |
+
size 134444
|