HopeV1.0 / app.py
Sumedh
Claude: voice back to ritu; add SARVAM_TEMPERATURE (expressiveness) setting
d33ef6b
Raw History Blame Contribute Delete
34.7 kB
#!/usr/bin/env python
# coding: utf-8
# HopeV1.0 - Gradio UI + WhatsApp API endpoint
# Server-side TTS (Sarvam / Kokoro, see "TTS SWITCH") and /whatsapp API for the bridge.
import io, json, wave, os, re, random, subprocess, tempfile, uuid, time
import numpy as np
import soundfile as sf
import gradio as gr
# 1) Import core (eager model loading happens here)
import coach_core
from coach_core import speech2text, llama_ask_question_and_record_student_answer
# ============================================================
# STORY LIBRARY
# ============================================================
STORY_LIBRARY = [
("Bathukamma festival is celebrated as part of Dussehra. The festival has historical and religious significance."
"Women carry Bathukamma , beautifully stacked with Tangedu, Gunugu and Chamanthi flowers to the meeting points."
"Making circles around Bathukamma, womenfolk sing songs related to our culture."
"Later, the Bathukamma are taken to the local tanks and streams to be immersed."),
("Two young men were friends. However, one friend was lame and could not walk. The other friend was blind and could not see."
"The two friends wished to go to a fair (mela). But how could they go to the fair?"
"The lame friend had an idea! he climbed on the shoulder of his blind friend."
"Then he showed the blind friend the way to the fair."
"The two friends enjoyed the fair. They then returned home, their friendhship even stronger."
"The moral of the story is that our friends can help us to succeed. Also, we can help our friends to succeed."),
("A hare and a turtle lived in the jungle. The hare could run fast but the turtle could only walk slowly."
"The hare challenged the turtle to a race."
"The hare was fast and left the turtle far behind."
"He decided to rest under a tree, but he fell asleep."
"The turtle walked past him and won the race."
"The moral of the story is SLOW and STEADY WINS THE RACE."),
("A thirsty crow found a pitcher with very little water in it. "
"He could not reach the water with his beak. "
"He had a clever idea and started dropping pebbles into the pitcher one by one. "
"The water level rose up, and the crow drank the water and flew away happily. "
"This shows that where there is a will, there is a way."),
("Once upon a time there lived a lion and lioness in a jungle."
"The lion was the king and the lioness was the queen of the jungle."
"There also lived a mouse in the jungle."
"He was small and was afraid of the big lion and lioness."
"One day the big lion was caught in a net."
"He was trapped in the net."
"He could not move."
"The small mouse saw the lion trapped."
"He ran to help the lion."
"The mouse used his teeth to bite the net and cut it open."
"The big lion was free!"
"The lioness said to the small mouse 'Thank you very much, small mouse'!"
"Then they became friends."),
("Rajaa Akbar had a grandaughter called Milli."
"Milli was 2 years old."
"Rajaa Akbar wanted Milli to always be happy and not be sad."
"He did everything Milli asked."
"One night Milli saw the moon in the sky."
"She told her daadaaji, Rajaa Akbar, 'I want to play with the moon.'"
"'Bring the moon to play with me.'"
"Rajaa Akbar was in a fix!"
"How could he get the moon to Milli?"
"He called his faithful servant Beerbal."
"Beerbal was wise but bringing the moon to the ground from the sky seemed impossible!"
"Beerbal thought long and hard to solve the problem."
"Suddenly, he had the answer!"),
("One sunny day, Ellie found a lost puppy in the park and decided to take it home."
"They became the best of friends, sharing adventures and lots of giggles."
"Every night, they would snuggle up and dream about their next day's fun.")
]
# ============================================================
# PATCH coach_core so it captures spoken text instead of playing locally
# ============================================================
_SPOKEN_CHUNKS = []
def _patched_play_to_student(text, wav_name='amy_said.wav', wav_file_only=False):
if not text or str(text).strip() == "":
return
t = re.sub(r"\*[^*]+\*", "", str(text))
try:
t = t.encode('cp1252', 'ignore').decode('cp1252')
except Exception:
pass
_SPOKEN_CHUNKS.append(t.strip())
def _patched_keep_playing():
return
coach_core.play_to_student = _patched_play_to_student
coach_core.keep_looking_for_audio_and_keep_playing = _patched_keep_playing
# ============================================================
# KOKORO TTS β€” English voice for kids
# Voice: af_bella (warm American female, child-friendly)
# Speed: 0.75 β€” slightly slower than normal, clear for children
# Output: WAV at 24 kHz via soundfile
# Requires: pip install kokoro>=0.9.4 + apt install espeak-ng
# ============================================================
TTS_DIR = "/tmp/tts_out"
os.makedirs(TTS_DIR, exist_ok=True)
DEFAULT_TTS_SPEED = float(os.getenv("TTS_SPEED", "0.75")) # 1.0 = normal
KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_bella") # override via env if needed
_kokoro_pipeline = None
def _get_kokoro():
global _kokoro_pipeline
if _kokoro_pipeline is None:
print("[Kokoro] Loading pipeline...", flush=True)
from kokoro import KPipeline
_kokoro_pipeline = KPipeline(lang_code="a") # 'a' = American English
print("[Kokoro] Pipeline ready.", flush=True)
return _kokoro_pipeline
# Pre-warm at startup so the first request is fast
try:
_get_kokoro()
print("[Kokoro] Pre-warm complete.", flush=True)
except Exception as _e:
print(f"[Kokoro] Pre-warm failed (will retry on first call): {_e}", flush=True)
def kokoro_tts_to_wav(text: str, speed: float = DEFAULT_TTS_SPEED):
"""Generate speech with Kokoro TTS. Returns path to WAV file, or None on failure."""
if not text or not text.strip():
return None
out_path = os.path.join(TTS_DIR, f"tts_{uuid.uuid4().hex}.wav")
try:
pipeline = _get_kokoro()
audio_chunks = []
for _, _, audio in pipeline(text, voice=KOKORO_VOICE, speed=speed):
audio_chunks.append(audio)
if not audio_chunks:
print("[Kokoro] No audio chunks generated.", flush=True)
return None
audio_np = np.concatenate(audio_chunks)
sf.write(out_path, audio_np, 24000)
size = os.path.getsize(out_path)
if size < 200:
print(f"[Kokoro] Output too small ({size} bytes): {out_path}", flush=True)
return None
print(f"[Kokoro] OK β€” {size} bytes, speed={speed:.2f} β†’ {out_path}", flush=True)
return out_path
except Exception as e:
import traceback
print(f"[Kokoro EXCEPTION] {e}\n{traceback.format_exc()}", flush=True)
return None
# ============================================================
# β˜… TTS SWITCH β€” one setting for ALL of Hope (WhatsApp + web browser UI)
#
# "sarvam" β†’ Sarvam Bulbul, Indian-accented English (needs SARVAM_API_KEY)
# "kokoro" β†’ local Kokoro-82M on the GPU (free, American accent)
# "browser" β†’ web UI uses the browser's built-in voice (old behaviour);
# WhatsApp can't use a browser voice, so it uses Kokoro
#
# Change it here, or override without a code change by setting the
# TTS_ENGINE variable in the HF Space settings (then restart the Space).
# If Sarvam is selected but fails (bad key, no credit, timeout), that reply
# falls back to Kokoro so the bot never goes silent.
# ============================================================
DEFAULT_TTS_ENGINE = "sarvam"
TTS_ENGINE = (os.getenv("TTS_ENGINE") or os.getenv("TTS_PROVIDER") or DEFAULT_TTS_ENGINE).strip().lower()
if TTS_ENGINE not in ("sarvam", "kokoro", "browser"):
print(f"[TTS] Unknown TTS_ENGINE={TTS_ENGINE!r} β€” using {DEFAULT_TTS_ENGINE}", flush=True)
TTS_ENGINE = DEFAULT_TTS_ENGINE
# ============================================================
# SARVAM BULBUL TTS β€” Indian-accented English
# Docs: https://docs.sarvam.ai/api-reference/text-to-speech/convert
# Env vars (HF Space secrets/variables):
# SARVAM_API_KEY secret, required for Sarvam (without it Kokoro is used)
# SARVAM_MODEL default "bulbul:v3" (or "bulbul:v4-flash" for style personas)
# SARVAM_SPEAKER default "ritu" (lowercase; must match the model β€”
# v3: ritu, priya, ishita, ...; v4-flash: persona ids like
# "nupur_en_kids", "aditi_en_stories", "ritu_en_edtech")
# SARVAM_PACE default 0.9 (0.5–2.0) β€” used by WhatsApp.
# The web UI uses its Voice Speed slider instead.
# SARVAM_TEMPERATURE default 0.6 (0.01–1.0) β€” higher = more expressive,
# but too high can add glitches. Try 0.7–0.85.
# Pick a voice by ear first (voice_audition.py in the whatsapp folder).
# Story audio is cached on disk, so each story is only paid for once per voice.
# ============================================================
import base64, hashlib, shutil
import requests
SARVAM_API_KEY = os.getenv("SARVAM_API_KEY", "").strip()
if TTS_ENGINE == "sarvam" and not SARVAM_API_KEY:
print("[TTS] TTS_ENGINE=sarvam but SARVAM_API_KEY is not set β€” Kokoro will be used", flush=True)
SARVAM_MODEL = os.getenv("SARVAM_MODEL", "bulbul:v3").strip()
SARVAM_SPEAKER = os.getenv("SARVAM_SPEAKER", "ritu").strip().lower()
SARVAM_PACE = min(2.0, max(0.5, float(os.getenv("SARVAM_PACE", "0.9"))))
SARVAM_TEMPERATURE = min(1.0, max(0.01, float(os.getenv("SARVAM_TEMPERATURE", "0.6"))))
SARVAM_URL = "https://api.sarvam.ai/text-to-speech"
SARVAM_SR = 24000
SARVAM_MAX_CHARS = 2000 # API limit is 2500; stay safely below
SEGMENT_GAP_SEC = 0.35 # short pause between greeting / story / question
TTS_CACHE_DIR = "/data/tts_cache" if os.path.isdir("/data") else "/tmp/tts_cache"
os.makedirs(TTS_CACHE_DIR, exist_ok=True)
print(f"[TTS] engine={TTS_ENGINE} model={SARVAM_MODEL} speaker={SARVAM_SPEAKER} "
f"pace={SARVAM_PACE} temp={SARVAM_TEMPERATURE} key_set={bool(SARVAM_API_KEY)}", flush=True)
def _split_for_sarvam(text: str, limit: int = SARVAM_MAX_CHARS):
"""Split text into chunks <= limit chars, preferring sentence boundaries."""
text = re.sub(r"\s+", " ", text).strip()
if len(text) <= limit:
return [text] if text else []
sentences = re.split(r"(?<=[.!?])\s+", text)
chunks, cur = [], ""
for s in sentences:
while len(s) > limit: # very long sentence: hard split on a space
cut = s.rfind(" ", 0, limit)
cut = cut if cut > 0 else limit
if cur:
chunks.append(cur); cur = ""
chunks.append(s[:cut].strip()); s = s[cut:].strip()
if len(cur) + len(s) + 1 > limit:
chunks.append(cur); cur = s
else:
cur = f"{cur} {s}".strip()
if cur:
chunks.append(cur)
return [c for c in chunks if c]
class _SarvamFatal(RuntimeError):
"""Bad key / bad request β€” retrying won't help."""
def _sarvam_chunk_audio(text: str, pace: float):
"""Call Sarvam for one chunk (<= limit). Returns float32 numpy array at SARVAM_SR."""
payload = {
"text": text,
"language_code": "en-IN",
"speaker": SARVAM_SPEAKER,
"model": SARVAM_MODEL,
"pace": pace,
"temperature": SARVAM_TEMPERATURE,
"speech_sample_rate": SARVAM_SR,
"output_audio_codec": "wav",
}
headers = {"api-subscription-key": SARVAM_API_KEY, "Content-Type": "application/json"}
last_err = None
for attempt in range(1, 3):
try:
r = requests.post(SARVAM_URL, json=payload, headers=headers, timeout=40)
if r.status_code in (400, 401, 403, 422): # not retryable
raise _SarvamFatal(f"Sarvam {r.status_code}: {r.text[:300]}")
r.raise_for_status()
audios = r.json().get("audios") or []
if not audios:
raise RuntimeError(f"Sarvam returned no audio: {r.text[:200]}")
parts = []
for b64 in audios:
data, sr = sf.read(io.BytesIO(base64.b64decode(b64)), dtype="float32")
if data.ndim > 1:
data = data.mean(axis=1)
if sr != SARVAM_SR:
import librosa
data = librosa.resample(data, orig_sr=sr, target_sr=SARVAM_SR)
parts.append(data)
return np.concatenate(parts)
except _SarvamFatal:
raise
except Exception as e:
last_err = e
print(f"[Sarvam] attempt {attempt} failed: {last_err}", flush=True)
time.sleep(1.5 * attempt)
raise RuntimeError(f"Sarvam failed after retries: {last_err}")
def _sarvam_segment_audio(text: str, cache: bool, pace: float):
"""Synthesize one logical segment (chunked as needed), with optional disk cache."""
cache_path = None
if cache:
key = hashlib.sha1(
f"{SARVAM_MODEL}|{SARVAM_SPEAKER}|{pace}|{SARVAM_TEMPERATURE}|{SARVAM_SR}|{text}".encode()
).hexdigest()
cache_path = os.path.join(TTS_CACHE_DIR, f"{key}.wav")
if os.path.exists(cache_path):
data, _ = sf.read(cache_path, dtype="float32")
print(f"[Sarvam] cache hit ({len(text)} chars)", flush=True)
return data
pieces = [_sarvam_chunk_audio(c, pace) for c in _split_for_sarvam(text)]
if not pieces:
return np.zeros(0, dtype="float32")
audio = np.concatenate(pieces)
if cache_path:
try:
sf.write(cache_path, audio, SARVAM_SR)
except Exception as e:
print(f"[Sarvam] cache write failed: {e}", flush=True)
return audio
def sarvam_tts_to_wav(segments, pace: float = None):
"""segments: list of (text, cacheable). Returns WAV path or None on failure."""
pace = SARVAM_PACE if pace is None else round(min(2.0, max(0.5, float(pace))), 2)
try:
t0 = time.time()
gap = np.zeros(int(SARVAM_SR * SEGMENT_GAP_SEC), dtype="float32")
out = []
for text, cacheable in segments:
if not text or not text.strip():
continue
seg = _sarvam_segment_audio(text.strip(), cache=cacheable, pace=pace)
if seg.size:
if out:
out.append(gap)
out.append(seg)
if not out:
return None
out_path = os.path.join(TTS_DIR, f"tts_{uuid.uuid4().hex}.wav")
sf.write(out_path, np.concatenate(out), SARVAM_SR)
print(f"[Sarvam] OK β€” {os.path.getsize(out_path)} bytes in {time.time()-t0:.1f}s "
f"pace={pace} β†’ {out_path}", flush=True)
return out_path
except Exception as e:
print(f"[Sarvam EXCEPTION] {e}", flush=True)
return None
def tts_to_wav(text: str, speed: float = DEFAULT_TTS_SPEED, segments=None, sarvam_pace=None):
"""
Main server-side TTS entry point, used by WhatsApp and the web UI.
Follows TTS_ENGINE: Sarvam (falls back to Kokoro on failure) or Kokoro.
("browser" has no server voice, so server-side calls use Kokoro.)
`segments` (optional): list of (text, cacheable) β€” used at session start so the
story audio is cached separately from the personalised greeting/question.
`speed` applies to Kokoro; Sarvam uses `sarvam_pace` if given, else SARVAM_PACE.
"""
if not text or not text.strip():
return None
if TTS_ENGINE == "sarvam" and SARVAM_API_KEY:
path = sarvam_tts_to_wav(segments or [(text, False)], pace=sarvam_pace)
if path:
return path
print("[TTS] Sarvam failed β€” falling back to Kokoro", flush=True)
return kokoro_tts_to_wav(text, speed=speed)
def start_segments(full_speech: str, story_text: str):
"""Split session-start speech into greeting / story / question so the story is cached."""
s_idx = full_speech.find(story_text) if story_text else -1
if s_idx >= 0:
return [
(full_speech[:s_idx], False),
(story_text, True),
(full_speech[s_idx + len(story_text):], False),
]
return [(full_speech, False)]
# ── Class-based speed + question count ──────────────────────
def class_to_tts_speed(student_class: int) -> float:
"""Uniform slow speed for all classes β€” clear and comfortable for all ages."""
return 0.75
def class_to_num_q(student_class: int) -> int:
"""More questions for higher classes."""
if student_class <= 2: return 2
elif student_class <= 4: return 3
elif student_class <= 6: return 4
else: return 5
# ============================================================
# HELPERS (shared by Gradio UI + WhatsApp API)
# ============================================================
TARGET_SR = 16000
def _numpy_to_wav_bytesio(sr_in: int, audio_np: np.ndarray) -> io.BytesIO:
x = audio_np.astype(np.float32)
if x.ndim > 1:
x = x.mean(axis=1)
x = np.clip(x, -1.0, 1.0)
if sr_in != TARGET_SR:
ratio = TARGET_SR / float(sr_in)
new_len = int(round(len(x) * ratio))
if new_len <= 1: new_len = 2
t_old = np.linspace(0.0, 1.0, num=len(x), endpoint=False, dtype=np.float32)
t_new = np.linspace(0.0, 1.0, num=new_len, endpoint=False, dtype=np.float32)
x = np.interp(t_new, t_old, x).astype(np.float32)
pcm16 = (x * 32767.0).astype(np.int16)
buf = io.BytesIO()
with wave.open(buf, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(TARGET_SR)
wf.writeframes(pcm16.tobytes())
buf.seek(0)
return buf
def _wav_file_to_bytesio(wav_path: str) -> io.BytesIO:
with open(wav_path, "rb") as f:
return io.BytesIO(f.read())
def _get_random_story():
return random.choice(STORY_LIBRARY)
def _generate_bot_reply(messages):
"""Run LLM, collect spoken chunks, return concatenated text."""
llama_input = coach_core.tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
_SPOKEN_CHUNKS.clear()
coach_core.llama_ask_question_and_record_student_answer(llama_input, record=False)
reply = " ".join(_SPOKEN_CHUNKS).strip() or "(No output)"
_SPOKEN_CHUNKS.clear()
return reply
# ============================================================
# SAFETY GUARDRAILS (word-filter safety net for a children's app)
# ============================================================
# Conservative list of clearly inappropriate terms (English + common romanized
# Hindi swears, since children may speak Hindi). Word-boundary matched, so normal
# story words are not affected. Keep this tight to avoid false positives.
_INAPPROPRIATE_WORDS = [
# English profanity / sexual
"fuck", "fucking", "shit", "bitch", "bastard", "asshole", "dick", "pussy",
"cunt", "slut", "whore", "sex", "porn", "nigger", "rape", "penis", "vagina",
"boobs", "nude", "naked",
# Common romanized Hindi swears
"bhenchod", "behenchod", "madarchod", "madarchood", "chutiya", "chutiyaa",
"gaand", "gandu", "lund", "lauda", "randi", "harami", "kutiya", "bhosdi",
"bhosdike", "chod", "chod do",
]
_INAPPROPRIATE_RE = re.compile(
r"\b(" + "|".join(re.escape(w) for w in _INAPPROPRIATE_WORDS) + r")\b",
re.IGNORECASE,
)
_SAFE_REDIRECT = (
"Oops, let's keep talking about our lovely story! "
"Can you tell me what your favourite part was?"
)
def _is_inappropriate(text: str) -> bool:
"""True if the text trips the profanity/inappropriate filter."""
if not text:
return False
return bool(_INAPPROPRIATE_RE.search(text))
def _sanitize_reply(text: str) -> str:
"""Guard the LLM output: if it slips and includes a flagged word, replace it
with a safe, friendly fallback so a child never hears it."""
if _is_inappropriate(text):
print("[GUARDRAIL] LLM reply tripped filter β€” using safe fallback.", flush=True)
return "Great job! Let's keep going with our story. What happened next?"
return text
# ============================================================
# SESSION LOGIC (pure functions, used by both UI and API)
# ============================================================
def _start_session_logic(story_text: str, num_questions: int, student_name: str = "Student"):
"""Returns (bot_question, full_speech_text, state_dict)."""
fixed_prompt = coach_core.fixed_prompt
target_q = int(num_questions)
messages = [
{"role": "system", "content": fixed_prompt},
{
"role": "user",
"content": (
f"The child's name is {student_name}. Warmly greet them by name and say you'll "
"talk about a fun story together. Then ask ONE short, simple question about the "
"story below. Ask only one question and keep it friendly.\n\n"
"===== STORY =====\n"
f"{story_text}\n"
"===== END STORY ====="
)
}
]
bot_question = _generate_bot_reply(messages)
messages.append({"role": "assistant", "content": bot_question})
greeting_text = f"Hi {student_name}! I am about to read a short story. Ready? Let's start."
full_speech_text = greeting_text + " " + story_text + " " + bot_question
state = {
"messages": messages,
"asked": 1,
"num_q": target_q,
}
return bot_question, full_speech_text, state
def _submit_answer_logic(state: dict, transcript: str):
"""Returns (reply_text, speech_text, new_state, is_complete)."""
messages = state["messages"]
current_q = state["asked"]
max_q = state["num_q"]
# Guardrail: if the child says something inappropriate, gently redirect WITHOUT
# sending it to the LLM, and do not advance the question count or end the session.
if _is_inappropriate(transcript):
print("[GUARDRAIL] Inappropriate input β€” redirecting, not advancing.", flush=True)
return _SAFE_REDIRECT, _SAFE_REDIRECT, state, False
appreciate_only = (current_q >= max_q)
if appreciate_only:
user_content = (
f"My answer is: {transcript}.\n\n"
"[END OF SESSION]\n"
"This was the last question. Give ONE warm, cheerful sentence praising how well "
"I did talking about the story, then say a friendly goodbye. "
"Do not ask any more questions. Do not correct my grammar or English."
)
else:
user_content = transcript
messages.append({"role": "user", "content": user_content})
reply_text = _sanitize_reply(_generate_bot_reply(messages))
messages.append({"role": "assistant", "content": reply_text})
state["messages"] = messages
if not appreciate_only:
state["asked"] += 1
return reply_text, reply_text, state, appreciate_only
# ============================================================
# GRADIO UI WRAPPERS
# Server voice (Sarvam/Kokoro) when TTS_ENGINE != "browser"; the returned
# audio path feeds an autoplaying gr.Audio. In "browser" mode it is None and
# the JS Web Speech API speaks instead.
# ============================================================
def _ui_tts(text, speed, segments=None):
if TTS_ENGINE == "browser" or not text:
return None
spd = float(speed) if speed else DEFAULT_TTS_SPEED
return tts_to_wav(text, speed=spd, segments=segments, sarvam_pace=spd)
def _start_session_ui(story_text, num_questions, speed=None):
bot_question, full_speech, state = _start_session_logic(story_text, num_questions)
target_q = state["num_q"]
display_text = f"(Question 1/{target_q}): {bot_question}"
audio = _ui_tts(full_speech, speed, segments=start_segments(full_speech, story_text))
return display_text, full_speech, json.dumps(state), audio
def _submit_answer_ui(state_json, mic_tuple, speed=None):
def _msg(display, speech, st):
return display, speech, st, None, _ui_tts(speech, speed)
if not state_json:
return _msg("Click 'Start session' first.", "Click start session first.", state_json)
if mic_tuple is None:
return _msg("I couldn't hear that. Please try again.", "I couldn't hear that.", state_json)
try:
sr_in, data_in = mic_tuple
buf = _numpy_to_wav_bytesio(sr_in, data_in)
transcript = speech2text(buf)
except Exception as e:
return f"Audio error: {e}", "Audio error", state_json, None, None
if not transcript:
return _msg("I couldn't catch that. Please try again.", "I couldn't catch that.", state_json)
state = json.loads(state_json)
reply_text, speech_text, new_state, is_complete = _submit_answer_logic(state, transcript)
if is_complete:
display_text = f"(Session Complete): {reply_text}"
else:
display_text = f"(Question {new_state['asked']}/{new_state['num_q']}): {reply_text}"
return display_text, speech_text, json.dumps(new_state), None, _ui_tts(speech_text, speed)
def _replay_ui(last_audio):
"""Re-emit the last reply under a new filename so the audio player autoplays again."""
if not last_audio or not os.path.exists(str(last_audio)):
return None
new_path = os.path.join(TTS_DIR, f"replay_{uuid.uuid4().hex}.wav")
try:
shutil.copy(str(last_audio), new_path)
return new_path
except Exception:
return last_audio
# ============================================================
# WHATSAPP API HANDLER
# Input:
# audio_file: path to WAV file (bridge has already converted from OGG)
# OR None / empty file for session start
# state_json: JSON string of session state ({"messages":[], "asked":0, "num_q":3})
# Output dict:
# { "bot_text": str, "audio_path": str (WAV on server disk), "state": str (json) }
# Bridge will download the audio_path file via Gradio and convert WAV->OGG for WhatsApp.
# ============================================================
def whatsapp_api_handler(audio_file, state_json):
t0 = time.time()
try:
state = json.loads(state_json) if state_json else {"messages": [], "asked": 0, "num_q": 3}
except Exception:
state = {"messages": [], "asked": 0, "num_q": 3}
asked = state.get("asked", 0)
student_name = state.get("student_name", "Student")
tts_speed = float(state.get("tts_speed", DEFAULT_TTS_SPEED))
story_index = state.get("story_index", None) # None β†’ random
# --- CASE 1: Start of session ---
if asked == 0:
# Use bridge-selected story or fall back to random
if story_index is not None:
try:
idx = int(story_index)
story_text = STORY_LIBRARY[idx] if 0 <= idx < len(STORY_LIBRARY) else _get_random_story()
except (ValueError, TypeError):
story_text = _get_random_story()
else:
story_text = _get_random_story()
num_q = state.get("num_q", 3)
bot_question, full_speech, new_state = _start_session_logic(story_text, num_q, student_name)
new_state["story"] = story_text
new_state["student_name"] = student_name
new_state["tts_speed"] = tts_speed
new_state["student_class"] = state.get("student_class", 0)
# Split into greeting / story / question so the story audio can be cached
wav_path = tts_to_wav(full_speech, speed=tts_speed,
segments=start_segments(full_speech, story_text))
print(f"[WA API] start_session done in {time.time()-t0:.1f}s, wav={wav_path}", flush=True)
return {
"bot_text": full_speech,
"audio_path": wav_path,
"state": json.dumps(new_state),
}
# --- CASE 2: Incoming audio answer ---
if not audio_file or not os.path.exists(str(audio_file)):
err_text = "I did not receive your voice. Please send a voice note."
return {
"bot_text": err_text,
"audio_path": tts_to_wav(err_text, speed=tts_speed),
"state": json.dumps(state),
}
# Transcribe the incoming WAV
try:
with open(str(audio_file), "rb") as f:
wav_buf = io.BytesIO(f.read())
transcript = speech2text(wav_buf)
except Exception as e:
print(f"[WA API] ASR error: {e}", flush=True)
err_text = "Sorry, I couldn't hear you clearly. Please try again."
return {
"bot_text": err_text,
"audio_path": tts_to_wav(err_text, speed=tts_speed),
"state": json.dumps(state),
}
if not transcript or len(transcript.strip()) < 2:
err_text = "I couldn't catch that. Please say it again a little louder."
return {
"bot_text": err_text,
"audio_path": tts_to_wav(err_text, speed=tts_speed),
"state": json.dumps(state),
}
print(f"[WA API] Transcript: {transcript}", flush=True)
# Feed to LLM
reply_text, speech_text, new_state, is_complete = _submit_answer_logic(state, transcript)
# TTS
wav_path = tts_to_wav(speech_text, speed=tts_speed)
# If session ended, set session_complete so bridge shows the post-session menu
if is_complete:
new_state = {
"messages": [],
"asked": 0,
"num_q": new_state.get("num_q", 3),
"session_complete": True,
"student_name": student_name,
"student_class": state.get("student_class", 0),
"tts_speed": tts_speed,
}
print(f"[WA API] reply done in {time.time()-t0:.1f}s, wav={wav_path}", flush=True)
return {
"bot_text": speech_text,
"audio_path": wav_path,
"state": json.dumps(new_state),
}
# ============================================================
# Browser TTS JS β€” only used by the web UI when TTS_ENGINE == "browser"
# ============================================================
JS_TTS_AUTOSPEAK = r"""
function (text, speed_val) {
try {
if (!text || !window.speechSynthesis) { return; }
const u = new SpeechSynthesisUtterance(text);
u.rate = speed_val ? parseFloat(speed_val) : 0.8;
u.pitch = 1.0;
u.lang = "en-US";
function speakNow() {
const voices = speechSynthesis.getVoices();
let best = null;
if (voices.length > 0) {
best = voices.find(v => v.lang.startsWith("en") && v.name.toLowerCase().includes("female"))
|| voices.find(v => v.lang.startsWith("en"));
}
if (best) u.voice = best;
speechSynthesis.cancel();
speechSynthesis.speak(u);
}
if (speechSynthesis.getVoices().length === 0) {
speechSynthesis.addEventListener('voiceschanged', speakNow, { once: true });
} else {
speakNow();
}
} catch (e) { console.error(e); }
}
"""
# ============================================================
# GRADIO APP
# ============================================================
with gr.Blocks(title="English Coach") as demo:
gr.Markdown("## 🎧 English Story Coach")
with gr.Accordion("πŸ“ Story & Settings", open=True):
rand_btn = gr.Button("🎲 Pick Random Story", size="sm")
with gr.Row():
story = gr.Textbox(label="Story", value=STORY_LIBRARY[0], lines=4)
with gr.Row():
num_q = gr.Slider(1, 10, value=3, step=1, label="Number of Questions")
speed_slider = gr.Slider(
minimum=0.3, maximum=1.2, value=0.8, step=0.1,
label="Voice Speed (Lower is Slower)"
)
state = gr.State()
hidden_speech_trigger = gr.Textbox(visible=False)
start_btn = gr.Button("▢️ Start Session", variant="primary")
bot_display = gr.Textbox(label="Bot Question", interactive=False)
use_server_voice = TTS_ENGINE != "browser"
teacher_audio = gr.Audio(
type="filepath", autoplay=True, interactive=False,
label=f"Teacher's voice ({TTS_ENGINE})", visible=use_server_voice,
)
replay_btn = gr.Button("πŸ”Š Replay Teacher's Voice", size="sm")
mic = gr.Audio(sources=["microphone"], type="numpy", label="Record your answer")
submit = gr.Button("Submit Answer")
rand_btn.click(fn=_get_random_story, inputs=None, outputs=story)
start_ev = start_btn.click(
fn=_start_session_ui,
inputs=[story, num_q, speed_slider],
outputs=[bot_display, hidden_speech_trigger, state, teacher_audio]
)
submit_ev = submit.click(
fn=_submit_answer_ui,
inputs=[state, mic, speed_slider],
outputs=[bot_display, hidden_speech_trigger, state, mic, teacher_audio]
)
if use_server_voice:
# Sarvam / Kokoro: audio is generated on the server and autoplays
replay_btn.click(fn=_replay_ui, inputs=[teacher_audio], outputs=[teacher_audio])
else:
# "browser": the browser's built-in voice speaks the text
start_ev.then(fn=None, inputs=[hidden_speech_trigger, speed_slider], js=JS_TTS_AUTOSPEAK)
submit_ev.then(fn=None, inputs=[hidden_speech_trigger, speed_slider], js=JS_TTS_AUTOSPEAK)
replay_btn.click(fn=None, inputs=[hidden_speech_trigger, speed_slider], js=JS_TTS_AUTOSPEAK)
# ---- WhatsApp API endpoint (hidden from UI) ----
# Gradio exposes any function with api_name set as a callable via gradio_client.
# ---- WhatsApp API endpoint (hidden from UI) ----
wa_audio_in = gr.Audio(type="filepath", visible=False, label="wa_audio_in")
wa_state_in = gr.Textbox(visible=False, label="wa_state_in")
wa_bot_text = gr.Textbox(visible=False, label="wa_bot_text")
wa_audio_out = gr.Audio(type="filepath", visible=False, label="wa_audio_out")
wa_state_out = gr.Textbox(visible=False, label="wa_state_out")
wa_trigger = gr.Button(visible=False)
def _whatsapp_api_wrapper(audio_file, state_json):
result = whatsapp_api_handler(audio_file, state_json)
return (
result.get("bot_text", ""),
result.get("audio_path", None),
result.get("state", ""),
)
wa_trigger.click(
fn=_whatsapp_api_wrapper,
inputs=[wa_audio_in, wa_state_in],
outputs=[wa_bot_text, wa_audio_out, wa_state_out],
api_name="whatsapp",
)
if __name__ == "__main__":
demo.queue(max_size=32).launch(server_name="0.0.0.0", server_port=7860, show_api=False, show_error=True)