Spaces:
Paused
Paused
Download app.py from HopeLLM/HopeV1.0: direct link, hf CLI and curl.
- Browser
- Download file 34.7 kB
-
https://huggingface.co/spaces/HopeLLM/HopeV1.0/resolve/main/app.py
- Command line
-
hf download hf://spaces/HopeLLM/HopeV1.0/app.py
-
curl -L -o app.py https://huggingface.co/spaces/HopeLLM/HopeV1.0/resolve/main/app.py
34.7 kB
| #!/usr/bin/env python | |
| # coding: utf-8 | |
| # HopeV1.0 - Gradio UI + WhatsApp API endpoint | |
| # Server-side TTS (Sarvam / Kokoro, see "TTS SWITCH") and /whatsapp API for the bridge. | |
| import io, json, wave, os, re, random, subprocess, tempfile, uuid, time | |
| import numpy as np | |
| import soundfile as sf | |
| import gradio as gr | |
| # 1) Import core (eager model loading happens here) | |
| import coach_core | |
| from coach_core import speech2text, llama_ask_question_and_record_student_answer | |
| # ============================================================ | |
| # STORY LIBRARY | |
| # ============================================================ | |
| STORY_LIBRARY = [ | |
| ("Bathukamma festival is celebrated as part of Dussehra. The festival has historical and religious significance." | |
| "Women carry Bathukamma , beautifully stacked with Tangedu, Gunugu and Chamanthi flowers to the meeting points." | |
| "Making circles around Bathukamma, womenfolk sing songs related to our culture." | |
| "Later, the Bathukamma are taken to the local tanks and streams to be immersed."), | |
| ("Two young men were friends. However, one friend was lame and could not walk. The other friend was blind and could not see." | |
| "The two friends wished to go to a fair (mela). But how could they go to the fair?" | |
| "The lame friend had an idea! he climbed on the shoulder of his blind friend." | |
| "Then he showed the blind friend the way to the fair." | |
| "The two friends enjoyed the fair. They then returned home, their friendhship even stronger." | |
| "The moral of the story is that our friends can help us to succeed. Also, we can help our friends to succeed."), | |
| ("A hare and a turtle lived in the jungle. The hare could run fast but the turtle could only walk slowly." | |
| "The hare challenged the turtle to a race." | |
| "The hare was fast and left the turtle far behind." | |
| "He decided to rest under a tree, but he fell asleep." | |
| "The turtle walked past him and won the race." | |
| "The moral of the story is SLOW and STEADY WINS THE RACE."), | |
| ("A thirsty crow found a pitcher with very little water in it. " | |
| "He could not reach the water with his beak. " | |
| "He had a clever idea and started dropping pebbles into the pitcher one by one. " | |
| "The water level rose up, and the crow drank the water and flew away happily. " | |
| "This shows that where there is a will, there is a way."), | |
| ("Once upon a time there lived a lion and lioness in a jungle." | |
| "The lion was the king and the lioness was the queen of the jungle." | |
| "There also lived a mouse in the jungle." | |
| "He was small and was afraid of the big lion and lioness." | |
| "One day the big lion was caught in a net." | |
| "He was trapped in the net." | |
| "He could not move." | |
| "The small mouse saw the lion trapped." | |
| "He ran to help the lion." | |
| "The mouse used his teeth to bite the net and cut it open." | |
| "The big lion was free!" | |
| "The lioness said to the small mouse 'Thank you very much, small mouse'!" | |
| "Then they became friends."), | |
| ("Rajaa Akbar had a grandaughter called Milli." | |
| "Milli was 2 years old." | |
| "Rajaa Akbar wanted Milli to always be happy and not be sad." | |
| "He did everything Milli asked." | |
| "One night Milli saw the moon in the sky." | |
| "She told her daadaaji, Rajaa Akbar, 'I want to play with the moon.'" | |
| "'Bring the moon to play with me.'" | |
| "Rajaa Akbar was in a fix!" | |
| "How could he get the moon to Milli?" | |
| "He called his faithful servant Beerbal." | |
| "Beerbal was wise but bringing the moon to the ground from the sky seemed impossible!" | |
| "Beerbal thought long and hard to solve the problem." | |
| "Suddenly, he had the answer!"), | |
| ("One sunny day, Ellie found a lost puppy in the park and decided to take it home." | |
| "They became the best of friends, sharing adventures and lots of giggles." | |
| "Every night, they would snuggle up and dream about their next day's fun.") | |
| ] | |
| # ============================================================ | |
| # PATCH coach_core so it captures spoken text instead of playing locally | |
| # ============================================================ | |
| _SPOKEN_CHUNKS = [] | |
| def _patched_play_to_student(text, wav_name='amy_said.wav', wav_file_only=False): | |
| if not text or str(text).strip() == "": | |
| return | |
| t = re.sub(r"\*[^*]+\*", "", str(text)) | |
| try: | |
| t = t.encode('cp1252', 'ignore').decode('cp1252') | |
| except Exception: | |
| pass | |
| _SPOKEN_CHUNKS.append(t.strip()) | |
| def _patched_keep_playing(): | |
| return | |
| coach_core.play_to_student = _patched_play_to_student | |
| coach_core.keep_looking_for_audio_and_keep_playing = _patched_keep_playing | |
| # ============================================================ | |
| # KOKORO TTS β English voice for kids | |
| # Voice: af_bella (warm American female, child-friendly) | |
| # Speed: 0.75 β slightly slower than normal, clear for children | |
| # Output: WAV at 24 kHz via soundfile | |
| # Requires: pip install kokoro>=0.9.4 + apt install espeak-ng | |
| # ============================================================ | |
| TTS_DIR = "/tmp/tts_out" | |
| os.makedirs(TTS_DIR, exist_ok=True) | |
| DEFAULT_TTS_SPEED = float(os.getenv("TTS_SPEED", "0.75")) # 1.0 = normal | |
| KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_bella") # override via env if needed | |
| _kokoro_pipeline = None | |
| def _get_kokoro(): | |
| global _kokoro_pipeline | |
| if _kokoro_pipeline is None: | |
| print("[Kokoro] Loading pipeline...", flush=True) | |
| from kokoro import KPipeline | |
| _kokoro_pipeline = KPipeline(lang_code="a") # 'a' = American English | |
| print("[Kokoro] Pipeline ready.", flush=True) | |
| return _kokoro_pipeline | |
| # Pre-warm at startup so the first request is fast | |
| try: | |
| _get_kokoro() | |
| print("[Kokoro] Pre-warm complete.", flush=True) | |
| except Exception as _e: | |
| print(f"[Kokoro] Pre-warm failed (will retry on first call): {_e}", flush=True) | |
| def kokoro_tts_to_wav(text: str, speed: float = DEFAULT_TTS_SPEED): | |
| """Generate speech with Kokoro TTS. Returns path to WAV file, or None on failure.""" | |
| if not text or not text.strip(): | |
| return None | |
| out_path = os.path.join(TTS_DIR, f"tts_{uuid.uuid4().hex}.wav") | |
| try: | |
| pipeline = _get_kokoro() | |
| audio_chunks = [] | |
| for _, _, audio in pipeline(text, voice=KOKORO_VOICE, speed=speed): | |
| audio_chunks.append(audio) | |
| if not audio_chunks: | |
| print("[Kokoro] No audio chunks generated.", flush=True) | |
| return None | |
| audio_np = np.concatenate(audio_chunks) | |
| sf.write(out_path, audio_np, 24000) | |
| size = os.path.getsize(out_path) | |
| if size < 200: | |
| print(f"[Kokoro] Output too small ({size} bytes): {out_path}", flush=True) | |
| return None | |
| print(f"[Kokoro] OK β {size} bytes, speed={speed:.2f} β {out_path}", flush=True) | |
| return out_path | |
| except Exception as e: | |
| import traceback | |
| print(f"[Kokoro EXCEPTION] {e}\n{traceback.format_exc()}", flush=True) | |
| return None | |
| # ============================================================ | |
| # β TTS SWITCH β one setting for ALL of Hope (WhatsApp + web browser UI) | |
| # | |
| # "sarvam" β Sarvam Bulbul, Indian-accented English (needs SARVAM_API_KEY) | |
| # "kokoro" β local Kokoro-82M on the GPU (free, American accent) | |
| # "browser" β web UI uses the browser's built-in voice (old behaviour); | |
| # WhatsApp can't use a browser voice, so it uses Kokoro | |
| # | |
| # Change it here, or override without a code change by setting the | |
| # TTS_ENGINE variable in the HF Space settings (then restart the Space). | |
| # If Sarvam is selected but fails (bad key, no credit, timeout), that reply | |
| # falls back to Kokoro so the bot never goes silent. | |
| # ============================================================ | |
| DEFAULT_TTS_ENGINE = "sarvam" | |
| TTS_ENGINE = (os.getenv("TTS_ENGINE") or os.getenv("TTS_PROVIDER") or DEFAULT_TTS_ENGINE).strip().lower() | |
| if TTS_ENGINE not in ("sarvam", "kokoro", "browser"): | |
| print(f"[TTS] Unknown TTS_ENGINE={TTS_ENGINE!r} β using {DEFAULT_TTS_ENGINE}", flush=True) | |
| TTS_ENGINE = DEFAULT_TTS_ENGINE | |
| # ============================================================ | |
| # SARVAM BULBUL TTS β Indian-accented English | |
| # Docs: https://docs.sarvam.ai/api-reference/text-to-speech/convert | |
| # Env vars (HF Space secrets/variables): | |
| # SARVAM_API_KEY secret, required for Sarvam (without it Kokoro is used) | |
| # SARVAM_MODEL default "bulbul:v3" (or "bulbul:v4-flash" for style personas) | |
| # SARVAM_SPEAKER default "ritu" (lowercase; must match the model β | |
| # v3: ritu, priya, ishita, ...; v4-flash: persona ids like | |
| # "nupur_en_kids", "aditi_en_stories", "ritu_en_edtech") | |
| # SARVAM_PACE default 0.9 (0.5β2.0) β used by WhatsApp. | |
| # The web UI uses its Voice Speed slider instead. | |
| # SARVAM_TEMPERATURE default 0.6 (0.01β1.0) β higher = more expressive, | |
| # but too high can add glitches. Try 0.7β0.85. | |
| # Pick a voice by ear first (voice_audition.py in the whatsapp folder). | |
| # Story audio is cached on disk, so each story is only paid for once per voice. | |
| # ============================================================ | |
| import base64, hashlib, shutil | |
| import requests | |
| SARVAM_API_KEY = os.getenv("SARVAM_API_KEY", "").strip() | |
| if TTS_ENGINE == "sarvam" and not SARVAM_API_KEY: | |
| print("[TTS] TTS_ENGINE=sarvam but SARVAM_API_KEY is not set β Kokoro will be used", flush=True) | |
| SARVAM_MODEL = os.getenv("SARVAM_MODEL", "bulbul:v3").strip() | |
| SARVAM_SPEAKER = os.getenv("SARVAM_SPEAKER", "ritu").strip().lower() | |
| SARVAM_PACE = min(2.0, max(0.5, float(os.getenv("SARVAM_PACE", "0.9")))) | |
| SARVAM_TEMPERATURE = min(1.0, max(0.01, float(os.getenv("SARVAM_TEMPERATURE", "0.6")))) | |
| SARVAM_URL = "https://api.sarvam.ai/text-to-speech" | |
| SARVAM_SR = 24000 | |
| SARVAM_MAX_CHARS = 2000 # API limit is 2500; stay safely below | |
| SEGMENT_GAP_SEC = 0.35 # short pause between greeting / story / question | |
| TTS_CACHE_DIR = "/data/tts_cache" if os.path.isdir("/data") else "/tmp/tts_cache" | |
| os.makedirs(TTS_CACHE_DIR, exist_ok=True) | |
| print(f"[TTS] engine={TTS_ENGINE} model={SARVAM_MODEL} speaker={SARVAM_SPEAKER} " | |
| f"pace={SARVAM_PACE} temp={SARVAM_TEMPERATURE} key_set={bool(SARVAM_API_KEY)}", flush=True) | |
| def _split_for_sarvam(text: str, limit: int = SARVAM_MAX_CHARS): | |
| """Split text into chunks <= limit chars, preferring sentence boundaries.""" | |
| text = re.sub(r"\s+", " ", text).strip() | |
| if len(text) <= limit: | |
| return [text] if text else [] | |
| sentences = re.split(r"(?<=[.!?])\s+", text) | |
| chunks, cur = [], "" | |
| for s in sentences: | |
| while len(s) > limit: # very long sentence: hard split on a space | |
| cut = s.rfind(" ", 0, limit) | |
| cut = cut if cut > 0 else limit | |
| if cur: | |
| chunks.append(cur); cur = "" | |
| chunks.append(s[:cut].strip()); s = s[cut:].strip() | |
| if len(cur) + len(s) + 1 > limit: | |
| chunks.append(cur); cur = s | |
| else: | |
| cur = f"{cur} {s}".strip() | |
| if cur: | |
| chunks.append(cur) | |
| return [c for c in chunks if c] | |
| class _SarvamFatal(RuntimeError): | |
| """Bad key / bad request β retrying won't help.""" | |
| def _sarvam_chunk_audio(text: str, pace: float): | |
| """Call Sarvam for one chunk (<= limit). Returns float32 numpy array at SARVAM_SR.""" | |
| payload = { | |
| "text": text, | |
| "language_code": "en-IN", | |
| "speaker": SARVAM_SPEAKER, | |
| "model": SARVAM_MODEL, | |
| "pace": pace, | |
| "temperature": SARVAM_TEMPERATURE, | |
| "speech_sample_rate": SARVAM_SR, | |
| "output_audio_codec": "wav", | |
| } | |
| headers = {"api-subscription-key": SARVAM_API_KEY, "Content-Type": "application/json"} | |
| last_err = None | |
| for attempt in range(1, 3): | |
| try: | |
| r = requests.post(SARVAM_URL, json=payload, headers=headers, timeout=40) | |
| if r.status_code in (400, 401, 403, 422): # not retryable | |
| raise _SarvamFatal(f"Sarvam {r.status_code}: {r.text[:300]}") | |
| r.raise_for_status() | |
| audios = r.json().get("audios") or [] | |
| if not audios: | |
| raise RuntimeError(f"Sarvam returned no audio: {r.text[:200]}") | |
| parts = [] | |
| for b64 in audios: | |
| data, sr = sf.read(io.BytesIO(base64.b64decode(b64)), dtype="float32") | |
| if data.ndim > 1: | |
| data = data.mean(axis=1) | |
| if sr != SARVAM_SR: | |
| import librosa | |
| data = librosa.resample(data, orig_sr=sr, target_sr=SARVAM_SR) | |
| parts.append(data) | |
| return np.concatenate(parts) | |
| except _SarvamFatal: | |
| raise | |
| except Exception as e: | |
| last_err = e | |
| print(f"[Sarvam] attempt {attempt} failed: {last_err}", flush=True) | |
| time.sleep(1.5 * attempt) | |
| raise RuntimeError(f"Sarvam failed after retries: {last_err}") | |
| def _sarvam_segment_audio(text: str, cache: bool, pace: float): | |
| """Synthesize one logical segment (chunked as needed), with optional disk cache.""" | |
| cache_path = None | |
| if cache: | |
| key = hashlib.sha1( | |
| f"{SARVAM_MODEL}|{SARVAM_SPEAKER}|{pace}|{SARVAM_TEMPERATURE}|{SARVAM_SR}|{text}".encode() | |
| ).hexdigest() | |
| cache_path = os.path.join(TTS_CACHE_DIR, f"{key}.wav") | |
| if os.path.exists(cache_path): | |
| data, _ = sf.read(cache_path, dtype="float32") | |
| print(f"[Sarvam] cache hit ({len(text)} chars)", flush=True) | |
| return data | |
| pieces = [_sarvam_chunk_audio(c, pace) for c in _split_for_sarvam(text)] | |
| if not pieces: | |
| return np.zeros(0, dtype="float32") | |
| audio = np.concatenate(pieces) | |
| if cache_path: | |
| try: | |
| sf.write(cache_path, audio, SARVAM_SR) | |
| except Exception as e: | |
| print(f"[Sarvam] cache write failed: {e}", flush=True) | |
| return audio | |
| def sarvam_tts_to_wav(segments, pace: float = None): | |
| """segments: list of (text, cacheable). Returns WAV path or None on failure.""" | |
| pace = SARVAM_PACE if pace is None else round(min(2.0, max(0.5, float(pace))), 2) | |
| try: | |
| t0 = time.time() | |
| gap = np.zeros(int(SARVAM_SR * SEGMENT_GAP_SEC), dtype="float32") | |
| out = [] | |
| for text, cacheable in segments: | |
| if not text or not text.strip(): | |
| continue | |
| seg = _sarvam_segment_audio(text.strip(), cache=cacheable, pace=pace) | |
| if seg.size: | |
| if out: | |
| out.append(gap) | |
| out.append(seg) | |
| if not out: | |
| return None | |
| out_path = os.path.join(TTS_DIR, f"tts_{uuid.uuid4().hex}.wav") | |
| sf.write(out_path, np.concatenate(out), SARVAM_SR) | |
| print(f"[Sarvam] OK β {os.path.getsize(out_path)} bytes in {time.time()-t0:.1f}s " | |
| f"pace={pace} β {out_path}", flush=True) | |
| return out_path | |
| except Exception as e: | |
| print(f"[Sarvam EXCEPTION] {e}", flush=True) | |
| return None | |
| def tts_to_wav(text: str, speed: float = DEFAULT_TTS_SPEED, segments=None, sarvam_pace=None): | |
| """ | |
| Main server-side TTS entry point, used by WhatsApp and the web UI. | |
| Follows TTS_ENGINE: Sarvam (falls back to Kokoro on failure) or Kokoro. | |
| ("browser" has no server voice, so server-side calls use Kokoro.) | |
| `segments` (optional): list of (text, cacheable) β used at session start so the | |
| story audio is cached separately from the personalised greeting/question. | |
| `speed` applies to Kokoro; Sarvam uses `sarvam_pace` if given, else SARVAM_PACE. | |
| """ | |
| if not text or not text.strip(): | |
| return None | |
| if TTS_ENGINE == "sarvam" and SARVAM_API_KEY: | |
| path = sarvam_tts_to_wav(segments or [(text, False)], pace=sarvam_pace) | |
| if path: | |
| return path | |
| print("[TTS] Sarvam failed β falling back to Kokoro", flush=True) | |
| return kokoro_tts_to_wav(text, speed=speed) | |
| def start_segments(full_speech: str, story_text: str): | |
| """Split session-start speech into greeting / story / question so the story is cached.""" | |
| s_idx = full_speech.find(story_text) if story_text else -1 | |
| if s_idx >= 0: | |
| return [ | |
| (full_speech[:s_idx], False), | |
| (story_text, True), | |
| (full_speech[s_idx + len(story_text):], False), | |
| ] | |
| return [(full_speech, False)] | |
| # ββ Class-based speed + question count ββββββββββββββββββββββ | |
| def class_to_tts_speed(student_class: int) -> float: | |
| """Uniform slow speed for all classes β clear and comfortable for all ages.""" | |
| return 0.75 | |
| def class_to_num_q(student_class: int) -> int: | |
| """More questions for higher classes.""" | |
| if student_class <= 2: return 2 | |
| elif student_class <= 4: return 3 | |
| elif student_class <= 6: return 4 | |
| else: return 5 | |
| # ============================================================ | |
| # HELPERS (shared by Gradio UI + WhatsApp API) | |
| # ============================================================ | |
| TARGET_SR = 16000 | |
| def _numpy_to_wav_bytesio(sr_in: int, audio_np: np.ndarray) -> io.BytesIO: | |
| x = audio_np.astype(np.float32) | |
| if x.ndim > 1: | |
| x = x.mean(axis=1) | |
| x = np.clip(x, -1.0, 1.0) | |
| if sr_in != TARGET_SR: | |
| ratio = TARGET_SR / float(sr_in) | |
| new_len = int(round(len(x) * ratio)) | |
| if new_len <= 1: new_len = 2 | |
| t_old = np.linspace(0.0, 1.0, num=len(x), endpoint=False, dtype=np.float32) | |
| t_new = np.linspace(0.0, 1.0, num=new_len, endpoint=False, dtype=np.float32) | |
| x = np.interp(t_new, t_old, x).astype(np.float32) | |
| pcm16 = (x * 32767.0).astype(np.int16) | |
| buf = io.BytesIO() | |
| with wave.open(buf, "wb") as wf: | |
| wf.setnchannels(1) | |
| wf.setsampwidth(2) | |
| wf.setframerate(TARGET_SR) | |
| wf.writeframes(pcm16.tobytes()) | |
| buf.seek(0) | |
| return buf | |
| def _wav_file_to_bytesio(wav_path: str) -> io.BytesIO: | |
| with open(wav_path, "rb") as f: | |
| return io.BytesIO(f.read()) | |
| def _get_random_story(): | |
| return random.choice(STORY_LIBRARY) | |
| def _generate_bot_reply(messages): | |
| """Run LLM, collect spoken chunks, return concatenated text.""" | |
| llama_input = coach_core.tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True | |
| ) | |
| _SPOKEN_CHUNKS.clear() | |
| coach_core.llama_ask_question_and_record_student_answer(llama_input, record=False) | |
| reply = " ".join(_SPOKEN_CHUNKS).strip() or "(No output)" | |
| _SPOKEN_CHUNKS.clear() | |
| return reply | |
| # ============================================================ | |
| # SAFETY GUARDRAILS (word-filter safety net for a children's app) | |
| # ============================================================ | |
| # Conservative list of clearly inappropriate terms (English + common romanized | |
| # Hindi swears, since children may speak Hindi). Word-boundary matched, so normal | |
| # story words are not affected. Keep this tight to avoid false positives. | |
| _INAPPROPRIATE_WORDS = [ | |
| # English profanity / sexual | |
| "fuck", "fucking", "shit", "bitch", "bastard", "asshole", "dick", "pussy", | |
| "cunt", "slut", "whore", "sex", "porn", "nigger", "rape", "penis", "vagina", | |
| "boobs", "nude", "naked", | |
| # Common romanized Hindi swears | |
| "bhenchod", "behenchod", "madarchod", "madarchood", "chutiya", "chutiyaa", | |
| "gaand", "gandu", "lund", "lauda", "randi", "harami", "kutiya", "bhosdi", | |
| "bhosdike", "chod", "chod do", | |
| ] | |
| _INAPPROPRIATE_RE = re.compile( | |
| r"\b(" + "|".join(re.escape(w) for w in _INAPPROPRIATE_WORDS) + r")\b", | |
| re.IGNORECASE, | |
| ) | |
| _SAFE_REDIRECT = ( | |
| "Oops, let's keep talking about our lovely story! " | |
| "Can you tell me what your favourite part was?" | |
| ) | |
| def _is_inappropriate(text: str) -> bool: | |
| """True if the text trips the profanity/inappropriate filter.""" | |
| if not text: | |
| return False | |
| return bool(_INAPPROPRIATE_RE.search(text)) | |
| def _sanitize_reply(text: str) -> str: | |
| """Guard the LLM output: if it slips and includes a flagged word, replace it | |
| with a safe, friendly fallback so a child never hears it.""" | |
| if _is_inappropriate(text): | |
| print("[GUARDRAIL] LLM reply tripped filter β using safe fallback.", flush=True) | |
| return "Great job! Let's keep going with our story. What happened next?" | |
| return text | |
| # ============================================================ | |
| # SESSION LOGIC (pure functions, used by both UI and API) | |
| # ============================================================ | |
| def _start_session_logic(story_text: str, num_questions: int, student_name: str = "Student"): | |
| """Returns (bot_question, full_speech_text, state_dict).""" | |
| fixed_prompt = coach_core.fixed_prompt | |
| target_q = int(num_questions) | |
| messages = [ | |
| {"role": "system", "content": fixed_prompt}, | |
| { | |
| "role": "user", | |
| "content": ( | |
| f"The child's name is {student_name}. Warmly greet them by name and say you'll " | |
| "talk about a fun story together. Then ask ONE short, simple question about the " | |
| "story below. Ask only one question and keep it friendly.\n\n" | |
| "===== STORY =====\n" | |
| f"{story_text}\n" | |
| "===== END STORY =====" | |
| ) | |
| } | |
| ] | |
| bot_question = _generate_bot_reply(messages) | |
| messages.append({"role": "assistant", "content": bot_question}) | |
| greeting_text = f"Hi {student_name}! I am about to read a short story. Ready? Let's start." | |
| full_speech_text = greeting_text + " " + story_text + " " + bot_question | |
| state = { | |
| "messages": messages, | |
| "asked": 1, | |
| "num_q": target_q, | |
| } | |
| return bot_question, full_speech_text, state | |
| def _submit_answer_logic(state: dict, transcript: str): | |
| """Returns (reply_text, speech_text, new_state, is_complete).""" | |
| messages = state["messages"] | |
| current_q = state["asked"] | |
| max_q = state["num_q"] | |
| # Guardrail: if the child says something inappropriate, gently redirect WITHOUT | |
| # sending it to the LLM, and do not advance the question count or end the session. | |
| if _is_inappropriate(transcript): | |
| print("[GUARDRAIL] Inappropriate input β redirecting, not advancing.", flush=True) | |
| return _SAFE_REDIRECT, _SAFE_REDIRECT, state, False | |
| appreciate_only = (current_q >= max_q) | |
| if appreciate_only: | |
| user_content = ( | |
| f"My answer is: {transcript}.\n\n" | |
| "[END OF SESSION]\n" | |
| "This was the last question. Give ONE warm, cheerful sentence praising how well " | |
| "I did talking about the story, then say a friendly goodbye. " | |
| "Do not ask any more questions. Do not correct my grammar or English." | |
| ) | |
| else: | |
| user_content = transcript | |
| messages.append({"role": "user", "content": user_content}) | |
| reply_text = _sanitize_reply(_generate_bot_reply(messages)) | |
| messages.append({"role": "assistant", "content": reply_text}) | |
| state["messages"] = messages | |
| if not appreciate_only: | |
| state["asked"] += 1 | |
| return reply_text, reply_text, state, appreciate_only | |
| # ============================================================ | |
| # GRADIO UI WRAPPERS | |
| # Server voice (Sarvam/Kokoro) when TTS_ENGINE != "browser"; the returned | |
| # audio path feeds an autoplaying gr.Audio. In "browser" mode it is None and | |
| # the JS Web Speech API speaks instead. | |
| # ============================================================ | |
| def _ui_tts(text, speed, segments=None): | |
| if TTS_ENGINE == "browser" or not text: | |
| return None | |
| spd = float(speed) if speed else DEFAULT_TTS_SPEED | |
| return tts_to_wav(text, speed=spd, segments=segments, sarvam_pace=spd) | |
| def _start_session_ui(story_text, num_questions, speed=None): | |
| bot_question, full_speech, state = _start_session_logic(story_text, num_questions) | |
| target_q = state["num_q"] | |
| display_text = f"(Question 1/{target_q}): {bot_question}" | |
| audio = _ui_tts(full_speech, speed, segments=start_segments(full_speech, story_text)) | |
| return display_text, full_speech, json.dumps(state), audio | |
| def _submit_answer_ui(state_json, mic_tuple, speed=None): | |
| def _msg(display, speech, st): | |
| return display, speech, st, None, _ui_tts(speech, speed) | |
| if not state_json: | |
| return _msg("Click 'Start session' first.", "Click start session first.", state_json) | |
| if mic_tuple is None: | |
| return _msg("I couldn't hear that. Please try again.", "I couldn't hear that.", state_json) | |
| try: | |
| sr_in, data_in = mic_tuple | |
| buf = _numpy_to_wav_bytesio(sr_in, data_in) | |
| transcript = speech2text(buf) | |
| except Exception as e: | |
| return f"Audio error: {e}", "Audio error", state_json, None, None | |
| if not transcript: | |
| return _msg("I couldn't catch that. Please try again.", "I couldn't catch that.", state_json) | |
| state = json.loads(state_json) | |
| reply_text, speech_text, new_state, is_complete = _submit_answer_logic(state, transcript) | |
| if is_complete: | |
| display_text = f"(Session Complete): {reply_text}" | |
| else: | |
| display_text = f"(Question {new_state['asked']}/{new_state['num_q']}): {reply_text}" | |
| return display_text, speech_text, json.dumps(new_state), None, _ui_tts(speech_text, speed) | |
| def _replay_ui(last_audio): | |
| """Re-emit the last reply under a new filename so the audio player autoplays again.""" | |
| if not last_audio or not os.path.exists(str(last_audio)): | |
| return None | |
| new_path = os.path.join(TTS_DIR, f"replay_{uuid.uuid4().hex}.wav") | |
| try: | |
| shutil.copy(str(last_audio), new_path) | |
| return new_path | |
| except Exception: | |
| return last_audio | |
| # ============================================================ | |
| # WHATSAPP API HANDLER | |
| # Input: | |
| # audio_file: path to WAV file (bridge has already converted from OGG) | |
| # OR None / empty file for session start | |
| # state_json: JSON string of session state ({"messages":[], "asked":0, "num_q":3}) | |
| # Output dict: | |
| # { "bot_text": str, "audio_path": str (WAV on server disk), "state": str (json) } | |
| # Bridge will download the audio_path file via Gradio and convert WAV->OGG for WhatsApp. | |
| # ============================================================ | |
| def whatsapp_api_handler(audio_file, state_json): | |
| t0 = time.time() | |
| try: | |
| state = json.loads(state_json) if state_json else {"messages": [], "asked": 0, "num_q": 3} | |
| except Exception: | |
| state = {"messages": [], "asked": 0, "num_q": 3} | |
| asked = state.get("asked", 0) | |
| student_name = state.get("student_name", "Student") | |
| tts_speed = float(state.get("tts_speed", DEFAULT_TTS_SPEED)) | |
| story_index = state.get("story_index", None) # None β random | |
| # --- CASE 1: Start of session --- | |
| if asked == 0: | |
| # Use bridge-selected story or fall back to random | |
| if story_index is not None: | |
| try: | |
| idx = int(story_index) | |
| story_text = STORY_LIBRARY[idx] if 0 <= idx < len(STORY_LIBRARY) else _get_random_story() | |
| except (ValueError, TypeError): | |
| story_text = _get_random_story() | |
| else: | |
| story_text = _get_random_story() | |
| num_q = state.get("num_q", 3) | |
| bot_question, full_speech, new_state = _start_session_logic(story_text, num_q, student_name) | |
| new_state["story"] = story_text | |
| new_state["student_name"] = student_name | |
| new_state["tts_speed"] = tts_speed | |
| new_state["student_class"] = state.get("student_class", 0) | |
| # Split into greeting / story / question so the story audio can be cached | |
| wav_path = tts_to_wav(full_speech, speed=tts_speed, | |
| segments=start_segments(full_speech, story_text)) | |
| print(f"[WA API] start_session done in {time.time()-t0:.1f}s, wav={wav_path}", flush=True) | |
| return { | |
| "bot_text": full_speech, | |
| "audio_path": wav_path, | |
| "state": json.dumps(new_state), | |
| } | |
| # --- CASE 2: Incoming audio answer --- | |
| if not audio_file or not os.path.exists(str(audio_file)): | |
| err_text = "I did not receive your voice. Please send a voice note." | |
| return { | |
| "bot_text": err_text, | |
| "audio_path": tts_to_wav(err_text, speed=tts_speed), | |
| "state": json.dumps(state), | |
| } | |
| # Transcribe the incoming WAV | |
| try: | |
| with open(str(audio_file), "rb") as f: | |
| wav_buf = io.BytesIO(f.read()) | |
| transcript = speech2text(wav_buf) | |
| except Exception as e: | |
| print(f"[WA API] ASR error: {e}", flush=True) | |
| err_text = "Sorry, I couldn't hear you clearly. Please try again." | |
| return { | |
| "bot_text": err_text, | |
| "audio_path": tts_to_wav(err_text, speed=tts_speed), | |
| "state": json.dumps(state), | |
| } | |
| if not transcript or len(transcript.strip()) < 2: | |
| err_text = "I couldn't catch that. Please say it again a little louder." | |
| return { | |
| "bot_text": err_text, | |
| "audio_path": tts_to_wav(err_text, speed=tts_speed), | |
| "state": json.dumps(state), | |
| } | |
| print(f"[WA API] Transcript: {transcript}", flush=True) | |
| # Feed to LLM | |
| reply_text, speech_text, new_state, is_complete = _submit_answer_logic(state, transcript) | |
| # TTS | |
| wav_path = tts_to_wav(speech_text, speed=tts_speed) | |
| # If session ended, set session_complete so bridge shows the post-session menu | |
| if is_complete: | |
| new_state = { | |
| "messages": [], | |
| "asked": 0, | |
| "num_q": new_state.get("num_q", 3), | |
| "session_complete": True, | |
| "student_name": student_name, | |
| "student_class": state.get("student_class", 0), | |
| "tts_speed": tts_speed, | |
| } | |
| print(f"[WA API] reply done in {time.time()-t0:.1f}s, wav={wav_path}", flush=True) | |
| return { | |
| "bot_text": speech_text, | |
| "audio_path": wav_path, | |
| "state": json.dumps(new_state), | |
| } | |
| # ============================================================ | |
| # Browser TTS JS β only used by the web UI when TTS_ENGINE == "browser" | |
| # ============================================================ | |
| JS_TTS_AUTOSPEAK = r""" | |
| function (text, speed_val) { | |
| try { | |
| if (!text || !window.speechSynthesis) { return; } | |
| const u = new SpeechSynthesisUtterance(text); | |
| u.rate = speed_val ? parseFloat(speed_val) : 0.8; | |
| u.pitch = 1.0; | |
| u.lang = "en-US"; | |
| function speakNow() { | |
| const voices = speechSynthesis.getVoices(); | |
| let best = null; | |
| if (voices.length > 0) { | |
| best = voices.find(v => v.lang.startsWith("en") && v.name.toLowerCase().includes("female")) | |
| || voices.find(v => v.lang.startsWith("en")); | |
| } | |
| if (best) u.voice = best; | |
| speechSynthesis.cancel(); | |
| speechSynthesis.speak(u); | |
| } | |
| if (speechSynthesis.getVoices().length === 0) { | |
| speechSynthesis.addEventListener('voiceschanged', speakNow, { once: true }); | |
| } else { | |
| speakNow(); | |
| } | |
| } catch (e) { console.error(e); } | |
| } | |
| """ | |
| # ============================================================ | |
| # GRADIO APP | |
| # ============================================================ | |
| with gr.Blocks(title="English Coach") as demo: | |
| gr.Markdown("## π§ English Story Coach") | |
| with gr.Accordion("π Story & Settings", open=True): | |
| rand_btn = gr.Button("π² Pick Random Story", size="sm") | |
| with gr.Row(): | |
| story = gr.Textbox(label="Story", value=STORY_LIBRARY[0], lines=4) | |
| with gr.Row(): | |
| num_q = gr.Slider(1, 10, value=3, step=1, label="Number of Questions") | |
| speed_slider = gr.Slider( | |
| minimum=0.3, maximum=1.2, value=0.8, step=0.1, | |
| label="Voice Speed (Lower is Slower)" | |
| ) | |
| state = gr.State() | |
| hidden_speech_trigger = gr.Textbox(visible=False) | |
| start_btn = gr.Button("βΆοΈ Start Session", variant="primary") | |
| bot_display = gr.Textbox(label="Bot Question", interactive=False) | |
| use_server_voice = TTS_ENGINE != "browser" | |
| teacher_audio = gr.Audio( | |
| type="filepath", autoplay=True, interactive=False, | |
| label=f"Teacher's voice ({TTS_ENGINE})", visible=use_server_voice, | |
| ) | |
| replay_btn = gr.Button("π Replay Teacher's Voice", size="sm") | |
| mic = gr.Audio(sources=["microphone"], type="numpy", label="Record your answer") | |
| submit = gr.Button("Submit Answer") | |
| rand_btn.click(fn=_get_random_story, inputs=None, outputs=story) | |
| start_ev = start_btn.click( | |
| fn=_start_session_ui, | |
| inputs=[story, num_q, speed_slider], | |
| outputs=[bot_display, hidden_speech_trigger, state, teacher_audio] | |
| ) | |
| submit_ev = submit.click( | |
| fn=_submit_answer_ui, | |
| inputs=[state, mic, speed_slider], | |
| outputs=[bot_display, hidden_speech_trigger, state, mic, teacher_audio] | |
| ) | |
| if use_server_voice: | |
| # Sarvam / Kokoro: audio is generated on the server and autoplays | |
| replay_btn.click(fn=_replay_ui, inputs=[teacher_audio], outputs=[teacher_audio]) | |
| else: | |
| # "browser": the browser's built-in voice speaks the text | |
| start_ev.then(fn=None, inputs=[hidden_speech_trigger, speed_slider], js=JS_TTS_AUTOSPEAK) | |
| submit_ev.then(fn=None, inputs=[hidden_speech_trigger, speed_slider], js=JS_TTS_AUTOSPEAK) | |
| replay_btn.click(fn=None, inputs=[hidden_speech_trigger, speed_slider], js=JS_TTS_AUTOSPEAK) | |
| # ---- WhatsApp API endpoint (hidden from UI) ---- | |
| # Gradio exposes any function with api_name set as a callable via gradio_client. | |
| # ---- WhatsApp API endpoint (hidden from UI) ---- | |
| wa_audio_in = gr.Audio(type="filepath", visible=False, label="wa_audio_in") | |
| wa_state_in = gr.Textbox(visible=False, label="wa_state_in") | |
| wa_bot_text = gr.Textbox(visible=False, label="wa_bot_text") | |
| wa_audio_out = gr.Audio(type="filepath", visible=False, label="wa_audio_out") | |
| wa_state_out = gr.Textbox(visible=False, label="wa_state_out") | |
| wa_trigger = gr.Button(visible=False) | |
| def _whatsapp_api_wrapper(audio_file, state_json): | |
| result = whatsapp_api_handler(audio_file, state_json) | |
| return ( | |
| result.get("bot_text", ""), | |
| result.get("audio_path", None), | |
| result.get("state", ""), | |
| ) | |
| wa_trigger.click( | |
| fn=_whatsapp_api_wrapper, | |
| inputs=[wa_audio_in, wa_state_in], | |
| outputs=[wa_bot_text, wa_audio_out, wa_state_out], | |
| api_name="whatsapp", | |
| ) | |
| if __name__ == "__main__": | |
| demo.queue(max_size=32).launch(server_name="0.0.0.0", server_port=7860, show_api=False, show_error=True) | |