Update app.py

app.py

@@ -27,6 +27,9 @@ FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true"
 for d in [VIDEOS_DIR, TTS_DIR, LIPSYNC_DIR]:
     d.mkdir(parents=True, exist_ok=True)
 
+import threading
+_float_lock = threading.Lock()
+
 # In-memory store for chunked generation results
 # session_id -> { "chunks": [...], "done": bool }
 _sessions = {}
@@ -218,8 +221,7 @@ async def chat(request: Request):
 async def chat_next(request: Request):
     """
     Generate TTS + lipsync for the next sentence in the session.
-    Called sequentially by the frontend — one chunk at a time.
-    Returns the audio URL, lipsync video URL, duration, and sentence text.
+    Heavy work runs in a thread so the server stays responsive for prefetch requests.
     """
     body = await request.json()
     session_id = body.get("session_id", "")
@@ -236,8 +238,34 @@ async def chat_next(request: Request):
     sentence = session["sentences"][idx]
     session["next_index"] = idx + 1
     is_last = (idx + 1 >= session["total"])
+    total = session["total"]
+
+    # Run the heavy TTS + FLOAT work in a thread
+    import asyncio
+    result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
+
+    if is_last:
+        session["done"] = True
+        if len(_sessions) > 10:
+            oldest = sorted(_sessions.keys(), key=lambda k: _sessions[k]["created"])
+            for k in oldest[:-10]:
+                del _sessions[k]
+
+    return JSONResponse({
+        "done": False,
+        "chunk_index": idx,
+        "total_chunks": total,
+        "is_last": is_last,
+        "sentence": sentence,
+        "audio_url": result["audio_url"],
+        "audio_duration": result["audio_duration"],
+        "lipsync_video_url": result["lipsync_video_url"],
+    })
 
-    logger.info(f"[{_ts()}] [CHUNK {idx+1}/{session['total']}] {sentence[:60]}...")
+
+def _generate_chunk(sentence, idx, total):
+    """Synchronous TTS + FLOAT generation (runs in thread pool)."""
+    logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] {sentence[:60]}...")
     t0 = _time.time()
 
     # 1. TTS
@@ -258,16 +286,17 @@ async def chat_next(request: Request):
         except Exception:
             audio_duration = 3.0
 
-    # 2. Lipsync
+    # 2. Lipsync (locked — GPU not thread-safe)
     lipsync_video_url = None
     if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
         try:
             from float_lipsync import get_lipsync
             lipsync = get_lipsync()
             if lipsync.ready:
-                t3 = _time.time()
-                lp = lipsync.generate(audio_path)
-                t4 = _time.time()
+                with _float_lock:
+                    t3 = _time.time()
+                    lp = lipsync.generate(audio_path)
+                    t4 = _time.time()
                 logger.info(f"[{_ts()}] [TIMING] Chunk {idx+1} FLOAT: {t4-t3:.2f}s")
                 if lp and os.path.exists(lp):
                     lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
@@ -277,24 +306,11 @@ async def chat_next(request: Request):
     total_time = _time.time() - t0
     logger.info(f"[{_ts()}] [TIMING] Chunk {idx+1} TOTAL: {total_time:.2f}s")
 
-    if is_last:
-        session["done"] = True
-        # Clean up old sessions (keep last 10)
-        if len(_sessions) > 10:
-            oldest = sorted(_sessions.keys(), key=lambda k: _sessions[k]["created"])
-            for k in oldest[:-10]:
-                del _sessions[k]
-
-    return JSONResponse({
-        "done": False,
-        "chunk_index": idx,
-        "total_chunks": session["total"],
-        "is_last": is_last,
-        "sentence": sentence,
+    return {
         "audio_url": audio_url,
         "audio_duration": audio_duration,
         "lipsync_video_url": lipsync_video_url,
-    })
+    }
 
 
 @app.post("/api/reset")