J Z commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -61,16 +61,21 @@ async def generate_greeting():
|
|
| 61 |
|
| 62 |
try:
|
| 63 |
from tts import generate_audio
|
|
|
|
| 64 |
audio_path = generate_audio(greeting_text)
|
|
|
|
|
|
|
| 65 |
audio_url = None
|
| 66 |
audio_duration = 0
|
| 67 |
|
| 68 |
if audio_path and os.path.exists(audio_path):
|
| 69 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
|
|
|
| 70 |
try:
|
| 71 |
r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
|
| 72 |
"-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
| 73 |
audio_duration = float(r.stdout.strip())
|
|
|
|
| 74 |
except Exception:
|
| 75 |
audio_duration = 5.0
|
| 76 |
|
|
@@ -80,9 +85,13 @@ async def generate_greeting():
|
|
| 80 |
from float_lipsync import get_lipsync
|
| 81 |
lipsync = get_lipsync()
|
| 82 |
if lipsync.ready:
|
|
|
|
| 83 |
lp = lipsync.generate(audio_path)
|
|
|
|
|
|
|
| 84 |
if lp and os.path.exists(lp):
|
| 85 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
|
|
|
| 86 |
except Exception as e:
|
| 87 |
logger.warning(f"[GREETING] FLOAT failed, greeting will be text-only: {e}")
|
| 88 |
|
|
@@ -275,6 +284,7 @@ async def chat_next(request: Request):
|
|
| 275 |
Generate TTS + lipsync for the next sentence in the session.
|
| 276 |
Heavy work runs in a thread so the server stays responsive for prefetch requests.
|
| 277 |
"""
|
|
|
|
| 278 |
body = await request.json()
|
| 279 |
session_id = body.get("session_id", "")
|
| 280 |
session = _sessions.get(session_id)
|
|
@@ -292,10 +302,15 @@ async def chat_next(request: Request):
|
|
| 292 |
is_last = (idx + 1 >= session["total"])
|
| 293 |
total = session["total"]
|
| 294 |
|
|
|
|
|
|
|
| 295 |
# Run the heavy TTS + FLOAT work in a thread
|
| 296 |
import asyncio
|
| 297 |
result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
|
| 298 |
|
|
|
|
|
|
|
|
|
|
| 299 |
if is_last:
|
| 300 |
session["done"] = True
|
| 301 |
if len(_sessions) > 10:
|
|
@@ -317,26 +332,33 @@ async def chat_next(request: Request):
|
|
| 317 |
|
| 318 |
def _generate_chunk(sentence, idx, total):
|
| 319 |
"""Synchronous TTS + FLOAT generation (runs in thread pool)."""
|
| 320 |
-
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] {sentence[:
|
| 321 |
t0 = _time.time()
|
| 322 |
|
| 323 |
-
# 1. TTS
|
| 324 |
from tts import generate_audio
|
| 325 |
-
|
| 326 |
audio_path = generate_audio(sentence)
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
audio_url = None
|
| 331 |
audio_duration = 0
|
|
|
|
| 332 |
if audio_path and os.path.exists(audio_path):
|
|
|
|
| 333 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
|
|
|
| 334 |
try:
|
| 335 |
r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
|
| 336 |
"-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
| 337 |
audio_duration = float(r.stdout.strip())
|
| 338 |
except Exception:
|
| 339 |
audio_duration = 3.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
|
| 341 |
# 2. Lipsync (locked — GPU not thread-safe)
|
| 342 |
lipsync_video_url = None
|
|
@@ -345,18 +367,25 @@ def _generate_chunk(sentence, idx, total):
|
|
| 345 |
from float_lipsync import get_lipsync
|
| 346 |
lipsync = get_lipsync()
|
| 347 |
if lipsync.ready:
|
|
|
|
| 348 |
with _float_lock:
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
| 350 |
lp = lipsync.generate(audio_path)
|
| 351 |
-
|
| 352 |
-
|
|
|
|
| 353 |
if lp and os.path.exists(lp):
|
|
|
|
| 354 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
|
|
|
| 355 |
except Exception as e:
|
| 356 |
-
logger.error(f"[{_ts()}] [CHUNK]
|
| 357 |
|
| 358 |
total_time = _time.time() - t0
|
| 359 |
-
logger.info(f"[{_ts()}] [
|
| 360 |
|
| 361 |
return {
|
| 362 |
"audio_url": audio_url,
|
|
@@ -590,14 +619,45 @@ async function sendMessage(){
|
|
| 590 |
if(!res.ok) throw new Error('HTTP '+res.status);
|
| 591 |
const data=await res.json();
|
| 592 |
|
| 593 |
-
//
|
| 594 |
typing.bubbleEl.classList.remove('typing');
|
| 595 |
typing.contentEl.classList.remove('typing-dots');
|
| 596 |
-
typing.contentEl.textContent=
|
| 597 |
-
|
| 598 |
|
| 599 |
const sessionId=data.session_id;
|
| 600 |
const totalChunks=data.total_chunks;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 601 |
|
| 602 |
videoStatus.textContent='generating 1/'+totalChunks+'...';
|
| 603 |
|
|
@@ -644,6 +704,10 @@ async function sendMessage(){
|
|
| 644 |
});
|
| 645 |
}
|
| 646 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 647 |
// Play lipsync video
|
| 648 |
if(chunk.lipsync_video_url){
|
| 649 |
await crossfadeTo(chunk.lipsync_video_url, true);
|
|
@@ -653,8 +717,14 @@ async function sendMessage(){
|
|
| 653 |
// No lipsync, just wait for audio to finish
|
| 654 |
await audioPromise;
|
| 655 |
}
|
|
|
|
|
|
|
|
|
|
| 656 |
}
|
| 657 |
|
|
|
|
|
|
|
|
|
|
| 658 |
// All chunks done, return to idle
|
| 659 |
isSpeaking=false;
|
| 660 |
playNextIdle();
|
|
@@ -663,6 +733,7 @@ async function sendMessage(){
|
|
| 663 |
typing.bubbleEl.classList.remove('typing');
|
| 664 |
typing.bubbleEl.classList.add('error');
|
| 665 |
typing.contentEl.classList.remove('typing-dots');
|
|
|
|
| 666 |
typing.contentEl.textContent='Error: '+err.message;
|
| 667 |
}
|
| 668 |
isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus();
|
|
|
|
| 61 |
|
| 62 |
try:
|
| 63 |
from tts import generate_audio
|
| 64 |
+
t_tts = _time.time()
|
| 65 |
audio_path = generate_audio(greeting_text)
|
| 66 |
+
t_tts_done = _time.time()
|
| 67 |
+
logger.info(f"[{_ts()}] [GREETING] TTS: {t_tts_done-t_tts:.2f}s")
|
| 68 |
audio_url = None
|
| 69 |
audio_duration = 0
|
| 70 |
|
| 71 |
if audio_path and os.path.exists(audio_path):
|
| 72 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
| 73 |
+
logger.info(f"[{_ts()}] [GREETING] Audio: {os.path.getsize(audio_path)/1024:.0f}KB")
|
| 74 |
try:
|
| 75 |
r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
|
| 76 |
"-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
| 77 |
audio_duration = float(r.stdout.strip())
|
| 78 |
+
logger.info(f"[{_ts()}] [GREETING] Audio duration: {audio_duration:.1f}s")
|
| 79 |
except Exception:
|
| 80 |
audio_duration = 5.0
|
| 81 |
|
|
|
|
| 85 |
from float_lipsync import get_lipsync
|
| 86 |
lipsync = get_lipsync()
|
| 87 |
if lipsync.ready:
|
| 88 |
+
t_float = _time.time()
|
| 89 |
lp = lipsync.generate(audio_path)
|
| 90 |
+
t_float_done = _time.time()
|
| 91 |
+
logger.info(f"[{_ts()}] [GREETING] FLOAT: {t_float_done-t_float:.2f}s")
|
| 92 |
if lp and os.path.exists(lp):
|
| 93 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
| 94 |
+
logger.info(f"[{_ts()}] [GREETING] Video: {os.path.getsize(lp)/1024:.0f}KB")
|
| 95 |
except Exception as e:
|
| 96 |
logger.warning(f"[GREETING] FLOAT failed, greeting will be text-only: {e}")
|
| 97 |
|
|
|
|
| 284 |
Generate TTS + lipsync for the next sentence in the session.
|
| 285 |
Heavy work runs in a thread so the server stays responsive for prefetch requests.
|
| 286 |
"""
|
| 287 |
+
t_req = _time.time()
|
| 288 |
body = await request.json()
|
| 289 |
session_id = body.get("session_id", "")
|
| 290 |
session = _sessions.get(session_id)
|
|
|
|
| 302 |
is_last = (idx + 1 >= session["total"])
|
| 303 |
total = session["total"]
|
| 304 |
|
| 305 |
+
logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} received")
|
| 306 |
+
|
| 307 |
# Run the heavy TTS + FLOAT work in a thread
|
| 308 |
import asyncio
|
| 309 |
result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
|
| 310 |
|
| 311 |
+
t_resp = _time.time()
|
| 312 |
+
logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} responding | request-to-response: {t_resp-t_req:.2f}s")
|
| 313 |
+
|
| 314 |
if is_last:
|
| 315 |
session["done"] = True
|
| 316 |
if len(_sessions) > 10:
|
|
|
|
| 332 |
|
| 333 |
def _generate_chunk(sentence, idx, total):
|
| 334 |
"""Synchronous TTS + FLOAT generation (runs in thread pool)."""
|
| 335 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] START | \"{sentence[:80]}\"")
|
| 336 |
t0 = _time.time()
|
| 337 |
|
| 338 |
+
# 1. TTS API call
|
| 339 |
from tts import generate_audio
|
| 340 |
+
t_tts_start = _time.time()
|
| 341 |
audio_path = generate_audio(sentence)
|
| 342 |
+
t_tts_end = _time.time()
|
| 343 |
+
|
|
|
|
| 344 |
audio_url = None
|
| 345 |
audio_duration = 0
|
| 346 |
+
audio_size = 0
|
| 347 |
if audio_path and os.path.exists(audio_path):
|
| 348 |
+
audio_size = os.path.getsize(audio_path)
|
| 349 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
| 350 |
+
t_probe_start = _time.time()
|
| 351 |
try:
|
| 352 |
r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
|
| 353 |
"-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
| 354 |
audio_duration = float(r.stdout.strip())
|
| 355 |
except Exception:
|
| 356 |
audio_duration = 3.0
|
| 357 |
+
t_probe_end = _time.time()
|
| 358 |
+
else:
|
| 359 |
+
t_probe_start = t_probe_end = _time.time()
|
| 360 |
+
|
| 361 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] TTS: {t_tts_end-t_tts_start:.2f}s | audio: {audio_duration:.1f}s, {audio_size/1024:.0f}KB | ffprobe: {t_probe_end-t_probe_start:.3f}s")
|
| 362 |
|
| 363 |
# 2. Lipsync (locked — GPU not thread-safe)
|
| 364 |
lipsync_video_url = None
|
|
|
|
| 367 |
from float_lipsync import get_lipsync
|
| 368 |
lipsync = get_lipsync()
|
| 369 |
if lipsync.ready:
|
| 370 |
+
t_lock_wait = _time.time()
|
| 371 |
with _float_lock:
|
| 372 |
+
t_lock_acquired = _time.time()
|
| 373 |
+
lock_wait = t_lock_acquired - t_lock_wait
|
| 374 |
+
if lock_wait > 0.01:
|
| 375 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] LOCK WAIT: {lock_wait:.2f}s")
|
| 376 |
lp = lipsync.generate(audio_path)
|
| 377 |
+
t_float_done = _time.time()
|
| 378 |
+
float_time = t_float_done - t_lock_acquired
|
| 379 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] FLOAT: {float_time:.2f}s (lock wait: {lock_wait:.2f}s)")
|
| 380 |
if lp and os.path.exists(lp):
|
| 381 |
+
lp_size = os.path.getsize(lp)
|
| 382 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
| 383 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] VIDEO: {lp_size/1024:.0f}KB")
|
| 384 |
except Exception as e:
|
| 385 |
+
logger.error(f"[{_ts()}] [CHUNK {idx+1}/{total}] LIPSYNC FAILED: {e}", exc_info=True)
|
| 386 |
|
| 387 |
total_time = _time.time() - t0
|
| 388 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] DONE | total: {total_time:.2f}s | tts: {t_tts_end-t_tts_start:.2f}s + float: {total_time-(t_tts_end-t_tts_start):.2f}s")
|
| 389 |
|
| 390 |
return {
|
| 391 |
"audio_url": audio_url,
|
|
|
|
| 619 |
if(!res.ok) throw new Error('HTTP '+res.status);
|
| 620 |
const data=await res.json();
|
| 621 |
|
| 622 |
+
// Replace typing indicator with empty bot bubble, ready for streaming
|
| 623 |
typing.bubbleEl.classList.remove('typing');
|
| 624 |
typing.contentEl.classList.remove('typing-dots');
|
| 625 |
+
typing.contentEl.textContent='';
|
| 626 |
+
typing.contentEl.classList.add('streaming-cursor');
|
| 627 |
|
| 628 |
const sessionId=data.session_id;
|
| 629 |
const totalChunks=data.total_chunks;
|
| 630 |
+
const fullText=data.text;
|
| 631 |
+
const words=fullText.split(/\s+/);
|
| 632 |
+
let wordsShown=0;
|
| 633 |
+
|
| 634 |
+
// Text streaming: reveal words over the given duration
|
| 635 |
+
function streamText(durationSec){
|
| 636 |
+
const wordsToShow=words.length - wordsShown;
|
| 637 |
+
if(wordsToShow<=0) return Promise.resolve();
|
| 638 |
+
const interval=Math.max(50, (durationSec*1000)/wordsToShow);
|
| 639 |
+
return new Promise((resolve)=>{
|
| 640 |
+
let i=0;
|
| 641 |
+
const timer=setInterval(()=>{
|
| 642 |
+
if(i>=wordsToShow){
|
| 643 |
+
clearInterval(timer);
|
| 644 |
+
resolve();
|
| 645 |
+
return;
|
| 646 |
+
}
|
| 647 |
+
wordsShown++;
|
| 648 |
+
typing.contentEl.textContent=words.slice(0,wordsShown).join(' ');
|
| 649 |
+
chatLog.scrollTop=chatLog.scrollHeight;
|
| 650 |
+
i++;
|
| 651 |
+
}, interval);
|
| 652 |
+
});
|
| 653 |
+
}
|
| 654 |
+
|
| 655 |
+
// Show all remaining text instantly (cleanup)
|
| 656 |
+
function showAllText(){
|
| 657 |
+
typing.contentEl.textContent=fullText;
|
| 658 |
+
typing.contentEl.classList.remove('streaming-cursor');
|
| 659 |
+
chatLog.scrollTop=chatLog.scrollHeight;
|
| 660 |
+
}
|
| 661 |
|
| 662 |
videoStatus.textContent='generating 1/'+totalChunks+'...';
|
| 663 |
|
|
|
|
| 704 |
});
|
| 705 |
}
|
| 706 |
|
| 707 |
+
// Stream text for this chunk's duration
|
| 708 |
+
const chunkDuration=chunk.audio_duration||3.0;
|
| 709 |
+
const textStreamPromise=streamText(chunkDuration);
|
| 710 |
+
|
| 711 |
// Play lipsync video
|
| 712 |
if(chunk.lipsync_video_url){
|
| 713 |
await crossfadeTo(chunk.lipsync_video_url, true);
|
|
|
|
| 717 |
// No lipsync, just wait for audio to finish
|
| 718 |
await audioPromise;
|
| 719 |
}
|
| 720 |
+
|
| 721 |
+
// Ensure text streaming for this chunk is done
|
| 722 |
+
await textStreamPromise;
|
| 723 |
}
|
| 724 |
|
| 725 |
+
// Ensure all text is shown
|
| 726 |
+
showAllText();
|
| 727 |
+
|
| 728 |
// All chunks done, return to idle
|
| 729 |
isSpeaking=false;
|
| 730 |
playNextIdle();
|
|
|
|
| 733 |
typing.bubbleEl.classList.remove('typing');
|
| 734 |
typing.bubbleEl.classList.add('error');
|
| 735 |
typing.contentEl.classList.remove('typing-dots');
|
| 736 |
+
typing.contentEl.classList.remove('streaming-cursor');
|
| 737 |
typing.contentEl.textContent='Error: '+err.message;
|
| 738 |
}
|
| 739 |
isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus();
|