J Z commited on
Commit
a8a2d90
·
verified ·
1 Parent(s): 0ac0147

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +85 -14
app.py CHANGED
@@ -61,16 +61,21 @@ async def generate_greeting():
61
 
62
  try:
63
  from tts import generate_audio
 
64
  audio_path = generate_audio(greeting_text)
 
 
65
  audio_url = None
66
  audio_duration = 0
67
 
68
  if audio_path and os.path.exists(audio_path):
69
  audio_url = f"/api/audio/{os.path.basename(audio_path)}"
 
70
  try:
71
  r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
72
  "-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
73
  audio_duration = float(r.stdout.strip())
 
74
  except Exception:
75
  audio_duration = 5.0
76
 
@@ -80,9 +85,13 @@ async def generate_greeting():
80
  from float_lipsync import get_lipsync
81
  lipsync = get_lipsync()
82
  if lipsync.ready:
 
83
  lp = lipsync.generate(audio_path)
 
 
84
  if lp and os.path.exists(lp):
85
  lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
 
86
  except Exception as e:
87
  logger.warning(f"[GREETING] FLOAT failed, greeting will be text-only: {e}")
88
 
@@ -275,6 +284,7 @@ async def chat_next(request: Request):
275
  Generate TTS + lipsync for the next sentence in the session.
276
  Heavy work runs in a thread so the server stays responsive for prefetch requests.
277
  """
 
278
  body = await request.json()
279
  session_id = body.get("session_id", "")
280
  session = _sessions.get(session_id)
@@ -292,10 +302,15 @@ async def chat_next(request: Request):
292
  is_last = (idx + 1 >= session["total"])
293
  total = session["total"]
294
 
 
 
295
  # Run the heavy TTS + FLOAT work in a thread
296
  import asyncio
297
  result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
298
 
 
 
 
299
  if is_last:
300
  session["done"] = True
301
  if len(_sessions) > 10:
@@ -317,26 +332,33 @@ async def chat_next(request: Request):
317
 
318
  def _generate_chunk(sentence, idx, total):
319
  """Synchronous TTS + FLOAT generation (runs in thread pool)."""
320
- logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] {sentence[:60]}...")
321
  t0 = _time.time()
322
 
323
- # 1. TTS
324
  from tts import generate_audio
325
- t1 = _time.time()
326
  audio_path = generate_audio(sentence)
327
- t2 = _time.time()
328
- logger.info(f"[{_ts()}] [TIMING] Chunk {idx+1} TTS: {t2-t1:.2f}s")
329
-
330
  audio_url = None
331
  audio_duration = 0
 
332
  if audio_path and os.path.exists(audio_path):
 
333
  audio_url = f"/api/audio/{os.path.basename(audio_path)}"
 
334
  try:
335
  r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
336
  "-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
337
  audio_duration = float(r.stdout.strip())
338
  except Exception:
339
  audio_duration = 3.0
 
 
 
 
 
340
 
341
  # 2. Lipsync (locked — GPU not thread-safe)
342
  lipsync_video_url = None
@@ -345,18 +367,25 @@ def _generate_chunk(sentence, idx, total):
345
  from float_lipsync import get_lipsync
346
  lipsync = get_lipsync()
347
  if lipsync.ready:
 
348
  with _float_lock:
349
- t3 = _time.time()
 
 
 
350
  lp = lipsync.generate(audio_path)
351
- t4 = _time.time()
352
- logger.info(f"[{_ts()}] [TIMING] Chunk {idx+1} FLOAT: {t4-t3:.2f}s")
 
353
  if lp and os.path.exists(lp):
 
354
  lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
 
355
  except Exception as e:
356
- logger.error(f"[{_ts()}] [CHUNK] Lipsync failed: {e}", exc_info=True)
357
 
358
  total_time = _time.time() - t0
359
- logger.info(f"[{_ts()}] [TIMING] Chunk {idx+1} TOTAL: {total_time:.2f}s")
360
 
361
  return {
362
  "audio_url": audio_url,
@@ -590,14 +619,45 @@ async function sendMessage(){
590
  if(!res.ok) throw new Error('HTTP '+res.status);
591
  const data=await res.json();
592
 
593
- // Show full text immediately
594
  typing.bubbleEl.classList.remove('typing');
595
  typing.contentEl.classList.remove('typing-dots');
596
- typing.contentEl.textContent=data.text;
597
- chatLog.scrollTop=chatLog.scrollHeight;
598
 
599
  const sessionId=data.session_id;
600
  const totalChunks=data.total_chunks;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
601
 
602
  videoStatus.textContent='generating 1/'+totalChunks+'...';
603
 
@@ -644,6 +704,10 @@ async function sendMessage(){
644
  });
645
  }
646
 
 
 
 
 
647
  // Play lipsync video
648
  if(chunk.lipsync_video_url){
649
  await crossfadeTo(chunk.lipsync_video_url, true);
@@ -653,8 +717,14 @@ async function sendMessage(){
653
  // No lipsync, just wait for audio to finish
654
  await audioPromise;
655
  }
 
 
 
656
  }
657
 
 
 
 
658
  // All chunks done, return to idle
659
  isSpeaking=false;
660
  playNextIdle();
@@ -663,6 +733,7 @@ async function sendMessage(){
663
  typing.bubbleEl.classList.remove('typing');
664
  typing.bubbleEl.classList.add('error');
665
  typing.contentEl.classList.remove('typing-dots');
 
666
  typing.contentEl.textContent='Error: '+err.message;
667
  }
668
  isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus();
 
61
 
62
  try:
63
  from tts import generate_audio
64
+ t_tts = _time.time()
65
  audio_path = generate_audio(greeting_text)
66
+ t_tts_done = _time.time()
67
+ logger.info(f"[{_ts()}] [GREETING] TTS: {t_tts_done-t_tts:.2f}s")
68
  audio_url = None
69
  audio_duration = 0
70
 
71
  if audio_path and os.path.exists(audio_path):
72
  audio_url = f"/api/audio/{os.path.basename(audio_path)}"
73
+ logger.info(f"[{_ts()}] [GREETING] Audio: {os.path.getsize(audio_path)/1024:.0f}KB")
74
  try:
75
  r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
76
  "-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
77
  audio_duration = float(r.stdout.strip())
78
+ logger.info(f"[{_ts()}] [GREETING] Audio duration: {audio_duration:.1f}s")
79
  except Exception:
80
  audio_duration = 5.0
81
 
 
85
  from float_lipsync import get_lipsync
86
  lipsync = get_lipsync()
87
  if lipsync.ready:
88
+ t_float = _time.time()
89
  lp = lipsync.generate(audio_path)
90
+ t_float_done = _time.time()
91
+ logger.info(f"[{_ts()}] [GREETING] FLOAT: {t_float_done-t_float:.2f}s")
92
  if lp and os.path.exists(lp):
93
  lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
94
+ logger.info(f"[{_ts()}] [GREETING] Video: {os.path.getsize(lp)/1024:.0f}KB")
95
  except Exception as e:
96
  logger.warning(f"[GREETING] FLOAT failed, greeting will be text-only: {e}")
97
 
 
284
  Generate TTS + lipsync for the next sentence in the session.
285
  Heavy work runs in a thread so the server stays responsive for prefetch requests.
286
  """
287
+ t_req = _time.time()
288
  body = await request.json()
289
  session_id = body.get("session_id", "")
290
  session = _sessions.get(session_id)
 
302
  is_last = (idx + 1 >= session["total"])
303
  total = session["total"]
304
 
305
+ logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} received")
306
+
307
  # Run the heavy TTS + FLOAT work in a thread
308
  import asyncio
309
  result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
310
 
311
+ t_resp = _time.time()
312
+ logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} responding | request-to-response: {t_resp-t_req:.2f}s")
313
+
314
  if is_last:
315
  session["done"] = True
316
  if len(_sessions) > 10:
 
332
 
333
  def _generate_chunk(sentence, idx, total):
334
  """Synchronous TTS + FLOAT generation (runs in thread pool)."""
335
+ logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] START | \"{sentence[:80]}\"")
336
  t0 = _time.time()
337
 
338
+ # 1. TTS API call
339
  from tts import generate_audio
340
+ t_tts_start = _time.time()
341
  audio_path = generate_audio(sentence)
342
+ t_tts_end = _time.time()
343
+
 
344
  audio_url = None
345
  audio_duration = 0
346
+ audio_size = 0
347
  if audio_path and os.path.exists(audio_path):
348
+ audio_size = os.path.getsize(audio_path)
349
  audio_url = f"/api/audio/{os.path.basename(audio_path)}"
350
+ t_probe_start = _time.time()
351
  try:
352
  r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
353
  "-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
354
  audio_duration = float(r.stdout.strip())
355
  except Exception:
356
  audio_duration = 3.0
357
+ t_probe_end = _time.time()
358
+ else:
359
+ t_probe_start = t_probe_end = _time.time()
360
+
361
+ logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] TTS: {t_tts_end-t_tts_start:.2f}s | audio: {audio_duration:.1f}s, {audio_size/1024:.0f}KB | ffprobe: {t_probe_end-t_probe_start:.3f}s")
362
 
363
  # 2. Lipsync (locked — GPU not thread-safe)
364
  lipsync_video_url = None
 
367
  from float_lipsync import get_lipsync
368
  lipsync = get_lipsync()
369
  if lipsync.ready:
370
+ t_lock_wait = _time.time()
371
  with _float_lock:
372
+ t_lock_acquired = _time.time()
373
+ lock_wait = t_lock_acquired - t_lock_wait
374
+ if lock_wait > 0.01:
375
+ logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] LOCK WAIT: {lock_wait:.2f}s")
376
  lp = lipsync.generate(audio_path)
377
+ t_float_done = _time.time()
378
+ float_time = t_float_done - t_lock_acquired
379
+ logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] FLOAT: {float_time:.2f}s (lock wait: {lock_wait:.2f}s)")
380
  if lp and os.path.exists(lp):
381
+ lp_size = os.path.getsize(lp)
382
  lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
383
+ logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] VIDEO: {lp_size/1024:.0f}KB")
384
  except Exception as e:
385
+ logger.error(f"[{_ts()}] [CHUNK {idx+1}/{total}] LIPSYNC FAILED: {e}", exc_info=True)
386
 
387
  total_time = _time.time() - t0
388
+ logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] DONE | total: {total_time:.2f}s | tts: {t_tts_end-t_tts_start:.2f}s + float: {total_time-(t_tts_end-t_tts_start):.2f}s")
389
 
390
  return {
391
  "audio_url": audio_url,
 
619
  if(!res.ok) throw new Error('HTTP '+res.status);
620
  const data=await res.json();
621
 
622
+ // Replace typing indicator with empty bot bubble, ready for streaming
623
  typing.bubbleEl.classList.remove('typing');
624
  typing.contentEl.classList.remove('typing-dots');
625
+ typing.contentEl.textContent='';
626
+ typing.contentEl.classList.add('streaming-cursor');
627
 
628
  const sessionId=data.session_id;
629
  const totalChunks=data.total_chunks;
630
+ const fullText=data.text;
631
+ const words=fullText.split(/\s+/);
632
+ let wordsShown=0;
633
+
634
+ // Text streaming: reveal words over the given duration
635
+ function streamText(durationSec){
636
+ const wordsToShow=words.length - wordsShown;
637
+ if(wordsToShow<=0) return Promise.resolve();
638
+ const interval=Math.max(50, (durationSec*1000)/wordsToShow);
639
+ return new Promise((resolve)=>{
640
+ let i=0;
641
+ const timer=setInterval(()=>{
642
+ if(i>=wordsToShow){
643
+ clearInterval(timer);
644
+ resolve();
645
+ return;
646
+ }
647
+ wordsShown++;
648
+ typing.contentEl.textContent=words.slice(0,wordsShown).join(' ');
649
+ chatLog.scrollTop=chatLog.scrollHeight;
650
+ i++;
651
+ }, interval);
652
+ });
653
+ }
654
+
655
+ // Show all remaining text instantly (cleanup)
656
+ function showAllText(){
657
+ typing.contentEl.textContent=fullText;
658
+ typing.contentEl.classList.remove('streaming-cursor');
659
+ chatLog.scrollTop=chatLog.scrollHeight;
660
+ }
661
 
662
  videoStatus.textContent='generating 1/'+totalChunks+'...';
663
 
 
704
  });
705
  }
706
 
707
+ // Stream text for this chunk's duration
708
+ const chunkDuration=chunk.audio_duration||3.0;
709
+ const textStreamPromise=streamText(chunkDuration);
710
+
711
  // Play lipsync video
712
  if(chunk.lipsync_video_url){
713
  await crossfadeTo(chunk.lipsync_video_url, true);
 
717
  // No lipsync, just wait for audio to finish
718
  await audioPromise;
719
  }
720
+
721
+ // Ensure text streaming for this chunk is done
722
+ await textStreamPromise;
723
  }
724
 
725
+ // Ensure all text is shown
726
+ showAllText();
727
+
728
  // All chunks done, return to idle
729
  isSpeaking=false;
730
  playNextIdle();
 
733
  typing.bubbleEl.classList.remove('typing');
734
  typing.bubbleEl.classList.add('error');
735
  typing.contentEl.classList.remove('typing-dots');
736
+ typing.contentEl.classList.remove('streaming-cursor');
737
  typing.contentEl.textContent='Error: '+err.message;
738
  }
739
  isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus();