Update app.py

app.py

@@ -35,11 +35,64 @@ _float_lock = threading.Lock()
 _sessions = {}
 
 
+GREETING_DATA = None  # Will hold {audio_url, lipsync_video_url, text} after startup
+
+
 @app.on_event("startup")
 async def startup():
+    global GREETING_DATA
     await download_videos()
     if FLOAT_ENABLED:
         await init_float()
+    # Generate a greeting to warm up FLOAT and play on first load
+    await generate_greeting()
+
+
+async def generate_greeting():
+    """Generate TTS + lipsync for a greeting message to warm up the model."""
+    global GREETING_DATA
+    import asyncio
+
+    greeting_text = "Good day! I am Charles Darwin, er, recreated in this curious digital form. Do feel free to ask me anything, umm, about the natural world."
+
+    logger.info(f"[{_ts()}] [GREETING] Generating warmup greeting...")
+    t0 = _time.time()
+
+    try:
+        from tts import generate_audio
+        audio_path = generate_audio(greeting_text)
+        audio_url = None
+        audio_duration = 0
+
+        if audio_path and os.path.exists(audio_path):
+            audio_url = f"/api/audio/{os.path.basename(audio_path)}"
+            try:
+                r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
+                                    "-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
+                audio_duration = float(r.stdout.strip())
+            except Exception:
+                audio_duration = 5.0
+
+        lipsync_video_url = None
+        if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
+            from float_lipsync import get_lipsync
+            lipsync = get_lipsync()
+            if lipsync.ready:
+                lp = lipsync.generate(audio_path)
+                if lp and os.path.exists(lp):
+                    lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
+
+        GREETING_DATA = {
+            "text": greeting_text,
+            "audio_url": audio_url,
+            "audio_duration": audio_duration,
+            "lipsync_video_url": lipsync_video_url,
+        }
+        logger.info(f"[{_ts()}] [GREETING] Ready in {_time.time()-t0:.2f}s (lipsync: {lipsync_video_url is not None})")
+
+    except Exception as e:
+        logger.error(f"[{_ts()}] [GREETING] Failed: {e}", exc_info=True)
+        GREETING_DATA = {"text": greeting_text, "audio_url": None, "audio_duration": 0, "lipsync_video_url": None}
 
 
 async def init_float():
@@ -96,10 +149,83 @@ async def init_float():
             "wav2vec_model_path": str(w2v), "audio2emotion_path": str(emo),
         })
         logger.info("[STARTUP] ✓ FLOAT ready!")
+
+        # Warmup: generate a greeting to prime GPU kernels
+        await _generate_warmup_greeting()
+
     except Exception as e:
         logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True)
 
 
+# Warmup greeting data (set during startup, served to frontend)
+_warmup_greeting = None
+
+GREETING_TEXT = "Good day! I am Charles Darwin, er, recreated in this curious digital form. Do feel free to ask me anything, umm, about the natural world or otherwise."
+
+async def _generate_warmup_greeting():
+    """Generate TTS + lipsync for a greeting to warm up the GPU."""
+    global _warmup_greeting
+    import asyncio
+    logger.info("[WARMUP] Generating greeting TTS + lipsync...")
+    t0 = _time.time()
+
+    try:
+        result = await asyncio.to_thread(_generate_warmup_sync)
+        _warmup_greeting = result
+        logger.info(f"[WARMUP] ✓ Greeting ready in {_time.time()-t0:.2f}s")
+    except Exception as e:
+        logger.error(f"[WARMUP] Failed: {e}", exc_info=True)
+        _warmup_greeting = None
+
+
+def _generate_warmup_sync():
+    """Synchronous warmup generation."""
+    from tts import generate_audio
+
+    t1 = _time.time()
+    audio_path = generate_audio(GREETING_TEXT)
+    t2 = _time.time()
+    logger.info(f"[WARMUP] TTS: {t2-t1:.2f}s")
+
+    audio_url = None
+    audio_duration = 0
+    if audio_path and os.path.exists(audio_path):
+        audio_url = f"/api/audio/{os.path.basename(audio_path)}"
+        try:
+            r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
+                                "-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
+            audio_duration = float(r.stdout.strip())
+        except Exception:
+            audio_duration = 5.0
+
+    lipsync_video_url = None
+    if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
+        from float_lipsync import get_lipsync
+        lipsync = get_lipsync()
+        if lipsync.ready:
+            t3 = _time.time()
+            lp = lipsync.generate(audio_path)
+            t4 = _time.time()
+            logger.info(f"[WARMUP] FLOAT: {t4-t3:.2f}s")
+            if lp and os.path.exists(lp):
+                lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
+
+    return {
+        "text": GREETING_TEXT,
+        "audio_url": audio_url,
+        "audio_duration": audio_duration,
+        "lipsync_video_url": lipsync_video_url,
+    }
+
+
+@app.get("/api/greeting")
+async def get_greeting():
+    """Return the pre-generated warmup greeting (or null if not ready)."""
+    if _warmup_greeting:
+        return JSONResponse(_warmup_greeting)
+    return JSONResponse({"text": GREETING_TEXT, "audio_url": None, "lipsync_video_url": None})
+
+
 async def download_videos():
     from huggingface_hub import HfApi, hf_hub_download
     import shutil
@@ -182,6 +308,16 @@ async def chat(request: Request):
 
     logger.info(f"[{_ts()}] [CHAT] User: {user_text}")
 
+    # Reset FLOAT reference to original for new message
+    if FLOAT_ENABLED:
+        try:
+            from float_lipsync import get_lipsync
+            lipsync = get_lipsync()
+            if lipsync.ready:
+                lipsync.reset_reference()
+        except Exception:
+            pass
+
     # 1. LLM (single call)
     t1 = _time.time()
     from llm import generate_darwin_response
@@ -265,7 +401,7 @@ async def chat_next(request: Request):
 
 def _generate_chunk(sentence, idx, total):
     """Synchronous TTS + FLOAT generation (runs in thread pool)."""
-    logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] {sentence[:60]}...")
+    logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] ({len(sentence.split())}w {len(sentence)}c) {sentence[:60]}...")
     t0 = _time.time()
 
     # 1. TTS
@@ -320,6 +456,14 @@ async def reset_chat():
     return JSONResponse({"status": "ok"})
 
 
+@app.get("/api/greeting")
+async def get_greeting():
+    """Return the pre-generated greeting (audio + lipsync video)."""
+    if GREETING_DATA:
+        return JSONResponse(GREETING_DATA)
+    return JSONResponse({"text": None})
+
+
 @app.get("/", response_class=HTMLResponse)
 async def index():
     return HTML_PAGE
@@ -339,7 +483,7 @@ body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;backgrou
 .video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center}
 .video-container{position:relative;width:100%;height:100%}
 .video-container video{width:100%;height:100%;object-fit:contain;display:block}
-#videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity .25s ease}
+#videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity .175s ease}
 #videoA{z-index:1;opacity:1}#videoB{z-index:0;opacity:0}
 .video-status{position:absolute;bottom:8px;right:12px;font-size:.7rem;color:rgba(255,255,255,.4);background:rgba(0,0,0,.5);padding:2px 8px;border-radius:4px;z-index:10}
 .chat-section{flex:1;display:flex;flex-direction:column;min-height:0;background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden}
@@ -387,51 +531,98 @@ body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;backgrou
 // ======================================================================
 // STATE
 // ======================================================================
-let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, lastPlayed=null, msgCounter=0;
+let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, isSpeaking=false, lastPlayed=null, msgCounter=0;
 const videoA=document.getElementById('videoA'), videoB=document.getElementById('videoB');
 const videoStatus=document.getElementById('videoStatus'), chatLog=document.getElementById('chatLog');
 const chatInput=document.getElementById('chatInput'), btnSend=document.getElementById('btnSend');
 const btnReset=document.getElementById('btnReset');
 
 // ======================================================================
-// VIDEO: Crossfade
+// VIDEO: Seamless crossfade with preloading
 // ======================================================================
 function getActiveVideo(){return currentVideo==='A'?videoA:videoB}
 function getInactiveVideo(){return currentVideo==='A'?videoB:videoA}
 
-function crossfadeTo(url, muted){
+// Preload a video into the inactive element (doesn't play yet)
+function preloadVideo(url){
     return new Promise((resolve)=>{
-        if(isTransitioning){ resolve(); return; }
-        isTransitioning=true;
-        const prev=getActiveVideo(), next=getInactiveVideo();
-        next.muted=(muted!==false);
-        next.src=url; next.load();
-        next.onloadeddata=function(){
-            next.style.zIndex='2'; prev.style.zIndex='1'; next.style.opacity='0';
+        const next=getInactiveVideo();
+        next.muted=true;
+        next.preload='auto';
+        next.src=url;
+        next.load();
+        next.oncanplaythrough=function(){
+            next.oncanplaythrough=null;
+            console.log('['+new Date().toISOString().substr(11,12)+'] preloaded: '+url.split('/').pop());
+            resolve(true);
+        };
+        next.onerror=function(){
+            next.onerror=null;
+            console.warn('[VIDEO] Preload failed:', url);
+            resolve(false);
+        };
+        // Timeout fallback — don't wait forever
+        setTimeout(()=>resolve(true), 3000);
+    });
+}
+
+// Swap: the inactive element is already loaded, just play and crossfade
+function swapToPreloaded(){
+    return new Promise((resolve)=>{
+        const prev=getActiveVideo();
+        const next=getInactiveVideo();
+
+        // If nothing preloaded, just resolve
+        if(!next.src || next.src===''){
+            resolve(); return;
+        }
+
+        next.style.zIndex='2';
+        prev.style.zIndex='1';
+        next.style.opacity='0';
+
+        next.play().then(()=>{
+            // Instant opacity swap (no slow fade — speed over beauty)
+            next.style.opacity='1';
+            prev.style.opacity='0';
+
+            // Clean up old video after brief overlap
+            setTimeout(()=>{
+                prev.pause();
+                prev.removeAttribute('src');
+                prev.load(); // reset
+                prev.style.zIndex='0';
+                next.style.zIndex='1';
+                currentVideo=currentVideo==='A'?'B':'A';
+                resolve();
+            }, 50);
+        }).catch(()=>{
+            next.muted=true;
             next.play().then(()=>{
                 next.style.opacity='1';
+                prev.style.opacity='0';
                 setTimeout(()=>{
-                    prev.style.opacity='0'; prev.pause(); prev.src='';
-                    prev.style.zIndex='0'; next.style.zIndex='1';
+                    prev.pause();
+                    prev.removeAttribute('src');
+                    prev.load();
+                    prev.style.zIndex='0';
+                    next.style.zIndex='1';
                     currentVideo=currentVideo==='A'?'B':'A';
-                    isTransitioning=false;
                     resolve();
-                },300);
-            }).catch(()=>{
-                next.muted=true;
-                next.play().then(()=>{
-                    next.style.opacity='1';
-                    setTimeout(()=>{
-                        prev.style.opacity='0'; prev.pause(); prev.src='';
-                        prev.style.zIndex='0'; next.style.zIndex='1';
-                        currentVideo=currentVideo==='A'?'B':'A';
-                        isTransitioning=false;
-                        resolve();
-                    },300);
-                }).catch(()=>{ isTransitioning=false; resolve(); });
-            });
-        };
-        next.onerror=function(){ isTransitioning=false; resolve(); };
+                }, 50);
+            }).catch(()=>{ resolve(); });
+        });
+    });
+}
+
+// Full crossfade (load + play) for idle videos and first chunk
+function crossfadeTo(url, muted){
+    return new Promise(async (resolve)=>{
+        await preloadVideo(url);
+        const next=getInactiveVideo();
+        next.muted=(muted!==false);
+        await swapToPreloaded();
+        resolve();
     });
 }
 
@@ -450,16 +641,15 @@ function playNextIdle(){
 }
 
 // Idle loop: when video ends and not speaking, play next idle
-videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isProcessing) playNextIdle(); });
-videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isProcessing) playNextIdle(); });
+videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isSpeaking) playNextIdle(); });
+videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isSpeaking) playNextIdle(); });
 
 // ======================================================================
-// VIDEO: Wait for current lipsync video to end
+// VIDEO: Wait for current video to end
 // ======================================================================
 function waitForVideoEnd(){
     return new Promise((resolve)=>{
         const active=getActiveVideo();
-        // If video is very short or already ended, resolve quickly
         if(active.ended || active.paused || !active.src){
             resolve(); return;
         }
@@ -518,6 +708,9 @@ async function sendMessage(){
 
         videoStatus.textContent='generating 1/'+totalChunks+'...';
 
+        // Idle videos keep playing while we wait for chunks
+        // isSpeaking stays false until we actually play a lipsync video
+
         // Helper to fetch a chunk from the server
         async function fetchChunk(sid){
             const r=await fetch('/api/chat/next',{
@@ -528,41 +721,74 @@ async function sendMessage(){
             return await r.json();
         }
 
-        // Step 2: Prefetch first chunk
-        let nextChunkPromise=fetchChunk(sessionId);
+        // Step 2: Fetch and play chunks with preloading
+        let nextChunk = await fetchChunk(sessionId);
 
         for(let i=0; i<totalChunks; i++){
-            videoStatus.textContent='waiting '+(i+1)+'/'+totalChunks+'...';
-
-            // Wait for the current chunk to be ready
-            const chunk=await nextChunkPromise;
+            const chunk = nextChunk;
             if(!chunk || chunk.done) break;
 
-            // Immediately start prefetching the NEXT chunk (runs in parallel with playback)
-            if(i+1 < totalChunks){
-                videoStatus.textContent='speaking '+(i+1)+'/'+totalChunks+' (prefetching '+(i+2)+')';
-                nextChunkPromise=fetchChunk(sessionId);
-            }
+            console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' ready');
+            videoStatus.textContent='playing '+(i+1)+'/'+totalChunks;
 
-            // Play audio
-            let audioPromise=null;
+            // Start fetching next chunk data from server (TTS+FLOAT generation)
+            let nextChunkPromise = (i+1 < totalChunks) ? fetchChunk(sessionId) : null;
+
+            // Play audio and get a promise for when it ends
+            let audioDonePromise = null;
             if(chunk.audio_url){
                 const audio=new Audio(chunk.audio_url);
-                audioPromise=new Promise((resolve)=>{
+                audioDonePromise = new Promise((resolve)=>{
                     audio.onended=resolve;
                     audio.onerror=resolve;
-                    audio.play().catch(()=>resolve());
+                    // Fallback timeout in case ended event doesn't fire
+                    const fallback = setTimeout(resolve, (chunk.audio_duration||5)*1000 + 500);
+                    audio.onended=()=>{ clearTimeout(fallback); resolve(); };
+                    audio.play().catch(()=>{ clearTimeout(fallback); resolve(); });
                 });
             }
 
-            // Play lipsync video
             if(chunk.lipsync_video_url){
-                await crossfadeTo(chunk.lipsync_video_url, true);
-                // Wait for this lipsync video to finish before next chunk
+                // --- HAS LIPSYNC VIDEO ---
+                isSpeaking=true; // stop idle loop for entire response
+                if(i===0){
+                    await crossfadeTo(chunk.lipsync_video_url, true);
+                } else {
+                    await swapToPreloaded();
+                }
+                console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' video playing');
+
+                // While video plays, wait for next chunk data and preload its video
+                if(nextChunkPromise){
+                    nextChunkPromise.then((nc)=>{
+                        if(nc && !nc.done && nc.lipsync_video_url){
+                            console.log('['+new Date().toISOString().substr(11,12)+'] preloading chunk '+(i+2));
+                            preloadVideo(nc.lipsync_video_url);
+                        }
+                    });
+                }
+
                 await waitForVideoEnd();
-            } else if(audioPromise){
-                // No lipsync, just wait for audio to finish
-                await audioPromise;
+                console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' video ended');
+                // isSpeaking stays true — no idle between chunks
+
+            } else {
+                // --- AUDIO ONLY (no lipsync) ---
+                console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' audio only');
+                if(audioDonePromise){
+                    await audioDonePromise;
+                } else if(chunk.audio_duration > 0){
+                    await new Promise(r=>setTimeout(r, chunk.audio_duration*1000));
+                } else {
+                    await new Promise(r=>setTimeout(r, 2000));
+                }
+            }
+
+            // Get the next chunk (should already be resolved)
+            if(nextChunkPromise){
+                nextChunk = await nextChunkPromise;
+            } else {
+                nextChunk = null;
             }
         }
 
@@ -575,7 +801,7 @@ async function sendMessage(){
         typing.contentEl.classList.remove('typing-dots');
         typing.contentEl.textContent='Error: '+err.message;
     }
-    isProcessing=false; btnSend.disabled=false; chatInput.focus();
+    isProcessing=false; isSpeaking=false; btnSend.disabled=false; chatInput.focus();
 }
 
 btnSend.addEventListener('click', sendMessage);
@@ -596,8 +822,33 @@ async function init(){
         if(!data.videos.length){ videoStatus.textContent='Downloading...'; setTimeout(init,3000); return; }
         videos=data.videos;
         videoStatus.textContent=videos.length+' idle clips loaded';
-        playNextIdle();
-        addMessage('bot','Good day! I am Charles Darwin, er, recreated in this curious digital form. Do feel free to ask me anything, umm, about the natural world or otherwise.');
+
+        // Check for pre-generated greeting
+        const greetRes=await fetch('/api/greeting');
+        const greeting=await greetRes.json();
+
+        if(greeting.lipsync_video_url){
+            // Play greeting lipsync video first
+            videoStatus.textContent='greeting...';
+            addMessage('bot', greeting.text);
+
+            // Play audio
+            if(greeting.audio_url){
+                const audio=new Audio(greeting.audio_url);
+                audio.play().catch(e=>console.warn('[AUDIO]',e));
+            }
+
+            // Play lipsync video
+            await crossfadeTo(greeting.lipsync_video_url, true);
+            await waitForVideoEnd();
+
+            // Then start idle loop
+            playNextIdle();
+        } else {
+            // No greeting video, just show text and start idle
+            addMessage('bot', greeting.text || 'Good day! I am Charles Darwin, er, recreated in this curious digital form. Do feel free to ask me anything, umm, about the natural world.');
+            playNextIdle();
+        }
     }catch(e){ videoStatus.textContent='Error: '+e.message; }
 }
 init();