Update app.py

app.py

@@ -182,6 +182,16 @@ async def chat(request: Request):
 
     logger.info(f"[{_ts()}] [CHAT] User: {user_text}")
 
+    # Reset FLOAT reference to original for new message
+    if FLOAT_ENABLED:
+        try:
+            from float_lipsync import get_lipsync
+            lipsync = get_lipsync()
+            if lipsync.ready:
+                lipsync.reset_reference()
+        except Exception:
+            pass
+
     # 1. LLM (single call)
     t1 = _time.time()
     from llm import generate_darwin_response
@@ -265,7 +275,7 @@ async def chat_next(request: Request):
 
 def _generate_chunk(sentence, idx, total):
     """Synchronous TTS + FLOAT generation (runs in thread pool)."""
-    logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] {sentence[:60]}...")
+    logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] ({len(sentence.split())}w {len(sentence)}c) {sentence[:60]}...")
     t0 = _time.time()
 
     # 1. TTS
@@ -295,7 +305,7 @@ def _generate_chunk(sentence, idx, total):
             if lipsync.ready:
                 with _float_lock:
                     t3 = _time.time()
-                    lp = lipsync.generate(audio_path)
+                    lp = lipsync.generate(audio_path, chain_frames=True)
                     t4 = _time.time()
                 logger.info(f"[{_ts()}] [TIMING] Chunk {idx+1} FLOAT: {t4-t3:.2f}s")
                 if lp and os.path.exists(lp):
@@ -339,7 +349,7 @@ body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;backgrou
 .video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center}
 .video-container{position:relative;width:100%;height:100%}
 .video-container video{width:100%;height:100%;object-fit:contain;display:block}
-#videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity .25s ease}
+#videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity .175s ease}
 #videoA{z-index:1;opacity:1}#videoB{z-index:0;opacity:0}
 .video-status{position:absolute;bottom:8px;right:12px;font-size:.7rem;color:rgba(255,255,255,.4);background:rgba(0,0,0,.5);padding:2px 8px;border-radius:4px;z-index:10}
 .chat-section{flex:1;display:flex;flex-direction:column;min-height:0;background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden}
@@ -394,44 +404,91 @@ const chatInput=document.getElementById('chatInput'), btnSend=document.getElemen
 const btnReset=document.getElementById('btnReset');
 
 // ======================================================================
-// VIDEO: Crossfade
+// VIDEO: Seamless crossfade with preloading
 // ======================================================================
 function getActiveVideo(){return currentVideo==='A'?videoA:videoB}
 function getInactiveVideo(){return currentVideo==='A'?videoB:videoA}
 
-function crossfadeTo(url, muted){
+// Preload a video into the inactive element (doesn't play yet)
+function preloadVideo(url){
     return new Promise((resolve)=>{
-        if(isTransitioning){ resolve(); return; }
-        isTransitioning=true;
-        const prev=getActiveVideo(), next=getInactiveVideo();
-        next.muted=(muted!==false);
-        next.src=url; next.load();
-        next.onloadeddata=function(){
-            next.style.zIndex='2'; prev.style.zIndex='1'; next.style.opacity='0';
+        const next=getInactiveVideo();
+        next.muted=true;
+        next.preload='auto';
+        next.src=url;
+        next.load();
+        next.oncanplaythrough=function(){
+            next.oncanplaythrough=null;
+            console.log('['+new Date().toISOString().substr(11,12)+'] preloaded: '+url.split('/').pop());
+            resolve(true);
+        };
+        next.onerror=function(){
+            next.onerror=null;
+            console.warn('[VIDEO] Preload failed:', url);
+            resolve(false);
+        };
+        // Timeout fallback — don't wait forever
+        setTimeout(()=>resolve(true), 3000);
+    });
+}
+
+// Swap: the inactive element is already loaded, just play and crossfade
+function swapToPreloaded(){
+    return new Promise((resolve)=>{
+        const prev=getActiveVideo();
+        const next=getInactiveVideo();
+
+        // If nothing preloaded, just resolve
+        if(!next.src || next.src===''){
+            resolve(); return;
+        }
+
+        next.style.zIndex='2';
+        prev.style.zIndex='1';
+        next.style.opacity='0';
+
+        next.play().then(()=>{
+            // Instant opacity swap (no slow fade — speed over beauty)
+            next.style.opacity='1';
+            prev.style.opacity='0';
+
+            // Clean up old video after brief overlap
+            setTimeout(()=>{
+                prev.pause();
+                prev.removeAttribute('src');
+                prev.load(); // reset
+                prev.style.zIndex='0';
+                next.style.zIndex='1';
+                currentVideo=currentVideo==='A'?'B':'A';
+                resolve();
+            }, 50);
+        }).catch(()=>{
+            next.muted=true;
             next.play().then(()=>{
                 next.style.opacity='1';
+                prev.style.opacity='0';
                 setTimeout(()=>{
-                    prev.style.opacity='0'; prev.pause(); prev.src='';
-                    prev.style.zIndex='0'; next.style.zIndex='1';
+                    prev.pause();
+                    prev.removeAttribute('src');
+                    prev.load();
+                    prev.style.zIndex='0';
+                    next.style.zIndex='1';
                     currentVideo=currentVideo==='A'?'B':'A';
-                    isTransitioning=false;
                     resolve();
-                },300);
-            }).catch(()=>{
-                next.muted=true;
-                next.play().then(()=>{
-                    next.style.opacity='1';
-                    setTimeout(()=>{
-                        prev.style.opacity='0'; prev.pause(); prev.src='';
-                        prev.style.zIndex='0'; next.style.zIndex='1';
-                        currentVideo=currentVideo==='A'?'B':'A';
-                        isTransitioning=false;
-                        resolve();
-                    },300);
-                }).catch(()=>{ isTransitioning=false; resolve(); });
-            });
-        };
-        next.onerror=function(){ isTransitioning=false; resolve(); };
+                }, 50);
+            }).catch(()=>{ resolve(); });
+        });
+    });
+}
+
+// Full crossfade (load + play) for idle videos and first chunk
+function crossfadeTo(url, muted){
+    return new Promise(async (resolve)=>{
+        await preloadVideo(url);
+        const next=getInactiveVideo();
+        next.muted=(muted!==false);
+        await swapToPreloaded();
+        resolve();
     });
 }
 
@@ -454,12 +511,11 @@ videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isProcessing) play
 videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isProcessing) playNextIdle(); });
 
 // ======================================================================
-// VIDEO: Wait for current lipsync video to end
+// VIDEO: Wait for current video to end
 // ======================================================================
 function waitForVideoEnd(){
     return new Promise((resolve)=>{
         const active=getActiveVideo();
-        // If video is very short or already ended, resolve quickly
         if(active.ended || active.paused || !active.src){
             resolve(); return;
         }
@@ -528,41 +584,74 @@ async function sendMessage(){
             return await r.json();
         }
 
-        // Step 2: Prefetch first chunk
-        let nextChunkPromise=fetchChunk(sessionId);
+        // Step 2: Fetch and play chunks with preloading
+        let nextChunk = await fetchChunk(sessionId);
 
         for(let i=0; i<totalChunks; i++){
-            videoStatus.textContent='waiting '+(i+1)+'/'+totalChunks+'...';
-
-            // Wait for the current chunk to be ready
-            const chunk=await nextChunkPromise;
+            const chunk = nextChunk;
             if(!chunk || chunk.done) break;
 
-            // Immediately start prefetching the NEXT chunk (runs in parallel with playback)
-            if(i+1 < totalChunks){
-                videoStatus.textContent='speaking '+(i+1)+'/'+totalChunks+' (prefetching '+(i+2)+')';
-                nextChunkPromise=fetchChunk(sessionId);
-            }
+            console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' ready');
+            videoStatus.textContent='playing '+(i+1)+'/'+totalChunks;
 
-            // Play audio
-            let audioPromise=null;
+            // Start fetching next chunk data from server (TTS+FLOAT generation)
+            let nextChunkPromise = (i+1 < totalChunks) ? fetchChunk(sessionId) : null;
+
+            // Play audio and get a promise for when it ends
+            let audioDonePromise = null;
             if(chunk.audio_url){
                 const audio=new Audio(chunk.audio_url);
-                audioPromise=new Promise((resolve)=>{
+                audioDonePromise = new Promise((resolve)=>{
                     audio.onended=resolve;
                     audio.onerror=resolve;
-                    audio.play().catch(()=>resolve());
+                    // Fallback timeout in case ended event doesn't fire
+                    const fallback = setTimeout(resolve, (chunk.audio_duration||5)*1000 + 500);
+                    audio.onended=()=>{ clearTimeout(fallback); resolve(); };
+                    audio.play().catch(()=>{ clearTimeout(fallback); resolve(); });
                 });
             }
 
-            // Play lipsync video
             if(chunk.lipsync_video_url){
-                await crossfadeTo(chunk.lipsync_video_url, true);
-                // Wait for this lipsync video to finish before next chunk
+                // --- HAS LIPSYNC VIDEO ---
+                if(i===0){
+                    await crossfadeTo(chunk.lipsync_video_url, true);
+                } else {
+                    await swapToPreloaded();
+                }
+                console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' video playing');
+
+                // While video plays, wait for next chunk data and preload its video
+                if(nextChunkPromise){
+                    nextChunkPromise.then((nc)=>{
+                        if(nc && !nc.done && nc.lipsync_video_url){
+                            console.log('['+new Date().toISOString().substr(11,12)+'] preloading chunk '+(i+2));
+                            preloadVideo(nc.lipsync_video_url);
+                        }
+                    });
+                }
+
                 await waitForVideoEnd();
-            } else if(audioPromise){
-                // No lipsync, just wait for audio to finish
-                await audioPromise;
+                console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' video ended');
+
+            } else {
+                // --- AUDIO ONLY (no lipsync) --- idle video keeps playing
+                isProcessing=false; // let idle loop continue
+                console.log('['+new Date().toISOString().substr(11,12)+'] chunk '+(i+1)+' audio only');
+                if(audioDonePromise){
+                    await audioDonePromise;
+                } else if(chunk.audio_duration > 0){
+                    await new Promise(r=>setTimeout(r, chunk.audio_duration*1000));
+                } else {
+                    await new Promise(r=>setTimeout(r, 2000));
+                }
+                isProcessing=true; // re-lock for next chunk
+            }
+
+            // Get the next chunk (should already be resolved)
+            if(nextChunkPromise){
+                nextChunk = await nextChunkPromise;
+            } else {
+                nextChunk = null;
             }
         }