J Z commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,17 +1,13 @@
|
|
| 1 |
"""
|
| 2 |
Avatar Chatbot - HuggingFace Spaces Edition
|
| 3 |
-
|
| 4 |
-
Phase 2: FLOAT lipsync (to be added)
|
| 5 |
"""
|
| 6 |
from fastapi import FastAPI, HTTPException, Request
|
| 7 |
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
|
| 8 |
-
from fastapi.staticfiles import StaticFiles
|
| 9 |
from pathlib import Path
|
| 10 |
import mimetypes
|
| 11 |
import os
|
| 12 |
import logging
|
| 13 |
-
import asyncio
|
| 14 |
-
import time
|
| 15 |
import subprocess
|
| 16 |
|
| 17 |
logging.basicConfig(level=logging.INFO)
|
|
@@ -19,19 +15,16 @@ logger = logging.getLogger(__name__)
|
|
| 19 |
|
| 20 |
app = FastAPI()
|
| 21 |
|
| 22 |
-
# --- Config ---
|
| 23 |
VIDEOS_DIR = Path("/tmp/videos")
|
| 24 |
TTS_DIR = Path("/tmp/tts_output")
|
| 25 |
LIPSYNC_DIR = Path("/tmp/lipsync_output")
|
| 26 |
REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot")
|
| 27 |
-
|
| 28 |
FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true"
|
| 29 |
|
| 30 |
for d in [VIDEOS_DIR, TTS_DIR, LIPSYNC_DIR]:
|
| 31 |
d.mkdir(parents=True, exist_ok=True)
|
| 32 |
|
| 33 |
|
| 34 |
-
# --- Startup: download idle videos + init FLOAT ---
|
| 35 |
@app.on_event("startup")
|
| 36 |
async def startup():
|
| 37 |
await download_videos()
|
|
@@ -40,102 +33,59 @@ async def startup():
|
|
| 40 |
|
| 41 |
|
| 42 |
async def init_float():
|
| 43 |
-
"""Initialize FLOAT lipsync model (runs once at startup)."""
|
| 44 |
import torch
|
| 45 |
logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}")
|
| 46 |
if not torch.cuda.is_available():
|
| 47 |
-
logger.warning("[STARTUP] No GPU
|
| 48 |
return
|
| 49 |
-
|
| 50 |
try:
|
| 51 |
from float_lipsync import get_lipsync
|
| 52 |
-
from huggingface_hub import hf_hub_download
|
| 53 |
import shutil
|
| 54 |
|
| 55 |
-
|
| 56 |
-
|
| 57 |
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
if not w2v_dir.exists():
|
| 61 |
logger.info("[STARTUP] Downloading wav2vec2-base-960h...")
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
)
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
logger.
|
| 92 |
-
except Exception as e:
|
| 93 |
-
logger.warning(f"[STARTUP] Could not download float.pth: {e}")
|
| 94 |
-
logger.warning("[STARTUP] Upload it to your Space repo under app/checkpoints/float.pth")
|
| 95 |
-
logger.warning("[STARTUP] FLOAT lipsync will be disabled until checkpoint is available")
|
| 96 |
-
return
|
| 97 |
-
|
| 98 |
-
# Check for reference image - download from repo if needed
|
| 99 |
-
assets_dir = Path("/app/assets")
|
| 100 |
-
assets_dir.mkdir(parents=True, exist_ok=True)
|
| 101 |
-
ref_image = assets_dir / "ref.png"
|
| 102 |
-
if not ref_image.exists() or ref_image.stat().st_size < 1024:
|
| 103 |
-
logger.info("[STARTUP] Looking for reference image in repo...")
|
| 104 |
-
try:
|
| 105 |
-
# Try common image extensions
|
| 106 |
-
for ext in ["ref.png", "ref.jpg", "ref.jpeg", "main2.png"]:
|
| 107 |
-
try:
|
| 108 |
-
downloaded = hf_hub_download(
|
| 109 |
-
repo_id=REPO_ID, repo_type="space",
|
| 110 |
-
filename=f"app/assets/{ext}",
|
| 111 |
-
local_dir="/tmp/hf_download",
|
| 112 |
-
)
|
| 113 |
-
dest = assets_dir / ext
|
| 114 |
-
shutil.copy2(downloaded, dest)
|
| 115 |
-
ref_image = dest
|
| 116 |
-
logger.info(f"[STARTUP] Reference image downloaded: {ext}")
|
| 117 |
-
break
|
| 118 |
-
except Exception:
|
| 119 |
-
continue
|
| 120 |
-
else:
|
| 121 |
-
logger.warning("[STARTUP] No reference image found in app/assets/")
|
| 122 |
-
logger.warning("[STARTUP] Upload a face image as app/assets/ref.png")
|
| 123 |
-
return
|
| 124 |
-
except Exception as e:
|
| 125 |
-
logger.warning(f"[STARTUP] Could not download reference image: {e}")
|
| 126 |
-
return
|
| 127 |
-
|
| 128 |
-
# Initialize FLOAT
|
| 129 |
-
lipsync = get_lipsync()
|
| 130 |
-
config = {
|
| 131 |
-
"ref_path": str(ref_image),
|
| 132 |
-
"ckpt_path": str(float_ckpt),
|
| 133 |
-
"wav2vec_model_path": str(w2v_dir),
|
| 134 |
-
"audio2emotion_path": str(emo_dir),
|
| 135 |
-
}
|
| 136 |
-
lipsync.initialize(config)
|
| 137 |
-
logger.info("[STARTUP] ✓ FLOAT lipsync system ready!")
|
| 138 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
except Exception as e:
|
| 140 |
logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True)
|
| 141 |
|
|
@@ -143,192 +93,110 @@ async def init_float():
|
|
| 143 |
async def download_videos():
|
| 144 |
from huggingface_hub import HfApi, hf_hub_download
|
| 145 |
import shutil
|
| 146 |
-
|
| 147 |
try:
|
| 148 |
api = HfApi()
|
| 149 |
files = api.list_repo_files(repo_id=REPO_ID, repo_type="space")
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
logger.info(f"Found {len(video_files)} video files in repo")
|
| 156 |
-
|
| 157 |
-
for vf in video_files:
|
| 158 |
-
filename = Path(vf).name
|
| 159 |
-
dest = VIDEOS_DIR / filename
|
| 160 |
-
if dest.exists() and dest.stat().st_size > 1024:
|
| 161 |
-
continue
|
| 162 |
logger.info(f"Downloading {vf}...")
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
)
|
| 167 |
-
shutil.copy2(downloaded, dest)
|
| 168 |
-
logger.info(f"Downloaded {filename}: {dest.stat().st_size} bytes")
|
| 169 |
except Exception as e:
|
| 170 |
logger.error(f"Error downloading videos: {e}")
|
| 171 |
|
| 172 |
|
| 173 |
-
# --- API: list idle videos ---
|
| 174 |
@app.get("/api/videos")
|
| 175 |
async def list_videos():
|
| 176 |
-
if not VIDEOS_DIR.exists():
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
if f.suffix.lower() in (".mp4",
|
| 181 |
-
|
| 182 |
-
return JSONResponse({"videos": videos})
|
| 183 |
|
| 184 |
|
| 185 |
-
# --- API: stream a video file ---
|
| 186 |
@app.get("/api/stream/{filename}")
|
| 187 |
async def stream_video(filename: str):
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
|
|
|
| 198 |
|
| 199 |
-
mime_type = mimetypes.guess_type(filename)[0] or "video/mp4"
|
| 200 |
-
file_size = video_path.stat().st_size
|
| 201 |
|
| 202 |
-
def iter_file():
|
| 203 |
-
with open(video_path, "rb") as f:
|
| 204 |
-
while chunk := f.read(1024 * 1024):
|
| 205 |
-
yield chunk
|
| 206 |
-
|
| 207 |
-
return StreamingResponse(
|
| 208 |
-
iter_file(),
|
| 209 |
-
media_type=mime_type,
|
| 210 |
-
headers={"Content-Length": str(file_size), "Accept-Ranges": "bytes"},
|
| 211 |
-
)
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
# --- API: stream audio file ---
|
| 215 |
@app.get("/api/audio/{filename}")
|
| 216 |
async def stream_audio(filename: str):
|
| 217 |
-
|
| 218 |
-
if not
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
|
|
|
|
|
|
| 223 |
|
| 224 |
-
def iter_file():
|
| 225 |
-
with open(audio_path, "rb") as f:
|
| 226 |
-
while chunk := f.read(1024 * 1024):
|
| 227 |
-
yield chunk
|
| 228 |
|
| 229 |
-
return StreamingResponse(
|
| 230 |
-
iter_file(),
|
| 231 |
-
media_type=mime_type,
|
| 232 |
-
headers={"Content-Length": str(file_size)},
|
| 233 |
-
)
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
# --- API: chat endpoint (returns text immediately, then streams lipsync chunks) ---
|
| 237 |
@app.post("/api/chat")
|
| 238 |
async def chat(request: Request):
|
| 239 |
-
"""
|
| 240 |
-
Step 1: LLM generates full response text.
|
| 241 |
-
Step 2: Returns text immediately so frontend can show it.
|
| 242 |
-
Step 3: Frontend then calls /api/chat/generate to get lipsync chunks via SSE.
|
| 243 |
-
"""
|
| 244 |
body = await request.json()
|
| 245 |
user_text = body.get("text", "").strip()
|
| 246 |
-
|
| 247 |
-
if not user_text:
|
| 248 |
-
raise HTTPException(status_code=400, detail="No text provided")
|
| 249 |
|
| 250 |
logger.info(f"[CHAT] User: {user_text}")
|
| 251 |
|
| 252 |
-
# 1. LLM
|
| 253 |
from llm import generate_darwin_response
|
| 254 |
llm_result = generate_darwin_response(user_text)
|
| 255 |
reply_text = llm_result['text']
|
| 256 |
logger.info(f"[CHAT] Darwin: {reply_text}")
|
| 257 |
|
| 258 |
-
# Split into sentences (by period only, not commas)
|
| 259 |
-
import re
|
| 260 |
-
raw_sentences = re.split(r'(?<=\.)\s+', reply_text)
|
| 261 |
-
sentences = [s.strip() for s in raw_sentences if s.strip()]
|
| 262 |
-
# If no periods found, treat whole text as one chunk
|
| 263 |
-
if not sentences:
|
| 264 |
-
sentences = [reply_text]
|
| 265 |
-
|
| 266 |
-
return JSONResponse({
|
| 267 |
-
"text": reply_text,
|
| 268 |
-
"emotion": llm_result.get('emotion', 'neutral'),
|
| 269 |
-
"sentences": sentences,
|
| 270 |
-
})
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
@app.post("/api/chat/generate")
|
| 274 |
-
async def chat_generate(request: Request):
|
| 275 |
-
"""
|
| 276 |
-
SSE endpoint: receives a sentence, generates TTS + lipsync, returns result.
|
| 277 |
-
Called once per sentence by the frontend.
|
| 278 |
-
"""
|
| 279 |
-
body = await request.json()
|
| 280 |
-
sentence = body.get("sentence", "").strip()
|
| 281 |
-
|
| 282 |
-
if not sentence:
|
| 283 |
-
raise HTTPException(status_code=400, detail="No sentence provided")
|
| 284 |
-
|
| 285 |
-
logger.info(f"[GEN] Sentence: {sentence[:60]}...")
|
| 286 |
-
|
| 287 |
-
# 1. TTS
|
| 288 |
from tts import generate_audio
|
| 289 |
-
audio_path = generate_audio(
|
| 290 |
audio_url = None
|
| 291 |
audio_duration = 0
|
| 292 |
-
|
| 293 |
if audio_path and os.path.exists(audio_path):
|
| 294 |
-
|
| 295 |
-
audio_url = f"/api/audio/{audio_filename}"
|
| 296 |
-
|
| 297 |
try:
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
capture_output=True, text=True, check=True
|
| 302 |
-
)
|
| 303 |
-
audio_duration = float(result.stdout.strip())
|
| 304 |
except Exception:
|
| 305 |
audio_duration = 3.0
|
| 306 |
|
| 307 |
-
# 2. Lipsync
|
| 308 |
lipsync_video_url = None
|
| 309 |
if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
|
| 310 |
try:
|
| 311 |
from float_lipsync import get_lipsync
|
| 312 |
lipsync = get_lipsync()
|
| 313 |
if lipsync.ready:
|
| 314 |
-
logger.info(
|
| 315 |
-
|
| 316 |
-
if
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
|
|
|
| 320 |
except Exception as e:
|
| 321 |
-
logger.error(f"[
|
| 322 |
|
| 323 |
return JSONResponse({
|
| 324 |
-
"
|
|
|
|
| 325 |
"audio_url": audio_url,
|
| 326 |
"audio_duration": audio_duration,
|
| 327 |
"lipsync_video_url": lipsync_video_url,
|
| 328 |
})
|
| 329 |
|
| 330 |
|
| 331 |
-
# --- API: reset conversation ---
|
| 332 |
@app.post("/api/reset")
|
| 333 |
async def reset_chat():
|
| 334 |
from llm import reset_conversation
|
|
@@ -336,7 +204,6 @@ async def reset_chat():
|
|
| 336 |
return JSONResponse({"status": "ok"})
|
| 337 |
|
| 338 |
|
| 339 |
-
# --- Main page ---
|
| 340 |
@app.get("/", response_class=HTMLResponse)
|
| 341 |
async def index():
|
| 342 |
return HTML_PAGE
|
|
@@ -346,197 +213,44 @@ HTML_PAGE = """\
|
|
| 346 |
<!DOCTYPE html>
|
| 347 |
<html lang="en">
|
| 348 |
<head>
|
| 349 |
-
<meta charset="UTF-8"
|
| 350 |
-
<meta name="viewport" content="width=device-width, initial-scale=1.0"
|
| 351 |
<title>Darwin Avatar Chatbot</title>
|
| 352 |
<style>
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
flex-shrink: 0;
|
| 384 |
-
display: flex;
|
| 385 |
-
align-items: center;
|
| 386 |
-
justify-content: center;
|
| 387 |
-
}
|
| 388 |
-
.video-container {
|
| 389 |
-
position: relative;
|
| 390 |
-
width: 100%;
|
| 391 |
-
height: 100%;
|
| 392 |
-
}
|
| 393 |
-
.video-container video {
|
| 394 |
-
width: 100%;
|
| 395 |
-
height: 100%;
|
| 396 |
-
object-fit: contain;
|
| 397 |
-
display: block;
|
| 398 |
-
}
|
| 399 |
-
#videoA, #videoB {
|
| 400 |
-
position: absolute;
|
| 401 |
-
top: 0; left: 0;
|
| 402 |
-
width: 100%; height: 100%;
|
| 403 |
-
object-fit: contain;
|
| 404 |
-
transition: opacity 0.25s ease;
|
| 405 |
-
}
|
| 406 |
-
#videoA { z-index: 1; opacity: 1; }
|
| 407 |
-
#videoB { z-index: 0; opacity: 0; }
|
| 408 |
-
.video-status {
|
| 409 |
-
position: absolute;
|
| 410 |
-
bottom: 8px;
|
| 411 |
-
right: 12px;
|
| 412 |
-
font-size: 0.7rem;
|
| 413 |
-
color: rgba(255,255,255,0.4);
|
| 414 |
-
background: rgba(0,0,0,0.5);
|
| 415 |
-
padding: 2px 8px;
|
| 416 |
-
border-radius: 4px;
|
| 417 |
-
z-index: 10;
|
| 418 |
-
}
|
| 419 |
-
/* --- CHAT LOG --- */
|
| 420 |
-
.chat-section {
|
| 421 |
-
flex: 1;
|
| 422 |
-
display: flex;
|
| 423 |
-
flex-direction: column;
|
| 424 |
-
min-height: 0;
|
| 425 |
-
background: rgba(15, 23, 42, 0.6);
|
| 426 |
-
border: 1px solid rgba(59, 130, 246, 0.15);
|
| 427 |
-
border-radius: 12px;
|
| 428 |
-
overflow: hidden;
|
| 429 |
-
}
|
| 430 |
-
.chat-log {
|
| 431 |
-
flex: 1;
|
| 432 |
-
overflow-y: auto;
|
| 433 |
-
padding: 1rem;
|
| 434 |
-
display: flex;
|
| 435 |
-
flex-direction: column;
|
| 436 |
-
gap: 0.5rem;
|
| 437 |
-
}
|
| 438 |
-
.chat-log::-webkit-scrollbar { width: 6px; }
|
| 439 |
-
.chat-log::-webkit-scrollbar-track { background: transparent; }
|
| 440 |
-
.chat-log::-webkit-scrollbar-thumb { background: #334155; border-radius: 3px; }
|
| 441 |
-
/* Message bubbles */
|
| 442 |
-
.msg { display: flex; width: 100%; }
|
| 443 |
-
.msg.user { justify-content: flex-end; }
|
| 444 |
-
.msg.bot { justify-content: flex-start; }
|
| 445 |
-
.bubble {
|
| 446 |
-
max-width: 85%;
|
| 447 |
-
padding: 10px 14px;
|
| 448 |
-
border-radius: 12px;
|
| 449 |
-
word-wrap: break-word;
|
| 450 |
-
font-size: 0.9rem;
|
| 451 |
-
line-height: 1.4;
|
| 452 |
-
}
|
| 453 |
-
.bubble.user {
|
| 454 |
-
background: linear-gradient(135deg, #1e40af, #3b82f6);
|
| 455 |
-
color: #fff;
|
| 456 |
-
border-bottom-right-radius: 4px;
|
| 457 |
-
}
|
| 458 |
-
.bubble.bot {
|
| 459 |
-
background: rgba(30, 41, 59, 0.8);
|
| 460 |
-
border: 1px solid rgba(59, 130, 246, 0.2);
|
| 461 |
-
color: #e2e8f0;
|
| 462 |
-
border-bottom-left-radius: 4px;
|
| 463 |
-
}
|
| 464 |
-
.bubble.bot .label { color: #60a5fa; font-weight: 600; font-size: 0.8rem; margin-bottom: 2px; }
|
| 465 |
-
.bubble.user .label { color: rgba(255,255,255,0.7); font-weight: 600; font-size: 0.8rem; margin-bottom: 2px; }
|
| 466 |
-
.bubble.typing {
|
| 467 |
-
color: #64748b;
|
| 468 |
-
font-style: italic;
|
| 469 |
-
}
|
| 470 |
-
.bubble.error {
|
| 471 |
-
background: rgba(127, 29, 29, 0.5);
|
| 472 |
-
border-color: rgba(239, 68, 68, 0.3);
|
| 473 |
-
color: #fca5a5;
|
| 474 |
-
}
|
| 475 |
-
@keyframes dots {
|
| 476 |
-
0%, 20% { content: ''; }
|
| 477 |
-
40% { content: '.'; }
|
| 478 |
-
60% { content: '..'; }
|
| 479 |
-
80%, 100% { content: '...'; }
|
| 480 |
-
}
|
| 481 |
-
.typing-dots::after {
|
| 482 |
-
content: '';
|
| 483 |
-
animation: dots 1.5s infinite;
|
| 484 |
-
}
|
| 485 |
-
.streaming-cursor::after {
|
| 486 |
-
content: '▊';
|
| 487 |
-
animation: blink 0.7s infinite;
|
| 488 |
-
color: #60a5fa;
|
| 489 |
-
}
|
| 490 |
-
@keyframes blink { 0%, 100% { opacity: 1; } 50% { opacity: 0; } }
|
| 491 |
-
/* --- INPUT BAR --- */
|
| 492 |
-
.input-bar {
|
| 493 |
-
display: flex;
|
| 494 |
-
gap: 0.5rem;
|
| 495 |
-
padding: 0.75rem 1rem;
|
| 496 |
-
background: rgba(15, 23, 42, 0.8);
|
| 497 |
-
border-top: 1px solid rgba(59, 130, 246, 0.1);
|
| 498 |
-
}
|
| 499 |
-
.input-bar input {
|
| 500 |
-
flex: 1;
|
| 501 |
-
background: rgba(30, 41, 59, 0.8);
|
| 502 |
-
border: 1px solid rgba(59, 130, 246, 0.25);
|
| 503 |
-
border-radius: 8px;
|
| 504 |
-
padding: 10px 14px;
|
| 505 |
-
color: #e2e8f0;
|
| 506 |
-
font-size: 0.9rem;
|
| 507 |
-
outline: none;
|
| 508 |
-
transition: border-color 0.2s;
|
| 509 |
-
}
|
| 510 |
-
.input-bar input:focus {
|
| 511 |
-
border-color: #3b82f6;
|
| 512 |
-
}
|
| 513 |
-
.input-bar input::placeholder { color: #64748b; }
|
| 514 |
-
.input-bar button {
|
| 515 |
-
padding: 10px 20px;
|
| 516 |
-
border: none;
|
| 517 |
-
border-radius: 8px;
|
| 518 |
-
font-size: 0.9rem;
|
| 519 |
-
cursor: pointer;
|
| 520 |
-
transition: all 0.15s;
|
| 521 |
-
font-weight: 500;
|
| 522 |
-
}
|
| 523 |
-
.btn-send {
|
| 524 |
-
background: #3b82f6;
|
| 525 |
-
color: #fff;
|
| 526 |
-
}
|
| 527 |
-
.btn-send:hover { background: #2563eb; }
|
| 528 |
-
.btn-send:disabled { background: #334155; color: #64748b; cursor: not-allowed; }
|
| 529 |
-
.btn-reset {
|
| 530 |
-
background: rgba(100, 116, 139, 0.3);
|
| 531 |
-
color: #94a3b8;
|
| 532 |
-
border: 1px solid rgba(100, 116, 139, 0.3);
|
| 533 |
-
}
|
| 534 |
-
.btn-reset:hover { background: rgba(100, 116, 139, 0.5); }
|
| 535 |
</style>
|
| 536 |
</head>
|
| 537 |
<body>
|
| 538 |
<div class="app-container">
|
| 539 |
-
<!-- VIDEO PLAYER with crossfade -->
|
| 540 |
<div class="video-section">
|
| 541 |
<div class="video-container">
|
| 542 |
<video id="videoA" muted autoplay playsinline></video>
|
|
@@ -544,306 +258,161 @@ HTML_PAGE = """\
|
|
| 544 |
</div>
|
| 545 |
<div class="video-status" id="videoStatus">Loading...</div>
|
| 546 |
</div>
|
| 547 |
-
<!-- CHAT -->
|
| 548 |
<div class="chat-section">
|
| 549 |
<div class="chat-log" id="chatLog"></div>
|
| 550 |
<div class="input-bar">
|
| 551 |
-
<input type="text" id="chatInput" placeholder="Ask Darwin something..."
|
| 552 |
-
autocomplete="off" />
|
| 553 |
<button class="btn-send" id="btnSend">Send</button>
|
| 554 |
-
<button class="btn-reset" id="btnReset" title="Reset conversation">
|
| 555 |
</div>
|
| 556 |
</div>
|
| 557 |
</div>
|
| 558 |
<script>
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
const
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
setTimeout(() => {
|
| 596 |
-
prev.style.opacity = '0';
|
| 597 |
-
prev.pause();
|
| 598 |
-
prev.src = '';
|
| 599 |
-
prev.style.zIndex = '0';
|
| 600 |
-
next.style.zIndex = '1';
|
| 601 |
-
currentVideo = currentVideo === 'A' ? 'B' : 'A';
|
| 602 |
-
isTransitioning = false;
|
| 603 |
-
}, 300);
|
| 604 |
-
}).catch(err => {
|
| 605 |
-
console.error('[VIDEO] Play failed:', err);
|
| 606 |
-
isTransitioning = false;
|
| 607 |
-
// Retry once on autoplay failure
|
| 608 |
-
next.muted = true;
|
| 609 |
-
next.play().then(() => {
|
| 610 |
-
next.style.opacity = '1';
|
| 611 |
-
setTimeout(() => {
|
| 612 |
-
prev.style.opacity = '0';
|
| 613 |
-
prev.pause();
|
| 614 |
-
prev.src = '';
|
| 615 |
-
prev.style.zIndex = '0';
|
| 616 |
-
next.style.zIndex = '1';
|
| 617 |
-
currentVideo = currentVideo === 'A' ? 'B' : 'A';
|
| 618 |
-
isTransitioning = false;
|
| 619 |
-
}, 300);
|
| 620 |
-
}).catch(e => {
|
| 621 |
-
console.error('[VIDEO] Muted retry also failed:', e);
|
| 622 |
-
isTransitioning = false;
|
| 623 |
-
});
|
| 624 |
});
|
| 625 |
};
|
| 626 |
-
next.onerror
|
| 627 |
-
console.error('[VIDEO] Load failed:', videoUrl);
|
| 628 |
-
isTransitioning = false;
|
| 629 |
-
};
|
| 630 |
}
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
if
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
} while (pick.filename === lastPlayed && videos.length > 1);
|
| 638 |
-
return pick;
|
| 639 |
-
}
|
| 640 |
-
function playNextIdle() {
|
| 641 |
-
const clip = pickRandomIdle();
|
| 642 |
-
if (!clip) return;
|
| 643 |
-
lastPlayed = clip.filename;
|
| 644 |
-
videoStatus.textContent = 'idle: ' + clip.name;
|
| 645 |
-
crossfadeTo('/api/stream/' + encodeURIComponent(clip.filename), true);
|
| 646 |
-
}
|
| 647 |
-
// When active video ends → play next idle OR next queued lipsync
|
| 648 |
-
videoA.addEventListener('ended', () => { if (currentVideo === 'A') onVideoEnded(); });
|
| 649 |
-
videoB.addEventListener('ended', () => { if (currentVideo === 'B') onVideoEnded(); });
|
| 650 |
-
let lipsyncQueue = [];
|
| 651 |
-
let playingLipsync = false;
|
| 652 |
-
function onVideoEnded() {
|
| 653 |
-
if (lipsyncQueue.length > 0 && !playingLipsync) {
|
| 654 |
-
playNextLipsync();
|
| 655 |
-
} else if (!playingLipsync) {
|
| 656 |
-
playNextIdle();
|
| 657 |
-
}
|
| 658 |
}
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
playingLipsync = true;
|
| 666 |
-
const item = lipsyncQueue.shift();
|
| 667 |
-
videoStatus.textContent = 'speaking...';
|
| 668 |
-
// Play audio
|
| 669 |
-
if (item.audio_url) {
|
| 670 |
-
const audio = new Audio(item.audio_url);
|
| 671 |
-
audio.play().catch(e => console.warn('[AUDIO] Play failed:', e));
|
| 672 |
-
}
|
| 673 |
-
if (item.lipsync_video_url) {
|
| 674 |
-
crossfadeTo(item.lipsync_video_url, true);
|
| 675 |
-
// The 'ended' event on the video will trigger onVideoEnded → playNextLipsync
|
| 676 |
-
// But we need to set playingLipsync=false when THIS lipsync video ends
|
| 677 |
-
const checkEnd = () => {
|
| 678 |
-
const active = getActiveVideo();
|
| 679 |
-
// Wait for crossfade to complete, then listen for ended
|
| 680 |
-
setTimeout(() => {
|
| 681 |
-
const nowActive = getActiveVideo();
|
| 682 |
-
const onEnd = () => {
|
| 683 |
-
nowActive.removeEventListener('ended', onEnd);
|
| 684 |
-
playingLipsync = false;
|
| 685 |
-
onVideoEnded(); // chain to next
|
| 686 |
-
};
|
| 687 |
-
nowActive.addEventListener('ended', onEnd);
|
| 688 |
-
}, 500);
|
| 689 |
-
};
|
| 690 |
-
checkEnd();
|
| 691 |
-
} else {
|
| 692 |
-
// No lipsync video, just play audio over idle, then move to next
|
| 693 |
-
if (item.audio_url) {
|
| 694 |
-
const waitMs = (item.audio_duration || 3) * 1000;
|
| 695 |
-
setTimeout(() => {
|
| 696 |
-
playingLipsync = false;
|
| 697 |
-
onVideoEnded();
|
| 698 |
-
}, waitMs);
|
| 699 |
-
} else {
|
| 700 |
-
playingLipsync = false;
|
| 701 |
-
onVideoEnded();
|
| 702 |
-
}
|
| 703 |
-
}
|
| 704 |
}
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
function addMessage(role, text,
|
| 710 |
-
const id
|
| 711 |
-
const
|
| 712 |
-
|
| 713 |
-
const
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
label.textContent = role === 'user' ? 'You' : 'Darwin';
|
| 719 |
-
const content = document.createElement('div');
|
| 720 |
-
content.className = 'content';
|
| 721 |
-
content.textContent = text;
|
| 722 |
-
bubble.appendChild(label);
|
| 723 |
-
bubble.appendChild(content);
|
| 724 |
-
wrapper.appendChild(bubble);
|
| 725 |
-
chatLog.appendChild(wrapper);
|
| 726 |
-
chatLog.scrollTop = chatLog.scrollHeight;
|
| 727 |
-
return { id, contentEl: content, bubbleEl: bubble };
|
| 728 |
}
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
contentEl.
|
| 733 |
-
return
|
| 734 |
}
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
let i
|
| 739 |
-
const
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
i++;
|
| 744 |
-
chatLog.scrollTop = chatLog.scrollHeight;
|
| 745 |
-
} else {
|
| 746 |
-
clearInterval(interval);
|
| 747 |
-
contentEl.classList.remove('streaming-cursor');
|
| 748 |
-
}
|
| 749 |
-
}, delay);
|
| 750 |
}
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
if (!text || isProcessing) return;
|
| 757 |
-
isProcessing = true;
|
| 758 |
-
btnSend.disabled = true;
|
| 759 |
-
chatInput.value = '';
|
| 760 |
-
// Show user message
|
| 761 |
addMessage('user', text);
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
try
|
| 765 |
-
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
headers: { 'Content-Type': 'application/json' },
|
| 769 |
-
body: JSON.stringify({ text })
|
| 770 |
});
|
| 771 |
-
if
|
| 772 |
-
const data
|
| 773 |
-
|
| 774 |
typing.bubbleEl.classList.remove('typing');
|
| 775 |
typing.contentEl.classList.remove('typing-dots');
|
| 776 |
-
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
// Queue the lipsync chunk
|
| 793 |
-
lipsyncQueue.push(chunk);
|
| 794 |
-
// If nothing is currently playing lipsync, start
|
| 795 |
-
if (!playingLipsync) {
|
| 796 |
-
playNextLipsync();
|
| 797 |
-
}
|
| 798 |
-
} catch (e) {
|
| 799 |
-
console.error('[CHAT] Chunk generation failed:', e);
|
| 800 |
-
}
|
| 801 |
}
|
| 802 |
-
}
|
| 803 |
typing.bubbleEl.classList.remove('typing');
|
| 804 |
typing.bubbleEl.classList.add('error');
|
| 805 |
typing.contentEl.classList.remove('typing-dots');
|
| 806 |
-
typing.contentEl.textContent
|
| 807 |
-
isProcessing = false;
|
| 808 |
-
btnSend.disabled = false;
|
| 809 |
}
|
|
|
|
| 810 |
}
|
|
|
|
| 811 |
btnSend.addEventListener('click', sendMessage);
|
| 812 |
-
chatInput.addEventListener('keydown',
|
| 813 |
-
|
| 814 |
-
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
});
|
| 818 |
-
btnReset.addEventListener('click', async () => {
|
| 819 |
-
await fetch('/api/reset', { method: 'POST' });
|
| 820 |
-
chatLog.innerHTML = '';
|
| 821 |
-
addMessage('bot', 'Ah, a fresh start! How may I assist you, my good fellow?');
|
| 822 |
});
|
| 823 |
-
|
| 824 |
-
|
| 825 |
-
|
| 826 |
-
|
| 827 |
-
|
| 828 |
-
|
| 829 |
-
|
| 830 |
-
|
| 831 |
-
videoStatus.textContent = 'Downloading idle videos...';
|
| 832 |
-
setTimeout(init, 3000);
|
| 833 |
-
return;
|
| 834 |
-
}
|
| 835 |
-
videos = data.videos;
|
| 836 |
-
videoStatus.textContent = videos.length + ' idle clips loaded';
|
| 837 |
-
// Start idle loop
|
| 838 |
playNextIdle();
|
| 839 |
-
|
| 840 |
-
|
| 841 |
-
'Good day! I am Charles Darwin, er, recreated in this curious digital form. ' +
|
| 842 |
-
'Do feel free to ask me anything, umm, about the natural world or otherwise.'
|
| 843 |
-
);
|
| 844 |
-
} catch (e) {
|
| 845 |
-
videoStatus.textContent = 'Error: ' + e.message;
|
| 846 |
-
}
|
| 847 |
}
|
| 848 |
init();
|
| 849 |
</script>
|
|
|
|
| 1 |
"""
|
| 2 |
Avatar Chatbot - HuggingFace Spaces Edition
|
| 3 |
+
Single response per message: LLM → TTS → FLOAT lipsync → play
|
|
|
|
| 4 |
"""
|
| 5 |
from fastapi import FastAPI, HTTPException, Request
|
| 6 |
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
import mimetypes
|
| 9 |
import os
|
| 10 |
import logging
|
|
|
|
|
|
|
| 11 |
import subprocess
|
| 12 |
|
| 13 |
logging.basicConfig(level=logging.INFO)
|
|
|
|
| 15 |
|
| 16 |
app = FastAPI()
|
| 17 |
|
|
|
|
| 18 |
VIDEOS_DIR = Path("/tmp/videos")
|
| 19 |
TTS_DIR = Path("/tmp/tts_output")
|
| 20 |
LIPSYNC_DIR = Path("/tmp/lipsync_output")
|
| 21 |
REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot")
|
|
|
|
| 22 |
FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true"
|
| 23 |
|
| 24 |
for d in [VIDEOS_DIR, TTS_DIR, LIPSYNC_DIR]:
|
| 25 |
d.mkdir(parents=True, exist_ok=True)
|
| 26 |
|
| 27 |
|
|
|
|
| 28 |
@app.on_event("startup")
|
| 29 |
async def startup():
|
| 30 |
await download_videos()
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
async def init_float():
|
|
|
|
| 36 |
import torch
|
| 37 |
logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}")
|
| 38 |
if not torch.cuda.is_available():
|
| 39 |
+
logger.warning("[STARTUP] No GPU - FLOAT disabled")
|
| 40 |
return
|
|
|
|
| 41 |
try:
|
| 42 |
from float_lipsync import get_lipsync
|
| 43 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 44 |
import shutil
|
| 45 |
|
| 46 |
+
ckpt = Path("/app/checkpoints")
|
| 47 |
+
ckpt.mkdir(parents=True, exist_ok=True)
|
| 48 |
|
| 49 |
+
w2v = ckpt / "wav2vec2-base-960h"
|
| 50 |
+
if not w2v.exists():
|
|
|
|
| 51 |
logger.info("[STARTUP] Downloading wav2vec2-base-960h...")
|
| 52 |
+
snapshot_download(repo_id="facebook/wav2vec2-base-960h", local_dir=str(w2v))
|
| 53 |
+
|
| 54 |
+
emo = ckpt / "wav2vec-english-speech-emotion-recognition"
|
| 55 |
+
if not emo.exists():
|
| 56 |
+
logger.info("[STARTUP] Downloading emotion model...")
|
| 57 |
+
snapshot_download(repo_id="r-f/wav2vec-english-speech-emotion-recognition", local_dir=str(emo))
|
| 58 |
+
|
| 59 |
+
fp = ckpt / "float.pth"
|
| 60 |
+
if not fp.exists() or fp.stat().st_size < 1024:
|
| 61 |
+
logger.info("[STARTUP] Downloading float.pth...")
|
| 62 |
+
dl = hf_hub_download(repo_id=REPO_ID, repo_type="space",
|
| 63 |
+
filename="app/checkpoints/float.pth", local_dir="/tmp/hf_download")
|
| 64 |
+
shutil.copy2(dl, fp)
|
| 65 |
+
logger.info(f"[STARTUP] float.pth: {fp.stat().st_size/1e6:.1f} MB")
|
| 66 |
+
|
| 67 |
+
assets = Path("/app/assets")
|
| 68 |
+
assets.mkdir(parents=True, exist_ok=True)
|
| 69 |
+
ref = assets / "ref.png"
|
| 70 |
+
if not ref.exists() or ref.stat().st_size < 1024:
|
| 71 |
+
for name in ["ref.png", "ref.jpg", "main2.png"]:
|
| 72 |
+
try:
|
| 73 |
+
dl = hf_hub_download(repo_id=REPO_ID, repo_type="space",
|
| 74 |
+
filename=f"app/assets/{name}", local_dir="/tmp/hf_download")
|
| 75 |
+
shutil.copy2(dl, assets / name)
|
| 76 |
+
ref = assets / name
|
| 77 |
+
break
|
| 78 |
+
except Exception:
|
| 79 |
+
continue
|
| 80 |
+
else:
|
| 81 |
+
logger.warning("[STARTUP] No ref image found"); return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
+
lipsync = get_lipsync()
|
| 84 |
+
lipsync.initialize({
|
| 85 |
+
"ref_path": str(ref), "ckpt_path": str(fp),
|
| 86 |
+
"wav2vec_model_path": str(w2v), "audio2emotion_path": str(emo),
|
| 87 |
+
})
|
| 88 |
+
logger.info("[STARTUP] ✓ FLOAT ready!")
|
| 89 |
except Exception as e:
|
| 90 |
logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True)
|
| 91 |
|
|
|
|
| 93 |
async def download_videos():
|
| 94 |
from huggingface_hub import HfApi, hf_hub_download
|
| 95 |
import shutil
|
|
|
|
| 96 |
try:
|
| 97 |
api = HfApi()
|
| 98 |
files = api.list_repo_files(repo_id=REPO_ID, repo_type="space")
|
| 99 |
+
vids = [f for f in files if f.startswith("app/videos/") and f.endswith((".mp4",".webm",".mkv",".mov"))]
|
| 100 |
+
logger.info(f"Found {len(vids)} video files")
|
| 101 |
+
for vf in vids:
|
| 102 |
+
dest = VIDEOS_DIR / Path(vf).name
|
| 103 |
+
if dest.exists() and dest.stat().st_size > 1024: continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
logger.info(f"Downloading {vf}...")
|
| 105 |
+
dl = hf_hub_download(repo_id=REPO_ID, repo_type="space", filename=vf, local_dir="/tmp/hf_download")
|
| 106 |
+
shutil.copy2(dl, dest)
|
| 107 |
+
logger.info(f"Downloaded {dest.name}: {dest.stat().st_size} bytes")
|
|
|
|
|
|
|
|
|
|
| 108 |
except Exception as e:
|
| 109 |
logger.error(f"Error downloading videos: {e}")
|
| 110 |
|
| 111 |
|
|
|
|
| 112 |
@app.get("/api/videos")
|
| 113 |
async def list_videos():
|
| 114 |
+
if not VIDEOS_DIR.exists(): return JSONResponse({"videos": []})
|
| 115 |
+
return JSONResponse({"videos": [
|
| 116 |
+
{"name": f.stem, "filename": f.name}
|
| 117 |
+
for f in sorted(VIDEOS_DIR.iterdir())
|
| 118 |
+
if f.suffix.lower() in (".mp4",".webm",".mkv",".mov") and f.stat().st_size > 1024
|
| 119 |
+
]})
|
|
|
|
| 120 |
|
| 121 |
|
|
|
|
| 122 |
@app.get("/api/stream/{filename}")
|
| 123 |
async def stream_video(filename: str):
|
| 124 |
+
path = None
|
| 125 |
+
for d in [VIDEOS_DIR, LIPSYNC_DIR, TTS_DIR]:
|
| 126 |
+
c = d / filename
|
| 127 |
+
if c.exists() and c.is_file(): path = c; break
|
| 128 |
+
if not path: raise HTTPException(404, "Not found")
|
| 129 |
+
mime = mimetypes.guess_type(filename)[0] or "video/mp4"
|
| 130 |
+
def gen():
|
| 131 |
+
with open(path, "rb") as f:
|
| 132 |
+
while chunk := f.read(1024*1024): yield chunk
|
| 133 |
+
return StreamingResponse(gen(), media_type=mime,
|
| 134 |
+
headers={"Content-Length": str(path.stat().st_size), "Accept-Ranges": "bytes"})
|
| 135 |
|
|
|
|
|
|
|
| 136 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
@app.get("/api/audio/{filename}")
|
| 138 |
async def stream_audio(filename: str):
|
| 139 |
+
path = TTS_DIR / filename
|
| 140 |
+
if not path.exists(): raise HTTPException(404, "Not found")
|
| 141 |
+
mime = mimetypes.guess_type(filename)[0] or "audio/wav"
|
| 142 |
+
def gen():
|
| 143 |
+
with open(path, "rb") as f:
|
| 144 |
+
while chunk := f.read(1024*1024): yield chunk
|
| 145 |
+
return StreamingResponse(gen(), media_type=mime,
|
| 146 |
+
headers={"Content-Length": str(path.stat().st_size)})
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
@app.post("/api/chat")
|
| 150 |
async def chat(request: Request):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
body = await request.json()
|
| 152 |
user_text = body.get("text", "").strip()
|
| 153 |
+
if not user_text: raise HTTPException(400, "No text")
|
|
|
|
|
|
|
| 154 |
|
| 155 |
logger.info(f"[CHAT] User: {user_text}")
|
| 156 |
|
|
|
|
| 157 |
from llm import generate_darwin_response
|
| 158 |
llm_result = generate_darwin_response(user_text)
|
| 159 |
reply_text = llm_result['text']
|
| 160 |
logger.info(f"[CHAT] Darwin: {reply_text}")
|
| 161 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
from tts import generate_audio
|
| 163 |
+
audio_path = generate_audio(reply_text)
|
| 164 |
audio_url = None
|
| 165 |
audio_duration = 0
|
|
|
|
| 166 |
if audio_path and os.path.exists(audio_path):
|
| 167 |
+
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
|
|
|
|
|
|
| 168 |
try:
|
| 169 |
+
r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
|
| 170 |
+
"-of","csv=p=0",audio_path], capture_output=True, text=True, check=True)
|
| 171 |
+
audio_duration = float(r.stdout.strip())
|
|
|
|
|
|
|
|
|
|
| 172 |
except Exception:
|
| 173 |
audio_duration = 3.0
|
| 174 |
|
|
|
|
| 175 |
lipsync_video_url = None
|
| 176 |
if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
|
| 177 |
try:
|
| 178 |
from float_lipsync import get_lipsync
|
| 179 |
lipsync = get_lipsync()
|
| 180 |
if lipsync.ready:
|
| 181 |
+
logger.info("[CHAT] Running FLOAT lipsync...")
|
| 182 |
+
lp = lipsync.generate(audio_path)
|
| 183 |
+
if lp and os.path.exists(lp):
|
| 184 |
+
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
| 185 |
+
logger.info(f"[CHAT] Lipsync: {lipsync_video_url}")
|
| 186 |
+
else:
|
| 187 |
+
logger.info("[CHAT] FLOAT not ready")
|
| 188 |
except Exception as e:
|
| 189 |
+
logger.error(f"[CHAT] Lipsync failed: {e}", exc_info=True)
|
| 190 |
|
| 191 |
return JSONResponse({
|
| 192 |
+
"text": reply_text,
|
| 193 |
+
"emotion": llm_result.get('emotion','neutral'),
|
| 194 |
"audio_url": audio_url,
|
| 195 |
"audio_duration": audio_duration,
|
| 196 |
"lipsync_video_url": lipsync_video_url,
|
| 197 |
})
|
| 198 |
|
| 199 |
|
|
|
|
| 200 |
@app.post("/api/reset")
|
| 201 |
async def reset_chat():
|
| 202 |
from llm import reset_conversation
|
|
|
|
| 204 |
return JSONResponse({"status": "ok"})
|
| 205 |
|
| 206 |
|
|
|
|
| 207 |
@app.get("/", response_class=HTMLResponse)
|
| 208 |
async def index():
|
| 209 |
return HTML_PAGE
|
|
|
|
| 213 |
<!DOCTYPE html>
|
| 214 |
<html lang="en">
|
| 215 |
<head>
|
| 216 |
+
<meta charset="UTF-8"/>
|
| 217 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
| 218 |
<title>Darwin Avatar Chatbot</title>
|
| 219 |
<style>
|
| 220 |
+
*{margin:0;padding:0;box-sizing:border-box}
|
| 221 |
+
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0f1c;color:#e2e8f0;height:100vh;display:flex;flex-direction:column;overflow:hidden}
|
| 222 |
+
.app-container{display:flex;flex-direction:column;height:100vh;max-width:900px;margin:0 auto;width:100%;padding:1rem;gap:.75rem}
|
| 223 |
+
.video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center}
|
| 224 |
+
.video-container{position:relative;width:100%;height:100%}
|
| 225 |
+
.video-container video{width:100%;height:100%;object-fit:contain;display:block}
|
| 226 |
+
#videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity .25s ease}
|
| 227 |
+
#videoA{z-index:1;opacity:1}#videoB{z-index:0;opacity:0}
|
| 228 |
+
.video-status{position:absolute;bottom:8px;right:12px;font-size:.7rem;color:rgba(255,255,255,.4);background:rgba(0,0,0,.5);padding:2px 8px;border-radius:4px;z-index:10}
|
| 229 |
+
.chat-section{flex:1;display:flex;flex-direction:column;min-height:0;background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden}
|
| 230 |
+
.chat-log{flex:1;overflow-y:auto;padding:1rem;display:flex;flex-direction:column;gap:.5rem}
|
| 231 |
+
.chat-log::-webkit-scrollbar{width:6px}.chat-log::-webkit-scrollbar-track{background:transparent}.chat-log::-webkit-scrollbar-thumb{background:#334155;border-radius:3px}
|
| 232 |
+
.msg{display:flex;width:100%}.msg.user{justify-content:flex-end}.msg.bot{justify-content:flex-start}
|
| 233 |
+
.bubble{max-width:85%;padding:10px 14px;border-radius:12px;word-wrap:break-word;font-size:.9rem;line-height:1.4}
|
| 234 |
+
.bubble.user{background:linear-gradient(135deg,#1e40af,#3b82f6);color:#fff;border-bottom-right-radius:4px}
|
| 235 |
+
.bubble.bot{background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.2);color:#e2e8f0;border-bottom-left-radius:4px}
|
| 236 |
+
.bubble.bot .label{color:#60a5fa;font-weight:600;font-size:.8rem;margin-bottom:2px}
|
| 237 |
+
.bubble.user .label{color:rgba(255,255,255,.7);font-weight:600;font-size:.8rem;margin-bottom:2px}
|
| 238 |
+
.bubble.typing{color:#64748b;font-style:italic}
|
| 239 |
+
.bubble.error{background:rgba(127,29,29,.5);border-color:rgba(239,68,68,.3);color:#fca5a5}
|
| 240 |
+
@keyframes dots{0%,20%{content:''}40%{content:'.'}60%{content:'..'}80%,100%{content:'...'}}
|
| 241 |
+
.typing-dots::after{content:'';animation:dots 1.5s infinite}
|
| 242 |
+
.streaming-cursor::after{content:'\\25CA';animation:blink .7s infinite;color:#60a5fa}
|
| 243 |
+
@keyframes blink{0%,100%{opacity:1}50%{opacity:0}}
|
| 244 |
+
.input-bar{display:flex;gap:.5rem;padding:.75rem 1rem;background:rgba(15,23,42,.8);border-top:1px solid rgba(59,130,246,.1)}
|
| 245 |
+
.input-bar input{flex:1;background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.25);border-radius:8px;padding:10px 14px;color:#e2e8f0;font-size:.9rem;outline:none;transition:border-color .2s}
|
| 246 |
+
.input-bar input:focus{border-color:#3b82f6}.input-bar input::placeholder{color:#64748b}
|
| 247 |
+
.input-bar button{padding:10px 20px;border:none;border-radius:8px;font-size:.9rem;cursor:pointer;transition:all .15s;font-weight:500}
|
| 248 |
+
.btn-send{background:#3b82f6;color:#fff}.btn-send:hover{background:#2563eb}.btn-send:disabled{background:#334155;color:#64748b;cursor:not-allowed}
|
| 249 |
+
.btn-reset{background:rgba(100,116,139,.3);color:#94a3b8;border:1px solid rgba(100,116,139,.3)}.btn-reset:hover{background:rgba(100,116,139,.5)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
</style>
|
| 251 |
</head>
|
| 252 |
<body>
|
| 253 |
<div class="app-container">
|
|
|
|
| 254 |
<div class="video-section">
|
| 255 |
<div class="video-container">
|
| 256 |
<video id="videoA" muted autoplay playsinline></video>
|
|
|
|
| 258 |
</div>
|
| 259 |
<div class="video-status" id="videoStatus">Loading...</div>
|
| 260 |
</div>
|
|
|
|
| 261 |
<div class="chat-section">
|
| 262 |
<div class="chat-log" id="chatLog"></div>
|
| 263 |
<div class="input-bar">
|
| 264 |
+
<input type="text" id="chatInput" placeholder="Ask Darwin something..." autocomplete="off"/>
|
|
|
|
| 265 |
<button class="btn-send" id="btnSend">Send</button>
|
| 266 |
+
<button class="btn-reset" id="btnReset" title="Reset conversation">↻</button>
|
| 267 |
</div>
|
| 268 |
</div>
|
| 269 |
</div>
|
| 270 |
<script>
|
| 271 |
+
let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, lastPlayed=null, msgCounter=0;
|
| 272 |
+
const videoA=document.getElementById('videoA'), videoB=document.getElementById('videoB');
|
| 273 |
+
const videoStatus=document.getElementById('videoStatus'), chatLog=document.getElementById('chatLog');
|
| 274 |
+
const chatInput=document.getElementById('chatInput'), btnSend=document.getElementById('btnSend');
|
| 275 |
+
const btnReset=document.getElementById('btnReset');
|
| 276 |
+
|
| 277 |
+
function getActiveVideo(){return currentVideo==='A'?videoA:videoB}
|
| 278 |
+
function getInactiveVideo(){return currentVideo==='A'?videoB:videoA}
|
| 279 |
+
|
| 280 |
+
function crossfadeTo(url, muted){
|
| 281 |
+
if(isTransitioning) return;
|
| 282 |
+
isTransitioning=true;
|
| 283 |
+
const prev=getActiveVideo(), next=getInactiveVideo();
|
| 284 |
+
next.muted=(muted!==false);
|
| 285 |
+
next.src=url; next.load();
|
| 286 |
+
next.onloadeddata=function(){
|
| 287 |
+
next.style.zIndex='2'; prev.style.zIndex='1'; next.style.opacity='0';
|
| 288 |
+
next.play().then(()=>{
|
| 289 |
+
next.style.opacity='1';
|
| 290 |
+
setTimeout(()=>{
|
| 291 |
+
prev.style.opacity='0'; prev.pause(); prev.src='';
|
| 292 |
+
prev.style.zIndex='0'; next.style.zIndex='1';
|
| 293 |
+
currentVideo=currentVideo==='A'?'B':'A';
|
| 294 |
+
isTransitioning=false;
|
| 295 |
+
},300);
|
| 296 |
+
}).catch(()=>{
|
| 297 |
+
next.muted=true;
|
| 298 |
+
next.play().then(()=>{
|
| 299 |
+
next.style.opacity='1';
|
| 300 |
+
setTimeout(()=>{
|
| 301 |
+
prev.style.opacity='0'; prev.pause(); prev.src='';
|
| 302 |
+
prev.style.zIndex='0'; next.style.zIndex='1';
|
| 303 |
+
currentVideo=currentVideo==='A'?'B':'A';
|
| 304 |
+
isTransitioning=false;
|
| 305 |
+
},300);
|
| 306 |
+
}).catch(()=>{ isTransitioning=false; });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
});
|
| 308 |
};
|
| 309 |
+
next.onerror=function(){ isTransitioning=false; };
|
|
|
|
|
|
|
|
|
|
| 310 |
}
|
| 311 |
+
|
| 312 |
+
function pickRandomIdle(){
|
| 313 |
+
if(!videos.length) return null;
|
| 314 |
+
if(videos.length===1) return videos[0];
|
| 315 |
+
let p; do{ p=videos[Math.floor(Math.random()*videos.length)]; }while(p.filename===lastPlayed&&videos.length>1);
|
| 316 |
+
return p;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
}
|
| 318 |
+
|
| 319 |
+
function playNextIdle(){
|
| 320 |
+
const c=pickRandomIdle(); if(!c) return;
|
| 321 |
+
lastPlayed=c.filename;
|
| 322 |
+
videoStatus.textContent='idle: '+c.name;
|
| 323 |
+
crossfadeTo('/api/stream/'+encodeURIComponent(c.filename), true);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
}
|
| 325 |
+
|
| 326 |
+
videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isProcessing) playNextIdle(); });
|
| 327 |
+
videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isProcessing) playNextIdle(); });
|
| 328 |
+
|
| 329 |
+
function addMessage(role, text, extra){
|
| 330 |
+
const id='msg_'+(msgCounter++);
|
| 331 |
+
const w=document.createElement('div'); w.className='msg '+role;
|
| 332 |
+
const b=document.createElement('div'); b.className='bubble '+role+(extra?' '+extra:''); b.id=id;
|
| 333 |
+
const l=document.createElement('div'); l.className='label'; l.textContent=role==='user'?'You':'Darwin';
|
| 334 |
+
const c=document.createElement('div'); c.className='content'; c.textContent=text;
|
| 335 |
+
b.appendChild(l); b.appendChild(c); w.appendChild(b);
|
| 336 |
+
chatLog.appendChild(w); chatLog.scrollTop=chatLog.scrollHeight;
|
| 337 |
+
return {id:id, contentEl:c, bubbleEl:b};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
}
|
| 339 |
+
|
| 340 |
+
function addTypingIndicator(){
|
| 341 |
+
const r=addMessage('bot','','typing');
|
| 342 |
+
r.contentEl.className='content typing-dots'; r.contentEl.textContent='thinking';
|
| 343 |
+
return r;
|
| 344 |
}
|
| 345 |
+
|
| 346 |
+
function streamText(el, text, ms){
|
| 347 |
+
el.className='content streaming-cursor'; el.textContent='';
|
| 348 |
+
let i=0; const d=ms/text.length;
|
| 349 |
+
const iv=setInterval(()=>{
|
| 350 |
+
if(i<text.length){ el.textContent+=text[i]; i++; chatLog.scrollTop=chatLog.scrollHeight; }
|
| 351 |
+
else{ clearInterval(iv); el.classList.remove('streaming-cursor'); }
|
| 352 |
+
}, d);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
}
|
| 354 |
+
|
| 355 |
+
async function sendMessage(){
|
| 356 |
+
const text=chatInput.value.trim();
|
| 357 |
+
if(!text||isProcessing) return;
|
| 358 |
+
isProcessing=true; btnSend.disabled=true; chatInput.value='';
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
addMessage('user', text);
|
| 360 |
+
const typing=addTypingIndicator();
|
| 361 |
+
|
| 362 |
+
try{
|
| 363 |
+
const res=await fetch('/api/chat',{
|
| 364 |
+
method:'POST', headers:{'Content-Type':'application/json'},
|
| 365 |
+
body:JSON.stringify({text:text})
|
|
|
|
|
|
|
| 366 |
});
|
| 367 |
+
if(!res.ok) throw new Error('HTTP '+res.status);
|
| 368 |
+
const data=await res.json();
|
| 369 |
+
|
| 370 |
typing.bubbleEl.classList.remove('typing');
|
| 371 |
typing.contentEl.classList.remove('typing-dots');
|
| 372 |
+
const dur=data.audio_duration?data.audio_duration*1000:2000;
|
| 373 |
+
streamText(typing.contentEl, data.text, dur);
|
| 374 |
+
|
| 375 |
+
if(data.audio_url){
|
| 376 |
+
const audio=new Audio(data.audio_url);
|
| 377 |
+
audio.play().catch(e=>console.warn('[AUDIO]',e));
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
if(data.lipsync_video_url){
|
| 381 |
+
videoStatus.textContent='speaking...';
|
| 382 |
+
crossfadeTo(data.lipsync_video_url, true);
|
| 383 |
+
setTimeout(()=>{
|
| 384 |
+
const active=getActiveVideo();
|
| 385 |
+
const onEnd=()=>{ active.removeEventListener('ended',onEnd); playNextIdle(); };
|
| 386 |
+
active.addEventListener('ended',onEnd);
|
| 387 |
+
},500);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
}
|
| 389 |
+
}catch(err){
|
| 390 |
typing.bubbleEl.classList.remove('typing');
|
| 391 |
typing.bubbleEl.classList.add('error');
|
| 392 |
typing.contentEl.classList.remove('typing-dots');
|
| 393 |
+
typing.contentEl.textContent='Error: '+err.message;
|
|
|
|
|
|
|
| 394 |
}
|
| 395 |
+
isProcessing=false; btnSend.disabled=false; chatInput.focus();
|
| 396 |
}
|
| 397 |
+
|
| 398 |
btnSend.addEventListener('click', sendMessage);
|
| 399 |
+
chatInput.addEventListener('keydown', e=>{ if(e.key==='Enter'&&!e.shiftKey){ e.preventDefault(); sendMessage(); }});
|
| 400 |
+
btnReset.addEventListener('click', async()=>{
|
| 401 |
+
await fetch('/api/reset',{method:'POST'});
|
| 402 |
+
chatLog.innerHTML='';
|
| 403 |
+
addMessage('bot','Ah, a fresh start! How may I assist you, my good fellow?');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
});
|
| 405 |
+
|
| 406 |
+
async function init(){
|
| 407 |
+
try{
|
| 408 |
+
const res=await fetch('/api/videos');
|
| 409 |
+
const data=await res.json();
|
| 410 |
+
if(!data.videos.length){ videoStatus.textContent='Downloading...'; setTimeout(init,3000); return; }
|
| 411 |
+
videos=data.videos;
|
| 412 |
+
videoStatus.textContent=videos.length+' idle clips loaded';
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 413 |
playNextIdle();
|
| 414 |
+
addMessage('bot','Good day! I am Charles Darwin, er, recreated in this curious digital form. Do feel free to ask me anything, umm, about the natural world or otherwise.');
|
| 415 |
+
}catch(e){ videoStatus.textContent='Error: '+e.message; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
}
|
| 417 |
init();
|
| 418 |
</script>
|