J Z commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
Avatar Chatbot - HuggingFace Spaces Edition
|
|
|
|
| 3 |
Chunked pipeline: LLM (once) → split sentences → TTS+FLOAT per chunk → stream to frontend
|
| 4 |
"""
|
| 5 |
from fastapi import FastAPI, HTTPException, Request
|
|
@@ -18,33 +19,136 @@ logger = logging.getLogger(__name__)
|
|
| 18 |
|
| 19 |
app = FastAPI()
|
| 20 |
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
| 22 |
TTS_DIR = Path("/tmp/tts_output")
|
| 23 |
LIPSYNC_DIR = Path("/tmp/lipsync_output")
|
| 24 |
REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot")
|
| 25 |
FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true"
|
| 26 |
|
| 27 |
-
for d in [
|
| 28 |
d.mkdir(parents=True, exist_ok=True)
|
| 29 |
|
| 30 |
|
| 31 |
def _ts():
|
| 32 |
-
"""Wall-clock timestamp for logs."""
|
| 33 |
return _time.strftime("%H:%M:%S", _time.gmtime()) + f".{int(_time.time()*1000)%1000:03d}"
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
import threading
|
| 36 |
_gpu_lock = threading.Lock()
|
| 37 |
|
| 38 |
-
# In-memory store for chunked generation results
|
| 39 |
_sessions = {}
|
| 40 |
-
|
| 41 |
-
# Pre-generated greeting (warmup)
|
| 42 |
GREETING_DATA = None
|
| 43 |
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
@app.on_event("startup")
|
| 46 |
async def startup():
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
await init_tts()
|
| 49 |
if FLOAT_ENABLED:
|
| 50 |
await init_float()
|
|
@@ -52,12 +156,10 @@ async def startup():
|
|
| 52 |
|
| 53 |
|
| 54 |
async def init_tts():
|
| 55 |
-
"""Initialize local Orpheus TTS model."""
|
| 56 |
logger.info(f"[{_ts()}] [STARTUP] Initializing TTS...")
|
| 57 |
t0 = _time.time()
|
| 58 |
try:
|
| 59 |
-
import groq_tts as tts_module
|
| 60 |
-
logger.info(f"[{_ts()}] [STARTUP] TTS module imported OK")
|
| 61 |
tts_module.initialize()
|
| 62 |
logger.info(f"[{_ts()}] [STARTUP] ✓ TTS ready in {_time.time()-t0:.2f}s")
|
| 63 |
except Exception as e:
|
|
@@ -65,31 +167,23 @@ async def init_tts():
|
|
| 65 |
|
| 66 |
|
| 67 |
async def generate_greeting():
|
| 68 |
-
"""Generate TTS + lipsync for a greeting to warm up FLOAT on first load."""
|
| 69 |
global GREETING_DATA
|
| 70 |
-
|
| 71 |
-
greeting_text = "Good day! I am Charles Darwin, recreated in this curious digital form. Do feel free to ask me anything about the natural world."
|
| 72 |
-
|
| 73 |
logger.info(f"[{_ts()}] [GREETING] Generating warmup greeting...")
|
| 74 |
t0 = _time.time()
|
| 75 |
|
| 76 |
try:
|
| 77 |
from groq_tts import generate_audio
|
| 78 |
-
t_tts = _time.time()
|
| 79 |
audio_path = generate_audio(greeting_text)
|
| 80 |
-
t_tts_done = _time.time()
|
| 81 |
-
logger.info(f"[{_ts()}] [GREETING] TTS: {t_tts_done-t_tts:.2f}s")
|
| 82 |
audio_url = None
|
| 83 |
audio_duration = 0
|
| 84 |
|
| 85 |
if audio_path and os.path.exists(audio_path):
|
| 86 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
| 87 |
-
logger.info(f"[{_ts()}] [GREETING] Audio: {os.path.getsize(audio_path)/1024:.0f}KB")
|
| 88 |
try:
|
| 89 |
r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
|
| 90 |
"-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
| 91 |
audio_duration = float(r.stdout.strip())
|
| 92 |
-
logger.info(f"[{_ts()}] [GREETING] Audio duration: {audio_duration:.1f}s")
|
| 93 |
except Exception:
|
| 94 |
audio_duration = 5.0
|
| 95 |
|
|
@@ -98,16 +192,12 @@ async def generate_greeting():
|
|
| 98 |
try:
|
| 99 |
from float_lipsync import get_lipsync
|
| 100 |
lipsync = get_lipsync()
|
| 101 |
-
if lipsync.ready:
|
| 102 |
-
t_float = _time.time()
|
| 103 |
lp = lipsync.generate(audio_path)
|
| 104 |
-
t_float_done = _time.time()
|
| 105 |
-
logger.info(f"[{_ts()}] [GREETING] FLOAT: {t_float_done-t_float:.2f}s")
|
| 106 |
if lp and os.path.exists(lp):
|
| 107 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
| 108 |
-
logger.info(f"[{_ts()}] [GREETING] Video: {os.path.getsize(lp)/1024:.0f}KB")
|
| 109 |
except Exception as e:
|
| 110 |
-
logger.warning(f"[GREETING] FLOAT failed
|
| 111 |
|
| 112 |
GREETING_DATA = {
|
| 113 |
"text": greeting_text,
|
|
@@ -126,8 +216,7 @@ async def init_float():
|
|
| 126 |
import torch
|
| 127 |
logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}")
|
| 128 |
if not torch.cuda.is_available():
|
| 129 |
-
logger.warning("[STARTUP] No GPU - FLOAT disabled")
|
| 130 |
-
return
|
| 131 |
try:
|
| 132 |
from float_lipsync import get_lipsync
|
| 133 |
from huggingface_hub import hf_hub_download, snapshot_download
|
|
@@ -154,25 +243,38 @@ async def init_float():
|
|
| 154 |
shutil.copy2(dl, fp)
|
| 155 |
logger.info(f"[STARTUP] float.pth: {fp.stat().st_size/1e6:.1f} MB")
|
| 156 |
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
if
|
| 161 |
-
for name in ["ref.png", "ref.jpg"
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
shutil.copy2(dl, assets / name)
|
| 166 |
-
ref = assets / name
|
| 167 |
break
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
lipsync = get_lipsync()
|
| 174 |
lipsync.initialize({
|
| 175 |
-
"ref_path":
|
| 176 |
"wav2vec_model_path": str(w2v), "audio2emotion_path": str(emo),
|
| 177 |
})
|
| 178 |
logger.info("[STARTUP] ✓ FLOAT ready!")
|
|
@@ -180,33 +282,57 @@ async def init_float():
|
|
| 180 |
logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True)
|
| 181 |
|
| 182 |
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
|
| 192 |
|
| 193 |
@app.get("/api/videos")
|
| 194 |
async def list_videos():
|
| 195 |
-
|
| 196 |
-
return JSONResponse({"videos":
|
| 197 |
-
{"name": f.stem, "filename": f.name}
|
| 198 |
-
for f in sorted(VIDEOS_DIR.iterdir())
|
| 199 |
-
if f.suffix.lower() in (".mp4",".webm",".mkv",".mov") and f.stat().st_size > 1024
|
| 200 |
-
]})
|
| 201 |
|
| 202 |
|
| 203 |
@app.get("/api/stream/{filename}")
|
| 204 |
async def stream_video(filename: str):
|
| 205 |
path = None
|
| 206 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
c = d / filename
|
| 208 |
-
if c.exists() and c.is_file():
|
| 209 |
-
|
|
|
|
|
|
|
|
|
|
| 210 |
mime = mimetypes.guess_type(filename)[0] or "video/mp4"
|
| 211 |
def gen():
|
| 212 |
with open(path, "rb") as f:
|
|
@@ -228,13 +354,8 @@ async def stream_audio(filename: str):
|
|
| 228 |
|
| 229 |
|
| 230 |
def split_sentences(text):
|
| 231 |
-
"""Split text into chunks, merging short sentences for better voice cloning quality.
|
| 232 |
-
Each chunk should be at least ~8 words so the model has enough text context."""
|
| 233 |
parts = re.split(r'(?<=[.!?])\s+', text)
|
| 234 |
parts = [s.strip() for s in parts if s.strip()]
|
| 235 |
-
|
| 236 |
-
# Merge sentences into longer chunks (minimum ~25 words per chunk)
|
| 237 |
-
# Voice cloning works much better with longer text context
|
| 238 |
MIN_WORDS = 25
|
| 239 |
merged = []
|
| 240 |
buffer = ""
|
|
@@ -243,28 +364,19 @@ def split_sentences(text):
|
|
| 243 |
buffer += " " + p
|
| 244 |
else:
|
| 245 |
buffer = p
|
| 246 |
-
# Flush if buffer is long enough
|
| 247 |
if len(buffer.split()) >= MIN_WORDS:
|
| 248 |
merged.append(buffer)
|
| 249 |
buffer = ""
|
| 250 |
-
# Don't leave a tiny orphan — merge it with last chunk
|
| 251 |
if buffer:
|
| 252 |
if merged:
|
| 253 |
merged[-1] += " " + buffer
|
| 254 |
else:
|
| 255 |
merged.append(buffer)
|
| 256 |
-
|
| 257 |
return merged if merged else [text]
|
| 258 |
|
| 259 |
|
| 260 |
@app.post("/api/chat")
|
| 261 |
async def chat(request: Request):
|
| 262 |
-
"""
|
| 263 |
-
Step 1: LLM generates full response (fast).
|
| 264 |
-
Step 2: Split into sentences.
|
| 265 |
-
Step 3: Return text + session_id immediately.
|
| 266 |
-
Step 4: Frontend polls /api/chat/next to get chunks as they're generated.
|
| 267 |
-
"""
|
| 268 |
t0 = _time.time()
|
| 269 |
body = await request.json()
|
| 270 |
user_text = body.get("text", "").strip()
|
|
@@ -272,22 +384,19 @@ async def chat(request: Request):
|
|
| 272 |
|
| 273 |
logger.info(f"[{_ts()}] [CHAT] User: {user_text}")
|
| 274 |
|
| 275 |
-
# 1. LLM (single call)
|
| 276 |
t1 = _time.time()
|
| 277 |
-
from llm import
|
| 278 |
-
llm_result =
|
| 279 |
reply_text = llm_result['text']
|
| 280 |
t2 = _time.time()
|
| 281 |
-
logger.info(f"[{_ts()}] [CHAT]
|
| 282 |
logger.info(f"[{_ts()}] [TIMING] LLM: {t2-t1:.2f}s")
|
| 283 |
|
| 284 |
-
# 2. Split into sentences
|
| 285 |
sentences = split_sentences(reply_text)
|
| 286 |
if not sentences:
|
| 287 |
sentences = [reply_text]
|
| 288 |
logger.info(f"[{_ts()}] [CHAT] Split into {len(sentences)} chunks")
|
| 289 |
|
| 290 |
-
# 3. Create session
|
| 291 |
session_id = str(uuid.uuid4())[:8]
|
| 292 |
_sessions[session_id] = {
|
| 293 |
"sentences": sentences,
|
|
@@ -297,8 +406,6 @@ async def chat(request: Request):
|
|
| 297 |
"created": _time.time(),
|
| 298 |
}
|
| 299 |
|
| 300 |
-
logger.info(f"[{_ts()}] [TIMING] Chat response ready: {_time.time()-t0:.2f}s")
|
| 301 |
-
|
| 302 |
return JSONResponse({
|
| 303 |
"text": reply_text,
|
| 304 |
"emotion": llm_result.get('emotion', 'neutral'),
|
|
@@ -309,17 +416,11 @@ async def chat(request: Request):
|
|
| 309 |
|
| 310 |
@app.post("/api/chat/next")
|
| 311 |
async def chat_next(request: Request):
|
| 312 |
-
"""
|
| 313 |
-
Generate TTS + lipsync for the next sentence in the session.
|
| 314 |
-
Heavy work runs in a thread so the server stays responsive for prefetch requests.
|
| 315 |
-
"""
|
| 316 |
t_req = _time.time()
|
| 317 |
body = await request.json()
|
| 318 |
session_id = body.get("session_id", "")
|
| 319 |
session = _sessions.get(session_id)
|
| 320 |
-
|
| 321 |
-
if not session:
|
| 322 |
-
raise HTTPException(404, "Session not found")
|
| 323 |
|
| 324 |
idx = session["next_index"]
|
| 325 |
if idx >= session["total"]:
|
|
@@ -333,7 +434,6 @@ async def chat_next(request: Request):
|
|
| 333 |
|
| 334 |
logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} received")
|
| 335 |
|
| 336 |
-
# Run the heavy TTS + FLOAT work in a thread
|
| 337 |
import asyncio
|
| 338 |
result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
|
| 339 |
|
|
@@ -360,31 +460,25 @@ async def chat_next(request: Request):
|
|
| 360 |
|
| 361 |
|
| 362 |
def _generate_chunk(sentence, idx, total):
|
| 363 |
-
"""Synchronous TTS + FLOAT generation (runs in thread pool)."""
|
| 364 |
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] START | \"{sentence[:80]}\"")
|
| 365 |
t0 = _time.time()
|
| 366 |
-
|
| 367 |
import torch
|
| 368 |
|
| 369 |
-
# Both TTS (Orpheus) and FLOAT use GPU — serialize with lock
|
| 370 |
with _gpu_lock:
|
| 371 |
-
# 1. TTS (local Orpheus - GPU)
|
| 372 |
from groq_tts import generate_audio
|
| 373 |
t_tts_start = _time.time()
|
| 374 |
audio_path = generate_audio(sentence)
|
| 375 |
t_tts_end = _time.time()
|
| 376 |
-
|
| 377 |
-
# FIX: Clear VRAM after massive LLM generation
|
| 378 |
if torch.cuda.is_available():
|
| 379 |
torch.cuda.empty_cache()
|
| 380 |
-
|
| 381 |
audio_url = None
|
| 382 |
audio_duration = 0
|
| 383 |
audio_size = 0
|
| 384 |
if audio_path and os.path.exists(audio_path):
|
| 385 |
audio_size = os.path.getsize(audio_path)
|
| 386 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
| 387 |
-
t_probe_start = _time.time()
|
| 388 |
try:
|
| 389 |
r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
|
| 390 |
"-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
|
@@ -393,11 +487,10 @@ def _generate_chunk(sentence, idx, total):
|
|
| 393 |
audio_duration = 3.0
|
| 394 |
t_probe_end = _time.time()
|
| 395 |
else:
|
| 396 |
-
|
| 397 |
|
| 398 |
-
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] TTS: {t_tts_end-t_tts_start:.2f}s | audio: {audio_duration:.1f}s, {audio_size/1024:.0f}KB
|
| 399 |
|
| 400 |
-
# 2. Lipsync (FLOAT - GPU)
|
| 401 |
lipsync_video_url = None
|
| 402 |
if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
|
| 403 |
try:
|
|
@@ -407,21 +500,18 @@ def _generate_chunk(sentence, idx, total):
|
|
| 407 |
t_float_start = _time.time()
|
| 408 |
lp = lipsync.generate(audio_path)
|
| 409 |
t_float_done = _time.time()
|
| 410 |
-
|
| 411 |
-
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] FLOAT: {float_time:.2f}s")
|
| 412 |
if lp and os.path.exists(lp):
|
| 413 |
-
lp_size = os.path.getsize(lp)
|
| 414 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
| 415 |
-
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] VIDEO: {
|
| 416 |
except Exception as e:
|
| 417 |
logger.error(f"[{_ts()}] [CHUNK {idx+1}/{total}] LIPSYNC FAILED: {e}", exc_info=True)
|
| 418 |
-
|
| 419 |
-
# FIX: Clear VRAM after FLOAT visual generation
|
| 420 |
if torch.cuda.is_available():
|
| 421 |
torch.cuda.empty_cache()
|
| 422 |
|
| 423 |
total_time = _time.time() - t0
|
| 424 |
-
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] DONE | total: {total_time:.2f}s
|
| 425 |
|
| 426 |
return {
|
| 427 |
"audio_url": audio_url,
|
|
@@ -429,6 +519,7 @@ def _generate_chunk(sentence, idx, total):
|
|
| 429 |
"lipsync_video_url": lipsync_video_url,
|
| 430 |
}
|
| 431 |
|
|
|
|
| 432 |
@app.post("/api/reset")
|
| 433 |
async def reset_chat():
|
| 434 |
from llm import reset_conversation
|
|
@@ -438,7 +529,6 @@ async def reset_chat():
|
|
| 438 |
|
| 439 |
@app.get("/api/greeting")
|
| 440 |
async def get_greeting():
|
| 441 |
-
"""Return the pre-generated greeting (audio + lipsync video)."""
|
| 442 |
if GREETING_DATA:
|
| 443 |
return JSONResponse(GREETING_DATA)
|
| 444 |
return JSONResponse({"text": None})
|
|
@@ -455,11 +545,15 @@ HTML_PAGE = """\
|
|
| 455 |
<head>
|
| 456 |
<meta charset="UTF-8"/>
|
| 457 |
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
| 458 |
-
<title>
|
| 459 |
<style>
|
| 460 |
*{margin:0;padding:0;box-sizing:border-box}
|
| 461 |
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0f1c;color:#e2e8f0;height:100vh;display:flex;flex-direction:column;overflow:hidden}
|
| 462 |
.app-container{display:flex;flex-direction:column;height:100vh;max-width:900px;margin:0 auto;width:100%;padding:1rem;gap:.75rem}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 463 |
.video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center}
|
| 464 |
.video-container{position:relative;width:100%;height:100%}
|
| 465 |
.video-container video{width:100%;height:100%;object-fit:contain;display:block}
|
|
@@ -491,6 +585,10 @@ body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;backgrou
|
|
| 491 |
</head>
|
| 492 |
<body>
|
| 493 |
<div class="app-container">
|
|
|
|
|
|
|
|
|
|
|
|
|
| 494 |
<div class="video-section">
|
| 495 |
<div class="video-container">
|
| 496 |
<video id="videoA" muted autoplay playsinline></video>
|
|
@@ -501,25 +599,20 @@ body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;backgrou
|
|
| 501 |
<div class="chat-section">
|
| 502 |
<div class="chat-log" id="chatLog"></div>
|
| 503 |
<div class="input-bar">
|
| 504 |
-
<input type="text" id="chatInput" placeholder="
|
| 505 |
<button class="btn-send" id="btnSend">Send</button>
|
| 506 |
<button class="btn-reset" id="btnReset" title="Reset conversation">↻</button>
|
| 507 |
</div>
|
| 508 |
</div>
|
| 509 |
</div>
|
| 510 |
<script>
|
| 511 |
-
// ======================================================================
|
| 512 |
-
// STATE
|
| 513 |
-
// ======================================================================
|
| 514 |
let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, lastPlayed=null, msgCounter=0;
|
|
|
|
| 515 |
const videoA=document.getElementById('videoA'), videoB=document.getElementById('videoB');
|
| 516 |
const videoStatus=document.getElementById('videoStatus'), chatLog=document.getElementById('chatLog');
|
| 517 |
const chatInput=document.getElementById('chatInput'), btnSend=document.getElementById('btnSend');
|
| 518 |
-
const btnReset=document.getElementById('btnReset');
|
| 519 |
|
| 520 |
-
// ======================================================================
|
| 521 |
-
// VIDEO: Crossfade
|
| 522 |
-
// ======================================================================
|
| 523 |
function getActiveVideo(){return currentVideo==='A'?videoA:videoB}
|
| 524 |
function getInactiveVideo(){return currentVideo==='A'?videoB:videoA}
|
| 525 |
|
|
@@ -529,48 +622,28 @@ function crossfadeTo(url, muted){
|
|
| 529 |
isTransitioning=true;
|
| 530 |
const prev=getActiveVideo(), next=getInactiveVideo();
|
| 531 |
next.muted=(muted!==false);
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
next.style.opacity='1';
|
| 535 |
-
next.style.zIndex='0';
|
| 536 |
-
prev.style.zIndex='1';
|
| 537 |
-
next.src=url;
|
| 538 |
-
next.load();
|
| 539 |
-
|
| 540 |
next.onloadeddata=function(){
|
| 541 |
next.play().then(()=>{
|
| 542 |
-
|
| 543 |
-
next.style.zIndex='2';
|
| 544 |
-
// Fade out the old video to reveal new one smoothly
|
| 545 |
-
prev.style.opacity='0';
|
| 546 |
-
|
| 547 |
setTimeout(()=>{
|
| 548 |
-
|
| 549 |
-
prev.
|
| 550 |
-
prev.removeAttribute('src');
|
| 551 |
-
prev.load();
|
| 552 |
-
prev.style.zIndex='0';
|
| 553 |
-
prev.style.opacity='1'; // reset for next use
|
| 554 |
next.style.zIndex='1';
|
| 555 |
currentVideo=currentVideo==='A'?'B':'A';
|
| 556 |
-
isTransitioning=false;
|
| 557 |
-
|
| 558 |
-
}, 180); // slightly longer than CSS transition
|
| 559 |
}).catch(()=>{
|
| 560 |
next.muted=true;
|
| 561 |
next.play().then(()=>{
|
| 562 |
-
next.style.zIndex='2';
|
| 563 |
-
prev.style.opacity='0';
|
| 564 |
setTimeout(()=>{
|
| 565 |
-
prev.pause();
|
| 566 |
-
prev.
|
| 567 |
-
prev.load();
|
| 568 |
-
prev.style.zIndex='0';
|
| 569 |
-
prev.style.opacity='1';
|
| 570 |
next.style.zIndex='1';
|
| 571 |
currentVideo=currentVideo==='A'?'B':'A';
|
| 572 |
-
isTransitioning=false;
|
| 573 |
-
resolve();
|
| 574 |
}, 180);
|
| 575 |
}).catch(()=>{ isTransitioning=false; resolve(); });
|
| 576 |
});
|
|
@@ -593,35 +666,24 @@ function playNextIdle(){
|
|
| 593 |
crossfadeTo('/api/stream/'+encodeURIComponent(c.filename), true);
|
| 594 |
}
|
| 595 |
|
| 596 |
-
// isSpeaking: true when lipsync chunks are actively playing
|
| 597 |
-
// idle loop continues when isProcessing but NOT isSpeaking
|
| 598 |
let isSpeaking=false;
|
| 599 |
-
|
| 600 |
videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isSpeaking) playNextIdle(); });
|
| 601 |
videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isSpeaking) playNextIdle(); });
|
| 602 |
|
| 603 |
-
// ======================================================================
|
| 604 |
-
// VIDEO: Wait for current video to end
|
| 605 |
-
// ======================================================================
|
| 606 |
function waitForVideoEnd(){
|
| 607 |
return new Promise((resolve)=>{
|
| 608 |
const active=getActiveVideo();
|
| 609 |
-
if(active.ended || active.paused || !active.src){
|
| 610 |
-
resolve(); return;
|
| 611 |
-
}
|
| 612 |
const onEnd=()=>{ active.removeEventListener('ended',onEnd); resolve(); };
|
| 613 |
active.addEventListener('ended', onEnd);
|
| 614 |
});
|
| 615 |
}
|
| 616 |
|
| 617 |
-
// ======================================================================
|
| 618 |
-
// CHAT: Messages
|
| 619 |
-
// ======================================================================
|
| 620 |
function addMessage(role, text, extra){
|
| 621 |
const id='msg_'+(msgCounter++);
|
| 622 |
const w=document.createElement('div'); w.className='msg '+role;
|
| 623 |
const b=document.createElement('div'); b.className='bubble '+role+(extra?' '+extra:''); b.id=id;
|
| 624 |
-
const l=document.createElement('div'); l.className='label'; l.textContent=role==='user'?'You':
|
| 625 |
const c=document.createElement('div'); c.className='content'; c.textContent=text;
|
| 626 |
b.appendChild(l); b.appendChild(c); w.appendChild(b);
|
| 627 |
chatLog.appendChild(w); chatLog.scrollTop=chatLog.scrollHeight;
|
|
@@ -634,9 +696,32 @@ function addTypingIndicator(){
|
|
| 634 |
return r;
|
| 635 |
}
|
| 636 |
|
| 637 |
-
//
|
| 638 |
-
|
| 639 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 640 |
async function sendMessage(){
|
| 641 |
const text=chatInput.value.trim();
|
| 642 |
if(!text||isProcessing) return;
|
|
@@ -645,8 +730,6 @@ async function sendMessage(){
|
|
| 645 |
const typing=addTypingIndicator();
|
| 646 |
|
| 647 |
try{
|
| 648 |
-
// Step 1: Get LLM response + session (fast, ~0.5s)
|
| 649 |
-
// Idle videos keep playing during this
|
| 650 |
const res=await fetch('/api/chat',{
|
| 651 |
method:'POST', headers:{'Content-Type':'application/json'},
|
| 652 |
body:JSON.stringify({text:text})
|
|
@@ -654,7 +737,6 @@ async function sendMessage(){
|
|
| 654 |
if(!res.ok) throw new Error('HTTP '+res.status);
|
| 655 |
const data=await res.json();
|
| 656 |
|
| 657 |
-
// Replace typing indicator with empty bot bubble, ready for streaming
|
| 658 |
typing.bubbleEl.classList.remove('typing');
|
| 659 |
typing.contentEl.classList.remove('typing-dots');
|
| 660 |
typing.contentEl.textContent='';
|
|
@@ -665,20 +747,14 @@ async function sendMessage(){
|
|
| 665 |
const fullText=data.text;
|
| 666 |
let displayedText='';
|
| 667 |
|
| 668 |
-
// Text streaming: reveal words of a specific sentence over the given duration
|
| 669 |
function streamChunkText(sentence, durationSec){
|
| 670 |
-
const chunkWords=sentence.split(/\s+/).filter(w=>w.length>0);
|
| 671 |
if(chunkWords.length===0) return Promise.resolve();
|
| 672 |
const interval=Math.max(50, (durationSec*1000)/chunkWords.length);
|
| 673 |
return new Promise((resolve)=>{
|
| 674 |
let i=0;
|
| 675 |
const timer=setInterval(()=>{
|
| 676 |
-
if(i>=chunkWords.length){
|
| 677 |
-
clearInterval(timer);
|
| 678 |
-
resolve();
|
| 679 |
-
return;
|
| 680 |
-
}
|
| 681 |
-
// Add space between previous text and new word
|
| 682 |
if(displayedText.length>0) displayedText+=' ';
|
| 683 |
displayedText+=chunkWords[i];
|
| 684 |
typing.contentEl.textContent=displayedText;
|
|
@@ -688,16 +764,12 @@ async function sendMessage(){
|
|
| 688 |
});
|
| 689 |
}
|
| 690 |
|
| 691 |
-
// Show all remaining text instantly (cleanup)
|
| 692 |
function showAllText(){
|
| 693 |
typing.contentEl.textContent=fullText;
|
| 694 |
typing.contentEl.classList.remove('streaming-cursor');
|
| 695 |
chatLog.scrollTop=chatLog.scrollHeight;
|
| 696 |
}
|
| 697 |
|
| 698 |
-
videoStatus.textContent='generating 1/'+totalChunks+'...';
|
| 699 |
-
|
| 700 |
-
// Helper to fetch a chunk from the server
|
| 701 |
async function fetchChunk(sid){
|
| 702 |
const r=await fetch('/api/chat/next',{
|
| 703 |
method:'POST', headers:{'Content-Type':'application/json'},
|
|
@@ -707,69 +779,47 @@ async function sendMessage(){
|
|
| 707 |
return await r.json();
|
| 708 |
}
|
| 709 |
|
| 710 |
-
// Step 2: Prefetch first chunk (idle keeps playing while we wait)
|
| 711 |
let nextChunkPromise=fetchChunk(sessionId);
|
| 712 |
|
| 713 |
for(let i=0; i<totalChunks; i++){
|
| 714 |
videoStatus.textContent='generating '+(i+1)+'/'+totalChunks+'...';
|
| 715 |
-
|
| 716 |
-
// Wait for the chunk to be ready
|
| 717 |
const chunk=await nextChunkPromise;
|
| 718 |
if(!chunk || chunk.done) break;
|
|
|
|
| 719 |
|
| 720 |
-
// Immediately start prefetching the NEXT chunk
|
| 721 |
-
if(i+1 < totalChunks){
|
| 722 |
-
nextChunkPromise=fetchChunk(sessionId);
|
| 723 |
-
}
|
| 724 |
-
|
| 725 |
-
// Now we have a chunk ready — stop idle loop and wait for current idle to finish
|
| 726 |
isSpeaking=true;
|
| 727 |
-
videoStatus.textContent='waiting for idle to finish...';
|
| 728 |
await waitForVideoEnd();
|
| 729 |
-
|
| 730 |
videoStatus.textContent='speaking '+(i+1)+'/'+totalChunks;
|
| 731 |
|
| 732 |
-
// Play audio
|
| 733 |
let audioPromise=null;
|
| 734 |
if(chunk.audio_url){
|
| 735 |
const audio=new Audio(chunk.audio_url);
|
| 736 |
audioPromise=new Promise((resolve)=>{
|
| 737 |
-
audio.onended=resolve;
|
| 738 |
-
audio.onerror=resolve;
|
| 739 |
audio.play().catch(()=>resolve());
|
| 740 |
});
|
| 741 |
}
|
| 742 |
|
| 743 |
-
// Stream text for this chunk's sentence over its audio duration
|
| 744 |
const chunkDuration=chunk.audio_duration||3.0;
|
| 745 |
const textStreamPromise=streamChunkText(chunk.sentence, chunkDuration);
|
| 746 |
|
| 747 |
-
// Play lipsync video
|
| 748 |
if(chunk.lipsync_video_url){
|
| 749 |
await crossfadeTo(chunk.lipsync_video_url, true);
|
| 750 |
-
// Wait for lipsync video to finish before next chunk
|
| 751 |
await waitForVideoEnd();
|
| 752 |
} else if(audioPromise){
|
| 753 |
-
// No lipsync, just wait for audio to finish
|
| 754 |
await audioPromise;
|
| 755 |
}
|
| 756 |
-
|
| 757 |
-
// Ensure text streaming for this chunk is done
|
| 758 |
await textStreamPromise;
|
| 759 |
}
|
| 760 |
|
| 761 |
-
// Ensure all text is shown
|
| 762 |
showAllText();
|
| 763 |
-
|
| 764 |
-
// All chunks done, return to idle
|
| 765 |
isSpeaking=false;
|
| 766 |
playNextIdle();
|
| 767 |
|
| 768 |
}catch(err){
|
| 769 |
typing.bubbleEl.classList.remove('typing');
|
| 770 |
typing.bubbleEl.classList.add('error');
|
| 771 |
-
typing.contentEl.classList.remove('typing-dots');
|
| 772 |
-
typing.contentEl.classList.remove('streaming-cursor');
|
| 773 |
typing.contentEl.textContent='Error: '+err.message;
|
| 774 |
}
|
| 775 |
isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus();
|
|
@@ -780,40 +830,46 @@ chatInput.addEventListener('keydown', e=>{ if(e.key==='Enter'&&!e.shiftKey){ e.p
|
|
| 780 |
btnReset.addEventListener('click', async()=>{
|
| 781 |
await fetch('/api/reset',{method:'POST'});
|
| 782 |
chatLog.innerHTML='';
|
| 783 |
-
addMessage('bot','
|
| 784 |
});
|
| 785 |
|
| 786 |
-
//
|
| 787 |
-
// INIT
|
| 788 |
-
// ======================================================================
|
| 789 |
async function init(){
|
| 790 |
try{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 791 |
const res=await fetch('/api/videos');
|
| 792 |
const data=await res.json();
|
| 793 |
-
if(!data.videos.length){ videoStatus.textContent='
|
| 794 |
-
videos=data.videos;
|
| 795 |
-
videoStatus.textContent=videos.length+' idle clips loaded';
|
| 796 |
|
| 797 |
-
// Check
|
| 798 |
const greetRes=await fetch('/api/greeting');
|
| 799 |
const greeting=await greetRes.json();
|
| 800 |
|
| 801 |
if(greeting.lipsync_video_url){
|
| 802 |
-
// Play greeting lipsync video first
|
| 803 |
videoStatus.textContent='greeting...';
|
| 804 |
addMessage('bot', greeting.text);
|
| 805 |
-
|
| 806 |
if(greeting.audio_url){
|
| 807 |
const audio=new Audio(greeting.audio_url);
|
| 808 |
audio.play().catch(e=>console.warn('[AUDIO]',e));
|
| 809 |
}
|
| 810 |
-
|
| 811 |
await crossfadeTo(greeting.lipsync_video_url, true);
|
| 812 |
await waitForVideoEnd();
|
| 813 |
playNextIdle();
|
| 814 |
} else {
|
| 815 |
-
|
| 816 |
-
addMessage('bot', greeting.text || 'Good day! I am Charles Darwin, recreated in this curious digital form. Do feel free to ask me anything about the natural world.');
|
| 817 |
playNextIdle();
|
| 818 |
}
|
| 819 |
}catch(e){ videoStatus.textContent='Error: '+e.message; }
|
|
|
|
| 1 |
"""
|
| 2 |
Avatar Chatbot - HuggingFace Spaces Edition
|
| 3 |
+
Avatar profile system: each avatar has ref.png, persona.txt, idlevideos/
|
| 4 |
Chunked pipeline: LLM (once) → split sentences → TTS+FLOAT per chunk → stream to frontend
|
| 5 |
"""
|
| 6 |
from fastapi import FastAPI, HTTPException, Request
|
|
|
|
| 19 |
|
| 20 |
app = FastAPI()
|
| 21 |
|
| 22 |
+
# ---- Avatar system ----
|
| 23 |
+
AVATARS_DIR = Path("/app/avatars")
|
| 24 |
+
_current_avatar = None # name of active avatar folder
|
| 25 |
+
|
| 26 |
TTS_DIR = Path("/tmp/tts_output")
|
| 27 |
LIPSYNC_DIR = Path("/tmp/lipsync_output")
|
| 28 |
REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot")
|
| 29 |
FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true"
|
| 30 |
|
| 31 |
+
for d in [TTS_DIR, LIPSYNC_DIR]:
|
| 32 |
d.mkdir(parents=True, exist_ok=True)
|
| 33 |
|
| 34 |
|
| 35 |
def _ts():
|
|
|
|
| 36 |
return _time.strftime("%H:%M:%S", _time.gmtime()) + f".{int(_time.time()*1000)%1000:03d}"
|
| 37 |
|
| 38 |
+
|
| 39 |
+
def _get_avatar_dir(name=None):
|
| 40 |
+
"""Get path to an avatar's directory."""
|
| 41 |
+
if name is None:
|
| 42 |
+
name = _current_avatar
|
| 43 |
+
if name:
|
| 44 |
+
return AVATARS_DIR / name
|
| 45 |
+
return None
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _get_idle_videos_dir():
|
| 49 |
+
"""Get the current avatar's idle videos directory."""
|
| 50 |
+
d = _get_avatar_dir()
|
| 51 |
+
if d:
|
| 52 |
+
return d / "idlevideos"
|
| 53 |
+
return Path("/tmp/videos")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _list_idle_videos():
|
| 57 |
+
"""List idle video files for the current avatar."""
|
| 58 |
+
d = _get_idle_videos_dir()
|
| 59 |
+
if not d.exists():
|
| 60 |
+
return []
|
| 61 |
+
return [
|
| 62 |
+
{"name": f.stem, "filename": f.name}
|
| 63 |
+
for f in sorted(d.iterdir())
|
| 64 |
+
if f.suffix.lower() in (".mp4", ".webm", ".mkv", ".mov") and f.stat().st_size > 1024
|
| 65 |
+
]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
import threading
|
| 69 |
_gpu_lock = threading.Lock()
|
| 70 |
|
|
|
|
| 71 |
_sessions = {}
|
|
|
|
|
|
|
| 72 |
GREETING_DATA = None
|
| 73 |
|
| 74 |
|
| 75 |
+
def _discover_avatars():
|
| 76 |
+
"""Find all avatar profiles in the avatars directory."""
|
| 77 |
+
if not AVATARS_DIR.exists():
|
| 78 |
+
logger.warning(f"[AVATARS] Directory not found: {AVATARS_DIR}")
|
| 79 |
+
return []
|
| 80 |
+
avatars = []
|
| 81 |
+
for d in sorted(AVATARS_DIR.iterdir()):
|
| 82 |
+
if d.is_dir():
|
| 83 |
+
has_ref = (d / "ref.png").exists() or (d / "ref.jpg").exists()
|
| 84 |
+
has_persona = (d / "persona.txt").exists()
|
| 85 |
+
has_videos = (d / "idlevideos").exists()
|
| 86 |
+
avatars.append({
|
| 87 |
+
"name": d.name,
|
| 88 |
+
"has_ref": has_ref,
|
| 89 |
+
"has_persona": has_persona,
|
| 90 |
+
"has_videos": has_videos,
|
| 91 |
+
})
|
| 92 |
+
return avatars
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _switch_avatar(avatar_name: str):
|
| 96 |
+
"""Switch to a different avatar profile."""
|
| 97 |
+
global _current_avatar
|
| 98 |
+
avatar_dir = AVATARS_DIR / avatar_name
|
| 99 |
+
if not avatar_dir.exists():
|
| 100 |
+
logger.error(f"[AVATARS] Avatar not found: {avatar_name}")
|
| 101 |
+
return False
|
| 102 |
+
|
| 103 |
+
_current_avatar = avatar_name
|
| 104 |
+
logger.info(f"[AVATARS] Switching to avatar: {avatar_name}")
|
| 105 |
+
|
| 106 |
+
# 1. Load persona into LLM
|
| 107 |
+
persona_path = avatar_dir / "persona.txt"
|
| 108 |
+
from llm import load_persona_from_file
|
| 109 |
+
persona_text = load_persona_from_file(str(persona_path))
|
| 110 |
+
logger.info(f"[AVATARS] Persona loaded: {persona_text[:80]}..." if persona_text else "[AVATARS] No persona.txt found")
|
| 111 |
+
|
| 112 |
+
# 2. Update FLOAT reference image
|
| 113 |
+
if FLOAT_ENABLED:
|
| 114 |
+
try:
|
| 115 |
+
from float_lipsync import get_lipsync
|
| 116 |
+
lipsync = get_lipsync()
|
| 117 |
+
if lipsync.ready:
|
| 118 |
+
ref_path = avatar_dir / "ref.png"
|
| 119 |
+
if not ref_path.exists():
|
| 120 |
+
ref_path = avatar_dir / "ref.jpg"
|
| 121 |
+
if ref_path.exists():
|
| 122 |
+
lipsync.update_reference_image(str(ref_path))
|
| 123 |
+
logger.info(f"[AVATARS] Reference image updated: {ref_path}")
|
| 124 |
+
else:
|
| 125 |
+
logger.warning(f"[AVATARS] No ref image in {avatar_dir}")
|
| 126 |
+
except Exception as e:
|
| 127 |
+
logger.error(f"[AVATARS] Failed to update ref image: {e}")
|
| 128 |
+
|
| 129 |
+
# 3. Idle videos — no action needed, _get_idle_videos_dir() uses _current_avatar dynamically
|
| 130 |
+
|
| 131 |
+
logger.info(f"[AVATARS] ✓ Switched to {avatar_name}")
|
| 132 |
+
return True
|
| 133 |
+
|
| 134 |
+
|
| 135 |
@app.on_event("startup")
|
| 136 |
async def startup():
|
| 137 |
+
# Discover avatars and set default
|
| 138 |
+
avatars = _discover_avatars()
|
| 139 |
+
logger.info(f"[STARTUP] Found {len(avatars)} avatars: {[a['name'] for a in avatars]}")
|
| 140 |
+
|
| 141 |
+
if avatars:
|
| 142 |
+
# Use first avatar as default
|
| 143 |
+
default = avatars[0]["name"]
|
| 144 |
+
# Load persona for default avatar (before FLOAT init so ref image is set)
|
| 145 |
+
global _current_avatar
|
| 146 |
+
_current_avatar = default
|
| 147 |
+
persona_path = AVATARS_DIR / default / "persona.txt"
|
| 148 |
+
from llm import load_persona_from_file
|
| 149 |
+
load_persona_from_file(str(persona_path))
|
| 150 |
+
logger.info(f"[STARTUP] Default avatar: {default}")
|
| 151 |
+
|
| 152 |
await init_tts()
|
| 153 |
if FLOAT_ENABLED:
|
| 154 |
await init_float()
|
|
|
|
| 156 |
|
| 157 |
|
| 158 |
async def init_tts():
|
|
|
|
| 159 |
logger.info(f"[{_ts()}] [STARTUP] Initializing TTS...")
|
| 160 |
t0 = _time.time()
|
| 161 |
try:
|
| 162 |
+
import groq_tts as tts_module
|
|
|
|
| 163 |
tts_module.initialize()
|
| 164 |
logger.info(f"[{_ts()}] [STARTUP] ✓ TTS ready in {_time.time()-t0:.2f}s")
|
| 165 |
except Exception as e:
|
|
|
|
| 167 |
|
| 168 |
|
| 169 |
async def generate_greeting():
|
|
|
|
| 170 |
global GREETING_DATA
|
| 171 |
+
greeting_text = "Hello! Feel free to ask me anything."
|
|
|
|
|
|
|
| 172 |
logger.info(f"[{_ts()}] [GREETING] Generating warmup greeting...")
|
| 173 |
t0 = _time.time()
|
| 174 |
|
| 175 |
try:
|
| 176 |
from groq_tts import generate_audio
|
|
|
|
| 177 |
audio_path = generate_audio(greeting_text)
|
|
|
|
|
|
|
| 178 |
audio_url = None
|
| 179 |
audio_duration = 0
|
| 180 |
|
| 181 |
if audio_path and os.path.exists(audio_path):
|
| 182 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
|
|
|
| 183 |
try:
|
| 184 |
r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
|
| 185 |
"-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
| 186 |
audio_duration = float(r.stdout.strip())
|
|
|
|
| 187 |
except Exception:
|
| 188 |
audio_duration = 5.0
|
| 189 |
|
|
|
|
| 192 |
try:
|
| 193 |
from float_lipsync import get_lipsync
|
| 194 |
lipsync = get_lipsync()
|
| 195 |
+
if lipsync.ready and audio_path:
|
|
|
|
| 196 |
lp = lipsync.generate(audio_path)
|
|
|
|
|
|
|
| 197 |
if lp and os.path.exists(lp):
|
| 198 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
|
|
|
| 199 |
except Exception as e:
|
| 200 |
+
logger.warning(f"[GREETING] FLOAT failed: {e}")
|
| 201 |
|
| 202 |
GREETING_DATA = {
|
| 203 |
"text": greeting_text,
|
|
|
|
| 216 |
import torch
|
| 217 |
logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}")
|
| 218 |
if not torch.cuda.is_available():
|
| 219 |
+
logger.warning("[STARTUP] No GPU - FLOAT disabled"); return
|
|
|
|
| 220 |
try:
|
| 221 |
from float_lipsync import get_lipsync
|
| 222 |
from huggingface_hub import hf_hub_download, snapshot_download
|
|
|
|
| 243 |
shutil.copy2(dl, fp)
|
| 244 |
logger.info(f"[STARTUP] float.pth: {fp.stat().st_size/1e6:.1f} MB")
|
| 245 |
|
| 246 |
+
# Use current avatar's ref.png for initial FLOAT setup
|
| 247 |
+
ref_path = None
|
| 248 |
+
avatar_dir = _get_avatar_dir()
|
| 249 |
+
if avatar_dir:
|
| 250 |
+
for name in ["ref.png", "ref.jpg"]:
|
| 251 |
+
p = avatar_dir / name
|
| 252 |
+
if p.exists():
|
| 253 |
+
ref_path = str(p)
|
|
|
|
|
|
|
| 254 |
break
|
| 255 |
+
|
| 256 |
+
# Fallback to legacy location
|
| 257 |
+
if not ref_path:
|
| 258 |
+
assets = Path("/app/assets")
|
| 259 |
+
assets.mkdir(parents=True, exist_ok=True)
|
| 260 |
+
ref = assets / "ref.png"
|
| 261 |
+
if not ref.exists() or ref.stat().st_size < 1024:
|
| 262 |
+
for name in ["ref.png", "ref.jpg", "main2.png"]:
|
| 263 |
+
try:
|
| 264 |
+
dl = hf_hub_download(repo_id=REPO_ID, repo_type="space",
|
| 265 |
+
filename=f"app/assets/{name}", local_dir="/tmp/hf_download")
|
| 266 |
+
shutil.copy2(dl, assets / name)
|
| 267 |
+
ref = assets / name
|
| 268 |
+
break
|
| 269 |
+
except Exception:
|
| 270 |
+
continue
|
| 271 |
+
else:
|
| 272 |
+
logger.warning("[STARTUP] No ref image found"); return
|
| 273 |
+
ref_path = str(ref)
|
| 274 |
|
| 275 |
lipsync = get_lipsync()
|
| 276 |
lipsync.initialize({
|
| 277 |
+
"ref_path": ref_path, "ckpt_path": str(fp),
|
| 278 |
"wav2vec_model_path": str(w2v), "audio2emotion_path": str(emo),
|
| 279 |
})
|
| 280 |
logger.info("[STARTUP] ✓ FLOAT ready!")
|
|
|
|
| 282 |
logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True)
|
| 283 |
|
| 284 |
|
| 285 |
+
# ---- Avatar API endpoints ----
|
| 286 |
+
|
| 287 |
+
@app.get("/api/avatars")
|
| 288 |
+
async def list_avatars():
|
| 289 |
+
"""List all available avatar profiles."""
|
| 290 |
+
avatars = _discover_avatars()
|
| 291 |
+
return JSONResponse({
|
| 292 |
+
"avatars": avatars,
|
| 293 |
+
"current": _current_avatar,
|
| 294 |
+
})
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
@app.post("/api/switch-avatar")
|
| 298 |
+
async def switch_avatar(request: Request):
|
| 299 |
+
"""Switch to a different avatar. Resets chat history."""
|
| 300 |
+
body = await request.json()
|
| 301 |
+
avatar_name = body.get("avatar", "").strip()
|
| 302 |
+
if not avatar_name:
|
| 303 |
+
raise HTTPException(400, "No avatar specified")
|
| 304 |
+
|
| 305 |
+
success = _switch_avatar(avatar_name)
|
| 306 |
+
if not success:
|
| 307 |
+
raise HTTPException(404, f"Avatar not found: {avatar_name}")
|
| 308 |
+
|
| 309 |
+
return JSONResponse({
|
| 310 |
+
"status": "ok",
|
| 311 |
+
"avatar": avatar_name,
|
| 312 |
+
"videos": _list_idle_videos(),
|
| 313 |
+
})
|
| 314 |
|
| 315 |
|
| 316 |
@app.get("/api/videos")
|
| 317 |
async def list_videos():
|
| 318 |
+
vids = _list_idle_videos()
|
| 319 |
+
return JSONResponse({"videos": vids})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
|
| 321 |
|
| 322 |
@app.get("/api/stream/{filename}")
|
| 323 |
async def stream_video(filename: str):
|
| 324 |
path = None
|
| 325 |
+
# Search in current avatar's idle videos, lipsync output, and TTS output
|
| 326 |
+
search_dirs = [_get_idle_videos_dir(), LIPSYNC_DIR, TTS_DIR]
|
| 327 |
+
for d in search_dirs:
|
| 328 |
+
if d is None:
|
| 329 |
+
continue
|
| 330 |
c = d / filename
|
| 331 |
+
if c.exists() and c.is_file():
|
| 332 |
+
path = c
|
| 333 |
+
break
|
| 334 |
+
if not path:
|
| 335 |
+
raise HTTPException(404, "Not found")
|
| 336 |
mime = mimetypes.guess_type(filename)[0] or "video/mp4"
|
| 337 |
def gen():
|
| 338 |
with open(path, "rb") as f:
|
|
|
|
| 354 |
|
| 355 |
|
| 356 |
def split_sentences(text):
|
|
|
|
|
|
|
| 357 |
parts = re.split(r'(?<=[.!?])\s+', text)
|
| 358 |
parts = [s.strip() for s in parts if s.strip()]
|
|
|
|
|
|
|
|
|
|
| 359 |
MIN_WORDS = 25
|
| 360 |
merged = []
|
| 361 |
buffer = ""
|
|
|
|
| 364 |
buffer += " " + p
|
| 365 |
else:
|
| 366 |
buffer = p
|
|
|
|
| 367 |
if len(buffer.split()) >= MIN_WORDS:
|
| 368 |
merged.append(buffer)
|
| 369 |
buffer = ""
|
|
|
|
| 370 |
if buffer:
|
| 371 |
if merged:
|
| 372 |
merged[-1] += " " + buffer
|
| 373 |
else:
|
| 374 |
merged.append(buffer)
|
|
|
|
| 375 |
return merged if merged else [text]
|
| 376 |
|
| 377 |
|
| 378 |
@app.post("/api/chat")
|
| 379 |
async def chat(request: Request):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 380 |
t0 = _time.time()
|
| 381 |
body = await request.json()
|
| 382 |
user_text = body.get("text", "").strip()
|
|
|
|
| 384 |
|
| 385 |
logger.info(f"[{_ts()}] [CHAT] User: {user_text}")
|
| 386 |
|
|
|
|
| 387 |
t1 = _time.time()
|
| 388 |
+
from llm import generate_response
|
| 389 |
+
llm_result = generate_response(user_text)
|
| 390 |
reply_text = llm_result['text']
|
| 391 |
t2 = _time.time()
|
| 392 |
+
logger.info(f"[{_ts()}] [CHAT] Reply: {reply_text}")
|
| 393 |
logger.info(f"[{_ts()}] [TIMING] LLM: {t2-t1:.2f}s")
|
| 394 |
|
|
|
|
| 395 |
sentences = split_sentences(reply_text)
|
| 396 |
if not sentences:
|
| 397 |
sentences = [reply_text]
|
| 398 |
logger.info(f"[{_ts()}] [CHAT] Split into {len(sentences)} chunks")
|
| 399 |
|
|
|
|
| 400 |
session_id = str(uuid.uuid4())[:8]
|
| 401 |
_sessions[session_id] = {
|
| 402 |
"sentences": sentences,
|
|
|
|
| 406 |
"created": _time.time(),
|
| 407 |
}
|
| 408 |
|
|
|
|
|
|
|
| 409 |
return JSONResponse({
|
| 410 |
"text": reply_text,
|
| 411 |
"emotion": llm_result.get('emotion', 'neutral'),
|
|
|
|
| 416 |
|
| 417 |
@app.post("/api/chat/next")
|
| 418 |
async def chat_next(request: Request):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 419 |
t_req = _time.time()
|
| 420 |
body = await request.json()
|
| 421 |
session_id = body.get("session_id", "")
|
| 422 |
session = _sessions.get(session_id)
|
| 423 |
+
if not session: raise HTTPException(404, "Session not found")
|
|
|
|
|
|
|
| 424 |
|
| 425 |
idx = session["next_index"]
|
| 426 |
if idx >= session["total"]:
|
|
|
|
| 434 |
|
| 435 |
logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} received")
|
| 436 |
|
|
|
|
| 437 |
import asyncio
|
| 438 |
result = await asyncio.to_thread(_generate_chunk, sentence, idx, total)
|
| 439 |
|
|
|
|
| 460 |
|
| 461 |
|
| 462 |
def _generate_chunk(sentence, idx, total):
|
|
|
|
| 463 |
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] START | \"{sentence[:80]}\"")
|
| 464 |
t0 = _time.time()
|
|
|
|
| 465 |
import torch
|
| 466 |
|
|
|
|
| 467 |
with _gpu_lock:
|
|
|
|
| 468 |
from groq_tts import generate_audio
|
| 469 |
t_tts_start = _time.time()
|
| 470 |
audio_path = generate_audio(sentence)
|
| 471 |
t_tts_end = _time.time()
|
| 472 |
+
|
|
|
|
| 473 |
if torch.cuda.is_available():
|
| 474 |
torch.cuda.empty_cache()
|
| 475 |
+
|
| 476 |
audio_url = None
|
| 477 |
audio_duration = 0
|
| 478 |
audio_size = 0
|
| 479 |
if audio_path and os.path.exists(audio_path):
|
| 480 |
audio_size = os.path.getsize(audio_path)
|
| 481 |
audio_url = f"/api/audio/{os.path.basename(audio_path)}"
|
|
|
|
| 482 |
try:
|
| 483 |
r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration",
|
| 484 |
"-of","csv=p=0", audio_path], capture_output=True, text=True, check=True)
|
|
|
|
| 487 |
audio_duration = 3.0
|
| 488 |
t_probe_end = _time.time()
|
| 489 |
else:
|
| 490 |
+
t_probe_end = _time.time()
|
| 491 |
|
| 492 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] TTS: {t_tts_end-t_tts_start:.2f}s | audio: {audio_duration:.1f}s, {audio_size/1024:.0f}KB")
|
| 493 |
|
|
|
|
| 494 |
lipsync_video_url = None
|
| 495 |
if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
|
| 496 |
try:
|
|
|
|
| 500 |
t_float_start = _time.time()
|
| 501 |
lp = lipsync.generate(audio_path)
|
| 502 |
t_float_done = _time.time()
|
| 503 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] FLOAT: {t_float_done-t_float_start:.2f}s")
|
|
|
|
| 504 |
if lp and os.path.exists(lp):
|
|
|
|
| 505 |
lipsync_video_url = f"/api/stream/{os.path.basename(lp)}"
|
| 506 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] VIDEO: {os.path.getsize(lp)/1024:.0f}KB")
|
| 507 |
except Exception as e:
|
| 508 |
logger.error(f"[{_ts()}] [CHUNK {idx+1}/{total}] LIPSYNC FAILED: {e}", exc_info=True)
|
| 509 |
+
|
|
|
|
| 510 |
if torch.cuda.is_available():
|
| 511 |
torch.cuda.empty_cache()
|
| 512 |
|
| 513 |
total_time = _time.time() - t0
|
| 514 |
+
logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] DONE | total: {total_time:.2f}s")
|
| 515 |
|
| 516 |
return {
|
| 517 |
"audio_url": audio_url,
|
|
|
|
| 519 |
"lipsync_video_url": lipsync_video_url,
|
| 520 |
}
|
| 521 |
|
| 522 |
+
|
| 523 |
@app.post("/api/reset")
|
| 524 |
async def reset_chat():
|
| 525 |
from llm import reset_conversation
|
|
|
|
| 529 |
|
| 530 |
@app.get("/api/greeting")
|
| 531 |
async def get_greeting():
|
|
|
|
| 532 |
if GREETING_DATA:
|
| 533 |
return JSONResponse(GREETING_DATA)
|
| 534 |
return JSONResponse({"text": None})
|
|
|
|
| 545 |
<head>
|
| 546 |
<meta charset="UTF-8"/>
|
| 547 |
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
| 548 |
+
<title>Avatar Chatbot</title>
|
| 549 |
<style>
|
| 550 |
*{margin:0;padding:0;box-sizing:border-box}
|
| 551 |
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0f1c;color:#e2e8f0;height:100vh;display:flex;flex-direction:column;overflow:hidden}
|
| 552 |
.app-container{display:flex;flex-direction:column;height:100vh;max-width:900px;margin:0 auto;width:100%;padding:1rem;gap:.75rem}
|
| 553 |
+
.top-bar{display:flex;align-items:center;gap:.75rem;flex-shrink:0}
|
| 554 |
+
.top-bar label{font-size:.85rem;color:#94a3b8}
|
| 555 |
+
.top-bar select{background:#1e293b;color:#e2e8f0;border:1px solid rgba(59,130,246,.3);border-radius:6px;padding:6px 10px;font-size:.85rem;outline:none;cursor:pointer}
|
| 556 |
+
.top-bar select:focus{border-color:#3b82f6}
|
| 557 |
.video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center}
|
| 558 |
.video-container{position:relative;width:100%;height:100%}
|
| 559 |
.video-container video{width:100%;height:100%;object-fit:contain;display:block}
|
|
|
|
| 585 |
</head>
|
| 586 |
<body>
|
| 587 |
<div class="app-container">
|
| 588 |
+
<div class="top-bar">
|
| 589 |
+
<label for="avatarSelect">Avatar:</label>
|
| 590 |
+
<select id="avatarSelect"></select>
|
| 591 |
+
</div>
|
| 592 |
<div class="video-section">
|
| 593 |
<div class="video-container">
|
| 594 |
<video id="videoA" muted autoplay playsinline></video>
|
|
|
|
| 599 |
<div class="chat-section">
|
| 600 |
<div class="chat-log" id="chatLog"></div>
|
| 601 |
<div class="input-bar">
|
| 602 |
+
<input type="text" id="chatInput" placeholder="Say something..." autocomplete="off"/>
|
| 603 |
<button class="btn-send" id="btnSend">Send</button>
|
| 604 |
<button class="btn-reset" id="btnReset" title="Reset conversation">↻</button>
|
| 605 |
</div>
|
| 606 |
</div>
|
| 607 |
</div>
|
| 608 |
<script>
|
|
|
|
|
|
|
|
|
|
| 609 |
let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, lastPlayed=null, msgCounter=0;
|
| 610 |
+
let currentAvatarName='Avatar';
|
| 611 |
const videoA=document.getElementById('videoA'), videoB=document.getElementById('videoB');
|
| 612 |
const videoStatus=document.getElementById('videoStatus'), chatLog=document.getElementById('chatLog');
|
| 613 |
const chatInput=document.getElementById('chatInput'), btnSend=document.getElementById('btnSend');
|
| 614 |
+
const btnReset=document.getElementById('btnReset'), avatarSelect=document.getElementById('avatarSelect');
|
| 615 |
|
|
|
|
|
|
|
|
|
|
| 616 |
function getActiveVideo(){return currentVideo==='A'?videoA:videoB}
|
| 617 |
function getInactiveVideo(){return currentVideo==='A'?videoB:videoA}
|
| 618 |
|
|
|
|
| 622 |
isTransitioning=true;
|
| 623 |
const prev=getActiveVideo(), next=getInactiveVideo();
|
| 624 |
next.muted=(muted!==false);
|
| 625 |
+
next.style.opacity='1'; next.style.zIndex='0'; prev.style.zIndex='1';
|
| 626 |
+
next.src=url; next.load();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 627 |
next.onloadeddata=function(){
|
| 628 |
next.play().then(()=>{
|
| 629 |
+
next.style.zIndex='2'; prev.style.opacity='0';
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
setTimeout(()=>{
|
| 631 |
+
prev.pause(); prev.removeAttribute('src'); prev.load();
|
| 632 |
+
prev.style.zIndex='0'; prev.style.opacity='1';
|
|
|
|
|
|
|
|
|
|
|
|
|
| 633 |
next.style.zIndex='1';
|
| 634 |
currentVideo=currentVideo==='A'?'B':'A';
|
| 635 |
+
isTransitioning=false; resolve();
|
| 636 |
+
}, 180);
|
|
|
|
| 637 |
}).catch(()=>{
|
| 638 |
next.muted=true;
|
| 639 |
next.play().then(()=>{
|
| 640 |
+
next.style.zIndex='2'; prev.style.opacity='0';
|
|
|
|
| 641 |
setTimeout(()=>{
|
| 642 |
+
prev.pause(); prev.removeAttribute('src'); prev.load();
|
| 643 |
+
prev.style.zIndex='0'; prev.style.opacity='1';
|
|
|
|
|
|
|
|
|
|
| 644 |
next.style.zIndex='1';
|
| 645 |
currentVideo=currentVideo==='A'?'B':'A';
|
| 646 |
+
isTransitioning=false; resolve();
|
|
|
|
| 647 |
}, 180);
|
| 648 |
}).catch(()=>{ isTransitioning=false; resolve(); });
|
| 649 |
});
|
|
|
|
| 666 |
crossfadeTo('/api/stream/'+encodeURIComponent(c.filename), true);
|
| 667 |
}
|
| 668 |
|
|
|
|
|
|
|
| 669 |
let isSpeaking=false;
|
|
|
|
| 670 |
videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isSpeaking) playNextIdle(); });
|
| 671 |
videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isSpeaking) playNextIdle(); });
|
| 672 |
|
|
|
|
|
|
|
|
|
|
| 673 |
function waitForVideoEnd(){
|
| 674 |
return new Promise((resolve)=>{
|
| 675 |
const active=getActiveVideo();
|
| 676 |
+
if(active.ended || active.paused || !active.src){ resolve(); return; }
|
|
|
|
|
|
|
| 677 |
const onEnd=()=>{ active.removeEventListener('ended',onEnd); resolve(); };
|
| 678 |
active.addEventListener('ended', onEnd);
|
| 679 |
});
|
| 680 |
}
|
| 681 |
|
|
|
|
|
|
|
|
|
|
| 682 |
function addMessage(role, text, extra){
|
| 683 |
const id='msg_'+(msgCounter++);
|
| 684 |
const w=document.createElement('div'); w.className='msg '+role;
|
| 685 |
const b=document.createElement('div'); b.className='bubble '+role+(extra?' '+extra:''); b.id=id;
|
| 686 |
+
const l=document.createElement('div'); l.className='label'; l.textContent=role==='user'?'You':currentAvatarName;
|
| 687 |
const c=document.createElement('div'); c.className='content'; c.textContent=text;
|
| 688 |
b.appendChild(l); b.appendChild(c); w.appendChild(b);
|
| 689 |
chatLog.appendChild(w); chatLog.scrollTop=chatLog.scrollHeight;
|
|
|
|
| 696 |
return r;
|
| 697 |
}
|
| 698 |
|
| 699 |
+
// ---- Avatar switching ----
|
| 700 |
+
avatarSelect.addEventListener('change', async ()=>{
|
| 701 |
+
const name=avatarSelect.value;
|
| 702 |
+
if(!name || isProcessing) return;
|
| 703 |
+
videoStatus.textContent='Switching avatar...';
|
| 704 |
+
btnSend.disabled=true;
|
| 705 |
+
try{
|
| 706 |
+
const res=await fetch('/api/switch-avatar',{
|
| 707 |
+
method:'POST', headers:{'Content-Type':'application/json'},
|
| 708 |
+
body:JSON.stringify({avatar:name})
|
| 709 |
+
});
|
| 710 |
+
const data=await res.json();
|
| 711 |
+
if(data.videos){
|
| 712 |
+
videos=data.videos;
|
| 713 |
+
}
|
| 714 |
+
currentAvatarName=name;
|
| 715 |
+
chatLog.innerHTML='';
|
| 716 |
+
addMessage('bot','Hi, I\\'m '+name+'. How can I help you?');
|
| 717 |
+
playNextIdle();
|
| 718 |
+
}catch(e){
|
| 719 |
+
videoStatus.textContent='Switch failed: '+e.message;
|
| 720 |
+
}
|
| 721 |
+
btnSend.disabled=false;
|
| 722 |
+
});
|
| 723 |
+
|
| 724 |
+
// ---- Chat ----
|
| 725 |
async function sendMessage(){
|
| 726 |
const text=chatInput.value.trim();
|
| 727 |
if(!text||isProcessing) return;
|
|
|
|
| 730 |
const typing=addTypingIndicator();
|
| 731 |
|
| 732 |
try{
|
|
|
|
|
|
|
| 733 |
const res=await fetch('/api/chat',{
|
| 734 |
method:'POST', headers:{'Content-Type':'application/json'},
|
| 735 |
body:JSON.stringify({text:text})
|
|
|
|
| 737 |
if(!res.ok) throw new Error('HTTP '+res.status);
|
| 738 |
const data=await res.json();
|
| 739 |
|
|
|
|
| 740 |
typing.bubbleEl.classList.remove('typing');
|
| 741 |
typing.contentEl.classList.remove('typing-dots');
|
| 742 |
typing.contentEl.textContent='';
|
|
|
|
| 747 |
const fullText=data.text;
|
| 748 |
let displayedText='';
|
| 749 |
|
|
|
|
| 750 |
function streamChunkText(sentence, durationSec){
|
| 751 |
+
const chunkWords=sentence.split(/\\s+/).filter(w=>w.length>0);
|
| 752 |
if(chunkWords.length===0) return Promise.resolve();
|
| 753 |
const interval=Math.max(50, (durationSec*1000)/chunkWords.length);
|
| 754 |
return new Promise((resolve)=>{
|
| 755 |
let i=0;
|
| 756 |
const timer=setInterval(()=>{
|
| 757 |
+
if(i>=chunkWords.length){ clearInterval(timer); resolve(); return; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 758 |
if(displayedText.length>0) displayedText+=' ';
|
| 759 |
displayedText+=chunkWords[i];
|
| 760 |
typing.contentEl.textContent=displayedText;
|
|
|
|
| 764 |
});
|
| 765 |
}
|
| 766 |
|
|
|
|
| 767 |
function showAllText(){
|
| 768 |
typing.contentEl.textContent=fullText;
|
| 769 |
typing.contentEl.classList.remove('streaming-cursor');
|
| 770 |
chatLog.scrollTop=chatLog.scrollHeight;
|
| 771 |
}
|
| 772 |
|
|
|
|
|
|
|
|
|
|
| 773 |
async function fetchChunk(sid){
|
| 774 |
const r=await fetch('/api/chat/next',{
|
| 775 |
method:'POST', headers:{'Content-Type':'application/json'},
|
|
|
|
| 779 |
return await r.json();
|
| 780 |
}
|
| 781 |
|
|
|
|
| 782 |
let nextChunkPromise=fetchChunk(sessionId);
|
| 783 |
|
| 784 |
for(let i=0; i<totalChunks; i++){
|
| 785 |
videoStatus.textContent='generating '+(i+1)+'/'+totalChunks+'...';
|
|
|
|
|
|
|
| 786 |
const chunk=await nextChunkPromise;
|
| 787 |
if(!chunk || chunk.done) break;
|
| 788 |
+
if(i+1 < totalChunks) nextChunkPromise=fetchChunk(sessionId);
|
| 789 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 790 |
isSpeaking=true;
|
|
|
|
| 791 |
await waitForVideoEnd();
|
|
|
|
| 792 |
videoStatus.textContent='speaking '+(i+1)+'/'+totalChunks;
|
| 793 |
|
|
|
|
| 794 |
let audioPromise=null;
|
| 795 |
if(chunk.audio_url){
|
| 796 |
const audio=new Audio(chunk.audio_url);
|
| 797 |
audioPromise=new Promise((resolve)=>{
|
| 798 |
+
audio.onended=resolve; audio.onerror=resolve;
|
|
|
|
| 799 |
audio.play().catch(()=>resolve());
|
| 800 |
});
|
| 801 |
}
|
| 802 |
|
|
|
|
| 803 |
const chunkDuration=chunk.audio_duration||3.0;
|
| 804 |
const textStreamPromise=streamChunkText(chunk.sentence, chunkDuration);
|
| 805 |
|
|
|
|
| 806 |
if(chunk.lipsync_video_url){
|
| 807 |
await crossfadeTo(chunk.lipsync_video_url, true);
|
|
|
|
| 808 |
await waitForVideoEnd();
|
| 809 |
} else if(audioPromise){
|
|
|
|
| 810 |
await audioPromise;
|
| 811 |
}
|
|
|
|
|
|
|
| 812 |
await textStreamPromise;
|
| 813 |
}
|
| 814 |
|
|
|
|
| 815 |
showAllText();
|
|
|
|
|
|
|
| 816 |
isSpeaking=false;
|
| 817 |
playNextIdle();
|
| 818 |
|
| 819 |
}catch(err){
|
| 820 |
typing.bubbleEl.classList.remove('typing');
|
| 821 |
typing.bubbleEl.classList.add('error');
|
| 822 |
+
typing.contentEl.classList.remove('typing-dots','streaming-cursor');
|
|
|
|
| 823 |
typing.contentEl.textContent='Error: '+err.message;
|
| 824 |
}
|
| 825 |
isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus();
|
|
|
|
| 830 |
btnReset.addEventListener('click', async()=>{
|
| 831 |
await fetch('/api/reset',{method:'POST'});
|
| 832 |
chatLog.innerHTML='';
|
| 833 |
+
addMessage('bot','Conversation reset. How can I help you?');
|
| 834 |
});
|
| 835 |
|
| 836 |
+
// ---- Init ----
|
|
|
|
|
|
|
| 837 |
async function init(){
|
| 838 |
try{
|
| 839 |
+
// Load avatars into dropdown
|
| 840 |
+
const avatarRes=await fetch('/api/avatars');
|
| 841 |
+
const avatarData=await avatarRes.json();
|
| 842 |
+
avatarSelect.innerHTML='';
|
| 843 |
+
for(const a of avatarData.avatars){
|
| 844 |
+
const opt=document.createElement('option');
|
| 845 |
+
opt.value=a.name; opt.textContent=a.name;
|
| 846 |
+
if(a.name===avatarData.current) opt.selected=true;
|
| 847 |
+
avatarSelect.appendChild(opt);
|
| 848 |
+
}
|
| 849 |
+
if(avatarData.current) currentAvatarName=avatarData.current;
|
| 850 |
+
|
| 851 |
+
// Load idle videos
|
| 852 |
const res=await fetch('/api/videos');
|
| 853 |
const data=await res.json();
|
| 854 |
+
if(!data.videos.length){ videoStatus.textContent='No idle videos...'; }
|
| 855 |
+
else{ videos=data.videos; videoStatus.textContent=videos.length+' idle clips loaded'; }
|
|
|
|
| 856 |
|
| 857 |
+
// Check greeting
|
| 858 |
const greetRes=await fetch('/api/greeting');
|
| 859 |
const greeting=await greetRes.json();
|
| 860 |
|
| 861 |
if(greeting.lipsync_video_url){
|
|
|
|
| 862 |
videoStatus.textContent='greeting...';
|
| 863 |
addMessage('bot', greeting.text);
|
|
|
|
| 864 |
if(greeting.audio_url){
|
| 865 |
const audio=new Audio(greeting.audio_url);
|
| 866 |
audio.play().catch(e=>console.warn('[AUDIO]',e));
|
| 867 |
}
|
|
|
|
| 868 |
await crossfadeTo(greeting.lipsync_video_url, true);
|
| 869 |
await waitForVideoEnd();
|
| 870 |
playNextIdle();
|
| 871 |
} else {
|
| 872 |
+
addMessage('bot', greeting.text || 'Hello! Feel free to ask me anything.');
|
|
|
|
| 873 |
playNextIdle();
|
| 874 |
}
|
| 875 |
}catch(e){ videoStatus.textContent='Error: '+e.message; }
|