J Z commited on
Commit
ceafffc
·
verified ·
1 Parent(s): 4084547

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -958
app.py DELETED
@@ -1,958 +0,0 @@
1
- """
2
- Avatar Chatbot - HuggingFace Spaces Edition
3
- Phase 1: Chat UI + Idle Video Loop + LLM (Groq) + TTS (Groq)
4
- Phase 2: FLOAT lipsync (to be added)
5
- """
6
- from fastapi import FastAPI, HTTPException, Request
7
- from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
8
- from fastapi.staticfiles import StaticFiles
9
- from pathlib import Path
10
- import mimetypes
11
- import os
12
- import logging
13
- import asyncio
14
- import time
15
- import subprocess
16
-
17
- logging.basicConfig(level=logging.INFO)
18
- logger = logging.getLogger(__name__)
19
-
20
- app = FastAPI()
21
-
22
- # --- Config ---
23
- VIDEOS_DIR = Path("/tmp/videos")
24
- TTS_DIR = Path("/tmp/tts_output")
25
- LIPSYNC_DIR = Path("/tmp/lipsync_output")
26
- REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot")
27
-
28
- FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true"
29
-
30
- for d in [VIDEOS_DIR, TTS_DIR, LIPSYNC_DIR]:
31
- d.mkdir(parents=True, exist_ok=True)
32
-
33
-
34
- # --- Startup: download idle videos + init FLOAT ---
35
- @app.on_event("startup")
36
- async def startup():
37
- await download_videos()
38
- if FLOAT_ENABLED:
39
- await init_float()
40
-
41
-
42
- async def init_float():
43
- """Initialize FLOAT lipsync model (runs once at startup)."""
44
- import torch
45
- logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}")
46
- if not torch.cuda.is_available():
47
- logger.warning("[STARTUP] No GPU detected - FLOAT lipsync disabled")
48
- return
49
-
50
- try:
51
- from float_lipsync import get_lipsync
52
- from huggingface_hub import hf_hub_download
53
- import shutil
54
-
55
- checkpoints_dir = Path("/app/checkpoints")
56
- checkpoints_dir.mkdir(parents=True, exist_ok=True)
57
-
58
- # Download wav2vec2-base-960h from HuggingFace
59
- w2v_dir = checkpoints_dir / "wav2vec2-base-960h"
60
- if not w2v_dir.exists():
61
- logger.info("[STARTUP] Downloading wav2vec2-base-960h...")
62
- from huggingface_hub import snapshot_download
63
- snapshot_download(
64
- repo_id="facebook/wav2vec2-base-960h",
65
- local_dir=str(w2v_dir),
66
- )
67
- logger.info("[STARTUP] wav2vec2-base-960h downloaded")
68
-
69
- # Download wav2vec-english-speech-emotion-recognition
70
- emo_dir = checkpoints_dir / "wav2vec-english-speech-emotion-recognition"
71
- if not emo_dir.exists():
72
- logger.info("[STARTUP] Downloading emotion recognition model...")
73
- from huggingface_hub import snapshot_download
74
- snapshot_download(
75
- repo_id="r-f/wav2vec-english-speech-emotion-recognition",
76
- local_dir=str(emo_dir),
77
- )
78
- logger.info("[STARTUP] Emotion model downloaded")
79
-
80
- # Check for float.pth - download from repo if not present (xet storage)
81
- float_ckpt = checkpoints_dir / "float.pth"
82
- if not float_ckpt.exists() or float_ckpt.stat().st_size < 1024:
83
- logger.info("[STARTUP] Downloading float.pth from repo...")
84
- try:
85
- downloaded = hf_hub_download(
86
- repo_id=REPO_ID, repo_type="space",
87
- filename="app/checkpoints/float.pth",
88
- local_dir="/tmp/hf_download",
89
- )
90
- shutil.copy2(downloaded, float_ckpt)
91
- logger.info(f"[STARTUP] float.pth downloaded: {float_ckpt.stat().st_size / 1e6:.1f} MB")
92
- except Exception as e:
93
- logger.warning(f"[STARTUP] Could not download float.pth: {e}")
94
- logger.warning("[STARTUP] Upload it to your Space repo under app/checkpoints/float.pth")
95
- logger.warning("[STARTUP] FLOAT lipsync will be disabled until checkpoint is available")
96
- return
97
-
98
- # Check for reference image - download from repo if needed
99
- assets_dir = Path("/app/assets")
100
- assets_dir.mkdir(parents=True, exist_ok=True)
101
- ref_image = assets_dir / "ref.png"
102
- if not ref_image.exists() or ref_image.stat().st_size < 1024:
103
- logger.info("[STARTUP] Looking for reference image in repo...")
104
- try:
105
- # Try common image extensions
106
- for ext in ["ref.png", "ref.jpg", "ref.jpeg", "main2.png"]:
107
- try:
108
- downloaded = hf_hub_download(
109
- repo_id=REPO_ID, repo_type="space",
110
- filename=f"app/assets/{ext}",
111
- local_dir="/tmp/hf_download",
112
- )
113
- dest = assets_dir / ext
114
- shutil.copy2(downloaded, dest)
115
- ref_image = dest
116
- logger.info(f"[STARTUP] Reference image downloaded: {ext}")
117
- break
118
- except Exception:
119
- continue
120
- else:
121
- logger.warning("[STARTUP] No reference image found in app/assets/")
122
- logger.warning("[STARTUP] Upload a face image as app/assets/ref.png")
123
- return
124
- except Exception as e:
125
- logger.warning(f"[STARTUP] Could not download reference image: {e}")
126
- return
127
-
128
- # Initialize FLOAT
129
- lipsync = get_lipsync()
130
- config = {
131
- "ref_path": str(ref_image),
132
- "ckpt_path": str(float_ckpt),
133
- "wav2vec_model_path": str(w2v_dir),
134
- "audio2emotion_path": str(emo_dir),
135
- }
136
- lipsync.initialize(config)
137
- logger.info("[STARTUP] ✓ FLOAT lipsync system ready!")
138
-
139
- except Exception as e:
140
- logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True)
141
-
142
-
143
- async def download_videos():
144
- from huggingface_hub import HfApi, hf_hub_download
145
- import shutil
146
-
147
- try:
148
- api = HfApi()
149
- files = api.list_repo_files(repo_id=REPO_ID, repo_type="space")
150
- video_files = [
151
- f for f in files
152
- if f.startswith("app/videos/")
153
- and any(f.endswith(ext) for ext in (".mp4", ".webm", ".mkv", ".mov"))
154
- ]
155
- logger.info(f"Found {len(video_files)} video files in repo")
156
-
157
- for vf in video_files:
158
- filename = Path(vf).name
159
- dest = VIDEOS_DIR / filename
160
- if dest.exists() and dest.stat().st_size > 1024:
161
- continue
162
- logger.info(f"Downloading {vf}...")
163
- downloaded = hf_hub_download(
164
- repo_id=REPO_ID, repo_type="space", filename=vf,
165
- local_dir="/tmp/hf_download",
166
- )
167
- shutil.copy2(downloaded, dest)
168
- logger.info(f"Downloaded {filename}: {dest.stat().st_size} bytes")
169
- except Exception as e:
170
- logger.error(f"Error downloading videos: {e}")
171
-
172
-
173
- # --- API: list idle videos ---
174
- @app.get("/api/videos")
175
- async def list_videos():
176
- if not VIDEOS_DIR.exists():
177
- return JSONResponse({"videos": []})
178
- videos = []
179
- for f in sorted(VIDEOS_DIR.iterdir()):
180
- if f.suffix.lower() in (".mp4", ".webm", ".mkv", ".mov") and f.stat().st_size > 1024:
181
- videos.append({"name": f.stem, "filename": f.name})
182
- return JSONResponse({"videos": videos})
183
-
184
-
185
- # --- API: stream a video file ---
186
- @app.get("/api/stream/{filename}")
187
- async def stream_video(filename: str):
188
- # Check both idle videos and lipsync output
189
- video_path = None
190
- for search_dir in [VIDEOS_DIR, LIPSYNC_DIR, TTS_DIR]:
191
- candidate = search_dir / filename
192
- if candidate.exists() and candidate.is_file():
193
- video_path = candidate
194
- break
195
-
196
- if not video_path:
197
- raise HTTPException(status_code=404, detail="Video not found")
198
-
199
- mime_type = mimetypes.guess_type(filename)[0] or "video/mp4"
200
- file_size = video_path.stat().st_size
201
-
202
- def iter_file():
203
- with open(video_path, "rb") as f:
204
- while chunk := f.read(1024 * 1024):
205
- yield chunk
206
-
207
- return StreamingResponse(
208
- iter_file(),
209
- media_type=mime_type,
210
- headers={"Content-Length": str(file_size), "Accept-Ranges": "bytes"},
211
- )
212
-
213
-
214
- # --- API: stream audio file ---
215
- @app.get("/api/audio/{filename}")
216
- async def stream_audio(filename: str):
217
- audio_path = TTS_DIR / filename
218
- if not audio_path.exists():
219
- raise HTTPException(status_code=404, detail="Audio not found")
220
-
221
- mime_type = mimetypes.guess_type(filename)[0] or "audio/wav"
222
- file_size = audio_path.stat().st_size
223
-
224
- def iter_file():
225
- with open(audio_path, "rb") as f:
226
- while chunk := f.read(1024 * 1024):
227
- yield chunk
228
-
229
- return StreamingResponse(
230
- iter_file(),
231
- media_type=mime_type,
232
- headers={"Content-Length": str(file_size)},
233
- )
234
-
235
-
236
- # --- API: chat endpoint (returns text immediately, then streams lipsync chunks) ---
237
- @app.post("/api/chat")
238
- async def chat(request: Request):
239
- """
240
- Step 1: LLM generates full response text.
241
- Step 2: Returns text immediately so frontend can show it.
242
- Step 3: Frontend then calls /api/chat/generate to get lipsync chunks via SSE.
243
- """
244
- body = await request.json()
245
- user_text = body.get("text", "").strip()
246
-
247
- if not user_text:
248
- raise HTTPException(status_code=400, detail="No text provided")
249
-
250
- logger.info(f"[CHAT] User: {user_text}")
251
-
252
- # 1. LLM
253
- from llm import generate_darwin_response
254
- llm_result = generate_darwin_response(user_text)
255
- reply_text = llm_result['text']
256
- logger.info(f"[CHAT] Darwin: {reply_text}")
257
-
258
- # Split into sentences (by period only, not commas)
259
- import re
260
- raw_sentences = re.split(r'(?<=\.)\s+', reply_text)
261
- sentences = [s.strip() for s in raw_sentences if s.strip()]
262
- # If no periods found, treat whole text as one chunk
263
- if not sentences:
264
- sentences = [reply_text]
265
-
266
- return JSONResponse({
267
- "text": reply_text,
268
- "emotion": llm_result.get('emotion', 'neutral'),
269
- "sentences": sentences,
270
- })
271
-
272
-
273
- @app.post("/api/chat/generate")
274
- async def chat_generate(request: Request):
275
- """
276
- SSE endpoint: receives a sentence, generates TTS + lipsync, returns result.
277
- Called once per sentence by the frontend.
278
- """
279
- body = await request.json()
280
- sentence = body.get("sentence", "").strip()
281
-
282
- if not sentence:
283
- raise HTTPException(status_code=400, detail="No sentence provided")
284
-
285
- logger.info(f"[GEN] Sentence: {sentence[:60]}...")
286
-
287
- # 1. TTS
288
- from tts import generate_audio
289
- audio_path = generate_audio(sentence)
290
- audio_url = None
291
- audio_duration = 0
292
-
293
- if audio_path and os.path.exists(audio_path):
294
- audio_filename = os.path.basename(audio_path)
295
- audio_url = f"/api/audio/{audio_filename}"
296
-
297
- try:
298
- result = subprocess.run(
299
- ["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
300
- "-of", "csv=p=0", audio_path],
301
- capture_output=True, text=True, check=True
302
- )
303
- audio_duration = float(result.stdout.strip())
304
- except Exception:
305
- audio_duration = 3.0
306
-
307
- # 2. Lipsync
308
- lipsync_video_url = None
309
- if FLOAT_ENABLED and audio_path and os.path.exists(audio_path):
310
- try:
311
- from float_lipsync import get_lipsync
312
- lipsync = get_lipsync()
313
- if lipsync.ready:
314
- logger.info(f"[GEN] Running FLOAT lipsync...")
315
- lipsync_path = lipsync.generate(audio_path)
316
- if lipsync_path and os.path.exists(lipsync_path):
317
- lipsync_filename = os.path.basename(lipsync_path)
318
- lipsync_video_url = f"/api/stream/{lipsync_filename}"
319
- logger.info(f"[GEN] Lipsync video: {lipsync_video_url}")
320
- except Exception as e:
321
- logger.error(f"[GEN] Lipsync failed: {e}", exc_info=True)
322
-
323
- return JSONResponse({
324
- "sentence": sentence,
325
- "audio_url": audio_url,
326
- "audio_duration": audio_duration,
327
- "lipsync_video_url": lipsync_video_url,
328
- })
329
-
330
-
331
- # --- API: reset conversation ---
332
- @app.post("/api/reset")
333
- async def reset_chat():
334
- from llm import reset_conversation
335
- reset_conversation()
336
- return JSONResponse({"status": "ok"})
337
-
338
-
339
- # --- Main page ---
340
- @app.get("/", response_class=HTMLResponse)
341
- async def index():
342
- return HTML_PAGE
343
-
344
-
345
- HTML_PAGE = """\
346
- <!DOCTYPE html>
347
- <html lang="en">
348
- <head>
349
- <meta charset="UTF-8" />
350
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
351
- <title>Darwin Avatar Chatbot</title>
352
- <style>
353
- * { margin: 0; padding: 0; box-sizing: border-box; }
354
-
355
- body {
356
- font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
357
- background: #0a0f1c;
358
- color: #e2e8f0;
359
- height: 100vh;
360
- display: flex;
361
- flex-direction: column;
362
- overflow: hidden;
363
- }
364
-
365
- /* --- LAYOUT --- */
366
- .app-container {
367
- display: flex;
368
- flex-direction: column;
369
- height: 100vh;
370
- max-width: 900px;
371
- margin: 0 auto;
372
- width: 100%;
373
- padding: 1rem;
374
- gap: 0.75rem;
375
- }
376
-
377
- /* --- VIDEO PLAYER --- */
378
- .video-section {
379
- position: relative;
380
- width: 100%;
381
- min-height: 300px;
382
- max-height: 45vh;
383
- background: #000;
384
- border-radius: 12px;
385
- overflow: hidden;
386
- flex-shrink: 0;
387
- display: flex;
388
- align-items: center;
389
- justify-content: center;
390
- }
391
-
392
- .video-container {
393
- position: relative;
394
- width: 100%;
395
- height: 100%;
396
- }
397
-
398
- .video-container video {
399
- width: 100%;
400
- height: 100%;
401
- object-fit: contain;
402
- display: block;
403
- }
404
-
405
- #videoA, #videoB {
406
- position: absolute;
407
- top: 0; left: 0;
408
- width: 100%; height: 100%;
409
- object-fit: contain;
410
- transition: opacity 0.25s ease;
411
- }
412
-
413
- #videoA { z-index: 1; opacity: 1; }
414
- #videoB { z-index: 0; opacity: 0; }
415
-
416
- .video-status {
417
- position: absolute;
418
- bottom: 8px;
419
- right: 12px;
420
- font-size: 0.7rem;
421
- color: rgba(255,255,255,0.4);
422
- background: rgba(0,0,0,0.5);
423
- padding: 2px 8px;
424
- border-radius: 4px;
425
- z-index: 10;
426
- }
427
-
428
- /* --- CHAT LOG --- */
429
- .chat-section {
430
- flex: 1;
431
- display: flex;
432
- flex-direction: column;
433
- min-height: 0;
434
- background: rgba(15, 23, 42, 0.6);
435
- border: 1px solid rgba(59, 130, 246, 0.15);
436
- border-radius: 12px;
437
- overflow: hidden;
438
- }
439
-
440
- .chat-log {
441
- flex: 1;
442
- overflow-y: auto;
443
- padding: 1rem;
444
- display: flex;
445
- flex-direction: column;
446
- gap: 0.5rem;
447
- }
448
-
449
- .chat-log::-webkit-scrollbar { width: 6px; }
450
- .chat-log::-webkit-scrollbar-track { background: transparent; }
451
- .chat-log::-webkit-scrollbar-thumb { background: #334155; border-radius: 3px; }
452
-
453
- /* Message bubbles */
454
- .msg { display: flex; width: 100%; }
455
- .msg.user { justify-content: flex-end; }
456
- .msg.bot { justify-content: flex-start; }
457
-
458
- .bubble {
459
- max-width: 85%;
460
- padding: 10px 14px;
461
- border-radius: 12px;
462
- word-wrap: break-word;
463
- font-size: 0.9rem;
464
- line-height: 1.4;
465
- }
466
-
467
- .bubble.user {
468
- background: linear-gradient(135deg, #1e40af, #3b82f6);
469
- color: #fff;
470
- border-bottom-right-radius: 4px;
471
- }
472
-
473
- .bubble.bot {
474
- background: rgba(30, 41, 59, 0.8);
475
- border: 1px solid rgba(59, 130, 246, 0.2);
476
- color: #e2e8f0;
477
- border-bottom-left-radius: 4px;
478
- }
479
-
480
- .bubble.bot .label { color: #60a5fa; font-weight: 600; font-size: 0.8rem; margin-bottom: 2px; }
481
- .bubble.user .label { color: rgba(255,255,255,0.7); font-weight: 600; font-size: 0.8rem; margin-bottom: 2px; }
482
-
483
- .bubble.typing {
484
- color: #64748b;
485
- font-style: italic;
486
- }
487
-
488
- .bubble.error {
489
- background: rgba(127, 29, 29, 0.5);
490
- border-color: rgba(239, 68, 68, 0.3);
491
- color: #fca5a5;
492
- }
493
-
494
- @keyframes dots {
495
- 0%, 20% { content: ''; }
496
- 40% { content: '.'; }
497
- 60% { content: '..'; }
498
- 80%, 100% { content: '...'; }
499
- }
500
- .typing-dots::after {
501
- content: '';
502
- animation: dots 1.5s infinite;
503
- }
504
-
505
- .streaming-cursor::after {
506
- content: '▊';
507
- animation: blink 0.7s infinite;
508
- color: #60a5fa;
509
- }
510
- @keyframes blink { 0%, 100% { opacity: 1; } 50% { opacity: 0; } }
511
-
512
- /* --- INPUT BAR --- */
513
- .input-bar {
514
- display: flex;
515
- gap: 0.5rem;
516
- padding: 0.75rem 1rem;
517
- background: rgba(15, 23, 42, 0.8);
518
- border-top: 1px solid rgba(59, 130, 246, 0.1);
519
- }
520
-
521
- .input-bar input {
522
- flex: 1;
523
- background: rgba(30, 41, 59, 0.8);
524
- border: 1px solid rgba(59, 130, 246, 0.25);
525
- border-radius: 8px;
526
- padding: 10px 14px;
527
- color: #e2e8f0;
528
- font-size: 0.9rem;
529
- outline: none;
530
- transition: border-color 0.2s;
531
- }
532
-
533
- .input-bar input:focus {
534
- border-color: #3b82f6;
535
- }
536
-
537
- .input-bar input::placeholder { color: #64748b; }
538
-
539
- .input-bar button {
540
- padding: 10px 20px;
541
- border: none;
542
- border-radius: 8px;
543
- font-size: 0.9rem;
544
- cursor: pointer;
545
- transition: all 0.15s;
546
- font-weight: 500;
547
- }
548
-
549
- .btn-send {
550
- background: #3b82f6;
551
- color: #fff;
552
- }
553
- .btn-send:hover { background: #2563eb; }
554
- .btn-send:disabled { background: #334155; color: #64748b; cursor: not-allowed; }
555
-
556
- .btn-reset {
557
- background: rgba(100, 116, 139, 0.3);
558
- color: #94a3b8;
559
- border: 1px solid rgba(100, 116, 139, 0.3);
560
- }
561
- .btn-reset:hover { background: rgba(100, 116, 139, 0.5); }
562
- </style>
563
- </head>
564
- <body>
565
-
566
- <div class="app-container">
567
-
568
- <!-- VIDEO PLAYER with crossfade -->
569
- <div class="video-section">
570
- <div class="video-container">
571
- <video id="videoA" muted autoplay playsinline></video>
572
- <video id="videoB" muted autoplay playsinline></video>
573
- </div>
574
- <div class="video-status" id="videoStatus">Loading...</div>
575
- </div>
576
-
577
- <!-- CHAT -->
578
- <div class="chat-section">
579
- <div class="chat-log" id="chatLog"></div>
580
- <div class="input-bar">
581
- <input type="text" id="chatInput" placeholder="Ask Darwin something..."
582
- autocomplete="off" />
583
- <button class="btn-send" id="btnSend">Send</button>
584
- <button class="btn-reset" id="btnReset" title="Reset conversation">↻</button>
585
- </div>
586
- </div>
587
-
588
- </div>
589
-
590
- <script>
591
- // ======================================================================
592
- // STATE
593
- // ======================================================================
594
- let videos = [];
595
- let currentVideo = 'A';
596
- let isTransitioning = false;
597
- let isProcessing = false;
598
- let lastPlayed = null;
599
-
600
- const videoA = document.getElementById('videoA');
601
- const videoB = document.getElementById('videoB');
602
- const videoStatus = document.getElementById('videoStatus');
603
- const chatLog = document.getElementById('chatLog');
604
- const chatInput = document.getElementById('chatInput');
605
- const btnSend = document.getElementById('btnSend');
606
- const btnReset = document.getElementById('btnReset');
607
-
608
- // ======================================================================
609
- // VIDEO: Crossfade System
610
- // ======================================================================
611
- function getActiveVideo() { return currentVideo === 'A' ? videoA : videoB; }
612
- function getInactiveVideo() { return currentVideo === 'A' ? videoB : videoA; }
613
-
614
- function crossfadeTo(videoUrl, isMuted) {
615
- if (isTransitioning) {
616
- console.log('[VIDEO] Already transitioning, queuing retry...');
617
- setTimeout(() => crossfadeTo(videoUrl, isMuted), 200);
618
- return;
619
- }
620
- isTransitioning = true;
621
-
622
- const prev = getActiveVideo();
623
- const next = getInactiveVideo();
624
-
625
- // Set muted state
626
- next.muted = (isMuted !== false);
627
- next.src = videoUrl;
628
- next.load();
629
-
630
- next.onloadeddata = function() {
631
- next.style.zIndex = '2';
632
- prev.style.zIndex = '1';
633
- next.style.opacity = '0';
634
-
635
- next.play().then(() => {
636
- next.style.opacity = '1';
637
-
638
- setTimeout(() => {
639
- prev.style.opacity = '0';
640
- prev.pause();
641
- prev.src = '';
642
- prev.style.zIndex = '0';
643
- next.style.zIndex = '1';
644
-
645
- // Swap active tracker BEFORE clearing transition flag
646
- currentVideo = currentVideo === 'A' ? 'B' : 'A';
647
- isTransitioning = false;
648
- console.log('[VIDEO] Crossfade done, active=', currentVideo, 'src=', videoUrl.split('/').pop());
649
- }, 300);
650
- }).catch(err => {
651
- console.error('[VIDEO] Play failed:', err);
652
- // Force muted retry
653
- next.muted = true;
654
- next.play().then(() => {
655
- next.style.opacity = '1';
656
- setTimeout(() => {
657
- prev.style.opacity = '0';
658
- prev.pause();
659
- prev.src = '';
660
- prev.style.zIndex = '0';
661
- next.style.zIndex = '1';
662
- currentVideo = currentVideo === 'A' ? 'B' : 'A';
663
- isTransitioning = false;
664
- }, 300);
665
- }).catch(e => {
666
- console.error('[VIDEO] Muted retry failed:', e);
667
- isTransitioning = false;
668
- // Fallback: just play next idle after a delay
669
- setTimeout(() => playNextIdle(), 1000);
670
- });
671
- });
672
- };
673
-
674
- next.onerror = function() {
675
- console.error('[VIDEO] Load failed:', videoUrl);
676
- isTransitioning = false;
677
- // Don't freeze — play next idle
678
- setTimeout(() => playNextIdle(), 500);
679
- };
680
- }
681
-
682
- function pickRandomIdle() {
683
- if (videos.length === 0) return null;
684
- if (videos.length === 1) return videos[0];
685
- let pick;
686
- do {
687
- pick = videos[Math.floor(Math.random() * videos.length)];
688
- } while (pick.filename === lastPlayed && videos.length > 1);
689
- return pick;
690
- }
691
-
692
- function playNextIdle() {
693
- const clip = pickRandomIdle();
694
- if (!clip) return;
695
- lastPlayed = clip.filename;
696
- videoStatus.textContent = 'idle: ' + clip.name;
697
- crossfadeTo('/api/stream/' + encodeURIComponent(clip.filename), true);
698
- }
699
-
700
- // When active video ends → decide what to play next
701
- videoA.addEventListener('ended', () => onVideoEnded('A'));
702
- videoB.addEventListener('ended', () => onVideoEnded('B'));
703
-
704
- let lipsyncQueue = [];
705
- let playingLipsync = false;
706
- let currentAudio = null;
707
-
708
- function onVideoEnded(which) {
709
- console.log('[VIDEO] ended on', which, 'currentVideo=', currentVideo,
710
- 'queue=', lipsyncQueue.length, 'playingLipsync=', playingLipsync);
711
-
712
- // Only handle if this is the currently active video
713
- if (which !== currentVideo) return;
714
-
715
- if (playingLipsync) {
716
- // Current lipsync clip finished — play next or return to idle
717
- playingLipsync = false;
718
- if (lipsyncQueue.length > 0) {
719
- playNextLipsync();
720
- } else {
721
- videoStatus.textContent = 'idle';
722
- playNextIdle();
723
- }
724
- } else {
725
- // Idle clip finished
726
- if (lipsyncQueue.length > 0) {
727
- playNextLipsync();
728
- } else {
729
- playNextIdle();
730
- }
731
- }
732
- }
733
-
734
- function playNextLipsync() {
735
- if (lipsyncQueue.length === 0) {
736
- playingLipsync = false;
737
- playNextIdle();
738
- return;
739
- }
740
-
741
- const item = lipsyncQueue.shift();
742
- console.log('[LIPSYNC] Playing chunk, remaining in queue:', lipsyncQueue.length);
743
-
744
- // Play audio
745
- if (item.audio_url) {
746
- if (currentAudio) {
747
- currentAudio.pause();
748
- currentAudio = null;
749
- }
750
- currentAudio = new Audio(item.audio_url);
751
- currentAudio.play().catch(e => console.warn('[AUDIO] Play failed:', e));
752
- }
753
-
754
- if (item.lipsync_video_url) {
755
- playingLipsync = true;
756
- videoStatus.textContent = 'speaking...';
757
- // Use crossfade — the ended event will fire onVideoEnded
758
- crossfadeTo(item.lipsync_video_url, true);
759
- } else {
760
- // No lipsync video — just let audio play over idle
761
- // Use audio duration as timeout, then check queue again
762
- const waitMs = (item.audio_duration || 3) * 1000;
763
- playingLipsync = true;
764
- videoStatus.textContent = 'speaking (audio only)...';
765
- setTimeout(() => {
766
- playingLipsync = false;
767
- if (lipsyncQueue.length > 0) {
768
- playNextLipsync();
769
- }
770
- // Don't force idle — let current video ended handler do it
771
- }, waitMs);
772
- }
773
- }
774
-
775
- // External trigger: called when a new chunk arrives and nothing is playing
776
- function tryStartLipsync() {
777
- if (!playingLipsync && lipsyncQueue.length > 0) {
778
- playNextLipsync();
779
- }
780
- }
781
-
782
- // ======================================================================
783
- // CHAT: Message rendering
784
- // ======================================================================
785
- let msgCounter = 0;
786
-
787
- function addMessage(role, text, extraClass = '') {
788
- const id = `msg_${msgCounter++}`;
789
- const wrapper = document.createElement('div');
790
- wrapper.className = `msg ${role}`;
791
-
792
- const bubble = document.createElement('div');
793
- bubble.className = `bubble ${role} ${extraClass}`;
794
- bubble.id = id;
795
-
796
- const label = document.createElement('div');
797
- label.className = 'label';
798
- label.textContent = role === 'user' ? 'You' : 'Darwin';
799
-
800
- const content = document.createElement('div');
801
- content.className = 'content';
802
- content.textContent = text;
803
-
804
- bubble.appendChild(label);
805
- bubble.appendChild(content);
806
- wrapper.appendChild(bubble);
807
- chatLog.appendChild(wrapper);
808
- chatLog.scrollTop = chatLog.scrollHeight;
809
- return { id, contentEl: content, bubbleEl: bubble };
810
- }
811
-
812
- function addTypingIndicator() {
813
- const { id, contentEl, bubbleEl } = addMessage('bot', '', 'typing');
814
- contentEl.className = 'content typing-dots';
815
- contentEl.textContent = 'thinking';
816
- return { id, contentEl, bubbleEl };
817
- }
818
-
819
- function streamText(contentEl, text, durationMs) {
820
- contentEl.className = 'content streaming-cursor';
821
- contentEl.textContent = '';
822
-
823
- let i = 0;
824
- const delay = durationMs / text.length;
825
-
826
- const interval = setInterval(() => {
827
- if (i < text.length) {
828
- contentEl.textContent += text[i];
829
- i++;
830
- chatLog.scrollTop = chatLog.scrollHeight;
831
- } else {
832
- clearInterval(interval);
833
- contentEl.classList.remove('streaming-cursor');
834
- }
835
- }, delay);
836
- }
837
-
838
- // ======================================================================
839
- // CHAT: Send message
840
- // ======================================================================
841
- async function sendMessage() {
842
- const text = chatInput.value.trim();
843
- if (!text || isProcessing) return;
844
-
845
- isProcessing = true;
846
- btnSend.disabled = true;
847
- chatInput.value = '';
848
-
849
- // Show user message
850
- addMessage('user', text);
851
-
852
- // Show typing indicator
853
- const typing = addTypingIndicator();
854
-
855
- try {
856
- // Step 1: Get LLM response (fast - just text)
857
- const res = await fetch('/api/chat', {
858
- method: 'POST',
859
- headers: { 'Content-Type': 'application/json' },
860
- body: JSON.stringify({ text })
861
- });
862
-
863
- if (!res.ok) throw new Error(`HTTP ${res.status}`);
864
- const data = await res.json();
865
-
866
- // Show the text immediately
867
- typing.bubbleEl.classList.remove('typing');
868
- typing.contentEl.classList.remove('typing-dots');
869
- typing.contentEl.textContent = data.text;
870
-
871
- // Re-enable input immediately so user can keep chatting
872
- isProcessing = false;
873
- btnSend.disabled = false;
874
- chatInput.focus();
875
-
876
- // Step 2: Generate TTS + lipsync for each sentence in background
877
- const sentences = data.sentences || [data.text];
878
- for (const sentence of sentences) {
879
- try {
880
- const genRes = await fetch('/api/chat/generate', {
881
- method: 'POST',
882
- headers: { 'Content-Type': 'application/json' },
883
- body: JSON.stringify({ sentence })
884
- });
885
-
886
- if (!genRes.ok) continue;
887
- const chunk = await genRes.json();
888
-
889
- // Queue the lipsync chunk
890
- lipsyncQueue.push(chunk);
891
-
892
- // If nothing is currently playing lipsync, start
893
- tryStartLipsync();
894
- } catch (e) {
895
- console.error('[CHAT] Chunk generation failed:', e);
896
- }
897
- }
898
-
899
- } catch (err) {
900
- typing.bubbleEl.classList.remove('typing');
901
- typing.bubbleEl.classList.add('error');
902
- typing.contentEl.classList.remove('typing-dots');
903
- typing.contentEl.textContent = 'Error: ' + err.message;
904
- isProcessing = false;
905
- btnSend.disabled = false;
906
- }
907
- }
908
-
909
- btnSend.addEventListener('click', sendMessage);
910
- chatInput.addEventListener('keydown', (e) => {
911
- if (e.key === 'Enter' && !e.shiftKey) {
912
- e.preventDefault();
913
- sendMessage();
914
- }
915
- });
916
-
917
- btnReset.addEventListener('click', async () => {
918
- await fetch('/api/reset', { method: 'POST' });
919
- chatLog.innerHTML = '';
920
- addMessage('bot', 'Ah, a fresh start! How may I assist you, my good fellow?');
921
- });
922
-
923
- // ======================================================================
924
- // INIT
925
- // ======================================================================
926
- async function init() {
927
- try {
928
- const res = await fetch('/api/videos');
929
- const data = await res.json();
930
-
931
- if (data.videos.length === 0) {
932
- videoStatus.textContent = 'Downloading idle videos...';
933
- setTimeout(init, 3000);
934
- return;
935
- }
936
-
937
- videos = data.videos;
938
- videoStatus.textContent = videos.length + ' idle clips loaded';
939
-
940
- // Start idle loop
941
- playNextIdle();
942
-
943
- // Welcome message
944
- addMessage('bot',
945
- 'Good day! I am Charles Darwin, er, recreated in this curious digital form. ' +
946
- 'Do feel free to ask me anything, umm, about the natural world or otherwise.'
947
- );
948
-
949
- } catch (e) {
950
- videoStatus.textContent = 'Error: ' + e.message;
951
- }
952
- }
953
-
954
- init();
955
- </script>
956
- </body>
957
- </html>
958
- """