Download app.py from AvatarResurrectionChatbot/AvatarChatbot: direct link, hf CLI and curl.
- Browser
- Download file 54 kB
-
https://huggingface.co/spaces/AvatarResurrectionChatbot/AvatarChatbot/resolve/main/app.py
- Command line
-
hf download hf://spaces/AvatarResurrectionChatbot/AvatarChatbot/app.py
-
curl -L -o app.py https://huggingface.co/spaces/AvatarResurrectionChatbot/AvatarChatbot/resolve/main/app.py
54 kB
| """ | |
| Avatar Chatbot - HuggingFace Spaces Edition | |
| Avatar profile system: each avatar has ref.png, persona.txt, idlevideos/ | |
| Chunked pipeline: LLM (once) → split sentences → TTS+FLOAT per chunk → stream to frontend | |
| """ | |
| from fastapi import FastAPI, HTTPException, Request | |
| from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse | |
| from pathlib import Path | |
| import mimetypes | |
| import os | |
| import re | |
| import logging | |
| import subprocess | |
| import time as _time | |
| import uuid | |
| import cv2 | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| app = FastAPI() | |
| # ---- Avatar system ---- | |
| AVATARS_DIR = Path("/app/avatars") | |
| _current_avatar = None # name of active avatar folder | |
| TTS_DIR = Path("/tmp/tts_output") | |
| LIPSYNC_DIR = Path("/tmp/lipsync_output") | |
| REPO_ID = os.environ.get("SPACE_ID", "ffbeqwbe/AvatarChatbot") | |
| FLOAT_ENABLED = os.environ.get("FLOAT_ENABLED", "true").lower() == "true" | |
| for d in [TTS_DIR, LIPSYNC_DIR]: | |
| d.mkdir(parents=True, exist_ok=True) | |
| def _ts(): | |
| return _time.strftime("%H:%M:%S", _time.gmtime()) + f".{int(_time.time()*1000)%1000:03d}" | |
| def _get_avatar_dir(name=None): | |
| """Get path to an avatar's directory.""" | |
| if name is None: | |
| name = _current_avatar | |
| if name: | |
| return AVATARS_DIR / name | |
| return None | |
| def _get_idle_videos_dir(): | |
| """Get the current avatar's idle videos directory.""" | |
| d = _get_avatar_dir() | |
| if d: | |
| return d / "idlevideos" | |
| return Path("/tmp/videos") | |
| def _list_idle_videos(): | |
| """List idle video files for the current avatar.""" | |
| d = _get_idle_videos_dir() | |
| if not d.exists(): | |
| return [] | |
| return [ | |
| {"name": f.stem, "filename": f.name} | |
| for f in sorted(d.iterdir()) | |
| if f.suffix.lower() in (".mp4", ".webm", ".mkv", ".mov") and f.stat().st_size > 1024 | |
| ] | |
| import threading | |
| _gpu_lock = threading.Lock() | |
| _sessions = {} | |
| GREETING_DATA = None | |
| def _discover_avatars(): | |
| """Find all avatar profiles in the avatars directory.""" | |
| if not AVATARS_DIR.exists(): | |
| logger.warning(f"[AVATARS] Directory not found: {AVATARS_DIR}") | |
| return [] | |
| avatars = [] | |
| for d in sorted(AVATARS_DIR.iterdir()): | |
| if d.is_dir(): | |
| has_ref = (d / "ref.png").exists() or (d / "ref.jpg").exists() | |
| has_persona = (d / "persona.txt").exists() | |
| has_videos = (d / "idlevideos").exists() | |
| avatars.append({ | |
| "name": d.name, | |
| "has_ref": has_ref, | |
| "has_persona": has_persona, | |
| "has_videos": has_videos, | |
| }) | |
| return avatars | |
| def _parse_avatar_config(avatar_dir): | |
| """Parse config.txt from avatar folder. Returns dict of key=value pairs.""" | |
| config = {} | |
| config_path = avatar_dir / "config.txt" | |
| if config_path.exists(): | |
| for line in open(config_path).readlines(): | |
| line = line.strip() | |
| if '=' in line and not line.startswith('#'): | |
| key, val = line.split('=', 1) | |
| config[key.strip().lower()] = val.strip() | |
| return config | |
| def _switch_avatar(avatar_name: str): | |
| """Switch to a different avatar profile.""" | |
| global _current_avatar | |
| avatar_dir = AVATARS_DIR / avatar_name | |
| if not avatar_dir.exists(): | |
| logger.error(f"[AVATARS] Avatar not found: {avatar_name}") | |
| return False | |
| _current_avatar = avatar_name | |
| logger.info(f"[AVATARS] Switching to avatar: {avatar_name}") | |
| # 0. Parse config.txt | |
| config = _parse_avatar_config(avatar_dir) | |
| # 1. Load persona into LLM | |
| persona_path = avatar_dir / "persona.txt" | |
| from llm import load_persona_from_file | |
| persona_text = load_persona_from_file(str(persona_path)) | |
| logger.info(f"[AVATARS] Persona loaded: {persona_text[:80]}..." if persona_text else "[AVATARS] No persona.txt found") | |
| # 2. Set TTS voice from config (default: diana) | |
| tts_voice = config.get("groqttsname", "diana") | |
| from groq_tts import set_voice | |
| set_voice(tts_voice) | |
| # 3. Update FLOAT reference image | |
| if FLOAT_ENABLED: | |
| try: | |
| from float_lipsync import get_lipsync | |
| lipsync = get_lipsync() | |
| if lipsync.ready: | |
| ref_path = avatar_dir / "ref.png" | |
| if not ref_path.exists(): | |
| ref_path = avatar_dir / "ref.jpg" | |
| if ref_path.exists(): | |
| lipsync.update_reference_image(str(ref_path)) | |
| logger.info(f"[AVATARS] Reference image updated: {ref_path}") | |
| else: | |
| logger.warning(f"[AVATARS] No ref image in {avatar_dir}") | |
| except Exception as e: | |
| logger.error(f"[AVATARS] Failed to update ref image: {e}") | |
| # 3. Idle videos — no action needed, _get_idle_videos_dir() uses _current_avatar dynamically | |
| logger.info(f"[AVATARS] ✓ Switched to {avatar_name}") | |
| return True | |
| async def startup(): | |
| # Discover avatars and set default | |
| avatars = _discover_avatars() | |
| logger.info(f"[STARTUP] Found {len(avatars)} avatars: {[a['name'] for a in avatars]}") | |
| if avatars: | |
| # Use first avatar as default | |
| default = avatars[0]["name"] | |
| # Load persona for default avatar (before FLOAT init so ref image is set) | |
| global _current_avatar | |
| _current_avatar = default | |
| persona_path = AVATARS_DIR / default / "persona.txt" | |
| from llm import load_persona_from_file | |
| load_persona_from_file(str(persona_path)) | |
| # Load config (TTS voice etc.) | |
| config = _parse_avatar_config(AVATARS_DIR / default) | |
| tts_voice = config.get("groqttsname", "diana") | |
| logger.info(f"[STARTUP] Default avatar: {default} | voice: {tts_voice}") | |
| await init_tts() | |
| # Apply default avatar's TTS voice (after TTS init) | |
| if avatars: | |
| config = _parse_avatar_config(AVATARS_DIR / _current_avatar) | |
| tts_voice = config.get("groqttsname", "diana") | |
| from groq_tts import set_voice | |
| set_voice(tts_voice) | |
| await init_stt() | |
| if FLOAT_ENABLED: | |
| await init_float() | |
| await generate_greeting() | |
| async def init_tts(): | |
| logger.info(f"[{_ts()}] [STARTUP] Initializing TTS...") | |
| t0 = _time.time() | |
| try: | |
| import groq_tts as tts_module | |
| tts_module.initialize() | |
| logger.info(f"[{_ts()}] [STARTUP] ✓ TTS ready in {_time.time()-t0:.2f}s") | |
| except Exception as e: | |
| logger.error(f"[{_ts()}] [STARTUP] TTS init failed: {type(e).__name__}: {e}", exc_info=True) | |
| async def generate_greeting(): | |
| global GREETING_DATA | |
| greeting_text = "Hello! Feel free to ask me anything." | |
| logger.info(f"[{_ts()}] [GREETING] Generating warmup greeting...") | |
| t0 = _time.time() | |
| try: | |
| from groq_tts import generate_audio | |
| audio_path = generate_audio(greeting_text) | |
| audio_url = None | |
| audio_duration = 0 | |
| if audio_path and os.path.exists(audio_path): | |
| audio_url = f"/api/audio/{os.path.basename(audio_path)}" | |
| try: | |
| r = subprocess.run(["ffprobe", "-v", "quiet", "-show_entries", "format=duration", | |
| "-of", "csv=p=0", audio_path], capture_output=True, text=True, check=True) | |
| audio_duration = float(r.stdout.strip()) | |
| except Exception: | |
| audio_duration = 5.0 | |
| lipsync_video_url = None | |
| if FLOAT_ENABLED: | |
| try: | |
| from float_lipsync import get_lipsync | |
| lipsync = get_lipsync() | |
| if lipsync.ready and audio_path: | |
| lp = lipsync.generate(audio_path) | |
| if lp and os.path.exists(lp): | |
| lipsync_video_url = f"/api/stream/{os.path.basename(lp)}" | |
| except Exception as e: | |
| logger.warning(f"[GREETING] FLOAT failed: {e}") | |
| GREETING_DATA = { | |
| "text": greeting_text, | |
| "audio_url": audio_url, | |
| "audio_duration": audio_duration, | |
| "lipsync_video_url": lipsync_video_url, | |
| } | |
| logger.info(f"[{_ts()}] [GREETING] Ready in {_time.time()-t0:.2f}s (lipsync: {lipsync_video_url is not None})") | |
| except Exception as e: | |
| logger.error(f"[{_ts()}] [GREETING] Failed: {e}", exc_info=True) | |
| GREETING_DATA = {"text": greeting_text, "audio_url": None, "audio_duration": 0, "lipsync_video_url": None} | |
| async def init_stt(): | |
| """Initialize Vosk speech-to-text.""" | |
| logger.info(f"[{_ts()}] [STARTUP] Initializing STT...") | |
| try: | |
| import vosk_stt | |
| if vosk_stt.initialize(): | |
| logger.info(f"[{_ts()}] [STARTUP] ✓ STT ready") | |
| else: | |
| logger.warning(f"[{_ts()}] [STARTUP] STT init failed (voice input disabled)") | |
| except Exception as e: | |
| logger.warning(f"[{_ts()}] [STARTUP] STT not available: {e}") | |
| async def init_float(): | |
| import torch | |
| logger.info(f"[STARTUP] CUDA available: {torch.cuda.is_available()}") | |
| if not torch.cuda.is_available(): | |
| logger.warning("[STARTUP] No GPU - FLOAT disabled"); return | |
| try: | |
| from float_lipsync import get_lipsync | |
| from huggingface_hub import hf_hub_download, snapshot_download | |
| import shutil | |
| ckpt = Path("/app/checkpoints") | |
| ckpt.mkdir(parents=True, exist_ok=True) | |
| w2v = ckpt / "wav2vec2-base-960h" | |
| if not w2v.exists(): | |
| logger.info("[STARTUP] Downloading wav2vec2-base-960h...") | |
| snapshot_download(repo_id="facebook/wav2vec2-base-960h", local_dir=str(w2v)) | |
| emo = ckpt / "wav2vec-english-speech-emotion-recognition" | |
| if not emo.exists(): | |
| logger.info("[STARTUP] Downloading emotion model...") | |
| snapshot_download(repo_id="r-f/wav2vec-english-speech-emotion-recognition", local_dir=str(emo)) | |
| fp = ckpt / "float.pth" | |
| if not fp.exists() or fp.stat().st_size < 1024: | |
| logger.info("[STARTUP] Downloading float.pth...") | |
| dl = hf_hub_download(repo_id=REPO_ID, repo_type="space", | |
| filename="app/checkpoints/float.pth", local_dir="/tmp/hf_download") | |
| shutil.copy2(dl, fp) | |
| logger.info(f"[STARTUP] float.pth: {fp.stat().st_size/1e6:.1f} MB") | |
| # Use current avatar's ref.png for initial FLOAT setup | |
| ref_path = None | |
| avatar_dir = _get_avatar_dir() | |
| if avatar_dir: | |
| for name in ["ref.png", "ref.jpg"]: | |
| p = avatar_dir / name | |
| if p.exists(): | |
| ref_path = str(p) | |
| break | |
| # Fallback to legacy location | |
| if not ref_path: | |
| assets = Path("/app/assets") | |
| assets.mkdir(parents=True, exist_ok=True) | |
| ref = assets / "ref.png" | |
| if not ref.exists() or ref.stat().st_size < 1024: | |
| for name in ["ref.png", "ref.jpg", "main2.png"]: | |
| try: | |
| dl = hf_hub_download(repo_id=REPO_ID, repo_type="space", | |
| filename=f"app/assets/{name}", local_dir="/tmp/hf_download") | |
| shutil.copy2(dl, assets / name) | |
| ref = assets / name | |
| break | |
| except Exception: | |
| continue | |
| else: | |
| logger.warning("[STARTUP] No ref image found"); return | |
| ref_path = str(ref) | |
| lipsync = get_lipsync() | |
| lipsync.initialize({ | |
| "ref_path": ref_path, "ckpt_path": str(fp), | |
| "wav2vec_model_path": str(w2v), "audio2emotion_path": str(emo), | |
| }) | |
| logger.info("[STARTUP] ✓ FLOAT ready!") | |
| except Exception as e: | |
| logger.error(f"[STARTUP] FLOAT init failed: {e}", exc_info=True) | |
| # ---- Avatar API endpoints ---- | |
| async def list_avatars(): | |
| """List all available avatar profiles.""" | |
| avatars = _discover_avatars() | |
| return JSONResponse({ | |
| "avatars": avatars, | |
| "current": _current_avatar, | |
| }) | |
| async def switch_avatar(request: Request): | |
| """Switch to a different avatar. Resets chat history.""" | |
| body = await request.json() | |
| avatar_name = body.get("avatar", "").strip() | |
| if not avatar_name: | |
| raise HTTPException(400, "No avatar specified") | |
| success = _switch_avatar(avatar_name) | |
| if not success: | |
| raise HTTPException(404, f"Avatar not found: {avatar_name}") | |
| return JSONResponse({ | |
| "status": "ok", | |
| "avatar": avatar_name, | |
| "videos": _list_idle_videos(), | |
| }) | |
| async def list_videos(): | |
| vids = _list_idle_videos() | |
| return JSONResponse({"videos": vids}) | |
| # ---- Avatar creation ---- | |
| _creation_status = {"active": False, "progress": 0, "total": 0, "message": "", "avatar_name": "", "done": False, "error": None} | |
| async def create_avatar(request: Request): | |
| """Create a new avatar from uploaded image + settings. Kicks off idle video generation.""" | |
| global _creation_status | |
| if _creation_status["active"]: | |
| raise HTTPException(409, "Avatar creation already in progress") | |
| import base64 | |
| body = await request.json() | |
| name = body.get("name", "").strip() | |
| voice = body.get("voice", "diana").strip() | |
| persona = body.get("persona", "").strip() | |
| image_b64 = body.get("image", "") | |
| if not name: | |
| raise HTTPException(400, "Name is required") | |
| if not image_b64: | |
| raise HTTPException(400, "Image is required") | |
| safe_name = re.sub(r'[^\w\-]', '', name) | |
| if not safe_name: | |
| raise HTTPException(400, "Invalid name") | |
| avatar_dir = AVATARS_DIR / safe_name | |
| if avatar_dir.exists(): | |
| raise HTTPException(409, f"Avatar '{safe_name}' already exists") | |
| avatar_dir.mkdir(parents=True, exist_ok=True) | |
| idle_dir = avatar_dir / "idlevideos" | |
| idle_dir.mkdir(exist_ok=True) | |
| try: | |
| if ',' in image_b64: | |
| image_b64 = image_b64.split(',', 1)[1] | |
| image_bytes = base64.b64decode(image_b64) | |
| ref_path = avatar_dir / "ref.png" | |
| import numpy as np | |
| nparr = np.frombuffer(image_bytes, np.uint8) | |
| img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) | |
| if img is None: | |
| raise ValueError("Could not decode image") | |
| cv2.imwrite(str(ref_path), img) | |
| logger.info(f"[CREATE] Saved ref image: {ref_path} ({img.shape})") | |
| except Exception as e: | |
| import shutil | |
| shutil.rmtree(str(avatar_dir), ignore_errors=True) | |
| raise HTTPException(400, f"Invalid image: {e}") | |
| with open(avatar_dir / "config.txt", "w") as f: | |
| f.write(f"groqttsname={voice}\n") | |
| with open(avatar_dir / "persona.txt", "w") as f: | |
| f.write(persona if persona else f"You are {name}.") | |
| logger.info(f"[CREATE] Avatar profile created: {safe_name} | voice: {voice}") | |
| _creation_status = { | |
| "active": True, "progress": 0, "total": 6, | |
| "message": "Starting idle video generation...", | |
| "avatar_name": safe_name, "done": False, "error": None | |
| } | |
| import asyncio | |
| asyncio.get_event_loop().run_in_executor(None, _run_idle_generation, safe_name, str(ref_path), str(idle_dir)) | |
| return JSONResponse({ | |
| "status": "started", | |
| "avatar": safe_name, | |
| "message": "Avatar created. Generating idle videos (this takes a few minutes)..." | |
| }) | |
| def _run_idle_generation(avatar_name, ref_path, idle_dir): | |
| """Background task: generate idle clips and switch to new avatar when done.""" | |
| global _creation_status | |
| try: | |
| from idle_generator import get_idle_generator | |
| generator = get_idle_generator() | |
| def on_progress(clip_idx, total, message): | |
| _creation_status["progress"] = clip_idx | |
| _creation_status["total"] = total | |
| _creation_status["message"] = message | |
| with _gpu_lock: | |
| generator.generate( | |
| ref_image_path=ref_path, | |
| output_dir=idle_dir, | |
| avatar_name=avatar_name, | |
| num_clips=6, | |
| progress_callback=on_progress, | |
| ) | |
| _switch_avatar(avatar_name) | |
| _creation_status["done"] = True | |
| _creation_status["message"] = f"Avatar '{avatar_name}' ready!" | |
| logger.info(f"[CREATE] ✓ Avatar '{avatar_name}' fully created and activated") | |
| except Exception as e: | |
| _creation_status["error"] = str(e) | |
| _creation_status["message"] = f"Error: {e}" | |
| logger.error(f"[CREATE] Failed: {e}", exc_info=True) | |
| finally: | |
| _creation_status["active"] = False | |
| async def create_avatar_status(): | |
| """Poll creation progress.""" | |
| return JSONResponse(_creation_status) | |
| async def stream_video(filename: str): | |
| path = None | |
| # Search in current avatar's idle videos, lipsync output, and TTS output | |
| search_dirs = [_get_idle_videos_dir(), LIPSYNC_DIR, TTS_DIR] | |
| for d in search_dirs: | |
| if d is None: | |
| continue | |
| c = d / filename | |
| if c.exists() and c.is_file(): | |
| path = c | |
| break | |
| if not path: | |
| raise HTTPException(404, "Not found") | |
| mime = mimetypes.guess_type(filename)[0] or "video/mp4" | |
| def gen(): | |
| with open(path, "rb") as f: | |
| while chunk := f.read(1024*1024): yield chunk | |
| return StreamingResponse(gen(), media_type=mime, | |
| headers={"Content-Length": str(path.stat().st_size), "Accept-Ranges": "bytes"}) | |
| async def stream_audio(filename: str): | |
| path = TTS_DIR / filename | |
| if not path.exists(): raise HTTPException(404, "Not found") | |
| mime = mimetypes.guess_type(filename)[0] or "audio/wav" | |
| def gen(): | |
| with open(path, "rb") as f: | |
| while chunk := f.read(1024*1024): yield chunk | |
| return StreamingResponse(gen(), media_type=mime, | |
| headers={"Content-Length": str(path.stat().st_size)}) | |
| def split_sentences(text): | |
| parts = re.split(r'(?<=[.!?])\s+', text) | |
| parts = [s.strip() for s in parts if s.strip()] | |
| MIN_WORDS = 25 | |
| merged = [] | |
| buffer = "" | |
| for p in parts: | |
| if buffer: | |
| buffer += " " + p | |
| else: | |
| buffer = p | |
| if len(buffer.split()) >= MIN_WORDS: | |
| merged.append(buffer) | |
| buffer = "" | |
| if buffer: | |
| if merged: | |
| merged[-1] += " " + buffer | |
| else: | |
| merged.append(buffer) | |
| return merged if merged else [text] | |
| async def chat(request: Request): | |
| t0 = _time.time() | |
| body = await request.json() | |
| user_text = body.get("text", "").strip() | |
| if not user_text: raise HTTPException(400, "No text") | |
| logger.info(f"[{_ts()}] [CHAT] User: {user_text}") | |
| t1 = _time.time() | |
| from llm import generate_response | |
| llm_result = generate_response(user_text) | |
| reply_text = llm_result['text'] # With voice direction tags — sent to TTS | |
| clean_text = llm_result.get('clean_text', reply_text) # Without tags — shown in UI | |
| t2 = _time.time() | |
| logger.info(f"[{_ts()}] [CHAT] Reply: {reply_text}") | |
| logger.info(f"[{_ts()}] [TIMING] LLM: {t2-t1:.2f}s") | |
| sentences = split_sentences(reply_text) | |
| if not sentences: | |
| sentences = [reply_text] | |
| logger.info(f"[{_ts()}] [CHAT] Split into {len(sentences)} chunks") | |
| session_id = str(uuid.uuid4())[:8] | |
| _sessions[session_id] = { | |
| "sentences": sentences, | |
| "next_index": 0, | |
| "total": len(sentences), | |
| "done": False, | |
| "created": _time.time(), | |
| } | |
| return JSONResponse({ | |
| "text": clean_text, | |
| "tts_text": reply_text, | |
| "emotion": llm_result.get('emotion', 'neutral'), | |
| "session_id": session_id, | |
| "total_chunks": len(sentences), | |
| }) | |
| async def chat_next(request: Request): | |
| t_req = _time.time() | |
| body = await request.json() | |
| session_id = body.get("session_id", "") | |
| session = _sessions.get(session_id) | |
| if not session: raise HTTPException(404, "Session not found") | |
| idx = session["next_index"] | |
| if idx >= session["total"]: | |
| session["done"] = True | |
| return JSONResponse({"done": True}) | |
| sentence = session["sentences"][idx] | |
| session["next_index"] = idx + 1 | |
| is_last = (idx + 1 >= session["total"]) | |
| total = session["total"] | |
| logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} received") | |
| import asyncio | |
| result = await asyncio.to_thread(_generate_chunk, sentence, idx, total) | |
| t_resp = _time.time() | |
| logger.info(f"[{_ts()}] [REQ] /api/chat/next chunk {idx+1}/{total} responding | request-to-response: {t_resp-t_req:.2f}s") | |
| if is_last: | |
| session["done"] = True | |
| if len(_sessions) > 10: | |
| oldest = sorted(_sessions.keys(), key=lambda k: _sessions[k]["created"]) | |
| for k in oldest[:-10]: | |
| del _sessions[k] | |
| # Strip voice direction tags for UI display | |
| import re as _re | |
| clean_sentence = _re.sub(r'\[[^\]]*\]\s*', '', sentence).strip() | |
| return JSONResponse({ | |
| "done": False, | |
| "chunk_index": idx, | |
| "total_chunks": total, | |
| "is_last": is_last, | |
| "sentence": clean_sentence, | |
| "audio_url": result["audio_url"], | |
| "audio_duration": result["audio_duration"], | |
| "lipsync_video_url": result["lipsync_video_url"], | |
| }) | |
| def _generate_chunk(sentence, idx, total): | |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] START | \"{sentence[:80]}\"") | |
| t0 = _time.time() | |
| import torch | |
| with _gpu_lock: | |
| from groq_tts import generate_audio | |
| t_tts_start = _time.time() | |
| audio_path = generate_audio(sentence) | |
| t_tts_end = _time.time() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| audio_url = None | |
| audio_duration = 0 | |
| audio_size = 0 | |
| if audio_path and os.path.exists(audio_path): | |
| audio_size = os.path.getsize(audio_path) | |
| audio_url = f"/api/audio/{os.path.basename(audio_path)}" | |
| try: | |
| r = subprocess.run(["ffprobe","-v","quiet","-show_entries","format=duration", | |
| "-of","csv=p=0", audio_path], capture_output=True, text=True, check=True) | |
| audio_duration = float(r.stdout.strip()) | |
| except Exception: | |
| audio_duration = 3.0 | |
| t_probe_end = _time.time() | |
| else: | |
| t_probe_end = _time.time() | |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] TTS: {t_tts_end-t_tts_start:.2f}s | audio: {audio_duration:.1f}s, {audio_size/1024:.0f}KB") | |
| lipsync_video_url = None | |
| if FLOAT_ENABLED and audio_path and os.path.exists(audio_path): | |
| try: | |
| from float_lipsync import get_lipsync | |
| lipsync = get_lipsync() | |
| if lipsync.ready: | |
| t_float_start = _time.time() | |
| lp = lipsync.generate(audio_path) | |
| t_float_done = _time.time() | |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] FLOAT: {t_float_done-t_float_start:.2f}s") | |
| if lp and os.path.exists(lp): | |
| lipsync_video_url = f"/api/stream/{os.path.basename(lp)}" | |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] VIDEO: {os.path.getsize(lp)/1024:.0f}KB") | |
| except Exception as e: | |
| logger.error(f"[{_ts()}] [CHUNK {idx+1}/{total}] LIPSYNC FAILED: {e}", exc_info=True) | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| total_time = _time.time() - t0 | |
| logger.info(f"[{_ts()}] [CHUNK {idx+1}/{total}] DONE | total: {total_time:.2f}s") | |
| return { | |
| "audio_url": audio_url, | |
| "audio_duration": audio_duration, | |
| "lipsync_video_url": lipsync_video_url, | |
| } | |
| async def reset_chat(): | |
| from llm import reset_conversation | |
| reset_conversation() | |
| return JSONResponse({"status": "ok"}) | |
| async def transcribe(request: Request): | |
| """Transcribe audio blob with Vosk. Accepts base64 JSON or raw body.""" | |
| try: | |
| content_type = request.headers.get("content-type", "") | |
| if "json" in content_type: | |
| body = await request.json() | |
| import base64 | |
| audio_bytes = base64.b64decode(body.get("audio", "")) | |
| audio_ct = body.get("content_type", "audio/webm") | |
| else: | |
| audio_bytes = await request.body() | |
| audio_ct = content_type or "audio/webm" | |
| if not audio_bytes or len(audio_bytes) < 500: | |
| return JSONResponse({"text": ""}) | |
| import asyncio | |
| import vosk_stt | |
| text = await asyncio.to_thread(vosk_stt.transcribe_audio, audio_bytes, audio_ct) | |
| return JSONResponse({"text": text}) | |
| except Exception as e: | |
| logger.error(f"[{_ts()}] [STT] Transcribe error: {e}", exc_info=True) | |
| return JSONResponse({"text": "", "error": str(e)}) | |
| async def get_greeting(): | |
| if GREETING_DATA: | |
| return JSONResponse(GREETING_DATA) | |
| return JSONResponse({"text": None}) | |
| async def index(): | |
| return HTML_PAGE | |
| HTML_PAGE = """\ | |
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"/> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"/> | |
| <title>Avatar Chatbot</title> | |
| <style> | |
| *{margin:0;padding:0;box-sizing:border-box} | |
| body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0f1c;color:#e2e8f0;height:100vh;display:flex;flex-direction:column;overflow:hidden} | |
| .app-container{display:flex;flex-direction:column;height:100vh;max-width:900px;margin:0 auto;width:100%;padding:1rem;gap:.75rem} | |
| .top-bar{display:flex;align-items:center;gap:.75rem;flex-shrink:0} | |
| .top-bar label{font-size:.85rem;color:#94a3b8} | |
| .top-bar select{background:#1e293b;color:#e2e8f0;border:1px solid rgba(59,130,246,.3);border-radius:6px;padding:6px 10px;font-size:.85rem;outline:none;cursor:pointer} | |
| .top-bar select:focus{border-color:#3b82f6} | |
| .video-section{position:relative;width:100%;min-height:300px;max-height:45vh;background:#000;border-radius:12px;overflow:hidden;flex-shrink:0;display:flex;align-items:center;justify-content:center} | |
| .video-container{position:relative;width:100%;height:100%} | |
| .video-container video{width:100%;height:100%;object-fit:contain;display:block} | |
| #videoA,#videoB{position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;transition:opacity 150ms ease} | |
| #videoA{z-index:1;opacity:1}#videoB{z-index:0;opacity:1} | |
| .video-status{position:absolute;bottom:8px;right:12px;font-size:.7rem;color:rgba(255,255,255,.4);background:rgba(0,0,0,.5);padding:2px 8px;border-radius:4px;z-index:10} | |
| .chat-section{flex:1;display:flex;flex-direction:column;min-height:0;background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden} | |
| .chat-log{flex:1;overflow-y:auto;padding:1rem;display:flex;flex-direction:column;gap:.5rem} | |
| .chat-log::-webkit-scrollbar{width:6px}.chat-log::-webkit-scrollbar-track{background:transparent}.chat-log::-webkit-scrollbar-thumb{background:#334155;border-radius:3px} | |
| .msg{display:flex;width:100%}.msg.user{justify-content:flex-end}.msg.bot{justify-content:flex-start} | |
| .bubble{max-width:85%;padding:10px 14px;border-radius:12px;word-wrap:break-word;font-size:.9rem;line-height:1.4} | |
| .bubble.user{background:linear-gradient(135deg,#1e40af,#3b82f6);color:#fff;border-bottom-right-radius:4px} | |
| .bubble.bot{background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.2);color:#e2e8f0;border-bottom-left-radius:4px} | |
| .bubble.bot .label{color:#60a5fa;font-weight:600;font-size:.8rem;margin-bottom:2px} | |
| .bubble.user .label{color:rgba(255,255,255,.7);font-weight:600;font-size:.8rem;margin-bottom:2px} | |
| .bubble.typing{color:#64748b;font-style:italic} | |
| .bubble.error{background:rgba(127,29,29,.5);border-color:rgba(239,68,68,.3);color:#fca5a5} | |
| @keyframes dots{0%,20%{content:''}40%{content:'.'}60%{content:'..'}80%,100%{content:'...'}} | |
| .typing-dots::after{content:'';animation:dots 1.5s infinite} | |
| .streaming-cursor::after{content:'\\25CA';animation:blink .7s infinite;color:#60a5fa} | |
| @keyframes blink{0%,100%{opacity:1}50%{opacity:0}} | |
| .input-bar{display:flex;gap:.5rem;padding:.75rem 1rem;background:rgba(15,23,42,.8);border-top:1px solid rgba(59,130,246,.1)} | |
| .input-bar input{flex:1;background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.25);border-radius:8px;padding:10px 14px;color:#e2e8f0;font-size:.9rem;outline:none;transition:border-color .2s} | |
| .input-bar input:focus{border-color:#3b82f6}.input-bar input::placeholder{color:#64748b} | |
| .input-bar button{padding:10px 20px;border:none;border-radius:8px;font-size:.9rem;cursor:pointer;transition:all .15s;font-weight:500} | |
| .btn-send{background:#3b82f6;color:#fff}.btn-send:hover{background:#2563eb}.btn-send:disabled{background:#334155;color:#64748b;cursor:not-allowed} | |
| .btn-reset{background:rgba(100,116,139,.3);color:#94a3b8;border:1px solid rgba(100,116,139,.3)}.btn-reset:hover{background:rgba(100,116,139,.5)} | |
| .btn-mic{background:rgba(100,116,139,.3);color:#94a3b8;border:1px solid rgba(100,116,139,.3);font-size:1.1rem;padding:10px 14px}.btn-mic:hover{background:rgba(100,116,139,.5)} | |
| .btn-mic.recording{background:rgba(239,68,68,.3);color:#fca5a5;border-color:rgba(239,68,68,.5);animation:pulse-rec 1.5s infinite} | |
| @keyframes pulse-rec{0%,100%{box-shadow:0 0 0 0 rgba(239,68,68,.3)}50%{box-shadow:0 0 0 8px rgba(239,68,68,0)}} | |
| .btn-clear{background:rgba(100,116,139,.2);color:#64748b;border:1px solid rgba(100,116,139,.2);padding:10px 12px;font-size:.85rem;display:none}.btn-clear:hover{background:rgba(100,116,139,.4);color:#94a3b8} | |
| .create-section{background:rgba(15,23,42,.6);border:1px solid rgba(59,130,246,.15);border-radius:12px;overflow:hidden;flex-shrink:0} | |
| .create-toggle{width:100%;padding:10px 14px;background:none;border:none;color:#94a3b8;font-size:.85rem;cursor:pointer;text-align:left;display:flex;justify-content:space-between;align-items:center} | |
| .create-toggle:hover{color:#e2e8f0} | |
| .create-body{padding:0 14px 14px;display:none;gap:.5rem;flex-direction:column} | |
| .create-body.open{display:flex} | |
| .create-body label{font-size:.8rem;color:#94a3b8;margin-top:.25rem} | |
| .create-body input,.create-body select,.create-body textarea{background:rgba(30,41,59,.8);border:1px solid rgba(59,130,246,.25);border-radius:6px;padding:8px 10px;color:#e2e8f0;font-size:.85rem;outline:none;font-family:inherit} | |
| .create-body textarea{resize:vertical;min-height:60px} | |
| .create-body input:focus,.create-body select:focus,.create-body textarea:focus{border-color:#3b82f6} | |
| .create-img-preview{max-width:120px;max-height:120px;border-radius:8px;margin-top:4px;display:none} | |
| .btn-create{padding:10px 20px;background:#059669;color:#fff;border:none;border-radius:8px;font-size:.85rem;cursor:pointer;font-weight:500;margin-top:.5rem}.btn-create:hover{background:#047857}.btn-create:disabled{background:#334155;color:#64748b;cursor:not-allowed} | |
| .create-progress{font-size:.8rem;color:#60a5fa;margin-top:.25rem;display:none} | |
| .create-progress .bar{height:4px;background:#1e293b;border-radius:2px;margin-top:4px;overflow:hidden} | |
| .create-progress .bar-fill{height:100%;background:#3b82f6;transition:width .3s;width:0%} | |
| </style> | |
| </head> | |
| <body> | |
| <div class="app-container"> | |
| <div class="top-bar"> | |
| <label for="avatarSelect">Avatar:</label> | |
| <select id="avatarSelect"></select> | |
| </div> | |
| <div class="video-section"> | |
| <div class="video-container"> | |
| <video id="videoA" muted autoplay playsinline></video> | |
| <video id="videoB" muted autoplay playsinline></video> | |
| </div> | |
| </div> | |
| <div class="chat-section"> | |
| <div class="chat-log" id="chatLog"></div> | |
| <div class="input-bar"> | |
| <input type="text" id="chatInput" placeholder="Say something..." autocomplete="off"/> | |
| <button class="btn-clear" id="btnClear" title="Clear input">✕</button> | |
| <button class="btn-send" id="btnSend">Send</button> | |
| <button class="btn-mic" id="btnMic" title="Voice input">🎤</button> | |
| <button class="btn-reset" id="btnReset" title="Reset conversation">↻</button> | |
| </div> | |
| </div> | |
| <div class="create-section"> | |
| <button class="create-toggle" id="createToggle">+ Create New Avatar <span id="createArrow">▶</span></button> | |
| <div class="create-body" id="createBody"> | |
| <label>Name</label> | |
| <input type="text" id="createName" placeholder="e.g. Einstein"/> | |
| <label>Face Image</label> | |
| <input type="file" id="createImage" accept="image/*"/> | |
| <img class="create-img-preview" id="createPreview"/> | |
| <label>Voice</label> | |
| <select id="createVoice"> | |
| <option value="diana">Diana (Female)</option> | |
| <option value="autumn">Autumn (Female)</option> | |
| <option value="hannah">Hannah (Female)</option> | |
| <option value="austin">Austin (Male)</option> | |
| <option value="daniel">Daniel (Male)</option> | |
| <option value="troy">Troy (Male)</option> | |
| </select> | |
| <label>Persona Description</label> | |
| <textarea id="createPersona" placeholder="Describe who this avatar is, how they speak, their background..."></textarea> | |
| <button class="btn-create" id="btnCreate">Create Avatar</button> | |
| <div class="create-progress" id="createProgress"> | |
| <span id="createProgressText">Starting...</span> | |
| <div class="bar"><div class="bar-fill" id="createProgressBar"></div></div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <script> | |
| let videos=[], currentVideo='A', isTransitioning=false, isProcessing=false, lastPlayed=null, msgCounter=0; | |
| let currentAvatarName='Avatar'; | |
| const videoA=document.getElementById('videoA'), videoB=document.getElementById('videoB'); | |
| const videoStatus={textContent:''}; // no overlay | |
| const chatLog=document.getElementById('chatLog'); | |
| const chatInput=document.getElementById('chatInput'), btnSend=document.getElementById('btnSend'); | |
| const btnReset=document.getElementById('btnReset'), avatarSelect=document.getElementById('avatarSelect'); | |
| function getActiveVideo(){return currentVideo==='A'?videoA:videoB} | |
| function getInactiveVideo(){return currentVideo==='A'?videoB:videoA} | |
| function crossfadeTo(url, muted){ | |
| return new Promise((resolve)=>{ | |
| if(isTransitioning){ resolve(); return; } | |
| isTransitioning=true; | |
| const prev=getActiveVideo(), next=getInactiveVideo(); | |
| next.muted=(muted!==false); | |
| next.style.opacity='1'; next.style.zIndex='0'; prev.style.zIndex='1'; | |
| next.src=url; next.preload='auto'; next.load(); | |
| next.oncanplaythrough=function(){ | |
| next.oncanplaythrough=null; | |
| next.play().then(()=>{ | |
| next.style.zIndex='2'; prev.style.opacity='0'; | |
| setTimeout(()=>{ | |
| prev.pause(); prev.removeAttribute('src'); prev.load(); | |
| prev.style.zIndex='0'; prev.style.opacity='1'; | |
| next.style.zIndex='1'; | |
| currentVideo=currentVideo==='A'?'B':'A'; | |
| isTransitioning=false; resolve(); | |
| }, 180); | |
| }).catch(()=>{ | |
| next.muted=true; | |
| next.play().then(()=>{ | |
| next.style.zIndex='2'; prev.style.opacity='0'; | |
| setTimeout(()=>{ | |
| prev.pause(); prev.removeAttribute('src'); prev.load(); | |
| prev.style.zIndex='0'; prev.style.opacity='1'; | |
| next.style.zIndex='1'; | |
| currentVideo=currentVideo==='A'?'B':'A'; | |
| isTransitioning=false; resolve(); | |
| }, 180); | |
| }).catch(()=>{ isTransitioning=false; resolve(); }); | |
| }); | |
| }; | |
| next.onerror=function(){ isTransitioning=false; resolve(); }; | |
| }); | |
| } | |
| function pickRandomIdle(){ | |
| if(!videos.length) return null; | |
| if(videos.length===1) return videos[0]; | |
| let p; do{ p=videos[Math.floor(Math.random()*videos.length)]; }while(p.filename===lastPlayed&&videos.length>1); | |
| return p; | |
| } | |
| function playNextIdle(){ | |
| const c=pickRandomIdle(); if(!c) return; | |
| lastPlayed=c.filename; | |
| videoStatus.textContent='idle: '+c.name; | |
| crossfadeTo('/api/stream/'+encodeURIComponent(c.filename), true); | |
| } | |
| let isSpeaking=false; | |
| videoA.addEventListener('ended',()=>{ if(currentVideo==='A'&&!isSpeaking) playNextIdle(); }); | |
| videoB.addEventListener('ended',()=>{ if(currentVideo==='B'&&!isSpeaking) playNextIdle(); }); | |
| function waitForVideoEnd(){ | |
| return new Promise((resolve)=>{ | |
| const active=getActiveVideo(); | |
| if(active.ended || active.paused || !active.src){ resolve(); return; } | |
| const onEnd=()=>{ active.removeEventListener('ended',onEnd); resolve(); }; | |
| active.addEventListener('ended', onEnd); | |
| }); | |
| } | |
| function addMessage(role, text, extra){ | |
| const id='msg_'+(msgCounter++); | |
| const w=document.createElement('div'); w.className='msg '+role; | |
| const b=document.createElement('div'); b.className='bubble '+role+(extra?' '+extra:''); b.id=id; | |
| const l=document.createElement('div'); l.className='label'; l.textContent=role==='user'?'You':currentAvatarName; | |
| const c=document.createElement('div'); c.className='content'; c.textContent=text; | |
| b.appendChild(l); b.appendChild(c); w.appendChild(b); | |
| chatLog.appendChild(w); chatLog.scrollTop=chatLog.scrollHeight; | |
| return {id:id, contentEl:c, bubbleEl:b}; | |
| } | |
| function addTypingIndicator(){ | |
| const r=addMessage('bot','','typing'); | |
| r.contentEl.className='content typing-dots'; r.contentEl.textContent='thinking'; | |
| return r; | |
| } | |
| // ---- Avatar switching ---- | |
| avatarSelect.addEventListener('change', async ()=>{ | |
| const name=avatarSelect.value; | |
| if(!name || isProcessing) return; | |
| videoStatus.textContent='Switching avatar...'; | |
| btnSend.disabled=true; | |
| try{ | |
| const res=await fetch('/api/switch-avatar',{ | |
| method:'POST', headers:{'Content-Type':'application/json'}, | |
| body:JSON.stringify({avatar:name}) | |
| }); | |
| const data=await res.json(); | |
| if(data.videos){ | |
| videos=data.videos; | |
| } | |
| currentAvatarName=name; | |
| chatLog.innerHTML=''; | |
| addMessage('bot','Hi, I\\'m '+name+'. How can I help you?'); | |
| playNextIdle(); | |
| }catch(e){ | |
| videoStatus.textContent='Switch failed: '+e.message; | |
| } | |
| btnSend.disabled=false; | |
| }); | |
| // ---- Chat ---- | |
| async function sendMessage(){ | |
| const text=chatInput.value.trim(); | |
| if(!text||isProcessing) return; | |
| isProcessing=true; btnSend.disabled=true; chatInput.value=''; btnClear.style.display='none'; | |
| addMessage('user', text); | |
| const typing=addTypingIndicator(); | |
| try{ | |
| const res=await fetch('/api/chat',{ | |
| method:'POST', headers:{'Content-Type':'application/json'}, | |
| body:JSON.stringify({text:text}) | |
| }); | |
| if(!res.ok) throw new Error('HTTP '+res.status); | |
| const data=await res.json(); | |
| typing.bubbleEl.classList.remove('typing'); | |
| typing.contentEl.classList.remove('typing-dots'); | |
| typing.contentEl.textContent=''; | |
| typing.contentEl.classList.add('streaming-cursor'); | |
| const sessionId=data.session_id; | |
| const totalChunks=data.total_chunks; | |
| const fullText=data.text; | |
| let displayedText=''; | |
| function streamChunkText(sentence, durationSec){ | |
| const chunkWords=sentence.split(/\\s+/).filter(w=>w.length>0); | |
| if(chunkWords.length===0) return Promise.resolve(); | |
| const interval=Math.max(50, (durationSec*1000)/chunkWords.length); | |
| return new Promise((resolve)=>{ | |
| let i=0; | |
| const timer=setInterval(()=>{ | |
| if(i>=chunkWords.length){ clearInterval(timer); resolve(); return; } | |
| if(displayedText.length>0) displayedText+=' '; | |
| displayedText+=chunkWords[i]; | |
| typing.contentEl.textContent=displayedText; | |
| chatLog.scrollTop=chatLog.scrollHeight; | |
| i++; | |
| }, interval); | |
| }); | |
| } | |
| function showAllText(){ | |
| typing.contentEl.textContent=fullText; | |
| typing.contentEl.classList.remove('streaming-cursor'); | |
| chatLog.scrollTop=chatLog.scrollHeight; | |
| } | |
| async function fetchChunk(sid){ | |
| const r=await fetch('/api/chat/next',{ | |
| method:'POST', headers:{'Content-Type':'application/json'}, | |
| body:JSON.stringify({session_id:sid}) | |
| }); | |
| if(!r.ok) return null; | |
| return await r.json(); | |
| } | |
| let nextChunkPromise=fetchChunk(sessionId); | |
| for(let i=0; i<totalChunks; i++){ | |
| videoStatus.textContent='generating '+(i+1)+'/'+totalChunks+'...'; | |
| const chunk=await nextChunkPromise; | |
| if(!chunk || chunk.done) break; | |
| if(i+1 < totalChunks) nextChunkPromise=fetchChunk(sessionId); | |
| isSpeaking=true; | |
| await waitForVideoEnd(); | |
| videoStatus.textContent='speaking '+(i+1)+'/'+totalChunks; | |
| let audioPromise=null; | |
| if(chunk.audio_url){ | |
| const audio=new Audio(chunk.audio_url); | |
| audioPromise=new Promise((resolve)=>{ | |
| audio.onended=resolve; audio.onerror=resolve; | |
| audio.play().catch(()=>resolve()); | |
| }); | |
| } | |
| const chunkDuration=chunk.audio_duration||3.0; | |
| const textStreamPromise=streamChunkText(chunk.sentence, chunkDuration); | |
| if(chunk.lipsync_video_url){ | |
| await crossfadeTo(chunk.lipsync_video_url, true); | |
| await waitForVideoEnd(); | |
| } else if(audioPromise){ | |
| await audioPromise; | |
| } | |
| await textStreamPromise; | |
| } | |
| showAllText(); | |
| isSpeaking=false; | |
| playNextIdle(); | |
| }catch(err){ | |
| typing.bubbleEl.classList.remove('typing'); | |
| typing.bubbleEl.classList.add('error'); | |
| typing.contentEl.classList.remove('typing-dots','streaming-cursor'); | |
| typing.contentEl.textContent='Error: '+err.message; | |
| } | |
| isSpeaking=false; isProcessing=false; btnSend.disabled=false; chatInput.focus(); | |
| } | |
| btnSend.addEventListener('click', sendMessage); | |
| chatInput.addEventListener('keydown', e=>{ if(e.key==='Enter'&&!e.shiftKey){ e.preventDefault(); if(!isRecording) sendMessage(); }}); | |
| // ---- Voice Input (Mic) ---- | |
| const btnMic=document.getElementById('btnMic'); | |
| const btnClear=document.getElementById('btnClear'); | |
| let mediaRecorder=null, isRecording=false, previewTimer=null, allChunks=[], micStream=null, micMimeType=''; | |
| // Show/hide clear button based on input content | |
| chatInput.addEventListener('input',()=>{ | |
| btnClear.style.display=chatInput.value?'block':'none'; | |
| }); | |
| btnClear.addEventListener('click',()=>{ | |
| chatInput.value=''; | |
| btnClear.style.display='none'; | |
| chatInput.focus(); | |
| }); | |
| // Helper: send accumulated audio blob to server for transcription | |
| async function transcribeBlob(blob){ | |
| const reader=new FileReader(); | |
| return new Promise((resolve)=>{ | |
| reader.onloadend=async()=>{ | |
| const b64=reader.result.split(',')[1]; | |
| try{ | |
| const res=await fetch('/api/transcribe',{ | |
| method:'POST', | |
| headers:{'Content-Type':'application/json'}, | |
| body:JSON.stringify({audio:b64, content_type:micMimeType||'audio/webm'}) | |
| }); | |
| const data=await res.json(); | |
| resolve(data.text||''); | |
| }catch(e){ resolve(''); } | |
| }; | |
| reader.readAsDataURL(blob); | |
| }); | |
| } | |
| btnMic.addEventListener('click', async()=>{ | |
| if(isRecording){ | |
| // ---- STOP recording ---- | |
| if(previewTimer) clearInterval(previewTimer); | |
| previewTimer=null; | |
| mediaRecorder.stop(); | |
| return; | |
| } | |
| // ---- START recording ---- | |
| try{ | |
| micStream=await navigator.mediaDevices.getUserMedia({audio:true}); | |
| micMimeType=MediaRecorder.isTypeSupported('audio/webm;codecs=opus')?'audio/webm;codecs=opus': | |
| MediaRecorder.isTypeSupported('audio/webm')?'audio/webm':''; | |
| mediaRecorder=new MediaRecorder(micStream, micMimeType?{mimeType:micMimeType}:{}); | |
| allChunks=[]; | |
| mediaRecorder.ondataavailable=(e)=>{ | |
| if(e.data.size>0) allChunks.push(e.data); | |
| }; | |
| mediaRecorder.onstop=async()=>{ | |
| isRecording=false; | |
| btnMic.classList.remove('recording'); | |
| btnMic.textContent=String.fromCodePoint(0x1F3A4); | |
| chatInput.placeholder='Transcribing...'; | |
| micStream.getTracks().forEach(t=>t.stop()); | |
| if(allChunks.length===0){ chatInput.placeholder='Say something...'; btnSend.disabled=false; return; } | |
| // Final transcription of entire recording | |
| const fullBlob=new Blob(allChunks, {type:micMimeType||'audio/webm'}); | |
| const text=await transcribeBlob(fullBlob); | |
| chatInput.value=text; | |
| chatInput.placeholder='Say something...'; | |
| btnClear.style.display=text?'block':'none'; | |
| btnSend.disabled=false; | |
| chatInput.focus(); | |
| }; | |
| // Start recording - request data every 500ms for live preview | |
| mediaRecorder.start(500); | |
| isRecording=true; | |
| btnMic.classList.add('recording'); | |
| btnMic.textContent=String.fromCodePoint(0x23F9); | |
| chatInput.value=''; | |
| chatInput.placeholder='Listening...'; | |
| btnSend.disabled=true; | |
| btnClear.style.display='none'; | |
| // Live preview: every 1.5s, transcribe what we have so far | |
| let previewBusy=false; | |
| previewTimer=setInterval(async()=>{ | |
| if(previewBusy||allChunks.length<2) return; | |
| previewBusy=true; | |
| try{ | |
| const previewBlob=new Blob(allChunks.slice(), {type:micMimeType||'audio/webm'}); | |
| if(previewBlob.size>2000){ | |
| const text=await transcribeBlob(previewBlob); | |
| if(isRecording && text){ | |
| chatInput.value=text; | |
| } | |
| } | |
| }catch(e){} | |
| previewBusy=false; | |
| }, 2000); | |
| }catch(err){ | |
| console.error('Mic access error:',err); | |
| alert('Could not access microphone. Please allow mic permission.'); | |
| } | |
| }); | |
| btnReset.addEventListener('click', async()=>{ | |
| await fetch('/api/reset',{method:'POST'}); | |
| chatLog.innerHTML=''; | |
| addMessage('bot','Conversation reset. How can I help you?'); | |
| }); | |
| // ---- Init ---- | |
| async function init(){ | |
| try{ | |
| // Load avatars into dropdown | |
| const avatarRes=await fetch('/api/avatars'); | |
| const avatarData=await avatarRes.json(); | |
| avatarSelect.innerHTML=''; | |
| for(const a of avatarData.avatars){ | |
| const opt=document.createElement('option'); | |
| opt.value=a.name; opt.textContent=a.name; | |
| if(a.name===avatarData.current) opt.selected=true; | |
| avatarSelect.appendChild(opt); | |
| } | |
| if(avatarData.current) currentAvatarName=avatarData.current; | |
| // Load idle videos | |
| const res=await fetch('/api/videos'); | |
| const data=await res.json(); | |
| if(!data.videos.length){ videoStatus.textContent='No idle videos...'; } | |
| else{ videos=data.videos; videoStatus.textContent=videos.length+' idle clips loaded'; } | |
| // Check greeting | |
| const greetRes=await fetch('/api/greeting'); | |
| const greeting=await greetRes.json(); | |
| if(greeting.lipsync_video_url){ | |
| videoStatus.textContent='greeting...'; | |
| addMessage('bot', greeting.text); | |
| if(greeting.audio_url){ | |
| const audio=new Audio(greeting.audio_url); | |
| audio.play().catch(e=>console.warn('[AUDIO]',e)); | |
| } | |
| await crossfadeTo(greeting.lipsync_video_url, true); | |
| await waitForVideoEnd(); | |
| playNextIdle(); | |
| } else { | |
| addMessage('bot', greeting.text || 'Hello! Feel free to ask me anything.'); | |
| playNextIdle(); | |
| } | |
| }catch(e){ videoStatus.textContent='Error: '+e.message; } | |
| } | |
| // ---- Create Avatar ---- | |
| const createToggle=document.getElementById('createToggle'); | |
| const createBody=document.getElementById('createBody'); | |
| const createArrow=document.getElementById('createArrow'); | |
| const createName=document.getElementById('createName'); | |
| const createImage=document.getElementById('createImage'); | |
| const createPreview=document.getElementById('createPreview'); | |
| const createVoice=document.getElementById('createVoice'); | |
| const createPersona=document.getElementById('createPersona'); | |
| const btnCreate=document.getElementById('btnCreate'); | |
| const createProgress=document.getElementById('createProgress'); | |
| const createProgressText=document.getElementById('createProgressText'); | |
| const createProgressBar=document.getElementById('createProgressBar'); | |
| createToggle.addEventListener('click',()=>{ | |
| const open=createBody.classList.toggle('open'); | |
| createArrow.innerHTML=open?'▼':'▶'; | |
| }); | |
| createImage.addEventListener('change',(e)=>{ | |
| const file=e.target.files[0]; | |
| if(file){ | |
| const reader=new FileReader(); | |
| reader.onload=(ev)=>{ | |
| createPreview.src=ev.target.result; | |
| createPreview.style.display='block'; | |
| }; | |
| reader.readAsDataURL(file); | |
| } | |
| }); | |
| btnCreate.addEventListener('click', async()=>{ | |
| const name=createName.value.trim(); | |
| const file=createImage.files[0]; | |
| const voice=createVoice.value; | |
| const persona=createPersona.value.trim(); | |
| if(!name){ alert('Please enter a name'); return; } | |
| if(!file){ alert('Please select a face image'); return; } | |
| const reader=new FileReader(); | |
| reader.onload=async(ev)=>{ | |
| const imageB64=ev.target.result; | |
| // Disable ALL user inputs during generation | |
| btnCreate.disabled=true; | |
| btnSend.disabled=true; | |
| chatInput.disabled=true; | |
| avatarSelect.disabled=true; | |
| btnReset.disabled=true; | |
| createProgress.style.display='block'; | |
| createProgressText.textContent='Creating avatar...'; | |
| createProgressBar.style.width='0%'; | |
| function reEnableInputs(){ | |
| btnCreate.disabled=false; | |
| btnSend.disabled=false; | |
| chatInput.disabled=false; | |
| avatarSelect.disabled=false; | |
| btnReset.disabled=false; | |
| } | |
| try{ | |
| const res=await fetch('/api/create-avatar',{ | |
| method:'POST', headers:{'Content-Type':'application/json'}, | |
| body:JSON.stringify({name:name, voice:voice, persona:persona, image:imageB64}) | |
| }); | |
| if(!res.ok){ | |
| const err=await res.json(); | |
| throw new Error(err.detail||'Creation failed'); | |
| } | |
| const pollInterval=setInterval(async()=>{ | |
| try{ | |
| const statusRes=await fetch('/api/create-avatar/status'); | |
| const status=await statusRes.json(); | |
| createProgressText.textContent=status.message; | |
| if(status.total>0){ | |
| createProgressBar.style.width=Math.round(status.progress/status.total*100)+'%'; | |
| } | |
| if(status.done){ | |
| clearInterval(pollInterval); | |
| createProgressBar.style.width='100%'; | |
| createProgressText.textContent='Avatar ready!'; | |
| const avatarRes=await fetch('/api/avatars'); | |
| const avatarData=await avatarRes.json(); | |
| avatarSelect.innerHTML=''; | |
| for(const a of avatarData.avatars){ | |
| const opt=document.createElement('option'); | |
| opt.value=a.name; opt.textContent=a.name; | |
| if(a.name===avatarData.current) opt.selected=true; | |
| avatarSelect.appendChild(opt); | |
| } | |
| currentAvatarName=avatarData.current||name; | |
| const vidRes=await fetch('/api/videos'); | |
| const vidData=await vidRes.json(); | |
| videos=vidData.videos; | |
| chatLog.innerHTML=''; | |
| addMessage('bot','Hi, I\\'m '+currentAvatarName+'. How can I help you?'); | |
| playNextIdle(); | |
| setTimeout(()=>{ | |
| reEnableInputs(); | |
| createProgress.style.display='none'; | |
| createName.value=''; | |
| createImage.value=''; | |
| createPreview.style.display='none'; | |
| createPersona.value=''; | |
| createBody.classList.remove('open'); | |
| createArrow.innerHTML='▶'; | |
| }, 2000); | |
| } | |
| if(status.error){ | |
| clearInterval(pollInterval); | |
| createProgressText.textContent='Error: '+status.error; | |
| reEnableInputs(); | |
| } | |
| }catch(pollErr){ | |
| console.error('Poll error:',pollErr); | |
| } | |
| }, 2000); | |
| }catch(err){ | |
| createProgressText.textContent='Error: '+err.message; | |
| reEnableInputs(); | |
| } | |
| }; | |
| reader.readAsDataURL(file); | |
| }); | |
| init(); | |
| </script> | |
| </body> | |
| </html> | |
| """ |