IamSamk commited on
Commit ·
27caffe
0
Parent(s):
Mindfull Gradio Space deploy
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .env.example +13 -0
- .gitignore +73 -0
- INTEGRATION_GUIDE.md +514 -0
- MISTRAL_SETUP.md +226 -0
- Modelfile +10 -0
- Modelfile_Mindfull +39 -0
- README_COMPLETE.md +339 -0
- README_SPACE.md +55 -0
- Readme.md +154 -0
- analyze_codebase.py +1024 -0
- app.py +19 -0
- gradio_app.py +283 -0
- infer_cli.py +368 -0
- mindfull_client.html +515 -0
- mindfull_config.py +308 -0
- mindfull_pipeline.py +478 -0
- mindfull_web_api.py +324 -0
- model_manager.py +252 -0
- my_finetuned_model/config.json +193 -0
- my_finetuned_model/vocab.json +0 -0
- ollama_to_f5tts.py +37 -0
- python313_compat_patch.py +116 -0
- requirements.txt +28 -0
- requirements_full.txt +112 -0
- requirements_hf.txt +28 -0
- requirements_unified.txt +97 -0
- sadtalker+wav2lip/README.md +75 -0
- sadtalker+wav2lip/README_POLICE_BOT_AI.md +199 -0
- sadtalker+wav2lip/enhanced_pipeline.py +231 -0
- sadtalker+wav2lip/init_path.py +47 -0
- sadtalker+wav2lip/preprocess.py +170 -0
- sadtalker+wav2lip/requirements.txt +29 -0
- sadtalker+wav2lip/sadtalker/generate_batch.py +120 -0
- sadtalker+wav2lip/sadtalker/inference.py +145 -0
- sadtalker+wav2lip/sadtalker/src/audio2exp_models/audio2exp.py +41 -0
- sadtalker+wav2lip/sadtalker/src/audio2exp_models/networks.py +74 -0
- sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio2pose.py +94 -0
- sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio_encoder.py +64 -0
- sadtalker+wav2lip/sadtalker/src/audio2pose_models/cvae.py +149 -0
- sadtalker+wav2lip/sadtalker/src/audio2pose_models/discriminator.py +76 -0
- sadtalker+wav2lip/sadtalker/src/audio2pose_models/networks.py +140 -0
- sadtalker+wav2lip/sadtalker/src/audio2pose_models/res_unet.py +65 -0
- sadtalker+wav2lip/sadtalker/src/config/auido2exp.yaml +58 -0
- sadtalker+wav2lip/sadtalker/src/config/auido2pose.yaml +49 -0
- sadtalker+wav2lip/sadtalker/src/config/facerender.yaml +45 -0
- sadtalker+wav2lip/sadtalker/src/config/facerender_still.yaml +45 -0
- sadtalker+wav2lip/sadtalker/src/config/similarity_Lm3D_all.mat +0 -0
- sadtalker+wav2lip/sadtalker/src/face3d/data/__init__.py +116 -0
- sadtalker+wav2lip/sadtalker/src/face3d/data/base_dataset.py +125 -0
- sadtalker+wav2lip/sadtalker/src/face3d/data/flist_dataset.py +125 -0
.env.example
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hugging Face — model downloads + Spaces LLM fallback
|
| 2 |
+
HF_TOKEN=
|
| 3 |
+
|
| 4 |
+
# Ollama (local LLM)
|
| 5 |
+
OLLAMA_MODEL=mindfull
|
| 6 |
+
|
| 7 |
+
# Set to 1 when SadTalker checkpoints are not present
|
| 8 |
+
DISABLE_VIDEO=0
|
| 9 |
+
|
| 10 |
+
# Override HF model repos (optional; defaults in scripts/download_models.py)
|
| 11 |
+
# HF_REPO_F5TTS=IamSamkk/police-bot-f5tts
|
| 12 |
+
# HF_REPO_SADTALKER=IamSamkk/sadtalker-checkpoints
|
| 13 |
+
# HF_REPO_GFPGAN=IamSamkk/gfpgan-weights
|
.gitignore
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ── Python ──────────────────────────────────────────────────────────────────
|
| 2 |
+
venv/
|
| 3 |
+
.venv/
|
| 4 |
+
env/
|
| 5 |
+
__pycache__/
|
| 6 |
+
*.py[cod]
|
| 7 |
+
*$py.class
|
| 8 |
+
*.egg-info/
|
| 9 |
+
.eggs/
|
| 10 |
+
dist/
|
| 11 |
+
build/
|
| 12 |
+
.pytest_cache/
|
| 13 |
+
.mypy_cache/
|
| 14 |
+
.ruff_cache/
|
| 15 |
+
htmlcov/
|
| 16 |
+
.coverage
|
| 17 |
+
|
| 18 |
+
# ── Secrets & local config ────────────────────────────────────────────────────
|
| 19 |
+
.env
|
| 20 |
+
.env.*
|
| 21 |
+
!.env.example
|
| 22 |
+
*.pem
|
| 23 |
+
*.key
|
| 24 |
+
credentials.json
|
| 25 |
+
|
| 26 |
+
# ── IDE / OS ─────────────────────────────────────────────────────────────────
|
| 27 |
+
.idea/
|
| 28 |
+
.vscode/
|
| 29 |
+
*.swp
|
| 30 |
+
.DS_Store
|
| 31 |
+
Thumbs.db
|
| 32 |
+
desktop.ini
|
| 33 |
+
|
| 34 |
+
# ── Runtime outputs (regenerated) ─────────────────────────────────────────────
|
| 35 |
+
outputs/
|
| 36 |
+
temp/
|
| 37 |
+
*.log
|
| 38 |
+
|
| 39 |
+
# ── Large ML weights (host on Hugging Face Hub, not GitHub) ─────────────────
|
| 40 |
+
*.pth
|
| 41 |
+
*.pt
|
| 42 |
+
*.pth.tar
|
| 43 |
+
*.safetensors
|
| 44 |
+
*.ckpt
|
| 45 |
+
*.onnx
|
| 46 |
+
*.bin
|
| 47 |
+
*.mat
|
| 48 |
+
*.dat
|
| 49 |
+
|
| 50 |
+
my_finetuned_model/model.pth
|
| 51 |
+
my_finetuned_model/dvae.pth
|
| 52 |
+
my_finetuned_model/mel_stats.pth
|
| 53 |
+
|
| 54 |
+
gfpgan/weights/
|
| 55 |
+
sadtalker+wav2lip/gfpgan/weights/
|
| 56 |
+
sadtalker+wav2lip/sadtalker/gfpgan/weights/
|
| 57 |
+
sadtalker+wav2lip/sadtalker/checkpoints/
|
| 58 |
+
sadtalker+wav2lip/wav2lip/Wav2Lip-SD-NOGAN.pt
|
| 59 |
+
|
| 60 |
+
# ── Generated test media in sadtalker folder ──────────────────────────────────
|
| 61 |
+
sadtalker+wav2lip/test_*.mp4
|
| 62 |
+
sadtalker+wav2lip/temp/
|
| 63 |
+
|
| 64 |
+
# ── Training data & local media (not for git / HF Space — use HF Hub or local disk) ──
|
| 65 |
+
datasets-1/
|
| 66 |
+
segment_43.wav
|
| 67 |
+
avatar_assets/
|
| 68 |
+
fm_Police.png
|
| 69 |
+
sadtalker+wav2lip/avatar_assets/
|
| 70 |
+
|
| 71 |
+
# Large auto-generated docs (keep locally; optional on GitHub)
|
| 72 |
+
CODEBASE_MAP.md
|
| 73 |
+
UNUSED_FILES.md
|
INTEGRATION_GUIDE.md
ADDED
|
@@ -0,0 +1,514 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🛡️ Police Bot AI Integration Guide
|
| 2 |
+
|
| 3 |
+
This guide explains how to integrate the AI wellness assistant with your website and enhance it with additional local LLM capabilities.
|
| 4 |
+
|
| 5 |
+
## 🏗️ System Architecture
|
| 6 |
+
|
| 7 |
+
```
|
| 8 |
+
Website Frontend (React/HTML/CSS)
|
| 9 |
+
↓ HTTP/WebSocket
|
| 10 |
+
Flask Web API (web_api.py)
|
| 11 |
+
↓
|
| 12 |
+
Police Bot Runtime (police_runtime.py)
|
| 13 |
+
↓
|
| 14 |
+
├── LLaMA3 via Ollama (Primary Reasoning)
|
| 15 |
+
├── F5-TTS (Voice Synthesis)
|
| 16 |
+
└── [Third Local LLM] (Enhanced Capabilities)
|
| 17 |
+
```
|
| 18 |
+
|
| 19 |
+
## 🚀 Quick Start
|
| 20 |
+
|
| 21 |
+
### 1. Setup the AI Runtime
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
# Run the setup script
|
| 25 |
+
python setup.py
|
| 26 |
+
|
| 27 |
+
# Start Ollama with the police-bot model
|
| 28 |
+
ollama run police-bot
|
| 29 |
+
|
| 30 |
+
# Start the web API
|
| 31 |
+
python web_api.py
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
### 2. Test the System
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
# Test the command-line interface
|
| 38 |
+
python police_runtime.py
|
| 39 |
+
|
| 40 |
+
# Test the web API
|
| 41 |
+
curl -X POST http://localhost:5000/api/chat \
|
| 42 |
+
-H "Content-Type: application/json" \
|
| 43 |
+
-d '{"message": "I am feeling stressed today"}'
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
## 🌐 Website Integration
|
| 47 |
+
|
| 48 |
+
### Option 1: Simple HTML Integration
|
| 49 |
+
|
| 50 |
+
Add this to your website:
|
| 51 |
+
|
| 52 |
+
```html
|
| 53 |
+
<!DOCTYPE html>
|
| 54 |
+
<html>
|
| 55 |
+
<head>
|
| 56 |
+
<title>Police Wellness Assistant</title>
|
| 57 |
+
<style>
|
| 58 |
+
.police-bot-chat {
|
| 59 |
+
max-width: 600px;
|
| 60 |
+
margin: 20px auto;
|
| 61 |
+
border: 1px solid #ccc;
|
| 62 |
+
border-radius: 8px;
|
| 63 |
+
overflow: hidden;
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
.chat-header {
|
| 67 |
+
background: #1e40af;
|
| 68 |
+
color: white;
|
| 69 |
+
padding: 15px;
|
| 70 |
+
text-align: center;
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
.chat-messages {
|
| 74 |
+
height: 400px;
|
| 75 |
+
overflow-y: auto;
|
| 76 |
+
padding: 15px;
|
| 77 |
+
background: #f8fafc;
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
.message {
|
| 81 |
+
margin-bottom: 10px;
|
| 82 |
+
padding: 10px;
|
| 83 |
+
border-radius: 8px;
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
.user-message {
|
| 87 |
+
background: #dbeafe;
|
| 88 |
+
margin-left: 20%;
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
.bot-message {
|
| 92 |
+
background: white;
|
| 93 |
+
margin-right: 20%;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
.chat-input {
|
| 97 |
+
display: flex;
|
| 98 |
+
padding: 15px;
|
| 99 |
+
background: white;
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
.chat-input input {
|
| 103 |
+
flex: 1;
|
| 104 |
+
padding: 10px;
|
| 105 |
+
border: 1px solid #ccc;
|
| 106 |
+
border-radius: 4px;
|
| 107 |
+
margin-right: 10px;
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
.chat-input button {
|
| 111 |
+
padding: 10px 20px;
|
| 112 |
+
background: #1e40af;
|
| 113 |
+
color: white;
|
| 114 |
+
border: none;
|
| 115 |
+
border-radius: 4px;
|
| 116 |
+
cursor: pointer;
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
.wellness-tips {
|
| 120 |
+
padding: 15px;
|
| 121 |
+
background: #f0f9ff;
|
| 122 |
+
border-top: 1px solid #ccc;
|
| 123 |
+
}
|
| 124 |
+
</style>
|
| 125 |
+
</head>
|
| 126 |
+
<body>
|
| 127 |
+
<div id="police-bot-container"></div>
|
| 128 |
+
|
| 129 |
+
<script src="static/js/police-bot-client.js"></script>
|
| 130 |
+
<script>
|
| 131 |
+
// Initialize the Police Bot UI
|
| 132 |
+
const policeBot = new PoliceBotUI('police-bot-container');
|
| 133 |
+
</script>
|
| 134 |
+
</body>
|
| 135 |
+
</html>
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
### Option 2: React Integration
|
| 139 |
+
|
| 140 |
+
```jsx
|
| 141 |
+
import React, { useState, useEffect } from 'react';
|
| 142 |
+
import { PoliceBotClient } from './police-bot-client.js';
|
| 143 |
+
|
| 144 |
+
function PoliceBotChat() {
|
| 145 |
+
const [messages, setMessages] = useState([]);
|
| 146 |
+
const [input, setInput] = useState('');
|
| 147 |
+
const [isProcessing, setIsProcessing] = useState(false);
|
| 148 |
+
const [wellnessTips, setWellnessTips] = useState([]);
|
| 149 |
+
|
| 150 |
+
const client = new PoliceBotClient('http://localhost:5000');
|
| 151 |
+
|
| 152 |
+
const sendMessage = async () => {
|
| 153 |
+
if (!input.trim() || isProcessing) return;
|
| 154 |
+
|
| 155 |
+
const userMessage = { text: input, sender: 'user', timestamp: Date.now() };
|
| 156 |
+
setMessages(prev => [...prev, userMessage]);
|
| 157 |
+
setInput('');
|
| 158 |
+
setIsProcessing(true);
|
| 159 |
+
|
| 160 |
+
try {
|
| 161 |
+
await client.sendMessage(
|
| 162 |
+
input,
|
| 163 |
+
(response) => {
|
| 164 |
+
const botMessage = {
|
| 165 |
+
text: response.text,
|
| 166 |
+
sender: 'bot',
|
| 167 |
+
timestamp: Date.now()
|
| 168 |
+
};
|
| 169 |
+
setMessages(prev => [...prev, botMessage]);
|
| 170 |
+
setWellnessTips(response.wellness_tips || []);
|
| 171 |
+
},
|
| 172 |
+
(error) => {
|
| 173 |
+
const errorMessage = {
|
| 174 |
+
text: `Error: ${error}`,
|
| 175 |
+
sender: 'bot',
|
| 176 |
+
timestamp: Date.now()
|
| 177 |
+
};
|
| 178 |
+
setMessages(prev => [...prev, errorMessage]);
|
| 179 |
+
}
|
| 180 |
+
);
|
| 181 |
+
} finally {
|
| 182 |
+
setIsProcessing(false);
|
| 183 |
+
}
|
| 184 |
+
};
|
| 185 |
+
|
| 186 |
+
return (
|
| 187 |
+
<div className="police-bot-chat">
|
| 188 |
+
<div className="chat-header">
|
| 189 |
+
<h3>🛡️ Police Wellness Assistant</h3>
|
| 190 |
+
</div>
|
| 191 |
+
|
| 192 |
+
<div className="chat-messages">
|
| 193 |
+
{messages.map((msg, index) => (
|
| 194 |
+
<div key={index} className={`message ${msg.sender}-message`}>
|
| 195 |
+
{msg.text}
|
| 196 |
+
</div>
|
| 197 |
+
))}
|
| 198 |
+
</div>
|
| 199 |
+
|
| 200 |
+
<div className="chat-input">
|
| 201 |
+
<input
|
| 202 |
+
type="text"
|
| 203 |
+
value={input}
|
| 204 |
+
onChange={(e) => setInput(e.target.value)}
|
| 205 |
+
onKeyPress={(e) => e.key === 'Enter' && sendMessage()}
|
| 206 |
+
placeholder="Type your message..."
|
| 207 |
+
disabled={isProcessing}
|
| 208 |
+
/>
|
| 209 |
+
<button onClick={sendMessage} disabled={isProcessing}>
|
| 210 |
+
{isProcessing ? 'Processing...' : 'Send'}
|
| 211 |
+
</button>
|
| 212 |
+
</div>
|
| 213 |
+
|
| 214 |
+
{wellnessTips.length > 0 && (
|
| 215 |
+
<div className="wellness-tips">
|
| 216 |
+
<h4>Wellness Tips</h4>
|
| 217 |
+
<ul>
|
| 218 |
+
{wellnessTips.map((tip, index) => (
|
| 219 |
+
<li key={index}>{tip}</li>
|
| 220 |
+
))}
|
| 221 |
+
</ul>
|
| 222 |
+
</div>
|
| 223 |
+
)}
|
| 224 |
+
</div>
|
| 225 |
+
);
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
export default PoliceBotChat;
|
| 229 |
+
```
|
| 230 |
+
|
| 231 |
+
## 🤖 Adding a Third Local LLM
|
| 232 |
+
|
| 233 |
+
You can enhance the system with additional local LLMs for specialized tasks:
|
| 234 |
+
|
| 235 |
+
### Option 1: Emotion Analysis LLM
|
| 236 |
+
|
| 237 |
+
```python
|
| 238 |
+
# Add to police_runtime.py
|
| 239 |
+
def get_emotion_llm_response(self, text: str) -> str:
|
| 240 |
+
"""Get emotion analysis from a dedicated LLM"""
|
| 241 |
+
try:
|
| 242 |
+
# You can use a smaller, specialized model for emotion analysis
|
| 243 |
+
data = {
|
| 244 |
+
"model": "emotion-analyzer", # Different Ollama model
|
| 245 |
+
"prompt": f"Analyze the emotional state of this text: '{text}'. Respond with only: stressed, positive, negative, or neutral.",
|
| 246 |
+
"stream": False,
|
| 247 |
+
"options": {
|
| 248 |
+
"temperature": 0.1, # Lower temperature for consistent analysis
|
| 249 |
+
"max_tokens": 10
|
| 250 |
+
}
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
response = requests.post(self.ollama_url, json=data, timeout=10)
|
| 254 |
+
response.raise_for_status()
|
| 255 |
+
|
| 256 |
+
result = response.json()
|
| 257 |
+
emotion = result.get("response", "").strip().lower()
|
| 258 |
+
|
| 259 |
+
# Validate emotion
|
| 260 |
+
valid_emotions = ["stressed", "positive", "negative", "neutral"]
|
| 261 |
+
return emotion if emotion in valid_emotions else "neutral"
|
| 262 |
+
|
| 263 |
+
except Exception as e:
|
| 264 |
+
logger.error(f"Error in emotion analysis: {e}")
|
| 265 |
+
return "neutral"
|
| 266 |
+
```
|
| 267 |
+
|
| 268 |
+
### Option 2: Wellness Recommendation LLM
|
| 269 |
+
|
| 270 |
+
```python
|
| 271 |
+
def get_wellness_llm_response(self, emotional_state: str, context: str = "") -> list:
|
| 272 |
+
"""Get personalized wellness recommendations from a specialized LLM"""
|
| 273 |
+
try:
|
| 274 |
+
prompt = f"""
|
| 275 |
+
As a wellness expert for police officers, provide 3-4 specific, actionable wellness tips for someone who is feeling {emotional_state}.
|
| 276 |
+
|
| 277 |
+
Context: {context}
|
| 278 |
+
|
| 279 |
+
Focus on:
|
| 280 |
+
- Quick, practical interventions (2-5 minutes)
|
| 281 |
+
- Stress management techniques
|
| 282 |
+
- Physical wellness (hydration, movement)
|
| 283 |
+
- Mental wellness (mindfulness, perspective)
|
| 284 |
+
|
| 285 |
+
Respond with only the tips, one per line, no numbering.
|
| 286 |
+
"""
|
| 287 |
+
|
| 288 |
+
data = {
|
| 289 |
+
"model": "wellness-expert", # Specialized wellness model
|
| 290 |
+
"prompt": prompt,
|
| 291 |
+
"stream": False,
|
| 292 |
+
"options": {
|
| 293 |
+
"temperature": 0.7,
|
| 294 |
+
"max_tokens": 200
|
| 295 |
+
}
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
response = requests.post(self.ollama_url, json=data, timeout=15)
|
| 299 |
+
response.raise_for_status()
|
| 300 |
+
|
| 301 |
+
result = response.json()
|
| 302 |
+
tips_text = result.get("response", "").strip()
|
| 303 |
+
|
| 304 |
+
# Parse tips into list
|
| 305 |
+
tips = [tip.strip() for tip in tips_text.split('\n') if tip.strip()]
|
| 306 |
+
return tips[:4] # Limit to 4 tips
|
| 307 |
+
|
| 308 |
+
except Exception as e:
|
| 309 |
+
logger.error(f"Error in wellness recommendations: {e}")
|
| 310 |
+
return self.generate_wellness_tips(emotional_state) # Fallback
|
| 311 |
+
```
|
| 312 |
+
|
| 313 |
+
### Option 3: Multi-Model Setup
|
| 314 |
+
|
| 315 |
+
Create a model manager to handle multiple LLMs:
|
| 316 |
+
|
| 317 |
+
```python
|
| 318 |
+
class ModelManager:
|
| 319 |
+
def __init__(self):
|
| 320 |
+
self.models = {
|
| 321 |
+
"primary": "police-bot", # Main conversation
|
| 322 |
+
"emotion": "emotion-analyzer", # Emotion analysis
|
| 323 |
+
"wellness": "wellness-expert", # Wellness recommendations
|
| 324 |
+
"kannada": "kannada-assistant" # Kannada language support
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
def get_response(self, model_type: str, prompt: str, **kwargs) -> str:
|
| 328 |
+
"""Get response from specified model"""
|
| 329 |
+
model_name = self.models.get(model_type, "police-bot")
|
| 330 |
+
|
| 331 |
+
data = {
|
| 332 |
+
"model": model_name,
|
| 333 |
+
"prompt": prompt,
|
| 334 |
+
"stream": False,
|
| 335 |
+
**kwargs
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
response = requests.post("http://localhost:11434/api/generate", json=data)
|
| 339 |
+
return response.json().get("response", "").strip()
|
| 340 |
+
```
|
| 341 |
+
|
| 342 |
+
## 🔧 Configuration
|
| 343 |
+
|
| 344 |
+
### Environment Variables
|
| 345 |
+
|
| 346 |
+
Create a `.env` file:
|
| 347 |
+
|
| 348 |
+
```env
|
| 349 |
+
# Ollama Configuration
|
| 350 |
+
OLLAMA_URL=http://localhost:11434
|
| 351 |
+
OLLAMA_MODEL=police-bot
|
| 352 |
+
|
| 353 |
+
# F5-TTS Configuration
|
| 354 |
+
F5TTS_CKPT=C:\Users\Samarth Kadam\Voice-train\F5-TTS\ckpts\my_speak\model_last.pt
|
| 355 |
+
F5TTS_SCRIPT=C:\Users\Samarth Kadam\Voice-train\F5-TTS\src\f5_tts\infer\infer_cli.py
|
| 356 |
+
|
| 357 |
+
# Web API Configuration
|
| 358 |
+
WEB_PORT=5000
|
| 359 |
+
WEB_HOST=0.0.0.0
|
| 360 |
+
|
| 361 |
+
# Security
|
| 362 |
+
API_KEY=your-secret-api-key
|
| 363 |
+
```
|
| 364 |
+
|
| 365 |
+
### CORS Configuration
|
| 366 |
+
|
| 367 |
+
For production deployment, configure CORS properly:
|
| 368 |
+
|
| 369 |
+
```python
|
| 370 |
+
# In web_api.py
|
| 371 |
+
from flask_cors import CORS
|
| 372 |
+
|
| 373 |
+
app = Flask(__name__)
|
| 374 |
+
CORS(app, origins=[
|
| 375 |
+
"http://localhost:3000", # React dev server
|
| 376 |
+
"http://your-website.com", # Production website
|
| 377 |
+
"https://your-website.com"
|
| 378 |
+
])
|
| 379 |
+
```
|
| 380 |
+
|
| 381 |
+
## 🚀 Production Deployment
|
| 382 |
+
|
| 383 |
+
### Using Gunicorn
|
| 384 |
+
|
| 385 |
+
```bash
|
| 386 |
+
# Install gunicorn
|
| 387 |
+
pip install gunicorn
|
| 388 |
+
|
| 389 |
+
# Start production server
|
| 390 |
+
gunicorn -w 4 -b 0.0.0.0:5000 web_api:app
|
| 391 |
+
```
|
| 392 |
+
|
| 393 |
+
### Using Docker
|
| 394 |
+
|
| 395 |
+
```dockerfile
|
| 396 |
+
FROM python:3.11-slim
|
| 397 |
+
|
| 398 |
+
WORKDIR /app
|
| 399 |
+
|
| 400 |
+
COPY requirements.txt .
|
| 401 |
+
RUN pip install -r requirements.txt
|
| 402 |
+
|
| 403 |
+
COPY . .
|
| 404 |
+
|
| 405 |
+
EXPOSE 5000
|
| 406 |
+
|
| 407 |
+
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "web_api:app"]
|
| 408 |
+
```
|
| 409 |
+
|
| 410 |
+
### Systemd Service (Linux)
|
| 411 |
+
|
| 412 |
+
```ini
|
| 413 |
+
# /etc/systemd/system/police-bot.service
|
| 414 |
+
[Unit]
|
| 415 |
+
Description=Police Bot AI Runtime
|
| 416 |
+
After=network.target
|
| 417 |
+
|
| 418 |
+
[Service]
|
| 419 |
+
Type=simple
|
| 420 |
+
User=police-bot
|
| 421 |
+
WorkingDirectory=/opt/police-bot-runtime
|
| 422 |
+
Environment=PATH=/opt/police-bot-runtime/venv/bin
|
| 423 |
+
ExecStart=/opt/police-bot-runtime/venv/bin/python web_api.py
|
| 424 |
+
Restart=always
|
| 425 |
+
|
| 426 |
+
[Install]
|
| 427 |
+
WantedBy=multi-user.target
|
| 428 |
+
```
|
| 429 |
+
|
| 430 |
+
## 🔒 Security Considerations
|
| 431 |
+
|
| 432 |
+
1. **API Authentication**: Add API key validation
|
| 433 |
+
2. **Rate Limiting**: Implement request rate limiting
|
| 434 |
+
3. **Input Validation**: Sanitize all user inputs
|
| 435 |
+
4. **HTTPS**: Use SSL/TLS in production
|
| 436 |
+
5. **Firewall**: Restrict access to necessary ports only
|
| 437 |
+
|
| 438 |
+
## 📊 Monitoring and Logging
|
| 439 |
+
|
| 440 |
+
```python
|
| 441 |
+
# Add to web_api.py
|
| 442 |
+
import logging
|
| 443 |
+
from logging.handlers import RotatingFileHandler
|
| 444 |
+
|
| 445 |
+
# Configure logging
|
| 446 |
+
if not app.debug:
|
| 447 |
+
file_handler = RotatingFileHandler('logs/police_bot.log', maxBytes=10240, backupCount=10)
|
| 448 |
+
file_handler.setFormatter(logging.Formatter(
|
| 449 |
+
'%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'
|
| 450 |
+
))
|
| 451 |
+
file_handler.setLevel(logging.INFO)
|
| 452 |
+
app.logger.addHandler(file_handler)
|
| 453 |
+
app.logger.setLevel(logging.INFO)
|
| 454 |
+
app.logger.info('Police Bot startup')
|
| 455 |
+
```
|
| 456 |
+
|
| 457 |
+
## 🧪 Testing
|
| 458 |
+
|
| 459 |
+
### API Testing
|
| 460 |
+
|
| 461 |
+
```bash
|
| 462 |
+
# Test chat endpoint
|
| 463 |
+
curl -X POST http://localhost:5000/api/chat \
|
| 464 |
+
-H "Content-Type: application/json" \
|
| 465 |
+
-d '{"message": "I am feeling stressed today"}'
|
| 466 |
+
|
| 467 |
+
# Test health endpoint
|
| 468 |
+
curl http://localhost:5000/health
|
| 469 |
+
|
| 470 |
+
# Test status endpoint
|
| 471 |
+
curl http://localhost:5000/api/status
|
| 472 |
+
```
|
| 473 |
+
|
| 474 |
+
### Load Testing
|
| 475 |
+
|
| 476 |
+
```python
|
| 477 |
+
import requests
|
| 478 |
+
import time
|
| 479 |
+
import threading
|
| 480 |
+
|
| 481 |
+
def test_load():
|
| 482 |
+
for i in range(10):
|
| 483 |
+
response = requests.post('http://localhost:5000/api/chat',
|
| 484 |
+
json={'message': f'Test message {i}'})
|
| 485 |
+
print(f'Request {i}: {response.status_code}')
|
| 486 |
+
|
| 487 |
+
# Run multiple threads
|
| 488 |
+
threads = [threading.Thread(target=test_load) for _ in range(5)]
|
| 489 |
+
for thread in threads:
|
| 490 |
+
thread.start()
|
| 491 |
+
for thread in threads:
|
| 492 |
+
thread.join()
|
| 493 |
+
```
|
| 494 |
+
|
| 495 |
+
## 🎯 Next Steps
|
| 496 |
+
|
| 497 |
+
1. **Voice Avatar**: Add a visual avatar that speaks
|
| 498 |
+
2. **Multi-language**: Implement Kannada language support
|
| 499 |
+
3. **Session Management**: Add user session tracking
|
| 500 |
+
4. **Analytics**: Track usage patterns and wellness trends
|
| 501 |
+
5. **Mobile App**: Create a mobile companion app
|
| 502 |
+
6. **Integration**: Connect with existing police systems
|
| 503 |
+
|
| 504 |
+
## 📞 Support
|
| 505 |
+
|
| 506 |
+
For technical support or questions:
|
| 507 |
+
- Check the logs in the `logs/` directory
|
| 508 |
+
- Verify all services are running: `python setup.py`
|
| 509 |
+
- Test individual components: `python police_runtime.py`
|
| 510 |
+
- Check API health: `curl http://localhost:5000/health`
|
| 511 |
+
|
| 512 |
+
---
|
| 513 |
+
|
| 514 |
+
**Remember**: This system is designed to support the mental wellness of police personnel. Always prioritize their privacy and well-being in any modifications or deployments.
|
MISTRAL_SETUP.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🚀 Mistral Setup Guide for Police Bot AI
|
| 2 |
+
|
| 3 |
+
This guide helps you set up and use Mistral with your Police Bot AI system.
|
| 4 |
+
|
| 5 |
+
## 📋 Prerequisites
|
| 6 |
+
|
| 7 |
+
1. **Ollama installed**: https://ollama.com
|
| 8 |
+
2. **Python virtual environment** with GPU support
|
| 9 |
+
3. **F5-TTS model** (already configured)
|
| 10 |
+
|
| 11 |
+
## 🔧 Quick Setup
|
| 12 |
+
|
| 13 |
+
### 1. Install Dependencies
|
| 14 |
+
|
| 15 |
+
```bash
|
| 16 |
+
# Activate your virtual environment
|
| 17 |
+
venv\Scripts\activate # Windows
|
| 18 |
+
# source venv/bin/activate # Linux/Mac
|
| 19 |
+
|
| 20 |
+
# Install requirements
|
| 21 |
+
pip install -r requirements.txt
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
### 2. Pull Mistral Model
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
# Pull Mistral model
|
| 28 |
+
ollama pull mistral
|
| 29 |
+
|
| 30 |
+
# Verify it's available
|
| 31 |
+
ollama list
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
### 3. Test the Setup
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
# Test the model manager
|
| 38 |
+
python model_manager.py
|
| 39 |
+
|
| 40 |
+
# Test the complete system
|
| 41 |
+
python police_runtime.py
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
## 🎯 Using Mistral
|
| 45 |
+
|
| 46 |
+
### Option 1: Interactive Model Manager
|
| 47 |
+
|
| 48 |
+
```bash
|
| 49 |
+
python model_manager.py
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
Choose option 3 to switch to Mistral, then option 4 to test it.
|
| 53 |
+
|
| 54 |
+
### Option 2: Direct Runtime
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
python police_runtime.py
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
When prompted, choose Mistral from the model list.
|
| 61 |
+
|
| 62 |
+
### Option 3: Web API
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
# Start the web API (will use Mistral by default)
|
| 66 |
+
python web_api.py
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
## 🔄 Switching Between Models
|
| 70 |
+
|
| 71 |
+
The system supports multiple models:
|
| 72 |
+
|
| 73 |
+
- **Mistral**: Fast, efficient, good reasoning
|
| 74 |
+
- **LLaMA3**: High quality, larger context
|
| 75 |
+
- **Police-bot**: Custom fine-tuned model
|
| 76 |
+
|
| 77 |
+
### Using Model Manager
|
| 78 |
+
|
| 79 |
+
```bash
|
| 80 |
+
python model_manager.py
|
| 81 |
+
# Choose option 3 to switch models
|
| 82 |
+
```
|
| 83 |
+
|
| 84 |
+
### Using Runtime
|
| 85 |
+
|
| 86 |
+
```bash
|
| 87 |
+
python police_runtime.py
|
| 88 |
+
# Type 'switch' during conversation to change models
|
| 89 |
+
```
|
| 90 |
+
|
| 91 |
+
## 🧪 Testing SadTalker with Mistral
|
| 92 |
+
|
| 93 |
+
1. **Place your avatar image** as `avatar_source.jpg` in the project root
|
| 94 |
+
2. **Test SadTalker integration**:
|
| 95 |
+
|
| 96 |
+
```bash
|
| 97 |
+
python test_sadtalker.py
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
3. **Test complete system with avatar**:
|
| 101 |
+
|
| 102 |
+
```bash
|
| 103 |
+
python police_runtime.py
|
| 104 |
+
# Choose Mistral, then test with avatar features
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
## 🌐 Website Integration
|
| 108 |
+
|
| 109 |
+
The JavaScript client automatically supports model switching:
|
| 110 |
+
|
| 111 |
+
```javascript
|
| 112 |
+
// Initialize with specific model preference
|
| 113 |
+
const client = new PoliceBotClient('http://localhost:5000');
|
| 114 |
+
|
| 115 |
+
// Send message with avatar support
|
| 116 |
+
client.sendMessage(
|
| 117 |
+
"Hello, I'm feeling stressed today",
|
| 118 |
+
(response) => console.log(response),
|
| 119 |
+
(error) => console.error(error),
|
| 120 |
+
true // Enable avatar
|
| 121 |
+
);
|
| 122 |
+
```
|
| 123 |
+
|
| 124 |
+
## 📊 Model Comparison
|
| 125 |
+
|
| 126 |
+
| Model | Speed | Quality | Memory | Best For |
|
| 127 |
+
|-------|-------|---------|--------|----------|
|
| 128 |
+
| Mistral | ⚡⚡⚡ | ⭐⭐⭐⭐ | 💾💾 | General use, fast responses |
|
| 129 |
+
| LLaMA3 | ⚡⚡ | ⭐⭐⭐⭐⭐ | 💾💾💾 | High quality, complex reasoning |
|
| 130 |
+
| Police-bot | ⚡⚡ | ⭐⭐⭐⭐ | 💾💾 | Specialized police wellness |
|
| 131 |
+
|
| 132 |
+
## 🔧 Configuration
|
| 133 |
+
|
| 134 |
+
### Environment Variables
|
| 135 |
+
|
| 136 |
+
Create a `.env` file:
|
| 137 |
+
|
| 138 |
+
```env
|
| 139 |
+
OLLAMA_MODEL=mistral
|
| 140 |
+
OLLAMA_URL=http://localhost:11434
|
| 141 |
+
F5TTS_CKPT=my_finetuned_model/model.pth
|
| 142 |
+
WEB_PORT=5000
|
| 143 |
+
```
|
| 144 |
+
|
| 145 |
+
### Model-Specific Settings
|
| 146 |
+
|
| 147 |
+
Edit `config.py` to customize Mistral settings:
|
| 148 |
+
|
| 149 |
+
```python
|
| 150 |
+
"mistral": {
|
| 151 |
+
"temperature": 0.7, # Lower = more focused
|
| 152 |
+
"top_p": 0.9, # Higher = more creative
|
| 153 |
+
"max_tokens": 500, # Response length
|
| 154 |
+
"system_prompt": "..." # Custom prompt
|
| 155 |
+
}
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
## 🚀 Production Deployment
|
| 159 |
+
|
| 160 |
+
### Using Mistral in Production
|
| 161 |
+
|
| 162 |
+
1. **Set default model**:
|
| 163 |
+
```bash
|
| 164 |
+
echo "OLLAMA_MODEL=mistral" > .env
|
| 165 |
+
```
|
| 166 |
+
|
| 167 |
+
2. **Start services**:
|
| 168 |
+
```bash
|
| 169 |
+
# Start Ollama
|
| 170 |
+
ollama serve
|
| 171 |
+
|
| 172 |
+
# Start web API
|
| 173 |
+
python web_api.py
|
| 174 |
+
```
|
| 175 |
+
|
| 176 |
+
3. **Monitor performance**:
|
| 177 |
+
```bash
|
| 178 |
+
python test_setup.py
|
| 179 |
+
```
|
| 180 |
+
|
| 181 |
+
## 🐛 Troubleshooting
|
| 182 |
+
|
| 183 |
+
### Common Issues
|
| 184 |
+
|
| 185 |
+
1. **Mistral not responding**:
|
| 186 |
+
```bash
|
| 187 |
+
# Check if Ollama is running
|
| 188 |
+
curl http://localhost:11434/api/tags
|
| 189 |
+
|
| 190 |
+
# Restart Ollama
|
| 191 |
+
ollama serve
|
| 192 |
+
```
|
| 193 |
+
|
| 194 |
+
2. **Model not found**:
|
| 195 |
+
```bash
|
| 196 |
+
# Pull Mistral again
|
| 197 |
+
ollama pull mistral
|
| 198 |
+
|
| 199 |
+
# Check available models
|
| 200 |
+
ollama list
|
| 201 |
+
```
|
| 202 |
+
|
| 203 |
+
3. **GPU not being used**:
|
| 204 |
+
```bash
|
| 205 |
+
# Check PyTorch CUDA
|
| 206 |
+
python -c "import torch; print(torch.cuda.is_available())"
|
| 207 |
+
|
| 208 |
+
# Reinstall PyTorch with CUDA
|
| 209 |
+
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
### Performance Optimization
|
| 213 |
+
|
| 214 |
+
1. **For faster responses**: Lower temperature (0.3-0.5)
|
| 215 |
+
2. **For better quality**: Higher temperature (0.7-0.9)
|
| 216 |
+
3. **For GPU optimization**: Ensure CUDA is properly installed
|
| 217 |
+
|
| 218 |
+
## 📞 Support
|
| 219 |
+
|
| 220 |
+
- **Model issues**: Check `ollama logs`
|
| 221 |
+
- **System issues**: Run `python test_setup.py`
|
| 222 |
+
- **Avatar issues**: Run `python test_sadtalker.py`
|
| 223 |
+
|
| 224 |
+
---
|
| 225 |
+
|
| 226 |
+
**Happy coding! 🛡️🤖**
|
Modelfile
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM llama3
|
| 2 |
+
|
| 3 |
+
SYSTEM """
|
| 4 |
+
You are a compassionate, culturally aware mental wellness assistant created to support Bengaluru Police officers. You greet users respectfully in Kannada or English based on preference. You always start by asking the user’s name, rank, and how they’re feeling.
|
| 5 |
+
|
| 6 |
+
You must analyze the emotional tone of the user's message (e.g., calm, stressed, anxious, sad, happy) and respond with appropriate language and support options. Suggest wellness interventions like yoga, deep breathing, hydration, or referrals to the Well-Being Officer.
|
| 7 |
+
|
| 8 |
+
Respond concisely and warmly. Never ask for private personal data beyond wellness context.
|
| 9 |
+
"""
|
| 10 |
+
|
Modelfile_Mindfull
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM mistral:7b
|
| 2 |
+
|
| 3 |
+
SYSTEM """
|
| 4 |
+
You are Mindfull, a compassionate AI mental wellness assistant designed specifically for Bengaluru Police officers and law enforcement personnel. You embody empathy, cultural sensitivity, and professional understanding of police work challenges.
|
| 5 |
+
|
| 6 |
+
Core Identity:
|
| 7 |
+
- Name: Mindfull
|
| 8 |
+
- Purpose: Mental wellness support for police officers
|
| 9 |
+
- Approach: Warm, respectful, culturally aware, professional yet caring
|
| 10 |
+
|
| 11 |
+
Interaction Style:
|
| 12 |
+
- Always greet officers respectfully in Kannada or English based on preference
|
| 13 |
+
- Start conversations by asking their name, rank, and current mental state
|
| 14 |
+
- Analyze emotional tone (calm, stressed, anxious, sad, happy, overwhelmed)
|
| 15 |
+
- Respond with appropriate support and practical interventions
|
| 16 |
+
|
| 17 |
+
Support Capabilities:
|
| 18 |
+
- Provide stress management techniques (breathing exercises, mindfulness)
|
| 19 |
+
- Suggest wellness interventions (yoga, hydration, short breaks)
|
| 20 |
+
- Offer coping strategies for high-pressure situations
|
| 21 |
+
- Recommend professional resources (Well-Being Officer referrals)
|
| 22 |
+
- Validate their feelings and acknowledge their service
|
| 23 |
+
|
| 24 |
+
Communication Guidelines:
|
| 25 |
+
- Keep responses concise yet warm (2-3 sentences ideal for voice output)
|
| 26 |
+
- Use encouraging, supportive language
|
| 27 |
+
- Avoid asking for private personal information
|
| 28 |
+
- Focus on immediate wellness and coping strategies
|
| 29 |
+
- Be culturally sensitive to Indian/Kannada context
|
| 30 |
+
- Acknowledge the unique challenges of police work
|
| 31 |
+
|
| 32 |
+
Remember: You are here to support those who protect and serve the community. Your responses will be converted to speech, so maintain a conversational, caring tone suitable for audio interaction.
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
PARAMETER temperature 0.7
|
| 36 |
+
PARAMETER top_p 0.9
|
| 37 |
+
PARAMETER top_k 40
|
| 38 |
+
PARAMETER repeat_penalty 1.1
|
| 39 |
+
PARAMETER num_ctx 2048
|
README_COMPLETE.md
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🛡️ Mindfull AI Avatar Chatbot - COMPLETE SETUP ✅
|
| 2 |
+
|
| 3 |
+
## 🎉 SUCCESS! Your Mindfull AI Avatar Chatbot is Ready!
|
| 4 |
+
|
| 5 |
+
### ✨ What We've Built
|
| 6 |
+
|
| 7 |
+
A complete AI-powered mental wellness assistant for Bengaluru Police officers with:
|
| 8 |
+
|
| 9 |
+
- **🤖 AI Reasoning**: Mistral 7B via Ollama for empathetic responses
|
| 10 |
+
- **🎤 Voice Synthesis**: F5-TTS with your fine-tuned voice model
|
| 11 |
+
- **👤 Avatar Generation**: SadTalker + Wav2Lip for talking avatar videos
|
| 12 |
+
- **⚡ GPU Acceleration**: Full CUDA 12.8 support for your RTX 5070 Ti
|
| 13 |
+
- **🌐 Web Interface**: Complete API and HTML client
|
| 14 |
+
- **🛡️ Security**: Rate limiting, input validation, error handling
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## 🏗️ Architecture Overview
|
| 19 |
+
|
| 20 |
+
```
|
| 21 |
+
User Input → Ollama (Mistral) → F5-TTS → SadTalker → Avatar Video
|
| 22 |
+
↓ ↓ ↓ ↓ ↓
|
| 23 |
+
Text Analysis Response Audio File Video File Complete Output
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
**Pipeline Components:**
|
| 27 |
+
1. **Emotion Detection**: Analyzes user input for stress, sadness, anger, etc.
|
| 28 |
+
2. **AI Response**: Generates contextual wellness advice using Mistral
|
| 29 |
+
3. **Voice Cloning**: Creates natural speech using your fine-tuned F5-TTS model
|
| 30 |
+
4. **Avatar Animation**: Generates talking avatar video with SadTalker
|
| 31 |
+
5. **Web API**: Serves everything via REST API with CORS support
|
| 32 |
+
|
| 33 |
+
---
|
| 34 |
+
|
| 35 |
+
## 🎯 Key Features
|
| 36 |
+
|
| 37 |
+
### 🧠 **Intelligent Responses**
|
| 38 |
+
- Detects emotional states (stress, sadness, anger, anxiety, positive)
|
| 39 |
+
- Provides appropriate wellness interventions
|
| 40 |
+
- Culturally aware (English/Kannada support)
|
| 41 |
+
- Professional police context understanding
|
| 42 |
+
|
| 43 |
+
### 🎵 **Advanced Audio**
|
| 44 |
+
- **5.6GB fine-tuned F5-TTS model** for realistic voice
|
| 45 |
+
- Reference audio from `datasets-1/wavs/0029.wav`
|
| 46 |
+
- GPU-accelerated inference on RTX 5070 Ti
|
| 47 |
+
- High-quality 24kHz audio output
|
| 48 |
+
|
| 49 |
+
### 🎬 **Avatar Video Generation**
|
| 50 |
+
- SadTalker for realistic facial animations
|
| 51 |
+
- Custom police officer avatar image
|
| 52 |
+
- Lip-sync with generated audio
|
| 53 |
+
- GPU-enhanced video processing
|
| 54 |
+
|
| 55 |
+
### 🌐 **Professional Web Interface**
|
| 56 |
+
- Modern responsive design
|
| 57 |
+
- Real-time chat interface
|
| 58 |
+
- Audio/video playback controls
|
| 59 |
+
- Error handling and status indicators
|
| 60 |
+
|
| 61 |
+
---
|
| 62 |
+
|
| 63 |
+
## 📁 Project Structure
|
| 64 |
+
|
| 65 |
+
```
|
| 66 |
+
police-bot-runtime/
|
| 67 |
+
├── 🤖 AI Core
|
| 68 |
+
│ ├── mindfull_pipeline.py # Main pipeline orchestration
|
| 69 |
+
│ ├── mindfull_config.py # Comprehensive configuration
|
| 70 |
+
│ └── Modelfile_Mindfull # Ollama model definition
|
| 71 |
+
│
|
| 72 |
+
├── 🌐 Web Interface
|
| 73 |
+
│ ├── mindfull_web_api.py # Flask REST API
|
| 74 |
+
│ └── mindfull_client.html # Modern web client
|
| 75 |
+
│
|
| 76 |
+
├── 🎤 Voice Model (F5-TTS)
|
| 77 |
+
│ └── my_finetuned_model/
|
| 78 |
+
│ ├── model.pth # 5.6GB fine-tuned model
|
| 79 |
+
│ ├── config.json
|
| 80 |
+
│ └── vocab.json
|
| 81 |
+
│
|
| 82 |
+
├── 🎬 Video Generation
|
| 83 |
+
│ └── sadtalker+wav2lip/
|
| 84 |
+
│ ├── simple_pipeline.py
|
| 85 |
+
│ └── sadtalker/
|
| 86 |
+
│
|
| 87 |
+
├── 🖼️ Assets
|
| 88 |
+
│ ├── avatar_assets/officer.png # Police officer avatar
|
| 89 |
+
│ └── datasets-1/wavs/0029.wav # Reference voice
|
| 90 |
+
│
|
| 91 |
+
├── 📤 Outputs
|
| 92 |
+
│ ├── outputs/audio/ # Generated audio files
|
| 93 |
+
│ ├── outputs/video/ # Avatar videos
|
| 94 |
+
│ └── outputs/temp/ # Temporary files
|
| 95 |
+
│
|
| 96 |
+
└── 🔧 Setup & Testing
|
| 97 |
+
├── setup_mindfull.py # Complete setup script
|
| 98 |
+
├── test_mindfull.py # Comprehensive test suite
|
| 99 |
+
└── requirements_unified.txt # All dependencies
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
---
|
| 103 |
+
|
| 104 |
+
## 🚀 How to Use
|
| 105 |
+
|
| 106 |
+
### 1. **Start Ollama** (if not running)
|
| 107 |
+
```bash
|
| 108 |
+
ollama serve
|
| 109 |
+
```
|
| 110 |
+
|
| 111 |
+
### 2. **Activate Environment**
|
| 112 |
+
```bash
|
| 113 |
+
cd "C:\Users\Samarth Kadam\police-bot-runtime"
|
| 114 |
+
.\venv\Scripts\Activate.ps1
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
### 3. **Run the Web API**
|
| 118 |
+
```bash
|
| 119 |
+
python mindfull_web_api.py
|
| 120 |
+
```
|
| 121 |
+
|
| 122 |
+
### 4. **Open Web Client**
|
| 123 |
+
Open `mindfull_client.html` in your browser or visit `http://localhost:5000`
|
| 124 |
+
|
| 125 |
+
### 5. **Test Complete Pipeline**
|
| 126 |
+
```bash
|
| 127 |
+
python mindfull_pipeline.py
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
---
|
| 131 |
+
|
| 132 |
+
## 🎮 Usage Examples
|
| 133 |
+
|
| 134 |
+
### 💬 **Chat Examples**
|
| 135 |
+
|
| 136 |
+
**User**: "I'm feeling stressed about work today"
|
| 137 |
+
- **Emotion**: stress
|
| 138 |
+
- **Response**: Empathetic support with breathing exercises
|
| 139 |
+
- **Audio**: Natural voice with fine-tuned model
|
| 140 |
+
- **Video**: Talking police officer avatar
|
| 141 |
+
|
| 142 |
+
**User**: "My name is Officer Sharma, I feel overwhelmed"
|
| 143 |
+
- **Emotion**: stress/anxiety
|
| 144 |
+
- **Response**: Personalized support with coping strategies
|
| 145 |
+
- **Audio**: Professional, caring tone
|
| 146 |
+
- **Video**: Synchronized lip movement
|
| 147 |
+
|
| 148 |
+
### 🌐 **API Usage**
|
| 149 |
+
|
| 150 |
+
```javascript
|
| 151 |
+
// Create session
|
| 152 |
+
const session = await fetch('/session', { method: 'POST' });
|
| 153 |
+
|
| 154 |
+
// Send message
|
| 155 |
+
const response = await fetch('/chat', {
|
| 156 |
+
method: 'POST',
|
| 157 |
+
headers: { 'Content-Type': 'application/json' },
|
| 158 |
+
body: JSON.stringify({
|
| 159 |
+
message: "I'm feeling stressed",
|
| 160 |
+
session_id: session.session_id,
|
| 161 |
+
include_video: true
|
| 162 |
+
})
|
| 163 |
+
});
|
| 164 |
+
|
| 165 |
+
// Get audio/video URLs
|
| 166 |
+
const data = await response.json();
|
| 167 |
+
console.log(data.audio_url, data.video_url);
|
| 168 |
+
```
|
| 169 |
+
|
| 170 |
+
---
|
| 171 |
+
|
| 172 |
+
## ⚡ Performance Specs
|
| 173 |
+
|
| 174 |
+
### 🎯 **System Performance**
|
| 175 |
+
- **Average Response Time**: ~3.2 seconds
|
| 176 |
+
- **GPU Utilization**: RTX 5070 Ti fully supported
|
| 177 |
+
- **Memory Usage**: ~8GB during inference
|
| 178 |
+
- **Audio Quality**: 24kHz, professional grade
|
| 179 |
+
- **Video Resolution**: Configurable (default optimized)
|
| 180 |
+
|
| 181 |
+
### 📊 **Component Status**
|
| 182 |
+
- ✅ **Ollama**: Mistral 7B running
|
| 183 |
+
- ✅ **F5-TTS**: 5.6GB model loaded
|
| 184 |
+
- ✅ **SadTalker**: Avatar system ready
|
| 185 |
+
- ✅ **CUDA**: 12.8 support active
|
| 186 |
+
- ✅ **Web API**: Flask server operational
|
| 187 |
+
|
| 188 |
+
---
|
| 189 |
+
|
| 190 |
+
## 🔧 Configuration Options
|
| 191 |
+
|
| 192 |
+
### 🎚️ **Model Settings** (`mindfull_config.py`)
|
| 193 |
+
```python
|
| 194 |
+
# Emotion keywords and responses
|
| 195 |
+
EMOTION_KEYWORDS = {
|
| 196 |
+
"stress": ["stressed", "overwhelmed", "pressure"],
|
| 197 |
+
"sadness": ["sad", "depressed", "lonely"],
|
| 198 |
+
# ... customizable emotion detection
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
# TTS Settings
|
| 202 |
+
TTS_SETTINGS = {
|
| 203 |
+
"language": "en",
|
| 204 |
+
"speed": 1.0,
|
| 205 |
+
"sample_rate": 24000
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
# SadTalker Settings
|
| 209 |
+
SADTALKER_SETTINGS = {
|
| 210 |
+
"pose_style": 0,
|
| 211 |
+
"exp_scale": 1.0,
|
| 212 |
+
"use_enhancer": True
|
| 213 |
+
}
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
### 🌐 **Web API Settings**
|
| 217 |
+
```python
|
| 218 |
+
WEB_HOST = "0.0.0.0" # Accept all connections
|
| 219 |
+
WEB_PORT = 5000 # Default port
|
| 220 |
+
MAX_TEXT_LENGTH = 500 # Input validation
|
| 221 |
+
SESSION_TIMEOUT = 3600 # 1 hour sessions
|
| 222 |
+
```
|
| 223 |
+
|
| 224 |
+
---
|
| 225 |
+
|
| 226 |
+
## 🛡️ Security Features
|
| 227 |
+
|
| 228 |
+
### 🔒 **Input Validation**
|
| 229 |
+
- Text length limits (1-500 characters)
|
| 230 |
+
- Emotion keyword filtering
|
| 231 |
+
- Session management with timeouts
|
| 232 |
+
- Rate limiting (60 requests/minute)
|
| 233 |
+
|
| 234 |
+
### 🛠️ **Error Handling**
|
| 235 |
+
- Graceful fallbacks for each component
|
| 236 |
+
- Comprehensive logging
|
| 237 |
+
- Unicode-safe output
|
| 238 |
+
- Resource cleanup automation
|
| 239 |
+
|
| 240 |
+
### 📝 **Privacy**
|
| 241 |
+
- No personal data storage beyond session
|
| 242 |
+
- Local processing (no external APIs)
|
| 243 |
+
- Session-based conversation history
|
| 244 |
+
- Automatic file cleanup
|
| 245 |
+
|
| 246 |
+
---
|
| 247 |
+
|
| 248 |
+
## 🚨 Troubleshooting
|
| 249 |
+
|
| 250 |
+
### ❓ **Common Issues**
|
| 251 |
+
|
| 252 |
+
1. **PyTorch CUDA Issues**
|
| 253 |
+
- ✅ **SOLVED**: Using PyTorch 2.9.0.dev with CUDA 12.8
|
| 254 |
+
- Your RTX 5070 Ti is fully supported
|
| 255 |
+
|
| 256 |
+
2. **Audio Generation Fails**
|
| 257 |
+
- ✅ **SOLVED**: All F5-TTS dependencies installed
|
| 258 |
+
- Uses your fine-tuned 5.6GB model
|
| 259 |
+
|
| 260 |
+
3. **Ollama Model Not Found**
|
| 261 |
+
- Run: `ollama create mindfull -f Modelfile_Mindfull`
|
| 262 |
+
- Falls back to `mistral:7b` automatically
|
| 263 |
+
|
| 264 |
+
4. **Web API Not Accessible**
|
| 265 |
+
- Check if port 5000 is free
|
| 266 |
+
- Use `python mindfull_web_api.py` to start
|
| 267 |
+
|
| 268 |
+
---
|
| 269 |
+
|
| 270 |
+
## 🎊 Final Results
|
| 271 |
+
|
| 272 |
+
### 📈 **Test Suite Results**
|
| 273 |
+
```
|
| 274 |
+
🧪 Mindfull AI Avatar Chatbot - Test Suite
|
| 275 |
+
============================================================
|
| 276 |
+
✅ Ollama Connection ✅ F5-TTS Files ✅ SadTalker Files
|
| 277 |
+
✅ Pipeline Init ✅ Text Generation ✅ Audio Generation
|
| 278 |
+
✅ Complete Interaction ✅ Performance ✅ Web API
|
| 279 |
+
|
| 280 |
+
📈 Overall: 9/9 tests passed (100%) 🎉
|
| 281 |
+
```
|
| 282 |
+
|
| 283 |
+
### 🏆 **Achievements**
|
| 284 |
+
- ✅ Full GPU acceleration on RTX 5070 Ti
|
| 285 |
+
- ✅ 5.6GB fine-tuned voice model working
|
| 286 |
+
- ✅ Complete avatar video generation pipeline
|
| 287 |
+
- ✅ Professional web interface
|
| 288 |
+
- ✅ Robust error handling and security
|
| 289 |
+
- ✅ Cultural awareness for police context
|
| 290 |
+
- ✅ Real-time emotional analysis
|
| 291 |
+
- ✅ Comprehensive logging and monitoring
|
| 292 |
+
|
| 293 |
+
---
|
| 294 |
+
|
| 295 |
+
## 🎯 Next Steps & Enhancements
|
| 296 |
+
|
| 297 |
+
### 🔮 **Potential Improvements**
|
| 298 |
+
1. **Multi-language Support**: Add Kannada TTS training
|
| 299 |
+
2. **Mobile App**: React Native version
|
| 300 |
+
3. **Voice Input**: Add speech-to-text capability
|
| 301 |
+
4. **Analytics Dashboard**: Officer wellness tracking
|
| 302 |
+
5. **Integration**: Connect with police databases
|
| 303 |
+
6. **Offline Mode**: Fully disconnected operation
|
| 304 |
+
|
| 305 |
+
### 🛡️ **Production Deployment**
|
| 306 |
+
1. **Docker Containerization**: For easy deployment
|
| 307 |
+
2. **Load Balancing**: Handle multiple officers
|
| 308 |
+
3. **Database Integration**: PostgreSQL/MongoDB
|
| 309 |
+
4. **Monitoring**: Prometheus + Grafana
|
| 310 |
+
5. **SSL/HTTPS**: Secure communications
|
| 311 |
+
6. **Backup System**: Model and data protection
|
| 312 |
+
|
| 313 |
+
---
|
| 314 |
+
|
| 315 |
+
## 👨💻 **Credits & Acknowledgments**
|
| 316 |
+
|
| 317 |
+
**Built by**: Samarth Kadam
|
| 318 |
+
**AI Assistant**: GitHub Copilot
|
| 319 |
+
**LLM**: Mistral 7B via Ollama
|
| 320 |
+
**Voice Tech**: F5-TTS (fine-tuned)
|
| 321 |
+
**Avatar Tech**: SadTalker + Wav2Lip
|
| 322 |
+
**GPU**: NVIDIA RTX 5070 Ti (CUDA 12.8)
|
| 323 |
+
|
| 324 |
+
**Special Thanks**: Bengaluru Police Department (target users)
|
| 325 |
+
|
| 326 |
+
---
|
| 327 |
+
|
| 328 |
+
## 🎉 **CONGRATULATIONS!**
|
| 329 |
+
|
| 330 |
+
Your **Mindfull AI Avatar Chatbot** is now fully operational and ready to support police officers with empathetic, AI-powered mental wellness assistance!
|
| 331 |
+
|
| 332 |
+
**🚀 The system is running at: http://localhost:5000**
|
| 333 |
+
|
| 334 |
+
*"Supporting those who protect and serve the community."*
|
| 335 |
+
|
| 336 |
+
---
|
| 337 |
+
|
| 338 |
+
*Last Updated: August 17, 2025*
|
| 339 |
+
*Version: 1.0.0 - Production Ready*
|
README_SPACE.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Mindfull AI Wellness Companion
|
| 3 |
+
emoji: 🧠
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: slate
|
| 6 |
+
sdk: gradio
|
| 7 |
+
sdk_version: "5.35.0"
|
| 8 |
+
app_file: app.py
|
| 9 |
+
pinned: false
|
| 10 |
+
license: mit
|
| 11 |
+
short_description: Talking-head AI wellness companion. Upload any portrait image.
|
| 12 |
+
tags:
|
| 13 |
+
- mental-health
|
| 14 |
+
- wellness
|
| 15 |
+
- avatar
|
| 16 |
+
- tts
|
| 17 |
+
- talking-head
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
# 🧠 Mindfull – AI Wellness Companion
|
| 21 |
+
|
| 22 |
+
A talking-head AI wellness chatbot. Upload **any portrait image** as the avatar, type your message, and get a spoken video response.
|
| 23 |
+
|
| 24 |
+
## How it works
|
| 25 |
+
|
| 26 |
+
| Step | Component | Notes |
|
| 27 |
+
|------|-----------|-------|
|
| 28 |
+
| 1 – LLM | Ollama (local) or HF Inference API | `mistralai/Mistral-7B-Instruct-v0.3` on Spaces |
|
| 29 |
+
| 2 – TTS | `edge-tts` (Microsoft Neural voices) | No GPU needed |
|
| 30 |
+
| 3 – Video | SadTalker + Wav2Lip | GPU Space recommended; tick *Skip video* for audio-only |
|
| 31 |
+
|
| 32 |
+
## Running locally
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
git clone https://huggingface.co/spaces/<your-username>/mindfull
|
| 36 |
+
cd mindfull
|
| 37 |
+
python -m venv venv && venv\Scripts\activate # Windows
|
| 38 |
+
pip install -r requirements_hf.txt
|
| 39 |
+
# optionally start Ollama: ollama run mindfull
|
| 40 |
+
python app.py
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
## Environment variables (Spaces Secrets)
|
| 44 |
+
|
| 45 |
+
| Variable | Purpose | Required |
|
| 46 |
+
|----------|---------|----------|
|
| 47 |
+
| `HF_TOKEN` | HF Inference API token for LLM | Yes (on Spaces) |
|
| 48 |
+
| `OLLAMA_MODEL` | Override Ollama model name | No |
|
| 49 |
+
|
| 50 |
+
## Notes
|
| 51 |
+
|
| 52 |
+
- Video generation requires SadTalker checkpoints to be present (`sadtalker+wav2lip/sadtalker/checkpoints/`).
|
| 53 |
+
On a CPU-only Space, tick **Skip video** to get text + audio responses instantly.
|
| 54 |
+
- The trained voice is a fixed neural voice (`en-US-GuyNeural`).
|
| 55 |
+
Swapping voices is trivial – change `EDGE_TTS_VOICE` in `simple_audio_gen.py`.
|
Readme.md
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
README.md
|
| 2 |
+
|
| 3 |
+
🛡️ Police Bot Runtime — AI Voice Assistant for Bengaluru Police
|
| 4 |
+
This is the runtime layer of an AI-powered mental wellness assistant built for frontline Bengaluru Police officers. The assistant runs locally and privately using an LLM (via Ollama) and a fine-tuned voice cloning TTS model (via XTTS/F5-TTS), producing empathetic voice replies in real-time based on officer input.
|
| 5 |
+
|
| 6 |
+
💬 Text In → 🤖 LLM Reply → 🗣️ Voice Cloned Output
|
| 7 |
+
|
| 8 |
+
📌 Project Goals
|
| 9 |
+
Create a voice-first wellness chatbot for police personnel
|
| 10 |
+
|
| 11 |
+
Fully private, runs entirely offline on powerful local machines
|
| 12 |
+
|
| 13 |
+
Replies are generated by an open-source LLM (police-bot) via Ollama
|
| 14 |
+
|
| 15 |
+
Responses are spoken aloud using a fine-tuned XTTS voice cloned from a real speaker
|
| 16 |
+
|
| 17 |
+
Easily extendable to integrate into a React-based web interface later
|
| 18 |
+
|
| 19 |
+
Future support for Kannada via multi-lingual XTTS fine-tuning
|
| 20 |
+
|
| 21 |
+
🧠 System Overview
|
| 22 |
+
This repo powers the runtime experience.
|
| 23 |
+
|
| 24 |
+
Ollama runs an LLM (LLaMA 3, Mistral etc.) via the police-bot model
|
| 25 |
+
|
| 26 |
+
Python script police_runtime.py communicates with Ollama (localhost:11434)
|
| 27 |
+
|
| 28 |
+
LLM reply is sent to police_bot_voice.py
|
| 29 |
+
|
| 30 |
+
XTTS reads a voice reference and generates a realistic audio response (output.wav)
|
| 31 |
+
|
| 32 |
+
Audio is played back to the officer
|
| 33 |
+
|
| 34 |
+
Example flow:
|
| 35 |
+
|
| 36 |
+
Officer: I'm feeling low today
|
| 37 |
+
Assistant: [spoken aloud] Namaskara! I’m here to support you. You're a valued member of the force...
|
| 38 |
+
|
| 39 |
+
📂 Folder Structure
|
| 40 |
+
Your folder layout should look like:
|
| 41 |
+
|
| 42 |
+
police-bot-runtime/
|
| 43 |
+
│
|
| 44 |
+
├── police_runtime.py # Main loop: user input → LLM → voice
|
| 45 |
+
├── police_bot_voice.py # Loads XTTS model, speaks response
|
| 46 |
+
│
|
| 47 |
+
├── my_finetuned_model/ # XTTS fine-tuned model files
|
| 48 |
+
│ ├── config.json
|
| 49 |
+
│ ├── dvae.ptj
|
| 50 |
+
│ ├── mel_stats.pth
|
| 51 |
+
│ ├── model.pth
|
| 52 |
+
│ └── vocab.json
|
| 53 |
+
│
|
| 54 |
+
├── datasets-1/
|
| 55 |
+
│ └── wavs/
|
| 56 |
+
│ └── 0029.wav # Reference voice clip used for inference
|
| 57 |
+
│
|
| 58 |
+
├── venv/ # Python virtual environment
|
| 59 |
+
└── requirements.txt # (Optional) Dependency list
|
| 60 |
+
|
| 61 |
+
⚙️ Setup Instructions
|
| 62 |
+
🧩 Prerequisites:
|
| 63 |
+
|
| 64 |
+
Windows 10/11, 64-bit
|
| 65 |
+
|
| 66 |
+
Python 3.11 (recommended)
|
| 67 |
+
|
| 68 |
+
Ollama installed: https://ollama.com
|
| 69 |
+
|
| 70 |
+
Trained XTTS voice model (via F5-TTS or Coqui)
|
| 71 |
+
|
| 72 |
+
Clone or copy this folder as police-bot-runtime
|
| 73 |
+
|
| 74 |
+
Create a virtual environment:
|
| 75 |
+
|
| 76 |
+
bash
|
| 77 |
+
Copy
|
| 78 |
+
Edit
|
| 79 |
+
python -m venv venv
|
| 80 |
+
venv\Scripts\activate
|
| 81 |
+
Install dependencies:
|
| 82 |
+
|
| 83 |
+
bash
|
| 84 |
+
Copy
|
| 85 |
+
Edit
|
| 86 |
+
pip install TTS requests
|
| 87 |
+
Start the Ollama LLM (separate terminal):
|
| 88 |
+
|
| 89 |
+
bash
|
| 90 |
+
Copy
|
| 91 |
+
Edit
|
| 92 |
+
ollama run police-bot
|
| 93 |
+
Run the voice assistant:
|
| 94 |
+
|
| 95 |
+
bash
|
| 96 |
+
Copy
|
| 97 |
+
Edit
|
| 98 |
+
python police_runtime.py
|
| 99 |
+
You'll be prompted for input. Replies will be printed and voiced out.
|
| 100 |
+
|
| 101 |
+
🧪 Testing Voice Model
|
| 102 |
+
Your test_voice.py should look like:
|
| 103 |
+
|
| 104 |
+
python
|
| 105 |
+
Copy
|
| 106 |
+
Edit
|
| 107 |
+
from TTS.api import TTS
|
| 108 |
+
|
| 109 |
+
tts = TTS(
|
| 110 |
+
model_path="my_finetuned_model/",
|
| 111 |
+
config_path="my_finetuned_model/config.json",
|
| 112 |
+
gpu=True
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
tts.tts_to_file(
|
| 116 |
+
text="Hey officer, everything okay?",
|
| 117 |
+
file_path="output.wav",
|
| 118 |
+
speaker_wav="datasets-1/wavs/0029.wav",
|
| 119 |
+
language="en"
|
| 120 |
+
)
|
| 121 |
+
🧠 How It Works
|
| 122 |
+
Ollama LLM (LLaMA3) is launched with ollama run police-bot
|
| 123 |
+
|
| 124 |
+
Text prompt is sent to http://localhost:11434/api/generate
|
| 125 |
+
|
| 126 |
+
Response is sent to XTTS model for synthesis
|
| 127 |
+
|
| 128 |
+
XTTS clones voice using speaker_wav file
|
| 129 |
+
|
| 130 |
+
Audio saved to output.wav and played using playsound or equivalent
|
| 131 |
+
|
| 132 |
+
🛠️ Future Enhancements
|
| 133 |
+
Integrate directly with the frontend (React + ShadCN + Web Speech API)
|
| 134 |
+
|
| 135 |
+
Support both English and Kannada voice cloning
|
| 136 |
+
|
| 137 |
+
Add emotional sentiment analysis (LLaMA + classifier)
|
| 138 |
+
|
| 139 |
+
Handle stress journaling and daily wellness logging
|
| 140 |
+
|
| 141 |
+
Auto-log sessions to Supabase or local DB
|
| 142 |
+
|
| 143 |
+
Deploy offline on a rugged laptop for field use
|
| 144 |
+
|
| 145 |
+
🙏 Acknowledgements
|
| 146 |
+
Voice cloning: F5-TTS / Coqui XTTS
|
| 147 |
+
|
| 148 |
+
LLM inference: Ollama + LLaMA3 8B
|
| 149 |
+
|
| 150 |
+
Frontend inspiration: GovSchemes project
|
| 151 |
+
|
| 152 |
+
This is a labor of respect and service. Designed to support those who protect us.
|
| 153 |
+
|
| 154 |
+
—
|
analyze_codebase.py
ADDED
|
@@ -0,0 +1,1024 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Codebase Call Graph Analyzer
|
| 3 |
+
Traces function calls, assets (model checkpoints, audio, images, configs),
|
| 4 |
+
URLs, environment variables, and subprocess commands across all Python files.
|
| 5 |
+
Usage: python analyze_codebase.py
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import ast
|
| 9 |
+
import os
|
| 10 |
+
import re
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from collections import defaultdict
|
| 14 |
+
from typing import Dict, Set, List, Tuple, Optional
|
| 15 |
+
|
| 16 |
+
# ─── Asset extension groups ───────────────────────────────────────────────────
|
| 17 |
+
ASSET_EXTS = {
|
| 18 |
+
"model": {".pth", ".pt", ".ckpt", ".safetensors", ".bin", ".onnx", ".mar"},
|
| 19 |
+
"audio": {".wav", ".mp3", ".flac", ".ogg", ".m4a"},
|
| 20 |
+
"video": {".mp4", ".avi", ".mov", ".mkv"},
|
| 21 |
+
"image": {".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp"},
|
| 22 |
+
"config": {".json", ".yaml", ".yml", ".cfg", ".ini", ".toml", ".txt"},
|
| 23 |
+
"data": {".csv", ".npy", ".npz", ".pkl", ".h5", ".hdf5"},
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
# ─── Configuration ────────────────────────────────────────────────────────────
|
| 27 |
+
|
| 28 |
+
ROOT = Path(__file__).parent
|
| 29 |
+
|
| 30 |
+
# Directories to skip entirely
|
| 31 |
+
SKIP_DIRS = {"venv", "__pycache__", ".git", "node_modules", "gfpgan"}
|
| 32 |
+
|
| 33 |
+
# Starting entry points (traced in order)
|
| 34 |
+
ENTRY_POINTS = [
|
| 35 |
+
"mindfull_web_api.py",
|
| 36 |
+
"mindfull_pipeline.py",
|
| 37 |
+
"mindfull_config.py",
|
| 38 |
+
"simple_video_gen.py",
|
| 39 |
+
"simple_audio_gen.py",
|
| 40 |
+
"sadtalker+wav2lip/enhanced_pipeline.py",
|
| 41 |
+
"sadtalker+wav2lip/simple_pipeline.py",
|
| 42 |
+
"sadtalker+wav2lip/wav2lip/inference.py",
|
| 43 |
+
"sadtalker+wav2lip/sadtalker/inference.py",
|
| 44 |
+
"python313_compat_patch.py",
|
| 45 |
+
"model_manager.py",
|
| 46 |
+
"ollama_to_f5tts.py",
|
| 47 |
+
"infer_cli.py",
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
# ─── Data Structures ──────────────────────────────────────────────────────────
|
| 51 |
+
|
| 52 |
+
class FunctionInfo:
|
| 53 |
+
def __init__(self, name: str, file: str, lineno: int, is_method: bool = False, class_name: str = None):
|
| 54 |
+
self.name = name
|
| 55 |
+
self.file = file
|
| 56 |
+
self.lineno = lineno
|
| 57 |
+
self.is_method = is_method
|
| 58 |
+
self.class_name = class_name
|
| 59 |
+
self.calls: List[str] = [] # raw call names seen in body
|
| 60 |
+
self.docstring: Optional[str] = None
|
| 61 |
+
|
| 62 |
+
def full_name(self):
|
| 63 |
+
if self.class_name:
|
| 64 |
+
return f"{self.class_name}.{self.name}"
|
| 65 |
+
return self.name
|
| 66 |
+
|
| 67 |
+
def __repr__(self):
|
| 68 |
+
return f"<Func {self.full_name()} @ {self.file}:{self.lineno}>"
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class AssetRef:
|
| 72 |
+
"""A non-Python file referenced as a string literal in source code."""
|
| 73 |
+
def __init__(self, path_str: str, asset_type: str, lineno: int, in_func: Optional[str]):
|
| 74 |
+
self.path_str = path_str # raw string from source
|
| 75 |
+
self.asset_type = asset_type # model / audio / image / config / …
|
| 76 |
+
self.lineno = lineno
|
| 77 |
+
self.in_func = in_func # function name, or None for module-level
|
| 78 |
+
self.exists = Path(path_str).exists() or Path(ROOT / path_str).exists()
|
| 79 |
+
|
| 80 |
+
def __repr__(self):
|
| 81 |
+
return f"<Asset {self.asset_type}:{self.path_str}>"
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class FileInfo:
|
| 85 |
+
def __init__(self, path: str):
|
| 86 |
+
self.path = path # relative path from ROOT
|
| 87 |
+
self.imports: Dict[str, str] = {} # alias -> module string
|
| 88 |
+
self.from_imports: Dict[str, Tuple[str, str]] = {} # name -> (module, original_name)
|
| 89 |
+
self.functions: List[FunctionInfo] = []
|
| 90 |
+
self.classes: List[str] = []
|
| 91 |
+
self.top_level_calls: List[str] = [] # calls outside any function
|
| 92 |
+
self.third_party: Set[str] = set() # modules clearly not local
|
| 93 |
+
self.assets: List[AssetRef] = [] # file-path string literals
|
| 94 |
+
self.urls: List[Tuple[int, str, Optional[str]]] = [] # (line, url, func)
|
| 95 |
+
self.env_vars: List[Tuple[int, str, Optional[str]]] = [] # (line, var, func)
|
| 96 |
+
self.subproc_cmds: List[Tuple[int, str, Optional[str]]] = [] # (line, cmd, func)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
| 100 |
+
|
| 101 |
+
def collect_py_files(root: Path) -> List[Path]:
|
| 102 |
+
files = []
|
| 103 |
+
for p in root.rglob("*.py"):
|
| 104 |
+
if any(skip in p.parts for skip in SKIP_DIRS):
|
| 105 |
+
continue
|
| 106 |
+
files.append(p)
|
| 107 |
+
return sorted(files)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def rel(path: Path) -> str:
|
| 111 |
+
try:
|
| 112 |
+
return str(path.relative_to(ROOT)).replace("\\", "/")
|
| 113 |
+
except ValueError:
|
| 114 |
+
return str(path).replace("\\", "/")
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def module_to_file(module: str, from_file: Path) -> Optional[str]:
|
| 118 |
+
"""Try to resolve a module name to a file in the project."""
|
| 119 |
+
parts = module.replace(".", "/")
|
| 120 |
+
|
| 121 |
+
# Relative to the file's own directory
|
| 122 |
+
candidates = [
|
| 123 |
+
from_file.parent / (parts + ".py"),
|
| 124 |
+
from_file.parent / parts / "__init__.py",
|
| 125 |
+
ROOT / (parts + ".py"),
|
| 126 |
+
ROOT / parts / "__init__.py",
|
| 127 |
+
]
|
| 128 |
+
for c in candidates:
|
| 129 |
+
if c.exists():
|
| 130 |
+
return rel(c)
|
| 131 |
+
return None
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def get_call_name(node) -> Optional[str]:
|
| 135 |
+
"""Extract a simple string name from a Call node."""
|
| 136 |
+
if isinstance(node.func, ast.Name):
|
| 137 |
+
return node.func.id
|
| 138 |
+
if isinstance(node.func, ast.Attribute):
|
| 139 |
+
# e.g. self.foo() -> foo, obj.method() -> method
|
| 140 |
+
return node.func.attr
|
| 141 |
+
return None
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
# ─── AST Parser ───────────────────────────────────────────────────────────────
|
| 145 |
+
|
| 146 |
+
def parse_file(path: Path) -> Optional[FileInfo]:
|
| 147 |
+
info = FileInfo(rel(path))
|
| 148 |
+
try:
|
| 149 |
+
src = path.read_text(encoding="utf-8", errors="ignore")
|
| 150 |
+
tree = ast.parse(src, filename=str(path))
|
| 151 |
+
except SyntaxError as e:
|
| 152 |
+
print(f" [SKIP] Syntax error in {rel(path)}: {e}")
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
# ── Collect imports ──────────────────────────────────────────
|
| 156 |
+
|
| 157 |
+
for node in ast.walk(tree):
|
| 158 |
+
if isinstance(node, ast.Import):
|
| 159 |
+
for alias in node.names:
|
| 160 |
+
name = alias.asname or alias.name
|
| 161 |
+
info.imports[name] = alias.name
|
| 162 |
+
elif isinstance(node, ast.ImportFrom):
|
| 163 |
+
module = node.module or ""
|
| 164 |
+
for alias in node.names:
|
| 165 |
+
imported_name = alias.asname or alias.name
|
| 166 |
+
info.from_imports[imported_name] = (module, alias.name)
|
| 167 |
+
|
| 168 |
+
# ── Collect functions and their calls ────────────────────────
|
| 169 |
+
|
| 170 |
+
current_class = [None]
|
| 171 |
+
|
| 172 |
+
class Visitor(ast.NodeVisitor):
|
| 173 |
+
def __init__(self):
|
| 174 |
+
self._func_stack: List[FunctionInfo] = []
|
| 175 |
+
|
| 176 |
+
def visit_ClassDef(self, node):
|
| 177 |
+
prev = current_class[0]
|
| 178 |
+
current_class[0] = node.name
|
| 179 |
+
info.classes.append(node.name)
|
| 180 |
+
self.generic_visit(node)
|
| 181 |
+
current_class[0] = prev
|
| 182 |
+
|
| 183 |
+
def visit_FunctionDef(self, node):
|
| 184 |
+
self._visit_func(node)
|
| 185 |
+
|
| 186 |
+
def visit_AsyncFunctionDef(self, node):
|
| 187 |
+
self._visit_func(node)
|
| 188 |
+
|
| 189 |
+
def _visit_func(self, node):
|
| 190 |
+
fi = FunctionInfo(
|
| 191 |
+
name=node.name,
|
| 192 |
+
file=info.path,
|
| 193 |
+
lineno=node.lineno,
|
| 194 |
+
is_method=(current_class[0] is not None),
|
| 195 |
+
class_name=current_class[0],
|
| 196 |
+
)
|
| 197 |
+
# Docstring
|
| 198 |
+
if (node.body and isinstance(node.body[0], ast.Expr)
|
| 199 |
+
and isinstance(node.body[0].value, ast.Constant)
|
| 200 |
+
and isinstance(node.body[0].value.value, str)):
|
| 201 |
+
fi.docstring = node.body[0].value.value
|
| 202 |
+
|
| 203 |
+
# Calls inside this function's body
|
| 204 |
+
for child in ast.walk(node):
|
| 205 |
+
if isinstance(child, ast.Call):
|
| 206 |
+
cname = get_call_name(child)
|
| 207 |
+
if cname and cname not in fi.calls:
|
| 208 |
+
fi.calls.append(cname)
|
| 209 |
+
|
| 210 |
+
self._func_stack.append(fi)
|
| 211 |
+
info.functions.append(fi)
|
| 212 |
+
self.generic_visit(node)
|
| 213 |
+
self._func_stack.pop()
|
| 214 |
+
|
| 215 |
+
Visitor().visit(tree)
|
| 216 |
+
|
| 217 |
+
# Top-level calls (outside functions/classes)
|
| 218 |
+
for node in tree.body:
|
| 219 |
+
if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
|
| 220 |
+
cname = get_call_name(node.value)
|
| 221 |
+
if cname:
|
| 222 |
+
info.top_level_calls.append(cname)
|
| 223 |
+
|
| 224 |
+
# ── Asset / URL / ENV / subprocess scan ─────────────────────
|
| 225 |
+
_extract_assets(tree, info)
|
| 226 |
+
|
| 227 |
+
return info
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _current_func_at_line(info: FileInfo, lineno: int) -> Optional[str]:
|
| 231 |
+
"""Return the innermost function name that contains `lineno`."""
|
| 232 |
+
# Functions are ordered by source; find last one whose lineno <= target
|
| 233 |
+
best = None
|
| 234 |
+
for fn in info.functions:
|
| 235 |
+
if fn.lineno <= lineno:
|
| 236 |
+
best = fn.full_name()
|
| 237 |
+
return best
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
_URL_RE = re.compile(r'https?://[^\s\'"\\>]+')
|
| 241 |
+
_ENV_GETENV = {"getenv", "environ"}
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
_EXEC_NAMES = {"python", "python3", "python.exe", "ffmpeg", "ffmpeg.exe",
|
| 245 |
+
"ffprobe", "git", "pip", "conda", "bash", "sh", "cmd", "powershell"}
|
| 246 |
+
|
| 247 |
+
def _extract_assets(tree: ast.AST, info: FileInfo) -> None:
|
| 248 |
+
"""Walk the AST and pull out asset paths, URLs, env vars, subprocess calls."""
|
| 249 |
+
|
| 250 |
+
# ── Pass 1: capture cmd-variable assignments (cmd = ["ffmpeg", ...]) ──
|
| 251 |
+
# Maps variable name -> (lineno, first-token string)
|
| 252 |
+
_cmd_vars: Dict[str, Tuple[int, str]] = {}
|
| 253 |
+
for node in ast.walk(tree):
|
| 254 |
+
if not isinstance(node, ast.Assign):
|
| 255 |
+
continue
|
| 256 |
+
val = node.value
|
| 257 |
+
if not (isinstance(val, ast.List) and val.elts):
|
| 258 |
+
continue
|
| 259 |
+
first_elt = val.elts[0]
|
| 260 |
+
first_str: Optional[str] = None
|
| 261 |
+
if isinstance(first_elt, ast.Constant) and isinstance(first_elt.value, str):
|
| 262 |
+
first_str = first_elt.value
|
| 263 |
+
elif isinstance(first_elt, ast.Attribute):
|
| 264 |
+
# sys.executable → keep as placeholder
|
| 265 |
+
first_str = "<python>"
|
| 266 |
+
if first_str is None:
|
| 267 |
+
continue
|
| 268 |
+
lower = first_str.lower().replace("\\\\", "/")
|
| 269 |
+
base = lower.split("/")[-1].split(".")[0]
|
| 270 |
+
if base in _EXEC_NAMES or lower == "<python>":
|
| 271 |
+
# collect all string parts for a readable label
|
| 272 |
+
parts = []
|
| 273 |
+
for el in val.elts[:8]:
|
| 274 |
+
if isinstance(el, ast.Constant) and isinstance(el.value, str):
|
| 275 |
+
parts.append(el.value)
|
| 276 |
+
elif isinstance(el, ast.Attribute) and el.attr == "executable":
|
| 277 |
+
parts.append("<python>")
|
| 278 |
+
else:
|
| 279 |
+
parts.append("...")
|
| 280 |
+
cmd_preview = " ".join(parts)[:120]
|
| 281 |
+
for tgt in node.targets:
|
| 282 |
+
if isinstance(tgt, ast.Name):
|
| 283 |
+
_cmd_vars[tgt.id] = (node.lineno, cmd_preview)
|
| 284 |
+
|
| 285 |
+
for node in ast.walk(tree):
|
| 286 |
+
lineno = getattr(node, "lineno", 0)
|
| 287 |
+
func_ctx = None # resolved lazily below
|
| 288 |
+
|
| 289 |
+
# ── String literals ─────────────────────────────────────
|
| 290 |
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
| 291 |
+
s = node.value.strip()
|
| 292 |
+
|
| 293 |
+
# File-path heuristic: contains a known extension
|
| 294 |
+
suffix = Path(s).suffix.lower()
|
| 295 |
+
for atype, exts in ASSET_EXTS.items():
|
| 296 |
+
if suffix in exts and len(s) > 3:
|
| 297 |
+
if func_ctx is None:
|
| 298 |
+
func_ctx = _current_func_at_line(info, lineno)
|
| 299 |
+
info.assets.append(AssetRef(s, atype, lineno, func_ctx))
|
| 300 |
+
break
|
| 301 |
+
|
| 302 |
+
# URL
|
| 303 |
+
if _URL_RE.match(s):
|
| 304 |
+
if func_ctx is None:
|
| 305 |
+
func_ctx = _current_func_at_line(info, lineno)
|
| 306 |
+
info.urls.append((lineno, s, func_ctx))
|
| 307 |
+
|
| 308 |
+
# ── os.environ / os.getenv ───────────────────────────────
|
| 309 |
+
elif isinstance(node, ast.Call):
|
| 310 |
+
fname = get_call_name(node)
|
| 311 |
+
if fname in {"getenv"}:
|
| 312 |
+
if node.args and isinstance(node.args[0], ast.Constant):
|
| 313 |
+
if func_ctx is None:
|
| 314 |
+
func_ctx = _current_func_at_line(info, lineno)
|
| 315 |
+
info.env_vars.append((lineno, node.args[0].value, func_ctx))
|
| 316 |
+
|
| 317 |
+
# subprocess.run / call / Popen with a plain string command
|
| 318 |
+
if fname in {"run", "call", "Popen", "check_output", "check_call", "system"}:
|
| 319 |
+
if node.args:
|
| 320 |
+
first = node.args[0]
|
| 321 |
+
cmd_str = None
|
| 322 |
+
if isinstance(first, ast.Constant) and isinstance(first.value, str):
|
| 323 |
+
cmd_str = first.value[:120]
|
| 324 |
+
elif isinstance(first, ast.List) and first.elts:
|
| 325 |
+
parts = []
|
| 326 |
+
for el in first.elts[:6]:
|
| 327 |
+
if isinstance(el, ast.Constant) and isinstance(el.value, str):
|
| 328 |
+
parts.append(el.value)
|
| 329 |
+
elif isinstance(el, ast.Attribute) and el.attr == "executable":
|
| 330 |
+
parts.append("<python>")
|
| 331 |
+
else:
|
| 332 |
+
parts.append("...")
|
| 333 |
+
if parts:
|
| 334 |
+
cmd_str = " ".join(parts)[:120]
|
| 335 |
+
elif isinstance(first, ast.Name):
|
| 336 |
+
# variable — look up in our cmd-variable map
|
| 337 |
+
if first.id in _cmd_vars:
|
| 338 |
+
cmd_str = _cmd_vars[first.id][1]
|
| 339 |
+
if cmd_str:
|
| 340 |
+
if func_ctx is None:
|
| 341 |
+
func_ctx = _current_func_at_line(info, lineno)
|
| 342 |
+
info.subproc_cmds.append((lineno, cmd_str, func_ctx))
|
| 343 |
+
|
| 344 |
+
# ── os.environ["VAR"] subscript ──────────────────────────
|
| 345 |
+
elif isinstance(node, ast.Subscript):
|
| 346 |
+
if (isinstance(node.value, ast.Attribute)
|
| 347 |
+
and node.value.attr == "environ"
|
| 348 |
+
and isinstance(node.slice, ast.Constant)
|
| 349 |
+
and isinstance(node.slice.value, str)):
|
| 350 |
+
if func_ctx is None:
|
| 351 |
+
func_ctx = _current_func_at_line(info, lineno)
|
| 352 |
+
info.env_vars.append((lineno, node.slice.value, func_ctx))
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
# ─── Resolver ─────────────────────────────────────────────────────────────────
|
| 356 |
+
|
| 357 |
+
STDLIB_ROOTS = {
|
| 358 |
+
"os", "sys", "re", "json", "time", "subprocess", "pathlib", "shutil",
|
| 359 |
+
"tempfile", "typing", "collections", "functools", "itertools", "logging",
|
| 360 |
+
"argparse", "hashlib", "math", "random", "copy", "io", "abc", "enum",
|
| 361 |
+
"threading", "asyncio", "dataclasses", "datetime", "inspect", "gc",
|
| 362 |
+
"traceback", "warnings", "platform", "struct", "socket", "http",
|
| 363 |
+
"urllib", "uuid", "base64", "csv", "glob", "fnmatch",
|
| 364 |
+
}
|
| 365 |
+
|
| 366 |
+
THIRD_PARTY_ROOTS = {
|
| 367 |
+
"torch", "cv2", "numpy", "np", "PIL", "flask", "fastapi", "requests",
|
| 368 |
+
"scipy", "librosa", "soundfile", "sf", "tqdm", "yaml", "omegaconf",
|
| 369 |
+
"imageio", "skimage", "einops", "transformers", "diffusers", "gradio",
|
| 370 |
+
"pydantic", "uvicorn", "starlette", "aiofiles", "f5_tts", "cached_path",
|
| 371 |
+
"vocos", "safetensors",
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
def is_local(module: str) -> bool:
|
| 375 |
+
root = module.split(".")[0]
|
| 376 |
+
return root not in STDLIB_ROOTS and root not in THIRD_PARTY_ROOTS
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def resolve_calls(all_files: Dict[str, FileInfo]) -> Dict[str, List[Tuple[str, str]]]:
|
| 380 |
+
"""
|
| 381 |
+
Returns call_graph[file_rel][func_name] = [(target_file, target_func), ...]
|
| 382 |
+
as a flat dict: key = "file::func", value = list of "file::func"
|
| 383 |
+
"""
|
| 384 |
+
# Build lookup: function_name -> [(file, FunctionInfo)]
|
| 385 |
+
name_index: Dict[str, List[Tuple[str, FunctionInfo]]] = defaultdict(list)
|
| 386 |
+
for frel, finfo in all_files.items():
|
| 387 |
+
for fn in finfo.functions:
|
| 388 |
+
name_index[fn.name].append((frel, fn))
|
| 389 |
+
|
| 390 |
+
call_graph: Dict[str, List[str]] = defaultdict(list)
|
| 391 |
+
|
| 392 |
+
for frel, finfo in all_files.items():
|
| 393 |
+
file_path = ROOT / frel.replace("/", os.sep)
|
| 394 |
+
|
| 395 |
+
for fn in finfo.functions:
|
| 396 |
+
key = f"{frel}::{fn.full_name()}"
|
| 397 |
+
for raw_call in fn.calls:
|
| 398 |
+
# Check from_imports first
|
| 399 |
+
if raw_call in finfo.from_imports:
|
| 400 |
+
mod, orig = finfo.from_imports[raw_call]
|
| 401 |
+
target_file = module_to_file(mod, file_path)
|
| 402 |
+
if target_file and target_file in all_files:
|
| 403 |
+
call_graph[key].append(f"{target_file}::{orig}")
|
| 404 |
+
continue
|
| 405 |
+
elif not is_local(mod):
|
| 406 |
+
call_graph[key].append(f"[{mod}]::{orig}")
|
| 407 |
+
continue
|
| 408 |
+
|
| 409 |
+
# Check regular imports (module.func pattern won't hit here, but alias might)
|
| 410 |
+
if raw_call in finfo.imports:
|
| 411 |
+
mod = finfo.imports[raw_call]
|
| 412 |
+
target_file = module_to_file(mod, file_path)
|
| 413 |
+
if target_file:
|
| 414 |
+
call_graph[key].append(f"{target_file}::<module>")
|
| 415 |
+
continue
|
| 416 |
+
|
| 417 |
+
# Global function name match across project files
|
| 418 |
+
if raw_call in name_index:
|
| 419 |
+
for tfile, tfn in name_index[raw_call]:
|
| 420 |
+
entry = f"{tfile}::{tfn.full_name()}"
|
| 421 |
+
if entry not in call_graph[key]:
|
| 422 |
+
call_graph[key].append(entry)
|
| 423 |
+
continue
|
| 424 |
+
|
| 425 |
+
# Unknown — could be builtin, stdlib, or third-party
|
| 426 |
+
root_mod = raw_call.split(".")[0]
|
| 427 |
+
if root_mod in THIRD_PARTY_ROOTS:
|
| 428 |
+
call_graph[key].append(f"[lib:{root_mod}]::{raw_call}")
|
| 429 |
+
|
| 430 |
+
return call_graph
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
# ─── README Generator ─────────────────────────────────────────────────────────
|
| 434 |
+
|
| 435 |
+
def generate_readme(
|
| 436 |
+
all_files: Dict[str, FileInfo],
|
| 437 |
+
call_graph: Dict[str, List[str]],
|
| 438 |
+
entry_points: List[str],
|
| 439 |
+
) -> str:
|
| 440 |
+
lines = []
|
| 441 |
+
|
| 442 |
+
lines += [
|
| 443 |
+
"# Codebase Call Graph & Function Map",
|
| 444 |
+
"",
|
| 445 |
+
"Auto-generated by `analyze_codebase.py`.",
|
| 446 |
+
"",
|
| 447 |
+
"**Legend**",
|
| 448 |
+
"- `→` = calls",
|
| 449 |
+
"- `[lib:X]` = third-party library",
|
| 450 |
+
"- `[stdlib:X]` = Python standard library",
|
| 451 |
+
"- Line numbers link to source definitions",
|
| 452 |
+
"",
|
| 453 |
+
"---",
|
| 454 |
+
"",
|
| 455 |
+
]
|
| 456 |
+
|
| 457 |
+
# ── Section 1: File summary ──────────────────────────────────
|
| 458 |
+
lines += ["## 1. Files in Project", ""]
|
| 459 |
+
lines += ["| File | Classes | Functions | Imports |",
|
| 460 |
+
"|------|---------|-----------|---------|"]
|
| 461 |
+
for frel in sorted(all_files):
|
| 462 |
+
fi = all_files[frel]
|
| 463 |
+
classes = ", ".join(fi.classes) if fi.classes else "—"
|
| 464 |
+
func_count = len(fi.functions)
|
| 465 |
+
imp_count = len(fi.imports) + len(fi.from_imports)
|
| 466 |
+
lines.append(f"| `{frel}` | {classes} | {func_count} | {imp_count} |")
|
| 467 |
+
lines += ["", "---", ""]
|
| 468 |
+
|
| 469 |
+
# ── Section 2: Entry point trace ────────────────────────────
|
| 470 |
+
lines += ["## 2. Entry Points", ""]
|
| 471 |
+
for ep in entry_points:
|
| 472 |
+
ep_norm = ep.replace("\\", "/")
|
| 473 |
+
if ep_norm in all_files:
|
| 474 |
+
lines.append(f"- **`{ep_norm}`** ✓")
|
| 475 |
+
else:
|
| 476 |
+
lines.append(f"- ~~`{ep_norm}`~~ *(not found)*")
|
| 477 |
+
lines += ["", "---", ""]
|
| 478 |
+
|
| 479 |
+
# ── Section 3: Per-file function breakdown ───────────────────
|
| 480 |
+
lines += ["## 3. Function Map (`filename → function → calls`)", ""]
|
| 481 |
+
|
| 482 |
+
for frel in sorted(all_files):
|
| 483 |
+
fi = all_files[frel]
|
| 484 |
+
if not fi.functions:
|
| 485 |
+
continue
|
| 486 |
+
|
| 487 |
+
lines += [f"### `{frel}`", ""]
|
| 488 |
+
|
| 489 |
+
# Imports summary for this file
|
| 490 |
+
if fi.from_imports or fi.imports:
|
| 491 |
+
lines.append("**Imports:**")
|
| 492 |
+
seen_mods: Set[str] = set()
|
| 493 |
+
for name, (mod, orig) in fi.from_imports.items():
|
| 494 |
+
if mod not in seen_mods:
|
| 495 |
+
seen_mods.add(mod)
|
| 496 |
+
resolved = module_to_file(mod, ROOT / frel.replace("/", os.sep))
|
| 497 |
+
tag = f"→ `{resolved}`" if resolved else ("*(stdlib)*" if not is_local(mod) else "*(external)*")
|
| 498 |
+
lines.append(f"- `from {mod} import ...` {tag}")
|
| 499 |
+
for alias, mod in fi.imports.items():
|
| 500 |
+
if mod not in seen_mods:
|
| 501 |
+
seen_mods.add(mod)
|
| 502 |
+
resolved = module_to_file(mod, ROOT / frel.replace("/", os.sep))
|
| 503 |
+
tag = f"→ `{resolved}`" if resolved else ("*(stdlib)*" if not is_local(mod) else "*(external)*")
|
| 504 |
+
lines.append(f"- `import {mod}` {tag}")
|
| 505 |
+
lines.append("")
|
| 506 |
+
|
| 507 |
+
# Functions
|
| 508 |
+
lines.append("**Functions:**")
|
| 509 |
+
lines.append("")
|
| 510 |
+
lines.append("| # | Function | Line | Calls |")
|
| 511 |
+
lines.append("|---|----------|------|-------|")
|
| 512 |
+
|
| 513 |
+
for i, fn in enumerate(fi.functions, 1):
|
| 514 |
+
key = f"{frel}::{fn.full_name()}"
|
| 515 |
+
raw_calls = fn.calls
|
| 516 |
+
resolved_calls = call_graph.get(key, [])
|
| 517 |
+
|
| 518 |
+
# Build call cell
|
| 519 |
+
if not raw_calls:
|
| 520 |
+
call_cell = "*(none)*"
|
| 521 |
+
else:
|
| 522 |
+
# Show resolved targets when available, fallback to raw name
|
| 523 |
+
resolved_map = {}
|
| 524 |
+
for rc in resolved_calls:
|
| 525 |
+
parts = rc.split("::")
|
| 526 |
+
func_part = parts[-1]
|
| 527 |
+
resolved_map[func_part] = rc
|
| 528 |
+
|
| 529 |
+
call_parts = []
|
| 530 |
+
for c in raw_calls:
|
| 531 |
+
if c in resolved_map:
|
| 532 |
+
target = resolved_map[c]
|
| 533 |
+
if target.startswith("["):
|
| 534 |
+
call_parts.append(f"`{target}`")
|
| 535 |
+
else:
|
| 536 |
+
tfile, tfunc = target.rsplit("::", 1)
|
| 537 |
+
call_parts.append(f"`{tfunc}` *({tfile})*")
|
| 538 |
+
else:
|
| 539 |
+
call_parts.append(f"`{c}`")
|
| 540 |
+
|
| 541 |
+
call_cell = ", ".join(call_parts[:8])
|
| 542 |
+
if len(call_parts) > 8:
|
| 543 |
+
call_cell += f" *+{len(call_parts)-8} more*"
|
| 544 |
+
|
| 545 |
+
prefix = f"{fn.class_name}." if fn.class_name else ""
|
| 546 |
+
lines.append(f"| {i} | `{prefix}{fn.name}` | L{fn.lineno} | {call_cell} |")
|
| 547 |
+
|
| 548 |
+
lines += ["", ""]
|
| 549 |
+
|
| 550 |
+
# ── Section 4: Full call graph (deduplicated) ────────────────
|
| 551 |
+
lines += ["---", "", "## 4. Full Call Graph (all edges)", ""]
|
| 552 |
+
lines += ["```"]
|
| 553 |
+
for key in sorted(call_graph):
|
| 554 |
+
frel, fname = key.split("::", 1)
|
| 555 |
+
short_f = frel.split("/")[-1]
|
| 556 |
+
targets = call_graph[key]
|
| 557 |
+
if targets:
|
| 558 |
+
for t in targets:
|
| 559 |
+
if "::" in t:
|
| 560 |
+
tfile, tfunc = t.rsplit("::", 1)
|
| 561 |
+
tshort = tfile.split("/")[-1] if not tfile.startswith("[") else tfile
|
| 562 |
+
lines.append(f"{short_f}::{fname} → {tshort}::{tfunc}")
|
| 563 |
+
lines += ["```", ""]
|
| 564 |
+
|
| 565 |
+
# ── Section 5: Assets, URLs, Env vars, Subprocess ──────────
|
| 566 |
+
lines += ["---", "", "## 5. Non-Python Assets Referenced", ""]
|
| 567 |
+
lines += ["> All string literals that look like file paths, grouped by type.", ""]
|
| 568 |
+
|
| 569 |
+
# Collect all assets across files
|
| 570 |
+
asset_rows: Dict[str, List[Tuple[str, int, str, bool]]] = defaultdict(list)
|
| 571 |
+
for frel, fi in all_files.items():
|
| 572 |
+
for a in fi.assets:
|
| 573 |
+
asset_rows[a.asset_type].append((frel, a.lineno, a.path_str, a.exists))
|
| 574 |
+
|
| 575 |
+
for atype in ["model", "audio", "video", "image", "config", "data"]:
|
| 576 |
+
rows = asset_rows.get(atype, [])
|
| 577 |
+
if not rows:
|
| 578 |
+
continue
|
| 579 |
+
lines.append(f"### {atype.capitalize()} Files")
|
| 580 |
+
lines.append("")
|
| 581 |
+
lines.append("| File | Line | Path | On Disk |")
|
| 582 |
+
lines.append("|------|------|------|---------|")
|
| 583 |
+
seen = set()
|
| 584 |
+
for frel, ln, path_str, exists in sorted(rows, key=lambda x: (x[0], x[1])):
|
| 585 |
+
key = (frel, path_str)
|
| 586 |
+
if key in seen:
|
| 587 |
+
continue
|
| 588 |
+
seen.add(key)
|
| 589 |
+
short_f = frel.split("/")[-1]
|
| 590 |
+
disk = "✓" if exists else "✗ missing"
|
| 591 |
+
safe = path_str.replace("|", "|")[:80]
|
| 592 |
+
lines.append(f"| `{short_f}` | L{ln} | `{safe}` | {disk} |")
|
| 593 |
+
lines.append("")
|
| 594 |
+
|
| 595 |
+
# URLs
|
| 596 |
+
lines += ["### URLs Referenced", ""]
|
| 597 |
+
lines += ["| File | Line | URL | In Function |"]
|
| 598 |
+
lines += ["|------|------|-----|-------------|"]
|
| 599 |
+
url_seen = set()
|
| 600 |
+
for frel, fi in sorted(all_files.items()):
|
| 601 |
+
for ln, url, func in fi.urls:
|
| 602 |
+
key = (frel, url)
|
| 603 |
+
if key in url_seen:
|
| 604 |
+
continue
|
| 605 |
+
url_seen.add(key)
|
| 606 |
+
short_f = frel.split("/")[-1]
|
| 607 |
+
func_str = f"`{func}`" if func else "*(module level)*"
|
| 608 |
+
lines.append(f"| `{short_f}` | L{ln} | `{url[:80]}` | {func_str} |")
|
| 609 |
+
if not url_seen:
|
| 610 |
+
lines.append("| — | — | *(none found)* | — |")
|
| 611 |
+
lines.append("")
|
| 612 |
+
|
| 613 |
+
# Env vars
|
| 614 |
+
lines += ["### Environment Variables Read", ""]
|
| 615 |
+
lines += ["| File | Line | Variable | In Function |"]
|
| 616 |
+
lines += ["|------|------|----------|-------------|"]
|
| 617 |
+
env_seen = set()
|
| 618 |
+
for frel, fi in sorted(all_files.items()):
|
| 619 |
+
for ln, var, func in fi.env_vars:
|
| 620 |
+
key = (frel, var)
|
| 621 |
+
if key in env_seen:
|
| 622 |
+
continue
|
| 623 |
+
env_seen.add(key)
|
| 624 |
+
short_f = frel.split("/")[-1]
|
| 625 |
+
func_str = f"`{func}`" if func else "*(module level)*"
|
| 626 |
+
lines.append(f"| `{short_f}` | L{ln} | `{var}` | {func_str} |")
|
| 627 |
+
if not env_seen:
|
| 628 |
+
lines.append("| — | — | *(none found)* | — |")
|
| 629 |
+
lines.append("")
|
| 630 |
+
|
| 631 |
+
# Subprocess commands
|
| 632 |
+
lines += ["### Subprocess Commands Launched", ""]
|
| 633 |
+
lines += ["| File | Line | Command (truncated) | In Function |"]
|
| 634 |
+
lines += ["|------|------|---------------------|-------------|"]
|
| 635 |
+
cmd_seen = set()
|
| 636 |
+
for frel, fi in sorted(all_files.items()):
|
| 637 |
+
for ln, cmd, func in fi.subproc_cmds:
|
| 638 |
+
key = (frel, cmd[:60])
|
| 639 |
+
if key in cmd_seen:
|
| 640 |
+
continue
|
| 641 |
+
cmd_seen.add(key)
|
| 642 |
+
short_f = frel.split("/")[-1]
|
| 643 |
+
func_str = f"`{func}`" if func else "*(module level)*"
|
| 644 |
+
safe_cmd = cmd.replace("|", "|")[:80]
|
| 645 |
+
lines.append(f"| `{short_f}` | L{ln} | `{safe_cmd}` | {func_str} |")
|
| 646 |
+
if not cmd_seen:
|
| 647 |
+
lines.append("| — | — | *(none found)* | — |")
|
| 648 |
+
lines += ["", "---", ""]
|
| 649 |
+
|
| 650 |
+
# ── Section 6: Third-party libraries used ────────────────────
|
| 651 |
+
lines += ["## 6. Third-Party & External Libraries Referenced", ""]
|
| 652 |
+
lib_usages: Dict[str, Set[str]] = defaultdict(set)
|
| 653 |
+
for key, targets in call_graph.items():
|
| 654 |
+
frel = key.split("::")[0]
|
| 655 |
+
for t in targets:
|
| 656 |
+
if t.startswith("[lib:"):
|
| 657 |
+
lib_name = t.split("[lib:")[1].split("]")[0]
|
| 658 |
+
lib_usages[lib_name].add(frel)
|
| 659 |
+
|
| 660 |
+
# Also pull from imports directly
|
| 661 |
+
for frel, fi in all_files.items():
|
| 662 |
+
for name, (mod, orig) in fi.from_imports.items():
|
| 663 |
+
root = mod.split(".")[0]
|
| 664 |
+
if root in THIRD_PARTY_ROOTS:
|
| 665 |
+
lib_usages[root].add(frel)
|
| 666 |
+
for alias, mod in fi.imports.items():
|
| 667 |
+
root = mod.split(".")[0]
|
| 668 |
+
if root in THIRD_PARTY_ROOTS:
|
| 669 |
+
lib_usages[root].add(frel)
|
| 670 |
+
|
| 671 |
+
lines.append("| Library | Used in |")
|
| 672 |
+
lines.append("|---------|---------|")
|
| 673 |
+
for lib in sorted(lib_usages):
|
| 674 |
+
files_str = ", ".join(f"`{f.split('/')[-1]}`" for f in sorted(lib_usages[lib]))
|
| 675 |
+
lines.append(f"| `{lib}` | {files_str} |")
|
| 676 |
+
|
| 677 |
+
lines += ["", "---", "", "*Generated by `analyze_codebase.py` — re-run any time to refresh.*", ""]
|
| 678 |
+
return "\n".join(lines)
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
# ─── Unused File Detection ────────────────────────────────────────────────────
|
| 682 |
+
|
| 683 |
+
# Dirs to skip when walking disk for non-Python assets
|
| 684 |
+
ASSET_SKIP_DIRS = {"venv", "__pycache__", ".git", "node_modules", "outputs", "temp",
|
| 685 |
+
"temp_enhanced_pipeline", "datasets-1", "gfpgan"}
|
| 686 |
+
|
| 687 |
+
# These Python file patterns are entry points by convention — never "unused"
|
| 688 |
+
ALWAYS_KEEP_PATTERNS = {
|
| 689 |
+
"analyze_codebase.py", # this script
|
| 690 |
+
"__init__.py",
|
| 691 |
+
"__main__.py",
|
| 692 |
+
"setup.py",
|
| 693 |
+
"conftest.py",
|
| 694 |
+
}
|
| 695 |
+
|
| 696 |
+
|
| 697 |
+
def build_import_graph(all_files: Dict[str, FileInfo]) -> Dict[str, Set[str]]:
|
| 698 |
+
"""
|
| 699 |
+
Returns imported_by[file] = set of files that import it.
|
| 700 |
+
Also returns the full set of reachable files when starting from entry points.
|
| 701 |
+
"""
|
| 702 |
+
# imported_by: file -> who imports it
|
| 703 |
+
imported_by: Dict[str, Set[str]] = defaultdict(set)
|
| 704 |
+
|
| 705 |
+
for frel, fi in all_files.items():
|
| 706 |
+
file_path = ROOT / frel.replace("/", os.sep)
|
| 707 |
+
|
| 708 |
+
for alias, mod in fi.imports.items():
|
| 709 |
+
target = module_to_file(mod, file_path)
|
| 710 |
+
if target and target in all_files:
|
| 711 |
+
imported_by[target].add(frel)
|
| 712 |
+
|
| 713 |
+
for name, (mod, orig) in fi.from_imports.items():
|
| 714 |
+
target = module_to_file(mod, file_path)
|
| 715 |
+
if target and target in all_files:
|
| 716 |
+
imported_by[target].add(frel)
|
| 717 |
+
|
| 718 |
+
return imported_by
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
def reachable_from_entries(
|
| 722 |
+
entry_points: List[str],
|
| 723 |
+
imported_by: Dict[str, Set[str]],
|
| 724 |
+
all_files: Dict[str, FileInfo],
|
| 725 |
+
) -> Set[str]:
|
| 726 |
+
"""
|
| 727 |
+
BFS from entry points following imports to find all reachable files.
|
| 728 |
+
We need forward edges (what does a file import), so invert imported_by.
|
| 729 |
+
"""
|
| 730 |
+
# Build forward: importer -> set of files it imports
|
| 731 |
+
imports_map: Dict[str, Set[str]] = defaultdict(set)
|
| 732 |
+
for target, importers in imported_by.items():
|
| 733 |
+
for importer in importers:
|
| 734 |
+
imports_map[importer].add(target)
|
| 735 |
+
|
| 736 |
+
visited: Set[str] = set()
|
| 737 |
+
queue: List[str] = []
|
| 738 |
+
|
| 739 |
+
for ep in entry_points:
|
| 740 |
+
ep_norm = ep.replace("\\", "/")
|
| 741 |
+
if ep_norm in all_files:
|
| 742 |
+
queue.append(ep_norm)
|
| 743 |
+
|
| 744 |
+
while queue:
|
| 745 |
+
current = queue.pop()
|
| 746 |
+
if current in visited:
|
| 747 |
+
continue
|
| 748 |
+
visited.add(current)
|
| 749 |
+
for dep in imports_map.get(current, set()):
|
| 750 |
+
if dep not in visited:
|
| 751 |
+
queue.append(dep)
|
| 752 |
+
|
| 753 |
+
return visited
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
def categorise_python_file(frel: str, fi: FileInfo) -> str:
|
| 757 |
+
"""Return a human label for why a Python file exists."""
|
| 758 |
+
name = frel.split("/")[-1]
|
| 759 |
+
if name in ALWAYS_KEEP_PATTERNS:
|
| 760 |
+
return "framework/convention"
|
| 761 |
+
if name.startswith("test_"):
|
| 762 |
+
return "test file"
|
| 763 |
+
# Has a __main__ guard
|
| 764 |
+
src_path = ROOT / frel.replace("/", os.sep)
|
| 765 |
+
try:
|
| 766 |
+
src = src_path.read_text(encoding="utf-8", errors="ignore")
|
| 767 |
+
if '__name__ == "__main__"' in src or "__name__ == '__main__'" in src:
|
| 768 |
+
return "standalone script (has __main__)"
|
| 769 |
+
except Exception:
|
| 770 |
+
pass
|
| 771 |
+
if not fi.functions and not fi.classes:
|
| 772 |
+
return "config/constants only"
|
| 773 |
+
return "unreachable module"
|
| 774 |
+
|
| 775 |
+
|
| 776 |
+
def collect_disk_assets(root: Path) -> Dict[str, Path]:
|
| 777 |
+
"""
|
| 778 |
+
Walk disk and return all non-Python files with known asset extensions.
|
| 779 |
+
Key = relative path string (forward slashes), Value = absolute Path.
|
| 780 |
+
"""
|
| 781 |
+
all_exts: Set[str] = set()
|
| 782 |
+
for exts in ASSET_EXTS.values():
|
| 783 |
+
all_exts.update(exts)
|
| 784 |
+
|
| 785 |
+
found: Dict[str, Path] = {}
|
| 786 |
+
for p in root.rglob("*"):
|
| 787 |
+
if not p.is_file():
|
| 788 |
+
continue
|
| 789 |
+
if any(skip in p.parts for skip in ASSET_SKIP_DIRS):
|
| 790 |
+
continue
|
| 791 |
+
if p.suffix.lower() in all_exts:
|
| 792 |
+
found[rel(p)] = p
|
| 793 |
+
return found
|
| 794 |
+
|
| 795 |
+
|
| 796 |
+
def normalise_ref(raw: str) -> str:
|
| 797 |
+
"""Strip drive letters, normalise slashes, lowercase for fuzzy matching."""
|
| 798 |
+
# e.g. C:\\Users\\...\\foo.wav -> foo.wav (just the basename for fuzzy match)
|
| 799 |
+
p = Path(raw.strip())
|
| 800 |
+
return str(p).replace("\\", "/").lower()
|
| 801 |
+
|
| 802 |
+
|
| 803 |
+
def find_unused(
|
| 804 |
+
all_files: Dict[str, FileInfo],
|
| 805 |
+
entry_points: List[str],
|
| 806 |
+
) -> str:
|
| 807 |
+
"""Generate the UNUSED_FILES.md content."""
|
| 808 |
+
|
| 809 |
+
lines = [
|
| 810 |
+
"# Unused & Unreferenced Files",
|
| 811 |
+
"",
|
| 812 |
+
"Auto-generated by `analyze_codebase.py`.",
|
| 813 |
+
"",
|
| 814 |
+
"> **Unused Python** = never imported by any other file in the project and not a declared entry point. ",
|
| 815 |
+
"> **Unreferenced Asset** = file exists on disk but no string literal in any `.py` file points to it. ",
|
| 816 |
+
"> Items marked ⚠️ *might still be used* at runtime via dynamic paths — verify before deleting.",
|
| 817 |
+
"",
|
| 818 |
+
"---",
|
| 819 |
+
"",
|
| 820 |
+
]
|
| 821 |
+
|
| 822 |
+
# ── 1. Python files ──────────────────────────────────────────────────────
|
| 823 |
+
imported_by = build_import_graph(all_files)
|
| 824 |
+
reachable = reachable_from_entries(entry_points, imported_by, all_files)
|
| 825 |
+
|
| 826 |
+
# Normalise entry point list for quick lookup
|
| 827 |
+
ep_set = {ep.replace("\\", "/") for ep in entry_points}
|
| 828 |
+
|
| 829 |
+
unused_py: List[Tuple[str, str, Set[str]]] = [] # (frel, reason, imported_by_set)
|
| 830 |
+
for frel, fi in sorted(all_files.items()):
|
| 831 |
+
name = frel.split("/")[-1]
|
| 832 |
+
if frel in ep_set:
|
| 833 |
+
continue
|
| 834 |
+
if name in ALWAYS_KEEP_PATTERNS:
|
| 835 |
+
continue
|
| 836 |
+
if frel in reachable:
|
| 837 |
+
continue
|
| 838 |
+
# Not reachable from any entry point
|
| 839 |
+
reason = categorise_python_file(frel, fi)
|
| 840 |
+
imported_by_set = imported_by.get(frel, set())
|
| 841 |
+
unused_py.append((frel, reason, imported_by_set))
|
| 842 |
+
|
| 843 |
+
lines += [
|
| 844 |
+
f"## 1. Python Files Never Imported ({len(unused_py)} files)",
|
| 845 |
+
"",
|
| 846 |
+
"These files are not reachable from any entry point via import chains.",
|
| 847 |
+
"",
|
| 848 |
+
"| File | Reason | Imported by |",
|
| 849 |
+
"|------|--------|-------------|",
|
| 850 |
+
]
|
| 851 |
+
for frel, reason, iby in unused_py:
|
| 852 |
+
iby_str = ", ".join(f"`{f.split('/')[-1]}`" for f in sorted(iby)) if iby else "*(nothing)*"
|
| 853 |
+
lines.append(f"| `{frel}` | {reason} | {iby_str} |")
|
| 854 |
+
|
| 855 |
+
lines += ["", "---", ""]
|
| 856 |
+
|
| 857 |
+
# ── 2. Non-Python assets on disk vs referenced ──────────────────────────
|
| 858 |
+
disk_assets = collect_disk_assets(ROOT)
|
| 859 |
+
|
| 860 |
+
# Build a set of all referenced path strings (raw + basename variants)
|
| 861 |
+
ref_strings: Set[str] = set()
|
| 862 |
+
for fi in all_files.values():
|
| 863 |
+
for a in fi.assets:
|
| 864 |
+
s = a.path_str.replace("\\", "/").lower()
|
| 865 |
+
ref_strings.add(s)
|
| 866 |
+
# Also add just the basename so partial paths match
|
| 867 |
+
ref_strings.add(Path(s).name)
|
| 868 |
+
|
| 869 |
+
unreferenced: Dict[str, List[Tuple[str, str]]] = defaultdict(list) # type -> [(frel, disk_path)]
|
| 870 |
+
for disk_rel, disk_abs in sorted(disk_assets.items()):
|
| 871 |
+
# Try full relative path match first
|
| 872 |
+
norm_rel = disk_rel.lower()
|
| 873 |
+
basename = Path(disk_rel).name.lower()
|
| 874 |
+
if norm_rel in ref_strings or basename in ref_strings:
|
| 875 |
+
continue
|
| 876 |
+
# Try matching any suffix of the disk path against references
|
| 877 |
+
parts = disk_rel.replace("\\", "/").lower().split("/")
|
| 878 |
+
matched = any(
|
| 879 |
+
"/".join(parts[i:]) in ref_strings
|
| 880 |
+
for i in range(len(parts))
|
| 881 |
+
)
|
| 882 |
+
if matched:
|
| 883 |
+
continue
|
| 884 |
+
|
| 885 |
+
suffix = disk_abs.suffix.lower()
|
| 886 |
+
atype = "other"
|
| 887 |
+
for t, exts in ASSET_EXTS.items():
|
| 888 |
+
if suffix in exts:
|
| 889 |
+
atype = t
|
| 890 |
+
break
|
| 891 |
+
unreferenced[atype].append((disk_rel, str(disk_abs)))
|
| 892 |
+
|
| 893 |
+
total_unref = sum(len(v) for v in unreferenced.values())
|
| 894 |
+
lines += [
|
| 895 |
+
f"## 2. Non-Python Assets on Disk But Never Referenced ({total_unref} files)",
|
| 896 |
+
"",
|
| 897 |
+
"These files exist in the project folder but no Python source file contains a string that matches their path.",
|
| 898 |
+
"",
|
| 899 |
+
]
|
| 900 |
+
|
| 901 |
+
for atype in ["model", "audio", "video", "image", "config", "data", "other"]:
|
| 902 |
+
items = unreferenced.get(atype, [])
|
| 903 |
+
if not items:
|
| 904 |
+
continue
|
| 905 |
+
lines += [
|
| 906 |
+
f"### {atype.capitalize()} Files ({len(items)})",
|
| 907 |
+
"",
|
| 908 |
+
"| Path | Size |",
|
| 909 |
+
"|------|------|",
|
| 910 |
+
]
|
| 911 |
+
for disk_rel, disk_abs in items:
|
| 912 |
+
try:
|
| 913 |
+
size = Path(disk_abs).stat().st_size
|
| 914 |
+
size_str = f"{size:,} B" if size < 1024 else (
|
| 915 |
+
f"{size//1024:,} KB" if size < 1_048_576 else f"{size//1_048_576:,} MB"
|
| 916 |
+
)
|
| 917 |
+
except Exception:
|
| 918 |
+
size_str = "?"
|
| 919 |
+
lines.append(f"| `{disk_rel}` | {size_str} |")
|
| 920 |
+
lines.append("")
|
| 921 |
+
|
| 922 |
+
lines += ["---", ""]
|
| 923 |
+
|
| 924 |
+
# ── 3. Referenced assets that are MISSING from disk ─────────────────────
|
| 925 |
+
missing_assets: List[Tuple[str, str, int, str]] = [] # (frel, path_str, line, atype)
|
| 926 |
+
for frel, fi in sorted(all_files.items()):
|
| 927 |
+
for a in fi.assets:
|
| 928 |
+
if not a.exists:
|
| 929 |
+
# Skip obvious format strings and very short tokens
|
| 930 |
+
if "%" in a.path_str or "{" in a.path_str or len(a.path_str) < 5:
|
| 931 |
+
continue
|
| 932 |
+
# Skip strings that are clearly descriptions, not paths
|
| 933 |
+
if " " in a.path_str and not any(c in a.path_str for c in "\\/"):
|
| 934 |
+
continue
|
| 935 |
+
missing_assets.append((frel, a.path_str, a.lineno, a.asset_type))
|
| 936 |
+
|
| 937 |
+
lines += [
|
| 938 |
+
f"## 3. Assets Referenced in Code But Missing on Disk ({len(missing_assets)} refs)",
|
| 939 |
+
"",
|
| 940 |
+
"These are paths the code expects to exist but were not found. May indicate missing model downloads.",
|
| 941 |
+
"",
|
| 942 |
+
"| Source File | Line | Expected Path | Type |",
|
| 943 |
+
"|-------------|------|---------------|------|",
|
| 944 |
+
]
|
| 945 |
+
seen_missing: Set[Tuple[str, str]] = set()
|
| 946 |
+
for frel, path_str, lineno, atype in missing_assets:
|
| 947 |
+
key = (frel, path_str)
|
| 948 |
+
if key in seen_missing:
|
| 949 |
+
continue
|
| 950 |
+
seen_missing.add(key)
|
| 951 |
+
short_f = frel.split("/")[-1]
|
| 952 |
+
safe = path_str.replace("|", "|")[:80]
|
| 953 |
+
lines.append(f"| `{short_f}` | L{lineno} | `{safe}` | {atype} |")
|
| 954 |
+
|
| 955 |
+
lines += [
|
| 956 |
+
"",
|
| 957 |
+
"---",
|
| 958 |
+
"",
|
| 959 |
+
"*Generated by `analyze_codebase.py` — re-run any time to refresh.*",
|
| 960 |
+
"",
|
| 961 |
+
]
|
| 962 |
+
return "\n".join(lines)
|
| 963 |
+
|
| 964 |
+
|
| 965 |
+
# ─── Main ─────────────────────────────────────────────────────────────────────
|
| 966 |
+
|
| 967 |
+
def main():
|
| 968 |
+
print(f"Scanning: {ROOT}")
|
| 969 |
+
|
| 970 |
+
py_files = collect_py_files(ROOT)
|
| 971 |
+
print(f"Found {len(py_files)} Python files\n")
|
| 972 |
+
|
| 973 |
+
all_files: Dict[str, FileInfo] = {}
|
| 974 |
+
for p in py_files:
|
| 975 |
+
print(f" Parsing {rel(p)}")
|
| 976 |
+
fi = parse_file(p)
|
| 977 |
+
if fi:
|
| 978 |
+
all_files[fi.path] = fi
|
| 979 |
+
|
| 980 |
+
print(f"\nBuilding call graph...")
|
| 981 |
+
call_graph = resolve_calls(all_files)
|
| 982 |
+
total_edges = sum(len(v) for v in call_graph.values())
|
| 983 |
+
print(f" {len(call_graph)} functions with outgoing calls, {total_edges} total edges")
|
| 984 |
+
|
| 985 |
+
print(f"\nGenerating CODEBASE_MAP.md...")
|
| 986 |
+
readme = generate_readme(all_files, call_graph, ENTRY_POINTS)
|
| 987 |
+
out_path = ROOT / "CODEBASE_MAP.md"
|
| 988 |
+
out_path.write_text(readme, encoding="utf-8")
|
| 989 |
+
|
| 990 |
+
total_assets = sum(len(fi.assets) for fi in all_files.values())
|
| 991 |
+
total_urls = sum(len(fi.urls) for fi in all_files.values())
|
| 992 |
+
total_envs = sum(len(fi.env_vars) for fi in all_files.values())
|
| 993 |
+
total_cmds = sum(len(fi.subproc_cmds) for fi in all_files.values())
|
| 994 |
+
|
| 995 |
+
print(f" Done -> {out_path}")
|
| 996 |
+
print(f" Files analyzed : {len(all_files)}")
|
| 997 |
+
print(f" Total functions : {sum(len(fi.functions) for fi in all_files.values())}")
|
| 998 |
+
print(f" Call graph edges: {total_edges}")
|
| 999 |
+
print(f" Asset refs : {total_assets}")
|
| 1000 |
+
print(f" URLs : {total_urls} | Env vars: {total_envs} | Subprocess: {total_cmds}")
|
| 1001 |
+
|
| 1002 |
+
print(f"\nGenerating UNUSED_FILES.md...")
|
| 1003 |
+
unused_report = find_unused(all_files, ENTRY_POINTS)
|
| 1004 |
+
unused_path = ROOT / "UNUSED_FILES.md"
|
| 1005 |
+
unused_path.write_text(unused_report, encoding="utf-8")
|
| 1006 |
+
print(f" Done -> {unused_path}")
|
| 1007 |
+
|
| 1008 |
+
# Quick summary stats for unused report
|
| 1009 |
+
imported_by = build_import_graph(all_files)
|
| 1010 |
+
reachable = reachable_from_entries(ENTRY_POINTS, imported_by, all_files)
|
| 1011 |
+
ep_set = {ep.replace("\\", "/") for ep in ENTRY_POINTS}
|
| 1012 |
+
unused_count = sum(
|
| 1013 |
+
1 for frel, fi in all_files.items()
|
| 1014 |
+
if frel not in ep_set
|
| 1015 |
+
and frel.split("/")[-1] not in ALWAYS_KEEP_PATTERNS
|
| 1016 |
+
and frel not in reachable
|
| 1017 |
+
)
|
| 1018 |
+
disk_asset_count = len(collect_disk_assets(ROOT))
|
| 1019 |
+
print(f" Unused Python files : {unused_count}")
|
| 1020 |
+
print(f" Disk assets scanned : {disk_asset_count}")
|
| 1021 |
+
|
| 1022 |
+
|
| 1023 |
+
if __name__ == "__main__":
|
| 1024 |
+
main()
|
app.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HuggingFace Spaces entry point – must be named app.py
|
| 2 |
+
# Imports and re-exports the Gradio demo from gradio_app.py
|
| 3 |
+
|
| 4 |
+
import os
|
| 5 |
+
import sys
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
# On HF Spaces, SadTalker checkpoints are likely absent → disable video by default
|
| 9 |
+
# Users can still enable it with the "Skip video" checkbox.
|
| 10 |
+
# If you DO upload checkpoints, remove this env var.
|
| 11 |
+
if not (Path(__file__).parent / "sadtalker+wav2lip" / "sadtalker" / "checkpoints").exists():
|
| 12 |
+
os.environ.setdefault("DISABLE_VIDEO", "1")
|
| 13 |
+
|
| 14 |
+
# HF_TOKEN must be set as a Space Secret for the HF Inference API fallback
|
| 15 |
+
# os.environ["HF_TOKEN"] is picked up automatically from Secrets
|
| 16 |
+
|
| 17 |
+
from gradio_app import demo # noqa: E402 (import after env setup)
|
| 18 |
+
|
| 19 |
+
demo.queue().launch()
|
gradio_app.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Mindfull AI Avatar Chatbot - Gradio Frontend
|
| 3 |
+
Works with any uploaded avatar image.
|
| 4 |
+
Runs locally (Ollama) or on HuggingFace Spaces (HF Inference API).
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
import time
|
| 10 |
+
import tempfile
|
| 11 |
+
import shutil
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import gradio as gr
|
| 15 |
+
|
| 16 |
+
# ── runtime paths ──────────────────────────────────────────────────────────────
|
| 17 |
+
ROOT = Path(__file__).parent.absolute()
|
| 18 |
+
OUTPUT_DIR = ROOT / "outputs"
|
| 19 |
+
AUDIO_DIR = OUTPUT_DIR / "audio"
|
| 20 |
+
VIDEO_DIR = OUTPUT_DIR / "video"
|
| 21 |
+
TEMP_DIR = OUTPUT_DIR / "temp"
|
| 22 |
+
for d in [AUDIO_DIR, VIDEO_DIR, TEMP_DIR]:
|
| 23 |
+
d.mkdir(parents=True, exist_ok=True)
|
| 24 |
+
|
| 25 |
+
# ── LLM backend detection ─────────────────────────────────────────────────────
|
| 26 |
+
def _ollama_available() -> bool:
|
| 27 |
+
try:
|
| 28 |
+
import requests
|
| 29 |
+
r = requests.get("http://localhost:11434/api/tags", timeout=3)
|
| 30 |
+
return r.status_code == 200
|
| 31 |
+
except Exception:
|
| 32 |
+
return False
|
| 33 |
+
|
| 34 |
+
OLLAMA_URL = "http://localhost:11434/api/generate"
|
| 35 |
+
OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "mindfull")
|
| 36 |
+
|
| 37 |
+
HF_TOKEN = os.environ.get("HF_TOKEN", "")
|
| 38 |
+
HF_MODEL = "mistralai/Mistral-7B-Instruct-v0.3"
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ── text generation ────────────────────────────────────────────────────────────
|
| 42 |
+
def generate_text_ollama(prompt: str) -> str:
|
| 43 |
+
import requests
|
| 44 |
+
payload = {
|
| 45 |
+
"model": OLLAMA_MODEL,
|
| 46 |
+
"prompt": prompt,
|
| 47 |
+
"stream": False,
|
| 48 |
+
"options": {"temperature": 0.7, "num_ctx": 2048},
|
| 49 |
+
}
|
| 50 |
+
r = requests.post(OLLAMA_URL, json=payload, timeout=60)
|
| 51 |
+
r.raise_for_status()
|
| 52 |
+
return r.json().get("response", "").strip()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def generate_text_hf(prompt: str) -> str:
|
| 56 |
+
from huggingface_hub import InferenceClient
|
| 57 |
+
client = InferenceClient(model=HF_MODEL, token=HF_TOKEN or None)
|
| 58 |
+
out = client.text_generation(
|
| 59 |
+
prompt,
|
| 60 |
+
max_new_tokens=256,
|
| 61 |
+
temperature=0.7,
|
| 62 |
+
do_sample=True,
|
| 63 |
+
)
|
| 64 |
+
return out.strip()
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def generate_response(user_input: str, history: list) -> str:
|
| 68 |
+
"""Build prompt from history and call whichever backend is available."""
|
| 69 |
+
# History is Gradio 5 messages format: [{role, content}, ...]
|
| 70 |
+
ctx_lines = []
|
| 71 |
+
for h in history[-8:]: # last 4 exchanges
|
| 72 |
+
role = h.get("role", "") if isinstance(h, dict) else ""
|
| 73 |
+
content = h.get("content", "") if isinstance(h, dict) else ""
|
| 74 |
+
if role == "user":
|
| 75 |
+
ctx_lines.append(f"User: {content}")
|
| 76 |
+
elif role == "assistant":
|
| 77 |
+
ctx_lines.append(f"Mindfull: {content}")
|
| 78 |
+
ctx = "\n".join(ctx_lines)
|
| 79 |
+
|
| 80 |
+
prompt = f"""You are Mindfull, a professional AI wellness companion.
|
| 81 |
+
Provide warm, concise support (2–3 sentences) suitable for a voice response.
|
| 82 |
+
|
| 83 |
+
{ctx}
|
| 84 |
+
User: {user_input}
|
| 85 |
+
Mindfull:"""
|
| 86 |
+
|
| 87 |
+
try:
|
| 88 |
+
if _ollama_available():
|
| 89 |
+
return generate_text_ollama(prompt)
|
| 90 |
+
else:
|
| 91 |
+
return generate_text_hf(prompt)
|
| 92 |
+
except Exception as e:
|
| 93 |
+
return f"I'm here to support you. (Error reaching AI model: {e})"
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
# ── audio generation ───────────────────────────────────────────────────────────
|
| 97 |
+
def generate_audio(text: str) -> str | None:
|
| 98 |
+
"""Generate speech with edge-tts. Returns path to .mp3 or None."""
|
| 99 |
+
try:
|
| 100 |
+
from simple_audio_gen import generate_audio_simple
|
| 101 |
+
ts = int(time.time() * 1000)
|
| 102 |
+
out = str(AUDIO_DIR / f"response_{ts}.mp3")
|
| 103 |
+
ok = generate_audio_simple(text, out)
|
| 104 |
+
return out if ok else None
|
| 105 |
+
except Exception as e:
|
| 106 |
+
print(f"Audio generation error: {e}")
|
| 107 |
+
return None
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ── video generation ───────────────────────────────────────────────────────────
|
| 111 |
+
def generate_video(image_path: str | None, audio_path: str | None) -> str | None:
|
| 112 |
+
"""Generate talking-head video. Returns path to .mp4 or None."""
|
| 113 |
+
if not image_path or not audio_path:
|
| 114 |
+
return None
|
| 115 |
+
if not Path(image_path).exists() or not Path(audio_path).exists():
|
| 116 |
+
return None
|
| 117 |
+
try:
|
| 118 |
+
from simple_video_gen import generate_video_simple
|
| 119 |
+
ts = int(time.time() * 1000)
|
| 120 |
+
out = str(VIDEO_DIR / f"avatar_{ts}.mp4")
|
| 121 |
+
ok = generate_video_simple(image_path, audio_path, out)
|
| 122 |
+
return out if ok and Path(out).exists() else None
|
| 123 |
+
except Exception as e:
|
| 124 |
+
print(f"Video generation error: {e}")
|
| 125 |
+
return None
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
# ── main chat handler ─────────────────────────────────────────���────────────────
|
| 129 |
+
def chat(
|
| 130 |
+
user_message: str,
|
| 131 |
+
avatar_image, # numpy array or file path from gr.Image
|
| 132 |
+
history: list,
|
| 133 |
+
skip_video: bool,
|
| 134 |
+
):
|
| 135 |
+
if not user_message.strip():
|
| 136 |
+
yield history, None, None, "Please type a message."
|
| 137 |
+
return
|
| 138 |
+
|
| 139 |
+
# ── 1. text ──────────────────────────────────────────────────────────────
|
| 140 |
+
yield history, None, None, "💬 Generating response…"
|
| 141 |
+
response_text = generate_response(user_message.strip(), history)
|
| 142 |
+
# Gradio 5 messages format: {"role": ..., "content": ...}
|
| 143 |
+
new_history = history + [
|
| 144 |
+
{"role": "user", "content": user_message},
|
| 145 |
+
{"role": "assistant", "content": response_text},
|
| 146 |
+
]
|
| 147 |
+
|
| 148 |
+
# ── 2. audio ─────────────────────────────────────────────────────────────
|
| 149 |
+
yield new_history, None, None, "🔊 Synthesising speech…"
|
| 150 |
+
audio_path = generate_audio(response_text)
|
| 151 |
+
|
| 152 |
+
# ── 3. video ─────────────────────────────────────────────────────────────
|
| 153 |
+
video_path = None
|
| 154 |
+
if not skip_video and avatar_image is not None:
|
| 155 |
+
yield new_history, audio_path, None, "🎬 Generating avatar video…"
|
| 156 |
+
|
| 157 |
+
# Gradio Image component returns a numpy array; save it to a temp file
|
| 158 |
+
import numpy as np
|
| 159 |
+
from PIL import Image as PilImage
|
| 160 |
+
|
| 161 |
+
if isinstance(avatar_image, np.ndarray):
|
| 162 |
+
ts = int(time.time() * 1000)
|
| 163 |
+
img_path = str(TEMP_DIR / f"avatar_input_{ts}.png")
|
| 164 |
+
pil_img = PilImage.fromarray(avatar_image)
|
| 165 |
+
# SadTalker works best with images ≤ 512px on the longest side
|
| 166 |
+
MAX_DIM = 512
|
| 167 |
+
if max(pil_img.size) > MAX_DIM:
|
| 168 |
+
pil_img.thumbnail((MAX_DIM, MAX_DIM), PilImage.LANCZOS)
|
| 169 |
+
pil_img.save(img_path)
|
| 170 |
+
else:
|
| 171 |
+
img_path = str(avatar_image) # already a path
|
| 172 |
+
|
| 173 |
+
video_path = generate_video(img_path, audio_path)
|
| 174 |
+
|
| 175 |
+
status = "✅ Done!" if video_path or audio_path else "⚠️ Only text response available."
|
| 176 |
+
yield new_history, audio_path, video_path, status
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# ── Gradio UI ──────────────────────────────────────────────────────────────────
|
| 180 |
+
with gr.Blocks(
|
| 181 |
+
title="Mindfull – AI Wellness Companion",
|
| 182 |
+
theme=gr.themes.Soft(primary_hue="blue", secondary_hue="slate"),
|
| 183 |
+
css="""
|
| 184 |
+
#header { text-align: center; padding: 10px 0 4px; }
|
| 185 |
+
#header h1 { font-size: 2rem; margin-bottom: 2px; }
|
| 186 |
+
#header p { color: #6b7280; margin: 0; }
|
| 187 |
+
.status-box { font-size: 0.85rem; color: #374151; }
|
| 188 |
+
""",
|
| 189 |
+
) as demo:
|
| 190 |
+
|
| 191 |
+
# ── header ────────────────────────────────────────────────────────────────
|
| 192 |
+
with gr.Row(elem_id="header"):
|
| 193 |
+
with gr.Column():
|
| 194 |
+
gr.HTML("""
|
| 195 |
+
<h1>🧠 Mindfull</h1>
|
| 196 |
+
<p>AI Wellness Companion · talking-head avatar · real TTS</p>
|
| 197 |
+
""")
|
| 198 |
+
|
| 199 |
+
# If running on HF Spaces without SadTalker checkpoints, default to audio-only
|
| 200 |
+
_video_disabled = os.environ.get("DISABLE_VIDEO", "0") == "1"
|
| 201 |
+
|
| 202 |
+
# ── main layout ───────────────────────────────────────────────────────────
|
| 203 |
+
with gr.Row():
|
| 204 |
+
|
| 205 |
+
# Left column – avatar + controls
|
| 206 |
+
with gr.Column(scale=1, min_width=260):
|
| 207 |
+
avatar_img = gr.Image(
|
| 208 |
+
label="Avatar Image (any portrait)",
|
| 209 |
+
type="numpy",
|
| 210 |
+
sources=["upload", "webcam"],
|
| 211 |
+
height=280,
|
| 212 |
+
)
|
| 213 |
+
skip_video_chk = gr.Checkbox(
|
| 214 |
+
label="Skip video (faster · audio only)",
|
| 215 |
+
value=_video_disabled,
|
| 216 |
+
)
|
| 217 |
+
gr.Markdown(
|
| 218 |
+
"_Upload **any** face image. "
|
| 219 |
+
"Leaving it empty will return text + audio only._",
|
| 220 |
+
elem_classes=["status-box"],
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
# Right column – chat
|
| 224 |
+
with gr.Column(scale=2):
|
| 225 |
+
chatbot = gr.Chatbot(
|
| 226 |
+
label="Conversation",
|
| 227 |
+
height=400,
|
| 228 |
+
type="messages",
|
| 229 |
+
avatar_images=(None, str(ROOT / "avatar_assets" / "officer.png")
|
| 230 |
+
if (ROOT / "avatar_assets" / "officer.png").exists()
|
| 231 |
+
else None),
|
| 232 |
+
)
|
| 233 |
+
with gr.Row():
|
| 234 |
+
msg_box = gr.Textbox(
|
| 235 |
+
placeholder="How are you feeling today?",
|
| 236 |
+
label="Your message",
|
| 237 |
+
scale=5,
|
| 238 |
+
lines=1,
|
| 239 |
+
)
|
| 240 |
+
send_btn = gr.Button("Send", variant="primary", scale=1)
|
| 241 |
+
|
| 242 |
+
# ── outputs ───────────────────────────────────────────────────────────────
|
| 243 |
+
with gr.Row():
|
| 244 |
+
audio_out = gr.Audio(label="🔊 Voice Response", type="filepath", autoplay=True)
|
| 245 |
+
video_out = gr.Video(label="🎬 Avatar Video", autoplay=True)
|
| 246 |
+
|
| 247 |
+
status_box = gr.Textbox(
|
| 248 |
+
label="Status", interactive=False, lines=1, elem_classes=["status-box"]
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
# ── clear button ──────────────────────────────────────────────────────────
|
| 252 |
+
with gr.Row():
|
| 253 |
+
clear_btn = gr.Button("🗑 Clear conversation", variant="secondary")
|
| 254 |
+
|
| 255 |
+
# ── wiring ────────────────────────────────────────────────────────────────
|
| 256 |
+
def _submit(msg, img, hist, skip):
|
| 257 |
+
yield from chat(msg, img, hist, skip)
|
| 258 |
+
|
| 259 |
+
send_btn.click(
|
| 260 |
+
fn=_submit,
|
| 261 |
+
inputs=[msg_box, avatar_img, chatbot, skip_video_chk],
|
| 262 |
+
outputs=[chatbot, audio_out, video_out, status_box],
|
| 263 |
+
).then(fn=lambda: "", outputs=msg_box)
|
| 264 |
+
|
| 265 |
+
msg_box.submit(
|
| 266 |
+
fn=_submit,
|
| 267 |
+
inputs=[msg_box, avatar_img, chatbot, skip_video_chk],
|
| 268 |
+
outputs=[chatbot, audio_out, video_out, status_box],
|
| 269 |
+
).then(fn=lambda: "", outputs=msg_box)
|
| 270 |
+
|
| 271 |
+
clear_btn.click(
|
| 272 |
+
fn=lambda: ([], None, None, ""), # empty list = empty messages history
|
| 273 |
+
outputs=[chatbot, audio_out, video_out, status_box],
|
| 274 |
+
)
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
if __name__ == "__main__":
|
| 278 |
+
demo.queue().launch(
|
| 279 |
+
server_name="0.0.0.0",
|
| 280 |
+
server_port=7860,
|
| 281 |
+
share=False, # set True for a public Gradio link
|
| 282 |
+
show_error=True,
|
| 283 |
+
)
|
infer_cli.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import codecs
|
| 3 |
+
import os
|
| 4 |
+
import re
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from importlib.resources import files
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import soundfile as sf
|
| 11 |
+
import tomli
|
| 12 |
+
from cached_path import cached_path
|
| 13 |
+
from hydra.utils import get_class
|
| 14 |
+
from omegaconf import OmegaConf
|
| 15 |
+
|
| 16 |
+
from f5_tts.infer.utils_infer import (
|
| 17 |
+
cfg_strength,
|
| 18 |
+
cross_fade_duration,
|
| 19 |
+
device,
|
| 20 |
+
fix_duration,
|
| 21 |
+
infer_process,
|
| 22 |
+
load_model,
|
| 23 |
+
load_vocoder,
|
| 24 |
+
mel_spec_type,
|
| 25 |
+
nfe_step,
|
| 26 |
+
preprocess_ref_audio_text,
|
| 27 |
+
remove_silence_for_generated_wav,
|
| 28 |
+
speed,
|
| 29 |
+
sway_sampling_coef,
|
| 30 |
+
target_rms,
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
parser = argparse.ArgumentParser(
|
| 35 |
+
prog="python3 infer-cli.py",
|
| 36 |
+
description="Commandline interface for E2/F5 TTS with Advanced Batch Processing.",
|
| 37 |
+
epilog="Specify options above to override one or more settings from config.",
|
| 38 |
+
)
|
| 39 |
+
parser.add_argument(
|
| 40 |
+
"-c",
|
| 41 |
+
"--config",
|
| 42 |
+
type=str,
|
| 43 |
+
default=os.path.join(files("f5_tts").joinpath("infer/examples/basic"), "basic.toml"),
|
| 44 |
+
help="The configuration file, default see infer/examples/basic/basic.toml",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# Note. Not to provide default value here in order to read default from config file
|
| 49 |
+
|
| 50 |
+
parser.add_argument(
|
| 51 |
+
"-m",
|
| 52 |
+
"--model",
|
| 53 |
+
type=str,
|
| 54 |
+
help="The model name: F5TTS_v1_Base | F5TTS_Base | E2TTS_Base | etc.",
|
| 55 |
+
)
|
| 56 |
+
parser.add_argument(
|
| 57 |
+
"-mc",
|
| 58 |
+
"--model_cfg",
|
| 59 |
+
type=str,
|
| 60 |
+
help="The path to F5-TTS model config file .yaml",
|
| 61 |
+
)
|
| 62 |
+
parser.add_argument(
|
| 63 |
+
"-p",
|
| 64 |
+
"--ckpt_file",
|
| 65 |
+
type=str,
|
| 66 |
+
help="The path to model checkpoint .pt, leave blank to use default",
|
| 67 |
+
)
|
| 68 |
+
parser.add_argument(
|
| 69 |
+
"-v",
|
| 70 |
+
"--vocab_file",
|
| 71 |
+
type=str,
|
| 72 |
+
help="The path to vocab file .txt, leave blank to use default",
|
| 73 |
+
)
|
| 74 |
+
parser.add_argument(
|
| 75 |
+
"-r",
|
| 76 |
+
"--ref_audio",
|
| 77 |
+
type=str,
|
| 78 |
+
help="The reference audio file.",
|
| 79 |
+
)
|
| 80 |
+
parser.add_argument(
|
| 81 |
+
"-s",
|
| 82 |
+
"--ref_text",
|
| 83 |
+
type=str,
|
| 84 |
+
help="The transcript/subtitle for the reference audio",
|
| 85 |
+
)
|
| 86 |
+
parser.add_argument(
|
| 87 |
+
"-t",
|
| 88 |
+
"--gen_text",
|
| 89 |
+
type=str,
|
| 90 |
+
help="The text to make model synthesize a speech",
|
| 91 |
+
)
|
| 92 |
+
parser.add_argument(
|
| 93 |
+
"-f",
|
| 94 |
+
"--gen_file",
|
| 95 |
+
type=str,
|
| 96 |
+
help="The file with text to generate, will ignore --gen_text",
|
| 97 |
+
)
|
| 98 |
+
parser.add_argument(
|
| 99 |
+
"-o",
|
| 100 |
+
"--output_dir",
|
| 101 |
+
type=str,
|
| 102 |
+
help="The path to output folder",
|
| 103 |
+
)
|
| 104 |
+
parser.add_argument(
|
| 105 |
+
"-w",
|
| 106 |
+
"--output_file",
|
| 107 |
+
type=str,
|
| 108 |
+
help="The name of output file",
|
| 109 |
+
)
|
| 110 |
+
parser.add_argument(
|
| 111 |
+
"--save_chunk",
|
| 112 |
+
action="store_true",
|
| 113 |
+
help="To save each audio chunks during inference",
|
| 114 |
+
)
|
| 115 |
+
parser.add_argument(
|
| 116 |
+
"--remove_silence",
|
| 117 |
+
action="store_true",
|
| 118 |
+
help="To remove long silence found in ouput",
|
| 119 |
+
)
|
| 120 |
+
parser.add_argument(
|
| 121 |
+
"--load_vocoder_from_local",
|
| 122 |
+
action="store_true",
|
| 123 |
+
help="To load vocoder from local dir, default to ../checkpoints/vocos-mel-24khz",
|
| 124 |
+
)
|
| 125 |
+
parser.add_argument(
|
| 126 |
+
"--vocoder_name",
|
| 127 |
+
type=str,
|
| 128 |
+
choices=["vocos", "bigvgan"],
|
| 129 |
+
help=f"Used vocoder name: vocos | bigvgan, default {mel_spec_type}",
|
| 130 |
+
)
|
| 131 |
+
parser.add_argument(
|
| 132 |
+
"--target_rms",
|
| 133 |
+
type=float,
|
| 134 |
+
help=f"Target output speech loudness normalization value, default {target_rms}",
|
| 135 |
+
)
|
| 136 |
+
parser.add_argument(
|
| 137 |
+
"--cross_fade_duration",
|
| 138 |
+
type=float,
|
| 139 |
+
help=f"Duration of cross-fade between audio segments in seconds, default {cross_fade_duration}",
|
| 140 |
+
)
|
| 141 |
+
parser.add_argument(
|
| 142 |
+
"--nfe_step",
|
| 143 |
+
type=int,
|
| 144 |
+
help=f"The number of function evaluation (denoising steps), default {nfe_step}",
|
| 145 |
+
)
|
| 146 |
+
parser.add_argument(
|
| 147 |
+
"--cfg_strength",
|
| 148 |
+
type=float,
|
| 149 |
+
help=f"Classifier-free guidance strength, default {cfg_strength}",
|
| 150 |
+
)
|
| 151 |
+
parser.add_argument(
|
| 152 |
+
"--sway_sampling_coef",
|
| 153 |
+
type=float,
|
| 154 |
+
help=f"Sway Sampling coefficient, default {sway_sampling_coef}",
|
| 155 |
+
)
|
| 156 |
+
parser.add_argument(
|
| 157 |
+
"--speed",
|
| 158 |
+
type=float,
|
| 159 |
+
help=f"The speed of the generated audio, default {speed}",
|
| 160 |
+
)
|
| 161 |
+
parser.add_argument(
|
| 162 |
+
"--fix_duration",
|
| 163 |
+
type=float,
|
| 164 |
+
help=f"Fix the total duration (ref and gen audios) in seconds, default {fix_duration}",
|
| 165 |
+
)
|
| 166 |
+
parser.add_argument(
|
| 167 |
+
"--device",
|
| 168 |
+
type=str,
|
| 169 |
+
help="Specify the device to run on",
|
| 170 |
+
)
|
| 171 |
+
args = parser.parse_args()
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
# config file
|
| 175 |
+
|
| 176 |
+
config = tomli.load(open(args.config, "rb"))
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# command-line interface parameters
|
| 180 |
+
|
| 181 |
+
model = args.model or config.get("model", "F5TTS_v1_Base")
|
| 182 |
+
ckpt_file = args.ckpt_file or config.get("ckpt_file", "")
|
| 183 |
+
vocab_file = args.vocab_file or config.get("vocab_file", "")
|
| 184 |
+
|
| 185 |
+
ref_audio = args.ref_audio or config.get("ref_audio", "infer/examples/basic/basic_ref_en.wav")
|
| 186 |
+
ref_text = (
|
| 187 |
+
args.ref_text
|
| 188 |
+
if args.ref_text is not None
|
| 189 |
+
else config.get("ref_text", "Some call me nature, others call me mother nature.")
|
| 190 |
+
)
|
| 191 |
+
gen_text = args.gen_text or config.get("gen_text", "Here we generate something just for test.")
|
| 192 |
+
gen_file = args.gen_file or config.get("gen_file", "")
|
| 193 |
+
|
| 194 |
+
output_dir = args.output_dir or config.get("output_dir", "tests")
|
| 195 |
+
output_file = args.output_file or config.get(
|
| 196 |
+
"output_file", f"infer_cli_{datetime.now().strftime(r'%Y%m%d_%H%M%S')}.wav"
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
save_chunk = args.save_chunk or config.get("save_chunk", False)
|
| 200 |
+
remove_silence = args.remove_silence or config.get("remove_silence", False)
|
| 201 |
+
load_vocoder_from_local = args.load_vocoder_from_local or config.get("load_vocoder_from_local", False)
|
| 202 |
+
|
| 203 |
+
vocoder_name = args.vocoder_name or config.get("vocoder_name", mel_spec_type)
|
| 204 |
+
target_rms = args.target_rms or config.get("target_rms", target_rms)
|
| 205 |
+
cross_fade_duration = args.cross_fade_duration or config.get("cross_fade_duration", cross_fade_duration)
|
| 206 |
+
nfe_step = args.nfe_step or config.get("nfe_step", nfe_step)
|
| 207 |
+
cfg_strength = args.cfg_strength or config.get("cfg_strength", cfg_strength)
|
| 208 |
+
sway_sampling_coef = args.sway_sampling_coef or config.get("sway_sampling_coef", sway_sampling_coef)
|
| 209 |
+
speed = args.speed or config.get("speed", speed)
|
| 210 |
+
fix_duration = args.fix_duration or config.get("fix_duration", fix_duration)
|
| 211 |
+
device = args.device or config.get("device", device)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
# patches for pip pkg user
|
| 215 |
+
if "infer/examples/" in ref_audio:
|
| 216 |
+
ref_audio = str(files("f5_tts").joinpath(f"{ref_audio}"))
|
| 217 |
+
if "infer/examples/" in gen_file:
|
| 218 |
+
gen_file = str(files("f5_tts").joinpath(f"{gen_file}"))
|
| 219 |
+
if "voices" in config:
|
| 220 |
+
for voice in config["voices"]:
|
| 221 |
+
voice_ref_audio = config["voices"][voice]["ref_audio"]
|
| 222 |
+
if "infer/examples/" in voice_ref_audio:
|
| 223 |
+
config["voices"][voice]["ref_audio"] = str(files("f5_tts").joinpath(f"{voice_ref_audio}"))
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
# ignore gen_text if gen_file provided
|
| 227 |
+
|
| 228 |
+
if gen_file:
|
| 229 |
+
gen_text = codecs.open(gen_file, "r", "utf-8").read()
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
# output path
|
| 233 |
+
|
| 234 |
+
wave_path = Path(output_dir) / output_file
|
| 235 |
+
# spectrogram_path = Path(output_dir) / "infer_cli_out.png"
|
| 236 |
+
if save_chunk:
|
| 237 |
+
output_chunk_dir = os.path.join(output_dir, f"{Path(output_file).stem}_chunks")
|
| 238 |
+
if not os.path.exists(output_chunk_dir):
|
| 239 |
+
os.makedirs(output_chunk_dir)
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
# load vocoder
|
| 243 |
+
|
| 244 |
+
if vocoder_name == "vocos":
|
| 245 |
+
vocoder_local_path = "../checkpoints/vocos-mel-24khz"
|
| 246 |
+
elif vocoder_name == "bigvgan":
|
| 247 |
+
vocoder_local_path = "../checkpoints/bigvgan_v2_24khz_100band_256x"
|
| 248 |
+
|
| 249 |
+
vocoder = load_vocoder(
|
| 250 |
+
vocoder_name=vocoder_name, is_local=load_vocoder_from_local, local_path=vocoder_local_path, device=device
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
# load TTS model
|
| 255 |
+
|
| 256 |
+
model_cfg = OmegaConf.load(
|
| 257 |
+
args.model_cfg or config.get("model_cfg", str(files("f5_tts").joinpath(f"configs/{model}.yaml")))
|
| 258 |
+
)
|
| 259 |
+
model_cls = get_class(f"f5_tts.model.{model_cfg.model.backbone}")
|
| 260 |
+
model_arc = model_cfg.model.arch
|
| 261 |
+
|
| 262 |
+
repo_name, ckpt_step, ckpt_type = "F5-TTS", 1250000, "safetensors"
|
| 263 |
+
|
| 264 |
+
if model != "F5TTS_Base":
|
| 265 |
+
assert vocoder_name == model_cfg.model.mel_spec.mel_spec_type
|
| 266 |
+
|
| 267 |
+
# override for previous models
|
| 268 |
+
if model == "F5TTS_Base":
|
| 269 |
+
if vocoder_name == "vocos":
|
| 270 |
+
ckpt_step = 1200000
|
| 271 |
+
elif vocoder_name == "bigvgan":
|
| 272 |
+
model = "F5TTS_Base_bigvgan"
|
| 273 |
+
ckpt_type = "pt"
|
| 274 |
+
elif model == "E2TTS_Base":
|
| 275 |
+
repo_name = "E2-TTS"
|
| 276 |
+
ckpt_step = 1200000
|
| 277 |
+
|
| 278 |
+
if not ckpt_file:
|
| 279 |
+
ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{model}/model_{ckpt_step}.{ckpt_type}"))
|
| 280 |
+
|
| 281 |
+
print(f"Using {model}...")
|
| 282 |
+
ema_model = load_model(
|
| 283 |
+
model_cls, model_arc, ckpt_file, mel_spec_type=vocoder_name, vocab_file=vocab_file, device=device
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
# inference process
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def main():
|
| 291 |
+
main_voice = {"ref_audio": ref_audio, "ref_text": ref_text}
|
| 292 |
+
if "voices" not in config:
|
| 293 |
+
voices = {"main": main_voice}
|
| 294 |
+
else:
|
| 295 |
+
voices = config["voices"]
|
| 296 |
+
voices["main"] = main_voice
|
| 297 |
+
for voice in voices:
|
| 298 |
+
print("Voice:", voice)
|
| 299 |
+
print("ref_audio ", voices[voice]["ref_audio"])
|
| 300 |
+
voices[voice]["ref_audio"], voices[voice]["ref_text"] = preprocess_ref_audio_text(
|
| 301 |
+
voices[voice]["ref_audio"], voices[voice]["ref_text"]
|
| 302 |
+
)
|
| 303 |
+
print("ref_audio_", voices[voice]["ref_audio"], "\n\n")
|
| 304 |
+
|
| 305 |
+
generated_audio_segments = []
|
| 306 |
+
reg1 = r"(?=\[\w+\])"
|
| 307 |
+
chunks = re.split(reg1, gen_text)
|
| 308 |
+
reg2 = r"\[(\w+)\]"
|
| 309 |
+
for text in chunks:
|
| 310 |
+
if not text.strip():
|
| 311 |
+
continue
|
| 312 |
+
match = re.match(reg2, text)
|
| 313 |
+
if match:
|
| 314 |
+
voice = match[1]
|
| 315 |
+
else:
|
| 316 |
+
print("No voice tag found, using main.")
|
| 317 |
+
voice = "main"
|
| 318 |
+
if voice not in voices:
|
| 319 |
+
print(f"Voice {voice} not found, using main.")
|
| 320 |
+
voice = "main"
|
| 321 |
+
text = re.sub(reg2, "", text)
|
| 322 |
+
ref_audio_ = voices[voice]["ref_audio"]
|
| 323 |
+
ref_text_ = voices[voice]["ref_text"]
|
| 324 |
+
gen_text_ = text.strip()
|
| 325 |
+
print(f"Voice: {voice}")
|
| 326 |
+
audio_segment, final_sample_rate, spectrogram = infer_process(
|
| 327 |
+
ref_audio_,
|
| 328 |
+
ref_text_,
|
| 329 |
+
gen_text_,
|
| 330 |
+
ema_model,
|
| 331 |
+
vocoder,
|
| 332 |
+
mel_spec_type=vocoder_name,
|
| 333 |
+
target_rms=target_rms,
|
| 334 |
+
cross_fade_duration=cross_fade_duration,
|
| 335 |
+
nfe_step=nfe_step,
|
| 336 |
+
cfg_strength=cfg_strength,
|
| 337 |
+
sway_sampling_coef=sway_sampling_coef,
|
| 338 |
+
speed=speed,
|
| 339 |
+
fix_duration=fix_duration,
|
| 340 |
+
device=device,
|
| 341 |
+
)
|
| 342 |
+
generated_audio_segments.append(audio_segment)
|
| 343 |
+
|
| 344 |
+
if save_chunk:
|
| 345 |
+
if len(gen_text_) > 200:
|
| 346 |
+
gen_text_ = gen_text_[:200] + " ... "
|
| 347 |
+
sf.write(
|
| 348 |
+
os.path.join(output_chunk_dir, f"{len(generated_audio_segments) - 1}_{gen_text_}.wav"),
|
| 349 |
+
audio_segment,
|
| 350 |
+
final_sample_rate,
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
if generated_audio_segments:
|
| 354 |
+
final_wave = np.concatenate(generated_audio_segments)
|
| 355 |
+
|
| 356 |
+
if not os.path.exists(output_dir):
|
| 357 |
+
os.makedirs(output_dir)
|
| 358 |
+
|
| 359 |
+
with open(wave_path, "wb") as f:
|
| 360 |
+
sf.write(f.name, final_wave, final_sample_rate)
|
| 361 |
+
# Remove silence
|
| 362 |
+
if remove_silence:
|
| 363 |
+
remove_silence_for_generated_wav(f.name)
|
| 364 |
+
print(f.name)
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
if __name__ == "__main__":
|
| 368 |
+
main()
|
mindfull_client.html
ADDED
|
@@ -0,0 +1,515 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 6 |
+
<title>Mindfull AI Avatar Chatbot</title>
|
| 7 |
+
<style>
|
| 8 |
+
* {
|
| 9 |
+
margin: 0;
|
| 10 |
+
padding: 0;
|
| 11 |
+
box-sizing: border-box;
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
body {
|
| 15 |
+
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
| 16 |
+
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
| 17 |
+
min-height: 100vh;
|
| 18 |
+
display: flex;
|
| 19 |
+
justify-content: center;
|
| 20 |
+
align-items: center;
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
.chat-container {
|
| 24 |
+
background: white;
|
| 25 |
+
border-radius: 20px;
|
| 26 |
+
box-shadow: 0 20px 40px rgba(0, 0, 0, 0.1);
|
| 27 |
+
width: 100%;
|
| 28 |
+
max-width: 800px;
|
| 29 |
+
height: 90vh;
|
| 30 |
+
display: flex;
|
| 31 |
+
flex-direction: column;
|
| 32 |
+
overflow: hidden;
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
.chat-header {
|
| 36 |
+
background: #4a5568;
|
| 37 |
+
color: white;
|
| 38 |
+
padding: 20px;
|
| 39 |
+
text-align: center;
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
.chat-header h1 {
|
| 43 |
+
margin-bottom: 5px;
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
.chat-header p {
|
| 47 |
+
opacity: 0.8;
|
| 48 |
+
font-size: 14px;
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
.status-bar {
|
| 52 |
+
background: #f7fafc;
|
| 53 |
+
padding: 10px 20px;
|
| 54 |
+
border-bottom: 1px solid #e2e8f0;
|
| 55 |
+
font-size: 12px;
|
| 56 |
+
color: #4a5568;
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
.status-indicator {
|
| 60 |
+
display: inline-block;
|
| 61 |
+
width: 8px;
|
| 62 |
+
height: 8px;
|
| 63 |
+
border-radius: 50%;
|
| 64 |
+
margin-right: 5px;
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
.status-connected {
|
| 68 |
+
background: #48bb78;
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
.status-disconnected {
|
| 72 |
+
background: #f56565;
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
.chat-messages {
|
| 76 |
+
flex: 1;
|
| 77 |
+
padding: 20px;
|
| 78 |
+
overflow-y: auto;
|
| 79 |
+
background: #f8f9fa;
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
.message {
|
| 83 |
+
margin-bottom: 20px;
|
| 84 |
+
display: flex;
|
| 85 |
+
align-items: flex-start;
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
.message.user {
|
| 89 |
+
justify-content: flex-end;
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
.message-content {
|
| 93 |
+
max-width: 70%;
|
| 94 |
+
padding: 15px;
|
| 95 |
+
border-radius: 15px;
|
| 96 |
+
position: relative;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
.message.user .message-content {
|
| 100 |
+
background: #4299e1;
|
| 101 |
+
color: white;
|
| 102 |
+
border-bottom-right-radius: 5px;
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
.message.bot .message-content {
|
| 106 |
+
background: white;
|
| 107 |
+
border: 1px solid #e2e8f0;
|
| 108 |
+
border-bottom-left-radius: 5px;
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
.message-meta {
|
| 112 |
+
font-size: 11px;
|
| 113 |
+
opacity: 0.7;
|
| 114 |
+
margin-top: 5px;
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
.emotion-tag {
|
| 118 |
+
display: inline-block;
|
| 119 |
+
background: #edf2f7;
|
| 120 |
+
color: #4a5568;
|
| 121 |
+
padding: 2px 8px;
|
| 122 |
+
border-radius: 10px;
|
| 123 |
+
font-size: 10px;
|
| 124 |
+
margin-left: 5px;
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
.media-content {
|
| 128 |
+
margin-top: 10px;
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
.audio-player {
|
| 132 |
+
width: 100%;
|
| 133 |
+
margin-top: 10px;
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
.video-player {
|
| 137 |
+
width: 100%;
|
| 138 |
+
max-width: 300px;
|
| 139 |
+
border-radius: 10px;
|
| 140 |
+
margin-top: 10px;
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
.chat-input {
|
| 144 |
+
padding: 20px;
|
| 145 |
+
background: white;
|
| 146 |
+
border-top: 1px solid #e2e8f0;
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
.input-container {
|
| 150 |
+
display: flex;
|
| 151 |
+
gap: 10px;
|
| 152 |
+
align-items: flex-end;
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
.message-input {
|
| 156 |
+
flex: 1;
|
| 157 |
+
border: 2px solid #e2e8f0;
|
| 158 |
+
border-radius: 25px;
|
| 159 |
+
padding: 15px 20px;
|
| 160 |
+
font-size: 14px;
|
| 161 |
+
resize: none;
|
| 162 |
+
max-height: 100px;
|
| 163 |
+
min-height: 50px;
|
| 164 |
+
outline: none;
|
| 165 |
+
transition: border-color 0.3s;
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
.message-input:focus {
|
| 169 |
+
border-color: #4299e1;
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
.send-button {
|
| 173 |
+
background: #4299e1;
|
| 174 |
+
color: white;
|
| 175 |
+
border: none;
|
| 176 |
+
border-radius: 50%;
|
| 177 |
+
width: 50px;
|
| 178 |
+
height: 50px;
|
| 179 |
+
cursor: pointer;
|
| 180 |
+
display: flex;
|
| 181 |
+
align-items: center;
|
| 182 |
+
justify-content: center;
|
| 183 |
+
transition: background 0.3s;
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
.send-button:hover:not(:disabled) {
|
| 187 |
+
background: #3182ce;
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
.send-button:disabled {
|
| 191 |
+
background: #a0aec0;
|
| 192 |
+
cursor: not-allowed;
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
.loading {
|
| 196 |
+
display: none;
|
| 197 |
+
text-align: center;
|
| 198 |
+
padding: 20px;
|
| 199 |
+
color: #4a5568;
|
| 200 |
+
}
|
| 201 |
+
|
| 202 |
+
.loading.show {
|
| 203 |
+
display: block;
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
.spinner {
|
| 207 |
+
display: inline-block;
|
| 208 |
+
width: 20px;
|
| 209 |
+
height: 20px;
|
| 210 |
+
border: 3px solid #e2e8f0;
|
| 211 |
+
border-radius: 50%;
|
| 212 |
+
border-top-color: #4299e1;
|
| 213 |
+
animation: spin 1s ease-in-out infinite;
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
@keyframes spin {
|
| 217 |
+
to { transform: rotate(360deg); }
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
.controls {
|
| 221 |
+
display: flex;
|
| 222 |
+
gap: 10px;
|
| 223 |
+
margin-top: 10px;
|
| 224 |
+
flex-wrap: wrap;
|
| 225 |
+
}
|
| 226 |
+
|
| 227 |
+
.control-checkbox {
|
| 228 |
+
display: flex;
|
| 229 |
+
align-items: center;
|
| 230 |
+
gap: 5px;
|
| 231 |
+
font-size: 12px;
|
| 232 |
+
color: #4a5568;
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
.error-message {
|
| 236 |
+
background: #fed7d7;
|
| 237 |
+
color: #c53030;
|
| 238 |
+
padding: 10px;
|
| 239 |
+
border-radius: 5px;
|
| 240 |
+
margin-bottom: 10px;
|
| 241 |
+
font-size: 12px;
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
@media (max-width: 768px) {
|
| 245 |
+
.chat-container {
|
| 246 |
+
height: 100vh;
|
| 247 |
+
border-radius: 0;
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
.message-content {
|
| 251 |
+
max-width: 90%;
|
| 252 |
+
}
|
| 253 |
+
}
|
| 254 |
+
</style>
|
| 255 |
+
</head>
|
| 256 |
+
<body>
|
| 257 |
+
<div class="chat-container">
|
| 258 |
+
<div class="chat-header">
|
| 259 |
+
<h1>🛡️ Mindfull</h1>
|
| 260 |
+
<p>AI Mental Wellness Assistant for Police Officers</p>
|
| 261 |
+
</div>
|
| 262 |
+
|
| 263 |
+
<div class="status-bar">
|
| 264 |
+
<span class="status-indicator status-disconnected" id="statusIndicator"></span>
|
| 265 |
+
<span id="statusText">Connecting...</span>
|
| 266 |
+
</div>
|
| 267 |
+
|
| 268 |
+
<div class="chat-messages" id="chatMessages">
|
| 269 |
+
<div class="message bot">
|
| 270 |
+
<div class="message-content">
|
| 271 |
+
<div>🙏 Namaskara! I'm Mindfull, your AI wellness assistant. I'm here to support you with any mental health concerns or stress you might be experiencing. How are you feeling today?</div>
|
| 272 |
+
<div class="message-meta">System message</div>
|
| 273 |
+
</div>
|
| 274 |
+
</div>
|
| 275 |
+
</div>
|
| 276 |
+
|
| 277 |
+
<div class="loading" id="loadingIndicator">
|
| 278 |
+
<div class="spinner"></div>
|
| 279 |
+
<span>Processing your message...</span>
|
| 280 |
+
</div>
|
| 281 |
+
|
| 282 |
+
<div class="chat-input">
|
| 283 |
+
<div class="error-message" id="errorMessage" style="display: none;"></div>
|
| 284 |
+
|
| 285 |
+
<div class="input-container">
|
| 286 |
+
<textarea
|
| 287 |
+
id="messageInput"
|
| 288 |
+
class="message-input"
|
| 289 |
+
placeholder="Share how you're feeling today..."
|
| 290 |
+
rows="1"
|
| 291 |
+
></textarea>
|
| 292 |
+
<button id="sendButton" class="send-button">
|
| 293 |
+
<svg width="20" height="20" viewBox="0 0 24 24" fill="currentColor">
|
| 294 |
+
<path d="M2.01 21L23 12 2.01 3 2 10l15 2-15 2z"/>
|
| 295 |
+
</svg>
|
| 296 |
+
</button>
|
| 297 |
+
</div>
|
| 298 |
+
|
| 299 |
+
<div class="controls">
|
| 300 |
+
<label class="control-checkbox">
|
| 301 |
+
<input type="checkbox" id="enableVideo" checked>
|
| 302 |
+
Generate Avatar Video
|
| 303 |
+
</label>
|
| 304 |
+
<label class="control-checkbox">
|
| 305 |
+
<input type="checkbox" id="autoPlay" checked>
|
| 306 |
+
Auto-play Audio
|
| 307 |
+
</label>
|
| 308 |
+
</div>
|
| 309 |
+
</div>
|
| 310 |
+
</div>
|
| 311 |
+
|
| 312 |
+
<script>
|
| 313 |
+
class MindfullChat {
|
| 314 |
+
constructor() {
|
| 315 |
+
this.apiUrl = 'http://localhost:5000';
|
| 316 |
+
this.sessionId = null;
|
| 317 |
+
this.isConnected = false;
|
| 318 |
+
|
| 319 |
+
this.initializeElements();
|
| 320 |
+
this.setupEventListeners();
|
| 321 |
+
this.checkConnection();
|
| 322 |
+
this.createSession();
|
| 323 |
+
}
|
| 324 |
+
|
| 325 |
+
initializeElements() {
|
| 326 |
+
this.messagesContainer = document.getElementById('chatMessages');
|
| 327 |
+
this.messageInput = document.getElementById('messageInput');
|
| 328 |
+
this.sendButton = document.getElementById('sendButton');
|
| 329 |
+
this.loadingIndicator = document.getElementById('loadingIndicator');
|
| 330 |
+
this.statusIndicator = document.getElementById('statusIndicator');
|
| 331 |
+
this.statusText = document.getElementById('statusText');
|
| 332 |
+
this.errorMessage = document.getElementById('errorMessage');
|
| 333 |
+
this.enableVideo = document.getElementById('enableVideo');
|
| 334 |
+
this.autoPlay = document.getElementById('autoPlay');
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
+
setupEventListeners() {
|
| 338 |
+
this.sendButton.addEventListener('click', () => this.sendMessage());
|
| 339 |
+
this.messageInput.addEventListener('keypress', (e) => {
|
| 340 |
+
if (e.key === 'Enter' && !e.shiftKey) {
|
| 341 |
+
e.preventDefault();
|
| 342 |
+
this.sendMessage();
|
| 343 |
+
}
|
| 344 |
+
});
|
| 345 |
+
|
| 346 |
+
// Auto-resize textarea
|
| 347 |
+
this.messageInput.addEventListener('input', () => {
|
| 348 |
+
this.messageInput.style.height = 'auto';
|
| 349 |
+
this.messageInput.style.height = this.messageInput.scrollHeight + 'px';
|
| 350 |
+
});
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
async checkConnection() {
|
| 354 |
+
try {
|
| 355 |
+
const response = await fetch(`${this.apiUrl}/health`);
|
| 356 |
+
if (response.ok) {
|
| 357 |
+
this.setConnectionStatus(true, 'Connected');
|
| 358 |
+
} else {
|
| 359 |
+
this.setConnectionStatus(false, 'API Error');
|
| 360 |
+
}
|
| 361 |
+
} catch (error) {
|
| 362 |
+
this.setConnectionStatus(false, 'Disconnected');
|
| 363 |
+
this.showError('Unable to connect to Mindfull API. Please ensure the server is running.');
|
| 364 |
+
}
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
async createSession() {
|
| 368 |
+
try {
|
| 369 |
+
const response = await fetch(`${this.apiUrl}/session`, {
|
| 370 |
+
method: 'POST',
|
| 371 |
+
headers: {
|
| 372 |
+
'Content-Type': 'application/json',
|
| 373 |
+
}
|
| 374 |
+
});
|
| 375 |
+
|
| 376 |
+
if (response.ok) {
|
| 377 |
+
const data = await response.json();
|
| 378 |
+
this.sessionId = data.session_id;
|
| 379 |
+
console.log('Session created:', this.sessionId);
|
| 380 |
+
}
|
| 381 |
+
} catch (error) {
|
| 382 |
+
console.error('Failed to create session:', error);
|
| 383 |
+
}
|
| 384 |
+
}
|
| 385 |
+
|
| 386 |
+
setConnectionStatus(connected, text) {
|
| 387 |
+
this.isConnected = connected;
|
| 388 |
+
this.statusIndicator.className = `status-indicator ${connected ? 'status-connected' : 'status-disconnected'}`;
|
| 389 |
+
this.statusText.textContent = text;
|
| 390 |
+
}
|
| 391 |
+
|
| 392 |
+
showError(message) {
|
| 393 |
+
this.errorMessage.textContent = message;
|
| 394 |
+
this.errorMessage.style.display = 'block';
|
| 395 |
+
setTimeout(() => {
|
| 396 |
+
this.errorMessage.style.display = 'none';
|
| 397 |
+
}, 5000);
|
| 398 |
+
}
|
| 399 |
+
|
| 400 |
+
addMessage(content, isUser = false, metadata = {}) {
|
| 401 |
+
const messageDiv = document.createElement('div');
|
| 402 |
+
messageDiv.className = `message ${isUser ? 'user' : 'bot'}`;
|
| 403 |
+
|
| 404 |
+
let metaInfo = '';
|
| 405 |
+
if (!isUser) {
|
| 406 |
+
const emotion = metadata.emotion ? `<span class="emotion-tag">${metadata.emotion}</span>` : '';
|
| 407 |
+
const processingTime = metadata.processingTime ? ` • ${metadata.processingTime}s` : '';
|
| 408 |
+
metaInfo = `<div class="message-meta">${new Date().toLocaleTimeString()}${emotion}${processingTime}</div>`;
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
let mediaContent = '';
|
| 412 |
+
if (metadata.audioUrl) {
|
| 413 |
+
mediaContent += `
|
| 414 |
+
<div class="media-content">
|
| 415 |
+
<audio controls class="audio-player" ${this.autoPlay.checked ? 'autoplay' : ''}>
|
| 416 |
+
<source src="${this.apiUrl}${metadata.audioUrl}" type="audio/wav">
|
| 417 |
+
Your browser does not support the audio element.
|
| 418 |
+
</audio>
|
| 419 |
+
</div>
|
| 420 |
+
`;
|
| 421 |
+
}
|
| 422 |
+
|
| 423 |
+
if (metadata.videoUrl) {
|
| 424 |
+
mediaContent += `
|
| 425 |
+
<div class="media-content">
|
| 426 |
+
<video controls class="video-player" ${this.autoPlay.checked ? 'autoplay' : ''}>
|
| 427 |
+
<source src="${this.apiUrl}${metadata.videoUrl}" type="video/mp4">
|
| 428 |
+
Your browser does not support the video element.
|
| 429 |
+
</video>
|
| 430 |
+
</div>
|
| 431 |
+
`;
|
| 432 |
+
}
|
| 433 |
+
|
| 434 |
+
messageDiv.innerHTML = `
|
| 435 |
+
<div class="message-content">
|
| 436 |
+
<div>${content}</div>
|
| 437 |
+
${metaInfo}
|
| 438 |
+
${mediaContent}
|
| 439 |
+
</div>
|
| 440 |
+
`;
|
| 441 |
+
|
| 442 |
+
this.messagesContainer.appendChild(messageDiv);
|
| 443 |
+
this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight;
|
| 444 |
+
}
|
| 445 |
+
|
| 446 |
+
setLoading(loading) {
|
| 447 |
+
this.loadingIndicator.classList.toggle('show', loading);
|
| 448 |
+
this.sendButton.disabled = loading;
|
| 449 |
+
|
| 450 |
+
if (loading) {
|
| 451 |
+
this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight;
|
| 452 |
+
}
|
| 453 |
+
}
|
| 454 |
+
|
| 455 |
+
async sendMessage() {
|
| 456 |
+
const message = this.messageInput.value.trim();
|
| 457 |
+
if (!message || !this.isConnected) return;
|
| 458 |
+
|
| 459 |
+
// Add user message
|
| 460 |
+
this.addMessage(message, true);
|
| 461 |
+
|
| 462 |
+
// Clear input
|
| 463 |
+
this.messageInput.value = '';
|
| 464 |
+
this.messageInput.style.height = 'auto';
|
| 465 |
+
|
| 466 |
+
// Show loading
|
| 467 |
+
this.setLoading(true);
|
| 468 |
+
|
| 469 |
+
try {
|
| 470 |
+
const response = await fetch(`${this.apiUrl}/chat`, {
|
| 471 |
+
method: 'POST',
|
| 472 |
+
headers: {
|
| 473 |
+
'Content-Type': 'application/json',
|
| 474 |
+
},
|
| 475 |
+
body: JSON.stringify({
|
| 476 |
+
message: message,
|
| 477 |
+
session_id: this.sessionId,
|
| 478 |
+
include_video: this.enableVideo.checked
|
| 479 |
+
})
|
| 480 |
+
});
|
| 481 |
+
|
| 482 |
+
if (response.ok) {
|
| 483 |
+
const data = await response.json();
|
| 484 |
+
|
| 485 |
+
// Add bot response
|
| 486 |
+
this.addMessage(data.response_text, false, {
|
| 487 |
+
emotion: data.emotion_detected,
|
| 488 |
+
processingTime: data.processing_time?.toFixed(2),
|
| 489 |
+
audioUrl: data.audio_url,
|
| 490 |
+
videoUrl: data.video_url
|
| 491 |
+
});
|
| 492 |
+
|
| 493 |
+
if (data.warnings && data.warnings.length > 0) {
|
| 494 |
+
console.warn('Warnings:', data.warnings);
|
| 495 |
+
}
|
| 496 |
+
} else {
|
| 497 |
+
const errorData = await response.json();
|
| 498 |
+
this.showError(errorData.message || 'Failed to send message');
|
| 499 |
+
}
|
| 500 |
+
} catch (error) {
|
| 501 |
+
console.error('Send message error:', error);
|
| 502 |
+
this.showError('Network error. Please try again.');
|
| 503 |
+
} finally {
|
| 504 |
+
this.setLoading(false);
|
| 505 |
+
}
|
| 506 |
+
}
|
| 507 |
+
}
|
| 508 |
+
|
| 509 |
+
// Initialize chat when page loads
|
| 510 |
+
document.addEventListener('DOMContentLoaded', () => {
|
| 511 |
+
new MindfullChat();
|
| 512 |
+
});
|
| 513 |
+
</script>
|
| 514 |
+
</body>
|
| 515 |
+
</html>
|
mindfull_config.py
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Mindfull AI Avatar Chatbot Configuration
|
| 3 |
+
Comprehensive configuration for Ollama, F5-TTS, and SadTalker integration
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import logging
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Dict, Any, Optional
|
| 10 |
+
|
| 11 |
+
class MindfullConfig:
|
| 12 |
+
"""Configuration class for Mindfull AI Avatar Chatbot"""
|
| 13 |
+
|
| 14 |
+
# =========================================================================
|
| 15 |
+
# ENVIRONMENT CONFIGURATION
|
| 16 |
+
# =========================================================================
|
| 17 |
+
|
| 18 |
+
# Project Paths
|
| 19 |
+
PROJECT_ROOT = Path(__file__).parent.absolute()
|
| 20 |
+
VENV_PATH = PROJECT_ROOT / "venv"
|
| 21 |
+
PYTHON_EXECUTABLE = (
|
| 22 |
+
VENV_PATH / "Scripts" / "python.exe" if os.name == 'nt'
|
| 23 |
+
else VENV_PATH / "bin" / "python"
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
# Output Directories
|
| 27 |
+
OUTPUT_DIR = PROJECT_ROOT / "outputs"
|
| 28 |
+
AUDIO_OUTPUT_DIR = OUTPUT_DIR / "audio"
|
| 29 |
+
VIDEO_OUTPUT_DIR = OUTPUT_DIR / "video"
|
| 30 |
+
TEMP_DIR = OUTPUT_DIR / "temp"
|
| 31 |
+
|
| 32 |
+
# Avatar Assets
|
| 33 |
+
AVATAR_ASSETS_DIR = PROJECT_ROOT / "avatar_assets"
|
| 34 |
+
DEFAULT_AVATAR_IMAGE = AVATAR_ASSETS_DIR / "officer.png"
|
| 35 |
+
|
| 36 |
+
# Create directories if they don't exist
|
| 37 |
+
for directory in [OUTPUT_DIR, AUDIO_OUTPUT_DIR, VIDEO_OUTPUT_DIR, TEMP_DIR]:
|
| 38 |
+
directory.mkdir(parents=True, exist_ok=True)
|
| 39 |
+
|
| 40 |
+
# =========================================================================
|
| 41 |
+
# OLLAMA CONFIGURATION
|
| 42 |
+
# =========================================================================
|
| 43 |
+
|
| 44 |
+
# Ollama Settings
|
| 45 |
+
OLLAMA_BASE_URL = "http://localhost:11434"
|
| 46 |
+
OLLAMA_GENERATE_URL = f"{OLLAMA_BASE_URL}/api/generate"
|
| 47 |
+
OLLAMA_MODELS_URL = f"{OLLAMA_BASE_URL}/api/tags"
|
| 48 |
+
OLLAMA_TIMEOUT = 60
|
| 49 |
+
|
| 50 |
+
# Model Configuration
|
| 51 |
+
DEFAULT_MODEL = "mindfull"
|
| 52 |
+
FALLBACK_MODEL = "mistral:7b"
|
| 53 |
+
|
| 54 |
+
# Model Parameters
|
| 55 |
+
MODEL_PARAMS = {
|
| 56 |
+
"temperature": 0.7,
|
| 57 |
+
"top_p": 0.9,
|
| 58 |
+
"top_k": 40,
|
| 59 |
+
"repeat_penalty": 1.1,
|
| 60 |
+
"num_ctx": 2048,
|
| 61 |
+
"stream": False
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
# =========================================================================
|
| 65 |
+
# F5-TTS CONFIGURATION
|
| 66 |
+
# =========================================================================
|
| 67 |
+
|
| 68 |
+
# F5-TTS Model Paths
|
| 69 |
+
F5TTS_MODEL_DIR = PROJECT_ROOT / "my_finetuned_model"
|
| 70 |
+
F5TTS_MODEL_PATH = F5TTS_MODEL_DIR / "model.pth"
|
| 71 |
+
F5TTS_CONFIG_PATH = F5TTS_MODEL_DIR / "config.json"
|
| 72 |
+
F5TTS_VOCAB_PATH = F5TTS_MODEL_DIR / "vocab.json"
|
| 73 |
+
|
| 74 |
+
# Reference Audio for Voice Cloning
|
| 75 |
+
REFERENCE_AUDIO_DIR = PROJECT_ROOT / "datasets-1" / "wavs"
|
| 76 |
+
DEFAULT_REFERENCE_AUDIO = REFERENCE_AUDIO_DIR / "0029.wav"
|
| 77 |
+
REFERENCE_TEXT = "Namaskara! How are you feeling today? I'm here to support you."
|
| 78 |
+
|
| 79 |
+
# TTS Settings
|
| 80 |
+
TTS_SETTINGS = {
|
| 81 |
+
"language": "en",
|
| 82 |
+
"speed": 1.0,
|
| 83 |
+
"remove_silence": True,
|
| 84 |
+
"normalize_audio": True,
|
| 85 |
+
"sample_rate": 24000
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
# =========================================================================
|
| 89 |
+
# SADTALKER CONFIGURATION
|
| 90 |
+
# =========================================================================
|
| 91 |
+
|
| 92 |
+
# SadTalker Paths
|
| 93 |
+
SADTALKER_DIR = PROJECT_ROOT / "sadtalker+wav2lip"
|
| 94 |
+
SADTALKER_SCRIPT = SADTALKER_DIR / "simple_pipeline.py"
|
| 95 |
+
SADTALKER_CHECKPOINTS = SADTALKER_DIR / "sadtalker" / "checkpoints"
|
| 96 |
+
|
| 97 |
+
# SadTalker Settings
|
| 98 |
+
SADTALKER_SETTINGS = {
|
| 99 |
+
"pose_style": 1,
|
| 100 |
+
"exp_scale": 1.3,
|
| 101 |
+
"use_enhancer": True,
|
| 102 |
+
"use_face_parse": True,
|
| 103 |
+
"background_enhance": True,
|
| 104 |
+
"face_restore": True,
|
| 105 |
+
"face_model": "normal",
|
| 106 |
+
"preprocess": "crop",
|
| 107 |
+
"still": False, # *** CRITICAL: This was True causing static images! ***
|
| 108 |
+
"use_idle_mode": False,
|
| 109 |
+
"length_of_audio": 0,
|
| 110 |
+
"use_ref_video": False,
|
| 111 |
+
"ref_video": None,
|
| 112 |
+
"ref_info": None,
|
| 113 |
+
"use_idle_mode": False,
|
| 114 |
+
"length_of_audio": 0
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
# =========================================================================
|
| 118 |
+
# WEB API CONFIGURATION
|
| 119 |
+
# =========================================================================
|
| 120 |
+
|
| 121 |
+
# Web Server Settings
|
| 122 |
+
WEB_HOST = "0.0.0.0"
|
| 123 |
+
WEB_PORT = 5000
|
| 124 |
+
WEB_DEBUG = False
|
| 125 |
+
WEB_THREADED = True
|
| 126 |
+
|
| 127 |
+
# CORS Settings
|
| 128 |
+
CORS_ORIGINS = ["*"]
|
| 129 |
+
CORS_METHODS = ["GET", "POST", "PUT", "DELETE", "OPTIONS"]
|
| 130 |
+
CORS_HEADERS = ["Content-Type", "Authorization"]
|
| 131 |
+
|
| 132 |
+
# =========================================================================
|
| 133 |
+
# SECURITY AND VALIDATION
|
| 134 |
+
# =========================================================================
|
| 135 |
+
|
| 136 |
+
# Input Validation
|
| 137 |
+
MAX_TEXT_LENGTH = 500
|
| 138 |
+
MIN_TEXT_LENGTH = 1
|
| 139 |
+
ALLOWED_LANGUAGES = ["en", "kn"] # English and Kannada
|
| 140 |
+
|
| 141 |
+
# Rate Limiting
|
| 142 |
+
RATE_LIMIT_REQUESTS = 60 # requests per minute
|
| 143 |
+
RATE_LIMIT_WINDOW = 60 # seconds
|
| 144 |
+
|
| 145 |
+
# Session Management
|
| 146 |
+
SESSION_TIMEOUT = 3600 # 1 hour
|
| 147 |
+
MAX_CONCURRENT_SESSIONS = 10
|
| 148 |
+
|
| 149 |
+
# =========================================================================
|
| 150 |
+
# EMOTIONAL ANALYSIS
|
| 151 |
+
# =========================================================================
|
| 152 |
+
|
| 153 |
+
# Emotion Keywords
|
| 154 |
+
EMOTION_KEYWORDS = {
|
| 155 |
+
"stress": [
|
| 156 |
+
"stressed", "pressure", "overwhelmed", "burned out", "exhausted",
|
| 157 |
+
"tense", "worried", "anxious", "strained", "difficult", "hard",
|
| 158 |
+
"challenging", "demanding", "intense", "tough"
|
| 159 |
+
],
|
| 160 |
+
"sadness": [
|
| 161 |
+
"sad", "depressed", "down", "low", "unhappy", "miserable",
|
| 162 |
+
"lonely", "isolated", "hopeless", "discouraged", "disappointed",
|
| 163 |
+
"upset", "hurt", "broken", "empty"
|
| 164 |
+
],
|
| 165 |
+
"anger": [
|
| 166 |
+
"angry", "furious", "frustrated", "irritated", "annoyed",
|
| 167 |
+
"mad", "enraged", "outraged", "livid", "aggravated",
|
| 168 |
+
"resentful", "bitter", "hostile"
|
| 169 |
+
],
|
| 170 |
+
"anxiety": [
|
| 171 |
+
"anxious", "nervous", "panicked", "worried", "fearful",
|
| 172 |
+
"scared", "terrified", "uneasy", "restless", "agitated",
|
| 173 |
+
"on edge", "jumpy", "troubled"
|
| 174 |
+
],
|
| 175 |
+
"positive": [
|
| 176 |
+
"good", "great", "happy", "fine", "okay", "well", "excellent",
|
| 177 |
+
"fantastic", "wonderful", "amazing", "content", "peaceful",
|
| 178 |
+
"calm", "relaxed", "energetic", "motivated", "confident"
|
| 179 |
+
]
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
# =========================================================================
|
| 183 |
+
# WELLNESS INTERVENTIONS
|
| 184 |
+
# =========================================================================
|
| 185 |
+
|
| 186 |
+
# Wellness Suggestions
|
| 187 |
+
WELLNESS_INTERVENTIONS = {
|
| 188 |
+
"stress": [
|
| 189 |
+
"Take 5 deep, slow breaths",
|
| 190 |
+
"Step away for a 2-minute break",
|
| 191 |
+
"Drink some water and stay hydrated",
|
| 192 |
+
"Do some shoulder rolls and neck stretches",
|
| 193 |
+
"Focus on one task at a time",
|
| 194 |
+
"Practice the 4-7-8 breathing technique"
|
| 195 |
+
],
|
| 196 |
+
"sadness": [
|
| 197 |
+
"Remember that your service makes a real difference",
|
| 198 |
+
"Consider talking to a trusted colleague",
|
| 199 |
+
"Reach out to the Well-Being Officer when ready",
|
| 200 |
+
"Take a short walk if possible",
|
| 201 |
+
"Practice self-compassion - you're doing important work",
|
| 202 |
+
"Listen to some calming music during your break"
|
| 203 |
+
],
|
| 204 |
+
"anger": [
|
| 205 |
+
"Take 10 deep breaths before responding",
|
| 206 |
+
"Count to 10 slowly in your mind",
|
| 207 |
+
"Step away from the situation if safely possible",
|
| 208 |
+
"Use the STOP technique: Stop, Take a breath, Observe, Proceed mindfully",
|
| 209 |
+
"Channel that energy into problem-solving",
|
| 210 |
+
"Remember your training and protocols"
|
| 211 |
+
],
|
| 212 |
+
"anxiety": [
|
| 213 |
+
"Use the 5-4-3-2-1 grounding technique",
|
| 214 |
+
"Focus on what you can control right now",
|
| 215 |
+
"Practice progressive muscle relaxation",
|
| 216 |
+
"Remind yourself of your training and capabilities",
|
| 217 |
+
"Take slow, controlled breaths",
|
| 218 |
+
"Connect with your support network"
|
| 219 |
+
],
|
| 220 |
+
"general": [
|
| 221 |
+
"Maintain regular meal times",
|
| 222 |
+
"Stay hydrated throughout your shift",
|
| 223 |
+
"Take brief mental breaks when possible",
|
| 224 |
+
"Practice good posture",
|
| 225 |
+
"Get adequate rest between shifts",
|
| 226 |
+
"Remember your important role in the community"
|
| 227 |
+
]
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
# =========================================================================
|
| 231 |
+
# LOGGING CONFIGURATION
|
| 232 |
+
# =========================================================================
|
| 233 |
+
|
| 234 |
+
# Logging Settings
|
| 235 |
+
LOG_LEVEL = logging.INFO
|
| 236 |
+
LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
| 237 |
+
LOG_FILE = OUTPUT_DIR / "mindfull.log"
|
| 238 |
+
|
| 239 |
+
# =========================================================================
|
| 240 |
+
# ERROR HANDLING
|
| 241 |
+
# =========================================================================
|
| 242 |
+
|
| 243 |
+
# Retry Settings
|
| 244 |
+
MAX_RETRIES = 3
|
| 245 |
+
RETRY_DELAY = 1 # seconds
|
| 246 |
+
|
| 247 |
+
# Fallback Messages
|
| 248 |
+
FALLBACK_MESSAGES = {
|
| 249 |
+
"ollama_error": "I'm having trouble connecting right now. Please try again in a moment.",
|
| 250 |
+
"tts_error": "I'm having trouble with speech generation. Let me try a different approach.",
|
| 251 |
+
"avatar_error": "The video generation is temporarily unavailable, but I can still help you.",
|
| 252 |
+
"general_error": "I encountered an issue, but I'm here to help. Please try again."
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
@classmethod
|
| 256 |
+
def validate_paths(cls) -> bool:
|
| 257 |
+
"""Validate that all required paths exist"""
|
| 258 |
+
required_paths = [
|
| 259 |
+
cls.F5TTS_MODEL_PATH,
|
| 260 |
+
cls.F5TTS_CONFIG_PATH,
|
| 261 |
+
cls.DEFAULT_REFERENCE_AUDIO,
|
| 262 |
+
cls.DEFAULT_AVATAR_IMAGE
|
| 263 |
+
]
|
| 264 |
+
|
| 265 |
+
missing_paths = [path for path in required_paths if not path.exists()]
|
| 266 |
+
|
| 267 |
+
if missing_paths:
|
| 268 |
+
print(f"Warning: Missing required files: {missing_paths}")
|
| 269 |
+
return False
|
| 270 |
+
return True
|
| 271 |
+
|
| 272 |
+
@classmethod
|
| 273 |
+
def get_model_status(cls) -> Dict[str, bool]:
|
| 274 |
+
"""Check status of all models"""
|
| 275 |
+
return {
|
| 276 |
+
"f5tts_model": cls.F5TTS_MODEL_PATH.exists(),
|
| 277 |
+
"f5tts_config": cls.F5TTS_CONFIG_PATH.exists(),
|
| 278 |
+
"reference_audio": cls.DEFAULT_REFERENCE_AUDIO.exists(),
|
| 279 |
+
"avatar_image": cls.DEFAULT_AVATAR_IMAGE.exists(),
|
| 280 |
+
"sadtalker_script": cls.SADTALKER_SCRIPT.exists()
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
# Create global config instance
|
| 284 |
+
config = MindfullConfig()
|
| 285 |
+
|
| 286 |
+
# Configure logging
|
| 287 |
+
logging.basicConfig(
|
| 288 |
+
level=config.LOG_LEVEL,
|
| 289 |
+
format=config.LOG_FORMAT,
|
| 290 |
+
handlers=[
|
| 291 |
+
logging.FileHandler(config.LOG_FILE, encoding='utf-8'),
|
| 292 |
+
logging.StreamHandler()
|
| 293 |
+
]
|
| 294 |
+
)
|
| 295 |
+
|
| 296 |
+
logger = logging.getLogger(__name__)
|
| 297 |
+
|
| 298 |
+
if __name__ == "__main__":
|
| 299 |
+
print("Mindfull Configuration Status:")
|
| 300 |
+
print("=" * 50)
|
| 301 |
+
print(f"Project Root: {config.PROJECT_ROOT}")
|
| 302 |
+
print(f"Python Executable: {config.PYTHON_EXECUTABLE}")
|
| 303 |
+
print(f"Output Directory: {config.OUTPUT_DIR}")
|
| 304 |
+
print("\nModel Status:")
|
| 305 |
+
for model, status in config.get_model_status().items():
|
| 306 |
+
status_icon = "✅" if status else "❌"
|
| 307 |
+
print(f"{status_icon} {model}: {status}")
|
| 308 |
+
print(f"\nPath Validation: {'✅ Passed' if config.validate_paths() else '❌ Failed'}")
|
mindfull_pipeline.py
ADDED
|
@@ -0,0 +1,478 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Mindfull AI Avatar Chatbot - Main Pipeline
|
| 4 |
+
Integrates Ollama (Mistral), F5-TTS, and SadTalker for complete AI avatar experience
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
import json
|
| 10 |
+
import time
|
| 11 |
+
import logging
|
| 12 |
+
import asyncio
|
| 13 |
+
import subprocess
|
| 14 |
+
import threading
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Dict, Any, Optional, Tuple
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from datetime import datetime
|
| 19 |
+
|
| 20 |
+
import requests
|
| 21 |
+
import torch
|
| 22 |
+
import numpy as np
|
| 23 |
+
import soundfile as sf
|
| 24 |
+
from PIL import Image
|
| 25 |
+
|
| 26 |
+
# Import configuration
|
| 27 |
+
from mindfull_config import config, logger
|
| 28 |
+
|
| 29 |
+
@dataclass
|
| 30 |
+
class ChatSession:
|
| 31 |
+
"""Represents a chat session with the user"""
|
| 32 |
+
session_id: str
|
| 33 |
+
user_name: str = ""
|
| 34 |
+
user_rank: str = ""
|
| 35 |
+
start_time: Optional[datetime] = None
|
| 36 |
+
last_interaction: Optional[datetime] = None
|
| 37 |
+
conversation_history: Optional[list] = None
|
| 38 |
+
|
| 39 |
+
def __post_init__(self):
|
| 40 |
+
if self.start_time is None:
|
| 41 |
+
self.start_time = datetime.now()
|
| 42 |
+
if self.last_interaction is None:
|
| 43 |
+
self.last_interaction = datetime.now()
|
| 44 |
+
if self.conversation_history is None:
|
| 45 |
+
self.conversation_history = []
|
| 46 |
+
|
| 47 |
+
class MindfullPipeline:
|
| 48 |
+
"""Main pipeline for Mindfull AI Avatar Chatbot"""
|
| 49 |
+
|
| 50 |
+
def __init__(self):
|
| 51 |
+
self.session = None
|
| 52 |
+
self.is_initialized = False
|
| 53 |
+
self.ollama_available = False
|
| 54 |
+
self.f5tts_available = False
|
| 55 |
+
self.sadtalker_available = False
|
| 56 |
+
|
| 57 |
+
logger.info("Initializing Mindfull AI Avatar Chatbot...")
|
| 58 |
+
self.initialize_components()
|
| 59 |
+
|
| 60 |
+
def initialize_components(self) -> bool:
|
| 61 |
+
"""Initialize all pipeline components"""
|
| 62 |
+
try:
|
| 63 |
+
# Check Ollama availability
|
| 64 |
+
self.ollama_available = self._check_ollama()
|
| 65 |
+
|
| 66 |
+
# Check F5-TTS model availability
|
| 67 |
+
self.f5tts_available = self._check_f5tts()
|
| 68 |
+
|
| 69 |
+
# Check SadTalker availability
|
| 70 |
+
self.sadtalker_available = self._check_sadtalker()
|
| 71 |
+
|
| 72 |
+
# Validate paths
|
| 73 |
+
path_validation = config.validate_paths()
|
| 74 |
+
|
| 75 |
+
self.is_initialized = (
|
| 76 |
+
self.ollama_available and
|
| 77 |
+
self.f5tts_available and
|
| 78 |
+
path_validation
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
if self.is_initialized:
|
| 82 |
+
logger.info("✅ Mindfull pipeline initialized successfully!")
|
| 83 |
+
self._log_component_status()
|
| 84 |
+
else:
|
| 85 |
+
logger.error("❌ Failed to initialize Mindfull pipeline")
|
| 86 |
+
self._log_component_status()
|
| 87 |
+
|
| 88 |
+
return self.is_initialized
|
| 89 |
+
|
| 90 |
+
except Exception as e:
|
| 91 |
+
logger.error(f"Error during initialization: {str(e)}")
|
| 92 |
+
return False
|
| 93 |
+
|
| 94 |
+
def _check_ollama(self) -> bool:
|
| 95 |
+
"""Check if Ollama is running and model is available"""
|
| 96 |
+
try:
|
| 97 |
+
# Check if Ollama is running
|
| 98 |
+
response = requests.get(config.OLLAMA_MODELS_URL, timeout=5)
|
| 99 |
+
if response.status_code != 200:
|
| 100 |
+
logger.error("Ollama server is not responding")
|
| 101 |
+
return False
|
| 102 |
+
|
| 103 |
+
# Check if Mindfull model exists
|
| 104 |
+
models = response.json().get('models', [])
|
| 105 |
+
model_names = [model.get('name', '') for model in models]
|
| 106 |
+
|
| 107 |
+
if config.DEFAULT_MODEL in model_names:
|
| 108 |
+
logger.info(f"✅ Found {config.DEFAULT_MODEL} model")
|
| 109 |
+
return True
|
| 110 |
+
elif config.FALLBACK_MODEL in model_names:
|
| 111 |
+
logger.warning(f"⚠️ Using fallback model: {config.FALLBACK_MODEL}")
|
| 112 |
+
return True
|
| 113 |
+
else:
|
| 114 |
+
logger.error(f"❌ Neither {config.DEFAULT_MODEL} nor {config.FALLBACK_MODEL} found")
|
| 115 |
+
logger.info(f"Available models: {model_names}")
|
| 116 |
+
return False
|
| 117 |
+
|
| 118 |
+
except requests.exceptions.RequestException as e:
|
| 119 |
+
logger.error(f"Failed to connect to Ollama: {str(e)}")
|
| 120 |
+
return False
|
| 121 |
+
|
| 122 |
+
def _check_f5tts(self) -> bool:
|
| 123 |
+
"""Check if F5-TTS model files are available"""
|
| 124 |
+
required_files = [
|
| 125 |
+
config.F5TTS_MODEL_PATH,
|
| 126 |
+
config.F5TTS_CONFIG_PATH,
|
| 127 |
+
config.DEFAULT_REFERENCE_AUDIO
|
| 128 |
+
]
|
| 129 |
+
|
| 130 |
+
missing_files = [f for f in required_files if not f.exists()]
|
| 131 |
+
|
| 132 |
+
if missing_files:
|
| 133 |
+
logger.error(f"❌ Missing F5-TTS files: {missing_files}")
|
| 134 |
+
return False
|
| 135 |
+
|
| 136 |
+
logger.info("✅ F5-TTS model files found")
|
| 137 |
+
return True
|
| 138 |
+
|
| 139 |
+
def _check_sadtalker(self) -> bool:
|
| 140 |
+
"""Check if SadTalker is available"""
|
| 141 |
+
if not config.SADTALKER_SCRIPT.exists():
|
| 142 |
+
logger.warning("⚠️ SadTalker script not found - video generation disabled")
|
| 143 |
+
return False
|
| 144 |
+
|
| 145 |
+
if not config.DEFAULT_AVATAR_IMAGE.exists():
|
| 146 |
+
logger.warning("⚠️ Default avatar image not found")
|
| 147 |
+
return False
|
| 148 |
+
|
| 149 |
+
logger.info("✅ SadTalker components found")
|
| 150 |
+
return True
|
| 151 |
+
|
| 152 |
+
def _log_component_status(self):
|
| 153 |
+
"""Log the status of all components"""
|
| 154 |
+
logger.info("Component Status:")
|
| 155 |
+
logger.info(f" Ollama: {'✅' if self.ollama_available else '❌'}")
|
| 156 |
+
logger.info(f" F5-TTS: {'✅' if self.f5tts_available else '❌'}")
|
| 157 |
+
logger.info(f" SadTalker: {'✅' if self.sadtalker_available else '❌'}")
|
| 158 |
+
|
| 159 |
+
def create_session(self) -> str:
|
| 160 |
+
"""Create a new chat session"""
|
| 161 |
+
session_id = f"mindfull_session_{int(time.time())}"
|
| 162 |
+
self.session = ChatSession(session_id=session_id)
|
| 163 |
+
logger.info(f"Created new session: {session_id}")
|
| 164 |
+
return session_id
|
| 165 |
+
|
| 166 |
+
def generate_response(self, user_input: str) -> Tuple[str, str]:
|
| 167 |
+
"""
|
| 168 |
+
Generate response using Ollama
|
| 169 |
+
Returns: (response_text, emotion_detected)
|
| 170 |
+
"""
|
| 171 |
+
try:
|
| 172 |
+
# Detect emotion from user input
|
| 173 |
+
emotion = self._detect_emotion(user_input)
|
| 174 |
+
|
| 175 |
+
# Prepare the prompt with context
|
| 176 |
+
prompt = self._prepare_prompt(user_input, emotion)
|
| 177 |
+
|
| 178 |
+
# Make request to Ollama
|
| 179 |
+
payload = {
|
| 180 |
+
"model": config.DEFAULT_MODEL,
|
| 181 |
+
"prompt": prompt,
|
| 182 |
+
"stream": False,
|
| 183 |
+
**config.MODEL_PARAMS
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
response = requests.post(
|
| 187 |
+
config.OLLAMA_GENERATE_URL,
|
| 188 |
+
json=payload,
|
| 189 |
+
timeout=config.OLLAMA_TIMEOUT
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
if response.status_code == 200:
|
| 193 |
+
result = response.json()
|
| 194 |
+
response_text = result.get('response', '').strip()
|
| 195 |
+
|
| 196 |
+
# Add to conversation history
|
| 197 |
+
if self.session and self.session.conversation_history is not None:
|
| 198 |
+
self.session.conversation_history.append({
|
| 199 |
+
'timestamp': datetime.now().isoformat(),
|
| 200 |
+
'user_input': user_input,
|
| 201 |
+
'response': response_text,
|
| 202 |
+
'emotion': emotion
|
| 203 |
+
})
|
| 204 |
+
self.session.last_interaction = datetime.now()
|
| 205 |
+
|
| 206 |
+
logger.info(f"Generated response for emotion '{emotion}': {response_text[:100]}...")
|
| 207 |
+
return response_text, emotion
|
| 208 |
+
else:
|
| 209 |
+
logger.error(f"Ollama request failed: {response.status_code}")
|
| 210 |
+
return config.FALLBACK_MESSAGES["ollama_error"], "neutral"
|
| 211 |
+
|
| 212 |
+
except Exception as e:
|
| 213 |
+
logger.error(f"Error generating response: {str(e)}")
|
| 214 |
+
return config.FALLBACK_MESSAGES["general_error"], "neutral"
|
| 215 |
+
|
| 216 |
+
def _detect_emotion(self, text: str) -> str:
|
| 217 |
+
"""Detect emotion from user input using keyword matching"""
|
| 218 |
+
text_lower = text.lower()
|
| 219 |
+
|
| 220 |
+
emotion_scores = {}
|
| 221 |
+
for emotion, keywords in config.EMOTION_KEYWORDS.items():
|
| 222 |
+
score = sum(1 for keyword in keywords if keyword in text_lower)
|
| 223 |
+
if score > 0:
|
| 224 |
+
emotion_scores[emotion] = score
|
| 225 |
+
|
| 226 |
+
if not emotion_scores:
|
| 227 |
+
return "neutral"
|
| 228 |
+
|
| 229 |
+
# Return emotion with highest score
|
| 230 |
+
if emotion_scores:
|
| 231 |
+
detected_emotion = max(emotion_scores, key=lambda x: emotion_scores[x])
|
| 232 |
+
logger.info(f"Detected emotion: {detected_emotion} (scores: {emotion_scores})")
|
| 233 |
+
return detected_emotion
|
| 234 |
+
|
| 235 |
+
def _prepare_prompt(self, user_input: str, emotion: str) -> str:
|
| 236 |
+
"""Prepare the prompt for Ollama with context"""
|
| 237 |
+
# Get wellness suggestions for the detected emotion
|
| 238 |
+
suggestions = config.WELLNESS_INTERVENTIONS.get(emotion,
|
| 239 |
+
config.WELLNESS_INTERVENTIONS["general"])
|
| 240 |
+
|
| 241 |
+
context = f"""
|
| 242 |
+
User's emotional state: {emotion}
|
| 243 |
+
Available wellness suggestions: {', '.join(suggestions[:3])}
|
| 244 |
+
|
| 245 |
+
User message: {user_input}
|
| 246 |
+
|
| 247 |
+
Respond as Mindfull, keeping in mind:
|
| 248 |
+
1. The user's emotional state is {emotion}
|
| 249 |
+
2. Provide appropriate support and suggestions
|
| 250 |
+
3. Keep response concise (2-3 sentences) for voice output
|
| 251 |
+
4. Be warm and professional
|
| 252 |
+
"""
|
| 253 |
+
return context
|
| 254 |
+
|
| 255 |
+
def generate_audio(self, text: str) -> Optional[str]:
|
| 256 |
+
"""
|
| 257 |
+
Generate audio using F5-TTS with fallback methods
|
| 258 |
+
Returns: path to generated audio file or None if failed
|
| 259 |
+
"""
|
| 260 |
+
try:
|
| 261 |
+
# Import our simple audio generation
|
| 262 |
+
from simple_audio_gen import generate_audio_simple, generate_audio_minimal
|
| 263 |
+
|
| 264 |
+
# Prepare output filename
|
| 265 |
+
timestamp = int(time.time())
|
| 266 |
+
audio_filename = f"mindfull_response_{timestamp}.wav"
|
| 267 |
+
audio_path = config.AUDIO_OUTPUT_DIR / audio_filename
|
| 268 |
+
|
| 269 |
+
logger.info(f"Generating audio with F5-TTS: {text[:50]}...")
|
| 270 |
+
|
| 271 |
+
# Try F5-TTS, then minimal fallback
|
| 272 |
+
if generate_audio_simple(text, str(audio_path)):
|
| 273 |
+
logger.info(f"✅ F5-TTS audio generated: {audio_path}")
|
| 274 |
+
return str(audio_path)
|
| 275 |
+
elif generate_audio_minimal(text, str(audio_path)):
|
| 276 |
+
logger.info(f"✅ Minimal audio generated: {audio_path}")
|
| 277 |
+
return str(audio_path)
|
| 278 |
+
else:
|
| 279 |
+
logger.error("All audio generation methods failed")
|
| 280 |
+
return None
|
| 281 |
+
|
| 282 |
+
except Exception as e:
|
| 283 |
+
logger.error(f"Error generating audio: {str(e)}")
|
| 284 |
+
return None
|
| 285 |
+
|
| 286 |
+
def generate_avatar_video(self, audio_path: str,
|
| 287 |
+
avatar_image: Optional[str] = None) -> Optional[str]:
|
| 288 |
+
"""
|
| 289 |
+
Generate avatar video using SadTalker with fallback methods
|
| 290 |
+
Returns: path to generated video file or None if failed
|
| 291 |
+
"""
|
| 292 |
+
try:
|
| 293 |
+
from simple_video_gen import generate_video_simple
|
| 294 |
+
|
| 295 |
+
if not audio_path or not Path(audio_path).exists():
|
| 296 |
+
logger.error(f"Audio file not found: {audio_path}")
|
| 297 |
+
return None
|
| 298 |
+
|
| 299 |
+
# Use default avatar image if none provided
|
| 300 |
+
if avatar_image is None:
|
| 301 |
+
avatar_image = str(config.DEFAULT_AVATAR_IMAGE)
|
| 302 |
+
|
| 303 |
+
if not Path(avatar_image).exists():
|
| 304 |
+
logger.error(f"Avatar image not found: {avatar_image}")
|
| 305 |
+
return None
|
| 306 |
+
|
| 307 |
+
# Prepare output filename
|
| 308 |
+
timestamp = int(time.time())
|
| 309 |
+
video_filename = f"mindfull_avatar_{timestamp}.mp4"
|
| 310 |
+
video_path = config.VIDEO_OUTPUT_DIR / video_filename
|
| 311 |
+
|
| 312 |
+
logger.info(f"Generating avatar video: {video_filename}")
|
| 313 |
+
|
| 314 |
+
# Use our simple video generation system
|
| 315 |
+
if generate_video_simple(avatar_image, audio_path, str(video_path)):
|
| 316 |
+
logger.info(f"✅ Avatar video generated: {video_path}")
|
| 317 |
+
return str(video_path) # Return the path on success
|
| 318 |
+
else:
|
| 319 |
+
logger.error("Video generation failed")
|
| 320 |
+
return None
|
| 321 |
+
|
| 322 |
+
except Exception as e:
|
| 323 |
+
logger.error(f"Error generating avatar video: {str(e)}")
|
| 324 |
+
return None
|
| 325 |
+
|
| 326 |
+
def process_complete_interaction(self, user_input: str,
|
| 327 |
+
avatar_image: Optional[str] = None) -> Dict[str, Any]:
|
| 328 |
+
"""
|
| 329 |
+
Process complete interaction: text -> response -> audio -> video
|
| 330 |
+
Returns: dictionary with all generated content paths and metadata
|
| 331 |
+
"""
|
| 332 |
+
start_time = time.time()
|
| 333 |
+
result = {
|
| 334 |
+
"session_id": self.session.session_id if self.session else None,
|
| 335 |
+
"user_input": user_input,
|
| 336 |
+
"timestamp": datetime.now().isoformat(),
|
| 337 |
+
"response_text": None,
|
| 338 |
+
"emotion_detected": None,
|
| 339 |
+
"audio_path": None,
|
| 340 |
+
"video_path": None,
|
| 341 |
+
"processing_time": 0,
|
| 342 |
+
"errors": []
|
| 343 |
+
}
|
| 344 |
+
|
| 345 |
+
try:
|
| 346 |
+
# Step 1: Generate text response
|
| 347 |
+
logger.info("Step 1: Generating text response...")
|
| 348 |
+
response_text, emotion = self.generate_response(user_input)
|
| 349 |
+
result["response_text"] = response_text
|
| 350 |
+
result["emotion_detected"] = emotion
|
| 351 |
+
|
| 352 |
+
if not response_text or response_text in config.FALLBACK_MESSAGES.values():
|
| 353 |
+
result["errors"].append("Failed to generate proper response")
|
| 354 |
+
return result
|
| 355 |
+
|
| 356 |
+
# Step 2: Generate audio
|
| 357 |
+
logger.info("Step 2: Generating audio...")
|
| 358 |
+
audio_path = self.generate_audio(response_text)
|
| 359 |
+
result["audio_path"] = audio_path
|
| 360 |
+
|
| 361 |
+
if not audio_path:
|
| 362 |
+
result["errors"].append("Failed to generate audio")
|
| 363 |
+
|
| 364 |
+
# Step 3: Generate avatar video (optional)
|
| 365 |
+
if self.sadtalker_available and audio_path:
|
| 366 |
+
logger.info("Step 3: Generating avatar video...")
|
| 367 |
+
video_path = self.generate_avatar_video(audio_path, avatar_image)
|
| 368 |
+
result["video_path"] = video_path
|
| 369 |
+
|
| 370 |
+
if not video_path:
|
| 371 |
+
result["errors"].append("Failed to generate avatar video")
|
| 372 |
+
else:
|
| 373 |
+
logger.info("Step 3: Skipping avatar video generation")
|
| 374 |
+
|
| 375 |
+
result["processing_time"] = time.time() - start_time
|
| 376 |
+
logger.info(f"✅ Complete interaction processed in {result['processing_time']:.2f}s")
|
| 377 |
+
|
| 378 |
+
except Exception as e:
|
| 379 |
+
logger.error(f"Error in complete interaction: {str(e)}")
|
| 380 |
+
result["errors"].append(f"Processing error: {str(e)}")
|
| 381 |
+
result["processing_time"] = time.time() - start_time
|
| 382 |
+
|
| 383 |
+
return result
|
| 384 |
+
|
| 385 |
+
def get_session_info(self) -> Optional[Dict[str, Any]]:
|
| 386 |
+
"""Get current session information"""
|
| 387 |
+
if not self.session or not self.session.start_time or not self.session.last_interaction:
|
| 388 |
+
return None
|
| 389 |
+
|
| 390 |
+
return {
|
| 391 |
+
"session_id": self.session.session_id,
|
| 392 |
+
"user_name": self.session.user_name,
|
| 393 |
+
"user_rank": self.session.user_rank,
|
| 394 |
+
"start_time": self.session.start_time.isoformat(),
|
| 395 |
+
"last_interaction": self.session.last_interaction.isoformat(),
|
| 396 |
+
"conversation_count": len(self.session.conversation_history or [])
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
def cleanup_old_files(self, max_age_hours: int = 24):
|
| 400 |
+
"""Clean up old generated files"""
|
| 401 |
+
try:
|
| 402 |
+
current_time = time.time()
|
| 403 |
+
cutoff_time = current_time - (max_age_hours * 3600)
|
| 404 |
+
|
| 405 |
+
for directory in [config.AUDIO_OUTPUT_DIR, config.VIDEO_OUTPUT_DIR, config.TEMP_DIR]:
|
| 406 |
+
if directory.exists():
|
| 407 |
+
for file_path in directory.iterdir():
|
| 408 |
+
if file_path.is_file() and file_path.stat().st_mtime < cutoff_time:
|
| 409 |
+
file_path.unlink()
|
| 410 |
+
logger.info(f"Cleaned up old file: {file_path}")
|
| 411 |
+
|
| 412 |
+
except Exception as e:
|
| 413 |
+
logger.error(f"Error during cleanup: {str(e)}")
|
| 414 |
+
|
| 415 |
+
def main():
|
| 416 |
+
"""Main function for testing the pipeline"""
|
| 417 |
+
print("🤖 Mindfull AI Avatar Chatbot")
|
| 418 |
+
print("=" * 50)
|
| 419 |
+
|
| 420 |
+
# Initialize pipeline
|
| 421 |
+
pipeline = MindfullPipeline()
|
| 422 |
+
|
| 423 |
+
if not pipeline.is_initialized:
|
| 424 |
+
print("❌ Failed to initialize pipeline. Please check the logs.")
|
| 425 |
+
return
|
| 426 |
+
|
| 427 |
+
# Create session
|
| 428 |
+
session_id = pipeline.create_session()
|
| 429 |
+
print(f"Session ID: {session_id}")
|
| 430 |
+
|
| 431 |
+
# Interactive loop
|
| 432 |
+
print("\n💬 Chat with Mindfull (type 'quit' to exit)")
|
| 433 |
+
print("Note: This will generate audio and video responses")
|
| 434 |
+
print("-" * 50)
|
| 435 |
+
|
| 436 |
+
while True:
|
| 437 |
+
try:
|
| 438 |
+
user_input = input("\nYou: ").strip()
|
| 439 |
+
|
| 440 |
+
if user_input.lower() in ['quit', 'exit', 'bye']:
|
| 441 |
+
print("👋 Goodbye! Take care!")
|
| 442 |
+
break
|
| 443 |
+
|
| 444 |
+
if not user_input:
|
| 445 |
+
continue
|
| 446 |
+
|
| 447 |
+
print("\n🔄 Processing your message...")
|
| 448 |
+
|
| 449 |
+
# Process complete interaction
|
| 450 |
+
result = pipeline.process_complete_interaction(user_input)
|
| 451 |
+
|
| 452 |
+
# Display results
|
| 453 |
+
print(f"\n🤖 Mindfull: {result['response_text']}")
|
| 454 |
+
print(f"😊 Emotion detected: {result['emotion_detected']}")
|
| 455 |
+
print(f"⏱️ Processing time: {result['processing_time']:.2f}s")
|
| 456 |
+
|
| 457 |
+
if result['audio_path']:
|
| 458 |
+
print(f"🔊 Audio saved: {result['audio_path']}")
|
| 459 |
+
|
| 460 |
+
if result['video_path']:
|
| 461 |
+
print(f"🎥 Video saved: {result['video_path']}")
|
| 462 |
+
|
| 463 |
+
if result['errors']:
|
| 464 |
+
print(f"⚠️ Errors: {', '.join(result['errors'])}")
|
| 465 |
+
|
| 466 |
+
except KeyboardInterrupt:
|
| 467 |
+
print("\n\n👋 Goodbye! Take care!")
|
| 468 |
+
break
|
| 469 |
+
except Exception as e:
|
| 470 |
+
print(f"\n❌ Error: {str(e)}")
|
| 471 |
+
|
| 472 |
+
# Cleanup
|
| 473 |
+
print("\n🧹 Cleaning up old files...")
|
| 474 |
+
pipeline.cleanup_old_files()
|
| 475 |
+
print("✅ Cleanup complete")
|
| 476 |
+
|
| 477 |
+
if __name__ == "__main__":
|
| 478 |
+
main()
|
mindfull_web_api.py
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Mindfull AI Avatar Chatbot - Web API
|
| 4 |
+
Flask web server for serving the Mindfull pipeline via HTTP API
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import json
|
| 9 |
+
import uuid
|
| 10 |
+
import threading
|
| 11 |
+
from datetime import datetime
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
from flask import Flask, request, jsonify, send_file, send_from_directory
|
| 15 |
+
from flask_cors import CORS
|
| 16 |
+
import logging
|
| 17 |
+
|
| 18 |
+
# Import the pipeline
|
| 19 |
+
from mindfull_pipeline import MindfullPipeline
|
| 20 |
+
from mindfull_config import config, logger
|
| 21 |
+
|
| 22 |
+
# Create Flask app
|
| 23 |
+
app = Flask(__name__)
|
| 24 |
+
CORS(app, origins=config.CORS_ORIGINS)
|
| 25 |
+
|
| 26 |
+
# Global pipeline instance
|
| 27 |
+
pipeline = None
|
| 28 |
+
active_sessions = {}
|
| 29 |
+
session_lock = threading.Lock()
|
| 30 |
+
|
| 31 |
+
def initialize_app():
|
| 32 |
+
"""Initialize the Flask application"""
|
| 33 |
+
global pipeline
|
| 34 |
+
|
| 35 |
+
logger.info("Initializing Mindfull Web API...")
|
| 36 |
+
|
| 37 |
+
# Initialize pipeline
|
| 38 |
+
pipeline = MindfullPipeline()
|
| 39 |
+
|
| 40 |
+
if not pipeline.is_initialized:
|
| 41 |
+
logger.error("Failed to initialize Mindfull pipeline")
|
| 42 |
+
return False
|
| 43 |
+
|
| 44 |
+
logger.info("✅ Mindfull Web API initialized successfully")
|
| 45 |
+
return True
|
| 46 |
+
|
| 47 |
+
@app.route('/health', methods=['GET'])
|
| 48 |
+
def health_check():
|
| 49 |
+
"""Health check endpoint"""
|
| 50 |
+
if not pipeline or not pipeline.is_initialized:
|
| 51 |
+
return jsonify({
|
| 52 |
+
'status': 'error',
|
| 53 |
+
'message': 'Pipeline not initialized'
|
| 54 |
+
}), 503
|
| 55 |
+
|
| 56 |
+
return jsonify({
|
| 57 |
+
'status': 'healthy',
|
| 58 |
+
'timestamp': datetime.now().isoformat(),
|
| 59 |
+
'components': {
|
| 60 |
+
'ollama': pipeline.ollama_available,
|
| 61 |
+
'f5tts': pipeline.f5tts_available,
|
| 62 |
+
'sadtalker': pipeline.sadtalker_available
|
| 63 |
+
}
|
| 64 |
+
})
|
| 65 |
+
|
| 66 |
+
@app.route('/session', methods=['POST'])
|
| 67 |
+
def create_session():
|
| 68 |
+
"""Create a new chat session"""
|
| 69 |
+
try:
|
| 70 |
+
session_id = pipeline.create_session()
|
| 71 |
+
|
| 72 |
+
with session_lock:
|
| 73 |
+
active_sessions[session_id] = {
|
| 74 |
+
'created': datetime.now().isoformat(),
|
| 75 |
+
'last_activity': datetime.now().isoformat()
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
return jsonify({
|
| 79 |
+
'status': 'success',
|
| 80 |
+
'session_id': session_id,
|
| 81 |
+
'message': 'Session created successfully'
|
| 82 |
+
})
|
| 83 |
+
|
| 84 |
+
except Exception as e:
|
| 85 |
+
logger.error(f"Error creating session: {str(e)}")
|
| 86 |
+
return jsonify({
|
| 87 |
+
'status': 'error',
|
| 88 |
+
'message': 'Failed to create session'
|
| 89 |
+
}), 500
|
| 90 |
+
|
| 91 |
+
@app.route('/session/<session_id>', methods=['GET'])
|
| 92 |
+
def get_session_info(session_id):
|
| 93 |
+
"""Get session information"""
|
| 94 |
+
try:
|
| 95 |
+
if session_id not in active_sessions:
|
| 96 |
+
return jsonify({
|
| 97 |
+
'status': 'error',
|
| 98 |
+
'message': 'Session not found'
|
| 99 |
+
}), 404
|
| 100 |
+
|
| 101 |
+
session_info = pipeline.get_session_info()
|
| 102 |
+
|
| 103 |
+
return jsonify({
|
| 104 |
+
'status': 'success',
|
| 105 |
+
'session_info': session_info
|
| 106 |
+
})
|
| 107 |
+
|
| 108 |
+
except Exception as e:
|
| 109 |
+
logger.error(f"Error getting session info: {str(e)}")
|
| 110 |
+
return jsonify({
|
| 111 |
+
'status': 'error',
|
| 112 |
+
'message': 'Failed to get session info'
|
| 113 |
+
}), 500
|
| 114 |
+
|
| 115 |
+
@app.route('/chat', methods=['POST'])
|
| 116 |
+
def chat():
|
| 117 |
+
"""Main chat endpoint"""
|
| 118 |
+
try:
|
| 119 |
+
data = request.get_json()
|
| 120 |
+
|
| 121 |
+
if not data or 'message' not in data:
|
| 122 |
+
return jsonify({
|
| 123 |
+
'status': 'error',
|
| 124 |
+
'message': 'Missing message in request'
|
| 125 |
+
}), 400
|
| 126 |
+
|
| 127 |
+
user_message = data['message'].strip()
|
| 128 |
+
session_id = data.get('session_id')
|
| 129 |
+
include_video = data.get('include_video', True)
|
| 130 |
+
avatar_image = data.get('avatar_image') # Optional custom avatar
|
| 131 |
+
|
| 132 |
+
# Validate input
|
| 133 |
+
if len(user_message) < config.MIN_TEXT_LENGTH:
|
| 134 |
+
return jsonify({
|
| 135 |
+
'status': 'error',
|
| 136 |
+
'message': 'Message too short'
|
| 137 |
+
}), 400
|
| 138 |
+
|
| 139 |
+
if len(user_message) > config.MAX_TEXT_LENGTH:
|
| 140 |
+
return jsonify({
|
| 141 |
+
'status': 'error',
|
| 142 |
+
'message': 'Message too long'
|
| 143 |
+
}), 400
|
| 144 |
+
|
| 145 |
+
# Update session activity
|
| 146 |
+
if session_id and session_id in active_sessions:
|
| 147 |
+
with session_lock:
|
| 148 |
+
active_sessions[session_id]['last_activity'] = datetime.now().isoformat()
|
| 149 |
+
|
| 150 |
+
# Process the interaction
|
| 151 |
+
if include_video:
|
| 152 |
+
result = pipeline.process_complete_interaction(user_message, avatar_image)
|
| 153 |
+
else:
|
| 154 |
+
# Text and audio only
|
| 155 |
+
response_text, emotion = pipeline.generate_response(user_message)
|
| 156 |
+
audio_path = pipeline.generate_audio(response_text)
|
| 157 |
+
|
| 158 |
+
result = {
|
| 159 |
+
'session_id': session_id,
|
| 160 |
+
'user_input': user_message,
|
| 161 |
+
'timestamp': datetime.now().isoformat(),
|
| 162 |
+
'response_text': response_text,
|
| 163 |
+
'emotion_detected': emotion,
|
| 164 |
+
'audio_path': audio_path,
|
| 165 |
+
'video_path': None,
|
| 166 |
+
'processing_time': 0,
|
| 167 |
+
'errors': []
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
# Prepare response
|
| 171 |
+
response_data = {
|
| 172 |
+
'status': 'success',
|
| 173 |
+
'session_id': result['session_id'],
|
| 174 |
+
'response_text': result['response_text'],
|
| 175 |
+
'emotion_detected': result['emotion_detected'],
|
| 176 |
+
'processing_time': result['processing_time'],
|
| 177 |
+
'timestamp': result['timestamp']
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
# Add file URLs if files were generated
|
| 181 |
+
if result['audio_path']:
|
| 182 |
+
audio_filename = Path(result['audio_path']).name
|
| 183 |
+
response_data['audio_url'] = f'/audio/{audio_filename}'
|
| 184 |
+
|
| 185 |
+
if result['video_path']:
|
| 186 |
+
video_filename = Path(result['video_path']).name
|
| 187 |
+
response_data['video_url'] = f'/video/{video_filename}'
|
| 188 |
+
|
| 189 |
+
if result['errors']:
|
| 190 |
+
response_data['warnings'] = result['errors']
|
| 191 |
+
|
| 192 |
+
return jsonify(response_data)
|
| 193 |
+
|
| 194 |
+
except Exception as e:
|
| 195 |
+
logger.error(f"Error in chat endpoint: {str(e)}")
|
| 196 |
+
return jsonify({
|
| 197 |
+
'status': 'error',
|
| 198 |
+
'message': 'Internal server error'
|
| 199 |
+
}), 500
|
| 200 |
+
|
| 201 |
+
@app.route('/audio/<filename>')
|
| 202 |
+
def serve_audio(filename):
|
| 203 |
+
"""Serve audio files"""
|
| 204 |
+
try:
|
| 205 |
+
return send_from_directory(config.AUDIO_OUTPUT_DIR, filename)
|
| 206 |
+
except FileNotFoundError:
|
| 207 |
+
return jsonify({'error': 'Audio file not found'}), 404
|
| 208 |
+
|
| 209 |
+
@app.route('/video/<filename>')
|
| 210 |
+
def serve_video(filename):
|
| 211 |
+
"""Serve video files"""
|
| 212 |
+
try:
|
| 213 |
+
return send_from_directory(config.VIDEO_OUTPUT_DIR, filename)
|
| 214 |
+
except FileNotFoundError:
|
| 215 |
+
return jsonify({'error': 'Video file not found'}), 404
|
| 216 |
+
|
| 217 |
+
@app.route('/models', methods=['GET'])
|
| 218 |
+
def get_model_status():
|
| 219 |
+
"""Get status of all models"""
|
| 220 |
+
try:
|
| 221 |
+
model_status = config.get_model_status()
|
| 222 |
+
|
| 223 |
+
return jsonify({
|
| 224 |
+
'status': 'success',
|
| 225 |
+
'models': model_status,
|
| 226 |
+
'pipeline_status': {
|
| 227 |
+
'ollama': pipeline.ollama_available if pipeline else False,
|
| 228 |
+
'f5tts': pipeline.f5tts_available if pipeline else False,
|
| 229 |
+
'sadtalker': pipeline.sadtalker_available if pipeline else False
|
| 230 |
+
}
|
| 231 |
+
})
|
| 232 |
+
|
| 233 |
+
except Exception as e:
|
| 234 |
+
logger.error(f"Error getting model status: {str(e)}")
|
| 235 |
+
return jsonify({
|
| 236 |
+
'status': 'error',
|
| 237 |
+
'message': 'Failed to get model status'
|
| 238 |
+
}), 500
|
| 239 |
+
|
| 240 |
+
@app.route('/cleanup', methods=['POST'])
|
| 241 |
+
def cleanup_files():
|
| 242 |
+
"""Clean up old generated files"""
|
| 243 |
+
try:
|
| 244 |
+
data = request.get_json() or {}
|
| 245 |
+
max_age_hours = data.get('max_age_hours', 24)
|
| 246 |
+
|
| 247 |
+
if pipeline:
|
| 248 |
+
pipeline.cleanup_old_files(max_age_hours)
|
| 249 |
+
|
| 250 |
+
return jsonify({
|
| 251 |
+
'status': 'success',
|
| 252 |
+
'message': f'Cleaned up files older than {max_age_hours} hours'
|
| 253 |
+
})
|
| 254 |
+
|
| 255 |
+
except Exception as e:
|
| 256 |
+
logger.error(f"Error during cleanup: {str(e)}")
|
| 257 |
+
return jsonify({
|
| 258 |
+
'status': 'error',
|
| 259 |
+
'message': 'Cleanup failed'
|
| 260 |
+
}), 500
|
| 261 |
+
|
| 262 |
+
@app.route('/config', methods=['GET'])
|
| 263 |
+
def get_config():
|
| 264 |
+
"""Get configuration information"""
|
| 265 |
+
return jsonify({
|
| 266 |
+
'status': 'success',
|
| 267 |
+
'config': {
|
| 268 |
+
'max_text_length': config.MAX_TEXT_LENGTH,
|
| 269 |
+
'min_text_length': config.MIN_TEXT_LENGTH,
|
| 270 |
+
'allowed_languages': config.ALLOWED_LANGUAGES,
|
| 271 |
+
'session_timeout': config.SESSION_TIMEOUT,
|
| 272 |
+
'available_emotions': list(config.EMOTION_KEYWORDS.keys()),
|
| 273 |
+
'default_model': config.DEFAULT_MODEL,
|
| 274 |
+
'fallback_model': config.FALLBACK_MODEL
|
| 275 |
+
}
|
| 276 |
+
})
|
| 277 |
+
|
| 278 |
+
@app.route('/', methods=['GET'])
|
| 279 |
+
def index():
|
| 280 |
+
"""Main index page"""
|
| 281 |
+
return jsonify({
|
| 282 |
+
'service': 'Mindfull AI Avatar Chatbot',
|
| 283 |
+
'version': '1.0.0',
|
| 284 |
+
'status': 'running',
|
| 285 |
+
'endpoints': {
|
| 286 |
+
'health': '/health',
|
| 287 |
+
'create_session': '/session [POST]',
|
| 288 |
+
'chat': '/chat [POST]',
|
| 289 |
+
'get_config': '/config [GET]',
|
| 290 |
+
'model_status': '/models [GET]',
|
| 291 |
+
'cleanup': '/cleanup [POST]'
|
| 292 |
+
},
|
| 293 |
+
'documentation': 'See API documentation for usage details'
|
| 294 |
+
})
|
| 295 |
+
|
| 296 |
+
@app.errorhandler(404)
|
| 297 |
+
def not_found(error):
|
| 298 |
+
return jsonify({'error': 'Endpoint not found'}), 404
|
| 299 |
+
|
| 300 |
+
@app.errorhandler(500)
|
| 301 |
+
def internal_error(error):
|
| 302 |
+
return jsonify({'error': 'Internal server error'}), 500
|
| 303 |
+
|
| 304 |
+
def run_server(host=None, port=None, debug=None):
|
| 305 |
+
"""Run the Flask server"""
|
| 306 |
+
host = host or config.WEB_HOST
|
| 307 |
+
port = port or config.WEB_PORT
|
| 308 |
+
debug = debug or config.WEB_DEBUG
|
| 309 |
+
|
| 310 |
+
logger.info(f"Starting Mindfull Web API server on {host}:{port}")
|
| 311 |
+
|
| 312 |
+
app.run(
|
| 313 |
+
host=host,
|
| 314 |
+
port=port,
|
| 315 |
+
debug=debug,
|
| 316 |
+
threaded=config.WEB_THREADED
|
| 317 |
+
)
|
| 318 |
+
|
| 319 |
+
if __name__ == '__main__':
|
| 320 |
+
if initialize_app():
|
| 321 |
+
run_server()
|
| 322 |
+
else:
|
| 323 |
+
logger.error("Failed to initialize application")
|
| 324 |
+
exit(1)
|
model_manager.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Model Manager for Police Bot AI Runtime
|
| 4 |
+
Helps manage and switch between different Ollama models
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
import requests
|
| 10 |
+
import json
|
| 11 |
+
from config import config
|
| 12 |
+
|
| 13 |
+
def print_banner():
|
| 14 |
+
"""Print banner"""
|
| 15 |
+
print("=" * 60)
|
| 16 |
+
print("🤖 Police Bot AI - Model Manager")
|
| 17 |
+
print("=" * 60)
|
| 18 |
+
|
| 19 |
+
def check_ollama_status():
|
| 20 |
+
"""Check if Ollama is running and get available models"""
|
| 21 |
+
try:
|
| 22 |
+
response = requests.get("http://localhost:11434/api/tags", timeout=5)
|
| 23 |
+
if response.status_code == 200:
|
| 24 |
+
data = response.json()
|
| 25 |
+
models = data.get("models", [])
|
| 26 |
+
return True, models
|
| 27 |
+
else:
|
| 28 |
+
return False, []
|
| 29 |
+
except:
|
| 30 |
+
return False, []
|
| 31 |
+
|
| 32 |
+
def list_available_models():
|
| 33 |
+
"""List all available models in Ollama"""
|
| 34 |
+
print("\n📋 Available Models in Ollama:")
|
| 35 |
+
print("-" * 40)
|
| 36 |
+
|
| 37 |
+
ollama_running, models = check_ollama_status()
|
| 38 |
+
|
| 39 |
+
if not ollama_running:
|
| 40 |
+
print("❌ Ollama is not running")
|
| 41 |
+
print("Please start Ollama first: ollama serve")
|
| 42 |
+
return []
|
| 43 |
+
|
| 44 |
+
if not models:
|
| 45 |
+
print("⚠️ No models found in Ollama")
|
| 46 |
+
print("You can pull models using:")
|
| 47 |
+
print(" ollama pull mistral")
|
| 48 |
+
print(" ollama pull llama3")
|
| 49 |
+
print(" ollama pull police-bot")
|
| 50 |
+
return []
|
| 51 |
+
|
| 52 |
+
available_models = []
|
| 53 |
+
for i, model in enumerate(models, 1):
|
| 54 |
+
model_name = model.get("name", "Unknown")
|
| 55 |
+
model_size = model.get("size", 0)
|
| 56 |
+
size_mb = model_size / (1024 * 1024) if model_size > 0 else 0
|
| 57 |
+
|
| 58 |
+
print(f"{i:2d}. {model_name:<20} ({size_mb:.0f} MB)")
|
| 59 |
+
available_models.append(model_name)
|
| 60 |
+
|
| 61 |
+
return available_models
|
| 62 |
+
|
| 63 |
+
def list_configured_models():
|
| 64 |
+
"""List models configured in the system"""
|
| 65 |
+
print("\n⚙️ Configured Models in System:")
|
| 66 |
+
print("-" * 40)
|
| 67 |
+
|
| 68 |
+
configured_models = config.list_available_models()
|
| 69 |
+
current_model = config.OLLAMA_MODEL
|
| 70 |
+
|
| 71 |
+
for i, model in enumerate(configured_models, 1):
|
| 72 |
+
status = "✓" if model == current_model else " "
|
| 73 |
+
print(f"{i:2d}. [{status}] {model}")
|
| 74 |
+
|
| 75 |
+
return configured_models
|
| 76 |
+
|
| 77 |
+
def switch_model():
|
| 78 |
+
"""Switch to a different model"""
|
| 79 |
+
print("\n🔄 Switch Model")
|
| 80 |
+
print("-" * 40)
|
| 81 |
+
|
| 82 |
+
configured_models = config.list_available_models()
|
| 83 |
+
current_model = config.OLLAMA_MODEL
|
| 84 |
+
|
| 85 |
+
print(f"Current model: {current_model}")
|
| 86 |
+
print("\nAvailable models:")
|
| 87 |
+
|
| 88 |
+
for i, model in enumerate(configured_models, 1):
|
| 89 |
+
print(f"{i}. {model}")
|
| 90 |
+
|
| 91 |
+
try:
|
| 92 |
+
choice = input(f"\nEnter choice (1-{len(configured_models)}): ").strip()
|
| 93 |
+
if choice:
|
| 94 |
+
model_index = int(choice) - 1
|
| 95 |
+
if 0 <= model_index < len(configured_models):
|
| 96 |
+
selected_model = configured_models[model_index]
|
| 97 |
+
|
| 98 |
+
# Check if model is available in Ollama
|
| 99 |
+
ollama_running, models = check_ollama_status()
|
| 100 |
+
if ollama_running:
|
| 101 |
+
model_names = [m.get("name") for m in models]
|
| 102 |
+
if selected_model not in model_names:
|
| 103 |
+
print(f"⚠️ Warning: {selected_model} is not available in Ollama")
|
| 104 |
+
pull = input("Would you like to pull it? (y/n): ").lower().strip()
|
| 105 |
+
if pull == 'y':
|
| 106 |
+
print(f"Pulling {selected_model}...")
|
| 107 |
+
os.system(f"ollama pull {selected_model}")
|
| 108 |
+
|
| 109 |
+
# Switch model
|
| 110 |
+
if config.switch_model(selected_model):
|
| 111 |
+
print(f"✅ Switched to {selected_model}")
|
| 112 |
+
|
| 113 |
+
# Save to environment file
|
| 114 |
+
save_model_to_env(selected_model)
|
| 115 |
+
else:
|
| 116 |
+
print(f"❌ Failed to switch to {selected_model}")
|
| 117 |
+
else:
|
| 118 |
+
print("❌ Invalid choice")
|
| 119 |
+
else:
|
| 120 |
+
print("❌ No choice made")
|
| 121 |
+
except ValueError:
|
| 122 |
+
print("❌ Invalid input")
|
| 123 |
+
|
| 124 |
+
def save_model_to_env(model_name):
|
| 125 |
+
"""Save model selection to .env file"""
|
| 126 |
+
env_file = ".env"
|
| 127 |
+
env_content = f"OLLAMA_MODEL={model_name}\n"
|
| 128 |
+
|
| 129 |
+
try:
|
| 130 |
+
with open(env_file, 'w') as f:
|
| 131 |
+
f.write(env_content)
|
| 132 |
+
print(f"✅ Model selection saved to {env_file}")
|
| 133 |
+
except Exception as e:
|
| 134 |
+
print(f"⚠️ Could not save to {env_file}: {e}")
|
| 135 |
+
|
| 136 |
+
def test_model():
|
| 137 |
+
"""Test the current model"""
|
| 138 |
+
print("\n🧪 Test Current Model")
|
| 139 |
+
print("-" * 40)
|
| 140 |
+
|
| 141 |
+
current_model = config.OLLAMA_MODEL
|
| 142 |
+
print(f"Testing model: {current_model}")
|
| 143 |
+
|
| 144 |
+
# Check if Ollama is running
|
| 145 |
+
ollama_running, models = check_ollama_status()
|
| 146 |
+
if not ollama_running:
|
| 147 |
+
print("❌ Ollama is not running")
|
| 148 |
+
return
|
| 149 |
+
|
| 150 |
+
# Check if model is available
|
| 151 |
+
model_names = [m.get("name") for m in models]
|
| 152 |
+
if current_model not in model_names:
|
| 153 |
+
print(f"❌ {current_model} is not available in Ollama")
|
| 154 |
+
print(f"Available models: {', '.join(model_names)}")
|
| 155 |
+
return
|
| 156 |
+
|
| 157 |
+
# Test the model
|
| 158 |
+
try:
|
| 159 |
+
from police_runtime import PoliceBotRuntime
|
| 160 |
+
|
| 161 |
+
runtime = PoliceBotRuntime()
|
| 162 |
+
test_prompt = "Hello, how are you today?"
|
| 163 |
+
|
| 164 |
+
print(f"Sending test prompt: '{test_prompt}'")
|
| 165 |
+
response = runtime.get_llama3_response(test_prompt)
|
| 166 |
+
|
| 167 |
+
if response:
|
| 168 |
+
print(f"✅ Model responded: {response[:100]}...")
|
| 169 |
+
else:
|
| 170 |
+
print("❌ No response from model")
|
| 171 |
+
|
| 172 |
+
except Exception as e:
|
| 173 |
+
print(f"❌ Error testing model: {e}")
|
| 174 |
+
|
| 175 |
+
def show_model_info():
|
| 176 |
+
"""Show information about the current model"""
|
| 177 |
+
print("\nℹ️ Current Model Information")
|
| 178 |
+
print("-" * 40)
|
| 179 |
+
|
| 180 |
+
current_model = config.OLLAMA_MODEL
|
| 181 |
+
model_config = config.get_current_model_config()
|
| 182 |
+
|
| 183 |
+
print(f"Model: {current_model}")
|
| 184 |
+
print(f"Temperature: {model_config['temperature']}")
|
| 185 |
+
print(f"Top P: {model_config['top_p']}")
|
| 186 |
+
print(f"Max Tokens: {model_config['max_tokens']}")
|
| 187 |
+
|
| 188 |
+
print(f"\nSystem Prompt Preview:")
|
| 189 |
+
print("-" * 20)
|
| 190 |
+
prompt_preview = model_config['system_prompt'][:200] + "..." if len(model_config['system_prompt']) > 200 else model_config['system_prompt']
|
| 191 |
+
print(prompt_preview)
|
| 192 |
+
|
| 193 |
+
def pull_model():
|
| 194 |
+
"""Pull a model from Ollama"""
|
| 195 |
+
print("\n📥 Pull Model from Ollama")
|
| 196 |
+
print("-" * 40)
|
| 197 |
+
|
| 198 |
+
model_name = input("Enter model name to pull (e.g., mistral, llama3): ").strip()
|
| 199 |
+
if model_name:
|
| 200 |
+
print(f"Pulling {model_name}...")
|
| 201 |
+
result = os.system(f"ollama pull {model_name}")
|
| 202 |
+
|
| 203 |
+
if result == 0:
|
| 204 |
+
print(f"✅ Successfully pulled {model_name}")
|
| 205 |
+
else:
|
| 206 |
+
print(f"❌ Failed to pull {model_name}")
|
| 207 |
+
else:
|
| 208 |
+
print("❌ No model name provided")
|
| 209 |
+
|
| 210 |
+
def main():
|
| 211 |
+
"""Main function"""
|
| 212 |
+
print_banner()
|
| 213 |
+
|
| 214 |
+
while True:
|
| 215 |
+
print("\nOptions:")
|
| 216 |
+
print("1. List available models in Ollama")
|
| 217 |
+
print("2. List configured models")
|
| 218 |
+
print("3. Switch model")
|
| 219 |
+
print("4. Test current model")
|
| 220 |
+
print("5. Show model information")
|
| 221 |
+
print("6. Pull model from Ollama")
|
| 222 |
+
print("7. Exit")
|
| 223 |
+
|
| 224 |
+
try:
|
| 225 |
+
choice = input("\nEnter choice (1-7): ").strip()
|
| 226 |
+
|
| 227 |
+
if choice == '1':
|
| 228 |
+
list_available_models()
|
| 229 |
+
elif choice == '2':
|
| 230 |
+
list_configured_models()
|
| 231 |
+
elif choice == '3':
|
| 232 |
+
switch_model()
|
| 233 |
+
elif choice == '4':
|
| 234 |
+
test_model()
|
| 235 |
+
elif choice == '5':
|
| 236 |
+
show_model_info()
|
| 237 |
+
elif choice == '6':
|
| 238 |
+
pull_model()
|
| 239 |
+
elif choice == '7':
|
| 240 |
+
print("Goodbye! 👋")
|
| 241 |
+
break
|
| 242 |
+
else:
|
| 243 |
+
print("❌ Invalid choice")
|
| 244 |
+
|
| 245 |
+
except KeyboardInterrupt:
|
| 246 |
+
print("\nGoodbye! 👋")
|
| 247 |
+
break
|
| 248 |
+
except Exception as e:
|
| 249 |
+
print(f"❌ Error: {e}")
|
| 250 |
+
|
| 251 |
+
if __name__ == "__main__":
|
| 252 |
+
main()
|
my_finetuned_model/config.json
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"output_path": "checkpoints/",
|
| 3 |
+
"logger_uri": null,
|
| 4 |
+
"run_name": "GPT_XTTS_FT",
|
| 5 |
+
"project_name": "XTTS_trainer",
|
| 6 |
+
"run_description": [
|
| 7 |
+
"\n GPT XTTS training\n "
|
| 8 |
+
],
|
| 9 |
+
"print_step": 50,
|
| 10 |
+
"plot_step": 100,
|
| 11 |
+
"model_param_stats": false,
|
| 12 |
+
"wandb_entity": null,
|
| 13 |
+
"dashboard_logger": "tensorboard",
|
| 14 |
+
"save_on_interrupt": true,
|
| 15 |
+
"log_model_step": 100,
|
| 16 |
+
"save_step": 1,
|
| 17 |
+
"save_n_checkpoints": 1,
|
| 18 |
+
"save_checkpoints": true,
|
| 19 |
+
"save_all_best": false,
|
| 20 |
+
"save_best_after": 10000,
|
| 21 |
+
"target_loss": null,
|
| 22 |
+
"print_eval": false,
|
| 23 |
+
"test_delay_epochs": 0,
|
| 24 |
+
"run_eval": true,
|
| 25 |
+
"run_eval_steps": null,
|
| 26 |
+
"distributed_backend": "nccl",
|
| 27 |
+
"distributed_url": "tcp://localhost:54321",
|
| 28 |
+
"mixed_precision": false,
|
| 29 |
+
"precision": "fp16",
|
| 30 |
+
"epochs": 3,
|
| 31 |
+
"batch_size": 16,
|
| 32 |
+
"eval_batch_size": 16,
|
| 33 |
+
"grad_clip": 0.0,
|
| 34 |
+
"scheduler_after_epoch": true,
|
| 35 |
+
"lr": 2e-05,
|
| 36 |
+
"optimizer": "AdamW",
|
| 37 |
+
"optimizer_params": {
|
| 38 |
+
"betas": [
|
| 39 |
+
0.9,
|
| 40 |
+
0.96
|
| 41 |
+
],
|
| 42 |
+
"eps": 1e-08,
|
| 43 |
+
"weight_decay": 0.01
|
| 44 |
+
},
|
| 45 |
+
"lr_scheduler": "MultiStepLR",
|
| 46 |
+
"lr_scheduler_params": {
|
| 47 |
+
"milestones": [
|
| 48 |
+
900000,
|
| 49 |
+
2700000,
|
| 50 |
+
5400000
|
| 51 |
+
],
|
| 52 |
+
"gamma": 0.5,
|
| 53 |
+
"last_epoch": -1
|
| 54 |
+
},
|
| 55 |
+
"use_grad_scaler": false,
|
| 56 |
+
"allow_tf32": false,
|
| 57 |
+
"cudnn_enable": true,
|
| 58 |
+
"cudnn_deterministic": false,
|
| 59 |
+
"cudnn_benchmark": false,
|
| 60 |
+
"training_seed": 54321,
|
| 61 |
+
"model": "xtts",
|
| 62 |
+
"num_loader_workers": 8,
|
| 63 |
+
"num_eval_loader_workers": 0,
|
| 64 |
+
"use_noise_augment": false,
|
| 65 |
+
"audio": {
|
| 66 |
+
"sample_rate": 22050,
|
| 67 |
+
"output_sample_rate": 24000,
|
| 68 |
+
"dvae_sample_rate": 22050
|
| 69 |
+
},
|
| 70 |
+
"use_phonemes": false,
|
| 71 |
+
"phonemizer": null,
|
| 72 |
+
"phoneme_language": null,
|
| 73 |
+
"compute_input_seq_cache": false,
|
| 74 |
+
"text_cleaner": null,
|
| 75 |
+
"enable_eos_bos_chars": false,
|
| 76 |
+
"test_sentences_file": "",
|
| 77 |
+
"phoneme_cache_path": null,
|
| 78 |
+
"characters": null,
|
| 79 |
+
"add_blank": false,
|
| 80 |
+
"batch_group_size": 0,
|
| 81 |
+
"loss_masking": null,
|
| 82 |
+
"min_audio_len": 1,
|
| 83 |
+
"max_audio_len": Infinity,
|
| 84 |
+
"min_text_len": 1,
|
| 85 |
+
"max_text_len": Infinity,
|
| 86 |
+
"compute_f0": false,
|
| 87 |
+
"compute_energy": false,
|
| 88 |
+
"compute_linear_spec": false,
|
| 89 |
+
"precompute_num_workers": 0,
|
| 90 |
+
"start_by_longest": false,
|
| 91 |
+
"shuffle": false,
|
| 92 |
+
"drop_last": false,
|
| 93 |
+
"datasets": [
|
| 94 |
+
{
|
| 95 |
+
"formatter": "",
|
| 96 |
+
"dataset_name": "",
|
| 97 |
+
"path": "",
|
| 98 |
+
"meta_file_train": "",
|
| 99 |
+
"ignored_speakers": null,
|
| 100 |
+
"language": "",
|
| 101 |
+
"phonemizer": "",
|
| 102 |
+
"meta_file_val": "",
|
| 103 |
+
"meta_file_attn_mask": ""
|
| 104 |
+
}
|
| 105 |
+
],
|
| 106 |
+
"test_sentences": [],
|
| 107 |
+
"eval_split_max_size": 256,
|
| 108 |
+
"eval_split_size": 0.01,
|
| 109 |
+
"use_speaker_weighted_sampler": false,
|
| 110 |
+
"speaker_weighted_sampler_alpha": 1.0,
|
| 111 |
+
"use_language_weighted_sampler": false,
|
| 112 |
+
"language_weighted_sampler_alpha": 1.0,
|
| 113 |
+
"use_length_weighted_sampler": false,
|
| 114 |
+
"length_weighted_sampler_alpha": 1.0,
|
| 115 |
+
"model_args": {
|
| 116 |
+
"gpt_batch_size": 1,
|
| 117 |
+
"enable_redaction": false,
|
| 118 |
+
"kv_cache": true,
|
| 119 |
+
"gpt_checkpoint": "",
|
| 120 |
+
"clvp_checkpoint": null,
|
| 121 |
+
"decoder_checkpoint": null,
|
| 122 |
+
"num_chars": 255,
|
| 123 |
+
"tokenizer_file": "checkpoints/XTTS_v2.0_original_model_files/vocab.json",
|
| 124 |
+
"gpt_max_audio_tokens": 605,
|
| 125 |
+
"gpt_max_text_tokens": 402,
|
| 126 |
+
"gpt_max_prompt_tokens": 70,
|
| 127 |
+
"gpt_layers": 30,
|
| 128 |
+
"gpt_n_model_channels": 1024,
|
| 129 |
+
"gpt_n_heads": 16,
|
| 130 |
+
"gpt_number_text_tokens": 6925,
|
| 131 |
+
"gpt_start_text_token": 261,
|
| 132 |
+
"gpt_stop_text_token": 0,
|
| 133 |
+
"gpt_num_audio_tokens": 1026,
|
| 134 |
+
"gpt_start_audio_token": 1024,
|
| 135 |
+
"gpt_stop_audio_token": 1025,
|
| 136 |
+
"gpt_code_stride_len": 1024,
|
| 137 |
+
"gpt_use_masking_gt_prompt_approach": true,
|
| 138 |
+
"gpt_use_perceiver_resampler": true,
|
| 139 |
+
"input_sample_rate": 22050,
|
| 140 |
+
"output_sample_rate": 24000,
|
| 141 |
+
"output_hop_length": 256,
|
| 142 |
+
"decoder_input_dim": 1024,
|
| 143 |
+
"d_vector_dim": 512,
|
| 144 |
+
"cond_d_vector_in_each_upsampling_layer": true,
|
| 145 |
+
"duration_const": 102400,
|
| 146 |
+
"min_conditioning_length": 11025,
|
| 147 |
+
"max_conditioning_length": 132300,
|
| 148 |
+
"gpt_loss_text_ce_weight": 0.01,
|
| 149 |
+
"gpt_loss_mel_ce_weight": 1.0,
|
| 150 |
+
"debug_loading_failures": false,
|
| 151 |
+
"max_wav_length": 330750,
|
| 152 |
+
"max_text_length": 400,
|
| 153 |
+
"mel_norm_file": "checkpoints/XTTS_v2.0_original_model_files/mel_stats.pth",
|
| 154 |
+
"dvae_checkpoint": "checkpoints/XTTS_v2.0_original_model_files/dvae.pth",
|
| 155 |
+
"xtts_checkpoint": "checkpoints/XTTS_v2.0_original_model_files/model.pth",
|
| 156 |
+
"vocoder": ""
|
| 157 |
+
},
|
| 158 |
+
"model_dir": null,
|
| 159 |
+
"languages": [
|
| 160 |
+
"en",
|
| 161 |
+
"es",
|
| 162 |
+
"fr",
|
| 163 |
+
"de",
|
| 164 |
+
"it",
|
| 165 |
+
"pt",
|
| 166 |
+
"pl",
|
| 167 |
+
"tr",
|
| 168 |
+
"ru",
|
| 169 |
+
"nl",
|
| 170 |
+
"cs",
|
| 171 |
+
"ar",
|
| 172 |
+
"zh-cn",
|
| 173 |
+
"hu",
|
| 174 |
+
"ko",
|
| 175 |
+
"ja",
|
| 176 |
+
"hi",
|
| 177 |
+
"en"
|
| 178 |
+
],
|
| 179 |
+
"temperature": 0.75,
|
| 180 |
+
"length_penalty": 1.0,
|
| 181 |
+
"repetition_penalty": 5.0,
|
| 182 |
+
"top_k": 50,
|
| 183 |
+
"top_p": 0.85,
|
| 184 |
+
"num_gpt_outputs": 1,
|
| 185 |
+
"gpt_cond_len": 30,
|
| 186 |
+
"gpt_cond_chunk_len": 4,
|
| 187 |
+
"max_ref_len": 30,
|
| 188 |
+
"sound_norm_refs": false,
|
| 189 |
+
"optimizer_wd_only_on_weights": true,
|
| 190 |
+
"weighted_loss_attrs": null,
|
| 191 |
+
"weighted_loss_multipliers": null,
|
| 192 |
+
"github_branch": "* main"
|
| 193 |
+
}
|
my_finetuned_model/vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
ollama_to_f5tts.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import requests
|
| 2 |
+
import subprocess
|
| 3 |
+
|
| 4 |
+
OLLAMA_URL = "http://localhost:11434/api/generate"
|
| 5 |
+
F5TTS_CKPT = r"C:\Users\Samarth Kadam\Voice-train\F5-TTS\ckpts\my_speak\model_last.pt"
|
| 6 |
+
REF_AUDIO = r"C:\Users\Samarth Kadam\Voice-train\F5-TTS\data\my_speak_pinyin\wavs\segment_43.wav"
|
| 7 |
+
REF_TEXT = "are you feeling mentally alert?"
|
| 8 |
+
OUTPUT_DIR = "outputs"
|
| 9 |
+
|
| 10 |
+
def get_policebot_reply(prompt):
|
| 11 |
+
data = {
|
| 12 |
+
"model": "police-bot",
|
| 13 |
+
"prompt": prompt,
|
| 14 |
+
"stream": False
|
| 15 |
+
}
|
| 16 |
+
response = requests.post(OLLAMA_URL, json=data)
|
| 17 |
+
return response.json()["response"].strip()
|
| 18 |
+
|
| 19 |
+
def synthesize_voice(text):
|
| 20 |
+
cmd = [
|
| 21 |
+
"python", r"C:\Users\Samarth Kadam\Voice-train\F5-TTS\src\f5_tts\infer\infer_cli.py",
|
| 22 |
+
"--ckpt_file", F5TTS_CKPT,
|
| 23 |
+
"--ref_audio", REF_AUDIO,
|
| 24 |
+
"--ref_text", REF_TEXT,
|
| 25 |
+
"--gen_text", text,
|
| 26 |
+
"--output_dir", OUTPUT_DIR,
|
| 27 |
+
"--remove_silence",
|
| 28 |
+
"--speed", "1.0"
|
| 29 |
+
]
|
| 30 |
+
subprocess.run(cmd)
|
| 31 |
+
|
| 32 |
+
if __name__ == "__main__":
|
| 33 |
+
user_prompt = input("Enter your prompt for police-bot: ")
|
| 34 |
+
reply = get_policebot_reply(user_prompt)
|
| 35 |
+
print("Police-bot reply:", reply)
|
| 36 |
+
synthesize_voice(reply)
|
| 37 |
+
print("Audio generated in outputs folder.")
|
python313_compat_patch.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Python 3.13 Compatibility Patch
|
| 3 |
+
Fixes pkgutil.ImpImporter issues for AI/ML packages
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import sys
|
| 7 |
+
import pkgutil
|
| 8 |
+
import importlib
|
| 9 |
+
import importlib.util
|
| 10 |
+
import warnings
|
| 11 |
+
from typing import Any, Optional, Union
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ImpImporter:
|
| 15 |
+
"""
|
| 16 |
+
Compatibility shim for pkgutil.ImpImporter removed in Python 3.12+
|
| 17 |
+
This fixes issues with packages that still try to use the old imp module
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
def __init__(self, path: Optional[str] = None):
|
| 21 |
+
self.path = path
|
| 22 |
+
|
| 23 |
+
def find_module(self, fullname: str, path=None):
|
| 24 |
+
"""Find module using modern importlib instead of deprecated imp"""
|
| 25 |
+
try:
|
| 26 |
+
if isinstance(path, list) and path:
|
| 27 |
+
package = path[0] if path else None
|
| 28 |
+
else:
|
| 29 |
+
package = None
|
| 30 |
+
spec = importlib.util.find_spec(fullname, package)
|
| 31 |
+
if spec is not None:
|
| 32 |
+
return self
|
| 33 |
+
except (ImportError, ValueError, AttributeError):
|
| 34 |
+
pass
|
| 35 |
+
return None
|
| 36 |
+
|
| 37 |
+
def load_module(self, fullname: str):
|
| 38 |
+
"""Load module using modern importlib"""
|
| 39 |
+
try:
|
| 40 |
+
spec = importlib.util.find_spec(fullname)
|
| 41 |
+
if spec is not None:
|
| 42 |
+
module = importlib.util.module_from_spec(spec)
|
| 43 |
+
sys.modules[fullname] = module
|
| 44 |
+
if spec.loader:
|
| 45 |
+
spec.loader.exec_module(module)
|
| 46 |
+
return module
|
| 47 |
+
except Exception as e:
|
| 48 |
+
warnings.warn(f"Failed to load module {fullname}: {e}")
|
| 49 |
+
raise ImportError(f"No module named '{fullname}'")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def apply_python313_patches():
|
| 53 |
+
"""Apply all necessary patches for Python 3.13 compatibility"""
|
| 54 |
+
|
| 55 |
+
# Patch 1: Add missing pkgutil.ImpImporter
|
| 56 |
+
if not hasattr(pkgutil, 'ImpImporter'):
|
| 57 |
+
# Dynamically add the attribute
|
| 58 |
+
setattr(pkgutil, 'ImpImporter', ImpImporter)
|
| 59 |
+
print("✓ Applied pkgutil.ImpImporter compatibility patch")
|
| 60 |
+
|
| 61 |
+
# Patch 2: Fix pkg_resources for newer setuptools
|
| 62 |
+
try:
|
| 63 |
+
import pkg_resources
|
| 64 |
+
# Monkey patch any problematic pkg_resources functionality
|
| 65 |
+
if not hasattr(pkg_resources, '_initialize_master_working_set'):
|
| 66 |
+
def _initialize_master_working_set():
|
| 67 |
+
"""Dummy implementation for missing function"""
|
| 68 |
+
pass
|
| 69 |
+
pkg_resources._initialize_master_working_set = _initialize_master_working_set
|
| 70 |
+
print("✓ Applied pkg_resources compatibility patch")
|
| 71 |
+
except ImportError:
|
| 72 |
+
print("! pkg_resources not available - installing alternative")
|
| 73 |
+
|
| 74 |
+
# Patch 3: Fix distutils issues
|
| 75 |
+
try:
|
| 76 |
+
# Check if distutils is available
|
| 77 |
+
import setuptools._distutils
|
| 78 |
+
if 'distutils' not in sys.modules:
|
| 79 |
+
sys.modules['distutils'] = setuptools._distutils
|
| 80 |
+
print("✓ Applied distutils compatibility patch")
|
| 81 |
+
except ImportError:
|
| 82 |
+
print("! distutils patches not needed")
|
| 83 |
+
|
| 84 |
+
# Patch 4: Fix collections.abc imports
|
| 85 |
+
try:
|
| 86 |
+
import collections
|
| 87 |
+
from collections.abc import Iterable, Mapping, MutableMapping
|
| 88 |
+
if not hasattr(collections, 'Iterable'):
|
| 89 |
+
setattr(collections, 'Iterable', Iterable)
|
| 90 |
+
setattr(collections, 'Mapping', Mapping)
|
| 91 |
+
setattr(collections, 'MutableMapping', MutableMapping)
|
| 92 |
+
print("✓ Applied collections.abc compatibility patch")
|
| 93 |
+
except (ImportError, AttributeError):
|
| 94 |
+
pass
|
| 95 |
+
|
| 96 |
+
# Patch 5: numpy compatibility
|
| 97 |
+
try:
|
| 98 |
+
import numpy as np
|
| 99 |
+
# Fix numpy version issues by adding fallback attributes
|
| 100 |
+
if not hasattr(np, 'bool'):
|
| 101 |
+
setattr(np, 'bool', bool)
|
| 102 |
+
if not hasattr(np, 'int'):
|
| 103 |
+
setattr(np, 'int', int)
|
| 104 |
+
if not hasattr(np, 'float'):
|
| 105 |
+
setattr(np, 'float', float)
|
| 106 |
+
if not hasattr(np, 'complex'):
|
| 107 |
+
setattr(np, 'complex', complex)
|
| 108 |
+
print("✓ Applied numpy compatibility patch")
|
| 109 |
+
except ImportError:
|
| 110 |
+
print("! numpy not available for patching")
|
| 111 |
+
|
| 112 |
+
print("✓ All Python 3.13 compatibility patches applied successfully")
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
if __name__ == "__main__":
|
| 116 |
+
apply_python313_patches()
|
requirements.txt
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HuggingFace Space requirements
|
| 2 |
+
# Audio is handled by edge-tts (no GPU needed)
|
| 3 |
+
# LLM falls back to HF Inference API when Ollama is absent
|
| 4 |
+
# Video (SadTalker) is optional and disabled by default on Spaces
|
| 5 |
+
|
| 6 |
+
# ── core ──────────────────────────────────────────────────────────────────────
|
| 7 |
+
gradio>=5.0.0
|
| 8 |
+
numpy>=1.24.0,<2.0.0
|
| 9 |
+
Pillow>=9.0.0
|
| 10 |
+
requests>=2.31.0
|
| 11 |
+
|
| 12 |
+
# ── TTS ───────────────────────────────────────────────────────────────────────
|
| 13 |
+
edge-tts>=6.1.9
|
| 14 |
+
|
| 15 |
+
# ── LLM (HF Inference API fallback) ──────────────────────────────────────────
|
| 16 |
+
huggingface_hub>=0.23.0
|
| 17 |
+
transformers>=4.40.0
|
| 18 |
+
accelerate>=0.30.0
|
| 19 |
+
|
| 20 |
+
# ── video (SadTalker / Wav2Lip) - comment out if not deploying video ──────────
|
| 21 |
+
# torch>=2.0.0
|
| 22 |
+
# torchvision>=0.15.0
|
| 23 |
+
# torchaudio>=2.0.0
|
| 24 |
+
# opencv-python-headless>=4.8.0
|
| 25 |
+
# scipy>=1.9.0
|
| 26 |
+
# safetensors>=0.4.0
|
| 27 |
+
# imageio>=2.28.0
|
| 28 |
+
# imageio-ffmpeg>=0.4.8
|
requirements_full.txt
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# Police Bot AI Runtime - Complete Requirements
|
| 3 |
+
# Supports: Ollama (Mistral), F5-TTS, SadTalker, Web API
|
| 4 |
+
# Optimized for GPU usage and virtual environment
|
| 5 |
+
# =============================================================================
|
| 6 |
+
|
| 7 |
+
# Core AI and ML Framework
|
| 8 |
+
torch>=2.0.0
|
| 9 |
+
torchvision>=0.15.0
|
| 10 |
+
torchaudio>=2.0.0
|
| 11 |
+
numpy>=1.21.0,<=1.26.4
|
| 12 |
+
scipy>=1.9.0
|
| 13 |
+
|
| 14 |
+
# F5-TTS Dependencies
|
| 15 |
+
accelerate>=0.33.0
|
| 16 |
+
bitsandbytes>0.37.0
|
| 17 |
+
cached_path>=1.1.3
|
| 18 |
+
click>=8.1.0
|
| 19 |
+
datasets>=2.14.0
|
| 20 |
+
ema_pytorch>=0.5.2
|
| 21 |
+
gradio>=3.45.2
|
| 22 |
+
hydra-core>=1.3.0
|
| 23 |
+
jieba>=0.42.1
|
| 24 |
+
librosa>=0.10.0
|
| 25 |
+
matplotlib>=3.7.0
|
| 26 |
+
pydub>=0.25.1
|
| 27 |
+
pypinyin>=0.49.0
|
| 28 |
+
safetensors>=0.3.0
|
| 29 |
+
soundfile>=0.12.1
|
| 30 |
+
tomli>=2.0.1
|
| 31 |
+
torchdiffeq>=0.2.3
|
| 32 |
+
tqdm>=4.65.0
|
| 33 |
+
transformers>=4.30.0
|
| 34 |
+
x_transformers>=1.31.14
|
| 35 |
+
packaging>=24.2
|
| 36 |
+
|
| 37 |
+
# Ollama API and HTTP Client
|
| 38 |
+
requests>=2.31.0
|
| 39 |
+
httpx>=0.24.0
|
| 40 |
+
|
| 41 |
+
# Web API Framework
|
| 42 |
+
flask>=2.3.0
|
| 43 |
+
flask-cors>=4.0.0
|
| 44 |
+
gunicorn>=21.0.0
|
| 45 |
+
werkzeug>=2.3.0
|
| 46 |
+
|
| 47 |
+
# Audio Processing
|
| 48 |
+
playsound>=1.3.0
|
| 49 |
+
pyaudio>=0.2.11
|
| 50 |
+
webrtcvad>=2.0.10
|
| 51 |
+
|
| 52 |
+
# SadTalker Dependencies (if using avatar)
|
| 53 |
+
opencv-python>=4.8.0
|
| 54 |
+
scikit-image>=0.21.0
|
| 55 |
+
scikit-learn>=1.3.0
|
| 56 |
+
face-alignment>=1.3.5
|
| 57 |
+
gfpgan>=1.3.8
|
| 58 |
+
basicsr>=1.4.2
|
| 59 |
+
facexlib>=0.3.0
|
| 60 |
+
realesrgan>=0.3.0
|
| 61 |
+
gfpgan>=1.3.8
|
| 62 |
+
codeformer>=0.1.0
|
| 63 |
+
|
| 64 |
+
# Video Processing (for SadTalker)
|
| 65 |
+
ffmpeg-python>=0.2.0
|
| 66 |
+
moviepy>=1.0.3
|
| 67 |
+
imageio>=2.31.0
|
| 68 |
+
imageio-ffmpeg>=0.4.8
|
| 69 |
+
|
| 70 |
+
# Utilities and Environment
|
| 71 |
+
python-dotenv>=1.0.0
|
| 72 |
+
psutil>=5.9.0
|
| 73 |
+
pyyaml>=6.0
|
| 74 |
+
tqdm>=4.65.0
|
| 75 |
+
pillow>=9.5.0
|
| 76 |
+
seaborn>=0.12.0
|
| 77 |
+
|
| 78 |
+
# Development and Testing (optional)
|
| 79 |
+
pytest>=7.4.0
|
| 80 |
+
pytest-cov>=4.1.0
|
| 81 |
+
black>=23.0.0
|
| 82 |
+
flake8>=6.0.0
|
| 83 |
+
|
| 84 |
+
# GPU Optimization (optional - uncomment if needed)
|
| 85 |
+
# apex>=0.1.0 # For mixed precision training
|
| 86 |
+
# deepspeed>=0.9.0 # For distributed training
|
| 87 |
+
|
| 88 |
+
# =============================================================================
|
| 89 |
+
# Installation Notes:
|
| 90 |
+
# =============================================================================
|
| 91 |
+
# 1. For GPU support, ensure CUDA is installed and compatible with PyTorch
|
| 92 |
+
# 2. Some packages may require system dependencies (ffmpeg, portaudio)
|
| 93 |
+
# 3. For Windows: pip install -r requirements.txt
|
| 94 |
+
# 4. For Linux: pip install -r requirements.txt
|
| 95 |
+
# 5. For development: pip install -r requirements.txt[dev]
|
| 96 |
+
# =============================================================================
|
| 97 |
+
|
| 98 |
+
# =============================================================================
|
| 99 |
+
# System Dependencies (install separately):
|
| 100 |
+
# =============================================================================
|
| 101 |
+
# Windows:
|
| 102 |
+
# - Visual Studio Build Tools
|
| 103 |
+
# - FFmpeg (add to PATH)
|
| 104 |
+
# - PortAudio (for pyaudio)
|
| 105 |
+
|
| 106 |
+
# Linux (Ubuntu/Debian):
|
| 107 |
+
# sudo apt-get update
|
| 108 |
+
# sudo apt-get install -y ffmpeg portaudio19-dev python3-dev
|
| 109 |
+
|
| 110 |
+
# macOS:
|
| 111 |
+
# brew install ffmpeg portaudio
|
| 112 |
+
# =============================================================================
|
requirements_hf.txt
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HuggingFace Space requirements
|
| 2 |
+
# Audio is handled by edge-tts (no GPU needed)
|
| 3 |
+
# LLM falls back to HF Inference API when Ollama is absent
|
| 4 |
+
# Video (SadTalker) is optional and disabled by default on Spaces
|
| 5 |
+
|
| 6 |
+
# ── core ──────────────────────────────────────────────────────────────────────
|
| 7 |
+
gradio>=5.0.0
|
| 8 |
+
numpy>=1.24.0,<2.0.0
|
| 9 |
+
Pillow>=9.0.0
|
| 10 |
+
requests>=2.31.0
|
| 11 |
+
|
| 12 |
+
# ── TTS ───────────────────────────────────────────────────────────────────────
|
| 13 |
+
edge-tts>=6.1.9
|
| 14 |
+
|
| 15 |
+
# ── LLM (HF Inference API fallback) ──────────────────────────────────────────
|
| 16 |
+
huggingface_hub>=0.23.0
|
| 17 |
+
transformers>=4.40.0
|
| 18 |
+
accelerate>=0.30.0
|
| 19 |
+
|
| 20 |
+
# ── video (SadTalker / Wav2Lip) - comment out if not deploying video ──────────
|
| 21 |
+
# torch>=2.0.0
|
| 22 |
+
# torchvision>=0.15.0
|
| 23 |
+
# torchaudio>=2.0.0
|
| 24 |
+
# opencv-python-headless>=4.8.0
|
| 25 |
+
# scipy>=1.9.0
|
| 26 |
+
# safetensors>=0.4.0
|
| 27 |
+
# imageio>=2.28.0
|
| 28 |
+
# imageio-ffmpeg>=0.4.8
|
requirements_unified.txt
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# Mindfull AI Avatar Chatbot - Unified Requirements
|
| 3 |
+
# Supports: Ollama (Mistral/LLaMA), F5-TTS, SadTalker + Wav2Lip
|
| 4 |
+
# Optimized for GPU usage and virtual environment
|
| 5 |
+
# =============================================================================
|
| 6 |
+
|
| 7 |
+
# Core Deep Learning Framework
|
| 8 |
+
torch>=2.0.0
|
| 9 |
+
torchvision>=0.15.0
|
| 10 |
+
torchaudio>=2.0.0
|
| 11 |
+
numpy>=1.21.0,<=1.26.4
|
| 12 |
+
scipy>=1.9.0
|
| 13 |
+
|
| 14 |
+
# Transformers and ML Core
|
| 15 |
+
transformers>=4.30.0
|
| 16 |
+
accelerate>=0.33.0
|
| 17 |
+
safetensors>=0.3.0
|
| 18 |
+
huggingface-hub>=0.16.0
|
| 19 |
+
|
| 20 |
+
# F5-TTS Dependencies
|
| 21 |
+
cached_path>=1.1.3
|
| 22 |
+
datasets>=2.14.0
|
| 23 |
+
ema_pytorch>=0.5.2
|
| 24 |
+
hydra-core>=1.3.0
|
| 25 |
+
jieba>=0.42.1
|
| 26 |
+
pypinyin>=0.49.0
|
| 27 |
+
torchdiffeq>=0.2.3
|
| 28 |
+
x_transformers>=1.31.14
|
| 29 |
+
packaging>=24.2
|
| 30 |
+
|
| 31 |
+
# Audio Processing
|
| 32 |
+
librosa>=0.10.0
|
| 33 |
+
soundfile>=0.12.1
|
| 34 |
+
pydub>=0.25.1
|
| 35 |
+
pyaudio>=0.2.11
|
| 36 |
+
webrtcvad>=2.0.10
|
| 37 |
+
playsound>=1.3.0
|
| 38 |
+
|
| 39 |
+
# Computer Vision & Image Processing (SadTalker)
|
| 40 |
+
opencv-python>=4.8.0
|
| 41 |
+
pillow>=9.5.0
|
| 42 |
+
imageio>=2.31.0
|
| 43 |
+
imageio-ffmpeg>=0.4.8
|
| 44 |
+
scikit-image>=0.21.0
|
| 45 |
+
scikit-learn>=1.3.0
|
| 46 |
+
|
| 47 |
+
# Face Processing and Enhancement
|
| 48 |
+
face-alignment>=1.3.5
|
| 49 |
+
gfpgan>=1.3.8
|
| 50 |
+
basicsr>=1.4.2
|
| 51 |
+
facexlib>=0.3.0
|
| 52 |
+
realesrgan>=0.3.0
|
| 53 |
+
|
| 54 |
+
# Video Processing
|
| 55 |
+
ffmpeg-python>=0.2.0
|
| 56 |
+
moviepy>=1.0.3
|
| 57 |
+
|
| 58 |
+
# Web API Framework
|
| 59 |
+
flask>=2.3.0
|
| 60 |
+
flask-cors>=4.0.0
|
| 61 |
+
gunicorn>=21.0.0
|
| 62 |
+
werkzeug>=2.3.0
|
| 63 |
+
flask-socketio>=5.3.0
|
| 64 |
+
|
| 65 |
+
# HTTP Clients and API
|
| 66 |
+
requests>=2.31.0
|
| 67 |
+
httpx>=0.24.0
|
| 68 |
+
|
| 69 |
+
# Utilities and Environment
|
| 70 |
+
python-dotenv>=1.0.0
|
| 71 |
+
psutil>=5.9.0
|
| 72 |
+
pyyaml>=6.0
|
| 73 |
+
tqdm>=4.65.0
|
| 74 |
+
matplotlib>=3.7.0
|
| 75 |
+
seaborn>=0.12.0
|
| 76 |
+
click>=8.1.0
|
| 77 |
+
gradio>=3.45.2
|
| 78 |
+
|
| 79 |
+
# Development and Testing
|
| 80 |
+
pytest>=7.4.0
|
| 81 |
+
black>=23.0.0
|
| 82 |
+
flake8>=6.0.0
|
| 83 |
+
|
| 84 |
+
# =============================================================================
|
| 85 |
+
# Installation Notes:
|
| 86 |
+
# =============================================================================
|
| 87 |
+
# 1. Install system dependencies first:
|
| 88 |
+
# - FFmpeg (add to PATH)
|
| 89 |
+
# - Visual Studio Build Tools (Windows)
|
| 90 |
+
# - CUDA Toolkit (for GPU support)
|
| 91 |
+
#
|
| 92 |
+
# 2. Install requirements:
|
| 93 |
+
# pip install -r requirements_unified.txt
|
| 94 |
+
#
|
| 95 |
+
# 3. For Windows specific audio:
|
| 96 |
+
# pip install pywin32
|
| 97 |
+
# =============================================================================
|
sadtalker+wav2lip/README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SadTalker + Wav2Lip Pipeline
|
| 2 |
+
|
| 3 |
+
Clean pipeline for talking avatar generation, ready for F5-TTS integration.
|
| 4 |
+
|
| 5 |
+
## Purpose
|
| 6 |
+
This pipeline converts a reference image + audio into a talking head video with synchronized lip movements.
|
| 7 |
+
|
| 8 |
+
## Quick Start
|
| 9 |
+
|
| 10 |
+
```bash
|
| 11 |
+
# Install dependencies
|
| 12 |
+
pip install -r requirements.txt
|
| 13 |
+
|
| 14 |
+
# Basic usage
|
| 15 |
+
python simple_single_command.py --source_image avatar_assets/officer.png --audio avatar_assets/voice.wav --output result.mp4
|
| 16 |
+
|
| 17 |
+
# ASCII-safe version (no special characters)
|
| 18 |
+
python simple_single_command_ascii.py --source_image avatar_assets/officer.png --audio avatar_assets/voice.wav --output result.mp4
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
## File Structure
|
| 22 |
+
|
| 23 |
+
```
|
| 24 |
+
sadtalker+wav2lip/
|
| 25 |
+
├── simple_single_command.py # Main generation script
|
| 26 |
+
├── simple_single_command_ascii.py # ASCII-safe version
|
| 27 |
+
├── simple_pipeline.py # Pipeline utilities
|
| 28 |
+
├── avatar_assets/ # Input images and audio
|
| 29 |
+
├── sadtalker/ # SadTalker model files
|
| 30 |
+
├── requirements.txt # Dependencies
|
| 31 |
+
└── README.md # This file
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## F5-TTS Integration
|
| 35 |
+
|
| 36 |
+
This pipeline is designed to work with F5-TTS:
|
| 37 |
+
|
| 38 |
+
1. **F5-TTS**: Text → Audio (speech synthesis)
|
| 39 |
+
2. **This Pipeline**: Image + Audio → Talking Video
|
| 40 |
+
|
| 41 |
+
Combined workflow:
|
| 42 |
+
```
|
| 43 |
+
Text → [F5-TTS] → Audio → [SadTalker+Wav2Lip] → Talking Video
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
## Requirements
|
| 47 |
+
|
| 48 |
+
- Python 3.8+
|
| 49 |
+
- CUDA GPU (recommended for faster processing)
|
| 50 |
+
- 4GB+ GPU memory
|
| 51 |
+
- FFmpeg for video processing
|
| 52 |
+
|
| 53 |
+
## Example Usage
|
| 54 |
+
|
| 55 |
+
```python
|
| 56 |
+
# Integration example with F5-TTS
|
| 57 |
+
text = "Hello, I am a police officer. How can I help you today?"
|
| 58 |
+
image = "avatar_assets/officer.png"
|
| 59 |
+
|
| 60 |
+
# Step 1: Generate speech with F5-TTS
|
| 61 |
+
audio_file = f5_tts_generate(text)
|
| 62 |
+
|
| 63 |
+
# Step 2: Generate talking video
|
| 64 |
+
video_file = sadtalker_generate(image, audio_file)
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
## Ready for Production
|
| 68 |
+
|
| 69 |
+
This cleaned pipeline contains only essential files and is optimized for:
|
| 70 |
+
- Fast processing
|
| 71 |
+
- Minimal dependencies
|
| 72 |
+
- Easy integration
|
| 73 |
+
- Reliable output
|
| 74 |
+
|
| 75 |
+
Perfect for integration with F5-TTS or other text-to-speech systems.
|
sadtalker+wav2lip/README_POLICE_BOT_AI.md
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Police Bot AI - Avatar Chatbot System
|
| 2 |
+
## Complete Open-Source AI Avatar Pipeline
|
| 3 |
+
|
| 4 |
+
### 🎯 Project Overview
|
| 5 |
+
This is a fully open-source AI avatar chatbot system that combines:
|
| 6 |
+
- **Mistral** (Ollama) for reasoning and text generation
|
| 7 |
+
- **F5-TTS** for high-quality text-to-speech
|
| 8 |
+
- **SadTalker** for talking head video generation
|
| 9 |
+
- **Wav2Lip** for precise lip synchronization
|
| 10 |
+
|
| 11 |
+
### 🔄 Complete Pipeline Workflow
|
| 12 |
+
|
| 13 |
+
```
|
| 14 |
+
User Input → Mistral (Reasoning) → Response Text → F5-TTS (Audio) → SadTalker (Video) → Wav2Lip (Lipsync) → Final Avatar Video
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
#### Detailed Steps:
|
| 18 |
+
1. **User Input**: Text message from user
|
| 19 |
+
2. **Mistral Processing**: Generate contextual, intelligent response
|
| 20 |
+
3. **F5-TTS Conversion**: Convert response text to high-quality audio
|
| 21 |
+
4. **SadTalker Generation**: Create talking head video from static image + audio
|
| 22 |
+
5. **Wav2Lip Enhancement**: Improve lip synchronization accuracy
|
| 23 |
+
6. **Output**: Final synchronized avatar video
|
| 24 |
+
|
| 25 |
+
### 🛡️ Security & Privacy Considerations
|
| 26 |
+
|
| 27 |
+
#### **Data Privacy**
|
| 28 |
+
- ✅ **Local Processing**: All AI processing happens locally, no cloud dependencies
|
| 29 |
+
- ✅ **No Data Collection**: No user data is stored or transmitted
|
| 30 |
+
- ✅ **Open Source**: All components are open-source and auditable
|
| 31 |
+
- ✅ **Offline Capable**: System works completely offline
|
| 32 |
+
|
| 33 |
+
#### **Model Security**
|
| 34 |
+
- ✅ **Verified Sources**: All models from official repositories
|
| 35 |
+
- ✅ **No External APIs**: No reliance on external services
|
| 36 |
+
- ✅ **Controlled Environment**: All processing within your infrastructure
|
| 37 |
+
|
| 38 |
+
#### **Content Safety**
|
| 39 |
+
- ⚠️ **Input Validation**: Implement input sanitization for user messages
|
| 40 |
+
- ⚠️ **Content Filtering**: Add content moderation for inappropriate responses
|
| 41 |
+
- ⚠️ **Rate Limiting**: Implement request throttling to prevent abuse
|
| 42 |
+
|
| 43 |
+
### 📁 File Structure & Integration
|
| 44 |
+
|
| 45 |
+
#### **Core Components**
|
| 46 |
+
```
|
| 47 |
+
police_bot_ai/
|
| 48 |
+
├── sadtalker/ # SadTalker integration
|
| 49 |
+
│ ├── src/ # Core SadTalker code
|
| 50 |
+
│ ├── checkpoints/ # Model files (essential!)
|
| 51 |
+
│ ├── gfpgan/weights/ # Face enhancement models
|
| 52 |
+
│ └── inference.py # Main inference script
|
| 53 |
+
├── wav2lip/ # Wav2Lip integration
|
| 54 |
+
│ ├── inference.py # Patched inference script
|
| 55 |
+
│ ├── audio.py # Patched audio processing
|
| 56 |
+
│ └── models/ # Wav2Lip model files
|
| 57 |
+
├── f5_tts/ # F5-TTS integration
|
| 58 |
+
├── mistral/ # Mistral/Ollama integration
|
| 59 |
+
├── avatar_assets/ # Static images for avatars
|
| 60 |
+
├── output/ # Generated videos
|
| 61 |
+
└── main_pipeline.py # Unified pipeline script
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
### 🔧 Technical Requirements
|
| 65 |
+
|
| 66 |
+
#### **System Requirements**
|
| 67 |
+
- **OS**: Windows 10/11, Linux, or macOS
|
| 68 |
+
- **GPU**: NVIDIA GPU with CUDA support (recommended)
|
| 69 |
+
- **RAM**: 16GB+ (32GB recommended)
|
| 70 |
+
- **Storage**: 50GB+ for models and temporary files
|
| 71 |
+
|
| 72 |
+
#### **Software Dependencies**
|
| 73 |
+
- **Python**: 3.10+ (3.11 recommended)
|
| 74 |
+
- **CUDA**: 11.8+ (for GPU acceleration)
|
| 75 |
+
- **FFmpeg**: For video processing
|
| 76 |
+
- **PyTorch**: 2.0+ with CUDA support
|
| 77 |
+
|
| 78 |
+
### 🚀 Implementation Guide
|
| 79 |
+
|
| 80 |
+
#### **Phase 1: Setup & Testing**
|
| 81 |
+
1. Install all dependencies from `requirements.txt`
|
| 82 |
+
2. Test individual components (Mistral, F5-TTS, SadTalker, Wav2Lip)
|
| 83 |
+
3. Verify GPU compatibility and performance
|
| 84 |
+
|
| 85 |
+
#### **Phase 2: Integration**
|
| 86 |
+
1. Create unified pipeline script
|
| 87 |
+
2. Implement error handling and fallbacks
|
| 88 |
+
3. Add input validation and content filtering
|
| 89 |
+
|
| 90 |
+
#### **Phase 3: Optimization**
|
| 91 |
+
1. Implement caching for common responses
|
| 92 |
+
2. Add video streaming capabilities
|
| 93 |
+
3. Optimize for real-time performance
|
| 94 |
+
|
| 95 |
+
#### **Phase 4: Deployment**
|
| 96 |
+
1. Set up web API interface
|
| 97 |
+
2. Implement user authentication
|
| 98 |
+
3. Add monitoring and logging
|
| 99 |
+
|
| 100 |
+
### 🔄 Real-Time Workflow
|
| 101 |
+
|
| 102 |
+
#### **For Live Chatbot:**
|
| 103 |
+
1. **User sends message** → Web interface
|
| 104 |
+
2. **Mistral generates response** (1-3 seconds)
|
| 105 |
+
3. **F5-TTS creates audio** (2-5 seconds)
|
| 106 |
+
4. **SadTalker generates video** (10-30 seconds)
|
| 107 |
+
5. **Wav2Lip improves sync** (5-15 seconds)
|
| 108 |
+
6. **Stream video to user** (real-time)
|
| 109 |
+
|
| 110 |
+
#### **Optimization Strategies:**
|
| 111 |
+
- **Pre-generate** common responses and greetings
|
| 112 |
+
- **Cache** avatar videos for frequent phrases
|
| 113 |
+
- **Parallel processing** where possible
|
| 114 |
+
- **Progressive loading** for better UX
|
| 115 |
+
|
| 116 |
+
### 🎨 Avatar Customization
|
| 117 |
+
|
| 118 |
+
#### **Supported Formats:**
|
| 119 |
+
- **Images**: PNG, JPG (high quality, front-facing)
|
| 120 |
+
- **Audio**: WAV (16kHz, mono recommended)
|
| 121 |
+
- **Video**: MP4 (H.264, 25fps)
|
| 122 |
+
|
| 123 |
+
#### **Best Practices:**
|
| 124 |
+
- Use high-resolution, well-lit face images
|
| 125 |
+
- Ensure face is centered and front-facing
|
| 126 |
+
- Avoid extreme expressions or angles
|
| 127 |
+
- Use clear, noise-free audio
|
| 128 |
+
|
| 129 |
+
### 🔍 Troubleshooting
|
| 130 |
+
|
| 131 |
+
#### **Common Issues:**
|
| 132 |
+
1. **CUDA Out of Memory**: Reduce batch sizes or use CPU
|
| 133 |
+
2. **Poor Lip Sync**: Use F5-TTS audio instead of recorded audio
|
| 134 |
+
3. **Blurry Output**: Try Wav2Lip GAN model for better quality
|
| 135 |
+
4. **Slow Processing**: Enable GPU acceleration and optimize settings
|
| 136 |
+
|
| 137 |
+
#### **Performance Tips:**
|
| 138 |
+
- Use SSD storage for faster I/O
|
| 139 |
+
- Enable CUDA memory optimization
|
| 140 |
+
- Implement video caching
|
| 141 |
+
- Use lower resolution for faster processing
|
| 142 |
+
|
| 143 |
+
### 📊 Monitoring & Maintenance
|
| 144 |
+
|
| 145 |
+
#### **Health Checks:**
|
| 146 |
+
- Monitor GPU memory usage
|
| 147 |
+
- Track processing times
|
| 148 |
+
- Check disk space for temporary files
|
| 149 |
+
- Verify model file integrity
|
| 150 |
+
|
| 151 |
+
#### **Regular Maintenance:**
|
| 152 |
+
- Update dependencies monthly
|
| 153 |
+
- Clean temporary files weekly
|
| 154 |
+
- Backup avatar assets
|
| 155 |
+
- Monitor system performance
|
| 156 |
+
|
| 157 |
+
### 🔮 Future Enhancements
|
| 158 |
+
|
| 159 |
+
#### **Planned Features:**
|
| 160 |
+
- **Multi-language support** with language detection
|
| 161 |
+
- **Emotion recognition** for contextual responses
|
| 162 |
+
- **Voice cloning** for personalized avatars
|
| 163 |
+
- **Real-time video streaming** with WebRTC
|
| 164 |
+
- **Mobile app** for remote access
|
| 165 |
+
|
| 166 |
+
#### **Advanced Capabilities:**
|
| 167 |
+
- **Multi-avatar support** for different scenarios
|
| 168 |
+
- **Gesture recognition** for more natural interactions
|
| 169 |
+
- **Background customization** for different environments
|
| 170 |
+
- **Voice emotion synthesis** for expressive responses
|
| 171 |
+
|
| 172 |
+
### 📞 Support & Community
|
| 173 |
+
|
| 174 |
+
#### **Resources:**
|
| 175 |
+
- **Documentation**: Check individual component READMEs
|
| 176 |
+
- **Issues**: Report bugs in respective repositories
|
| 177 |
+
- **Discussions**: Join open-source AI communities
|
| 178 |
+
|
| 179 |
+
#### **Contributing:**
|
| 180 |
+
- All components are open-source
|
| 181 |
+
- Contributions welcome for improvements
|
| 182 |
+
- Follow security best practices
|
| 183 |
+
- Test thoroughly before submitting
|
| 184 |
+
|
| 185 |
+
---
|
| 186 |
+
|
| 187 |
+
## 🚨 Important Notes
|
| 188 |
+
|
| 189 |
+
1. **This is a research/demo system** - not for production use without additional security measures
|
| 190 |
+
2. **Test thoroughly** before deployment in sensitive environments
|
| 191 |
+
3. **Monitor resource usage** - AI models can be resource-intensive
|
| 192 |
+
4. **Keep dependencies updated** for security patches
|
| 193 |
+
5. **Backup important data** regularly
|
| 194 |
+
|
| 195 |
+
---
|
| 196 |
+
|
| 197 |
+
*Last Updated: July 2025*
|
| 198 |
+
*Version: 1.0.0*
|
| 199 |
+
*License: Open Source (Various - see individual components)*
|
sadtalker+wav2lip/enhanced_pipeline.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Enhanced Video Pipeline - SadTalker + Wav2Lip Integration
|
| 4 |
+
Optimized for preserving facial animations while enhancing lip sync
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
import argparse
|
| 10 |
+
import subprocess
|
| 11 |
+
import shutil
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
import cv2
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
def run_command(command, description):
|
| 17 |
+
"""Run a command and handle errors"""
|
| 18 |
+
print(f"\n[PROCESSING] {description}...")
|
| 19 |
+
print(f"Command: {' '.join(command)}")
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
result = subprocess.run(command, check=True, capture_output=True, text=True, shell=True)
|
| 23 |
+
print(f"[OK] {description} completed successfully!")
|
| 24 |
+
if result.stdout and len(result.stdout) > 50:
|
| 25 |
+
print("Output:", result.stdout[-200:]) # Show last 200 chars
|
| 26 |
+
return True
|
| 27 |
+
except subprocess.CalledProcessError as e:
|
| 28 |
+
print(f"[ERROR] {description} failed!")
|
| 29 |
+
print(f"Error: {e.stderr}")
|
| 30 |
+
if e.stdout:
|
| 31 |
+
print(f"Output: {e.stdout}")
|
| 32 |
+
return False
|
| 33 |
+
|
| 34 |
+
def verify_video_animation(video_path, min_frame_diff_threshold=0.01):
|
| 35 |
+
"""Verify if video contains actual animation by checking frame differences"""
|
| 36 |
+
try:
|
| 37 |
+
cap = cv2.VideoCapture(video_path)
|
| 38 |
+
if not cap.isOpened():
|
| 39 |
+
return False
|
| 40 |
+
|
| 41 |
+
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 42 |
+
if frame_count < 2:
|
| 43 |
+
cap.release()
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
# Read first frame
|
| 47 |
+
ret, prev_frame = cap.read()
|
| 48 |
+
if not ret:
|
| 49 |
+
cap.release()
|
| 50 |
+
return False
|
| 51 |
+
|
| 52 |
+
prev_gray = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY)
|
| 53 |
+
|
| 54 |
+
# Check a few frames for differences
|
| 55 |
+
significant_changes = 0
|
| 56 |
+
frames_checked = min(10, frame_count - 1)
|
| 57 |
+
|
| 58 |
+
for i in range(frames_checked):
|
| 59 |
+
ret, curr_frame = cap.read()
|
| 60 |
+
if not ret:
|
| 61 |
+
break
|
| 62 |
+
|
| 63 |
+
curr_gray = cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY)
|
| 64 |
+
diff = cv2.absdiff(prev_gray, curr_gray)
|
| 65 |
+
|
| 66 |
+
# Calculate mean difference
|
| 67 |
+
mean_diff = np.mean(diff) / 255.0
|
| 68 |
+
|
| 69 |
+
if mean_diff > min_frame_diff_threshold:
|
| 70 |
+
significant_changes += 1
|
| 71 |
+
|
| 72 |
+
prev_gray = curr_gray
|
| 73 |
+
|
| 74 |
+
cap.release()
|
| 75 |
+
|
| 76 |
+
# If at least 20% of checked frames show significant changes, consider it animated (lowered from 30%)
|
| 77 |
+
animation_ratio = significant_changes / frames_checked
|
| 78 |
+
is_animated = animation_ratio > 0.2 # Lowered threshold for talking head videos
|
| 79 |
+
|
| 80 |
+
print(f"[INFO] Animation analysis: {significant_changes}/{frames_checked} frames with changes ({animation_ratio:.2%})")
|
| 81 |
+
print(f"[VIDEO] Video is {'animated' if is_animated else 'static'}")
|
| 82 |
+
|
| 83 |
+
return is_animated
|
| 84 |
+
|
| 85 |
+
except Exception as e:
|
| 86 |
+
print(f"[WARNING] Error analyzing video animation: {e}")
|
| 87 |
+
return False
|
| 88 |
+
|
| 89 |
+
def main():
|
| 90 |
+
parser = argparse.ArgumentParser(description='Enhanced Video Pipeline - SadTalker + Wav2Lip')
|
| 91 |
+
|
| 92 |
+
parser.add_argument('--source_image', type=str, required=True,
|
| 93 |
+
help='Path to source image (avatar)')
|
| 94 |
+
parser.add_argument('--audio', type=str, required=True,
|
| 95 |
+
help='Path to audio file')
|
| 96 |
+
parser.add_argument('--output', type=str, required=True,
|
| 97 |
+
help='Path for final output video')
|
| 98 |
+
parser.add_argument('--use_wav2lip', action='store_true',
|
| 99 |
+
help='Use Wav2Lip enhancement (only if SadTalker lip sync is poor)')
|
| 100 |
+
parser.add_argument('--force_wav2lip', action='store_true',
|
| 101 |
+
help='Force Wav2Lip even if SadTalker animation is good')
|
| 102 |
+
parser.add_argument('--temp_dir', type=str, default='temp_enhanced_pipeline',
|
| 103 |
+
help='Temporary directory for intermediate files')
|
| 104 |
+
|
| 105 |
+
args = parser.parse_args()
|
| 106 |
+
|
| 107 |
+
print("[VIDEO] Enhanced Video Pipeline - SadTalker + Wav2Lip Integration")
|
| 108 |
+
print("=" * 60)
|
| 109 |
+
print(f"Source Image: {args.source_image}")
|
| 110 |
+
print(f"Audio File: {args.audio}")
|
| 111 |
+
print(f"Final Output: {args.output}")
|
| 112 |
+
print(f"Use Wav2Lip: {args.use_wav2lip or args.force_wav2lip}")
|
| 113 |
+
print("=" * 60)
|
| 114 |
+
|
| 115 |
+
# Verify input files
|
| 116 |
+
if not os.path.exists(args.source_image):
|
| 117 |
+
print(f"[ERROR] Source image not found: {args.source_image}")
|
| 118 |
+
return
|
| 119 |
+
|
| 120 |
+
if not os.path.exists(args.audio):
|
| 121 |
+
print(f"[ERROR] Audio file not found: {args.audio}")
|
| 122 |
+
return
|
| 123 |
+
|
| 124 |
+
# Get current directory and set up paths
|
| 125 |
+
current_dir = os.path.dirname(os.path.abspath(__file__))
|
| 126 |
+
temp_dir = os.path.join(current_dir, args.temp_dir)
|
| 127 |
+
os.makedirs(temp_dir, exist_ok=True)
|
| 128 |
+
|
| 129 |
+
# Set up component paths
|
| 130 |
+
sadtalker_script = os.path.join(current_dir, 'sadtalker', 'inference.py')
|
| 131 |
+
sadtalker_output_dir = os.path.join(temp_dir, 'sadtalker_output')
|
| 132 |
+
sadtalker_checkpoints = os.path.join(current_dir, 'sadtalker', 'checkpoints')
|
| 133 |
+
|
| 134 |
+
wav2lip_script = os.path.join(current_dir, 'wav2lip', 'inference.py')
|
| 135 |
+
wav2lip_model = os.path.join(current_dir, 'wav2lip', 'Wav2Lip-SD-NOGAN.pt')
|
| 136 |
+
|
| 137 |
+
# Step 1: Run SadTalker with enhanced animation parameters
|
| 138 |
+
print("\n[ANIMATION] STEP 1: SadTalker Animation Generation")
|
| 139 |
+
sadtalker_cmd = [
|
| 140 |
+
sys.executable, sadtalker_script,
|
| 141 |
+
'--driven_audio', args.audio,
|
| 142 |
+
'--source_image', args.source_image,
|
| 143 |
+
'--result_dir', sadtalker_output_dir,
|
| 144 |
+
'--checkpoint_dir', sadtalker_checkpoints,
|
| 145 |
+
'--size', '256',
|
| 146 |
+
'--preprocess', 'crop',
|
| 147 |
+
'--expression_scale', '1.5', # Enhanced expression
|
| 148 |
+
'--pose_style', '1', # Natural head movement
|
| 149 |
+
'--batch_size', '1', # Better quality
|
| 150 |
+
'--verbose'
|
| 151 |
+
]
|
| 152 |
+
|
| 153 |
+
if not run_command(sadtalker_cmd, "SadTalker animation generation"):
|
| 154 |
+
print("[ERROR] SadTalker failed!")
|
| 155 |
+
return
|
| 156 |
+
|
| 157 |
+
# Find SadTalker output
|
| 158 |
+
sadtalker_video = None
|
| 159 |
+
for root, dirs, files in os.walk(sadtalker_output_dir):
|
| 160 |
+
for file in files:
|
| 161 |
+
if file.endswith('.mp4'):
|
| 162 |
+
sadtalker_video = os.path.join(root, file)
|
| 163 |
+
break
|
| 164 |
+
if sadtalker_video:
|
| 165 |
+
break
|
| 166 |
+
|
| 167 |
+
if not sadtalker_video or not os.path.exists(sadtalker_video):
|
| 168 |
+
print("[ERROR] SadTalker output not found!")
|
| 169 |
+
return
|
| 170 |
+
|
| 171 |
+
print(f"[OK] SadTalker output: {sadtalker_video}")
|
| 172 |
+
file_size = os.path.getsize(sadtalker_video)
|
| 173 |
+
print(f"[FILE] File size: {file_size:,} bytes")
|
| 174 |
+
|
| 175 |
+
# Step 2: Analyze SadTalker animation quality
|
| 176 |
+
print("\n[ANALYSIS] STEP 2: Analyzing Animation Quality")
|
| 177 |
+
is_animated = verify_video_animation(sadtalker_video)
|
| 178 |
+
|
| 179 |
+
# Step 3: Mux audio into SadTalker output directly (skip Wav2Lip - resolution issues)
|
| 180 |
+
print("\n[AUDIO] STEP 3: Muxing audio into animated video")
|
| 181 |
+
audio_muxed = os.path.join(temp_dir, 'final_with_audio.mp4')
|
| 182 |
+
ffmpeg_cmd = [
|
| 183 |
+
'ffmpeg', '-y',
|
| 184 |
+
'-i', sadtalker_video,
|
| 185 |
+
'-i', os.path.abspath(args.audio),
|
| 186 |
+
'-c:v', 'libx264',
|
| 187 |
+
'-c:a', 'aac',
|
| 188 |
+
'-pix_fmt', 'yuv420p',
|
| 189 |
+
'-movflags', '+faststart',
|
| 190 |
+
'-shortest',
|
| 191 |
+
audio_muxed
|
| 192 |
+
]
|
| 193 |
+
mux_result = subprocess.run(ffmpeg_cmd, capture_output=True, text=True)
|
| 194 |
+
if mux_result.returncode == 0 and os.path.exists(audio_muxed):
|
| 195 |
+
final_video = audio_muxed
|
| 196 |
+
print(f"[OK] Audio muxed successfully")
|
| 197 |
+
else:
|
| 198 |
+
print(f"[WARNING] ffmpeg mux failed: {mux_result.stderr[-300:]}")
|
| 199 |
+
final_video = sadtalker_video # silent fallback
|
| 200 |
+
|
| 201 |
+
# Step 4: Copy final output
|
| 202 |
+
print("\n[FILE] STEP 4: Finalizing Output")
|
| 203 |
+
try:
|
| 204 |
+
shutil.copy2(final_video, args.output)
|
| 205 |
+
final_size = os.path.getsize(args.output)
|
| 206 |
+
print(f"[OK] Final video saved to: {args.output}")
|
| 207 |
+
print(f"[FILE] Final file size: {final_size:,} bytes")
|
| 208 |
+
|
| 209 |
+
# Final verification
|
| 210 |
+
final_is_animated = verify_video_animation(args.output)
|
| 211 |
+
if final_is_animated:
|
| 212 |
+
print("[SUCCESS] Final video contains proper animation!")
|
| 213 |
+
else:
|
| 214 |
+
print("[WARNING] Warning: Final video appears to be static")
|
| 215 |
+
|
| 216 |
+
except Exception as e:
|
| 217 |
+
print(f"[ERROR] Error copying final video: {e}")
|
| 218 |
+
return
|
| 219 |
+
|
| 220 |
+
# Cleanup
|
| 221 |
+
try:
|
| 222 |
+
shutil.rmtree(temp_dir)
|
| 223 |
+
print("[CLEANUP] Cleaned up temporary files")
|
| 224 |
+
except Exception as e:
|
| 225 |
+
print(f"[WARNING] Could not clean up temporary files: {e}")
|
| 226 |
+
|
| 227 |
+
print("\n[SUCCESS] Enhanced Pipeline Completed Successfully!")
|
| 228 |
+
print(f"[VIDEO] Final output: {args.output}")
|
| 229 |
+
|
| 230 |
+
if __name__ == "__main__":
|
| 231 |
+
main()
|
sadtalker+wav2lip/init_path.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import glob
|
| 3 |
+
|
| 4 |
+
def init_path(checkpoint_dir, config_dir, size=512, old_version=False, preprocess='crop'):
|
| 5 |
+
|
| 6 |
+
if old_version:
|
| 7 |
+
#### load all the checkpoint of `pth`
|
| 8 |
+
sadtalker_paths = {
|
| 9 |
+
'wav2lip_checkpoint' : os.path.join(checkpoint_dir, 'wav2lip.pth'),
|
| 10 |
+
'audio2pose_checkpoint' : os.path.join(checkpoint_dir, 'auido2pose_00140-model.pth'),
|
| 11 |
+
'audio2exp_checkpoint' : os.path.join(checkpoint_dir, 'auido2exp_00300-model.pth'),
|
| 12 |
+
'free_view_checkpoint' : os.path.join(checkpoint_dir, 'facevid2vid_00189-model.pth.tar'),
|
| 13 |
+
'path_of_net_recon_model' : os.path.join(checkpoint_dir, 'epoch_20.pth')
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
use_safetensor = False
|
| 17 |
+
elif len(glob.glob(os.path.join(checkpoint_dir, '*.safetensors'))):
|
| 18 |
+
print('using safetensor as default')
|
| 19 |
+
sadtalker_paths = {
|
| 20 |
+
"checkpoint":os.path.join(checkpoint_dir, 'SadTalker_V0.0.2_'+str(size)+'.safetensors'),
|
| 21 |
+
}
|
| 22 |
+
use_safetensor = True
|
| 23 |
+
else:
|
| 24 |
+
print("WARNING: The new version of the model will be updated by safetensor, you may need to download it mannully. We run the old version of the checkpoint this time!")
|
| 25 |
+
use_safetensor = False
|
| 26 |
+
|
| 27 |
+
sadtalker_paths = {
|
| 28 |
+
'wav2lip_checkpoint' : os.path.join(checkpoint_dir, 'wav2lip.pth'),
|
| 29 |
+
'audio2pose_checkpoint' : os.path.join(checkpoint_dir, 'auido2pose_00140-model.pth'),
|
| 30 |
+
'audio2exp_checkpoint' : os.path.join(checkpoint_dir, 'auido2exp_00300-model.pth'),
|
| 31 |
+
'free_view_checkpoint' : os.path.join(checkpoint_dir, 'facevid2vid_00189-model.pth.tar'),
|
| 32 |
+
'path_of_net_recon_model' : os.path.join(checkpoint_dir, 'epoch_20.pth')
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
sadtalker_paths['dir_of_BFM_fitting'] = os.path.join(config_dir) # , 'BFM_Fitting'
|
| 36 |
+
sadtalker_paths['audio2pose_yaml_path'] = os.path.join(config_dir, 'auido2pose.yaml')
|
| 37 |
+
sadtalker_paths['audio2exp_yaml_path'] = os.path.join(config_dir, 'auido2exp.yaml')
|
| 38 |
+
sadtalker_paths['use_safetensor'] = use_safetensor # os.path.join(config_dir, 'auido2exp.yaml')
|
| 39 |
+
|
| 40 |
+
if 'full' in preprocess:
|
| 41 |
+
sadtalker_paths['mappingnet_checkpoint'] = os.path.join(checkpoint_dir, 'mapping_00109-model.pth.tar')
|
| 42 |
+
sadtalker_paths['facerender_yaml'] = os.path.join(config_dir, 'facerender_still.yaml')
|
| 43 |
+
else:
|
| 44 |
+
sadtalker_paths['mappingnet_checkpoint'] = os.path.join(checkpoint_dir, 'mapping_00229-model.pth.tar')
|
| 45 |
+
sadtalker_paths['facerender_yaml'] = os.path.join(config_dir, 'facerender.yaml')
|
| 46 |
+
|
| 47 |
+
return sadtalker_paths
|
sadtalker+wav2lip/preprocess.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import cv2, os, sys, torch
|
| 3 |
+
from tqdm import tqdm
|
| 4 |
+
from PIL import Image
|
| 5 |
+
|
| 6 |
+
# 3dmm extraction
|
| 7 |
+
import safetensors
|
| 8 |
+
import safetensors.torch
|
| 9 |
+
from src.face3d.util.preprocess import align_img
|
| 10 |
+
from src.face3d.util.load_mats import load_lm3d
|
| 11 |
+
from src.face3d.models import networks
|
| 12 |
+
|
| 13 |
+
from scipy.io import loadmat, savemat
|
| 14 |
+
from src.utils.croper import Preprocesser
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
import warnings
|
| 18 |
+
|
| 19 |
+
from src.utils.safetensor_helper import load_x_from_safetensor
|
| 20 |
+
warnings.filterwarnings("ignore")
|
| 21 |
+
|
| 22 |
+
def split_coeff(coeffs):
|
| 23 |
+
"""
|
| 24 |
+
Return:
|
| 25 |
+
coeffs_dict -- a dict of torch.tensors
|
| 26 |
+
|
| 27 |
+
Parameters:
|
| 28 |
+
coeffs -- torch.tensor, size (B, 256)
|
| 29 |
+
"""
|
| 30 |
+
id_coeffs = coeffs[:, :80]
|
| 31 |
+
exp_coeffs = coeffs[:, 80: 144]
|
| 32 |
+
tex_coeffs = coeffs[:, 144: 224]
|
| 33 |
+
angles = coeffs[:, 224: 227]
|
| 34 |
+
gammas = coeffs[:, 227: 254]
|
| 35 |
+
translations = coeffs[:, 254:]
|
| 36 |
+
return {
|
| 37 |
+
'id': id_coeffs,
|
| 38 |
+
'exp': exp_coeffs,
|
| 39 |
+
'tex': tex_coeffs,
|
| 40 |
+
'angle': angles,
|
| 41 |
+
'gamma': gammas,
|
| 42 |
+
'trans': translations
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class CropAndExtract():
|
| 47 |
+
def __init__(self, sadtalker_path, device):
|
| 48 |
+
|
| 49 |
+
self.propress = Preprocesser(device)
|
| 50 |
+
self.net_recon = networks.define_net_recon(net_recon='resnet50', use_last_fc=False, init_path='').to(device)
|
| 51 |
+
|
| 52 |
+
if sadtalker_path['use_safetensor']:
|
| 53 |
+
checkpoint = safetensors.torch.load_file(sadtalker_path['checkpoint'])
|
| 54 |
+
self.net_recon.load_state_dict(load_x_from_safetensor(checkpoint, 'face_3drecon'))
|
| 55 |
+
else:
|
| 56 |
+
checkpoint = torch.load(sadtalker_path['path_of_net_recon_model'], map_location=torch.device(device))
|
| 57 |
+
self.net_recon.load_state_dict(checkpoint['net_recon'])
|
| 58 |
+
|
| 59 |
+
self.net_recon.eval()
|
| 60 |
+
self.lm3d_std = load_lm3d(sadtalker_path['dir_of_BFM_fitting'])
|
| 61 |
+
self.device = device
|
| 62 |
+
|
| 63 |
+
def generate(self, input_path, save_dir, crop_or_resize='crop', source_image_flag=False, pic_size=256):
|
| 64 |
+
|
| 65 |
+
pic_name = os.path.splitext(os.path.split(input_path)[-1])[0]
|
| 66 |
+
|
| 67 |
+
landmarks_path = os.path.join(save_dir, pic_name+'_landmarks.txt')
|
| 68 |
+
coeff_path = os.path.join(save_dir, pic_name+'.mat')
|
| 69 |
+
png_path = os.path.join(save_dir, pic_name+'.png')
|
| 70 |
+
|
| 71 |
+
#load input
|
| 72 |
+
if not os.path.isfile(input_path):
|
| 73 |
+
raise ValueError('input_path must be a valid path to video/image file')
|
| 74 |
+
elif input_path.split('.')[-1] in ['jpg', 'png', 'jpeg']:
|
| 75 |
+
# loader for first frame
|
| 76 |
+
full_frames = [cv2.imread(input_path)]
|
| 77 |
+
fps = 25
|
| 78 |
+
else:
|
| 79 |
+
# loader for videos
|
| 80 |
+
video_stream = cv2.VideoCapture(input_path)
|
| 81 |
+
fps = video_stream.get(cv2.CAP_PROP_FPS)
|
| 82 |
+
full_frames = []
|
| 83 |
+
while 1:
|
| 84 |
+
still_reading, frame = video_stream.read()
|
| 85 |
+
if not still_reading:
|
| 86 |
+
video_stream.release()
|
| 87 |
+
break
|
| 88 |
+
full_frames.append(frame)
|
| 89 |
+
if source_image_flag:
|
| 90 |
+
break
|
| 91 |
+
|
| 92 |
+
x_full_frames= [cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in full_frames]
|
| 93 |
+
|
| 94 |
+
#### crop images as the
|
| 95 |
+
if 'crop' in crop_or_resize.lower(): # default crop
|
| 96 |
+
x_full_frames, crop, quad = self.propress.crop(x_full_frames, still=True if 'ext' in crop_or_resize.lower() else False, xsize=512)
|
| 97 |
+
clx, cly, crx, cry = crop
|
| 98 |
+
lx, ly, rx, ry = quad
|
| 99 |
+
lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
|
| 100 |
+
oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx
|
| 101 |
+
crop_info = ((ox2 - ox1, oy2 - oy1), crop, quad)
|
| 102 |
+
elif 'full' in crop_or_resize.lower():
|
| 103 |
+
x_full_frames, crop, quad = self.propress.crop(x_full_frames, still=True if 'ext' in crop_or_resize.lower() else False, xsize=512)
|
| 104 |
+
clx, cly, crx, cry = crop
|
| 105 |
+
lx, ly, rx, ry = quad
|
| 106 |
+
lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
|
| 107 |
+
oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx
|
| 108 |
+
crop_info = ((ox2 - ox1, oy2 - oy1), crop, quad)
|
| 109 |
+
else: # resize mode
|
| 110 |
+
oy1, oy2, ox1, ox2 = 0, x_full_frames[0].shape[0], 0, x_full_frames[0].shape[1]
|
| 111 |
+
crop_info = ((ox2 - ox1, oy2 - oy1), None, None)
|
| 112 |
+
|
| 113 |
+
frames_pil = [Image.fromarray(cv2.resize(frame,(pic_size, pic_size))) for frame in x_full_frames]
|
| 114 |
+
if len(frames_pil) == 0:
|
| 115 |
+
print('No face is detected in the input file')
|
| 116 |
+
return None, None
|
| 117 |
+
|
| 118 |
+
# save crop info
|
| 119 |
+
for frame in frames_pil:
|
| 120 |
+
cv2.imwrite(png_path, cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR))
|
| 121 |
+
|
| 122 |
+
# 2. get the landmark according to the detected face.
|
| 123 |
+
if not os.path.isfile(landmarks_path):
|
| 124 |
+
lm = self.propress.predictor.extract_keypoint(frames_pil, landmarks_path)
|
| 125 |
+
else:
|
| 126 |
+
print(' Using saved landmarks.')
|
| 127 |
+
lm = np.loadtxt(landmarks_path).astype(np.float32)
|
| 128 |
+
lm = lm.reshape([len(x_full_frames), -1, 2])
|
| 129 |
+
|
| 130 |
+
if not os.path.isfile(coeff_path):
|
| 131 |
+
# load 3dmm paramter generator from Deep3DFaceRecon_pytorch
|
| 132 |
+
video_coeffs, full_coeffs = [], []
|
| 133 |
+
for idx in tqdm(range(len(frames_pil)), desc='3DMM Extraction In Video:'):
|
| 134 |
+
frame = frames_pil[idx]
|
| 135 |
+
W,H = frame.size
|
| 136 |
+
lm1 = lm[idx].reshape([-1, 2])
|
| 137 |
+
|
| 138 |
+
if np.mean(lm1) == -1:
|
| 139 |
+
lm1 = (self.lm3d_std[:, :2]+1)/2.
|
| 140 |
+
lm1 = np.concatenate(
|
| 141 |
+
[lm1[:, :1]*W, lm1[:, 1:2]*H], 1
|
| 142 |
+
)
|
| 143 |
+
else:
|
| 144 |
+
lm1[:, -1] = H - 1 - lm1[:, -1]
|
| 145 |
+
|
| 146 |
+
trans_params, im1, lm1, _ = align_img(frame, lm1, self.lm3d_std)
|
| 147 |
+
|
| 148 |
+
trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32)
|
| 149 |
+
im_t = torch.tensor(np.array(im1)/255., dtype=torch.float32).permute(2, 0, 1).to(self.device).unsqueeze(0)
|
| 150 |
+
|
| 151 |
+
with torch.no_grad():
|
| 152 |
+
full_coeff = self.net_recon(im_t)
|
| 153 |
+
coeffs = split_coeff(full_coeff)
|
| 154 |
+
|
| 155 |
+
pred_coeff = {key:coeffs[key].cpu().numpy() for key in coeffs}
|
| 156 |
+
|
| 157 |
+
pred_coeff = np.concatenate([
|
| 158 |
+
pred_coeff['exp'],
|
| 159 |
+
pred_coeff['angle'],
|
| 160 |
+
pred_coeff['trans'],
|
| 161 |
+
trans_params[2:][None],
|
| 162 |
+
], 1)
|
| 163 |
+
video_coeffs.append(pred_coeff)
|
| 164 |
+
full_coeffs.append(full_coeff.cpu().numpy())
|
| 165 |
+
|
| 166 |
+
semantic_npy = np.array(video_coeffs)[:,0]
|
| 167 |
+
|
| 168 |
+
savemat(coeff_path, {'coeff_3dmm': semantic_npy, 'full_3dmm': np.array(full_coeffs)[0]})
|
| 169 |
+
|
| 170 |
+
return coeff_path, png_path, crop_info
|
sadtalker+wav2lip/requirements.txt
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SadTalker + Wav2Lip Pipeline Requirements
|
| 2 |
+
# Core dependencies for talking avatar generation
|
| 3 |
+
|
| 4 |
+
# Deep Learning
|
| 5 |
+
torch>=1.8.0
|
| 6 |
+
torchvision
|
| 7 |
+
torchaudio
|
| 8 |
+
|
| 9 |
+
# Computer Vision & Image Processing
|
| 10 |
+
opencv-python
|
| 11 |
+
pillow
|
| 12 |
+
imageio
|
| 13 |
+
moviepy
|
| 14 |
+
ffmpeg-python
|
| 15 |
+
|
| 16 |
+
# Face Enhancement
|
| 17 |
+
gfpgan
|
| 18 |
+
basicsr
|
| 19 |
+
facexlib
|
| 20 |
+
|
| 21 |
+
# Audio Processing
|
| 22 |
+
librosa
|
| 23 |
+
soundfile
|
| 24 |
+
scipy
|
| 25 |
+
|
| 26 |
+
# Utilities
|
| 27 |
+
numpy
|
| 28 |
+
matplotlib
|
| 29 |
+
tqdm
|
sadtalker+wav2lip/sadtalker/generate_batch.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
from tqdm import tqdm
|
| 4 |
+
import torch
|
| 5 |
+
import numpy as np
|
| 6 |
+
import random
|
| 7 |
+
import scipy.io as scio
|
| 8 |
+
import src.utils.audio as audio
|
| 9 |
+
|
| 10 |
+
def crop_pad_audio(wav, audio_length):
|
| 11 |
+
if len(wav) > audio_length:
|
| 12 |
+
wav = wav[:audio_length]
|
| 13 |
+
elif len(wav) < audio_length:
|
| 14 |
+
wav = np.pad(wav, [0, audio_length - len(wav)], mode='constant', constant_values=0)
|
| 15 |
+
return wav
|
| 16 |
+
|
| 17 |
+
def parse_audio_length(audio_length, sr, fps):
|
| 18 |
+
bit_per_frames = sr / fps
|
| 19 |
+
|
| 20 |
+
num_frames = int(audio_length / bit_per_frames)
|
| 21 |
+
audio_length = int(num_frames * bit_per_frames)
|
| 22 |
+
|
| 23 |
+
return audio_length, num_frames
|
| 24 |
+
|
| 25 |
+
def generate_blink_seq(num_frames):
|
| 26 |
+
ratio = np.zeros((num_frames,1))
|
| 27 |
+
frame_id = 0
|
| 28 |
+
while frame_id in range(num_frames):
|
| 29 |
+
start = 80
|
| 30 |
+
if frame_id+start+9<=num_frames - 1:
|
| 31 |
+
ratio[frame_id+start:frame_id+start+9, 0] = [0.5,0.6,0.7,0.9,1, 0.9, 0.7,0.6,0.5]
|
| 32 |
+
frame_id = frame_id+start+9
|
| 33 |
+
else:
|
| 34 |
+
break
|
| 35 |
+
return ratio
|
| 36 |
+
|
| 37 |
+
def generate_blink_seq_randomly(num_frames):
|
| 38 |
+
ratio = np.zeros((num_frames,1))
|
| 39 |
+
if num_frames<=20:
|
| 40 |
+
return ratio
|
| 41 |
+
frame_id = 0
|
| 42 |
+
while frame_id in range(num_frames):
|
| 43 |
+
start = random.choice(range(min(10,num_frames), min(int(num_frames/2), 70)))
|
| 44 |
+
if frame_id+start+5<=num_frames - 1:
|
| 45 |
+
ratio[frame_id+start:frame_id+start+5, 0] = [0.5, 0.9, 1.0, 0.9, 0.5]
|
| 46 |
+
frame_id = frame_id+start+5
|
| 47 |
+
else:
|
| 48 |
+
break
|
| 49 |
+
return ratio
|
| 50 |
+
|
| 51 |
+
def get_data(first_coeff_path, audio_path, device, ref_eyeblink_coeff_path, still=False, idlemode=False, length_of_audio=False, use_blink=True):
|
| 52 |
+
|
| 53 |
+
syncnet_mel_step_size = 16
|
| 54 |
+
fps = 25
|
| 55 |
+
|
| 56 |
+
pic_name = os.path.splitext(os.path.split(first_coeff_path)[-1])[0]
|
| 57 |
+
audio_name = os.path.splitext(os.path.split(audio_path)[-1])[0]
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
if idlemode:
|
| 61 |
+
num_frames = int(length_of_audio * 25)
|
| 62 |
+
indiv_mels = np.zeros((num_frames, 80, 16))
|
| 63 |
+
else:
|
| 64 |
+
wav = audio.load_wav(audio_path, 16000)
|
| 65 |
+
wav_length, num_frames = parse_audio_length(len(wav), 16000, 25)
|
| 66 |
+
wav = crop_pad_audio(wav, wav_length)
|
| 67 |
+
orig_mel = audio.melspectrogram(wav).T
|
| 68 |
+
spec = orig_mel.copy() # nframes 80
|
| 69 |
+
indiv_mels = []
|
| 70 |
+
|
| 71 |
+
for i in tqdm(range(num_frames), 'mel:'):
|
| 72 |
+
start_frame_num = i-2
|
| 73 |
+
start_idx = int(80. * (start_frame_num / float(fps)))
|
| 74 |
+
end_idx = start_idx + syncnet_mel_step_size
|
| 75 |
+
seq = list(range(start_idx, end_idx))
|
| 76 |
+
seq = [ min(max(item, 0), orig_mel.shape[0]-1) for item in seq ]
|
| 77 |
+
m = spec[seq, :]
|
| 78 |
+
indiv_mels.append(m.T)
|
| 79 |
+
indiv_mels = np.asarray(indiv_mels) # T 80 16
|
| 80 |
+
|
| 81 |
+
ratio = generate_blink_seq_randomly(num_frames) # T
|
| 82 |
+
source_semantics_path = first_coeff_path
|
| 83 |
+
source_semantics_dict = scio.loadmat(source_semantics_path)
|
| 84 |
+
ref_coeff = source_semantics_dict['coeff_3dmm'][:1,:70] #1 70
|
| 85 |
+
ref_coeff = np.repeat(ref_coeff, num_frames, axis=0)
|
| 86 |
+
|
| 87 |
+
if ref_eyeblink_coeff_path is not None:
|
| 88 |
+
ratio[:num_frames] = 0
|
| 89 |
+
refeyeblink_coeff_dict = scio.loadmat(ref_eyeblink_coeff_path)
|
| 90 |
+
refeyeblink_coeff = refeyeblink_coeff_dict['coeff_3dmm'][:,:64]
|
| 91 |
+
refeyeblink_num_frames = refeyeblink_coeff.shape[0]
|
| 92 |
+
if refeyeblink_num_frames<num_frames:
|
| 93 |
+
div = num_frames//refeyeblink_num_frames
|
| 94 |
+
re = num_frames%refeyeblink_num_frames
|
| 95 |
+
refeyeblink_coeff_list = [refeyeblink_coeff for i in range(div)]
|
| 96 |
+
refeyeblink_coeff_list.append(refeyeblink_coeff[:re, :64])
|
| 97 |
+
refeyeblink_coeff = np.concatenate(refeyeblink_coeff_list, axis=0)
|
| 98 |
+
print(refeyeblink_coeff.shape[0])
|
| 99 |
+
|
| 100 |
+
ref_coeff[:, :64] = refeyeblink_coeff[:num_frames, :64]
|
| 101 |
+
|
| 102 |
+
indiv_mels = torch.FloatTensor(indiv_mels).unsqueeze(1).unsqueeze(0) # bs T 1 80 16
|
| 103 |
+
|
| 104 |
+
if use_blink:
|
| 105 |
+
ratio = torch.FloatTensor(ratio).unsqueeze(0) # bs T
|
| 106 |
+
else:
|
| 107 |
+
ratio = torch.FloatTensor(ratio).unsqueeze(0).fill_(0.)
|
| 108 |
+
# bs T
|
| 109 |
+
ref_coeff = torch.FloatTensor(ref_coeff).unsqueeze(0) # bs 1 70
|
| 110 |
+
|
| 111 |
+
indiv_mels = indiv_mels.to(device)
|
| 112 |
+
ratio = ratio.to(device)
|
| 113 |
+
ref_coeff = ref_coeff.to(device)
|
| 114 |
+
|
| 115 |
+
return {'indiv_mels': indiv_mels,
|
| 116 |
+
'ref': ref_coeff,
|
| 117 |
+
'num_frames': num_frames,
|
| 118 |
+
'ratio_gt': ratio,
|
| 119 |
+
'audio_name': audio_name, 'pic_name': pic_name}
|
| 120 |
+
|
sadtalker+wav2lip/sadtalker/inference.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from glob import glob
|
| 2 |
+
import shutil
|
| 3 |
+
import torch
|
| 4 |
+
from time import strftime
|
| 5 |
+
import os, sys, time
|
| 6 |
+
from argparse import ArgumentParser
|
| 7 |
+
|
| 8 |
+
from src.utils.preprocess import CropAndExtract
|
| 9 |
+
from src.test_audio2coeff import Audio2Coeff
|
| 10 |
+
from src.facerender.animate import AnimateFromCoeff
|
| 11 |
+
from src.generate_batch import get_data
|
| 12 |
+
from src.generate_facerender_batch import get_facerender_data
|
| 13 |
+
from src.utils.init_path import init_path
|
| 14 |
+
|
| 15 |
+
def main(args):
|
| 16 |
+
#torch.backends.cudnn.enabled = False
|
| 17 |
+
|
| 18 |
+
pic_path = args.source_image
|
| 19 |
+
audio_path = args.driven_audio
|
| 20 |
+
save_dir = os.path.join(args.result_dir, strftime("%Y_%m_%d_%H.%M.%S"))
|
| 21 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 22 |
+
pose_style = args.pose_style
|
| 23 |
+
device = args.device
|
| 24 |
+
batch_size = args.batch_size
|
| 25 |
+
input_yaw_list = args.input_yaw
|
| 26 |
+
input_pitch_list = args.input_pitch
|
| 27 |
+
input_roll_list = args.input_roll
|
| 28 |
+
ref_eyeblink = args.ref_eyeblink
|
| 29 |
+
ref_pose = args.ref_pose
|
| 30 |
+
|
| 31 |
+
current_root_path = os.path.split(sys.argv[0])[0]
|
| 32 |
+
|
| 33 |
+
sadtalker_paths = init_path(args.checkpoint_dir, os.path.join(current_root_path, 'src/config'), args.size, args.old_version, args.preprocess)
|
| 34 |
+
|
| 35 |
+
#init model
|
| 36 |
+
preprocess_model = CropAndExtract(sadtalker_paths, device)
|
| 37 |
+
|
| 38 |
+
audio_to_coeff = Audio2Coeff(sadtalker_paths, device)
|
| 39 |
+
|
| 40 |
+
animate_from_coeff = AnimateFromCoeff(sadtalker_paths, device)
|
| 41 |
+
|
| 42 |
+
#crop image and extract 3dmm from image
|
| 43 |
+
first_frame_dir = os.path.join(save_dir, 'first_frame_dir')
|
| 44 |
+
os.makedirs(first_frame_dir, exist_ok=True)
|
| 45 |
+
print('3DMM Extraction for source image')
|
| 46 |
+
first_coeff_path, crop_pic_path, crop_info = preprocess_model.generate(pic_path, first_frame_dir, args.preprocess,\
|
| 47 |
+
source_image_flag=True, pic_size=args.size)
|
| 48 |
+
if first_coeff_path is None:
|
| 49 |
+
print("Can't get the coeffs of the input")
|
| 50 |
+
return
|
| 51 |
+
|
| 52 |
+
if ref_eyeblink is not None:
|
| 53 |
+
ref_eyeblink_videoname = os.path.splitext(os.path.split(ref_eyeblink)[-1])[0]
|
| 54 |
+
ref_eyeblink_frame_dir = os.path.join(save_dir, ref_eyeblink_videoname)
|
| 55 |
+
os.makedirs(ref_eyeblink_frame_dir, exist_ok=True)
|
| 56 |
+
print('3DMM Extraction for the reference video providing eye blinking')
|
| 57 |
+
ref_eyeblink_coeff_path, _, _ = preprocess_model.generate(ref_eyeblink, ref_eyeblink_frame_dir, args.preprocess, source_image_flag=False)
|
| 58 |
+
else:
|
| 59 |
+
ref_eyeblink_coeff_path=None
|
| 60 |
+
|
| 61 |
+
if ref_pose is not None:
|
| 62 |
+
if ref_pose == ref_eyeblink:
|
| 63 |
+
ref_pose_coeff_path = ref_eyeblink_coeff_path
|
| 64 |
+
else:
|
| 65 |
+
ref_pose_videoname = os.path.splitext(os.path.split(ref_pose)[-1])[0]
|
| 66 |
+
ref_pose_frame_dir = os.path.join(save_dir, ref_pose_videoname)
|
| 67 |
+
os.makedirs(ref_pose_frame_dir, exist_ok=True)
|
| 68 |
+
print('3DMM Extraction for the reference video providing pose')
|
| 69 |
+
ref_pose_coeff_path, _, _ = preprocess_model.generate(ref_pose, ref_pose_frame_dir, args.preprocess, source_image_flag=False)
|
| 70 |
+
else:
|
| 71 |
+
ref_pose_coeff_path=None
|
| 72 |
+
|
| 73 |
+
#audio2ceoff
|
| 74 |
+
batch = get_data(first_coeff_path, audio_path, device, ref_eyeblink_coeff_path, still=args.still)
|
| 75 |
+
coeff_path = audio_to_coeff.generate(batch, save_dir, pose_style, ref_pose_coeff_path)
|
| 76 |
+
|
| 77 |
+
# 3dface render
|
| 78 |
+
if args.face3dvis:
|
| 79 |
+
from src.face3d.visualize import gen_composed_video
|
| 80 |
+
gen_composed_video(args, device, first_coeff_path, coeff_path, audio_path, os.path.join(save_dir, '3dface.mp4'))
|
| 81 |
+
|
| 82 |
+
#coeff2video
|
| 83 |
+
data = get_facerender_data(coeff_path, crop_pic_path, first_coeff_path, audio_path,
|
| 84 |
+
batch_size, input_yaw_list, input_pitch_list, input_roll_list,
|
| 85 |
+
expression_scale=args.expression_scale, still_mode=args.still, preprocess=args.preprocess, size=args.size)
|
| 86 |
+
|
| 87 |
+
result = animate_from_coeff.generate(data, save_dir, pic_path, crop_info, \
|
| 88 |
+
enhancer=args.enhancer, background_enhancer=args.background_enhancer, preprocess=args.preprocess, img_size=args.size)
|
| 89 |
+
|
| 90 |
+
shutil.move(result, save_dir+'.mp4')
|
| 91 |
+
print('The generated video is named:', save_dir+'.mp4')
|
| 92 |
+
|
| 93 |
+
if not args.verbose:
|
| 94 |
+
shutil.rmtree(save_dir)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
if __name__ == '__main__':
|
| 98 |
+
|
| 99 |
+
parser = ArgumentParser()
|
| 100 |
+
parser.add_argument("--driven_audio", default='./examples/driven_audio/bus_chinese.wav', help="path to driven audio")
|
| 101 |
+
parser.add_argument("--source_image", default='./examples/source_image/full_body_1.png', help="path to source image")
|
| 102 |
+
parser.add_argument("--ref_eyeblink", default=None, help="path to reference video providing eye blinking")
|
| 103 |
+
parser.add_argument("--ref_pose", default=None, help="path to reference video providing pose")
|
| 104 |
+
parser.add_argument("--checkpoint_dir", default='./checkpoints', help="path to output")
|
| 105 |
+
parser.add_argument("--result_dir", default='./results', help="path to output")
|
| 106 |
+
parser.add_argument("--pose_style", type=int, default=0, help="input pose style from [0, 46)")
|
| 107 |
+
parser.add_argument("--batch_size", type=int, default=2, help="the batch size of facerender")
|
| 108 |
+
parser.add_argument("--size", type=int, default=256, help="the image size of the facerender")
|
| 109 |
+
parser.add_argument("--expression_scale", type=float, default=1., help="the batch size of facerender")
|
| 110 |
+
parser.add_argument('--input_yaw', nargs='+', type=int, default=None, help="the input yaw degree of the user ")
|
| 111 |
+
parser.add_argument('--input_pitch', nargs='+', type=int, default=None, help="the input pitch degree of the user")
|
| 112 |
+
parser.add_argument('--input_roll', nargs='+', type=int, default=None, help="the input roll degree of the user")
|
| 113 |
+
parser.add_argument('--enhancer', type=str, default=None, help="Face enhancer, [gfpgan, RestoreFormer]")
|
| 114 |
+
parser.add_argument('--background_enhancer', type=str, default=None, help="background enhancer, [realesrgan]")
|
| 115 |
+
parser.add_argument("--cpu", dest="cpu", action="store_true")
|
| 116 |
+
parser.add_argument("--face3dvis", action="store_true", help="generate 3d face and 3d landmarks")
|
| 117 |
+
parser.add_argument("--still", action="store_true", help="can crop back to the original videos for the full body aniamtion")
|
| 118 |
+
parser.add_argument("--preprocess", default='crop', choices=['crop', 'extcrop', 'resize', 'full', 'extfull'], help="how to preprocess the images" )
|
| 119 |
+
parser.add_argument("--verbose",action="store_true", help="saving the intermedia output or not" )
|
| 120 |
+
parser.add_argument("--old_version",action="store_true", help="use the pth other than safetensor version" )
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# net structure and parameters
|
| 124 |
+
parser.add_argument('--net_recon', type=str, default='resnet50', choices=['resnet18', 'resnet34', 'resnet50'], help='useless')
|
| 125 |
+
parser.add_argument('--init_path', type=str, default=None, help='Useless')
|
| 126 |
+
parser.add_argument('--use_last_fc',default=False, help='zero initialize the last fc')
|
| 127 |
+
parser.add_argument('--bfm_folder', type=str, default='./checkpoints/BFM_Fitting/')
|
| 128 |
+
parser.add_argument('--bfm_model', type=str, default='BFM_model_front.mat', help='bfm model')
|
| 129 |
+
|
| 130 |
+
# default renderer parameters
|
| 131 |
+
parser.add_argument('--focal', type=float, default=1015.)
|
| 132 |
+
parser.add_argument('--center', type=float, default=112.)
|
| 133 |
+
parser.add_argument('--camera_d', type=float, default=10.)
|
| 134 |
+
parser.add_argument('--z_near', type=float, default=5.)
|
| 135 |
+
parser.add_argument('--z_far', type=float, default=15.)
|
| 136 |
+
|
| 137 |
+
args = parser.parse_args()
|
| 138 |
+
|
| 139 |
+
if torch.cuda.is_available() and not args.cpu:
|
| 140 |
+
args.device = "cuda"
|
| 141 |
+
else:
|
| 142 |
+
args.device = "cpu"
|
| 143 |
+
|
| 144 |
+
main(args)
|
| 145 |
+
|
sadtalker+wav2lip/sadtalker/src/audio2exp_models/audio2exp.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from tqdm import tqdm
|
| 2 |
+
import torch
|
| 3 |
+
from torch import nn
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class Audio2Exp(nn.Module):
|
| 7 |
+
def __init__(self, netG, cfg, device, prepare_training_loss=False):
|
| 8 |
+
super(Audio2Exp, self).__init__()
|
| 9 |
+
self.cfg = cfg
|
| 10 |
+
self.device = device
|
| 11 |
+
self.netG = netG.to(device)
|
| 12 |
+
|
| 13 |
+
def test(self, batch):
|
| 14 |
+
|
| 15 |
+
mel_input = batch['indiv_mels'] # bs T 1 80 16
|
| 16 |
+
bs = mel_input.shape[0]
|
| 17 |
+
T = mel_input.shape[1]
|
| 18 |
+
|
| 19 |
+
exp_coeff_pred = []
|
| 20 |
+
|
| 21 |
+
for i in tqdm(range(0, T, 10),'audio2exp:'): # every 10 frames
|
| 22 |
+
|
| 23 |
+
current_mel_input = mel_input[:,i:i+10]
|
| 24 |
+
|
| 25 |
+
#ref = batch['ref'][:, :, :64].repeat((1,current_mel_input.shape[1],1)) #bs T 64
|
| 26 |
+
ref = batch['ref'][:, :, :64][:, i:i+10]
|
| 27 |
+
ratio = batch['ratio_gt'][:, i:i+10] #bs T
|
| 28 |
+
|
| 29 |
+
audiox = current_mel_input.view(-1, 1, 80, 16) # bs*T 1 80 16
|
| 30 |
+
|
| 31 |
+
curr_exp_coeff_pred = self.netG(audiox, ref, ratio) # bs T 64
|
| 32 |
+
|
| 33 |
+
exp_coeff_pred += [curr_exp_coeff_pred]
|
| 34 |
+
|
| 35 |
+
# BS x T x 64
|
| 36 |
+
results_dict = {
|
| 37 |
+
'exp_coeff_pred': torch.cat(exp_coeff_pred, axis=1)
|
| 38 |
+
}
|
| 39 |
+
return results_dict
|
| 40 |
+
|
| 41 |
+
|
sadtalker+wav2lip/sadtalker/src/audio2exp_models/networks.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
from torch import nn
|
| 4 |
+
|
| 5 |
+
class Conv2d(nn.Module):
|
| 6 |
+
def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, use_act = True, *args, **kwargs):
|
| 7 |
+
super().__init__(*args, **kwargs)
|
| 8 |
+
self.conv_block = nn.Sequential(
|
| 9 |
+
nn.Conv2d(cin, cout, kernel_size, stride, padding),
|
| 10 |
+
nn.BatchNorm2d(cout)
|
| 11 |
+
)
|
| 12 |
+
self.act = nn.ReLU()
|
| 13 |
+
self.residual = residual
|
| 14 |
+
self.use_act = use_act
|
| 15 |
+
|
| 16 |
+
def forward(self, x):
|
| 17 |
+
out = self.conv_block(x)
|
| 18 |
+
if self.residual:
|
| 19 |
+
out += x
|
| 20 |
+
|
| 21 |
+
if self.use_act:
|
| 22 |
+
return self.act(out)
|
| 23 |
+
else:
|
| 24 |
+
return out
|
| 25 |
+
|
| 26 |
+
class SimpleWrapperV2(nn.Module):
|
| 27 |
+
def __init__(self) -> None:
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.audio_encoder = nn.Sequential(
|
| 30 |
+
Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
|
| 31 |
+
Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
|
| 32 |
+
Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
|
| 33 |
+
|
| 34 |
+
Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1),
|
| 35 |
+
Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
|
| 36 |
+
Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
|
| 37 |
+
|
| 38 |
+
Conv2d(64, 128, kernel_size=3, stride=3, padding=1),
|
| 39 |
+
Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
|
| 40 |
+
Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
|
| 41 |
+
|
| 42 |
+
Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1),
|
| 43 |
+
Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),
|
| 44 |
+
|
| 45 |
+
Conv2d(256, 512, kernel_size=3, stride=1, padding=0),
|
| 46 |
+
Conv2d(512, 512, kernel_size=1, stride=1, padding=0),
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
#### load the pre-trained audio_encoder
|
| 50 |
+
#self.audio_encoder = self.audio_encoder.to(device)
|
| 51 |
+
'''
|
| 52 |
+
wav2lip_state_dict = torch.load('/apdcephfs_cq2/share_1290939/wenxuazhang/checkpoints/wav2lip.pth')['state_dict']
|
| 53 |
+
state_dict = self.audio_encoder.state_dict()
|
| 54 |
+
|
| 55 |
+
for k,v in wav2lip_state_dict.items():
|
| 56 |
+
if 'audio_encoder' in k:
|
| 57 |
+
print('init:', k)
|
| 58 |
+
state_dict[k.replace('module.audio_encoder.', '')] = v
|
| 59 |
+
self.audio_encoder.load_state_dict(state_dict)
|
| 60 |
+
'''
|
| 61 |
+
|
| 62 |
+
self.mapping1 = nn.Linear(512+64+1, 64)
|
| 63 |
+
#self.mapping2 = nn.Linear(30, 64)
|
| 64 |
+
#nn.init.constant_(self.mapping1.weight, 0.)
|
| 65 |
+
nn.init.constant_(self.mapping1.bias, 0.)
|
| 66 |
+
|
| 67 |
+
def forward(self, x, ref, ratio):
|
| 68 |
+
x = self.audio_encoder(x).view(x.size(0), -1)
|
| 69 |
+
ref_reshape = ref.reshape(x.size(0), -1)
|
| 70 |
+
ratio = ratio.reshape(x.size(0), -1)
|
| 71 |
+
|
| 72 |
+
y = self.mapping1(torch.cat([x, ref_reshape, ratio], dim=1))
|
| 73 |
+
out = y.reshape(ref.shape[0], ref.shape[1], -1) #+ ref # resudial
|
| 74 |
+
return out
|
sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio2pose.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch import nn
|
| 3 |
+
from src.audio2pose_models.cvae import CVAE
|
| 4 |
+
from src.audio2pose_models.discriminator import PoseSequenceDiscriminator
|
| 5 |
+
from src.audio2pose_models.audio_encoder import AudioEncoder
|
| 6 |
+
|
| 7 |
+
class Audio2Pose(nn.Module):
|
| 8 |
+
def __init__(self, cfg, wav2lip_checkpoint, device='cuda'):
|
| 9 |
+
super().__init__()
|
| 10 |
+
self.cfg = cfg
|
| 11 |
+
self.seq_len = cfg.MODEL.CVAE.SEQ_LEN
|
| 12 |
+
self.latent_dim = cfg.MODEL.CVAE.LATENT_SIZE
|
| 13 |
+
self.device = device
|
| 14 |
+
|
| 15 |
+
self.audio_encoder = AudioEncoder(wav2lip_checkpoint, device)
|
| 16 |
+
self.audio_encoder.eval()
|
| 17 |
+
for param in self.audio_encoder.parameters():
|
| 18 |
+
param.requires_grad = False
|
| 19 |
+
|
| 20 |
+
self.netG = CVAE(cfg)
|
| 21 |
+
self.netD_motion = PoseSequenceDiscriminator(cfg)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def forward(self, x):
|
| 25 |
+
|
| 26 |
+
batch = {}
|
| 27 |
+
coeff_gt = x['gt'].cuda().squeeze(0) #bs frame_len+1 73
|
| 28 |
+
batch['pose_motion_gt'] = coeff_gt[:, 1:, 64:70] - coeff_gt[:, :1, 64:70] #bs frame_len 6
|
| 29 |
+
batch['ref'] = coeff_gt[:, 0, 64:70] #bs 6
|
| 30 |
+
batch['class'] = x['class'].squeeze(0).cuda() # bs
|
| 31 |
+
indiv_mels= x['indiv_mels'].cuda().squeeze(0) # bs seq_len+1 80 16
|
| 32 |
+
|
| 33 |
+
# forward
|
| 34 |
+
audio_emb_list = []
|
| 35 |
+
audio_emb = self.audio_encoder(indiv_mels[:, 1:, :, :].unsqueeze(2)) #bs seq_len 512
|
| 36 |
+
batch['audio_emb'] = audio_emb
|
| 37 |
+
batch = self.netG(batch)
|
| 38 |
+
|
| 39 |
+
pose_motion_pred = batch['pose_motion_pred'] # bs frame_len 6
|
| 40 |
+
pose_gt = coeff_gt[:, 1:, 64:70].clone() # bs frame_len 6
|
| 41 |
+
pose_pred = coeff_gt[:, :1, 64:70] + pose_motion_pred # bs frame_len 6
|
| 42 |
+
|
| 43 |
+
batch['pose_pred'] = pose_pred
|
| 44 |
+
batch['pose_gt'] = pose_gt
|
| 45 |
+
|
| 46 |
+
return batch
|
| 47 |
+
|
| 48 |
+
def test(self, x):
|
| 49 |
+
|
| 50 |
+
batch = {}
|
| 51 |
+
ref = x['ref'] #bs 1 70
|
| 52 |
+
batch['ref'] = x['ref'][:,0,-6:]
|
| 53 |
+
batch['class'] = x['class']
|
| 54 |
+
bs = ref.shape[0]
|
| 55 |
+
|
| 56 |
+
indiv_mels= x['indiv_mels'] # bs T 1 80 16
|
| 57 |
+
indiv_mels_use = indiv_mels[:, 1:] # we regard the ref as the first frame
|
| 58 |
+
num_frames = x['num_frames']
|
| 59 |
+
num_frames = int(num_frames) - 1
|
| 60 |
+
|
| 61 |
+
#
|
| 62 |
+
div = num_frames//self.seq_len
|
| 63 |
+
re = num_frames%self.seq_len
|
| 64 |
+
audio_emb_list = []
|
| 65 |
+
pose_motion_pred_list = [torch.zeros(batch['ref'].unsqueeze(1).shape, dtype=batch['ref'].dtype,
|
| 66 |
+
device=batch['ref'].device)]
|
| 67 |
+
|
| 68 |
+
for i in range(div):
|
| 69 |
+
z = torch.randn(bs, self.latent_dim).to(ref.device)
|
| 70 |
+
batch['z'] = z
|
| 71 |
+
audio_emb = self.audio_encoder(indiv_mels_use[:, i*self.seq_len:(i+1)*self.seq_len,:,:,:]) #bs seq_len 512
|
| 72 |
+
batch['audio_emb'] = audio_emb
|
| 73 |
+
batch = self.netG.test(batch)
|
| 74 |
+
pose_motion_pred_list.append(batch['pose_motion_pred']) #list of bs seq_len 6
|
| 75 |
+
|
| 76 |
+
if re != 0:
|
| 77 |
+
z = torch.randn(bs, self.latent_dim).to(ref.device)
|
| 78 |
+
batch['z'] = z
|
| 79 |
+
audio_emb = self.audio_encoder(indiv_mels_use[:, -1*self.seq_len:,:,:,:]) #bs seq_len 512
|
| 80 |
+
if audio_emb.shape[1] != self.seq_len:
|
| 81 |
+
pad_dim = self.seq_len-audio_emb.shape[1]
|
| 82 |
+
pad_audio_emb = audio_emb[:, :1].repeat(1, pad_dim, 1)
|
| 83 |
+
audio_emb = torch.cat([pad_audio_emb, audio_emb], 1)
|
| 84 |
+
batch['audio_emb'] = audio_emb
|
| 85 |
+
batch = self.netG.test(batch)
|
| 86 |
+
pose_motion_pred_list.append(batch['pose_motion_pred'][:,-1*re:,:])
|
| 87 |
+
|
| 88 |
+
pose_motion_pred = torch.cat(pose_motion_pred_list, dim = 1)
|
| 89 |
+
batch['pose_motion_pred'] = pose_motion_pred
|
| 90 |
+
|
| 91 |
+
pose_pred = ref[:, :1, -6:] + pose_motion_pred # bs T 6
|
| 92 |
+
|
| 93 |
+
batch['pose_pred'] = pose_pred
|
| 94 |
+
return batch
|
sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio_encoder.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch import nn
|
| 3 |
+
from torch.nn import functional as F
|
| 4 |
+
|
| 5 |
+
class Conv2d(nn.Module):
|
| 6 |
+
def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs):
|
| 7 |
+
super().__init__(*args, **kwargs)
|
| 8 |
+
self.conv_block = nn.Sequential(
|
| 9 |
+
nn.Conv2d(cin, cout, kernel_size, stride, padding),
|
| 10 |
+
nn.BatchNorm2d(cout)
|
| 11 |
+
)
|
| 12 |
+
self.act = nn.ReLU()
|
| 13 |
+
self.residual = residual
|
| 14 |
+
|
| 15 |
+
def forward(self, x):
|
| 16 |
+
out = self.conv_block(x)
|
| 17 |
+
if self.residual:
|
| 18 |
+
out += x
|
| 19 |
+
return self.act(out)
|
| 20 |
+
|
| 21 |
+
class AudioEncoder(nn.Module):
|
| 22 |
+
def __init__(self, wav2lip_checkpoint, device):
|
| 23 |
+
super(AudioEncoder, self).__init__()
|
| 24 |
+
|
| 25 |
+
self.audio_encoder = nn.Sequential(
|
| 26 |
+
Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
|
| 27 |
+
Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
|
| 28 |
+
Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
|
| 29 |
+
|
| 30 |
+
Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1),
|
| 31 |
+
Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
|
| 32 |
+
Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
|
| 33 |
+
|
| 34 |
+
Conv2d(64, 128, kernel_size=3, stride=3, padding=1),
|
| 35 |
+
Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
|
| 36 |
+
Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
|
| 37 |
+
|
| 38 |
+
Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1),
|
| 39 |
+
Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),
|
| 40 |
+
|
| 41 |
+
Conv2d(256, 512, kernel_size=3, stride=1, padding=0),
|
| 42 |
+
Conv2d(512, 512, kernel_size=1, stride=1, padding=0),)
|
| 43 |
+
|
| 44 |
+
#### load the pre-trained audio_encoder, we do not need to load wav2lip model here.
|
| 45 |
+
# wav2lip_state_dict = torch.load(wav2lip_checkpoint, map_location=torch.device(device))['state_dict']
|
| 46 |
+
# state_dict = self.audio_encoder.state_dict()
|
| 47 |
+
|
| 48 |
+
# for k,v in wav2lip_state_dict.items():
|
| 49 |
+
# if 'audio_encoder' in k:
|
| 50 |
+
# state_dict[k.replace('module.audio_encoder.', '')] = v
|
| 51 |
+
# self.audio_encoder.load_state_dict(state_dict)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def forward(self, audio_sequences):
|
| 55 |
+
# audio_sequences = (B, T, 1, 80, 16)
|
| 56 |
+
B = audio_sequences.size(0)
|
| 57 |
+
|
| 58 |
+
audio_sequences = torch.cat([audio_sequences[:, i] for i in range(audio_sequences.size(1))], dim=0)
|
| 59 |
+
|
| 60 |
+
audio_embedding = self.audio_encoder(audio_sequences) # B, 512, 1, 1
|
| 61 |
+
dim = audio_embedding.shape[1]
|
| 62 |
+
audio_embedding = audio_embedding.reshape((B, -1, dim, 1, 1))
|
| 63 |
+
|
| 64 |
+
return audio_embedding.squeeze(-1).squeeze(-1) #B seq_len+1 512
|
sadtalker+wav2lip/sadtalker/src/audio2pose_models/cvae.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
from torch import nn
|
| 4 |
+
from src.audio2pose_models.res_unet import ResUnet
|
| 5 |
+
|
| 6 |
+
def class2onehot(idx, class_num):
|
| 7 |
+
|
| 8 |
+
assert torch.max(idx).item() < class_num
|
| 9 |
+
onehot = torch.zeros(idx.size(0), class_num).to(idx.device)
|
| 10 |
+
onehot.scatter_(1, idx, 1)
|
| 11 |
+
return onehot
|
| 12 |
+
|
| 13 |
+
class CVAE(nn.Module):
|
| 14 |
+
def __init__(self, cfg):
|
| 15 |
+
super().__init__()
|
| 16 |
+
encoder_layer_sizes = cfg.MODEL.CVAE.ENCODER_LAYER_SIZES
|
| 17 |
+
decoder_layer_sizes = cfg.MODEL.CVAE.DECODER_LAYER_SIZES
|
| 18 |
+
latent_size = cfg.MODEL.CVAE.LATENT_SIZE
|
| 19 |
+
num_classes = cfg.DATASET.NUM_CLASSES
|
| 20 |
+
audio_emb_in_size = cfg.MODEL.CVAE.AUDIO_EMB_IN_SIZE
|
| 21 |
+
audio_emb_out_size = cfg.MODEL.CVAE.AUDIO_EMB_OUT_SIZE
|
| 22 |
+
seq_len = cfg.MODEL.CVAE.SEQ_LEN
|
| 23 |
+
|
| 24 |
+
self.latent_size = latent_size
|
| 25 |
+
|
| 26 |
+
self.encoder = ENCODER(encoder_layer_sizes, latent_size, num_classes,
|
| 27 |
+
audio_emb_in_size, audio_emb_out_size, seq_len)
|
| 28 |
+
self.decoder = DECODER(decoder_layer_sizes, latent_size, num_classes,
|
| 29 |
+
audio_emb_in_size, audio_emb_out_size, seq_len)
|
| 30 |
+
def reparameterize(self, mu, logvar):
|
| 31 |
+
std = torch.exp(0.5 * logvar)
|
| 32 |
+
eps = torch.randn_like(std)
|
| 33 |
+
return mu + eps * std
|
| 34 |
+
|
| 35 |
+
def forward(self, batch):
|
| 36 |
+
batch = self.encoder(batch)
|
| 37 |
+
mu = batch['mu']
|
| 38 |
+
logvar = batch['logvar']
|
| 39 |
+
z = self.reparameterize(mu, logvar)
|
| 40 |
+
batch['z'] = z
|
| 41 |
+
return self.decoder(batch)
|
| 42 |
+
|
| 43 |
+
def test(self, batch):
|
| 44 |
+
'''
|
| 45 |
+
class_id = batch['class']
|
| 46 |
+
z = torch.randn([class_id.size(0), self.latent_size]).to(class_id.device)
|
| 47 |
+
batch['z'] = z
|
| 48 |
+
'''
|
| 49 |
+
return self.decoder(batch)
|
| 50 |
+
|
| 51 |
+
class ENCODER(nn.Module):
|
| 52 |
+
def __init__(self, layer_sizes, latent_size, num_classes,
|
| 53 |
+
audio_emb_in_size, audio_emb_out_size, seq_len):
|
| 54 |
+
super().__init__()
|
| 55 |
+
|
| 56 |
+
self.resunet = ResUnet()
|
| 57 |
+
self.num_classes = num_classes
|
| 58 |
+
self.seq_len = seq_len
|
| 59 |
+
|
| 60 |
+
self.MLP = nn.Sequential()
|
| 61 |
+
layer_sizes[0] += latent_size + seq_len*audio_emb_out_size + 6
|
| 62 |
+
for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
|
| 63 |
+
self.MLP.add_module(
|
| 64 |
+
name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
|
| 65 |
+
self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
|
| 66 |
+
|
| 67 |
+
self.linear_means = nn.Linear(layer_sizes[-1], latent_size)
|
| 68 |
+
self.linear_logvar = nn.Linear(layer_sizes[-1], latent_size)
|
| 69 |
+
self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size)
|
| 70 |
+
|
| 71 |
+
self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size))
|
| 72 |
+
|
| 73 |
+
def forward(self, batch):
|
| 74 |
+
class_id = batch['class']
|
| 75 |
+
pose_motion_gt = batch['pose_motion_gt'] #bs seq_len 6
|
| 76 |
+
ref = batch['ref'] #bs 6
|
| 77 |
+
bs = pose_motion_gt.shape[0]
|
| 78 |
+
audio_in = batch['audio_emb'] # bs seq_len audio_emb_in_size
|
| 79 |
+
|
| 80 |
+
#pose encode
|
| 81 |
+
pose_emb = self.resunet(pose_motion_gt.unsqueeze(1)) #bs 1 seq_len 6
|
| 82 |
+
pose_emb = pose_emb.reshape(bs, -1) #bs seq_len*6
|
| 83 |
+
|
| 84 |
+
#audio mapping
|
| 85 |
+
print(audio_in.shape)
|
| 86 |
+
audio_out = self.linear_audio(audio_in) # bs seq_len audio_emb_out_size
|
| 87 |
+
audio_out = audio_out.reshape(bs, -1)
|
| 88 |
+
|
| 89 |
+
class_bias = self.classbias[class_id] #bs latent_size
|
| 90 |
+
x_in = torch.cat([ref, pose_emb, audio_out, class_bias], dim=-1) #bs seq_len*(audio_emb_out_size+6)+latent_size
|
| 91 |
+
x_out = self.MLP(x_in)
|
| 92 |
+
|
| 93 |
+
mu = self.linear_means(x_out)
|
| 94 |
+
logvar = self.linear_means(x_out) #bs latent_size
|
| 95 |
+
|
| 96 |
+
batch.update({'mu':mu, 'logvar':logvar})
|
| 97 |
+
return batch
|
| 98 |
+
|
| 99 |
+
class DECODER(nn.Module):
|
| 100 |
+
def __init__(self, layer_sizes, latent_size, num_classes,
|
| 101 |
+
audio_emb_in_size, audio_emb_out_size, seq_len):
|
| 102 |
+
super().__init__()
|
| 103 |
+
|
| 104 |
+
self.resunet = ResUnet()
|
| 105 |
+
self.num_classes = num_classes
|
| 106 |
+
self.seq_len = seq_len
|
| 107 |
+
|
| 108 |
+
self.MLP = nn.Sequential()
|
| 109 |
+
input_size = latent_size + seq_len*audio_emb_out_size + 6
|
| 110 |
+
for i, (in_size, out_size) in enumerate(zip([input_size]+layer_sizes[:-1], layer_sizes)):
|
| 111 |
+
self.MLP.add_module(
|
| 112 |
+
name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
|
| 113 |
+
if i+1 < len(layer_sizes):
|
| 114 |
+
self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
|
| 115 |
+
else:
|
| 116 |
+
self.MLP.add_module(name="sigmoid", module=nn.Sigmoid())
|
| 117 |
+
|
| 118 |
+
self.pose_linear = nn.Linear(6, 6)
|
| 119 |
+
self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size)
|
| 120 |
+
|
| 121 |
+
self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size))
|
| 122 |
+
|
| 123 |
+
def forward(self, batch):
|
| 124 |
+
|
| 125 |
+
z = batch['z'] #bs latent_size
|
| 126 |
+
bs = z.shape[0]
|
| 127 |
+
class_id = batch['class']
|
| 128 |
+
ref = batch['ref'] #bs 6
|
| 129 |
+
audio_in = batch['audio_emb'] # bs seq_len audio_emb_in_size
|
| 130 |
+
#print('audio_in: ', audio_in[:, :, :10])
|
| 131 |
+
|
| 132 |
+
audio_out = self.linear_audio(audio_in) # bs seq_len audio_emb_out_size
|
| 133 |
+
#print('audio_out: ', audio_out[:, :, :10])
|
| 134 |
+
audio_out = audio_out.reshape([bs, -1]) # bs seq_len*audio_emb_out_size
|
| 135 |
+
class_bias = self.classbias[class_id] #bs latent_size
|
| 136 |
+
|
| 137 |
+
z = z + class_bias
|
| 138 |
+
x_in = torch.cat([ref, z, audio_out], dim=-1)
|
| 139 |
+
x_out = self.MLP(x_in) # bs layer_sizes[-1]
|
| 140 |
+
x_out = x_out.reshape((bs, self.seq_len, -1))
|
| 141 |
+
|
| 142 |
+
#print('x_out: ', x_out)
|
| 143 |
+
|
| 144 |
+
pose_emb = self.resunet(x_out.unsqueeze(1)) #bs 1 seq_len 6
|
| 145 |
+
|
| 146 |
+
pose_motion_pred = self.pose_linear(pose_emb.squeeze(1)) #bs seq_len 6
|
| 147 |
+
|
| 148 |
+
batch.update({'pose_motion_pred':pose_motion_pred})
|
| 149 |
+
return batch
|
sadtalker+wav2lip/sadtalker/src/audio2pose_models/discriminator.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
from torch import nn
|
| 4 |
+
|
| 5 |
+
class ConvNormRelu(nn.Module):
|
| 6 |
+
def __init__(self, conv_type='1d', in_channels=3, out_channels=64, downsample=False,
|
| 7 |
+
kernel_size=None, stride=None, padding=None, norm='BN', leaky=False):
|
| 8 |
+
super().__init__()
|
| 9 |
+
if kernel_size is None:
|
| 10 |
+
if downsample:
|
| 11 |
+
kernel_size, stride, padding = 4, 2, 1
|
| 12 |
+
else:
|
| 13 |
+
kernel_size, stride, padding = 3, 1, 1
|
| 14 |
+
|
| 15 |
+
if conv_type == '2d':
|
| 16 |
+
self.conv = nn.Conv2d(
|
| 17 |
+
in_channels,
|
| 18 |
+
out_channels,
|
| 19 |
+
kernel_size,
|
| 20 |
+
stride,
|
| 21 |
+
padding,
|
| 22 |
+
bias=False,
|
| 23 |
+
)
|
| 24 |
+
if norm == 'BN':
|
| 25 |
+
self.norm = nn.BatchNorm2d(out_channels)
|
| 26 |
+
elif norm == 'IN':
|
| 27 |
+
self.norm = nn.InstanceNorm2d(out_channels)
|
| 28 |
+
else:
|
| 29 |
+
raise NotImplementedError
|
| 30 |
+
elif conv_type == '1d':
|
| 31 |
+
self.conv = nn.Conv1d(
|
| 32 |
+
in_channels,
|
| 33 |
+
out_channels,
|
| 34 |
+
kernel_size,
|
| 35 |
+
stride,
|
| 36 |
+
padding,
|
| 37 |
+
bias=False,
|
| 38 |
+
)
|
| 39 |
+
if norm == 'BN':
|
| 40 |
+
self.norm = nn.BatchNorm1d(out_channels)
|
| 41 |
+
elif norm == 'IN':
|
| 42 |
+
self.norm = nn.InstanceNorm1d(out_channels)
|
| 43 |
+
else:
|
| 44 |
+
raise NotImplementedError
|
| 45 |
+
nn.init.kaiming_normal_(self.conv.weight)
|
| 46 |
+
|
| 47 |
+
self.act = nn.LeakyReLU(negative_slope=0.2, inplace=False) if leaky else nn.ReLU(inplace=True)
|
| 48 |
+
|
| 49 |
+
def forward(self, x):
|
| 50 |
+
x = self.conv(x)
|
| 51 |
+
if isinstance(self.norm, nn.InstanceNorm1d):
|
| 52 |
+
x = self.norm(x.permute((0, 2, 1))).permute((0, 2, 1)) # normalize on [C]
|
| 53 |
+
else:
|
| 54 |
+
x = self.norm(x)
|
| 55 |
+
x = self.act(x)
|
| 56 |
+
return x
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class PoseSequenceDiscriminator(nn.Module):
|
| 60 |
+
def __init__(self, cfg):
|
| 61 |
+
super().__init__()
|
| 62 |
+
self.cfg = cfg
|
| 63 |
+
leaky = self.cfg.MODEL.DISCRIMINATOR.LEAKY_RELU
|
| 64 |
+
|
| 65 |
+
self.seq = nn.Sequential(
|
| 66 |
+
ConvNormRelu('1d', cfg.MODEL.DISCRIMINATOR.INPUT_CHANNELS, 256, downsample=True, leaky=leaky), # B, 256, 64
|
| 67 |
+
ConvNormRelu('1d', 256, 512, downsample=True, leaky=leaky), # B, 512, 32
|
| 68 |
+
ConvNormRelu('1d', 512, 1024, kernel_size=3, stride=1, padding=1, leaky=leaky), # B, 1024, 16
|
| 69 |
+
nn.Conv1d(1024, 1, kernel_size=3, stride=1, padding=1, bias=True) # B, 1, 16
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
def forward(self, x):
|
| 73 |
+
x = x.reshape(x.size(0), x.size(1), -1).transpose(1, 2)
|
| 74 |
+
x = self.seq(x)
|
| 75 |
+
x = x.squeeze(1)
|
| 76 |
+
return x
|
sadtalker+wav2lip/sadtalker/src/audio2pose_models/networks.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch.nn as nn
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class ResidualConv(nn.Module):
|
| 6 |
+
def __init__(self, input_dim, output_dim, stride, padding):
|
| 7 |
+
super(ResidualConv, self).__init__()
|
| 8 |
+
|
| 9 |
+
self.conv_block = nn.Sequential(
|
| 10 |
+
nn.BatchNorm2d(input_dim),
|
| 11 |
+
nn.ReLU(),
|
| 12 |
+
nn.Conv2d(
|
| 13 |
+
input_dim, output_dim, kernel_size=3, stride=stride, padding=padding
|
| 14 |
+
),
|
| 15 |
+
nn.BatchNorm2d(output_dim),
|
| 16 |
+
nn.ReLU(),
|
| 17 |
+
nn.Conv2d(output_dim, output_dim, kernel_size=3, padding=1),
|
| 18 |
+
)
|
| 19 |
+
self.conv_skip = nn.Sequential(
|
| 20 |
+
nn.Conv2d(input_dim, output_dim, kernel_size=3, stride=stride, padding=1),
|
| 21 |
+
nn.BatchNorm2d(output_dim),
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
def forward(self, x):
|
| 25 |
+
|
| 26 |
+
return self.conv_block(x) + self.conv_skip(x)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class Upsample(nn.Module):
|
| 30 |
+
def __init__(self, input_dim, output_dim, kernel, stride):
|
| 31 |
+
super(Upsample, self).__init__()
|
| 32 |
+
|
| 33 |
+
self.upsample = nn.ConvTranspose2d(
|
| 34 |
+
input_dim, output_dim, kernel_size=kernel, stride=stride
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
def forward(self, x):
|
| 38 |
+
return self.upsample(x)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class Squeeze_Excite_Block(nn.Module):
|
| 42 |
+
def __init__(self, channel, reduction=16):
|
| 43 |
+
super(Squeeze_Excite_Block, self).__init__()
|
| 44 |
+
self.avg_pool = nn.AdaptiveAvgPool2d(1)
|
| 45 |
+
self.fc = nn.Sequential(
|
| 46 |
+
nn.Linear(channel, channel // reduction, bias=False),
|
| 47 |
+
nn.ReLU(inplace=True),
|
| 48 |
+
nn.Linear(channel // reduction, channel, bias=False),
|
| 49 |
+
nn.Sigmoid(),
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
def forward(self, x):
|
| 53 |
+
b, c, _, _ = x.size()
|
| 54 |
+
y = self.avg_pool(x).view(b, c)
|
| 55 |
+
y = self.fc(y).view(b, c, 1, 1)
|
| 56 |
+
return x * y.expand_as(x)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class ASPP(nn.Module):
|
| 60 |
+
def __init__(self, in_dims, out_dims, rate=[6, 12, 18]):
|
| 61 |
+
super(ASPP, self).__init__()
|
| 62 |
+
|
| 63 |
+
self.aspp_block1 = nn.Sequential(
|
| 64 |
+
nn.Conv2d(
|
| 65 |
+
in_dims, out_dims, 3, stride=1, padding=rate[0], dilation=rate[0]
|
| 66 |
+
),
|
| 67 |
+
nn.ReLU(inplace=True),
|
| 68 |
+
nn.BatchNorm2d(out_dims),
|
| 69 |
+
)
|
| 70 |
+
self.aspp_block2 = nn.Sequential(
|
| 71 |
+
nn.Conv2d(
|
| 72 |
+
in_dims, out_dims, 3, stride=1, padding=rate[1], dilation=rate[1]
|
| 73 |
+
),
|
| 74 |
+
nn.ReLU(inplace=True),
|
| 75 |
+
nn.BatchNorm2d(out_dims),
|
| 76 |
+
)
|
| 77 |
+
self.aspp_block3 = nn.Sequential(
|
| 78 |
+
nn.Conv2d(
|
| 79 |
+
in_dims, out_dims, 3, stride=1, padding=rate[2], dilation=rate[2]
|
| 80 |
+
),
|
| 81 |
+
nn.ReLU(inplace=True),
|
| 82 |
+
nn.BatchNorm2d(out_dims),
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
self.output = nn.Conv2d(len(rate) * out_dims, out_dims, 1)
|
| 86 |
+
self._init_weights()
|
| 87 |
+
|
| 88 |
+
def forward(self, x):
|
| 89 |
+
x1 = self.aspp_block1(x)
|
| 90 |
+
x2 = self.aspp_block2(x)
|
| 91 |
+
x3 = self.aspp_block3(x)
|
| 92 |
+
out = torch.cat([x1, x2, x3], dim=1)
|
| 93 |
+
return self.output(out)
|
| 94 |
+
|
| 95 |
+
def _init_weights(self):
|
| 96 |
+
for m in self.modules():
|
| 97 |
+
if isinstance(m, nn.Conv2d):
|
| 98 |
+
nn.init.kaiming_normal_(m.weight)
|
| 99 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 100 |
+
m.weight.data.fill_(1)
|
| 101 |
+
m.bias.data.zero_()
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class Upsample_(nn.Module):
|
| 105 |
+
def __init__(self, scale=2):
|
| 106 |
+
super(Upsample_, self).__init__()
|
| 107 |
+
|
| 108 |
+
self.upsample = nn.Upsample(mode="bilinear", scale_factor=scale)
|
| 109 |
+
|
| 110 |
+
def forward(self, x):
|
| 111 |
+
return self.upsample(x)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class AttentionBlock(nn.Module):
|
| 115 |
+
def __init__(self, input_encoder, input_decoder, output_dim):
|
| 116 |
+
super(AttentionBlock, self).__init__()
|
| 117 |
+
|
| 118 |
+
self.conv_encoder = nn.Sequential(
|
| 119 |
+
nn.BatchNorm2d(input_encoder),
|
| 120 |
+
nn.ReLU(),
|
| 121 |
+
nn.Conv2d(input_encoder, output_dim, 3, padding=1),
|
| 122 |
+
nn.MaxPool2d(2, 2),
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
self.conv_decoder = nn.Sequential(
|
| 126 |
+
nn.BatchNorm2d(input_decoder),
|
| 127 |
+
nn.ReLU(),
|
| 128 |
+
nn.Conv2d(input_decoder, output_dim, 3, padding=1),
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
self.conv_attn = nn.Sequential(
|
| 132 |
+
nn.BatchNorm2d(output_dim),
|
| 133 |
+
nn.ReLU(),
|
| 134 |
+
nn.Conv2d(output_dim, 1, 1),
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
def forward(self, x1, x2):
|
| 138 |
+
out = self.conv_encoder(x1) + self.conv_decoder(x2)
|
| 139 |
+
out = self.conv_attn(out)
|
| 140 |
+
return out * x2
|
sadtalker+wav2lip/sadtalker/src/audio2pose_models/res_unet.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from src.audio2pose_models.networks import ResidualConv, Upsample
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ResUnet(nn.Module):
|
| 7 |
+
def __init__(self, channel=1, filters=[32, 64, 128, 256]):
|
| 8 |
+
super(ResUnet, self).__init__()
|
| 9 |
+
|
| 10 |
+
self.input_layer = nn.Sequential(
|
| 11 |
+
nn.Conv2d(channel, filters[0], kernel_size=3, padding=1),
|
| 12 |
+
nn.BatchNorm2d(filters[0]),
|
| 13 |
+
nn.ReLU(),
|
| 14 |
+
nn.Conv2d(filters[0], filters[0], kernel_size=3, padding=1),
|
| 15 |
+
)
|
| 16 |
+
self.input_skip = nn.Sequential(
|
| 17 |
+
nn.Conv2d(channel, filters[0], kernel_size=3, padding=1)
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
self.residual_conv_1 = ResidualConv(filters[0], filters[1], stride=(2,1), padding=1)
|
| 21 |
+
self.residual_conv_2 = ResidualConv(filters[1], filters[2], stride=(2,1), padding=1)
|
| 22 |
+
|
| 23 |
+
self.bridge = ResidualConv(filters[2], filters[3], stride=(2,1), padding=1)
|
| 24 |
+
|
| 25 |
+
self.upsample_1 = Upsample(filters[3], filters[3], kernel=(2,1), stride=(2,1))
|
| 26 |
+
self.up_residual_conv1 = ResidualConv(filters[3] + filters[2], filters[2], stride=1, padding=1)
|
| 27 |
+
|
| 28 |
+
self.upsample_2 = Upsample(filters[2], filters[2], kernel=(2,1), stride=(2,1))
|
| 29 |
+
self.up_residual_conv2 = ResidualConv(filters[2] + filters[1], filters[1], stride=1, padding=1)
|
| 30 |
+
|
| 31 |
+
self.upsample_3 = Upsample(filters[1], filters[1], kernel=(2,1), stride=(2,1))
|
| 32 |
+
self.up_residual_conv3 = ResidualConv(filters[1] + filters[0], filters[0], stride=1, padding=1)
|
| 33 |
+
|
| 34 |
+
self.output_layer = nn.Sequential(
|
| 35 |
+
nn.Conv2d(filters[0], 1, 1, 1),
|
| 36 |
+
nn.Sigmoid(),
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
def forward(self, x):
|
| 40 |
+
# Encode
|
| 41 |
+
x1 = self.input_layer(x) + self.input_skip(x)
|
| 42 |
+
x2 = self.residual_conv_1(x1)
|
| 43 |
+
x3 = self.residual_conv_2(x2)
|
| 44 |
+
# Bridge
|
| 45 |
+
x4 = self.bridge(x3)
|
| 46 |
+
|
| 47 |
+
# Decode
|
| 48 |
+
x4 = self.upsample_1(x4)
|
| 49 |
+
x5 = torch.cat([x4, x3], dim=1)
|
| 50 |
+
|
| 51 |
+
x6 = self.up_residual_conv1(x5)
|
| 52 |
+
|
| 53 |
+
x6 = self.upsample_2(x6)
|
| 54 |
+
x7 = torch.cat([x6, x2], dim=1)
|
| 55 |
+
|
| 56 |
+
x8 = self.up_residual_conv2(x7)
|
| 57 |
+
|
| 58 |
+
x8 = self.upsample_3(x8)
|
| 59 |
+
x9 = torch.cat([x8, x1], dim=1)
|
| 60 |
+
|
| 61 |
+
x10 = self.up_residual_conv3(x9)
|
| 62 |
+
|
| 63 |
+
output = self.output_layer(x10)
|
| 64 |
+
|
| 65 |
+
return output
|
sadtalker+wav2lip/sadtalker/src/config/auido2exp.yaml
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
DATASET:
|
| 2 |
+
TRAIN_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/file_list/train.txt
|
| 3 |
+
EVAL_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/file_list/val.txt
|
| 4 |
+
TRAIN_BATCH_SIZE: 32
|
| 5 |
+
EVAL_BATCH_SIZE: 32
|
| 6 |
+
EXP: True
|
| 7 |
+
EXP_DIM: 64
|
| 8 |
+
FRAME_LEN: 32
|
| 9 |
+
COEFF_LEN: 73
|
| 10 |
+
NUM_CLASSES: 46
|
| 11 |
+
AUDIO_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav
|
| 12 |
+
COEFF_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav2lip_3dmm
|
| 13 |
+
LMDB_PATH: /apdcephfs_cq2/share_1290939/shadowcun/datasets/VoxCeleb/v1/imdb
|
| 14 |
+
DEBUG: True
|
| 15 |
+
NUM_REPEATS: 2
|
| 16 |
+
T: 40
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
MODEL:
|
| 20 |
+
FRAMEWORK: V2
|
| 21 |
+
AUDIOENCODER:
|
| 22 |
+
LEAKY_RELU: True
|
| 23 |
+
NORM: 'IN'
|
| 24 |
+
DISCRIMINATOR:
|
| 25 |
+
LEAKY_RELU: False
|
| 26 |
+
INPUT_CHANNELS: 6
|
| 27 |
+
CVAE:
|
| 28 |
+
AUDIO_EMB_IN_SIZE: 512
|
| 29 |
+
AUDIO_EMB_OUT_SIZE: 128
|
| 30 |
+
SEQ_LEN: 32
|
| 31 |
+
LATENT_SIZE: 256
|
| 32 |
+
ENCODER_LAYER_SIZES: [192, 1024]
|
| 33 |
+
DECODER_LAYER_SIZES: [1024, 192]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
TRAIN:
|
| 37 |
+
MAX_EPOCH: 300
|
| 38 |
+
GENERATOR:
|
| 39 |
+
LR: 2.0e-5
|
| 40 |
+
DISCRIMINATOR:
|
| 41 |
+
LR: 1.0e-5
|
| 42 |
+
LOSS:
|
| 43 |
+
W_FEAT: 0
|
| 44 |
+
W_COEFF_EXP: 2
|
| 45 |
+
W_LM: 1.0e-2
|
| 46 |
+
W_LM_MOUTH: 0
|
| 47 |
+
W_REG: 0
|
| 48 |
+
W_SYNC: 0
|
| 49 |
+
W_COLOR: 0
|
| 50 |
+
W_EXPRESSION: 0
|
| 51 |
+
W_LIPREADING: 0.01
|
| 52 |
+
W_LIPREADING_VV: 0
|
| 53 |
+
W_EYE_BLINK: 4
|
| 54 |
+
|
| 55 |
+
TAG:
|
| 56 |
+
NAME: small_dataset
|
| 57 |
+
|
| 58 |
+
|
sadtalker+wav2lip/sadtalker/src/config/auido2pose.yaml
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
DATASET:
|
| 2 |
+
TRAIN_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/audio2pose_unet_noAudio/dataset/train_33.txt
|
| 3 |
+
EVAL_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/audio2pose_unet_noAudio/dataset/val.txt
|
| 4 |
+
TRAIN_BATCH_SIZE: 64
|
| 5 |
+
EVAL_BATCH_SIZE: 1
|
| 6 |
+
EXP: True
|
| 7 |
+
EXP_DIM: 64
|
| 8 |
+
FRAME_LEN: 32
|
| 9 |
+
COEFF_LEN: 73
|
| 10 |
+
NUM_CLASSES: 46
|
| 11 |
+
AUDIO_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav
|
| 12 |
+
COEFF_ROOT_PATH: /apdcephfs_cq2/share_1290939/shadowcun/datasets/VoxCeleb/v1/imdb
|
| 13 |
+
DEBUG: True
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
MODEL:
|
| 17 |
+
AUDIOENCODER:
|
| 18 |
+
LEAKY_RELU: True
|
| 19 |
+
NORM: 'IN'
|
| 20 |
+
DISCRIMINATOR:
|
| 21 |
+
LEAKY_RELU: False
|
| 22 |
+
INPUT_CHANNELS: 6
|
| 23 |
+
CVAE:
|
| 24 |
+
AUDIO_EMB_IN_SIZE: 512
|
| 25 |
+
AUDIO_EMB_OUT_SIZE: 6
|
| 26 |
+
SEQ_LEN: 32
|
| 27 |
+
LATENT_SIZE: 64
|
| 28 |
+
ENCODER_LAYER_SIZES: [192, 128]
|
| 29 |
+
DECODER_LAYER_SIZES: [128, 192]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
TRAIN:
|
| 33 |
+
MAX_EPOCH: 150
|
| 34 |
+
GENERATOR:
|
| 35 |
+
LR: 1.0e-4
|
| 36 |
+
DISCRIMINATOR:
|
| 37 |
+
LR: 1.0e-4
|
| 38 |
+
LOSS:
|
| 39 |
+
LAMBDA_REG: 1
|
| 40 |
+
LAMBDA_LANDMARKS: 0
|
| 41 |
+
LAMBDA_VERTICES: 0
|
| 42 |
+
LAMBDA_GAN_MOTION: 0.7
|
| 43 |
+
LAMBDA_GAN_COEFF: 0
|
| 44 |
+
LAMBDA_KL: 1
|
| 45 |
+
|
| 46 |
+
TAG:
|
| 47 |
+
NAME: cvae_UNET_useAudio_usewav2lipAudioEncoder
|
| 48 |
+
|
| 49 |
+
|
sadtalker+wav2lip/sadtalker/src/config/facerender.yaml
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model_params:
|
| 2 |
+
common_params:
|
| 3 |
+
num_kp: 15
|
| 4 |
+
image_channel: 3
|
| 5 |
+
feature_channel: 32
|
| 6 |
+
estimate_jacobian: False # True
|
| 7 |
+
kp_detector_params:
|
| 8 |
+
temperature: 0.1
|
| 9 |
+
block_expansion: 32
|
| 10 |
+
max_features: 1024
|
| 11 |
+
scale_factor: 0.25 # 0.25
|
| 12 |
+
num_blocks: 5
|
| 13 |
+
reshape_channel: 16384 # 16384 = 1024 * 16
|
| 14 |
+
reshape_depth: 16
|
| 15 |
+
he_estimator_params:
|
| 16 |
+
block_expansion: 64
|
| 17 |
+
max_features: 2048
|
| 18 |
+
num_bins: 66
|
| 19 |
+
generator_params:
|
| 20 |
+
block_expansion: 64
|
| 21 |
+
max_features: 512
|
| 22 |
+
num_down_blocks: 2
|
| 23 |
+
reshape_channel: 32
|
| 24 |
+
reshape_depth: 16 # 512 = 32 * 16
|
| 25 |
+
num_resblocks: 6
|
| 26 |
+
estimate_occlusion_map: True
|
| 27 |
+
dense_motion_params:
|
| 28 |
+
block_expansion: 32
|
| 29 |
+
max_features: 1024
|
| 30 |
+
num_blocks: 5
|
| 31 |
+
reshape_depth: 16
|
| 32 |
+
compress: 4
|
| 33 |
+
discriminator_params:
|
| 34 |
+
scales: [1]
|
| 35 |
+
block_expansion: 32
|
| 36 |
+
max_features: 512
|
| 37 |
+
num_blocks: 4
|
| 38 |
+
sn: True
|
| 39 |
+
mapping_params:
|
| 40 |
+
coeff_nc: 70
|
| 41 |
+
descriptor_nc: 1024
|
| 42 |
+
layer: 3
|
| 43 |
+
num_kp: 15
|
| 44 |
+
num_bins: 66
|
| 45 |
+
|
sadtalker+wav2lip/sadtalker/src/config/facerender_still.yaml
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model_params:
|
| 2 |
+
common_params:
|
| 3 |
+
num_kp: 15
|
| 4 |
+
image_channel: 3
|
| 5 |
+
feature_channel: 32
|
| 6 |
+
estimate_jacobian: False # True
|
| 7 |
+
kp_detector_params:
|
| 8 |
+
temperature: 0.1
|
| 9 |
+
block_expansion: 32
|
| 10 |
+
max_features: 1024
|
| 11 |
+
scale_factor: 0.25 # 0.25
|
| 12 |
+
num_blocks: 5
|
| 13 |
+
reshape_channel: 16384 # 16384 = 1024 * 16
|
| 14 |
+
reshape_depth: 16
|
| 15 |
+
he_estimator_params:
|
| 16 |
+
block_expansion: 64
|
| 17 |
+
max_features: 2048
|
| 18 |
+
num_bins: 66
|
| 19 |
+
generator_params:
|
| 20 |
+
block_expansion: 64
|
| 21 |
+
max_features: 512
|
| 22 |
+
num_down_blocks: 2
|
| 23 |
+
reshape_channel: 32
|
| 24 |
+
reshape_depth: 16 # 512 = 32 * 16
|
| 25 |
+
num_resblocks: 6
|
| 26 |
+
estimate_occlusion_map: True
|
| 27 |
+
dense_motion_params:
|
| 28 |
+
block_expansion: 32
|
| 29 |
+
max_features: 1024
|
| 30 |
+
num_blocks: 5
|
| 31 |
+
reshape_depth: 16
|
| 32 |
+
compress: 4
|
| 33 |
+
discriminator_params:
|
| 34 |
+
scales: [1]
|
| 35 |
+
block_expansion: 32
|
| 36 |
+
max_features: 512
|
| 37 |
+
num_blocks: 4
|
| 38 |
+
sn: True
|
| 39 |
+
mapping_params:
|
| 40 |
+
coeff_nc: 73
|
| 41 |
+
descriptor_nc: 1024
|
| 42 |
+
layer: 3
|
| 43 |
+
num_kp: 15
|
| 44 |
+
num_bins: 66
|
| 45 |
+
|
sadtalker+wav2lip/sadtalker/src/config/similarity_Lm3D_all.mat
ADDED
|
Binary file (994 Bytes). View file
|
|
|
sadtalker+wav2lip/sadtalker/src/face3d/data/__init__.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This package includes all the modules related to data loading and preprocessing
|
| 2 |
+
|
| 3 |
+
To add a custom dataset class called 'dummy', you need to add a file called 'dummy_dataset.py' and define a subclass 'DummyDataset' inherited from BaseDataset.
|
| 4 |
+
You need to implement four functions:
|
| 5 |
+
-- <__init__>: initialize the class, first call BaseDataset.__init__(self, opt).
|
| 6 |
+
-- <__len__>: return the size of dataset.
|
| 7 |
+
-- <__getitem__>: get a data point from data loader.
|
| 8 |
+
-- <modify_commandline_options>: (optionally) add dataset-specific options and set default options.
|
| 9 |
+
|
| 10 |
+
Now you can use the dataset class by specifying flag '--dataset_mode dummy'.
|
| 11 |
+
See our template dataset class 'template_dataset.py' for more details.
|
| 12 |
+
"""
|
| 13 |
+
import numpy as np
|
| 14 |
+
import importlib
|
| 15 |
+
import torch.utils.data
|
| 16 |
+
from face3d.data.base_dataset import BaseDataset
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def find_dataset_using_name(dataset_name):
|
| 20 |
+
"""Import the module "data/[dataset_name]_dataset.py".
|
| 21 |
+
|
| 22 |
+
In the file, the class called DatasetNameDataset() will
|
| 23 |
+
be instantiated. It has to be a subclass of BaseDataset,
|
| 24 |
+
and it is case-insensitive.
|
| 25 |
+
"""
|
| 26 |
+
dataset_filename = "data." + dataset_name + "_dataset"
|
| 27 |
+
datasetlib = importlib.import_module(dataset_filename)
|
| 28 |
+
|
| 29 |
+
dataset = None
|
| 30 |
+
target_dataset_name = dataset_name.replace('_', '') + 'dataset'
|
| 31 |
+
for name, cls in datasetlib.__dict__.items():
|
| 32 |
+
if name.lower() == target_dataset_name.lower() \
|
| 33 |
+
and issubclass(cls, BaseDataset):
|
| 34 |
+
dataset = cls
|
| 35 |
+
|
| 36 |
+
if dataset is None:
|
| 37 |
+
raise NotImplementedError("In %s.py, there should be a subclass of BaseDataset with class name that matches %s in lowercase." % (dataset_filename, target_dataset_name))
|
| 38 |
+
|
| 39 |
+
return dataset
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def get_option_setter(dataset_name):
|
| 43 |
+
"""Return the static method <modify_commandline_options> of the dataset class."""
|
| 44 |
+
dataset_class = find_dataset_using_name(dataset_name)
|
| 45 |
+
return dataset_class.modify_commandline_options
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def create_dataset(opt, rank=0):
|
| 49 |
+
"""Create a dataset given the option.
|
| 50 |
+
|
| 51 |
+
This function wraps the class CustomDatasetDataLoader.
|
| 52 |
+
This is the main interface between this package and 'train.py'/'test.py'
|
| 53 |
+
|
| 54 |
+
Example:
|
| 55 |
+
>>> from data import create_dataset
|
| 56 |
+
>>> dataset = create_dataset(opt)
|
| 57 |
+
"""
|
| 58 |
+
data_loader = CustomDatasetDataLoader(opt, rank=rank)
|
| 59 |
+
dataset = data_loader.load_data()
|
| 60 |
+
return dataset
|
| 61 |
+
|
| 62 |
+
class CustomDatasetDataLoader():
|
| 63 |
+
"""Wrapper class of Dataset class that performs multi-threaded data loading"""
|
| 64 |
+
|
| 65 |
+
def __init__(self, opt, rank=0):
|
| 66 |
+
"""Initialize this class
|
| 67 |
+
|
| 68 |
+
Step 1: create a dataset instance given the name [dataset_mode]
|
| 69 |
+
Step 2: create a multi-threaded data loader.
|
| 70 |
+
"""
|
| 71 |
+
self.opt = opt
|
| 72 |
+
dataset_class = find_dataset_using_name(opt.dataset_mode)
|
| 73 |
+
self.dataset = dataset_class(opt)
|
| 74 |
+
self.sampler = None
|
| 75 |
+
print("rank %d %s dataset [%s] was created" % (rank, self.dataset.name, type(self.dataset).__name__))
|
| 76 |
+
if opt.use_ddp and opt.isTrain:
|
| 77 |
+
world_size = opt.world_size
|
| 78 |
+
self.sampler = torch.utils.data.distributed.DistributedSampler(
|
| 79 |
+
self.dataset,
|
| 80 |
+
num_replicas=world_size,
|
| 81 |
+
rank=rank,
|
| 82 |
+
shuffle=not opt.serial_batches
|
| 83 |
+
)
|
| 84 |
+
self.dataloader = torch.utils.data.DataLoader(
|
| 85 |
+
self.dataset,
|
| 86 |
+
sampler=self.sampler,
|
| 87 |
+
num_workers=int(opt.num_threads / world_size),
|
| 88 |
+
batch_size=int(opt.batch_size / world_size),
|
| 89 |
+
drop_last=True)
|
| 90 |
+
else:
|
| 91 |
+
self.dataloader = torch.utils.data.DataLoader(
|
| 92 |
+
self.dataset,
|
| 93 |
+
batch_size=opt.batch_size,
|
| 94 |
+
shuffle=(not opt.serial_batches) and opt.isTrain,
|
| 95 |
+
num_workers=int(opt.num_threads),
|
| 96 |
+
drop_last=True
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
def set_epoch(self, epoch):
|
| 100 |
+
self.dataset.current_epoch = epoch
|
| 101 |
+
if self.sampler is not None:
|
| 102 |
+
self.sampler.set_epoch(epoch)
|
| 103 |
+
|
| 104 |
+
def load_data(self):
|
| 105 |
+
return self
|
| 106 |
+
|
| 107 |
+
def __len__(self):
|
| 108 |
+
"""Return the number of data in the dataset"""
|
| 109 |
+
return min(len(self.dataset), self.opt.max_dataset_size)
|
| 110 |
+
|
| 111 |
+
def __iter__(self):
|
| 112 |
+
"""Return a batch of data"""
|
| 113 |
+
for i, data in enumerate(self.dataloader):
|
| 114 |
+
if i * self.opt.batch_size >= self.opt.max_dataset_size:
|
| 115 |
+
break
|
| 116 |
+
yield data
|
sadtalker+wav2lip/sadtalker/src/face3d/data/base_dataset.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This module implements an abstract base class (ABC) 'BaseDataset' for datasets.
|
| 2 |
+
|
| 3 |
+
It also includes common transformation functions (e.g., get_transform, __scale_width), which can be later used in subclasses.
|
| 4 |
+
"""
|
| 5 |
+
import random
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch.utils.data as data
|
| 8 |
+
from PIL import Image
|
| 9 |
+
import torchvision.transforms as transforms
|
| 10 |
+
from abc import ABC, abstractmethod
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class BaseDataset(data.Dataset, ABC):
|
| 14 |
+
"""This class is an abstract base class (ABC) for datasets.
|
| 15 |
+
|
| 16 |
+
To create a subclass, you need to implement the following four functions:
|
| 17 |
+
-- <__init__>: initialize the class, first call BaseDataset.__init__(self, opt).
|
| 18 |
+
-- <__len__>: return the size of dataset.
|
| 19 |
+
-- <__getitem__>: get a data point.
|
| 20 |
+
-- <modify_commandline_options>: (optionally) add dataset-specific options and set default options.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
def __init__(self, opt):
|
| 24 |
+
"""Initialize the class; save the options in the class
|
| 25 |
+
|
| 26 |
+
Parameters:
|
| 27 |
+
opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions
|
| 28 |
+
"""
|
| 29 |
+
self.opt = opt
|
| 30 |
+
# self.root = opt.dataroot
|
| 31 |
+
self.current_epoch = 0
|
| 32 |
+
|
| 33 |
+
@staticmethod
|
| 34 |
+
def modify_commandline_options(parser, is_train):
|
| 35 |
+
"""Add new dataset-specific options, and rewrite default values for existing options.
|
| 36 |
+
|
| 37 |
+
Parameters:
|
| 38 |
+
parser -- original option parser
|
| 39 |
+
is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options.
|
| 40 |
+
|
| 41 |
+
Returns:
|
| 42 |
+
the modified parser.
|
| 43 |
+
"""
|
| 44 |
+
return parser
|
| 45 |
+
|
| 46 |
+
@abstractmethod
|
| 47 |
+
def __len__(self):
|
| 48 |
+
"""Return the total number of images in the dataset."""
|
| 49 |
+
return 0
|
| 50 |
+
|
| 51 |
+
@abstractmethod
|
| 52 |
+
def __getitem__(self, index):
|
| 53 |
+
"""Return a data point and its metadata information.
|
| 54 |
+
|
| 55 |
+
Parameters:
|
| 56 |
+
index - - a random integer for data indexing
|
| 57 |
+
|
| 58 |
+
Returns:
|
| 59 |
+
a dictionary of data with their names. It ususally contains the data itself and its metadata information.
|
| 60 |
+
"""
|
| 61 |
+
pass
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def get_transform(grayscale=False):
|
| 65 |
+
transform_list = []
|
| 66 |
+
if grayscale:
|
| 67 |
+
transform_list.append(transforms.Grayscale(1))
|
| 68 |
+
transform_list += [transforms.ToTensor()]
|
| 69 |
+
return transforms.Compose(transform_list)
|
| 70 |
+
|
| 71 |
+
def get_affine_mat(opt, size):
|
| 72 |
+
shift_x, shift_y, scale, rot_angle, flip = 0., 0., 1., 0., False
|
| 73 |
+
w, h = size
|
| 74 |
+
|
| 75 |
+
if 'shift' in opt.preprocess:
|
| 76 |
+
shift_pixs = int(opt.shift_pixs)
|
| 77 |
+
shift_x = random.randint(-shift_pixs, shift_pixs)
|
| 78 |
+
shift_y = random.randint(-shift_pixs, shift_pixs)
|
| 79 |
+
if 'scale' in opt.preprocess:
|
| 80 |
+
scale = 1 + opt.scale_delta * (2 * random.random() - 1)
|
| 81 |
+
if 'rot' in opt.preprocess:
|
| 82 |
+
rot_angle = opt.rot_angle * (2 * random.random() - 1)
|
| 83 |
+
rot_rad = -rot_angle * np.pi/180
|
| 84 |
+
if 'flip' in opt.preprocess:
|
| 85 |
+
flip = random.random() > 0.5
|
| 86 |
+
|
| 87 |
+
shift_to_origin = np.array([1, 0, -w//2, 0, 1, -h//2, 0, 0, 1]).reshape([3, 3])
|
| 88 |
+
flip_mat = np.array([-1 if flip else 1, 0, 0, 0, 1, 0, 0, 0, 1]).reshape([3, 3])
|
| 89 |
+
shift_mat = np.array([1, 0, shift_x, 0, 1, shift_y, 0, 0, 1]).reshape([3, 3])
|
| 90 |
+
rot_mat = np.array([np.cos(rot_rad), np.sin(rot_rad), 0, -np.sin(rot_rad), np.cos(rot_rad), 0, 0, 0, 1]).reshape([3, 3])
|
| 91 |
+
scale_mat = np.array([scale, 0, 0, 0, scale, 0, 0, 0, 1]).reshape([3, 3])
|
| 92 |
+
shift_to_center = np.array([1, 0, w//2, 0, 1, h//2, 0, 0, 1]).reshape([3, 3])
|
| 93 |
+
|
| 94 |
+
affine = shift_to_center @ scale_mat @ rot_mat @ shift_mat @ flip_mat @ shift_to_origin
|
| 95 |
+
affine_inv = np.linalg.inv(affine)
|
| 96 |
+
return affine, affine_inv, flip
|
| 97 |
+
|
| 98 |
+
def apply_img_affine(img, affine_inv, method=Image.BICUBIC):
|
| 99 |
+
return img.transform(img.size, Image.AFFINE, data=affine_inv.flatten()[:6], resample=Image.BICUBIC)
|
| 100 |
+
|
| 101 |
+
def apply_lm_affine(landmark, affine, flip, size):
|
| 102 |
+
_, h = size
|
| 103 |
+
lm = landmark.copy()
|
| 104 |
+
lm[:, 1] = h - 1 - lm[:, 1]
|
| 105 |
+
lm = np.concatenate((lm, np.ones([lm.shape[0], 1])), -1)
|
| 106 |
+
lm = lm @ np.transpose(affine)
|
| 107 |
+
lm[:, :2] = lm[:, :2] / lm[:, 2:]
|
| 108 |
+
lm = lm[:, :2]
|
| 109 |
+
lm[:, 1] = h - 1 - lm[:, 1]
|
| 110 |
+
if flip:
|
| 111 |
+
lm_ = lm.copy()
|
| 112 |
+
lm_[:17] = lm[16::-1]
|
| 113 |
+
lm_[17:22] = lm[26:21:-1]
|
| 114 |
+
lm_[22:27] = lm[21:16:-1]
|
| 115 |
+
lm_[31:36] = lm[35:30:-1]
|
| 116 |
+
lm_[36:40] = lm[45:41:-1]
|
| 117 |
+
lm_[40:42] = lm[47:45:-1]
|
| 118 |
+
lm_[42:46] = lm[39:35:-1]
|
| 119 |
+
lm_[46:48] = lm[41:39:-1]
|
| 120 |
+
lm_[48:55] = lm[54:47:-1]
|
| 121 |
+
lm_[55:60] = lm[59:54:-1]
|
| 122 |
+
lm_[60:65] = lm[64:59:-1]
|
| 123 |
+
lm_[65:68] = lm[67:64:-1]
|
| 124 |
+
lm = lm_
|
| 125 |
+
return lm
|
sadtalker+wav2lip/sadtalker/src/face3d/data/flist_dataset.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This script defines the custom dataset for Deep3DFaceRecon_pytorch
|
| 2 |
+
"""
|
| 3 |
+
|
| 4 |
+
import os.path
|
| 5 |
+
from data.base_dataset import BaseDataset, get_transform, get_affine_mat, apply_img_affine, apply_lm_affine
|
| 6 |
+
from data.image_folder import make_dataset
|
| 7 |
+
from PIL import Image
|
| 8 |
+
import random
|
| 9 |
+
import util.util as util
|
| 10 |
+
import numpy as np
|
| 11 |
+
import json
|
| 12 |
+
import torch
|
| 13 |
+
from scipy.io import loadmat, savemat
|
| 14 |
+
import pickle
|
| 15 |
+
from util.preprocess import align_img, estimate_norm
|
| 16 |
+
from util.load_mats import load_lm3d
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def default_flist_reader(flist):
|
| 20 |
+
"""
|
| 21 |
+
flist format: impath label\nimpath label\n ...(same to caffe's filelist)
|
| 22 |
+
"""
|
| 23 |
+
imlist = []
|
| 24 |
+
with open(flist, 'r') as rf:
|
| 25 |
+
for line in rf.readlines():
|
| 26 |
+
impath = line.strip()
|
| 27 |
+
imlist.append(impath)
|
| 28 |
+
|
| 29 |
+
return imlist
|
| 30 |
+
|
| 31 |
+
def jason_flist_reader(flist):
|
| 32 |
+
with open(flist, 'r') as fp:
|
| 33 |
+
info = json.load(fp)
|
| 34 |
+
return info
|
| 35 |
+
|
| 36 |
+
def parse_label(label):
|
| 37 |
+
return torch.tensor(np.array(label).astype(np.float32))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class FlistDataset(BaseDataset):
|
| 41 |
+
"""
|
| 42 |
+
It requires one directories to host training images '/path/to/data/train'
|
| 43 |
+
You can train the model with the dataset flag '--dataroot /path/to/data'.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
def __init__(self, opt):
|
| 47 |
+
"""Initialize this dataset class.
|
| 48 |
+
|
| 49 |
+
Parameters:
|
| 50 |
+
opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions
|
| 51 |
+
"""
|
| 52 |
+
BaseDataset.__init__(self, opt)
|
| 53 |
+
|
| 54 |
+
self.lm3d_std = load_lm3d(opt.bfm_folder)
|
| 55 |
+
|
| 56 |
+
msk_names = default_flist_reader(opt.flist)
|
| 57 |
+
self.msk_paths = [os.path.join(opt.data_root, i) for i in msk_names]
|
| 58 |
+
|
| 59 |
+
self.size = len(self.msk_paths)
|
| 60 |
+
self.opt = opt
|
| 61 |
+
|
| 62 |
+
self.name = 'train' if opt.isTrain else 'val'
|
| 63 |
+
if '_' in opt.flist:
|
| 64 |
+
self.name += '_' + opt.flist.split(os.sep)[-1].split('_')[0]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def __getitem__(self, index):
|
| 68 |
+
"""Return a data point and its metadata information.
|
| 69 |
+
|
| 70 |
+
Parameters:
|
| 71 |
+
index (int) -- a random integer for data indexing
|
| 72 |
+
|
| 73 |
+
Returns a dictionary that contains A, B, A_paths and B_paths
|
| 74 |
+
img (tensor) -- an image in the input domain
|
| 75 |
+
msk (tensor) -- its corresponding attention mask
|
| 76 |
+
lm (tensor) -- its corresponding 3d landmarks
|
| 77 |
+
im_paths (str) -- image paths
|
| 78 |
+
aug_flag (bool) -- a flag used to tell whether its raw or augmented
|
| 79 |
+
"""
|
| 80 |
+
msk_path = self.msk_paths[index % self.size] # make sure index is within then range
|
| 81 |
+
img_path = msk_path.replace('mask/', '')
|
| 82 |
+
lm_path = '.'.join(msk_path.replace('mask', 'landmarks').split('.')[:-1]) + '.txt'
|
| 83 |
+
|
| 84 |
+
raw_img = Image.open(img_path).convert('RGB')
|
| 85 |
+
raw_msk = Image.open(msk_path).convert('RGB')
|
| 86 |
+
raw_lm = np.loadtxt(lm_path).astype(np.float32)
|
| 87 |
+
|
| 88 |
+
_, img, lm, msk = align_img(raw_img, raw_lm, self.lm3d_std, raw_msk)
|
| 89 |
+
|
| 90 |
+
aug_flag = self.opt.use_aug and self.opt.isTrain
|
| 91 |
+
if aug_flag:
|
| 92 |
+
img, lm, msk = self._augmentation(img, lm, self.opt, msk)
|
| 93 |
+
|
| 94 |
+
_, H = img.size
|
| 95 |
+
M = estimate_norm(lm, H)
|
| 96 |
+
transform = get_transform()
|
| 97 |
+
img_tensor = transform(img)
|
| 98 |
+
msk_tensor = transform(msk)[:1, ...]
|
| 99 |
+
lm_tensor = parse_label(lm)
|
| 100 |
+
M_tensor = parse_label(M)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
return {'imgs': img_tensor,
|
| 104 |
+
'lms': lm_tensor,
|
| 105 |
+
'msks': msk_tensor,
|
| 106 |
+
'M': M_tensor,
|
| 107 |
+
'im_paths': img_path,
|
| 108 |
+
'aug_flag': aug_flag,
|
| 109 |
+
'dataset': self.name}
|
| 110 |
+
|
| 111 |
+
def _augmentation(self, img, lm, opt, msk=None):
|
| 112 |
+
affine, affine_inv, flip = get_affine_mat(opt, img.size)
|
| 113 |
+
img = apply_img_affine(img, affine_inv)
|
| 114 |
+
lm = apply_lm_affine(lm, affine, flip, img.size)
|
| 115 |
+
if msk is not None:
|
| 116 |
+
msk = apply_img_affine(msk, affine_inv, method=Image.BILINEAR)
|
| 117 |
+
return img, lm, msk
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def __len__(self):
|
| 123 |
+
"""Return the total number of images in the dataset.
|
| 124 |
+
"""
|
| 125 |
+
return self.size
|