IamSamk commited on
Commit
27caffe
·
0 Parent(s):

Mindfull Gradio Space deploy

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .env.example +13 -0
  2. .gitignore +73 -0
  3. INTEGRATION_GUIDE.md +514 -0
  4. MISTRAL_SETUP.md +226 -0
  5. Modelfile +10 -0
  6. Modelfile_Mindfull +39 -0
  7. README_COMPLETE.md +339 -0
  8. README_SPACE.md +55 -0
  9. Readme.md +154 -0
  10. analyze_codebase.py +1024 -0
  11. app.py +19 -0
  12. gradio_app.py +283 -0
  13. infer_cli.py +368 -0
  14. mindfull_client.html +515 -0
  15. mindfull_config.py +308 -0
  16. mindfull_pipeline.py +478 -0
  17. mindfull_web_api.py +324 -0
  18. model_manager.py +252 -0
  19. my_finetuned_model/config.json +193 -0
  20. my_finetuned_model/vocab.json +0 -0
  21. ollama_to_f5tts.py +37 -0
  22. python313_compat_patch.py +116 -0
  23. requirements.txt +28 -0
  24. requirements_full.txt +112 -0
  25. requirements_hf.txt +28 -0
  26. requirements_unified.txt +97 -0
  27. sadtalker+wav2lip/README.md +75 -0
  28. sadtalker+wav2lip/README_POLICE_BOT_AI.md +199 -0
  29. sadtalker+wav2lip/enhanced_pipeline.py +231 -0
  30. sadtalker+wav2lip/init_path.py +47 -0
  31. sadtalker+wav2lip/preprocess.py +170 -0
  32. sadtalker+wav2lip/requirements.txt +29 -0
  33. sadtalker+wav2lip/sadtalker/generate_batch.py +120 -0
  34. sadtalker+wav2lip/sadtalker/inference.py +145 -0
  35. sadtalker+wav2lip/sadtalker/src/audio2exp_models/audio2exp.py +41 -0
  36. sadtalker+wav2lip/sadtalker/src/audio2exp_models/networks.py +74 -0
  37. sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio2pose.py +94 -0
  38. sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio_encoder.py +64 -0
  39. sadtalker+wav2lip/sadtalker/src/audio2pose_models/cvae.py +149 -0
  40. sadtalker+wav2lip/sadtalker/src/audio2pose_models/discriminator.py +76 -0
  41. sadtalker+wav2lip/sadtalker/src/audio2pose_models/networks.py +140 -0
  42. sadtalker+wav2lip/sadtalker/src/audio2pose_models/res_unet.py +65 -0
  43. sadtalker+wav2lip/sadtalker/src/config/auido2exp.yaml +58 -0
  44. sadtalker+wav2lip/sadtalker/src/config/auido2pose.yaml +49 -0
  45. sadtalker+wav2lip/sadtalker/src/config/facerender.yaml +45 -0
  46. sadtalker+wav2lip/sadtalker/src/config/facerender_still.yaml +45 -0
  47. sadtalker+wav2lip/sadtalker/src/config/similarity_Lm3D_all.mat +0 -0
  48. sadtalker+wav2lip/sadtalker/src/face3d/data/__init__.py +116 -0
  49. sadtalker+wav2lip/sadtalker/src/face3d/data/base_dataset.py +125 -0
  50. sadtalker+wav2lip/sadtalker/src/face3d/data/flist_dataset.py +125 -0
.env.example ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Hugging Face — model downloads + Spaces LLM fallback
2
+ HF_TOKEN=
3
+
4
+ # Ollama (local LLM)
5
+ OLLAMA_MODEL=mindfull
6
+
7
+ # Set to 1 when SadTalker checkpoints are not present
8
+ DISABLE_VIDEO=0
9
+
10
+ # Override HF model repos (optional; defaults in scripts/download_models.py)
11
+ # HF_REPO_F5TTS=IamSamkk/police-bot-f5tts
12
+ # HF_REPO_SADTALKER=IamSamkk/sadtalker-checkpoints
13
+ # HF_REPO_GFPGAN=IamSamkk/gfpgan-weights
.gitignore ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ── Python ──────────────────────────────────────────────────────────────────
2
+ venv/
3
+ .venv/
4
+ env/
5
+ __pycache__/
6
+ *.py[cod]
7
+ *$py.class
8
+ *.egg-info/
9
+ .eggs/
10
+ dist/
11
+ build/
12
+ .pytest_cache/
13
+ .mypy_cache/
14
+ .ruff_cache/
15
+ htmlcov/
16
+ .coverage
17
+
18
+ # ── Secrets & local config ────────────────────────────────────────────────────
19
+ .env
20
+ .env.*
21
+ !.env.example
22
+ *.pem
23
+ *.key
24
+ credentials.json
25
+
26
+ # ── IDE / OS ─────────────────────────────────────────────────────────────────
27
+ .idea/
28
+ .vscode/
29
+ *.swp
30
+ .DS_Store
31
+ Thumbs.db
32
+ desktop.ini
33
+
34
+ # ── Runtime outputs (regenerated) ─────────────────────────────────────────────
35
+ outputs/
36
+ temp/
37
+ *.log
38
+
39
+ # ── Large ML weights (host on Hugging Face Hub, not GitHub) ─────────────────
40
+ *.pth
41
+ *.pt
42
+ *.pth.tar
43
+ *.safetensors
44
+ *.ckpt
45
+ *.onnx
46
+ *.bin
47
+ *.mat
48
+ *.dat
49
+
50
+ my_finetuned_model/model.pth
51
+ my_finetuned_model/dvae.pth
52
+ my_finetuned_model/mel_stats.pth
53
+
54
+ gfpgan/weights/
55
+ sadtalker+wav2lip/gfpgan/weights/
56
+ sadtalker+wav2lip/sadtalker/gfpgan/weights/
57
+ sadtalker+wav2lip/sadtalker/checkpoints/
58
+ sadtalker+wav2lip/wav2lip/Wav2Lip-SD-NOGAN.pt
59
+
60
+ # ── Generated test media in sadtalker folder ──────────────────────────────────
61
+ sadtalker+wav2lip/test_*.mp4
62
+ sadtalker+wav2lip/temp/
63
+
64
+ # ── Training data & local media (not for git / HF Space — use HF Hub or local disk) ──
65
+ datasets-1/
66
+ segment_43.wav
67
+ avatar_assets/
68
+ fm_Police.png
69
+ sadtalker+wav2lip/avatar_assets/
70
+
71
+ # Large auto-generated docs (keep locally; optional on GitHub)
72
+ CODEBASE_MAP.md
73
+ UNUSED_FILES.md
INTEGRATION_GUIDE.md ADDED
@@ -0,0 +1,514 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🛡️ Police Bot AI Integration Guide
2
+
3
+ This guide explains how to integrate the AI wellness assistant with your website and enhance it with additional local LLM capabilities.
4
+
5
+ ## 🏗️ System Architecture
6
+
7
+ ```
8
+ Website Frontend (React/HTML/CSS)
9
+ ↓ HTTP/WebSocket
10
+ Flask Web API (web_api.py)
11
+ ↓
12
+ Police Bot Runtime (police_runtime.py)
13
+ ↓
14
+ ├── LLaMA3 via Ollama (Primary Reasoning)
15
+ ├── F5-TTS (Voice Synthesis)
16
+ └── [Third Local LLM] (Enhanced Capabilities)
17
+ ```
18
+
19
+ ## 🚀 Quick Start
20
+
21
+ ### 1. Setup the AI Runtime
22
+
23
+ ```bash
24
+ # Run the setup script
25
+ python setup.py
26
+
27
+ # Start Ollama with the police-bot model
28
+ ollama run police-bot
29
+
30
+ # Start the web API
31
+ python web_api.py
32
+ ```
33
+
34
+ ### 2. Test the System
35
+
36
+ ```bash
37
+ # Test the command-line interface
38
+ python police_runtime.py
39
+
40
+ # Test the web API
41
+ curl -X POST http://localhost:5000/api/chat \
42
+ -H "Content-Type: application/json" \
43
+ -d '{"message": "I am feeling stressed today"}'
44
+ ```
45
+
46
+ ## 🌐 Website Integration
47
+
48
+ ### Option 1: Simple HTML Integration
49
+
50
+ Add this to your website:
51
+
52
+ ```html
53
+ <!DOCTYPE html>
54
+ <html>
55
+ <head>
56
+ <title>Police Wellness Assistant</title>
57
+ <style>
58
+ .police-bot-chat {
59
+ max-width: 600px;
60
+ margin: 20px auto;
61
+ border: 1px solid #ccc;
62
+ border-radius: 8px;
63
+ overflow: hidden;
64
+ }
65
+
66
+ .chat-header {
67
+ background: #1e40af;
68
+ color: white;
69
+ padding: 15px;
70
+ text-align: center;
71
+ }
72
+
73
+ .chat-messages {
74
+ height: 400px;
75
+ overflow-y: auto;
76
+ padding: 15px;
77
+ background: #f8fafc;
78
+ }
79
+
80
+ .message {
81
+ margin-bottom: 10px;
82
+ padding: 10px;
83
+ border-radius: 8px;
84
+ }
85
+
86
+ .user-message {
87
+ background: #dbeafe;
88
+ margin-left: 20%;
89
+ }
90
+
91
+ .bot-message {
92
+ background: white;
93
+ margin-right: 20%;
94
+ }
95
+
96
+ .chat-input {
97
+ display: flex;
98
+ padding: 15px;
99
+ background: white;
100
+ }
101
+
102
+ .chat-input input {
103
+ flex: 1;
104
+ padding: 10px;
105
+ border: 1px solid #ccc;
106
+ border-radius: 4px;
107
+ margin-right: 10px;
108
+ }
109
+
110
+ .chat-input button {
111
+ padding: 10px 20px;
112
+ background: #1e40af;
113
+ color: white;
114
+ border: none;
115
+ border-radius: 4px;
116
+ cursor: pointer;
117
+ }
118
+
119
+ .wellness-tips {
120
+ padding: 15px;
121
+ background: #f0f9ff;
122
+ border-top: 1px solid #ccc;
123
+ }
124
+ </style>
125
+ </head>
126
+ <body>
127
+ <div id="police-bot-container"></div>
128
+
129
+ <script src="static/js/police-bot-client.js"></script>
130
+ <script>
131
+ // Initialize the Police Bot UI
132
+ const policeBot = new PoliceBotUI('police-bot-container');
133
+ </script>
134
+ </body>
135
+ </html>
136
+ ```
137
+
138
+ ### Option 2: React Integration
139
+
140
+ ```jsx
141
+ import React, { useState, useEffect } from 'react';
142
+ import { PoliceBotClient } from './police-bot-client.js';
143
+
144
+ function PoliceBotChat() {
145
+ const [messages, setMessages] = useState([]);
146
+ const [input, setInput] = useState('');
147
+ const [isProcessing, setIsProcessing] = useState(false);
148
+ const [wellnessTips, setWellnessTips] = useState([]);
149
+
150
+ const client = new PoliceBotClient('http://localhost:5000');
151
+
152
+ const sendMessage = async () => {
153
+ if (!input.trim() || isProcessing) return;
154
+
155
+ const userMessage = { text: input, sender: 'user', timestamp: Date.now() };
156
+ setMessages(prev => [...prev, userMessage]);
157
+ setInput('');
158
+ setIsProcessing(true);
159
+
160
+ try {
161
+ await client.sendMessage(
162
+ input,
163
+ (response) => {
164
+ const botMessage = {
165
+ text: response.text,
166
+ sender: 'bot',
167
+ timestamp: Date.now()
168
+ };
169
+ setMessages(prev => [...prev, botMessage]);
170
+ setWellnessTips(response.wellness_tips || []);
171
+ },
172
+ (error) => {
173
+ const errorMessage = {
174
+ text: `Error: ${error}`,
175
+ sender: 'bot',
176
+ timestamp: Date.now()
177
+ };
178
+ setMessages(prev => [...prev, errorMessage]);
179
+ }
180
+ );
181
+ } finally {
182
+ setIsProcessing(false);
183
+ }
184
+ };
185
+
186
+ return (
187
+ <div className="police-bot-chat">
188
+ <div className="chat-header">
189
+ <h3>🛡️ Police Wellness Assistant</h3>
190
+ </div>
191
+
192
+ <div className="chat-messages">
193
+ {messages.map((msg, index) => (
194
+ <div key={index} className={`message ${msg.sender}-message`}>
195
+ {msg.text}
196
+ </div>
197
+ ))}
198
+ </div>
199
+
200
+ <div className="chat-input">
201
+ <input
202
+ type="text"
203
+ value={input}
204
+ onChange={(e) => setInput(e.target.value)}
205
+ onKeyPress={(e) => e.key === 'Enter' && sendMessage()}
206
+ placeholder="Type your message..."
207
+ disabled={isProcessing}
208
+ />
209
+ <button onClick={sendMessage} disabled={isProcessing}>
210
+ {isProcessing ? 'Processing...' : 'Send'}
211
+ </button>
212
+ </div>
213
+
214
+ {wellnessTips.length > 0 && (
215
+ <div className="wellness-tips">
216
+ <h4>Wellness Tips</h4>
217
+ <ul>
218
+ {wellnessTips.map((tip, index) => (
219
+ <li key={index}>{tip}</li>
220
+ ))}
221
+ </ul>
222
+ </div>
223
+ )}
224
+ </div>
225
+ );
226
+ }
227
+
228
+ export default PoliceBotChat;
229
+ ```
230
+
231
+ ## 🤖 Adding a Third Local LLM
232
+
233
+ You can enhance the system with additional local LLMs for specialized tasks:
234
+
235
+ ### Option 1: Emotion Analysis LLM
236
+
237
+ ```python
238
+ # Add to police_runtime.py
239
+ def get_emotion_llm_response(self, text: str) -> str:
240
+ """Get emotion analysis from a dedicated LLM"""
241
+ try:
242
+ # You can use a smaller, specialized model for emotion analysis
243
+ data = {
244
+ "model": "emotion-analyzer", # Different Ollama model
245
+ "prompt": f"Analyze the emotional state of this text: '{text}'. Respond with only: stressed, positive, negative, or neutral.",
246
+ "stream": False,
247
+ "options": {
248
+ "temperature": 0.1, # Lower temperature for consistent analysis
249
+ "max_tokens": 10
250
+ }
251
+ }
252
+
253
+ response = requests.post(self.ollama_url, json=data, timeout=10)
254
+ response.raise_for_status()
255
+
256
+ result = response.json()
257
+ emotion = result.get("response", "").strip().lower()
258
+
259
+ # Validate emotion
260
+ valid_emotions = ["stressed", "positive", "negative", "neutral"]
261
+ return emotion if emotion in valid_emotions else "neutral"
262
+
263
+ except Exception as e:
264
+ logger.error(f"Error in emotion analysis: {e}")
265
+ return "neutral"
266
+ ```
267
+
268
+ ### Option 2: Wellness Recommendation LLM
269
+
270
+ ```python
271
+ def get_wellness_llm_response(self, emotional_state: str, context: str = "") -> list:
272
+ """Get personalized wellness recommendations from a specialized LLM"""
273
+ try:
274
+ prompt = f"""
275
+ As a wellness expert for police officers, provide 3-4 specific, actionable wellness tips for someone who is feeling {emotional_state}.
276
+
277
+ Context: {context}
278
+
279
+ Focus on:
280
+ - Quick, practical interventions (2-5 minutes)
281
+ - Stress management techniques
282
+ - Physical wellness (hydration, movement)
283
+ - Mental wellness (mindfulness, perspective)
284
+
285
+ Respond with only the tips, one per line, no numbering.
286
+ """
287
+
288
+ data = {
289
+ "model": "wellness-expert", # Specialized wellness model
290
+ "prompt": prompt,
291
+ "stream": False,
292
+ "options": {
293
+ "temperature": 0.7,
294
+ "max_tokens": 200
295
+ }
296
+ }
297
+
298
+ response = requests.post(self.ollama_url, json=data, timeout=15)
299
+ response.raise_for_status()
300
+
301
+ result = response.json()
302
+ tips_text = result.get("response", "").strip()
303
+
304
+ # Parse tips into list
305
+ tips = [tip.strip() for tip in tips_text.split('\n') if tip.strip()]
306
+ return tips[:4] # Limit to 4 tips
307
+
308
+ except Exception as e:
309
+ logger.error(f"Error in wellness recommendations: {e}")
310
+ return self.generate_wellness_tips(emotional_state) # Fallback
311
+ ```
312
+
313
+ ### Option 3: Multi-Model Setup
314
+
315
+ Create a model manager to handle multiple LLMs:
316
+
317
+ ```python
318
+ class ModelManager:
319
+ def __init__(self):
320
+ self.models = {
321
+ "primary": "police-bot", # Main conversation
322
+ "emotion": "emotion-analyzer", # Emotion analysis
323
+ "wellness": "wellness-expert", # Wellness recommendations
324
+ "kannada": "kannada-assistant" # Kannada language support
325
+ }
326
+
327
+ def get_response(self, model_type: str, prompt: str, **kwargs) -> str:
328
+ """Get response from specified model"""
329
+ model_name = self.models.get(model_type, "police-bot")
330
+
331
+ data = {
332
+ "model": model_name,
333
+ "prompt": prompt,
334
+ "stream": False,
335
+ **kwargs
336
+ }
337
+
338
+ response = requests.post("http://localhost:11434/api/generate", json=data)
339
+ return response.json().get("response", "").strip()
340
+ ```
341
+
342
+ ## 🔧 Configuration
343
+
344
+ ### Environment Variables
345
+
346
+ Create a `.env` file:
347
+
348
+ ```env
349
+ # Ollama Configuration
350
+ OLLAMA_URL=http://localhost:11434
351
+ OLLAMA_MODEL=police-bot
352
+
353
+ # F5-TTS Configuration
354
+ F5TTS_CKPT=C:\Users\Samarth Kadam\Voice-train\F5-TTS\ckpts\my_speak\model_last.pt
355
+ F5TTS_SCRIPT=C:\Users\Samarth Kadam\Voice-train\F5-TTS\src\f5_tts\infer\infer_cli.py
356
+
357
+ # Web API Configuration
358
+ WEB_PORT=5000
359
+ WEB_HOST=0.0.0.0
360
+
361
+ # Security
362
+ API_KEY=your-secret-api-key
363
+ ```
364
+
365
+ ### CORS Configuration
366
+
367
+ For production deployment, configure CORS properly:
368
+
369
+ ```python
370
+ # In web_api.py
371
+ from flask_cors import CORS
372
+
373
+ app = Flask(__name__)
374
+ CORS(app, origins=[
375
+ "http://localhost:3000", # React dev server
376
+ "http://your-website.com", # Production website
377
+ "https://your-website.com"
378
+ ])
379
+ ```
380
+
381
+ ## 🚀 Production Deployment
382
+
383
+ ### Using Gunicorn
384
+
385
+ ```bash
386
+ # Install gunicorn
387
+ pip install gunicorn
388
+
389
+ # Start production server
390
+ gunicorn -w 4 -b 0.0.0.0:5000 web_api:app
391
+ ```
392
+
393
+ ### Using Docker
394
+
395
+ ```dockerfile
396
+ FROM python:3.11-slim
397
+
398
+ WORKDIR /app
399
+
400
+ COPY requirements.txt .
401
+ RUN pip install -r requirements.txt
402
+
403
+ COPY . .
404
+
405
+ EXPOSE 5000
406
+
407
+ CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "web_api:app"]
408
+ ```
409
+
410
+ ### Systemd Service (Linux)
411
+
412
+ ```ini
413
+ # /etc/systemd/system/police-bot.service
414
+ [Unit]
415
+ Description=Police Bot AI Runtime
416
+ After=network.target
417
+
418
+ [Service]
419
+ Type=simple
420
+ User=police-bot
421
+ WorkingDirectory=/opt/police-bot-runtime
422
+ Environment=PATH=/opt/police-bot-runtime/venv/bin
423
+ ExecStart=/opt/police-bot-runtime/venv/bin/python web_api.py
424
+ Restart=always
425
+
426
+ [Install]
427
+ WantedBy=multi-user.target
428
+ ```
429
+
430
+ ## 🔒 Security Considerations
431
+
432
+ 1. **API Authentication**: Add API key validation
433
+ 2. **Rate Limiting**: Implement request rate limiting
434
+ 3. **Input Validation**: Sanitize all user inputs
435
+ 4. **HTTPS**: Use SSL/TLS in production
436
+ 5. **Firewall**: Restrict access to necessary ports only
437
+
438
+ ## 📊 Monitoring and Logging
439
+
440
+ ```python
441
+ # Add to web_api.py
442
+ import logging
443
+ from logging.handlers import RotatingFileHandler
444
+
445
+ # Configure logging
446
+ if not app.debug:
447
+ file_handler = RotatingFileHandler('logs/police_bot.log', maxBytes=10240, backupCount=10)
448
+ file_handler.setFormatter(logging.Formatter(
449
+ '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'
450
+ ))
451
+ file_handler.setLevel(logging.INFO)
452
+ app.logger.addHandler(file_handler)
453
+ app.logger.setLevel(logging.INFO)
454
+ app.logger.info('Police Bot startup')
455
+ ```
456
+
457
+ ## 🧪 Testing
458
+
459
+ ### API Testing
460
+
461
+ ```bash
462
+ # Test chat endpoint
463
+ curl -X POST http://localhost:5000/api/chat \
464
+ -H "Content-Type: application/json" \
465
+ -d '{"message": "I am feeling stressed today"}'
466
+
467
+ # Test health endpoint
468
+ curl http://localhost:5000/health
469
+
470
+ # Test status endpoint
471
+ curl http://localhost:5000/api/status
472
+ ```
473
+
474
+ ### Load Testing
475
+
476
+ ```python
477
+ import requests
478
+ import time
479
+ import threading
480
+
481
+ def test_load():
482
+ for i in range(10):
483
+ response = requests.post('http://localhost:5000/api/chat',
484
+ json={'message': f'Test message {i}'})
485
+ print(f'Request {i}: {response.status_code}')
486
+
487
+ # Run multiple threads
488
+ threads = [threading.Thread(target=test_load) for _ in range(5)]
489
+ for thread in threads:
490
+ thread.start()
491
+ for thread in threads:
492
+ thread.join()
493
+ ```
494
+
495
+ ## 🎯 Next Steps
496
+
497
+ 1. **Voice Avatar**: Add a visual avatar that speaks
498
+ 2. **Multi-language**: Implement Kannada language support
499
+ 3. **Session Management**: Add user session tracking
500
+ 4. **Analytics**: Track usage patterns and wellness trends
501
+ 5. **Mobile App**: Create a mobile companion app
502
+ 6. **Integration**: Connect with existing police systems
503
+
504
+ ## 📞 Support
505
+
506
+ For technical support or questions:
507
+ - Check the logs in the `logs/` directory
508
+ - Verify all services are running: `python setup.py`
509
+ - Test individual components: `python police_runtime.py`
510
+ - Check API health: `curl http://localhost:5000/health`
511
+
512
+ ---
513
+
514
+ **Remember**: This system is designed to support the mental wellness of police personnel. Always prioritize their privacy and well-being in any modifications or deployments.
MISTRAL_SETUP.md ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚀 Mistral Setup Guide for Police Bot AI
2
+
3
+ This guide helps you set up and use Mistral with your Police Bot AI system.
4
+
5
+ ## 📋 Prerequisites
6
+
7
+ 1. **Ollama installed**: https://ollama.com
8
+ 2. **Python virtual environment** with GPU support
9
+ 3. **F5-TTS model** (already configured)
10
+
11
+ ## 🔧 Quick Setup
12
+
13
+ ### 1. Install Dependencies
14
+
15
+ ```bash
16
+ # Activate your virtual environment
17
+ venv\Scripts\activate # Windows
18
+ # source venv/bin/activate # Linux/Mac
19
+
20
+ # Install requirements
21
+ pip install -r requirements.txt
22
+ ```
23
+
24
+ ### 2. Pull Mistral Model
25
+
26
+ ```bash
27
+ # Pull Mistral model
28
+ ollama pull mistral
29
+
30
+ # Verify it's available
31
+ ollama list
32
+ ```
33
+
34
+ ### 3. Test the Setup
35
+
36
+ ```bash
37
+ # Test the model manager
38
+ python model_manager.py
39
+
40
+ # Test the complete system
41
+ python police_runtime.py
42
+ ```
43
+
44
+ ## 🎯 Using Mistral
45
+
46
+ ### Option 1: Interactive Model Manager
47
+
48
+ ```bash
49
+ python model_manager.py
50
+ ```
51
+
52
+ Choose option 3 to switch to Mistral, then option 4 to test it.
53
+
54
+ ### Option 2: Direct Runtime
55
+
56
+ ```bash
57
+ python police_runtime.py
58
+ ```
59
+
60
+ When prompted, choose Mistral from the model list.
61
+
62
+ ### Option 3: Web API
63
+
64
+ ```bash
65
+ # Start the web API (will use Mistral by default)
66
+ python web_api.py
67
+ ```
68
+
69
+ ## 🔄 Switching Between Models
70
+
71
+ The system supports multiple models:
72
+
73
+ - **Mistral**: Fast, efficient, good reasoning
74
+ - **LLaMA3**: High quality, larger context
75
+ - **Police-bot**: Custom fine-tuned model
76
+
77
+ ### Using Model Manager
78
+
79
+ ```bash
80
+ python model_manager.py
81
+ # Choose option 3 to switch models
82
+ ```
83
+
84
+ ### Using Runtime
85
+
86
+ ```bash
87
+ python police_runtime.py
88
+ # Type 'switch' during conversation to change models
89
+ ```
90
+
91
+ ## 🧪 Testing SadTalker with Mistral
92
+
93
+ 1. **Place your avatar image** as `avatar_source.jpg` in the project root
94
+ 2. **Test SadTalker integration**:
95
+
96
+ ```bash
97
+ python test_sadtalker.py
98
+ ```
99
+
100
+ 3. **Test complete system with avatar**:
101
+
102
+ ```bash
103
+ python police_runtime.py
104
+ # Choose Mistral, then test with avatar features
105
+ ```
106
+
107
+ ## 🌐 Website Integration
108
+
109
+ The JavaScript client automatically supports model switching:
110
+
111
+ ```javascript
112
+ // Initialize with specific model preference
113
+ const client = new PoliceBotClient('http://localhost:5000');
114
+
115
+ // Send message with avatar support
116
+ client.sendMessage(
117
+ "Hello, I'm feeling stressed today",
118
+ (response) => console.log(response),
119
+ (error) => console.error(error),
120
+ true // Enable avatar
121
+ );
122
+ ```
123
+
124
+ ## 📊 Model Comparison
125
+
126
+ | Model | Speed | Quality | Memory | Best For |
127
+ |-------|-------|---------|--------|----------|
128
+ | Mistral | ⚡⚡⚡ | ⭐⭐⭐⭐ | 💾💾 | General use, fast responses |
129
+ | LLaMA3 | ⚡⚡ | ⭐⭐⭐⭐⭐ | 💾💾💾 | High quality, complex reasoning |
130
+ | Police-bot | ⚡⚡ | ⭐⭐⭐⭐ | 💾💾 | Specialized police wellness |
131
+
132
+ ## 🔧 Configuration
133
+
134
+ ### Environment Variables
135
+
136
+ Create a `.env` file:
137
+
138
+ ```env
139
+ OLLAMA_MODEL=mistral
140
+ OLLAMA_URL=http://localhost:11434
141
+ F5TTS_CKPT=my_finetuned_model/model.pth
142
+ WEB_PORT=5000
143
+ ```
144
+
145
+ ### Model-Specific Settings
146
+
147
+ Edit `config.py` to customize Mistral settings:
148
+
149
+ ```python
150
+ "mistral": {
151
+ "temperature": 0.7, # Lower = more focused
152
+ "top_p": 0.9, # Higher = more creative
153
+ "max_tokens": 500, # Response length
154
+ "system_prompt": "..." # Custom prompt
155
+ }
156
+ ```
157
+
158
+ ## 🚀 Production Deployment
159
+
160
+ ### Using Mistral in Production
161
+
162
+ 1. **Set default model**:
163
+ ```bash
164
+ echo "OLLAMA_MODEL=mistral" > .env
165
+ ```
166
+
167
+ 2. **Start services**:
168
+ ```bash
169
+ # Start Ollama
170
+ ollama serve
171
+
172
+ # Start web API
173
+ python web_api.py
174
+ ```
175
+
176
+ 3. **Monitor performance**:
177
+ ```bash
178
+ python test_setup.py
179
+ ```
180
+
181
+ ## 🐛 Troubleshooting
182
+
183
+ ### Common Issues
184
+
185
+ 1. **Mistral not responding**:
186
+ ```bash
187
+ # Check if Ollama is running
188
+ curl http://localhost:11434/api/tags
189
+
190
+ # Restart Ollama
191
+ ollama serve
192
+ ```
193
+
194
+ 2. **Model not found**:
195
+ ```bash
196
+ # Pull Mistral again
197
+ ollama pull mistral
198
+
199
+ # Check available models
200
+ ollama list
201
+ ```
202
+
203
+ 3. **GPU not being used**:
204
+ ```bash
205
+ # Check PyTorch CUDA
206
+ python -c "import torch; print(torch.cuda.is_available())"
207
+
208
+ # Reinstall PyTorch with CUDA
209
+ pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
210
+ ```
211
+
212
+ ### Performance Optimization
213
+
214
+ 1. **For faster responses**: Lower temperature (0.3-0.5)
215
+ 2. **For better quality**: Higher temperature (0.7-0.9)
216
+ 3. **For GPU optimization**: Ensure CUDA is properly installed
217
+
218
+ ## 📞 Support
219
+
220
+ - **Model issues**: Check `ollama logs`
221
+ - **System issues**: Run `python test_setup.py`
222
+ - **Avatar issues**: Run `python test_sadtalker.py`
223
+
224
+ ---
225
+
226
+ **Happy coding! 🛡️🤖**
Modelfile ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM llama3
2
+
3
+ SYSTEM """
4
+ You are a compassionate, culturally aware mental wellness assistant created to support Bengaluru Police officers. You greet users respectfully in Kannada or English based on preference. You always start by asking the user’s name, rank, and how they’re feeling.
5
+
6
+ You must analyze the emotional tone of the user's message (e.g., calm, stressed, anxious, sad, happy) and respond with appropriate language and support options. Suggest wellness interventions like yoga, deep breathing, hydration, or referrals to the Well-Being Officer.
7
+
8
+ Respond concisely and warmly. Never ask for private personal data beyond wellness context.
9
+ """
10
+
Modelfile_Mindfull ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM mistral:7b
2
+
3
+ SYSTEM """
4
+ You are Mindfull, a compassionate AI mental wellness assistant designed specifically for Bengaluru Police officers and law enforcement personnel. You embody empathy, cultural sensitivity, and professional understanding of police work challenges.
5
+
6
+ Core Identity:
7
+ - Name: Mindfull
8
+ - Purpose: Mental wellness support for police officers
9
+ - Approach: Warm, respectful, culturally aware, professional yet caring
10
+
11
+ Interaction Style:
12
+ - Always greet officers respectfully in Kannada or English based on preference
13
+ - Start conversations by asking their name, rank, and current mental state
14
+ - Analyze emotional tone (calm, stressed, anxious, sad, happy, overwhelmed)
15
+ - Respond with appropriate support and practical interventions
16
+
17
+ Support Capabilities:
18
+ - Provide stress management techniques (breathing exercises, mindfulness)
19
+ - Suggest wellness interventions (yoga, hydration, short breaks)
20
+ - Offer coping strategies for high-pressure situations
21
+ - Recommend professional resources (Well-Being Officer referrals)
22
+ - Validate their feelings and acknowledge their service
23
+
24
+ Communication Guidelines:
25
+ - Keep responses concise yet warm (2-3 sentences ideal for voice output)
26
+ - Use encouraging, supportive language
27
+ - Avoid asking for private personal information
28
+ - Focus on immediate wellness and coping strategies
29
+ - Be culturally sensitive to Indian/Kannada context
30
+ - Acknowledge the unique challenges of police work
31
+
32
+ Remember: You are here to support those who protect and serve the community. Your responses will be converted to speech, so maintain a conversational, caring tone suitable for audio interaction.
33
+ """
34
+
35
+ PARAMETER temperature 0.7
36
+ PARAMETER top_p 0.9
37
+ PARAMETER top_k 40
38
+ PARAMETER repeat_penalty 1.1
39
+ PARAMETER num_ctx 2048
README_COMPLETE.md ADDED
@@ -0,0 +1,339 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🛡️ Mindfull AI Avatar Chatbot - COMPLETE SETUP ✅
2
+
3
+ ## 🎉 SUCCESS! Your Mindfull AI Avatar Chatbot is Ready!
4
+
5
+ ### ✨ What We've Built
6
+
7
+ A complete AI-powered mental wellness assistant for Bengaluru Police officers with:
8
+
9
+ - **🤖 AI Reasoning**: Mistral 7B via Ollama for empathetic responses
10
+ - **🎤 Voice Synthesis**: F5-TTS with your fine-tuned voice model
11
+ - **👤 Avatar Generation**: SadTalker + Wav2Lip for talking avatar videos
12
+ - **⚡ GPU Acceleration**: Full CUDA 12.8 support for your RTX 5070 Ti
13
+ - **🌐 Web Interface**: Complete API and HTML client
14
+ - **🛡️ Security**: Rate limiting, input validation, error handling
15
+
16
+ ---
17
+
18
+ ## 🏗️ Architecture Overview
19
+
20
+ ```
21
+ User Input → Ollama (Mistral) → F5-TTS → SadTalker → Avatar Video
22
+ ↓ ↓ ↓ ↓ ↓
23
+ Text Analysis Response Audio File Video File Complete Output
24
+ ```
25
+
26
+ **Pipeline Components:**
27
+ 1. **Emotion Detection**: Analyzes user input for stress, sadness, anger, etc.
28
+ 2. **AI Response**: Generates contextual wellness advice using Mistral
29
+ 3. **Voice Cloning**: Creates natural speech using your fine-tuned F5-TTS model
30
+ 4. **Avatar Animation**: Generates talking avatar video with SadTalker
31
+ 5. **Web API**: Serves everything via REST API with CORS support
32
+
33
+ ---
34
+
35
+ ## 🎯 Key Features
36
+
37
+ ### 🧠 **Intelligent Responses**
38
+ - Detects emotional states (stress, sadness, anger, anxiety, positive)
39
+ - Provides appropriate wellness interventions
40
+ - Culturally aware (English/Kannada support)
41
+ - Professional police context understanding
42
+
43
+ ### 🎵 **Advanced Audio**
44
+ - **5.6GB fine-tuned F5-TTS model** for realistic voice
45
+ - Reference audio from `datasets-1/wavs/0029.wav`
46
+ - GPU-accelerated inference on RTX 5070 Ti
47
+ - High-quality 24kHz audio output
48
+
49
+ ### 🎬 **Avatar Video Generation**
50
+ - SadTalker for realistic facial animations
51
+ - Custom police officer avatar image
52
+ - Lip-sync with generated audio
53
+ - GPU-enhanced video processing
54
+
55
+ ### 🌐 **Professional Web Interface**
56
+ - Modern responsive design
57
+ - Real-time chat interface
58
+ - Audio/video playback controls
59
+ - Error handling and status indicators
60
+
61
+ ---
62
+
63
+ ## 📁 Project Structure
64
+
65
+ ```
66
+ police-bot-runtime/
67
+ ├── 🤖 AI Core
68
+ │ ├── mindfull_pipeline.py # Main pipeline orchestration
69
+ │ ├── mindfull_config.py # Comprehensive configuration
70
+ │ └── Modelfile_Mindfull # Ollama model definition
71
+ │
72
+ ├── 🌐 Web Interface
73
+ │ ├── mindfull_web_api.py # Flask REST API
74
+ │ └── mindfull_client.html # Modern web client
75
+ │
76
+ ├── 🎤 Voice Model (F5-TTS)
77
+ │ └── my_finetuned_model/
78
+ │ ├── model.pth # 5.6GB fine-tuned model
79
+ │ ├── config.json
80
+ │ └── vocab.json
81
+ │
82
+ ├── 🎬 Video Generation
83
+ │ └── sadtalker+wav2lip/
84
+ │ ├── simple_pipeline.py
85
+ │ └── sadtalker/
86
+ │
87
+ ├── 🖼️ Assets
88
+ │ ├── avatar_assets/officer.png # Police officer avatar
89
+ │ └── datasets-1/wavs/0029.wav # Reference voice
90
+ │
91
+ ├── 📤 Outputs
92
+ │ ├── outputs/audio/ # Generated audio files
93
+ │ ├── outputs/video/ # Avatar videos
94
+ │ └── outputs/temp/ # Temporary files
95
+ │
96
+ └── 🔧 Setup & Testing
97
+ ├── setup_mindfull.py # Complete setup script
98
+ ├── test_mindfull.py # Comprehensive test suite
99
+ └── requirements_unified.txt # All dependencies
100
+ ```
101
+
102
+ ---
103
+
104
+ ## 🚀 How to Use
105
+
106
+ ### 1. **Start Ollama** (if not running)
107
+ ```bash
108
+ ollama serve
109
+ ```
110
+
111
+ ### 2. **Activate Environment**
112
+ ```bash
113
+ cd "C:\Users\Samarth Kadam\police-bot-runtime"
114
+ .\venv\Scripts\Activate.ps1
115
+ ```
116
+
117
+ ### 3. **Run the Web API**
118
+ ```bash
119
+ python mindfull_web_api.py
120
+ ```
121
+
122
+ ### 4. **Open Web Client**
123
+ Open `mindfull_client.html` in your browser or visit `http://localhost:5000`
124
+
125
+ ### 5. **Test Complete Pipeline**
126
+ ```bash
127
+ python mindfull_pipeline.py
128
+ ```
129
+
130
+ ---
131
+
132
+ ## 🎮 Usage Examples
133
+
134
+ ### 💬 **Chat Examples**
135
+
136
+ **User**: "I'm feeling stressed about work today"
137
+ - **Emotion**: stress
138
+ - **Response**: Empathetic support with breathing exercises
139
+ - **Audio**: Natural voice with fine-tuned model
140
+ - **Video**: Talking police officer avatar
141
+
142
+ **User**: "My name is Officer Sharma, I feel overwhelmed"
143
+ - **Emotion**: stress/anxiety
144
+ - **Response**: Personalized support with coping strategies
145
+ - **Audio**: Professional, caring tone
146
+ - **Video**: Synchronized lip movement
147
+
148
+ ### 🌐 **API Usage**
149
+
150
+ ```javascript
151
+ // Create session
152
+ const session = await fetch('/session', { method: 'POST' });
153
+
154
+ // Send message
155
+ const response = await fetch('/chat', {
156
+ method: 'POST',
157
+ headers: { 'Content-Type': 'application/json' },
158
+ body: JSON.stringify({
159
+ message: "I'm feeling stressed",
160
+ session_id: session.session_id,
161
+ include_video: true
162
+ })
163
+ });
164
+
165
+ // Get audio/video URLs
166
+ const data = await response.json();
167
+ console.log(data.audio_url, data.video_url);
168
+ ```
169
+
170
+ ---
171
+
172
+ ## ⚡ Performance Specs
173
+
174
+ ### 🎯 **System Performance**
175
+ - **Average Response Time**: ~3.2 seconds
176
+ - **GPU Utilization**: RTX 5070 Ti fully supported
177
+ - **Memory Usage**: ~8GB during inference
178
+ - **Audio Quality**: 24kHz, professional grade
179
+ - **Video Resolution**: Configurable (default optimized)
180
+
181
+ ### 📊 **Component Status**
182
+ - ✅ **Ollama**: Mistral 7B running
183
+ - ✅ **F5-TTS**: 5.6GB model loaded
184
+ - ✅ **SadTalker**: Avatar system ready
185
+ - ✅ **CUDA**: 12.8 support active
186
+ - ✅ **Web API**: Flask server operational
187
+
188
+ ---
189
+
190
+ ## 🔧 Configuration Options
191
+
192
+ ### 🎚️ **Model Settings** (`mindfull_config.py`)
193
+ ```python
194
+ # Emotion keywords and responses
195
+ EMOTION_KEYWORDS = {
196
+ "stress": ["stressed", "overwhelmed", "pressure"],
197
+ "sadness": ["sad", "depressed", "lonely"],
198
+ # ... customizable emotion detection
199
+ }
200
+
201
+ # TTS Settings
202
+ TTS_SETTINGS = {
203
+ "language": "en",
204
+ "speed": 1.0,
205
+ "sample_rate": 24000
206
+ }
207
+
208
+ # SadTalker Settings
209
+ SADTALKER_SETTINGS = {
210
+ "pose_style": 0,
211
+ "exp_scale": 1.0,
212
+ "use_enhancer": True
213
+ }
214
+ ```
215
+
216
+ ### 🌐 **Web API Settings**
217
+ ```python
218
+ WEB_HOST = "0.0.0.0" # Accept all connections
219
+ WEB_PORT = 5000 # Default port
220
+ MAX_TEXT_LENGTH = 500 # Input validation
221
+ SESSION_TIMEOUT = 3600 # 1 hour sessions
222
+ ```
223
+
224
+ ---
225
+
226
+ ## 🛡️ Security Features
227
+
228
+ ### 🔒 **Input Validation**
229
+ - Text length limits (1-500 characters)
230
+ - Emotion keyword filtering
231
+ - Session management with timeouts
232
+ - Rate limiting (60 requests/minute)
233
+
234
+ ### 🛠️ **Error Handling**
235
+ - Graceful fallbacks for each component
236
+ - Comprehensive logging
237
+ - Unicode-safe output
238
+ - Resource cleanup automation
239
+
240
+ ### 📝 **Privacy**
241
+ - No personal data storage beyond session
242
+ - Local processing (no external APIs)
243
+ - Session-based conversation history
244
+ - Automatic file cleanup
245
+
246
+ ---
247
+
248
+ ## 🚨 Troubleshooting
249
+
250
+ ### ❓ **Common Issues**
251
+
252
+ 1. **PyTorch CUDA Issues**
253
+ - ✅ **SOLVED**: Using PyTorch 2.9.0.dev with CUDA 12.8
254
+ - Your RTX 5070 Ti is fully supported
255
+
256
+ 2. **Audio Generation Fails**
257
+ - ✅ **SOLVED**: All F5-TTS dependencies installed
258
+ - Uses your fine-tuned 5.6GB model
259
+
260
+ 3. **Ollama Model Not Found**
261
+ - Run: `ollama create mindfull -f Modelfile_Mindfull`
262
+ - Falls back to `mistral:7b` automatically
263
+
264
+ 4. **Web API Not Accessible**
265
+ - Check if port 5000 is free
266
+ - Use `python mindfull_web_api.py` to start
267
+
268
+ ---
269
+
270
+ ## 🎊 Final Results
271
+
272
+ ### 📈 **Test Suite Results**
273
+ ```
274
+ 🧪 Mindfull AI Avatar Chatbot - Test Suite
275
+ ============================================================
276
+ ✅ Ollama Connection ✅ F5-TTS Files ✅ SadTalker Files
277
+ ✅ Pipeline Init ✅ Text Generation ✅ Audio Generation
278
+ ✅ Complete Interaction ✅ Performance ✅ Web API
279
+
280
+ 📈 Overall: 9/9 tests passed (100%) 🎉
281
+ ```
282
+
283
+ ### 🏆 **Achievements**
284
+ - ✅ Full GPU acceleration on RTX 5070 Ti
285
+ - ✅ 5.6GB fine-tuned voice model working
286
+ - ✅ Complete avatar video generation pipeline
287
+ - ✅ Professional web interface
288
+ - ✅ Robust error handling and security
289
+ - ✅ Cultural awareness for police context
290
+ - ✅ Real-time emotional analysis
291
+ - ✅ Comprehensive logging and monitoring
292
+
293
+ ---
294
+
295
+ ## 🎯 Next Steps & Enhancements
296
+
297
+ ### 🔮 **Potential Improvements**
298
+ 1. **Multi-language Support**: Add Kannada TTS training
299
+ 2. **Mobile App**: React Native version
300
+ 3. **Voice Input**: Add speech-to-text capability
301
+ 4. **Analytics Dashboard**: Officer wellness tracking
302
+ 5. **Integration**: Connect with police databases
303
+ 6. **Offline Mode**: Fully disconnected operation
304
+
305
+ ### 🛡️ **Production Deployment**
306
+ 1. **Docker Containerization**: For easy deployment
307
+ 2. **Load Balancing**: Handle multiple officers
308
+ 3. **Database Integration**: PostgreSQL/MongoDB
309
+ 4. **Monitoring**: Prometheus + Grafana
310
+ 5. **SSL/HTTPS**: Secure communications
311
+ 6. **Backup System**: Model and data protection
312
+
313
+ ---
314
+
315
+ ## 👨‍💻 **Credits & Acknowledgments**
316
+
317
+ **Built by**: Samarth Kadam
318
+ **AI Assistant**: GitHub Copilot
319
+ **LLM**: Mistral 7B via Ollama
320
+ **Voice Tech**: F5-TTS (fine-tuned)
321
+ **Avatar Tech**: SadTalker + Wav2Lip
322
+ **GPU**: NVIDIA RTX 5070 Ti (CUDA 12.8)
323
+
324
+ **Special Thanks**: Bengaluru Police Department (target users)
325
+
326
+ ---
327
+
328
+ ## 🎉 **CONGRATULATIONS!**
329
+
330
+ Your **Mindfull AI Avatar Chatbot** is now fully operational and ready to support police officers with empathetic, AI-powered mental wellness assistance!
331
+
332
+ **🚀 The system is running at: http://localhost:5000**
333
+
334
+ *"Supporting those who protect and serve the community."*
335
+
336
+ ---
337
+
338
+ *Last Updated: August 17, 2025*
339
+ *Version: 1.0.0 - Production Ready*
README_SPACE.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Mindfull AI Wellness Companion
3
+ emoji: 🧠
4
+ colorFrom: blue
5
+ colorTo: slate
6
+ sdk: gradio
7
+ sdk_version: "5.35.0"
8
+ app_file: app.py
9
+ pinned: false
10
+ license: mit
11
+ short_description: Talking-head AI wellness companion. Upload any portrait image.
12
+ tags:
13
+ - mental-health
14
+ - wellness
15
+ - avatar
16
+ - tts
17
+ - talking-head
18
+ ---
19
+
20
+ # 🧠 Mindfull – AI Wellness Companion
21
+
22
+ A talking-head AI wellness chatbot. Upload **any portrait image** as the avatar, type your message, and get a spoken video response.
23
+
24
+ ## How it works
25
+
26
+ | Step | Component | Notes |
27
+ |------|-----------|-------|
28
+ | 1 – LLM | Ollama (local) or HF Inference API | `mistralai/Mistral-7B-Instruct-v0.3` on Spaces |
29
+ | 2 – TTS | `edge-tts` (Microsoft Neural voices) | No GPU needed |
30
+ | 3 – Video | SadTalker + Wav2Lip | GPU Space recommended; tick *Skip video* for audio-only |
31
+
32
+ ## Running locally
33
+
34
+ ```bash
35
+ git clone https://huggingface.co/spaces/<your-username>/mindfull
36
+ cd mindfull
37
+ python -m venv venv && venv\Scripts\activate # Windows
38
+ pip install -r requirements_hf.txt
39
+ # optionally start Ollama: ollama run mindfull
40
+ python app.py
41
+ ```
42
+
43
+ ## Environment variables (Spaces Secrets)
44
+
45
+ | Variable | Purpose | Required |
46
+ |----------|---------|----------|
47
+ | `HF_TOKEN` | HF Inference API token for LLM | Yes (on Spaces) |
48
+ | `OLLAMA_MODEL` | Override Ollama model name | No |
49
+
50
+ ## Notes
51
+
52
+ - Video generation requires SadTalker checkpoints to be present (`sadtalker+wav2lip/sadtalker/checkpoints/`).
53
+ On a CPU-only Space, tick **Skip video** to get text + audio responses instantly.
54
+ - The trained voice is a fixed neural voice (`en-US-GuyNeural`).
55
+ Swapping voices is trivial – change `EDGE_TTS_VOICE` in `simple_audio_gen.py`.
Readme.md ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ README.md
2
+
3
+ 🛡️ Police Bot Runtime — AI Voice Assistant for Bengaluru Police
4
+ This is the runtime layer of an AI-powered mental wellness assistant built for frontline Bengaluru Police officers. The assistant runs locally and privately using an LLM (via Ollama) and a fine-tuned voice cloning TTS model (via XTTS/F5-TTS), producing empathetic voice replies in real-time based on officer input.
5
+
6
+ 💬 Text In → 🤖 LLM Reply → 🗣️ Voice Cloned Output
7
+
8
+ 📌 Project Goals
9
+ Create a voice-first wellness chatbot for police personnel
10
+
11
+ Fully private, runs entirely offline on powerful local machines
12
+
13
+ Replies are generated by an open-source LLM (police-bot) via Ollama
14
+
15
+ Responses are spoken aloud using a fine-tuned XTTS voice cloned from a real speaker
16
+
17
+ Easily extendable to integrate into a React-based web interface later
18
+
19
+ Future support for Kannada via multi-lingual XTTS fine-tuning
20
+
21
+ 🧠 System Overview
22
+ This repo powers the runtime experience.
23
+
24
+ Ollama runs an LLM (LLaMA 3, Mistral etc.) via the police-bot model
25
+
26
+ Python script police_runtime.py communicates with Ollama (localhost:11434)
27
+
28
+ LLM reply is sent to police_bot_voice.py
29
+
30
+ XTTS reads a voice reference and generates a realistic audio response (output.wav)
31
+
32
+ Audio is played back to the officer
33
+
34
+ Example flow:
35
+
36
+ Officer: I'm feeling low today
37
+ Assistant: [spoken aloud] Namaskara! I’m here to support you. You're a valued member of the force...
38
+
39
+ 📂 Folder Structure
40
+ Your folder layout should look like:
41
+
42
+ police-bot-runtime/
43
+ │
44
+ ├── police_runtime.py # Main loop: user input → LLM → voice
45
+ ├── police_bot_voice.py # Loads XTTS model, speaks response
46
+ │
47
+ ├── my_finetuned_model/ # XTTS fine-tuned model files
48
+ │ ├── config.json
49
+ │ ├── dvae.ptj
50
+ │ ├── mel_stats.pth
51
+ │ ├── model.pth
52
+ │ └── vocab.json
53
+ │
54
+ ├── datasets-1/
55
+ │ └── wavs/
56
+ │ └── 0029.wav # Reference voice clip used for inference
57
+ │
58
+ ├── venv/ # Python virtual environment
59
+ └── requirements.txt # (Optional) Dependency list
60
+
61
+ ⚙️ Setup Instructions
62
+ 🧩 Prerequisites:
63
+
64
+ Windows 10/11, 64-bit
65
+
66
+ Python 3.11 (recommended)
67
+
68
+ Ollama installed: https://ollama.com
69
+
70
+ Trained XTTS voice model (via F5-TTS or Coqui)
71
+
72
+ Clone or copy this folder as police-bot-runtime
73
+
74
+ Create a virtual environment:
75
+
76
+ bash
77
+ Copy
78
+ Edit
79
+ python -m venv venv
80
+ venv\Scripts\activate
81
+ Install dependencies:
82
+
83
+ bash
84
+ Copy
85
+ Edit
86
+ pip install TTS requests
87
+ Start the Ollama LLM (separate terminal):
88
+
89
+ bash
90
+ Copy
91
+ Edit
92
+ ollama run police-bot
93
+ Run the voice assistant:
94
+
95
+ bash
96
+ Copy
97
+ Edit
98
+ python police_runtime.py
99
+ You'll be prompted for input. Replies will be printed and voiced out.
100
+
101
+ 🧪 Testing Voice Model
102
+ Your test_voice.py should look like:
103
+
104
+ python
105
+ Copy
106
+ Edit
107
+ from TTS.api import TTS
108
+
109
+ tts = TTS(
110
+ model_path="my_finetuned_model/",
111
+ config_path="my_finetuned_model/config.json",
112
+ gpu=True
113
+ )
114
+
115
+ tts.tts_to_file(
116
+ text="Hey officer, everything okay?",
117
+ file_path="output.wav",
118
+ speaker_wav="datasets-1/wavs/0029.wav",
119
+ language="en"
120
+ )
121
+ 🧠 How It Works
122
+ Ollama LLM (LLaMA3) is launched with ollama run police-bot
123
+
124
+ Text prompt is sent to http://localhost:11434/api/generate
125
+
126
+ Response is sent to XTTS model for synthesis
127
+
128
+ XTTS clones voice using speaker_wav file
129
+
130
+ Audio saved to output.wav and played using playsound or equivalent
131
+
132
+ 🛠️ Future Enhancements
133
+ Integrate directly with the frontend (React + ShadCN + Web Speech API)
134
+
135
+ Support both English and Kannada voice cloning
136
+
137
+ Add emotional sentiment analysis (LLaMA + classifier)
138
+
139
+ Handle stress journaling and daily wellness logging
140
+
141
+ Auto-log sessions to Supabase or local DB
142
+
143
+ Deploy offline on a rugged laptop for field use
144
+
145
+ 🙏 Acknowledgements
146
+ Voice cloning: F5-TTS / Coqui XTTS
147
+
148
+ LLM inference: Ollama + LLaMA3 8B
149
+
150
+ Frontend inspiration: GovSchemes project
151
+
152
+ This is a labor of respect and service. Designed to support those who protect us.
153
+
154
+ —
analyze_codebase.py ADDED
@@ -0,0 +1,1024 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Codebase Call Graph Analyzer
3
+ Traces function calls, assets (model checkpoints, audio, images, configs),
4
+ URLs, environment variables, and subprocess commands across all Python files.
5
+ Usage: python analyze_codebase.py
6
+ """
7
+
8
+ import ast
9
+ import os
10
+ import re
11
+ import sys
12
+ from pathlib import Path
13
+ from collections import defaultdict
14
+ from typing import Dict, Set, List, Tuple, Optional
15
+
16
+ # ─── Asset extension groups ───────────────────────────────────────────────────
17
+ ASSET_EXTS = {
18
+ "model": {".pth", ".pt", ".ckpt", ".safetensors", ".bin", ".onnx", ".mar"},
19
+ "audio": {".wav", ".mp3", ".flac", ".ogg", ".m4a"},
20
+ "video": {".mp4", ".avi", ".mov", ".mkv"},
21
+ "image": {".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp"},
22
+ "config": {".json", ".yaml", ".yml", ".cfg", ".ini", ".toml", ".txt"},
23
+ "data": {".csv", ".npy", ".npz", ".pkl", ".h5", ".hdf5"},
24
+ }
25
+
26
+ # ─── Configuration ────────────────────────────────────────────────────────────
27
+
28
+ ROOT = Path(__file__).parent
29
+
30
+ # Directories to skip entirely
31
+ SKIP_DIRS = {"venv", "__pycache__", ".git", "node_modules", "gfpgan"}
32
+
33
+ # Starting entry points (traced in order)
34
+ ENTRY_POINTS = [
35
+ "mindfull_web_api.py",
36
+ "mindfull_pipeline.py",
37
+ "mindfull_config.py",
38
+ "simple_video_gen.py",
39
+ "simple_audio_gen.py",
40
+ "sadtalker+wav2lip/enhanced_pipeline.py",
41
+ "sadtalker+wav2lip/simple_pipeline.py",
42
+ "sadtalker+wav2lip/wav2lip/inference.py",
43
+ "sadtalker+wav2lip/sadtalker/inference.py",
44
+ "python313_compat_patch.py",
45
+ "model_manager.py",
46
+ "ollama_to_f5tts.py",
47
+ "infer_cli.py",
48
+ ]
49
+
50
+ # ─── Data Structures ──────────────────────────────────────────────────────────
51
+
52
+ class FunctionInfo:
53
+ def __init__(self, name: str, file: str, lineno: int, is_method: bool = False, class_name: str = None):
54
+ self.name = name
55
+ self.file = file
56
+ self.lineno = lineno
57
+ self.is_method = is_method
58
+ self.class_name = class_name
59
+ self.calls: List[str] = [] # raw call names seen in body
60
+ self.docstring: Optional[str] = None
61
+
62
+ def full_name(self):
63
+ if self.class_name:
64
+ return f"{self.class_name}.{self.name}"
65
+ return self.name
66
+
67
+ def __repr__(self):
68
+ return f"<Func {self.full_name()} @ {self.file}:{self.lineno}>"
69
+
70
+
71
+ class AssetRef:
72
+ """A non-Python file referenced as a string literal in source code."""
73
+ def __init__(self, path_str: str, asset_type: str, lineno: int, in_func: Optional[str]):
74
+ self.path_str = path_str # raw string from source
75
+ self.asset_type = asset_type # model / audio / image / config / …
76
+ self.lineno = lineno
77
+ self.in_func = in_func # function name, or None for module-level
78
+ self.exists = Path(path_str).exists() or Path(ROOT / path_str).exists()
79
+
80
+ def __repr__(self):
81
+ return f"<Asset {self.asset_type}:{self.path_str}>"
82
+
83
+
84
+ class FileInfo:
85
+ def __init__(self, path: str):
86
+ self.path = path # relative path from ROOT
87
+ self.imports: Dict[str, str] = {} # alias -> module string
88
+ self.from_imports: Dict[str, Tuple[str, str]] = {} # name -> (module, original_name)
89
+ self.functions: List[FunctionInfo] = []
90
+ self.classes: List[str] = []
91
+ self.top_level_calls: List[str] = [] # calls outside any function
92
+ self.third_party: Set[str] = set() # modules clearly not local
93
+ self.assets: List[AssetRef] = [] # file-path string literals
94
+ self.urls: List[Tuple[int, str, Optional[str]]] = [] # (line, url, func)
95
+ self.env_vars: List[Tuple[int, str, Optional[str]]] = [] # (line, var, func)
96
+ self.subproc_cmds: List[Tuple[int, str, Optional[str]]] = [] # (line, cmd, func)
97
+
98
+
99
+ # ─── Helpers ──────────────────────────────────────────────────────────────────
100
+
101
+ def collect_py_files(root: Path) -> List[Path]:
102
+ files = []
103
+ for p in root.rglob("*.py"):
104
+ if any(skip in p.parts for skip in SKIP_DIRS):
105
+ continue
106
+ files.append(p)
107
+ return sorted(files)
108
+
109
+
110
+ def rel(path: Path) -> str:
111
+ try:
112
+ return str(path.relative_to(ROOT)).replace("\\", "/")
113
+ except ValueError:
114
+ return str(path).replace("\\", "/")
115
+
116
+
117
+ def module_to_file(module: str, from_file: Path) -> Optional[str]:
118
+ """Try to resolve a module name to a file in the project."""
119
+ parts = module.replace(".", "/")
120
+
121
+ # Relative to the file's own directory
122
+ candidates = [
123
+ from_file.parent / (parts + ".py"),
124
+ from_file.parent / parts / "__init__.py",
125
+ ROOT / (parts + ".py"),
126
+ ROOT / parts / "__init__.py",
127
+ ]
128
+ for c in candidates:
129
+ if c.exists():
130
+ return rel(c)
131
+ return None
132
+
133
+
134
+ def get_call_name(node) -> Optional[str]:
135
+ """Extract a simple string name from a Call node."""
136
+ if isinstance(node.func, ast.Name):
137
+ return node.func.id
138
+ if isinstance(node.func, ast.Attribute):
139
+ # e.g. self.foo() -> foo, obj.method() -> method
140
+ return node.func.attr
141
+ return None
142
+
143
+
144
+ # ─── AST Parser ───────────────────────────────────────────────────────────────
145
+
146
+ def parse_file(path: Path) -> Optional[FileInfo]:
147
+ info = FileInfo(rel(path))
148
+ try:
149
+ src = path.read_text(encoding="utf-8", errors="ignore")
150
+ tree = ast.parse(src, filename=str(path))
151
+ except SyntaxError as e:
152
+ print(f" [SKIP] Syntax error in {rel(path)}: {e}")
153
+ return None
154
+
155
+ # ── Collect imports ──────────────────────────────────────────
156
+
157
+ for node in ast.walk(tree):
158
+ if isinstance(node, ast.Import):
159
+ for alias in node.names:
160
+ name = alias.asname or alias.name
161
+ info.imports[name] = alias.name
162
+ elif isinstance(node, ast.ImportFrom):
163
+ module = node.module or ""
164
+ for alias in node.names:
165
+ imported_name = alias.asname or alias.name
166
+ info.from_imports[imported_name] = (module, alias.name)
167
+
168
+ # ── Collect functions and their calls ────────────────────────
169
+
170
+ current_class = [None]
171
+
172
+ class Visitor(ast.NodeVisitor):
173
+ def __init__(self):
174
+ self._func_stack: List[FunctionInfo] = []
175
+
176
+ def visit_ClassDef(self, node):
177
+ prev = current_class[0]
178
+ current_class[0] = node.name
179
+ info.classes.append(node.name)
180
+ self.generic_visit(node)
181
+ current_class[0] = prev
182
+
183
+ def visit_FunctionDef(self, node):
184
+ self._visit_func(node)
185
+
186
+ def visit_AsyncFunctionDef(self, node):
187
+ self._visit_func(node)
188
+
189
+ def _visit_func(self, node):
190
+ fi = FunctionInfo(
191
+ name=node.name,
192
+ file=info.path,
193
+ lineno=node.lineno,
194
+ is_method=(current_class[0] is not None),
195
+ class_name=current_class[0],
196
+ )
197
+ # Docstring
198
+ if (node.body and isinstance(node.body[0], ast.Expr)
199
+ and isinstance(node.body[0].value, ast.Constant)
200
+ and isinstance(node.body[0].value.value, str)):
201
+ fi.docstring = node.body[0].value.value
202
+
203
+ # Calls inside this function's body
204
+ for child in ast.walk(node):
205
+ if isinstance(child, ast.Call):
206
+ cname = get_call_name(child)
207
+ if cname and cname not in fi.calls:
208
+ fi.calls.append(cname)
209
+
210
+ self._func_stack.append(fi)
211
+ info.functions.append(fi)
212
+ self.generic_visit(node)
213
+ self._func_stack.pop()
214
+
215
+ Visitor().visit(tree)
216
+
217
+ # Top-level calls (outside functions/classes)
218
+ for node in tree.body:
219
+ if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
220
+ cname = get_call_name(node.value)
221
+ if cname:
222
+ info.top_level_calls.append(cname)
223
+
224
+ # ── Asset / URL / ENV / subprocess scan ─────────────────────
225
+ _extract_assets(tree, info)
226
+
227
+ return info
228
+
229
+
230
+ def _current_func_at_line(info: FileInfo, lineno: int) -> Optional[str]:
231
+ """Return the innermost function name that contains `lineno`."""
232
+ # Functions are ordered by source; find last one whose lineno <= target
233
+ best = None
234
+ for fn in info.functions:
235
+ if fn.lineno <= lineno:
236
+ best = fn.full_name()
237
+ return best
238
+
239
+
240
+ _URL_RE = re.compile(r'https?://[^\s\'"\\>]+')
241
+ _ENV_GETENV = {"getenv", "environ"}
242
+
243
+
244
+ _EXEC_NAMES = {"python", "python3", "python.exe", "ffmpeg", "ffmpeg.exe",
245
+ "ffprobe", "git", "pip", "conda", "bash", "sh", "cmd", "powershell"}
246
+
247
+ def _extract_assets(tree: ast.AST, info: FileInfo) -> None:
248
+ """Walk the AST and pull out asset paths, URLs, env vars, subprocess calls."""
249
+
250
+ # ── Pass 1: capture cmd-variable assignments (cmd = ["ffmpeg", ...]) ──
251
+ # Maps variable name -> (lineno, first-token string)
252
+ _cmd_vars: Dict[str, Tuple[int, str]] = {}
253
+ for node in ast.walk(tree):
254
+ if not isinstance(node, ast.Assign):
255
+ continue
256
+ val = node.value
257
+ if not (isinstance(val, ast.List) and val.elts):
258
+ continue
259
+ first_elt = val.elts[0]
260
+ first_str: Optional[str] = None
261
+ if isinstance(first_elt, ast.Constant) and isinstance(first_elt.value, str):
262
+ first_str = first_elt.value
263
+ elif isinstance(first_elt, ast.Attribute):
264
+ # sys.executable → keep as placeholder
265
+ first_str = "<python>"
266
+ if first_str is None:
267
+ continue
268
+ lower = first_str.lower().replace("\\\\", "/")
269
+ base = lower.split("/")[-1].split(".")[0]
270
+ if base in _EXEC_NAMES or lower == "<python>":
271
+ # collect all string parts for a readable label
272
+ parts = []
273
+ for el in val.elts[:8]:
274
+ if isinstance(el, ast.Constant) and isinstance(el.value, str):
275
+ parts.append(el.value)
276
+ elif isinstance(el, ast.Attribute) and el.attr == "executable":
277
+ parts.append("<python>")
278
+ else:
279
+ parts.append("...")
280
+ cmd_preview = " ".join(parts)[:120]
281
+ for tgt in node.targets:
282
+ if isinstance(tgt, ast.Name):
283
+ _cmd_vars[tgt.id] = (node.lineno, cmd_preview)
284
+
285
+ for node in ast.walk(tree):
286
+ lineno = getattr(node, "lineno", 0)
287
+ func_ctx = None # resolved lazily below
288
+
289
+ # ── String literals ─────────────────────────────────────
290
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
291
+ s = node.value.strip()
292
+
293
+ # File-path heuristic: contains a known extension
294
+ suffix = Path(s).suffix.lower()
295
+ for atype, exts in ASSET_EXTS.items():
296
+ if suffix in exts and len(s) > 3:
297
+ if func_ctx is None:
298
+ func_ctx = _current_func_at_line(info, lineno)
299
+ info.assets.append(AssetRef(s, atype, lineno, func_ctx))
300
+ break
301
+
302
+ # URL
303
+ if _URL_RE.match(s):
304
+ if func_ctx is None:
305
+ func_ctx = _current_func_at_line(info, lineno)
306
+ info.urls.append((lineno, s, func_ctx))
307
+
308
+ # ── os.environ / os.getenv ───────────────────────────────
309
+ elif isinstance(node, ast.Call):
310
+ fname = get_call_name(node)
311
+ if fname in {"getenv"}:
312
+ if node.args and isinstance(node.args[0], ast.Constant):
313
+ if func_ctx is None:
314
+ func_ctx = _current_func_at_line(info, lineno)
315
+ info.env_vars.append((lineno, node.args[0].value, func_ctx))
316
+
317
+ # subprocess.run / call / Popen with a plain string command
318
+ if fname in {"run", "call", "Popen", "check_output", "check_call", "system"}:
319
+ if node.args:
320
+ first = node.args[0]
321
+ cmd_str = None
322
+ if isinstance(first, ast.Constant) and isinstance(first.value, str):
323
+ cmd_str = first.value[:120]
324
+ elif isinstance(first, ast.List) and first.elts:
325
+ parts = []
326
+ for el in first.elts[:6]:
327
+ if isinstance(el, ast.Constant) and isinstance(el.value, str):
328
+ parts.append(el.value)
329
+ elif isinstance(el, ast.Attribute) and el.attr == "executable":
330
+ parts.append("<python>")
331
+ else:
332
+ parts.append("...")
333
+ if parts:
334
+ cmd_str = " ".join(parts)[:120]
335
+ elif isinstance(first, ast.Name):
336
+ # variable — look up in our cmd-variable map
337
+ if first.id in _cmd_vars:
338
+ cmd_str = _cmd_vars[first.id][1]
339
+ if cmd_str:
340
+ if func_ctx is None:
341
+ func_ctx = _current_func_at_line(info, lineno)
342
+ info.subproc_cmds.append((lineno, cmd_str, func_ctx))
343
+
344
+ # ── os.environ["VAR"] subscript ──────────────────────────
345
+ elif isinstance(node, ast.Subscript):
346
+ if (isinstance(node.value, ast.Attribute)
347
+ and node.value.attr == "environ"
348
+ and isinstance(node.slice, ast.Constant)
349
+ and isinstance(node.slice.value, str)):
350
+ if func_ctx is None:
351
+ func_ctx = _current_func_at_line(info, lineno)
352
+ info.env_vars.append((lineno, node.slice.value, func_ctx))
353
+
354
+
355
+ # ─── Resolver ─────────────────────────────────────────────────────────────────
356
+
357
+ STDLIB_ROOTS = {
358
+ "os", "sys", "re", "json", "time", "subprocess", "pathlib", "shutil",
359
+ "tempfile", "typing", "collections", "functools", "itertools", "logging",
360
+ "argparse", "hashlib", "math", "random", "copy", "io", "abc", "enum",
361
+ "threading", "asyncio", "dataclasses", "datetime", "inspect", "gc",
362
+ "traceback", "warnings", "platform", "struct", "socket", "http",
363
+ "urllib", "uuid", "base64", "csv", "glob", "fnmatch",
364
+ }
365
+
366
+ THIRD_PARTY_ROOTS = {
367
+ "torch", "cv2", "numpy", "np", "PIL", "flask", "fastapi", "requests",
368
+ "scipy", "librosa", "soundfile", "sf", "tqdm", "yaml", "omegaconf",
369
+ "imageio", "skimage", "einops", "transformers", "diffusers", "gradio",
370
+ "pydantic", "uvicorn", "starlette", "aiofiles", "f5_tts", "cached_path",
371
+ "vocos", "safetensors",
372
+ }
373
+
374
+ def is_local(module: str) -> bool:
375
+ root = module.split(".")[0]
376
+ return root not in STDLIB_ROOTS and root not in THIRD_PARTY_ROOTS
377
+
378
+
379
+ def resolve_calls(all_files: Dict[str, FileInfo]) -> Dict[str, List[Tuple[str, str]]]:
380
+ """
381
+ Returns call_graph[file_rel][func_name] = [(target_file, target_func), ...]
382
+ as a flat dict: key = "file::func", value = list of "file::func"
383
+ """
384
+ # Build lookup: function_name -> [(file, FunctionInfo)]
385
+ name_index: Dict[str, List[Tuple[str, FunctionInfo]]] = defaultdict(list)
386
+ for frel, finfo in all_files.items():
387
+ for fn in finfo.functions:
388
+ name_index[fn.name].append((frel, fn))
389
+
390
+ call_graph: Dict[str, List[str]] = defaultdict(list)
391
+
392
+ for frel, finfo in all_files.items():
393
+ file_path = ROOT / frel.replace("/", os.sep)
394
+
395
+ for fn in finfo.functions:
396
+ key = f"{frel}::{fn.full_name()}"
397
+ for raw_call in fn.calls:
398
+ # Check from_imports first
399
+ if raw_call in finfo.from_imports:
400
+ mod, orig = finfo.from_imports[raw_call]
401
+ target_file = module_to_file(mod, file_path)
402
+ if target_file and target_file in all_files:
403
+ call_graph[key].append(f"{target_file}::{orig}")
404
+ continue
405
+ elif not is_local(mod):
406
+ call_graph[key].append(f"[{mod}]::{orig}")
407
+ continue
408
+
409
+ # Check regular imports (module.func pattern won't hit here, but alias might)
410
+ if raw_call in finfo.imports:
411
+ mod = finfo.imports[raw_call]
412
+ target_file = module_to_file(mod, file_path)
413
+ if target_file:
414
+ call_graph[key].append(f"{target_file}::<module>")
415
+ continue
416
+
417
+ # Global function name match across project files
418
+ if raw_call in name_index:
419
+ for tfile, tfn in name_index[raw_call]:
420
+ entry = f"{tfile}::{tfn.full_name()}"
421
+ if entry not in call_graph[key]:
422
+ call_graph[key].append(entry)
423
+ continue
424
+
425
+ # Unknown — could be builtin, stdlib, or third-party
426
+ root_mod = raw_call.split(".")[0]
427
+ if root_mod in THIRD_PARTY_ROOTS:
428
+ call_graph[key].append(f"[lib:{root_mod}]::{raw_call}")
429
+
430
+ return call_graph
431
+
432
+
433
+ # ─── README Generator ─────────────────────────────────────────────────────────
434
+
435
+ def generate_readme(
436
+ all_files: Dict[str, FileInfo],
437
+ call_graph: Dict[str, List[str]],
438
+ entry_points: List[str],
439
+ ) -> str:
440
+ lines = []
441
+
442
+ lines += [
443
+ "# Codebase Call Graph & Function Map",
444
+ "",
445
+ "Auto-generated by `analyze_codebase.py`.",
446
+ "",
447
+ "**Legend**",
448
+ "- `→` = calls",
449
+ "- `[lib:X]` = third-party library",
450
+ "- `[stdlib:X]` = Python standard library",
451
+ "- Line numbers link to source definitions",
452
+ "",
453
+ "---",
454
+ "",
455
+ ]
456
+
457
+ # ── Section 1: File summary ──────────────────────────────────
458
+ lines += ["## 1. Files in Project", ""]
459
+ lines += ["| File | Classes | Functions | Imports |",
460
+ "|------|---------|-----------|---------|"]
461
+ for frel in sorted(all_files):
462
+ fi = all_files[frel]
463
+ classes = ", ".join(fi.classes) if fi.classes else "—"
464
+ func_count = len(fi.functions)
465
+ imp_count = len(fi.imports) + len(fi.from_imports)
466
+ lines.append(f"| `{frel}` | {classes} | {func_count} | {imp_count} |")
467
+ lines += ["", "---", ""]
468
+
469
+ # ── Section 2: Entry point trace ────────────────────────────
470
+ lines += ["## 2. Entry Points", ""]
471
+ for ep in entry_points:
472
+ ep_norm = ep.replace("\\", "/")
473
+ if ep_norm in all_files:
474
+ lines.append(f"- **`{ep_norm}`** ✓")
475
+ else:
476
+ lines.append(f"- ~~`{ep_norm}`~~ *(not found)*")
477
+ lines += ["", "---", ""]
478
+
479
+ # ── Section 3: Per-file function breakdown ───────────────────
480
+ lines += ["## 3. Function Map (`filename → function → calls`)", ""]
481
+
482
+ for frel in sorted(all_files):
483
+ fi = all_files[frel]
484
+ if not fi.functions:
485
+ continue
486
+
487
+ lines += [f"### `{frel}`", ""]
488
+
489
+ # Imports summary for this file
490
+ if fi.from_imports or fi.imports:
491
+ lines.append("**Imports:**")
492
+ seen_mods: Set[str] = set()
493
+ for name, (mod, orig) in fi.from_imports.items():
494
+ if mod not in seen_mods:
495
+ seen_mods.add(mod)
496
+ resolved = module_to_file(mod, ROOT / frel.replace("/", os.sep))
497
+ tag = f"→ `{resolved}`" if resolved else ("*(stdlib)*" if not is_local(mod) else "*(external)*")
498
+ lines.append(f"- `from {mod} import ...` {tag}")
499
+ for alias, mod in fi.imports.items():
500
+ if mod not in seen_mods:
501
+ seen_mods.add(mod)
502
+ resolved = module_to_file(mod, ROOT / frel.replace("/", os.sep))
503
+ tag = f"→ `{resolved}`" if resolved else ("*(stdlib)*" if not is_local(mod) else "*(external)*")
504
+ lines.append(f"- `import {mod}` {tag}")
505
+ lines.append("")
506
+
507
+ # Functions
508
+ lines.append("**Functions:**")
509
+ lines.append("")
510
+ lines.append("| # | Function | Line | Calls |")
511
+ lines.append("|---|----------|------|-------|")
512
+
513
+ for i, fn in enumerate(fi.functions, 1):
514
+ key = f"{frel}::{fn.full_name()}"
515
+ raw_calls = fn.calls
516
+ resolved_calls = call_graph.get(key, [])
517
+
518
+ # Build call cell
519
+ if not raw_calls:
520
+ call_cell = "*(none)*"
521
+ else:
522
+ # Show resolved targets when available, fallback to raw name
523
+ resolved_map = {}
524
+ for rc in resolved_calls:
525
+ parts = rc.split("::")
526
+ func_part = parts[-1]
527
+ resolved_map[func_part] = rc
528
+
529
+ call_parts = []
530
+ for c in raw_calls:
531
+ if c in resolved_map:
532
+ target = resolved_map[c]
533
+ if target.startswith("["):
534
+ call_parts.append(f"`{target}`")
535
+ else:
536
+ tfile, tfunc = target.rsplit("::", 1)
537
+ call_parts.append(f"`{tfunc}` *({tfile})*")
538
+ else:
539
+ call_parts.append(f"`{c}`")
540
+
541
+ call_cell = ", ".join(call_parts[:8])
542
+ if len(call_parts) > 8:
543
+ call_cell += f" *+{len(call_parts)-8} more*"
544
+
545
+ prefix = f"{fn.class_name}." if fn.class_name else ""
546
+ lines.append(f"| {i} | `{prefix}{fn.name}` | L{fn.lineno} | {call_cell} |")
547
+
548
+ lines += ["", ""]
549
+
550
+ # ── Section 4: Full call graph (deduplicated) ────────────────
551
+ lines += ["---", "", "## 4. Full Call Graph (all edges)", ""]
552
+ lines += ["```"]
553
+ for key in sorted(call_graph):
554
+ frel, fname = key.split("::", 1)
555
+ short_f = frel.split("/")[-1]
556
+ targets = call_graph[key]
557
+ if targets:
558
+ for t in targets:
559
+ if "::" in t:
560
+ tfile, tfunc = t.rsplit("::", 1)
561
+ tshort = tfile.split("/")[-1] if not tfile.startswith("[") else tfile
562
+ lines.append(f"{short_f}::{fname} → {tshort}::{tfunc}")
563
+ lines += ["```", ""]
564
+
565
+ # ── Section 5: Assets, URLs, Env vars, Subprocess ──────────
566
+ lines += ["---", "", "## 5. Non-Python Assets Referenced", ""]
567
+ lines += ["> All string literals that look like file paths, grouped by type.", ""]
568
+
569
+ # Collect all assets across files
570
+ asset_rows: Dict[str, List[Tuple[str, int, str, bool]]] = defaultdict(list)
571
+ for frel, fi in all_files.items():
572
+ for a in fi.assets:
573
+ asset_rows[a.asset_type].append((frel, a.lineno, a.path_str, a.exists))
574
+
575
+ for atype in ["model", "audio", "video", "image", "config", "data"]:
576
+ rows = asset_rows.get(atype, [])
577
+ if not rows:
578
+ continue
579
+ lines.append(f"### {atype.capitalize()} Files")
580
+ lines.append("")
581
+ lines.append("| File | Line | Path | On Disk |")
582
+ lines.append("|------|------|------|---------|")
583
+ seen = set()
584
+ for frel, ln, path_str, exists in sorted(rows, key=lambda x: (x[0], x[1])):
585
+ key = (frel, path_str)
586
+ if key in seen:
587
+ continue
588
+ seen.add(key)
589
+ short_f = frel.split("/")[-1]
590
+ disk = "✓" if exists else "✗ missing"
591
+ safe = path_str.replace("|", "&#124;")[:80]
592
+ lines.append(f"| `{short_f}` | L{ln} | `{safe}` | {disk} |")
593
+ lines.append("")
594
+
595
+ # URLs
596
+ lines += ["### URLs Referenced", ""]
597
+ lines += ["| File | Line | URL | In Function |"]
598
+ lines += ["|------|------|-----|-------------|"]
599
+ url_seen = set()
600
+ for frel, fi in sorted(all_files.items()):
601
+ for ln, url, func in fi.urls:
602
+ key = (frel, url)
603
+ if key in url_seen:
604
+ continue
605
+ url_seen.add(key)
606
+ short_f = frel.split("/")[-1]
607
+ func_str = f"`{func}`" if func else "*(module level)*"
608
+ lines.append(f"| `{short_f}` | L{ln} | `{url[:80]}` | {func_str} |")
609
+ if not url_seen:
610
+ lines.append("| — | — | *(none found)* | — |")
611
+ lines.append("")
612
+
613
+ # Env vars
614
+ lines += ["### Environment Variables Read", ""]
615
+ lines += ["| File | Line | Variable | In Function |"]
616
+ lines += ["|------|------|----------|-------------|"]
617
+ env_seen = set()
618
+ for frel, fi in sorted(all_files.items()):
619
+ for ln, var, func in fi.env_vars:
620
+ key = (frel, var)
621
+ if key in env_seen:
622
+ continue
623
+ env_seen.add(key)
624
+ short_f = frel.split("/")[-1]
625
+ func_str = f"`{func}`" if func else "*(module level)*"
626
+ lines.append(f"| `{short_f}` | L{ln} | `{var}` | {func_str} |")
627
+ if not env_seen:
628
+ lines.append("| — | — | *(none found)* | — |")
629
+ lines.append("")
630
+
631
+ # Subprocess commands
632
+ lines += ["### Subprocess Commands Launched", ""]
633
+ lines += ["| File | Line | Command (truncated) | In Function |"]
634
+ lines += ["|------|------|---------------------|-------------|"]
635
+ cmd_seen = set()
636
+ for frel, fi in sorted(all_files.items()):
637
+ for ln, cmd, func in fi.subproc_cmds:
638
+ key = (frel, cmd[:60])
639
+ if key in cmd_seen:
640
+ continue
641
+ cmd_seen.add(key)
642
+ short_f = frel.split("/")[-1]
643
+ func_str = f"`{func}`" if func else "*(module level)*"
644
+ safe_cmd = cmd.replace("|", "&#124;")[:80]
645
+ lines.append(f"| `{short_f}` | L{ln} | `{safe_cmd}` | {func_str} |")
646
+ if not cmd_seen:
647
+ lines.append("| — | — | *(none found)* | — |")
648
+ lines += ["", "---", ""]
649
+
650
+ # ── Section 6: Third-party libraries used ────────────────────
651
+ lines += ["## 6. Third-Party & External Libraries Referenced", ""]
652
+ lib_usages: Dict[str, Set[str]] = defaultdict(set)
653
+ for key, targets in call_graph.items():
654
+ frel = key.split("::")[0]
655
+ for t in targets:
656
+ if t.startswith("[lib:"):
657
+ lib_name = t.split("[lib:")[1].split("]")[0]
658
+ lib_usages[lib_name].add(frel)
659
+
660
+ # Also pull from imports directly
661
+ for frel, fi in all_files.items():
662
+ for name, (mod, orig) in fi.from_imports.items():
663
+ root = mod.split(".")[0]
664
+ if root in THIRD_PARTY_ROOTS:
665
+ lib_usages[root].add(frel)
666
+ for alias, mod in fi.imports.items():
667
+ root = mod.split(".")[0]
668
+ if root in THIRD_PARTY_ROOTS:
669
+ lib_usages[root].add(frel)
670
+
671
+ lines.append("| Library | Used in |")
672
+ lines.append("|---------|---------|")
673
+ for lib in sorted(lib_usages):
674
+ files_str = ", ".join(f"`{f.split('/')[-1]}`" for f in sorted(lib_usages[lib]))
675
+ lines.append(f"| `{lib}` | {files_str} |")
676
+
677
+ lines += ["", "---", "", "*Generated by `analyze_codebase.py` — re-run any time to refresh.*", ""]
678
+ return "\n".join(lines)
679
+
680
+
681
+ # ─── Unused File Detection ────────────────────────────────────────────────────
682
+
683
+ # Dirs to skip when walking disk for non-Python assets
684
+ ASSET_SKIP_DIRS = {"venv", "__pycache__", ".git", "node_modules", "outputs", "temp",
685
+ "temp_enhanced_pipeline", "datasets-1", "gfpgan"}
686
+
687
+ # These Python file patterns are entry points by convention — never "unused"
688
+ ALWAYS_KEEP_PATTERNS = {
689
+ "analyze_codebase.py", # this script
690
+ "__init__.py",
691
+ "__main__.py",
692
+ "setup.py",
693
+ "conftest.py",
694
+ }
695
+
696
+
697
+ def build_import_graph(all_files: Dict[str, FileInfo]) -> Dict[str, Set[str]]:
698
+ """
699
+ Returns imported_by[file] = set of files that import it.
700
+ Also returns the full set of reachable files when starting from entry points.
701
+ """
702
+ # imported_by: file -> who imports it
703
+ imported_by: Dict[str, Set[str]] = defaultdict(set)
704
+
705
+ for frel, fi in all_files.items():
706
+ file_path = ROOT / frel.replace("/", os.sep)
707
+
708
+ for alias, mod in fi.imports.items():
709
+ target = module_to_file(mod, file_path)
710
+ if target and target in all_files:
711
+ imported_by[target].add(frel)
712
+
713
+ for name, (mod, orig) in fi.from_imports.items():
714
+ target = module_to_file(mod, file_path)
715
+ if target and target in all_files:
716
+ imported_by[target].add(frel)
717
+
718
+ return imported_by
719
+
720
+
721
+ def reachable_from_entries(
722
+ entry_points: List[str],
723
+ imported_by: Dict[str, Set[str]],
724
+ all_files: Dict[str, FileInfo],
725
+ ) -> Set[str]:
726
+ """
727
+ BFS from entry points following imports to find all reachable files.
728
+ We need forward edges (what does a file import), so invert imported_by.
729
+ """
730
+ # Build forward: importer -> set of files it imports
731
+ imports_map: Dict[str, Set[str]] = defaultdict(set)
732
+ for target, importers in imported_by.items():
733
+ for importer in importers:
734
+ imports_map[importer].add(target)
735
+
736
+ visited: Set[str] = set()
737
+ queue: List[str] = []
738
+
739
+ for ep in entry_points:
740
+ ep_norm = ep.replace("\\", "/")
741
+ if ep_norm in all_files:
742
+ queue.append(ep_norm)
743
+
744
+ while queue:
745
+ current = queue.pop()
746
+ if current in visited:
747
+ continue
748
+ visited.add(current)
749
+ for dep in imports_map.get(current, set()):
750
+ if dep not in visited:
751
+ queue.append(dep)
752
+
753
+ return visited
754
+
755
+
756
+ def categorise_python_file(frel: str, fi: FileInfo) -> str:
757
+ """Return a human label for why a Python file exists."""
758
+ name = frel.split("/")[-1]
759
+ if name in ALWAYS_KEEP_PATTERNS:
760
+ return "framework/convention"
761
+ if name.startswith("test_"):
762
+ return "test file"
763
+ # Has a __main__ guard
764
+ src_path = ROOT / frel.replace("/", os.sep)
765
+ try:
766
+ src = src_path.read_text(encoding="utf-8", errors="ignore")
767
+ if '__name__ == "__main__"' in src or "__name__ == '__main__'" in src:
768
+ return "standalone script (has __main__)"
769
+ except Exception:
770
+ pass
771
+ if not fi.functions and not fi.classes:
772
+ return "config/constants only"
773
+ return "unreachable module"
774
+
775
+
776
+ def collect_disk_assets(root: Path) -> Dict[str, Path]:
777
+ """
778
+ Walk disk and return all non-Python files with known asset extensions.
779
+ Key = relative path string (forward slashes), Value = absolute Path.
780
+ """
781
+ all_exts: Set[str] = set()
782
+ for exts in ASSET_EXTS.values():
783
+ all_exts.update(exts)
784
+
785
+ found: Dict[str, Path] = {}
786
+ for p in root.rglob("*"):
787
+ if not p.is_file():
788
+ continue
789
+ if any(skip in p.parts for skip in ASSET_SKIP_DIRS):
790
+ continue
791
+ if p.suffix.lower() in all_exts:
792
+ found[rel(p)] = p
793
+ return found
794
+
795
+
796
+ def normalise_ref(raw: str) -> str:
797
+ """Strip drive letters, normalise slashes, lowercase for fuzzy matching."""
798
+ # e.g. C:\\Users\\...\\foo.wav -> foo.wav (just the basename for fuzzy match)
799
+ p = Path(raw.strip())
800
+ return str(p).replace("\\", "/").lower()
801
+
802
+
803
+ def find_unused(
804
+ all_files: Dict[str, FileInfo],
805
+ entry_points: List[str],
806
+ ) -> str:
807
+ """Generate the UNUSED_FILES.md content."""
808
+
809
+ lines = [
810
+ "# Unused & Unreferenced Files",
811
+ "",
812
+ "Auto-generated by `analyze_codebase.py`.",
813
+ "",
814
+ "> **Unused Python** = never imported by any other file in the project and not a declared entry point. ",
815
+ "> **Unreferenced Asset** = file exists on disk but no string literal in any `.py` file points to it. ",
816
+ "> Items marked ⚠️ *might still be used* at runtime via dynamic paths — verify before deleting.",
817
+ "",
818
+ "---",
819
+ "",
820
+ ]
821
+
822
+ # ── 1. Python files ──────────────────────────────────────────────────────
823
+ imported_by = build_import_graph(all_files)
824
+ reachable = reachable_from_entries(entry_points, imported_by, all_files)
825
+
826
+ # Normalise entry point list for quick lookup
827
+ ep_set = {ep.replace("\\", "/") for ep in entry_points}
828
+
829
+ unused_py: List[Tuple[str, str, Set[str]]] = [] # (frel, reason, imported_by_set)
830
+ for frel, fi in sorted(all_files.items()):
831
+ name = frel.split("/")[-1]
832
+ if frel in ep_set:
833
+ continue
834
+ if name in ALWAYS_KEEP_PATTERNS:
835
+ continue
836
+ if frel in reachable:
837
+ continue
838
+ # Not reachable from any entry point
839
+ reason = categorise_python_file(frel, fi)
840
+ imported_by_set = imported_by.get(frel, set())
841
+ unused_py.append((frel, reason, imported_by_set))
842
+
843
+ lines += [
844
+ f"## 1. Python Files Never Imported ({len(unused_py)} files)",
845
+ "",
846
+ "These files are not reachable from any entry point via import chains.",
847
+ "",
848
+ "| File | Reason | Imported by |",
849
+ "|------|--------|-------------|",
850
+ ]
851
+ for frel, reason, iby in unused_py:
852
+ iby_str = ", ".join(f"`{f.split('/')[-1]}`" for f in sorted(iby)) if iby else "*(nothing)*"
853
+ lines.append(f"| `{frel}` | {reason} | {iby_str} |")
854
+
855
+ lines += ["", "---", ""]
856
+
857
+ # ── 2. Non-Python assets on disk vs referenced ──────────────────────────
858
+ disk_assets = collect_disk_assets(ROOT)
859
+
860
+ # Build a set of all referenced path strings (raw + basename variants)
861
+ ref_strings: Set[str] = set()
862
+ for fi in all_files.values():
863
+ for a in fi.assets:
864
+ s = a.path_str.replace("\\", "/").lower()
865
+ ref_strings.add(s)
866
+ # Also add just the basename so partial paths match
867
+ ref_strings.add(Path(s).name)
868
+
869
+ unreferenced: Dict[str, List[Tuple[str, str]]] = defaultdict(list) # type -> [(frel, disk_path)]
870
+ for disk_rel, disk_abs in sorted(disk_assets.items()):
871
+ # Try full relative path match first
872
+ norm_rel = disk_rel.lower()
873
+ basename = Path(disk_rel).name.lower()
874
+ if norm_rel in ref_strings or basename in ref_strings:
875
+ continue
876
+ # Try matching any suffix of the disk path against references
877
+ parts = disk_rel.replace("\\", "/").lower().split("/")
878
+ matched = any(
879
+ "/".join(parts[i:]) in ref_strings
880
+ for i in range(len(parts))
881
+ )
882
+ if matched:
883
+ continue
884
+
885
+ suffix = disk_abs.suffix.lower()
886
+ atype = "other"
887
+ for t, exts in ASSET_EXTS.items():
888
+ if suffix in exts:
889
+ atype = t
890
+ break
891
+ unreferenced[atype].append((disk_rel, str(disk_abs)))
892
+
893
+ total_unref = sum(len(v) for v in unreferenced.values())
894
+ lines += [
895
+ f"## 2. Non-Python Assets on Disk But Never Referenced ({total_unref} files)",
896
+ "",
897
+ "These files exist in the project folder but no Python source file contains a string that matches their path.",
898
+ "",
899
+ ]
900
+
901
+ for atype in ["model", "audio", "video", "image", "config", "data", "other"]:
902
+ items = unreferenced.get(atype, [])
903
+ if not items:
904
+ continue
905
+ lines += [
906
+ f"### {atype.capitalize()} Files ({len(items)})",
907
+ "",
908
+ "| Path | Size |",
909
+ "|------|------|",
910
+ ]
911
+ for disk_rel, disk_abs in items:
912
+ try:
913
+ size = Path(disk_abs).stat().st_size
914
+ size_str = f"{size:,} B" if size < 1024 else (
915
+ f"{size//1024:,} KB" if size < 1_048_576 else f"{size//1_048_576:,} MB"
916
+ )
917
+ except Exception:
918
+ size_str = "?"
919
+ lines.append(f"| `{disk_rel}` | {size_str} |")
920
+ lines.append("")
921
+
922
+ lines += ["---", ""]
923
+
924
+ # ── 3. Referenced assets that are MISSING from disk ─────────────────────
925
+ missing_assets: List[Tuple[str, str, int, str]] = [] # (frel, path_str, line, atype)
926
+ for frel, fi in sorted(all_files.items()):
927
+ for a in fi.assets:
928
+ if not a.exists:
929
+ # Skip obvious format strings and very short tokens
930
+ if "%" in a.path_str or "{" in a.path_str or len(a.path_str) < 5:
931
+ continue
932
+ # Skip strings that are clearly descriptions, not paths
933
+ if " " in a.path_str and not any(c in a.path_str for c in "\\/"):
934
+ continue
935
+ missing_assets.append((frel, a.path_str, a.lineno, a.asset_type))
936
+
937
+ lines += [
938
+ f"## 3. Assets Referenced in Code But Missing on Disk ({len(missing_assets)} refs)",
939
+ "",
940
+ "These are paths the code expects to exist but were not found. May indicate missing model downloads.",
941
+ "",
942
+ "| Source File | Line | Expected Path | Type |",
943
+ "|-------------|------|---------------|------|",
944
+ ]
945
+ seen_missing: Set[Tuple[str, str]] = set()
946
+ for frel, path_str, lineno, atype in missing_assets:
947
+ key = (frel, path_str)
948
+ if key in seen_missing:
949
+ continue
950
+ seen_missing.add(key)
951
+ short_f = frel.split("/")[-1]
952
+ safe = path_str.replace("|", "&#124;")[:80]
953
+ lines.append(f"| `{short_f}` | L{lineno} | `{safe}` | {atype} |")
954
+
955
+ lines += [
956
+ "",
957
+ "---",
958
+ "",
959
+ "*Generated by `analyze_codebase.py` — re-run any time to refresh.*",
960
+ "",
961
+ ]
962
+ return "\n".join(lines)
963
+
964
+
965
+ # ─── Main ─────────────────────────────────────────────────────────────────────
966
+
967
+ def main():
968
+ print(f"Scanning: {ROOT}")
969
+
970
+ py_files = collect_py_files(ROOT)
971
+ print(f"Found {len(py_files)} Python files\n")
972
+
973
+ all_files: Dict[str, FileInfo] = {}
974
+ for p in py_files:
975
+ print(f" Parsing {rel(p)}")
976
+ fi = parse_file(p)
977
+ if fi:
978
+ all_files[fi.path] = fi
979
+
980
+ print(f"\nBuilding call graph...")
981
+ call_graph = resolve_calls(all_files)
982
+ total_edges = sum(len(v) for v in call_graph.values())
983
+ print(f" {len(call_graph)} functions with outgoing calls, {total_edges} total edges")
984
+
985
+ print(f"\nGenerating CODEBASE_MAP.md...")
986
+ readme = generate_readme(all_files, call_graph, ENTRY_POINTS)
987
+ out_path = ROOT / "CODEBASE_MAP.md"
988
+ out_path.write_text(readme, encoding="utf-8")
989
+
990
+ total_assets = sum(len(fi.assets) for fi in all_files.values())
991
+ total_urls = sum(len(fi.urls) for fi in all_files.values())
992
+ total_envs = sum(len(fi.env_vars) for fi in all_files.values())
993
+ total_cmds = sum(len(fi.subproc_cmds) for fi in all_files.values())
994
+
995
+ print(f" Done -> {out_path}")
996
+ print(f" Files analyzed : {len(all_files)}")
997
+ print(f" Total functions : {sum(len(fi.functions) for fi in all_files.values())}")
998
+ print(f" Call graph edges: {total_edges}")
999
+ print(f" Asset refs : {total_assets}")
1000
+ print(f" URLs : {total_urls} | Env vars: {total_envs} | Subprocess: {total_cmds}")
1001
+
1002
+ print(f"\nGenerating UNUSED_FILES.md...")
1003
+ unused_report = find_unused(all_files, ENTRY_POINTS)
1004
+ unused_path = ROOT / "UNUSED_FILES.md"
1005
+ unused_path.write_text(unused_report, encoding="utf-8")
1006
+ print(f" Done -> {unused_path}")
1007
+
1008
+ # Quick summary stats for unused report
1009
+ imported_by = build_import_graph(all_files)
1010
+ reachable = reachable_from_entries(ENTRY_POINTS, imported_by, all_files)
1011
+ ep_set = {ep.replace("\\", "/") for ep in ENTRY_POINTS}
1012
+ unused_count = sum(
1013
+ 1 for frel, fi in all_files.items()
1014
+ if frel not in ep_set
1015
+ and frel.split("/")[-1] not in ALWAYS_KEEP_PATTERNS
1016
+ and frel not in reachable
1017
+ )
1018
+ disk_asset_count = len(collect_disk_assets(ROOT))
1019
+ print(f" Unused Python files : {unused_count}")
1020
+ print(f" Disk assets scanned : {disk_asset_count}")
1021
+
1022
+
1023
+ if __name__ == "__main__":
1024
+ main()
app.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # HuggingFace Spaces entry point – must be named app.py
2
+ # Imports and re-exports the Gradio demo from gradio_app.py
3
+
4
+ import os
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ # On HF Spaces, SadTalker checkpoints are likely absent → disable video by default
9
+ # Users can still enable it with the "Skip video" checkbox.
10
+ # If you DO upload checkpoints, remove this env var.
11
+ if not (Path(__file__).parent / "sadtalker+wav2lip" / "sadtalker" / "checkpoints").exists():
12
+ os.environ.setdefault("DISABLE_VIDEO", "1")
13
+
14
+ # HF_TOKEN must be set as a Space Secret for the HF Inference API fallback
15
+ # os.environ["HF_TOKEN"] is picked up automatically from Secrets
16
+
17
+ from gradio_app import demo # noqa: E402 (import after env setup)
18
+
19
+ demo.queue().launch()
gradio_app.py ADDED
@@ -0,0 +1,283 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Mindfull AI Avatar Chatbot - Gradio Frontend
3
+ Works with any uploaded avatar image.
4
+ Runs locally (Ollama) or on HuggingFace Spaces (HF Inference API).
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import time
10
+ import tempfile
11
+ import shutil
12
+ from pathlib import Path
13
+
14
+ import gradio as gr
15
+
16
+ # ── runtime paths ──────────────────────────────────────────────────────────────
17
+ ROOT = Path(__file__).parent.absolute()
18
+ OUTPUT_DIR = ROOT / "outputs"
19
+ AUDIO_DIR = OUTPUT_DIR / "audio"
20
+ VIDEO_DIR = OUTPUT_DIR / "video"
21
+ TEMP_DIR = OUTPUT_DIR / "temp"
22
+ for d in [AUDIO_DIR, VIDEO_DIR, TEMP_DIR]:
23
+ d.mkdir(parents=True, exist_ok=True)
24
+
25
+ # ── LLM backend detection ─────────────────────────────────────────────────────
26
+ def _ollama_available() -> bool:
27
+ try:
28
+ import requests
29
+ r = requests.get("http://localhost:11434/api/tags", timeout=3)
30
+ return r.status_code == 200
31
+ except Exception:
32
+ return False
33
+
34
+ OLLAMA_URL = "http://localhost:11434/api/generate"
35
+ OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "mindfull")
36
+
37
+ HF_TOKEN = os.environ.get("HF_TOKEN", "")
38
+ HF_MODEL = "mistralai/Mistral-7B-Instruct-v0.3"
39
+
40
+
41
+ # ── text generation ────────────────────────────────────────────────────────────
42
+ def generate_text_ollama(prompt: str) -> str:
43
+ import requests
44
+ payload = {
45
+ "model": OLLAMA_MODEL,
46
+ "prompt": prompt,
47
+ "stream": False,
48
+ "options": {"temperature": 0.7, "num_ctx": 2048},
49
+ }
50
+ r = requests.post(OLLAMA_URL, json=payload, timeout=60)
51
+ r.raise_for_status()
52
+ return r.json().get("response", "").strip()
53
+
54
+
55
+ def generate_text_hf(prompt: str) -> str:
56
+ from huggingface_hub import InferenceClient
57
+ client = InferenceClient(model=HF_MODEL, token=HF_TOKEN or None)
58
+ out = client.text_generation(
59
+ prompt,
60
+ max_new_tokens=256,
61
+ temperature=0.7,
62
+ do_sample=True,
63
+ )
64
+ return out.strip()
65
+
66
+
67
+ def generate_response(user_input: str, history: list) -> str:
68
+ """Build prompt from history and call whichever backend is available."""
69
+ # History is Gradio 5 messages format: [{role, content}, ...]
70
+ ctx_lines = []
71
+ for h in history[-8:]: # last 4 exchanges
72
+ role = h.get("role", "") if isinstance(h, dict) else ""
73
+ content = h.get("content", "") if isinstance(h, dict) else ""
74
+ if role == "user":
75
+ ctx_lines.append(f"User: {content}")
76
+ elif role == "assistant":
77
+ ctx_lines.append(f"Mindfull: {content}")
78
+ ctx = "\n".join(ctx_lines)
79
+
80
+ prompt = f"""You are Mindfull, a professional AI wellness companion.
81
+ Provide warm, concise support (2–3 sentences) suitable for a voice response.
82
+
83
+ {ctx}
84
+ User: {user_input}
85
+ Mindfull:"""
86
+
87
+ try:
88
+ if _ollama_available():
89
+ return generate_text_ollama(prompt)
90
+ else:
91
+ return generate_text_hf(prompt)
92
+ except Exception as e:
93
+ return f"I'm here to support you. (Error reaching AI model: {e})"
94
+
95
+
96
+ # ── audio generation ───────────────────────────────────────────────────────────
97
+ def generate_audio(text: str) -> str | None:
98
+ """Generate speech with edge-tts. Returns path to .mp3 or None."""
99
+ try:
100
+ from simple_audio_gen import generate_audio_simple
101
+ ts = int(time.time() * 1000)
102
+ out = str(AUDIO_DIR / f"response_{ts}.mp3")
103
+ ok = generate_audio_simple(text, out)
104
+ return out if ok else None
105
+ except Exception as e:
106
+ print(f"Audio generation error: {e}")
107
+ return None
108
+
109
+
110
+ # ── video generation ───────────────────────────────────────────────────────────
111
+ def generate_video(image_path: str | None, audio_path: str | None) -> str | None:
112
+ """Generate talking-head video. Returns path to .mp4 or None."""
113
+ if not image_path or not audio_path:
114
+ return None
115
+ if not Path(image_path).exists() or not Path(audio_path).exists():
116
+ return None
117
+ try:
118
+ from simple_video_gen import generate_video_simple
119
+ ts = int(time.time() * 1000)
120
+ out = str(VIDEO_DIR / f"avatar_{ts}.mp4")
121
+ ok = generate_video_simple(image_path, audio_path, out)
122
+ return out if ok and Path(out).exists() else None
123
+ except Exception as e:
124
+ print(f"Video generation error: {e}")
125
+ return None
126
+
127
+
128
+ # ── main chat handler ─────────────────────────────────────────���────────────────
129
+ def chat(
130
+ user_message: str,
131
+ avatar_image, # numpy array or file path from gr.Image
132
+ history: list,
133
+ skip_video: bool,
134
+ ):
135
+ if not user_message.strip():
136
+ yield history, None, None, "Please type a message."
137
+ return
138
+
139
+ # ── 1. text ──────────────────────────────────────────────────────────────
140
+ yield history, None, None, "💬 Generating response…"
141
+ response_text = generate_response(user_message.strip(), history)
142
+ # Gradio 5 messages format: {"role": ..., "content": ...}
143
+ new_history = history + [
144
+ {"role": "user", "content": user_message},
145
+ {"role": "assistant", "content": response_text},
146
+ ]
147
+
148
+ # ── 2. audio ─────────────────────────────────────────────────────────────
149
+ yield new_history, None, None, "🔊 Synthesising speech…"
150
+ audio_path = generate_audio(response_text)
151
+
152
+ # ── 3. video ─────────────────────────────────────────────────────────────
153
+ video_path = None
154
+ if not skip_video and avatar_image is not None:
155
+ yield new_history, audio_path, None, "🎬 Generating avatar video…"
156
+
157
+ # Gradio Image component returns a numpy array; save it to a temp file
158
+ import numpy as np
159
+ from PIL import Image as PilImage
160
+
161
+ if isinstance(avatar_image, np.ndarray):
162
+ ts = int(time.time() * 1000)
163
+ img_path = str(TEMP_DIR / f"avatar_input_{ts}.png")
164
+ pil_img = PilImage.fromarray(avatar_image)
165
+ # SadTalker works best with images ≤ 512px on the longest side
166
+ MAX_DIM = 512
167
+ if max(pil_img.size) > MAX_DIM:
168
+ pil_img.thumbnail((MAX_DIM, MAX_DIM), PilImage.LANCZOS)
169
+ pil_img.save(img_path)
170
+ else:
171
+ img_path = str(avatar_image) # already a path
172
+
173
+ video_path = generate_video(img_path, audio_path)
174
+
175
+ status = "✅ Done!" if video_path or audio_path else "⚠️ Only text response available."
176
+ yield new_history, audio_path, video_path, status
177
+
178
+
179
+ # ── Gradio UI ──────────────────────────────────────────────────────────────────
180
+ with gr.Blocks(
181
+ title="Mindfull – AI Wellness Companion",
182
+ theme=gr.themes.Soft(primary_hue="blue", secondary_hue="slate"),
183
+ css="""
184
+ #header { text-align: center; padding: 10px 0 4px; }
185
+ #header h1 { font-size: 2rem; margin-bottom: 2px; }
186
+ #header p { color: #6b7280; margin: 0; }
187
+ .status-box { font-size: 0.85rem; color: #374151; }
188
+ """,
189
+ ) as demo:
190
+
191
+ # ── header ────────────────────────────────────────────────────────────────
192
+ with gr.Row(elem_id="header"):
193
+ with gr.Column():
194
+ gr.HTML("""
195
+ <h1>🧠 Mindfull</h1>
196
+ <p>AI Wellness Companion · talking-head avatar · real TTS</p>
197
+ """)
198
+
199
+ # If running on HF Spaces without SadTalker checkpoints, default to audio-only
200
+ _video_disabled = os.environ.get("DISABLE_VIDEO", "0") == "1"
201
+
202
+ # ── main layout ───────────────────────────────────────────────────────────
203
+ with gr.Row():
204
+
205
+ # Left column – avatar + controls
206
+ with gr.Column(scale=1, min_width=260):
207
+ avatar_img = gr.Image(
208
+ label="Avatar Image (any portrait)",
209
+ type="numpy",
210
+ sources=["upload", "webcam"],
211
+ height=280,
212
+ )
213
+ skip_video_chk = gr.Checkbox(
214
+ label="Skip video (faster · audio only)",
215
+ value=_video_disabled,
216
+ )
217
+ gr.Markdown(
218
+ "_Upload **any** face image. "
219
+ "Leaving it empty will return text + audio only._",
220
+ elem_classes=["status-box"],
221
+ )
222
+
223
+ # Right column – chat
224
+ with gr.Column(scale=2):
225
+ chatbot = gr.Chatbot(
226
+ label="Conversation",
227
+ height=400,
228
+ type="messages",
229
+ avatar_images=(None, str(ROOT / "avatar_assets" / "officer.png")
230
+ if (ROOT / "avatar_assets" / "officer.png").exists()
231
+ else None),
232
+ )
233
+ with gr.Row():
234
+ msg_box = gr.Textbox(
235
+ placeholder="How are you feeling today?",
236
+ label="Your message",
237
+ scale=5,
238
+ lines=1,
239
+ )
240
+ send_btn = gr.Button("Send", variant="primary", scale=1)
241
+
242
+ # ── outputs ───────────────────────────────────────────────────────────────
243
+ with gr.Row():
244
+ audio_out = gr.Audio(label="🔊 Voice Response", type="filepath", autoplay=True)
245
+ video_out = gr.Video(label="🎬 Avatar Video", autoplay=True)
246
+
247
+ status_box = gr.Textbox(
248
+ label="Status", interactive=False, lines=1, elem_classes=["status-box"]
249
+ )
250
+
251
+ # ── clear button ──────────────────────────────────────────────────────────
252
+ with gr.Row():
253
+ clear_btn = gr.Button("🗑 Clear conversation", variant="secondary")
254
+
255
+ # ── wiring ────────────────────────────────────────────────────────────────
256
+ def _submit(msg, img, hist, skip):
257
+ yield from chat(msg, img, hist, skip)
258
+
259
+ send_btn.click(
260
+ fn=_submit,
261
+ inputs=[msg_box, avatar_img, chatbot, skip_video_chk],
262
+ outputs=[chatbot, audio_out, video_out, status_box],
263
+ ).then(fn=lambda: "", outputs=msg_box)
264
+
265
+ msg_box.submit(
266
+ fn=_submit,
267
+ inputs=[msg_box, avatar_img, chatbot, skip_video_chk],
268
+ outputs=[chatbot, audio_out, video_out, status_box],
269
+ ).then(fn=lambda: "", outputs=msg_box)
270
+
271
+ clear_btn.click(
272
+ fn=lambda: ([], None, None, ""), # empty list = empty messages history
273
+ outputs=[chatbot, audio_out, video_out, status_box],
274
+ )
275
+
276
+
277
+ if __name__ == "__main__":
278
+ demo.queue().launch(
279
+ server_name="0.0.0.0",
280
+ server_port=7860,
281
+ share=False, # set True for a public Gradio link
282
+ show_error=True,
283
+ )
infer_cli.py ADDED
@@ -0,0 +1,368 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import codecs
3
+ import os
4
+ import re
5
+ from datetime import datetime
6
+ from importlib.resources import files
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+ import soundfile as sf
11
+ import tomli
12
+ from cached_path import cached_path
13
+ from hydra.utils import get_class
14
+ from omegaconf import OmegaConf
15
+
16
+ from f5_tts.infer.utils_infer import (
17
+ cfg_strength,
18
+ cross_fade_duration,
19
+ device,
20
+ fix_duration,
21
+ infer_process,
22
+ load_model,
23
+ load_vocoder,
24
+ mel_spec_type,
25
+ nfe_step,
26
+ preprocess_ref_audio_text,
27
+ remove_silence_for_generated_wav,
28
+ speed,
29
+ sway_sampling_coef,
30
+ target_rms,
31
+ )
32
+
33
+
34
+ parser = argparse.ArgumentParser(
35
+ prog="python3 infer-cli.py",
36
+ description="Commandline interface for E2/F5 TTS with Advanced Batch Processing.",
37
+ epilog="Specify options above to override one or more settings from config.",
38
+ )
39
+ parser.add_argument(
40
+ "-c",
41
+ "--config",
42
+ type=str,
43
+ default=os.path.join(files("f5_tts").joinpath("infer/examples/basic"), "basic.toml"),
44
+ help="The configuration file, default see infer/examples/basic/basic.toml",
45
+ )
46
+
47
+
48
+ # Note. Not to provide default value here in order to read default from config file
49
+
50
+ parser.add_argument(
51
+ "-m",
52
+ "--model",
53
+ type=str,
54
+ help="The model name: F5TTS_v1_Base | F5TTS_Base | E2TTS_Base | etc.",
55
+ )
56
+ parser.add_argument(
57
+ "-mc",
58
+ "--model_cfg",
59
+ type=str,
60
+ help="The path to F5-TTS model config file .yaml",
61
+ )
62
+ parser.add_argument(
63
+ "-p",
64
+ "--ckpt_file",
65
+ type=str,
66
+ help="The path to model checkpoint .pt, leave blank to use default",
67
+ )
68
+ parser.add_argument(
69
+ "-v",
70
+ "--vocab_file",
71
+ type=str,
72
+ help="The path to vocab file .txt, leave blank to use default",
73
+ )
74
+ parser.add_argument(
75
+ "-r",
76
+ "--ref_audio",
77
+ type=str,
78
+ help="The reference audio file.",
79
+ )
80
+ parser.add_argument(
81
+ "-s",
82
+ "--ref_text",
83
+ type=str,
84
+ help="The transcript/subtitle for the reference audio",
85
+ )
86
+ parser.add_argument(
87
+ "-t",
88
+ "--gen_text",
89
+ type=str,
90
+ help="The text to make model synthesize a speech",
91
+ )
92
+ parser.add_argument(
93
+ "-f",
94
+ "--gen_file",
95
+ type=str,
96
+ help="The file with text to generate, will ignore --gen_text",
97
+ )
98
+ parser.add_argument(
99
+ "-o",
100
+ "--output_dir",
101
+ type=str,
102
+ help="The path to output folder",
103
+ )
104
+ parser.add_argument(
105
+ "-w",
106
+ "--output_file",
107
+ type=str,
108
+ help="The name of output file",
109
+ )
110
+ parser.add_argument(
111
+ "--save_chunk",
112
+ action="store_true",
113
+ help="To save each audio chunks during inference",
114
+ )
115
+ parser.add_argument(
116
+ "--remove_silence",
117
+ action="store_true",
118
+ help="To remove long silence found in ouput",
119
+ )
120
+ parser.add_argument(
121
+ "--load_vocoder_from_local",
122
+ action="store_true",
123
+ help="To load vocoder from local dir, default to ../checkpoints/vocos-mel-24khz",
124
+ )
125
+ parser.add_argument(
126
+ "--vocoder_name",
127
+ type=str,
128
+ choices=["vocos", "bigvgan"],
129
+ help=f"Used vocoder name: vocos | bigvgan, default {mel_spec_type}",
130
+ )
131
+ parser.add_argument(
132
+ "--target_rms",
133
+ type=float,
134
+ help=f"Target output speech loudness normalization value, default {target_rms}",
135
+ )
136
+ parser.add_argument(
137
+ "--cross_fade_duration",
138
+ type=float,
139
+ help=f"Duration of cross-fade between audio segments in seconds, default {cross_fade_duration}",
140
+ )
141
+ parser.add_argument(
142
+ "--nfe_step",
143
+ type=int,
144
+ help=f"The number of function evaluation (denoising steps), default {nfe_step}",
145
+ )
146
+ parser.add_argument(
147
+ "--cfg_strength",
148
+ type=float,
149
+ help=f"Classifier-free guidance strength, default {cfg_strength}",
150
+ )
151
+ parser.add_argument(
152
+ "--sway_sampling_coef",
153
+ type=float,
154
+ help=f"Sway Sampling coefficient, default {sway_sampling_coef}",
155
+ )
156
+ parser.add_argument(
157
+ "--speed",
158
+ type=float,
159
+ help=f"The speed of the generated audio, default {speed}",
160
+ )
161
+ parser.add_argument(
162
+ "--fix_duration",
163
+ type=float,
164
+ help=f"Fix the total duration (ref and gen audios) in seconds, default {fix_duration}",
165
+ )
166
+ parser.add_argument(
167
+ "--device",
168
+ type=str,
169
+ help="Specify the device to run on",
170
+ )
171
+ args = parser.parse_args()
172
+
173
+
174
+ # config file
175
+
176
+ config = tomli.load(open(args.config, "rb"))
177
+
178
+
179
+ # command-line interface parameters
180
+
181
+ model = args.model or config.get("model", "F5TTS_v1_Base")
182
+ ckpt_file = args.ckpt_file or config.get("ckpt_file", "")
183
+ vocab_file = args.vocab_file or config.get("vocab_file", "")
184
+
185
+ ref_audio = args.ref_audio or config.get("ref_audio", "infer/examples/basic/basic_ref_en.wav")
186
+ ref_text = (
187
+ args.ref_text
188
+ if args.ref_text is not None
189
+ else config.get("ref_text", "Some call me nature, others call me mother nature.")
190
+ )
191
+ gen_text = args.gen_text or config.get("gen_text", "Here we generate something just for test.")
192
+ gen_file = args.gen_file or config.get("gen_file", "")
193
+
194
+ output_dir = args.output_dir or config.get("output_dir", "tests")
195
+ output_file = args.output_file or config.get(
196
+ "output_file", f"infer_cli_{datetime.now().strftime(r'%Y%m%d_%H%M%S')}.wav"
197
+ )
198
+
199
+ save_chunk = args.save_chunk or config.get("save_chunk", False)
200
+ remove_silence = args.remove_silence or config.get("remove_silence", False)
201
+ load_vocoder_from_local = args.load_vocoder_from_local or config.get("load_vocoder_from_local", False)
202
+
203
+ vocoder_name = args.vocoder_name or config.get("vocoder_name", mel_spec_type)
204
+ target_rms = args.target_rms or config.get("target_rms", target_rms)
205
+ cross_fade_duration = args.cross_fade_duration or config.get("cross_fade_duration", cross_fade_duration)
206
+ nfe_step = args.nfe_step or config.get("nfe_step", nfe_step)
207
+ cfg_strength = args.cfg_strength or config.get("cfg_strength", cfg_strength)
208
+ sway_sampling_coef = args.sway_sampling_coef or config.get("sway_sampling_coef", sway_sampling_coef)
209
+ speed = args.speed or config.get("speed", speed)
210
+ fix_duration = args.fix_duration or config.get("fix_duration", fix_duration)
211
+ device = args.device or config.get("device", device)
212
+
213
+
214
+ # patches for pip pkg user
215
+ if "infer/examples/" in ref_audio:
216
+ ref_audio = str(files("f5_tts").joinpath(f"{ref_audio}"))
217
+ if "infer/examples/" in gen_file:
218
+ gen_file = str(files("f5_tts").joinpath(f"{gen_file}"))
219
+ if "voices" in config:
220
+ for voice in config["voices"]:
221
+ voice_ref_audio = config["voices"][voice]["ref_audio"]
222
+ if "infer/examples/" in voice_ref_audio:
223
+ config["voices"][voice]["ref_audio"] = str(files("f5_tts").joinpath(f"{voice_ref_audio}"))
224
+
225
+
226
+ # ignore gen_text if gen_file provided
227
+
228
+ if gen_file:
229
+ gen_text = codecs.open(gen_file, "r", "utf-8").read()
230
+
231
+
232
+ # output path
233
+
234
+ wave_path = Path(output_dir) / output_file
235
+ # spectrogram_path = Path(output_dir) / "infer_cli_out.png"
236
+ if save_chunk:
237
+ output_chunk_dir = os.path.join(output_dir, f"{Path(output_file).stem}_chunks")
238
+ if not os.path.exists(output_chunk_dir):
239
+ os.makedirs(output_chunk_dir)
240
+
241
+
242
+ # load vocoder
243
+
244
+ if vocoder_name == "vocos":
245
+ vocoder_local_path = "../checkpoints/vocos-mel-24khz"
246
+ elif vocoder_name == "bigvgan":
247
+ vocoder_local_path = "../checkpoints/bigvgan_v2_24khz_100band_256x"
248
+
249
+ vocoder = load_vocoder(
250
+ vocoder_name=vocoder_name, is_local=load_vocoder_from_local, local_path=vocoder_local_path, device=device
251
+ )
252
+
253
+
254
+ # load TTS model
255
+
256
+ model_cfg = OmegaConf.load(
257
+ args.model_cfg or config.get("model_cfg", str(files("f5_tts").joinpath(f"configs/{model}.yaml")))
258
+ )
259
+ model_cls = get_class(f"f5_tts.model.{model_cfg.model.backbone}")
260
+ model_arc = model_cfg.model.arch
261
+
262
+ repo_name, ckpt_step, ckpt_type = "F5-TTS", 1250000, "safetensors"
263
+
264
+ if model != "F5TTS_Base":
265
+ assert vocoder_name == model_cfg.model.mel_spec.mel_spec_type
266
+
267
+ # override for previous models
268
+ if model == "F5TTS_Base":
269
+ if vocoder_name == "vocos":
270
+ ckpt_step = 1200000
271
+ elif vocoder_name == "bigvgan":
272
+ model = "F5TTS_Base_bigvgan"
273
+ ckpt_type = "pt"
274
+ elif model == "E2TTS_Base":
275
+ repo_name = "E2-TTS"
276
+ ckpt_step = 1200000
277
+
278
+ if not ckpt_file:
279
+ ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{model}/model_{ckpt_step}.{ckpt_type}"))
280
+
281
+ print(f"Using {model}...")
282
+ ema_model = load_model(
283
+ model_cls, model_arc, ckpt_file, mel_spec_type=vocoder_name, vocab_file=vocab_file, device=device
284
+ )
285
+
286
+
287
+ # inference process
288
+
289
+
290
+ def main():
291
+ main_voice = {"ref_audio": ref_audio, "ref_text": ref_text}
292
+ if "voices" not in config:
293
+ voices = {"main": main_voice}
294
+ else:
295
+ voices = config["voices"]
296
+ voices["main"] = main_voice
297
+ for voice in voices:
298
+ print("Voice:", voice)
299
+ print("ref_audio ", voices[voice]["ref_audio"])
300
+ voices[voice]["ref_audio"], voices[voice]["ref_text"] = preprocess_ref_audio_text(
301
+ voices[voice]["ref_audio"], voices[voice]["ref_text"]
302
+ )
303
+ print("ref_audio_", voices[voice]["ref_audio"], "\n\n")
304
+
305
+ generated_audio_segments = []
306
+ reg1 = r"(?=\[\w+\])"
307
+ chunks = re.split(reg1, gen_text)
308
+ reg2 = r"\[(\w+)\]"
309
+ for text in chunks:
310
+ if not text.strip():
311
+ continue
312
+ match = re.match(reg2, text)
313
+ if match:
314
+ voice = match[1]
315
+ else:
316
+ print("No voice tag found, using main.")
317
+ voice = "main"
318
+ if voice not in voices:
319
+ print(f"Voice {voice} not found, using main.")
320
+ voice = "main"
321
+ text = re.sub(reg2, "", text)
322
+ ref_audio_ = voices[voice]["ref_audio"]
323
+ ref_text_ = voices[voice]["ref_text"]
324
+ gen_text_ = text.strip()
325
+ print(f"Voice: {voice}")
326
+ audio_segment, final_sample_rate, spectrogram = infer_process(
327
+ ref_audio_,
328
+ ref_text_,
329
+ gen_text_,
330
+ ema_model,
331
+ vocoder,
332
+ mel_spec_type=vocoder_name,
333
+ target_rms=target_rms,
334
+ cross_fade_duration=cross_fade_duration,
335
+ nfe_step=nfe_step,
336
+ cfg_strength=cfg_strength,
337
+ sway_sampling_coef=sway_sampling_coef,
338
+ speed=speed,
339
+ fix_duration=fix_duration,
340
+ device=device,
341
+ )
342
+ generated_audio_segments.append(audio_segment)
343
+
344
+ if save_chunk:
345
+ if len(gen_text_) > 200:
346
+ gen_text_ = gen_text_[:200] + " ... "
347
+ sf.write(
348
+ os.path.join(output_chunk_dir, f"{len(generated_audio_segments) - 1}_{gen_text_}.wav"),
349
+ audio_segment,
350
+ final_sample_rate,
351
+ )
352
+
353
+ if generated_audio_segments:
354
+ final_wave = np.concatenate(generated_audio_segments)
355
+
356
+ if not os.path.exists(output_dir):
357
+ os.makedirs(output_dir)
358
+
359
+ with open(wave_path, "wb") as f:
360
+ sf.write(f.name, final_wave, final_sample_rate)
361
+ # Remove silence
362
+ if remove_silence:
363
+ remove_silence_for_generated_wav(f.name)
364
+ print(f.name)
365
+
366
+
367
+ if __name__ == "__main__":
368
+ main()
mindfull_client.html ADDED
@@ -0,0 +1,515 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>Mindfull AI Avatar Chatbot</title>
7
+ <style>
8
+ * {
9
+ margin: 0;
10
+ padding: 0;
11
+ box-sizing: border-box;
12
+ }
13
+
14
+ body {
15
+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
16
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
17
+ min-height: 100vh;
18
+ display: flex;
19
+ justify-content: center;
20
+ align-items: center;
21
+ }
22
+
23
+ .chat-container {
24
+ background: white;
25
+ border-radius: 20px;
26
+ box-shadow: 0 20px 40px rgba(0, 0, 0, 0.1);
27
+ width: 100%;
28
+ max-width: 800px;
29
+ height: 90vh;
30
+ display: flex;
31
+ flex-direction: column;
32
+ overflow: hidden;
33
+ }
34
+
35
+ .chat-header {
36
+ background: #4a5568;
37
+ color: white;
38
+ padding: 20px;
39
+ text-align: center;
40
+ }
41
+
42
+ .chat-header h1 {
43
+ margin-bottom: 5px;
44
+ }
45
+
46
+ .chat-header p {
47
+ opacity: 0.8;
48
+ font-size: 14px;
49
+ }
50
+
51
+ .status-bar {
52
+ background: #f7fafc;
53
+ padding: 10px 20px;
54
+ border-bottom: 1px solid #e2e8f0;
55
+ font-size: 12px;
56
+ color: #4a5568;
57
+ }
58
+
59
+ .status-indicator {
60
+ display: inline-block;
61
+ width: 8px;
62
+ height: 8px;
63
+ border-radius: 50%;
64
+ margin-right: 5px;
65
+ }
66
+
67
+ .status-connected {
68
+ background: #48bb78;
69
+ }
70
+
71
+ .status-disconnected {
72
+ background: #f56565;
73
+ }
74
+
75
+ .chat-messages {
76
+ flex: 1;
77
+ padding: 20px;
78
+ overflow-y: auto;
79
+ background: #f8f9fa;
80
+ }
81
+
82
+ .message {
83
+ margin-bottom: 20px;
84
+ display: flex;
85
+ align-items: flex-start;
86
+ }
87
+
88
+ .message.user {
89
+ justify-content: flex-end;
90
+ }
91
+
92
+ .message-content {
93
+ max-width: 70%;
94
+ padding: 15px;
95
+ border-radius: 15px;
96
+ position: relative;
97
+ }
98
+
99
+ .message.user .message-content {
100
+ background: #4299e1;
101
+ color: white;
102
+ border-bottom-right-radius: 5px;
103
+ }
104
+
105
+ .message.bot .message-content {
106
+ background: white;
107
+ border: 1px solid #e2e8f0;
108
+ border-bottom-left-radius: 5px;
109
+ }
110
+
111
+ .message-meta {
112
+ font-size: 11px;
113
+ opacity: 0.7;
114
+ margin-top: 5px;
115
+ }
116
+
117
+ .emotion-tag {
118
+ display: inline-block;
119
+ background: #edf2f7;
120
+ color: #4a5568;
121
+ padding: 2px 8px;
122
+ border-radius: 10px;
123
+ font-size: 10px;
124
+ margin-left: 5px;
125
+ }
126
+
127
+ .media-content {
128
+ margin-top: 10px;
129
+ }
130
+
131
+ .audio-player {
132
+ width: 100%;
133
+ margin-top: 10px;
134
+ }
135
+
136
+ .video-player {
137
+ width: 100%;
138
+ max-width: 300px;
139
+ border-radius: 10px;
140
+ margin-top: 10px;
141
+ }
142
+
143
+ .chat-input {
144
+ padding: 20px;
145
+ background: white;
146
+ border-top: 1px solid #e2e8f0;
147
+ }
148
+
149
+ .input-container {
150
+ display: flex;
151
+ gap: 10px;
152
+ align-items: flex-end;
153
+ }
154
+
155
+ .message-input {
156
+ flex: 1;
157
+ border: 2px solid #e2e8f0;
158
+ border-radius: 25px;
159
+ padding: 15px 20px;
160
+ font-size: 14px;
161
+ resize: none;
162
+ max-height: 100px;
163
+ min-height: 50px;
164
+ outline: none;
165
+ transition: border-color 0.3s;
166
+ }
167
+
168
+ .message-input:focus {
169
+ border-color: #4299e1;
170
+ }
171
+
172
+ .send-button {
173
+ background: #4299e1;
174
+ color: white;
175
+ border: none;
176
+ border-radius: 50%;
177
+ width: 50px;
178
+ height: 50px;
179
+ cursor: pointer;
180
+ display: flex;
181
+ align-items: center;
182
+ justify-content: center;
183
+ transition: background 0.3s;
184
+ }
185
+
186
+ .send-button:hover:not(:disabled) {
187
+ background: #3182ce;
188
+ }
189
+
190
+ .send-button:disabled {
191
+ background: #a0aec0;
192
+ cursor: not-allowed;
193
+ }
194
+
195
+ .loading {
196
+ display: none;
197
+ text-align: center;
198
+ padding: 20px;
199
+ color: #4a5568;
200
+ }
201
+
202
+ .loading.show {
203
+ display: block;
204
+ }
205
+
206
+ .spinner {
207
+ display: inline-block;
208
+ width: 20px;
209
+ height: 20px;
210
+ border: 3px solid #e2e8f0;
211
+ border-radius: 50%;
212
+ border-top-color: #4299e1;
213
+ animation: spin 1s ease-in-out infinite;
214
+ }
215
+
216
+ @keyframes spin {
217
+ to { transform: rotate(360deg); }
218
+ }
219
+
220
+ .controls {
221
+ display: flex;
222
+ gap: 10px;
223
+ margin-top: 10px;
224
+ flex-wrap: wrap;
225
+ }
226
+
227
+ .control-checkbox {
228
+ display: flex;
229
+ align-items: center;
230
+ gap: 5px;
231
+ font-size: 12px;
232
+ color: #4a5568;
233
+ }
234
+
235
+ .error-message {
236
+ background: #fed7d7;
237
+ color: #c53030;
238
+ padding: 10px;
239
+ border-radius: 5px;
240
+ margin-bottom: 10px;
241
+ font-size: 12px;
242
+ }
243
+
244
+ @media (max-width: 768px) {
245
+ .chat-container {
246
+ height: 100vh;
247
+ border-radius: 0;
248
+ }
249
+
250
+ .message-content {
251
+ max-width: 90%;
252
+ }
253
+ }
254
+ </style>
255
+ </head>
256
+ <body>
257
+ <div class="chat-container">
258
+ <div class="chat-header">
259
+ <h1>🛡️ Mindfull</h1>
260
+ <p>AI Mental Wellness Assistant for Police Officers</p>
261
+ </div>
262
+
263
+ <div class="status-bar">
264
+ <span class="status-indicator status-disconnected" id="statusIndicator"></span>
265
+ <span id="statusText">Connecting...</span>
266
+ </div>
267
+
268
+ <div class="chat-messages" id="chatMessages">
269
+ <div class="message bot">
270
+ <div class="message-content">
271
+ <div>🙏 Namaskara! I'm Mindfull, your AI wellness assistant. I'm here to support you with any mental health concerns or stress you might be experiencing. How are you feeling today?</div>
272
+ <div class="message-meta">System message</div>
273
+ </div>
274
+ </div>
275
+ </div>
276
+
277
+ <div class="loading" id="loadingIndicator">
278
+ <div class="spinner"></div>
279
+ <span>Processing your message...</span>
280
+ </div>
281
+
282
+ <div class="chat-input">
283
+ <div class="error-message" id="errorMessage" style="display: none;"></div>
284
+
285
+ <div class="input-container">
286
+ <textarea
287
+ id="messageInput"
288
+ class="message-input"
289
+ placeholder="Share how you're feeling today..."
290
+ rows="1"
291
+ ></textarea>
292
+ <button id="sendButton" class="send-button">
293
+ <svg width="20" height="20" viewBox="0 0 24 24" fill="currentColor">
294
+ <path d="M2.01 21L23 12 2.01 3 2 10l15 2-15 2z"/>
295
+ </svg>
296
+ </button>
297
+ </div>
298
+
299
+ <div class="controls">
300
+ <label class="control-checkbox">
301
+ <input type="checkbox" id="enableVideo" checked>
302
+ Generate Avatar Video
303
+ </label>
304
+ <label class="control-checkbox">
305
+ <input type="checkbox" id="autoPlay" checked>
306
+ Auto-play Audio
307
+ </label>
308
+ </div>
309
+ </div>
310
+ </div>
311
+
312
+ <script>
313
+ class MindfullChat {
314
+ constructor() {
315
+ this.apiUrl = 'http://localhost:5000';
316
+ this.sessionId = null;
317
+ this.isConnected = false;
318
+
319
+ this.initializeElements();
320
+ this.setupEventListeners();
321
+ this.checkConnection();
322
+ this.createSession();
323
+ }
324
+
325
+ initializeElements() {
326
+ this.messagesContainer = document.getElementById('chatMessages');
327
+ this.messageInput = document.getElementById('messageInput');
328
+ this.sendButton = document.getElementById('sendButton');
329
+ this.loadingIndicator = document.getElementById('loadingIndicator');
330
+ this.statusIndicator = document.getElementById('statusIndicator');
331
+ this.statusText = document.getElementById('statusText');
332
+ this.errorMessage = document.getElementById('errorMessage');
333
+ this.enableVideo = document.getElementById('enableVideo');
334
+ this.autoPlay = document.getElementById('autoPlay');
335
+ }
336
+
337
+ setupEventListeners() {
338
+ this.sendButton.addEventListener('click', () => this.sendMessage());
339
+ this.messageInput.addEventListener('keypress', (e) => {
340
+ if (e.key === 'Enter' && !e.shiftKey) {
341
+ e.preventDefault();
342
+ this.sendMessage();
343
+ }
344
+ });
345
+
346
+ // Auto-resize textarea
347
+ this.messageInput.addEventListener('input', () => {
348
+ this.messageInput.style.height = 'auto';
349
+ this.messageInput.style.height = this.messageInput.scrollHeight + 'px';
350
+ });
351
+ }
352
+
353
+ async checkConnection() {
354
+ try {
355
+ const response = await fetch(`${this.apiUrl}/health`);
356
+ if (response.ok) {
357
+ this.setConnectionStatus(true, 'Connected');
358
+ } else {
359
+ this.setConnectionStatus(false, 'API Error');
360
+ }
361
+ } catch (error) {
362
+ this.setConnectionStatus(false, 'Disconnected');
363
+ this.showError('Unable to connect to Mindfull API. Please ensure the server is running.');
364
+ }
365
+ }
366
+
367
+ async createSession() {
368
+ try {
369
+ const response = await fetch(`${this.apiUrl}/session`, {
370
+ method: 'POST',
371
+ headers: {
372
+ 'Content-Type': 'application/json',
373
+ }
374
+ });
375
+
376
+ if (response.ok) {
377
+ const data = await response.json();
378
+ this.sessionId = data.session_id;
379
+ console.log('Session created:', this.sessionId);
380
+ }
381
+ } catch (error) {
382
+ console.error('Failed to create session:', error);
383
+ }
384
+ }
385
+
386
+ setConnectionStatus(connected, text) {
387
+ this.isConnected = connected;
388
+ this.statusIndicator.className = `status-indicator ${connected ? 'status-connected' : 'status-disconnected'}`;
389
+ this.statusText.textContent = text;
390
+ }
391
+
392
+ showError(message) {
393
+ this.errorMessage.textContent = message;
394
+ this.errorMessage.style.display = 'block';
395
+ setTimeout(() => {
396
+ this.errorMessage.style.display = 'none';
397
+ }, 5000);
398
+ }
399
+
400
+ addMessage(content, isUser = false, metadata = {}) {
401
+ const messageDiv = document.createElement('div');
402
+ messageDiv.className = `message ${isUser ? 'user' : 'bot'}`;
403
+
404
+ let metaInfo = '';
405
+ if (!isUser) {
406
+ const emotion = metadata.emotion ? `<span class="emotion-tag">${metadata.emotion}</span>` : '';
407
+ const processingTime = metadata.processingTime ? ` • ${metadata.processingTime}s` : '';
408
+ metaInfo = `<div class="message-meta">${new Date().toLocaleTimeString()}${emotion}${processingTime}</div>`;
409
+ }
410
+
411
+ let mediaContent = '';
412
+ if (metadata.audioUrl) {
413
+ mediaContent += `
414
+ <div class="media-content">
415
+ <audio controls class="audio-player" ${this.autoPlay.checked ? 'autoplay' : ''}>
416
+ <source src="${this.apiUrl}${metadata.audioUrl}" type="audio/wav">
417
+ Your browser does not support the audio element.
418
+ </audio>
419
+ </div>
420
+ `;
421
+ }
422
+
423
+ if (metadata.videoUrl) {
424
+ mediaContent += `
425
+ <div class="media-content">
426
+ <video controls class="video-player" ${this.autoPlay.checked ? 'autoplay' : ''}>
427
+ <source src="${this.apiUrl}${metadata.videoUrl}" type="video/mp4">
428
+ Your browser does not support the video element.
429
+ </video>
430
+ </div>
431
+ `;
432
+ }
433
+
434
+ messageDiv.innerHTML = `
435
+ <div class="message-content">
436
+ <div>${content}</div>
437
+ ${metaInfo}
438
+ ${mediaContent}
439
+ </div>
440
+ `;
441
+
442
+ this.messagesContainer.appendChild(messageDiv);
443
+ this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight;
444
+ }
445
+
446
+ setLoading(loading) {
447
+ this.loadingIndicator.classList.toggle('show', loading);
448
+ this.sendButton.disabled = loading;
449
+
450
+ if (loading) {
451
+ this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight;
452
+ }
453
+ }
454
+
455
+ async sendMessage() {
456
+ const message = this.messageInput.value.trim();
457
+ if (!message || !this.isConnected) return;
458
+
459
+ // Add user message
460
+ this.addMessage(message, true);
461
+
462
+ // Clear input
463
+ this.messageInput.value = '';
464
+ this.messageInput.style.height = 'auto';
465
+
466
+ // Show loading
467
+ this.setLoading(true);
468
+
469
+ try {
470
+ const response = await fetch(`${this.apiUrl}/chat`, {
471
+ method: 'POST',
472
+ headers: {
473
+ 'Content-Type': 'application/json',
474
+ },
475
+ body: JSON.stringify({
476
+ message: message,
477
+ session_id: this.sessionId,
478
+ include_video: this.enableVideo.checked
479
+ })
480
+ });
481
+
482
+ if (response.ok) {
483
+ const data = await response.json();
484
+
485
+ // Add bot response
486
+ this.addMessage(data.response_text, false, {
487
+ emotion: data.emotion_detected,
488
+ processingTime: data.processing_time?.toFixed(2),
489
+ audioUrl: data.audio_url,
490
+ videoUrl: data.video_url
491
+ });
492
+
493
+ if (data.warnings && data.warnings.length > 0) {
494
+ console.warn('Warnings:', data.warnings);
495
+ }
496
+ } else {
497
+ const errorData = await response.json();
498
+ this.showError(errorData.message || 'Failed to send message');
499
+ }
500
+ } catch (error) {
501
+ console.error('Send message error:', error);
502
+ this.showError('Network error. Please try again.');
503
+ } finally {
504
+ this.setLoading(false);
505
+ }
506
+ }
507
+ }
508
+
509
+ // Initialize chat when page loads
510
+ document.addEventListener('DOMContentLoaded', () => {
511
+ new MindfullChat();
512
+ });
513
+ </script>
514
+ </body>
515
+ </html>
mindfull_config.py ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Mindfull AI Avatar Chatbot Configuration
3
+ Comprehensive configuration for Ollama, F5-TTS, and SadTalker integration
4
+ """
5
+
6
+ import os
7
+ import logging
8
+ from pathlib import Path
9
+ from typing import Dict, Any, Optional
10
+
11
+ class MindfullConfig:
12
+ """Configuration class for Mindfull AI Avatar Chatbot"""
13
+
14
+ # =========================================================================
15
+ # ENVIRONMENT CONFIGURATION
16
+ # =========================================================================
17
+
18
+ # Project Paths
19
+ PROJECT_ROOT = Path(__file__).parent.absolute()
20
+ VENV_PATH = PROJECT_ROOT / "venv"
21
+ PYTHON_EXECUTABLE = (
22
+ VENV_PATH / "Scripts" / "python.exe" if os.name == 'nt'
23
+ else VENV_PATH / "bin" / "python"
24
+ )
25
+
26
+ # Output Directories
27
+ OUTPUT_DIR = PROJECT_ROOT / "outputs"
28
+ AUDIO_OUTPUT_DIR = OUTPUT_DIR / "audio"
29
+ VIDEO_OUTPUT_DIR = OUTPUT_DIR / "video"
30
+ TEMP_DIR = OUTPUT_DIR / "temp"
31
+
32
+ # Avatar Assets
33
+ AVATAR_ASSETS_DIR = PROJECT_ROOT / "avatar_assets"
34
+ DEFAULT_AVATAR_IMAGE = AVATAR_ASSETS_DIR / "officer.png"
35
+
36
+ # Create directories if they don't exist
37
+ for directory in [OUTPUT_DIR, AUDIO_OUTPUT_DIR, VIDEO_OUTPUT_DIR, TEMP_DIR]:
38
+ directory.mkdir(parents=True, exist_ok=True)
39
+
40
+ # =========================================================================
41
+ # OLLAMA CONFIGURATION
42
+ # =========================================================================
43
+
44
+ # Ollama Settings
45
+ OLLAMA_BASE_URL = "http://localhost:11434"
46
+ OLLAMA_GENERATE_URL = f"{OLLAMA_BASE_URL}/api/generate"
47
+ OLLAMA_MODELS_URL = f"{OLLAMA_BASE_URL}/api/tags"
48
+ OLLAMA_TIMEOUT = 60
49
+
50
+ # Model Configuration
51
+ DEFAULT_MODEL = "mindfull"
52
+ FALLBACK_MODEL = "mistral:7b"
53
+
54
+ # Model Parameters
55
+ MODEL_PARAMS = {
56
+ "temperature": 0.7,
57
+ "top_p": 0.9,
58
+ "top_k": 40,
59
+ "repeat_penalty": 1.1,
60
+ "num_ctx": 2048,
61
+ "stream": False
62
+ }
63
+
64
+ # =========================================================================
65
+ # F5-TTS CONFIGURATION
66
+ # =========================================================================
67
+
68
+ # F5-TTS Model Paths
69
+ F5TTS_MODEL_DIR = PROJECT_ROOT / "my_finetuned_model"
70
+ F5TTS_MODEL_PATH = F5TTS_MODEL_DIR / "model.pth"
71
+ F5TTS_CONFIG_PATH = F5TTS_MODEL_DIR / "config.json"
72
+ F5TTS_VOCAB_PATH = F5TTS_MODEL_DIR / "vocab.json"
73
+
74
+ # Reference Audio for Voice Cloning
75
+ REFERENCE_AUDIO_DIR = PROJECT_ROOT / "datasets-1" / "wavs"
76
+ DEFAULT_REFERENCE_AUDIO = REFERENCE_AUDIO_DIR / "0029.wav"
77
+ REFERENCE_TEXT = "Namaskara! How are you feeling today? I'm here to support you."
78
+
79
+ # TTS Settings
80
+ TTS_SETTINGS = {
81
+ "language": "en",
82
+ "speed": 1.0,
83
+ "remove_silence": True,
84
+ "normalize_audio": True,
85
+ "sample_rate": 24000
86
+ }
87
+
88
+ # =========================================================================
89
+ # SADTALKER CONFIGURATION
90
+ # =========================================================================
91
+
92
+ # SadTalker Paths
93
+ SADTALKER_DIR = PROJECT_ROOT / "sadtalker+wav2lip"
94
+ SADTALKER_SCRIPT = SADTALKER_DIR / "simple_pipeline.py"
95
+ SADTALKER_CHECKPOINTS = SADTALKER_DIR / "sadtalker" / "checkpoints"
96
+
97
+ # SadTalker Settings
98
+ SADTALKER_SETTINGS = {
99
+ "pose_style": 1,
100
+ "exp_scale": 1.3,
101
+ "use_enhancer": True,
102
+ "use_face_parse": True,
103
+ "background_enhance": True,
104
+ "face_restore": True,
105
+ "face_model": "normal",
106
+ "preprocess": "crop",
107
+ "still": False, # *** CRITICAL: This was True causing static images! ***
108
+ "use_idle_mode": False,
109
+ "length_of_audio": 0,
110
+ "use_ref_video": False,
111
+ "ref_video": None,
112
+ "ref_info": None,
113
+ "use_idle_mode": False,
114
+ "length_of_audio": 0
115
+ }
116
+
117
+ # =========================================================================
118
+ # WEB API CONFIGURATION
119
+ # =========================================================================
120
+
121
+ # Web Server Settings
122
+ WEB_HOST = "0.0.0.0"
123
+ WEB_PORT = 5000
124
+ WEB_DEBUG = False
125
+ WEB_THREADED = True
126
+
127
+ # CORS Settings
128
+ CORS_ORIGINS = ["*"]
129
+ CORS_METHODS = ["GET", "POST", "PUT", "DELETE", "OPTIONS"]
130
+ CORS_HEADERS = ["Content-Type", "Authorization"]
131
+
132
+ # =========================================================================
133
+ # SECURITY AND VALIDATION
134
+ # =========================================================================
135
+
136
+ # Input Validation
137
+ MAX_TEXT_LENGTH = 500
138
+ MIN_TEXT_LENGTH = 1
139
+ ALLOWED_LANGUAGES = ["en", "kn"] # English and Kannada
140
+
141
+ # Rate Limiting
142
+ RATE_LIMIT_REQUESTS = 60 # requests per minute
143
+ RATE_LIMIT_WINDOW = 60 # seconds
144
+
145
+ # Session Management
146
+ SESSION_TIMEOUT = 3600 # 1 hour
147
+ MAX_CONCURRENT_SESSIONS = 10
148
+
149
+ # =========================================================================
150
+ # EMOTIONAL ANALYSIS
151
+ # =========================================================================
152
+
153
+ # Emotion Keywords
154
+ EMOTION_KEYWORDS = {
155
+ "stress": [
156
+ "stressed", "pressure", "overwhelmed", "burned out", "exhausted",
157
+ "tense", "worried", "anxious", "strained", "difficult", "hard",
158
+ "challenging", "demanding", "intense", "tough"
159
+ ],
160
+ "sadness": [
161
+ "sad", "depressed", "down", "low", "unhappy", "miserable",
162
+ "lonely", "isolated", "hopeless", "discouraged", "disappointed",
163
+ "upset", "hurt", "broken", "empty"
164
+ ],
165
+ "anger": [
166
+ "angry", "furious", "frustrated", "irritated", "annoyed",
167
+ "mad", "enraged", "outraged", "livid", "aggravated",
168
+ "resentful", "bitter", "hostile"
169
+ ],
170
+ "anxiety": [
171
+ "anxious", "nervous", "panicked", "worried", "fearful",
172
+ "scared", "terrified", "uneasy", "restless", "agitated",
173
+ "on edge", "jumpy", "troubled"
174
+ ],
175
+ "positive": [
176
+ "good", "great", "happy", "fine", "okay", "well", "excellent",
177
+ "fantastic", "wonderful", "amazing", "content", "peaceful",
178
+ "calm", "relaxed", "energetic", "motivated", "confident"
179
+ ]
180
+ }
181
+
182
+ # =========================================================================
183
+ # WELLNESS INTERVENTIONS
184
+ # =========================================================================
185
+
186
+ # Wellness Suggestions
187
+ WELLNESS_INTERVENTIONS = {
188
+ "stress": [
189
+ "Take 5 deep, slow breaths",
190
+ "Step away for a 2-minute break",
191
+ "Drink some water and stay hydrated",
192
+ "Do some shoulder rolls and neck stretches",
193
+ "Focus on one task at a time",
194
+ "Practice the 4-7-8 breathing technique"
195
+ ],
196
+ "sadness": [
197
+ "Remember that your service makes a real difference",
198
+ "Consider talking to a trusted colleague",
199
+ "Reach out to the Well-Being Officer when ready",
200
+ "Take a short walk if possible",
201
+ "Practice self-compassion - you're doing important work",
202
+ "Listen to some calming music during your break"
203
+ ],
204
+ "anger": [
205
+ "Take 10 deep breaths before responding",
206
+ "Count to 10 slowly in your mind",
207
+ "Step away from the situation if safely possible",
208
+ "Use the STOP technique: Stop, Take a breath, Observe, Proceed mindfully",
209
+ "Channel that energy into problem-solving",
210
+ "Remember your training and protocols"
211
+ ],
212
+ "anxiety": [
213
+ "Use the 5-4-3-2-1 grounding technique",
214
+ "Focus on what you can control right now",
215
+ "Practice progressive muscle relaxation",
216
+ "Remind yourself of your training and capabilities",
217
+ "Take slow, controlled breaths",
218
+ "Connect with your support network"
219
+ ],
220
+ "general": [
221
+ "Maintain regular meal times",
222
+ "Stay hydrated throughout your shift",
223
+ "Take brief mental breaks when possible",
224
+ "Practice good posture",
225
+ "Get adequate rest between shifts",
226
+ "Remember your important role in the community"
227
+ ]
228
+ }
229
+
230
+ # =========================================================================
231
+ # LOGGING CONFIGURATION
232
+ # =========================================================================
233
+
234
+ # Logging Settings
235
+ LOG_LEVEL = logging.INFO
236
+ LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
237
+ LOG_FILE = OUTPUT_DIR / "mindfull.log"
238
+
239
+ # =========================================================================
240
+ # ERROR HANDLING
241
+ # =========================================================================
242
+
243
+ # Retry Settings
244
+ MAX_RETRIES = 3
245
+ RETRY_DELAY = 1 # seconds
246
+
247
+ # Fallback Messages
248
+ FALLBACK_MESSAGES = {
249
+ "ollama_error": "I'm having trouble connecting right now. Please try again in a moment.",
250
+ "tts_error": "I'm having trouble with speech generation. Let me try a different approach.",
251
+ "avatar_error": "The video generation is temporarily unavailable, but I can still help you.",
252
+ "general_error": "I encountered an issue, but I'm here to help. Please try again."
253
+ }
254
+
255
+ @classmethod
256
+ def validate_paths(cls) -> bool:
257
+ """Validate that all required paths exist"""
258
+ required_paths = [
259
+ cls.F5TTS_MODEL_PATH,
260
+ cls.F5TTS_CONFIG_PATH,
261
+ cls.DEFAULT_REFERENCE_AUDIO,
262
+ cls.DEFAULT_AVATAR_IMAGE
263
+ ]
264
+
265
+ missing_paths = [path for path in required_paths if not path.exists()]
266
+
267
+ if missing_paths:
268
+ print(f"Warning: Missing required files: {missing_paths}")
269
+ return False
270
+ return True
271
+
272
+ @classmethod
273
+ def get_model_status(cls) -> Dict[str, bool]:
274
+ """Check status of all models"""
275
+ return {
276
+ "f5tts_model": cls.F5TTS_MODEL_PATH.exists(),
277
+ "f5tts_config": cls.F5TTS_CONFIG_PATH.exists(),
278
+ "reference_audio": cls.DEFAULT_REFERENCE_AUDIO.exists(),
279
+ "avatar_image": cls.DEFAULT_AVATAR_IMAGE.exists(),
280
+ "sadtalker_script": cls.SADTALKER_SCRIPT.exists()
281
+ }
282
+
283
+ # Create global config instance
284
+ config = MindfullConfig()
285
+
286
+ # Configure logging
287
+ logging.basicConfig(
288
+ level=config.LOG_LEVEL,
289
+ format=config.LOG_FORMAT,
290
+ handlers=[
291
+ logging.FileHandler(config.LOG_FILE, encoding='utf-8'),
292
+ logging.StreamHandler()
293
+ ]
294
+ )
295
+
296
+ logger = logging.getLogger(__name__)
297
+
298
+ if __name__ == "__main__":
299
+ print("Mindfull Configuration Status:")
300
+ print("=" * 50)
301
+ print(f"Project Root: {config.PROJECT_ROOT}")
302
+ print(f"Python Executable: {config.PYTHON_EXECUTABLE}")
303
+ print(f"Output Directory: {config.OUTPUT_DIR}")
304
+ print("\nModel Status:")
305
+ for model, status in config.get_model_status().items():
306
+ status_icon = "✅" if status else "❌"
307
+ print(f"{status_icon} {model}: {status}")
308
+ print(f"\nPath Validation: {'✅ Passed' if config.validate_paths() else '❌ Failed'}")
mindfull_pipeline.py ADDED
@@ -0,0 +1,478 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Mindfull AI Avatar Chatbot - Main Pipeline
4
+ Integrates Ollama (Mistral), F5-TTS, and SadTalker for complete AI avatar experience
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import json
10
+ import time
11
+ import logging
12
+ import asyncio
13
+ import subprocess
14
+ import threading
15
+ from pathlib import Path
16
+ from typing import Dict, Any, Optional, Tuple
17
+ from dataclasses import dataclass
18
+ from datetime import datetime
19
+
20
+ import requests
21
+ import torch
22
+ import numpy as np
23
+ import soundfile as sf
24
+ from PIL import Image
25
+
26
+ # Import configuration
27
+ from mindfull_config import config, logger
28
+
29
+ @dataclass
30
+ class ChatSession:
31
+ """Represents a chat session with the user"""
32
+ session_id: str
33
+ user_name: str = ""
34
+ user_rank: str = ""
35
+ start_time: Optional[datetime] = None
36
+ last_interaction: Optional[datetime] = None
37
+ conversation_history: Optional[list] = None
38
+
39
+ def __post_init__(self):
40
+ if self.start_time is None:
41
+ self.start_time = datetime.now()
42
+ if self.last_interaction is None:
43
+ self.last_interaction = datetime.now()
44
+ if self.conversation_history is None:
45
+ self.conversation_history = []
46
+
47
+ class MindfullPipeline:
48
+ """Main pipeline for Mindfull AI Avatar Chatbot"""
49
+
50
+ def __init__(self):
51
+ self.session = None
52
+ self.is_initialized = False
53
+ self.ollama_available = False
54
+ self.f5tts_available = False
55
+ self.sadtalker_available = False
56
+
57
+ logger.info("Initializing Mindfull AI Avatar Chatbot...")
58
+ self.initialize_components()
59
+
60
+ def initialize_components(self) -> bool:
61
+ """Initialize all pipeline components"""
62
+ try:
63
+ # Check Ollama availability
64
+ self.ollama_available = self._check_ollama()
65
+
66
+ # Check F5-TTS model availability
67
+ self.f5tts_available = self._check_f5tts()
68
+
69
+ # Check SadTalker availability
70
+ self.sadtalker_available = self._check_sadtalker()
71
+
72
+ # Validate paths
73
+ path_validation = config.validate_paths()
74
+
75
+ self.is_initialized = (
76
+ self.ollama_available and
77
+ self.f5tts_available and
78
+ path_validation
79
+ )
80
+
81
+ if self.is_initialized:
82
+ logger.info("✅ Mindfull pipeline initialized successfully!")
83
+ self._log_component_status()
84
+ else:
85
+ logger.error("❌ Failed to initialize Mindfull pipeline")
86
+ self._log_component_status()
87
+
88
+ return self.is_initialized
89
+
90
+ except Exception as e:
91
+ logger.error(f"Error during initialization: {str(e)}")
92
+ return False
93
+
94
+ def _check_ollama(self) -> bool:
95
+ """Check if Ollama is running and model is available"""
96
+ try:
97
+ # Check if Ollama is running
98
+ response = requests.get(config.OLLAMA_MODELS_URL, timeout=5)
99
+ if response.status_code != 200:
100
+ logger.error("Ollama server is not responding")
101
+ return False
102
+
103
+ # Check if Mindfull model exists
104
+ models = response.json().get('models', [])
105
+ model_names = [model.get('name', '') for model in models]
106
+
107
+ if config.DEFAULT_MODEL in model_names:
108
+ logger.info(f"✅ Found {config.DEFAULT_MODEL} model")
109
+ return True
110
+ elif config.FALLBACK_MODEL in model_names:
111
+ logger.warning(f"⚠️ Using fallback model: {config.FALLBACK_MODEL}")
112
+ return True
113
+ else:
114
+ logger.error(f"❌ Neither {config.DEFAULT_MODEL} nor {config.FALLBACK_MODEL} found")
115
+ logger.info(f"Available models: {model_names}")
116
+ return False
117
+
118
+ except requests.exceptions.RequestException as e:
119
+ logger.error(f"Failed to connect to Ollama: {str(e)}")
120
+ return False
121
+
122
+ def _check_f5tts(self) -> bool:
123
+ """Check if F5-TTS model files are available"""
124
+ required_files = [
125
+ config.F5TTS_MODEL_PATH,
126
+ config.F5TTS_CONFIG_PATH,
127
+ config.DEFAULT_REFERENCE_AUDIO
128
+ ]
129
+
130
+ missing_files = [f for f in required_files if not f.exists()]
131
+
132
+ if missing_files:
133
+ logger.error(f"❌ Missing F5-TTS files: {missing_files}")
134
+ return False
135
+
136
+ logger.info("✅ F5-TTS model files found")
137
+ return True
138
+
139
+ def _check_sadtalker(self) -> bool:
140
+ """Check if SadTalker is available"""
141
+ if not config.SADTALKER_SCRIPT.exists():
142
+ logger.warning("⚠️ SadTalker script not found - video generation disabled")
143
+ return False
144
+
145
+ if not config.DEFAULT_AVATAR_IMAGE.exists():
146
+ logger.warning("⚠️ Default avatar image not found")
147
+ return False
148
+
149
+ logger.info("✅ SadTalker components found")
150
+ return True
151
+
152
+ def _log_component_status(self):
153
+ """Log the status of all components"""
154
+ logger.info("Component Status:")
155
+ logger.info(f" Ollama: {'✅' if self.ollama_available else '❌'}")
156
+ logger.info(f" F5-TTS: {'✅' if self.f5tts_available else '❌'}")
157
+ logger.info(f" SadTalker: {'✅' if self.sadtalker_available else '❌'}")
158
+
159
+ def create_session(self) -> str:
160
+ """Create a new chat session"""
161
+ session_id = f"mindfull_session_{int(time.time())}"
162
+ self.session = ChatSession(session_id=session_id)
163
+ logger.info(f"Created new session: {session_id}")
164
+ return session_id
165
+
166
+ def generate_response(self, user_input: str) -> Tuple[str, str]:
167
+ """
168
+ Generate response using Ollama
169
+ Returns: (response_text, emotion_detected)
170
+ """
171
+ try:
172
+ # Detect emotion from user input
173
+ emotion = self._detect_emotion(user_input)
174
+
175
+ # Prepare the prompt with context
176
+ prompt = self._prepare_prompt(user_input, emotion)
177
+
178
+ # Make request to Ollama
179
+ payload = {
180
+ "model": config.DEFAULT_MODEL,
181
+ "prompt": prompt,
182
+ "stream": False,
183
+ **config.MODEL_PARAMS
184
+ }
185
+
186
+ response = requests.post(
187
+ config.OLLAMA_GENERATE_URL,
188
+ json=payload,
189
+ timeout=config.OLLAMA_TIMEOUT
190
+ )
191
+
192
+ if response.status_code == 200:
193
+ result = response.json()
194
+ response_text = result.get('response', '').strip()
195
+
196
+ # Add to conversation history
197
+ if self.session and self.session.conversation_history is not None:
198
+ self.session.conversation_history.append({
199
+ 'timestamp': datetime.now().isoformat(),
200
+ 'user_input': user_input,
201
+ 'response': response_text,
202
+ 'emotion': emotion
203
+ })
204
+ self.session.last_interaction = datetime.now()
205
+
206
+ logger.info(f"Generated response for emotion '{emotion}': {response_text[:100]}...")
207
+ return response_text, emotion
208
+ else:
209
+ logger.error(f"Ollama request failed: {response.status_code}")
210
+ return config.FALLBACK_MESSAGES["ollama_error"], "neutral"
211
+
212
+ except Exception as e:
213
+ logger.error(f"Error generating response: {str(e)}")
214
+ return config.FALLBACK_MESSAGES["general_error"], "neutral"
215
+
216
+ def _detect_emotion(self, text: str) -> str:
217
+ """Detect emotion from user input using keyword matching"""
218
+ text_lower = text.lower()
219
+
220
+ emotion_scores = {}
221
+ for emotion, keywords in config.EMOTION_KEYWORDS.items():
222
+ score = sum(1 for keyword in keywords if keyword in text_lower)
223
+ if score > 0:
224
+ emotion_scores[emotion] = score
225
+
226
+ if not emotion_scores:
227
+ return "neutral"
228
+
229
+ # Return emotion with highest score
230
+ if emotion_scores:
231
+ detected_emotion = max(emotion_scores, key=lambda x: emotion_scores[x])
232
+ logger.info(f"Detected emotion: {detected_emotion} (scores: {emotion_scores})")
233
+ return detected_emotion
234
+
235
+ def _prepare_prompt(self, user_input: str, emotion: str) -> str:
236
+ """Prepare the prompt for Ollama with context"""
237
+ # Get wellness suggestions for the detected emotion
238
+ suggestions = config.WELLNESS_INTERVENTIONS.get(emotion,
239
+ config.WELLNESS_INTERVENTIONS["general"])
240
+
241
+ context = f"""
242
+ User's emotional state: {emotion}
243
+ Available wellness suggestions: {', '.join(suggestions[:3])}
244
+
245
+ User message: {user_input}
246
+
247
+ Respond as Mindfull, keeping in mind:
248
+ 1. The user's emotional state is {emotion}
249
+ 2. Provide appropriate support and suggestions
250
+ 3. Keep response concise (2-3 sentences) for voice output
251
+ 4. Be warm and professional
252
+ """
253
+ return context
254
+
255
+ def generate_audio(self, text: str) -> Optional[str]:
256
+ """
257
+ Generate audio using F5-TTS with fallback methods
258
+ Returns: path to generated audio file or None if failed
259
+ """
260
+ try:
261
+ # Import our simple audio generation
262
+ from simple_audio_gen import generate_audio_simple, generate_audio_minimal
263
+
264
+ # Prepare output filename
265
+ timestamp = int(time.time())
266
+ audio_filename = f"mindfull_response_{timestamp}.wav"
267
+ audio_path = config.AUDIO_OUTPUT_DIR / audio_filename
268
+
269
+ logger.info(f"Generating audio with F5-TTS: {text[:50]}...")
270
+
271
+ # Try F5-TTS, then minimal fallback
272
+ if generate_audio_simple(text, str(audio_path)):
273
+ logger.info(f"✅ F5-TTS audio generated: {audio_path}")
274
+ return str(audio_path)
275
+ elif generate_audio_minimal(text, str(audio_path)):
276
+ logger.info(f"✅ Minimal audio generated: {audio_path}")
277
+ return str(audio_path)
278
+ else:
279
+ logger.error("All audio generation methods failed")
280
+ return None
281
+
282
+ except Exception as e:
283
+ logger.error(f"Error generating audio: {str(e)}")
284
+ return None
285
+
286
+ def generate_avatar_video(self, audio_path: str,
287
+ avatar_image: Optional[str] = None) -> Optional[str]:
288
+ """
289
+ Generate avatar video using SadTalker with fallback methods
290
+ Returns: path to generated video file or None if failed
291
+ """
292
+ try:
293
+ from simple_video_gen import generate_video_simple
294
+
295
+ if not audio_path or not Path(audio_path).exists():
296
+ logger.error(f"Audio file not found: {audio_path}")
297
+ return None
298
+
299
+ # Use default avatar image if none provided
300
+ if avatar_image is None:
301
+ avatar_image = str(config.DEFAULT_AVATAR_IMAGE)
302
+
303
+ if not Path(avatar_image).exists():
304
+ logger.error(f"Avatar image not found: {avatar_image}")
305
+ return None
306
+
307
+ # Prepare output filename
308
+ timestamp = int(time.time())
309
+ video_filename = f"mindfull_avatar_{timestamp}.mp4"
310
+ video_path = config.VIDEO_OUTPUT_DIR / video_filename
311
+
312
+ logger.info(f"Generating avatar video: {video_filename}")
313
+
314
+ # Use our simple video generation system
315
+ if generate_video_simple(avatar_image, audio_path, str(video_path)):
316
+ logger.info(f"✅ Avatar video generated: {video_path}")
317
+ return str(video_path) # Return the path on success
318
+ else:
319
+ logger.error("Video generation failed")
320
+ return None
321
+
322
+ except Exception as e:
323
+ logger.error(f"Error generating avatar video: {str(e)}")
324
+ return None
325
+
326
+ def process_complete_interaction(self, user_input: str,
327
+ avatar_image: Optional[str] = None) -> Dict[str, Any]:
328
+ """
329
+ Process complete interaction: text -> response -> audio -> video
330
+ Returns: dictionary with all generated content paths and metadata
331
+ """
332
+ start_time = time.time()
333
+ result = {
334
+ "session_id": self.session.session_id if self.session else None,
335
+ "user_input": user_input,
336
+ "timestamp": datetime.now().isoformat(),
337
+ "response_text": None,
338
+ "emotion_detected": None,
339
+ "audio_path": None,
340
+ "video_path": None,
341
+ "processing_time": 0,
342
+ "errors": []
343
+ }
344
+
345
+ try:
346
+ # Step 1: Generate text response
347
+ logger.info("Step 1: Generating text response...")
348
+ response_text, emotion = self.generate_response(user_input)
349
+ result["response_text"] = response_text
350
+ result["emotion_detected"] = emotion
351
+
352
+ if not response_text or response_text in config.FALLBACK_MESSAGES.values():
353
+ result["errors"].append("Failed to generate proper response")
354
+ return result
355
+
356
+ # Step 2: Generate audio
357
+ logger.info("Step 2: Generating audio...")
358
+ audio_path = self.generate_audio(response_text)
359
+ result["audio_path"] = audio_path
360
+
361
+ if not audio_path:
362
+ result["errors"].append("Failed to generate audio")
363
+
364
+ # Step 3: Generate avatar video (optional)
365
+ if self.sadtalker_available and audio_path:
366
+ logger.info("Step 3: Generating avatar video...")
367
+ video_path = self.generate_avatar_video(audio_path, avatar_image)
368
+ result["video_path"] = video_path
369
+
370
+ if not video_path:
371
+ result["errors"].append("Failed to generate avatar video")
372
+ else:
373
+ logger.info("Step 3: Skipping avatar video generation")
374
+
375
+ result["processing_time"] = time.time() - start_time
376
+ logger.info(f"✅ Complete interaction processed in {result['processing_time']:.2f}s")
377
+
378
+ except Exception as e:
379
+ logger.error(f"Error in complete interaction: {str(e)}")
380
+ result["errors"].append(f"Processing error: {str(e)}")
381
+ result["processing_time"] = time.time() - start_time
382
+
383
+ return result
384
+
385
+ def get_session_info(self) -> Optional[Dict[str, Any]]:
386
+ """Get current session information"""
387
+ if not self.session or not self.session.start_time or not self.session.last_interaction:
388
+ return None
389
+
390
+ return {
391
+ "session_id": self.session.session_id,
392
+ "user_name": self.session.user_name,
393
+ "user_rank": self.session.user_rank,
394
+ "start_time": self.session.start_time.isoformat(),
395
+ "last_interaction": self.session.last_interaction.isoformat(),
396
+ "conversation_count": len(self.session.conversation_history or [])
397
+ }
398
+
399
+ def cleanup_old_files(self, max_age_hours: int = 24):
400
+ """Clean up old generated files"""
401
+ try:
402
+ current_time = time.time()
403
+ cutoff_time = current_time - (max_age_hours * 3600)
404
+
405
+ for directory in [config.AUDIO_OUTPUT_DIR, config.VIDEO_OUTPUT_DIR, config.TEMP_DIR]:
406
+ if directory.exists():
407
+ for file_path in directory.iterdir():
408
+ if file_path.is_file() and file_path.stat().st_mtime < cutoff_time:
409
+ file_path.unlink()
410
+ logger.info(f"Cleaned up old file: {file_path}")
411
+
412
+ except Exception as e:
413
+ logger.error(f"Error during cleanup: {str(e)}")
414
+
415
+ def main():
416
+ """Main function for testing the pipeline"""
417
+ print("🤖 Mindfull AI Avatar Chatbot")
418
+ print("=" * 50)
419
+
420
+ # Initialize pipeline
421
+ pipeline = MindfullPipeline()
422
+
423
+ if not pipeline.is_initialized:
424
+ print("❌ Failed to initialize pipeline. Please check the logs.")
425
+ return
426
+
427
+ # Create session
428
+ session_id = pipeline.create_session()
429
+ print(f"Session ID: {session_id}")
430
+
431
+ # Interactive loop
432
+ print("\n💬 Chat with Mindfull (type 'quit' to exit)")
433
+ print("Note: This will generate audio and video responses")
434
+ print("-" * 50)
435
+
436
+ while True:
437
+ try:
438
+ user_input = input("\nYou: ").strip()
439
+
440
+ if user_input.lower() in ['quit', 'exit', 'bye']:
441
+ print("👋 Goodbye! Take care!")
442
+ break
443
+
444
+ if not user_input:
445
+ continue
446
+
447
+ print("\n🔄 Processing your message...")
448
+
449
+ # Process complete interaction
450
+ result = pipeline.process_complete_interaction(user_input)
451
+
452
+ # Display results
453
+ print(f"\n🤖 Mindfull: {result['response_text']}")
454
+ print(f"😊 Emotion detected: {result['emotion_detected']}")
455
+ print(f"⏱️ Processing time: {result['processing_time']:.2f}s")
456
+
457
+ if result['audio_path']:
458
+ print(f"🔊 Audio saved: {result['audio_path']}")
459
+
460
+ if result['video_path']:
461
+ print(f"🎥 Video saved: {result['video_path']}")
462
+
463
+ if result['errors']:
464
+ print(f"⚠️ Errors: {', '.join(result['errors'])}")
465
+
466
+ except KeyboardInterrupt:
467
+ print("\n\n👋 Goodbye! Take care!")
468
+ break
469
+ except Exception as e:
470
+ print(f"\n❌ Error: {str(e)}")
471
+
472
+ # Cleanup
473
+ print("\n🧹 Cleaning up old files...")
474
+ pipeline.cleanup_old_files()
475
+ print("✅ Cleanup complete")
476
+
477
+ if __name__ == "__main__":
478
+ main()
mindfull_web_api.py ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Mindfull AI Avatar Chatbot - Web API
4
+ Flask web server for serving the Mindfull pipeline via HTTP API
5
+ """
6
+
7
+ import os
8
+ import json
9
+ import uuid
10
+ import threading
11
+ from datetime import datetime
12
+ from pathlib import Path
13
+
14
+ from flask import Flask, request, jsonify, send_file, send_from_directory
15
+ from flask_cors import CORS
16
+ import logging
17
+
18
+ # Import the pipeline
19
+ from mindfull_pipeline import MindfullPipeline
20
+ from mindfull_config import config, logger
21
+
22
+ # Create Flask app
23
+ app = Flask(__name__)
24
+ CORS(app, origins=config.CORS_ORIGINS)
25
+
26
+ # Global pipeline instance
27
+ pipeline = None
28
+ active_sessions = {}
29
+ session_lock = threading.Lock()
30
+
31
+ def initialize_app():
32
+ """Initialize the Flask application"""
33
+ global pipeline
34
+
35
+ logger.info("Initializing Mindfull Web API...")
36
+
37
+ # Initialize pipeline
38
+ pipeline = MindfullPipeline()
39
+
40
+ if not pipeline.is_initialized:
41
+ logger.error("Failed to initialize Mindfull pipeline")
42
+ return False
43
+
44
+ logger.info("✅ Mindfull Web API initialized successfully")
45
+ return True
46
+
47
+ @app.route('/health', methods=['GET'])
48
+ def health_check():
49
+ """Health check endpoint"""
50
+ if not pipeline or not pipeline.is_initialized:
51
+ return jsonify({
52
+ 'status': 'error',
53
+ 'message': 'Pipeline not initialized'
54
+ }), 503
55
+
56
+ return jsonify({
57
+ 'status': 'healthy',
58
+ 'timestamp': datetime.now().isoformat(),
59
+ 'components': {
60
+ 'ollama': pipeline.ollama_available,
61
+ 'f5tts': pipeline.f5tts_available,
62
+ 'sadtalker': pipeline.sadtalker_available
63
+ }
64
+ })
65
+
66
+ @app.route('/session', methods=['POST'])
67
+ def create_session():
68
+ """Create a new chat session"""
69
+ try:
70
+ session_id = pipeline.create_session()
71
+
72
+ with session_lock:
73
+ active_sessions[session_id] = {
74
+ 'created': datetime.now().isoformat(),
75
+ 'last_activity': datetime.now().isoformat()
76
+ }
77
+
78
+ return jsonify({
79
+ 'status': 'success',
80
+ 'session_id': session_id,
81
+ 'message': 'Session created successfully'
82
+ })
83
+
84
+ except Exception as e:
85
+ logger.error(f"Error creating session: {str(e)}")
86
+ return jsonify({
87
+ 'status': 'error',
88
+ 'message': 'Failed to create session'
89
+ }), 500
90
+
91
+ @app.route('/session/<session_id>', methods=['GET'])
92
+ def get_session_info(session_id):
93
+ """Get session information"""
94
+ try:
95
+ if session_id not in active_sessions:
96
+ return jsonify({
97
+ 'status': 'error',
98
+ 'message': 'Session not found'
99
+ }), 404
100
+
101
+ session_info = pipeline.get_session_info()
102
+
103
+ return jsonify({
104
+ 'status': 'success',
105
+ 'session_info': session_info
106
+ })
107
+
108
+ except Exception as e:
109
+ logger.error(f"Error getting session info: {str(e)}")
110
+ return jsonify({
111
+ 'status': 'error',
112
+ 'message': 'Failed to get session info'
113
+ }), 500
114
+
115
+ @app.route('/chat', methods=['POST'])
116
+ def chat():
117
+ """Main chat endpoint"""
118
+ try:
119
+ data = request.get_json()
120
+
121
+ if not data or 'message' not in data:
122
+ return jsonify({
123
+ 'status': 'error',
124
+ 'message': 'Missing message in request'
125
+ }), 400
126
+
127
+ user_message = data['message'].strip()
128
+ session_id = data.get('session_id')
129
+ include_video = data.get('include_video', True)
130
+ avatar_image = data.get('avatar_image') # Optional custom avatar
131
+
132
+ # Validate input
133
+ if len(user_message) < config.MIN_TEXT_LENGTH:
134
+ return jsonify({
135
+ 'status': 'error',
136
+ 'message': 'Message too short'
137
+ }), 400
138
+
139
+ if len(user_message) > config.MAX_TEXT_LENGTH:
140
+ return jsonify({
141
+ 'status': 'error',
142
+ 'message': 'Message too long'
143
+ }), 400
144
+
145
+ # Update session activity
146
+ if session_id and session_id in active_sessions:
147
+ with session_lock:
148
+ active_sessions[session_id]['last_activity'] = datetime.now().isoformat()
149
+
150
+ # Process the interaction
151
+ if include_video:
152
+ result = pipeline.process_complete_interaction(user_message, avatar_image)
153
+ else:
154
+ # Text and audio only
155
+ response_text, emotion = pipeline.generate_response(user_message)
156
+ audio_path = pipeline.generate_audio(response_text)
157
+
158
+ result = {
159
+ 'session_id': session_id,
160
+ 'user_input': user_message,
161
+ 'timestamp': datetime.now().isoformat(),
162
+ 'response_text': response_text,
163
+ 'emotion_detected': emotion,
164
+ 'audio_path': audio_path,
165
+ 'video_path': None,
166
+ 'processing_time': 0,
167
+ 'errors': []
168
+ }
169
+
170
+ # Prepare response
171
+ response_data = {
172
+ 'status': 'success',
173
+ 'session_id': result['session_id'],
174
+ 'response_text': result['response_text'],
175
+ 'emotion_detected': result['emotion_detected'],
176
+ 'processing_time': result['processing_time'],
177
+ 'timestamp': result['timestamp']
178
+ }
179
+
180
+ # Add file URLs if files were generated
181
+ if result['audio_path']:
182
+ audio_filename = Path(result['audio_path']).name
183
+ response_data['audio_url'] = f'/audio/{audio_filename}'
184
+
185
+ if result['video_path']:
186
+ video_filename = Path(result['video_path']).name
187
+ response_data['video_url'] = f'/video/{video_filename}'
188
+
189
+ if result['errors']:
190
+ response_data['warnings'] = result['errors']
191
+
192
+ return jsonify(response_data)
193
+
194
+ except Exception as e:
195
+ logger.error(f"Error in chat endpoint: {str(e)}")
196
+ return jsonify({
197
+ 'status': 'error',
198
+ 'message': 'Internal server error'
199
+ }), 500
200
+
201
+ @app.route('/audio/<filename>')
202
+ def serve_audio(filename):
203
+ """Serve audio files"""
204
+ try:
205
+ return send_from_directory(config.AUDIO_OUTPUT_DIR, filename)
206
+ except FileNotFoundError:
207
+ return jsonify({'error': 'Audio file not found'}), 404
208
+
209
+ @app.route('/video/<filename>')
210
+ def serve_video(filename):
211
+ """Serve video files"""
212
+ try:
213
+ return send_from_directory(config.VIDEO_OUTPUT_DIR, filename)
214
+ except FileNotFoundError:
215
+ return jsonify({'error': 'Video file not found'}), 404
216
+
217
+ @app.route('/models', methods=['GET'])
218
+ def get_model_status():
219
+ """Get status of all models"""
220
+ try:
221
+ model_status = config.get_model_status()
222
+
223
+ return jsonify({
224
+ 'status': 'success',
225
+ 'models': model_status,
226
+ 'pipeline_status': {
227
+ 'ollama': pipeline.ollama_available if pipeline else False,
228
+ 'f5tts': pipeline.f5tts_available if pipeline else False,
229
+ 'sadtalker': pipeline.sadtalker_available if pipeline else False
230
+ }
231
+ })
232
+
233
+ except Exception as e:
234
+ logger.error(f"Error getting model status: {str(e)}")
235
+ return jsonify({
236
+ 'status': 'error',
237
+ 'message': 'Failed to get model status'
238
+ }), 500
239
+
240
+ @app.route('/cleanup', methods=['POST'])
241
+ def cleanup_files():
242
+ """Clean up old generated files"""
243
+ try:
244
+ data = request.get_json() or {}
245
+ max_age_hours = data.get('max_age_hours', 24)
246
+
247
+ if pipeline:
248
+ pipeline.cleanup_old_files(max_age_hours)
249
+
250
+ return jsonify({
251
+ 'status': 'success',
252
+ 'message': f'Cleaned up files older than {max_age_hours} hours'
253
+ })
254
+
255
+ except Exception as e:
256
+ logger.error(f"Error during cleanup: {str(e)}")
257
+ return jsonify({
258
+ 'status': 'error',
259
+ 'message': 'Cleanup failed'
260
+ }), 500
261
+
262
+ @app.route('/config', methods=['GET'])
263
+ def get_config():
264
+ """Get configuration information"""
265
+ return jsonify({
266
+ 'status': 'success',
267
+ 'config': {
268
+ 'max_text_length': config.MAX_TEXT_LENGTH,
269
+ 'min_text_length': config.MIN_TEXT_LENGTH,
270
+ 'allowed_languages': config.ALLOWED_LANGUAGES,
271
+ 'session_timeout': config.SESSION_TIMEOUT,
272
+ 'available_emotions': list(config.EMOTION_KEYWORDS.keys()),
273
+ 'default_model': config.DEFAULT_MODEL,
274
+ 'fallback_model': config.FALLBACK_MODEL
275
+ }
276
+ })
277
+
278
+ @app.route('/', methods=['GET'])
279
+ def index():
280
+ """Main index page"""
281
+ return jsonify({
282
+ 'service': 'Mindfull AI Avatar Chatbot',
283
+ 'version': '1.0.0',
284
+ 'status': 'running',
285
+ 'endpoints': {
286
+ 'health': '/health',
287
+ 'create_session': '/session [POST]',
288
+ 'chat': '/chat [POST]',
289
+ 'get_config': '/config [GET]',
290
+ 'model_status': '/models [GET]',
291
+ 'cleanup': '/cleanup [POST]'
292
+ },
293
+ 'documentation': 'See API documentation for usage details'
294
+ })
295
+
296
+ @app.errorhandler(404)
297
+ def not_found(error):
298
+ return jsonify({'error': 'Endpoint not found'}), 404
299
+
300
+ @app.errorhandler(500)
301
+ def internal_error(error):
302
+ return jsonify({'error': 'Internal server error'}), 500
303
+
304
+ def run_server(host=None, port=None, debug=None):
305
+ """Run the Flask server"""
306
+ host = host or config.WEB_HOST
307
+ port = port or config.WEB_PORT
308
+ debug = debug or config.WEB_DEBUG
309
+
310
+ logger.info(f"Starting Mindfull Web API server on {host}:{port}")
311
+
312
+ app.run(
313
+ host=host,
314
+ port=port,
315
+ debug=debug,
316
+ threaded=config.WEB_THREADED
317
+ )
318
+
319
+ if __name__ == '__main__':
320
+ if initialize_app():
321
+ run_server()
322
+ else:
323
+ logger.error("Failed to initialize application")
324
+ exit(1)
model_manager.py ADDED
@@ -0,0 +1,252 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Model Manager for Police Bot AI Runtime
4
+ Helps manage and switch between different Ollama models
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import requests
10
+ import json
11
+ from config import config
12
+
13
+ def print_banner():
14
+ """Print banner"""
15
+ print("=" * 60)
16
+ print("🤖 Police Bot AI - Model Manager")
17
+ print("=" * 60)
18
+
19
+ def check_ollama_status():
20
+ """Check if Ollama is running and get available models"""
21
+ try:
22
+ response = requests.get("http://localhost:11434/api/tags", timeout=5)
23
+ if response.status_code == 200:
24
+ data = response.json()
25
+ models = data.get("models", [])
26
+ return True, models
27
+ else:
28
+ return False, []
29
+ except:
30
+ return False, []
31
+
32
+ def list_available_models():
33
+ """List all available models in Ollama"""
34
+ print("\n📋 Available Models in Ollama:")
35
+ print("-" * 40)
36
+
37
+ ollama_running, models = check_ollama_status()
38
+
39
+ if not ollama_running:
40
+ print("❌ Ollama is not running")
41
+ print("Please start Ollama first: ollama serve")
42
+ return []
43
+
44
+ if not models:
45
+ print("⚠️ No models found in Ollama")
46
+ print("You can pull models using:")
47
+ print(" ollama pull mistral")
48
+ print(" ollama pull llama3")
49
+ print(" ollama pull police-bot")
50
+ return []
51
+
52
+ available_models = []
53
+ for i, model in enumerate(models, 1):
54
+ model_name = model.get("name", "Unknown")
55
+ model_size = model.get("size", 0)
56
+ size_mb = model_size / (1024 * 1024) if model_size > 0 else 0
57
+
58
+ print(f"{i:2d}. {model_name:<20} ({size_mb:.0f} MB)")
59
+ available_models.append(model_name)
60
+
61
+ return available_models
62
+
63
+ def list_configured_models():
64
+ """List models configured in the system"""
65
+ print("\n⚙️ Configured Models in System:")
66
+ print("-" * 40)
67
+
68
+ configured_models = config.list_available_models()
69
+ current_model = config.OLLAMA_MODEL
70
+
71
+ for i, model in enumerate(configured_models, 1):
72
+ status = "✓" if model == current_model else " "
73
+ print(f"{i:2d}. [{status}] {model}")
74
+
75
+ return configured_models
76
+
77
+ def switch_model():
78
+ """Switch to a different model"""
79
+ print("\n🔄 Switch Model")
80
+ print("-" * 40)
81
+
82
+ configured_models = config.list_available_models()
83
+ current_model = config.OLLAMA_MODEL
84
+
85
+ print(f"Current model: {current_model}")
86
+ print("\nAvailable models:")
87
+
88
+ for i, model in enumerate(configured_models, 1):
89
+ print(f"{i}. {model}")
90
+
91
+ try:
92
+ choice = input(f"\nEnter choice (1-{len(configured_models)}): ").strip()
93
+ if choice:
94
+ model_index = int(choice) - 1
95
+ if 0 <= model_index < len(configured_models):
96
+ selected_model = configured_models[model_index]
97
+
98
+ # Check if model is available in Ollama
99
+ ollama_running, models = check_ollama_status()
100
+ if ollama_running:
101
+ model_names = [m.get("name") for m in models]
102
+ if selected_model not in model_names:
103
+ print(f"⚠️ Warning: {selected_model} is not available in Ollama")
104
+ pull = input("Would you like to pull it? (y/n): ").lower().strip()
105
+ if pull == 'y':
106
+ print(f"Pulling {selected_model}...")
107
+ os.system(f"ollama pull {selected_model}")
108
+
109
+ # Switch model
110
+ if config.switch_model(selected_model):
111
+ print(f"✅ Switched to {selected_model}")
112
+
113
+ # Save to environment file
114
+ save_model_to_env(selected_model)
115
+ else:
116
+ print(f"❌ Failed to switch to {selected_model}")
117
+ else:
118
+ print("❌ Invalid choice")
119
+ else:
120
+ print("❌ No choice made")
121
+ except ValueError:
122
+ print("❌ Invalid input")
123
+
124
+ def save_model_to_env(model_name):
125
+ """Save model selection to .env file"""
126
+ env_file = ".env"
127
+ env_content = f"OLLAMA_MODEL={model_name}\n"
128
+
129
+ try:
130
+ with open(env_file, 'w') as f:
131
+ f.write(env_content)
132
+ print(f"✅ Model selection saved to {env_file}")
133
+ except Exception as e:
134
+ print(f"⚠️ Could not save to {env_file}: {e}")
135
+
136
+ def test_model():
137
+ """Test the current model"""
138
+ print("\n🧪 Test Current Model")
139
+ print("-" * 40)
140
+
141
+ current_model = config.OLLAMA_MODEL
142
+ print(f"Testing model: {current_model}")
143
+
144
+ # Check if Ollama is running
145
+ ollama_running, models = check_ollama_status()
146
+ if not ollama_running:
147
+ print("❌ Ollama is not running")
148
+ return
149
+
150
+ # Check if model is available
151
+ model_names = [m.get("name") for m in models]
152
+ if current_model not in model_names:
153
+ print(f"❌ {current_model} is not available in Ollama")
154
+ print(f"Available models: {', '.join(model_names)}")
155
+ return
156
+
157
+ # Test the model
158
+ try:
159
+ from police_runtime import PoliceBotRuntime
160
+
161
+ runtime = PoliceBotRuntime()
162
+ test_prompt = "Hello, how are you today?"
163
+
164
+ print(f"Sending test prompt: '{test_prompt}'")
165
+ response = runtime.get_llama3_response(test_prompt)
166
+
167
+ if response:
168
+ print(f"✅ Model responded: {response[:100]}...")
169
+ else:
170
+ print("❌ No response from model")
171
+
172
+ except Exception as e:
173
+ print(f"❌ Error testing model: {e}")
174
+
175
+ def show_model_info():
176
+ """Show information about the current model"""
177
+ print("\nℹ️ Current Model Information")
178
+ print("-" * 40)
179
+
180
+ current_model = config.OLLAMA_MODEL
181
+ model_config = config.get_current_model_config()
182
+
183
+ print(f"Model: {current_model}")
184
+ print(f"Temperature: {model_config['temperature']}")
185
+ print(f"Top P: {model_config['top_p']}")
186
+ print(f"Max Tokens: {model_config['max_tokens']}")
187
+
188
+ print(f"\nSystem Prompt Preview:")
189
+ print("-" * 20)
190
+ prompt_preview = model_config['system_prompt'][:200] + "..." if len(model_config['system_prompt']) > 200 else model_config['system_prompt']
191
+ print(prompt_preview)
192
+
193
+ def pull_model():
194
+ """Pull a model from Ollama"""
195
+ print("\n📥 Pull Model from Ollama")
196
+ print("-" * 40)
197
+
198
+ model_name = input("Enter model name to pull (e.g., mistral, llama3): ").strip()
199
+ if model_name:
200
+ print(f"Pulling {model_name}...")
201
+ result = os.system(f"ollama pull {model_name}")
202
+
203
+ if result == 0:
204
+ print(f"✅ Successfully pulled {model_name}")
205
+ else:
206
+ print(f"❌ Failed to pull {model_name}")
207
+ else:
208
+ print("❌ No model name provided")
209
+
210
+ def main():
211
+ """Main function"""
212
+ print_banner()
213
+
214
+ while True:
215
+ print("\nOptions:")
216
+ print("1. List available models in Ollama")
217
+ print("2. List configured models")
218
+ print("3. Switch model")
219
+ print("4. Test current model")
220
+ print("5. Show model information")
221
+ print("6. Pull model from Ollama")
222
+ print("7. Exit")
223
+
224
+ try:
225
+ choice = input("\nEnter choice (1-7): ").strip()
226
+
227
+ if choice == '1':
228
+ list_available_models()
229
+ elif choice == '2':
230
+ list_configured_models()
231
+ elif choice == '3':
232
+ switch_model()
233
+ elif choice == '4':
234
+ test_model()
235
+ elif choice == '5':
236
+ show_model_info()
237
+ elif choice == '6':
238
+ pull_model()
239
+ elif choice == '7':
240
+ print("Goodbye! 👋")
241
+ break
242
+ else:
243
+ print("❌ Invalid choice")
244
+
245
+ except KeyboardInterrupt:
246
+ print("\nGoodbye! 👋")
247
+ break
248
+ except Exception as e:
249
+ print(f"❌ Error: {e}")
250
+
251
+ if __name__ == "__main__":
252
+ main()
my_finetuned_model/config.json ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_path": "checkpoints/",
3
+ "logger_uri": null,
4
+ "run_name": "GPT_XTTS_FT",
5
+ "project_name": "XTTS_trainer",
6
+ "run_description": [
7
+ "\n GPT XTTS training\n "
8
+ ],
9
+ "print_step": 50,
10
+ "plot_step": 100,
11
+ "model_param_stats": false,
12
+ "wandb_entity": null,
13
+ "dashboard_logger": "tensorboard",
14
+ "save_on_interrupt": true,
15
+ "log_model_step": 100,
16
+ "save_step": 1,
17
+ "save_n_checkpoints": 1,
18
+ "save_checkpoints": true,
19
+ "save_all_best": false,
20
+ "save_best_after": 10000,
21
+ "target_loss": null,
22
+ "print_eval": false,
23
+ "test_delay_epochs": 0,
24
+ "run_eval": true,
25
+ "run_eval_steps": null,
26
+ "distributed_backend": "nccl",
27
+ "distributed_url": "tcp://localhost:54321",
28
+ "mixed_precision": false,
29
+ "precision": "fp16",
30
+ "epochs": 3,
31
+ "batch_size": 16,
32
+ "eval_batch_size": 16,
33
+ "grad_clip": 0.0,
34
+ "scheduler_after_epoch": true,
35
+ "lr": 2e-05,
36
+ "optimizer": "AdamW",
37
+ "optimizer_params": {
38
+ "betas": [
39
+ 0.9,
40
+ 0.96
41
+ ],
42
+ "eps": 1e-08,
43
+ "weight_decay": 0.01
44
+ },
45
+ "lr_scheduler": "MultiStepLR",
46
+ "lr_scheduler_params": {
47
+ "milestones": [
48
+ 900000,
49
+ 2700000,
50
+ 5400000
51
+ ],
52
+ "gamma": 0.5,
53
+ "last_epoch": -1
54
+ },
55
+ "use_grad_scaler": false,
56
+ "allow_tf32": false,
57
+ "cudnn_enable": true,
58
+ "cudnn_deterministic": false,
59
+ "cudnn_benchmark": false,
60
+ "training_seed": 54321,
61
+ "model": "xtts",
62
+ "num_loader_workers": 8,
63
+ "num_eval_loader_workers": 0,
64
+ "use_noise_augment": false,
65
+ "audio": {
66
+ "sample_rate": 22050,
67
+ "output_sample_rate": 24000,
68
+ "dvae_sample_rate": 22050
69
+ },
70
+ "use_phonemes": false,
71
+ "phonemizer": null,
72
+ "phoneme_language": null,
73
+ "compute_input_seq_cache": false,
74
+ "text_cleaner": null,
75
+ "enable_eos_bos_chars": false,
76
+ "test_sentences_file": "",
77
+ "phoneme_cache_path": null,
78
+ "characters": null,
79
+ "add_blank": false,
80
+ "batch_group_size": 0,
81
+ "loss_masking": null,
82
+ "min_audio_len": 1,
83
+ "max_audio_len": Infinity,
84
+ "min_text_len": 1,
85
+ "max_text_len": Infinity,
86
+ "compute_f0": false,
87
+ "compute_energy": false,
88
+ "compute_linear_spec": false,
89
+ "precompute_num_workers": 0,
90
+ "start_by_longest": false,
91
+ "shuffle": false,
92
+ "drop_last": false,
93
+ "datasets": [
94
+ {
95
+ "formatter": "",
96
+ "dataset_name": "",
97
+ "path": "",
98
+ "meta_file_train": "",
99
+ "ignored_speakers": null,
100
+ "language": "",
101
+ "phonemizer": "",
102
+ "meta_file_val": "",
103
+ "meta_file_attn_mask": ""
104
+ }
105
+ ],
106
+ "test_sentences": [],
107
+ "eval_split_max_size": 256,
108
+ "eval_split_size": 0.01,
109
+ "use_speaker_weighted_sampler": false,
110
+ "speaker_weighted_sampler_alpha": 1.0,
111
+ "use_language_weighted_sampler": false,
112
+ "language_weighted_sampler_alpha": 1.0,
113
+ "use_length_weighted_sampler": false,
114
+ "length_weighted_sampler_alpha": 1.0,
115
+ "model_args": {
116
+ "gpt_batch_size": 1,
117
+ "enable_redaction": false,
118
+ "kv_cache": true,
119
+ "gpt_checkpoint": "",
120
+ "clvp_checkpoint": null,
121
+ "decoder_checkpoint": null,
122
+ "num_chars": 255,
123
+ "tokenizer_file": "checkpoints/XTTS_v2.0_original_model_files/vocab.json",
124
+ "gpt_max_audio_tokens": 605,
125
+ "gpt_max_text_tokens": 402,
126
+ "gpt_max_prompt_tokens": 70,
127
+ "gpt_layers": 30,
128
+ "gpt_n_model_channels": 1024,
129
+ "gpt_n_heads": 16,
130
+ "gpt_number_text_tokens": 6925,
131
+ "gpt_start_text_token": 261,
132
+ "gpt_stop_text_token": 0,
133
+ "gpt_num_audio_tokens": 1026,
134
+ "gpt_start_audio_token": 1024,
135
+ "gpt_stop_audio_token": 1025,
136
+ "gpt_code_stride_len": 1024,
137
+ "gpt_use_masking_gt_prompt_approach": true,
138
+ "gpt_use_perceiver_resampler": true,
139
+ "input_sample_rate": 22050,
140
+ "output_sample_rate": 24000,
141
+ "output_hop_length": 256,
142
+ "decoder_input_dim": 1024,
143
+ "d_vector_dim": 512,
144
+ "cond_d_vector_in_each_upsampling_layer": true,
145
+ "duration_const": 102400,
146
+ "min_conditioning_length": 11025,
147
+ "max_conditioning_length": 132300,
148
+ "gpt_loss_text_ce_weight": 0.01,
149
+ "gpt_loss_mel_ce_weight": 1.0,
150
+ "debug_loading_failures": false,
151
+ "max_wav_length": 330750,
152
+ "max_text_length": 400,
153
+ "mel_norm_file": "checkpoints/XTTS_v2.0_original_model_files/mel_stats.pth",
154
+ "dvae_checkpoint": "checkpoints/XTTS_v2.0_original_model_files/dvae.pth",
155
+ "xtts_checkpoint": "checkpoints/XTTS_v2.0_original_model_files/model.pth",
156
+ "vocoder": ""
157
+ },
158
+ "model_dir": null,
159
+ "languages": [
160
+ "en",
161
+ "es",
162
+ "fr",
163
+ "de",
164
+ "it",
165
+ "pt",
166
+ "pl",
167
+ "tr",
168
+ "ru",
169
+ "nl",
170
+ "cs",
171
+ "ar",
172
+ "zh-cn",
173
+ "hu",
174
+ "ko",
175
+ "ja",
176
+ "hi",
177
+ "en"
178
+ ],
179
+ "temperature": 0.75,
180
+ "length_penalty": 1.0,
181
+ "repetition_penalty": 5.0,
182
+ "top_k": 50,
183
+ "top_p": 0.85,
184
+ "num_gpt_outputs": 1,
185
+ "gpt_cond_len": 30,
186
+ "gpt_cond_chunk_len": 4,
187
+ "max_ref_len": 30,
188
+ "sound_norm_refs": false,
189
+ "optimizer_wd_only_on_weights": true,
190
+ "weighted_loss_attrs": null,
191
+ "weighted_loss_multipliers": null,
192
+ "github_branch": "* main"
193
+ }
my_finetuned_model/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
ollama_to_f5tts.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ import subprocess
3
+
4
+ OLLAMA_URL = "http://localhost:11434/api/generate"
5
+ F5TTS_CKPT = r"C:\Users\Samarth Kadam\Voice-train\F5-TTS\ckpts\my_speak\model_last.pt"
6
+ REF_AUDIO = r"C:\Users\Samarth Kadam\Voice-train\F5-TTS\data\my_speak_pinyin\wavs\segment_43.wav"
7
+ REF_TEXT = "are you feeling mentally alert?"
8
+ OUTPUT_DIR = "outputs"
9
+
10
+ def get_policebot_reply(prompt):
11
+ data = {
12
+ "model": "police-bot",
13
+ "prompt": prompt,
14
+ "stream": False
15
+ }
16
+ response = requests.post(OLLAMA_URL, json=data)
17
+ return response.json()["response"].strip()
18
+
19
+ def synthesize_voice(text):
20
+ cmd = [
21
+ "python", r"C:\Users\Samarth Kadam\Voice-train\F5-TTS\src\f5_tts\infer\infer_cli.py",
22
+ "--ckpt_file", F5TTS_CKPT,
23
+ "--ref_audio", REF_AUDIO,
24
+ "--ref_text", REF_TEXT,
25
+ "--gen_text", text,
26
+ "--output_dir", OUTPUT_DIR,
27
+ "--remove_silence",
28
+ "--speed", "1.0"
29
+ ]
30
+ subprocess.run(cmd)
31
+
32
+ if __name__ == "__main__":
33
+ user_prompt = input("Enter your prompt for police-bot: ")
34
+ reply = get_policebot_reply(user_prompt)
35
+ print("Police-bot reply:", reply)
36
+ synthesize_voice(reply)
37
+ print("Audio generated in outputs folder.")
python313_compat_patch.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Python 3.13 Compatibility Patch
3
+ Fixes pkgutil.ImpImporter issues for AI/ML packages
4
+ """
5
+
6
+ import sys
7
+ import pkgutil
8
+ import importlib
9
+ import importlib.util
10
+ import warnings
11
+ from typing import Any, Optional, Union
12
+
13
+
14
+ class ImpImporter:
15
+ """
16
+ Compatibility shim for pkgutil.ImpImporter removed in Python 3.12+
17
+ This fixes issues with packages that still try to use the old imp module
18
+ """
19
+
20
+ def __init__(self, path: Optional[str] = None):
21
+ self.path = path
22
+
23
+ def find_module(self, fullname: str, path=None):
24
+ """Find module using modern importlib instead of deprecated imp"""
25
+ try:
26
+ if isinstance(path, list) and path:
27
+ package = path[0] if path else None
28
+ else:
29
+ package = None
30
+ spec = importlib.util.find_spec(fullname, package)
31
+ if spec is not None:
32
+ return self
33
+ except (ImportError, ValueError, AttributeError):
34
+ pass
35
+ return None
36
+
37
+ def load_module(self, fullname: str):
38
+ """Load module using modern importlib"""
39
+ try:
40
+ spec = importlib.util.find_spec(fullname)
41
+ if spec is not None:
42
+ module = importlib.util.module_from_spec(spec)
43
+ sys.modules[fullname] = module
44
+ if spec.loader:
45
+ spec.loader.exec_module(module)
46
+ return module
47
+ except Exception as e:
48
+ warnings.warn(f"Failed to load module {fullname}: {e}")
49
+ raise ImportError(f"No module named '{fullname}'")
50
+
51
+
52
+ def apply_python313_patches():
53
+ """Apply all necessary patches for Python 3.13 compatibility"""
54
+
55
+ # Patch 1: Add missing pkgutil.ImpImporter
56
+ if not hasattr(pkgutil, 'ImpImporter'):
57
+ # Dynamically add the attribute
58
+ setattr(pkgutil, 'ImpImporter', ImpImporter)
59
+ print("✓ Applied pkgutil.ImpImporter compatibility patch")
60
+
61
+ # Patch 2: Fix pkg_resources for newer setuptools
62
+ try:
63
+ import pkg_resources
64
+ # Monkey patch any problematic pkg_resources functionality
65
+ if not hasattr(pkg_resources, '_initialize_master_working_set'):
66
+ def _initialize_master_working_set():
67
+ """Dummy implementation for missing function"""
68
+ pass
69
+ pkg_resources._initialize_master_working_set = _initialize_master_working_set
70
+ print("✓ Applied pkg_resources compatibility patch")
71
+ except ImportError:
72
+ print("! pkg_resources not available - installing alternative")
73
+
74
+ # Patch 3: Fix distutils issues
75
+ try:
76
+ # Check if distutils is available
77
+ import setuptools._distutils
78
+ if 'distutils' not in sys.modules:
79
+ sys.modules['distutils'] = setuptools._distutils
80
+ print("✓ Applied distutils compatibility patch")
81
+ except ImportError:
82
+ print("! distutils patches not needed")
83
+
84
+ # Patch 4: Fix collections.abc imports
85
+ try:
86
+ import collections
87
+ from collections.abc import Iterable, Mapping, MutableMapping
88
+ if not hasattr(collections, 'Iterable'):
89
+ setattr(collections, 'Iterable', Iterable)
90
+ setattr(collections, 'Mapping', Mapping)
91
+ setattr(collections, 'MutableMapping', MutableMapping)
92
+ print("✓ Applied collections.abc compatibility patch")
93
+ except (ImportError, AttributeError):
94
+ pass
95
+
96
+ # Patch 5: numpy compatibility
97
+ try:
98
+ import numpy as np
99
+ # Fix numpy version issues by adding fallback attributes
100
+ if not hasattr(np, 'bool'):
101
+ setattr(np, 'bool', bool)
102
+ if not hasattr(np, 'int'):
103
+ setattr(np, 'int', int)
104
+ if not hasattr(np, 'float'):
105
+ setattr(np, 'float', float)
106
+ if not hasattr(np, 'complex'):
107
+ setattr(np, 'complex', complex)
108
+ print("✓ Applied numpy compatibility patch")
109
+ except ImportError:
110
+ print("! numpy not available for patching")
111
+
112
+ print("✓ All Python 3.13 compatibility patches applied successfully")
113
+
114
+
115
+ if __name__ == "__main__":
116
+ apply_python313_patches()
requirements.txt ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # HuggingFace Space requirements
2
+ # Audio is handled by edge-tts (no GPU needed)
3
+ # LLM falls back to HF Inference API when Ollama is absent
4
+ # Video (SadTalker) is optional and disabled by default on Spaces
5
+
6
+ # ── core ──────────────────────────────────────────────────────────────────────
7
+ gradio>=5.0.0
8
+ numpy>=1.24.0,<2.0.0
9
+ Pillow>=9.0.0
10
+ requests>=2.31.0
11
+
12
+ # ── TTS ───────────────────────────────────────────────────────────────────────
13
+ edge-tts>=6.1.9
14
+
15
+ # ── LLM (HF Inference API fallback) ──────────────────────────────────────────
16
+ huggingface_hub>=0.23.0
17
+ transformers>=4.40.0
18
+ accelerate>=0.30.0
19
+
20
+ # ── video (SadTalker / Wav2Lip) - comment out if not deploying video ──────────
21
+ # torch>=2.0.0
22
+ # torchvision>=0.15.0
23
+ # torchaudio>=2.0.0
24
+ # opencv-python-headless>=4.8.0
25
+ # scipy>=1.9.0
26
+ # safetensors>=0.4.0
27
+ # imageio>=2.28.0
28
+ # imageio-ffmpeg>=0.4.8
requirements_full.txt ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # =============================================================================
2
+ # Police Bot AI Runtime - Complete Requirements
3
+ # Supports: Ollama (Mistral), F5-TTS, SadTalker, Web API
4
+ # Optimized for GPU usage and virtual environment
5
+ # =============================================================================
6
+
7
+ # Core AI and ML Framework
8
+ torch>=2.0.0
9
+ torchvision>=0.15.0
10
+ torchaudio>=2.0.0
11
+ numpy>=1.21.0,<=1.26.4
12
+ scipy>=1.9.0
13
+
14
+ # F5-TTS Dependencies
15
+ accelerate>=0.33.0
16
+ bitsandbytes>0.37.0
17
+ cached_path>=1.1.3
18
+ click>=8.1.0
19
+ datasets>=2.14.0
20
+ ema_pytorch>=0.5.2
21
+ gradio>=3.45.2
22
+ hydra-core>=1.3.0
23
+ jieba>=0.42.1
24
+ librosa>=0.10.0
25
+ matplotlib>=3.7.0
26
+ pydub>=0.25.1
27
+ pypinyin>=0.49.0
28
+ safetensors>=0.3.0
29
+ soundfile>=0.12.1
30
+ tomli>=2.0.1
31
+ torchdiffeq>=0.2.3
32
+ tqdm>=4.65.0
33
+ transformers>=4.30.0
34
+ x_transformers>=1.31.14
35
+ packaging>=24.2
36
+
37
+ # Ollama API and HTTP Client
38
+ requests>=2.31.0
39
+ httpx>=0.24.0
40
+
41
+ # Web API Framework
42
+ flask>=2.3.0
43
+ flask-cors>=4.0.0
44
+ gunicorn>=21.0.0
45
+ werkzeug>=2.3.0
46
+
47
+ # Audio Processing
48
+ playsound>=1.3.0
49
+ pyaudio>=0.2.11
50
+ webrtcvad>=2.0.10
51
+
52
+ # SadTalker Dependencies (if using avatar)
53
+ opencv-python>=4.8.0
54
+ scikit-image>=0.21.0
55
+ scikit-learn>=1.3.0
56
+ face-alignment>=1.3.5
57
+ gfpgan>=1.3.8
58
+ basicsr>=1.4.2
59
+ facexlib>=0.3.0
60
+ realesrgan>=0.3.0
61
+ gfpgan>=1.3.8
62
+ codeformer>=0.1.0
63
+
64
+ # Video Processing (for SadTalker)
65
+ ffmpeg-python>=0.2.0
66
+ moviepy>=1.0.3
67
+ imageio>=2.31.0
68
+ imageio-ffmpeg>=0.4.8
69
+
70
+ # Utilities and Environment
71
+ python-dotenv>=1.0.0
72
+ psutil>=5.9.0
73
+ pyyaml>=6.0
74
+ tqdm>=4.65.0
75
+ pillow>=9.5.0
76
+ seaborn>=0.12.0
77
+
78
+ # Development and Testing (optional)
79
+ pytest>=7.4.0
80
+ pytest-cov>=4.1.0
81
+ black>=23.0.0
82
+ flake8>=6.0.0
83
+
84
+ # GPU Optimization (optional - uncomment if needed)
85
+ # apex>=0.1.0 # For mixed precision training
86
+ # deepspeed>=0.9.0 # For distributed training
87
+
88
+ # =============================================================================
89
+ # Installation Notes:
90
+ # =============================================================================
91
+ # 1. For GPU support, ensure CUDA is installed and compatible with PyTorch
92
+ # 2. Some packages may require system dependencies (ffmpeg, portaudio)
93
+ # 3. For Windows: pip install -r requirements.txt
94
+ # 4. For Linux: pip install -r requirements.txt
95
+ # 5. For development: pip install -r requirements.txt[dev]
96
+ # =============================================================================
97
+
98
+ # =============================================================================
99
+ # System Dependencies (install separately):
100
+ # =============================================================================
101
+ # Windows:
102
+ # - Visual Studio Build Tools
103
+ # - FFmpeg (add to PATH)
104
+ # - PortAudio (for pyaudio)
105
+
106
+ # Linux (Ubuntu/Debian):
107
+ # sudo apt-get update
108
+ # sudo apt-get install -y ffmpeg portaudio19-dev python3-dev
109
+
110
+ # macOS:
111
+ # brew install ffmpeg portaudio
112
+ # =============================================================================
requirements_hf.txt ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # HuggingFace Space requirements
2
+ # Audio is handled by edge-tts (no GPU needed)
3
+ # LLM falls back to HF Inference API when Ollama is absent
4
+ # Video (SadTalker) is optional and disabled by default on Spaces
5
+
6
+ # ── core ──────────────────────────────────────────────────────────────────────
7
+ gradio>=5.0.0
8
+ numpy>=1.24.0,<2.0.0
9
+ Pillow>=9.0.0
10
+ requests>=2.31.0
11
+
12
+ # ── TTS ───────────────────────────────────────────────────────────────────────
13
+ edge-tts>=6.1.9
14
+
15
+ # ── LLM (HF Inference API fallback) ──────────────────────────────────────────
16
+ huggingface_hub>=0.23.0
17
+ transformers>=4.40.0
18
+ accelerate>=0.30.0
19
+
20
+ # ── video (SadTalker / Wav2Lip) - comment out if not deploying video ──────────
21
+ # torch>=2.0.0
22
+ # torchvision>=0.15.0
23
+ # torchaudio>=2.0.0
24
+ # opencv-python-headless>=4.8.0
25
+ # scipy>=1.9.0
26
+ # safetensors>=0.4.0
27
+ # imageio>=2.28.0
28
+ # imageio-ffmpeg>=0.4.8
requirements_unified.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # =============================================================================
2
+ # Mindfull AI Avatar Chatbot - Unified Requirements
3
+ # Supports: Ollama (Mistral/LLaMA), F5-TTS, SadTalker + Wav2Lip
4
+ # Optimized for GPU usage and virtual environment
5
+ # =============================================================================
6
+
7
+ # Core Deep Learning Framework
8
+ torch>=2.0.0
9
+ torchvision>=0.15.0
10
+ torchaudio>=2.0.0
11
+ numpy>=1.21.0,<=1.26.4
12
+ scipy>=1.9.0
13
+
14
+ # Transformers and ML Core
15
+ transformers>=4.30.0
16
+ accelerate>=0.33.0
17
+ safetensors>=0.3.0
18
+ huggingface-hub>=0.16.0
19
+
20
+ # F5-TTS Dependencies
21
+ cached_path>=1.1.3
22
+ datasets>=2.14.0
23
+ ema_pytorch>=0.5.2
24
+ hydra-core>=1.3.0
25
+ jieba>=0.42.1
26
+ pypinyin>=0.49.0
27
+ torchdiffeq>=0.2.3
28
+ x_transformers>=1.31.14
29
+ packaging>=24.2
30
+
31
+ # Audio Processing
32
+ librosa>=0.10.0
33
+ soundfile>=0.12.1
34
+ pydub>=0.25.1
35
+ pyaudio>=0.2.11
36
+ webrtcvad>=2.0.10
37
+ playsound>=1.3.0
38
+
39
+ # Computer Vision & Image Processing (SadTalker)
40
+ opencv-python>=4.8.0
41
+ pillow>=9.5.0
42
+ imageio>=2.31.0
43
+ imageio-ffmpeg>=0.4.8
44
+ scikit-image>=0.21.0
45
+ scikit-learn>=1.3.0
46
+
47
+ # Face Processing and Enhancement
48
+ face-alignment>=1.3.5
49
+ gfpgan>=1.3.8
50
+ basicsr>=1.4.2
51
+ facexlib>=0.3.0
52
+ realesrgan>=0.3.0
53
+
54
+ # Video Processing
55
+ ffmpeg-python>=0.2.0
56
+ moviepy>=1.0.3
57
+
58
+ # Web API Framework
59
+ flask>=2.3.0
60
+ flask-cors>=4.0.0
61
+ gunicorn>=21.0.0
62
+ werkzeug>=2.3.0
63
+ flask-socketio>=5.3.0
64
+
65
+ # HTTP Clients and API
66
+ requests>=2.31.0
67
+ httpx>=0.24.0
68
+
69
+ # Utilities and Environment
70
+ python-dotenv>=1.0.0
71
+ psutil>=5.9.0
72
+ pyyaml>=6.0
73
+ tqdm>=4.65.0
74
+ matplotlib>=3.7.0
75
+ seaborn>=0.12.0
76
+ click>=8.1.0
77
+ gradio>=3.45.2
78
+
79
+ # Development and Testing
80
+ pytest>=7.4.0
81
+ black>=23.0.0
82
+ flake8>=6.0.0
83
+
84
+ # =============================================================================
85
+ # Installation Notes:
86
+ # =============================================================================
87
+ # 1. Install system dependencies first:
88
+ # - FFmpeg (add to PATH)
89
+ # - Visual Studio Build Tools (Windows)
90
+ # - CUDA Toolkit (for GPU support)
91
+ #
92
+ # 2. Install requirements:
93
+ # pip install -r requirements_unified.txt
94
+ #
95
+ # 3. For Windows specific audio:
96
+ # pip install pywin32
97
+ # =============================================================================
sadtalker+wav2lip/README.md ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SadTalker + Wav2Lip Pipeline
2
+
3
+ Clean pipeline for talking avatar generation, ready for F5-TTS integration.
4
+
5
+ ## Purpose
6
+ This pipeline converts a reference image + audio into a talking head video with synchronized lip movements.
7
+
8
+ ## Quick Start
9
+
10
+ ```bash
11
+ # Install dependencies
12
+ pip install -r requirements.txt
13
+
14
+ # Basic usage
15
+ python simple_single_command.py --source_image avatar_assets/officer.png --audio avatar_assets/voice.wav --output result.mp4
16
+
17
+ # ASCII-safe version (no special characters)
18
+ python simple_single_command_ascii.py --source_image avatar_assets/officer.png --audio avatar_assets/voice.wav --output result.mp4
19
+ ```
20
+
21
+ ## File Structure
22
+
23
+ ```
24
+ sadtalker+wav2lip/
25
+ ├── simple_single_command.py # Main generation script
26
+ ├── simple_single_command_ascii.py # ASCII-safe version
27
+ ├── simple_pipeline.py # Pipeline utilities
28
+ ├── avatar_assets/ # Input images and audio
29
+ ├── sadtalker/ # SadTalker model files
30
+ ├── requirements.txt # Dependencies
31
+ └── README.md # This file
32
+ ```
33
+
34
+ ## F5-TTS Integration
35
+
36
+ This pipeline is designed to work with F5-TTS:
37
+
38
+ 1. **F5-TTS**: Text → Audio (speech synthesis)
39
+ 2. **This Pipeline**: Image + Audio → Talking Video
40
+
41
+ Combined workflow:
42
+ ```
43
+ Text → [F5-TTS] → Audio → [SadTalker+Wav2Lip] → Talking Video
44
+ ```
45
+
46
+ ## Requirements
47
+
48
+ - Python 3.8+
49
+ - CUDA GPU (recommended for faster processing)
50
+ - 4GB+ GPU memory
51
+ - FFmpeg for video processing
52
+
53
+ ## Example Usage
54
+
55
+ ```python
56
+ # Integration example with F5-TTS
57
+ text = "Hello, I am a police officer. How can I help you today?"
58
+ image = "avatar_assets/officer.png"
59
+
60
+ # Step 1: Generate speech with F5-TTS
61
+ audio_file = f5_tts_generate(text)
62
+
63
+ # Step 2: Generate talking video
64
+ video_file = sadtalker_generate(image, audio_file)
65
+ ```
66
+
67
+ ## Ready for Production
68
+
69
+ This cleaned pipeline contains only essential files and is optimized for:
70
+ - Fast processing
71
+ - Minimal dependencies
72
+ - Easy integration
73
+ - Reliable output
74
+
75
+ Perfect for integration with F5-TTS or other text-to-speech systems.
sadtalker+wav2lip/README_POLICE_BOT_AI.md ADDED
@@ -0,0 +1,199 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Police Bot AI - Avatar Chatbot System
2
+ ## Complete Open-Source AI Avatar Pipeline
3
+
4
+ ### 🎯 Project Overview
5
+ This is a fully open-source AI avatar chatbot system that combines:
6
+ - **Mistral** (Ollama) for reasoning and text generation
7
+ - **F5-TTS** for high-quality text-to-speech
8
+ - **SadTalker** for talking head video generation
9
+ - **Wav2Lip** for precise lip synchronization
10
+
11
+ ### 🔄 Complete Pipeline Workflow
12
+
13
+ ```
14
+ User Input → Mistral (Reasoning) → Response Text → F5-TTS (Audio) → SadTalker (Video) → Wav2Lip (Lipsync) → Final Avatar Video
15
+ ```
16
+
17
+ #### Detailed Steps:
18
+ 1. **User Input**: Text message from user
19
+ 2. **Mistral Processing**: Generate contextual, intelligent response
20
+ 3. **F5-TTS Conversion**: Convert response text to high-quality audio
21
+ 4. **SadTalker Generation**: Create talking head video from static image + audio
22
+ 5. **Wav2Lip Enhancement**: Improve lip synchronization accuracy
23
+ 6. **Output**: Final synchronized avatar video
24
+
25
+ ### 🛡️ Security & Privacy Considerations
26
+
27
+ #### **Data Privacy**
28
+ - ✅ **Local Processing**: All AI processing happens locally, no cloud dependencies
29
+ - ✅ **No Data Collection**: No user data is stored or transmitted
30
+ - ✅ **Open Source**: All components are open-source and auditable
31
+ - ✅ **Offline Capable**: System works completely offline
32
+
33
+ #### **Model Security**
34
+ - ✅ **Verified Sources**: All models from official repositories
35
+ - ✅ **No External APIs**: No reliance on external services
36
+ - ✅ **Controlled Environment**: All processing within your infrastructure
37
+
38
+ #### **Content Safety**
39
+ - ⚠️ **Input Validation**: Implement input sanitization for user messages
40
+ - ⚠️ **Content Filtering**: Add content moderation for inappropriate responses
41
+ - ⚠️ **Rate Limiting**: Implement request throttling to prevent abuse
42
+
43
+ ### 📁 File Structure & Integration
44
+
45
+ #### **Core Components**
46
+ ```
47
+ police_bot_ai/
48
+ ├── sadtalker/ # SadTalker integration
49
+ │ ├── src/ # Core SadTalker code
50
+ │ ├── checkpoints/ # Model files (essential!)
51
+ │ ├── gfpgan/weights/ # Face enhancement models
52
+ │ └── inference.py # Main inference script
53
+ ├── wav2lip/ # Wav2Lip integration
54
+ │ ├── inference.py # Patched inference script
55
+ │ ├── audio.py # Patched audio processing
56
+ │ └── models/ # Wav2Lip model files
57
+ ├── f5_tts/ # F5-TTS integration
58
+ ├── mistral/ # Mistral/Ollama integration
59
+ ├── avatar_assets/ # Static images for avatars
60
+ ├── output/ # Generated videos
61
+ └── main_pipeline.py # Unified pipeline script
62
+ ```
63
+
64
+ ### 🔧 Technical Requirements
65
+
66
+ #### **System Requirements**
67
+ - **OS**: Windows 10/11, Linux, or macOS
68
+ - **GPU**: NVIDIA GPU with CUDA support (recommended)
69
+ - **RAM**: 16GB+ (32GB recommended)
70
+ - **Storage**: 50GB+ for models and temporary files
71
+
72
+ #### **Software Dependencies**
73
+ - **Python**: 3.10+ (3.11 recommended)
74
+ - **CUDA**: 11.8+ (for GPU acceleration)
75
+ - **FFmpeg**: For video processing
76
+ - **PyTorch**: 2.0+ with CUDA support
77
+
78
+ ### 🚀 Implementation Guide
79
+
80
+ #### **Phase 1: Setup & Testing**
81
+ 1. Install all dependencies from `requirements.txt`
82
+ 2. Test individual components (Mistral, F5-TTS, SadTalker, Wav2Lip)
83
+ 3. Verify GPU compatibility and performance
84
+
85
+ #### **Phase 2: Integration**
86
+ 1. Create unified pipeline script
87
+ 2. Implement error handling and fallbacks
88
+ 3. Add input validation and content filtering
89
+
90
+ #### **Phase 3: Optimization**
91
+ 1. Implement caching for common responses
92
+ 2. Add video streaming capabilities
93
+ 3. Optimize for real-time performance
94
+
95
+ #### **Phase 4: Deployment**
96
+ 1. Set up web API interface
97
+ 2. Implement user authentication
98
+ 3. Add monitoring and logging
99
+
100
+ ### 🔄 Real-Time Workflow
101
+
102
+ #### **For Live Chatbot:**
103
+ 1. **User sends message** → Web interface
104
+ 2. **Mistral generates response** (1-3 seconds)
105
+ 3. **F5-TTS creates audio** (2-5 seconds)
106
+ 4. **SadTalker generates video** (10-30 seconds)
107
+ 5. **Wav2Lip improves sync** (5-15 seconds)
108
+ 6. **Stream video to user** (real-time)
109
+
110
+ #### **Optimization Strategies:**
111
+ - **Pre-generate** common responses and greetings
112
+ - **Cache** avatar videos for frequent phrases
113
+ - **Parallel processing** where possible
114
+ - **Progressive loading** for better UX
115
+
116
+ ### 🎨 Avatar Customization
117
+
118
+ #### **Supported Formats:**
119
+ - **Images**: PNG, JPG (high quality, front-facing)
120
+ - **Audio**: WAV (16kHz, mono recommended)
121
+ - **Video**: MP4 (H.264, 25fps)
122
+
123
+ #### **Best Practices:**
124
+ - Use high-resolution, well-lit face images
125
+ - Ensure face is centered and front-facing
126
+ - Avoid extreme expressions or angles
127
+ - Use clear, noise-free audio
128
+
129
+ ### 🔍 Troubleshooting
130
+
131
+ #### **Common Issues:**
132
+ 1. **CUDA Out of Memory**: Reduce batch sizes or use CPU
133
+ 2. **Poor Lip Sync**: Use F5-TTS audio instead of recorded audio
134
+ 3. **Blurry Output**: Try Wav2Lip GAN model for better quality
135
+ 4. **Slow Processing**: Enable GPU acceleration and optimize settings
136
+
137
+ #### **Performance Tips:**
138
+ - Use SSD storage for faster I/O
139
+ - Enable CUDA memory optimization
140
+ - Implement video caching
141
+ - Use lower resolution for faster processing
142
+
143
+ ### 📊 Monitoring & Maintenance
144
+
145
+ #### **Health Checks:**
146
+ - Monitor GPU memory usage
147
+ - Track processing times
148
+ - Check disk space for temporary files
149
+ - Verify model file integrity
150
+
151
+ #### **Regular Maintenance:**
152
+ - Update dependencies monthly
153
+ - Clean temporary files weekly
154
+ - Backup avatar assets
155
+ - Monitor system performance
156
+
157
+ ### 🔮 Future Enhancements
158
+
159
+ #### **Planned Features:**
160
+ - **Multi-language support** with language detection
161
+ - **Emotion recognition** for contextual responses
162
+ - **Voice cloning** for personalized avatars
163
+ - **Real-time video streaming** with WebRTC
164
+ - **Mobile app** for remote access
165
+
166
+ #### **Advanced Capabilities:**
167
+ - **Multi-avatar support** for different scenarios
168
+ - **Gesture recognition** for more natural interactions
169
+ - **Background customization** for different environments
170
+ - **Voice emotion synthesis** for expressive responses
171
+
172
+ ### 📞 Support & Community
173
+
174
+ #### **Resources:**
175
+ - **Documentation**: Check individual component READMEs
176
+ - **Issues**: Report bugs in respective repositories
177
+ - **Discussions**: Join open-source AI communities
178
+
179
+ #### **Contributing:**
180
+ - All components are open-source
181
+ - Contributions welcome for improvements
182
+ - Follow security best practices
183
+ - Test thoroughly before submitting
184
+
185
+ ---
186
+
187
+ ## 🚨 Important Notes
188
+
189
+ 1. **This is a research/demo system** - not for production use without additional security measures
190
+ 2. **Test thoroughly** before deployment in sensitive environments
191
+ 3. **Monitor resource usage** - AI models can be resource-intensive
192
+ 4. **Keep dependencies updated** for security patches
193
+ 5. **Backup important data** regularly
194
+
195
+ ---
196
+
197
+ *Last Updated: July 2025*
198
+ *Version: 1.0.0*
199
+ *License: Open Source (Various - see individual components)*
sadtalker+wav2lip/enhanced_pipeline.py ADDED
@@ -0,0 +1,231 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Enhanced Video Pipeline - SadTalker + Wav2Lip Integration
4
+ Optimized for preserving facial animations while enhancing lip sync
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import argparse
10
+ import subprocess
11
+ import shutil
12
+ from pathlib import Path
13
+ import cv2
14
+ import numpy as np
15
+
16
+ def run_command(command, description):
17
+ """Run a command and handle errors"""
18
+ print(f"\n[PROCESSING] {description}...")
19
+ print(f"Command: {' '.join(command)}")
20
+
21
+ try:
22
+ result = subprocess.run(command, check=True, capture_output=True, text=True, shell=True)
23
+ print(f"[OK] {description} completed successfully!")
24
+ if result.stdout and len(result.stdout) > 50:
25
+ print("Output:", result.stdout[-200:]) # Show last 200 chars
26
+ return True
27
+ except subprocess.CalledProcessError as e:
28
+ print(f"[ERROR] {description} failed!")
29
+ print(f"Error: {e.stderr}")
30
+ if e.stdout:
31
+ print(f"Output: {e.stdout}")
32
+ return False
33
+
34
+ def verify_video_animation(video_path, min_frame_diff_threshold=0.01):
35
+ """Verify if video contains actual animation by checking frame differences"""
36
+ try:
37
+ cap = cv2.VideoCapture(video_path)
38
+ if not cap.isOpened():
39
+ return False
40
+
41
+ frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
42
+ if frame_count < 2:
43
+ cap.release()
44
+ return False
45
+
46
+ # Read first frame
47
+ ret, prev_frame = cap.read()
48
+ if not ret:
49
+ cap.release()
50
+ return False
51
+
52
+ prev_gray = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY)
53
+
54
+ # Check a few frames for differences
55
+ significant_changes = 0
56
+ frames_checked = min(10, frame_count - 1)
57
+
58
+ for i in range(frames_checked):
59
+ ret, curr_frame = cap.read()
60
+ if not ret:
61
+ break
62
+
63
+ curr_gray = cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY)
64
+ diff = cv2.absdiff(prev_gray, curr_gray)
65
+
66
+ # Calculate mean difference
67
+ mean_diff = np.mean(diff) / 255.0
68
+
69
+ if mean_diff > min_frame_diff_threshold:
70
+ significant_changes += 1
71
+
72
+ prev_gray = curr_gray
73
+
74
+ cap.release()
75
+
76
+ # If at least 20% of checked frames show significant changes, consider it animated (lowered from 30%)
77
+ animation_ratio = significant_changes / frames_checked
78
+ is_animated = animation_ratio > 0.2 # Lowered threshold for talking head videos
79
+
80
+ print(f"[INFO] Animation analysis: {significant_changes}/{frames_checked} frames with changes ({animation_ratio:.2%})")
81
+ print(f"[VIDEO] Video is {'animated' if is_animated else 'static'}")
82
+
83
+ return is_animated
84
+
85
+ except Exception as e:
86
+ print(f"[WARNING] Error analyzing video animation: {e}")
87
+ return False
88
+
89
+ def main():
90
+ parser = argparse.ArgumentParser(description='Enhanced Video Pipeline - SadTalker + Wav2Lip')
91
+
92
+ parser.add_argument('--source_image', type=str, required=True,
93
+ help='Path to source image (avatar)')
94
+ parser.add_argument('--audio', type=str, required=True,
95
+ help='Path to audio file')
96
+ parser.add_argument('--output', type=str, required=True,
97
+ help='Path for final output video')
98
+ parser.add_argument('--use_wav2lip', action='store_true',
99
+ help='Use Wav2Lip enhancement (only if SadTalker lip sync is poor)')
100
+ parser.add_argument('--force_wav2lip', action='store_true',
101
+ help='Force Wav2Lip even if SadTalker animation is good')
102
+ parser.add_argument('--temp_dir', type=str, default='temp_enhanced_pipeline',
103
+ help='Temporary directory for intermediate files')
104
+
105
+ args = parser.parse_args()
106
+
107
+ print("[VIDEO] Enhanced Video Pipeline - SadTalker + Wav2Lip Integration")
108
+ print("=" * 60)
109
+ print(f"Source Image: {args.source_image}")
110
+ print(f"Audio File: {args.audio}")
111
+ print(f"Final Output: {args.output}")
112
+ print(f"Use Wav2Lip: {args.use_wav2lip or args.force_wav2lip}")
113
+ print("=" * 60)
114
+
115
+ # Verify input files
116
+ if not os.path.exists(args.source_image):
117
+ print(f"[ERROR] Source image not found: {args.source_image}")
118
+ return
119
+
120
+ if not os.path.exists(args.audio):
121
+ print(f"[ERROR] Audio file not found: {args.audio}")
122
+ return
123
+
124
+ # Get current directory and set up paths
125
+ current_dir = os.path.dirname(os.path.abspath(__file__))
126
+ temp_dir = os.path.join(current_dir, args.temp_dir)
127
+ os.makedirs(temp_dir, exist_ok=True)
128
+
129
+ # Set up component paths
130
+ sadtalker_script = os.path.join(current_dir, 'sadtalker', 'inference.py')
131
+ sadtalker_output_dir = os.path.join(temp_dir, 'sadtalker_output')
132
+ sadtalker_checkpoints = os.path.join(current_dir, 'sadtalker', 'checkpoints')
133
+
134
+ wav2lip_script = os.path.join(current_dir, 'wav2lip', 'inference.py')
135
+ wav2lip_model = os.path.join(current_dir, 'wav2lip', 'Wav2Lip-SD-NOGAN.pt')
136
+
137
+ # Step 1: Run SadTalker with enhanced animation parameters
138
+ print("\n[ANIMATION] STEP 1: SadTalker Animation Generation")
139
+ sadtalker_cmd = [
140
+ sys.executable, sadtalker_script,
141
+ '--driven_audio', args.audio,
142
+ '--source_image', args.source_image,
143
+ '--result_dir', sadtalker_output_dir,
144
+ '--checkpoint_dir', sadtalker_checkpoints,
145
+ '--size', '256',
146
+ '--preprocess', 'crop',
147
+ '--expression_scale', '1.5', # Enhanced expression
148
+ '--pose_style', '1', # Natural head movement
149
+ '--batch_size', '1', # Better quality
150
+ '--verbose'
151
+ ]
152
+
153
+ if not run_command(sadtalker_cmd, "SadTalker animation generation"):
154
+ print("[ERROR] SadTalker failed!")
155
+ return
156
+
157
+ # Find SadTalker output
158
+ sadtalker_video = None
159
+ for root, dirs, files in os.walk(sadtalker_output_dir):
160
+ for file in files:
161
+ if file.endswith('.mp4'):
162
+ sadtalker_video = os.path.join(root, file)
163
+ break
164
+ if sadtalker_video:
165
+ break
166
+
167
+ if not sadtalker_video or not os.path.exists(sadtalker_video):
168
+ print("[ERROR] SadTalker output not found!")
169
+ return
170
+
171
+ print(f"[OK] SadTalker output: {sadtalker_video}")
172
+ file_size = os.path.getsize(sadtalker_video)
173
+ print(f"[FILE] File size: {file_size:,} bytes")
174
+
175
+ # Step 2: Analyze SadTalker animation quality
176
+ print("\n[ANALYSIS] STEP 2: Analyzing Animation Quality")
177
+ is_animated = verify_video_animation(sadtalker_video)
178
+
179
+ # Step 3: Mux audio into SadTalker output directly (skip Wav2Lip - resolution issues)
180
+ print("\n[AUDIO] STEP 3: Muxing audio into animated video")
181
+ audio_muxed = os.path.join(temp_dir, 'final_with_audio.mp4')
182
+ ffmpeg_cmd = [
183
+ 'ffmpeg', '-y',
184
+ '-i', sadtalker_video,
185
+ '-i', os.path.abspath(args.audio),
186
+ '-c:v', 'libx264',
187
+ '-c:a', 'aac',
188
+ '-pix_fmt', 'yuv420p',
189
+ '-movflags', '+faststart',
190
+ '-shortest',
191
+ audio_muxed
192
+ ]
193
+ mux_result = subprocess.run(ffmpeg_cmd, capture_output=True, text=True)
194
+ if mux_result.returncode == 0 and os.path.exists(audio_muxed):
195
+ final_video = audio_muxed
196
+ print(f"[OK] Audio muxed successfully")
197
+ else:
198
+ print(f"[WARNING] ffmpeg mux failed: {mux_result.stderr[-300:]}")
199
+ final_video = sadtalker_video # silent fallback
200
+
201
+ # Step 4: Copy final output
202
+ print("\n[FILE] STEP 4: Finalizing Output")
203
+ try:
204
+ shutil.copy2(final_video, args.output)
205
+ final_size = os.path.getsize(args.output)
206
+ print(f"[OK] Final video saved to: {args.output}")
207
+ print(f"[FILE] Final file size: {final_size:,} bytes")
208
+
209
+ # Final verification
210
+ final_is_animated = verify_video_animation(args.output)
211
+ if final_is_animated:
212
+ print("[SUCCESS] Final video contains proper animation!")
213
+ else:
214
+ print("[WARNING] Warning: Final video appears to be static")
215
+
216
+ except Exception as e:
217
+ print(f"[ERROR] Error copying final video: {e}")
218
+ return
219
+
220
+ # Cleanup
221
+ try:
222
+ shutil.rmtree(temp_dir)
223
+ print("[CLEANUP] Cleaned up temporary files")
224
+ except Exception as e:
225
+ print(f"[WARNING] Could not clean up temporary files: {e}")
226
+
227
+ print("\n[SUCCESS] Enhanced Pipeline Completed Successfully!")
228
+ print(f"[VIDEO] Final output: {args.output}")
229
+
230
+ if __name__ == "__main__":
231
+ main()
sadtalker+wav2lip/init_path.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import glob
3
+
4
+ def init_path(checkpoint_dir, config_dir, size=512, old_version=False, preprocess='crop'):
5
+
6
+ if old_version:
7
+ #### load all the checkpoint of `pth`
8
+ sadtalker_paths = {
9
+ 'wav2lip_checkpoint' : os.path.join(checkpoint_dir, 'wav2lip.pth'),
10
+ 'audio2pose_checkpoint' : os.path.join(checkpoint_dir, 'auido2pose_00140-model.pth'),
11
+ 'audio2exp_checkpoint' : os.path.join(checkpoint_dir, 'auido2exp_00300-model.pth'),
12
+ 'free_view_checkpoint' : os.path.join(checkpoint_dir, 'facevid2vid_00189-model.pth.tar'),
13
+ 'path_of_net_recon_model' : os.path.join(checkpoint_dir, 'epoch_20.pth')
14
+ }
15
+
16
+ use_safetensor = False
17
+ elif len(glob.glob(os.path.join(checkpoint_dir, '*.safetensors'))):
18
+ print('using safetensor as default')
19
+ sadtalker_paths = {
20
+ "checkpoint":os.path.join(checkpoint_dir, 'SadTalker_V0.0.2_'+str(size)+'.safetensors'),
21
+ }
22
+ use_safetensor = True
23
+ else:
24
+ print("WARNING: The new version of the model will be updated by safetensor, you may need to download it mannully. We run the old version of the checkpoint this time!")
25
+ use_safetensor = False
26
+
27
+ sadtalker_paths = {
28
+ 'wav2lip_checkpoint' : os.path.join(checkpoint_dir, 'wav2lip.pth'),
29
+ 'audio2pose_checkpoint' : os.path.join(checkpoint_dir, 'auido2pose_00140-model.pth'),
30
+ 'audio2exp_checkpoint' : os.path.join(checkpoint_dir, 'auido2exp_00300-model.pth'),
31
+ 'free_view_checkpoint' : os.path.join(checkpoint_dir, 'facevid2vid_00189-model.pth.tar'),
32
+ 'path_of_net_recon_model' : os.path.join(checkpoint_dir, 'epoch_20.pth')
33
+ }
34
+
35
+ sadtalker_paths['dir_of_BFM_fitting'] = os.path.join(config_dir) # , 'BFM_Fitting'
36
+ sadtalker_paths['audio2pose_yaml_path'] = os.path.join(config_dir, 'auido2pose.yaml')
37
+ sadtalker_paths['audio2exp_yaml_path'] = os.path.join(config_dir, 'auido2exp.yaml')
38
+ sadtalker_paths['use_safetensor'] = use_safetensor # os.path.join(config_dir, 'auido2exp.yaml')
39
+
40
+ if 'full' in preprocess:
41
+ sadtalker_paths['mappingnet_checkpoint'] = os.path.join(checkpoint_dir, 'mapping_00109-model.pth.tar')
42
+ sadtalker_paths['facerender_yaml'] = os.path.join(config_dir, 'facerender_still.yaml')
43
+ else:
44
+ sadtalker_paths['mappingnet_checkpoint'] = os.path.join(checkpoint_dir, 'mapping_00229-model.pth.tar')
45
+ sadtalker_paths['facerender_yaml'] = os.path.join(config_dir, 'facerender.yaml')
46
+
47
+ return sadtalker_paths
sadtalker+wav2lip/preprocess.py ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import cv2, os, sys, torch
3
+ from tqdm import tqdm
4
+ from PIL import Image
5
+
6
+ # 3dmm extraction
7
+ import safetensors
8
+ import safetensors.torch
9
+ from src.face3d.util.preprocess import align_img
10
+ from src.face3d.util.load_mats import load_lm3d
11
+ from src.face3d.models import networks
12
+
13
+ from scipy.io import loadmat, savemat
14
+ from src.utils.croper import Preprocesser
15
+
16
+
17
+ import warnings
18
+
19
+ from src.utils.safetensor_helper import load_x_from_safetensor
20
+ warnings.filterwarnings("ignore")
21
+
22
+ def split_coeff(coeffs):
23
+ """
24
+ Return:
25
+ coeffs_dict -- a dict of torch.tensors
26
+
27
+ Parameters:
28
+ coeffs -- torch.tensor, size (B, 256)
29
+ """
30
+ id_coeffs = coeffs[:, :80]
31
+ exp_coeffs = coeffs[:, 80: 144]
32
+ tex_coeffs = coeffs[:, 144: 224]
33
+ angles = coeffs[:, 224: 227]
34
+ gammas = coeffs[:, 227: 254]
35
+ translations = coeffs[:, 254:]
36
+ return {
37
+ 'id': id_coeffs,
38
+ 'exp': exp_coeffs,
39
+ 'tex': tex_coeffs,
40
+ 'angle': angles,
41
+ 'gamma': gammas,
42
+ 'trans': translations
43
+ }
44
+
45
+
46
+ class CropAndExtract():
47
+ def __init__(self, sadtalker_path, device):
48
+
49
+ self.propress = Preprocesser(device)
50
+ self.net_recon = networks.define_net_recon(net_recon='resnet50', use_last_fc=False, init_path='').to(device)
51
+
52
+ if sadtalker_path['use_safetensor']:
53
+ checkpoint = safetensors.torch.load_file(sadtalker_path['checkpoint'])
54
+ self.net_recon.load_state_dict(load_x_from_safetensor(checkpoint, 'face_3drecon'))
55
+ else:
56
+ checkpoint = torch.load(sadtalker_path['path_of_net_recon_model'], map_location=torch.device(device))
57
+ self.net_recon.load_state_dict(checkpoint['net_recon'])
58
+
59
+ self.net_recon.eval()
60
+ self.lm3d_std = load_lm3d(sadtalker_path['dir_of_BFM_fitting'])
61
+ self.device = device
62
+
63
+ def generate(self, input_path, save_dir, crop_or_resize='crop', source_image_flag=False, pic_size=256):
64
+
65
+ pic_name = os.path.splitext(os.path.split(input_path)[-1])[0]
66
+
67
+ landmarks_path = os.path.join(save_dir, pic_name+'_landmarks.txt')
68
+ coeff_path = os.path.join(save_dir, pic_name+'.mat')
69
+ png_path = os.path.join(save_dir, pic_name+'.png')
70
+
71
+ #load input
72
+ if not os.path.isfile(input_path):
73
+ raise ValueError('input_path must be a valid path to video/image file')
74
+ elif input_path.split('.')[-1] in ['jpg', 'png', 'jpeg']:
75
+ # loader for first frame
76
+ full_frames = [cv2.imread(input_path)]
77
+ fps = 25
78
+ else:
79
+ # loader for videos
80
+ video_stream = cv2.VideoCapture(input_path)
81
+ fps = video_stream.get(cv2.CAP_PROP_FPS)
82
+ full_frames = []
83
+ while 1:
84
+ still_reading, frame = video_stream.read()
85
+ if not still_reading:
86
+ video_stream.release()
87
+ break
88
+ full_frames.append(frame)
89
+ if source_image_flag:
90
+ break
91
+
92
+ x_full_frames= [cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in full_frames]
93
+
94
+ #### crop images as the
95
+ if 'crop' in crop_or_resize.lower(): # default crop
96
+ x_full_frames, crop, quad = self.propress.crop(x_full_frames, still=True if 'ext' in crop_or_resize.lower() else False, xsize=512)
97
+ clx, cly, crx, cry = crop
98
+ lx, ly, rx, ry = quad
99
+ lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
100
+ oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx
101
+ crop_info = ((ox2 - ox1, oy2 - oy1), crop, quad)
102
+ elif 'full' in crop_or_resize.lower():
103
+ x_full_frames, crop, quad = self.propress.crop(x_full_frames, still=True if 'ext' in crop_or_resize.lower() else False, xsize=512)
104
+ clx, cly, crx, cry = crop
105
+ lx, ly, rx, ry = quad
106
+ lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
107
+ oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx
108
+ crop_info = ((ox2 - ox1, oy2 - oy1), crop, quad)
109
+ else: # resize mode
110
+ oy1, oy2, ox1, ox2 = 0, x_full_frames[0].shape[0], 0, x_full_frames[0].shape[1]
111
+ crop_info = ((ox2 - ox1, oy2 - oy1), None, None)
112
+
113
+ frames_pil = [Image.fromarray(cv2.resize(frame,(pic_size, pic_size))) for frame in x_full_frames]
114
+ if len(frames_pil) == 0:
115
+ print('No face is detected in the input file')
116
+ return None, None
117
+
118
+ # save crop info
119
+ for frame in frames_pil:
120
+ cv2.imwrite(png_path, cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR))
121
+
122
+ # 2. get the landmark according to the detected face.
123
+ if not os.path.isfile(landmarks_path):
124
+ lm = self.propress.predictor.extract_keypoint(frames_pil, landmarks_path)
125
+ else:
126
+ print(' Using saved landmarks.')
127
+ lm = np.loadtxt(landmarks_path).astype(np.float32)
128
+ lm = lm.reshape([len(x_full_frames), -1, 2])
129
+
130
+ if not os.path.isfile(coeff_path):
131
+ # load 3dmm paramter generator from Deep3DFaceRecon_pytorch
132
+ video_coeffs, full_coeffs = [], []
133
+ for idx in tqdm(range(len(frames_pil)), desc='3DMM Extraction In Video:'):
134
+ frame = frames_pil[idx]
135
+ W,H = frame.size
136
+ lm1 = lm[idx].reshape([-1, 2])
137
+
138
+ if np.mean(lm1) == -1:
139
+ lm1 = (self.lm3d_std[:, :2]+1)/2.
140
+ lm1 = np.concatenate(
141
+ [lm1[:, :1]*W, lm1[:, 1:2]*H], 1
142
+ )
143
+ else:
144
+ lm1[:, -1] = H - 1 - lm1[:, -1]
145
+
146
+ trans_params, im1, lm1, _ = align_img(frame, lm1, self.lm3d_std)
147
+
148
+ trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32)
149
+ im_t = torch.tensor(np.array(im1)/255., dtype=torch.float32).permute(2, 0, 1).to(self.device).unsqueeze(0)
150
+
151
+ with torch.no_grad():
152
+ full_coeff = self.net_recon(im_t)
153
+ coeffs = split_coeff(full_coeff)
154
+
155
+ pred_coeff = {key:coeffs[key].cpu().numpy() for key in coeffs}
156
+
157
+ pred_coeff = np.concatenate([
158
+ pred_coeff['exp'],
159
+ pred_coeff['angle'],
160
+ pred_coeff['trans'],
161
+ trans_params[2:][None],
162
+ ], 1)
163
+ video_coeffs.append(pred_coeff)
164
+ full_coeffs.append(full_coeff.cpu().numpy())
165
+
166
+ semantic_npy = np.array(video_coeffs)[:,0]
167
+
168
+ savemat(coeff_path, {'coeff_3dmm': semantic_npy, 'full_3dmm': np.array(full_coeffs)[0]})
169
+
170
+ return coeff_path, png_path, crop_info
sadtalker+wav2lip/requirements.txt ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SadTalker + Wav2Lip Pipeline Requirements
2
+ # Core dependencies for talking avatar generation
3
+
4
+ # Deep Learning
5
+ torch>=1.8.0
6
+ torchvision
7
+ torchaudio
8
+
9
+ # Computer Vision & Image Processing
10
+ opencv-python
11
+ pillow
12
+ imageio
13
+ moviepy
14
+ ffmpeg-python
15
+
16
+ # Face Enhancement
17
+ gfpgan
18
+ basicsr
19
+ facexlib
20
+
21
+ # Audio Processing
22
+ librosa
23
+ soundfile
24
+ scipy
25
+
26
+ # Utilities
27
+ numpy
28
+ matplotlib
29
+ tqdm
sadtalker+wav2lip/sadtalker/generate_batch.py ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ from tqdm import tqdm
4
+ import torch
5
+ import numpy as np
6
+ import random
7
+ import scipy.io as scio
8
+ import src.utils.audio as audio
9
+
10
+ def crop_pad_audio(wav, audio_length):
11
+ if len(wav) > audio_length:
12
+ wav = wav[:audio_length]
13
+ elif len(wav) < audio_length:
14
+ wav = np.pad(wav, [0, audio_length - len(wav)], mode='constant', constant_values=0)
15
+ return wav
16
+
17
+ def parse_audio_length(audio_length, sr, fps):
18
+ bit_per_frames = sr / fps
19
+
20
+ num_frames = int(audio_length / bit_per_frames)
21
+ audio_length = int(num_frames * bit_per_frames)
22
+
23
+ return audio_length, num_frames
24
+
25
+ def generate_blink_seq(num_frames):
26
+ ratio = np.zeros((num_frames,1))
27
+ frame_id = 0
28
+ while frame_id in range(num_frames):
29
+ start = 80
30
+ if frame_id+start+9<=num_frames - 1:
31
+ ratio[frame_id+start:frame_id+start+9, 0] = [0.5,0.6,0.7,0.9,1, 0.9, 0.7,0.6,0.5]
32
+ frame_id = frame_id+start+9
33
+ else:
34
+ break
35
+ return ratio
36
+
37
+ def generate_blink_seq_randomly(num_frames):
38
+ ratio = np.zeros((num_frames,1))
39
+ if num_frames<=20:
40
+ return ratio
41
+ frame_id = 0
42
+ while frame_id in range(num_frames):
43
+ start = random.choice(range(min(10,num_frames), min(int(num_frames/2), 70)))
44
+ if frame_id+start+5<=num_frames - 1:
45
+ ratio[frame_id+start:frame_id+start+5, 0] = [0.5, 0.9, 1.0, 0.9, 0.5]
46
+ frame_id = frame_id+start+5
47
+ else:
48
+ break
49
+ return ratio
50
+
51
+ def get_data(first_coeff_path, audio_path, device, ref_eyeblink_coeff_path, still=False, idlemode=False, length_of_audio=False, use_blink=True):
52
+
53
+ syncnet_mel_step_size = 16
54
+ fps = 25
55
+
56
+ pic_name = os.path.splitext(os.path.split(first_coeff_path)[-1])[0]
57
+ audio_name = os.path.splitext(os.path.split(audio_path)[-1])[0]
58
+
59
+
60
+ if idlemode:
61
+ num_frames = int(length_of_audio * 25)
62
+ indiv_mels = np.zeros((num_frames, 80, 16))
63
+ else:
64
+ wav = audio.load_wav(audio_path, 16000)
65
+ wav_length, num_frames = parse_audio_length(len(wav), 16000, 25)
66
+ wav = crop_pad_audio(wav, wav_length)
67
+ orig_mel = audio.melspectrogram(wav).T
68
+ spec = orig_mel.copy() # nframes 80
69
+ indiv_mels = []
70
+
71
+ for i in tqdm(range(num_frames), 'mel:'):
72
+ start_frame_num = i-2
73
+ start_idx = int(80. * (start_frame_num / float(fps)))
74
+ end_idx = start_idx + syncnet_mel_step_size
75
+ seq = list(range(start_idx, end_idx))
76
+ seq = [ min(max(item, 0), orig_mel.shape[0]-1) for item in seq ]
77
+ m = spec[seq, :]
78
+ indiv_mels.append(m.T)
79
+ indiv_mels = np.asarray(indiv_mels) # T 80 16
80
+
81
+ ratio = generate_blink_seq_randomly(num_frames) # T
82
+ source_semantics_path = first_coeff_path
83
+ source_semantics_dict = scio.loadmat(source_semantics_path)
84
+ ref_coeff = source_semantics_dict['coeff_3dmm'][:1,:70] #1 70
85
+ ref_coeff = np.repeat(ref_coeff, num_frames, axis=0)
86
+
87
+ if ref_eyeblink_coeff_path is not None:
88
+ ratio[:num_frames] = 0
89
+ refeyeblink_coeff_dict = scio.loadmat(ref_eyeblink_coeff_path)
90
+ refeyeblink_coeff = refeyeblink_coeff_dict['coeff_3dmm'][:,:64]
91
+ refeyeblink_num_frames = refeyeblink_coeff.shape[0]
92
+ if refeyeblink_num_frames<num_frames:
93
+ div = num_frames//refeyeblink_num_frames
94
+ re = num_frames%refeyeblink_num_frames
95
+ refeyeblink_coeff_list = [refeyeblink_coeff for i in range(div)]
96
+ refeyeblink_coeff_list.append(refeyeblink_coeff[:re, :64])
97
+ refeyeblink_coeff = np.concatenate(refeyeblink_coeff_list, axis=0)
98
+ print(refeyeblink_coeff.shape[0])
99
+
100
+ ref_coeff[:, :64] = refeyeblink_coeff[:num_frames, :64]
101
+
102
+ indiv_mels = torch.FloatTensor(indiv_mels).unsqueeze(1).unsqueeze(0) # bs T 1 80 16
103
+
104
+ if use_blink:
105
+ ratio = torch.FloatTensor(ratio).unsqueeze(0) # bs T
106
+ else:
107
+ ratio = torch.FloatTensor(ratio).unsqueeze(0).fill_(0.)
108
+ # bs T
109
+ ref_coeff = torch.FloatTensor(ref_coeff).unsqueeze(0) # bs 1 70
110
+
111
+ indiv_mels = indiv_mels.to(device)
112
+ ratio = ratio.to(device)
113
+ ref_coeff = ref_coeff.to(device)
114
+
115
+ return {'indiv_mels': indiv_mels,
116
+ 'ref': ref_coeff,
117
+ 'num_frames': num_frames,
118
+ 'ratio_gt': ratio,
119
+ 'audio_name': audio_name, 'pic_name': pic_name}
120
+
sadtalker+wav2lip/sadtalker/inference.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from glob import glob
2
+ import shutil
3
+ import torch
4
+ from time import strftime
5
+ import os, sys, time
6
+ from argparse import ArgumentParser
7
+
8
+ from src.utils.preprocess import CropAndExtract
9
+ from src.test_audio2coeff import Audio2Coeff
10
+ from src.facerender.animate import AnimateFromCoeff
11
+ from src.generate_batch import get_data
12
+ from src.generate_facerender_batch import get_facerender_data
13
+ from src.utils.init_path import init_path
14
+
15
+ def main(args):
16
+ #torch.backends.cudnn.enabled = False
17
+
18
+ pic_path = args.source_image
19
+ audio_path = args.driven_audio
20
+ save_dir = os.path.join(args.result_dir, strftime("%Y_%m_%d_%H.%M.%S"))
21
+ os.makedirs(save_dir, exist_ok=True)
22
+ pose_style = args.pose_style
23
+ device = args.device
24
+ batch_size = args.batch_size
25
+ input_yaw_list = args.input_yaw
26
+ input_pitch_list = args.input_pitch
27
+ input_roll_list = args.input_roll
28
+ ref_eyeblink = args.ref_eyeblink
29
+ ref_pose = args.ref_pose
30
+
31
+ current_root_path = os.path.split(sys.argv[0])[0]
32
+
33
+ sadtalker_paths = init_path(args.checkpoint_dir, os.path.join(current_root_path, 'src/config'), args.size, args.old_version, args.preprocess)
34
+
35
+ #init model
36
+ preprocess_model = CropAndExtract(sadtalker_paths, device)
37
+
38
+ audio_to_coeff = Audio2Coeff(sadtalker_paths, device)
39
+
40
+ animate_from_coeff = AnimateFromCoeff(sadtalker_paths, device)
41
+
42
+ #crop image and extract 3dmm from image
43
+ first_frame_dir = os.path.join(save_dir, 'first_frame_dir')
44
+ os.makedirs(first_frame_dir, exist_ok=True)
45
+ print('3DMM Extraction for source image')
46
+ first_coeff_path, crop_pic_path, crop_info = preprocess_model.generate(pic_path, first_frame_dir, args.preprocess,\
47
+ source_image_flag=True, pic_size=args.size)
48
+ if first_coeff_path is None:
49
+ print("Can't get the coeffs of the input")
50
+ return
51
+
52
+ if ref_eyeblink is not None:
53
+ ref_eyeblink_videoname = os.path.splitext(os.path.split(ref_eyeblink)[-1])[0]
54
+ ref_eyeblink_frame_dir = os.path.join(save_dir, ref_eyeblink_videoname)
55
+ os.makedirs(ref_eyeblink_frame_dir, exist_ok=True)
56
+ print('3DMM Extraction for the reference video providing eye blinking')
57
+ ref_eyeblink_coeff_path, _, _ = preprocess_model.generate(ref_eyeblink, ref_eyeblink_frame_dir, args.preprocess, source_image_flag=False)
58
+ else:
59
+ ref_eyeblink_coeff_path=None
60
+
61
+ if ref_pose is not None:
62
+ if ref_pose == ref_eyeblink:
63
+ ref_pose_coeff_path = ref_eyeblink_coeff_path
64
+ else:
65
+ ref_pose_videoname = os.path.splitext(os.path.split(ref_pose)[-1])[0]
66
+ ref_pose_frame_dir = os.path.join(save_dir, ref_pose_videoname)
67
+ os.makedirs(ref_pose_frame_dir, exist_ok=True)
68
+ print('3DMM Extraction for the reference video providing pose')
69
+ ref_pose_coeff_path, _, _ = preprocess_model.generate(ref_pose, ref_pose_frame_dir, args.preprocess, source_image_flag=False)
70
+ else:
71
+ ref_pose_coeff_path=None
72
+
73
+ #audio2ceoff
74
+ batch = get_data(first_coeff_path, audio_path, device, ref_eyeblink_coeff_path, still=args.still)
75
+ coeff_path = audio_to_coeff.generate(batch, save_dir, pose_style, ref_pose_coeff_path)
76
+
77
+ # 3dface render
78
+ if args.face3dvis:
79
+ from src.face3d.visualize import gen_composed_video
80
+ gen_composed_video(args, device, first_coeff_path, coeff_path, audio_path, os.path.join(save_dir, '3dface.mp4'))
81
+
82
+ #coeff2video
83
+ data = get_facerender_data(coeff_path, crop_pic_path, first_coeff_path, audio_path,
84
+ batch_size, input_yaw_list, input_pitch_list, input_roll_list,
85
+ expression_scale=args.expression_scale, still_mode=args.still, preprocess=args.preprocess, size=args.size)
86
+
87
+ result = animate_from_coeff.generate(data, save_dir, pic_path, crop_info, \
88
+ enhancer=args.enhancer, background_enhancer=args.background_enhancer, preprocess=args.preprocess, img_size=args.size)
89
+
90
+ shutil.move(result, save_dir+'.mp4')
91
+ print('The generated video is named:', save_dir+'.mp4')
92
+
93
+ if not args.verbose:
94
+ shutil.rmtree(save_dir)
95
+
96
+
97
+ if __name__ == '__main__':
98
+
99
+ parser = ArgumentParser()
100
+ parser.add_argument("--driven_audio", default='./examples/driven_audio/bus_chinese.wav', help="path to driven audio")
101
+ parser.add_argument("--source_image", default='./examples/source_image/full_body_1.png', help="path to source image")
102
+ parser.add_argument("--ref_eyeblink", default=None, help="path to reference video providing eye blinking")
103
+ parser.add_argument("--ref_pose", default=None, help="path to reference video providing pose")
104
+ parser.add_argument("--checkpoint_dir", default='./checkpoints', help="path to output")
105
+ parser.add_argument("--result_dir", default='./results', help="path to output")
106
+ parser.add_argument("--pose_style", type=int, default=0, help="input pose style from [0, 46)")
107
+ parser.add_argument("--batch_size", type=int, default=2, help="the batch size of facerender")
108
+ parser.add_argument("--size", type=int, default=256, help="the image size of the facerender")
109
+ parser.add_argument("--expression_scale", type=float, default=1., help="the batch size of facerender")
110
+ parser.add_argument('--input_yaw', nargs='+', type=int, default=None, help="the input yaw degree of the user ")
111
+ parser.add_argument('--input_pitch', nargs='+', type=int, default=None, help="the input pitch degree of the user")
112
+ parser.add_argument('--input_roll', nargs='+', type=int, default=None, help="the input roll degree of the user")
113
+ parser.add_argument('--enhancer', type=str, default=None, help="Face enhancer, [gfpgan, RestoreFormer]")
114
+ parser.add_argument('--background_enhancer', type=str, default=None, help="background enhancer, [realesrgan]")
115
+ parser.add_argument("--cpu", dest="cpu", action="store_true")
116
+ parser.add_argument("--face3dvis", action="store_true", help="generate 3d face and 3d landmarks")
117
+ parser.add_argument("--still", action="store_true", help="can crop back to the original videos for the full body aniamtion")
118
+ parser.add_argument("--preprocess", default='crop', choices=['crop', 'extcrop', 'resize', 'full', 'extfull'], help="how to preprocess the images" )
119
+ parser.add_argument("--verbose",action="store_true", help="saving the intermedia output or not" )
120
+ parser.add_argument("--old_version",action="store_true", help="use the pth other than safetensor version" )
121
+
122
+
123
+ # net structure and parameters
124
+ parser.add_argument('--net_recon', type=str, default='resnet50', choices=['resnet18', 'resnet34', 'resnet50'], help='useless')
125
+ parser.add_argument('--init_path', type=str, default=None, help='Useless')
126
+ parser.add_argument('--use_last_fc',default=False, help='zero initialize the last fc')
127
+ parser.add_argument('--bfm_folder', type=str, default='./checkpoints/BFM_Fitting/')
128
+ parser.add_argument('--bfm_model', type=str, default='BFM_model_front.mat', help='bfm model')
129
+
130
+ # default renderer parameters
131
+ parser.add_argument('--focal', type=float, default=1015.)
132
+ parser.add_argument('--center', type=float, default=112.)
133
+ parser.add_argument('--camera_d', type=float, default=10.)
134
+ parser.add_argument('--z_near', type=float, default=5.)
135
+ parser.add_argument('--z_far', type=float, default=15.)
136
+
137
+ args = parser.parse_args()
138
+
139
+ if torch.cuda.is_available() and not args.cpu:
140
+ args.device = "cuda"
141
+ else:
142
+ args.device = "cpu"
143
+
144
+ main(args)
145
+
sadtalker+wav2lip/sadtalker/src/audio2exp_models/audio2exp.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from tqdm import tqdm
2
+ import torch
3
+ from torch import nn
4
+
5
+
6
+ class Audio2Exp(nn.Module):
7
+ def __init__(self, netG, cfg, device, prepare_training_loss=False):
8
+ super(Audio2Exp, self).__init__()
9
+ self.cfg = cfg
10
+ self.device = device
11
+ self.netG = netG.to(device)
12
+
13
+ def test(self, batch):
14
+
15
+ mel_input = batch['indiv_mels'] # bs T 1 80 16
16
+ bs = mel_input.shape[0]
17
+ T = mel_input.shape[1]
18
+
19
+ exp_coeff_pred = []
20
+
21
+ for i in tqdm(range(0, T, 10),'audio2exp:'): # every 10 frames
22
+
23
+ current_mel_input = mel_input[:,i:i+10]
24
+
25
+ #ref = batch['ref'][:, :, :64].repeat((1,current_mel_input.shape[1],1)) #bs T 64
26
+ ref = batch['ref'][:, :, :64][:, i:i+10]
27
+ ratio = batch['ratio_gt'][:, i:i+10] #bs T
28
+
29
+ audiox = current_mel_input.view(-1, 1, 80, 16) # bs*T 1 80 16
30
+
31
+ curr_exp_coeff_pred = self.netG(audiox, ref, ratio) # bs T 64
32
+
33
+ exp_coeff_pred += [curr_exp_coeff_pred]
34
+
35
+ # BS x T x 64
36
+ results_dict = {
37
+ 'exp_coeff_pred': torch.cat(exp_coeff_pred, axis=1)
38
+ }
39
+ return results_dict
40
+
41
+
sadtalker+wav2lip/sadtalker/src/audio2exp_models/networks.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+ from torch import nn
4
+
5
+ class Conv2d(nn.Module):
6
+ def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, use_act = True, *args, **kwargs):
7
+ super().__init__(*args, **kwargs)
8
+ self.conv_block = nn.Sequential(
9
+ nn.Conv2d(cin, cout, kernel_size, stride, padding),
10
+ nn.BatchNorm2d(cout)
11
+ )
12
+ self.act = nn.ReLU()
13
+ self.residual = residual
14
+ self.use_act = use_act
15
+
16
+ def forward(self, x):
17
+ out = self.conv_block(x)
18
+ if self.residual:
19
+ out += x
20
+
21
+ if self.use_act:
22
+ return self.act(out)
23
+ else:
24
+ return out
25
+
26
+ class SimpleWrapperV2(nn.Module):
27
+ def __init__(self) -> None:
28
+ super().__init__()
29
+ self.audio_encoder = nn.Sequential(
30
+ Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
31
+ Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
32
+ Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
33
+
34
+ Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1),
35
+ Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
36
+ Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
37
+
38
+ Conv2d(64, 128, kernel_size=3, stride=3, padding=1),
39
+ Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
40
+ Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
41
+
42
+ Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1),
43
+ Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),
44
+
45
+ Conv2d(256, 512, kernel_size=3, stride=1, padding=0),
46
+ Conv2d(512, 512, kernel_size=1, stride=1, padding=0),
47
+ )
48
+
49
+ #### load the pre-trained audio_encoder
50
+ #self.audio_encoder = self.audio_encoder.to(device)
51
+ '''
52
+ wav2lip_state_dict = torch.load('/apdcephfs_cq2/share_1290939/wenxuazhang/checkpoints/wav2lip.pth')['state_dict']
53
+ state_dict = self.audio_encoder.state_dict()
54
+
55
+ for k,v in wav2lip_state_dict.items():
56
+ if 'audio_encoder' in k:
57
+ print('init:', k)
58
+ state_dict[k.replace('module.audio_encoder.', '')] = v
59
+ self.audio_encoder.load_state_dict(state_dict)
60
+ '''
61
+
62
+ self.mapping1 = nn.Linear(512+64+1, 64)
63
+ #self.mapping2 = nn.Linear(30, 64)
64
+ #nn.init.constant_(self.mapping1.weight, 0.)
65
+ nn.init.constant_(self.mapping1.bias, 0.)
66
+
67
+ def forward(self, x, ref, ratio):
68
+ x = self.audio_encoder(x).view(x.size(0), -1)
69
+ ref_reshape = ref.reshape(x.size(0), -1)
70
+ ratio = ratio.reshape(x.size(0), -1)
71
+
72
+ y = self.mapping1(torch.cat([x, ref_reshape, ratio], dim=1))
73
+ out = y.reshape(ref.shape[0], ref.shape[1], -1) #+ ref # resudial
74
+ return out
sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio2pose.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from torch import nn
3
+ from src.audio2pose_models.cvae import CVAE
4
+ from src.audio2pose_models.discriminator import PoseSequenceDiscriminator
5
+ from src.audio2pose_models.audio_encoder import AudioEncoder
6
+
7
+ class Audio2Pose(nn.Module):
8
+ def __init__(self, cfg, wav2lip_checkpoint, device='cuda'):
9
+ super().__init__()
10
+ self.cfg = cfg
11
+ self.seq_len = cfg.MODEL.CVAE.SEQ_LEN
12
+ self.latent_dim = cfg.MODEL.CVAE.LATENT_SIZE
13
+ self.device = device
14
+
15
+ self.audio_encoder = AudioEncoder(wav2lip_checkpoint, device)
16
+ self.audio_encoder.eval()
17
+ for param in self.audio_encoder.parameters():
18
+ param.requires_grad = False
19
+
20
+ self.netG = CVAE(cfg)
21
+ self.netD_motion = PoseSequenceDiscriminator(cfg)
22
+
23
+
24
+ def forward(self, x):
25
+
26
+ batch = {}
27
+ coeff_gt = x['gt'].cuda().squeeze(0) #bs frame_len+1 73
28
+ batch['pose_motion_gt'] = coeff_gt[:, 1:, 64:70] - coeff_gt[:, :1, 64:70] #bs frame_len 6
29
+ batch['ref'] = coeff_gt[:, 0, 64:70] #bs 6
30
+ batch['class'] = x['class'].squeeze(0).cuda() # bs
31
+ indiv_mels= x['indiv_mels'].cuda().squeeze(0) # bs seq_len+1 80 16
32
+
33
+ # forward
34
+ audio_emb_list = []
35
+ audio_emb = self.audio_encoder(indiv_mels[:, 1:, :, :].unsqueeze(2)) #bs seq_len 512
36
+ batch['audio_emb'] = audio_emb
37
+ batch = self.netG(batch)
38
+
39
+ pose_motion_pred = batch['pose_motion_pred'] # bs frame_len 6
40
+ pose_gt = coeff_gt[:, 1:, 64:70].clone() # bs frame_len 6
41
+ pose_pred = coeff_gt[:, :1, 64:70] + pose_motion_pred # bs frame_len 6
42
+
43
+ batch['pose_pred'] = pose_pred
44
+ batch['pose_gt'] = pose_gt
45
+
46
+ return batch
47
+
48
+ def test(self, x):
49
+
50
+ batch = {}
51
+ ref = x['ref'] #bs 1 70
52
+ batch['ref'] = x['ref'][:,0,-6:]
53
+ batch['class'] = x['class']
54
+ bs = ref.shape[0]
55
+
56
+ indiv_mels= x['indiv_mels'] # bs T 1 80 16
57
+ indiv_mels_use = indiv_mels[:, 1:] # we regard the ref as the first frame
58
+ num_frames = x['num_frames']
59
+ num_frames = int(num_frames) - 1
60
+
61
+ #
62
+ div = num_frames//self.seq_len
63
+ re = num_frames%self.seq_len
64
+ audio_emb_list = []
65
+ pose_motion_pred_list = [torch.zeros(batch['ref'].unsqueeze(1).shape, dtype=batch['ref'].dtype,
66
+ device=batch['ref'].device)]
67
+
68
+ for i in range(div):
69
+ z = torch.randn(bs, self.latent_dim).to(ref.device)
70
+ batch['z'] = z
71
+ audio_emb = self.audio_encoder(indiv_mels_use[:, i*self.seq_len:(i+1)*self.seq_len,:,:,:]) #bs seq_len 512
72
+ batch['audio_emb'] = audio_emb
73
+ batch = self.netG.test(batch)
74
+ pose_motion_pred_list.append(batch['pose_motion_pred']) #list of bs seq_len 6
75
+
76
+ if re != 0:
77
+ z = torch.randn(bs, self.latent_dim).to(ref.device)
78
+ batch['z'] = z
79
+ audio_emb = self.audio_encoder(indiv_mels_use[:, -1*self.seq_len:,:,:,:]) #bs seq_len 512
80
+ if audio_emb.shape[1] != self.seq_len:
81
+ pad_dim = self.seq_len-audio_emb.shape[1]
82
+ pad_audio_emb = audio_emb[:, :1].repeat(1, pad_dim, 1)
83
+ audio_emb = torch.cat([pad_audio_emb, audio_emb], 1)
84
+ batch['audio_emb'] = audio_emb
85
+ batch = self.netG.test(batch)
86
+ pose_motion_pred_list.append(batch['pose_motion_pred'][:,-1*re:,:])
87
+
88
+ pose_motion_pred = torch.cat(pose_motion_pred_list, dim = 1)
89
+ batch['pose_motion_pred'] = pose_motion_pred
90
+
91
+ pose_pred = ref[:, :1, -6:] + pose_motion_pred # bs T 6
92
+
93
+ batch['pose_pred'] = pose_pred
94
+ return batch
sadtalker+wav2lip/sadtalker/src/audio2pose_models/audio_encoder.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from torch import nn
3
+ from torch.nn import functional as F
4
+
5
+ class Conv2d(nn.Module):
6
+ def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs):
7
+ super().__init__(*args, **kwargs)
8
+ self.conv_block = nn.Sequential(
9
+ nn.Conv2d(cin, cout, kernel_size, stride, padding),
10
+ nn.BatchNorm2d(cout)
11
+ )
12
+ self.act = nn.ReLU()
13
+ self.residual = residual
14
+
15
+ def forward(self, x):
16
+ out = self.conv_block(x)
17
+ if self.residual:
18
+ out += x
19
+ return self.act(out)
20
+
21
+ class AudioEncoder(nn.Module):
22
+ def __init__(self, wav2lip_checkpoint, device):
23
+ super(AudioEncoder, self).__init__()
24
+
25
+ self.audio_encoder = nn.Sequential(
26
+ Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
27
+ Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
28
+ Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
29
+
30
+ Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1),
31
+ Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
32
+ Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
33
+
34
+ Conv2d(64, 128, kernel_size=3, stride=3, padding=1),
35
+ Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
36
+ Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
37
+
38
+ Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1),
39
+ Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),
40
+
41
+ Conv2d(256, 512, kernel_size=3, stride=1, padding=0),
42
+ Conv2d(512, 512, kernel_size=1, stride=1, padding=0),)
43
+
44
+ #### load the pre-trained audio_encoder, we do not need to load wav2lip model here.
45
+ # wav2lip_state_dict = torch.load(wav2lip_checkpoint, map_location=torch.device(device))['state_dict']
46
+ # state_dict = self.audio_encoder.state_dict()
47
+
48
+ # for k,v in wav2lip_state_dict.items():
49
+ # if 'audio_encoder' in k:
50
+ # state_dict[k.replace('module.audio_encoder.', '')] = v
51
+ # self.audio_encoder.load_state_dict(state_dict)
52
+
53
+
54
+ def forward(self, audio_sequences):
55
+ # audio_sequences = (B, T, 1, 80, 16)
56
+ B = audio_sequences.size(0)
57
+
58
+ audio_sequences = torch.cat([audio_sequences[:, i] for i in range(audio_sequences.size(1))], dim=0)
59
+
60
+ audio_embedding = self.audio_encoder(audio_sequences) # B, 512, 1, 1
61
+ dim = audio_embedding.shape[1]
62
+ audio_embedding = audio_embedding.reshape((B, -1, dim, 1, 1))
63
+
64
+ return audio_embedding.squeeze(-1).squeeze(-1) #B seq_len+1 512
sadtalker+wav2lip/sadtalker/src/audio2pose_models/cvae.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+ from torch import nn
4
+ from src.audio2pose_models.res_unet import ResUnet
5
+
6
+ def class2onehot(idx, class_num):
7
+
8
+ assert torch.max(idx).item() < class_num
9
+ onehot = torch.zeros(idx.size(0), class_num).to(idx.device)
10
+ onehot.scatter_(1, idx, 1)
11
+ return onehot
12
+
13
+ class CVAE(nn.Module):
14
+ def __init__(self, cfg):
15
+ super().__init__()
16
+ encoder_layer_sizes = cfg.MODEL.CVAE.ENCODER_LAYER_SIZES
17
+ decoder_layer_sizes = cfg.MODEL.CVAE.DECODER_LAYER_SIZES
18
+ latent_size = cfg.MODEL.CVAE.LATENT_SIZE
19
+ num_classes = cfg.DATASET.NUM_CLASSES
20
+ audio_emb_in_size = cfg.MODEL.CVAE.AUDIO_EMB_IN_SIZE
21
+ audio_emb_out_size = cfg.MODEL.CVAE.AUDIO_EMB_OUT_SIZE
22
+ seq_len = cfg.MODEL.CVAE.SEQ_LEN
23
+
24
+ self.latent_size = latent_size
25
+
26
+ self.encoder = ENCODER(encoder_layer_sizes, latent_size, num_classes,
27
+ audio_emb_in_size, audio_emb_out_size, seq_len)
28
+ self.decoder = DECODER(decoder_layer_sizes, latent_size, num_classes,
29
+ audio_emb_in_size, audio_emb_out_size, seq_len)
30
+ def reparameterize(self, mu, logvar):
31
+ std = torch.exp(0.5 * logvar)
32
+ eps = torch.randn_like(std)
33
+ return mu + eps * std
34
+
35
+ def forward(self, batch):
36
+ batch = self.encoder(batch)
37
+ mu = batch['mu']
38
+ logvar = batch['logvar']
39
+ z = self.reparameterize(mu, logvar)
40
+ batch['z'] = z
41
+ return self.decoder(batch)
42
+
43
+ def test(self, batch):
44
+ '''
45
+ class_id = batch['class']
46
+ z = torch.randn([class_id.size(0), self.latent_size]).to(class_id.device)
47
+ batch['z'] = z
48
+ '''
49
+ return self.decoder(batch)
50
+
51
+ class ENCODER(nn.Module):
52
+ def __init__(self, layer_sizes, latent_size, num_classes,
53
+ audio_emb_in_size, audio_emb_out_size, seq_len):
54
+ super().__init__()
55
+
56
+ self.resunet = ResUnet()
57
+ self.num_classes = num_classes
58
+ self.seq_len = seq_len
59
+
60
+ self.MLP = nn.Sequential()
61
+ layer_sizes[0] += latent_size + seq_len*audio_emb_out_size + 6
62
+ for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
63
+ self.MLP.add_module(
64
+ name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
65
+ self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
66
+
67
+ self.linear_means = nn.Linear(layer_sizes[-1], latent_size)
68
+ self.linear_logvar = nn.Linear(layer_sizes[-1], latent_size)
69
+ self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size)
70
+
71
+ self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size))
72
+
73
+ def forward(self, batch):
74
+ class_id = batch['class']
75
+ pose_motion_gt = batch['pose_motion_gt'] #bs seq_len 6
76
+ ref = batch['ref'] #bs 6
77
+ bs = pose_motion_gt.shape[0]
78
+ audio_in = batch['audio_emb'] # bs seq_len audio_emb_in_size
79
+
80
+ #pose encode
81
+ pose_emb = self.resunet(pose_motion_gt.unsqueeze(1)) #bs 1 seq_len 6
82
+ pose_emb = pose_emb.reshape(bs, -1) #bs seq_len*6
83
+
84
+ #audio mapping
85
+ print(audio_in.shape)
86
+ audio_out = self.linear_audio(audio_in) # bs seq_len audio_emb_out_size
87
+ audio_out = audio_out.reshape(bs, -1)
88
+
89
+ class_bias = self.classbias[class_id] #bs latent_size
90
+ x_in = torch.cat([ref, pose_emb, audio_out, class_bias], dim=-1) #bs seq_len*(audio_emb_out_size+6)+latent_size
91
+ x_out = self.MLP(x_in)
92
+
93
+ mu = self.linear_means(x_out)
94
+ logvar = self.linear_means(x_out) #bs latent_size
95
+
96
+ batch.update({'mu':mu, 'logvar':logvar})
97
+ return batch
98
+
99
+ class DECODER(nn.Module):
100
+ def __init__(self, layer_sizes, latent_size, num_classes,
101
+ audio_emb_in_size, audio_emb_out_size, seq_len):
102
+ super().__init__()
103
+
104
+ self.resunet = ResUnet()
105
+ self.num_classes = num_classes
106
+ self.seq_len = seq_len
107
+
108
+ self.MLP = nn.Sequential()
109
+ input_size = latent_size + seq_len*audio_emb_out_size + 6
110
+ for i, (in_size, out_size) in enumerate(zip([input_size]+layer_sizes[:-1], layer_sizes)):
111
+ self.MLP.add_module(
112
+ name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
113
+ if i+1 < len(layer_sizes):
114
+ self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
115
+ else:
116
+ self.MLP.add_module(name="sigmoid", module=nn.Sigmoid())
117
+
118
+ self.pose_linear = nn.Linear(6, 6)
119
+ self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size)
120
+
121
+ self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size))
122
+
123
+ def forward(self, batch):
124
+
125
+ z = batch['z'] #bs latent_size
126
+ bs = z.shape[0]
127
+ class_id = batch['class']
128
+ ref = batch['ref'] #bs 6
129
+ audio_in = batch['audio_emb'] # bs seq_len audio_emb_in_size
130
+ #print('audio_in: ', audio_in[:, :, :10])
131
+
132
+ audio_out = self.linear_audio(audio_in) # bs seq_len audio_emb_out_size
133
+ #print('audio_out: ', audio_out[:, :, :10])
134
+ audio_out = audio_out.reshape([bs, -1]) # bs seq_len*audio_emb_out_size
135
+ class_bias = self.classbias[class_id] #bs latent_size
136
+
137
+ z = z + class_bias
138
+ x_in = torch.cat([ref, z, audio_out], dim=-1)
139
+ x_out = self.MLP(x_in) # bs layer_sizes[-1]
140
+ x_out = x_out.reshape((bs, self.seq_len, -1))
141
+
142
+ #print('x_out: ', x_out)
143
+
144
+ pose_emb = self.resunet(x_out.unsqueeze(1)) #bs 1 seq_len 6
145
+
146
+ pose_motion_pred = self.pose_linear(pose_emb.squeeze(1)) #bs seq_len 6
147
+
148
+ batch.update({'pose_motion_pred':pose_motion_pred})
149
+ return batch
sadtalker+wav2lip/sadtalker/src/audio2pose_models/discriminator.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+ from torch import nn
4
+
5
+ class ConvNormRelu(nn.Module):
6
+ def __init__(self, conv_type='1d', in_channels=3, out_channels=64, downsample=False,
7
+ kernel_size=None, stride=None, padding=None, norm='BN', leaky=False):
8
+ super().__init__()
9
+ if kernel_size is None:
10
+ if downsample:
11
+ kernel_size, stride, padding = 4, 2, 1
12
+ else:
13
+ kernel_size, stride, padding = 3, 1, 1
14
+
15
+ if conv_type == '2d':
16
+ self.conv = nn.Conv2d(
17
+ in_channels,
18
+ out_channels,
19
+ kernel_size,
20
+ stride,
21
+ padding,
22
+ bias=False,
23
+ )
24
+ if norm == 'BN':
25
+ self.norm = nn.BatchNorm2d(out_channels)
26
+ elif norm == 'IN':
27
+ self.norm = nn.InstanceNorm2d(out_channels)
28
+ else:
29
+ raise NotImplementedError
30
+ elif conv_type == '1d':
31
+ self.conv = nn.Conv1d(
32
+ in_channels,
33
+ out_channels,
34
+ kernel_size,
35
+ stride,
36
+ padding,
37
+ bias=False,
38
+ )
39
+ if norm == 'BN':
40
+ self.norm = nn.BatchNorm1d(out_channels)
41
+ elif norm == 'IN':
42
+ self.norm = nn.InstanceNorm1d(out_channels)
43
+ else:
44
+ raise NotImplementedError
45
+ nn.init.kaiming_normal_(self.conv.weight)
46
+
47
+ self.act = nn.LeakyReLU(negative_slope=0.2, inplace=False) if leaky else nn.ReLU(inplace=True)
48
+
49
+ def forward(self, x):
50
+ x = self.conv(x)
51
+ if isinstance(self.norm, nn.InstanceNorm1d):
52
+ x = self.norm(x.permute((0, 2, 1))).permute((0, 2, 1)) # normalize on [C]
53
+ else:
54
+ x = self.norm(x)
55
+ x = self.act(x)
56
+ return x
57
+
58
+
59
+ class PoseSequenceDiscriminator(nn.Module):
60
+ def __init__(self, cfg):
61
+ super().__init__()
62
+ self.cfg = cfg
63
+ leaky = self.cfg.MODEL.DISCRIMINATOR.LEAKY_RELU
64
+
65
+ self.seq = nn.Sequential(
66
+ ConvNormRelu('1d', cfg.MODEL.DISCRIMINATOR.INPUT_CHANNELS, 256, downsample=True, leaky=leaky), # B, 256, 64
67
+ ConvNormRelu('1d', 256, 512, downsample=True, leaky=leaky), # B, 512, 32
68
+ ConvNormRelu('1d', 512, 1024, kernel_size=3, stride=1, padding=1, leaky=leaky), # B, 1024, 16
69
+ nn.Conv1d(1024, 1, kernel_size=3, stride=1, padding=1, bias=True) # B, 1, 16
70
+ )
71
+
72
+ def forward(self, x):
73
+ x = x.reshape(x.size(0), x.size(1), -1).transpose(1, 2)
74
+ x = self.seq(x)
75
+ x = x.squeeze(1)
76
+ return x
sadtalker+wav2lip/sadtalker/src/audio2pose_models/networks.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch.nn as nn
2
+ import torch
3
+
4
+
5
+ class ResidualConv(nn.Module):
6
+ def __init__(self, input_dim, output_dim, stride, padding):
7
+ super(ResidualConv, self).__init__()
8
+
9
+ self.conv_block = nn.Sequential(
10
+ nn.BatchNorm2d(input_dim),
11
+ nn.ReLU(),
12
+ nn.Conv2d(
13
+ input_dim, output_dim, kernel_size=3, stride=stride, padding=padding
14
+ ),
15
+ nn.BatchNorm2d(output_dim),
16
+ nn.ReLU(),
17
+ nn.Conv2d(output_dim, output_dim, kernel_size=3, padding=1),
18
+ )
19
+ self.conv_skip = nn.Sequential(
20
+ nn.Conv2d(input_dim, output_dim, kernel_size=3, stride=stride, padding=1),
21
+ nn.BatchNorm2d(output_dim),
22
+ )
23
+
24
+ def forward(self, x):
25
+
26
+ return self.conv_block(x) + self.conv_skip(x)
27
+
28
+
29
+ class Upsample(nn.Module):
30
+ def __init__(self, input_dim, output_dim, kernel, stride):
31
+ super(Upsample, self).__init__()
32
+
33
+ self.upsample = nn.ConvTranspose2d(
34
+ input_dim, output_dim, kernel_size=kernel, stride=stride
35
+ )
36
+
37
+ def forward(self, x):
38
+ return self.upsample(x)
39
+
40
+
41
+ class Squeeze_Excite_Block(nn.Module):
42
+ def __init__(self, channel, reduction=16):
43
+ super(Squeeze_Excite_Block, self).__init__()
44
+ self.avg_pool = nn.AdaptiveAvgPool2d(1)
45
+ self.fc = nn.Sequential(
46
+ nn.Linear(channel, channel // reduction, bias=False),
47
+ nn.ReLU(inplace=True),
48
+ nn.Linear(channel // reduction, channel, bias=False),
49
+ nn.Sigmoid(),
50
+ )
51
+
52
+ def forward(self, x):
53
+ b, c, _, _ = x.size()
54
+ y = self.avg_pool(x).view(b, c)
55
+ y = self.fc(y).view(b, c, 1, 1)
56
+ return x * y.expand_as(x)
57
+
58
+
59
+ class ASPP(nn.Module):
60
+ def __init__(self, in_dims, out_dims, rate=[6, 12, 18]):
61
+ super(ASPP, self).__init__()
62
+
63
+ self.aspp_block1 = nn.Sequential(
64
+ nn.Conv2d(
65
+ in_dims, out_dims, 3, stride=1, padding=rate[0], dilation=rate[0]
66
+ ),
67
+ nn.ReLU(inplace=True),
68
+ nn.BatchNorm2d(out_dims),
69
+ )
70
+ self.aspp_block2 = nn.Sequential(
71
+ nn.Conv2d(
72
+ in_dims, out_dims, 3, stride=1, padding=rate[1], dilation=rate[1]
73
+ ),
74
+ nn.ReLU(inplace=True),
75
+ nn.BatchNorm2d(out_dims),
76
+ )
77
+ self.aspp_block3 = nn.Sequential(
78
+ nn.Conv2d(
79
+ in_dims, out_dims, 3, stride=1, padding=rate[2], dilation=rate[2]
80
+ ),
81
+ nn.ReLU(inplace=True),
82
+ nn.BatchNorm2d(out_dims),
83
+ )
84
+
85
+ self.output = nn.Conv2d(len(rate) * out_dims, out_dims, 1)
86
+ self._init_weights()
87
+
88
+ def forward(self, x):
89
+ x1 = self.aspp_block1(x)
90
+ x2 = self.aspp_block2(x)
91
+ x3 = self.aspp_block3(x)
92
+ out = torch.cat([x1, x2, x3], dim=1)
93
+ return self.output(out)
94
+
95
+ def _init_weights(self):
96
+ for m in self.modules():
97
+ if isinstance(m, nn.Conv2d):
98
+ nn.init.kaiming_normal_(m.weight)
99
+ elif isinstance(m, nn.BatchNorm2d):
100
+ m.weight.data.fill_(1)
101
+ m.bias.data.zero_()
102
+
103
+
104
+ class Upsample_(nn.Module):
105
+ def __init__(self, scale=2):
106
+ super(Upsample_, self).__init__()
107
+
108
+ self.upsample = nn.Upsample(mode="bilinear", scale_factor=scale)
109
+
110
+ def forward(self, x):
111
+ return self.upsample(x)
112
+
113
+
114
+ class AttentionBlock(nn.Module):
115
+ def __init__(self, input_encoder, input_decoder, output_dim):
116
+ super(AttentionBlock, self).__init__()
117
+
118
+ self.conv_encoder = nn.Sequential(
119
+ nn.BatchNorm2d(input_encoder),
120
+ nn.ReLU(),
121
+ nn.Conv2d(input_encoder, output_dim, 3, padding=1),
122
+ nn.MaxPool2d(2, 2),
123
+ )
124
+
125
+ self.conv_decoder = nn.Sequential(
126
+ nn.BatchNorm2d(input_decoder),
127
+ nn.ReLU(),
128
+ nn.Conv2d(input_decoder, output_dim, 3, padding=1),
129
+ )
130
+
131
+ self.conv_attn = nn.Sequential(
132
+ nn.BatchNorm2d(output_dim),
133
+ nn.ReLU(),
134
+ nn.Conv2d(output_dim, 1, 1),
135
+ )
136
+
137
+ def forward(self, x1, x2):
138
+ out = self.conv_encoder(x1) + self.conv_decoder(x2)
139
+ out = self.conv_attn(out)
140
+ return out * x2
sadtalker+wav2lip/sadtalker/src/audio2pose_models/res_unet.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from src.audio2pose_models.networks import ResidualConv, Upsample
4
+
5
+
6
+ class ResUnet(nn.Module):
7
+ def __init__(self, channel=1, filters=[32, 64, 128, 256]):
8
+ super(ResUnet, self).__init__()
9
+
10
+ self.input_layer = nn.Sequential(
11
+ nn.Conv2d(channel, filters[0], kernel_size=3, padding=1),
12
+ nn.BatchNorm2d(filters[0]),
13
+ nn.ReLU(),
14
+ nn.Conv2d(filters[0], filters[0], kernel_size=3, padding=1),
15
+ )
16
+ self.input_skip = nn.Sequential(
17
+ nn.Conv2d(channel, filters[0], kernel_size=3, padding=1)
18
+ )
19
+
20
+ self.residual_conv_1 = ResidualConv(filters[0], filters[1], stride=(2,1), padding=1)
21
+ self.residual_conv_2 = ResidualConv(filters[1], filters[2], stride=(2,1), padding=1)
22
+
23
+ self.bridge = ResidualConv(filters[2], filters[3], stride=(2,1), padding=1)
24
+
25
+ self.upsample_1 = Upsample(filters[3], filters[3], kernel=(2,1), stride=(2,1))
26
+ self.up_residual_conv1 = ResidualConv(filters[3] + filters[2], filters[2], stride=1, padding=1)
27
+
28
+ self.upsample_2 = Upsample(filters[2], filters[2], kernel=(2,1), stride=(2,1))
29
+ self.up_residual_conv2 = ResidualConv(filters[2] + filters[1], filters[1], stride=1, padding=1)
30
+
31
+ self.upsample_3 = Upsample(filters[1], filters[1], kernel=(2,1), stride=(2,1))
32
+ self.up_residual_conv3 = ResidualConv(filters[1] + filters[0], filters[0], stride=1, padding=1)
33
+
34
+ self.output_layer = nn.Sequential(
35
+ nn.Conv2d(filters[0], 1, 1, 1),
36
+ nn.Sigmoid(),
37
+ )
38
+
39
+ def forward(self, x):
40
+ # Encode
41
+ x1 = self.input_layer(x) + self.input_skip(x)
42
+ x2 = self.residual_conv_1(x1)
43
+ x3 = self.residual_conv_2(x2)
44
+ # Bridge
45
+ x4 = self.bridge(x3)
46
+
47
+ # Decode
48
+ x4 = self.upsample_1(x4)
49
+ x5 = torch.cat([x4, x3], dim=1)
50
+
51
+ x6 = self.up_residual_conv1(x5)
52
+
53
+ x6 = self.upsample_2(x6)
54
+ x7 = torch.cat([x6, x2], dim=1)
55
+
56
+ x8 = self.up_residual_conv2(x7)
57
+
58
+ x8 = self.upsample_3(x8)
59
+ x9 = torch.cat([x8, x1], dim=1)
60
+
61
+ x10 = self.up_residual_conv3(x9)
62
+
63
+ output = self.output_layer(x10)
64
+
65
+ return output
sadtalker+wav2lip/sadtalker/src/config/auido2exp.yaml ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DATASET:
2
+ TRAIN_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/file_list/train.txt
3
+ EVAL_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/file_list/val.txt
4
+ TRAIN_BATCH_SIZE: 32
5
+ EVAL_BATCH_SIZE: 32
6
+ EXP: True
7
+ EXP_DIM: 64
8
+ FRAME_LEN: 32
9
+ COEFF_LEN: 73
10
+ NUM_CLASSES: 46
11
+ AUDIO_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav
12
+ COEFF_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav2lip_3dmm
13
+ LMDB_PATH: /apdcephfs_cq2/share_1290939/shadowcun/datasets/VoxCeleb/v1/imdb
14
+ DEBUG: True
15
+ NUM_REPEATS: 2
16
+ T: 40
17
+
18
+
19
+ MODEL:
20
+ FRAMEWORK: V2
21
+ AUDIOENCODER:
22
+ LEAKY_RELU: True
23
+ NORM: 'IN'
24
+ DISCRIMINATOR:
25
+ LEAKY_RELU: False
26
+ INPUT_CHANNELS: 6
27
+ CVAE:
28
+ AUDIO_EMB_IN_SIZE: 512
29
+ AUDIO_EMB_OUT_SIZE: 128
30
+ SEQ_LEN: 32
31
+ LATENT_SIZE: 256
32
+ ENCODER_LAYER_SIZES: [192, 1024]
33
+ DECODER_LAYER_SIZES: [1024, 192]
34
+
35
+
36
+ TRAIN:
37
+ MAX_EPOCH: 300
38
+ GENERATOR:
39
+ LR: 2.0e-5
40
+ DISCRIMINATOR:
41
+ LR: 1.0e-5
42
+ LOSS:
43
+ W_FEAT: 0
44
+ W_COEFF_EXP: 2
45
+ W_LM: 1.0e-2
46
+ W_LM_MOUTH: 0
47
+ W_REG: 0
48
+ W_SYNC: 0
49
+ W_COLOR: 0
50
+ W_EXPRESSION: 0
51
+ W_LIPREADING: 0.01
52
+ W_LIPREADING_VV: 0
53
+ W_EYE_BLINK: 4
54
+
55
+ TAG:
56
+ NAME: small_dataset
57
+
58
+
sadtalker+wav2lip/sadtalker/src/config/auido2pose.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DATASET:
2
+ TRAIN_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/audio2pose_unet_noAudio/dataset/train_33.txt
3
+ EVAL_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/audio2pose_unet_noAudio/dataset/val.txt
4
+ TRAIN_BATCH_SIZE: 64
5
+ EVAL_BATCH_SIZE: 1
6
+ EXP: True
7
+ EXP_DIM: 64
8
+ FRAME_LEN: 32
9
+ COEFF_LEN: 73
10
+ NUM_CLASSES: 46
11
+ AUDIO_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav
12
+ COEFF_ROOT_PATH: /apdcephfs_cq2/share_1290939/shadowcun/datasets/VoxCeleb/v1/imdb
13
+ DEBUG: True
14
+
15
+
16
+ MODEL:
17
+ AUDIOENCODER:
18
+ LEAKY_RELU: True
19
+ NORM: 'IN'
20
+ DISCRIMINATOR:
21
+ LEAKY_RELU: False
22
+ INPUT_CHANNELS: 6
23
+ CVAE:
24
+ AUDIO_EMB_IN_SIZE: 512
25
+ AUDIO_EMB_OUT_SIZE: 6
26
+ SEQ_LEN: 32
27
+ LATENT_SIZE: 64
28
+ ENCODER_LAYER_SIZES: [192, 128]
29
+ DECODER_LAYER_SIZES: [128, 192]
30
+
31
+
32
+ TRAIN:
33
+ MAX_EPOCH: 150
34
+ GENERATOR:
35
+ LR: 1.0e-4
36
+ DISCRIMINATOR:
37
+ LR: 1.0e-4
38
+ LOSS:
39
+ LAMBDA_REG: 1
40
+ LAMBDA_LANDMARKS: 0
41
+ LAMBDA_VERTICES: 0
42
+ LAMBDA_GAN_MOTION: 0.7
43
+ LAMBDA_GAN_COEFF: 0
44
+ LAMBDA_KL: 1
45
+
46
+ TAG:
47
+ NAME: cvae_UNET_useAudio_usewav2lipAudioEncoder
48
+
49
+
sadtalker+wav2lip/sadtalker/src/config/facerender.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model_params:
2
+ common_params:
3
+ num_kp: 15
4
+ image_channel: 3
5
+ feature_channel: 32
6
+ estimate_jacobian: False # True
7
+ kp_detector_params:
8
+ temperature: 0.1
9
+ block_expansion: 32
10
+ max_features: 1024
11
+ scale_factor: 0.25 # 0.25
12
+ num_blocks: 5
13
+ reshape_channel: 16384 # 16384 = 1024 * 16
14
+ reshape_depth: 16
15
+ he_estimator_params:
16
+ block_expansion: 64
17
+ max_features: 2048
18
+ num_bins: 66
19
+ generator_params:
20
+ block_expansion: 64
21
+ max_features: 512
22
+ num_down_blocks: 2
23
+ reshape_channel: 32
24
+ reshape_depth: 16 # 512 = 32 * 16
25
+ num_resblocks: 6
26
+ estimate_occlusion_map: True
27
+ dense_motion_params:
28
+ block_expansion: 32
29
+ max_features: 1024
30
+ num_blocks: 5
31
+ reshape_depth: 16
32
+ compress: 4
33
+ discriminator_params:
34
+ scales: [1]
35
+ block_expansion: 32
36
+ max_features: 512
37
+ num_blocks: 4
38
+ sn: True
39
+ mapping_params:
40
+ coeff_nc: 70
41
+ descriptor_nc: 1024
42
+ layer: 3
43
+ num_kp: 15
44
+ num_bins: 66
45
+
sadtalker+wav2lip/sadtalker/src/config/facerender_still.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model_params:
2
+ common_params:
3
+ num_kp: 15
4
+ image_channel: 3
5
+ feature_channel: 32
6
+ estimate_jacobian: False # True
7
+ kp_detector_params:
8
+ temperature: 0.1
9
+ block_expansion: 32
10
+ max_features: 1024
11
+ scale_factor: 0.25 # 0.25
12
+ num_blocks: 5
13
+ reshape_channel: 16384 # 16384 = 1024 * 16
14
+ reshape_depth: 16
15
+ he_estimator_params:
16
+ block_expansion: 64
17
+ max_features: 2048
18
+ num_bins: 66
19
+ generator_params:
20
+ block_expansion: 64
21
+ max_features: 512
22
+ num_down_blocks: 2
23
+ reshape_channel: 32
24
+ reshape_depth: 16 # 512 = 32 * 16
25
+ num_resblocks: 6
26
+ estimate_occlusion_map: True
27
+ dense_motion_params:
28
+ block_expansion: 32
29
+ max_features: 1024
30
+ num_blocks: 5
31
+ reshape_depth: 16
32
+ compress: 4
33
+ discriminator_params:
34
+ scales: [1]
35
+ block_expansion: 32
36
+ max_features: 512
37
+ num_blocks: 4
38
+ sn: True
39
+ mapping_params:
40
+ coeff_nc: 73
41
+ descriptor_nc: 1024
42
+ layer: 3
43
+ num_kp: 15
44
+ num_bins: 66
45
+
sadtalker+wav2lip/sadtalker/src/config/similarity_Lm3D_all.mat ADDED
Binary file (994 Bytes). View file
 
sadtalker+wav2lip/sadtalker/src/face3d/data/__init__.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """This package includes all the modules related to data loading and preprocessing
2
+
3
+ To add a custom dataset class called 'dummy', you need to add a file called 'dummy_dataset.py' and define a subclass 'DummyDataset' inherited from BaseDataset.
4
+ You need to implement four functions:
5
+ -- <__init__>: initialize the class, first call BaseDataset.__init__(self, opt).
6
+ -- <__len__>: return the size of dataset.
7
+ -- <__getitem__>: get a data point from data loader.
8
+ -- <modify_commandline_options>: (optionally) add dataset-specific options and set default options.
9
+
10
+ Now you can use the dataset class by specifying flag '--dataset_mode dummy'.
11
+ See our template dataset class 'template_dataset.py' for more details.
12
+ """
13
+ import numpy as np
14
+ import importlib
15
+ import torch.utils.data
16
+ from face3d.data.base_dataset import BaseDataset
17
+
18
+
19
+ def find_dataset_using_name(dataset_name):
20
+ """Import the module "data/[dataset_name]_dataset.py".
21
+
22
+ In the file, the class called DatasetNameDataset() will
23
+ be instantiated. It has to be a subclass of BaseDataset,
24
+ and it is case-insensitive.
25
+ """
26
+ dataset_filename = "data." + dataset_name + "_dataset"
27
+ datasetlib = importlib.import_module(dataset_filename)
28
+
29
+ dataset = None
30
+ target_dataset_name = dataset_name.replace('_', '') + 'dataset'
31
+ for name, cls in datasetlib.__dict__.items():
32
+ if name.lower() == target_dataset_name.lower() \
33
+ and issubclass(cls, BaseDataset):
34
+ dataset = cls
35
+
36
+ if dataset is None:
37
+ raise NotImplementedError("In %s.py, there should be a subclass of BaseDataset with class name that matches %s in lowercase." % (dataset_filename, target_dataset_name))
38
+
39
+ return dataset
40
+
41
+
42
+ def get_option_setter(dataset_name):
43
+ """Return the static method <modify_commandline_options> of the dataset class."""
44
+ dataset_class = find_dataset_using_name(dataset_name)
45
+ return dataset_class.modify_commandline_options
46
+
47
+
48
+ def create_dataset(opt, rank=0):
49
+ """Create a dataset given the option.
50
+
51
+ This function wraps the class CustomDatasetDataLoader.
52
+ This is the main interface between this package and 'train.py'/'test.py'
53
+
54
+ Example:
55
+ >>> from data import create_dataset
56
+ >>> dataset = create_dataset(opt)
57
+ """
58
+ data_loader = CustomDatasetDataLoader(opt, rank=rank)
59
+ dataset = data_loader.load_data()
60
+ return dataset
61
+
62
+ class CustomDatasetDataLoader():
63
+ """Wrapper class of Dataset class that performs multi-threaded data loading"""
64
+
65
+ def __init__(self, opt, rank=0):
66
+ """Initialize this class
67
+
68
+ Step 1: create a dataset instance given the name [dataset_mode]
69
+ Step 2: create a multi-threaded data loader.
70
+ """
71
+ self.opt = opt
72
+ dataset_class = find_dataset_using_name(opt.dataset_mode)
73
+ self.dataset = dataset_class(opt)
74
+ self.sampler = None
75
+ print("rank %d %s dataset [%s] was created" % (rank, self.dataset.name, type(self.dataset).__name__))
76
+ if opt.use_ddp and opt.isTrain:
77
+ world_size = opt.world_size
78
+ self.sampler = torch.utils.data.distributed.DistributedSampler(
79
+ self.dataset,
80
+ num_replicas=world_size,
81
+ rank=rank,
82
+ shuffle=not opt.serial_batches
83
+ )
84
+ self.dataloader = torch.utils.data.DataLoader(
85
+ self.dataset,
86
+ sampler=self.sampler,
87
+ num_workers=int(opt.num_threads / world_size),
88
+ batch_size=int(opt.batch_size / world_size),
89
+ drop_last=True)
90
+ else:
91
+ self.dataloader = torch.utils.data.DataLoader(
92
+ self.dataset,
93
+ batch_size=opt.batch_size,
94
+ shuffle=(not opt.serial_batches) and opt.isTrain,
95
+ num_workers=int(opt.num_threads),
96
+ drop_last=True
97
+ )
98
+
99
+ def set_epoch(self, epoch):
100
+ self.dataset.current_epoch = epoch
101
+ if self.sampler is not None:
102
+ self.sampler.set_epoch(epoch)
103
+
104
+ def load_data(self):
105
+ return self
106
+
107
+ def __len__(self):
108
+ """Return the number of data in the dataset"""
109
+ return min(len(self.dataset), self.opt.max_dataset_size)
110
+
111
+ def __iter__(self):
112
+ """Return a batch of data"""
113
+ for i, data in enumerate(self.dataloader):
114
+ if i * self.opt.batch_size >= self.opt.max_dataset_size:
115
+ break
116
+ yield data
sadtalker+wav2lip/sadtalker/src/face3d/data/base_dataset.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """This module implements an abstract base class (ABC) 'BaseDataset' for datasets.
2
+
3
+ It also includes common transformation functions (e.g., get_transform, __scale_width), which can be later used in subclasses.
4
+ """
5
+ import random
6
+ import numpy as np
7
+ import torch.utils.data as data
8
+ from PIL import Image
9
+ import torchvision.transforms as transforms
10
+ from abc import ABC, abstractmethod
11
+
12
+
13
+ class BaseDataset(data.Dataset, ABC):
14
+ """This class is an abstract base class (ABC) for datasets.
15
+
16
+ To create a subclass, you need to implement the following four functions:
17
+ -- <__init__>: initialize the class, first call BaseDataset.__init__(self, opt).
18
+ -- <__len__>: return the size of dataset.
19
+ -- <__getitem__>: get a data point.
20
+ -- <modify_commandline_options>: (optionally) add dataset-specific options and set default options.
21
+ """
22
+
23
+ def __init__(self, opt):
24
+ """Initialize the class; save the options in the class
25
+
26
+ Parameters:
27
+ opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions
28
+ """
29
+ self.opt = opt
30
+ # self.root = opt.dataroot
31
+ self.current_epoch = 0
32
+
33
+ @staticmethod
34
+ def modify_commandline_options(parser, is_train):
35
+ """Add new dataset-specific options, and rewrite default values for existing options.
36
+
37
+ Parameters:
38
+ parser -- original option parser
39
+ is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options.
40
+
41
+ Returns:
42
+ the modified parser.
43
+ """
44
+ return parser
45
+
46
+ @abstractmethod
47
+ def __len__(self):
48
+ """Return the total number of images in the dataset."""
49
+ return 0
50
+
51
+ @abstractmethod
52
+ def __getitem__(self, index):
53
+ """Return a data point and its metadata information.
54
+
55
+ Parameters:
56
+ index - - a random integer for data indexing
57
+
58
+ Returns:
59
+ a dictionary of data with their names. It ususally contains the data itself and its metadata information.
60
+ """
61
+ pass
62
+
63
+
64
+ def get_transform(grayscale=False):
65
+ transform_list = []
66
+ if grayscale:
67
+ transform_list.append(transforms.Grayscale(1))
68
+ transform_list += [transforms.ToTensor()]
69
+ return transforms.Compose(transform_list)
70
+
71
+ def get_affine_mat(opt, size):
72
+ shift_x, shift_y, scale, rot_angle, flip = 0., 0., 1., 0., False
73
+ w, h = size
74
+
75
+ if 'shift' in opt.preprocess:
76
+ shift_pixs = int(opt.shift_pixs)
77
+ shift_x = random.randint(-shift_pixs, shift_pixs)
78
+ shift_y = random.randint(-shift_pixs, shift_pixs)
79
+ if 'scale' in opt.preprocess:
80
+ scale = 1 + opt.scale_delta * (2 * random.random() - 1)
81
+ if 'rot' in opt.preprocess:
82
+ rot_angle = opt.rot_angle * (2 * random.random() - 1)
83
+ rot_rad = -rot_angle * np.pi/180
84
+ if 'flip' in opt.preprocess:
85
+ flip = random.random() > 0.5
86
+
87
+ shift_to_origin = np.array([1, 0, -w//2, 0, 1, -h//2, 0, 0, 1]).reshape([3, 3])
88
+ flip_mat = np.array([-1 if flip else 1, 0, 0, 0, 1, 0, 0, 0, 1]).reshape([3, 3])
89
+ shift_mat = np.array([1, 0, shift_x, 0, 1, shift_y, 0, 0, 1]).reshape([3, 3])
90
+ rot_mat = np.array([np.cos(rot_rad), np.sin(rot_rad), 0, -np.sin(rot_rad), np.cos(rot_rad), 0, 0, 0, 1]).reshape([3, 3])
91
+ scale_mat = np.array([scale, 0, 0, 0, scale, 0, 0, 0, 1]).reshape([3, 3])
92
+ shift_to_center = np.array([1, 0, w//2, 0, 1, h//2, 0, 0, 1]).reshape([3, 3])
93
+
94
+ affine = shift_to_center @ scale_mat @ rot_mat @ shift_mat @ flip_mat @ shift_to_origin
95
+ affine_inv = np.linalg.inv(affine)
96
+ return affine, affine_inv, flip
97
+
98
+ def apply_img_affine(img, affine_inv, method=Image.BICUBIC):
99
+ return img.transform(img.size, Image.AFFINE, data=affine_inv.flatten()[:6], resample=Image.BICUBIC)
100
+
101
+ def apply_lm_affine(landmark, affine, flip, size):
102
+ _, h = size
103
+ lm = landmark.copy()
104
+ lm[:, 1] = h - 1 - lm[:, 1]
105
+ lm = np.concatenate((lm, np.ones([lm.shape[0], 1])), -1)
106
+ lm = lm @ np.transpose(affine)
107
+ lm[:, :2] = lm[:, :2] / lm[:, 2:]
108
+ lm = lm[:, :2]
109
+ lm[:, 1] = h - 1 - lm[:, 1]
110
+ if flip:
111
+ lm_ = lm.copy()
112
+ lm_[:17] = lm[16::-1]
113
+ lm_[17:22] = lm[26:21:-1]
114
+ lm_[22:27] = lm[21:16:-1]
115
+ lm_[31:36] = lm[35:30:-1]
116
+ lm_[36:40] = lm[45:41:-1]
117
+ lm_[40:42] = lm[47:45:-1]
118
+ lm_[42:46] = lm[39:35:-1]
119
+ lm_[46:48] = lm[41:39:-1]
120
+ lm_[48:55] = lm[54:47:-1]
121
+ lm_[55:60] = lm[59:54:-1]
122
+ lm_[60:65] = lm[64:59:-1]
123
+ lm_[65:68] = lm[67:64:-1]
124
+ lm = lm_
125
+ return lm
sadtalker+wav2lip/sadtalker/src/face3d/data/flist_dataset.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """This script defines the custom dataset for Deep3DFaceRecon_pytorch
2
+ """
3
+
4
+ import os.path
5
+ from data.base_dataset import BaseDataset, get_transform, get_affine_mat, apply_img_affine, apply_lm_affine
6
+ from data.image_folder import make_dataset
7
+ from PIL import Image
8
+ import random
9
+ import util.util as util
10
+ import numpy as np
11
+ import json
12
+ import torch
13
+ from scipy.io import loadmat, savemat
14
+ import pickle
15
+ from util.preprocess import align_img, estimate_norm
16
+ from util.load_mats import load_lm3d
17
+
18
+
19
+ def default_flist_reader(flist):
20
+ """
21
+ flist format: impath label\nimpath label\n ...(same to caffe's filelist)
22
+ """
23
+ imlist = []
24
+ with open(flist, 'r') as rf:
25
+ for line in rf.readlines():
26
+ impath = line.strip()
27
+ imlist.append(impath)
28
+
29
+ return imlist
30
+
31
+ def jason_flist_reader(flist):
32
+ with open(flist, 'r') as fp:
33
+ info = json.load(fp)
34
+ return info
35
+
36
+ def parse_label(label):
37
+ return torch.tensor(np.array(label).astype(np.float32))
38
+
39
+
40
+ class FlistDataset(BaseDataset):
41
+ """
42
+ It requires one directories to host training images '/path/to/data/train'
43
+ You can train the model with the dataset flag '--dataroot /path/to/data'.
44
+ """
45
+
46
+ def __init__(self, opt):
47
+ """Initialize this dataset class.
48
+
49
+ Parameters:
50
+ opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions
51
+ """
52
+ BaseDataset.__init__(self, opt)
53
+
54
+ self.lm3d_std = load_lm3d(opt.bfm_folder)
55
+
56
+ msk_names = default_flist_reader(opt.flist)
57
+ self.msk_paths = [os.path.join(opt.data_root, i) for i in msk_names]
58
+
59
+ self.size = len(self.msk_paths)
60
+ self.opt = opt
61
+
62
+ self.name = 'train' if opt.isTrain else 'val'
63
+ if '_' in opt.flist:
64
+ self.name += '_' + opt.flist.split(os.sep)[-1].split('_')[0]
65
+
66
+
67
+ def __getitem__(self, index):
68
+ """Return a data point and its metadata information.
69
+
70
+ Parameters:
71
+ index (int) -- a random integer for data indexing
72
+
73
+ Returns a dictionary that contains A, B, A_paths and B_paths
74
+ img (tensor) -- an image in the input domain
75
+ msk (tensor) -- its corresponding attention mask
76
+ lm (tensor) -- its corresponding 3d landmarks
77
+ im_paths (str) -- image paths
78
+ aug_flag (bool) -- a flag used to tell whether its raw or augmented
79
+ """
80
+ msk_path = self.msk_paths[index % self.size] # make sure index is within then range
81
+ img_path = msk_path.replace('mask/', '')
82
+ lm_path = '.'.join(msk_path.replace('mask', 'landmarks').split('.')[:-1]) + '.txt'
83
+
84
+ raw_img = Image.open(img_path).convert('RGB')
85
+ raw_msk = Image.open(msk_path).convert('RGB')
86
+ raw_lm = np.loadtxt(lm_path).astype(np.float32)
87
+
88
+ _, img, lm, msk = align_img(raw_img, raw_lm, self.lm3d_std, raw_msk)
89
+
90
+ aug_flag = self.opt.use_aug and self.opt.isTrain
91
+ if aug_flag:
92
+ img, lm, msk = self._augmentation(img, lm, self.opt, msk)
93
+
94
+ _, H = img.size
95
+ M = estimate_norm(lm, H)
96
+ transform = get_transform()
97
+ img_tensor = transform(img)
98
+ msk_tensor = transform(msk)[:1, ...]
99
+ lm_tensor = parse_label(lm)
100
+ M_tensor = parse_label(M)
101
+
102
+
103
+ return {'imgs': img_tensor,
104
+ 'lms': lm_tensor,
105
+ 'msks': msk_tensor,
106
+ 'M': M_tensor,
107
+ 'im_paths': img_path,
108
+ 'aug_flag': aug_flag,
109
+ 'dataset': self.name}
110
+
111
+ def _augmentation(self, img, lm, opt, msk=None):
112
+ affine, affine_inv, flip = get_affine_mat(opt, img.size)
113
+ img = apply_img_affine(img, affine_inv)
114
+ lm = apply_lm_affine(lm, affine, flip, img.size)
115
+ if msk is not None:
116
+ msk = apply_img_affine(msk, affine_inv, method=Image.BILINEAR)
117
+ return img, lm, msk
118
+
119
+
120
+
121
+
122
+ def __len__(self):
123
+ """Return the total number of images in the dataset.
124
+ """
125
+ return self.size