APEXPRIT commited on
Commit
59f824e
·
verified ·
1 Parent(s): 4e9a5c4

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .dockerignore +22 -0
  2. Dockerfile +36 -0
  3. README.md +360 -10
  4. api.py +129 -0
  5. app.py +596 -0
  6. colab_deception_analysis.ipynb +203 -0
  7. create_pretrained_model.py +276 -0
  8. inference.py +323 -0
  9. models/.gitkeep +1 -0
  10. requirements.txt +16 -0
  11. src/__init__.py +4 -0
  12. src/__pycache__/__init__.cpython-314.pyc +0 -0
  13. src/__pycache__/pipeline.cpython-314.pyc +0 -0
  14. src/audio/__init__.py +6 -0
  15. src/audio/__pycache__/__init__.cpython-314.pyc +0 -0
  16. src/audio/extractor.py +248 -0
  17. src/audio/fusion.py +108 -0
  18. src/evaluation/__init__.py +13 -0
  19. src/evaluation/__pycache__/__init__.cpython-314.pyc +0 -0
  20. src/evaluation/__pycache__/ablation.cpython-314.pyc +0 -0
  21. src/evaluation/__pycache__/cross_validation.cpython-314.pyc +0 -0
  22. src/evaluation/__pycache__/metrics.cpython-314.pyc +0 -0
  23. src/evaluation/__pycache__/visualizer.cpython-314.pyc +0 -0
  24. src/evaluation/ablation.py +294 -0
  25. src/evaluation/cross_validation.py +303 -0
  26. src/evaluation/metrics.py +356 -0
  27. src/evaluation/visualizer.py +396 -0
  28. src/explainability/__init__.py +6 -0
  29. src/explainability/__pycache__/__init__.cpython-314.pyc +0 -0
  30. src/explainability/__pycache__/explainer.cpython-314.pyc +0 -0
  31. src/explainability/__pycache__/visualizer.cpython-314.pyc +0 -0
  32. src/explainability/explainer.py +513 -0
  33. src/explainability/visualizer.py +743 -0
  34. src/facial/__init__.py +6 -0
  35. src/facial/__pycache__/__init__.cpython-314.pyc +0 -0
  36. src/facial/extractor.py +341 -0
  37. src/facial/visualizer.py +135 -0
  38. src/model/__init__.py +6 -0
  39. src/model/__pycache__/__init__.cpython-314.pyc +0 -0
  40. src/model/calibration.py +265 -0
  41. src/model/classifier.py +330 -0
  42. src/pipeline.py +524 -0
  43. src/temporal/__init__.py +6 -0
  44. src/temporal/__pycache__/__init__.cpython-314.pyc +0 -0
  45. src/temporal/analyzer.py +298 -0
  46. src/temporal/visualizer.py +201 -0
  47. src/utils/__init__.py +15 -0
  48. src/utils/__pycache__/__init__.cpython-314.pyc +0 -0
  49. src/utils/__pycache__/config.cpython-314.pyc +0 -0
  50. src/utils/__pycache__/io.cpython-314.pyc +0 -0
.dockerignore ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.py[cod]
3
+ *$py.class
4
+ .env
5
+ .venv
6
+ env/
7
+ venv/
8
+ ENV/
9
+ venv.bak/
10
+ env.bak/
11
+ .git/
12
+ .github/
13
+ .vscode/
14
+ data/
15
+ results/
16
+ results_dir/
17
+ notebooks/
18
+ docs/
19
+ *.mp4
20
+ *.wav
21
+ *.csv
22
+ !pretraining_dataset.csv
Dockerfile ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use an official Python runtime as a parent image
2
+ FROM python:3.10-slim
3
+
4
+ # Set environment variables
5
+ ENV PYTHONDONTWRITEBYTECODE 1
6
+ ENV PYTHONUNBUFFERED 1
7
+
8
+ # Install system dependencies
9
+ RUN apt-get update && apt-get install -y \
10
+ ffmpeg \
11
+ libgl1-mesa-glx \
12
+ libglib2.0-0 \
13
+ && rm -rf /var/lib/apt/lists/*
14
+
15
+ # Set the working directory in the container
16
+ WORKDIR /app
17
+
18
+ # Copy the requirements file into the container
19
+ COPY requirements.txt .
20
+
21
+ # Install dependencies
22
+ # We use opencv-python-headless for the server environment
23
+ RUN pip install --no-cache-dir -r requirements.txt
24
+ RUN pip install --no-cache-dir opencv-python-headless
25
+
26
+ # Copy the rest of the application code
27
+ COPY . .
28
+
29
+ # Ensure models directory exists
30
+ RUN mkdir -p models
31
+
32
+ # Expose the port the app runs on (Hugging Face default)
33
+ EXPOSE 7860
34
+
35
+ # Command to run the application
36
+ CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "7860"]
README.md CHANGED
@@ -1,10 +1,360 @@
1
- ---
2
- title: Deception Analysis Api
3
- emoji: 🌖
4
- colorFrom: yellow
5
- colorTo: red
6
- sdk: docker
7
- pinned: false
8
- ---
9
-
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🕵️‍♂️ Explainable Multimodal Deception Analysis System
2
+
3
+ A comprehensive system for detecting deception in video interviews using facial micro-expressions, audio speech patterns, temporal dynamics, and SHAP-based explainability. Works with or without real datasets using synthetic data generation.
4
+
5
+ ## 🚀 Quick Start (Google Colab - No Local Installation)
6
+
7
+ **Run everything in the cloud with one click:**
8
+
9
+ [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/apexprit/deception-analysis/blob/main/colab_deception_analysis.ipynb)
10
+
11
+ ### Steps to Run in Google Colab:
12
+
13
+ 1. **Open the Colab notebook** using the link above
14
+ 2. **Click "Runtime" → "Run all"** (Ctrl+F9)
15
+ 3. **Wait for installation** (~2-3 minutes)
16
+ 4. **Use the Gradio interface** that appears at the bottom
17
+
18
+ > [!TIP]
19
+ > **Working locally?** Make sure to `git push` your latest changes to GitHub before running in Colab, as the notebook clones the repository directly from the `main` branch.
20
+
21
+ The notebook will:
22
+ - ✅ Install all dependencies automatically
23
+ - ✅ Generate realistic synthetic deception data
24
+ - ✅ Train a deception detection model (~85% accuracy)
25
+ - ✅ Launch interactive web interface
26
+ - ✅ Provide sample analysis with visualizations
27
+
28
+ ## 📋 Features
29
+
30
+ ### 🔍 **Multimodal Analysis**
31
+ - **Facial Features**: 16 micro-expressions (eye openness, mouth movement, brow tension, head dynamics)
32
+ - **Audio Features**: 30+ speech patterns (pitch variation, pauses, MFCC coefficients, spectral features)
33
+ - **Cross-modal Features**: Audio-visual sync, pause-gaze coupling, stress composite
34
+ - **Temporal Analysis**: Frame-by-frame probability tracking, spike detection, suspicious intervals
35
+
36
+ ### 🧠 **Explainability**
37
+ - **SHAP-based feature importance** with behavioral interpretations
38
+ - **Feature group analysis** (facial vs. audio vs. cross-modal)
39
+ - **Human-readable explanations** of deception indicators
40
+ - **Publication-quality visualizations**
41
+
42
+ ### 🎯 **Subject Adaptation**
43
+ - **Personal calibration** to reduce inter-subject variability
44
+ - **Baseline establishment** for individual behavioral patterns
45
+ - **Adaptive thresholding** based on subject history
46
+
47
+ ### 📊 **Evaluation & Validation**
48
+ - **Comprehensive metrics**: Accuracy, Precision, Recall, F1, AUC-ROC
49
+ - **Cross-validation**: Stratified, subject-aware, leave-one-subject-out
50
+ - **Ablation studies**: Feature group importance analysis
51
+ - **Bootstrap confidence intervals** for robust metrics
52
+
53
+ ## 🏗️ Architecture
54
+
55
+ ```
56
+ Video Input
57
+ │
58
+ ├── Facial Feature Extraction (MediaPipe Face Mesh)
59
+ │ └── 16 micro-expression features
60
+ │
61
+ ├── Audio Feature Extraction (librosa)
62
+ │ └── 30+ speech pattern features
63
+ │
64
+ └── Multimodal Fusion
65
+ └── Temporal alignment + cross-modal features
66
+ │
67
+ ├── Deception Classifier (Gradient Boosting)
68
+ │ └── Probability prediction (0-1)
69
+ │
70
+ ├── Subject Calibration
71
+ │ └── Personalized baseline adjustment
72
+ │
73
+ ├── Temporal Analysis
74
+ │ └── Spike detection, interval localization
75
+ │
76
+ └── Explainability Engine (SHAP)
77
+ └── Feature importance + behavioral interpretation
78
+ ```
79
+
80
+ ## 📁 Project Structure
81
+
82
+ ```
83
+ deception-analysis/
84
+ ├── app.py # Gradio web interface
85
+ ├── train.py # CLI training script
86
+ ├── inference.py # CLI inference script
87
+ ├── create_pretrained_model.py # Generate pre-trained model
88
+ ├── requirements.txt # Dependencies
89
+ ├── colab_deception_analysis.ipynb # Google Colab notebook
90
+ ├── notebooks/
91
+ │ ├── demo.ipynb # Full workflow demonstration
92
+ │ └── demo_with_synthetic_data.ipynb # Synthetic data demo
93
+ ├── data/ # Generated synthetic data
94
+ ├── models/ # Pre-trained models
95
+ └── src/ # Core modules
96
+ ├── pipeline.py # Main orchestration
97
+ ├── facial/ # Facial analysis (extractor, visualizer)
98
+ ├── audio/ # Audio analysis (extractor, fusion)
99
+ ├── temporal/ # Temporal analysis (analyzer, visualizer)
100
+ ├── model/ # Classifier + calibration
101
+ ├── explainability/ # SHAP explanations (explainer, visualizer)
102
+ ├── evaluation/ # Metrics, CV, ablation, plots
103
+ └── utils/ # Config, I/O, synthetic data, downloader
104
+ ```
105
+
106
+ ## 🛠️ Local Installation (Optional)
107
+
108
+ If you want to run locally:
109
+
110
+ ```bash
111
+ # Clone repository
112
+ git clone https://github.com/apexprit/deception-analysis.git
113
+ cd deception-analysis
114
+
115
+ # Install dependencies
116
+ pip install -r requirements.txt
117
+
118
+ # Generate synthetic data and pre-trained model
119
+ python create_pretrained_model.py
120
+
121
+ # Train on synthetic data
122
+ python train.py --data_dir ./data/synthetic --labels_file ./data/synthetic/synthetic_labels.csv
123
+
124
+ # Run inference (demo mode)
125
+ python inference.py --model ./models/deception_model.pkl --video_dir ./data/sample/
126
+
127
+ # Launch web interface
128
+ python app.py --port 7860
129
+ ```
130
+
131
+ ## 📊 Usage Examples
132
+
133
+ ### 1. **Basic Analysis**
134
+ ```python
135
+ from src.pipeline import DeceptionPipeline
136
+ from src.utils.config import AppConfig
137
+
138
+ # Initialize pipeline
139
+ config = AppConfig()
140
+ pipeline = DeceptionPipeline(config)
141
+
142
+ # Analyze a video
143
+ result = pipeline.analyze_video(
144
+ video_path="interview.mp4",
145
+ subject_id="S001",
146
+ generate_visualizations=True,
147
+ output_dir="./results"
148
+ )
149
+
150
+ print(f"Deception Probability: {result['deception_probability']:.3f}")
151
+ print(f"Prediction: {result['prediction']}")
152
+ print(f"Confidence: {result['confidence']:.3f}")
153
+ ```
154
+
155
+ ### 2. **Batch Processing**
156
+ ```python
157
+ # Analyze multiple videos
158
+ results = pipeline.analyze_batch(
159
+ video_paths=["video1.mp4", "video2.mp4", "video3.mp4"],
160
+ subject_ids=["S001", "S002", "S003"],
161
+ generate_visualizations=False,
162
+ output_dir="./batch_results"
163
+ )
164
+ ```
165
+
166
+ ### 3. **Model Training**
167
+ ```python
168
+ # Train on your dataset
169
+ train_result = pipeline.train(
170
+ video_paths=["train1.mp4", "train2.mp4", "train3.mp4"],
171
+ labels=[0, 1, 0], # 0=truthful, 1=deceptive
172
+ subject_ids=["S001", "S002", "S003"],
173
+ output_dir="./models"
174
+ )
175
+
176
+ print(f"Training Accuracy: {train_result['accuracy']:.3f}")
177
+ print(f"Training F1: {train_result['f1']:.3f}")
178
+ ```
179
+
180
+ ### 4. **Synthetic Data Generation**
181
+ ```python
182
+ from src.utils.synthetic_data import SyntheticDataGenerator
183
+
184
+ # Generate synthetic dataset
185
+ generator = SyntheticDataGenerator()
186
+ features, labels, subject_ids = generator.generate_dataset(
187
+ n_truthful=500,
188
+ n_deceptive=500
189
+ )
190
+
191
+ # Save for training
192
+ generator.save_synthetic_dataset(
193
+ output_dir="./data/synthetic",
194
+ n_truthful=500,
195
+ n_deceptive=500,
196
+ dataset_name="my_dataset"
197
+ )
198
+ ```
199
+
200
+ ## 🧪 Evaluation
201
+
202
+ ```python
203
+ from src.evaluation.metrics import DeceptionMetrics
204
+ from src.evaluation.cross_validation import CrossValidator
205
+ from src.evaluation.ablation import AblationStudy
206
+
207
+ # Compute metrics
208
+ metrics = DeceptionMetrics()
209
+ all_metrics = metrics.compute_all_metrics(y_true, y_pred, y_prob)
210
+
211
+ # Cross-validation
212
+ cv = CrossValidator()
213
+ cv_results = cv.run_cross_validation(
214
+ classifier=DeceptionClassifier,
215
+ X=features,
216
+ y=labels,
217
+ n_splits=5,
218
+ method='stratified'
219
+ )
220
+
221
+ # Ablation study
222
+ ablation = AblationStudy(feature_names, feature_groups)
223
+ ablation_results = ablation.run_ablation(
224
+ classifier_class=DeceptionClassifier,
225
+ X=features,
226
+ y=labels,
227
+ feature_names=feature_names,
228
+ feature_groups=feature_groups,
229
+ n_splits=5
230
+ )
231
+ ```
232
+
233
+ ## 🔧 Configuration
234
+
235
+ The system is highly configurable via `src/utils/config.py`:
236
+
237
+ ```python
238
+ from src.utils.config import AppConfig
239
+
240
+ config = AppConfig(
241
+ project=ProjectConfig(
242
+ data_dir="./data",
243
+ model_dir="./models",
244
+ results_dir="./results",
245
+ random_seed=42
246
+ ),
247
+ feature=FeatureConfig(
248
+ fps=15,
249
+ face_detection_confidence=0.5,
250
+ landmark_refinement=True
251
+ ),
252
+ audio=AudioConfig(
253
+ sample_rate=16000,
254
+ n_fft=2048,
255
+ hop_length=512,
256
+ n_mfcc=13
257
+ ),
258
+ model=ModelConfig(
259
+ model_type="gradient_boosting",
260
+ n_estimators=100,
261
+ learning_rate=0.1,
262
+ max_depth=3
263
+ ),
264
+ temporal=TemporalConfig(
265
+ spike_threshold=0.7,
266
+ min_spike_duration=0.5,
267
+ smoothing_window=5
268
+ )
269
+ )
270
+ ```
271
+
272
+ ## 📈 Performance
273
+
274
+ With synthetic data (1000 samples, 80/20 train/test split):
275
+ - **Accuracy**: ~85%
276
+ - **F1-Score**: ~0.83
277
+ - **AUC-ROC**: ~0.91
278
+ - **Precision**: ~0.82
279
+ - **Recall**: ~0.84
280
+
281
+ **Feature Importance** (top 5):
282
+ 1. `pause_count` (audio) - More pauses indicate deception
283
+ 2. `eye_left_openness` (facial) - Less eye openness indicates deception
284
+ 3. `pitch_variation_coefficient` (audio) - Higher variation indicates deception
285
+ 4. `brow_tension` (facial) - Higher tension indicates deception
286
+ 5. `audio_visual_sync` (cross-modal) - Less sync indicates deception
287
+
288
+ ## 🚀 Deployment Options
289
+
290
+ ### 1. **Google Colab** (Recommended for no local setup)
291
+ - Open `colab_deception_analysis.ipynb` in Colab
292
+ - Run all cells
293
+ - Use the Gradio interface
294
+
295
+ ### 2. **Local Python Environment**
296
+ ```bash
297
+ pip install -r requirements.txt
298
+ python app.py
299
+ ```
300
+
301
+ ### 3. **Docker Container**
302
+ ```dockerfile
303
+ FROM python:3.9
304
+ COPY . /app
305
+ WORKDIR /app
306
+ RUN pip install -r requirements.txt
307
+ CMD ["python", "app.py", "--host", "0.0.0.0", "--port", "7860"]
308
+ ```
309
+
310
+ ### 4. **Cloud Services**
311
+ - **Hugging Face Spaces**: Deploy Gradio app
312
+ - **Vercel/Heroku**: Deploy as web service
313
+ - **AWS SageMaker**: Deploy as ML endpoint
314
+ - **Google Cloud Run**: Containerized deployment
315
+
316
+ ## 📚 Research Basis
317
+
318
+ The system implements features based on established deception detection research:
319
+
320
+ 1. **Facial Indicators** (Ekman, 2009):
321
+ - Micro-expressions (brief facial expressions)
322
+ - Eye gaze avoidance
323
+ - Facial asymmetry
324
+
325
+ 2. **Vocal Indicators** (DePaulo et al., 2003):
326
+ - Higher pitch variation
327
+ - More speech errors
328
+ - Longer response latency
329
+
330
+ 3. **Multimodal Integration** (Pérez-Rosas & Mihalcea, 2014):
331
+ - Audio-visual feature fusion
332
+ - Temporal alignment
333
+ - Cross-modal correlation analysis
334
+
335
+ ## ⚠️ Ethical Considerations
336
+
337
+ 1. **Privacy**: Video/audio data should be anonymized and stored securely
338
+ 2. **Consent**: Subjects must provide informed consent for analysis
339
+ 3. **Bias**: Models may have demographic biases - regular auditing required
340
+ 4. **Interpretation**: Results are probabilistic indicators, not definitive truth
341
+ 5. **Context**: Consider cultural and situational factors in interpretation
342
+
343
+ ## 🤝 Contributing
344
+
345
+ 1. Fork the repository
346
+ 2. Create a feature branch (`git checkout -b feature/amazing-feature`)
347
+ 3. Commit changes (`git commit -m 'Add amazing feature'`)
348
+ 4. Push to branch (`git push origin feature/amazing-feature`)
349
+ 5. Open a Pull Request
350
+
351
+ ## 📄 License
352
+
353
+ MIT License - see LICENSE file for details
354
+
355
+ ## 📧 Contact
356
+
357
+ For questions or support, please open an issue on GitHub.
358
+
359
+ ---
360
+
api.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import tempfile
3
+ import logging
4
+ from typing import Optional
5
+
6
+ from fastapi import FastAPI, UploadFile, File, Form, HTTPException
7
+ from fastapi.responses import JSONResponse
8
+ from fastapi.middleware.cors import CORSMiddleware
9
+
10
+ from src.pipeline import DeceptionPipeline
11
+ from src.utils.config import get_default_config
12
+
13
+ logging.basicConfig(level=logging.INFO)
14
+ logger = logging.getLogger(__name__)
15
+
16
+ app = FastAPI(
17
+ title="Deception Analysis API",
18
+ description="API for Explainable Multimodal Deception Analysis for Mobile Apps"
19
+ )
20
+
21
+ # Enable CORS so the Flutter app can connect easily
22
+ app.add_middleware(
23
+ CORSMiddleware,
24
+ allow_origins=["*"],
25
+ allow_credentials=True,
26
+ allow_methods=["*"],
27
+ allow_headers=["*"],
28
+ )
29
+
30
+ # Global pipeline instance
31
+ pipeline = None
32
+
33
+ @app.on_event("startup")
34
+ async def startup_event():
35
+ global pipeline
36
+ try:
37
+ logger.info("Initializing Deception ML Pipeline...")
38
+ config = get_default_config()
39
+ pipeline = DeceptionPipeline(config)
40
+
41
+ # Load the pre-trained model if it exists
42
+ model_path = os.path.join("models", "deception_model.pkl")
43
+ if os.path.exists(model_path):
44
+ pipeline.load_model(model_path)
45
+ logger.info("Pre-trained model loaded successfully.")
46
+ else:
47
+ logger.warning(f"No pre-trained model found at {model_path}. You must generate it first!")
48
+
49
+ logger.info("Pipeline initialized successfully.")
50
+ except Exception as e:
51
+ logger.error(f"Failed to initialize pipeline: {e}")
52
+
53
+ @app.post("/analyze")
54
+ async def analyze_video_endpoint(
55
+ file: UploadFile = File(...),
56
+ subject_id: Optional[str] = Form(None)
57
+ ):
58
+ if not pipeline:
59
+ raise HTTPException(status_code=500, detail="ML Pipeline not initialized yet")
60
+
61
+ # Create temporary directory for the uploaded video
62
+ temp_dir = tempfile.mkdtemp(prefix="deception_api_")
63
+ video_path = os.path.join(temp_dir, file.filename)
64
+
65
+ try:
66
+ # Save uploaded video to disk
67
+ with open(video_path, "wb") as f:
68
+ content = await file.read()
69
+ f.write(content)
70
+
71
+ logger.info(f"Analyzing video: {file.filename} (Subject: {subject_id})")
72
+
73
+ # Run pipeline
74
+ result = pipeline.analyze_video(
75
+ video_path=video_path,
76
+ subject_id=subject_id,
77
+ generate_visualizations=False, # We don't need Matplotlib images, just JSON data
78
+ output_dir=temp_dir
79
+ )
80
+
81
+ if "error" in result:
82
+ raise HTTPException(status_code=400, detail=result["error"])
83
+
84
+ # Format clean JSON response for the Flutter app
85
+ response_data = {
86
+ "status": "success",
87
+ "prediction": result.get("prediction", "unknown"),
88
+ "verdict": result.get("prediction", "unknown"), # added for Flutter
89
+ "probability": result.get("deception_probability", 0.5),
90
+ "confidence": result.get("confidence", 0.0),
91
+ "is_deceptive": result.get("prediction") == "deceptive",
92
+ }
93
+
94
+ # Add temporal analysis (to draw the timeline graph in Flutter)
95
+ if "temporal_analysis" in result:
96
+ response_data["temporal"] = {
97
+ "timestamps": result["temporal_analysis"].get("timestamps", []),
98
+ "probabilities": result["temporal_analysis"].get("probability_trajectory", []),
99
+ "suspicious_intervals": result["temporal_analysis"].get("suspicious_intervals", [])
100
+ }
101
+
102
+ # Add SHAP explainability (to draw the Feature Importance chart in Flutter)
103
+ if "explanation" in result:
104
+ fi = result["explanation"].get("feature_importance", {})
105
+ top_indicators = [{"feature": k, "impact": v} for k, v in list(fi.items())[:5]]
106
+
107
+ response_data["explanation"] = {
108
+ "top_indicators": top_indicators,
109
+ "feature_importance": fi,
110
+ "behavioral_explanations": result["explanation"].get("behavioral_explanations", {})
111
+ }
112
+
113
+ return JSONResponse(content=response_data)
114
+
115
+ except Exception as e:
116
+ logger.error(f"Analysis failed: {str(e)}")
117
+ raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
118
+ finally:
119
+ # Clean up the heavy video file to save server storage
120
+ try:
121
+ if os.path.exists(video_path):
122
+ os.remove(video_path)
123
+ os.rmdir(temp_dir)
124
+ except:
125
+ pass
126
+
127
+ @app.get("/health")
128
+ async def health_check():
129
+ return {"status": "online", "pipeline_ready": pipeline is not None}
app.py ADDED
@@ -0,0 +1,596 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Gradio demo application for the Explainable Multimodal Deception Analysis System."""
2
+
3
+ import argparse
4
+ import logging
5
+ import os
6
+ import tempfile
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Dict, Any
10
+ import json
11
+
12
+ import gradio as gr
13
+ import matplotlib.pyplot as plt
14
+ import numpy as np
15
+ import pandas as pd
16
+ from matplotlib.figure import Figure
17
+
18
+ from src.pipeline import DeceptionPipeline
19
+ from src.utils.config import AppConfig, get_default_config
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ # Global state
24
+ pipeline: Optional[DeceptionPipeline] = None
25
+ config: Optional[AppConfig] = None
26
+ current_model_path: Optional[str] = None
27
+
28
+ # Custom CSS for styling
29
+ custom_css = """
30
+ /* Deception assessment styling */
31
+ .deceptive-badge {
32
+ background-color: #ffcccc;
33
+ color: #b30000;
34
+ padding: 8px 16px;
35
+ border-radius: 20px;
36
+ font-weight: bold;
37
+ border: 2px solid #ff6666;
38
+ display: inline-block;
39
+ margin: 5px;
40
+ }
41
+ .truthful-badge {
42
+ background-color: #ccffcc;
43
+ color: #006600;
44
+ padding: 8px 16px;
45
+ border-radius: 20px;
46
+ font-weight: bold;
47
+ border: 2px solid #66cc66;
48
+ display: inline-block;
49
+ margin: 5px;
50
+ }
51
+ .error-badge {
52
+ background-color: #ffebcc;
53
+ color: #cc6600;
54
+ padding: 8px 16px;
55
+ border-radius: 20px;
56
+ font-weight: bold;
57
+ border: 2px solid #ff9933;
58
+ display: inline-block;
59
+ margin: 5px;
60
+ }
61
+ /* Professional font choices */
62
+ body {
63
+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
64
+ }
65
+ /* Proper spacing */
66
+ .gr-container {
67
+ padding: 20px;
68
+ }
69
+ .gr-tab {
70
+ padding: 20px;
71
+ }
72
+ /* Highlighted suspicious intervals */
73
+ .suspicious-interval {
74
+ background-color: #fff0f0;
75
+ border-left: 4px solid #ff6666;
76
+ padding: 8px 12px;
77
+ margin: 5px 0;
78
+ }
79
+ /* Progress bar styling */
80
+ .gr-progress-bar {
81
+ height: 8px;
82
+ border-radius: 4px;
83
+ }
84
+ """
85
+
86
+ def initialize_pipeline(config_path: Optional[str] = None) -> DeceptionPipeline:
87
+ """Initialize or reinitialize the pipeline."""
88
+ global pipeline, config
89
+ try:
90
+ if config_path and os.path.exists(config_path):
91
+ # In a real implementation, you'd load config from file
92
+ logger.warning(f"Config file loading not implemented, using default config")
93
+ config = get_default_config()
94
+ pipeline = DeceptionPipeline(config)
95
+ logger.info("Pipeline initialized with default config.")
96
+ return pipeline
97
+ except Exception as e:
98
+ logger.error(f"Failed to initialize pipeline: {e}")
99
+ raise
100
+
101
+ def get_pipeline() -> DeceptionPipeline:
102
+ """Get or create the pipeline instance."""
103
+ global pipeline
104
+ if pipeline is None:
105
+ pipeline = initialize_pipeline()
106
+ return pipeline
107
+
108
+ def analyze_video(
109
+ video_file: Optional[str],
110
+ subject_id: Optional[str],
111
+ threshold: float,
112
+ progress=gr.Progress()
113
+ ) -> Tuple[Dict[str, Any], Optional[Figure], Optional[Figure], Optional[pd.DataFrame], Optional[pd.DataFrame]]:
114
+ """Run deception analysis on uploaded video."""
115
+ if video_file is None:
116
+ return {
117
+ "error": "No video file provided"
118
+ }, None, None, None, None
119
+
120
+ # Save uploaded video to a temporary file
121
+ temp_dir = tempfile.mkdtemp(prefix="deception_video_")
122
+ try:
123
+ video_path = os.path.join(temp_dir, "uploaded_video.mp4")
124
+ # Copy the uploaded file
125
+ with open(video_file, 'rb') as src, open(video_path, 'wb') as dst:
126
+ dst.write(src.read())
127
+
128
+ progress(0.1, desc="Initializing pipeline...")
129
+ pipeline = get_pipeline()
130
+
131
+ progress(0.2, desc="Analyzing video (this may take a minute)...")
132
+ # Call the pipeline's analyze_video method
133
+ result = pipeline.analyze_video(
134
+ video_path=video_path,
135
+ subject_id=subject_id if subject_id else None,
136
+ generate_visualizations=True,
137
+ output_dir=temp_dir
138
+ )
139
+
140
+ progress(0.8, desc="Processing results...")
141
+
142
+ # Check for error
143
+ if "error" in result:
144
+ return {
145
+ "error": result["error"]
146
+ }, None, None, None, None
147
+
148
+ # Create temporal analysis plot
149
+ temporal_fig = None
150
+ if "temporal_analysis" in result and "timestamps" in result["temporal_analysis"]:
151
+ try:
152
+ temporal_fig = create_temporal_plot(
153
+ result["temporal_analysis"]["timestamps"],
154
+ result["temporal_analysis"]["probability_trajectory"],
155
+ result["temporal_analysis"].get("suspicious_intervals", [])
156
+ )
157
+ except Exception as e:
158
+ logger.warning(f"Failed to create temporal plot: {e}")
159
+
160
+ # Create feature importance plot
161
+ feature_fig = None
162
+ if "explanation" in result and "feature_importance" in result["explanation"]:
163
+ try:
164
+ feature_fig = create_feature_importance_plot(
165
+ result["explanation"]["feature_importance"]
166
+ )
167
+ except Exception as e:
168
+ logger.warning(f"Failed to create feature importance plot: {e}")
169
+
170
+ # Prepare behavioral indicators dataframe
171
+ behavioral_df = None
172
+ if "explanation" in result and "behavioral_explanations" in result["explanation"]:
173
+ try:
174
+ behavioral_df = create_behavioral_dataframe(
175
+ result["explanation"]["behavioral_explanations"]
176
+ )
177
+ except Exception as e:
178
+ logger.warning(f"Failed to create behavioral dataframe: {e}")
179
+
180
+ # Prepare suspicious intervals dataframe
181
+ suspicious_df = None
182
+ if "temporal_analysis" in result and "suspicious_intervals" in result["temporal_analysis"]:
183
+ try:
184
+ suspicious_df = create_suspicious_intervals_dataframe(
185
+ result["temporal_analysis"]["suspicious_intervals"]
186
+ )
187
+ except Exception as e:
188
+ logger.warning(f"Failed to create suspicious intervals dataframe: {e}")
189
+
190
+ # Format overall assessment
191
+ assessment = {
192
+ "prediction": result.get("prediction", "unknown"),
193
+ "probability": result.get("deception_probability", 0.5),
194
+ "confidence": result.get("confidence", 0.0),
195
+ "calibrated": result.get("calibrated", False),
196
+ "threshold": threshold,
197
+ "is_deceptive": result.get("prediction") == "deceptive" if "prediction" in result else False
198
+ }
199
+
200
+ progress(1.0, desc="Analysis complete!")
201
+ return assessment, temporal_fig, feature_fig, behavioral_df, suspicious_df
202
+
203
+ except Exception as e:
204
+ logger.error(f"Analysis failed: {e}")
205
+ return {
206
+ "error": f"Analysis failed: {str(e)}"
207
+ }, None, None, None, None
208
+ finally:
209
+ # Clean up temp directory after a short delay (or keep for debugging)
210
+ # shutil.rmtree(temp_dir, ignore_errors=True)
211
+ pass
212
+
213
+ def create_temporal_plot(timestamps, probabilities, suspicious_intervals) -> Figure:
214
+ """Create matplotlib figure for temporal analysis."""
215
+ fig, ax = plt.subplots(figsize=(10, 5))
216
+ ax.plot(timestamps, probabilities, 'b-', linewidth=2, label='Deception Probability')
217
+ ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5, label='Neutral (0.5)')
218
+ ax.set_xlabel('Time (seconds)', fontsize=12)
219
+ ax.set_ylabel('Deception Probability', fontsize=12)
220
+ ax.set_title('Deception Probability Over Time', fontsize=14, fontweight='bold')
221
+ ax.grid(True, alpha=0.3)
222
+ ax.set_ylim(0, 1)
223
+
224
+ # Highlight suspicious intervals
225
+ for interval in suspicious_intervals:
226
+ if len(interval) >= 2:
227
+ start, end = interval[0], interval[1]
228
+ ax.axvspan(start, end, alpha=0.2, color='red', label='Suspicious Interval' if 'Suspicious Interval' not in ax.get_legend_handles_labels()[1] else '')
229
+
230
+ ax.legend(loc='upper right')
231
+ fig.tight_layout()
232
+ return fig
233
+
234
+ def create_feature_importance_plot(feature_importance: Dict[str, float]) -> Figure:
235
+ """Create matplotlib figure for top feature importance."""
236
+ # Sort features by absolute importance
237
+ sorted_features = sorted(feature_importance.items(), key=lambda x: abs(x[1]), reverse=True)
238
+ top_features = sorted_features[:10] # Top 10 features
239
+
240
+ if not top_features:
241
+ # Create empty plot
242
+ fig, ax = plt.subplots(figsize=(8, 5))
243
+ ax.text(0.5, 0.5, 'No feature importance data available',
244
+ ha='center', va='center', fontsize=12)
245
+ ax.set_xlim(0, 1)
246
+ ax.set_ylim(0, 1)
247
+ ax.axis('off')
248
+ return fig
249
+
250
+ features, importances = zip(*top_features)
251
+ # Shorten long feature names for display
252
+ display_features = [f[:30] + '...' if len(f) > 30 else f for f in features]
253
+
254
+ fig, ax = plt.subplots(figsize=(10, 6))
255
+ colors = ['red' if imp < 0 else 'green' for imp in importances]
256
+ y_pos = np.arange(len(display_features))
257
+ ax.barh(y_pos, importances, color=colors, alpha=0.7)
258
+ ax.set_yticks(y_pos)
259
+ ax.set_yticklabels(display_features, fontsize=10)
260
+ ax.set_xlabel('SHAP Value (Impact on Prediction)', fontsize=12)
261
+ ax.set_title('Top 10 Feature Importance for Deception Prediction', fontsize=14, fontweight='bold')
262
+ ax.grid(True, alpha=0.3, axis='x')
263
+ fig.tight_layout()
264
+ return fig
265
+
266
+ def create_behavioral_dataframe(behavioral_explanations: Dict[str, Any]) -> pd.DataFrame:
267
+ """Create dataframe for behavioral indicators."""
268
+ rows = []
269
+ for category, indicators in behavioral_explanations.items():
270
+ if isinstance(indicators, dict):
271
+ for indicator, value in indicators.items():
272
+ rows.append({
273
+ 'Category': category,
274
+ 'Indicator': indicator,
275
+ 'Value': value,
276
+ 'Interpretation': interpret_behavioral_indicator(category, indicator, value)
277
+ })
278
+ else:
279
+ rows.append({
280
+ 'Category': category,
281
+ 'Indicator': 'Overall',
282
+ 'Value': indicators,
283
+ 'Interpretation': interpret_behavioral_indicator(category, 'Overall', indicators)
284
+ })
285
+
286
+ if rows:
287
+ df = pd.DataFrame(rows)
288
+ return df
289
+ return pd.DataFrame(columns=['Category', 'Indicator', 'Value', 'Interpretation'])
290
+
291
+ def interpret_behavioral_indicator(category: str, indicator: str, value: Any) -> str:
292
+ """Provide human-readable interpretation of behavioral indicators."""
293
+ # Simplified interpretation logic
294
+ try:
295
+ val = float(value)
296
+ if category == 'facial':
297
+ if 'eye_contact' in indicator.lower():
298
+ return "Low eye contact may indicate deception" if val < 0.5 else "Normal eye contact"
299
+ elif 'blink_rate' in indicator.lower():
300
+ return "Elevated blink rate may indicate stress" if val > 0.7 else "Normal blink rate"
301
+ elif category == 'audio':
302
+ if 'pitch_variability' in indicator.lower():
303
+ return "Reduced pitch variability may indicate rehearsed speech" if val < 0.3 else "Normal pitch variability"
304
+ elif 'speech_rate' in indicator.lower():
305
+ return "Increased speech rate may indicate anxiety" if val > 1.2 else "Normal speech rate"
306
+ elif category == 'cross_modal':
307
+ return "Incongruence between facial and audio cues detected" if val > 0.6 else "Consistent multimodal behavior"
308
+ except:
309
+ pass
310
+ return "No specific interpretation available"
311
+
312
+ def create_suspicious_intervals_dataframe(suspicious_intervals: list) -> pd.DataFrame:
313
+ """Create dataframe for suspicious intervals."""
314
+ if not suspicious_intervals:
315
+ return pd.DataFrame(columns=['Start Time (s)', 'End Time (s)', 'Duration (s)', 'Max Probability', 'Severity'])
316
+
317
+ rows = []
318
+ for i, interval in enumerate(suspicious_intervals):
319
+ if len(interval) >= 4:
320
+ start, end, max_prob, severity = interval[0], interval[1], interval[2], interval[3]
321
+ elif len(interval) >= 2:
322
+ start, end = interval[0], interval[1]
323
+ max_prob = 0.8 # default
324
+ severity = 'medium'
325
+ else:
326
+ continue
327
+
328
+ rows.append({
329
+ 'Start Time (s)': f"{start:.2f}",
330
+ 'End Time (s)': f"{end:.2f}",
331
+ 'Duration (s)': f"{end - start:.2f}",
332
+ 'Max Probability': f"{max_prob:.3f}",
333
+ 'Severity': severity
334
+ })
335
+
336
+ df = pd.DataFrame(rows)
337
+ return df
338
+
339
+ def load_model(model_path: str, progress=gr.Progress()) -> Dict[str, Any]:
340
+ """Load a pre-trained model."""
341
+ global pipeline, current_model_path
342
+ try:
343
+ progress(0.3, desc="Checking model file...")
344
+ if not os.path.exists(model_path):
345
+ return {"error": f"Model file not found: {model_path}"}
346
+
347
+ pipeline = get_pipeline()
348
+ progress(0.6, desc="Loading model...")
349
+ # Assuming pipeline has a load_model method
350
+ if hasattr(pipeline, 'load_model'):
351
+ pipeline.load_model(model_path)
352
+ else:
353
+ # Try to load via classifier
354
+ if hasattr(pipeline.classifier, 'load'):
355
+ pipeline.classifier.load(model_path)
356
+ else:
357
+ return {"error": "Model loading not implemented in pipeline"}
358
+
359
+ current_model_path = model_path
360
+ progress(1.0, desc="Model loaded successfully!")
361
+ return {
362
+ "status": "success",
363
+ "model_path": model_path,
364
+ "feature_count": getattr(pipeline.classifier, 'n_features_', 'unknown'),
365
+ "is_fitted": getattr(pipeline.classifier, 'is_fitted_', False)
366
+ }
367
+ except Exception as e:
368
+ logger.error(f"Failed to load model: {e}")
369
+ return {"error": f"Failed to load model: {str(e)}"}
370
+
371
+ def get_model_info() -> Dict[str, Any]:
372
+ """Return current model information."""
373
+ global pipeline, current_model_path
374
+ if pipeline is None:
375
+ return {"status": "No pipeline initialized"}
376
+
377
+ classifier = pipeline.classifier
378
+ return {
379
+ "model_path": current_model_path or "Not loaded",
380
+ "is_fitted": getattr(classifier, 'is_fitted_', False),
381
+ "feature_count": getattr(classifier, 'n_features_', 'unknown'),
382
+ "model_type": type(classifier).__name__,
383
+ "config": {
384
+ "decision_threshold": pipeline.config.model.decision_threshold,
385
+ "n_estimators": pipeline.config.model.n_estimators,
386
+ "calibration_method": pipeline.config.model.calibration_method
387
+ }
388
+ }
389
+
390
+ def update_settings(
391
+ fps: int,
392
+ audio_segment_length: int,
393
+ generate_visualizations: bool,
394
+ output_dir: str
395
+ ) -> Dict[str, Any]:
396
+ """Update pipeline settings."""
397
+ global pipeline, config
398
+ try:
399
+ # Note: AppConfig is frozen (immutable), so we cannot modify it directly.
400
+ # In a real implementation, we would create a new config with updated values.
401
+ # For this demo, we'll just store the settings and apply them when pipeline is reinitialized.
402
+ settings = {
403
+ "fps": fps,
404
+ "audio_segment_length": audio_segment_length,
405
+ "generate_visualizations": generate_visualizations,
406
+ "output_dir": output_dir if output_dir.strip() else "Not specified (will use temp directory)",
407
+ "note": "Settings will be applied on next pipeline initialization"
408
+ }
409
+ logger.info(f"Settings updated: {settings}")
410
+ return {"status": "Settings updated (note: config is immutable, requires pipeline reinitialization)", "settings": settings}
411
+ except Exception as e:
412
+ logger.error(f"Failed to update settings: {e}")
413
+ return {"error": str(e)}
414
+
415
+ def format_assessment_html(assessment: Dict[str, Any]) -> str:
416
+ """Format assessment results as HTML with styling."""
417
+ if "error" in assessment:
418
+ return f"""
419
+ <div class="error-badge">
420
+ <h3>❌ Error</h3>
421
+ <p>{assessment['error']}</p>
422
+ </div>
423
+ """
424
+
425
+ prediction = assessment.get("prediction", "unknown")
426
+ probability = assessment.get("probability", 0.5)
427
+ confidence = assessment.get("confidence", 0.0)
428
+ calibrated = assessment.get("calibrated", False)
429
+ threshold = assessment.get("threshold", 0.5)
430
+
431
+ if prediction == "deceptive":
432
+ badge_class = "deceptive-badge"
433
+ verdict = "DECEPTIVE"
434
+ icon = "⚠️"
435
+ elif prediction == "truthful":
436
+ badge_class = "truthful-badge"
437
+ verdict = "TRUTHFUL"
438
+ icon = "✅"
439
+ else:
440
+ badge_class = "error-badge"
441
+ verdict = "UNKNOWN"
442
+ icon = "❓"
443
+
444
+ calibrated_text = " (calibrated)" if calibrated else ""
445
+
446
+ return f"""
447
+ <div class="{badge_class}">
448
+ <h2>{icon} {verdict}{calibrated_text}</h2>
449
+ <p><strong>Deception Probability:</strong> {probability:.3f}</p>
450
+ <p><strong>Confidence:</strong> {confidence:.3f}</p>
451
+ <p><strong>Decision Threshold:</strong> {threshold:.2f}</p>
452
+ <p><strong>Calibrated:</strong> {calibrated}</p>
453
+ </div>
454
+ """
455
+
456
+
457
+ # ============================================================================
458
+ # Gradio UI Definition
459
+ # ============================================================================
460
+
461
+ def create_ui():
462
+ """Create the Gradio interface with tabs."""
463
+ with gr.Blocks(css=custom_css, title="Explainable Multimodal Deception Analysis") as demo:
464
+ gr.Markdown("# 🔍 Explainable Multimodal Deception Analysis System")
465
+ gr.Markdown("Upload a video to analyze for deception indicators using facial, audio, and cross-modal features.")
466
+
467
+ with gr.Tabs():
468
+ # Tab 1: Analyze Video
469
+ with gr.TabItem("🔍 Analyze Video"):
470
+ with gr.Row():
471
+ with gr.Column(scale=1):
472
+ video_input = gr.Video(label="Upload Video", sources=["upload"], type="filepath")
473
+ subject_id = gr.Textbox(label="Subject ID (optional)", placeholder="Enter subject identifier")
474
+ threshold_slider = gr.Slider(minimum=0.0, maximum=1.0, value=0.5, step=0.05, label="Decision Threshold", info="Higher values make the model more conservative (less likely to classify as deceptive)")
475
+ analyze_btn = gr.Button("Analyze Video", variant="primary")
476
+
477
+ with gr.Column(scale=2):
478
+ assessment_html = gr.HTML(label="Deception Assessment")
479
+ with gr.Row():
480
+ temporal_plot = gr.Plot(label="Temporal Analysis")
481
+ feature_importance_plot = gr.Plot(label="Feature Importance")
482
+ with gr.Row():
483
+ behavioral_df = gr.Dataframe(label="Behavioral Indicators", headers=["Category", "Indicator", "Value", "Interpretation"])
484
+ suspicious_df = gr.Dataframe(label="Suspicious Intervals", headers=["Start Time (s)", "End Time (s)", "Duration (s)", "Max Probability", "Severity"])
485
+
486
+ analyze_btn.click(
487
+ fn=analyze_video,
488
+ inputs=[video_input, subject_id, threshold_slider],
489
+ outputs=[assessment_html, temporal_plot, feature_importance_plot, behavioral_df, suspicious_df]
490
+ )
491
+
492
+ # Tab 2: Model Info
493
+ with gr.TabItem("📊 Model Info"):
494
+ with gr.Row():
495
+ with gr.Column():
496
+ model_path_input = gr.Textbox(label="Model Path", placeholder="Path to trained model file (.pkl, .joblib, etc.)")
497
+ load_model_btn = gr.Button("Load Model", variant="secondary")
498
+ model_status = gr.JSON(label="Model Status")
499
+ gr.Markdown("### Current Model Information")
500
+ model_info_json = gr.JSON(label="Model Info")
501
+ refresh_btn = gr.Button("Refresh Model Info")
502
+
503
+ load_model_btn.click(
504
+ fn=load_model,
505
+ inputs=[model_path_input],
506
+ outputs=[model_status]
507
+ )
508
+ refresh_btn.click(
509
+ fn=get_model_info,
510
+ inputs=[],
511
+ outputs=[model_info_json]
512
+ )
513
+
514
+ # Tab 3: Settings
515
+ with gr.TabItem("⚙️ Settings"):
516
+ with gr.Row():
517
+ with gr.Column():
518
+ fps_slider = gr.Slider(minimum=1, maximum=60, value=30, step=1, label="Frames Per Second (FPS) for processing")
519
+ audio_segment_slider = gr.Slider(minimum=1, maximum=10, value=3, step=1, label="Audio Segment Length (seconds)")
520
+ visualization_checkbox = gr.Checkbox(value=True, label="Generate Visualizations")
521
+ output_dir_input = gr.Textbox(label="Output Directory (optional)", placeholder="Leave empty for temporary directory")
522
+ update_settings_btn = gr.Button("Update Settings", variant="secondary")
523
+ settings_status = gr.JSON(label="Settings Status")
524
+
525
+ update_settings_btn.click(
526
+ fn=update_settings,
527
+ inputs=[fps_slider, audio_segment_slider, visualization_checkbox, output_dir_input],
528
+ outputs=[settings_status]
529
+ )
530
+
531
+ # Tab 4: About
532
+ with gr.TabItem("📖 About"):
533
+ gr.Markdown("""
534
+ ## About This System
535
+
536
+ This is an **Explainable Multimodal Deception Analysis System** that uses:
537
+ - **Facial features**: Micro‑expressions, eye contact, blink rate, head pose
538
+ - **Audio features**: Pitch variability, speech rate, voice stress, spectral features
539
+ - **Cross‑modal fusion**: Combining facial and audio cues for robust detection
540
+
541
+ ### How It Works
542
+ 1. Upload a video of a person speaking
543
+ 2. The system extracts facial and audio features frame‑by‑frame
544
+ 3. A trained classifier predicts deception probability over time
545
+ 4. SHAP explanations highlight the most influential features
546
+ 5. Suspicious intervals are flagged for closer inspection
547
+
548
+ ### Interpretation
549
+ - **Deceptive**: Probability > decision threshold (adjustable)
550
+ - **Truthful**: Probability ≤ decision threshold
551
+ - **Confidence**: Model's certainty in the prediction
552
+
553
+ ### Technical Details
554
+ - Built with Python, OpenCV, Librosa, and Scikit‑learn
555
+ - Uses a Random Forest classifier with calibration
556
+ - Real‑time visualization with Matplotlib
557
+ - Gradio‑based web interface
558
+
559
+ ### Disclaimer
560
+ This tool is for research and demonstration purposes only.
561
+ It should not be used as the sole basis for any legal, security, or psychological assessment.
562
+ """)
563
+
564
+ gr.Markdown("---")
565
+ gr.Markdown("**Note**: Analysis may take several minutes depending on video length and system resources.")
566
+
567
+ return demo
568
+
569
+
570
+ def main():
571
+ """Main entry point with CLI argument parsing."""
572
+ parser = argparse.ArgumentParser(description="Gradio demo for Explainable Multimodal Deception Analysis")
573
+ parser.add_argument("--share", action="store_true", help="Create a public share link")
574
+ parser.add_argument("--port", type=int, default=7860, help="Port to run the server on")
575
+ parser.add_argument("--config", type=str, help="Path to configuration file")
576
+ args = parser.parse_args()
577
+
578
+ logging.basicConfig(level=logging.INFO)
579
+
580
+ # Initialize pipeline with optional config
581
+ try:
582
+ initialize_pipeline(args.config)
583
+ logger.info("Pipeline initialized successfully")
584
+ except Exception as e:
585
+ logger.warning(f"Pipeline initialization failed: {e}. Will initialize on first use.")
586
+
587
+ demo = create_ui()
588
+ demo.launch(
589
+ share=args.share,
590
+ server_port=args.port,
591
+ server_name="0.0.0.0"
592
+ )
593
+
594
+
595
+ if __name__ == "__main__":
596
+ main()
colab_deception_analysis.ipynb ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {
6
+ "id": "view-in-github"
7
+ },
8
+ "source": [
9
+ "<a href=\"https://colab.research.google.com/github/apexprit/deception-analysis/blob/main/colab_deception_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
10
+ ]
11
+ },
12
+ {
13
+ "cell_type": "markdown",
14
+ "metadata": {
15
+ "id": "X3jW0N2vQe7p"
16
+ },
17
+ "source": [
18
+ "# \ud83d\udd75\ufe0f\u200d\u2642\ufe0f Multimodal Deception Analysis - Google Colab\n",
19
+ "\n",
20
+ "**Run this entire deception detection system in the cloud.**\n",
21
+ "\n",
22
+ "This notebook provides a complete environment for detecting deception using:\n",
23
+ "- **Facial micro-expressions** (Mediapipe Face Mesh)\n",
24
+ "- **Audio speech patterns** (Librosa)\n",
25
+ "- **SHAP-based explainability**\n",
26
+ "\n",
27
+ "## \ud83d\ude80 Quick Start\n",
28
+ "1. **Runtime \u2192 Run all** (Ctrl+F9).\n",
29
+ "2. If the session restarts, just click **Run all** again.\n",
30
+ "3. **Scroll to the bottom** of the last cell to find the UI."
31
+ ]
32
+ },
33
+ {
34
+ "cell_type": "markdown",
35
+ "metadata": {
36
+ "id": "8h2L5U7nQe7q"
37
+ },
38
+ "source": [
39
+ "## 1. Environment Setup"
40
+ ]
41
+ },
42
+ {
43
+ "cell_type": "code",
44
+ "execution_count": null,
45
+ "metadata": {
46
+ "id": "7hQ2nVv-Qe7r"
47
+ },
48
+ "outputs": [],
49
+ "source": [
50
+ "#@title Install Dependencies\n",
51
+ "print(\"\ud83d\udee0\ufe0f Installing project dependencies...\")\n",
52
+ "\n",
53
+ "!pip install -q mediapipe==0.10.11\n",
54
+ "!pip install -q opencv-python librosa gradio shap joblib tqdm pandas numpy scikit-learn\n",
55
+ "\n",
56
+ "print(\"\u2705 Installation complete!\")"
57
+ ]
58
+ },
59
+ {
60
+ "cell_type": "code",
61
+ "execution_count": null,
62
+ "metadata": {
63
+ "id": "q2Q2nVv-Qe7s"
64
+ },
65
+ "outputs": [],
66
+ "source": [
67
+ "#@title Clone & Setup Path\n",
68
+ "import os\n",
69
+ "import sys\n",
70
+ "import shutil\n",
71
+ "\n",
72
+ "if os.path.exists('/content/deception-analysis'):\n",
73
+ " shutil.rmtree('/content/deception-analysis')\n",
74
+ "\n",
75
+ "!git clone -q https://github.com/apexprit/deception-analysis.git /content/deception-analysis\n",
76
+ "sys.path.insert(0, '/content/deception-analysis')\n",
77
+ "\n",
78
+ "print(\"\u2705 Project ready!\")"
79
+ ]
80
+ },
81
+ {
82
+ "cell_type": "markdown",
83
+ "metadata": {
84
+ "id": "MhQ2nVv-Qe7t"
85
+ },
86
+ "source": [
87
+ "## 2. Train & Launch App"
88
+ ]
89
+ },
90
+ {
91
+ "cell_type": "code",
92
+ "execution_count": null,
93
+ "metadata": {
94
+ "id": "NhQ2nVv-Qe7u"
95
+ },
96
+ "outputs": [],
97
+ "source": [
98
+ "#@title Generate Model & Start Web App\n",
99
+ "import sys\n",
100
+ "import os\n",
101
+ "import pandas as pd\n",
102
+ "import numpy as np\n",
103
+ "sys.path.insert(0, '/content/deception-analysis')\n",
104
+ "\n",
105
+ "from src.utils.synthetic_data import SyntheticDataGenerator\n",
106
+ "from src.utils.config import get_default_config\n",
107
+ "from src.model.classifier import DeceptionClassifier\n",
108
+ "import gradio as gr\n",
109
+ "from src.pipeline import DeceptionPipeline\n",
110
+ "\n",
111
+ "print(\"\ud83d\udd27 Loading configuration...\")\n",
112
+ "config = get_default_config()\n",
113
+ "\n",
114
+ "print(\"\ud83d\udd27 Generating synthetic dataset for model training...\")\n",
115
+ "generator = SyntheticDataGenerator(config, seed=42)\n",
116
+ "features_df, labels, _ = generator.generate_dataset(n_truthful=300, n_deceptive=300)\n",
117
+ "\n",
118
+ "print(\"\ud83d\udcc8 Training classifier...\")\n",
119
+ "classifier = DeceptionClassifier(config.model)\n",
120
+ "X, feature_names = classifier.prepare_features(features_df)\n",
121
+ "metrics = classifier.train(X, labels, feature_names)\n",
122
+ "print(f\"\u2705 Training complete! Accuracy: {metrics.get('train_accuracy', 0):.3f}\")\n",
123
+ "\n",
124
+ "model_dir = '/content/deception-analysis/models'\n",
125
+ "os.makedirs(model_dir, exist_ok=True)\n",
126
+ "model_path = os.path.join(model_dir, 'deception_model.pkl')\n",
127
+ "classifier.save(model_path)\n",
128
+ "\n",
129
+ "print(\"\\n\ud83d\ude80 Initializing Analysis Pipeline...\")\n",
130
+ "pipeline = DeceptionPipeline(config)\n",
131
+ "pipeline.load_model(model_path)\n",
132
+ "\n",
133
+ "def analyze(video, subject_id):\n",
134
+ " if not video: return \"Please upload a video.\", None, None\n",
135
+ " \n",
136
+ " output_results = '/content/results'\n",
137
+ " os.makedirs(output_results, exist_ok=True)\n",
138
+ " \n",
139
+ " res = pipeline.analyze_video(\n",
140
+ " video, \n",
141
+ " subject_id=subject_id or \"S01\", \n",
142
+ " generate_visualizations=True, \n",
143
+ " output_dir=output_results\n",
144
+ " )\n",
145
+ " \n",
146
+ " if \"error\" in res:\n",
147
+ " return f\"### Error during analysis:\\n{res['error']}\", None, None\n",
148
+ " \n",
149
+ " summary = f\"### Result: {res['prediction'].upper()}\\n- Prob: {res['deception_probability']:.3f}\\n- Conf: {res['confidence']:.3f}\"\n",
150
+ " viz = res.get('visualizations', {})\n",
151
+ " \n",
152
+ " return summary, viz.get('temporal_trajectory'), viz.get('shap_summary')\n",
153
+ "\n",
154
+ "print(\"\ud83d\udce1 Building Interface...\")\n",
155
+ "with gr.Blocks(theme=gr.themes.Soft()) as demo:\n",
156
+ " gr.Markdown(\"# \ud83d\udd75\ufe0f\u200d\u2642\ufe0f Multimodal Deception Analysis\")\n",
157
+ " gr.Markdown(\"Analyze facial micro-expressions and audio patterns for deception detection.\")\n",
158
+ " \n",
159
+ " with gr.Row():\n",
160
+ " with gr.Column():\n",
161
+ " v_in = gr.Video(label=\"Interview Video\")\n",
162
+ " s_in = gr.Textbox(label=\"Subject ID\", placeholder=\"e.g. S01\")\n",
163
+ " btn = gr.Button(\"Analyze Video\", variant=\"primary\")\n",
164
+ " with gr.Column():\n",
165
+ " sum_out = gr.Markdown(label=\"Assessment Summary\")\n",
166
+ " with gr.Tabs():\n",
167
+ " with gr.TabItem(\"Temporal\"): \n",
168
+ " traj_out = gr.Image(label=\"Probability Trajectory\")\n",
169
+ " with gr.TabItem(\"Explainability\"): \n",
170
+ " shap_out = gr.Image(label=\"SHAP Feature Importance\")\n",
171
+ " \n",
172
+ " btn.click(analyze, [v_in, s_in], [sum_out, traj_out, shap_out])\n",
173
+ "\n",
174
+ "print(\"\\n\u2728 APP IS READY! Scroll down to see the interface below.\")\n",
175
+ "demo.launch(share=True, debug=True)"
176
+ ]
177
+ }
178
+ ],
179
+ "metadata": {
180
+ "colab": {
181
+ "provenance": []
182
+ },
183
+ "kernelspec": {
184
+ "display_name": "Python 3",
185
+ "language": "python",
186
+ "name": "python3"
187
+ },
188
+ "language_info": {
189
+ "codemirror_mode": {
190
+ "name": "ipython",
191
+ "version": 3
192
+ },
193
+ "file_extension": ".py",
194
+ "mimetype": "text/x-python",
195
+ "name": "python",
196
+ "nbconvert_exporter": "python",
197
+ "pygments_lexer": "ipython3",
198
+ "version": "3.8.10"
199
+ }
200
+ },
201
+ "nbformat": 4,
202
+ "nbformat_minor": 0
203
+ }
create_pretrained_model.py ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Create and save a pre-trained deception detection model using synthetic data.
2
+
3
+ This script generates a realistic synthetic dataset, trains a deception classifier,
4
+ and saves it as a pre-trained model for demonstration purposes.
5
+ """
6
+
7
+ import json
8
+ import os
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ import pandas as pd
13
+
14
+ # Add src to path
15
+ sys.path.insert(0, str(Path(__file__).parent))
16
+
17
+ from src.utils.config import AppConfig, get_default_config
18
+ from src.utils.synthetic_data import SyntheticDataGenerator
19
+ from src.model.classifier import DeceptionClassifier
20
+ from src.model.calibration import SubjectCalibrator
21
+ from src.explainability.explainer import DeceptionExplainer
22
+
23
+
24
+ def create_pretrained_model(output_dir: str = "./models",
25
+ n_samples: int = 1000,
26
+ test_size: float = 0.2,
27
+ random_state: int = 42):
28
+ """Create and save a pre-trained deception detection model.
29
+
30
+ Args:
31
+ output_dir: Directory to save model and related files
32
+ n_samples: Total number of synthetic samples to generate
33
+ test_size: Proportion of data for testing
34
+ random_state: Random seed for reproducibility
35
+ """
36
+ print("=" * 60)
37
+ print("Creating Pre-trained Deception Detection Model")
38
+ print("=" * 60)
39
+
40
+ # Create output directory
41
+ os.makedirs(output_dir, exist_ok=True)
42
+
43
+ # Initialize components
44
+ config = AppConfig()
45
+ generator = SyntheticDataGenerator(config, seed=random_state)
46
+
47
+ print(f"\n1. Generating synthetic dataset ({n_samples} samples)...")
48
+ # Generate balanced dataset
49
+ n_truthful = n_samples // 2
50
+ n_deceptive = n_samples // 2
51
+
52
+ features, labels, subject_ids = generator.generate_dataset(
53
+ n_truthful=n_truthful,
54
+ n_deceptive=n_deceptive
55
+ )
56
+
57
+ print(f" Generated {len(features)} samples")
58
+ print(f" Truthful: {sum(labels == 0)}, Deceptive: {sum(labels == 1)}")
59
+ print(f" Features: {len(features.columns)}")
60
+ print(f" Unique subjects: {len(set(subject_ids))}")
61
+
62
+ # Save dataset
63
+ dataset_path = os.path.join(output_dir, "pretraining_dataset.csv")
64
+ dataset_with_labels = features.copy()
65
+ dataset_with_labels['label'] = labels
66
+ dataset_with_labels['subject_id'] = subject_ids
67
+ dataset_with_labels.to_csv(dataset_path, index=False)
68
+ print(f" Dataset saved to: {dataset_path}")
69
+
70
+ print("\n2. Training deception classifier...")
71
+ # Initialize and train classifier
72
+ classifier = DeceptionClassifier(config.model)
73
+
74
+ # Split data
75
+ from sklearn.model_selection import train_test_split
76
+ X_train, X_test, y_train, y_test = train_test_split(
77
+ features, labels,
78
+ test_size=test_size,
79
+ random_state=random_state,
80
+ stratify=labels
81
+ )
82
+
83
+ print(f" Training samples: {len(X_train)}")
84
+ print(f" Test samples: {len(X_test)}")
85
+
86
+ # Train
87
+ train_result = classifier.train(X_train, y_train, list(features.columns))
88
+
89
+ # Evaluate
90
+ from sklearn.metrics import accuracy_score, f1_score, roc_auc_score
91
+ y_pred = classifier.predict(X_test)
92
+ y_prob = classifier.predict_proba(X_test)[:, 1]
93
+
94
+ test_accuracy = accuracy_score(y_test, y_pred)
95
+ test_f1 = f1_score(y_test, y_pred)
96
+ test_auc = roc_auc_score(y_test, y_prob)
97
+
98
+ print(f" Training accuracy: {train_result.get('train_accuracy', 0):.3f}")
99
+ print(f" Test accuracy: {test_accuracy:.3f}")
100
+ print(f" Test F1-score: {test_f1:.3f}")
101
+ print(f" Test AUC-ROC: {test_auc:.3f}")
102
+
103
+ print("\n3. Training subject calibrator...")
104
+ # Train calibrator on all data
105
+ calibrator = SubjectCalibrator(classifier, config)
106
+
107
+ # Create subject profiles (in real scenario, would use baseline videos)
108
+ subject_profiles = {}
109
+ for subject_id in set(subject_ids):
110
+ subject_mask = [sid == subject_id for sid in subject_ids]
111
+ if sum(subject_mask) >= 5: # Need enough samples per subject
112
+ subject_features = features[subject_mask]
113
+ calibrator.add_subject_profile(subject_id, subject_features)
114
+ subject_profiles[subject_id] = {
115
+ 'n_samples': sum(subject_mask),
116
+ 'baseline_updated': True
117
+ }
118
+
119
+ print(f" Created profiles for {len(subject_profiles)} subjects")
120
+
121
+ print("\n4. Training explainability model...")
122
+ # Train explainer on subset for efficiency
123
+ explainer = DeceptionExplainer(classifier)
124
+
125
+ # Use smaller subset for SHAP training (computationally expensive)
126
+ if len(X_train) > 200:
127
+ X_explain = X_train.iloc[:200]
128
+ else:
129
+ X_explain = X_train
130
+
131
+ print(f" Initializing SHAP explainer...")
132
+ # Explainer initializes lazily
133
+ explainer._init_shap_explainer()
134
+
135
+ print("\n5. Saving model and components...")
136
+ # Save classifier
137
+ model_path = os.path.join(output_dir, "deception_model.pkl")
138
+ classifier.save(model_path)
139
+ print(f" Classifier saved to: {model_path}")
140
+
141
+ # Save calibrator
142
+ calibrator_path = os.path.join(output_dir, "subject_calibrator.json")
143
+ calibrator.save_profiles(calibrator_path)
144
+ print(f" Calibrator saved to: {calibrator_path}")
145
+
146
+ # Explainer does not need saving
147
+ explainer_path = "N/A"
148
+
149
+ # Save feature names
150
+ feature_names_path = os.path.join(output_dir, "feature_names.json")
151
+ with open(feature_names_path, 'w') as f:
152
+ json.dump(list(features.columns), f, indent=2)
153
+ print(f" Feature names saved to: {feature_names_path}")
154
+
155
+ # Save model metadata
156
+ metadata = {
157
+ 'model_name': 'DeceptionDetectionModel',
158
+ 'version': '1.0.0',
159
+ 'description': 'Pre-trained deception detection model using synthetic data',
160
+ 'training_date': pd.Timestamp.now().isoformat(),
161
+ 'training_samples': len(X_train),
162
+ 'test_samples': len(X_test),
163
+ 'n_features': len(features.columns),
164
+ 'performance_metrics': {
165
+ 'training_accuracy': float(train_result.get('train_accuracy', 0)),
166
+ 'training_f1': 0.0, # Not returned by train()
167
+ 'test_accuracy': float(test_accuracy),
168
+ 'test_f1': float(test_f1),
169
+ 'test_auc_roc': float(test_auc)
170
+ },
171
+ 'feature_groups': {
172
+ 'facial': generator.FACIAL_FEATURES,
173
+ 'audio': generator.AUDIO_FEATURES,
174
+ 'cross_modal': generator.CROSS_MODAL_FEATURES
175
+ },
176
+ 'model_parameters': {
177
+ 'model_type': config.model.model_type,
178
+ 'random_state': random_state,
179
+ 'n_estimators': config.model.n_estimators,
180
+ 'learning_rate': config.model.learning_rate,
181
+ 'max_depth': config.model.max_depth
182
+ }
183
+ }
184
+
185
+ metadata_path = os.path.join(output_dir, "model_metadata.json")
186
+ with open(metadata_path, 'w') as f:
187
+ json.dump(metadata, f, indent=2)
188
+ print(f" Metadata saved to: {metadata_path}")
189
+
190
+ # Create a sample inference script
191
+ inference_script = '''"""Sample inference with pre-trained deception model."""
192
+
193
+ import json
194
+ import pandas as pd
195
+ from src.model.classifier import DeceptionClassifier
196
+ from src.model.calibration import SubjectCalibrator
197
+ from src.explainability.explainer import DeceptionExplainer
198
+
199
+ # Load model and components
200
+ model = DeceptionClassifier()
201
+ model.load("./models/deception_model.pkl")
202
+ from src.utils.config import get_default_config
203
+ calibrator = SubjectCalibrator(model, get_default_config())
204
+ calibrator.load_profiles("./models/subject_calibrator.json")
205
+ explainer = DeceptionExplainer(model)
206
+
207
+ # Load feature names
208
+ with open("./models/feature_names.json", "r") as f:
209
+ feature_names = json.load(f)
210
+
211
+ print(f"Model loaded with {len(feature_names)} features")
212
+
213
+ # Example: Create sample features (replace with real data)
214
+ import numpy as np
215
+ sample_features = pd.DataFrame({
216
+ feature: np.random.random() for feature in feature_names
217
+ }, index=[0])
218
+
219
+ # Predict
220
+ prediction = model.predict(sample_features)
221
+ probability = model.predict_proba(sample_features)[0, 1]
222
+
223
+ print(f"Prediction: {'Deceptive' if prediction[0] == 1 else 'Truthful'}")
224
+ print(f"Probability of deception: {probability:.3f}")
225
+
226
+ # If subject ID is known, calibrate
227
+ subject_id = "S001"
228
+ if subject_id in calibrator.subject_profiles:
229
+ calibrated_prob = calibrator.calibrate_prediction(probability, subject_id)
230
+ print(f"Calibrated probability: {calibrated_prob:.3f}")
231
+ '''
232
+
233
+ inference_path = os.path.join(output_dir, "sample_inference.py")
234
+ with open(inference_path, 'w') as f:
235
+ f.write(inference_script)
236
+ print(f" Sample inference script: {inference_path}")
237
+
238
+ print("\n" + "=" * 60)
239
+ print("✅ Pre-trained model creation complete!")
240
+ print("=" * 60)
241
+ print(f"\nModel files saved to: {output_dir}")
242
+ print("\nTo use the model:")
243
+ print(f" 1. Load classifier: DeceptionClassifier.load('{model_path}')")
244
+ print(f" 2. Load calibrator: SubjectCalibrator.load('{calibrator_path}')")
245
+ print(f" 3. Load explainer: DeceptionExplainer.load('{explainer_path}')")
246
+ print(f"\nOr use the pipeline:")
247
+ print(f" from src.pipeline import DeceptionPipeline")
248
+ print(f" pipeline = DeceptionPipeline()")
249
+ print(f" pipeline.load_model('{model_path}')")
250
+
251
+ return {
252
+ 'model_path': model_path,
253
+ 'calibrator_path': calibrator_path,
254
+ 'explainer_path': explainer_path,
255
+ 'metadata_path': metadata_path,
256
+ 'dataset_path': dataset_path
257
+ }
258
+
259
+
260
+ if __name__ == "__main__":
261
+ import argparse
262
+
263
+ parser = argparse.ArgumentParser(description="Create pre-trained deception detection model")
264
+ parser.add_argument("--output-dir", default="./models", help="Output directory for model files")
265
+ parser.add_argument("--n-samples", type=int, default=1000, help="Number of synthetic samples")
266
+ parser.add_argument("--test-size", type=float, default=0.2, help="Test set proportion")
267
+ parser.add_argument("--seed", type=int, default=42, help="Random seed")
268
+
269
+ args = parser.parse_args()
270
+
271
+ create_pretrained_model(
272
+ output_dir=args.output_dir,
273
+ n_samples=args.n_samples,
274
+ test_size=args.test_size,
275
+ random_state=args.seed
276
+ )
inference.py ADDED
@@ -0,0 +1,323 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ CLI script for running deception analysis on new videos.
4
+
5
+ Usage:
6
+ python inference.py --video ./data/interview.mp4 --model ./models/deception_model.pkl --subject_id S01 --visualize --output_dir ./results
7
+ python inference.py --video_dir ./data/batch --model ./models/deception_model.pkl --subject_file subjects.csv --output_dir ./results
8
+ """
9
+
10
+ import argparse
11
+ import json
12
+ import logging
13
+ import sys
14
+ from pathlib import Path
15
+ from typing import List, Optional, Dict, Any
16
+
17
+ import pandas as pd
18
+
19
+ from src.pipeline import DeceptionPipeline
20
+ from src.utils.config import AppConfig, get_default_config
21
+
22
+
23
+ def setup_logging(verbose: bool = False) -> None:
24
+ """Configure logging level and format."""
25
+ level = logging.DEBUG if verbose else logging.INFO
26
+ logging.basicConfig(
27
+ level=level,
28
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
29
+ handlers=[logging.StreamHandler(sys.stdout)],
30
+ )
31
+
32
+
33
+ def load_config(config_path: Optional[str]) -> AppConfig:
34
+ """Load configuration from JSON file or use defaults."""
35
+ if config_path is None:
36
+ return get_default_config()
37
+ try:
38
+ with open(config_path, "r") as f:
39
+ config_dict = json.load(f)
40
+ config = get_default_config()
41
+ from dataclasses import fields
42
+ for field in fields(config):
43
+ if field.name in config_dict:
44
+ setattr(config, field.name, config_dict[field.name])
45
+ logging.info(f"Loaded configuration from {config_path}")
46
+ return config
47
+ except Exception as e:
48
+ logging.warning(f"Failed to load config from {config_path}: {e}. Using defaults.")
49
+ return get_default_config()
50
+
51
+
52
+ def load_subject_mapping(subject_file: str) -> Dict[str, str]:
53
+ """Load CSV mapping video_path -> subject_id."""
54
+ df = pd.read_csv(subject_file)
55
+ if "video_path" not in df.columns or "subject_id" not in df.columns:
56
+ raise ValueError("CSV must contain columns 'video_path' and 'subject_id'")
57
+ return dict(zip(df["video_path"], df["subject_id"]))
58
+
59
+
60
+ def find_videos(video_dir: str, extensions: List[str] = [".mp4", ".avi", ".mov", ".mkv"]) -> List[str]:
61
+ """Recursively find video files in directory."""
62
+ video_dir = Path(video_dir)
63
+ video_paths = []
64
+ for ext in extensions:
65
+ video_paths.extend(video_dir.rglob(f"*{ext}"))
66
+ return sorted([str(p) for p in video_paths])
67
+
68
+
69
+ def print_result_summary(result: Dict[str, Any], threshold: float = 0.5) -> None:
70
+ """Print a formatted summary of analysis result."""
71
+ prob = result.get("deception_probability", 0.5)
72
+ pred = result.get("prediction", "unknown")
73
+ confidence = result.get("confidence", 0.0)
74
+ calibrated = result.get("calibrated", False)
75
+ print("\n" + "="*60)
76
+ print("DECEPTION ANALYSIS RESULT")
77
+ print("="*60)
78
+ print(f"Prediction: {pred.upper()}")
79
+ print(f"Deception Probability: {prob:.3f} (threshold = {threshold})")
80
+ print(f"Confidence: {confidence:.3f}")
81
+ if calibrated:
82
+ print("Calibration: Applied (subject-specific)")
83
+ else:
84
+ print("Calibration: Not applied")
85
+ # Key indicators
86
+ temporal = result.get("temporal_analysis", {})
87
+ if "suspicious_intervals" in temporal and temporal["suspicious_intervals"]:
88
+ intervals = temporal["suspicious_intervals"]
89
+ print(f"Suspicious Intervals: {len(intervals)} detected")
90
+ if "spikes" in temporal and temporal["spikes"]:
91
+ print(f"Probability Spikes: {len(temporal['spikes'])}")
92
+ # Feature importance
93
+ explanation = result.get("explanation", {})
94
+ if "top_features" in explanation:
95
+ top = explanation["top_features"][:3]
96
+ print("Top Contributing Features:")
97
+ for feat, imp in top:
98
+ print(f" - {feat}: {imp:.3f}")
99
+ print("="*60)
100
+
101
+
102
+ def main():
103
+ parser = argparse.ArgumentParser(
104
+ description="Run deception analysis on video(s).",
105
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
106
+ )
107
+ # Input options
108
+ input_group = parser.add_mutually_exclusive_group(required=True)
109
+ input_group.add_argument(
110
+ "--video",
111
+ help="Single video path to analyze.",
112
+ )
113
+ input_group.add_argument(
114
+ "--video_dir",
115
+ help="Directory containing videos to analyze (batch mode).",
116
+ )
117
+
118
+ # Model and config
119
+ parser.add_argument(
120
+ "--model",
121
+ required=True,
122
+ help="Path to trained model file.",
123
+ )
124
+ parser.add_argument(
125
+ "--config",
126
+ help="Path to JSON config file (optional).",
127
+ )
128
+ parser.add_argument(
129
+ "--threshold",
130
+ type=float,
131
+ default=0.5,
132
+ help="Deception probability threshold.",
133
+ )
134
+
135
+ # Subject identification
136
+ parser.add_argument(
137
+ "--subject_id",
138
+ help="Subject identifier for calibration (single video mode).",
139
+ )
140
+ parser.add_argument(
141
+ "--subject_file",
142
+ help="CSV mapping video_path to subject_id (batch mode).",
143
+ )
144
+
145
+ # Output options
146
+ parser.add_argument(
147
+ "--output_dir",
148
+ default="./results",
149
+ help="Directory to save results and visualizations.",
150
+ )
151
+ parser.add_argument(
152
+ "--visualize",
153
+ action="store_true",
154
+ help="Generate all visualizations.",
155
+ )
156
+ parser.add_argument(
157
+ "--report",
158
+ action="store_true",
159
+ help="Generate comprehensive JSON report.",
160
+ )
161
+ parser.add_argument(
162
+ "--verbose",
163
+ action="store_true",
164
+ help="Print detailed output.",
165
+ )
166
+
167
+ args = parser.parse_args()
168
+
169
+ setup_logging(args.verbose)
170
+
171
+ # Validate model path
172
+ model_path = Path(args.model)
173
+ if not model_path.exists():
174
+ logging.error(f"Model file does not exist: {model_path}")
175
+ sys.exit(1)
176
+
177
+ # Prepare output directory
178
+ output_dir = Path(args.output_dir)
179
+ output_dir.mkdir(parents=True, exist_ok=True)
180
+
181
+ # Load configuration
182
+ config = load_config(args.config)
183
+ # Override decision threshold if provided
184
+ config.model.decision_threshold = args.threshold
185
+ logging.info(f"Decision threshold set to {args.threshold}")
186
+
187
+ # Initialize pipeline
188
+ pipeline = DeceptionPipeline(config)
189
+
190
+ # Load pre-trained model
191
+ try:
192
+ pipeline.load_model(str(model_path))
193
+ logging.info(f"Model loaded from {model_path}")
194
+ except Exception as e:
195
+ logging.error(f"Failed to load model: {e}")
196
+ sys.exit(1)
197
+
198
+ # Determine video list and subject IDs
199
+ video_paths = []
200
+ subject_ids = None
201
+
202
+ if args.video:
203
+ video_paths = [args.video]
204
+ if args.subject_id:
205
+ subject_ids = [args.subject_id]
206
+ elif args.subject_file:
207
+ logging.warning("--subject_file ignored in single video mode. Use --subject_id.")
208
+ else: # video_dir
209
+ video_paths = find_videos(args.video_dir)
210
+ if not video_paths:
211
+ logging.error(f"No video files found in {args.video_dir}")
212
+ sys.exit(1)
213
+ logging.info(f"Found {len(video_paths)} videos in {args.video_dir}")
214
+ if args.subject_file:
215
+ try:
216
+ mapping = load_subject_mapping(args.subject_file)
217
+ subject_ids = [mapping.get(Path(v).name, None) for v in video_paths]
218
+ # If mapping uses relative paths, we could match by basename
219
+ # For simplicity, we assume mapping uses video file names.
220
+ except Exception as e:
221
+ logging.error(f"Failed to load subject mapping: {e}")
222
+ sys.exit(1)
223
+ elif args.subject_id:
224
+ logging.warning("--subject_id ignored in batch mode. Use --subject_file.")
225
+ subject_ids = None
226
+
227
+ # Run analysis
228
+ results = []
229
+ if len(video_paths) == 1:
230
+ # Single video analysis
231
+ video_path = video_paths[0]
232
+ subject_id = subject_ids[0] if subject_ids else None
233
+ logging.info(f"Analyzing video: {video_path}")
234
+ try:
235
+ result = pipeline.analyze_video(
236
+ video_path=video_path,
237
+ subject_id=subject_id,
238
+ generate_visualizations=args.visualize,
239
+ output_dir=str(output_dir),
240
+ )
241
+ results.append(result)
242
+ # Print summary
243
+ print_result_summary(result, args.threshold)
244
+ # Generate report if requested
245
+ if args.report:
246
+ report_path = output_dir / f"{Path(video_path).stem}_report.json"
247
+ pipeline.generate_report(result, str(report_path))
248
+ logging.info(f"Report saved to {report_path}")
249
+ except Exception as e:
250
+ logging.error(f"Analysis failed for {video_path}: {e}")
251
+ sys.exit(1)
252
+ else:
253
+ # Batch analysis
254
+ logging.info(f"Starting batch analysis of {len(video_paths)} videos...")
255
+ try:
256
+ results = pipeline.analyze_batch(
257
+ video_paths=video_paths,
258
+ subject_ids=subject_ids,
259
+ generate_visualizations=args.visualize,
260
+ output_dir=str(output_dir),
261
+ )
262
+ logging.info("Batch analysis completed.")
263
+ except Exception as e:
264
+ logging.error(f"Batch analysis failed: {e}")
265
+ sys.exit(1)
266
+
267
+ # Print batch summary
268
+ print("\n" + "="*60)
269
+ print("BATCH ANALYSIS SUMMARY")
270
+ print("="*60)
271
+ deceptive_count = sum(1 for r in results if r.get("prediction") == "deceptive")
272
+ total = len(results)
273
+ print(f"Videos analyzed: {total}")
274
+ print(f"Deceptive predictions: {deceptive_count} ({deceptive_count/total*100:.1f}%)")
275
+ print(f"Truthful predictions: {total - deceptive_count}")
276
+ avg_prob = sum(r.get("deception_probability", 0.5) for r in results) / total
277
+ print(f"Average deception probability: {avg_prob:.3f}")
278
+ print("="*60)
279
+
280
+ # Save batch results to CSV
281
+ batch_csv = output_dir / "batch_results.csv"
282
+ rows = []
283
+ for vid, res in zip(video_paths, results):
284
+ rows.append({
285
+ "video_path": vid,
286
+ "prediction": res.get("prediction", "unknown"),
287
+ "deception_probability": res.get("deception_probability", 0.5),
288
+ "confidence": res.get("confidence", 0.0),
289
+ "calibrated": res.get("calibrated", False),
290
+ })
291
+ df = pd.DataFrame(rows)
292
+ df.to_csv(batch_csv, index=False)
293
+ logging.info(f"Batch results saved to {batch_csv}")
294
+
295
+ # Generate report for each video if requested
296
+ if args.report:
297
+ for vid, res in zip(video_paths, results):
298
+ report_path = output_dir / f"{Path(vid).stem}_report.json"
299
+ pipeline.generate_report(res, str(report_path))
300
+ logging.info(f"Individual reports saved to {output_dir}")
301
+
302
+ # Save overall results as JSON
303
+ if results:
304
+ results_json = output_dir / "analysis_results.json"
305
+ # Convert DataFrames to dict for serialization (skip them)
306
+ serializable = []
307
+ for res in results:
308
+ ser = {}
309
+ for k, v in res.items():
310
+ if isinstance(v, pd.DataFrame):
311
+ ser[k] = v.to_dict(orient="records")
312
+ else:
313
+ ser[k] = v
314
+ serializable.append(ser)
315
+ with open(results_json, "w") as f:
316
+ json.dump(serializable, f, indent=2, default=str)
317
+ logging.info(f"Full results saved to {results_json}")
318
+
319
+ logging.info("Analysis completed successfully.")
320
+
321
+
322
+ if __name__ == "__main__":
323
+ main()
models/.gitkeep ADDED
@@ -0,0 +1 @@
 
 
1
+
requirements.txt ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ opencv-python>=4.8.0
2
+ mediapipe>=0.10.0
3
+ librosa>=0.10.0
4
+ scikit-learn>=1.3.0
5
+ matplotlib>=3.7.0
6
+ seaborn>=0.12.0
7
+ gradio>=4.0.0
8
+ numpy>=1.24.0
9
+ pandas>=2.0.0
10
+ shap>=0.43.0
11
+ joblib>=1.3.0
12
+ tqdm>=4.65.0
13
+ scipy>=1.11.0
14
+ fastapi>=0.100.0
15
+ uvicorn>=0.23.0
16
+ python-multipart>=0.0.6
src/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ """Explainable Multimodal Deception Analysis System."""
2
+
3
+ # We avoid eager imports of DeceptionPipeline here to prevent side-effects
4
+ # (like MediaPipe initialization) when only utility modules are needed.
src/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (215 Bytes). View file
 
src/__pycache__/pipeline.cpython-314.pyc ADDED
Binary file (24.6 kB). View file
 
src/audio/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """Audio analysis module for speech pattern detection."""
2
+
3
+ from .extractor import AudioFeatureExtractor
4
+ from .fusion import MultimodalFusion
5
+
6
+ __all__ = ['AudioFeatureExtractor', 'MultimodalFusion']
src/audio/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (328 Bytes). View file
 
src/audio/extractor.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Audio and speech feature extraction using librosa."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Dict, List, Optional
6
+
7
+ import librosa
8
+ import numpy as np
9
+ import pandas as pd
10
+
11
+ from src.utils.config import AudioConfig, get_default_config
12
+
13
+
14
+ class AudioFeatureExtractor:
15
+ """Extracts speech and audio features for deception analysis using librosa."""
16
+
17
+ def __init__(self, config: Optional[AudioConfig] = None):
18
+ """Initialize the audio extractor.
19
+
20
+ Args:
21
+ config: Optional audio configuration. Defaults to application audio config.
22
+ """
23
+
24
+ self.config = config or get_default_config().audio
25
+
26
+ @staticmethod
27
+ def _finite_or_zero(value: float) -> float:
28
+ """Convert non-finite numeric outputs to zero."""
29
+
30
+ return float(value) if np.isfinite(value) else 0.0
31
+
32
+ def _empty_features(self) -> Dict[str, float]:
33
+ """Return a zero-filled feature dictionary for invalid audio."""
34
+
35
+ return {name: 0.0 for name in self.get_feature_names()}
36
+
37
+ def extract_pitch_variation(self, y: np.ndarray, sr: int) -> Dict[str, float]:
38
+ """Extract pitch/f0 features using probabilistic YIN.
39
+
40
+ Args:
41
+ y: Audio waveform.
42
+ sr: Sample rate.
43
+
44
+ Returns:
45
+ Dictionary with pitch summary and voiced-frame ratio.
46
+ """
47
+
48
+ if y.size < self.config.hop_length or np.allclose(y, 0.0):
49
+ return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
50
+ try:
51
+ f0, _, voiced_prob = librosa.pyin(y, fmin=65.0, fmax=2093.0, sr=sr, hop_length=self.config.hop_length)
52
+ except Exception:
53
+ return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
54
+ voiced = f0[np.isfinite(f0)]
55
+ if voiced.size == 0:
56
+ return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
57
+ mean = float(np.mean(voiced))
58
+ std = float(np.std(voiced))
59
+ # Pitch elevation and instability can reflect stress or high cognitive load during deceptive speech.
60
+ return {
61
+ "pitch_mean": mean,
62
+ "pitch_std": std,
63
+ "pitch_range": float(np.max(voiced) - np.min(voiced)),
64
+ "pitch_variation_coefficient": self._finite_or_zero(std / mean if mean else 0.0),
65
+ "voiced_ratio": float(np.mean(np.asarray(voiced_prob) > 0.5)) if voiced_prob is not None else float(voiced.size / f0.size),
66
+ }
67
+
68
+ def extract_pause_analysis(self, y: np.ndarray, sr: int) -> Dict[str, float]:
69
+ """Analyze speech pauses and timing with adaptive energy VAD.
70
+
71
+ Args:
72
+ y: Audio waveform.
73
+ sr: Sample rate.
74
+
75
+ Returns:
76
+ Dictionary with pause count, duration, rate, and speech ratio.
77
+ """
78
+
79
+ duration = len(y) / float(sr) if sr else 0.0
80
+ if y.size == 0 or duration <= 0.0:
81
+ return {"pause_count": 0.0, "avg_pause_duration": 0.0, "max_pause_duration": 0.0, "pause_rate": 0.0, "speech_ratio": 0.0}
82
+ rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
83
+ if rms.size == 0 or np.allclose(rms, 0.0):
84
+ return {"pause_count": 1.0, "avg_pause_duration": duration, "max_pause_duration": duration, "pause_rate": 60.0 / duration, "speech_ratio": 0.0}
85
+ threshold = max(float(np.median(rms) * 0.6), float(np.max(rms) * 0.03))
86
+ silent = rms <= threshold
87
+ frame_seconds = self.config.hop_length / float(sr)
88
+ min_pause_frames = max(1, int(np.ceil(0.3 / frame_seconds)))
89
+ pauses: List[float] = []
90
+ run = 0
91
+ for is_silent in silent:
92
+ if is_silent:
93
+ run += 1
94
+ else:
95
+ if run >= min_pause_frames:
96
+ pauses.append(run * frame_seconds)
97
+ run = 0
98
+ if run >= min_pause_frames:
99
+ pauses.append(run * frame_seconds)
100
+ pause_time = float(np.sum(pauses)) if pauses else 0.0
101
+ # Pauses and hesitations are deception-relevant when they indicate formulation difficulty.
102
+ return {
103
+ "pause_count": float(len(pauses)),
104
+ "avg_pause_duration": float(np.mean(pauses)) if pauses else 0.0,
105
+ "max_pause_duration": float(np.max(pauses)) if pauses else 0.0,
106
+ "pause_rate": float(len(pauses) / (duration / 60.0)) if duration > 0 else 0.0,
107
+ "speech_ratio": float(np.clip(1.0 - pause_time / duration, 0.0, 1.0)),
108
+ }
109
+
110
+ def extract_speech_energy(self, y: np.ndarray, sr: int) -> Dict[str, float]:
111
+ """Extract speech energy and spectral intensity features.
112
+
113
+ Args:
114
+ y: Audio waveform.
115
+ sr: Sample rate.
116
+
117
+ Returns:
118
+ Dictionary with RMS energy and spectral descriptors.
119
+ """
120
+
121
+ if y.size == 0:
122
+ return {"energy_mean": 0.0, "energy_std": 0.0, "energy_range": 0.0, "energy_variation": 0.0, "spectral_centroid_mean": 0.0, "spectral_bandwidth_mean": 0.0}
123
+ rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
124
+ mean = float(np.mean(rms)) if rms.size else 0.0
125
+ std = float(np.std(rms)) if rms.size else 0.0
126
+ centroid = librosa.feature.spectral_centroid(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
127
+ bandwidth = librosa.feature.spectral_bandwidth(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
128
+ # Vocal intensity spikes can pair with facial stress markers in multimodal deception cues.
129
+ return {
130
+ "energy_mean": mean,
131
+ "energy_std": std,
132
+ "energy_range": float(np.max(rms) - np.min(rms)) if rms.size else 0.0,
133
+ "energy_variation": self._finite_or_zero(std / mean if mean else 0.0),
134
+ "spectral_centroid_mean": float(np.mean(centroid)) if centroid.size else 0.0,
135
+ "spectral_bandwidth_mean": float(np.mean(bandwidth)) if bandwidth.size else 0.0,
136
+ }
137
+
138
+ def extract_mfcc_features(self, y: np.ndarray, sr: int) -> Dict[str, float]:
139
+ """Extract MFCC and delta-MFCC speech characterization features.
140
+
141
+ Args:
142
+ y: Audio waveform.
143
+ sr: Sample rate.
144
+
145
+ Returns:
146
+ Dictionary containing MFCC means, standard deviations, and delta means.
147
+ """
148
+
149
+ features: Dict[str, float] = {}
150
+ if y.size == 0:
151
+ for idx in range(self.config.n_mfcc):
152
+ features[f"mfcc_{idx}_mean"] = 0.0
153
+ features[f"mfcc_{idx}_std"] = 0.0
154
+ features[f"delta_mfcc_{idx}_mean"] = 0.0
155
+ return features
156
+ mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=self.config.n_mfcc, n_fft=self.config.n_fft, hop_length=self.config.hop_length)
157
+ delta = librosa.feature.delta(mfcc) if mfcc.shape[1] >= 3 else np.zeros_like(mfcc)
158
+ # MFCCs capture voice quality shifts that may accompany stress or controlled speech.
159
+ for idx in range(self.config.n_mfcc):
160
+ features[f"mfcc_{idx}_mean"] = float(np.mean(mfcc[idx]))
161
+ features[f"mfcc_{idx}_std"] = float(np.std(mfcc[idx]))
162
+ features[f"delta_mfcc_{idx}_mean"] = float(np.mean(delta[idx]))
163
+ return features
164
+
165
+ def process_audio(self, audio_path: str) -> Dict[str, float]:
166
+ """Process a complete audio file and extract all features.
167
+
168
+ Args:
169
+ audio_path: Path to an audio file readable by librosa.
170
+
171
+ Returns:
172
+ Combined audio feature dictionary.
173
+ """
174
+
175
+ try:
176
+ y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
177
+ except Exception:
178
+ return self._empty_features()
179
+ if y.size == 0:
180
+ return self._empty_features()
181
+ features: Dict[str, float] = {}
182
+ features.update(self.extract_pitch_variation(y, sr))
183
+ features.update(self.extract_pause_analysis(y, sr))
184
+ features.update(self.extract_speech_energy(y, sr))
185
+ features.update(self.extract_mfcc_features(y, sr))
186
+ return {name: self._finite_or_zero(float(features.get(name, 0.0))) for name in self.get_feature_names()}
187
+
188
+ def process_audio_segments(self, audio_path: str, segment_duration: float = 2.0) -> pd.DataFrame:
189
+ """Process audio in fixed-duration segments for temporal analysis.
190
+
191
+ Args:
192
+ audio_path: Path to an audio file readable by librosa.
193
+ segment_duration: Segment length in seconds.
194
+
195
+ Returns:
196
+ DataFrame with segment index, timing, and audio feature columns.
197
+ """
198
+
199
+ try:
200
+ y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
201
+ except Exception:
202
+ columns = ["segment_idx", "start_time", "end_time"] + self.get_feature_names()
203
+ return pd.DataFrame(columns=columns)
204
+ segment_samples = max(1, int(segment_duration * sr))
205
+ records: List[Dict[str, float]] = []
206
+ for segment_idx, start in enumerate(range(0, len(y), segment_samples)):
207
+ end = min(start + segment_samples, len(y))
208
+ segment = y[start:end]
209
+ record: Dict[str, float] = {
210
+ "segment_idx": float(segment_idx),
211
+ "start_time": start / float(sr),
212
+ "end_time": end / float(sr),
213
+ }
214
+ if segment.size < max(16, self.config.hop_length // 2):
215
+ record.update(self._empty_features())
216
+ else:
217
+ record.update(self.extract_pitch_variation(segment, sr))
218
+ record.update(self.extract_pause_analysis(segment, sr))
219
+ record.update(self.extract_speech_energy(segment, sr))
220
+ record.update(self.extract_mfcc_features(segment, sr))
221
+ records.append(record)
222
+ return pd.DataFrame(records, columns=["segment_idx", "start_time", "end_time"] + self.get_feature_names())
223
+
224
+ def get_feature_names(self) -> List[str]:
225
+ """Return ordered names of all produced audio features."""
226
+
227
+ names = [
228
+ "pitch_mean",
229
+ "pitch_std",
230
+ "pitch_range",
231
+ "pitch_variation_coefficient",
232
+ "voiced_ratio",
233
+ "pause_count",
234
+ "avg_pause_duration",
235
+ "max_pause_duration",
236
+ "pause_rate",
237
+ "speech_ratio",
238
+ "energy_mean",
239
+ "energy_std",
240
+ "energy_range",
241
+ "energy_variation",
242
+ "spectral_centroid_mean",
243
+ "spectral_bandwidth_mean",
244
+ ]
245
+ for idx in range(self.config.n_mfcc):
246
+ names.extend([f"mfcc_{idx}_mean", f"mfcc_{idx}_std", f"delta_mfcc_{idx}_mean"])
247
+ return names
248
+
src/audio/fusion.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Multimodal fusion utilities for facial and audio features."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import List, Optional
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+
10
+ from src.utils.config import AppConfig, get_default_config
11
+
12
+
13
+ class MultimodalFusion:
14
+ """Fuses facial and audio features for multimodal deception analysis."""
15
+
16
+ def __init__(self, config: Optional[AppConfig] = None):
17
+ """Initialize the fusion module.
18
+
19
+ Args:
20
+ config: Optional application configuration.
21
+ """
22
+
23
+ self.config = config or get_default_config()
24
+
25
+ def align_features(self, facial_df: pd.DataFrame, audio_df: pd.DataFrame, video_fps: float = 30.0) -> pd.DataFrame:
26
+ """Align facial frame-level and audio segment-level features by time.
27
+
28
+ Args:
29
+ facial_df: Frame-level facial feature DataFrame.
30
+ audio_df: Segment-level audio feature DataFrame with start and end times.
31
+ video_fps: Fallback video FPS if timestamps are not present.
32
+
33
+ Returns:
34
+ Combined DataFrame aligned to facial frame granularity.
35
+ """
36
+
37
+ if facial_df.empty:
38
+ return facial_df.copy()
39
+ aligned = facial_df.copy().reset_index(drop=True)
40
+ if "timestamp" not in aligned.columns:
41
+ aligned["timestamp"] = aligned.get("frame_idx", pd.Series(aligned.index)).astype(float) / float(video_fps)
42
+ if audio_df.empty:
43
+ return aligned
44
+
45
+ audio_features = [col for col in audio_df.columns if col not in {"segment_idx", "start_time", "end_time"}]
46
+ for column in audio_features:
47
+ aligned[column] = np.nan
48
+
49
+ for idx, timestamp in aligned["timestamp"].items():
50
+ matches = audio_df[(audio_df["start_time"] <= timestamp) & (timestamp < audio_df["end_time"])]
51
+ if matches.empty and timestamp == aligned["timestamp"].iloc[-1]:
52
+ matches = audio_df[audio_df["end_time"] <= timestamp].tail(1)
53
+ if not matches.empty:
54
+ aligned.loc[idx, audio_features] = matches.iloc[0][audio_features].to_numpy(dtype=float)
55
+ return aligned.ffill().bfill()
56
+
57
+ @staticmethod
58
+ def _rolling_corr(left: pd.Series, right: pd.Series, window: int = 30) -> pd.Series:
59
+ """Compute robust rolling correlation with finite fallbacks."""
60
+
61
+ corr = left.rolling(window=window, min_periods=max(3, window // 5)).corr(right)
62
+ return corr.replace([np.inf, -np.inf], np.nan).fillna(0.0)
63
+
64
+ @staticmethod
65
+ def _zscore(series: pd.Series) -> pd.Series:
66
+ """Return z-scored values with zero variance protection."""
67
+
68
+ std = float(series.std(skipna=True) or 0.0)
69
+ if std <= 1e-8:
70
+ return pd.Series(np.zeros(len(series)), index=series.index)
71
+ return (series.fillna(series.mean()) - float(series.mean())) / std
72
+
73
+ def extract_fusion_features(self, aligned_df: pd.DataFrame) -> pd.DataFrame:
74
+ """Compute cross-modal fusion features.
75
+
76
+ Args:
77
+ aligned_df: DataFrame containing aligned facial and audio features.
78
+
79
+ Returns:
80
+ DataFrame with original columns plus fusion-specific features.
81
+ """
82
+
83
+ fused = aligned_df.copy()
84
+ if fused.empty:
85
+ for name in self.get_fusion_feature_names():
86
+ fused[name] = []
87
+ return fused
88
+
89
+ mouth = fused.get("mouth_openness", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
90
+ energy = fused.get("energy_mean", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
91
+ eye = fused.get("eye_left_openness", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
92
+ pause = fused.get("pause_count", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
93
+ pitch = fused.get("pitch_mean", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
94
+ brow = fused.get("brow_tension", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
95
+
96
+ # Audio-visual synchrony flags mismatches between visible articulation and speech energy.
97
+ fused["audio_visual_sync"] = self._rolling_corr(mouth, energy)
98
+ # Pause-gaze coupling captures whether vocal hesitation coincides with eye behavior changes.
99
+ fused["pause_gaze_coupling"] = self._rolling_corr(eye, pause)
100
+ # Composite stress combines vocal pitch, facial brow tension, and energy spikes.
101
+ fused["stress_composite"] = (0.4 * self._zscore(pitch) + 0.35 * self._zscore(brow) + 0.25 * self._zscore(energy)).astype(float)
102
+ return fused
103
+
104
+ def get_fusion_feature_names(self) -> List[str]:
105
+ """Return names of fusion-specific features."""
106
+
107
+ return ["audio_visual_sync", "pause_gaze_coupling", "stress_composite"]
108
+
src/evaluation/__init__.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evaluation module for model assessment and validation."""
2
+
3
+ from .metrics import DeceptionMetrics
4
+ from .cross_validation import CrossValidator
5
+ from .ablation import AblationStudy
6
+ from .visualizer import EvaluationVisualizer
7
+
8
+ __all__ = [
9
+ 'DeceptionMetrics',
10
+ 'CrossValidator',
11
+ 'AblationStudy',
12
+ 'EvaluationVisualizer',
13
+ ]
src/evaluation/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (448 Bytes). View file
 
src/evaluation/__pycache__/ablation.cpython-314.pyc ADDED
Binary file (12.3 kB). View file
 
src/evaluation/__pycache__/cross_validation.cpython-314.pyc ADDED
Binary file (12.7 kB). View file
 
src/evaluation/__pycache__/metrics.cpython-314.pyc ADDED
Binary file (15.1 kB). View file
 
src/evaluation/__pycache__/visualizer.cpython-314.pyc ADDED
Binary file (18.4 kB). View file
 
src/evaluation/ablation.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Ablation study for feature group importance in deception detection.
2
+
3
+ This module provides systematic removal of feature groups (facial, audio,
4
+ temporal, cross‑modal) to quantify their contribution to model performance.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import logging
10
+ from typing import Any, Dict, List, Optional, Tuple, Union
11
+
12
+ import numpy as np
13
+ import pandas as pd
14
+ from sklearn.base import BaseEstimator, clone
15
+
16
+ from .cross_validation import CrossValidator
17
+ from .metrics import DeceptionMetrics
18
+
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+
23
+ class AblationStudy:
24
+ """Conduct ablation studies by removing feature groups.
25
+
26
+ This class evaluates the contribution of different feature groups
27
+ (facial, audio, temporal, cross‑modal) to overall model performance.
28
+
29
+ Attributes:
30
+ feature_names: List of all feature names in the dataset.
31
+ feature_groups: Dictionary mapping group names to lists of feature indices or names.
32
+ random_state: Random seed for reproducibility.
33
+ n_splits: Number of cross‑validation folds.
34
+ metrics: Instance of DeceptionMetrics for evaluation.
35
+ """
36
+
37
+ def __init__(
38
+ self,
39
+ feature_names: List[str],
40
+ feature_groups: Dict[str, List[str]],
41
+ random_state: int = 42,
42
+ n_splits: int = 5,
43
+ metrics: Optional[DeceptionMetrics] = None,
44
+ ) -> None:
45
+ """Initialize ablation study.
46
+
47
+ Args:
48
+ feature_names: List of all feature names in the dataset.
49
+ feature_groups: Dictionary mapping group names to lists of feature names.
50
+ Expected keys: 'facial', 'audio', 'temporal', 'cross_modal'.
51
+ random_state: Random seed for reproducibility.
52
+ n_splits: Number of cross‑validation folds.
53
+ metrics: Optional DeceptionMetrics instance.
54
+ """
55
+ self.feature_names = feature_names
56
+ self.feature_groups = feature_groups
57
+ self.random_state = random_state
58
+ self.n_splits = n_splits
59
+ self.metrics = metrics or DeceptionMetrics(random_state=random_state)
60
+
61
+ # Validate that all group features are in feature_names
62
+ for group_name, features in feature_groups.items():
63
+ for f in features:
64
+ if f not in feature_names:
65
+ logger.warning(
66
+ f"Feature '{f}' from group '{group_name}' not found in feature_names"
67
+ )
68
+
69
+ # Build index mapping
70
+ self.feature_to_index = {name: i for i, name in enumerate(feature_names)}
71
+ self.group_indices = {}
72
+ for group_name, features in feature_groups.items():
73
+ indices = []
74
+ for f in features:
75
+ if f in self.feature_to_index:
76
+ indices.append(self.feature_to_index[f])
77
+ self.group_indices[group_name] = indices
78
+
79
+ def _select_features(
80
+ self,
81
+ X: Union[np.ndarray, pd.DataFrame],
82
+ include_groups: Optional[List[str]] = None,
83
+ exclude_groups: Optional[List[str]] = None,
84
+ ) -> Union[np.ndarray, pd.DataFrame]:
85
+ """Select features based on included/excluded groups.
86
+
87
+ Args:
88
+ X: Original feature matrix (n_samples × n_features).
89
+ include_groups: List of group names to keep (all others removed).
90
+ exclude_groups: List of group names to remove (all others kept).
91
+
92
+ Returns:
93
+ Feature matrix with selected columns.
94
+ """
95
+ if include_groups is not None and exclude_groups is not None:
96
+ raise ValueError("Cannot specify both include_groups and exclude_groups")
97
+
98
+ if isinstance(X, pd.DataFrame):
99
+ columns = X.columns.tolist()
100
+ else:
101
+ columns = list(range(X.shape[1]))
102
+
103
+ # Determine which indices to keep
104
+ if include_groups is not None:
105
+ keep_indices = set()
106
+ for group in include_groups:
107
+ keep_indices.update(self.group_indices.get(group, []))
108
+ elif exclude_groups is not None:
109
+ all_indices = set(range(len(self.feature_names)))
110
+ remove_indices = set()
111
+ for group in exclude_groups:
112
+ remove_indices.update(self.group_indices.get(group, []))
113
+ keep_indices = all_indices - remove_indices
114
+ else:
115
+ # Keep all features
116
+ keep_indices = set(range(len(self.feature_names)))
117
+
118
+ # Convert to sorted list
119
+ keep_indices = sorted(keep_indices)
120
+
121
+ if isinstance(X, pd.DataFrame):
122
+ selected_cols = [columns[i] for i in keep_indices]
123
+ return X[selected_cols]
124
+ else:
125
+ return X[:, keep_indices]
126
+
127
+ def run_ablation(
128
+ self,
129
+ classifier_class: type,
130
+ X: Union[np.ndarray, pd.DataFrame],
131
+ y: Union[np.ndarray, List[int]],
132
+ feature_names: Optional[List[str]] = None,
133
+ feature_groups: Optional[Dict[str, List[str]]] = None,
134
+ n_splits: Optional[int] = None,
135
+ subject_ids: Optional[Union[np.ndarray, List[str]]] = None,
136
+ cv_method: str = "stratified",
137
+ ) -> Dict[str, Dict[str, Any]]:
138
+ """Run ablation study across feature groups.
139
+
140
+ Args:
141
+ classifier_class: A scikit‑learn classifier class (not instance).
142
+ X: Feature matrix.
143
+ y: Target labels.
144
+ feature_names: Override stored feature names.
145
+ feature_groups: Override stored feature groups.
146
+ n_splits: Override default number of folds.
147
+ subject_ids: Optional subject IDs for subject‑aware CV.
148
+ cv_method: Cross‑validation method ('stratified', 'subject_aware', 'loso').
149
+
150
+ Returns:
151
+ Dictionary with keys:
152
+ - full_model: metrics with all features
153
+ - minus_facial: metrics without facial features
154
+ - minus_audio: metrics without audio features
155
+ - minus_temporal: metrics without temporal features
156
+ - minus_cross_modal: metrics without cross‑modal features
157
+ - facial_only: metrics with only facial features
158
+ - audio_only: metrics with only audio features
159
+ - temporal_only: metrics with only temporal features
160
+ Each value is a dict with 'mean_metrics', 'std_metrics', 'per_fold_metrics'.
161
+ """
162
+ if feature_names is not None:
163
+ self.feature_names = feature_names
164
+ if feature_groups is not None:
165
+ self.feature_groups = feature_groups
166
+ self.group_indices = {}
167
+ for group_name, features in feature_groups.items():
168
+ indices = []
169
+ for f in features:
170
+ if f in self.feature_to_index:
171
+ indices.append(self.feature_to_index[f])
172
+ self.group_indices[group_name] = indices
173
+
174
+ n_splits = n_splits or self.n_splits
175
+ cv = CrossValidator(
176
+ random_state=self.random_state,
177
+ n_splits=n_splits,
178
+ metrics=self.metrics,
179
+ )
180
+
181
+ # Define ablation conditions
182
+ conditions = {
183
+ "full_model": {"include_groups": None, "exclude_groups": None},
184
+ "minus_facial": {"exclude_groups": ["facial"]},
185
+ "minus_audio": {"exclude_groups": ["audio"]},
186
+ "minus_temporal": {"exclude_groups": ["temporal"]},
187
+ "minus_cross_modal": {"exclude_groups": ["cross_modal"]},
188
+ "facial_only": {"include_groups": ["facial"]},
189
+ "audio_only": {"include_groups": ["audio"]},
190
+ "temporal_only": {"include_groups": ["temporal"]},
191
+ }
192
+
193
+ results = {}
194
+
195
+ for cond_name, cond_args in conditions.items():
196
+ logger.info(f"Running ablation condition: {cond_name}")
197
+
198
+ # Select features
199
+ X_cond = self._select_features(
200
+ X,
201
+ include_groups=cond_args.get("include_groups"),
202
+ exclude_groups=cond_args.get("exclude_groups"),
203
+ )
204
+
205
+ # Skip if no features remain
206
+ if X_cond.shape[1] == 0:
207
+ logger.warning(f"No features left for condition {cond_name}, skipping")
208
+ results[cond_name] = {
209
+ "mean_metrics": {},
210
+ "std_metrics": {},
211
+ "per_fold_metrics": [],
212
+ }
213
+ continue
214
+
215
+ # Run cross‑validation
216
+ classifier = classifier_class()
217
+ cv_result = cv.run_cross_validation(
218
+ classifier,
219
+ X_cond,
220
+ y,
221
+ n_splits=n_splits,
222
+ subject_ids=subject_ids,
223
+ method=cv_method,
224
+ return_predictions=False,
225
+ )
226
+
227
+ results[cond_name] = {
228
+ "mean_metrics": cv_result["mean_metrics"],
229
+ "std_metrics": cv_result["std_metrics"],
230
+ "per_fold_metrics": cv_result["per_fold_metrics"],
231
+ "n_features": X_cond.shape[1],
232
+ }
233
+
234
+ logger.info("Ablation study completed.")
235
+ return results
236
+
237
+ def compute_feature_importance_ranking(
238
+ self,
239
+ classifier: BaseEstimator,
240
+ feature_names: Optional[List[str]] = None,
241
+ ) -> List[Tuple[str, float]]:
242
+ """Compute feature importance ranking from a trained classifier.
243
+
244
+ Supports classifiers with `feature_importances_` attribute (tree‑based)
245
+ or `coef_` attribute (linear models).
246
+
247
+ Args:
248
+ classifier: Trained scikit‑learn classifier.
249
+ feature_names: Override stored feature names.
250
+
251
+ Returns:
252
+ List of (feature_name, importance_score) sorted descending.
253
+ """
254
+ if feature_names is None:
255
+ feature_names = self.feature_names
256
+
257
+ if hasattr(classifier, "feature_importances_"):
258
+ importances = classifier.feature_importances_
259
+ elif hasattr(classifier, "coef_"):
260
+ # For linear models, take absolute coefficients averaged across classes
261
+ coef = classifier.coef_
262
+ if coef.ndim == 2:
263
+ importances = np.mean(np.abs(coef), axis=0)
264
+ else:
265
+ importances = np.abs(coef)
266
+ else:
267
+ logger.warning(
268
+ "Classifier has no feature_importances_ or coef_; returning uniform importances"
269
+ )
270
+ importances = np.ones(len(feature_names))
271
+
272
+ # Ensure length matches
273
+ if len(importances) != len(feature_names):
274
+ logger.error(
275
+ f"Importance length ({len(importances)}) != feature count ({len(feature_names)})"
276
+ )
277
+ # Truncate or pad
278
+ if len(importances) > len(feature_names):
279
+ importances = importances[: len(feature_names)]
280
+ else:
281
+ importances = np.pad(
282
+ importances,
283
+ (0, len(feature_names) - len(importances)),
284
+ mode="constant",
285
+ constant_values=0.0,
286
+ )
287
+
288
+ ranked = sorted(
289
+ zip(feature_names, importances),
290
+ key=lambda x: x[1],
291
+ reverse=True,
292
+ )
293
+ logger.info(f"Computed feature importance ranking for {len(ranked)} features")
294
+ return ranked
src/evaluation/cross_validation.py ADDED
@@ -0,0 +1,303 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Cross‑validation strategies for deception detection.
2
+
3
+ This module provides subject‑aware and stratified cross‑validation
4
+ splits suitable for multimodal deception analysis.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import logging
10
+ from typing import Any, Dict, List, Optional, Tuple, Union
11
+
12
+ import numpy as np
13
+ import pandas as pd
14
+ from sklearn.model_selection import (
15
+ StratifiedKFold,
16
+ GroupKFold,
17
+ LeaveOneGroupOut,
18
+ cross_val_predict,
19
+ )
20
+ from sklearn.base import BaseEstimator
21
+
22
+ from .metrics import DeceptionMetrics
23
+
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+
28
+ class CrossValidator:
29
+ """Cross‑validation with stratification and subject‑aware splits.
30
+
31
+ This class provides methods for performing k‑fold cross‑validation
32
+ while respecting subject boundaries (no data leakage) and maintaining
33
+ class balance.
34
+
35
+ Attributes:
36
+ random_state: Random seed for reproducibility.
37
+ n_splits: Default number of folds.
38
+ metrics: Instance of DeceptionMetrics for evaluation.
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ random_state: int = 42,
44
+ n_splits: int = 5,
45
+ metrics: Optional[DeceptionMetrics] = None,
46
+ ) -> None:
47
+ """Initialize cross‑validator.
48
+
49
+ Args:
50
+ random_state: Random seed for reproducibility.
51
+ n_splits: Default number of folds.
52
+ metrics: Optional DeceptionMetrics instance. If None, a default is created.
53
+ """
54
+ self.random_state = random_state
55
+ self.n_splits = n_splits
56
+ self.metrics = metrics or DeceptionMetrics(random_state=random_state)
57
+
58
+ def stratified_k_fold(
59
+ self,
60
+ X: Union[np.ndarray, pd.DataFrame],
61
+ y: Union[np.ndarray, List[int]],
62
+ n_splits: Optional[int] = None,
63
+ random_state: Optional[int] = None,
64
+ ) -> List[Tuple[np.ndarray, np.ndarray]]:
65
+ """Generate stratified k‑fold indices.
66
+
67
+ Args:
68
+ X: Feature matrix (any shape, only length matters).
69
+ y: Target labels used for stratification.
70
+ n_splits: Override default number of folds.
71
+ random_state: Override default random seed.
72
+
73
+ Returns:
74
+ List of (train_indices, test_indices) for each fold.
75
+ """
76
+ n_splits = n_splits or self.n_splits
77
+ random_state = random_state or self.random_state
78
+
79
+ y = np.asarray(y)
80
+ skf = StratifiedKFold(
81
+ n_splits=n_splits, shuffle=True, random_state=random_state
82
+ )
83
+ splits = []
84
+ for train_idx, test_idx in skf.split(X, y):
85
+ splits.append((train_idx, test_idx))
86
+
87
+ logger.info(
88
+ f"Generated {n_splits} stratified folds "
89
+ f"(random_state={random_state}, n_samples={len(y)})"
90
+ )
91
+ return splits
92
+
93
+ def subject_aware_split(
94
+ self,
95
+ X: Union[np.ndarray, pd.DataFrame],
96
+ y: Union[np.ndarray, List[int]],
97
+ subject_ids: Union[np.ndarray, List[str]],
98
+ n_splits: Optional[int] = None,
99
+ ) -> List[Tuple[np.ndarray, np.ndarray]]:
100
+ """Generate k‑fold splits where the same subject does not appear in both train and test.
101
+
102
+ Uses GroupKFold with subject_ids as groups.
103
+
104
+ Args:
105
+ X: Feature matrix.
106
+ y: Target labels (unused for grouping, kept for compatibility).
107
+ subject_ids: Subject identifier for each sample.
108
+ n_splits: Override default number of folds.
109
+
110
+ Returns:
111
+ List of (train_indices, test_indices) for each fold.
112
+ """
113
+ n_splits = n_splits or self.n_splits
114
+ subject_ids = np.asarray(subject_ids)
115
+
116
+ # Ensure we have enough unique groups
117
+ unique_subjects = np.unique(subject_ids)
118
+ if len(unique_subjects) < n_splits:
119
+ logger.warning(
120
+ f"Only {len(unique_subjects)} unique subjects, reducing n_splits to {len(unique_subjects)}"
121
+ )
122
+ n_splits = len(unique_subjects)
123
+
124
+ gkf = GroupKFold(n_splits=n_splits)
125
+ splits = []
126
+ for train_idx, test_idx in gkf.split(X, y, groups=subject_ids):
127
+ splits.append((train_idx, test_idx))
128
+
129
+ logger.info(
130
+ f"Generated {n_splits} subject‑aware folds "
131
+ f"(unique subjects={len(unique_subjects)})"
132
+ )
133
+ return splits
134
+
135
+ def leave_one_subject_out(
136
+ self,
137
+ X: Union[np.ndarray, pd.DataFrame],
138
+ y: Union[np.ndarray, List[int]],
139
+ subject_ids: Union[np.ndarray, List[str]],
140
+ ) -> List[Tuple[np.ndarray, np.ndarray]]:
141
+ """Leave‑One‑Subject‑Out (LOSO) cross‑validation generator.
142
+
143
+ Each fold uses all subjects except one as training, and the left‑out
144
+ subject as test.
145
+
146
+ Args:
147
+ X: Feature matrix.
148
+ y: Target labels.
149
+ subject_ids: Subject identifier for each sample.
150
+
151
+ Returns:
152
+ List of (train_indices, test_indices) for each fold.
153
+ """
154
+ subject_ids = np.asarray(subject_ids)
155
+ unique_subjects = np.unique(subject_ids)
156
+
157
+ splits = []
158
+ for subj in unique_subjects:
159
+ test_mask = subject_ids == subj
160
+ train_mask = ~test_mask
161
+ train_idx = np.where(train_mask)[0]
162
+ test_idx = np.where(test_mask)[0]
163
+ splits.append((train_idx, test_idx))
164
+
165
+ logger.info(
166
+ f"Generated {len(splits)} LOSO folds (unique subjects={len(unique_subjects)})"
167
+ )
168
+ return splits
169
+
170
+ def run_cross_validation(
171
+ self,
172
+ classifier: BaseEstimator,
173
+ X: Union[np.ndarray, pd.DataFrame],
174
+ y: Union[np.ndarray, List[int]],
175
+ n_splits: Optional[int] = None,
176
+ subject_ids: Optional[Union[np.ndarray, List[str]]] = None,
177
+ method: str = "stratified",
178
+ return_predictions: bool = True,
179
+ ) -> Dict[str, Any]:
180
+ """Run cross‑validation and compute metrics across folds.
181
+
182
+ Args:
183
+ classifier: A scikit‑learn compatible classifier.
184
+ X: Feature matrix.
185
+ y: Target labels.
186
+ n_splits: Override default number of folds.
187
+ subject_ids: Required for 'subject_aware' and 'loso' methods.
188
+ method: One of 'stratified', 'subject_aware', 'loso'.
189
+ return_predictions: If True, aggregate predictions across folds.
190
+
191
+ Returns:
192
+ Dictionary with:
193
+ - mean_metrics: dict of mean metric values across folds
194
+ - std_metrics: dict of standard deviations
195
+ - per_fold_metrics: list of metric dicts for each fold
196
+ - aggregated_predictions: dict with y_true, y_pred, y_prob (if return_predictions)
197
+ - fold_indices: list of (train_idx, test_idx) for each fold
198
+ """
199
+ n_splits = n_splits or self.n_splits
200
+ y = np.asarray(y)
201
+
202
+ # Generate fold indices
203
+ if method == "stratified":
204
+ folds = self.stratified_k_fold(X, y, n_splits=n_splits)
205
+ elif method == "subject_aware":
206
+ if subject_ids is None:
207
+ raise ValueError("subject_ids required for subject_aware method")
208
+ folds = self.subject_aware_split(X, y, subject_ids, n_splits=n_splits)
209
+ elif method == "loso":
210
+ if subject_ids is None:
211
+ raise ValueError("subject_ids required for loso method")
212
+ folds = self.leave_one_subject_out(X, y, subject_ids)
213
+ else:
214
+ raise ValueError(
215
+ f"Unknown method '{method}'. Choose from 'stratified', 'subject_aware', 'loso'."
216
+ )
217
+
218
+ per_fold_metrics = []
219
+ all_y_true = []
220
+ all_y_pred = []
221
+ all_y_prob = []
222
+ fold_indices = []
223
+
224
+ for fold_idx, (train_idx, test_idx) in enumerate(folds):
225
+ logger.info(f"Fold {fold_idx+1}/{len(folds)}: train={len(train_idx)}, test={len(test_idx)}")
226
+
227
+ X_train, X_test = X[train_idx], X[test_idx]
228
+ y_train, y_test = y[train_idx], y[test_idx]
229
+
230
+ # Train classifier
231
+ classifier.fit(X_train, y_train)
232
+
233
+ # Predict
234
+ y_pred = classifier.predict(X_test)
235
+ y_prob = (
236
+ classifier.predict_proba(X_test)[:, 1]
237
+ if hasattr(classifier, "predict_proba")
238
+ else None
239
+ )
240
+
241
+ # Compute metrics
242
+ metrics = self.metrics.compute_all_metrics(y_test, y_pred, y_prob)
243
+ per_fold_metrics.append(metrics)
244
+
245
+ # Aggregate for overall analysis
246
+ all_y_true.extend(y_test.tolist())
247
+ all_y_pred.extend(y_pred.tolist())
248
+ if y_prob is not None:
249
+ all_y_prob.extend(y_prob.tolist())
250
+
251
+ fold_indices.append((train_idx.tolist(), test_idx.tolist()))
252
+
253
+ # Compute mean and std across folds
254
+ metric_names = [
255
+ "accuracy",
256
+ "precision",
257
+ "recall",
258
+ "f1",
259
+ "specificity",
260
+ "balanced_accuracy",
261
+ ]
262
+ if any("auc_roc" in m for m in per_fold_metrics):
263
+ metric_names.append("auc_roc")
264
+ if any("average_precision" in m for m in per_fold_metrics):
265
+ metric_names.append("average_precision")
266
+
267
+ mean_metrics = {}
268
+ std_metrics = {}
269
+ for name in metric_names:
270
+ values = []
271
+ for fold_metrics in per_fold_metrics:
272
+ if name in fold_metrics:
273
+ values.append(fold_metrics[name])
274
+ if values:
275
+ mean_metrics[name] = float(np.mean(values))
276
+ std_metrics[name] = float(np.std(values))
277
+ else:
278
+ mean_metrics[name] = np.nan
279
+ std_metrics[name] = np.nan
280
+
281
+ # Aggregate predictions
282
+ aggregated = {}
283
+ if return_predictions:
284
+ aggregated["y_true"] = all_y_true
285
+ aggregated["y_pred"] = all_y_pred
286
+ if all_y_prob:
287
+ aggregated["y_prob"] = all_y_prob
288
+
289
+ result = {
290
+ "mean_metrics": mean_metrics,
291
+ "std_metrics": std_metrics,
292
+ "per_fold_metrics": per_fold_metrics,
293
+ "aggregated_predictions": aggregated,
294
+ "fold_indices": fold_indices,
295
+ "method": method,
296
+ "n_splits": len(folds),
297
+ }
298
+
299
+ logger.info(
300
+ f"Cross‑validation completed ({method}, {len(folds)} folds). "
301
+ f"Mean accuracy = {mean_metrics.get('accuracy', 0):.3f} ± {std_metrics.get('accuracy', 0):.3f}"
302
+ )
303
+ return result
src/evaluation/metrics.py ADDED
@@ -0,0 +1,356 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Comprehensive evaluation metrics for deception detection.
2
+
3
+ This module provides a DeceptionMetrics class that computes a wide range of
4
+ classification metrics, fairness metrics, and confidence intervals for
5
+ evaluating deception detection models.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from typing import Any, Dict, List, Optional, Tuple, Union
12
+
13
+ import numpy as np
14
+ import pandas as pd
15
+ from sklearn.metrics import (
16
+ accuracy_score,
17
+ precision_score,
18
+ recall_score,
19
+ f1_score,
20
+ roc_auc_score,
21
+ average_precision_score,
22
+ confusion_matrix,
23
+ classification_report,
24
+ balanced_accuracy_score,
25
+ )
26
+ from sklearn.utils import resample
27
+ from scipy import stats
28
+
29
+
30
+ logger = logging.getLogger(__name__)
31
+
32
+
33
+ class DeceptionMetrics:
34
+ """Compute and aggregate evaluation metrics for deception detection.
35
+
36
+ This class provides methods for computing standard classification metrics,
37
+ per-subject metrics, fairness metrics, and bootstrap confidence intervals.
38
+
39
+ Attributes:
40
+ random_state: Random seed for reproducibility.
41
+ n_bootstrap: Number of bootstrap samples for confidence intervals.
42
+ ci_level: Confidence level for intervals (default 0.95).
43
+ """
44
+
45
+ def __init__(
46
+ self,
47
+ random_state: int = 42,
48
+ n_bootstrap: int = 1000,
49
+ ci_level: float = 0.95,
50
+ ) -> None:
51
+ """Initialize metrics calculator.
52
+
53
+ Args:
54
+ random_state: Random seed for reproducibility.
55
+ n_bootstrap: Number of bootstrap samples for confidence intervals.
56
+ ci_level: Confidence level for intervals (default 0.95).
57
+ """
58
+ self.random_state = random_state
59
+ self.n_bootstrap = n_bootstrap
60
+ self.ci_level = ci_level
61
+ np.random.seed(random_state)
62
+
63
+ def compute_all_metrics(
64
+ self,
65
+ y_true: Union[np.ndarray, List[int]],
66
+ y_pred: Union[np.ndarray, List[int]],
67
+ y_prob: Optional[Union[np.ndarray, List[float]]] = None,
68
+ ) -> Dict[str, Any]:
69
+ """Compute comprehensive classification metrics.
70
+
71
+ Args:
72
+ y_true: Ground truth binary labels (0 = truthful, 1 = deceptive).
73
+ y_pred: Predicted binary labels.
74
+ y_prob: Predicted probabilities for the positive class (deceptive).
75
+ If provided, ROC‑AUC and average precision are computed.
76
+
77
+ Returns:
78
+ Dictionary containing:
79
+ - accuracy, precision, recall, f1, specificity
80
+ - balanced_accuracy
81
+ - auc_roc, average_precision (if y_prob provided)
82
+ - confusion_matrix (as 2x2 array)
83
+ - classification_report (as dict)
84
+ """
85
+ y_true = np.asarray(y_true)
86
+ y_pred = np.asarray(y_pred)
87
+
88
+ if y_true.ndim != 1:
89
+ raise ValueError(f"y_true must be 1‑D, got shape {y_true.shape}")
90
+ if y_pred.ndim != 1:
91
+ raise ValueError(f"y_pred must be 1‑D, got shape {y_pred.shape}")
92
+ if len(y_true) != len(y_pred):
93
+ raise ValueError(
94
+ f"Length mismatch: y_true ({len(y_true)}) vs y_pred ({len(y_pred)})"
95
+ )
96
+
97
+ # Basic metrics
98
+ accuracy = accuracy_score(y_true, y_pred)
99
+ precision = precision_score(y_true, y_pred, zero_division=0)
100
+ recall = recall_score(y_true, y_pred, zero_division=0)
101
+ f1 = f1_score(y_true, y_pred, zero_division=0)
102
+ balanced_acc = balanced_accuracy_score(y_true, y_pred)
103
+
104
+ # Specificity (true negative rate)
105
+ tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
106
+ specificity = tn / (tn + fp) if (tn + fp) > 0 else 0.0
107
+
108
+ # Probability‑based metrics
109
+ auc_roc = None
110
+ avg_precision = None
111
+ if y_prob is not None:
112
+ y_prob = np.asarray(y_prob)
113
+ if y_prob.ndim == 2 and y_prob.shape[1] == 2:
114
+ y_prob = y_prob[:, 1] # take positive class probabilities
115
+ try:
116
+ auc_roc = roc_auc_score(y_true, y_prob)
117
+ except ValueError as e:
118
+ logger.warning(f"Could not compute ROC‑AUC: {e}")
119
+ auc_roc = None
120
+ try:
121
+ avg_precision = average_precision_score(y_true, y_prob)
122
+ except ValueError as e:
123
+ logger.warning(f"Could not compute average precision: {e}")
124
+ avg_precision = None
125
+
126
+ # Confusion matrix
127
+ cm = confusion_matrix(y_true, y_pred)
128
+
129
+ # Classification report as dict
130
+ report = classification_report(
131
+ y_true, y_pred, output_dict=True, zero_division=0
132
+ )
133
+
134
+ metrics = {
135
+ "accuracy": float(accuracy),
136
+ "precision": float(precision),
137
+ "recall": float(recall),
138
+ "f1": float(f1),
139
+ "specificity": float(specificity),
140
+ "balanced_accuracy": float(balanced_acc),
141
+ "confusion_matrix": cm.tolist(),
142
+ "classification_report": report,
143
+ "support": int(len(y_true)),
144
+ }
145
+ if auc_roc is not None:
146
+ metrics["auc_roc"] = float(auc_roc)
147
+ if avg_precision is not None:
148
+ metrics["average_precision"] = float(avg_precision)
149
+
150
+ logger.info(
151
+ f"Computed metrics: accuracy={accuracy:.3f}, f1={f1:.3f}, "
152
+ f"recall={recall:.3f}, precision={precision:.3f}"
153
+ )
154
+ return metrics
155
+
156
+ def compute_per_subject_metrics(
157
+ self,
158
+ y_true: Union[np.ndarray, List[int]],
159
+ y_pred: Union[np.ndarray, List[int]],
160
+ subject_ids: Union[np.ndarray, List[str]],
161
+ y_prob: Optional[Union[np.ndarray, List[float]]] = None,
162
+ ) -> Dict[str, Dict[str, Any]]:
163
+ """Compute metrics separately for each subject.
164
+
165
+ Args:
166
+ y_true: Ground truth binary labels.
167
+ y_pred: Predicted binary labels.
168
+ subject_ids: Subject identifier for each sample.
169
+ y_prob: Predicted probabilities (optional).
170
+
171
+ Returns:
172
+ Dictionary mapping subject_id -> metrics dict.
173
+ """
174
+ y_true = np.asarray(y_true)
175
+ y_pred = np.asarray(y_pred)
176
+ subject_ids = np.asarray(subject_ids)
177
+
178
+ if len(y_true) != len(subject_ids):
179
+ raise ValueError(
180
+ f"Length mismatch: y_true ({len(y_true)}) vs subject_ids ({len(subject_ids)})"
181
+ )
182
+
183
+ unique_subjects = np.unique(subject_ids)
184
+ subject_metrics = {}
185
+
186
+ for subj in unique_subjects:
187
+ mask = subject_ids == subj
188
+ if np.sum(mask) == 0:
189
+ continue
190
+ subj_y_true = y_true[mask]
191
+ subj_y_pred = y_pred[mask]
192
+ subj_y_prob = y_prob[mask] if y_prob is not None else None
193
+
194
+ try:
195
+ metrics = self.compute_all_metrics(
196
+ subj_y_true, subj_y_pred, subj_y_prob
197
+ )
198
+ subject_metrics[subj] = metrics
199
+ except Exception as e:
200
+ logger.warning(f"Failed to compute metrics for subject {subj}: {e}")
201
+ subject_metrics[subj] = {"error": str(e)}
202
+
203
+ logger.info(f"Computed per‑subject metrics for {len(subject_metrics)} subjects")
204
+ return subject_metrics
205
+
206
+ def compute_fairness_metrics(
207
+ self,
208
+ y_true: Union[np.ndarray, List[int]],
209
+ y_pred: Union[np.ndarray, List[int]],
210
+ sensitive_groups: Union[np.ndarray, List[str]],
211
+ ) -> Dict[str, float]:
212
+ """Compute fairness metrics across sensitive groups.
213
+
214
+ Currently implemented:
215
+ - Demographic parity difference: max difference in positive rate across groups.
216
+ - Equalized odds difference: max difference in TPR and FPR across groups.
217
+
218
+ Args:
219
+ y_true: Ground truth binary labels.
220
+ y_pred: Predicted binary labels.
221
+ sensitive_groups: Group membership for each sample (e.g., gender, ethnicity).
222
+
223
+ Returns:
224
+ Dictionary with fairness metrics.
225
+ """
226
+ y_true = np.asarray(y_true)
227
+ y_pred = np.asarray(y_pred)
228
+ sensitive_groups = np.asarray(sensitive_groups)
229
+
230
+ unique_groups = np.unique(sensitive_groups)
231
+ group_stats = {}
232
+
233
+ for group in unique_groups:
234
+ mask = sensitive_groups == group
235
+ if np.sum(mask) == 0:
236
+ continue
237
+ group_y_true = y_true[mask]
238
+ group_y_pred = y_pred[mask]
239
+
240
+ # Positive rate (predicted)
241
+ pos_rate = np.mean(group_y_pred)
242
+
243
+ # True positive rate and false positive rate
244
+ tn, fp, fn, tp = confusion_matrix(group_y_true, group_y_pred).ravel()
245
+ tpr = tp / (tp + fn) if (tp + fn) > 0 else 0.0
246
+ fpr = fp / (fp + tn) if (fp + tn) > 0 else 0.0
247
+
248
+ group_stats[group] = {
249
+ "positive_rate": pos_rate,
250
+ "tpr": tpr,
251
+ "fpr": fpr,
252
+ "support": int(np.sum(mask)),
253
+ }
254
+
255
+ # Demographic parity difference
256
+ pos_rates = [stats["positive_rate"] for stats in group_stats.values()]
257
+ demographic_parity_diff = max(pos_rates) - min(pos_rates) if pos_rates else 0.0
258
+
259
+ # Equalized odds differences (max TPR diff, max FPR diff)
260
+ tprs = [stats["tpr"] for stats in group_stats.values()]
261
+ fprs = [stats["fpr"] for stats in group_stats.values()]
262
+ equalized_odds_tpr_diff = max(tprs) - min(tprs) if tprs else 0.0
263
+ equalized_odds_fpr_diff = max(fprs) - min(fprs) if fprs else 0.0
264
+
265
+ fairness = {
266
+ "demographic_parity_difference": float(demographic_parity_diff),
267
+ "equalized_odds_tpr_difference": float(equalized_odds_tpr_diff),
268
+ "equalized_odds_fpr_difference": float(equalized_odds_fpr_diff),
269
+ "group_stats": group_stats,
270
+ }
271
+ logger.info(
272
+ f"Fairness metrics: demographic parity diff = {demographic_parity_diff:.3f}, "
273
+ f"equalized odds TPR diff = {equalized_odds_tpr_diff:.3f}"
274
+ )
275
+ return fairness
276
+
277
+ def bootstrap_confidence_intervals(
278
+ self,
279
+ y_true: Union[np.ndarray, List[int]],
280
+ y_pred: Union[np.ndarray, List[int]],
281
+ y_prob: Optional[Union[np.ndarray, List[float]]] = None,
282
+ n_bootstrap: Optional[int] = None,
283
+ ci: Optional[float] = None,
284
+ ) -> Dict[str, Dict[str, float]]:
285
+ """Compute bootstrap confidence intervals for metrics.
286
+
287
+ Args:
288
+ y_true: Ground truth binary labels.
289
+ y_pred: Predicted binary labels.
290
+ y_prob: Predicted probabilities (optional).
291
+ n_bootstrap: Override default number of bootstrap samples.
292
+ ci: Override default confidence level.
293
+
294
+ Returns:
295
+ Dictionary mapping metric name -> {'lower': ..., 'upper': ..., 'mean': ...}.
296
+ """
297
+ n_bootstrap = n_bootstrap or self.n_bootstrap
298
+ ci_level = ci or self.ci_level
299
+ y_true = np.asarray(y_true)
300
+ y_pred = np.asarray(y_pred)
301
+ y_prob = np.asarray(y_prob) if y_prob is not None else None
302
+
303
+ n_samples = len(y_true)
304
+ metric_names = [
305
+ "accuracy",
306
+ "precision",
307
+ "recall",
308
+ "f1",
309
+ "specificity",
310
+ "balanced_accuracy",
311
+ ]
312
+ if y_prob is not None:
313
+ metric_names.extend(["auc_roc", "average_precision"])
314
+
315
+ bootstrap_metrics = {name: [] for name in metric_names}
316
+
317
+ for i in range(n_bootstrap):
318
+ indices = resample(
319
+ np.arange(n_samples),
320
+ replace=True,
321
+ n_samples=n_samples,
322
+ random_state=self.random_state + i,
323
+ )
324
+ b_y_true = y_true[indices]
325
+ b_y_pred = y_pred[indices]
326
+ b_y_prob = y_prob[indices] if y_prob is not None else None
327
+
328
+ try:
329
+ metrics = self.compute_all_metrics(b_y_true, b_y_pred, b_y_prob)
330
+ for name in metric_names:
331
+ if name in metrics:
332
+ bootstrap_metrics[name].append(metrics[name])
333
+ except Exception as e:
334
+ logger.warning(f"Bootstrap iteration {i} failed: {e}")
335
+
336
+ # Compute confidence intervals
337
+ ci_results = {}
338
+ alpha = (1 - ci_level) / 2
339
+ for name, values in bootstrap_metrics.items():
340
+ if len(values) == 0:
341
+ ci_results[name] = {"lower": np.nan, "upper": np.nan, "mean": np.nan}
342
+ continue
343
+ lower = np.percentile(values, 100 * alpha)
344
+ upper = np.percentile(values, 100 * (1 - alpha))
345
+ mean = np.mean(values)
346
+ ci_results[name] = {
347
+ "lower": float(lower),
348
+ "upper": float(upper),
349
+ "mean": float(mean),
350
+ }
351
+
352
+ logger.info(
353
+ f"Computed bootstrap CIs for {len(ci_results)} metrics "
354
+ f"({n_bootstrap} samples, CI={ci_level})"
355
+ )
356
+ return ci_results
src/evaluation/visualizer.py ADDED
@@ -0,0 +1,396 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Visualization tools for deception detection evaluation.
2
+
3
+ This module provides plotting functions for confusion matrices, ROC curves,
4
+ precision‑recall curves, cross‑validation results, ablation studies, and
5
+ subject‑wise performance.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from typing import Any, Dict, List, Optional, Tuple, Union
12
+
13
+ import matplotlib.pyplot as plt
14
+ import numpy as np
15
+ import pandas as pd
16
+ import seaborn as sns
17
+ from matplotlib.figure import Figure
18
+ from sklearn.metrics import (
19
+ roc_curve,
20
+ auc,
21
+ precision_recall_curve,
22
+ average_precision_score,
23
+ confusion_matrix,
24
+ )
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ class EvaluationVisualizer:
30
+ """Create publication‑quality plots for evaluation results.
31
+
32
+ All methods accept a `save_path` argument; if provided, the figure is saved
33
+ to that path. Otherwise, the figure is returned for further customization.
34
+
35
+ Attributes:
36
+ style: Matplotlib style to use (default 'seaborn‑whitegrid').
37
+ figsize: Default figure size as (width, height).
38
+ dpi: Resolution for saved figures.
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ style: str = "seaborn‑whitegrid",
44
+ figsize: Tuple[float, float] = (10, 8),
45
+ dpi: int = 150,
46
+ ) -> None:
47
+ """Initialize visualizer.
48
+
49
+ Args:
50
+ style: Matplotlib style name.
51
+ figsize: Default figure size (width, height) in inches.
52
+ dpi: Dots per inch for saved figures.
53
+ """
54
+ self.style = style
55
+ self.figsize = figsize
56
+ self.dpi = dpi
57
+ plt.style.use(style)
58
+
59
+ def plot_confusion_matrix(
60
+ self,
61
+ cm: Union[np.ndarray, List[List[int]]],
62
+ labels: List[str],
63
+ normalize: bool = False,
64
+ title: str = "Confusion Matrix",
65
+ cmap: str = "Blues",
66
+ save_path: Optional[str] = None,
67
+ ) -> Figure:
68
+ """Plot a confusion matrix.
69
+
70
+ Args:
71
+ cm: 2×2 or N×N confusion matrix.
72
+ labels: List of class labels (e.g., ['Truthful', 'Deceptive']).
73
+ normalize: If True, show proportions instead of counts.
74
+ title: Plot title.
75
+ cmap: Colormap for the heatmap.
76
+ save_path: If provided, save figure to this path.
77
+
78
+ Returns:
79
+ Matplotlib Figure object.
80
+ """
81
+ cm = np.asarray(cm)
82
+ if normalize:
83
+ cm = cm.astype("float") / cm.sum(axis=1, keepdims=True)
84
+ fmt = ".2f"
85
+ else:
86
+ fmt = "d"
87
+
88
+ fig, ax = plt.subplots(figsize=self.figsize)
89
+ sns.heatmap(
90
+ cm,
91
+ annot=True,
92
+ fmt=fmt,
93
+ cmap=cmap,
94
+ xticklabels=labels,
95
+ yticklabels=labels,
96
+ ax=ax,
97
+ cbar_kws={"label": "Normalized frequency" if normalize else "Count"},
98
+ )
99
+ ax.set_xlabel("Predicted label")
100
+ ax.set_ylabel("True label")
101
+ ax.set_title(title)
102
+
103
+ if save_path:
104
+ fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
105
+ logger.info(f"Saved confusion matrix to {save_path}")
106
+
107
+ return fig
108
+
109
+ def plot_roc_curve(
110
+ self,
111
+ y_true: Union[np.ndarray, List[int]],
112
+ y_prob: Union[np.ndarray, List[float]],
113
+ label: str = "Model",
114
+ title: str = "ROC Curve",
115
+ save_path: Optional[str] = None,
116
+ ) -> Figure:
117
+ """Plot ROC curve with AUC.
118
+
119
+ Args:
120
+ y_true: Ground truth binary labels.
121
+ y_prob: Predicted probabilities for the positive class.
122
+ label: Label for the curve (used in legend).
123
+ title: Plot title.
124
+ save_path: If provided, save figure to this path.
125
+
126
+ Returns:
127
+ Matplotlib Figure object.
128
+ """
129
+ y_true = np.asarray(y_true)
130
+ y_prob = np.asarray(y_prob)
131
+
132
+ fpr, tpr, _ = roc_curve(y_true, y_prob)
133
+ roc_auc = auc(fpr, tpr)
134
+
135
+ fig, ax = plt.subplots(figsize=self.figsize)
136
+ ax.plot(fpr, tpr, lw=2, label=f"{label} (AUC = {roc_auc:.3f})")
137
+ ax.plot([0, 1], [0, 1], color="gray", lw=1, linestyle="--", label="Random")
138
+ ax.set_xlim([0.0, 1.0])
139
+ ax.set_ylim([0.0, 1.05])
140
+ ax.set_xlabel("False Positive Rate")
141
+ ax.set_ylabel("True Positive Rate")
142
+ ax.set_title(title)
143
+ ax.legend(loc="lower right")
144
+ ax.grid(True, alpha=0.3)
145
+
146
+ if save_path:
147
+ fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
148
+ logger.info(f"Saved ROC curve to {save_path}")
149
+
150
+ return fig
151
+
152
+ def plot_precision_recall_curve(
153
+ self,
154
+ y_true: Union[np.ndarray, List[int]],
155
+ y_prob: Union[np.ndarray, List[float]],
156
+ label: str = "Model",
157
+ title: str = "Precision‑Recall Curve",
158
+ save_path: Optional[str] = None,
159
+ ) -> Figure:
160
+ """Plot precision‑recall curve with average precision.
161
+
162
+ Args:
163
+ y_true: Ground truth binary labels.
164
+ y_prob: Predicted probabilities for the positive class.
165
+ label: Label for the curve (used in legend).
166
+ title: Plot title.
167
+ save_path: If provided, save figure to this path.
168
+
169
+ Returns:
170
+ Matplotlib Figure object.
171
+ """
172
+ y_true = np.asarray(y_true)
173
+ y_prob = np.asarray(y_prob)
174
+
175
+ precision, recall, _ = precision_recall_curve(y_true, y_prob)
176
+ avg_precision = average_precision_score(y_true, y_prob)
177
+
178
+ fig, ax = plt.subplots(figsize=self.figsize)
179
+ ax.plot(recall, precision, lw=2, label=f"{label} (AP = {avg_precision:.3f})")
180
+ ax.set_xlim([0.0, 1.0])
181
+ ax.set_ylim([0.0, 1.05])
182
+ ax.set_xlabel("Recall")
183
+ ax.set_ylabel("Precision")
184
+ ax.set_title(title)
185
+ ax.legend(loc="lower left")
186
+ ax.grid(True, alpha=0.3)
187
+
188
+ if save_path:
189
+ fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
190
+ logger.info(f"Saved precision‑recall curve to {save_path}")
191
+
192
+ return fig
193
+
194
+ def plot_cross_validation_results(
195
+ self,
196
+ cv_results: Dict[str, Any],
197
+ metrics: Optional[List[str]] = None,
198
+ title: str = "Cross‑Validation Performance",
199
+ save_path: Optional[str] = None,
200
+ ) -> Figure:
201
+ """Plot box‑whisker or bar chart of metrics across CV folds.
202
+
203
+ Args:
204
+ cv_results: Dictionary as returned by CrossValidator.run_cross_validation.
205
+ Must contain 'per_fold_metrics' (list of dicts).
206
+ metrics: List of metric names to plot. If None, defaults to
207
+ ['accuracy', 'precision', 'recall', 'f1', 'auc_roc'].
208
+ title: Plot title.
209
+ save_path: If provided, save figure to this path.
210
+
211
+ Returns:
212
+ Matplotlib Figure object.
213
+ """
214
+ if "per_fold_metrics" not in cv_results:
215
+ raise ValueError("cv_results must contain 'per_fold_metrics'")
216
+
217
+ per_fold = cv_results["per_fold_metrics"]
218
+ if metrics is None:
219
+ metrics = ["accuracy", "precision", "recall", "f1"]
220
+ # Add AUC if present in any fold
221
+ if any("auc_roc" in fold for fold in per_fold):
222
+ metrics.append("auc_roc")
223
+
224
+ # Build DataFrame of metric values across folds
225
+ data = []
226
+ for fold_idx, fold_metrics in enumerate(per_fold):
227
+ for metric in metrics:
228
+ if metric in fold_metrics:
229
+ data.append(
230
+ {
231
+ "fold": f"Fold {fold_idx+1}",
232
+ "metric": metric,
233
+ "value": fold_metrics[metric],
234
+ }
235
+ )
236
+
237
+ if not data:
238
+ logger.warning("No metric data to plot")
239
+ fig, ax = plt.subplots(figsize=self.figsize)
240
+ ax.text(0.5, 0.5, "No data", ha="center", va="center")
241
+ return fig
242
+
243
+ df = pd.DataFrame(data)
244
+
245
+ fig, ax = plt.subplots(figsize=self.figsize)
246
+ sns.boxplot(data=df, x="metric", y="value", ax=ax, palette="Set2")
247
+ sns.stripplot(
248
+ data=df,
249
+ x="metric",
250
+ y="value",
251
+ ax=ax,
252
+ color="black",
253
+ alpha=0.7,
254
+ jitter=True,
255
+ )
256
+ ax.set_xlabel("Metric")
257
+ ax.set_ylabel("Value")
258
+ ax.set_title(title)
259
+ ax.grid(True, alpha=0.3)
260
+
261
+ if save_path:
262
+ fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
263
+ logger.info(f"Saved cross‑validation results plot to {save_path}")
264
+
265
+ return fig
266
+
267
+ def plot_ablation_results(
268
+ self,
269
+ ablation_results: Dict[str, Dict[str, Any]],
270
+ metric: str = "accuracy",
271
+ title: str = "Ablation Study",
272
+ save_path: Optional[str] = None,
273
+ ) -> Figure:
274
+ """Plot bar chart comparing performance across ablation conditions.
275
+
276
+ Args:
277
+ ablation_results: Dictionary as returned by AblationStudy.run_ablation.
278
+ Keys are condition names, each containing 'mean_metrics' dict.
279
+ metric: Metric to compare across conditions.
280
+ title: Plot title.
281
+ save_path: If provided, save figure to this path.
282
+
283
+ Returns:
284
+ Matplotlib Figure object.
285
+ """
286
+ conditions = []
287
+ means = []
288
+ stds = []
289
+
290
+ for cond_name, cond_data in ablation_results.items():
291
+ if "mean_metrics" in cond_data and metric in cond_data["mean_metrics"]:
292
+ conditions.append(cond_name.replace("_", " ").title())
293
+ means.append(cond_data["mean_metrics"][metric])
294
+ stds.append(cond_data.get("std_metrics", {}).get(metric, 0.0))
295
+
296
+ if not conditions:
297
+ logger.warning(f"Metric '{metric}' not found in any condition")
298
+ fig, ax = plt.subplots(figsize=self.figsize)
299
+ ax.text(0.5, 0.5, "No data", ha="center", va="center")
300
+ return fig
301
+
302
+ x = np.arange(len(conditions))
303
+ fig, ax = plt.subplots(figsize=self.figsize)
304
+ bars = ax.bar(x, means, yerr=stds, capsize=5, color="steelblue", alpha=0.8)
305
+ ax.set_xticks(x)
306
+ ax.set_xticklabels(conditions, rotation=45, ha="right")
307
+ ax.set_ylabel(metric.replace("_", " ").title())
308
+ ax.set_title(title)
309
+ ax.grid(True, alpha=0.3, axis="y")
310
+
311
+ # Add value labels on top of bars
312
+ for bar, mean_val in zip(bars, means):
313
+ height = bar.get_height()
314
+ ax.text(
315
+ bar.get_x() + bar.get_width() / 2,
316
+ height + 0.01,
317
+ f"{mean_val:.3f}",
318
+ ha="center",
319
+ va="bottom",
320
+ fontsize=9,
321
+ )
322
+
323
+ if save_path:
324
+ fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
325
+ logger.info(f"Saved ablation results plot to {save_path}")
326
+
327
+ return fig
328
+
329
+ def plot_subject_performance(
330
+ self,
331
+ per_subject_metrics: Dict[str, Dict[str, Any]],
332
+ metric: str = "accuracy",
333
+ title: str = "Subject‑wise Performance",
334
+ sort_by: str = "value", # 'value' or 'subject'
335
+ save_path: Optional[str] = None,
336
+ ) -> Figure:
337
+ """Plot bar chart of a chosen metric across subjects.
338
+
339
+ Args:
340
+ per_subject_metrics: Dictionary mapping subject_id -> metrics dict.
341
+ metric: Metric to plot (must exist in each subject's dict).
342
+ title: Plot title.
343
+ sort_by: 'value' to sort bars by metric value, 'subject' to keep original order.
344
+ save_path: If provided, save figure to this path.
345
+
346
+ Returns:
347
+ Matplotlib Figure object.
348
+ """
349
+ subjects = []
350
+ values = []
351
+
352
+ for subj, metrics in per_subject_metrics.items():
353
+ if metric in metrics:
354
+ subjects.append(subj)
355
+ values.append(metrics[metric])
356
+ else:
357
+ logger.warning(f"Metric '{metric}' not found for subject {subj}")
358
+
359
+ if not subjects:
360
+ logger.warning("No subject data to plot")
361
+ fig, ax = plt.subplots(figsize=self.figsize)
362
+ ax.text(0.5, 0.5, "No data", ha="center", va="center")
363
+ return fig
364
+
365
+ if sort_by == "value":
366
+ # Sort by value descending
367
+ sorted_pairs = sorted(zip(subjects, values), key=lambda x: x[1], reverse=True)
368
+ subjects, values = zip(*sorted_pairs)
369
+ # else keep original order
370
+
371
+ x = np.arange(len(subjects))
372
+ fig, ax = plt.subplots(figsize=(max(10, len(subjects) * 0.5), 6))
373
+ bars = ax.bar(x, values, color="teal", alpha=0.7)
374
+ ax.set_xticks(x)
375
+ ax.set_xticklabels(subjects, rotation=90, ha="center")
376
+ ax.set_ylabel(metric.replace("_", " ").title())
377
+ ax.set_title(title)
378
+ ax.grid(True, alpha=0.3, axis="y")
379
+
380
+ # Add value labels
381
+ for bar, val in zip(bars, values):
382
+ height = bar.get_height()
383
+ ax.text(
384
+ bar.get_x() + bar.get_width() / 2,
385
+ height + 0.005,
386
+ f"{val:.3f}",
387
+ ha="center",
388
+ va="bottom",
389
+ fontsize=8,
390
+ )
391
+
392
+ if save_path:
393
+ fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
394
+ logger.info(f"Saved subject performance plot to {save_path}")
395
+
396
+ return fig
src/explainability/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """Explainability module for SHAP-based feature interpretation."""
2
+
3
+ from .explainer import DeceptionExplainer
4
+ from .visualizer import ExplainabilityVisualizer
5
+
6
+ __all__ = ['DeceptionExplainer', 'ExplainabilityVisualizer']
src/explainability/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (355 Bytes). View file
 
src/explainability/__pycache__/explainer.cpython-314.pyc ADDED
Binary file (24 kB). View file
 
src/explainability/__pycache__/visualizer.cpython-314.pyc ADDED
Binary file (36.1 kB). View file
 
src/explainability/explainer.py ADDED
@@ -0,0 +1,513 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Explainability module for deception detection using SHAP and feature importance.
2
+
3
+ This module provides interpretable explanations for deception predictions by
4
+ combining SHAP values, feature importance, and domain knowledge to produce
5
+ human-readable behavioral explanations.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import warnings
11
+ from typing import Any, Dict, List, Optional, Tuple, Union
12
+
13
+ import numpy as np
14
+ import pandas as pd
15
+ import shap
16
+ from sklearn.inspection import permutation_importance
17
+ from sklearn.utils import check_array
18
+
19
+ from src.model.classifier import DeceptionClassifier
20
+
21
+
22
+ class DeceptionExplainer:
23
+ """Generates interpretable explanations for deception predictions.
24
+
25
+ Combines SHAP values, feature importance, and domain knowledge
26
+ to produce human-readable behavioral explanations.
27
+ """
28
+
29
+ # Domain knowledge mapping: feature name -> behavioral interpretation
30
+ FEATURE_INTERPRETATIONS = {
31
+ 'eye_left_openness': 'Left eye openness (reduced blinking or gaze aversion may indicate deception)',
32
+ 'eye_right_openness': 'Right eye openness (asymmetric eye behavior can signal cognitive load)',
33
+ 'mouth_openness': 'Mouth openness (suppressed or exaggerated expressions may indicate deception)',
34
+ 'mouth_width': 'Mouth width changes (forced smiles show different width patterns)',
35
+ 'mouth_asymmetry': 'Mouth asymmetry (asymmetric expressions are harder to fake)',
36
+ 'left_brow_height': 'Left eyebrow height (raised brows can indicate surprise or concern)',
37
+ 'right_brow_height': 'Right eyebrow height (asymmetric brow activity linked to deception)',
38
+ 'brow_asymmetry': 'Eyebrow asymmetry (involuntary micro-expressions often appear asymmetrically)',
39
+ 'brow_tension': 'Eyebrow tension (furrowed brows indicate cognitive effort or stress)',
40
+ 'nose_wrinkle': 'Nose wrinkle (disgust micro-expression, hard to fake voluntarily)',
41
+ 'nostril_flare': 'Nostril flare (stress indicator, autonomic nervous system response)',
42
+ 'nose_asymmetry': 'Nose asymmetry (subtle asymmetries reveal involuntary expressions)',
43
+ 'head_pitch': 'Head pitch (nodding patterns differ between truthful and deceptive responses)',
44
+ 'head_yaw': 'Head yaw (head turning can indicate avoidance behavior)',
45
+ 'head_roll': 'Head tilt (unusual tilt patterns may indicate discomfort)',
46
+ 'head_displacement': 'Head movement (excessive stillness or fidgeting both signal stress)',
47
+ 'pitch_mean': 'Average pitch (elevated pitch is a stress indicator)',
48
+ 'pitch_std': 'Pitch variability (reduced variability suggests rehearsed responses)',
49
+ 'pitch_range': 'Pitch range (constricted range indicates tension)',
50
+ 'pitch_variation_coefficient': 'Pitch variation coefficient (normalized pitch instability)',
51
+ 'voiced_ratio': 'Voiced speech ratio (reduced voicing suggests hesitation)',
52
+ 'pause_count': 'Number of pauses (increased pausing indicates cognitive load)',
53
+ 'avg_pause_duration': 'Average pause length (longer pauses suggest response fabrication)',
54
+ 'max_pause_duration': 'Longest pause (extended silence before answering is suspicious)',
55
+ 'pause_rate': 'Pauses per minute (higher rate indicates cognitive effort)',
56
+ 'speech_ratio': 'Speech ratio (less speech time relative to pauses suggests evasion)',
57
+ 'energy_mean': 'Average speech energy (reduced energy may indicate withdrawal)',
58
+ 'energy_std': 'Energy variability (high variability suggests emotional instability)',
59
+ 'energy_range': 'Energy range (wide range indicates emotional dysregulation)',
60
+ 'energy_variation': 'Energy variation coefficient (normalized energy instability)',
61
+ 'spectral_centroid_mean': 'Spectral centroid (brightness of voice, shifts under stress)',
62
+ 'spectral_bandwidth_mean': 'Spectral bandwidth (voice quality changes under deception)',
63
+ 'audio_visual_sync': 'Audio-visual synchrony (desynchronization suggests incongruent behavior)',
64
+ 'pause_gaze_coupling': 'Pause-gaze coupling (looking away while pausing is a deception cue)',
65
+ 'stress_composite': 'Stress composite (combined physiological stress indicator)',
66
+ }
67
+
68
+ FEATURE_GROUPS = {
69
+ 'Eye Behavior': ['eye_left_openness', 'eye_right_openness'],
70
+ 'Mouth Dynamics': ['mouth_openness', 'mouth_width', 'mouth_asymmetry'],
71
+ 'Eyebrow Activity': ['left_brow_height', 'right_brow_height', 'brow_asymmetry', 'brow_tension'],
72
+ 'Nose Movement': ['nose_wrinkle', 'nostril_flare', 'nose_asymmetry'],
73
+ 'Head Dynamics': ['head_pitch', 'head_yaw', 'head_roll', 'head_displacement'],
74
+ 'Voice Pitch': ['pitch_mean', 'pitch_std', 'pitch_range', 'pitch_variation_coefficient'],
75
+ 'Speech Timing': ['pause_count', 'avg_pause_duration', 'max_pause_duration', 'pause_rate', 'speech_ratio', 'voiced_ratio'],
76
+ 'Voice Energy': ['energy_mean', 'energy_std', 'energy_range', 'energy_variation', 'spectral_centroid_mean', 'spectral_bandwidth_mean'],
77
+ 'Cross-Modal': ['audio_visual_sync', 'pause_gaze_coupling', 'stress_composite'],
78
+ }
79
+
80
+ def __init__(self, classifier: DeceptionClassifier):
81
+ """Initialize with a trained DeceptionClassifier.
82
+
83
+ Args:
84
+ classifier: A trained DeceptionClassifier instance.
85
+
86
+ Raises:
87
+ ValueError: If classifier is not trained.
88
+ """
89
+ if not classifier.is_fitted_:
90
+ warnings.warn("Classifier must be trained before creating explainer. Explainer will not work until trained.")
91
+
92
+ self.classifier = classifier
93
+ self._shap_explainer = None
94
+ self._feature_names = classifier.feature_names_
95
+
96
+ def _init_shap_explainer(self) -> Optional[shap.TreeExplainer]:
97
+ """Lazy-initialize SHAP TreeExplainer from the trained model.
98
+
99
+ Returns:
100
+ shap.TreeExplainer: Initialized SHAP explainer, or None on failure.
101
+
102
+ Notes:
103
+ Uses feature_perturbation='tree_path_dependent' for tree-based models.
104
+ On failure, warns and returns None.
105
+ """
106
+ if self._shap_explainer is None:
107
+ try:
108
+ # Use the underlying GradientBoostingClassifier (not calibrated model)
109
+ # Research rationale: TreeExplainer works best with tree-based models
110
+ # and provides exact SHAP values for gradient boosting.
111
+ self._shap_explainer = shap.TreeExplainer(
112
+ model=self.classifier.model,
113
+ feature_perturbation='tree_path_dependent'
114
+ )
115
+ except Exception as e:
116
+ warnings.warn(
117
+ f"Failed to initialize SHAP TreeExplainer: {e}. "
118
+ "Falling back to permutation importance.",
119
+ RuntimeWarning
120
+ )
121
+ self._shap_explainer = None
122
+
123
+ return self._shap_explainer
124
+
125
+ def compute_shap_values(self, X: np.ndarray, feature_names: List[str]) -> np.ndarray:
126
+ """Compute SHAP values using TreeExplainer.
127
+
128
+ Args:
129
+ X: Feature matrix of shape (n_samples, n_features).
130
+ feature_names: Ordered list of feature names matching columns of X.
131
+
132
+ Returns:
133
+ np.ndarray: SHAP values array of shape (n_samples, n_features).
134
+
135
+ Notes:
136
+ - If SHAP fails, falls back to permutation importance.
137
+ - Returns absolute SHAP values (magnitude of contribution).
138
+ """
139
+ X = check_array(X, ensure_2d=True, dtype=np.float64)
140
+
141
+ # Ensure feature names match
142
+ if len(feature_names) != X.shape[1]:
143
+ raise ValueError(
144
+ f"Feature names length ({len(feature_names)}) "
145
+ f"does not match X columns ({X.shape[1]})"
146
+ )
147
+
148
+ # Scale features using classifier's scaler
149
+ X_scaled = self.classifier.scaler.transform(X)
150
+
151
+ try:
152
+ explainer = self._init_shap_explainer()
153
+ if explainer is None:
154
+ raise RuntimeError("SHAP explainer could not be initialized.")
155
+
156
+ shap_values = explainer.shap_values(X_scaled)
157
+
158
+ # For binary classification, SHAP returns list of arrays for each class
159
+ # We take the SHAP values for the positive class (deception)
160
+ if isinstance(shap_values, list) and len(shap_values) == 2:
161
+ shap_values = shap_values[1] # Positive class (deception)
162
+ elif isinstance(shap_values, np.ndarray) and shap_values.ndim == 3:
163
+ shap_values = shap_values[:, :, 1] # Positive class for multi-output
164
+ except Exception as e:
165
+ warnings.warn(
166
+ f"SHAP computation failed: {e}. Falling back to permutation importance.",
167
+ RuntimeWarning
168
+ )
169
+ # Fallback: compute permutation importance as proxy for feature contribution
170
+ shap_values = self._compute_fallback_importance(X_scaled, feature_names)
171
+
172
+ return np.asarray(shap_values, dtype=np.float64)
173
+
174
+ def _compute_fallback_importance(
175
+ self, X_scaled: np.ndarray, feature_names: List[str]
176
+ ) -> np.ndarray:
177
+ """Compute permutation importance as fallback when SHAP fails.
178
+
179
+ Args:
180
+ X_scaled: Scaled feature matrix.
181
+ feature_names: Feature names.
182
+
183
+ Returns:
184
+ np.ndarray: Importance values shaped (n_samples, n_features).
185
+ """
186
+ # Use permutation importance to get global feature importance
187
+ # Research rationale: permutation importance measures decrease in model
188
+ # performance when a feature is randomly shuffled, providing a robust
189
+ # alternative to SHAP for feature contribution estimation.
190
+ try:
191
+ result = permutation_importance(
192
+ estimator=self.classifier.model,
193
+ X=X_scaled,
194
+ y=self.classifier.model.predict(X_scaled), # Use predicted labels
195
+ n_repeats=10,
196
+ random_state=42,
197
+ n_jobs=-1
198
+ )
199
+ # Broadcast global importance to each sample
200
+ importance = result.importances_mean
201
+ importance = np.clip(importance, 0, None) # Ensure non-negative
202
+ importance = importance / (np.sum(importance) + 1e-10) # Normalize
203
+
204
+ # Create per-sample importance by adding small noise
205
+ # This approximates sample-specific contributions
206
+ n_samples = X_scaled.shape[0]
207
+ noise = np.random.normal(0, 0.01 * importance.std(), (n_samples, len(importance)))
208
+ shap_values = importance.reshape(1, -1) + noise
209
+ return shap_values
210
+ except Exception:
211
+ # Ultimate fallback: uniform importance
212
+ n_samples, n_features = X_scaled.shape
213
+ return np.ones((n_samples, n_features)) / n_features
214
+
215
+ def get_feature_importance(
216
+ self, X: np.ndarray, feature_names: List[str], top_k: int = 10
217
+ ) -> pd.DataFrame:
218
+ """Get ranked feature importance combining model importance and SHAP.
219
+
220
+ Args:
221
+ X: Feature matrix of shape (n_samples, n_features).
222
+ feature_names: Ordered list of feature names.
223
+ top_k: Number of top features to return (default: 10).
224
+
225
+ Returns:
226
+ pd.DataFrame: Columns: ['feature', 'importance', 'shap_importance',
227
+ 'combined_score', 'rank', 'interpretation']
228
+ """
229
+ X = check_array(X, ensure_2d=True, dtype=np.float64)
230
+
231
+ # 1. Get model's built-in feature importance
232
+ try:
233
+ model_importance = self.classifier.get_feature_importance()
234
+ model_importance_dict = dict(
235
+ zip(model_importance['feature'], model_importance['importance'])
236
+ )
237
+ except Exception:
238
+ model_importance_dict = {}
239
+
240
+ # 2. Compute SHAP importance (mean absolute SHAP value per feature)
241
+ shap_values = self.compute_shap_values(X, feature_names)
242
+ shap_importance = np.mean(np.abs(shap_values), axis=0)
243
+
244
+ # 3. Combine scores
245
+ combined_scores = []
246
+ for i, feat in enumerate(feature_names):
247
+ model_imp = model_importance_dict.get(feat, 0.0)
248
+ shap_imp = shap_importance[i] if i < len(shap_importance) else 0.0
249
+
250
+ # Research rationale: weighted average gives more weight to SHAP
251
+ # because it captures non-linear interactions better than Gini importance.
252
+ combined = 0.3 * model_imp + 0.7 * shap_imp
253
+
254
+ combined_scores.append({
255
+ 'feature': feat,
256
+ 'importance': float(model_imp),
257
+ 'shap_importance': float(shap_imp),
258
+ 'combined_score': float(combined),
259
+ 'interpretation': self.FEATURE_INTERPRETATIONS.get(feat, 'No interpretation available.')
260
+ })
261
+
262
+ df = pd.DataFrame(combined_scores)
263
+ df = df.sort_values('combined_score', ascending=False).reset_index(drop=True)
264
+ df['rank'] = range(1, len(df) + 1)
265
+
266
+ # Limit to top_k
267
+ if top_k > 0:
268
+ df = df.head(top_k)
269
+
270
+ return df
271
+
272
+ def get_ranked_indicators(
273
+ self, X: np.ndarray, feature_names: List[str], top_k: int = 5
274
+ ) -> List[Dict]:
275
+ """Get ranked deception indicators for a single sample.
276
+
277
+ Args:
278
+ X: Feature matrix of shape (1, n_features) for a single sample.
279
+ feature_names: Ordered list of feature names.
280
+ top_k: Number of top indicators to return (default: 5).
281
+
282
+ Returns:
283
+ List[Dict]: Each dict contains:
284
+ - 'feature': feature name
285
+ - 'value': original feature value
286
+ - 'shap_value': SHAP contribution value
287
+ - 'abs_contribution': absolute SHAP magnitude
288
+ - 'direction': 'toward_deception' or 'toward_truth'
289
+ - 'interpretation': behavioral interpretation
290
+ """
291
+ if X.ndim != 2:
292
+ raise ValueError("X must be 2D. Use shape (1, n_features) for single sample.")
293
+
294
+ if X.shape[0] != 1:
295
+ warnings.warn(
296
+ "Multiple samples provided. Using first sample for indicator ranking.",
297
+ UserWarning
298
+ )
299
+ X = X[:1]
300
+
301
+ # Compute SHAP values for the sample
302
+ shap_values = self.compute_shap_values(X, feature_names)
303
+ shap_sample = shap_values[0]
304
+
305
+ # Get original feature values
306
+ feature_values = X[0]
307
+
308
+ # Create indicator dictionaries
309
+ indicators = []
310
+ for i, feat in enumerate(feature_names):
311
+ shap_val = shap_sample[i]
312
+ feat_val = feature_values[i]
313
+
314
+ # Determine direction based on SHAP sign
315
+ # Positive SHAP pushes prediction toward deception (class 1)
316
+ direction = 'toward_deception' if shap_val > 0 else 'toward_truth'
317
+
318
+ indicators.append({
319
+ 'feature': feat,
320
+ 'value': float(feat_val),
321
+ 'shap_value': float(shap_val),
322
+ 'abs_contribution': float(np.abs(shap_val)),
323
+ 'direction': direction,
324
+ 'interpretation': self.FEATURE_INTERPRETATIONS.get(feat, 'No interpretation available.')
325
+ })
326
+
327
+ # Sort by absolute contribution (most influential first)
328
+ indicators.sort(key=lambda x: x['abs_contribution'], reverse=True)
329
+
330
+ # Return top_k indicators
331
+ return indicators[:top_k]
332
+
333
+ def explain_prediction(
334
+ self,
335
+ X: np.ndarray,
336
+ feature_names: List[str],
337
+ prediction: float,
338
+ subject_id: Optional[str] = None
339
+ ) -> Dict:
340
+ """Generate complete explanation for a single prediction.
341
+
342
+ Args:
343
+ X: Feature matrix of shape (1, n_features) for a single sample.
344
+ feature_names: Ordered list of feature names.
345
+ prediction: Deception probability (0-1).
346
+ subject_id: Optional subject identifier for contextualization.
347
+
348
+ Returns:
349
+ Dict: Explanation dictionary with keys:
350
+ - 'verdict': 'truthful' or 'deceptive' based on threshold 0.5
351
+ - 'confidence': absolute distance from decision boundary
352
+ - 'deception_probability': prediction probability
353
+ - 'top_indicators': list of top indicators (from get_ranked_indicators)
354
+ - 'group_contributions': dict of SHAP contributions per feature group
355
+ - 'behavioral_summary': human-readable summary text
356
+ - 'feature_values': dict of original feature values
357
+ - 'subject_id': subject identifier if provided
358
+ """
359
+ if X.ndim != 2 or X.shape[0] != 1:
360
+ raise ValueError("X must be a single sample with shape (1, n_features).")
361
+
362
+ # Compute SHAP values
363
+ shap_values = self.compute_shap_values(X, feature_names)
364
+
365
+ # Determine verdict
366
+ verdict = 'deceptive' if prediction >= 0.5 else 'truthful'
367
+ confidence = abs(prediction - 0.5) * 2 # Scale to 0-1
368
+
369
+ # Get top indicators
370
+ top_indicators = self.get_ranked_indicators(X, feature_names, top_k=5)
371
+
372
+ # Compute group contributions
373
+ group_contributions = self.explain_group_contributions(shap_values, feature_names)
374
+
375
+ # Generate behavioral summary
376
+ behavioral_summary = self.generate_behavioral_summary(
377
+ top_indicators, prediction, group_contributions
378
+ )
379
+
380
+ # Prepare feature values dictionary
381
+ feature_values = {
382
+ feat: float(val) for feat, val in zip(feature_names, X[0])
383
+ }
384
+
385
+ return {
386
+ 'verdict': verdict,
387
+ 'confidence': float(confidence),
388
+ 'deception_probability': float(prediction),
389
+ 'top_indicators': top_indicators,
390
+ 'group_contributions': group_contributions,
391
+ 'behavioral_summary': behavioral_summary,
392
+ 'feature_values': feature_values,
393
+ 'subject_id': subject_id
394
+ }
395
+
396
+ def generate_behavioral_summary(
397
+ self,
398
+ top_indicators: List[Dict],
399
+ prediction: float,
400
+ group_contributions: Dict[str, float]
401
+ ) -> str:
402
+ """Generate human-readable behavioral explanation paragraph.
403
+
404
+ Args:
405
+ top_indicators: Top indicators from get_ranked_indicators.
406
+ prediction: Deception probability (0-1).
407
+ group_contributions: Normalized SHAP contributions per feature group.
408
+
409
+ Returns:
410
+ str: Human-readable summary paragraph.
411
+
412
+ Example:
413
+ "The subject shows elevated deception probability (0.82). Key indicators include
414
+ increased pause duration (SHAP: +0.12), which suggests cognitive load during
415
+ response fabrication, and reduced audio-visual synchrony (SHAP: +0.09),
416
+ indicating incongruent verbal and non-verbal behavior."
417
+ """
418
+ # Determine deception level
419
+ if prediction >= 0.7:
420
+ level = "strongly elevated"
421
+ elif prediction >= 0.6:
422
+ level = "elevated"
423
+ elif prediction >= 0.5:
424
+ level = "slightly elevated"
425
+ elif prediction >= 0.4:
426
+ level = "slightly reduced"
427
+ elif prediction >= 0.3:
428
+ level = "reduced"
429
+ else:
430
+ level = "strongly reduced"
431
+
432
+ # Build summary opening
433
+ summary_parts = [
434
+ f"The subject shows {level} deception probability ({prediction:.2f})."
435
+ ]
436
+
437
+ # Add top indicators
438
+ if top_indicators:
439
+ indicator_desc = []
440
+ for idx, ind in enumerate(top_indicators[:3]): # top 3
441
+ feat = ind['feature']
442
+ shap_val = ind['shap_value']
443
+ direction = ind['direction']
444
+ interp = self.FEATURE_INTERPRETATIONS.get(feat, '')
445
+ # Extract the main behavioral insight (first sentence)
446
+ if interp:
447
+ insight = interp.split('.')[0] + '.'
448
+ else:
449
+ insight = "This feature contributes to the prediction."
450
+
451
+ # Format SHAP value with sign
452
+ sign = '+' if shap_val >= 0 else ''
453
+ indicator_desc.append(
454
+ f"{feat.replace('_', ' ')} (SHAP: {sign}{shap_val:.2f}), "
455
+ f"which {insight.lower()}"
456
+ )
457
+
458
+ if indicator_desc:
459
+ summary_parts.append("Key indicators include " + ", ".join(indicator_desc) + ".")
460
+
461
+ # Add group contributions
462
+ top_group = max(group_contributions.items(), key=lambda x: x[1]) if group_contributions else None
463
+ if top_group:
464
+ group_name, contrib = top_group
465
+ summary_parts.append(
466
+ f"The {group_name} feature group contributed the most ({contrib:.1%}) to the prediction, "
467
+ f"suggesting that {group_name.lower()} played a dominant role."
468
+ )
469
+
470
+ # Join into a single paragraph
471
+ return " ".join(summary_parts)
472
+
473
+ def explain_group_contributions(
474
+ self, shap_values: np.ndarray, feature_names: List[str]
475
+ ) -> Dict[str, float]:
476
+ """Aggregate SHAP contribution per feature group, normalized to sum to 1.0.
477
+
478
+ Args:
479
+ shap_values: SHAP values array of shape (n_samples, n_features).
480
+ feature_names: Ordered list of feature names.
481
+
482
+ Returns:
483
+ Dict[str, float]: Mapping from group name to normalized contribution.
484
+ """
485
+ # Ensure single sample (take first if multiple)
486
+ if shap_values.ndim == 2 and shap_values.shape[0] > 1:
487
+ shap_values = shap_values[0:1]
488
+ shap_abs = np.abs(shap_values).mean(axis=0) # average across samples (usually 1)
489
+
490
+ # Map feature to group
491
+ feature_to_group = {}
492
+ for group, features in self.FEATURE_GROUPS.items():
493
+ for feat in features:
494
+ feature_to_group[feat] = group
495
+
496
+ # Aggregate contributions
497
+ group_sums = {}
498
+ for i, feat in enumerate(feature_names):
499
+ if i >= len(shap_abs):
500
+ continue
501
+ group = feature_to_group.get(feat)
502
+ if group is None:
503
+ # Assign to 'Other' group
504
+ group = 'Other'
505
+ group_sums[group] = group_sums.get(group, 0.0) + shap_abs[i]
506
+
507
+ # Normalize to sum to 1.0
508
+ total = sum(group_sums.values())
509
+ if total > 0:
510
+ group_contributions = {k: v / total for k, v in group_sums.items()}
511
+ else:
512
+ group_contributions = {k: 0.0 for k in group_sums.keys()}
513
+ return group_contributions
src/explainability/visualizer.py ADDED
@@ -0,0 +1,743 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Publication-quality explainability visualizations for deception detection."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import warnings
6
+ from typing import Dict, List, Optional, Tuple, Any
7
+
8
+ import matplotlib.pyplot as plt
9
+ import numpy as np
10
+ import pandas as pd
11
+ import seaborn as sns
12
+ from matplotlib import gridspec
13
+ from matplotlib.cm import ScalarMappable
14
+ from matplotlib.colors import Normalize
15
+
16
+
17
+ class ExplainabilityVisualizer:
18
+ """Publication-quality explainability visualizations."""
19
+
20
+ def __init__(self):
21
+ """Set up matplotlib style for publication plots.
22
+
23
+ Configures seaborn style, font sizes, and color palette.
24
+ """
25
+ # Use seaborn style with white background
26
+ sns.set_style("whitegrid")
27
+ sns.set_context("paper", font_scale=1.2)
28
+
29
+ # Set publication-quality defaults
30
+ plt.rcParams.update({
31
+ 'font.size': 10,
32
+ 'axes.titlesize': 12,
33
+ 'axes.labelsize': 10,
34
+ 'xtick.labelsize': 9,
35
+ 'ytick.labelsize': 9,
36
+ 'legend.fontsize': 9,
37
+ 'figure.titlesize': 14,
38
+ 'figure.dpi': 100,
39
+ 'savefig.dpi': 300,
40
+ 'savefig.bbox': 'tight',
41
+ 'savefig.pad_inches': 0.1,
42
+ 'font.family': 'sans-serif',
43
+ 'font.sans-serif': ['Arial', 'DejaVu Sans', 'Liberation Sans'],
44
+ })
45
+
46
+ # Define colorblind-friendly palette
47
+ self.cb_palette = sns.color_palette("colorblind")
48
+ self.importance_color = self.cb_palette[0] # blue
49
+ self.shap_color = self.cb_palette[2] # red
50
+ self.truth_color = self.cb_palette[1] # green
51
+ self.deception_color = self.cb_palette[3] # orange
52
+
53
+ # Store current figure reference
54
+ self._current_fig = None
55
+
56
+ def plot_feature_importance(
57
+ self,
58
+ importance_df: pd.DataFrame,
59
+ top_k: int = 15,
60
+ save_path: Optional[str] = None,
61
+ ) -> plt.Figure:
62
+ """Horizontal bar chart comparing model importance vs SHAP importance.
63
+
64
+ Args:
65
+ importance_df: DataFrame with columns ['feature', 'importance',
66
+ 'shap_importance', 'combined_score', 'rank', 'interpretation'].
67
+ top_k: Number of top features to display (default 15).
68
+ save_path: Optional path to save figure at 300 DPI.
69
+
70
+ Returns:
71
+ matplotlib.figure.Figure: The created figure.
72
+
73
+ Raises:
74
+ ValueError: If required columns are missing or DataFrame is empty.
75
+ """
76
+ if importance_df.empty:
77
+ raise ValueError("importance_df is empty")
78
+
79
+ required_cols = {'feature', 'importance', 'shap_importance'}
80
+ missing = required_cols - set(importance_df.columns)
81
+ if missing:
82
+ raise ValueError(f"Missing columns: {missing}")
83
+
84
+ # Sort by combined_score if available, else by importance
85
+ if 'combined_score' in importance_df.columns:
86
+ df = importance_df.sort_values('combined_score', ascending=False).head(top_k)
87
+ else:
88
+ df = importance_df.sort_values('importance', ascending=False).head(top_k)
89
+
90
+ # Ensure we have at least one feature
91
+ if df.empty:
92
+ warnings.warn("No features to plot after filtering")
93
+ fig, ax = plt.subplots(figsize=(8, 4))
94
+ ax.text(0.5, 0.5, "No feature importance data",
95
+ ha='center', va='center', transform=ax.transAxes)
96
+ ax.set_axis_off()
97
+ if save_path:
98
+ fig.savefig(save_path, dpi=300)
99
+ return fig
100
+
101
+ # Create horizontal positions
102
+ y_pos = np.arange(len(df))
103
+ bar_height = 0.35
104
+
105
+ fig, ax = plt.subplots(figsize=(10, max(6, len(df) * 0.4)))
106
+
107
+ # Plot side-by-side bars
108
+ bars1 = ax.barh(y_pos - bar_height/2, df['importance'],
109
+ height=bar_height, color=self.importance_color,
110
+ label='Model Importance', edgecolor='black', linewidth=0.5)
111
+ bars2 = ax.barh(y_pos + bar_height/2, df['shap_importance'],
112
+ height=bar_height, color=self.shap_color,
113
+ label='SHAP Importance', edgecolor='black', linewidth=0.5)
114
+
115
+ # Add feature names
116
+ ax.set_yticks(y_pos)
117
+ ax.set_yticklabels(df['feature'])
118
+
119
+ # Add value labels on bars
120
+ for bar in bars1:
121
+ width = bar.get_width()
122
+ if width > 0:
123
+ ax.text(width + 0.01, bar.get_y() + bar.get_height()/2,
124
+ f'{width:.3f}', va='center', ha='left', fontsize=8)
125
+
126
+ for bar in bars2:
127
+ width = bar.get_width()
128
+ if width > 0:
129
+ ax.text(width + 0.01, bar.get_y() + bar.get_height()/2,
130
+ f'{width:.3f}', va='center', ha='left', fontsize=8)
131
+
132
+ ax.set_xlabel('Importance Score')
133
+ ax.set_title('Feature Importance Comparison (Model vs SHAP)')
134
+ ax.legend(loc='lower right')
135
+
136
+ # Add interpretation as hover-like annotations (optional)
137
+ if 'interpretation' in df.columns:
138
+ # Add interpretation as text on the right side if space
139
+ max_x = max(df['importance'].max(), df['shap_importance'].max()) * 1.3
140
+ for i, (_, row) in enumerate(df.iterrows()):
141
+ if isinstance(row['interpretation'], str) and row['interpretation']:
142
+ ax.text(max_x, i, f" {row['interpretation']}",
143
+ va='center', fontsize=7, alpha=0.7,
144
+ bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.5))
145
+
146
+ ax.invert_yaxis() # Highest importance at top
147
+ plt.tight_layout()
148
+
149
+ if save_path:
150
+ fig.savefig(save_path, dpi=300)
151
+
152
+ self._current_fig = fig
153
+ return fig
154
+
155
+ def plot_shap_summary(
156
+ self,
157
+ shap_values: np.ndarray,
158
+ feature_names: List[str],
159
+ X: np.ndarray,
160
+ top_k: int = 15,
161
+ save_path: Optional[str] = None,
162
+ ) -> plt.Figure:
163
+ """SHAP beeswarm-style summary plot.
164
+
165
+ Each dot represents one sample, colored by feature value,
166
+ x-position is SHAP value.
167
+
168
+ Args:
169
+ shap_values: 2D array of SHAP values (n_samples, n_features).
170
+ feature_names: List of feature names.
171
+ X: 2D array of feature values (n_samples, n_features).
172
+ top_k: Number of top features to display (default 15).
173
+ save_path: Optional path to save figure at 300 DPI.
174
+
175
+ Returns:
176
+ matplotlib.figure.Figure: The created figure.
177
+
178
+ Raises:
179
+ ValueError: If dimensions mismatch.
180
+ """
181
+ if shap_values.shape[1] != len(feature_names):
182
+ raise ValueError("shap_values columns must match feature_names length")
183
+ if X.shape[1] != len(feature_names):
184
+ raise ValueError("X columns must match feature_names length")
185
+
186
+ n_samples, n_features = shap_values.shape
187
+ if n_samples == 0 or n_features == 0:
188
+ warnings.warn("No data to plot")
189
+ fig, ax = plt.subplots(figsize=(8, 4))
190
+ ax.text(0.5, 0.5, "No SHAP data",
191
+ ha='center', va='center', transform=ax.transAxes)
192
+ ax.set_axis_off()
193
+ if save_path:
194
+ fig.savefig(save_path, dpi=300)
195
+ return fig
196
+
197
+ # Compute mean absolute SHAP to rank features
198
+ mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
199
+ top_indices = np.argsort(mean_abs_shap)[-top_k:][::-1]
200
+
201
+ # Prepare data for plotting
202
+ top_shap = shap_values[:, top_indices]
203
+ top_X = X[:, top_indices]
204
+ top_names = [feature_names[i] for i in top_indices]
205
+
206
+ # Create figure
207
+ fig, ax = plt.subplots(figsize=(10, max(6, top_k * 0.5)))
208
+
209
+ # Create colormap for feature values
210
+ cmap = plt.cm.coolwarm
211
+ norm = Normalize(vmin=np.nanmin(top_X), vmax=np.nanmax(top_X))
212
+
213
+ # Plot each feature
214
+ y_positions = np.arange(len(top_names))
215
+
216
+ for i, (name, shap_vals, x_vals) in enumerate(zip(top_names, top_shap.T, top_X.T)):
217
+ y = y_positions[i]
218
+
219
+ # Add jitter for visibility
220
+ jitter = np.random.normal(0, 0.05, size=len(shap_vals))
221
+
222
+ # Scatter plot
223
+ scatter = ax.scatter(
224
+ shap_vals,
225
+ y + jitter,
226
+ c=x_vals,
227
+ cmap=cmap,
228
+ norm=norm,
229
+ s=20,
230
+ alpha=0.6,
231
+ edgecolors='black',
232
+ linewidths=0.3,
233
+ )
234
+
235
+ # Add colorbar
236
+ sm = ScalarMappable(cmap=cmap, norm=norm)
237
+ sm.set_array([])
238
+ cbar = plt.colorbar(sm, ax=ax, pad=0.01)
239
+ cbar.set_label('Feature Value', fontsize=9)
240
+
241
+ # Add vertical line at zero
242
+ ax.axvline(x=0, color='black', linestyle='--', linewidth=0.5, alpha=0.5)
243
+
244
+ # Set labels and ticks
245
+ ax.set_yticks(y_positions)
246
+ ax.set_yticklabels(top_names)
247
+ ax.set_xlabel('SHAP Value (impact on prediction)')
248
+ ax.set_ylabel('Feature')
249
+ ax.set_title(f'SHAP Summary Plot (Top {top_k} Features)')
250
+
251
+ # Add mean absolute SHAP annotation
252
+ for i, idx in enumerate(top_indices):
253
+ ax.text(0.02, y_positions[i], f'mean|SHAP|={mean_abs_shap[idx]:.3f}',
254
+ transform=ax.get_yaxis_transform(),
255
+ fontsize=7, va='center', bbox=dict(boxstyle='round,pad=0.2',
256
+ facecolor='white', alpha=0.7))
257
+
258
+ plt.tight_layout()
259
+
260
+ if save_path:
261
+ fig.savefig(save_path, dpi=300)
262
+
263
+ self._current_fig = fig
264
+ return fig
265
+
266
+ def plot_shap_waterfall(
267
+ self,
268
+ shap_values: np.ndarray,
269
+ feature_names: List[str],
270
+ base_value: float,
271
+ top_k: int = 10,
272
+ save_path: Optional[str] = None,
273
+ ) -> plt.Figure:
274
+ """Waterfall plot for a single prediction's SHAP explanation.
275
+
276
+ Red bars = features pushing toward deception (positive SHAP)
277
+ Blue bars = features pushing toward truth (negative SHAP)
278
+
279
+ Args:
280
+ shap_values: 1D array of SHAP values for a single sample.
281
+ feature_names: List of feature names.
282
+ base_value: Base (expected) value from SHAP explainer.
283
+ top_k: Number of top features to display (default 10).
284
+ save_path: Optional path to save figure at 300 DPI.
285
+
286
+ Returns:
287
+ matplotlib.figure.Figure: The created figure.
288
+ """
289
+ if shap_values.ndim != 1:
290
+ if shap_values.ndim == 2 and shap_values.shape[0] == 1:
291
+ shap_values = shap_values[0]
292
+ else:
293
+ raise ValueError("shap_values must be 1D or 2D with single sample")
294
+
295
+ if len(shap_values) != len(feature_names):
296
+ raise ValueError("shap_values length must match feature_names")
297
+
298
+ # Select top K features by absolute SHAP
299
+ abs_shap = np.abs(shap_values)
300
+ top_indices = np.argsort(abs_shap)[-top_k:][::-1]
301
+
302
+ # Sort by SHAP value (deception to truth)
303
+ sorted_indices = top_indices[np.argsort(shap_values[top_indices])[::-1]]
304
+
305
+ top_shap = shap_values[sorted_indices]
306
+ top_names = [feature_names[i] for i in sorted_indices]
307
+
308
+ # Compute cumulative values for waterfall
309
+ cumulative = base_value + np.cumsum(top_shap)
310
+ cumulative = np.insert(cumulative, 0, base_value)
311
+
312
+ # Create figure
313
+ fig, ax = plt.subplots(figsize=(10, max(6, top_k * 0.6)))
314
+
315
+ # Plot bars
316
+ for i, (shap_val, name) in enumerate(zip(top_shap, top_names)):
317
+ color = self.deception_color if shap_val > 0 else self.truth_color
318
+ ax.barh(i, shap_val, left=cumulative[i], color=color,
319
+ edgecolor='black', linewidth=0.5, height=0.8)
320
+
321
+ # Add connecting lines
322
+ for i in range(len(cumulative) - 1):
323
+ ax.plot([cumulative[i], cumulative[i+1]], [i, i],
324
+ color='gray', linewidth=0.5, linestyle='--', alpha=0.5)
325
+
326
+ # Add base value and final prediction
327
+ final_value = cumulative[-1]
328
+ ax.axvline(x=base_value, color='black', linestyle=':', linewidth=1,
329
+ label=f'Base Value: {base_value:.3f}')
330
+ ax.axvline(x=final_value, color='red', linestyle=':', linewidth=1,
331
+ label=f'Final Prediction: {final_value:.3f}')
332
+
333
+ # Add feature labels
334
+ ax.set_yticks(np.arange(len(top_names)))
335
+ ax.set_yticklabels(top_names)
336
+
337
+ # Add value annotations
338
+ for i, (shap_val, cum_val) in enumerate(zip(top_shap, cumulative[1:])):
339
+ ax.text(cum_val, i, f'{shap_val:+.3f}',
340
+ va='center', ha='left' if shap_val > 0 else 'right',
341
+ fontsize=8, fontweight='bold',
342
+ bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.8))
343
+
344
+ ax.set_xlabel('Model Output (Deception Probability)')
345
+ ax.set_title('SHAP Waterfall Explanation')
346
+ ax.legend(loc='upper right')
347
+
348
+ # Add interpretation
349
+ ax.text(0.02, 0.98, f'Base: {base_value:.3f} → Final: {final_value:.3f}',
350
+ transform=ax.transAxes, fontsize=9, va='top',
351
+ bbox=dict(boxstyle='round,pad=0.3', facecolor='white', alpha=0.9))
352
+
353
+ plt.tight_layout()
354
+
355
+ if save_path:
356
+ fig.savefig(save_path, dpi=300)
357
+
358
+ self._current_fig = fig
359
+ return fig
360
+
361
+ def plot_group_contributions(
362
+ self,
363
+ group_contributions: Dict[str, float],
364
+ save_path: Optional[str] = None,
365
+ ) -> plt.Figure:
366
+ """Donut chart of feature group contributions.
367
+
368
+ Args:
369
+ group_contributions: Maps group name to normalized contribution (sums to 1.0).
370
+ save_path: Optional path to save figure at 300 DPI.
371
+
372
+ Returns:
373
+ matplotlib.figure.Figure: The created figure.
374
+ """
375
+ if not group_contributions:
376
+ warnings.warn("Empty group_contributions dictionary")
377
+ fig, ax = plt.subplots(figsize=(6, 6))
378
+ ax.text(0.5, 0.5, "No group contribution data",
379
+ ha='center', va='center', transform=ax.transAxes)
380
+ ax.set_axis_off()
381
+ if save_path:
382
+ fig.savefig(save_path, dpi=300)
383
+ return fig
384
+
385
+ # Normalize contributions to sum to 1.0
386
+ total = sum(group_contributions.values())
387
+ if total == 0:
388
+ # Equal distribution if all zero
389
+ groups = list(group_contributions.keys())
390
+ contributions = [1.0 / len(groups)] * len(groups)
391
+ else:
392
+ groups = list(group_contributions.keys())
393
+ contributions = [group_contributions[g] / total for g in groups]
394
+
395
+ # Sort by contribution
396
+ sorted_indices = np.argsort(contributions)[::-1]
397
+ groups = [groups[i] for i in sorted_indices]
398
+ contributions = [contributions[i] for i in sorted_indices]
399
+
400
+ # Create donut chart
401
+ fig, ax = plt.subplots(figsize=(8, 8))
402
+
403
+ # Use colorblind-friendly palette
404
+ colors = sns.color_palette("colorblind", len(groups))
405
+
406
+ # Outer pie
407
+ wedges, texts, autotexts = ax.pie(
408
+ contributions,
409
+ labels=groups,
410
+ colors=colors,
411
+ autopct='%1.1f%%',
412
+ startangle=90,
413
+ pctdistance=0.85,
414
+ textprops={'fontsize': 9},
415
+ )
416
+
417
+ # Draw circle for donut effect
418
+ centre_circle = plt.Circle((0, 0), 0.70, fc='white')
419
+ ax.add_artist(centre_circle)
420
+
421
+ # Equal aspect ratio
422
+ ax.axis('equal')
423
+
424
+ # Add title
425
+ ax.set_title('Feature Group Contributions', fontsize=12)
426
+
427
+ plt.tight_layout()
428
+
429
+ if save_path:
430
+ fig.savefig(save_path, dpi=300)
431
+
432
+ self._current_fig = fig
433
+ return fig
434
+
435
+ def plot_shap_summary(
436
+ self,
437
+ shap_values: np.ndarray,
438
+ feature_names: List[str],
439
+ plot_type: str = 'bar',
440
+ top_k: int = 15,
441
+ save_path: Optional[str] = None,
442
+ ) -> plt.Figure:
443
+ """SHAP summary plot (bar or beeswarm).
444
+
445
+ Args:
446
+ shap_values: 2D array of SHAP values (n_samples, n_features).
447
+ feature_names: List of feature names.
448
+ plot_type: 'bar' for bar chart of mean absolute SHAP,
449
+ 'beeswarm' for beeswarm plot (requires X).
450
+ top_k: Number of top features to display (default 15).
451
+ save_path: Optional path to save figure at 300 DPI.
452
+
453
+ Returns:
454
+ matplotlib.figure.Figure: The created figure.
455
+
456
+ Raises:
457
+ ValueError: If dimensions mismatch or unsupported plot_type.
458
+ """
459
+ if shap_values.shape[1] != len(feature_names):
460
+ raise ValueError("shap_values columns must match feature_names length")
461
+
462
+ n_samples, n_features = shap_values.shape
463
+ if n_samples == 0 or n_features == 0:
464
+ warnings.warn("No data to plot")
465
+ fig, ax = plt.subplots(figsize=(8, 4))
466
+ ax.text(0.5, 0.5, "No SHAP data",
467
+ ha='center', va='center', transform=ax.transAxes)
468
+ ax.set_axis_off()
469
+ if save_path:
470
+ fig.savefig(save_path, dpi=300)
471
+ return fig
472
+
473
+ # Compute mean absolute SHAP to rank features
474
+ mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
475
+ top_indices = np.argsort(mean_abs_shap)[-top_k:][::-1]
476
+ top_means = mean_abs_shap[top_indices]
477
+ top_names = [feature_names[i] for i in top_indices]
478
+
479
+ if plot_type == 'bar':
480
+ fig, ax = plt.subplots(figsize=(10, max(6, top_k * 0.5)))
481
+ y_pos = np.arange(len(top_names))
482
+ ax.barh(y_pos, top_means, color=self.shap_color, edgecolor='black', linewidth=0.5)
483
+ ax.set_yticks(y_pos)
484
+ ax.set_yticklabels(top_names)
485
+ ax.set_xlabel('Mean |SHAP|')
486
+ ax.set_title(f'SHAP Feature Importance (Top {top_k})')
487
+ ax.invert_yaxis()
488
+ plt.tight_layout()
489
+ elif plot_type == 'beeswarm':
490
+ # Requires X (feature values) which we don't have; fallback to bar
491
+ warnings.warn("Beeswarm plot requires X matrix; falling back to bar plot.")
492
+ return self.plot_shap_summary(shap_values, feature_names, plot_type='bar', top_k=top_k, save_path=save_path)
493
+ else:
494
+ raise ValueError(f"Unsupported plot_type: {plot_type}")
495
+
496
+ if save_path:
497
+ fig.savefig(save_path, dpi=300)
498
+
499
+ self._current_fig = fig
500
+ return fig
501
+
502
+ def plot_feature_groups(
503
+ self,
504
+ shap_values: np.ndarray,
505
+ feature_names: List[str],
506
+ feature_groups: Dict[str, List[str]],
507
+ save_path: Optional[str] = None,
508
+ ) -> plt.Figure:
509
+ """Grouped feature contribution plot.
510
+
511
+ Args:
512
+ shap_values: 2D array of SHAP values (n_samples, n_features).
513
+ feature_names: List of feature names.
514
+ feature_groups: Mapping from group name to list of feature names.
515
+ save_path: Optional path to save figure at 300 DPI.
516
+
517
+ Returns:
518
+ matplotlib.figure.Figure: The created figure.
519
+ """
520
+ # Compute mean absolute SHAP per feature
521
+ mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
522
+ # Aggregate per group
523
+ group_sums = {}
524
+ for group, features in feature_groups.items():
525
+ total = 0.0
526
+ for feat in features:
527
+ if feat in feature_names:
528
+ idx = feature_names.index(feat)
529
+ total += mean_abs_shap[idx]
530
+ group_sums[group] = total
531
+
532
+ # Normalize to percentages
533
+ total = sum(group_sums.values())
534
+ if total == 0:
535
+ group_percents = {k: 0.0 for k in group_sums.keys()}
536
+ else:
537
+ group_percents = {k: v / total * 100 for k, v in group_sums.items()}
538
+
539
+ # Sort groups by contribution
540
+ sorted_groups = sorted(group_percents.items(), key=lambda x: x[1], reverse=True)
541
+ groups = [g for g, _ in sorted_groups]
542
+ percents = [p for _, p in sorted_groups]
543
+
544
+ fig, ax = plt.subplots(figsize=(10, max(6, len(groups) * 0.5)))
545
+ y_pos = np.arange(len(groups))
546
+ colors = sns.color_palette("colorblind", len(groups))
547
+ ax.barh(y_pos, percents, color=colors, edgecolor='black', linewidth=0.5)
548
+ ax.set_yticks(y_pos)
549
+ ax.set_yticklabels(groups)
550
+ ax.set_xlabel('Contribution (%)')
551
+ ax.set_title('Feature Group Contributions (SHAP)')
552
+ # Add percentage labels
553
+ for i, p in enumerate(percents):
554
+ ax.text(p + 0.5, i, f'{p:.1f}%', va='center', fontsize=9)
555
+
556
+ ax.invert_yaxis()
557
+ plt.tight_layout()
558
+
559
+ if save_path:
560
+ fig.savefig(save_path, dpi=300)
561
+
562
+ self._current_fig = fig
563
+ return fig
564
+
565
+ def plot_behavioral_interpretation(
566
+ self,
567
+ behavioral_explanations: List[Dict[str, Any]],
568
+ save_path: Optional[str] = None,
569
+ ) -> plt.Figure:
570
+ """Human-readable behavioral interpretation visualization.
571
+
572
+ Args:
573
+ behavioral_explanations: List of dicts with keys 'feature', 'value',
574
+ 'shap_value', 'direction', 'interpretation'.
575
+ save_path: Optional path to save figure at 300 DPI.
576
+
577
+ Returns:
578
+ matplotlib.figure.Figure: The created figure.
579
+ """
580
+ if not behavioral_explanations:
581
+ warnings.warn("Empty behavioral_explanations list")
582
+ fig, ax = plt.subplots(figsize=(8, 4))
583
+ ax.text(0.5, 0.5, "No behavioral explanation data",
584
+ ha='center', va='center', transform=ax.transAxes)
585
+ ax.set_axis_off()
586
+ if save_path:
587
+ fig.savefig(save_path, dpi=300)
588
+ return fig
589
+
590
+ # Limit to top 10 for readability
591
+ top_explanations = behavioral_explanations[:10]
592
+ n = len(top_explanations)
593
+
594
+ fig, ax = plt.subplots(figsize=(12, max(6, n * 0.8)))
595
+ ax.axis('off')
596
+
597
+ # Create a table-like visualization
598
+ y_pos = np.arange(n) * 1.2
599
+ for i, exp in enumerate(top_explanations):
600
+ feature = exp.get('feature', 'Unknown')
601
+ value = exp.get('value', 0.0)
602
+ shap_val = exp.get('shap_value', 0.0)
603
+ direction = exp.get('direction', 'neutral')
604
+ interpretation = exp.get('interpretation', '')
605
+
606
+ # Determine color based on direction
607
+ if direction == 'toward_deception':
608
+ color = self.deception_color
609
+ dir_text = '→ Deception'
610
+ elif direction == 'toward_truth':
611
+ color = self.truth_color
612
+ dir_text = '→ Truth'
613
+ else:
614
+ color = 'gray'
615
+ dir_text = 'Neutral'
616
+
617
+ # Feature name and value
618
+ ax.text(0.1, y_pos[i], f'{feature}', fontsize=10, fontweight='bold',
619
+ verticalalignment='center')
620
+ ax.text(0.3, y_pos[i], f'{value:.3f}', fontsize=9,
621
+ verticalalignment='center')
622
+ # SHAP value with sign
623
+ shap_sign = '+' if shap_val >= 0 else ''
624
+ ax.text(0.45, y_pos[i], f'{shap_sign}{shap_val:.3f}', fontsize=9,
625
+ color=color, verticalalignment='center')
626
+ # Direction arrow
627
+ ax.text(0.6, y_pos[i], dir_text, fontsize=9, color=color,
628
+ verticalalignment='center')
629
+ # Interpretation (wrap)
630
+ ax.text(0.75, y_pos[i], interpretation, fontsize=8, wrap=True,
631
+ verticalalignment='center', style='italic')
632
+
633
+ # Add column headers
634
+ ax.text(0.1, y_pos[-1] + 1.5, 'Feature', fontsize=10, fontweight='bold')
635
+ ax.text(0.3, y_pos[-1] + 1.5, 'Value', fontsize=10, fontweight='bold')
636
+ ax.text(0.45, y_pos[-1] + 1.5, 'SHAP', fontsize=10, fontweight='bold')
637
+ ax.text(0.6, y_pos[-1] + 1.5, 'Direction', fontsize=10, fontweight='bold')
638
+ ax.text(0.75, y_pos[-1] + 1.5, 'Interpretation', fontsize=10, fontweight='bold')
639
+
640
+ ax.set_xlim(0, 1)
641
+ ax.set_ylim(-1, y_pos[-1] + 3)
642
+ ax.set_title('Behavioral Interpretation', fontsize=14, pad=20)
643
+
644
+ plt.tight_layout()
645
+
646
+ if save_path:
647
+ fig.savefig(save_path, dpi=300)
648
+
649
+ self._current_fig = fig
650
+ return fig
651
+
652
+ def plot_explanation_dashboard(
653
+ self,
654
+ shap_values: np.ndarray,
655
+ feature_names: List[str],
656
+ feature_groups: Dict[str, List[str]],
657
+ behavioral_explanations: List[Dict[str, Any]],
658
+ save_path: Optional[str] = None,
659
+ ) -> plt.Figure:
660
+ """Combined dashboard with all plots.
661
+
662
+ Args:
663
+ shap_values: 2D array of SHAP values (n_samples, n_features).
664
+ feature_names: List of feature names.
665
+ feature_groups: Mapping from group name to list of feature names.
666
+ behavioral_explanations: List of behavioral explanation dicts.
667
+ save_path: Optional path to save figure at 300 DPI.
668
+
669
+ Returns:
670
+ matplotlib.figure.Figure: The created figure.
671
+ """
672
+ fig = plt.figure(figsize=(16, 12))
673
+ gs = gridspec.GridSpec(2, 2, figure=fig, hspace=0.3, wspace=0.3)
674
+
675
+ # 1. SHAP summary bar plot
676
+ ax1 = fig.add_subplot(gs[0, 0])
677
+ mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
678
+ top_k = 10
679
+ top_indices = np.argsort(mean_abs_shap)[-top_k:][::-1]
680
+ top_means = mean_abs_shap[top_indices]
681
+ top_names = [feature_names[i] for i in top_indices]
682
+ y_pos = np.arange(len(top_names))
683
+ ax1.barh(y_pos, top_means, color=self.shap_color, edgecolor='black', linewidth=0.5)
684
+ ax1.set_yticks(y_pos)
685
+ ax1.set_yticklabels(top_names)
686
+ ax1.set_xlabel('Mean |SHAP|')
687
+ ax1.set_title('SHAP Feature Importance (Top 10)')
688
+ ax1.invert_yaxis()
689
+
690
+ # 2. Feature group contributions
691
+ ax2 = fig.add_subplot(gs[0, 1])
692
+ group_sums = {}
693
+ for group, features in feature_groups.items():
694
+ total = 0.0
695
+ for feat in features:
696
+ if feat in feature_names:
697
+ idx = feature_names.index(feat)
698
+ total += mean_abs_shap[idx]
699
+ group_sums[group] = total
700
+ total = sum(group_sums.values())
701
+ if total == 0:
702
+ group_percents = {k: 0.0 for k in group_sums.keys()}
703
+ else:
704
+ group_percents = {k: v / total * 100 for k, v in group_sums.items()}
705
+ sorted_groups = sorted(group_percents.items(), key=lambda x: x[1], reverse=True)
706
+ groups = [g for g, _ in sorted_groups]
707
+ percents = [p for _, p in sorted_groups]
708
+ colors = sns.color_palette("colorblind", len(groups))
709
+ ax2.barh(np.arange(len(groups)), percents, color=colors, edgecolor='black', linewidth=0.5)
710
+ ax2.set_yticks(np.arange(len(groups)))
711
+ ax2.set_yticklabels(groups)
712
+ ax2.set_xlabel('Contribution (%)')
713
+ ax2.set_title('Feature Group Contributions')
714
+ ax2.invert_yaxis()
715
+
716
+ # 3. Behavioral interpretation table
717
+ ax3 = fig.add_subplot(gs[1, :])
718
+ ax3.axis('off')
719
+ if behavioral_explanations:
720
+ top_explanations = behavioral_explanations[:5]
721
+ y_pos = np.arange(len(top_explanations)) * 1.5
722
+ for i, exp in enumerate(top_explanations):
723
+ feature = exp.get('feature', 'Unknown')
724
+ shap_val = exp.get('shap_value', 0.0)
725
+ direction = exp.get('direction', 'neutral')
726
+ interpretation = exp.get('interpretation', '')
727
+ color = self.deception_color if direction == 'toward_deception' else self.truth_color
728
+ ax3.text(0.1, y_pos[i], f'{feature}', fontsize=10, fontweight='bold')
729
+ ax3.text(0.3, y_pos[i], f'{shap_val:+.3f}', fontsize=9, color=color)
730
+ ax3.text(0.5, y_pos[i], direction, fontsize=9, color=color)
731
+ ax3.text(0.7, y_pos[i], interpretation, fontsize=8, wrap=True, style='italic')
732
+ ax3.set_title('Top Behavioral Indicators', fontsize=12, pad=10)
733
+ else:
734
+ ax3.text(0.5, 0.5, 'No behavioral explanations', ha='center', va='center', fontsize=12)
735
+
736
+ fig.suptitle('Explainability Dashboard', fontsize=16, y=0.98)
737
+ plt.tight_layout(rect=[0, 0, 1, 0.96])
738
+
739
+ if save_path:
740
+ fig.savefig(save_path, dpi=300)
741
+
742
+ self._current_fig = fig
743
+ return fig
src/facial/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """Facial analysis module for micro-expression detection."""
2
+
3
+ from .extractor import FacialFeatureExtractor
4
+ from .visualizer import FacialVisualizer
5
+
6
+ __all__ = ['FacialFeatureExtractor', 'FacialVisualizer']
src/facial/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (337 Bytes). View file
 
src/facial/extractor.py ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Facial micro-expression feature extraction using MediaPipe Face Mesh."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Dict, List, Optional, Tuple
6
+
7
+ import cv2
8
+ import numpy as np
9
+ import pandas as pd
10
+ from tqdm import tqdm
11
+
12
+ from src.utils.config import FeatureConfig, get_default_config
13
+ from src.utils.io import load_video
14
+
15
+
16
+ class FacialFeatureExtractor:
17
+ """Extracts micro-expression features from video using MediaPipe Face Mesh."""
18
+
19
+ def _get_mp_solutions(self):
20
+ """Lazily load mediapipe to avoid global import errors."""
21
+ try:
22
+ import mediapipe as mp
23
+ return mp.solutions
24
+ except Exception as e:
25
+ raise ImportError(
26
+ "MediaPipe 'solutions' module not found. Original error: " + str(e)
27
+ )
28
+
29
+ _LEFT_EYE = [33, 160, 158, 133, 153, 144]
30
+ _RIGHT_EYE = [362, 385, 387, 263, 373, 380]
31
+ _LEFT_BROW = [70, 63, 105, 66, 107]
32
+ _RIGHT_BROW = [336, 296, 334, 293, 300]
33
+ _NOSE = [1, 168, 98, 327]
34
+ _HEAD_POINTS = [1, 33, 263, 61, 291, 199]
35
+ _FEATURE_NAMES = [
36
+ "eye_left_openness",
37
+ "eye_right_openness",
38
+ "mouth_openness",
39
+ "mouth_width",
40
+ "mouth_asymmetry",
41
+ "left_brow_height",
42
+ "right_brow_height",
43
+ "brow_asymmetry",
44
+ "brow_tension",
45
+ "nose_wrinkle",
46
+ "nostril_flare",
47
+ "nose_asymmetry",
48
+ "head_pitch",
49
+ "head_yaw",
50
+ "head_roll",
51
+ "head_displacement",
52
+ ]
53
+
54
+ def __init__(self, config: Optional[FeatureConfig] = None):
55
+ """Initialize the MediaPipe Face Mesh extractor.
56
+
57
+ Gracefully handles missing MediaPipe dependencies by setting an 'available' flag.
58
+ """
59
+
60
+ self.config = config or get_default_config().features
61
+ self.available = False
62
+ self.face_mesh = None
63
+
64
+ try:
65
+ solutions = self._get_mp_solutions()
66
+ self.mp_face_mesh = solutions.face_mesh
67
+ self.face_mesh = self.mp_face_mesh.FaceMesh(
68
+ static_image_mode=False,
69
+ max_num_faces=1,
70
+ refine_landmarks=True,
71
+ min_detection_confidence=0.5,
72
+ min_tracking_confidence=0.5,
73
+ )
74
+ self.available = True
75
+ except (ImportError, AttributeError) as e:
76
+ print(f"\u26a0\ufe0f WARNING: Facial features disabled. MediaPipe error: {e}")
77
+
78
+ @staticmethod
79
+ def _distance(point_a: np.ndarray, point_b: np.ndarray) -> float:
80
+ """Return Euclidean distance between two landmark points."""
81
+ return float(np.linalg.norm(point_a[:3] - point_b[:3]))
82
+
83
+ @staticmethod
84
+ def _face_diagonal(landmarks: np.ndarray) -> float:
85
+ """Return face bounding-box diagonal for scale normalization."""
86
+ if landmarks.size == 0:
87
+ return 1.0
88
+ mins = np.nanmin(landmarks[:, :2], axis=0)
89
+ maxs = np.nanmax(landmarks[:, :2], axis=0)
90
+ diagonal = float(np.linalg.norm(maxs - mins))
91
+ return diagonal if diagonal > 1e-6 else 1.0
92
+
93
+ @staticmethod
94
+ def _safe_ratio(numerator: float, denominator: float) -> float:
95
+ """Return a bounded ratio while avoiding division by zero."""
96
+ if denominator <= 1e-8 or not np.isfinite(denominator):
97
+ return 0.0
98
+ value = numerator / denominator
99
+ return float(np.clip(value, 0.0, 1.0))
100
+
101
+ def _extract_landmarks(self, frame: np.ndarray) -> Optional[np.ndarray]:
102
+ """Run Face Mesh on a BGR frame and return normalized landmarks."""
103
+
104
+ if not self.available or self.face_mesh is None or frame is None or frame.size == 0:
105
+ return None
106
+ gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
107
+ if float(cv2.Laplacian(gray, cv2.CV_64F).var()) < 5.0:
108
+ # Extremely blurry frames can cause unstable micro-expression measurements.
109
+ return None
110
+
111
+ rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
112
+ results = self.face_mesh.process(rgb_frame)
113
+ if not results.multi_face_landmarks:
114
+ return None
115
+
116
+ height, width = frame.shape[:2]
117
+ landmarks = np.array(
118
+ [[point.x * width, point.y * height, point.z * width] for point in results.multi_face_landmarks[0].landmark],
119
+ dtype=np.float32,
120
+ )
121
+ if self._face_diagonal(landmarks) < 25.0:
122
+ # Very small faces do not provide reliable deception-relevant movements.
123
+ return None
124
+ return landmarks
125
+
126
+ def extract_eye_openness(self, landmarks: np.ndarray) -> Tuple[float, float]:
127
+ """Calculate left and right eye openness ratios.
128
+
129
+ Args:
130
+ landmarks: Face Mesh landmarks with shape ``(468, 3)``.
131
+
132
+ Returns:
133
+ Tuple containing left and right Eye Aspect Ratio values.
134
+ """
135
+
136
+ def ear(indices: List[int]) -> float:
137
+ p1, p2, p3, p4, p5, p6 = landmarks[indices]
138
+ vertical = self._distance(p2, p6) + self._distance(p3, p5)
139
+ horizontal = 2.0 * self._distance(p1, p4)
140
+ # Blink rate and eye narrowing are behavioral cues often reviewed for stress or concealment.
141
+ return self._safe_ratio(vertical, horizontal)
142
+
143
+ return ear(self._LEFT_EYE), ear(self._RIGHT_EYE)
144
+
145
+ def extract_mouth_movement(self, landmarks: np.ndarray) -> Dict[str, float]:
146
+ """Calculate scale-normalized mouth movement features.
147
+
148
+ Args:
149
+ landmarks: Face Mesh landmarks with shape ``(468, 3)``.
150
+
151
+ Returns:
152
+ Dictionary containing mouth openness, width, and asymmetry.
153
+ """
154
+
155
+ diagonal = self._face_diagonal(landmarks)
156
+ upper_lip, lower_lip = landmarks[13], landmarks[14]
157
+ left_corner, right_corner = landmarks[61], landmarks[291]
158
+ center = (upper_lip + lower_lip) / 2.0
159
+ left_distance = self._distance(center, left_corner)
160
+ right_distance = self._distance(center, right_corner)
161
+ # Mouth tension, dry-mouth behavior, and articulation changes can accompany cognitive load.
162
+ return {
163
+ "mouth_openness": self._distance(upper_lip, lower_lip) / diagonal,
164
+ "mouth_width": self._distance(left_corner, right_corner) / diagonal,
165
+ "mouth_asymmetry": abs(left_distance - right_distance) / diagonal,
166
+ }
167
+
168
+ def extract_eyebrow_tension(self, landmarks: np.ndarray) -> Dict[str, float]:
169
+ """Calculate eyebrow raise, asymmetry, and squeeze features.
170
+
171
+ Args:
172
+ landmarks: Face Mesh landmarks with shape ``(468, 3)``.
173
+
174
+ Returns:
175
+ Dictionary containing brow height, asymmetry, and tension.
176
+ """
177
+
178
+ diagonal = self._face_diagonal(landmarks)
179
+ nose_bridge = landmarks[168]
180
+ left_brow = landmarks[self._LEFT_BROW]
181
+ right_brow = landmarks[self._RIGHT_BROW]
182
+ left_height = float(np.mean(nose_bridge[1] - left_brow[:, 1]) / diagonal)
183
+ right_height = float(np.mean(nose_bridge[1] - right_brow[:, 1]) / diagonal)
184
+ inner_distance = self._distance(landmarks[107], landmarks[336]) / diagonal
185
+ # Brow squeeze and asymmetry are micro-expression proxies for stress, concern, or suppression.
186
+ return {
187
+ "left_brow_height": left_height,
188
+ "right_brow_height": right_height,
189
+ "brow_asymmetry": abs(left_height - right_height),
190
+ "brow_tension": max(0.0, 1.0 - inner_distance),
191
+ }
192
+
193
+ def extract_nose_movement(self, landmarks: np.ndarray) -> Dict[str, float]:
194
+ """Calculate nose wrinkle, nostril flare, and asymmetry features.
195
+
196
+ Args:
197
+ landmarks: Face Mesh landmarks with shape ``(468, 3)``.
198
+
199
+ Returns:
200
+ Dictionary containing nose movement measurements.
201
+ """
202
+
203
+ diagonal = self._face_diagonal(landmarks)
204
+ nose_tip, nose_bridge = landmarks[1], landmarks[168]
205
+ left_nostril, right_nostril = landmarks[98], landmarks[327]
206
+ center = (left_nostril + right_nostril) / 2.0
207
+ # Nose wrinkling or nostril flare can indicate disgust, arousal, or regulated emotion.
208
+ return {
209
+ "nose_wrinkle": abs(float(nose_tip[1] - nose_bridge[1])) / diagonal,
210
+ "nostril_flare": self._distance(left_nostril, right_nostril) / diagonal,
211
+ "nose_asymmetry": abs(self._distance(center, left_nostril) - self._distance(center, right_nostril)) / diagonal,
212
+ }
213
+
214
+ def extract_head_dynamics(
215
+ self, landmarks: np.ndarray, prev_landmarks: Optional[np.ndarray] = None
216
+ ) -> Dict[str, float]:
217
+ """Calculate approximate head pose and inter-frame movement.
218
+
219
+ Args:
220
+ landmarks: Current Face Mesh landmarks with shape ``(468, 3)``.
221
+ prev_landmarks: Optional previous frame landmarks for displacement.
222
+
223
+ Returns:
224
+ Dictionary containing pitch, yaw, roll, and displacement.
225
+ """
226
+
227
+ diagonal = self._face_diagonal(landmarks)
228
+ left_eye = landmarks[33]
229
+ right_eye = landmarks[263]
230
+ nose_tip = landmarks[1]
231
+ chin = landmarks[199]
232
+ mouth_left = landmarks[61]
233
+ mouth_right = landmarks[291]
234
+
235
+ eye_center = (left_eye + right_eye) / 2.0
236
+ mouth_center = (mouth_left + mouth_right) / 2.0
237
+ eye_vector = right_eye - left_eye
238
+ face_height = max(self._distance(eye_center, chin), 1e-6)
239
+ face_width = max(self._distance(left_eye, right_eye), 1e-6)
240
+
241
+ roll = float(np.degrees(np.arctan2(eye_vector[1], eye_vector[0])))
242
+ yaw = float(((nose_tip[0] - eye_center[0]) / face_width) * 45.0)
243
+ pitch = float(((nose_tip[1] - mouth_center[1]) / face_height) * 45.0)
244
+ displacement = 0.0
245
+ if prev_landmarks is not None and prev_landmarks.shape == landmarks.shape:
246
+ # Sudden head movement or freezing can be relevant when paired with verbal hesitation.
247
+ displacement = float(np.mean(np.linalg.norm(landmarks[self._HEAD_POINTS] - prev_landmarks[self._HEAD_POINTS], axis=1)) / diagonal)
248
+
249
+ return {
250
+ "head_pitch": pitch,
251
+ "head_yaw": yaw,
252
+ "head_roll": roll,
253
+ "head_displacement": displacement,
254
+ }
255
+
256
+ def process_frame(
257
+ self, frame: np.ndarray, prev_landmarks: Optional[np.ndarray] = None
258
+ ) -> Optional[Dict[str, float]]:
259
+ """Process a single BGR frame and extract all facial features.
260
+
261
+ Args:
262
+ frame: BGR video frame.
263
+ prev_landmarks: Optional previous frame landmarks for dynamics.
264
+
265
+ Returns:
266
+ Combined feature dictionary, or ``None`` when no reliable face is detected.
267
+ """
268
+
269
+ landmarks = self._extract_landmarks(frame)
270
+ if landmarks is None:
271
+ return None
272
+
273
+ left_eye, right_eye = self.extract_eye_openness(landmarks)
274
+ features: Dict[str, float] = {
275
+ "eye_left_openness": left_eye,
276
+ "eye_right_openness": right_eye,
277
+ }
278
+ features.update(self.extract_mouth_movement(landmarks))
279
+ features.update(self.extract_eyebrow_tension(landmarks))
280
+ features.update(self.extract_nose_movement(landmarks))
281
+ features.update(self.extract_head_dynamics(landmarks, prev_landmarks))
282
+ features["_landmarks"] = landmarks # Internal handoff for process_video only.
283
+ return features
284
+
285
+ def process_video(self, video_path: str, max_frames: Optional[int] = None) -> pd.DataFrame:
286
+ """Process a video and extract frame-level facial features.
287
+
288
+ Args:
289
+ video_path: Path to a video file.
290
+ max_frames: Optional maximum number of frames to process.
291
+
292
+ Returns:
293
+ DataFrame with frame index, timestamp, and feature columns. Missing
294
+ detections are represented with ``NaN`` feature values.
295
+ """
296
+
297
+ capture, metadata = load_video(video_path)
298
+ fps = float(metadata.get("fps") or 30.0)
299
+ total_frames = int(metadata.get("frame_count") or 0)
300
+ limit = min(total_frames, max_frames) if max_frames is not None and total_frames else max_frames or total_frames
301
+ records: List[Dict[str, float]] = []
302
+ prev_landmarks: Optional[np.ndarray] = None
303
+
304
+ try:
305
+ progress = tqdm(total=limit if limit else None, desc="Extracting facial features")
306
+ frame_idx = 0
307
+ while True:
308
+ if max_frames is not None and frame_idx >= max_frames:
309
+ break
310
+ ok, frame = capture.read()
311
+ if not ok:
312
+ break
313
+ record: Dict[str, float] = {"frame_idx": float(frame_idx), "timestamp": frame_idx / fps}
314
+ features = self.process_frame(frame, prev_landmarks)
315
+ if features is None:
316
+ record.update({name: np.nan for name in self.get_feature_names()})
317
+ prev_landmarks = None
318
+ else:
319
+ landmarks = features.pop("_landmarks")
320
+ record.update(features)
321
+ prev_landmarks = landmarks
322
+ records.append(record)
323
+ frame_idx += 1
324
+ progress.update(1)
325
+ progress.close()
326
+ finally:
327
+ capture.release()
328
+
329
+ columns = ["frame_idx", "timestamp"] + self.get_feature_names()
330
+ return pd.DataFrame(records, columns=columns)
331
+
332
+ def get_feature_names(self) -> List[str]:
333
+ """Return ordered names of all produced facial features."""
334
+
335
+ return list(self._FEATURE_NAMES)
336
+
337
+ def close(self) -> None:
338
+ """Release MediaPipe resources."""
339
+
340
+ self.face_mesh.close()
341
+
src/facial/visualizer.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Visualization utilities for facial landmarks and feature timelines."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Dict, Optional
6
+
7
+ import cv2
8
+ import matplotlib.pyplot as plt
9
+ import numpy as np
10
+ import pandas as pd
11
+ import seaborn as sns
12
+
13
+
14
+ class FacialVisualizer:
15
+ """Visualizes facial landmarks and feature values on video frames."""
16
+
17
+ def __init__(self):
18
+ """Define landmark groups and display thresholds for overlays."""
19
+
20
+ self.eye_landmarks = [33, 160, 158, 133, 153, 144, 362, 385, 387, 263, 373, 380]
21
+ self.mouth_landmarks = [13, 14, 61, 291]
22
+ self.eyebrow_landmarks = [70, 63, 105, 66, 107, 336, 296, 334, 293, 300]
23
+ self.eye_threshold = 0.22
24
+ self.mouth_threshold = 0.35
25
+ self.brow_threshold = 0.18
26
+
27
+ @staticmethod
28
+ def _as_array(landmarks) -> np.ndarray:
29
+ """Convert supported landmark representations to an array."""
30
+
31
+ if landmarks is None:
32
+ return np.empty((0, 3), dtype=np.float32)
33
+ if isinstance(landmarks, np.ndarray):
34
+ return landmarks
35
+ if hasattr(landmarks, "landmark"):
36
+ return np.array([[point.x, point.y, point.z] for point in landmarks.landmark], dtype=np.float32)
37
+ return np.asarray(landmarks, dtype=np.float32)
38
+
39
+ def draw_landmarks(self, frame: np.ndarray, landmarks, feature_values: Dict[str, float]) -> np.ndarray:
40
+ """Draw face mesh landmarks and highlight active feature groups.
41
+
42
+ Args:
43
+ frame: BGR image frame to annotate.
44
+ landmarks: Landmark array or MediaPipe landmark list.
45
+ feature_values: Dictionary of extracted facial feature values.
46
+
47
+ Returns:
48
+ Annotated frame copy.
49
+ """
50
+
51
+ output = frame.copy()
52
+ points = self._as_array(landmarks)
53
+ if points.size == 0:
54
+ cv2.putText(output, "No face detected", (20, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2)
55
+ return output
56
+
57
+ height, width = output.shape[:2]
58
+ if np.nanmax(points[:, :2]) <= 1.5:
59
+ draw_points = np.column_stack((points[:, 0] * width, points[:, 1] * height)).astype(int)
60
+ else:
61
+ draw_points = points[:, :2].astype(int)
62
+
63
+ for x_coord, y_coord in draw_points:
64
+ cv2.circle(output, (int(x_coord), int(y_coord)), 1, (0, 180, 0), -1)
65
+
66
+ eye_open = min(
67
+ float(feature_values.get("eye_left_openness", 1.0)),
68
+ float(feature_values.get("eye_right_openness", 1.0)),
69
+ )
70
+ if eye_open < self.eye_threshold:
71
+ # Eye closure highlights possible blinking or avoidance-related stress markers.
72
+ for idx in self.eye_landmarks:
73
+ cv2.circle(output, tuple(draw_points[idx]), 3, (255, 0, 0), -1)
74
+
75
+ if float(feature_values.get("mouth_openness", 0.0)) > self.mouth_threshold:
76
+ # Mouth movement highlights speech articulation shifts and possible hesitation.
77
+ for idx in self.mouth_landmarks:
78
+ cv2.circle(output, tuple(draw_points[idx]), 4, (0, 0, 255), -1)
79
+
80
+ if float(feature_values.get("brow_tension", 0.0)) > self.brow_threshold:
81
+ # Brow tension highlights facial stress or emotion-suppression cues.
82
+ for idx in self.eyebrow_landmarks:
83
+ cv2.circle(output, tuple(draw_points[idx]), 3, (0, 255, 255), -1)
84
+
85
+ overlay_items = [
86
+ ("Eye", eye_open),
87
+ ("Mouth", float(feature_values.get("mouth_openness", np.nan))),
88
+ ("Brow", float(feature_values.get("brow_tension", np.nan))),
89
+ ("Yaw", float(feature_values.get("head_yaw", np.nan))),
90
+ ]
91
+ for row, (name, value) in enumerate(overlay_items):
92
+ cv2.putText(output, f"{name}: {value:.3f}", (20, 30 + row * 25), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 255), 2)
93
+ return output
94
+
95
+ def create_feature_timeline(self, features_df: pd.DataFrame, save_path: Optional[str] = None) -> plt.Figure:
96
+ """Create a multi-panel timeline plot of facial features.
97
+
98
+ Args:
99
+ features_df: DataFrame containing frame-level facial features.
100
+ save_path: Optional path where the figure should be saved.
101
+
102
+ Returns:
103
+ Matplotlib figure containing four synchronized subplots.
104
+ """
105
+
106
+ sns.set_theme(style="whitegrid")
107
+ time = features_df["timestamp"] if "timestamp" in features_df else features_df.index
108
+ fig, axes = plt.subplots(4, 1, figsize=(12, 10), sharex=True)
109
+
110
+ axes[0].plot(time, features_df.get("eye_left_openness"), label="Left eye")
111
+ axes[0].plot(time, features_df.get("eye_right_openness"), label="Right eye")
112
+ axes[0].set_ylabel("Eye openness")
113
+ axes[0].legend(loc="best")
114
+
115
+ axes[1].plot(time, features_df.get("mouth_openness"), label="Openness")
116
+ axes[1].plot(time, features_df.get("mouth_width"), label="Width")
117
+ axes[1].set_ylabel("Mouth movement")
118
+ axes[1].legend(loc="best")
119
+
120
+ axes[2].plot(time, features_df.get("brow_tension"), label="Brow tension", color="tab:orange")
121
+ axes[2].set_ylabel("Eyebrow tension")
122
+ axes[2].legend(loc="best")
123
+
124
+ for column in ["head_pitch", "head_yaw", "head_roll"]:
125
+ axes[3].plot(time, features_df.get(column), label=column.replace("head_", "").title())
126
+ axes[3].set_ylabel("Degrees")
127
+ axes[3].set_xlabel("Time (s)")
128
+ axes[3].legend(loc="best")
129
+
130
+ fig.suptitle("Facial Feature Timeline")
131
+ fig.tight_layout()
132
+ if save_path:
133
+ fig.savefig(save_path, dpi=300, bbox_inches="tight")
134
+ return fig
135
+
src/model/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """Model module for deception classification and calibration."""
2
+
3
+ from .classifier import DeceptionClassifier
4
+ from .calibration import SubjectCalibrator
5
+
6
+ __all__ = ['DeceptionClassifier', 'SubjectCalibrator']
src/model/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (340 Bytes). View file
 
src/model/calibration.py ADDED
@@ -0,0 +1,265 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Subject-adaptive calibration utilities for deception detection.
2
+
3
+ This module implements personal calibration to account for inter-subject
4
+ behavioral variability, a major confound in affective/deception modeling.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from dataclasses import asdict
11
+ from pathlib import Path
12
+ from typing import Dict, List, Optional
13
+
14
+ import numpy as np
15
+ import pandas as pd
16
+ from sklearn.base import clone
17
+
18
+ from src.model.classifier import DeceptionClassifier
19
+ from src.utils.config import AppConfig, get_default_config
20
+ from src.utils.io import ensure_dir
21
+
22
+
23
+ class SubjectCalibrator:
24
+ """Subject-specific calibration for deception detection.
25
+
26
+ Adapts a global model to individual subjects by learning personal baselines
27
+ and applying feature-space plus score-space adjustments.
28
+ """
29
+
30
+ def __init__(self, base_classifier: DeceptionClassifier, config: Optional[AppConfig] = None):
31
+ """Initialize calibrator.
32
+
33
+ Args:
34
+ base_classifier: Trained global classifier.
35
+ config: Optional app-level configuration.
36
+ """
37
+
38
+ self.base_classifier = base_classifier
39
+ self.config: AppConfig = config or get_default_config()
40
+ self.subject_profiles: Dict[str, Dict[str, float]] = {}
41
+
42
+ def compute_baseline(self, subject_features: pd.DataFrame, subject_id: str) -> Dict[str, float]:
43
+ """Compute a subject's baseline behavior from truthful samples.
44
+
45
+ Args:
46
+ subject_features: DataFrame containing subject feature rows.
47
+ subject_id: Subject identifier.
48
+
49
+ Returns:
50
+ Dict[str, float]: Baseline profile with mean/std per feature.
51
+ """
52
+
53
+ if subject_features is None or subject_features.empty:
54
+ return {"subject_id": subject_id, "n_samples": 0.0}
55
+
56
+ X, feature_names = self.base_classifier.prepare_features(subject_features)
57
+ if X.size == 0 or len(feature_names) == 0:
58
+ return {"subject_id": subject_id, "n_samples": float(len(subject_features))}
59
+
60
+ means = np.nanmean(X, axis=0)
61
+ stds = np.nanstd(X, axis=0)
62
+
63
+ profile: Dict[str, float] = {
64
+ "subject_id": subject_id, # type: ignore[assignment]
65
+ "n_samples": float(X.shape[0]),
66
+ }
67
+ for i, name in enumerate(feature_names):
68
+ profile[f"{name}_mean"] = float(means[i])
69
+ profile[f"{name}_std"] = float(max(stds[i], 1e-8))
70
+
71
+ # Baseline deception tendency is used for threshold adaptation.
72
+ if self.base_classifier.is_fitted_:
73
+ scores = self.base_classifier.predict_deception_score(X)
74
+ profile["baseline_deception_mean"] = float(np.mean(scores))
75
+ profile["baseline_deception_std"] = float(max(np.std(scores), 1e-8))
76
+ else:
77
+ profile["baseline_deception_mean"] = 0.5
78
+ profile["baseline_deception_std"] = 0.1
79
+
80
+ return profile
81
+
82
+ def calibrate_features(
83
+ self,
84
+ features: np.ndarray,
85
+ subject_id: str,
86
+ feature_names: List[str],
87
+ ) -> np.ndarray:
88
+ """Apply subject-specific z-score normalization.
89
+
90
+ Args:
91
+ features: Raw feature array.
92
+ subject_id: Subject identifier.
93
+ feature_names: Ordered feature names.
94
+
95
+ Returns:
96
+ np.ndarray: Calibrated feature array.
97
+ """
98
+
99
+ X = np.asarray(features, dtype=np.float64)
100
+ if X.size == 0:
101
+ return X
102
+
103
+ profile = self.subject_profiles.get(subject_id)
104
+ if profile is None:
105
+ return X
106
+
107
+ calibrated = X.copy()
108
+ for i, name in enumerate(feature_names):
109
+ mean_key = f"{name}_mean"
110
+ std_key = f"{name}_std"
111
+ if mean_key in profile and std_key in profile:
112
+ mu = float(profile[mean_key])
113
+ sigma = float(profile[std_key])
114
+ calibrated[:, i] = (calibrated[:, i] - mu) / (sigma + 1e-8)
115
+
116
+ return calibrated
117
+
118
+ def calibrate_prediction(self, raw_proba: float, subject_id: str) -> float:
119
+ """Apply subject-specific logistic threshold adjustment.
120
+
121
+ Args:
122
+ raw_proba: Raw deception probability from global model.
123
+ subject_id: Subject identifier.
124
+
125
+ Returns:
126
+ float: Calibrated deception probability in [0, 1].
127
+ """
128
+
129
+ p = float(np.clip(raw_proba, 1e-6, 1.0 - 1e-6))
130
+ profile = self.subject_profiles.get(subject_id)
131
+ if profile is None:
132
+ return p
133
+
134
+ baseline_mean = float(profile.get("baseline_deception_mean", 0.5))
135
+ baseline_std = float(profile.get("baseline_deception_std", 0.1))
136
+ n_samples = float(profile.get("n_samples", 0.0))
137
+
138
+ # Research rationale: move the decision surface in logit space, where
139
+ # additive shifts correspond to multiplicative odds adjustment.
140
+ confidence = float(np.clip(np.log1p(n_samples) / np.log1p(200.0), 0.0, 1.0))
141
+ shift_strength = confidence * np.clip((baseline_mean - 0.5) / (baseline_std + 1e-8), -2.0, 2.0)
142
+ shift_strength *= 0.15
143
+
144
+ logit = np.log(p / (1.0 - p))
145
+ adjusted = logit - shift_strength
146
+ calibrated = 1.0 / (1.0 + np.exp(-adjusted))
147
+ return float(np.clip(calibrated, 0.0, 1.0))
148
+
149
+ def add_subject_profile(self, subject_id: str, truthful_features: pd.DataFrame) -> None:
150
+ """Register a subject profile using truthful behavior samples.
151
+
152
+ Args:
153
+ subject_id: Subject identifier.
154
+ truthful_features: Truthful-only feature dataframe for the subject.
155
+ """
156
+
157
+ profile = self.compute_baseline(truthful_features, subject_id)
158
+ self.subject_profiles[subject_id] = profile
159
+
160
+ def save_profiles(self, path: str) -> None:
161
+ """Save all subject profiles to a JSON file.
162
+
163
+ Args:
164
+ path: Output JSON path.
165
+ """
166
+
167
+ out_path = Path(path)
168
+ ensure_dir(out_path.parent)
169
+ with out_path.open("w", encoding="utf-8") as f:
170
+ json.dump(self.subject_profiles, f, indent=2, sort_keys=True)
171
+
172
+ def load_profiles(self, path: str) -> None:
173
+ """Load subject profiles from JSON file.
174
+
175
+ Args:
176
+ path: Input JSON path.
177
+ """
178
+
179
+ in_path = Path(path)
180
+ if not in_path.exists():
181
+ raise FileNotFoundError(f"Subject profile file not found: {in_path}")
182
+ with in_path.open("r", encoding="utf-8") as f:
183
+ data = json.load(f)
184
+ if not isinstance(data, dict):
185
+ raise ValueError("Invalid subject profile format. Expected JSON object.")
186
+ self.subject_profiles = data
187
+
188
+ def get_subject_ids(self) -> List[str]:
189
+ """Return registered subject identifiers."""
190
+
191
+ return sorted(self.subject_profiles.keys())
192
+
193
+ def calibrate_subject_model(
194
+ self,
195
+ subject_id: str,
196
+ subject_features: np.ndarray,
197
+ subject_labels: np.ndarray,
198
+ feature_names: List[str],
199
+ ) -> DeceptionClassifier:
200
+ """Fine-tune a personal model for a specific subject.
201
+
202
+ Hard examples (where base model disagrees) are upweighted to specialize
203
+ the model for subject-specific decision boundaries.
204
+
205
+ Args:
206
+ subject_id: Subject identifier.
207
+ subject_features: Subject feature matrix.
208
+ subject_labels: Subject labels.
209
+ feature_names: Feature names used by the model.
210
+
211
+ Returns:
212
+ DeceptionClassifier: Subject-adapted classifier.
213
+ """
214
+
215
+ X = np.asarray(subject_features, dtype=np.float64)
216
+ y = np.asarray(subject_labels).astype(int)
217
+ if X.ndim != 2 or y.ndim != 1 or len(X) != len(y):
218
+ raise ValueError("Invalid subject data shapes for calibration.")
219
+ if len(y) == 0:
220
+ raise ValueError("Cannot calibrate subject model with zero samples.")
221
+
222
+ personal = DeceptionClassifier(config=self.base_classifier.config)
223
+
224
+ # Transfer model state when available to keep inductive bias from global data.
225
+ if self.base_classifier.is_fitted_:
226
+ try:
227
+ personal.scaler = clone(self.base_classifier.scaler)
228
+ personal.scaler.mean_ = np.copy(self.base_classifier.scaler.mean_)
229
+ personal.scaler.scale_ = np.copy(self.base_classifier.scaler.scale_)
230
+ personal.scaler.var_ = np.copy(self.base_classifier.scaler.var_)
231
+ personal.scaler.n_features_in_ = self.base_classifier.scaler.n_features_in_
232
+ if hasattr(self.base_classifier.scaler, "n_samples_seen_"):
233
+ personal.scaler.n_samples_seen_ = self.base_classifier.scaler.n_samples_seen_
234
+ except Exception:
235
+ # Fallback to refitting scaler below.
236
+ pass
237
+
238
+ # Subject-space calibration (if profile exists) before fine-tuning.
239
+ X_cal = self.calibrate_features(X, subject_id, feature_names)
240
+
241
+ # Derive sample weights emphasizing hard examples.
242
+ weights = np.ones(len(y), dtype=np.float64)
243
+ if self.base_classifier.is_fitted_:
244
+ base_pred = self.base_classifier.predict(X)
245
+ disagreement = (base_pred != y).astype(np.float64)
246
+ weights += 2.0 * disagreement
247
+
248
+ X_scaled = personal.scaler.fit_transform(X_cal)
249
+ personal.model.fit(X_scaled, y, sample_weight=weights)
250
+
251
+ # Calibrator requires both classes and enough support.
252
+ unique = np.unique(y)
253
+ if len(unique) > 1 and len(y) >= 10:
254
+ personal.calibrated_model.fit(X_scaled, y, sample_weight=weights)
255
+
256
+ personal.feature_names_ = list(feature_names)
257
+ personal.is_fitted_ = True
258
+ personal.training_metadata_ = {
259
+ "subject_id": subject_id,
260
+ "n_samples": float(len(y)),
261
+ "class_distribution": {int(c): int((y == c).sum()) for c in unique},
262
+ "config": asdict(personal.config),
263
+ }
264
+ return personal
265
+
src/model/classifier.py ADDED
@@ -0,0 +1,330 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Core Gradient Boosting classifier for deception detection.
2
+
3
+ This module implements a robust, production-oriented classifier wrapper that
4
+ standardizes features, handles missing values, performs probability calibration,
5
+ and persists all metadata required for reproducible inference.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import asdict
11
+ from typing import Any, Dict, List, Optional, Tuple
12
+
13
+ import numpy as np
14
+ import pandas as pd
15
+ from sklearn.calibration import CalibratedClassifierCV
16
+ from sklearn.ensemble import GradientBoostingClassifier
17
+ from sklearn.metrics import accuracy_score, log_loss, roc_auc_score
18
+ from sklearn.model_selection import cross_val_score
19
+ from sklearn.pipeline import make_pipeline
20
+ from sklearn.preprocessing import StandardScaler
21
+
22
+ from src.utils.config import ModelConfig
23
+ from src.utils.io import load_model as io_load_model
24
+ from src.utils.io import save_model as io_save_model
25
+
26
+
27
+ class DeceptionClassifier:
28
+ """Gradient Boosting-based deception classifier with probability calibration.
29
+
30
+ Uses `sklearn` GradientBoostingClassifier as the base estimator and wraps it
31
+ with calibration for better probabilistic interpretation.
32
+ """
33
+
34
+ def __init__(self, config: Optional[ModelConfig] = None):
35
+ """Initialize classifier with model config parameters.
36
+
37
+ Args:
38
+ config: Optional model configuration. If not provided, defaults are used.
39
+ """
40
+
41
+ self.config: ModelConfig = config or ModelConfig()
42
+ self.model = GradientBoostingClassifier(
43
+ n_estimators=self.config.n_estimators,
44
+ learning_rate=self.config.learning_rate,
45
+ max_depth=self.config.max_depth,
46
+ min_samples_split=self.config.min_samples_split,
47
+ min_samples_leaf=self.config.min_samples_leaf,
48
+ subsample=self.config.subsample,
49
+ random_state=self.config.random_state,
50
+ )
51
+ self.scaler = StandardScaler()
52
+
53
+ calibration_method = getattr(self.config, "calibration_method", "sigmoid")
54
+ self.calibrated_model = CalibratedClassifierCV(
55
+ estimator=GradientBoostingClassifier(
56
+ n_estimators=self.config.n_estimators,
57
+ learning_rate=self.config.learning_rate,
58
+ max_depth=self.config.max_depth,
59
+ min_samples_split=self.config.min_samples_split,
60
+ min_samples_leaf=self.config.min_samples_leaf,
61
+ subsample=self.config.subsample,
62
+ random_state=self.config.random_state,
63
+ ),
64
+ method=calibration_method,
65
+ cv=min(max(2, getattr(self.config, "cross_validation_folds", 5)), 5),
66
+ )
67
+
68
+ self.feature_names_: Optional[List[str]] = None
69
+ self.is_fitted_: bool = False
70
+ self.training_metadata_: Dict[str, Any] = {}
71
+
72
+ def prepare_features(
73
+ self,
74
+ features_df: pd.DataFrame,
75
+ feature_columns: Optional[List[str]] = None,
76
+ ) -> Tuple[np.ndarray, List[str]]:
77
+ """Prepare feature matrix from a feature dataframe.
78
+
79
+ Args:
80
+ features_df: DataFrame with extracted features.
81
+ feature_columns: Optional explicit feature column list.
82
+
83
+ Returns:
84
+ Tuple[np.ndarray, List[str]]: Feature matrix and ordered feature names.
85
+ """
86
+
87
+ if features_df is None or features_df.empty:
88
+ return np.empty((0, 0), dtype=np.float64), []
89
+
90
+ excluded_columns = {
91
+ "frame_idx",
92
+ "timestamp",
93
+ "segment_idx",
94
+ "start_time",
95
+ "end_time",
96
+ "label",
97
+ "subject_id",
98
+ }
99
+
100
+ if feature_columns is None:
101
+ if self.is_fitted_ and self.feature_names_ is not None:
102
+ feature_columns = self.feature_names_
103
+ else:
104
+ candidate_cols = [c for c in features_df.columns if c not in excluded_columns]
105
+ # Research rationale: constraining to numeric-only columns prevents accidental
106
+ # leakage of identifiers and ensures stable scaling behavior.
107
+ feature_columns = [
108
+ c for c in candidate_cols if pd.api.types.is_numeric_dtype(features_df[c])
109
+ ]
110
+
111
+ if not feature_columns:
112
+ return np.empty((len(features_df), 0), dtype=np.float64), []
113
+
114
+ missing_cols = set(feature_columns) - set(features_df.columns)
115
+ if missing_cols:
116
+ for col in missing_cols:
117
+ features_df[col] = 0.0
118
+
119
+ selected = features_df[feature_columns].copy()
120
+ for col in selected.columns:
121
+ selected[col] = pd.to_numeric(selected[col], errors="coerce")
122
+
123
+ selected = selected.replace([np.inf, -np.inf], np.nan)
124
+ medians = selected.median(numeric_only=True)
125
+ selected = selected.fillna(medians)
126
+ selected = selected.fillna(0.0)
127
+
128
+ # Clip to robust quantiles to reduce effect of extreme outliers.
129
+ lower = selected.quantile(0.01)
130
+ upper = selected.quantile(0.99)
131
+ selected = selected.clip(lower=lower, upper=upper, axis=1)
132
+
133
+ return selected.to_numpy(dtype=np.float64), list(selected.columns)
134
+
135
+ def train(self, X: np.ndarray, y: np.ndarray, feature_names: List[str]) -> Dict[str, float]:
136
+ """Train the gradient boosting classifier.
137
+
138
+ Args:
139
+ X: Feature matrix of shape (n_samples, n_features).
140
+ y: Binary labels (0=truthful, 1=deceptive).
141
+ feature_names: Ordered feature names.
142
+
143
+ Returns:
144
+ Dict[str, float]: Training metrics and dataset summary.
145
+ """
146
+
147
+ X = np.asarray(X, dtype=np.float64)
148
+ y = np.asarray(y).astype(int)
149
+
150
+ if X.ndim != 2:
151
+ raise ValueError("X must be a 2D array.")
152
+ if y.ndim != 1:
153
+ raise ValueError("y must be a 1D array.")
154
+ if len(X) != len(y):
155
+ raise ValueError("X and y must have the same number of samples.")
156
+ if len(y) == 0:
157
+ raise ValueError("Cannot train with zero samples.")
158
+
159
+ self.feature_names_ = list(feature_names)
160
+
161
+ X_scaled = self.scaler.fit_transform(X)
162
+
163
+ unique_classes, counts = np.unique(y, return_counts=True)
164
+ class_distribution = {int(k): int(v) for k, v in zip(unique_classes, counts)}
165
+
166
+ # Research rationale: inverse-frequency weighting mitigates skewed priors and
167
+ # helps gradient boosting focus on under-represented deceptive/truthful samples.
168
+ class_weights = {
169
+ cls: len(y) / (len(unique_classes) * count)
170
+ for cls, count in zip(unique_classes, counts)
171
+ if count > 0
172
+ }
173
+ sample_weight = np.array([class_weights.get(label, 1.0) for label in y], dtype=np.float64)
174
+
175
+ self.model.fit(X_scaled, y, sample_weight=sample_weight)
176
+
177
+ can_calibrate = len(unique_classes) > 1 and len(y) >= max(10, len(unique_classes) * 3)
178
+ if can_calibrate:
179
+ self.calibrated_model.fit(X_scaled, y, sample_weight=sample_weight)
180
+
181
+ # Research rationale: setting is_fitted_ prior to metric computation allows
182
+ # reuse of internal predict/predict_proba methods for training performance tracking.
183
+ self.is_fitted_ = True
184
+
185
+ train_preds = self.predict(X, check_fitted=False)
186
+ train_probs = self.predict_proba(X, check_fitted=False)
187
+
188
+ metrics: Dict[str, float] = {
189
+ "train_accuracy": float(accuracy_score(y, train_preds)),
190
+ "n_samples": float(len(y)),
191
+ "n_features": float(X.shape[1]),
192
+ "class_distribution": class_distribution, # type: ignore[assignment]
193
+ }
194
+
195
+ try:
196
+ metrics["train_log_loss"] = float(log_loss(y, train_probs, labels=[0, 1]))
197
+ except ValueError:
198
+ metrics["train_log_loss"] = float("nan")
199
+
200
+ if len(unique_classes) > 1:
201
+ try:
202
+ metrics["train_auc"] = float(roc_auc_score(y, train_probs[:, 1]))
203
+ except ValueError:
204
+ metrics["train_auc"] = float("nan")
205
+ else:
206
+ metrics["train_auc"] = float("nan")
207
+
208
+ cv_folds = min(getattr(self.config, "cross_validation_folds", 5), len(y))
209
+ if cv_folds >= 2 and len(unique_classes) > 1:
210
+ pipeline = make_pipeline(
211
+ StandardScaler(),
212
+ GradientBoostingClassifier(
213
+ n_estimators=self.config.n_estimators,
214
+ learning_rate=self.config.learning_rate,
215
+ max_depth=self.config.max_depth,
216
+ min_samples_split=self.config.min_samples_split,
217
+ min_samples_leaf=self.config.min_samples_leaf,
218
+ subsample=self.config.subsample,
219
+ random_state=self.config.random_state,
220
+ ),
221
+ )
222
+ cv_scores = cross_val_score(pipeline, X, y, cv=cv_folds, scoring="accuracy")
223
+ metrics["cv_accuracy_mean"] = float(np.mean(cv_scores))
224
+ metrics["cv_accuracy_std"] = float(np.std(cv_scores))
225
+ else:
226
+ metrics["cv_accuracy_mean"] = float("nan")
227
+ metrics["cv_accuracy_std"] = float("nan")
228
+
229
+ self.training_metadata_ = metrics
230
+ return metrics
231
+
232
+ def predict(self, X: np.ndarray, check_fitted: bool = True) -> np.ndarray:
233
+ """Predict binary deception labels for input features.
234
+
235
+ Args:
236
+ X: Input feature matrix.
237
+ check_fitted: Whether to verify that the model is fitted.
238
+ """
239
+
240
+ if check_fitted and not self.is_fitted_:
241
+ raise ValueError("Model is not trained. Call train() first.")
242
+
243
+ X_scaled = self.scaler.transform(np.asarray(X, dtype=np.float64))
244
+ if hasattr(self.calibrated_model, "calibrated_classifiers_"):
245
+ return self.calibrated_model.predict(X_scaled)
246
+ return self.model.predict(X_scaled)
247
+
248
+ def predict_proba(self, X: np.ndarray, check_fitted: bool = True) -> np.ndarray:
249
+ """Predict deception probabilities for input features.
250
+
251
+ Args:
252
+ X: Input feature matrix.
253
+ check_fitted: Whether to verify that the model is fitted.
254
+
255
+ Returns:
256
+ np.ndarray: Probability matrix [truthful_prob, deceptive_prob].
257
+ """
258
+
259
+ if check_fitted and not self.is_fitted_:
260
+ raise ValueError("Model is not trained. Call train() first.")
261
+
262
+ X_scaled = self.scaler.transform(np.asarray(X, dtype=np.float64))
263
+ if hasattr(self.calibrated_model, "calibrated_classifiers_"):
264
+ return self.calibrated_model.predict_proba(X_scaled)
265
+ return self.model.predict_proba(X_scaled)
266
+
267
+ def predict_deception_score(self, X: np.ndarray, check_fitted: bool = True) -> np.ndarray:
268
+ """Predict scalar deception score in [0, 1] for each sample."""
269
+
270
+ proba = self.predict_proba(X, check_fitted=check_fitted)
271
+ return proba[:, 1]
272
+
273
+ def get_feature_importance(self) -> pd.DataFrame:
274
+ """Get sorted feature importance values from the trained base model."""
275
+
276
+ if not self.is_fitted_:
277
+ raise ValueError("Model is not trained. Call train() first.")
278
+
279
+ if self.feature_names_ is None:
280
+ raise ValueError("Feature names are unavailable.")
281
+
282
+ importances = getattr(self.model, "feature_importances_", None)
283
+ if importances is None:
284
+ raise ValueError("Model does not expose feature importances.")
285
+
286
+ importance_df = pd.DataFrame(
287
+ {"feature": self.feature_names_, "importance": np.asarray(importances, dtype=np.float64)}
288
+ )
289
+ return importance_df.sort_values("importance", ascending=False).reset_index(drop=True)
290
+
291
+ def save(self, path: str) -> None:
292
+ """Save model, scaler, calibration wrapper, and metadata using joblib."""
293
+
294
+ payload = {
295
+ "model": self.model,
296
+ "calibrated_model": self.calibrated_model,
297
+ "scaler": self.scaler,
298
+ "feature_names": self.feature_names_,
299
+ "is_fitted": self.is_fitted_,
300
+ "config": asdict(self.config),
301
+ "training_metadata": self.training_metadata_,
302
+ }
303
+ io_save_model(payload, path)
304
+
305
+ def load(self, path: str) -> None:
306
+ """Load model, scaler, calibration wrapper, and metadata from file."""
307
+
308
+ payload = io_load_model(path)
309
+ self.model = payload["model"]
310
+ self.calibrated_model = payload.get("calibrated_model", self.calibrated_model)
311
+ self.scaler = payload["scaler"]
312
+ self.feature_names_ = payload.get("feature_names")
313
+ self.is_fitted_ = bool(payload.get("is_fitted", True))
314
+ self.training_metadata_ = dict(payload.get("training_metadata", {}))
315
+
316
+ config_dict = payload.get("config", {})
317
+ if isinstance(config_dict, dict):
318
+ self.config = ModelConfig(**config_dict)
319
+
320
+ def get_model_info(self) -> Dict[str, Any]:
321
+ """Return model metadata and configuration details."""
322
+
323
+ return {
324
+ "is_fitted": self.is_fitted_,
325
+ "n_features": 0 if self.feature_names_ is None else len(self.feature_names_),
326
+ "feature_names": self.feature_names_ or [],
327
+ "config": asdict(self.config),
328
+ "training_metadata": self.training_metadata_,
329
+ }
330
+
src/pipeline.py ADDED
@@ -0,0 +1,524 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Main pipeline for multimodal deception analysis."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import tempfile
7
+ from pathlib import Path
8
+ from typing import Dict, List, Optional, Union
9
+
10
+ import pandas as pd
11
+ from tqdm import tqdm
12
+
13
+ from src.audio.extractor import AudioFeatureExtractor
14
+ from src.audio.fusion import MultimodalFusion
15
+ from src.explainability.explainer import DeceptionExplainer
16
+ from src.explainability.visualizer import ExplainabilityVisualizer
17
+ from src.facial.extractor import FacialFeatureExtractor
18
+ from src.facial.visualizer import FacialVisualizer
19
+ from src.model.calibration import SubjectCalibrator
20
+ from src.model.classifier import DeceptionClassifier
21
+ from src.temporal.analyzer import TemporalAnalyzer
22
+ from src.temporal.visualizer import TemporalVisualizer
23
+ from src.utils.config import AppConfig, get_default_config
24
+ from src.utils.io import extract_audio_from_video, save_results
25
+
26
+
27
+ class DeceptionPipeline:
28
+ """Main pipeline for multimodal deception analysis."""
29
+
30
+ def __init__(self, config: Optional[AppConfig] = None):
31
+ """Initialize pipeline with all sub-modules.
32
+
33
+ Args:
34
+ config: Optional application configuration. If None, default config is used.
35
+ """
36
+ self.config = config or get_default_config()
37
+ self.logger = logging.getLogger(__name__)
38
+
39
+ # Initialize all components with config
40
+ self.facial_extractor = FacialFeatureExtractor(self.config.features)
41
+ self.audio_extractor = AudioFeatureExtractor(self.config.audio)
42
+ self.fusion = MultimodalFusion(self.config)
43
+ self.classifier = DeceptionClassifier(self.config.model)
44
+ self.calibrator = SubjectCalibrator(self.classifier, self.config)
45
+ self.temporal_analyzer = TemporalAnalyzer(self.config.temporal)
46
+ self.explainer = DeceptionExplainer(self.classifier)
47
+
48
+ # Visualizers (no config needed)
49
+ self.facial_visualizer = FacialVisualizer()
50
+ self.temporal_visualizer = TemporalVisualizer()
51
+ self.explainability_visualizer = ExplainabilityVisualizer()
52
+
53
+ self.logger.info("DeceptionPipeline initialized with default config.")
54
+
55
+ def analyze_video(
56
+ self,
57
+ video_path: str,
58
+ subject_id: Optional[str] = None,
59
+ generate_visualizations: bool = True,
60
+ output_dir: Optional[str] = None,
61
+ ) -> dict:
62
+ """Full analysis pipeline for a single video.
63
+
64
+ Steps:
65
+ 1. Extract facial features from video
66
+ 2. Extract audio from video (using io utilities), then extract audio features
67
+ 3. Fuse facial and audio features (align + cross-modal)
68
+ 4. Run classifier prediction
69
+ 5. Apply subject calibration if subject_id provided
70
+ 6. Run temporal analysis on probability trajectory
71
+ 7. Generate explanations using SHAP
72
+ 8. Generate visualizations if requested
73
+ 9. Compile results into a comprehensive report dict
74
+
75
+ Args:
76
+ video_path: Path to the video file.
77
+ subject_id: Optional subject identifier for calibration.
78
+ generate_visualizations: Whether to generate visualization plots.
79
+ output_dir: Directory to save visualizations and intermediate results.
80
+ If None, a temporary directory is used.
81
+
82
+ Returns:
83
+ dict with keys:
84
+ - 'deception_probability': float
85
+ - 'prediction': str ('truthful' or 'deceptive')
86
+ - 'confidence': float
87
+ - 'facial_features': DataFrame
88
+ - 'audio_features': DataFrame
89
+ - 'fused_features': DataFrame
90
+ - 'temporal_analysis': dict
91
+ - 'explanation': dict
92
+ - 'calibrated': bool
93
+ - 'visualizations': dict of file paths (if generated)
94
+ - 'metadata': dict
95
+ """
96
+ self.logger.info(f"Starting analysis of video: {video_path}")
97
+ result: Dict[str, Union[float, str, pd.DataFrame, dict, bool, list]] = {}
98
+ visualizations = {}
99
+ metadata = {"video_path": video_path, "subject_id": subject_id}
100
+
101
+ # Determine output directory
102
+ if output_dir is None:
103
+ output_dir = tempfile.mkdtemp(prefix="deception_analysis_")
104
+ output_path = Path(output_dir)
105
+ output_path.mkdir(parents=True, exist_ok=True)
106
+
107
+ try:
108
+ # 1. Extract facial features
109
+ self.logger.info("Extracting facial features...")
110
+ facial_df = self.facial_extractor.process_video(video_path)
111
+ result["facial_features"] = facial_df
112
+ metadata["facial_frames"] = len(facial_df)
113
+ self.logger.info(f"Extracted {len(facial_df)} facial frames.")
114
+
115
+ # 2. Extract audio features
116
+ self.logger.info("Extracting audio features...")
117
+ # Extract audio from video
118
+ audio_temp = output_path / "audio.wav"
119
+ try:
120
+ audio_path = extract_audio_from_video(video_path, audio_temp)
121
+ except Exception as e:
122
+ self.logger.warning(f"Audio extraction failed: {e}. Using empty audio features.")
123
+ audio_df = pd.DataFrame()
124
+ else:
125
+ # Extract audio features
126
+ audio_df = self.audio_extractor.process_audio_segments(
127
+ str(audio_path), segment_duration=2.0
128
+ )
129
+ result["audio_features"] = audio_df
130
+ metadata["audio_segments"] = len(audio_df)
131
+
132
+ # 3. Fuse features
133
+ self.logger.info("Fusing facial and audio features...")
134
+ aligned_df = self.fusion.align_features(
135
+ facial_df, audio_df, video_fps=self.config.features.fps
136
+ )
137
+ fused_df = self.fusion.extract_fusion_features(aligned_df)
138
+ result["fused_features"] = fused_df
139
+
140
+ # 4. Run classifier prediction
141
+ self.logger.info("Running classifier prediction...")
142
+ if not self.classifier.is_fitted_:
143
+ self.logger.warning("Classifier not trained. Using default probability 0.5.")
144
+ proba = 0.5
145
+ prediction = "truthful"
146
+ confidence = 0.5
147
+ per_frame_probs = pd.Series([0.5] * len(fused_df))
148
+ else:
149
+ # Prepare features for classification
150
+ X, feature_names = self.classifier.prepare_features(fused_df)
151
+ if X.size == 0:
152
+ self.logger.warning("No features for classification.")
153
+ proba = 0.5
154
+ prediction = "truthful"
155
+ confidence = 0.5
156
+ per_frame_probs = pd.Series([0.5] * len(fused_df))
157
+ else:
158
+ # Overall prediction (average across frames)
159
+ proba = float(self.classifier.predict_deception_score(X).mean())
160
+ prediction = "deceptive" if proba >= self.config.model.decision_threshold else "truthful"
161
+ confidence = abs(proba - 0.5) * 2 # map 0.5->0, 0/1->1
162
+ # Per-frame probabilities for temporal analysis
163
+ per_frame_probs = pd.Series(self.classifier.predict_deception_score(X))
164
+
165
+ result["deception_probability"] = proba
166
+ result["prediction"] = prediction
167
+ result["confidence"] = confidence
168
+ metadata["decision_threshold"] = self.config.model.decision_threshold
169
+
170
+ # 5. Subject calibration
171
+ calibrated = False
172
+ if subject_id is not None and self.classifier.is_fitted_:
173
+ self.logger.info(f"Applying subject calibration for {subject_id}...")
174
+ try:
175
+ # Update baseline with current features
176
+ self.calibrator.add_subject_profile(subject_id, fused_df)
177
+ # Calibrate predictions
178
+ import numpy as np
179
+ calibrated_preds = np.array([
180
+ self.calibrator.calibrate_prediction(p, subject_id)
181
+ for p in per_frame_probs.values
182
+ ])
183
+ if calibrated_preds is not None:
184
+ calibrated_proba = float(calibrated_preds.mean())
185
+ calibrated_pred = (
186
+ "deceptive"
187
+ if calibrated_proba >= self.config.model.decision_threshold
188
+ else "truthful"
189
+ )
190
+ result["deception_probability"] = calibrated_proba
191
+ result["prediction"] = calibrated_pred
192
+ result["confidence"] = abs(calibrated_proba - 0.5) * 2
193
+ calibrated = True
194
+ per_frame_probs = pd.Series(calibrated_preds)
195
+ except Exception as e:
196
+ self.logger.error(f"Subject calibration failed: {e}")
197
+ result["calibrated"] = calibrated
198
+
199
+ # 6. Temporal analysis
200
+ self.logger.info("Performing temporal analysis...")
201
+ timestamps = fused_df.get("timestamp", pd.Series(range(len(fused_df))))
202
+ temporal_result = self.temporal_analyzer.analyze(
203
+ per_frame_probs.values, timestamps.values
204
+ )
205
+
206
+ # Map temporal results for API
207
+ if isinstance(timestamps, pd.Series):
208
+ temporal_result["timestamps"] = timestamps.tolist()
209
+ else:
210
+ temporal_result["timestamps"] = list(timestamps)
211
+
212
+ if "smoothed_probabilities" in temporal_result:
213
+ import numpy as np
214
+ if isinstance(temporal_result["smoothed_probabilities"], np.ndarray):
215
+ temporal_result["probability_trajectory"] = temporal_result["smoothed_probabilities"].tolist()
216
+ else:
217
+ temporal_result["probability_trajectory"] = list(temporal_result["smoothed_probabilities"])
218
+
219
+ result["temporal_analysis"] = temporal_result
220
+
221
+ # 7. Generate explanations
222
+ self.logger.info("Generating explanations...")
223
+ if self.classifier.is_fitted_ and X.size > 0:
224
+ try:
225
+ # Explainer expects a single sample (1, n_features).
226
+ # We use the mean of all frames to explain the overall video prediction.
227
+ import numpy as np
228
+ X_mean = np.nanmean(X, axis=0, keepdims=True)
229
+ explanation = self.explainer.explain_prediction(
230
+ X=X_mean,
231
+ feature_names=feature_names,
232
+ prediction=proba,
233
+ subject_id=subject_id
234
+ )
235
+ result["explanation"] = explanation
236
+ except Exception as e:
237
+ self.logger.warning(f"Explanation generation failed: {e}")
238
+ result["explanation"] = {"error": str(e)}
239
+ else:
240
+ result["explanation"] = {"info": "Classifier not trained or no features."}
241
+
242
+ # 8. Generate visualizations
243
+ if generate_visualizations:
244
+ self.logger.info("Generating visualizations...")
245
+ try:
246
+ # Facial landmarks visualization
247
+ facial_viz_path = output_path / "facial_landmarks.png"
248
+ self.facial_visualizer.plot_landmarks(facial_df, str(facial_viz_path))
249
+ visualizations["facial_landmarks"] = str(facial_viz_path)
250
+ except Exception as e:
251
+ self.logger.warning(f"Facial visualization failed: {e}")
252
+
253
+ try:
254
+ # Temporal probability trajectory
255
+ temporal_viz_path = output_path / "temporal_trajectory.png"
256
+ self.temporal_visualizer.plot_trajectory(
257
+ per_frame_probs.values,
258
+ timestamps.values,
259
+ str(temporal_viz_path),
260
+ )
261
+ visualizations["temporal_trajectory"] = str(temporal_viz_path)
262
+ except Exception as e:
263
+ self.logger.warning(f"Temporal visualization failed: {e}")
264
+
265
+ if self.classifier.is_fitted_ and X.size > 0:
266
+ try:
267
+ # SHAP summary plot
268
+ shap_viz_path = output_path / "shap_summary.png"
269
+ self.explainability_visualizer.plot_shap_summary(
270
+ self.explainer, X, str(shap_viz_path)
271
+ )
272
+ visualizations["shap_summary"] = str(shap_viz_path)
273
+ except Exception as e:
274
+ self.logger.warning(f"SHAP visualization failed: {e}")
275
+
276
+ result["visualizations"] = visualizations
277
+
278
+ # 9. Metadata
279
+ metadata.update(
280
+ {
281
+ "video_fps": self.config.features.fps,
282
+ "audio_sample_rate": self.config.audio.sample_rate,
283
+ "pipeline_version": "1.0",
284
+ }
285
+ )
286
+ result["metadata"] = metadata
287
+
288
+ self.logger.info(f"Analysis completed. Prediction: {prediction} (prob={proba:.3f})")
289
+ return result
290
+
291
+ except Exception as e:
292
+ self.logger.error(f"Pipeline analysis failed: {e}")
293
+ # Return partial results with error flag
294
+ error_result = {
295
+ "error": str(e),
296
+ "deception_probability": 0.5,
297
+ "prediction": "error",
298
+ "confidence": 0.0,
299
+ "facial_features": pd.DataFrame(),
300
+ "audio_features": pd.DataFrame(),
301
+ "fused_features": pd.DataFrame(),
302
+ "temporal_analysis": {},
303
+ "explanation": {},
304
+ "calibrated": False,
305
+ "visualizations": {},
306
+ "metadata": metadata,
307
+ }
308
+ return error_result
309
+
310
+ def analyze_batch(
311
+ self,
312
+ video_paths: List[str],
313
+ subject_ids: Optional[List[str]] = None,
314
+ generate_visualizations: bool = False,
315
+ output_dir: Optional[str] = None,
316
+ ) -> List[dict]:
317
+ """Analyze multiple videos. Wraps analyze_video with progress tracking.
318
+
319
+ Args:
320
+ video_paths: List of video file paths.
321
+ subject_ids: Optional list of subject IDs (same length as video_paths).
322
+ generate_visualizations: Whether to generate visualizations per video.
323
+ output_dir: Base output directory. Subdirectories will be created per video.
324
+
325
+ Returns:
326
+ List of analysis result dicts, one per video.
327
+ """
328
+ if subject_ids is not None and len(subject_ids) != len(video_paths):
329
+ raise ValueError("subject_ids must have same length as video_paths")
330
+
331
+ results = []
332
+ for idx, video_path in enumerate(tqdm(video_paths, desc="Analyzing videos")):
333
+ subject_id = subject_ids[idx] if subject_ids else None
334
+ video_output_dir = None
335
+ if output_dir is not None:
336
+ video_name = Path(video_path).stem
337
+ video_output_dir = str(Path(output_dir) / video_name)
338
+ Path(video_output_dir).mkdir(parents=True, exist_ok=True)
339
+
340
+ result = self.analyze_video(
341
+ video_path,
342
+ subject_id=subject_id,
343
+ generate_visualizations=generate_visualizations,
344
+ output_dir=video_output_dir,
345
+ )
346
+ results.append(result)
347
+
348
+ return results
349
+
350
+ def train(
351
+ self,
352
+ video_paths: List[str],
353
+ labels: List[int],
354
+ subject_ids: Optional[List[str]] = None,
355
+ output_dir: Optional[str] = None,
356
+ ) -> dict:
357
+ """Train the classifier on labeled video data.
358
+
359
+ Steps:
360
+ 1. Extract features from all videos
361
+ 2. Fuse features
362
+ 3. Train classifier
363
+ 4. Compute training metrics
364
+ 5. Save model if output_dir provided
365
+
366
+ Args:
367
+ video_paths: List of video file paths.
368
+ labels: Binary labels (0=truthful, 1=deceptive).
369
+ subject_ids: Optional list of subject IDs for calibration baseline.
370
+ output_dir: Directory to save trained model and training results.
371
+
372
+ Returns:
373
+ dict with training metrics and model info.
374
+ """
375
+ if len(video_paths) != len(labels):
376
+ raise ValueError("video_paths and labels must have same length")
377
+ if subject_ids is not None and len(subject_ids) != len(video_paths):
378
+ raise ValueError("subject_ids must have same length as video_paths")
379
+
380
+ self.logger.info(f"Starting training on {len(video_paths)} videos.")
381
+
382
+ # 1. Extract features from all videos
383
+ all_fused_dfs = []
384
+ for idx, video_path in enumerate(tqdm(video_paths, desc="Extracting features")):
385
+ try:
386
+ # Extract facial features
387
+ facial_df = self.facial_extractor.process_video(video_path)
388
+
389
+ # Extract audio features
390
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
391
+ audio_path = extract_audio_from_video(video_path, tmp.name)
392
+ audio_df = self.audio_extractor.process_audio_segments(str(audio_path), segment_duration=2.0)
393
+
394
+ # Fuse
395
+ aligned_df = self.fusion.align_features(
396
+ facial_df, audio_df, video_fps=self.config.features.fps
397
+ )
398
+ fused_df = self.fusion.extract_fusion_features(aligned_df)
399
+ fused_df["label"] = labels[idx]
400
+ if subject_ids is not None:
401
+ fused_df["subject_id"] = subject_ids[idx]
402
+ all_fused_dfs.append(fused_df)
403
+ except Exception as e:
404
+ self.logger.warning(f"Failed to extract features from {video_path}: {e}")
405
+ continue
406
+
407
+ if not all_fused_dfs:
408
+ raise ValueError("No features extracted from any video.")
409
+
410
+ # Concatenate all features
411
+ full_df = pd.concat(all_fused_dfs, ignore_index=True)
412
+
413
+ # 2. Prepare feature matrix and labels
414
+ X, feature_names = self.classifier.prepare_features(full_df)
415
+ y = full_df["label"].values
416
+
417
+ if X.size == 0:
418
+ raise ValueError("No valid features after preparation.")
419
+
420
+ # 3. Train classifier
421
+ self.logger.info(f"Training classifier with {X.shape[0]} samples, {X.shape[1]} features.")
422
+ metrics = self.classifier.train(X, y, feature_names)
423
+
424
+ # 4. Update calibrator with subject baselines if subject_ids provided
425
+ if subject_ids is not None and self.classifier.is_fitted_:
426
+ self.logger.info("Updating subject baselines...")
427
+ for subject_id in set(subject_ids):
428
+ subject_mask = full_df["subject_id"] == subject_id
429
+ if subject_mask.any():
430
+ subject_features = full_df[subject_mask]
431
+ try:
432
+ self.calibrator.add_subject_profile(subject_id, subject_features)
433
+ except Exception as e:
434
+ self.logger.warning(f"Failed to update baseline for {subject_id}: {e}")
435
+
436
+ # 5. Save model if output_dir provided
437
+ if output_dir is not None:
438
+ model_path = Path(output_dir) / "deception_classifier.joblib"
439
+ self.classifier.save(str(model_path))
440
+ metrics["model_path"] = str(model_path)
441
+ self.logger.info(f"Model saved to {model_path}")
442
+
443
+ # Save training results
444
+ results_path = Path(output_dir) / "training_results.json"
445
+ save_results(metrics, str(results_path))
446
+
447
+ self.logger.info("Training completed.")
448
+ return metrics
449
+
450
+ def generate_report(self, analysis_result: dict, output_path: str) -> str:
451
+ """Generate a comprehensive text/JSON report from analysis results.
452
+
453
+ Creates a formatted report including:
454
+ - Overall deception assessment
455
+ - Key behavioral indicators
456
+ - Temporal analysis summary
457
+ - Feature importance highlights
458
+ - Suspicious intervals if detected
459
+
460
+ Args:
461
+ analysis_result: Result dict from analyze_video.
462
+ output_path: Path to save the report (JSON).
463
+
464
+ Returns:
465
+ Path to the saved report file.
466
+ """
467
+ from src.utils.io import save_results
468
+
469
+ # Build report structure
470
+ report = {
471
+ "summary": {
472
+ "prediction": analysis_result.get("prediction", "unknown"),
473
+ "deception_probability": analysis_result.get("deception_probability", 0.0),
474
+ "confidence": analysis_result.get("confidence", 0.0),
475
+ "calibrated": analysis_result.get("calibrated", False),
476
+ },
477
+ "temporal_analysis": analysis_result.get("temporal_analysis", {}),
478
+ "explanation": analysis_result.get("explanation", {}),
479
+ "metadata": analysis_result.get("metadata", {}),
480
+ }
481
+
482
+ # Add feature importance if available
483
+ if self.classifier.is_fitted_:
484
+ try:
485
+ importance_df = self.classifier.get_feature_importance()
486
+ top_features = importance_df.head(10).to_dict(orient="records")
487
+ report["feature_importance"] = top_features
488
+ except Exception as e:
489
+ self.logger.warning(f"Could not compute feature importance: {e}")
490
+ report["feature_importance"] = []
491
+
492
+ # Add suspicious intervals from temporal analysis
493
+ temporal = analysis_result.get("temporal_analysis", {})
494
+ if "suspicious_intervals" in temporal:
495
+ report["suspicious_intervals"] = temporal["suspicious_intervals"]
496
+ if "spikes" in temporal:
497
+ report["spikes"] = temporal["spikes"]
498
+
499
+ # Save report as JSON
500
+ save_results(report, output_path)
501
+ self.logger.info(f"Report saved to {output_path}")
502
+ return output_path
503
+
504
+ def load_model(self, model_path: str) -> None:
505
+ """Load a pre-trained classifier model.
506
+
507
+ Args:
508
+ model_path: Path to the saved model file.
509
+ """
510
+ self.classifier.load(model_path)
511
+ # Re-initialize explainer with the loaded classifier
512
+ self.explainer = DeceptionExplainer(self.classifier)
513
+ # Re-initialize calibrator with the loaded classifier
514
+ self.calibrator = SubjectCalibrator(self.classifier, self.config)
515
+ self.logger.info(f"Model loaded from {model_path}")
516
+
517
+ def save_model(self, model_path: str) -> None:
518
+ """Save the current classifier model.
519
+
520
+ Args:
521
+ model_path: Path where the model should be saved.
522
+ """
523
+ self.classifier.save(model_path)
524
+ self.logger.info(f"Model saved to {model_path}")
src/temporal/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ """Temporal analysis module for deception trajectory tracking."""
2
+
3
+ from .analyzer import TemporalAnalyzer
4
+ from .visualizer import TemporalVisualizer
5
+
6
+ __all__ = ['TemporalAnalyzer', 'TemporalVisualizer']
src/temporal/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (339 Bytes). View file
 
src/temporal/analyzer.py ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Temporal deception probability analysis utilities.
2
+
3
+ This module is model-agnostic and operates on probability trajectories to detect
4
+ spikes, contiguous suspicious regions, and aggregate temporal statistics.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Dict, List, Optional
10
+
11
+ import numpy as np
12
+
13
+ from src.utils.config import TemporalConfig
14
+
15
+
16
+ class TemporalAnalyzer:
17
+ """Frame-by-frame temporal deception analysis with spike detection."""
18
+
19
+ def __init__(self, config: Optional[TemporalConfig] = None):
20
+ """Initialize temporal analyzer.
21
+
22
+ Args:
23
+ config: Optional temporal configuration.
24
+ """
25
+
26
+ self.config = config or TemporalConfig()
27
+ self.window_size = int(getattr(self.config, "window_size", 30))
28
+ self.spike_threshold = float(getattr(self.config, "spike_threshold", 0.7))
29
+ self.min_suspicious_duration = int(getattr(self.config, "min_suspicious_duration", 10))
30
+ self.smoothing_alpha = float(getattr(self.config, "smoothing_alpha", 0.3))
31
+
32
+ def smooth_probabilities(self, probabilities: np.ndarray) -> np.ndarray:
33
+ """Apply exponential moving average smoothing.
34
+
35
+ Args:
36
+ probabilities: Raw deception probabilities.
37
+
38
+ Returns:
39
+ np.ndarray: Smoothed probabilities.
40
+ """
41
+
42
+ probs = np.asarray(probabilities, dtype=np.float64)
43
+ if probs.size == 0:
44
+ return np.array([], dtype=np.float64)
45
+ if probs.size == 1:
46
+ return np.clip(probs.copy(), 0.0, 1.0)
47
+
48
+ probs = np.nan_to_num(probs, nan=0.0, posinf=1.0, neginf=0.0)
49
+ probs = np.clip(probs, 0.0, 1.0)
50
+
51
+ alpha = float(np.clip(self.smoothing_alpha, 1e-3, 1.0))
52
+ smoothed = np.empty_like(probs)
53
+ smoothed[0] = probs[0]
54
+ for i in range(1, len(probs)):
55
+ # EMA is used because it denoises while preserving abrupt changes,
56
+ # which are behaviorally meaningful in deception dynamics.
57
+ smoothed[i] = alpha * probs[i] + (1.0 - alpha) * smoothed[i - 1]
58
+
59
+ return smoothed
60
+
61
+ def detect_spikes(self, probabilities: np.ndarray, timestamps: np.ndarray) -> List[Dict]:
62
+ """Detect high-confidence temporal spikes.
63
+
64
+ Args:
65
+ probabilities: Raw deception probability sequence.
66
+ timestamps: Timestamps aligned with probabilities.
67
+
68
+ Returns:
69
+ List[Dict]: Spike descriptors.
70
+ """
71
+
72
+ probs = np.asarray(probabilities, dtype=np.float64)
73
+ times = np.asarray(timestamps, dtype=np.float64)
74
+ n = min(len(probs), len(times))
75
+ if n == 0:
76
+ return []
77
+
78
+ probs = probs[:n]
79
+ times = times[:n]
80
+ smoothed = self.smooth_probabilities(probs)
81
+
82
+ if n == 1:
83
+ if smoothed[0] >= self.spike_threshold:
84
+ return [
85
+ {
86
+ "frame_idx": 0,
87
+ "timestamp": float(times[0]),
88
+ "probability": float(smoothed[0]),
89
+ "magnitude": float(smoothed[0] - self.spike_threshold),
90
+ "rate_of_change": 0.0,
91
+ }
92
+ ]
93
+ return []
94
+
95
+ deriv = np.diff(smoothed, prepend=smoothed[0])
96
+ deriv_std = float(np.std(deriv))
97
+ deriv_threshold = 2.0 * deriv_std
98
+
99
+ spikes: List[Dict] = []
100
+ for i in range(n):
101
+ prob_high = smoothed[i] >= self.spike_threshold
102
+ roc_high = abs(deriv[i]) >= deriv_threshold if deriv_std > 0 else False
103
+ if prob_high or roc_high:
104
+ spikes.append(
105
+ {
106
+ "frame_idx": int(i),
107
+ "timestamp": float(times[i]),
108
+ "probability": float(smoothed[i]),
109
+ "magnitude": float(max(0.0, smoothed[i] - self.spike_threshold)),
110
+ "rate_of_change": float(deriv[i]),
111
+ }
112
+ )
113
+
114
+ return spikes
115
+
116
+ def localize_suspicious_intervals(
117
+ self,
118
+ probabilities: np.ndarray,
119
+ timestamps: np.ndarray,
120
+ ) -> List[Dict]:
121
+ """Identify contiguous suspicious intervals.
122
+
123
+ Args:
124
+ probabilities: Raw deception probability sequence.
125
+ timestamps: Timestamps aligned with probabilities.
126
+
127
+ Returns:
128
+ List[Dict]: Interval dictionaries.
129
+ """
130
+
131
+ probs = np.asarray(probabilities, dtype=np.float64)
132
+ times = np.asarray(timestamps, dtype=np.float64)
133
+ n = min(len(probs), len(times))
134
+ if n == 0:
135
+ return []
136
+
137
+ probs = probs[:n]
138
+ times = times[:n]
139
+ smoothed = self.smooth_probabilities(probs)
140
+ mask = smoothed >= self.spike_threshold
141
+
142
+ if not np.any(mask):
143
+ return []
144
+
145
+ padded = np.pad(mask.astype(np.int8), (1, 1), mode="constant")
146
+ transitions = np.diff(padded)
147
+ starts = np.where(transitions == 1)[0]
148
+ ends = np.where(transitions == -1)[0] - 1
149
+
150
+ intervals: List[Dict] = []
151
+ for start, end in zip(starts, ends):
152
+ duration_frames = int(end - start + 1)
153
+ if duration_frames < self.min_suspicious_duration:
154
+ continue
155
+
156
+ segment = smoothed[start : end + 1]
157
+ avg_p = float(np.mean(segment))
158
+ max_p = float(np.max(segment))
159
+ if max_p < 0.8:
160
+ severity = "low"
161
+ elif max_p < 0.9:
162
+ severity = "medium"
163
+ else:
164
+ severity = "high"
165
+
166
+ start_t = float(times[start])
167
+ end_t = float(times[end])
168
+ duration_s = float(max(0.0, end_t - start_t))
169
+
170
+ intervals.append(
171
+ {
172
+ "start_frame": int(start),
173
+ "end_frame": int(end),
174
+ "start_time": start_t,
175
+ "end_time": end_t,
176
+ "duration_frames": duration_frames,
177
+ "duration_seconds": duration_s,
178
+ "avg_probability": avg_p,
179
+ "max_probability": max_p,
180
+ "severity": severity,
181
+ }
182
+ )
183
+
184
+ return intervals
185
+
186
+ def compute_temporal_features(
187
+ self,
188
+ probabilities: np.ndarray,
189
+ timestamps: np.ndarray,
190
+ ) -> Dict[str, float]:
191
+ """Compute aggregate temporal statistics.
192
+
193
+ Args:
194
+ probabilities: Raw deception probability sequence.
195
+ timestamps: Timestamps aligned with probabilities.
196
+
197
+ Returns:
198
+ Dict[str, float]: Aggregate temporal metrics.
199
+ """
200
+
201
+ probs = np.asarray(probabilities, dtype=np.float64)
202
+ times = np.asarray(timestamps, dtype=np.float64)
203
+ n = min(len(probs), len(times))
204
+ if n == 0:
205
+ return {
206
+ "mean_probability": 0.0,
207
+ "max_probability": 0.0,
208
+ "std_probability": 0.0,
209
+ "spike_count": 0.0,
210
+ "suspicious_time_ratio": 0.0,
211
+ "longest_suspicious_duration": 0.0,
212
+ "transition_count": 0.0,
213
+ "temporal_entropy": 0.0,
214
+ "trend_slope": 0.0,
215
+ }
216
+
217
+ probs = probs[:n]
218
+ times = times[:n]
219
+ smoothed = self.smooth_probabilities(probs)
220
+
221
+ spikes = self.detect_spikes(probs, times)
222
+ intervals = self.localize_suspicious_intervals(probs, times)
223
+
224
+ total_duration = float(max(1e-8, times[-1] - times[0])) if n > 1 else 1e-8
225
+ suspicious_duration = float(sum(item["duration_seconds"] for item in intervals))
226
+ longest_duration = float(max([item["duration_seconds"] for item in intervals], default=0.0))
227
+
228
+ binary = (smoothed >= 0.5).astype(np.int8)
229
+ transition_count = float(np.sum(np.abs(np.diff(binary)))) if n > 1 else 0.0
230
+
231
+ hist, _ = np.histogram(smoothed, bins=10, range=(0.0, 1.0), density=True)
232
+ hist = hist[hist > 0]
233
+ entropy = float(-np.sum(hist * np.log2(hist + 1e-12))) if hist.size else 0.0
234
+
235
+ if n > 1 and np.std(times) > 0:
236
+ slope = float(np.polyfit(times, smoothed, deg=1)[0])
237
+ else:
238
+ slope = 0.0
239
+
240
+ return {
241
+ "mean_probability": float(np.mean(smoothed)),
242
+ "max_probability": float(np.max(smoothed)),
243
+ "std_probability": float(np.std(smoothed)),
244
+ "spike_count": float(len(spikes)),
245
+ "suspicious_time_ratio": float(np.clip(suspicious_duration / total_duration, 0.0, 1.0)),
246
+ "longest_suspicious_duration": longest_duration,
247
+ "transition_count": transition_count,
248
+ "temporal_entropy": entropy,
249
+ "trend_slope": slope,
250
+ }
251
+
252
+ def analyze(self, probabilities: np.ndarray, timestamps: np.ndarray, fps: float = 30.0) -> Dict:
253
+ """Perform complete temporal analysis.
254
+
255
+ Args:
256
+ probabilities: Raw deception probability sequence.
257
+ timestamps: Optional timestamp sequence. If empty, generated using fps.
258
+ fps: Frame rate for synthetic timestamps.
259
+
260
+ Returns:
261
+ Dict: Full temporal analysis output.
262
+ """
263
+
264
+ probs = np.asarray(probabilities, dtype=np.float64)
265
+ times = np.asarray(timestamps, dtype=np.float64)
266
+
267
+ if probs.size == 0:
268
+ return {
269
+ "smoothed_probabilities": np.array([], dtype=np.float64),
270
+ "spikes": [],
271
+ "suspicious_intervals": [],
272
+ "temporal_features": self.compute_temporal_features(np.array([]), np.array([])),
273
+ "verdict": "truthful",
274
+ "confidence": 0.0,
275
+ }
276
+
277
+ if times.size == 0 or len(times) != len(probs):
278
+ safe_fps = fps if fps > 0 else 30.0
279
+ times = np.arange(len(probs), dtype=np.float64) / float(safe_fps)
280
+
281
+ smoothed = self.smooth_probabilities(probs)
282
+ spikes = self.detect_spikes(probs, times)
283
+ intervals = self.localize_suspicious_intervals(probs, times)
284
+ features = self.compute_temporal_features(probs, times)
285
+
286
+ mean_prob = float(features.get("mean_probability", 0.0))
287
+ verdict = "deceptive" if mean_prob > 0.5 else "truthful"
288
+ confidence = float(np.clip(abs(mean_prob - 0.5) * 2.0, 0.0, 1.0))
289
+
290
+ return {
291
+ "smoothed_probabilities": smoothed,
292
+ "spikes": spikes,
293
+ "suspicious_intervals": intervals,
294
+ "temporal_features": features,
295
+ "verdict": verdict,
296
+ "confidence": confidence,
297
+ }
298
+
src/temporal/visualizer.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Visualization helpers for temporal deception analysis outputs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Dict, List, Optional
6
+
7
+ import matplotlib.pyplot as plt
8
+ import numpy as np
9
+
10
+ from src.utils.config import TemporalConfig
11
+
12
+
13
+ class TemporalVisualizer:
14
+ """Publication-quality temporal analysis visualizations."""
15
+
16
+ def __init__(self):
17
+ """Initialize plotting style for publication-quality figures."""
18
+
19
+ plt.style.use("seaborn-v0_8-whitegrid")
20
+ plt.rcParams.update(
21
+ {
22
+ "figure.dpi": 300,
23
+ "savefig.dpi": 300,
24
+ "axes.titlesize": 14,
25
+ "axes.labelsize": 12,
26
+ "legend.fontsize": 10,
27
+ "xtick.labelsize": 10,
28
+ "ytick.labelsize": 10,
29
+ "font.size": 11,
30
+ }
31
+ )
32
+ self._temporal_cfg = TemporalConfig()
33
+
34
+ def plot_deception_timeline(
35
+ self,
36
+ probabilities: np.ndarray,
37
+ timestamps: np.ndarray,
38
+ smoothed: np.ndarray,
39
+ suspicious_intervals: List[Dict],
40
+ spikes: List[Dict],
41
+ save_path: Optional[str] = None,
42
+ ) -> plt.Figure:
43
+ """Create the main deception timeline plot."""
44
+
45
+ probs = np.asarray(probabilities, dtype=np.float64)
46
+ times = np.asarray(timestamps, dtype=np.float64)
47
+ smooth = np.asarray(smoothed, dtype=np.float64)
48
+
49
+ n = min(len(probs), len(times), len(smooth))
50
+ probs = probs[:n]
51
+ times = times[:n]
52
+ smooth = smooth[:n]
53
+
54
+ fig, ax = plt.subplots(figsize=(12, 5))
55
+
56
+ if n > 0:
57
+ ax.plot(times, probs, color="#93c5fd", linewidth=1.2, alpha=0.8, label="Raw probability")
58
+ ax.plot(times, smooth, color="#1e3a8a", linewidth=2.0, label="Smoothed probability")
59
+
60
+ threshold = float(getattr(self._temporal_cfg, "spike_threshold", 0.7))
61
+ ax.axhline(threshold, color="#dc2626", linestyle="--", linewidth=1.4, label="Spike threshold")
62
+ ax.axhline(0.5, color="#6b7280", linestyle="--", linewidth=1.2, label="Decision boundary")
63
+
64
+ for i, interval in enumerate(suspicious_intervals):
65
+ start_t = float(interval.get("start_time", 0.0))
66
+ end_t = float(interval.get("end_time", start_t))
67
+ ax.axvspan(
68
+ start_t,
69
+ end_t,
70
+ color="#fecaca",
71
+ alpha=0.35,
72
+ label="Suspicious interval" if i == 0 else None,
73
+ )
74
+
75
+ if spikes:
76
+ spike_x = [float(item.get("timestamp", 0.0)) for item in spikes]
77
+ spike_y = [float(item.get("probability", 0.0)) for item in spikes]
78
+ ax.scatter(
79
+ spike_x,
80
+ spike_y,
81
+ marker="^",
82
+ color="#ef4444",
83
+ edgecolors="black",
84
+ linewidths=0.3,
85
+ s=40,
86
+ zorder=5,
87
+ label="Spikes",
88
+ )
89
+
90
+ ax.set_title("Temporal Deception Analysis")
91
+ ax.set_xlabel("Time (seconds)")
92
+ ax.set_ylabel("Deception Probability")
93
+ ax.set_ylim(0.0, 1.02)
94
+ ax.legend(loc="best")
95
+ fig.tight_layout()
96
+
97
+ if save_path:
98
+ fig.savefig(save_path, bbox_inches="tight")
99
+
100
+ return fig
101
+
102
+ def plot_suspicious_regions(
103
+ self,
104
+ suspicious_intervals: List[Dict],
105
+ total_duration: float,
106
+ save_path: Optional[str] = None,
107
+ ) -> plt.Figure:
108
+ """Create a Gantt-style suspicious interval chart."""
109
+
110
+ fig, ax = plt.subplots(figsize=(11, 4))
111
+
112
+ severity_color = {
113
+ "low": "#facc15",
114
+ "medium": "#fb923c",
115
+ "high": "#ef4444",
116
+ }
117
+
118
+ if not suspicious_intervals:
119
+ ax.barh([0], [max(total_duration, 1.0)], left=[0.0], color="#e5e7eb", alpha=0.4)
120
+ ax.text(0.5 * max(total_duration, 1.0), 0, "No suspicious intervals", ha="center", va="center")
121
+ ax.set_yticks([0])
122
+ ax.set_yticklabels(["N/A"])
123
+ else:
124
+ for idx, interval in enumerate(suspicious_intervals):
125
+ start = float(interval.get("start_time", 0.0))
126
+ end = float(interval.get("end_time", start))
127
+ width = max(0.0, end - start)
128
+ severity = str(interval.get("severity", "low")).lower()
129
+ color = severity_color.get(severity, "#facc15")
130
+ ax.barh(idx, width, left=start, color=color, edgecolor="black", alpha=0.85)
131
+
132
+ ax.set_yticks(np.arange(len(suspicious_intervals)))
133
+ ax.set_yticklabels([f"Interval {i + 1}" for i in range(len(suspicious_intervals))])
134
+
135
+ ax.set_xlim(0.0, max(float(total_duration), 1.0))
136
+ ax.set_xlabel("Time (seconds)")
137
+ ax.set_ylabel("Suspicious Intervals")
138
+ ax.set_title("Suspicious Regions Timeline")
139
+ fig.tight_layout()
140
+
141
+ if save_path:
142
+ fig.savefig(save_path, bbox_inches="tight")
143
+
144
+ return fig
145
+
146
+ def plot_temporal_summary(
147
+ self,
148
+ temporal_features: Dict[str, float],
149
+ save_path: Optional[str] = None,
150
+ ) -> plt.Figure:
151
+ """Create a multi-panel summary dashboard of temporal statistics."""
152
+
153
+ fig, axes = plt.subplots(1, 3, figsize=(15, 4.5))
154
+
155
+ # Panel 1: synthetic distribution from mean/std when raw sequence is unavailable.
156
+ mean_p = float(temporal_features.get("mean_probability", 0.0))
157
+ std_p = float(max(temporal_features.get("std_probability", 0.0), 1e-3))
158
+ synthetic = np.clip(np.random.normal(loc=mean_p, scale=std_p, size=800), 0.0, 1.0)
159
+ axes[0].hist(synthetic, bins=20, color="#60a5fa", edgecolor="black", alpha=0.85)
160
+ axes[0].axvline(mean_p, color="#1d4ed8", linestyle="--", linewidth=2)
161
+ axes[0].set_title("Probability Distribution")
162
+ axes[0].set_xlabel("Deception Probability")
163
+ axes[0].set_ylabel("Count")
164
+
165
+ # Panel 2: key metrics.
166
+ key_names = [
167
+ "mean_probability",
168
+ "max_probability",
169
+ "spike_count",
170
+ "suspicious_time_ratio",
171
+ "longest_suspicious_duration",
172
+ ]
173
+ key_vals = [float(temporal_features.get(name, 0.0)) for name in key_names]
174
+ axes[1].barh(np.arange(len(key_names)), key_vals, color="#34d399", edgecolor="black", alpha=0.85)
175
+ axes[1].set_yticks(np.arange(len(key_names)))
176
+ axes[1].set_yticklabels([name.replace("_", " ") for name in key_names])
177
+ axes[1].set_title("Key Temporal Metrics")
178
+ axes[1].set_xlabel("Value")
179
+
180
+ # Panel 3: severity pie chart proxy from max probability.
181
+ max_p = float(temporal_features.get("max_probability", 0.0))
182
+ if max_p < 0.7:
183
+ sev = [1.0, 0.0, 0.0]
184
+ elif max_p < 0.85:
185
+ sev = [0.4, 0.6, 0.0]
186
+ else:
187
+ sev = [0.2, 0.3, 0.5]
188
+ axes[2].pie(
189
+ sev,
190
+ labels=["Low", "Medium", "High"],
191
+ colors=["#facc15", "#fb923c", "#ef4444"],
192
+ autopct="%1.0f%%",
193
+ startangle=90,
194
+ )
195
+ axes[2].set_title("Severity Breakdown")
196
+
197
+ fig.tight_layout()
198
+ if save_path:
199
+ fig.savefig(save_path, bbox_inches="tight")
200
+ return fig
201
+
src/utils/__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Utility modules for configuration and I/O operations."""
2
+
3
+ from .config import AppConfig, ProjectConfig, FeatureConfig, AudioConfig, ModelConfig, TemporalConfig
4
+ from .io import (
5
+ load_video as VideoLoader,
6
+ extract_audio_from_video as AudioExtractor,
7
+ save_model as ModelSerializer,
8
+ save_results as ResultsSaver,
9
+ )
10
+
11
+ __all__ = [
12
+ 'AppConfig', 'ProjectConfig', 'FeatureConfig', 'AudioConfig',
13
+ 'ModelConfig', 'TemporalConfig',
14
+ 'VideoLoader', 'AudioExtractor', 'ModelSerializer', 'ResultsSaver'
15
+ ]
src/utils/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (609 Bytes). View file
 
src/utils/__pycache__/config.cpython-314.pyc ADDED
Binary file (8.43 kB). View file
 
src/utils/__pycache__/io.cpython-314.pyc ADDED
Binary file (6.94 kB). View file