Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +22 -0
- Dockerfile +36 -0
- README.md +360 -10
- api.py +129 -0
- app.py +596 -0
- colab_deception_analysis.ipynb +203 -0
- create_pretrained_model.py +276 -0
- inference.py +323 -0
- models/.gitkeep +1 -0
- requirements.txt +16 -0
- src/__init__.py +4 -0
- src/__pycache__/__init__.cpython-314.pyc +0 -0
- src/__pycache__/pipeline.cpython-314.pyc +0 -0
- src/audio/__init__.py +6 -0
- src/audio/__pycache__/__init__.cpython-314.pyc +0 -0
- src/audio/extractor.py +248 -0
- src/audio/fusion.py +108 -0
- src/evaluation/__init__.py +13 -0
- src/evaluation/__pycache__/__init__.cpython-314.pyc +0 -0
- src/evaluation/__pycache__/ablation.cpython-314.pyc +0 -0
- src/evaluation/__pycache__/cross_validation.cpython-314.pyc +0 -0
- src/evaluation/__pycache__/metrics.cpython-314.pyc +0 -0
- src/evaluation/__pycache__/visualizer.cpython-314.pyc +0 -0
- src/evaluation/ablation.py +294 -0
- src/evaluation/cross_validation.py +303 -0
- src/evaluation/metrics.py +356 -0
- src/evaluation/visualizer.py +396 -0
- src/explainability/__init__.py +6 -0
- src/explainability/__pycache__/__init__.cpython-314.pyc +0 -0
- src/explainability/__pycache__/explainer.cpython-314.pyc +0 -0
- src/explainability/__pycache__/visualizer.cpython-314.pyc +0 -0
- src/explainability/explainer.py +513 -0
- src/explainability/visualizer.py +743 -0
- src/facial/__init__.py +6 -0
- src/facial/__pycache__/__init__.cpython-314.pyc +0 -0
- src/facial/extractor.py +341 -0
- src/facial/visualizer.py +135 -0
- src/model/__init__.py +6 -0
- src/model/__pycache__/__init__.cpython-314.pyc +0 -0
- src/model/calibration.py +265 -0
- src/model/classifier.py +330 -0
- src/pipeline.py +524 -0
- src/temporal/__init__.py +6 -0
- src/temporal/__pycache__/__init__.cpython-314.pyc +0 -0
- src/temporal/analyzer.py +298 -0
- src/temporal/visualizer.py +201 -0
- src/utils/__init__.py +15 -0
- src/utils/__pycache__/__init__.cpython-314.pyc +0 -0
- src/utils/__pycache__/config.cpython-314.pyc +0 -0
- src/utils/__pycache__/io.cpython-314.pyc +0 -0
.dockerignore
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
*$py.class
|
| 4 |
+
.env
|
| 5 |
+
.venv
|
| 6 |
+
env/
|
| 7 |
+
venv/
|
| 8 |
+
ENV/
|
| 9 |
+
venv.bak/
|
| 10 |
+
env.bak/
|
| 11 |
+
.git/
|
| 12 |
+
.github/
|
| 13 |
+
.vscode/
|
| 14 |
+
data/
|
| 15 |
+
results/
|
| 16 |
+
results_dir/
|
| 17 |
+
notebooks/
|
| 18 |
+
docs/
|
| 19 |
+
*.mp4
|
| 20 |
+
*.wav
|
| 21 |
+
*.csv
|
| 22 |
+
!pretraining_dataset.csv
|
Dockerfile
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Use an official Python runtime as a parent image
|
| 2 |
+
FROM python:3.10-slim
|
| 3 |
+
|
| 4 |
+
# Set environment variables
|
| 5 |
+
ENV PYTHONDONTWRITEBYTECODE 1
|
| 6 |
+
ENV PYTHONUNBUFFERED 1
|
| 7 |
+
|
| 8 |
+
# Install system dependencies
|
| 9 |
+
RUN apt-get update && apt-get install -y \
|
| 10 |
+
ffmpeg \
|
| 11 |
+
libgl1-mesa-glx \
|
| 12 |
+
libglib2.0-0 \
|
| 13 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 14 |
+
|
| 15 |
+
# Set the working directory in the container
|
| 16 |
+
WORKDIR /app
|
| 17 |
+
|
| 18 |
+
# Copy the requirements file into the container
|
| 19 |
+
COPY requirements.txt .
|
| 20 |
+
|
| 21 |
+
# Install dependencies
|
| 22 |
+
# We use opencv-python-headless for the server environment
|
| 23 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 24 |
+
RUN pip install --no-cache-dir opencv-python-headless
|
| 25 |
+
|
| 26 |
+
# Copy the rest of the application code
|
| 27 |
+
COPY . .
|
| 28 |
+
|
| 29 |
+
# Ensure models directory exists
|
| 30 |
+
RUN mkdir -p models
|
| 31 |
+
|
| 32 |
+
# Expose the port the app runs on (Hugging Face default)
|
| 33 |
+
EXPOSE 7860
|
| 34 |
+
|
| 35 |
+
# Command to run the application
|
| 36 |
+
CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,10 +1,360 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🕵️♂️ Explainable Multimodal Deception Analysis System
|
| 2 |
+
|
| 3 |
+
A comprehensive system for detecting deception in video interviews using facial micro-expressions, audio speech patterns, temporal dynamics, and SHAP-based explainability. Works with or without real datasets using synthetic data generation.
|
| 4 |
+
|
| 5 |
+
## 🚀 Quick Start (Google Colab - No Local Installation)
|
| 6 |
+
|
| 7 |
+
**Run everything in the cloud with one click:**
|
| 8 |
+
|
| 9 |
+
[](https://colab.research.google.com/github/apexprit/deception-analysis/blob/main/colab_deception_analysis.ipynb)
|
| 10 |
+
|
| 11 |
+
### Steps to Run in Google Colab:
|
| 12 |
+
|
| 13 |
+
1. **Open the Colab notebook** using the link above
|
| 14 |
+
2. **Click "Runtime" → "Run all"** (Ctrl+F9)
|
| 15 |
+
3. **Wait for installation** (~2-3 minutes)
|
| 16 |
+
4. **Use the Gradio interface** that appears at the bottom
|
| 17 |
+
|
| 18 |
+
> [!TIP]
|
| 19 |
+
> **Working locally?** Make sure to `git push` your latest changes to GitHub before running in Colab, as the notebook clones the repository directly from the `main` branch.
|
| 20 |
+
|
| 21 |
+
The notebook will:
|
| 22 |
+
- ✅ Install all dependencies automatically
|
| 23 |
+
- ✅ Generate realistic synthetic deception data
|
| 24 |
+
- ✅ Train a deception detection model (~85% accuracy)
|
| 25 |
+
- ✅ Launch interactive web interface
|
| 26 |
+
- ✅ Provide sample analysis with visualizations
|
| 27 |
+
|
| 28 |
+
## 📋 Features
|
| 29 |
+
|
| 30 |
+
### 🔍 **Multimodal Analysis**
|
| 31 |
+
- **Facial Features**: 16 micro-expressions (eye openness, mouth movement, brow tension, head dynamics)
|
| 32 |
+
- **Audio Features**: 30+ speech patterns (pitch variation, pauses, MFCC coefficients, spectral features)
|
| 33 |
+
- **Cross-modal Features**: Audio-visual sync, pause-gaze coupling, stress composite
|
| 34 |
+
- **Temporal Analysis**: Frame-by-frame probability tracking, spike detection, suspicious intervals
|
| 35 |
+
|
| 36 |
+
### 🧠 **Explainability**
|
| 37 |
+
- **SHAP-based feature importance** with behavioral interpretations
|
| 38 |
+
- **Feature group analysis** (facial vs. audio vs. cross-modal)
|
| 39 |
+
- **Human-readable explanations** of deception indicators
|
| 40 |
+
- **Publication-quality visualizations**
|
| 41 |
+
|
| 42 |
+
### 🎯 **Subject Adaptation**
|
| 43 |
+
- **Personal calibration** to reduce inter-subject variability
|
| 44 |
+
- **Baseline establishment** for individual behavioral patterns
|
| 45 |
+
- **Adaptive thresholding** based on subject history
|
| 46 |
+
|
| 47 |
+
### 📊 **Evaluation & Validation**
|
| 48 |
+
- **Comprehensive metrics**: Accuracy, Precision, Recall, F1, AUC-ROC
|
| 49 |
+
- **Cross-validation**: Stratified, subject-aware, leave-one-subject-out
|
| 50 |
+
- **Ablation studies**: Feature group importance analysis
|
| 51 |
+
- **Bootstrap confidence intervals** for robust metrics
|
| 52 |
+
|
| 53 |
+
## 🏗️ Architecture
|
| 54 |
+
|
| 55 |
+
```
|
| 56 |
+
Video Input
|
| 57 |
+
│
|
| 58 |
+
├── Facial Feature Extraction (MediaPipe Face Mesh)
|
| 59 |
+
│ └── 16 micro-expression features
|
| 60 |
+
│
|
| 61 |
+
├── Audio Feature Extraction (librosa)
|
| 62 |
+
│ └── 30+ speech pattern features
|
| 63 |
+
│
|
| 64 |
+
└── Multimodal Fusion
|
| 65 |
+
└── Temporal alignment + cross-modal features
|
| 66 |
+
│
|
| 67 |
+
├── Deception Classifier (Gradient Boosting)
|
| 68 |
+
│ └── Probability prediction (0-1)
|
| 69 |
+
│
|
| 70 |
+
├── Subject Calibration
|
| 71 |
+
│ └── Personalized baseline adjustment
|
| 72 |
+
│
|
| 73 |
+
├── Temporal Analysis
|
| 74 |
+
│ └── Spike detection, interval localization
|
| 75 |
+
│
|
| 76 |
+
└── Explainability Engine (SHAP)
|
| 77 |
+
└── Feature importance + behavioral interpretation
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
## 📁 Project Structure
|
| 81 |
+
|
| 82 |
+
```
|
| 83 |
+
deception-analysis/
|
| 84 |
+
├── app.py # Gradio web interface
|
| 85 |
+
├── train.py # CLI training script
|
| 86 |
+
├── inference.py # CLI inference script
|
| 87 |
+
├── create_pretrained_model.py # Generate pre-trained model
|
| 88 |
+
├── requirements.txt # Dependencies
|
| 89 |
+
├── colab_deception_analysis.ipynb # Google Colab notebook
|
| 90 |
+
├── notebooks/
|
| 91 |
+
│ ├── demo.ipynb # Full workflow demonstration
|
| 92 |
+
│ └── demo_with_synthetic_data.ipynb # Synthetic data demo
|
| 93 |
+
├── data/ # Generated synthetic data
|
| 94 |
+
├── models/ # Pre-trained models
|
| 95 |
+
└── src/ # Core modules
|
| 96 |
+
├── pipeline.py # Main orchestration
|
| 97 |
+
├── facial/ # Facial analysis (extractor, visualizer)
|
| 98 |
+
├── audio/ # Audio analysis (extractor, fusion)
|
| 99 |
+
├── temporal/ # Temporal analysis (analyzer, visualizer)
|
| 100 |
+
├── model/ # Classifier + calibration
|
| 101 |
+
├── explainability/ # SHAP explanations (explainer, visualizer)
|
| 102 |
+
├── evaluation/ # Metrics, CV, ablation, plots
|
| 103 |
+
└── utils/ # Config, I/O, synthetic data, downloader
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
## 🛠️ Local Installation (Optional)
|
| 107 |
+
|
| 108 |
+
If you want to run locally:
|
| 109 |
+
|
| 110 |
+
```bash
|
| 111 |
+
# Clone repository
|
| 112 |
+
git clone https://github.com/apexprit/deception-analysis.git
|
| 113 |
+
cd deception-analysis
|
| 114 |
+
|
| 115 |
+
# Install dependencies
|
| 116 |
+
pip install -r requirements.txt
|
| 117 |
+
|
| 118 |
+
# Generate synthetic data and pre-trained model
|
| 119 |
+
python create_pretrained_model.py
|
| 120 |
+
|
| 121 |
+
# Train on synthetic data
|
| 122 |
+
python train.py --data_dir ./data/synthetic --labels_file ./data/synthetic/synthetic_labels.csv
|
| 123 |
+
|
| 124 |
+
# Run inference (demo mode)
|
| 125 |
+
python inference.py --model ./models/deception_model.pkl --video_dir ./data/sample/
|
| 126 |
+
|
| 127 |
+
# Launch web interface
|
| 128 |
+
python app.py --port 7860
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
## 📊 Usage Examples
|
| 132 |
+
|
| 133 |
+
### 1. **Basic Analysis**
|
| 134 |
+
```python
|
| 135 |
+
from src.pipeline import DeceptionPipeline
|
| 136 |
+
from src.utils.config import AppConfig
|
| 137 |
+
|
| 138 |
+
# Initialize pipeline
|
| 139 |
+
config = AppConfig()
|
| 140 |
+
pipeline = DeceptionPipeline(config)
|
| 141 |
+
|
| 142 |
+
# Analyze a video
|
| 143 |
+
result = pipeline.analyze_video(
|
| 144 |
+
video_path="interview.mp4",
|
| 145 |
+
subject_id="S001",
|
| 146 |
+
generate_visualizations=True,
|
| 147 |
+
output_dir="./results"
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
print(f"Deception Probability: {result['deception_probability']:.3f}")
|
| 151 |
+
print(f"Prediction: {result['prediction']}")
|
| 152 |
+
print(f"Confidence: {result['confidence']:.3f}")
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
### 2. **Batch Processing**
|
| 156 |
+
```python
|
| 157 |
+
# Analyze multiple videos
|
| 158 |
+
results = pipeline.analyze_batch(
|
| 159 |
+
video_paths=["video1.mp4", "video2.mp4", "video3.mp4"],
|
| 160 |
+
subject_ids=["S001", "S002", "S003"],
|
| 161 |
+
generate_visualizations=False,
|
| 162 |
+
output_dir="./batch_results"
|
| 163 |
+
)
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
### 3. **Model Training**
|
| 167 |
+
```python
|
| 168 |
+
# Train on your dataset
|
| 169 |
+
train_result = pipeline.train(
|
| 170 |
+
video_paths=["train1.mp4", "train2.mp4", "train3.mp4"],
|
| 171 |
+
labels=[0, 1, 0], # 0=truthful, 1=deceptive
|
| 172 |
+
subject_ids=["S001", "S002", "S003"],
|
| 173 |
+
output_dir="./models"
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
print(f"Training Accuracy: {train_result['accuracy']:.3f}")
|
| 177 |
+
print(f"Training F1: {train_result['f1']:.3f}")
|
| 178 |
+
```
|
| 179 |
+
|
| 180 |
+
### 4. **Synthetic Data Generation**
|
| 181 |
+
```python
|
| 182 |
+
from src.utils.synthetic_data import SyntheticDataGenerator
|
| 183 |
+
|
| 184 |
+
# Generate synthetic dataset
|
| 185 |
+
generator = SyntheticDataGenerator()
|
| 186 |
+
features, labels, subject_ids = generator.generate_dataset(
|
| 187 |
+
n_truthful=500,
|
| 188 |
+
n_deceptive=500
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
# Save for training
|
| 192 |
+
generator.save_synthetic_dataset(
|
| 193 |
+
output_dir="./data/synthetic",
|
| 194 |
+
n_truthful=500,
|
| 195 |
+
n_deceptive=500,
|
| 196 |
+
dataset_name="my_dataset"
|
| 197 |
+
)
|
| 198 |
+
```
|
| 199 |
+
|
| 200 |
+
## 🧪 Evaluation
|
| 201 |
+
|
| 202 |
+
```python
|
| 203 |
+
from src.evaluation.metrics import DeceptionMetrics
|
| 204 |
+
from src.evaluation.cross_validation import CrossValidator
|
| 205 |
+
from src.evaluation.ablation import AblationStudy
|
| 206 |
+
|
| 207 |
+
# Compute metrics
|
| 208 |
+
metrics = DeceptionMetrics()
|
| 209 |
+
all_metrics = metrics.compute_all_metrics(y_true, y_pred, y_prob)
|
| 210 |
+
|
| 211 |
+
# Cross-validation
|
| 212 |
+
cv = CrossValidator()
|
| 213 |
+
cv_results = cv.run_cross_validation(
|
| 214 |
+
classifier=DeceptionClassifier,
|
| 215 |
+
X=features,
|
| 216 |
+
y=labels,
|
| 217 |
+
n_splits=5,
|
| 218 |
+
method='stratified'
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
# Ablation study
|
| 222 |
+
ablation = AblationStudy(feature_names, feature_groups)
|
| 223 |
+
ablation_results = ablation.run_ablation(
|
| 224 |
+
classifier_class=DeceptionClassifier,
|
| 225 |
+
X=features,
|
| 226 |
+
y=labels,
|
| 227 |
+
feature_names=feature_names,
|
| 228 |
+
feature_groups=feature_groups,
|
| 229 |
+
n_splits=5
|
| 230 |
+
)
|
| 231 |
+
```
|
| 232 |
+
|
| 233 |
+
## 🔧 Configuration
|
| 234 |
+
|
| 235 |
+
The system is highly configurable via `src/utils/config.py`:
|
| 236 |
+
|
| 237 |
+
```python
|
| 238 |
+
from src.utils.config import AppConfig
|
| 239 |
+
|
| 240 |
+
config = AppConfig(
|
| 241 |
+
project=ProjectConfig(
|
| 242 |
+
data_dir="./data",
|
| 243 |
+
model_dir="./models",
|
| 244 |
+
results_dir="./results",
|
| 245 |
+
random_seed=42
|
| 246 |
+
),
|
| 247 |
+
feature=FeatureConfig(
|
| 248 |
+
fps=15,
|
| 249 |
+
face_detection_confidence=0.5,
|
| 250 |
+
landmark_refinement=True
|
| 251 |
+
),
|
| 252 |
+
audio=AudioConfig(
|
| 253 |
+
sample_rate=16000,
|
| 254 |
+
n_fft=2048,
|
| 255 |
+
hop_length=512,
|
| 256 |
+
n_mfcc=13
|
| 257 |
+
),
|
| 258 |
+
model=ModelConfig(
|
| 259 |
+
model_type="gradient_boosting",
|
| 260 |
+
n_estimators=100,
|
| 261 |
+
learning_rate=0.1,
|
| 262 |
+
max_depth=3
|
| 263 |
+
),
|
| 264 |
+
temporal=TemporalConfig(
|
| 265 |
+
spike_threshold=0.7,
|
| 266 |
+
min_spike_duration=0.5,
|
| 267 |
+
smoothing_window=5
|
| 268 |
+
)
|
| 269 |
+
)
|
| 270 |
+
```
|
| 271 |
+
|
| 272 |
+
## 📈 Performance
|
| 273 |
+
|
| 274 |
+
With synthetic data (1000 samples, 80/20 train/test split):
|
| 275 |
+
- **Accuracy**: ~85%
|
| 276 |
+
- **F1-Score**: ~0.83
|
| 277 |
+
- **AUC-ROC**: ~0.91
|
| 278 |
+
- **Precision**: ~0.82
|
| 279 |
+
- **Recall**: ~0.84
|
| 280 |
+
|
| 281 |
+
**Feature Importance** (top 5):
|
| 282 |
+
1. `pause_count` (audio) - More pauses indicate deception
|
| 283 |
+
2. `eye_left_openness` (facial) - Less eye openness indicates deception
|
| 284 |
+
3. `pitch_variation_coefficient` (audio) - Higher variation indicates deception
|
| 285 |
+
4. `brow_tension` (facial) - Higher tension indicates deception
|
| 286 |
+
5. `audio_visual_sync` (cross-modal) - Less sync indicates deception
|
| 287 |
+
|
| 288 |
+
## 🚀 Deployment Options
|
| 289 |
+
|
| 290 |
+
### 1. **Google Colab** (Recommended for no local setup)
|
| 291 |
+
- Open `colab_deception_analysis.ipynb` in Colab
|
| 292 |
+
- Run all cells
|
| 293 |
+
- Use the Gradio interface
|
| 294 |
+
|
| 295 |
+
### 2. **Local Python Environment**
|
| 296 |
+
```bash
|
| 297 |
+
pip install -r requirements.txt
|
| 298 |
+
python app.py
|
| 299 |
+
```
|
| 300 |
+
|
| 301 |
+
### 3. **Docker Container**
|
| 302 |
+
```dockerfile
|
| 303 |
+
FROM python:3.9
|
| 304 |
+
COPY . /app
|
| 305 |
+
WORKDIR /app
|
| 306 |
+
RUN pip install -r requirements.txt
|
| 307 |
+
CMD ["python", "app.py", "--host", "0.0.0.0", "--port", "7860"]
|
| 308 |
+
```
|
| 309 |
+
|
| 310 |
+
### 4. **Cloud Services**
|
| 311 |
+
- **Hugging Face Spaces**: Deploy Gradio app
|
| 312 |
+
- **Vercel/Heroku**: Deploy as web service
|
| 313 |
+
- **AWS SageMaker**: Deploy as ML endpoint
|
| 314 |
+
- **Google Cloud Run**: Containerized deployment
|
| 315 |
+
|
| 316 |
+
## 📚 Research Basis
|
| 317 |
+
|
| 318 |
+
The system implements features based on established deception detection research:
|
| 319 |
+
|
| 320 |
+
1. **Facial Indicators** (Ekman, 2009):
|
| 321 |
+
- Micro-expressions (brief facial expressions)
|
| 322 |
+
- Eye gaze avoidance
|
| 323 |
+
- Facial asymmetry
|
| 324 |
+
|
| 325 |
+
2. **Vocal Indicators** (DePaulo et al., 2003):
|
| 326 |
+
- Higher pitch variation
|
| 327 |
+
- More speech errors
|
| 328 |
+
- Longer response latency
|
| 329 |
+
|
| 330 |
+
3. **Multimodal Integration** (Pérez-Rosas & Mihalcea, 2014):
|
| 331 |
+
- Audio-visual feature fusion
|
| 332 |
+
- Temporal alignment
|
| 333 |
+
- Cross-modal correlation analysis
|
| 334 |
+
|
| 335 |
+
## ⚠️ Ethical Considerations
|
| 336 |
+
|
| 337 |
+
1. **Privacy**: Video/audio data should be anonymized and stored securely
|
| 338 |
+
2. **Consent**: Subjects must provide informed consent for analysis
|
| 339 |
+
3. **Bias**: Models may have demographic biases - regular auditing required
|
| 340 |
+
4. **Interpretation**: Results are probabilistic indicators, not definitive truth
|
| 341 |
+
5. **Context**: Consider cultural and situational factors in interpretation
|
| 342 |
+
|
| 343 |
+
## 🤝 Contributing
|
| 344 |
+
|
| 345 |
+
1. Fork the repository
|
| 346 |
+
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
| 347 |
+
3. Commit changes (`git commit -m 'Add amazing feature'`)
|
| 348 |
+
4. Push to branch (`git push origin feature/amazing-feature`)
|
| 349 |
+
5. Open a Pull Request
|
| 350 |
+
|
| 351 |
+
## 📄 License
|
| 352 |
+
|
| 353 |
+
MIT License - see LICENSE file for details
|
| 354 |
+
|
| 355 |
+
## 📧 Contact
|
| 356 |
+
|
| 357 |
+
For questions or support, please open an issue on GitHub.
|
| 358 |
+
|
| 359 |
+
---
|
| 360 |
+
|
api.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import tempfile
|
| 3 |
+
import logging
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
from fastapi import FastAPI, UploadFile, File, Form, HTTPException
|
| 7 |
+
from fastapi.responses import JSONResponse
|
| 8 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 9 |
+
|
| 10 |
+
from src.pipeline import DeceptionPipeline
|
| 11 |
+
from src.utils.config import get_default_config
|
| 12 |
+
|
| 13 |
+
logging.basicConfig(level=logging.INFO)
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
app = FastAPI(
|
| 17 |
+
title="Deception Analysis API",
|
| 18 |
+
description="API for Explainable Multimodal Deception Analysis for Mobile Apps"
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
# Enable CORS so the Flutter app can connect easily
|
| 22 |
+
app.add_middleware(
|
| 23 |
+
CORSMiddleware,
|
| 24 |
+
allow_origins=["*"],
|
| 25 |
+
allow_credentials=True,
|
| 26 |
+
allow_methods=["*"],
|
| 27 |
+
allow_headers=["*"],
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
# Global pipeline instance
|
| 31 |
+
pipeline = None
|
| 32 |
+
|
| 33 |
+
@app.on_event("startup")
|
| 34 |
+
async def startup_event():
|
| 35 |
+
global pipeline
|
| 36 |
+
try:
|
| 37 |
+
logger.info("Initializing Deception ML Pipeline...")
|
| 38 |
+
config = get_default_config()
|
| 39 |
+
pipeline = DeceptionPipeline(config)
|
| 40 |
+
|
| 41 |
+
# Load the pre-trained model if it exists
|
| 42 |
+
model_path = os.path.join("models", "deception_model.pkl")
|
| 43 |
+
if os.path.exists(model_path):
|
| 44 |
+
pipeline.load_model(model_path)
|
| 45 |
+
logger.info("Pre-trained model loaded successfully.")
|
| 46 |
+
else:
|
| 47 |
+
logger.warning(f"No pre-trained model found at {model_path}. You must generate it first!")
|
| 48 |
+
|
| 49 |
+
logger.info("Pipeline initialized successfully.")
|
| 50 |
+
except Exception as e:
|
| 51 |
+
logger.error(f"Failed to initialize pipeline: {e}")
|
| 52 |
+
|
| 53 |
+
@app.post("/analyze")
|
| 54 |
+
async def analyze_video_endpoint(
|
| 55 |
+
file: UploadFile = File(...),
|
| 56 |
+
subject_id: Optional[str] = Form(None)
|
| 57 |
+
):
|
| 58 |
+
if not pipeline:
|
| 59 |
+
raise HTTPException(status_code=500, detail="ML Pipeline not initialized yet")
|
| 60 |
+
|
| 61 |
+
# Create temporary directory for the uploaded video
|
| 62 |
+
temp_dir = tempfile.mkdtemp(prefix="deception_api_")
|
| 63 |
+
video_path = os.path.join(temp_dir, file.filename)
|
| 64 |
+
|
| 65 |
+
try:
|
| 66 |
+
# Save uploaded video to disk
|
| 67 |
+
with open(video_path, "wb") as f:
|
| 68 |
+
content = await file.read()
|
| 69 |
+
f.write(content)
|
| 70 |
+
|
| 71 |
+
logger.info(f"Analyzing video: {file.filename} (Subject: {subject_id})")
|
| 72 |
+
|
| 73 |
+
# Run pipeline
|
| 74 |
+
result = pipeline.analyze_video(
|
| 75 |
+
video_path=video_path,
|
| 76 |
+
subject_id=subject_id,
|
| 77 |
+
generate_visualizations=False, # We don't need Matplotlib images, just JSON data
|
| 78 |
+
output_dir=temp_dir
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
if "error" in result:
|
| 82 |
+
raise HTTPException(status_code=400, detail=result["error"])
|
| 83 |
+
|
| 84 |
+
# Format clean JSON response for the Flutter app
|
| 85 |
+
response_data = {
|
| 86 |
+
"status": "success",
|
| 87 |
+
"prediction": result.get("prediction", "unknown"),
|
| 88 |
+
"verdict": result.get("prediction", "unknown"), # added for Flutter
|
| 89 |
+
"probability": result.get("deception_probability", 0.5),
|
| 90 |
+
"confidence": result.get("confidence", 0.0),
|
| 91 |
+
"is_deceptive": result.get("prediction") == "deceptive",
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
# Add temporal analysis (to draw the timeline graph in Flutter)
|
| 95 |
+
if "temporal_analysis" in result:
|
| 96 |
+
response_data["temporal"] = {
|
| 97 |
+
"timestamps": result["temporal_analysis"].get("timestamps", []),
|
| 98 |
+
"probabilities": result["temporal_analysis"].get("probability_trajectory", []),
|
| 99 |
+
"suspicious_intervals": result["temporal_analysis"].get("suspicious_intervals", [])
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
# Add SHAP explainability (to draw the Feature Importance chart in Flutter)
|
| 103 |
+
if "explanation" in result:
|
| 104 |
+
fi = result["explanation"].get("feature_importance", {})
|
| 105 |
+
top_indicators = [{"feature": k, "impact": v} for k, v in list(fi.items())[:5]]
|
| 106 |
+
|
| 107 |
+
response_data["explanation"] = {
|
| 108 |
+
"top_indicators": top_indicators,
|
| 109 |
+
"feature_importance": fi,
|
| 110 |
+
"behavioral_explanations": result["explanation"].get("behavioral_explanations", {})
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
return JSONResponse(content=response_data)
|
| 114 |
+
|
| 115 |
+
except Exception as e:
|
| 116 |
+
logger.error(f"Analysis failed: {str(e)}")
|
| 117 |
+
raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
|
| 118 |
+
finally:
|
| 119 |
+
# Clean up the heavy video file to save server storage
|
| 120 |
+
try:
|
| 121 |
+
if os.path.exists(video_path):
|
| 122 |
+
os.remove(video_path)
|
| 123 |
+
os.rmdir(temp_dir)
|
| 124 |
+
except:
|
| 125 |
+
pass
|
| 126 |
+
|
| 127 |
+
@app.get("/health")
|
| 128 |
+
async def health_check():
|
| 129 |
+
return {"status": "online", "pipeline_ready": pipeline is not None}
|
app.py
ADDED
|
@@ -0,0 +1,596 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gradio demo application for the Explainable Multimodal Deception Analysis System."""
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import tempfile
|
| 7 |
+
import shutil
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Optional, Tuple, Dict, Any
|
| 10 |
+
import json
|
| 11 |
+
|
| 12 |
+
import gradio as gr
|
| 13 |
+
import matplotlib.pyplot as plt
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
from matplotlib.figure import Figure
|
| 17 |
+
|
| 18 |
+
from src.pipeline import DeceptionPipeline
|
| 19 |
+
from src.utils.config import AppConfig, get_default_config
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
# Global state
|
| 24 |
+
pipeline: Optional[DeceptionPipeline] = None
|
| 25 |
+
config: Optional[AppConfig] = None
|
| 26 |
+
current_model_path: Optional[str] = None
|
| 27 |
+
|
| 28 |
+
# Custom CSS for styling
|
| 29 |
+
custom_css = """
|
| 30 |
+
/* Deception assessment styling */
|
| 31 |
+
.deceptive-badge {
|
| 32 |
+
background-color: #ffcccc;
|
| 33 |
+
color: #b30000;
|
| 34 |
+
padding: 8px 16px;
|
| 35 |
+
border-radius: 20px;
|
| 36 |
+
font-weight: bold;
|
| 37 |
+
border: 2px solid #ff6666;
|
| 38 |
+
display: inline-block;
|
| 39 |
+
margin: 5px;
|
| 40 |
+
}
|
| 41 |
+
.truthful-badge {
|
| 42 |
+
background-color: #ccffcc;
|
| 43 |
+
color: #006600;
|
| 44 |
+
padding: 8px 16px;
|
| 45 |
+
border-radius: 20px;
|
| 46 |
+
font-weight: bold;
|
| 47 |
+
border: 2px solid #66cc66;
|
| 48 |
+
display: inline-block;
|
| 49 |
+
margin: 5px;
|
| 50 |
+
}
|
| 51 |
+
.error-badge {
|
| 52 |
+
background-color: #ffebcc;
|
| 53 |
+
color: #cc6600;
|
| 54 |
+
padding: 8px 16px;
|
| 55 |
+
border-radius: 20px;
|
| 56 |
+
font-weight: bold;
|
| 57 |
+
border: 2px solid #ff9933;
|
| 58 |
+
display: inline-block;
|
| 59 |
+
margin: 5px;
|
| 60 |
+
}
|
| 61 |
+
/* Professional font choices */
|
| 62 |
+
body {
|
| 63 |
+
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
| 64 |
+
}
|
| 65 |
+
/* Proper spacing */
|
| 66 |
+
.gr-container {
|
| 67 |
+
padding: 20px;
|
| 68 |
+
}
|
| 69 |
+
.gr-tab {
|
| 70 |
+
padding: 20px;
|
| 71 |
+
}
|
| 72 |
+
/* Highlighted suspicious intervals */
|
| 73 |
+
.suspicious-interval {
|
| 74 |
+
background-color: #fff0f0;
|
| 75 |
+
border-left: 4px solid #ff6666;
|
| 76 |
+
padding: 8px 12px;
|
| 77 |
+
margin: 5px 0;
|
| 78 |
+
}
|
| 79 |
+
/* Progress bar styling */
|
| 80 |
+
.gr-progress-bar {
|
| 81 |
+
height: 8px;
|
| 82 |
+
border-radius: 4px;
|
| 83 |
+
}
|
| 84 |
+
"""
|
| 85 |
+
|
| 86 |
+
def initialize_pipeline(config_path: Optional[str] = None) -> DeceptionPipeline:
|
| 87 |
+
"""Initialize or reinitialize the pipeline."""
|
| 88 |
+
global pipeline, config
|
| 89 |
+
try:
|
| 90 |
+
if config_path and os.path.exists(config_path):
|
| 91 |
+
# In a real implementation, you'd load config from file
|
| 92 |
+
logger.warning(f"Config file loading not implemented, using default config")
|
| 93 |
+
config = get_default_config()
|
| 94 |
+
pipeline = DeceptionPipeline(config)
|
| 95 |
+
logger.info("Pipeline initialized with default config.")
|
| 96 |
+
return pipeline
|
| 97 |
+
except Exception as e:
|
| 98 |
+
logger.error(f"Failed to initialize pipeline: {e}")
|
| 99 |
+
raise
|
| 100 |
+
|
| 101 |
+
def get_pipeline() -> DeceptionPipeline:
|
| 102 |
+
"""Get or create the pipeline instance."""
|
| 103 |
+
global pipeline
|
| 104 |
+
if pipeline is None:
|
| 105 |
+
pipeline = initialize_pipeline()
|
| 106 |
+
return pipeline
|
| 107 |
+
|
| 108 |
+
def analyze_video(
|
| 109 |
+
video_file: Optional[str],
|
| 110 |
+
subject_id: Optional[str],
|
| 111 |
+
threshold: float,
|
| 112 |
+
progress=gr.Progress()
|
| 113 |
+
) -> Tuple[Dict[str, Any], Optional[Figure], Optional[Figure], Optional[pd.DataFrame], Optional[pd.DataFrame]]:
|
| 114 |
+
"""Run deception analysis on uploaded video."""
|
| 115 |
+
if video_file is None:
|
| 116 |
+
return {
|
| 117 |
+
"error": "No video file provided"
|
| 118 |
+
}, None, None, None, None
|
| 119 |
+
|
| 120 |
+
# Save uploaded video to a temporary file
|
| 121 |
+
temp_dir = tempfile.mkdtemp(prefix="deception_video_")
|
| 122 |
+
try:
|
| 123 |
+
video_path = os.path.join(temp_dir, "uploaded_video.mp4")
|
| 124 |
+
# Copy the uploaded file
|
| 125 |
+
with open(video_file, 'rb') as src, open(video_path, 'wb') as dst:
|
| 126 |
+
dst.write(src.read())
|
| 127 |
+
|
| 128 |
+
progress(0.1, desc="Initializing pipeline...")
|
| 129 |
+
pipeline = get_pipeline()
|
| 130 |
+
|
| 131 |
+
progress(0.2, desc="Analyzing video (this may take a minute)...")
|
| 132 |
+
# Call the pipeline's analyze_video method
|
| 133 |
+
result = pipeline.analyze_video(
|
| 134 |
+
video_path=video_path,
|
| 135 |
+
subject_id=subject_id if subject_id else None,
|
| 136 |
+
generate_visualizations=True,
|
| 137 |
+
output_dir=temp_dir
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
progress(0.8, desc="Processing results...")
|
| 141 |
+
|
| 142 |
+
# Check for error
|
| 143 |
+
if "error" in result:
|
| 144 |
+
return {
|
| 145 |
+
"error": result["error"]
|
| 146 |
+
}, None, None, None, None
|
| 147 |
+
|
| 148 |
+
# Create temporal analysis plot
|
| 149 |
+
temporal_fig = None
|
| 150 |
+
if "temporal_analysis" in result and "timestamps" in result["temporal_analysis"]:
|
| 151 |
+
try:
|
| 152 |
+
temporal_fig = create_temporal_plot(
|
| 153 |
+
result["temporal_analysis"]["timestamps"],
|
| 154 |
+
result["temporal_analysis"]["probability_trajectory"],
|
| 155 |
+
result["temporal_analysis"].get("suspicious_intervals", [])
|
| 156 |
+
)
|
| 157 |
+
except Exception as e:
|
| 158 |
+
logger.warning(f"Failed to create temporal plot: {e}")
|
| 159 |
+
|
| 160 |
+
# Create feature importance plot
|
| 161 |
+
feature_fig = None
|
| 162 |
+
if "explanation" in result and "feature_importance" in result["explanation"]:
|
| 163 |
+
try:
|
| 164 |
+
feature_fig = create_feature_importance_plot(
|
| 165 |
+
result["explanation"]["feature_importance"]
|
| 166 |
+
)
|
| 167 |
+
except Exception as e:
|
| 168 |
+
logger.warning(f"Failed to create feature importance plot: {e}")
|
| 169 |
+
|
| 170 |
+
# Prepare behavioral indicators dataframe
|
| 171 |
+
behavioral_df = None
|
| 172 |
+
if "explanation" in result and "behavioral_explanations" in result["explanation"]:
|
| 173 |
+
try:
|
| 174 |
+
behavioral_df = create_behavioral_dataframe(
|
| 175 |
+
result["explanation"]["behavioral_explanations"]
|
| 176 |
+
)
|
| 177 |
+
except Exception as e:
|
| 178 |
+
logger.warning(f"Failed to create behavioral dataframe: {e}")
|
| 179 |
+
|
| 180 |
+
# Prepare suspicious intervals dataframe
|
| 181 |
+
suspicious_df = None
|
| 182 |
+
if "temporal_analysis" in result and "suspicious_intervals" in result["temporal_analysis"]:
|
| 183 |
+
try:
|
| 184 |
+
suspicious_df = create_suspicious_intervals_dataframe(
|
| 185 |
+
result["temporal_analysis"]["suspicious_intervals"]
|
| 186 |
+
)
|
| 187 |
+
except Exception as e:
|
| 188 |
+
logger.warning(f"Failed to create suspicious intervals dataframe: {e}")
|
| 189 |
+
|
| 190 |
+
# Format overall assessment
|
| 191 |
+
assessment = {
|
| 192 |
+
"prediction": result.get("prediction", "unknown"),
|
| 193 |
+
"probability": result.get("deception_probability", 0.5),
|
| 194 |
+
"confidence": result.get("confidence", 0.0),
|
| 195 |
+
"calibrated": result.get("calibrated", False),
|
| 196 |
+
"threshold": threshold,
|
| 197 |
+
"is_deceptive": result.get("prediction") == "deceptive" if "prediction" in result else False
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
progress(1.0, desc="Analysis complete!")
|
| 201 |
+
return assessment, temporal_fig, feature_fig, behavioral_df, suspicious_df
|
| 202 |
+
|
| 203 |
+
except Exception as e:
|
| 204 |
+
logger.error(f"Analysis failed: {e}")
|
| 205 |
+
return {
|
| 206 |
+
"error": f"Analysis failed: {str(e)}"
|
| 207 |
+
}, None, None, None, None
|
| 208 |
+
finally:
|
| 209 |
+
# Clean up temp directory after a short delay (or keep for debugging)
|
| 210 |
+
# shutil.rmtree(temp_dir, ignore_errors=True)
|
| 211 |
+
pass
|
| 212 |
+
|
| 213 |
+
def create_temporal_plot(timestamps, probabilities, suspicious_intervals) -> Figure:
|
| 214 |
+
"""Create matplotlib figure for temporal analysis."""
|
| 215 |
+
fig, ax = plt.subplots(figsize=(10, 5))
|
| 216 |
+
ax.plot(timestamps, probabilities, 'b-', linewidth=2, label='Deception Probability')
|
| 217 |
+
ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5, label='Neutral (0.5)')
|
| 218 |
+
ax.set_xlabel('Time (seconds)', fontsize=12)
|
| 219 |
+
ax.set_ylabel('Deception Probability', fontsize=12)
|
| 220 |
+
ax.set_title('Deception Probability Over Time', fontsize=14, fontweight='bold')
|
| 221 |
+
ax.grid(True, alpha=0.3)
|
| 222 |
+
ax.set_ylim(0, 1)
|
| 223 |
+
|
| 224 |
+
# Highlight suspicious intervals
|
| 225 |
+
for interval in suspicious_intervals:
|
| 226 |
+
if len(interval) >= 2:
|
| 227 |
+
start, end = interval[0], interval[1]
|
| 228 |
+
ax.axvspan(start, end, alpha=0.2, color='red', label='Suspicious Interval' if 'Suspicious Interval' not in ax.get_legend_handles_labels()[1] else '')
|
| 229 |
+
|
| 230 |
+
ax.legend(loc='upper right')
|
| 231 |
+
fig.tight_layout()
|
| 232 |
+
return fig
|
| 233 |
+
|
| 234 |
+
def create_feature_importance_plot(feature_importance: Dict[str, float]) -> Figure:
|
| 235 |
+
"""Create matplotlib figure for top feature importance."""
|
| 236 |
+
# Sort features by absolute importance
|
| 237 |
+
sorted_features = sorted(feature_importance.items(), key=lambda x: abs(x[1]), reverse=True)
|
| 238 |
+
top_features = sorted_features[:10] # Top 10 features
|
| 239 |
+
|
| 240 |
+
if not top_features:
|
| 241 |
+
# Create empty plot
|
| 242 |
+
fig, ax = plt.subplots(figsize=(8, 5))
|
| 243 |
+
ax.text(0.5, 0.5, 'No feature importance data available',
|
| 244 |
+
ha='center', va='center', fontsize=12)
|
| 245 |
+
ax.set_xlim(0, 1)
|
| 246 |
+
ax.set_ylim(0, 1)
|
| 247 |
+
ax.axis('off')
|
| 248 |
+
return fig
|
| 249 |
+
|
| 250 |
+
features, importances = zip(*top_features)
|
| 251 |
+
# Shorten long feature names for display
|
| 252 |
+
display_features = [f[:30] + '...' if len(f) > 30 else f for f in features]
|
| 253 |
+
|
| 254 |
+
fig, ax = plt.subplots(figsize=(10, 6))
|
| 255 |
+
colors = ['red' if imp < 0 else 'green' for imp in importances]
|
| 256 |
+
y_pos = np.arange(len(display_features))
|
| 257 |
+
ax.barh(y_pos, importances, color=colors, alpha=0.7)
|
| 258 |
+
ax.set_yticks(y_pos)
|
| 259 |
+
ax.set_yticklabels(display_features, fontsize=10)
|
| 260 |
+
ax.set_xlabel('SHAP Value (Impact on Prediction)', fontsize=12)
|
| 261 |
+
ax.set_title('Top 10 Feature Importance for Deception Prediction', fontsize=14, fontweight='bold')
|
| 262 |
+
ax.grid(True, alpha=0.3, axis='x')
|
| 263 |
+
fig.tight_layout()
|
| 264 |
+
return fig
|
| 265 |
+
|
| 266 |
+
def create_behavioral_dataframe(behavioral_explanations: Dict[str, Any]) -> pd.DataFrame:
|
| 267 |
+
"""Create dataframe for behavioral indicators."""
|
| 268 |
+
rows = []
|
| 269 |
+
for category, indicators in behavioral_explanations.items():
|
| 270 |
+
if isinstance(indicators, dict):
|
| 271 |
+
for indicator, value in indicators.items():
|
| 272 |
+
rows.append({
|
| 273 |
+
'Category': category,
|
| 274 |
+
'Indicator': indicator,
|
| 275 |
+
'Value': value,
|
| 276 |
+
'Interpretation': interpret_behavioral_indicator(category, indicator, value)
|
| 277 |
+
})
|
| 278 |
+
else:
|
| 279 |
+
rows.append({
|
| 280 |
+
'Category': category,
|
| 281 |
+
'Indicator': 'Overall',
|
| 282 |
+
'Value': indicators,
|
| 283 |
+
'Interpretation': interpret_behavioral_indicator(category, 'Overall', indicators)
|
| 284 |
+
})
|
| 285 |
+
|
| 286 |
+
if rows:
|
| 287 |
+
df = pd.DataFrame(rows)
|
| 288 |
+
return df
|
| 289 |
+
return pd.DataFrame(columns=['Category', 'Indicator', 'Value', 'Interpretation'])
|
| 290 |
+
|
| 291 |
+
def interpret_behavioral_indicator(category: str, indicator: str, value: Any) -> str:
|
| 292 |
+
"""Provide human-readable interpretation of behavioral indicators."""
|
| 293 |
+
# Simplified interpretation logic
|
| 294 |
+
try:
|
| 295 |
+
val = float(value)
|
| 296 |
+
if category == 'facial':
|
| 297 |
+
if 'eye_contact' in indicator.lower():
|
| 298 |
+
return "Low eye contact may indicate deception" if val < 0.5 else "Normal eye contact"
|
| 299 |
+
elif 'blink_rate' in indicator.lower():
|
| 300 |
+
return "Elevated blink rate may indicate stress" if val > 0.7 else "Normal blink rate"
|
| 301 |
+
elif category == 'audio':
|
| 302 |
+
if 'pitch_variability' in indicator.lower():
|
| 303 |
+
return "Reduced pitch variability may indicate rehearsed speech" if val < 0.3 else "Normal pitch variability"
|
| 304 |
+
elif 'speech_rate' in indicator.lower():
|
| 305 |
+
return "Increased speech rate may indicate anxiety" if val > 1.2 else "Normal speech rate"
|
| 306 |
+
elif category == 'cross_modal':
|
| 307 |
+
return "Incongruence between facial and audio cues detected" if val > 0.6 else "Consistent multimodal behavior"
|
| 308 |
+
except:
|
| 309 |
+
pass
|
| 310 |
+
return "No specific interpretation available"
|
| 311 |
+
|
| 312 |
+
def create_suspicious_intervals_dataframe(suspicious_intervals: list) -> pd.DataFrame:
|
| 313 |
+
"""Create dataframe for suspicious intervals."""
|
| 314 |
+
if not suspicious_intervals:
|
| 315 |
+
return pd.DataFrame(columns=['Start Time (s)', 'End Time (s)', 'Duration (s)', 'Max Probability', 'Severity'])
|
| 316 |
+
|
| 317 |
+
rows = []
|
| 318 |
+
for i, interval in enumerate(suspicious_intervals):
|
| 319 |
+
if len(interval) >= 4:
|
| 320 |
+
start, end, max_prob, severity = interval[0], interval[1], interval[2], interval[3]
|
| 321 |
+
elif len(interval) >= 2:
|
| 322 |
+
start, end = interval[0], interval[1]
|
| 323 |
+
max_prob = 0.8 # default
|
| 324 |
+
severity = 'medium'
|
| 325 |
+
else:
|
| 326 |
+
continue
|
| 327 |
+
|
| 328 |
+
rows.append({
|
| 329 |
+
'Start Time (s)': f"{start:.2f}",
|
| 330 |
+
'End Time (s)': f"{end:.2f}",
|
| 331 |
+
'Duration (s)': f"{end - start:.2f}",
|
| 332 |
+
'Max Probability': f"{max_prob:.3f}",
|
| 333 |
+
'Severity': severity
|
| 334 |
+
})
|
| 335 |
+
|
| 336 |
+
df = pd.DataFrame(rows)
|
| 337 |
+
return df
|
| 338 |
+
|
| 339 |
+
def load_model(model_path: str, progress=gr.Progress()) -> Dict[str, Any]:
|
| 340 |
+
"""Load a pre-trained model."""
|
| 341 |
+
global pipeline, current_model_path
|
| 342 |
+
try:
|
| 343 |
+
progress(0.3, desc="Checking model file...")
|
| 344 |
+
if not os.path.exists(model_path):
|
| 345 |
+
return {"error": f"Model file not found: {model_path}"}
|
| 346 |
+
|
| 347 |
+
pipeline = get_pipeline()
|
| 348 |
+
progress(0.6, desc="Loading model...")
|
| 349 |
+
# Assuming pipeline has a load_model method
|
| 350 |
+
if hasattr(pipeline, 'load_model'):
|
| 351 |
+
pipeline.load_model(model_path)
|
| 352 |
+
else:
|
| 353 |
+
# Try to load via classifier
|
| 354 |
+
if hasattr(pipeline.classifier, 'load'):
|
| 355 |
+
pipeline.classifier.load(model_path)
|
| 356 |
+
else:
|
| 357 |
+
return {"error": "Model loading not implemented in pipeline"}
|
| 358 |
+
|
| 359 |
+
current_model_path = model_path
|
| 360 |
+
progress(1.0, desc="Model loaded successfully!")
|
| 361 |
+
return {
|
| 362 |
+
"status": "success",
|
| 363 |
+
"model_path": model_path,
|
| 364 |
+
"feature_count": getattr(pipeline.classifier, 'n_features_', 'unknown'),
|
| 365 |
+
"is_fitted": getattr(pipeline.classifier, 'is_fitted_', False)
|
| 366 |
+
}
|
| 367 |
+
except Exception as e:
|
| 368 |
+
logger.error(f"Failed to load model: {e}")
|
| 369 |
+
return {"error": f"Failed to load model: {str(e)}"}
|
| 370 |
+
|
| 371 |
+
def get_model_info() -> Dict[str, Any]:
|
| 372 |
+
"""Return current model information."""
|
| 373 |
+
global pipeline, current_model_path
|
| 374 |
+
if pipeline is None:
|
| 375 |
+
return {"status": "No pipeline initialized"}
|
| 376 |
+
|
| 377 |
+
classifier = pipeline.classifier
|
| 378 |
+
return {
|
| 379 |
+
"model_path": current_model_path or "Not loaded",
|
| 380 |
+
"is_fitted": getattr(classifier, 'is_fitted_', False),
|
| 381 |
+
"feature_count": getattr(classifier, 'n_features_', 'unknown'),
|
| 382 |
+
"model_type": type(classifier).__name__,
|
| 383 |
+
"config": {
|
| 384 |
+
"decision_threshold": pipeline.config.model.decision_threshold,
|
| 385 |
+
"n_estimators": pipeline.config.model.n_estimators,
|
| 386 |
+
"calibration_method": pipeline.config.model.calibration_method
|
| 387 |
+
}
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
def update_settings(
|
| 391 |
+
fps: int,
|
| 392 |
+
audio_segment_length: int,
|
| 393 |
+
generate_visualizations: bool,
|
| 394 |
+
output_dir: str
|
| 395 |
+
) -> Dict[str, Any]:
|
| 396 |
+
"""Update pipeline settings."""
|
| 397 |
+
global pipeline, config
|
| 398 |
+
try:
|
| 399 |
+
# Note: AppConfig is frozen (immutable), so we cannot modify it directly.
|
| 400 |
+
# In a real implementation, we would create a new config with updated values.
|
| 401 |
+
# For this demo, we'll just store the settings and apply them when pipeline is reinitialized.
|
| 402 |
+
settings = {
|
| 403 |
+
"fps": fps,
|
| 404 |
+
"audio_segment_length": audio_segment_length,
|
| 405 |
+
"generate_visualizations": generate_visualizations,
|
| 406 |
+
"output_dir": output_dir if output_dir.strip() else "Not specified (will use temp directory)",
|
| 407 |
+
"note": "Settings will be applied on next pipeline initialization"
|
| 408 |
+
}
|
| 409 |
+
logger.info(f"Settings updated: {settings}")
|
| 410 |
+
return {"status": "Settings updated (note: config is immutable, requires pipeline reinitialization)", "settings": settings}
|
| 411 |
+
except Exception as e:
|
| 412 |
+
logger.error(f"Failed to update settings: {e}")
|
| 413 |
+
return {"error": str(e)}
|
| 414 |
+
|
| 415 |
+
def format_assessment_html(assessment: Dict[str, Any]) -> str:
|
| 416 |
+
"""Format assessment results as HTML with styling."""
|
| 417 |
+
if "error" in assessment:
|
| 418 |
+
return f"""
|
| 419 |
+
<div class="error-badge">
|
| 420 |
+
<h3>❌ Error</h3>
|
| 421 |
+
<p>{assessment['error']}</p>
|
| 422 |
+
</div>
|
| 423 |
+
"""
|
| 424 |
+
|
| 425 |
+
prediction = assessment.get("prediction", "unknown")
|
| 426 |
+
probability = assessment.get("probability", 0.5)
|
| 427 |
+
confidence = assessment.get("confidence", 0.0)
|
| 428 |
+
calibrated = assessment.get("calibrated", False)
|
| 429 |
+
threshold = assessment.get("threshold", 0.5)
|
| 430 |
+
|
| 431 |
+
if prediction == "deceptive":
|
| 432 |
+
badge_class = "deceptive-badge"
|
| 433 |
+
verdict = "DECEPTIVE"
|
| 434 |
+
icon = "⚠️"
|
| 435 |
+
elif prediction == "truthful":
|
| 436 |
+
badge_class = "truthful-badge"
|
| 437 |
+
verdict = "TRUTHFUL"
|
| 438 |
+
icon = "✅"
|
| 439 |
+
else:
|
| 440 |
+
badge_class = "error-badge"
|
| 441 |
+
verdict = "UNKNOWN"
|
| 442 |
+
icon = "❓"
|
| 443 |
+
|
| 444 |
+
calibrated_text = " (calibrated)" if calibrated else ""
|
| 445 |
+
|
| 446 |
+
return f"""
|
| 447 |
+
<div class="{badge_class}">
|
| 448 |
+
<h2>{icon} {verdict}{calibrated_text}</h2>
|
| 449 |
+
<p><strong>Deception Probability:</strong> {probability:.3f}</p>
|
| 450 |
+
<p><strong>Confidence:</strong> {confidence:.3f}</p>
|
| 451 |
+
<p><strong>Decision Threshold:</strong> {threshold:.2f}</p>
|
| 452 |
+
<p><strong>Calibrated:</strong> {calibrated}</p>
|
| 453 |
+
</div>
|
| 454 |
+
"""
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
# ============================================================================
|
| 458 |
+
# Gradio UI Definition
|
| 459 |
+
# ============================================================================
|
| 460 |
+
|
| 461 |
+
def create_ui():
|
| 462 |
+
"""Create the Gradio interface with tabs."""
|
| 463 |
+
with gr.Blocks(css=custom_css, title="Explainable Multimodal Deception Analysis") as demo:
|
| 464 |
+
gr.Markdown("# 🔍 Explainable Multimodal Deception Analysis System")
|
| 465 |
+
gr.Markdown("Upload a video to analyze for deception indicators using facial, audio, and cross-modal features.")
|
| 466 |
+
|
| 467 |
+
with gr.Tabs():
|
| 468 |
+
# Tab 1: Analyze Video
|
| 469 |
+
with gr.TabItem("🔍 Analyze Video"):
|
| 470 |
+
with gr.Row():
|
| 471 |
+
with gr.Column(scale=1):
|
| 472 |
+
video_input = gr.Video(label="Upload Video", sources=["upload"], type="filepath")
|
| 473 |
+
subject_id = gr.Textbox(label="Subject ID (optional)", placeholder="Enter subject identifier")
|
| 474 |
+
threshold_slider = gr.Slider(minimum=0.0, maximum=1.0, value=0.5, step=0.05, label="Decision Threshold", info="Higher values make the model more conservative (less likely to classify as deceptive)")
|
| 475 |
+
analyze_btn = gr.Button("Analyze Video", variant="primary")
|
| 476 |
+
|
| 477 |
+
with gr.Column(scale=2):
|
| 478 |
+
assessment_html = gr.HTML(label="Deception Assessment")
|
| 479 |
+
with gr.Row():
|
| 480 |
+
temporal_plot = gr.Plot(label="Temporal Analysis")
|
| 481 |
+
feature_importance_plot = gr.Plot(label="Feature Importance")
|
| 482 |
+
with gr.Row():
|
| 483 |
+
behavioral_df = gr.Dataframe(label="Behavioral Indicators", headers=["Category", "Indicator", "Value", "Interpretation"])
|
| 484 |
+
suspicious_df = gr.Dataframe(label="Suspicious Intervals", headers=["Start Time (s)", "End Time (s)", "Duration (s)", "Max Probability", "Severity"])
|
| 485 |
+
|
| 486 |
+
analyze_btn.click(
|
| 487 |
+
fn=analyze_video,
|
| 488 |
+
inputs=[video_input, subject_id, threshold_slider],
|
| 489 |
+
outputs=[assessment_html, temporal_plot, feature_importance_plot, behavioral_df, suspicious_df]
|
| 490 |
+
)
|
| 491 |
+
|
| 492 |
+
# Tab 2: Model Info
|
| 493 |
+
with gr.TabItem("📊 Model Info"):
|
| 494 |
+
with gr.Row():
|
| 495 |
+
with gr.Column():
|
| 496 |
+
model_path_input = gr.Textbox(label="Model Path", placeholder="Path to trained model file (.pkl, .joblib, etc.)")
|
| 497 |
+
load_model_btn = gr.Button("Load Model", variant="secondary")
|
| 498 |
+
model_status = gr.JSON(label="Model Status")
|
| 499 |
+
gr.Markdown("### Current Model Information")
|
| 500 |
+
model_info_json = gr.JSON(label="Model Info")
|
| 501 |
+
refresh_btn = gr.Button("Refresh Model Info")
|
| 502 |
+
|
| 503 |
+
load_model_btn.click(
|
| 504 |
+
fn=load_model,
|
| 505 |
+
inputs=[model_path_input],
|
| 506 |
+
outputs=[model_status]
|
| 507 |
+
)
|
| 508 |
+
refresh_btn.click(
|
| 509 |
+
fn=get_model_info,
|
| 510 |
+
inputs=[],
|
| 511 |
+
outputs=[model_info_json]
|
| 512 |
+
)
|
| 513 |
+
|
| 514 |
+
# Tab 3: Settings
|
| 515 |
+
with gr.TabItem("⚙️ Settings"):
|
| 516 |
+
with gr.Row():
|
| 517 |
+
with gr.Column():
|
| 518 |
+
fps_slider = gr.Slider(minimum=1, maximum=60, value=30, step=1, label="Frames Per Second (FPS) for processing")
|
| 519 |
+
audio_segment_slider = gr.Slider(minimum=1, maximum=10, value=3, step=1, label="Audio Segment Length (seconds)")
|
| 520 |
+
visualization_checkbox = gr.Checkbox(value=True, label="Generate Visualizations")
|
| 521 |
+
output_dir_input = gr.Textbox(label="Output Directory (optional)", placeholder="Leave empty for temporary directory")
|
| 522 |
+
update_settings_btn = gr.Button("Update Settings", variant="secondary")
|
| 523 |
+
settings_status = gr.JSON(label="Settings Status")
|
| 524 |
+
|
| 525 |
+
update_settings_btn.click(
|
| 526 |
+
fn=update_settings,
|
| 527 |
+
inputs=[fps_slider, audio_segment_slider, visualization_checkbox, output_dir_input],
|
| 528 |
+
outputs=[settings_status]
|
| 529 |
+
)
|
| 530 |
+
|
| 531 |
+
# Tab 4: About
|
| 532 |
+
with gr.TabItem("📖 About"):
|
| 533 |
+
gr.Markdown("""
|
| 534 |
+
## About This System
|
| 535 |
+
|
| 536 |
+
This is an **Explainable Multimodal Deception Analysis System** that uses:
|
| 537 |
+
- **Facial features**: Micro‑expressions, eye contact, blink rate, head pose
|
| 538 |
+
- **Audio features**: Pitch variability, speech rate, voice stress, spectral features
|
| 539 |
+
- **Cross‑modal fusion**: Combining facial and audio cues for robust detection
|
| 540 |
+
|
| 541 |
+
### How It Works
|
| 542 |
+
1. Upload a video of a person speaking
|
| 543 |
+
2. The system extracts facial and audio features frame‑by‑frame
|
| 544 |
+
3. A trained classifier predicts deception probability over time
|
| 545 |
+
4. SHAP explanations highlight the most influential features
|
| 546 |
+
5. Suspicious intervals are flagged for closer inspection
|
| 547 |
+
|
| 548 |
+
### Interpretation
|
| 549 |
+
- **Deceptive**: Probability > decision threshold (adjustable)
|
| 550 |
+
- **Truthful**: Probability ≤ decision threshold
|
| 551 |
+
- **Confidence**: Model's certainty in the prediction
|
| 552 |
+
|
| 553 |
+
### Technical Details
|
| 554 |
+
- Built with Python, OpenCV, Librosa, and Scikit‑learn
|
| 555 |
+
- Uses a Random Forest classifier with calibration
|
| 556 |
+
- Real‑time visualization with Matplotlib
|
| 557 |
+
- Gradio‑based web interface
|
| 558 |
+
|
| 559 |
+
### Disclaimer
|
| 560 |
+
This tool is for research and demonstration purposes only.
|
| 561 |
+
It should not be used as the sole basis for any legal, security, or psychological assessment.
|
| 562 |
+
""")
|
| 563 |
+
|
| 564 |
+
gr.Markdown("---")
|
| 565 |
+
gr.Markdown("**Note**: Analysis may take several minutes depending on video length and system resources.")
|
| 566 |
+
|
| 567 |
+
return demo
|
| 568 |
+
|
| 569 |
+
|
| 570 |
+
def main():
|
| 571 |
+
"""Main entry point with CLI argument parsing."""
|
| 572 |
+
parser = argparse.ArgumentParser(description="Gradio demo for Explainable Multimodal Deception Analysis")
|
| 573 |
+
parser.add_argument("--share", action="store_true", help="Create a public share link")
|
| 574 |
+
parser.add_argument("--port", type=int, default=7860, help="Port to run the server on")
|
| 575 |
+
parser.add_argument("--config", type=str, help="Path to configuration file")
|
| 576 |
+
args = parser.parse_args()
|
| 577 |
+
|
| 578 |
+
logging.basicConfig(level=logging.INFO)
|
| 579 |
+
|
| 580 |
+
# Initialize pipeline with optional config
|
| 581 |
+
try:
|
| 582 |
+
initialize_pipeline(args.config)
|
| 583 |
+
logger.info("Pipeline initialized successfully")
|
| 584 |
+
except Exception as e:
|
| 585 |
+
logger.warning(f"Pipeline initialization failed: {e}. Will initialize on first use.")
|
| 586 |
+
|
| 587 |
+
demo = create_ui()
|
| 588 |
+
demo.launch(
|
| 589 |
+
share=args.share,
|
| 590 |
+
server_port=args.port,
|
| 591 |
+
server_name="0.0.0.0"
|
| 592 |
+
)
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
if __name__ == "__main__":
|
| 596 |
+
main()
|
colab_deception_analysis.ipynb
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {
|
| 6 |
+
"id": "view-in-github"
|
| 7 |
+
},
|
| 8 |
+
"source": [
|
| 9 |
+
"<a href=\"https://colab.research.google.com/github/apexprit/deception-analysis/blob/main/colab_deception_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
| 10 |
+
]
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"cell_type": "markdown",
|
| 14 |
+
"metadata": {
|
| 15 |
+
"id": "X3jW0N2vQe7p"
|
| 16 |
+
},
|
| 17 |
+
"source": [
|
| 18 |
+
"# \ud83d\udd75\ufe0f\u200d\u2642\ufe0f Multimodal Deception Analysis - Google Colab\n",
|
| 19 |
+
"\n",
|
| 20 |
+
"**Run this entire deception detection system in the cloud.**\n",
|
| 21 |
+
"\n",
|
| 22 |
+
"This notebook provides a complete environment for detecting deception using:\n",
|
| 23 |
+
"- **Facial micro-expressions** (Mediapipe Face Mesh)\n",
|
| 24 |
+
"- **Audio speech patterns** (Librosa)\n",
|
| 25 |
+
"- **SHAP-based explainability**\n",
|
| 26 |
+
"\n",
|
| 27 |
+
"## \ud83d\ude80 Quick Start\n",
|
| 28 |
+
"1. **Runtime \u2192 Run all** (Ctrl+F9).\n",
|
| 29 |
+
"2. If the session restarts, just click **Run all** again.\n",
|
| 30 |
+
"3. **Scroll to the bottom** of the last cell to find the UI."
|
| 31 |
+
]
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"cell_type": "markdown",
|
| 35 |
+
"metadata": {
|
| 36 |
+
"id": "8h2L5U7nQe7q"
|
| 37 |
+
},
|
| 38 |
+
"source": [
|
| 39 |
+
"## 1. Environment Setup"
|
| 40 |
+
]
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"cell_type": "code",
|
| 44 |
+
"execution_count": null,
|
| 45 |
+
"metadata": {
|
| 46 |
+
"id": "7hQ2nVv-Qe7r"
|
| 47 |
+
},
|
| 48 |
+
"outputs": [],
|
| 49 |
+
"source": [
|
| 50 |
+
"#@title Install Dependencies\n",
|
| 51 |
+
"print(\"\ud83d\udee0\ufe0f Installing project dependencies...\")\n",
|
| 52 |
+
"\n",
|
| 53 |
+
"!pip install -q mediapipe==0.10.11\n",
|
| 54 |
+
"!pip install -q opencv-python librosa gradio shap joblib tqdm pandas numpy scikit-learn\n",
|
| 55 |
+
"\n",
|
| 56 |
+
"print(\"\u2705 Installation complete!\")"
|
| 57 |
+
]
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"cell_type": "code",
|
| 61 |
+
"execution_count": null,
|
| 62 |
+
"metadata": {
|
| 63 |
+
"id": "q2Q2nVv-Qe7s"
|
| 64 |
+
},
|
| 65 |
+
"outputs": [],
|
| 66 |
+
"source": [
|
| 67 |
+
"#@title Clone & Setup Path\n",
|
| 68 |
+
"import os\n",
|
| 69 |
+
"import sys\n",
|
| 70 |
+
"import shutil\n",
|
| 71 |
+
"\n",
|
| 72 |
+
"if os.path.exists('/content/deception-analysis'):\n",
|
| 73 |
+
" shutil.rmtree('/content/deception-analysis')\n",
|
| 74 |
+
"\n",
|
| 75 |
+
"!git clone -q https://github.com/apexprit/deception-analysis.git /content/deception-analysis\n",
|
| 76 |
+
"sys.path.insert(0, '/content/deception-analysis')\n",
|
| 77 |
+
"\n",
|
| 78 |
+
"print(\"\u2705 Project ready!\")"
|
| 79 |
+
]
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"cell_type": "markdown",
|
| 83 |
+
"metadata": {
|
| 84 |
+
"id": "MhQ2nVv-Qe7t"
|
| 85 |
+
},
|
| 86 |
+
"source": [
|
| 87 |
+
"## 2. Train & Launch App"
|
| 88 |
+
]
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"cell_type": "code",
|
| 92 |
+
"execution_count": null,
|
| 93 |
+
"metadata": {
|
| 94 |
+
"id": "NhQ2nVv-Qe7u"
|
| 95 |
+
},
|
| 96 |
+
"outputs": [],
|
| 97 |
+
"source": [
|
| 98 |
+
"#@title Generate Model & Start Web App\n",
|
| 99 |
+
"import sys\n",
|
| 100 |
+
"import os\n",
|
| 101 |
+
"import pandas as pd\n",
|
| 102 |
+
"import numpy as np\n",
|
| 103 |
+
"sys.path.insert(0, '/content/deception-analysis')\n",
|
| 104 |
+
"\n",
|
| 105 |
+
"from src.utils.synthetic_data import SyntheticDataGenerator\n",
|
| 106 |
+
"from src.utils.config import get_default_config\n",
|
| 107 |
+
"from src.model.classifier import DeceptionClassifier\n",
|
| 108 |
+
"import gradio as gr\n",
|
| 109 |
+
"from src.pipeline import DeceptionPipeline\n",
|
| 110 |
+
"\n",
|
| 111 |
+
"print(\"\ud83d\udd27 Loading configuration...\")\n",
|
| 112 |
+
"config = get_default_config()\n",
|
| 113 |
+
"\n",
|
| 114 |
+
"print(\"\ud83d\udd27 Generating synthetic dataset for model training...\")\n",
|
| 115 |
+
"generator = SyntheticDataGenerator(config, seed=42)\n",
|
| 116 |
+
"features_df, labels, _ = generator.generate_dataset(n_truthful=300, n_deceptive=300)\n",
|
| 117 |
+
"\n",
|
| 118 |
+
"print(\"\ud83d\udcc8 Training classifier...\")\n",
|
| 119 |
+
"classifier = DeceptionClassifier(config.model)\n",
|
| 120 |
+
"X, feature_names = classifier.prepare_features(features_df)\n",
|
| 121 |
+
"metrics = classifier.train(X, labels, feature_names)\n",
|
| 122 |
+
"print(f\"\u2705 Training complete! Accuracy: {metrics.get('train_accuracy', 0):.3f}\")\n",
|
| 123 |
+
"\n",
|
| 124 |
+
"model_dir = '/content/deception-analysis/models'\n",
|
| 125 |
+
"os.makedirs(model_dir, exist_ok=True)\n",
|
| 126 |
+
"model_path = os.path.join(model_dir, 'deception_model.pkl')\n",
|
| 127 |
+
"classifier.save(model_path)\n",
|
| 128 |
+
"\n",
|
| 129 |
+
"print(\"\\n\ud83d\ude80 Initializing Analysis Pipeline...\")\n",
|
| 130 |
+
"pipeline = DeceptionPipeline(config)\n",
|
| 131 |
+
"pipeline.load_model(model_path)\n",
|
| 132 |
+
"\n",
|
| 133 |
+
"def analyze(video, subject_id):\n",
|
| 134 |
+
" if not video: return \"Please upload a video.\", None, None\n",
|
| 135 |
+
" \n",
|
| 136 |
+
" output_results = '/content/results'\n",
|
| 137 |
+
" os.makedirs(output_results, exist_ok=True)\n",
|
| 138 |
+
" \n",
|
| 139 |
+
" res = pipeline.analyze_video(\n",
|
| 140 |
+
" video, \n",
|
| 141 |
+
" subject_id=subject_id or \"S01\", \n",
|
| 142 |
+
" generate_visualizations=True, \n",
|
| 143 |
+
" output_dir=output_results\n",
|
| 144 |
+
" )\n",
|
| 145 |
+
" \n",
|
| 146 |
+
" if \"error\" in res:\n",
|
| 147 |
+
" return f\"### Error during analysis:\\n{res['error']}\", None, None\n",
|
| 148 |
+
" \n",
|
| 149 |
+
" summary = f\"### Result: {res['prediction'].upper()}\\n- Prob: {res['deception_probability']:.3f}\\n- Conf: {res['confidence']:.3f}\"\n",
|
| 150 |
+
" viz = res.get('visualizations', {})\n",
|
| 151 |
+
" \n",
|
| 152 |
+
" return summary, viz.get('temporal_trajectory'), viz.get('shap_summary')\n",
|
| 153 |
+
"\n",
|
| 154 |
+
"print(\"\ud83d\udce1 Building Interface...\")\n",
|
| 155 |
+
"with gr.Blocks(theme=gr.themes.Soft()) as demo:\n",
|
| 156 |
+
" gr.Markdown(\"# \ud83d\udd75\ufe0f\u200d\u2642\ufe0f Multimodal Deception Analysis\")\n",
|
| 157 |
+
" gr.Markdown(\"Analyze facial micro-expressions and audio patterns for deception detection.\")\n",
|
| 158 |
+
" \n",
|
| 159 |
+
" with gr.Row():\n",
|
| 160 |
+
" with gr.Column():\n",
|
| 161 |
+
" v_in = gr.Video(label=\"Interview Video\")\n",
|
| 162 |
+
" s_in = gr.Textbox(label=\"Subject ID\", placeholder=\"e.g. S01\")\n",
|
| 163 |
+
" btn = gr.Button(\"Analyze Video\", variant=\"primary\")\n",
|
| 164 |
+
" with gr.Column():\n",
|
| 165 |
+
" sum_out = gr.Markdown(label=\"Assessment Summary\")\n",
|
| 166 |
+
" with gr.Tabs():\n",
|
| 167 |
+
" with gr.TabItem(\"Temporal\"): \n",
|
| 168 |
+
" traj_out = gr.Image(label=\"Probability Trajectory\")\n",
|
| 169 |
+
" with gr.TabItem(\"Explainability\"): \n",
|
| 170 |
+
" shap_out = gr.Image(label=\"SHAP Feature Importance\")\n",
|
| 171 |
+
" \n",
|
| 172 |
+
" btn.click(analyze, [v_in, s_in], [sum_out, traj_out, shap_out])\n",
|
| 173 |
+
"\n",
|
| 174 |
+
"print(\"\\n\u2728 APP IS READY! Scroll down to see the interface below.\")\n",
|
| 175 |
+
"demo.launch(share=True, debug=True)"
|
| 176 |
+
]
|
| 177 |
+
}
|
| 178 |
+
],
|
| 179 |
+
"metadata": {
|
| 180 |
+
"colab": {
|
| 181 |
+
"provenance": []
|
| 182 |
+
},
|
| 183 |
+
"kernelspec": {
|
| 184 |
+
"display_name": "Python 3",
|
| 185 |
+
"language": "python",
|
| 186 |
+
"name": "python3"
|
| 187 |
+
},
|
| 188 |
+
"language_info": {
|
| 189 |
+
"codemirror_mode": {
|
| 190 |
+
"name": "ipython",
|
| 191 |
+
"version": 3
|
| 192 |
+
},
|
| 193 |
+
"file_extension": ".py",
|
| 194 |
+
"mimetype": "text/x-python",
|
| 195 |
+
"name": "python",
|
| 196 |
+
"nbconvert_exporter": "python",
|
| 197 |
+
"pygments_lexer": "ipython3",
|
| 198 |
+
"version": "3.8.10"
|
| 199 |
+
}
|
| 200 |
+
},
|
| 201 |
+
"nbformat": 4,
|
| 202 |
+
"nbformat_minor": 0
|
| 203 |
+
}
|
create_pretrained_model.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Create and save a pre-trained deception detection model using synthetic data.
|
| 2 |
+
|
| 3 |
+
This script generates a realistic synthetic dataset, trains a deception classifier,
|
| 4 |
+
and saves it as a pre-trained model for demonstration purposes.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
# Add src to path
|
| 15 |
+
sys.path.insert(0, str(Path(__file__).parent))
|
| 16 |
+
|
| 17 |
+
from src.utils.config import AppConfig, get_default_config
|
| 18 |
+
from src.utils.synthetic_data import SyntheticDataGenerator
|
| 19 |
+
from src.model.classifier import DeceptionClassifier
|
| 20 |
+
from src.model.calibration import SubjectCalibrator
|
| 21 |
+
from src.explainability.explainer import DeceptionExplainer
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def create_pretrained_model(output_dir: str = "./models",
|
| 25 |
+
n_samples: int = 1000,
|
| 26 |
+
test_size: float = 0.2,
|
| 27 |
+
random_state: int = 42):
|
| 28 |
+
"""Create and save a pre-trained deception detection model.
|
| 29 |
+
|
| 30 |
+
Args:
|
| 31 |
+
output_dir: Directory to save model and related files
|
| 32 |
+
n_samples: Total number of synthetic samples to generate
|
| 33 |
+
test_size: Proportion of data for testing
|
| 34 |
+
random_state: Random seed for reproducibility
|
| 35 |
+
"""
|
| 36 |
+
print("=" * 60)
|
| 37 |
+
print("Creating Pre-trained Deception Detection Model")
|
| 38 |
+
print("=" * 60)
|
| 39 |
+
|
| 40 |
+
# Create output directory
|
| 41 |
+
os.makedirs(output_dir, exist_ok=True)
|
| 42 |
+
|
| 43 |
+
# Initialize components
|
| 44 |
+
config = AppConfig()
|
| 45 |
+
generator = SyntheticDataGenerator(config, seed=random_state)
|
| 46 |
+
|
| 47 |
+
print(f"\n1. Generating synthetic dataset ({n_samples} samples)...")
|
| 48 |
+
# Generate balanced dataset
|
| 49 |
+
n_truthful = n_samples // 2
|
| 50 |
+
n_deceptive = n_samples // 2
|
| 51 |
+
|
| 52 |
+
features, labels, subject_ids = generator.generate_dataset(
|
| 53 |
+
n_truthful=n_truthful,
|
| 54 |
+
n_deceptive=n_deceptive
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
print(f" Generated {len(features)} samples")
|
| 58 |
+
print(f" Truthful: {sum(labels == 0)}, Deceptive: {sum(labels == 1)}")
|
| 59 |
+
print(f" Features: {len(features.columns)}")
|
| 60 |
+
print(f" Unique subjects: {len(set(subject_ids))}")
|
| 61 |
+
|
| 62 |
+
# Save dataset
|
| 63 |
+
dataset_path = os.path.join(output_dir, "pretraining_dataset.csv")
|
| 64 |
+
dataset_with_labels = features.copy()
|
| 65 |
+
dataset_with_labels['label'] = labels
|
| 66 |
+
dataset_with_labels['subject_id'] = subject_ids
|
| 67 |
+
dataset_with_labels.to_csv(dataset_path, index=False)
|
| 68 |
+
print(f" Dataset saved to: {dataset_path}")
|
| 69 |
+
|
| 70 |
+
print("\n2. Training deception classifier...")
|
| 71 |
+
# Initialize and train classifier
|
| 72 |
+
classifier = DeceptionClassifier(config.model)
|
| 73 |
+
|
| 74 |
+
# Split data
|
| 75 |
+
from sklearn.model_selection import train_test_split
|
| 76 |
+
X_train, X_test, y_train, y_test = train_test_split(
|
| 77 |
+
features, labels,
|
| 78 |
+
test_size=test_size,
|
| 79 |
+
random_state=random_state,
|
| 80 |
+
stratify=labels
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
print(f" Training samples: {len(X_train)}")
|
| 84 |
+
print(f" Test samples: {len(X_test)}")
|
| 85 |
+
|
| 86 |
+
# Train
|
| 87 |
+
train_result = classifier.train(X_train, y_train, list(features.columns))
|
| 88 |
+
|
| 89 |
+
# Evaluate
|
| 90 |
+
from sklearn.metrics import accuracy_score, f1_score, roc_auc_score
|
| 91 |
+
y_pred = classifier.predict(X_test)
|
| 92 |
+
y_prob = classifier.predict_proba(X_test)[:, 1]
|
| 93 |
+
|
| 94 |
+
test_accuracy = accuracy_score(y_test, y_pred)
|
| 95 |
+
test_f1 = f1_score(y_test, y_pred)
|
| 96 |
+
test_auc = roc_auc_score(y_test, y_prob)
|
| 97 |
+
|
| 98 |
+
print(f" Training accuracy: {train_result.get('train_accuracy', 0):.3f}")
|
| 99 |
+
print(f" Test accuracy: {test_accuracy:.3f}")
|
| 100 |
+
print(f" Test F1-score: {test_f1:.3f}")
|
| 101 |
+
print(f" Test AUC-ROC: {test_auc:.3f}")
|
| 102 |
+
|
| 103 |
+
print("\n3. Training subject calibrator...")
|
| 104 |
+
# Train calibrator on all data
|
| 105 |
+
calibrator = SubjectCalibrator(classifier, config)
|
| 106 |
+
|
| 107 |
+
# Create subject profiles (in real scenario, would use baseline videos)
|
| 108 |
+
subject_profiles = {}
|
| 109 |
+
for subject_id in set(subject_ids):
|
| 110 |
+
subject_mask = [sid == subject_id for sid in subject_ids]
|
| 111 |
+
if sum(subject_mask) >= 5: # Need enough samples per subject
|
| 112 |
+
subject_features = features[subject_mask]
|
| 113 |
+
calibrator.add_subject_profile(subject_id, subject_features)
|
| 114 |
+
subject_profiles[subject_id] = {
|
| 115 |
+
'n_samples': sum(subject_mask),
|
| 116 |
+
'baseline_updated': True
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
print(f" Created profiles for {len(subject_profiles)} subjects")
|
| 120 |
+
|
| 121 |
+
print("\n4. Training explainability model...")
|
| 122 |
+
# Train explainer on subset for efficiency
|
| 123 |
+
explainer = DeceptionExplainer(classifier)
|
| 124 |
+
|
| 125 |
+
# Use smaller subset for SHAP training (computationally expensive)
|
| 126 |
+
if len(X_train) > 200:
|
| 127 |
+
X_explain = X_train.iloc[:200]
|
| 128 |
+
else:
|
| 129 |
+
X_explain = X_train
|
| 130 |
+
|
| 131 |
+
print(f" Initializing SHAP explainer...")
|
| 132 |
+
# Explainer initializes lazily
|
| 133 |
+
explainer._init_shap_explainer()
|
| 134 |
+
|
| 135 |
+
print("\n5. Saving model and components...")
|
| 136 |
+
# Save classifier
|
| 137 |
+
model_path = os.path.join(output_dir, "deception_model.pkl")
|
| 138 |
+
classifier.save(model_path)
|
| 139 |
+
print(f" Classifier saved to: {model_path}")
|
| 140 |
+
|
| 141 |
+
# Save calibrator
|
| 142 |
+
calibrator_path = os.path.join(output_dir, "subject_calibrator.json")
|
| 143 |
+
calibrator.save_profiles(calibrator_path)
|
| 144 |
+
print(f" Calibrator saved to: {calibrator_path}")
|
| 145 |
+
|
| 146 |
+
# Explainer does not need saving
|
| 147 |
+
explainer_path = "N/A"
|
| 148 |
+
|
| 149 |
+
# Save feature names
|
| 150 |
+
feature_names_path = os.path.join(output_dir, "feature_names.json")
|
| 151 |
+
with open(feature_names_path, 'w') as f:
|
| 152 |
+
json.dump(list(features.columns), f, indent=2)
|
| 153 |
+
print(f" Feature names saved to: {feature_names_path}")
|
| 154 |
+
|
| 155 |
+
# Save model metadata
|
| 156 |
+
metadata = {
|
| 157 |
+
'model_name': 'DeceptionDetectionModel',
|
| 158 |
+
'version': '1.0.0',
|
| 159 |
+
'description': 'Pre-trained deception detection model using synthetic data',
|
| 160 |
+
'training_date': pd.Timestamp.now().isoformat(),
|
| 161 |
+
'training_samples': len(X_train),
|
| 162 |
+
'test_samples': len(X_test),
|
| 163 |
+
'n_features': len(features.columns),
|
| 164 |
+
'performance_metrics': {
|
| 165 |
+
'training_accuracy': float(train_result.get('train_accuracy', 0)),
|
| 166 |
+
'training_f1': 0.0, # Not returned by train()
|
| 167 |
+
'test_accuracy': float(test_accuracy),
|
| 168 |
+
'test_f1': float(test_f1),
|
| 169 |
+
'test_auc_roc': float(test_auc)
|
| 170 |
+
},
|
| 171 |
+
'feature_groups': {
|
| 172 |
+
'facial': generator.FACIAL_FEATURES,
|
| 173 |
+
'audio': generator.AUDIO_FEATURES,
|
| 174 |
+
'cross_modal': generator.CROSS_MODAL_FEATURES
|
| 175 |
+
},
|
| 176 |
+
'model_parameters': {
|
| 177 |
+
'model_type': config.model.model_type,
|
| 178 |
+
'random_state': random_state,
|
| 179 |
+
'n_estimators': config.model.n_estimators,
|
| 180 |
+
'learning_rate': config.model.learning_rate,
|
| 181 |
+
'max_depth': config.model.max_depth
|
| 182 |
+
}
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
metadata_path = os.path.join(output_dir, "model_metadata.json")
|
| 186 |
+
with open(metadata_path, 'w') as f:
|
| 187 |
+
json.dump(metadata, f, indent=2)
|
| 188 |
+
print(f" Metadata saved to: {metadata_path}")
|
| 189 |
+
|
| 190 |
+
# Create a sample inference script
|
| 191 |
+
inference_script = '''"""Sample inference with pre-trained deception model."""
|
| 192 |
+
|
| 193 |
+
import json
|
| 194 |
+
import pandas as pd
|
| 195 |
+
from src.model.classifier import DeceptionClassifier
|
| 196 |
+
from src.model.calibration import SubjectCalibrator
|
| 197 |
+
from src.explainability.explainer import DeceptionExplainer
|
| 198 |
+
|
| 199 |
+
# Load model and components
|
| 200 |
+
model = DeceptionClassifier()
|
| 201 |
+
model.load("./models/deception_model.pkl")
|
| 202 |
+
from src.utils.config import get_default_config
|
| 203 |
+
calibrator = SubjectCalibrator(model, get_default_config())
|
| 204 |
+
calibrator.load_profiles("./models/subject_calibrator.json")
|
| 205 |
+
explainer = DeceptionExplainer(model)
|
| 206 |
+
|
| 207 |
+
# Load feature names
|
| 208 |
+
with open("./models/feature_names.json", "r") as f:
|
| 209 |
+
feature_names = json.load(f)
|
| 210 |
+
|
| 211 |
+
print(f"Model loaded with {len(feature_names)} features")
|
| 212 |
+
|
| 213 |
+
# Example: Create sample features (replace with real data)
|
| 214 |
+
import numpy as np
|
| 215 |
+
sample_features = pd.DataFrame({
|
| 216 |
+
feature: np.random.random() for feature in feature_names
|
| 217 |
+
}, index=[0])
|
| 218 |
+
|
| 219 |
+
# Predict
|
| 220 |
+
prediction = model.predict(sample_features)
|
| 221 |
+
probability = model.predict_proba(sample_features)[0, 1]
|
| 222 |
+
|
| 223 |
+
print(f"Prediction: {'Deceptive' if prediction[0] == 1 else 'Truthful'}")
|
| 224 |
+
print(f"Probability of deception: {probability:.3f}")
|
| 225 |
+
|
| 226 |
+
# If subject ID is known, calibrate
|
| 227 |
+
subject_id = "S001"
|
| 228 |
+
if subject_id in calibrator.subject_profiles:
|
| 229 |
+
calibrated_prob = calibrator.calibrate_prediction(probability, subject_id)
|
| 230 |
+
print(f"Calibrated probability: {calibrated_prob:.3f}")
|
| 231 |
+
'''
|
| 232 |
+
|
| 233 |
+
inference_path = os.path.join(output_dir, "sample_inference.py")
|
| 234 |
+
with open(inference_path, 'w') as f:
|
| 235 |
+
f.write(inference_script)
|
| 236 |
+
print(f" Sample inference script: {inference_path}")
|
| 237 |
+
|
| 238 |
+
print("\n" + "=" * 60)
|
| 239 |
+
print("✅ Pre-trained model creation complete!")
|
| 240 |
+
print("=" * 60)
|
| 241 |
+
print(f"\nModel files saved to: {output_dir}")
|
| 242 |
+
print("\nTo use the model:")
|
| 243 |
+
print(f" 1. Load classifier: DeceptionClassifier.load('{model_path}')")
|
| 244 |
+
print(f" 2. Load calibrator: SubjectCalibrator.load('{calibrator_path}')")
|
| 245 |
+
print(f" 3. Load explainer: DeceptionExplainer.load('{explainer_path}')")
|
| 246 |
+
print(f"\nOr use the pipeline:")
|
| 247 |
+
print(f" from src.pipeline import DeceptionPipeline")
|
| 248 |
+
print(f" pipeline = DeceptionPipeline()")
|
| 249 |
+
print(f" pipeline.load_model('{model_path}')")
|
| 250 |
+
|
| 251 |
+
return {
|
| 252 |
+
'model_path': model_path,
|
| 253 |
+
'calibrator_path': calibrator_path,
|
| 254 |
+
'explainer_path': explainer_path,
|
| 255 |
+
'metadata_path': metadata_path,
|
| 256 |
+
'dataset_path': dataset_path
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
if __name__ == "__main__":
|
| 261 |
+
import argparse
|
| 262 |
+
|
| 263 |
+
parser = argparse.ArgumentParser(description="Create pre-trained deception detection model")
|
| 264 |
+
parser.add_argument("--output-dir", default="./models", help="Output directory for model files")
|
| 265 |
+
parser.add_argument("--n-samples", type=int, default=1000, help="Number of synthetic samples")
|
| 266 |
+
parser.add_argument("--test-size", type=float, default=0.2, help="Test set proportion")
|
| 267 |
+
parser.add_argument("--seed", type=int, default=42, help="Random seed")
|
| 268 |
+
|
| 269 |
+
args = parser.parse_args()
|
| 270 |
+
|
| 271 |
+
create_pretrained_model(
|
| 272 |
+
output_dir=args.output_dir,
|
| 273 |
+
n_samples=args.n_samples,
|
| 274 |
+
test_size=args.test_size,
|
| 275 |
+
random_state=args.seed
|
| 276 |
+
)
|
inference.py
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
CLI script for running deception analysis on new videos.
|
| 4 |
+
|
| 5 |
+
Usage:
|
| 6 |
+
python inference.py --video ./data/interview.mp4 --model ./models/deception_model.pkl --subject_id S01 --visualize --output_dir ./results
|
| 7 |
+
python inference.py --video_dir ./data/batch --model ./models/deception_model.pkl --subject_file subjects.csv --output_dir ./results
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import logging
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from typing import List, Optional, Dict, Any
|
| 16 |
+
|
| 17 |
+
import pandas as pd
|
| 18 |
+
|
| 19 |
+
from src.pipeline import DeceptionPipeline
|
| 20 |
+
from src.utils.config import AppConfig, get_default_config
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def setup_logging(verbose: bool = False) -> None:
|
| 24 |
+
"""Configure logging level and format."""
|
| 25 |
+
level = logging.DEBUG if verbose else logging.INFO
|
| 26 |
+
logging.basicConfig(
|
| 27 |
+
level=level,
|
| 28 |
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
| 29 |
+
handlers=[logging.StreamHandler(sys.stdout)],
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def load_config(config_path: Optional[str]) -> AppConfig:
|
| 34 |
+
"""Load configuration from JSON file or use defaults."""
|
| 35 |
+
if config_path is None:
|
| 36 |
+
return get_default_config()
|
| 37 |
+
try:
|
| 38 |
+
with open(config_path, "r") as f:
|
| 39 |
+
config_dict = json.load(f)
|
| 40 |
+
config = get_default_config()
|
| 41 |
+
from dataclasses import fields
|
| 42 |
+
for field in fields(config):
|
| 43 |
+
if field.name in config_dict:
|
| 44 |
+
setattr(config, field.name, config_dict[field.name])
|
| 45 |
+
logging.info(f"Loaded configuration from {config_path}")
|
| 46 |
+
return config
|
| 47 |
+
except Exception as e:
|
| 48 |
+
logging.warning(f"Failed to load config from {config_path}: {e}. Using defaults.")
|
| 49 |
+
return get_default_config()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def load_subject_mapping(subject_file: str) -> Dict[str, str]:
|
| 53 |
+
"""Load CSV mapping video_path -> subject_id."""
|
| 54 |
+
df = pd.read_csv(subject_file)
|
| 55 |
+
if "video_path" not in df.columns or "subject_id" not in df.columns:
|
| 56 |
+
raise ValueError("CSV must contain columns 'video_path' and 'subject_id'")
|
| 57 |
+
return dict(zip(df["video_path"], df["subject_id"]))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def find_videos(video_dir: str, extensions: List[str] = [".mp4", ".avi", ".mov", ".mkv"]) -> List[str]:
|
| 61 |
+
"""Recursively find video files in directory."""
|
| 62 |
+
video_dir = Path(video_dir)
|
| 63 |
+
video_paths = []
|
| 64 |
+
for ext in extensions:
|
| 65 |
+
video_paths.extend(video_dir.rglob(f"*{ext}"))
|
| 66 |
+
return sorted([str(p) for p in video_paths])
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def print_result_summary(result: Dict[str, Any], threshold: float = 0.5) -> None:
|
| 70 |
+
"""Print a formatted summary of analysis result."""
|
| 71 |
+
prob = result.get("deception_probability", 0.5)
|
| 72 |
+
pred = result.get("prediction", "unknown")
|
| 73 |
+
confidence = result.get("confidence", 0.0)
|
| 74 |
+
calibrated = result.get("calibrated", False)
|
| 75 |
+
print("\n" + "="*60)
|
| 76 |
+
print("DECEPTION ANALYSIS RESULT")
|
| 77 |
+
print("="*60)
|
| 78 |
+
print(f"Prediction: {pred.upper()}")
|
| 79 |
+
print(f"Deception Probability: {prob:.3f} (threshold = {threshold})")
|
| 80 |
+
print(f"Confidence: {confidence:.3f}")
|
| 81 |
+
if calibrated:
|
| 82 |
+
print("Calibration: Applied (subject-specific)")
|
| 83 |
+
else:
|
| 84 |
+
print("Calibration: Not applied")
|
| 85 |
+
# Key indicators
|
| 86 |
+
temporal = result.get("temporal_analysis", {})
|
| 87 |
+
if "suspicious_intervals" in temporal and temporal["suspicious_intervals"]:
|
| 88 |
+
intervals = temporal["suspicious_intervals"]
|
| 89 |
+
print(f"Suspicious Intervals: {len(intervals)} detected")
|
| 90 |
+
if "spikes" in temporal and temporal["spikes"]:
|
| 91 |
+
print(f"Probability Spikes: {len(temporal['spikes'])}")
|
| 92 |
+
# Feature importance
|
| 93 |
+
explanation = result.get("explanation", {})
|
| 94 |
+
if "top_features" in explanation:
|
| 95 |
+
top = explanation["top_features"][:3]
|
| 96 |
+
print("Top Contributing Features:")
|
| 97 |
+
for feat, imp in top:
|
| 98 |
+
print(f" - {feat}: {imp:.3f}")
|
| 99 |
+
print("="*60)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def main():
|
| 103 |
+
parser = argparse.ArgumentParser(
|
| 104 |
+
description="Run deception analysis on video(s).",
|
| 105 |
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
| 106 |
+
)
|
| 107 |
+
# Input options
|
| 108 |
+
input_group = parser.add_mutually_exclusive_group(required=True)
|
| 109 |
+
input_group.add_argument(
|
| 110 |
+
"--video",
|
| 111 |
+
help="Single video path to analyze.",
|
| 112 |
+
)
|
| 113 |
+
input_group.add_argument(
|
| 114 |
+
"--video_dir",
|
| 115 |
+
help="Directory containing videos to analyze (batch mode).",
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
# Model and config
|
| 119 |
+
parser.add_argument(
|
| 120 |
+
"--model",
|
| 121 |
+
required=True,
|
| 122 |
+
help="Path to trained model file.",
|
| 123 |
+
)
|
| 124 |
+
parser.add_argument(
|
| 125 |
+
"--config",
|
| 126 |
+
help="Path to JSON config file (optional).",
|
| 127 |
+
)
|
| 128 |
+
parser.add_argument(
|
| 129 |
+
"--threshold",
|
| 130 |
+
type=float,
|
| 131 |
+
default=0.5,
|
| 132 |
+
help="Deception probability threshold.",
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
# Subject identification
|
| 136 |
+
parser.add_argument(
|
| 137 |
+
"--subject_id",
|
| 138 |
+
help="Subject identifier for calibration (single video mode).",
|
| 139 |
+
)
|
| 140 |
+
parser.add_argument(
|
| 141 |
+
"--subject_file",
|
| 142 |
+
help="CSV mapping video_path to subject_id (batch mode).",
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
# Output options
|
| 146 |
+
parser.add_argument(
|
| 147 |
+
"--output_dir",
|
| 148 |
+
default="./results",
|
| 149 |
+
help="Directory to save results and visualizations.",
|
| 150 |
+
)
|
| 151 |
+
parser.add_argument(
|
| 152 |
+
"--visualize",
|
| 153 |
+
action="store_true",
|
| 154 |
+
help="Generate all visualizations.",
|
| 155 |
+
)
|
| 156 |
+
parser.add_argument(
|
| 157 |
+
"--report",
|
| 158 |
+
action="store_true",
|
| 159 |
+
help="Generate comprehensive JSON report.",
|
| 160 |
+
)
|
| 161 |
+
parser.add_argument(
|
| 162 |
+
"--verbose",
|
| 163 |
+
action="store_true",
|
| 164 |
+
help="Print detailed output.",
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
args = parser.parse_args()
|
| 168 |
+
|
| 169 |
+
setup_logging(args.verbose)
|
| 170 |
+
|
| 171 |
+
# Validate model path
|
| 172 |
+
model_path = Path(args.model)
|
| 173 |
+
if not model_path.exists():
|
| 174 |
+
logging.error(f"Model file does not exist: {model_path}")
|
| 175 |
+
sys.exit(1)
|
| 176 |
+
|
| 177 |
+
# Prepare output directory
|
| 178 |
+
output_dir = Path(args.output_dir)
|
| 179 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 180 |
+
|
| 181 |
+
# Load configuration
|
| 182 |
+
config = load_config(args.config)
|
| 183 |
+
# Override decision threshold if provided
|
| 184 |
+
config.model.decision_threshold = args.threshold
|
| 185 |
+
logging.info(f"Decision threshold set to {args.threshold}")
|
| 186 |
+
|
| 187 |
+
# Initialize pipeline
|
| 188 |
+
pipeline = DeceptionPipeline(config)
|
| 189 |
+
|
| 190 |
+
# Load pre-trained model
|
| 191 |
+
try:
|
| 192 |
+
pipeline.load_model(str(model_path))
|
| 193 |
+
logging.info(f"Model loaded from {model_path}")
|
| 194 |
+
except Exception as e:
|
| 195 |
+
logging.error(f"Failed to load model: {e}")
|
| 196 |
+
sys.exit(1)
|
| 197 |
+
|
| 198 |
+
# Determine video list and subject IDs
|
| 199 |
+
video_paths = []
|
| 200 |
+
subject_ids = None
|
| 201 |
+
|
| 202 |
+
if args.video:
|
| 203 |
+
video_paths = [args.video]
|
| 204 |
+
if args.subject_id:
|
| 205 |
+
subject_ids = [args.subject_id]
|
| 206 |
+
elif args.subject_file:
|
| 207 |
+
logging.warning("--subject_file ignored in single video mode. Use --subject_id.")
|
| 208 |
+
else: # video_dir
|
| 209 |
+
video_paths = find_videos(args.video_dir)
|
| 210 |
+
if not video_paths:
|
| 211 |
+
logging.error(f"No video files found in {args.video_dir}")
|
| 212 |
+
sys.exit(1)
|
| 213 |
+
logging.info(f"Found {len(video_paths)} videos in {args.video_dir}")
|
| 214 |
+
if args.subject_file:
|
| 215 |
+
try:
|
| 216 |
+
mapping = load_subject_mapping(args.subject_file)
|
| 217 |
+
subject_ids = [mapping.get(Path(v).name, None) for v in video_paths]
|
| 218 |
+
# If mapping uses relative paths, we could match by basename
|
| 219 |
+
# For simplicity, we assume mapping uses video file names.
|
| 220 |
+
except Exception as e:
|
| 221 |
+
logging.error(f"Failed to load subject mapping: {e}")
|
| 222 |
+
sys.exit(1)
|
| 223 |
+
elif args.subject_id:
|
| 224 |
+
logging.warning("--subject_id ignored in batch mode. Use --subject_file.")
|
| 225 |
+
subject_ids = None
|
| 226 |
+
|
| 227 |
+
# Run analysis
|
| 228 |
+
results = []
|
| 229 |
+
if len(video_paths) == 1:
|
| 230 |
+
# Single video analysis
|
| 231 |
+
video_path = video_paths[0]
|
| 232 |
+
subject_id = subject_ids[0] if subject_ids else None
|
| 233 |
+
logging.info(f"Analyzing video: {video_path}")
|
| 234 |
+
try:
|
| 235 |
+
result = pipeline.analyze_video(
|
| 236 |
+
video_path=video_path,
|
| 237 |
+
subject_id=subject_id,
|
| 238 |
+
generate_visualizations=args.visualize,
|
| 239 |
+
output_dir=str(output_dir),
|
| 240 |
+
)
|
| 241 |
+
results.append(result)
|
| 242 |
+
# Print summary
|
| 243 |
+
print_result_summary(result, args.threshold)
|
| 244 |
+
# Generate report if requested
|
| 245 |
+
if args.report:
|
| 246 |
+
report_path = output_dir / f"{Path(video_path).stem}_report.json"
|
| 247 |
+
pipeline.generate_report(result, str(report_path))
|
| 248 |
+
logging.info(f"Report saved to {report_path}")
|
| 249 |
+
except Exception as e:
|
| 250 |
+
logging.error(f"Analysis failed for {video_path}: {e}")
|
| 251 |
+
sys.exit(1)
|
| 252 |
+
else:
|
| 253 |
+
# Batch analysis
|
| 254 |
+
logging.info(f"Starting batch analysis of {len(video_paths)} videos...")
|
| 255 |
+
try:
|
| 256 |
+
results = pipeline.analyze_batch(
|
| 257 |
+
video_paths=video_paths,
|
| 258 |
+
subject_ids=subject_ids,
|
| 259 |
+
generate_visualizations=args.visualize,
|
| 260 |
+
output_dir=str(output_dir),
|
| 261 |
+
)
|
| 262 |
+
logging.info("Batch analysis completed.")
|
| 263 |
+
except Exception as e:
|
| 264 |
+
logging.error(f"Batch analysis failed: {e}")
|
| 265 |
+
sys.exit(1)
|
| 266 |
+
|
| 267 |
+
# Print batch summary
|
| 268 |
+
print("\n" + "="*60)
|
| 269 |
+
print("BATCH ANALYSIS SUMMARY")
|
| 270 |
+
print("="*60)
|
| 271 |
+
deceptive_count = sum(1 for r in results if r.get("prediction") == "deceptive")
|
| 272 |
+
total = len(results)
|
| 273 |
+
print(f"Videos analyzed: {total}")
|
| 274 |
+
print(f"Deceptive predictions: {deceptive_count} ({deceptive_count/total*100:.1f}%)")
|
| 275 |
+
print(f"Truthful predictions: {total - deceptive_count}")
|
| 276 |
+
avg_prob = sum(r.get("deception_probability", 0.5) for r in results) / total
|
| 277 |
+
print(f"Average deception probability: {avg_prob:.3f}")
|
| 278 |
+
print("="*60)
|
| 279 |
+
|
| 280 |
+
# Save batch results to CSV
|
| 281 |
+
batch_csv = output_dir / "batch_results.csv"
|
| 282 |
+
rows = []
|
| 283 |
+
for vid, res in zip(video_paths, results):
|
| 284 |
+
rows.append({
|
| 285 |
+
"video_path": vid,
|
| 286 |
+
"prediction": res.get("prediction", "unknown"),
|
| 287 |
+
"deception_probability": res.get("deception_probability", 0.5),
|
| 288 |
+
"confidence": res.get("confidence", 0.0),
|
| 289 |
+
"calibrated": res.get("calibrated", False),
|
| 290 |
+
})
|
| 291 |
+
df = pd.DataFrame(rows)
|
| 292 |
+
df.to_csv(batch_csv, index=False)
|
| 293 |
+
logging.info(f"Batch results saved to {batch_csv}")
|
| 294 |
+
|
| 295 |
+
# Generate report for each video if requested
|
| 296 |
+
if args.report:
|
| 297 |
+
for vid, res in zip(video_paths, results):
|
| 298 |
+
report_path = output_dir / f"{Path(vid).stem}_report.json"
|
| 299 |
+
pipeline.generate_report(res, str(report_path))
|
| 300 |
+
logging.info(f"Individual reports saved to {output_dir}")
|
| 301 |
+
|
| 302 |
+
# Save overall results as JSON
|
| 303 |
+
if results:
|
| 304 |
+
results_json = output_dir / "analysis_results.json"
|
| 305 |
+
# Convert DataFrames to dict for serialization (skip them)
|
| 306 |
+
serializable = []
|
| 307 |
+
for res in results:
|
| 308 |
+
ser = {}
|
| 309 |
+
for k, v in res.items():
|
| 310 |
+
if isinstance(v, pd.DataFrame):
|
| 311 |
+
ser[k] = v.to_dict(orient="records")
|
| 312 |
+
else:
|
| 313 |
+
ser[k] = v
|
| 314 |
+
serializable.append(ser)
|
| 315 |
+
with open(results_json, "w") as f:
|
| 316 |
+
json.dump(serializable, f, indent=2, default=str)
|
| 317 |
+
logging.info(f"Full results saved to {results_json}")
|
| 318 |
+
|
| 319 |
+
logging.info("Analysis completed successfully.")
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
if __name__ == "__main__":
|
| 323 |
+
main()
|
models/.gitkeep
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
requirements.txt
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
opencv-python>=4.8.0
|
| 2 |
+
mediapipe>=0.10.0
|
| 3 |
+
librosa>=0.10.0
|
| 4 |
+
scikit-learn>=1.3.0
|
| 5 |
+
matplotlib>=3.7.0
|
| 6 |
+
seaborn>=0.12.0
|
| 7 |
+
gradio>=4.0.0
|
| 8 |
+
numpy>=1.24.0
|
| 9 |
+
pandas>=2.0.0
|
| 10 |
+
shap>=0.43.0
|
| 11 |
+
joblib>=1.3.0
|
| 12 |
+
tqdm>=4.65.0
|
| 13 |
+
scipy>=1.11.0
|
| 14 |
+
fastapi>=0.100.0
|
| 15 |
+
uvicorn>=0.23.0
|
| 16 |
+
python-multipart>=0.0.6
|
src/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explainable Multimodal Deception Analysis System."""
|
| 2 |
+
|
| 3 |
+
# We avoid eager imports of DeceptionPipeline here to prevent side-effects
|
| 4 |
+
# (like MediaPipe initialization) when only utility modules are needed.
|
src/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (215 Bytes). View file
|
|
|
src/__pycache__/pipeline.cpython-314.pyc
ADDED
|
Binary file (24.6 kB). View file
|
|
|
src/audio/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio analysis module for speech pattern detection."""
|
| 2 |
+
|
| 3 |
+
from .extractor import AudioFeatureExtractor
|
| 4 |
+
from .fusion import MultimodalFusion
|
| 5 |
+
|
| 6 |
+
__all__ = ['AudioFeatureExtractor', 'MultimodalFusion']
|
src/audio/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (328 Bytes). View file
|
|
|
src/audio/extractor.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio and speech feature extraction using librosa."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Dict, List, Optional
|
| 6 |
+
|
| 7 |
+
import librosa
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pandas as pd
|
| 10 |
+
|
| 11 |
+
from src.utils.config import AudioConfig, get_default_config
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class AudioFeatureExtractor:
|
| 15 |
+
"""Extracts speech and audio features for deception analysis using librosa."""
|
| 16 |
+
|
| 17 |
+
def __init__(self, config: Optional[AudioConfig] = None):
|
| 18 |
+
"""Initialize the audio extractor.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
config: Optional audio configuration. Defaults to application audio config.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
self.config = config or get_default_config().audio
|
| 25 |
+
|
| 26 |
+
@staticmethod
|
| 27 |
+
def _finite_or_zero(value: float) -> float:
|
| 28 |
+
"""Convert non-finite numeric outputs to zero."""
|
| 29 |
+
|
| 30 |
+
return float(value) if np.isfinite(value) else 0.0
|
| 31 |
+
|
| 32 |
+
def _empty_features(self) -> Dict[str, float]:
|
| 33 |
+
"""Return a zero-filled feature dictionary for invalid audio."""
|
| 34 |
+
|
| 35 |
+
return {name: 0.0 for name in self.get_feature_names()}
|
| 36 |
+
|
| 37 |
+
def extract_pitch_variation(self, y: np.ndarray, sr: int) -> Dict[str, float]:
|
| 38 |
+
"""Extract pitch/f0 features using probabilistic YIN.
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
y: Audio waveform.
|
| 42 |
+
sr: Sample rate.
|
| 43 |
+
|
| 44 |
+
Returns:
|
| 45 |
+
Dictionary with pitch summary and voiced-frame ratio.
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
if y.size < self.config.hop_length or np.allclose(y, 0.0):
|
| 49 |
+
return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
|
| 50 |
+
try:
|
| 51 |
+
f0, _, voiced_prob = librosa.pyin(y, fmin=65.0, fmax=2093.0, sr=sr, hop_length=self.config.hop_length)
|
| 52 |
+
except Exception:
|
| 53 |
+
return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
|
| 54 |
+
voiced = f0[np.isfinite(f0)]
|
| 55 |
+
if voiced.size == 0:
|
| 56 |
+
return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
|
| 57 |
+
mean = float(np.mean(voiced))
|
| 58 |
+
std = float(np.std(voiced))
|
| 59 |
+
# Pitch elevation and instability can reflect stress or high cognitive load during deceptive speech.
|
| 60 |
+
return {
|
| 61 |
+
"pitch_mean": mean,
|
| 62 |
+
"pitch_std": std,
|
| 63 |
+
"pitch_range": float(np.max(voiced) - np.min(voiced)),
|
| 64 |
+
"pitch_variation_coefficient": self._finite_or_zero(std / mean if mean else 0.0),
|
| 65 |
+
"voiced_ratio": float(np.mean(np.asarray(voiced_prob) > 0.5)) if voiced_prob is not None else float(voiced.size / f0.size),
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
def extract_pause_analysis(self, y: np.ndarray, sr: int) -> Dict[str, float]:
|
| 69 |
+
"""Analyze speech pauses and timing with adaptive energy VAD.
|
| 70 |
+
|
| 71 |
+
Args:
|
| 72 |
+
y: Audio waveform.
|
| 73 |
+
sr: Sample rate.
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
Dictionary with pause count, duration, rate, and speech ratio.
|
| 77 |
+
"""
|
| 78 |
+
|
| 79 |
+
duration = len(y) / float(sr) if sr else 0.0
|
| 80 |
+
if y.size == 0 or duration <= 0.0:
|
| 81 |
+
return {"pause_count": 0.0, "avg_pause_duration": 0.0, "max_pause_duration": 0.0, "pause_rate": 0.0, "speech_ratio": 0.0}
|
| 82 |
+
rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
|
| 83 |
+
if rms.size == 0 or np.allclose(rms, 0.0):
|
| 84 |
+
return {"pause_count": 1.0, "avg_pause_duration": duration, "max_pause_duration": duration, "pause_rate": 60.0 / duration, "speech_ratio": 0.0}
|
| 85 |
+
threshold = max(float(np.median(rms) * 0.6), float(np.max(rms) * 0.03))
|
| 86 |
+
silent = rms <= threshold
|
| 87 |
+
frame_seconds = self.config.hop_length / float(sr)
|
| 88 |
+
min_pause_frames = max(1, int(np.ceil(0.3 / frame_seconds)))
|
| 89 |
+
pauses: List[float] = []
|
| 90 |
+
run = 0
|
| 91 |
+
for is_silent in silent:
|
| 92 |
+
if is_silent:
|
| 93 |
+
run += 1
|
| 94 |
+
else:
|
| 95 |
+
if run >= min_pause_frames:
|
| 96 |
+
pauses.append(run * frame_seconds)
|
| 97 |
+
run = 0
|
| 98 |
+
if run >= min_pause_frames:
|
| 99 |
+
pauses.append(run * frame_seconds)
|
| 100 |
+
pause_time = float(np.sum(pauses)) if pauses else 0.0
|
| 101 |
+
# Pauses and hesitations are deception-relevant when they indicate formulation difficulty.
|
| 102 |
+
return {
|
| 103 |
+
"pause_count": float(len(pauses)),
|
| 104 |
+
"avg_pause_duration": float(np.mean(pauses)) if pauses else 0.0,
|
| 105 |
+
"max_pause_duration": float(np.max(pauses)) if pauses else 0.0,
|
| 106 |
+
"pause_rate": float(len(pauses) / (duration / 60.0)) if duration > 0 else 0.0,
|
| 107 |
+
"speech_ratio": float(np.clip(1.0 - pause_time / duration, 0.0, 1.0)),
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
def extract_speech_energy(self, y: np.ndarray, sr: int) -> Dict[str, float]:
|
| 111 |
+
"""Extract speech energy and spectral intensity features.
|
| 112 |
+
|
| 113 |
+
Args:
|
| 114 |
+
y: Audio waveform.
|
| 115 |
+
sr: Sample rate.
|
| 116 |
+
|
| 117 |
+
Returns:
|
| 118 |
+
Dictionary with RMS energy and spectral descriptors.
|
| 119 |
+
"""
|
| 120 |
+
|
| 121 |
+
if y.size == 0:
|
| 122 |
+
return {"energy_mean": 0.0, "energy_std": 0.0, "energy_range": 0.0, "energy_variation": 0.0, "spectral_centroid_mean": 0.0, "spectral_bandwidth_mean": 0.0}
|
| 123 |
+
rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
|
| 124 |
+
mean = float(np.mean(rms)) if rms.size else 0.0
|
| 125 |
+
std = float(np.std(rms)) if rms.size else 0.0
|
| 126 |
+
centroid = librosa.feature.spectral_centroid(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
|
| 127 |
+
bandwidth = librosa.feature.spectral_bandwidth(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
|
| 128 |
+
# Vocal intensity spikes can pair with facial stress markers in multimodal deception cues.
|
| 129 |
+
return {
|
| 130 |
+
"energy_mean": mean,
|
| 131 |
+
"energy_std": std,
|
| 132 |
+
"energy_range": float(np.max(rms) - np.min(rms)) if rms.size else 0.0,
|
| 133 |
+
"energy_variation": self._finite_or_zero(std / mean if mean else 0.0),
|
| 134 |
+
"spectral_centroid_mean": float(np.mean(centroid)) if centroid.size else 0.0,
|
| 135 |
+
"spectral_bandwidth_mean": float(np.mean(bandwidth)) if bandwidth.size else 0.0,
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
def extract_mfcc_features(self, y: np.ndarray, sr: int) -> Dict[str, float]:
|
| 139 |
+
"""Extract MFCC and delta-MFCC speech characterization features.
|
| 140 |
+
|
| 141 |
+
Args:
|
| 142 |
+
y: Audio waveform.
|
| 143 |
+
sr: Sample rate.
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
Dictionary containing MFCC means, standard deviations, and delta means.
|
| 147 |
+
"""
|
| 148 |
+
|
| 149 |
+
features: Dict[str, float] = {}
|
| 150 |
+
if y.size == 0:
|
| 151 |
+
for idx in range(self.config.n_mfcc):
|
| 152 |
+
features[f"mfcc_{idx}_mean"] = 0.0
|
| 153 |
+
features[f"mfcc_{idx}_std"] = 0.0
|
| 154 |
+
features[f"delta_mfcc_{idx}_mean"] = 0.0
|
| 155 |
+
return features
|
| 156 |
+
mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=self.config.n_mfcc, n_fft=self.config.n_fft, hop_length=self.config.hop_length)
|
| 157 |
+
delta = librosa.feature.delta(mfcc) if mfcc.shape[1] >= 3 else np.zeros_like(mfcc)
|
| 158 |
+
# MFCCs capture voice quality shifts that may accompany stress or controlled speech.
|
| 159 |
+
for idx in range(self.config.n_mfcc):
|
| 160 |
+
features[f"mfcc_{idx}_mean"] = float(np.mean(mfcc[idx]))
|
| 161 |
+
features[f"mfcc_{idx}_std"] = float(np.std(mfcc[idx]))
|
| 162 |
+
features[f"delta_mfcc_{idx}_mean"] = float(np.mean(delta[idx]))
|
| 163 |
+
return features
|
| 164 |
+
|
| 165 |
+
def process_audio(self, audio_path: str) -> Dict[str, float]:
|
| 166 |
+
"""Process a complete audio file and extract all features.
|
| 167 |
+
|
| 168 |
+
Args:
|
| 169 |
+
audio_path: Path to an audio file readable by librosa.
|
| 170 |
+
|
| 171 |
+
Returns:
|
| 172 |
+
Combined audio feature dictionary.
|
| 173 |
+
"""
|
| 174 |
+
|
| 175 |
+
try:
|
| 176 |
+
y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
|
| 177 |
+
except Exception:
|
| 178 |
+
return self._empty_features()
|
| 179 |
+
if y.size == 0:
|
| 180 |
+
return self._empty_features()
|
| 181 |
+
features: Dict[str, float] = {}
|
| 182 |
+
features.update(self.extract_pitch_variation(y, sr))
|
| 183 |
+
features.update(self.extract_pause_analysis(y, sr))
|
| 184 |
+
features.update(self.extract_speech_energy(y, sr))
|
| 185 |
+
features.update(self.extract_mfcc_features(y, sr))
|
| 186 |
+
return {name: self._finite_or_zero(float(features.get(name, 0.0))) for name in self.get_feature_names()}
|
| 187 |
+
|
| 188 |
+
def process_audio_segments(self, audio_path: str, segment_duration: float = 2.0) -> pd.DataFrame:
|
| 189 |
+
"""Process audio in fixed-duration segments for temporal analysis.
|
| 190 |
+
|
| 191 |
+
Args:
|
| 192 |
+
audio_path: Path to an audio file readable by librosa.
|
| 193 |
+
segment_duration: Segment length in seconds.
|
| 194 |
+
|
| 195 |
+
Returns:
|
| 196 |
+
DataFrame with segment index, timing, and audio feature columns.
|
| 197 |
+
"""
|
| 198 |
+
|
| 199 |
+
try:
|
| 200 |
+
y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
|
| 201 |
+
except Exception:
|
| 202 |
+
columns = ["segment_idx", "start_time", "end_time"] + self.get_feature_names()
|
| 203 |
+
return pd.DataFrame(columns=columns)
|
| 204 |
+
segment_samples = max(1, int(segment_duration * sr))
|
| 205 |
+
records: List[Dict[str, float]] = []
|
| 206 |
+
for segment_idx, start in enumerate(range(0, len(y), segment_samples)):
|
| 207 |
+
end = min(start + segment_samples, len(y))
|
| 208 |
+
segment = y[start:end]
|
| 209 |
+
record: Dict[str, float] = {
|
| 210 |
+
"segment_idx": float(segment_idx),
|
| 211 |
+
"start_time": start / float(sr),
|
| 212 |
+
"end_time": end / float(sr),
|
| 213 |
+
}
|
| 214 |
+
if segment.size < max(16, self.config.hop_length // 2):
|
| 215 |
+
record.update(self._empty_features())
|
| 216 |
+
else:
|
| 217 |
+
record.update(self.extract_pitch_variation(segment, sr))
|
| 218 |
+
record.update(self.extract_pause_analysis(segment, sr))
|
| 219 |
+
record.update(self.extract_speech_energy(segment, sr))
|
| 220 |
+
record.update(self.extract_mfcc_features(segment, sr))
|
| 221 |
+
records.append(record)
|
| 222 |
+
return pd.DataFrame(records, columns=["segment_idx", "start_time", "end_time"] + self.get_feature_names())
|
| 223 |
+
|
| 224 |
+
def get_feature_names(self) -> List[str]:
|
| 225 |
+
"""Return ordered names of all produced audio features."""
|
| 226 |
+
|
| 227 |
+
names = [
|
| 228 |
+
"pitch_mean",
|
| 229 |
+
"pitch_std",
|
| 230 |
+
"pitch_range",
|
| 231 |
+
"pitch_variation_coefficient",
|
| 232 |
+
"voiced_ratio",
|
| 233 |
+
"pause_count",
|
| 234 |
+
"avg_pause_duration",
|
| 235 |
+
"max_pause_duration",
|
| 236 |
+
"pause_rate",
|
| 237 |
+
"speech_ratio",
|
| 238 |
+
"energy_mean",
|
| 239 |
+
"energy_std",
|
| 240 |
+
"energy_range",
|
| 241 |
+
"energy_variation",
|
| 242 |
+
"spectral_centroid_mean",
|
| 243 |
+
"spectral_bandwidth_mean",
|
| 244 |
+
]
|
| 245 |
+
for idx in range(self.config.n_mfcc):
|
| 246 |
+
names.extend([f"mfcc_{idx}_mean", f"mfcc_{idx}_std", f"delta_mfcc_{idx}_mean"])
|
| 247 |
+
return names
|
| 248 |
+
|
src/audio/fusion.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Multimodal fusion utilities for facial and audio features."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import List, Optional
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import pandas as pd
|
| 9 |
+
|
| 10 |
+
from src.utils.config import AppConfig, get_default_config
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class MultimodalFusion:
|
| 14 |
+
"""Fuses facial and audio features for multimodal deception analysis."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, config: Optional[AppConfig] = None):
|
| 17 |
+
"""Initialize the fusion module.
|
| 18 |
+
|
| 19 |
+
Args:
|
| 20 |
+
config: Optional application configuration.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
self.config = config or get_default_config()
|
| 24 |
+
|
| 25 |
+
def align_features(self, facial_df: pd.DataFrame, audio_df: pd.DataFrame, video_fps: float = 30.0) -> pd.DataFrame:
|
| 26 |
+
"""Align facial frame-level and audio segment-level features by time.
|
| 27 |
+
|
| 28 |
+
Args:
|
| 29 |
+
facial_df: Frame-level facial feature DataFrame.
|
| 30 |
+
audio_df: Segment-level audio feature DataFrame with start and end times.
|
| 31 |
+
video_fps: Fallback video FPS if timestamps are not present.
|
| 32 |
+
|
| 33 |
+
Returns:
|
| 34 |
+
Combined DataFrame aligned to facial frame granularity.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
if facial_df.empty:
|
| 38 |
+
return facial_df.copy()
|
| 39 |
+
aligned = facial_df.copy().reset_index(drop=True)
|
| 40 |
+
if "timestamp" not in aligned.columns:
|
| 41 |
+
aligned["timestamp"] = aligned.get("frame_idx", pd.Series(aligned.index)).astype(float) / float(video_fps)
|
| 42 |
+
if audio_df.empty:
|
| 43 |
+
return aligned
|
| 44 |
+
|
| 45 |
+
audio_features = [col for col in audio_df.columns if col not in {"segment_idx", "start_time", "end_time"}]
|
| 46 |
+
for column in audio_features:
|
| 47 |
+
aligned[column] = np.nan
|
| 48 |
+
|
| 49 |
+
for idx, timestamp in aligned["timestamp"].items():
|
| 50 |
+
matches = audio_df[(audio_df["start_time"] <= timestamp) & (timestamp < audio_df["end_time"])]
|
| 51 |
+
if matches.empty and timestamp == aligned["timestamp"].iloc[-1]:
|
| 52 |
+
matches = audio_df[audio_df["end_time"] <= timestamp].tail(1)
|
| 53 |
+
if not matches.empty:
|
| 54 |
+
aligned.loc[idx, audio_features] = matches.iloc[0][audio_features].to_numpy(dtype=float)
|
| 55 |
+
return aligned.ffill().bfill()
|
| 56 |
+
|
| 57 |
+
@staticmethod
|
| 58 |
+
def _rolling_corr(left: pd.Series, right: pd.Series, window: int = 30) -> pd.Series:
|
| 59 |
+
"""Compute robust rolling correlation with finite fallbacks."""
|
| 60 |
+
|
| 61 |
+
corr = left.rolling(window=window, min_periods=max(3, window // 5)).corr(right)
|
| 62 |
+
return corr.replace([np.inf, -np.inf], np.nan).fillna(0.0)
|
| 63 |
+
|
| 64 |
+
@staticmethod
|
| 65 |
+
def _zscore(series: pd.Series) -> pd.Series:
|
| 66 |
+
"""Return z-scored values with zero variance protection."""
|
| 67 |
+
|
| 68 |
+
std = float(series.std(skipna=True) or 0.0)
|
| 69 |
+
if std <= 1e-8:
|
| 70 |
+
return pd.Series(np.zeros(len(series)), index=series.index)
|
| 71 |
+
return (series.fillna(series.mean()) - float(series.mean())) / std
|
| 72 |
+
|
| 73 |
+
def extract_fusion_features(self, aligned_df: pd.DataFrame) -> pd.DataFrame:
|
| 74 |
+
"""Compute cross-modal fusion features.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
aligned_df: DataFrame containing aligned facial and audio features.
|
| 78 |
+
|
| 79 |
+
Returns:
|
| 80 |
+
DataFrame with original columns plus fusion-specific features.
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
fused = aligned_df.copy()
|
| 84 |
+
if fused.empty:
|
| 85 |
+
for name in self.get_fusion_feature_names():
|
| 86 |
+
fused[name] = []
|
| 87 |
+
return fused
|
| 88 |
+
|
| 89 |
+
mouth = fused.get("mouth_openness", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
|
| 90 |
+
energy = fused.get("energy_mean", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
|
| 91 |
+
eye = fused.get("eye_left_openness", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
|
| 92 |
+
pause = fused.get("pause_count", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
|
| 93 |
+
pitch = fused.get("pitch_mean", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
|
| 94 |
+
brow = fused.get("brow_tension", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
|
| 95 |
+
|
| 96 |
+
# Audio-visual synchrony flags mismatches between visible articulation and speech energy.
|
| 97 |
+
fused["audio_visual_sync"] = self._rolling_corr(mouth, energy)
|
| 98 |
+
# Pause-gaze coupling captures whether vocal hesitation coincides with eye behavior changes.
|
| 99 |
+
fused["pause_gaze_coupling"] = self._rolling_corr(eye, pause)
|
| 100 |
+
# Composite stress combines vocal pitch, facial brow tension, and energy spikes.
|
| 101 |
+
fused["stress_composite"] = (0.4 * self._zscore(pitch) + 0.35 * self._zscore(brow) + 0.25 * self._zscore(energy)).astype(float)
|
| 102 |
+
return fused
|
| 103 |
+
|
| 104 |
+
def get_fusion_feature_names(self) -> List[str]:
|
| 105 |
+
"""Return names of fusion-specific features."""
|
| 106 |
+
|
| 107 |
+
return ["audio_visual_sync", "pause_gaze_coupling", "stress_composite"]
|
| 108 |
+
|
src/evaluation/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluation module for model assessment and validation."""
|
| 2 |
+
|
| 3 |
+
from .metrics import DeceptionMetrics
|
| 4 |
+
from .cross_validation import CrossValidator
|
| 5 |
+
from .ablation import AblationStudy
|
| 6 |
+
from .visualizer import EvaluationVisualizer
|
| 7 |
+
|
| 8 |
+
__all__ = [
|
| 9 |
+
'DeceptionMetrics',
|
| 10 |
+
'CrossValidator',
|
| 11 |
+
'AblationStudy',
|
| 12 |
+
'EvaluationVisualizer',
|
| 13 |
+
]
|
src/evaluation/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (448 Bytes). View file
|
|
|
src/evaluation/__pycache__/ablation.cpython-314.pyc
ADDED
|
Binary file (12.3 kB). View file
|
|
|
src/evaluation/__pycache__/cross_validation.cpython-314.pyc
ADDED
|
Binary file (12.7 kB). View file
|
|
|
src/evaluation/__pycache__/metrics.cpython-314.pyc
ADDED
|
Binary file (15.1 kB). View file
|
|
|
src/evaluation/__pycache__/visualizer.cpython-314.pyc
ADDED
|
Binary file (18.4 kB). View file
|
|
|
src/evaluation/ablation.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Ablation study for feature group importance in deception detection.
|
| 2 |
+
|
| 3 |
+
This module provides systematic removal of feature groups (facial, audio,
|
| 4 |
+
temporal, cross‑modal) to quantify their contribution to model performance.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import logging
|
| 10 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import pandas as pd
|
| 14 |
+
from sklearn.base import BaseEstimator, clone
|
| 15 |
+
|
| 16 |
+
from .cross_validation import CrossValidator
|
| 17 |
+
from .metrics import DeceptionMetrics
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class AblationStudy:
|
| 24 |
+
"""Conduct ablation studies by removing feature groups.
|
| 25 |
+
|
| 26 |
+
This class evaluates the contribution of different feature groups
|
| 27 |
+
(facial, audio, temporal, cross‑modal) to overall model performance.
|
| 28 |
+
|
| 29 |
+
Attributes:
|
| 30 |
+
feature_names: List of all feature names in the dataset.
|
| 31 |
+
feature_groups: Dictionary mapping group names to lists of feature indices or names.
|
| 32 |
+
random_state: Random seed for reproducibility.
|
| 33 |
+
n_splits: Number of cross‑validation folds.
|
| 34 |
+
metrics: Instance of DeceptionMetrics for evaluation.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def __init__(
|
| 38 |
+
self,
|
| 39 |
+
feature_names: List[str],
|
| 40 |
+
feature_groups: Dict[str, List[str]],
|
| 41 |
+
random_state: int = 42,
|
| 42 |
+
n_splits: int = 5,
|
| 43 |
+
metrics: Optional[DeceptionMetrics] = None,
|
| 44 |
+
) -> None:
|
| 45 |
+
"""Initialize ablation study.
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
feature_names: List of all feature names in the dataset.
|
| 49 |
+
feature_groups: Dictionary mapping group names to lists of feature names.
|
| 50 |
+
Expected keys: 'facial', 'audio', 'temporal', 'cross_modal'.
|
| 51 |
+
random_state: Random seed for reproducibility.
|
| 52 |
+
n_splits: Number of cross‑validation folds.
|
| 53 |
+
metrics: Optional DeceptionMetrics instance.
|
| 54 |
+
"""
|
| 55 |
+
self.feature_names = feature_names
|
| 56 |
+
self.feature_groups = feature_groups
|
| 57 |
+
self.random_state = random_state
|
| 58 |
+
self.n_splits = n_splits
|
| 59 |
+
self.metrics = metrics or DeceptionMetrics(random_state=random_state)
|
| 60 |
+
|
| 61 |
+
# Validate that all group features are in feature_names
|
| 62 |
+
for group_name, features in feature_groups.items():
|
| 63 |
+
for f in features:
|
| 64 |
+
if f not in feature_names:
|
| 65 |
+
logger.warning(
|
| 66 |
+
f"Feature '{f}' from group '{group_name}' not found in feature_names"
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
# Build index mapping
|
| 70 |
+
self.feature_to_index = {name: i for i, name in enumerate(feature_names)}
|
| 71 |
+
self.group_indices = {}
|
| 72 |
+
for group_name, features in feature_groups.items():
|
| 73 |
+
indices = []
|
| 74 |
+
for f in features:
|
| 75 |
+
if f in self.feature_to_index:
|
| 76 |
+
indices.append(self.feature_to_index[f])
|
| 77 |
+
self.group_indices[group_name] = indices
|
| 78 |
+
|
| 79 |
+
def _select_features(
|
| 80 |
+
self,
|
| 81 |
+
X: Union[np.ndarray, pd.DataFrame],
|
| 82 |
+
include_groups: Optional[List[str]] = None,
|
| 83 |
+
exclude_groups: Optional[List[str]] = None,
|
| 84 |
+
) -> Union[np.ndarray, pd.DataFrame]:
|
| 85 |
+
"""Select features based on included/excluded groups.
|
| 86 |
+
|
| 87 |
+
Args:
|
| 88 |
+
X: Original feature matrix (n_samples × n_features).
|
| 89 |
+
include_groups: List of group names to keep (all others removed).
|
| 90 |
+
exclude_groups: List of group names to remove (all others kept).
|
| 91 |
+
|
| 92 |
+
Returns:
|
| 93 |
+
Feature matrix with selected columns.
|
| 94 |
+
"""
|
| 95 |
+
if include_groups is not None and exclude_groups is not None:
|
| 96 |
+
raise ValueError("Cannot specify both include_groups and exclude_groups")
|
| 97 |
+
|
| 98 |
+
if isinstance(X, pd.DataFrame):
|
| 99 |
+
columns = X.columns.tolist()
|
| 100 |
+
else:
|
| 101 |
+
columns = list(range(X.shape[1]))
|
| 102 |
+
|
| 103 |
+
# Determine which indices to keep
|
| 104 |
+
if include_groups is not None:
|
| 105 |
+
keep_indices = set()
|
| 106 |
+
for group in include_groups:
|
| 107 |
+
keep_indices.update(self.group_indices.get(group, []))
|
| 108 |
+
elif exclude_groups is not None:
|
| 109 |
+
all_indices = set(range(len(self.feature_names)))
|
| 110 |
+
remove_indices = set()
|
| 111 |
+
for group in exclude_groups:
|
| 112 |
+
remove_indices.update(self.group_indices.get(group, []))
|
| 113 |
+
keep_indices = all_indices - remove_indices
|
| 114 |
+
else:
|
| 115 |
+
# Keep all features
|
| 116 |
+
keep_indices = set(range(len(self.feature_names)))
|
| 117 |
+
|
| 118 |
+
# Convert to sorted list
|
| 119 |
+
keep_indices = sorted(keep_indices)
|
| 120 |
+
|
| 121 |
+
if isinstance(X, pd.DataFrame):
|
| 122 |
+
selected_cols = [columns[i] for i in keep_indices]
|
| 123 |
+
return X[selected_cols]
|
| 124 |
+
else:
|
| 125 |
+
return X[:, keep_indices]
|
| 126 |
+
|
| 127 |
+
def run_ablation(
|
| 128 |
+
self,
|
| 129 |
+
classifier_class: type,
|
| 130 |
+
X: Union[np.ndarray, pd.DataFrame],
|
| 131 |
+
y: Union[np.ndarray, List[int]],
|
| 132 |
+
feature_names: Optional[List[str]] = None,
|
| 133 |
+
feature_groups: Optional[Dict[str, List[str]]] = None,
|
| 134 |
+
n_splits: Optional[int] = None,
|
| 135 |
+
subject_ids: Optional[Union[np.ndarray, List[str]]] = None,
|
| 136 |
+
cv_method: str = "stratified",
|
| 137 |
+
) -> Dict[str, Dict[str, Any]]:
|
| 138 |
+
"""Run ablation study across feature groups.
|
| 139 |
+
|
| 140 |
+
Args:
|
| 141 |
+
classifier_class: A scikit‑learn classifier class (not instance).
|
| 142 |
+
X: Feature matrix.
|
| 143 |
+
y: Target labels.
|
| 144 |
+
feature_names: Override stored feature names.
|
| 145 |
+
feature_groups: Override stored feature groups.
|
| 146 |
+
n_splits: Override default number of folds.
|
| 147 |
+
subject_ids: Optional subject IDs for subject‑aware CV.
|
| 148 |
+
cv_method: Cross‑validation method ('stratified', 'subject_aware', 'loso').
|
| 149 |
+
|
| 150 |
+
Returns:
|
| 151 |
+
Dictionary with keys:
|
| 152 |
+
- full_model: metrics with all features
|
| 153 |
+
- minus_facial: metrics without facial features
|
| 154 |
+
- minus_audio: metrics without audio features
|
| 155 |
+
- minus_temporal: metrics without temporal features
|
| 156 |
+
- minus_cross_modal: metrics without cross‑modal features
|
| 157 |
+
- facial_only: metrics with only facial features
|
| 158 |
+
- audio_only: metrics with only audio features
|
| 159 |
+
- temporal_only: metrics with only temporal features
|
| 160 |
+
Each value is a dict with 'mean_metrics', 'std_metrics', 'per_fold_metrics'.
|
| 161 |
+
"""
|
| 162 |
+
if feature_names is not None:
|
| 163 |
+
self.feature_names = feature_names
|
| 164 |
+
if feature_groups is not None:
|
| 165 |
+
self.feature_groups = feature_groups
|
| 166 |
+
self.group_indices = {}
|
| 167 |
+
for group_name, features in feature_groups.items():
|
| 168 |
+
indices = []
|
| 169 |
+
for f in features:
|
| 170 |
+
if f in self.feature_to_index:
|
| 171 |
+
indices.append(self.feature_to_index[f])
|
| 172 |
+
self.group_indices[group_name] = indices
|
| 173 |
+
|
| 174 |
+
n_splits = n_splits or self.n_splits
|
| 175 |
+
cv = CrossValidator(
|
| 176 |
+
random_state=self.random_state,
|
| 177 |
+
n_splits=n_splits,
|
| 178 |
+
metrics=self.metrics,
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
# Define ablation conditions
|
| 182 |
+
conditions = {
|
| 183 |
+
"full_model": {"include_groups": None, "exclude_groups": None},
|
| 184 |
+
"minus_facial": {"exclude_groups": ["facial"]},
|
| 185 |
+
"minus_audio": {"exclude_groups": ["audio"]},
|
| 186 |
+
"minus_temporal": {"exclude_groups": ["temporal"]},
|
| 187 |
+
"minus_cross_modal": {"exclude_groups": ["cross_modal"]},
|
| 188 |
+
"facial_only": {"include_groups": ["facial"]},
|
| 189 |
+
"audio_only": {"include_groups": ["audio"]},
|
| 190 |
+
"temporal_only": {"include_groups": ["temporal"]},
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
results = {}
|
| 194 |
+
|
| 195 |
+
for cond_name, cond_args in conditions.items():
|
| 196 |
+
logger.info(f"Running ablation condition: {cond_name}")
|
| 197 |
+
|
| 198 |
+
# Select features
|
| 199 |
+
X_cond = self._select_features(
|
| 200 |
+
X,
|
| 201 |
+
include_groups=cond_args.get("include_groups"),
|
| 202 |
+
exclude_groups=cond_args.get("exclude_groups"),
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
# Skip if no features remain
|
| 206 |
+
if X_cond.shape[1] == 0:
|
| 207 |
+
logger.warning(f"No features left for condition {cond_name}, skipping")
|
| 208 |
+
results[cond_name] = {
|
| 209 |
+
"mean_metrics": {},
|
| 210 |
+
"std_metrics": {},
|
| 211 |
+
"per_fold_metrics": [],
|
| 212 |
+
}
|
| 213 |
+
continue
|
| 214 |
+
|
| 215 |
+
# Run cross‑validation
|
| 216 |
+
classifier = classifier_class()
|
| 217 |
+
cv_result = cv.run_cross_validation(
|
| 218 |
+
classifier,
|
| 219 |
+
X_cond,
|
| 220 |
+
y,
|
| 221 |
+
n_splits=n_splits,
|
| 222 |
+
subject_ids=subject_ids,
|
| 223 |
+
method=cv_method,
|
| 224 |
+
return_predictions=False,
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
results[cond_name] = {
|
| 228 |
+
"mean_metrics": cv_result["mean_metrics"],
|
| 229 |
+
"std_metrics": cv_result["std_metrics"],
|
| 230 |
+
"per_fold_metrics": cv_result["per_fold_metrics"],
|
| 231 |
+
"n_features": X_cond.shape[1],
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
logger.info("Ablation study completed.")
|
| 235 |
+
return results
|
| 236 |
+
|
| 237 |
+
def compute_feature_importance_ranking(
|
| 238 |
+
self,
|
| 239 |
+
classifier: BaseEstimator,
|
| 240 |
+
feature_names: Optional[List[str]] = None,
|
| 241 |
+
) -> List[Tuple[str, float]]:
|
| 242 |
+
"""Compute feature importance ranking from a trained classifier.
|
| 243 |
+
|
| 244 |
+
Supports classifiers with `feature_importances_` attribute (tree‑based)
|
| 245 |
+
or `coef_` attribute (linear models).
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
classifier: Trained scikit‑learn classifier.
|
| 249 |
+
feature_names: Override stored feature names.
|
| 250 |
+
|
| 251 |
+
Returns:
|
| 252 |
+
List of (feature_name, importance_score) sorted descending.
|
| 253 |
+
"""
|
| 254 |
+
if feature_names is None:
|
| 255 |
+
feature_names = self.feature_names
|
| 256 |
+
|
| 257 |
+
if hasattr(classifier, "feature_importances_"):
|
| 258 |
+
importances = classifier.feature_importances_
|
| 259 |
+
elif hasattr(classifier, "coef_"):
|
| 260 |
+
# For linear models, take absolute coefficients averaged across classes
|
| 261 |
+
coef = classifier.coef_
|
| 262 |
+
if coef.ndim == 2:
|
| 263 |
+
importances = np.mean(np.abs(coef), axis=0)
|
| 264 |
+
else:
|
| 265 |
+
importances = np.abs(coef)
|
| 266 |
+
else:
|
| 267 |
+
logger.warning(
|
| 268 |
+
"Classifier has no feature_importances_ or coef_; returning uniform importances"
|
| 269 |
+
)
|
| 270 |
+
importances = np.ones(len(feature_names))
|
| 271 |
+
|
| 272 |
+
# Ensure length matches
|
| 273 |
+
if len(importances) != len(feature_names):
|
| 274 |
+
logger.error(
|
| 275 |
+
f"Importance length ({len(importances)}) != feature count ({len(feature_names)})"
|
| 276 |
+
)
|
| 277 |
+
# Truncate or pad
|
| 278 |
+
if len(importances) > len(feature_names):
|
| 279 |
+
importances = importances[: len(feature_names)]
|
| 280 |
+
else:
|
| 281 |
+
importances = np.pad(
|
| 282 |
+
importances,
|
| 283 |
+
(0, len(feature_names) - len(importances)),
|
| 284 |
+
mode="constant",
|
| 285 |
+
constant_values=0.0,
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
ranked = sorted(
|
| 289 |
+
zip(feature_names, importances),
|
| 290 |
+
key=lambda x: x[1],
|
| 291 |
+
reverse=True,
|
| 292 |
+
)
|
| 293 |
+
logger.info(f"Computed feature importance ranking for {len(ranked)} features")
|
| 294 |
+
return ranked
|
src/evaluation/cross_validation.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cross‑validation strategies for deception detection.
|
| 2 |
+
|
| 3 |
+
This module provides subject‑aware and stratified cross‑validation
|
| 4 |
+
splits suitable for multimodal deception analysis.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import logging
|
| 10 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import pandas as pd
|
| 14 |
+
from sklearn.model_selection import (
|
| 15 |
+
StratifiedKFold,
|
| 16 |
+
GroupKFold,
|
| 17 |
+
LeaveOneGroupOut,
|
| 18 |
+
cross_val_predict,
|
| 19 |
+
)
|
| 20 |
+
from sklearn.base import BaseEstimator
|
| 21 |
+
|
| 22 |
+
from .metrics import DeceptionMetrics
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
logger = logging.getLogger(__name__)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class CrossValidator:
|
| 29 |
+
"""Cross‑validation with stratification and subject‑aware splits.
|
| 30 |
+
|
| 31 |
+
This class provides methods for performing k‑fold cross‑validation
|
| 32 |
+
while respecting subject boundaries (no data leakage) and maintaining
|
| 33 |
+
class balance.
|
| 34 |
+
|
| 35 |
+
Attributes:
|
| 36 |
+
random_state: Random seed for reproducibility.
|
| 37 |
+
n_splits: Default number of folds.
|
| 38 |
+
metrics: Instance of DeceptionMetrics for evaluation.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
def __init__(
|
| 42 |
+
self,
|
| 43 |
+
random_state: int = 42,
|
| 44 |
+
n_splits: int = 5,
|
| 45 |
+
metrics: Optional[DeceptionMetrics] = None,
|
| 46 |
+
) -> None:
|
| 47 |
+
"""Initialize cross‑validator.
|
| 48 |
+
|
| 49 |
+
Args:
|
| 50 |
+
random_state: Random seed for reproducibility.
|
| 51 |
+
n_splits: Default number of folds.
|
| 52 |
+
metrics: Optional DeceptionMetrics instance. If None, a default is created.
|
| 53 |
+
"""
|
| 54 |
+
self.random_state = random_state
|
| 55 |
+
self.n_splits = n_splits
|
| 56 |
+
self.metrics = metrics or DeceptionMetrics(random_state=random_state)
|
| 57 |
+
|
| 58 |
+
def stratified_k_fold(
|
| 59 |
+
self,
|
| 60 |
+
X: Union[np.ndarray, pd.DataFrame],
|
| 61 |
+
y: Union[np.ndarray, List[int]],
|
| 62 |
+
n_splits: Optional[int] = None,
|
| 63 |
+
random_state: Optional[int] = None,
|
| 64 |
+
) -> List[Tuple[np.ndarray, np.ndarray]]:
|
| 65 |
+
"""Generate stratified k‑fold indices.
|
| 66 |
+
|
| 67 |
+
Args:
|
| 68 |
+
X: Feature matrix (any shape, only length matters).
|
| 69 |
+
y: Target labels used for stratification.
|
| 70 |
+
n_splits: Override default number of folds.
|
| 71 |
+
random_state: Override default random seed.
|
| 72 |
+
|
| 73 |
+
Returns:
|
| 74 |
+
List of (train_indices, test_indices) for each fold.
|
| 75 |
+
"""
|
| 76 |
+
n_splits = n_splits or self.n_splits
|
| 77 |
+
random_state = random_state or self.random_state
|
| 78 |
+
|
| 79 |
+
y = np.asarray(y)
|
| 80 |
+
skf = StratifiedKFold(
|
| 81 |
+
n_splits=n_splits, shuffle=True, random_state=random_state
|
| 82 |
+
)
|
| 83 |
+
splits = []
|
| 84 |
+
for train_idx, test_idx in skf.split(X, y):
|
| 85 |
+
splits.append((train_idx, test_idx))
|
| 86 |
+
|
| 87 |
+
logger.info(
|
| 88 |
+
f"Generated {n_splits} stratified folds "
|
| 89 |
+
f"(random_state={random_state}, n_samples={len(y)})"
|
| 90 |
+
)
|
| 91 |
+
return splits
|
| 92 |
+
|
| 93 |
+
def subject_aware_split(
|
| 94 |
+
self,
|
| 95 |
+
X: Union[np.ndarray, pd.DataFrame],
|
| 96 |
+
y: Union[np.ndarray, List[int]],
|
| 97 |
+
subject_ids: Union[np.ndarray, List[str]],
|
| 98 |
+
n_splits: Optional[int] = None,
|
| 99 |
+
) -> List[Tuple[np.ndarray, np.ndarray]]:
|
| 100 |
+
"""Generate k‑fold splits where the same subject does not appear in both train and test.
|
| 101 |
+
|
| 102 |
+
Uses GroupKFold with subject_ids as groups.
|
| 103 |
+
|
| 104 |
+
Args:
|
| 105 |
+
X: Feature matrix.
|
| 106 |
+
y: Target labels (unused for grouping, kept for compatibility).
|
| 107 |
+
subject_ids: Subject identifier for each sample.
|
| 108 |
+
n_splits: Override default number of folds.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
List of (train_indices, test_indices) for each fold.
|
| 112 |
+
"""
|
| 113 |
+
n_splits = n_splits or self.n_splits
|
| 114 |
+
subject_ids = np.asarray(subject_ids)
|
| 115 |
+
|
| 116 |
+
# Ensure we have enough unique groups
|
| 117 |
+
unique_subjects = np.unique(subject_ids)
|
| 118 |
+
if len(unique_subjects) < n_splits:
|
| 119 |
+
logger.warning(
|
| 120 |
+
f"Only {len(unique_subjects)} unique subjects, reducing n_splits to {len(unique_subjects)}"
|
| 121 |
+
)
|
| 122 |
+
n_splits = len(unique_subjects)
|
| 123 |
+
|
| 124 |
+
gkf = GroupKFold(n_splits=n_splits)
|
| 125 |
+
splits = []
|
| 126 |
+
for train_idx, test_idx in gkf.split(X, y, groups=subject_ids):
|
| 127 |
+
splits.append((train_idx, test_idx))
|
| 128 |
+
|
| 129 |
+
logger.info(
|
| 130 |
+
f"Generated {n_splits} subject‑aware folds "
|
| 131 |
+
f"(unique subjects={len(unique_subjects)})"
|
| 132 |
+
)
|
| 133 |
+
return splits
|
| 134 |
+
|
| 135 |
+
def leave_one_subject_out(
|
| 136 |
+
self,
|
| 137 |
+
X: Union[np.ndarray, pd.DataFrame],
|
| 138 |
+
y: Union[np.ndarray, List[int]],
|
| 139 |
+
subject_ids: Union[np.ndarray, List[str]],
|
| 140 |
+
) -> List[Tuple[np.ndarray, np.ndarray]]:
|
| 141 |
+
"""Leave‑One‑Subject‑Out (LOSO) cross‑validation generator.
|
| 142 |
+
|
| 143 |
+
Each fold uses all subjects except one as training, and the left‑out
|
| 144 |
+
subject as test.
|
| 145 |
+
|
| 146 |
+
Args:
|
| 147 |
+
X: Feature matrix.
|
| 148 |
+
y: Target labels.
|
| 149 |
+
subject_ids: Subject identifier for each sample.
|
| 150 |
+
|
| 151 |
+
Returns:
|
| 152 |
+
List of (train_indices, test_indices) for each fold.
|
| 153 |
+
"""
|
| 154 |
+
subject_ids = np.asarray(subject_ids)
|
| 155 |
+
unique_subjects = np.unique(subject_ids)
|
| 156 |
+
|
| 157 |
+
splits = []
|
| 158 |
+
for subj in unique_subjects:
|
| 159 |
+
test_mask = subject_ids == subj
|
| 160 |
+
train_mask = ~test_mask
|
| 161 |
+
train_idx = np.where(train_mask)[0]
|
| 162 |
+
test_idx = np.where(test_mask)[0]
|
| 163 |
+
splits.append((train_idx, test_idx))
|
| 164 |
+
|
| 165 |
+
logger.info(
|
| 166 |
+
f"Generated {len(splits)} LOSO folds (unique subjects={len(unique_subjects)})"
|
| 167 |
+
)
|
| 168 |
+
return splits
|
| 169 |
+
|
| 170 |
+
def run_cross_validation(
|
| 171 |
+
self,
|
| 172 |
+
classifier: BaseEstimator,
|
| 173 |
+
X: Union[np.ndarray, pd.DataFrame],
|
| 174 |
+
y: Union[np.ndarray, List[int]],
|
| 175 |
+
n_splits: Optional[int] = None,
|
| 176 |
+
subject_ids: Optional[Union[np.ndarray, List[str]]] = None,
|
| 177 |
+
method: str = "stratified",
|
| 178 |
+
return_predictions: bool = True,
|
| 179 |
+
) -> Dict[str, Any]:
|
| 180 |
+
"""Run cross‑validation and compute metrics across folds.
|
| 181 |
+
|
| 182 |
+
Args:
|
| 183 |
+
classifier: A scikit‑learn compatible classifier.
|
| 184 |
+
X: Feature matrix.
|
| 185 |
+
y: Target labels.
|
| 186 |
+
n_splits: Override default number of folds.
|
| 187 |
+
subject_ids: Required for 'subject_aware' and 'loso' methods.
|
| 188 |
+
method: One of 'stratified', 'subject_aware', 'loso'.
|
| 189 |
+
return_predictions: If True, aggregate predictions across folds.
|
| 190 |
+
|
| 191 |
+
Returns:
|
| 192 |
+
Dictionary with:
|
| 193 |
+
- mean_metrics: dict of mean metric values across folds
|
| 194 |
+
- std_metrics: dict of standard deviations
|
| 195 |
+
- per_fold_metrics: list of metric dicts for each fold
|
| 196 |
+
- aggregated_predictions: dict with y_true, y_pred, y_prob (if return_predictions)
|
| 197 |
+
- fold_indices: list of (train_idx, test_idx) for each fold
|
| 198 |
+
"""
|
| 199 |
+
n_splits = n_splits or self.n_splits
|
| 200 |
+
y = np.asarray(y)
|
| 201 |
+
|
| 202 |
+
# Generate fold indices
|
| 203 |
+
if method == "stratified":
|
| 204 |
+
folds = self.stratified_k_fold(X, y, n_splits=n_splits)
|
| 205 |
+
elif method == "subject_aware":
|
| 206 |
+
if subject_ids is None:
|
| 207 |
+
raise ValueError("subject_ids required for subject_aware method")
|
| 208 |
+
folds = self.subject_aware_split(X, y, subject_ids, n_splits=n_splits)
|
| 209 |
+
elif method == "loso":
|
| 210 |
+
if subject_ids is None:
|
| 211 |
+
raise ValueError("subject_ids required for loso method")
|
| 212 |
+
folds = self.leave_one_subject_out(X, y, subject_ids)
|
| 213 |
+
else:
|
| 214 |
+
raise ValueError(
|
| 215 |
+
f"Unknown method '{method}'. Choose from 'stratified', 'subject_aware', 'loso'."
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
per_fold_metrics = []
|
| 219 |
+
all_y_true = []
|
| 220 |
+
all_y_pred = []
|
| 221 |
+
all_y_prob = []
|
| 222 |
+
fold_indices = []
|
| 223 |
+
|
| 224 |
+
for fold_idx, (train_idx, test_idx) in enumerate(folds):
|
| 225 |
+
logger.info(f"Fold {fold_idx+1}/{len(folds)}: train={len(train_idx)}, test={len(test_idx)}")
|
| 226 |
+
|
| 227 |
+
X_train, X_test = X[train_idx], X[test_idx]
|
| 228 |
+
y_train, y_test = y[train_idx], y[test_idx]
|
| 229 |
+
|
| 230 |
+
# Train classifier
|
| 231 |
+
classifier.fit(X_train, y_train)
|
| 232 |
+
|
| 233 |
+
# Predict
|
| 234 |
+
y_pred = classifier.predict(X_test)
|
| 235 |
+
y_prob = (
|
| 236 |
+
classifier.predict_proba(X_test)[:, 1]
|
| 237 |
+
if hasattr(classifier, "predict_proba")
|
| 238 |
+
else None
|
| 239 |
+
)
|
| 240 |
+
|
| 241 |
+
# Compute metrics
|
| 242 |
+
metrics = self.metrics.compute_all_metrics(y_test, y_pred, y_prob)
|
| 243 |
+
per_fold_metrics.append(metrics)
|
| 244 |
+
|
| 245 |
+
# Aggregate for overall analysis
|
| 246 |
+
all_y_true.extend(y_test.tolist())
|
| 247 |
+
all_y_pred.extend(y_pred.tolist())
|
| 248 |
+
if y_prob is not None:
|
| 249 |
+
all_y_prob.extend(y_prob.tolist())
|
| 250 |
+
|
| 251 |
+
fold_indices.append((train_idx.tolist(), test_idx.tolist()))
|
| 252 |
+
|
| 253 |
+
# Compute mean and std across folds
|
| 254 |
+
metric_names = [
|
| 255 |
+
"accuracy",
|
| 256 |
+
"precision",
|
| 257 |
+
"recall",
|
| 258 |
+
"f1",
|
| 259 |
+
"specificity",
|
| 260 |
+
"balanced_accuracy",
|
| 261 |
+
]
|
| 262 |
+
if any("auc_roc" in m for m in per_fold_metrics):
|
| 263 |
+
metric_names.append("auc_roc")
|
| 264 |
+
if any("average_precision" in m for m in per_fold_metrics):
|
| 265 |
+
metric_names.append("average_precision")
|
| 266 |
+
|
| 267 |
+
mean_metrics = {}
|
| 268 |
+
std_metrics = {}
|
| 269 |
+
for name in metric_names:
|
| 270 |
+
values = []
|
| 271 |
+
for fold_metrics in per_fold_metrics:
|
| 272 |
+
if name in fold_metrics:
|
| 273 |
+
values.append(fold_metrics[name])
|
| 274 |
+
if values:
|
| 275 |
+
mean_metrics[name] = float(np.mean(values))
|
| 276 |
+
std_metrics[name] = float(np.std(values))
|
| 277 |
+
else:
|
| 278 |
+
mean_metrics[name] = np.nan
|
| 279 |
+
std_metrics[name] = np.nan
|
| 280 |
+
|
| 281 |
+
# Aggregate predictions
|
| 282 |
+
aggregated = {}
|
| 283 |
+
if return_predictions:
|
| 284 |
+
aggregated["y_true"] = all_y_true
|
| 285 |
+
aggregated["y_pred"] = all_y_pred
|
| 286 |
+
if all_y_prob:
|
| 287 |
+
aggregated["y_prob"] = all_y_prob
|
| 288 |
+
|
| 289 |
+
result = {
|
| 290 |
+
"mean_metrics": mean_metrics,
|
| 291 |
+
"std_metrics": std_metrics,
|
| 292 |
+
"per_fold_metrics": per_fold_metrics,
|
| 293 |
+
"aggregated_predictions": aggregated,
|
| 294 |
+
"fold_indices": fold_indices,
|
| 295 |
+
"method": method,
|
| 296 |
+
"n_splits": len(folds),
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
logger.info(
|
| 300 |
+
f"Cross‑validation completed ({method}, {len(folds)} folds). "
|
| 301 |
+
f"Mean accuracy = {mean_metrics.get('accuracy', 0):.3f} ± {std_metrics.get('accuracy', 0):.3f}"
|
| 302 |
+
)
|
| 303 |
+
return result
|
src/evaluation/metrics.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Comprehensive evaluation metrics for deception detection.
|
| 2 |
+
|
| 3 |
+
This module provides a DeceptionMetrics class that computes a wide range of
|
| 4 |
+
classification metrics, fairness metrics, and confidence intervals for
|
| 5 |
+
evaluating deception detection models.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import pandas as pd
|
| 15 |
+
from sklearn.metrics import (
|
| 16 |
+
accuracy_score,
|
| 17 |
+
precision_score,
|
| 18 |
+
recall_score,
|
| 19 |
+
f1_score,
|
| 20 |
+
roc_auc_score,
|
| 21 |
+
average_precision_score,
|
| 22 |
+
confusion_matrix,
|
| 23 |
+
classification_report,
|
| 24 |
+
balanced_accuracy_score,
|
| 25 |
+
)
|
| 26 |
+
from sklearn.utils import resample
|
| 27 |
+
from scipy import stats
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
logger = logging.getLogger(__name__)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class DeceptionMetrics:
|
| 34 |
+
"""Compute and aggregate evaluation metrics for deception detection.
|
| 35 |
+
|
| 36 |
+
This class provides methods for computing standard classification metrics,
|
| 37 |
+
per-subject metrics, fairness metrics, and bootstrap confidence intervals.
|
| 38 |
+
|
| 39 |
+
Attributes:
|
| 40 |
+
random_state: Random seed for reproducibility.
|
| 41 |
+
n_bootstrap: Number of bootstrap samples for confidence intervals.
|
| 42 |
+
ci_level: Confidence level for intervals (default 0.95).
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
def __init__(
|
| 46 |
+
self,
|
| 47 |
+
random_state: int = 42,
|
| 48 |
+
n_bootstrap: int = 1000,
|
| 49 |
+
ci_level: float = 0.95,
|
| 50 |
+
) -> None:
|
| 51 |
+
"""Initialize metrics calculator.
|
| 52 |
+
|
| 53 |
+
Args:
|
| 54 |
+
random_state: Random seed for reproducibility.
|
| 55 |
+
n_bootstrap: Number of bootstrap samples for confidence intervals.
|
| 56 |
+
ci_level: Confidence level for intervals (default 0.95).
|
| 57 |
+
"""
|
| 58 |
+
self.random_state = random_state
|
| 59 |
+
self.n_bootstrap = n_bootstrap
|
| 60 |
+
self.ci_level = ci_level
|
| 61 |
+
np.random.seed(random_state)
|
| 62 |
+
|
| 63 |
+
def compute_all_metrics(
|
| 64 |
+
self,
|
| 65 |
+
y_true: Union[np.ndarray, List[int]],
|
| 66 |
+
y_pred: Union[np.ndarray, List[int]],
|
| 67 |
+
y_prob: Optional[Union[np.ndarray, List[float]]] = None,
|
| 68 |
+
) -> Dict[str, Any]:
|
| 69 |
+
"""Compute comprehensive classification metrics.
|
| 70 |
+
|
| 71 |
+
Args:
|
| 72 |
+
y_true: Ground truth binary labels (0 = truthful, 1 = deceptive).
|
| 73 |
+
y_pred: Predicted binary labels.
|
| 74 |
+
y_prob: Predicted probabilities for the positive class (deceptive).
|
| 75 |
+
If provided, ROC‑AUC and average precision are computed.
|
| 76 |
+
|
| 77 |
+
Returns:
|
| 78 |
+
Dictionary containing:
|
| 79 |
+
- accuracy, precision, recall, f1, specificity
|
| 80 |
+
- balanced_accuracy
|
| 81 |
+
- auc_roc, average_precision (if y_prob provided)
|
| 82 |
+
- confusion_matrix (as 2x2 array)
|
| 83 |
+
- classification_report (as dict)
|
| 84 |
+
"""
|
| 85 |
+
y_true = np.asarray(y_true)
|
| 86 |
+
y_pred = np.asarray(y_pred)
|
| 87 |
+
|
| 88 |
+
if y_true.ndim != 1:
|
| 89 |
+
raise ValueError(f"y_true must be 1‑D, got shape {y_true.shape}")
|
| 90 |
+
if y_pred.ndim != 1:
|
| 91 |
+
raise ValueError(f"y_pred must be 1‑D, got shape {y_pred.shape}")
|
| 92 |
+
if len(y_true) != len(y_pred):
|
| 93 |
+
raise ValueError(
|
| 94 |
+
f"Length mismatch: y_true ({len(y_true)}) vs y_pred ({len(y_pred)})"
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
# Basic metrics
|
| 98 |
+
accuracy = accuracy_score(y_true, y_pred)
|
| 99 |
+
precision = precision_score(y_true, y_pred, zero_division=0)
|
| 100 |
+
recall = recall_score(y_true, y_pred, zero_division=0)
|
| 101 |
+
f1 = f1_score(y_true, y_pred, zero_division=0)
|
| 102 |
+
balanced_acc = balanced_accuracy_score(y_true, y_pred)
|
| 103 |
+
|
| 104 |
+
# Specificity (true negative rate)
|
| 105 |
+
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
|
| 106 |
+
specificity = tn / (tn + fp) if (tn + fp) > 0 else 0.0
|
| 107 |
+
|
| 108 |
+
# Probability‑based metrics
|
| 109 |
+
auc_roc = None
|
| 110 |
+
avg_precision = None
|
| 111 |
+
if y_prob is not None:
|
| 112 |
+
y_prob = np.asarray(y_prob)
|
| 113 |
+
if y_prob.ndim == 2 and y_prob.shape[1] == 2:
|
| 114 |
+
y_prob = y_prob[:, 1] # take positive class probabilities
|
| 115 |
+
try:
|
| 116 |
+
auc_roc = roc_auc_score(y_true, y_prob)
|
| 117 |
+
except ValueError as e:
|
| 118 |
+
logger.warning(f"Could not compute ROC‑AUC: {e}")
|
| 119 |
+
auc_roc = None
|
| 120 |
+
try:
|
| 121 |
+
avg_precision = average_precision_score(y_true, y_prob)
|
| 122 |
+
except ValueError as e:
|
| 123 |
+
logger.warning(f"Could not compute average precision: {e}")
|
| 124 |
+
avg_precision = None
|
| 125 |
+
|
| 126 |
+
# Confusion matrix
|
| 127 |
+
cm = confusion_matrix(y_true, y_pred)
|
| 128 |
+
|
| 129 |
+
# Classification report as dict
|
| 130 |
+
report = classification_report(
|
| 131 |
+
y_true, y_pred, output_dict=True, zero_division=0
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
metrics = {
|
| 135 |
+
"accuracy": float(accuracy),
|
| 136 |
+
"precision": float(precision),
|
| 137 |
+
"recall": float(recall),
|
| 138 |
+
"f1": float(f1),
|
| 139 |
+
"specificity": float(specificity),
|
| 140 |
+
"balanced_accuracy": float(balanced_acc),
|
| 141 |
+
"confusion_matrix": cm.tolist(),
|
| 142 |
+
"classification_report": report,
|
| 143 |
+
"support": int(len(y_true)),
|
| 144 |
+
}
|
| 145 |
+
if auc_roc is not None:
|
| 146 |
+
metrics["auc_roc"] = float(auc_roc)
|
| 147 |
+
if avg_precision is not None:
|
| 148 |
+
metrics["average_precision"] = float(avg_precision)
|
| 149 |
+
|
| 150 |
+
logger.info(
|
| 151 |
+
f"Computed metrics: accuracy={accuracy:.3f}, f1={f1:.3f}, "
|
| 152 |
+
f"recall={recall:.3f}, precision={precision:.3f}"
|
| 153 |
+
)
|
| 154 |
+
return metrics
|
| 155 |
+
|
| 156 |
+
def compute_per_subject_metrics(
|
| 157 |
+
self,
|
| 158 |
+
y_true: Union[np.ndarray, List[int]],
|
| 159 |
+
y_pred: Union[np.ndarray, List[int]],
|
| 160 |
+
subject_ids: Union[np.ndarray, List[str]],
|
| 161 |
+
y_prob: Optional[Union[np.ndarray, List[float]]] = None,
|
| 162 |
+
) -> Dict[str, Dict[str, Any]]:
|
| 163 |
+
"""Compute metrics separately for each subject.
|
| 164 |
+
|
| 165 |
+
Args:
|
| 166 |
+
y_true: Ground truth binary labels.
|
| 167 |
+
y_pred: Predicted binary labels.
|
| 168 |
+
subject_ids: Subject identifier for each sample.
|
| 169 |
+
y_prob: Predicted probabilities (optional).
|
| 170 |
+
|
| 171 |
+
Returns:
|
| 172 |
+
Dictionary mapping subject_id -> metrics dict.
|
| 173 |
+
"""
|
| 174 |
+
y_true = np.asarray(y_true)
|
| 175 |
+
y_pred = np.asarray(y_pred)
|
| 176 |
+
subject_ids = np.asarray(subject_ids)
|
| 177 |
+
|
| 178 |
+
if len(y_true) != len(subject_ids):
|
| 179 |
+
raise ValueError(
|
| 180 |
+
f"Length mismatch: y_true ({len(y_true)}) vs subject_ids ({len(subject_ids)})"
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
unique_subjects = np.unique(subject_ids)
|
| 184 |
+
subject_metrics = {}
|
| 185 |
+
|
| 186 |
+
for subj in unique_subjects:
|
| 187 |
+
mask = subject_ids == subj
|
| 188 |
+
if np.sum(mask) == 0:
|
| 189 |
+
continue
|
| 190 |
+
subj_y_true = y_true[mask]
|
| 191 |
+
subj_y_pred = y_pred[mask]
|
| 192 |
+
subj_y_prob = y_prob[mask] if y_prob is not None else None
|
| 193 |
+
|
| 194 |
+
try:
|
| 195 |
+
metrics = self.compute_all_metrics(
|
| 196 |
+
subj_y_true, subj_y_pred, subj_y_prob
|
| 197 |
+
)
|
| 198 |
+
subject_metrics[subj] = metrics
|
| 199 |
+
except Exception as e:
|
| 200 |
+
logger.warning(f"Failed to compute metrics for subject {subj}: {e}")
|
| 201 |
+
subject_metrics[subj] = {"error": str(e)}
|
| 202 |
+
|
| 203 |
+
logger.info(f"Computed per‑subject metrics for {len(subject_metrics)} subjects")
|
| 204 |
+
return subject_metrics
|
| 205 |
+
|
| 206 |
+
def compute_fairness_metrics(
|
| 207 |
+
self,
|
| 208 |
+
y_true: Union[np.ndarray, List[int]],
|
| 209 |
+
y_pred: Union[np.ndarray, List[int]],
|
| 210 |
+
sensitive_groups: Union[np.ndarray, List[str]],
|
| 211 |
+
) -> Dict[str, float]:
|
| 212 |
+
"""Compute fairness metrics across sensitive groups.
|
| 213 |
+
|
| 214 |
+
Currently implemented:
|
| 215 |
+
- Demographic parity difference: max difference in positive rate across groups.
|
| 216 |
+
- Equalized odds difference: max difference in TPR and FPR across groups.
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
y_true: Ground truth binary labels.
|
| 220 |
+
y_pred: Predicted binary labels.
|
| 221 |
+
sensitive_groups: Group membership for each sample (e.g., gender, ethnicity).
|
| 222 |
+
|
| 223 |
+
Returns:
|
| 224 |
+
Dictionary with fairness metrics.
|
| 225 |
+
"""
|
| 226 |
+
y_true = np.asarray(y_true)
|
| 227 |
+
y_pred = np.asarray(y_pred)
|
| 228 |
+
sensitive_groups = np.asarray(sensitive_groups)
|
| 229 |
+
|
| 230 |
+
unique_groups = np.unique(sensitive_groups)
|
| 231 |
+
group_stats = {}
|
| 232 |
+
|
| 233 |
+
for group in unique_groups:
|
| 234 |
+
mask = sensitive_groups == group
|
| 235 |
+
if np.sum(mask) == 0:
|
| 236 |
+
continue
|
| 237 |
+
group_y_true = y_true[mask]
|
| 238 |
+
group_y_pred = y_pred[mask]
|
| 239 |
+
|
| 240 |
+
# Positive rate (predicted)
|
| 241 |
+
pos_rate = np.mean(group_y_pred)
|
| 242 |
+
|
| 243 |
+
# True positive rate and false positive rate
|
| 244 |
+
tn, fp, fn, tp = confusion_matrix(group_y_true, group_y_pred).ravel()
|
| 245 |
+
tpr = tp / (tp + fn) if (tp + fn) > 0 else 0.0
|
| 246 |
+
fpr = fp / (fp + tn) if (fp + tn) > 0 else 0.0
|
| 247 |
+
|
| 248 |
+
group_stats[group] = {
|
| 249 |
+
"positive_rate": pos_rate,
|
| 250 |
+
"tpr": tpr,
|
| 251 |
+
"fpr": fpr,
|
| 252 |
+
"support": int(np.sum(mask)),
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
# Demographic parity difference
|
| 256 |
+
pos_rates = [stats["positive_rate"] for stats in group_stats.values()]
|
| 257 |
+
demographic_parity_diff = max(pos_rates) - min(pos_rates) if pos_rates else 0.0
|
| 258 |
+
|
| 259 |
+
# Equalized odds differences (max TPR diff, max FPR diff)
|
| 260 |
+
tprs = [stats["tpr"] for stats in group_stats.values()]
|
| 261 |
+
fprs = [stats["fpr"] for stats in group_stats.values()]
|
| 262 |
+
equalized_odds_tpr_diff = max(tprs) - min(tprs) if tprs else 0.0
|
| 263 |
+
equalized_odds_fpr_diff = max(fprs) - min(fprs) if fprs else 0.0
|
| 264 |
+
|
| 265 |
+
fairness = {
|
| 266 |
+
"demographic_parity_difference": float(demographic_parity_diff),
|
| 267 |
+
"equalized_odds_tpr_difference": float(equalized_odds_tpr_diff),
|
| 268 |
+
"equalized_odds_fpr_difference": float(equalized_odds_fpr_diff),
|
| 269 |
+
"group_stats": group_stats,
|
| 270 |
+
}
|
| 271 |
+
logger.info(
|
| 272 |
+
f"Fairness metrics: demographic parity diff = {demographic_parity_diff:.3f}, "
|
| 273 |
+
f"equalized odds TPR diff = {equalized_odds_tpr_diff:.3f}"
|
| 274 |
+
)
|
| 275 |
+
return fairness
|
| 276 |
+
|
| 277 |
+
def bootstrap_confidence_intervals(
|
| 278 |
+
self,
|
| 279 |
+
y_true: Union[np.ndarray, List[int]],
|
| 280 |
+
y_pred: Union[np.ndarray, List[int]],
|
| 281 |
+
y_prob: Optional[Union[np.ndarray, List[float]]] = None,
|
| 282 |
+
n_bootstrap: Optional[int] = None,
|
| 283 |
+
ci: Optional[float] = None,
|
| 284 |
+
) -> Dict[str, Dict[str, float]]:
|
| 285 |
+
"""Compute bootstrap confidence intervals for metrics.
|
| 286 |
+
|
| 287 |
+
Args:
|
| 288 |
+
y_true: Ground truth binary labels.
|
| 289 |
+
y_pred: Predicted binary labels.
|
| 290 |
+
y_prob: Predicted probabilities (optional).
|
| 291 |
+
n_bootstrap: Override default number of bootstrap samples.
|
| 292 |
+
ci: Override default confidence level.
|
| 293 |
+
|
| 294 |
+
Returns:
|
| 295 |
+
Dictionary mapping metric name -> {'lower': ..., 'upper': ..., 'mean': ...}.
|
| 296 |
+
"""
|
| 297 |
+
n_bootstrap = n_bootstrap or self.n_bootstrap
|
| 298 |
+
ci_level = ci or self.ci_level
|
| 299 |
+
y_true = np.asarray(y_true)
|
| 300 |
+
y_pred = np.asarray(y_pred)
|
| 301 |
+
y_prob = np.asarray(y_prob) if y_prob is not None else None
|
| 302 |
+
|
| 303 |
+
n_samples = len(y_true)
|
| 304 |
+
metric_names = [
|
| 305 |
+
"accuracy",
|
| 306 |
+
"precision",
|
| 307 |
+
"recall",
|
| 308 |
+
"f1",
|
| 309 |
+
"specificity",
|
| 310 |
+
"balanced_accuracy",
|
| 311 |
+
]
|
| 312 |
+
if y_prob is not None:
|
| 313 |
+
metric_names.extend(["auc_roc", "average_precision"])
|
| 314 |
+
|
| 315 |
+
bootstrap_metrics = {name: [] for name in metric_names}
|
| 316 |
+
|
| 317 |
+
for i in range(n_bootstrap):
|
| 318 |
+
indices = resample(
|
| 319 |
+
np.arange(n_samples),
|
| 320 |
+
replace=True,
|
| 321 |
+
n_samples=n_samples,
|
| 322 |
+
random_state=self.random_state + i,
|
| 323 |
+
)
|
| 324 |
+
b_y_true = y_true[indices]
|
| 325 |
+
b_y_pred = y_pred[indices]
|
| 326 |
+
b_y_prob = y_prob[indices] if y_prob is not None else None
|
| 327 |
+
|
| 328 |
+
try:
|
| 329 |
+
metrics = self.compute_all_metrics(b_y_true, b_y_pred, b_y_prob)
|
| 330 |
+
for name in metric_names:
|
| 331 |
+
if name in metrics:
|
| 332 |
+
bootstrap_metrics[name].append(metrics[name])
|
| 333 |
+
except Exception as e:
|
| 334 |
+
logger.warning(f"Bootstrap iteration {i} failed: {e}")
|
| 335 |
+
|
| 336 |
+
# Compute confidence intervals
|
| 337 |
+
ci_results = {}
|
| 338 |
+
alpha = (1 - ci_level) / 2
|
| 339 |
+
for name, values in bootstrap_metrics.items():
|
| 340 |
+
if len(values) == 0:
|
| 341 |
+
ci_results[name] = {"lower": np.nan, "upper": np.nan, "mean": np.nan}
|
| 342 |
+
continue
|
| 343 |
+
lower = np.percentile(values, 100 * alpha)
|
| 344 |
+
upper = np.percentile(values, 100 * (1 - alpha))
|
| 345 |
+
mean = np.mean(values)
|
| 346 |
+
ci_results[name] = {
|
| 347 |
+
"lower": float(lower),
|
| 348 |
+
"upper": float(upper),
|
| 349 |
+
"mean": float(mean),
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
logger.info(
|
| 353 |
+
f"Computed bootstrap CIs for {len(ci_results)} metrics "
|
| 354 |
+
f"({n_bootstrap} samples, CI={ci_level})"
|
| 355 |
+
)
|
| 356 |
+
return ci_results
|
src/evaluation/visualizer.py
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Visualization tools for deception detection evaluation.
|
| 2 |
+
|
| 3 |
+
This module provides plotting functions for confusion matrices, ROC curves,
|
| 4 |
+
precision‑recall curves, cross‑validation results, ablation studies, and
|
| 5 |
+
subject‑wise performance.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import matplotlib.pyplot as plt
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
import seaborn as sns
|
| 17 |
+
from matplotlib.figure import Figure
|
| 18 |
+
from sklearn.metrics import (
|
| 19 |
+
roc_curve,
|
| 20 |
+
auc,
|
| 21 |
+
precision_recall_curve,
|
| 22 |
+
average_precision_score,
|
| 23 |
+
confusion_matrix,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
logger = logging.getLogger(__name__)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class EvaluationVisualizer:
|
| 30 |
+
"""Create publication‑quality plots for evaluation results.
|
| 31 |
+
|
| 32 |
+
All methods accept a `save_path` argument; if provided, the figure is saved
|
| 33 |
+
to that path. Otherwise, the figure is returned for further customization.
|
| 34 |
+
|
| 35 |
+
Attributes:
|
| 36 |
+
style: Matplotlib style to use (default 'seaborn‑whitegrid').
|
| 37 |
+
figsize: Default figure size as (width, height).
|
| 38 |
+
dpi: Resolution for saved figures.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
def __init__(
|
| 42 |
+
self,
|
| 43 |
+
style: str = "seaborn‑whitegrid",
|
| 44 |
+
figsize: Tuple[float, float] = (10, 8),
|
| 45 |
+
dpi: int = 150,
|
| 46 |
+
) -> None:
|
| 47 |
+
"""Initialize visualizer.
|
| 48 |
+
|
| 49 |
+
Args:
|
| 50 |
+
style: Matplotlib style name.
|
| 51 |
+
figsize: Default figure size (width, height) in inches.
|
| 52 |
+
dpi: Dots per inch for saved figures.
|
| 53 |
+
"""
|
| 54 |
+
self.style = style
|
| 55 |
+
self.figsize = figsize
|
| 56 |
+
self.dpi = dpi
|
| 57 |
+
plt.style.use(style)
|
| 58 |
+
|
| 59 |
+
def plot_confusion_matrix(
|
| 60 |
+
self,
|
| 61 |
+
cm: Union[np.ndarray, List[List[int]]],
|
| 62 |
+
labels: List[str],
|
| 63 |
+
normalize: bool = False,
|
| 64 |
+
title: str = "Confusion Matrix",
|
| 65 |
+
cmap: str = "Blues",
|
| 66 |
+
save_path: Optional[str] = None,
|
| 67 |
+
) -> Figure:
|
| 68 |
+
"""Plot a confusion matrix.
|
| 69 |
+
|
| 70 |
+
Args:
|
| 71 |
+
cm: 2×2 or N×N confusion matrix.
|
| 72 |
+
labels: List of class labels (e.g., ['Truthful', 'Deceptive']).
|
| 73 |
+
normalize: If True, show proportions instead of counts.
|
| 74 |
+
title: Plot title.
|
| 75 |
+
cmap: Colormap for the heatmap.
|
| 76 |
+
save_path: If provided, save figure to this path.
|
| 77 |
+
|
| 78 |
+
Returns:
|
| 79 |
+
Matplotlib Figure object.
|
| 80 |
+
"""
|
| 81 |
+
cm = np.asarray(cm)
|
| 82 |
+
if normalize:
|
| 83 |
+
cm = cm.astype("float") / cm.sum(axis=1, keepdims=True)
|
| 84 |
+
fmt = ".2f"
|
| 85 |
+
else:
|
| 86 |
+
fmt = "d"
|
| 87 |
+
|
| 88 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 89 |
+
sns.heatmap(
|
| 90 |
+
cm,
|
| 91 |
+
annot=True,
|
| 92 |
+
fmt=fmt,
|
| 93 |
+
cmap=cmap,
|
| 94 |
+
xticklabels=labels,
|
| 95 |
+
yticklabels=labels,
|
| 96 |
+
ax=ax,
|
| 97 |
+
cbar_kws={"label": "Normalized frequency" if normalize else "Count"},
|
| 98 |
+
)
|
| 99 |
+
ax.set_xlabel("Predicted label")
|
| 100 |
+
ax.set_ylabel("True label")
|
| 101 |
+
ax.set_title(title)
|
| 102 |
+
|
| 103 |
+
if save_path:
|
| 104 |
+
fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
|
| 105 |
+
logger.info(f"Saved confusion matrix to {save_path}")
|
| 106 |
+
|
| 107 |
+
return fig
|
| 108 |
+
|
| 109 |
+
def plot_roc_curve(
|
| 110 |
+
self,
|
| 111 |
+
y_true: Union[np.ndarray, List[int]],
|
| 112 |
+
y_prob: Union[np.ndarray, List[float]],
|
| 113 |
+
label: str = "Model",
|
| 114 |
+
title: str = "ROC Curve",
|
| 115 |
+
save_path: Optional[str] = None,
|
| 116 |
+
) -> Figure:
|
| 117 |
+
"""Plot ROC curve with AUC.
|
| 118 |
+
|
| 119 |
+
Args:
|
| 120 |
+
y_true: Ground truth binary labels.
|
| 121 |
+
y_prob: Predicted probabilities for the positive class.
|
| 122 |
+
label: Label for the curve (used in legend).
|
| 123 |
+
title: Plot title.
|
| 124 |
+
save_path: If provided, save figure to this path.
|
| 125 |
+
|
| 126 |
+
Returns:
|
| 127 |
+
Matplotlib Figure object.
|
| 128 |
+
"""
|
| 129 |
+
y_true = np.asarray(y_true)
|
| 130 |
+
y_prob = np.asarray(y_prob)
|
| 131 |
+
|
| 132 |
+
fpr, tpr, _ = roc_curve(y_true, y_prob)
|
| 133 |
+
roc_auc = auc(fpr, tpr)
|
| 134 |
+
|
| 135 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 136 |
+
ax.plot(fpr, tpr, lw=2, label=f"{label} (AUC = {roc_auc:.3f})")
|
| 137 |
+
ax.plot([0, 1], [0, 1], color="gray", lw=1, linestyle="--", label="Random")
|
| 138 |
+
ax.set_xlim([0.0, 1.0])
|
| 139 |
+
ax.set_ylim([0.0, 1.05])
|
| 140 |
+
ax.set_xlabel("False Positive Rate")
|
| 141 |
+
ax.set_ylabel("True Positive Rate")
|
| 142 |
+
ax.set_title(title)
|
| 143 |
+
ax.legend(loc="lower right")
|
| 144 |
+
ax.grid(True, alpha=0.3)
|
| 145 |
+
|
| 146 |
+
if save_path:
|
| 147 |
+
fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
|
| 148 |
+
logger.info(f"Saved ROC curve to {save_path}")
|
| 149 |
+
|
| 150 |
+
return fig
|
| 151 |
+
|
| 152 |
+
def plot_precision_recall_curve(
|
| 153 |
+
self,
|
| 154 |
+
y_true: Union[np.ndarray, List[int]],
|
| 155 |
+
y_prob: Union[np.ndarray, List[float]],
|
| 156 |
+
label: str = "Model",
|
| 157 |
+
title: str = "Precision‑Recall Curve",
|
| 158 |
+
save_path: Optional[str] = None,
|
| 159 |
+
) -> Figure:
|
| 160 |
+
"""Plot precision‑recall curve with average precision.
|
| 161 |
+
|
| 162 |
+
Args:
|
| 163 |
+
y_true: Ground truth binary labels.
|
| 164 |
+
y_prob: Predicted probabilities for the positive class.
|
| 165 |
+
label: Label for the curve (used in legend).
|
| 166 |
+
title: Plot title.
|
| 167 |
+
save_path: If provided, save figure to this path.
|
| 168 |
+
|
| 169 |
+
Returns:
|
| 170 |
+
Matplotlib Figure object.
|
| 171 |
+
"""
|
| 172 |
+
y_true = np.asarray(y_true)
|
| 173 |
+
y_prob = np.asarray(y_prob)
|
| 174 |
+
|
| 175 |
+
precision, recall, _ = precision_recall_curve(y_true, y_prob)
|
| 176 |
+
avg_precision = average_precision_score(y_true, y_prob)
|
| 177 |
+
|
| 178 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 179 |
+
ax.plot(recall, precision, lw=2, label=f"{label} (AP = {avg_precision:.3f})")
|
| 180 |
+
ax.set_xlim([0.0, 1.0])
|
| 181 |
+
ax.set_ylim([0.0, 1.05])
|
| 182 |
+
ax.set_xlabel("Recall")
|
| 183 |
+
ax.set_ylabel("Precision")
|
| 184 |
+
ax.set_title(title)
|
| 185 |
+
ax.legend(loc="lower left")
|
| 186 |
+
ax.grid(True, alpha=0.3)
|
| 187 |
+
|
| 188 |
+
if save_path:
|
| 189 |
+
fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
|
| 190 |
+
logger.info(f"Saved precision‑recall curve to {save_path}")
|
| 191 |
+
|
| 192 |
+
return fig
|
| 193 |
+
|
| 194 |
+
def plot_cross_validation_results(
|
| 195 |
+
self,
|
| 196 |
+
cv_results: Dict[str, Any],
|
| 197 |
+
metrics: Optional[List[str]] = None,
|
| 198 |
+
title: str = "Cross‑Validation Performance",
|
| 199 |
+
save_path: Optional[str] = None,
|
| 200 |
+
) -> Figure:
|
| 201 |
+
"""Plot box‑whisker or bar chart of metrics across CV folds.
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
cv_results: Dictionary as returned by CrossValidator.run_cross_validation.
|
| 205 |
+
Must contain 'per_fold_metrics' (list of dicts).
|
| 206 |
+
metrics: List of metric names to plot. If None, defaults to
|
| 207 |
+
['accuracy', 'precision', 'recall', 'f1', 'auc_roc'].
|
| 208 |
+
title: Plot title.
|
| 209 |
+
save_path: If provided, save figure to this path.
|
| 210 |
+
|
| 211 |
+
Returns:
|
| 212 |
+
Matplotlib Figure object.
|
| 213 |
+
"""
|
| 214 |
+
if "per_fold_metrics" not in cv_results:
|
| 215 |
+
raise ValueError("cv_results must contain 'per_fold_metrics'")
|
| 216 |
+
|
| 217 |
+
per_fold = cv_results["per_fold_metrics"]
|
| 218 |
+
if metrics is None:
|
| 219 |
+
metrics = ["accuracy", "precision", "recall", "f1"]
|
| 220 |
+
# Add AUC if present in any fold
|
| 221 |
+
if any("auc_roc" in fold for fold in per_fold):
|
| 222 |
+
metrics.append("auc_roc")
|
| 223 |
+
|
| 224 |
+
# Build DataFrame of metric values across folds
|
| 225 |
+
data = []
|
| 226 |
+
for fold_idx, fold_metrics in enumerate(per_fold):
|
| 227 |
+
for metric in metrics:
|
| 228 |
+
if metric in fold_metrics:
|
| 229 |
+
data.append(
|
| 230 |
+
{
|
| 231 |
+
"fold": f"Fold {fold_idx+1}",
|
| 232 |
+
"metric": metric,
|
| 233 |
+
"value": fold_metrics[metric],
|
| 234 |
+
}
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
if not data:
|
| 238 |
+
logger.warning("No metric data to plot")
|
| 239 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 240 |
+
ax.text(0.5, 0.5, "No data", ha="center", va="center")
|
| 241 |
+
return fig
|
| 242 |
+
|
| 243 |
+
df = pd.DataFrame(data)
|
| 244 |
+
|
| 245 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 246 |
+
sns.boxplot(data=df, x="metric", y="value", ax=ax, palette="Set2")
|
| 247 |
+
sns.stripplot(
|
| 248 |
+
data=df,
|
| 249 |
+
x="metric",
|
| 250 |
+
y="value",
|
| 251 |
+
ax=ax,
|
| 252 |
+
color="black",
|
| 253 |
+
alpha=0.7,
|
| 254 |
+
jitter=True,
|
| 255 |
+
)
|
| 256 |
+
ax.set_xlabel("Metric")
|
| 257 |
+
ax.set_ylabel("Value")
|
| 258 |
+
ax.set_title(title)
|
| 259 |
+
ax.grid(True, alpha=0.3)
|
| 260 |
+
|
| 261 |
+
if save_path:
|
| 262 |
+
fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
|
| 263 |
+
logger.info(f"Saved cross‑validation results plot to {save_path}")
|
| 264 |
+
|
| 265 |
+
return fig
|
| 266 |
+
|
| 267 |
+
def plot_ablation_results(
|
| 268 |
+
self,
|
| 269 |
+
ablation_results: Dict[str, Dict[str, Any]],
|
| 270 |
+
metric: str = "accuracy",
|
| 271 |
+
title: str = "Ablation Study",
|
| 272 |
+
save_path: Optional[str] = None,
|
| 273 |
+
) -> Figure:
|
| 274 |
+
"""Plot bar chart comparing performance across ablation conditions.
|
| 275 |
+
|
| 276 |
+
Args:
|
| 277 |
+
ablation_results: Dictionary as returned by AblationStudy.run_ablation.
|
| 278 |
+
Keys are condition names, each containing 'mean_metrics' dict.
|
| 279 |
+
metric: Metric to compare across conditions.
|
| 280 |
+
title: Plot title.
|
| 281 |
+
save_path: If provided, save figure to this path.
|
| 282 |
+
|
| 283 |
+
Returns:
|
| 284 |
+
Matplotlib Figure object.
|
| 285 |
+
"""
|
| 286 |
+
conditions = []
|
| 287 |
+
means = []
|
| 288 |
+
stds = []
|
| 289 |
+
|
| 290 |
+
for cond_name, cond_data in ablation_results.items():
|
| 291 |
+
if "mean_metrics" in cond_data and metric in cond_data["mean_metrics"]:
|
| 292 |
+
conditions.append(cond_name.replace("_", " ").title())
|
| 293 |
+
means.append(cond_data["mean_metrics"][metric])
|
| 294 |
+
stds.append(cond_data.get("std_metrics", {}).get(metric, 0.0))
|
| 295 |
+
|
| 296 |
+
if not conditions:
|
| 297 |
+
logger.warning(f"Metric '{metric}' not found in any condition")
|
| 298 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 299 |
+
ax.text(0.5, 0.5, "No data", ha="center", va="center")
|
| 300 |
+
return fig
|
| 301 |
+
|
| 302 |
+
x = np.arange(len(conditions))
|
| 303 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 304 |
+
bars = ax.bar(x, means, yerr=stds, capsize=5, color="steelblue", alpha=0.8)
|
| 305 |
+
ax.set_xticks(x)
|
| 306 |
+
ax.set_xticklabels(conditions, rotation=45, ha="right")
|
| 307 |
+
ax.set_ylabel(metric.replace("_", " ").title())
|
| 308 |
+
ax.set_title(title)
|
| 309 |
+
ax.grid(True, alpha=0.3, axis="y")
|
| 310 |
+
|
| 311 |
+
# Add value labels on top of bars
|
| 312 |
+
for bar, mean_val in zip(bars, means):
|
| 313 |
+
height = bar.get_height()
|
| 314 |
+
ax.text(
|
| 315 |
+
bar.get_x() + bar.get_width() / 2,
|
| 316 |
+
height + 0.01,
|
| 317 |
+
f"{mean_val:.3f}",
|
| 318 |
+
ha="center",
|
| 319 |
+
va="bottom",
|
| 320 |
+
fontsize=9,
|
| 321 |
+
)
|
| 322 |
+
|
| 323 |
+
if save_path:
|
| 324 |
+
fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
|
| 325 |
+
logger.info(f"Saved ablation results plot to {save_path}")
|
| 326 |
+
|
| 327 |
+
return fig
|
| 328 |
+
|
| 329 |
+
def plot_subject_performance(
|
| 330 |
+
self,
|
| 331 |
+
per_subject_metrics: Dict[str, Dict[str, Any]],
|
| 332 |
+
metric: str = "accuracy",
|
| 333 |
+
title: str = "Subject‑wise Performance",
|
| 334 |
+
sort_by: str = "value", # 'value' or 'subject'
|
| 335 |
+
save_path: Optional[str] = None,
|
| 336 |
+
) -> Figure:
|
| 337 |
+
"""Plot bar chart of a chosen metric across subjects.
|
| 338 |
+
|
| 339 |
+
Args:
|
| 340 |
+
per_subject_metrics: Dictionary mapping subject_id -> metrics dict.
|
| 341 |
+
metric: Metric to plot (must exist in each subject's dict).
|
| 342 |
+
title: Plot title.
|
| 343 |
+
sort_by: 'value' to sort bars by metric value, 'subject' to keep original order.
|
| 344 |
+
save_path: If provided, save figure to this path.
|
| 345 |
+
|
| 346 |
+
Returns:
|
| 347 |
+
Matplotlib Figure object.
|
| 348 |
+
"""
|
| 349 |
+
subjects = []
|
| 350 |
+
values = []
|
| 351 |
+
|
| 352 |
+
for subj, metrics in per_subject_metrics.items():
|
| 353 |
+
if metric in metrics:
|
| 354 |
+
subjects.append(subj)
|
| 355 |
+
values.append(metrics[metric])
|
| 356 |
+
else:
|
| 357 |
+
logger.warning(f"Metric '{metric}' not found for subject {subj}")
|
| 358 |
+
|
| 359 |
+
if not subjects:
|
| 360 |
+
logger.warning("No subject data to plot")
|
| 361 |
+
fig, ax = plt.subplots(figsize=self.figsize)
|
| 362 |
+
ax.text(0.5, 0.5, "No data", ha="center", va="center")
|
| 363 |
+
return fig
|
| 364 |
+
|
| 365 |
+
if sort_by == "value":
|
| 366 |
+
# Sort by value descending
|
| 367 |
+
sorted_pairs = sorted(zip(subjects, values), key=lambda x: x[1], reverse=True)
|
| 368 |
+
subjects, values = zip(*sorted_pairs)
|
| 369 |
+
# else keep original order
|
| 370 |
+
|
| 371 |
+
x = np.arange(len(subjects))
|
| 372 |
+
fig, ax = plt.subplots(figsize=(max(10, len(subjects) * 0.5), 6))
|
| 373 |
+
bars = ax.bar(x, values, color="teal", alpha=0.7)
|
| 374 |
+
ax.set_xticks(x)
|
| 375 |
+
ax.set_xticklabels(subjects, rotation=90, ha="center")
|
| 376 |
+
ax.set_ylabel(metric.replace("_", " ").title())
|
| 377 |
+
ax.set_title(title)
|
| 378 |
+
ax.grid(True, alpha=0.3, axis="y")
|
| 379 |
+
|
| 380 |
+
# Add value labels
|
| 381 |
+
for bar, val in zip(bars, values):
|
| 382 |
+
height = bar.get_height()
|
| 383 |
+
ax.text(
|
| 384 |
+
bar.get_x() + bar.get_width() / 2,
|
| 385 |
+
height + 0.005,
|
| 386 |
+
f"{val:.3f}",
|
| 387 |
+
ha="center",
|
| 388 |
+
va="bottom",
|
| 389 |
+
fontsize=8,
|
| 390 |
+
)
|
| 391 |
+
|
| 392 |
+
if save_path:
|
| 393 |
+
fig.savefig(save_path, dpi=self.dpi, bbox_inches="tight")
|
| 394 |
+
logger.info(f"Saved subject performance plot to {save_path}")
|
| 395 |
+
|
| 396 |
+
return fig
|
src/explainability/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explainability module for SHAP-based feature interpretation."""
|
| 2 |
+
|
| 3 |
+
from .explainer import DeceptionExplainer
|
| 4 |
+
from .visualizer import ExplainabilityVisualizer
|
| 5 |
+
|
| 6 |
+
__all__ = ['DeceptionExplainer', 'ExplainabilityVisualizer']
|
src/explainability/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (355 Bytes). View file
|
|
|
src/explainability/__pycache__/explainer.cpython-314.pyc
ADDED
|
Binary file (24 kB). View file
|
|
|
src/explainability/__pycache__/visualizer.cpython-314.pyc
ADDED
|
Binary file (36.1 kB). View file
|
|
|
src/explainability/explainer.py
ADDED
|
@@ -0,0 +1,513 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explainability module for deception detection using SHAP and feature importance.
|
| 2 |
+
|
| 3 |
+
This module provides interpretable explanations for deception predictions by
|
| 4 |
+
combining SHAP values, feature importance, and domain knowledge to produce
|
| 5 |
+
human-readable behavioral explanations.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import warnings
|
| 11 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import pandas as pd
|
| 15 |
+
import shap
|
| 16 |
+
from sklearn.inspection import permutation_importance
|
| 17 |
+
from sklearn.utils import check_array
|
| 18 |
+
|
| 19 |
+
from src.model.classifier import DeceptionClassifier
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class DeceptionExplainer:
|
| 23 |
+
"""Generates interpretable explanations for deception predictions.
|
| 24 |
+
|
| 25 |
+
Combines SHAP values, feature importance, and domain knowledge
|
| 26 |
+
to produce human-readable behavioral explanations.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
# Domain knowledge mapping: feature name -> behavioral interpretation
|
| 30 |
+
FEATURE_INTERPRETATIONS = {
|
| 31 |
+
'eye_left_openness': 'Left eye openness (reduced blinking or gaze aversion may indicate deception)',
|
| 32 |
+
'eye_right_openness': 'Right eye openness (asymmetric eye behavior can signal cognitive load)',
|
| 33 |
+
'mouth_openness': 'Mouth openness (suppressed or exaggerated expressions may indicate deception)',
|
| 34 |
+
'mouth_width': 'Mouth width changes (forced smiles show different width patterns)',
|
| 35 |
+
'mouth_asymmetry': 'Mouth asymmetry (asymmetric expressions are harder to fake)',
|
| 36 |
+
'left_brow_height': 'Left eyebrow height (raised brows can indicate surprise or concern)',
|
| 37 |
+
'right_brow_height': 'Right eyebrow height (asymmetric brow activity linked to deception)',
|
| 38 |
+
'brow_asymmetry': 'Eyebrow asymmetry (involuntary micro-expressions often appear asymmetrically)',
|
| 39 |
+
'brow_tension': 'Eyebrow tension (furrowed brows indicate cognitive effort or stress)',
|
| 40 |
+
'nose_wrinkle': 'Nose wrinkle (disgust micro-expression, hard to fake voluntarily)',
|
| 41 |
+
'nostril_flare': 'Nostril flare (stress indicator, autonomic nervous system response)',
|
| 42 |
+
'nose_asymmetry': 'Nose asymmetry (subtle asymmetries reveal involuntary expressions)',
|
| 43 |
+
'head_pitch': 'Head pitch (nodding patterns differ between truthful and deceptive responses)',
|
| 44 |
+
'head_yaw': 'Head yaw (head turning can indicate avoidance behavior)',
|
| 45 |
+
'head_roll': 'Head tilt (unusual tilt patterns may indicate discomfort)',
|
| 46 |
+
'head_displacement': 'Head movement (excessive stillness or fidgeting both signal stress)',
|
| 47 |
+
'pitch_mean': 'Average pitch (elevated pitch is a stress indicator)',
|
| 48 |
+
'pitch_std': 'Pitch variability (reduced variability suggests rehearsed responses)',
|
| 49 |
+
'pitch_range': 'Pitch range (constricted range indicates tension)',
|
| 50 |
+
'pitch_variation_coefficient': 'Pitch variation coefficient (normalized pitch instability)',
|
| 51 |
+
'voiced_ratio': 'Voiced speech ratio (reduced voicing suggests hesitation)',
|
| 52 |
+
'pause_count': 'Number of pauses (increased pausing indicates cognitive load)',
|
| 53 |
+
'avg_pause_duration': 'Average pause length (longer pauses suggest response fabrication)',
|
| 54 |
+
'max_pause_duration': 'Longest pause (extended silence before answering is suspicious)',
|
| 55 |
+
'pause_rate': 'Pauses per minute (higher rate indicates cognitive effort)',
|
| 56 |
+
'speech_ratio': 'Speech ratio (less speech time relative to pauses suggests evasion)',
|
| 57 |
+
'energy_mean': 'Average speech energy (reduced energy may indicate withdrawal)',
|
| 58 |
+
'energy_std': 'Energy variability (high variability suggests emotional instability)',
|
| 59 |
+
'energy_range': 'Energy range (wide range indicates emotional dysregulation)',
|
| 60 |
+
'energy_variation': 'Energy variation coefficient (normalized energy instability)',
|
| 61 |
+
'spectral_centroid_mean': 'Spectral centroid (brightness of voice, shifts under stress)',
|
| 62 |
+
'spectral_bandwidth_mean': 'Spectral bandwidth (voice quality changes under deception)',
|
| 63 |
+
'audio_visual_sync': 'Audio-visual synchrony (desynchronization suggests incongruent behavior)',
|
| 64 |
+
'pause_gaze_coupling': 'Pause-gaze coupling (looking away while pausing is a deception cue)',
|
| 65 |
+
'stress_composite': 'Stress composite (combined physiological stress indicator)',
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
FEATURE_GROUPS = {
|
| 69 |
+
'Eye Behavior': ['eye_left_openness', 'eye_right_openness'],
|
| 70 |
+
'Mouth Dynamics': ['mouth_openness', 'mouth_width', 'mouth_asymmetry'],
|
| 71 |
+
'Eyebrow Activity': ['left_brow_height', 'right_brow_height', 'brow_asymmetry', 'brow_tension'],
|
| 72 |
+
'Nose Movement': ['nose_wrinkle', 'nostril_flare', 'nose_asymmetry'],
|
| 73 |
+
'Head Dynamics': ['head_pitch', 'head_yaw', 'head_roll', 'head_displacement'],
|
| 74 |
+
'Voice Pitch': ['pitch_mean', 'pitch_std', 'pitch_range', 'pitch_variation_coefficient'],
|
| 75 |
+
'Speech Timing': ['pause_count', 'avg_pause_duration', 'max_pause_duration', 'pause_rate', 'speech_ratio', 'voiced_ratio'],
|
| 76 |
+
'Voice Energy': ['energy_mean', 'energy_std', 'energy_range', 'energy_variation', 'spectral_centroid_mean', 'spectral_bandwidth_mean'],
|
| 77 |
+
'Cross-Modal': ['audio_visual_sync', 'pause_gaze_coupling', 'stress_composite'],
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
def __init__(self, classifier: DeceptionClassifier):
|
| 81 |
+
"""Initialize with a trained DeceptionClassifier.
|
| 82 |
+
|
| 83 |
+
Args:
|
| 84 |
+
classifier: A trained DeceptionClassifier instance.
|
| 85 |
+
|
| 86 |
+
Raises:
|
| 87 |
+
ValueError: If classifier is not trained.
|
| 88 |
+
"""
|
| 89 |
+
if not classifier.is_fitted_:
|
| 90 |
+
warnings.warn("Classifier must be trained before creating explainer. Explainer will not work until trained.")
|
| 91 |
+
|
| 92 |
+
self.classifier = classifier
|
| 93 |
+
self._shap_explainer = None
|
| 94 |
+
self._feature_names = classifier.feature_names_
|
| 95 |
+
|
| 96 |
+
def _init_shap_explainer(self) -> Optional[shap.TreeExplainer]:
|
| 97 |
+
"""Lazy-initialize SHAP TreeExplainer from the trained model.
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
shap.TreeExplainer: Initialized SHAP explainer, or None on failure.
|
| 101 |
+
|
| 102 |
+
Notes:
|
| 103 |
+
Uses feature_perturbation='tree_path_dependent' for tree-based models.
|
| 104 |
+
On failure, warns and returns None.
|
| 105 |
+
"""
|
| 106 |
+
if self._shap_explainer is None:
|
| 107 |
+
try:
|
| 108 |
+
# Use the underlying GradientBoostingClassifier (not calibrated model)
|
| 109 |
+
# Research rationale: TreeExplainer works best with tree-based models
|
| 110 |
+
# and provides exact SHAP values for gradient boosting.
|
| 111 |
+
self._shap_explainer = shap.TreeExplainer(
|
| 112 |
+
model=self.classifier.model,
|
| 113 |
+
feature_perturbation='tree_path_dependent'
|
| 114 |
+
)
|
| 115 |
+
except Exception as e:
|
| 116 |
+
warnings.warn(
|
| 117 |
+
f"Failed to initialize SHAP TreeExplainer: {e}. "
|
| 118 |
+
"Falling back to permutation importance.",
|
| 119 |
+
RuntimeWarning
|
| 120 |
+
)
|
| 121 |
+
self._shap_explainer = None
|
| 122 |
+
|
| 123 |
+
return self._shap_explainer
|
| 124 |
+
|
| 125 |
+
def compute_shap_values(self, X: np.ndarray, feature_names: List[str]) -> np.ndarray:
|
| 126 |
+
"""Compute SHAP values using TreeExplainer.
|
| 127 |
+
|
| 128 |
+
Args:
|
| 129 |
+
X: Feature matrix of shape (n_samples, n_features).
|
| 130 |
+
feature_names: Ordered list of feature names matching columns of X.
|
| 131 |
+
|
| 132 |
+
Returns:
|
| 133 |
+
np.ndarray: SHAP values array of shape (n_samples, n_features).
|
| 134 |
+
|
| 135 |
+
Notes:
|
| 136 |
+
- If SHAP fails, falls back to permutation importance.
|
| 137 |
+
- Returns absolute SHAP values (magnitude of contribution).
|
| 138 |
+
"""
|
| 139 |
+
X = check_array(X, ensure_2d=True, dtype=np.float64)
|
| 140 |
+
|
| 141 |
+
# Ensure feature names match
|
| 142 |
+
if len(feature_names) != X.shape[1]:
|
| 143 |
+
raise ValueError(
|
| 144 |
+
f"Feature names length ({len(feature_names)}) "
|
| 145 |
+
f"does not match X columns ({X.shape[1]})"
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
# Scale features using classifier's scaler
|
| 149 |
+
X_scaled = self.classifier.scaler.transform(X)
|
| 150 |
+
|
| 151 |
+
try:
|
| 152 |
+
explainer = self._init_shap_explainer()
|
| 153 |
+
if explainer is None:
|
| 154 |
+
raise RuntimeError("SHAP explainer could not be initialized.")
|
| 155 |
+
|
| 156 |
+
shap_values = explainer.shap_values(X_scaled)
|
| 157 |
+
|
| 158 |
+
# For binary classification, SHAP returns list of arrays for each class
|
| 159 |
+
# We take the SHAP values for the positive class (deception)
|
| 160 |
+
if isinstance(shap_values, list) and len(shap_values) == 2:
|
| 161 |
+
shap_values = shap_values[1] # Positive class (deception)
|
| 162 |
+
elif isinstance(shap_values, np.ndarray) and shap_values.ndim == 3:
|
| 163 |
+
shap_values = shap_values[:, :, 1] # Positive class for multi-output
|
| 164 |
+
except Exception as e:
|
| 165 |
+
warnings.warn(
|
| 166 |
+
f"SHAP computation failed: {e}. Falling back to permutation importance.",
|
| 167 |
+
RuntimeWarning
|
| 168 |
+
)
|
| 169 |
+
# Fallback: compute permutation importance as proxy for feature contribution
|
| 170 |
+
shap_values = self._compute_fallback_importance(X_scaled, feature_names)
|
| 171 |
+
|
| 172 |
+
return np.asarray(shap_values, dtype=np.float64)
|
| 173 |
+
|
| 174 |
+
def _compute_fallback_importance(
|
| 175 |
+
self, X_scaled: np.ndarray, feature_names: List[str]
|
| 176 |
+
) -> np.ndarray:
|
| 177 |
+
"""Compute permutation importance as fallback when SHAP fails.
|
| 178 |
+
|
| 179 |
+
Args:
|
| 180 |
+
X_scaled: Scaled feature matrix.
|
| 181 |
+
feature_names: Feature names.
|
| 182 |
+
|
| 183 |
+
Returns:
|
| 184 |
+
np.ndarray: Importance values shaped (n_samples, n_features).
|
| 185 |
+
"""
|
| 186 |
+
# Use permutation importance to get global feature importance
|
| 187 |
+
# Research rationale: permutation importance measures decrease in model
|
| 188 |
+
# performance when a feature is randomly shuffled, providing a robust
|
| 189 |
+
# alternative to SHAP for feature contribution estimation.
|
| 190 |
+
try:
|
| 191 |
+
result = permutation_importance(
|
| 192 |
+
estimator=self.classifier.model,
|
| 193 |
+
X=X_scaled,
|
| 194 |
+
y=self.classifier.model.predict(X_scaled), # Use predicted labels
|
| 195 |
+
n_repeats=10,
|
| 196 |
+
random_state=42,
|
| 197 |
+
n_jobs=-1
|
| 198 |
+
)
|
| 199 |
+
# Broadcast global importance to each sample
|
| 200 |
+
importance = result.importances_mean
|
| 201 |
+
importance = np.clip(importance, 0, None) # Ensure non-negative
|
| 202 |
+
importance = importance / (np.sum(importance) + 1e-10) # Normalize
|
| 203 |
+
|
| 204 |
+
# Create per-sample importance by adding small noise
|
| 205 |
+
# This approximates sample-specific contributions
|
| 206 |
+
n_samples = X_scaled.shape[0]
|
| 207 |
+
noise = np.random.normal(0, 0.01 * importance.std(), (n_samples, len(importance)))
|
| 208 |
+
shap_values = importance.reshape(1, -1) + noise
|
| 209 |
+
return shap_values
|
| 210 |
+
except Exception:
|
| 211 |
+
# Ultimate fallback: uniform importance
|
| 212 |
+
n_samples, n_features = X_scaled.shape
|
| 213 |
+
return np.ones((n_samples, n_features)) / n_features
|
| 214 |
+
|
| 215 |
+
def get_feature_importance(
|
| 216 |
+
self, X: np.ndarray, feature_names: List[str], top_k: int = 10
|
| 217 |
+
) -> pd.DataFrame:
|
| 218 |
+
"""Get ranked feature importance combining model importance and SHAP.
|
| 219 |
+
|
| 220 |
+
Args:
|
| 221 |
+
X: Feature matrix of shape (n_samples, n_features).
|
| 222 |
+
feature_names: Ordered list of feature names.
|
| 223 |
+
top_k: Number of top features to return (default: 10).
|
| 224 |
+
|
| 225 |
+
Returns:
|
| 226 |
+
pd.DataFrame: Columns: ['feature', 'importance', 'shap_importance',
|
| 227 |
+
'combined_score', 'rank', 'interpretation']
|
| 228 |
+
"""
|
| 229 |
+
X = check_array(X, ensure_2d=True, dtype=np.float64)
|
| 230 |
+
|
| 231 |
+
# 1. Get model's built-in feature importance
|
| 232 |
+
try:
|
| 233 |
+
model_importance = self.classifier.get_feature_importance()
|
| 234 |
+
model_importance_dict = dict(
|
| 235 |
+
zip(model_importance['feature'], model_importance['importance'])
|
| 236 |
+
)
|
| 237 |
+
except Exception:
|
| 238 |
+
model_importance_dict = {}
|
| 239 |
+
|
| 240 |
+
# 2. Compute SHAP importance (mean absolute SHAP value per feature)
|
| 241 |
+
shap_values = self.compute_shap_values(X, feature_names)
|
| 242 |
+
shap_importance = np.mean(np.abs(shap_values), axis=0)
|
| 243 |
+
|
| 244 |
+
# 3. Combine scores
|
| 245 |
+
combined_scores = []
|
| 246 |
+
for i, feat in enumerate(feature_names):
|
| 247 |
+
model_imp = model_importance_dict.get(feat, 0.0)
|
| 248 |
+
shap_imp = shap_importance[i] if i < len(shap_importance) else 0.0
|
| 249 |
+
|
| 250 |
+
# Research rationale: weighted average gives more weight to SHAP
|
| 251 |
+
# because it captures non-linear interactions better than Gini importance.
|
| 252 |
+
combined = 0.3 * model_imp + 0.7 * shap_imp
|
| 253 |
+
|
| 254 |
+
combined_scores.append({
|
| 255 |
+
'feature': feat,
|
| 256 |
+
'importance': float(model_imp),
|
| 257 |
+
'shap_importance': float(shap_imp),
|
| 258 |
+
'combined_score': float(combined),
|
| 259 |
+
'interpretation': self.FEATURE_INTERPRETATIONS.get(feat, 'No interpretation available.')
|
| 260 |
+
})
|
| 261 |
+
|
| 262 |
+
df = pd.DataFrame(combined_scores)
|
| 263 |
+
df = df.sort_values('combined_score', ascending=False).reset_index(drop=True)
|
| 264 |
+
df['rank'] = range(1, len(df) + 1)
|
| 265 |
+
|
| 266 |
+
# Limit to top_k
|
| 267 |
+
if top_k > 0:
|
| 268 |
+
df = df.head(top_k)
|
| 269 |
+
|
| 270 |
+
return df
|
| 271 |
+
|
| 272 |
+
def get_ranked_indicators(
|
| 273 |
+
self, X: np.ndarray, feature_names: List[str], top_k: int = 5
|
| 274 |
+
) -> List[Dict]:
|
| 275 |
+
"""Get ranked deception indicators for a single sample.
|
| 276 |
+
|
| 277 |
+
Args:
|
| 278 |
+
X: Feature matrix of shape (1, n_features) for a single sample.
|
| 279 |
+
feature_names: Ordered list of feature names.
|
| 280 |
+
top_k: Number of top indicators to return (default: 5).
|
| 281 |
+
|
| 282 |
+
Returns:
|
| 283 |
+
List[Dict]: Each dict contains:
|
| 284 |
+
- 'feature': feature name
|
| 285 |
+
- 'value': original feature value
|
| 286 |
+
- 'shap_value': SHAP contribution value
|
| 287 |
+
- 'abs_contribution': absolute SHAP magnitude
|
| 288 |
+
- 'direction': 'toward_deception' or 'toward_truth'
|
| 289 |
+
- 'interpretation': behavioral interpretation
|
| 290 |
+
"""
|
| 291 |
+
if X.ndim != 2:
|
| 292 |
+
raise ValueError("X must be 2D. Use shape (1, n_features) for single sample.")
|
| 293 |
+
|
| 294 |
+
if X.shape[0] != 1:
|
| 295 |
+
warnings.warn(
|
| 296 |
+
"Multiple samples provided. Using first sample for indicator ranking.",
|
| 297 |
+
UserWarning
|
| 298 |
+
)
|
| 299 |
+
X = X[:1]
|
| 300 |
+
|
| 301 |
+
# Compute SHAP values for the sample
|
| 302 |
+
shap_values = self.compute_shap_values(X, feature_names)
|
| 303 |
+
shap_sample = shap_values[0]
|
| 304 |
+
|
| 305 |
+
# Get original feature values
|
| 306 |
+
feature_values = X[0]
|
| 307 |
+
|
| 308 |
+
# Create indicator dictionaries
|
| 309 |
+
indicators = []
|
| 310 |
+
for i, feat in enumerate(feature_names):
|
| 311 |
+
shap_val = shap_sample[i]
|
| 312 |
+
feat_val = feature_values[i]
|
| 313 |
+
|
| 314 |
+
# Determine direction based on SHAP sign
|
| 315 |
+
# Positive SHAP pushes prediction toward deception (class 1)
|
| 316 |
+
direction = 'toward_deception' if shap_val > 0 else 'toward_truth'
|
| 317 |
+
|
| 318 |
+
indicators.append({
|
| 319 |
+
'feature': feat,
|
| 320 |
+
'value': float(feat_val),
|
| 321 |
+
'shap_value': float(shap_val),
|
| 322 |
+
'abs_contribution': float(np.abs(shap_val)),
|
| 323 |
+
'direction': direction,
|
| 324 |
+
'interpretation': self.FEATURE_INTERPRETATIONS.get(feat, 'No interpretation available.')
|
| 325 |
+
})
|
| 326 |
+
|
| 327 |
+
# Sort by absolute contribution (most influential first)
|
| 328 |
+
indicators.sort(key=lambda x: x['abs_contribution'], reverse=True)
|
| 329 |
+
|
| 330 |
+
# Return top_k indicators
|
| 331 |
+
return indicators[:top_k]
|
| 332 |
+
|
| 333 |
+
def explain_prediction(
|
| 334 |
+
self,
|
| 335 |
+
X: np.ndarray,
|
| 336 |
+
feature_names: List[str],
|
| 337 |
+
prediction: float,
|
| 338 |
+
subject_id: Optional[str] = None
|
| 339 |
+
) -> Dict:
|
| 340 |
+
"""Generate complete explanation for a single prediction.
|
| 341 |
+
|
| 342 |
+
Args:
|
| 343 |
+
X: Feature matrix of shape (1, n_features) for a single sample.
|
| 344 |
+
feature_names: Ordered list of feature names.
|
| 345 |
+
prediction: Deception probability (0-1).
|
| 346 |
+
subject_id: Optional subject identifier for contextualization.
|
| 347 |
+
|
| 348 |
+
Returns:
|
| 349 |
+
Dict: Explanation dictionary with keys:
|
| 350 |
+
- 'verdict': 'truthful' or 'deceptive' based on threshold 0.5
|
| 351 |
+
- 'confidence': absolute distance from decision boundary
|
| 352 |
+
- 'deception_probability': prediction probability
|
| 353 |
+
- 'top_indicators': list of top indicators (from get_ranked_indicators)
|
| 354 |
+
- 'group_contributions': dict of SHAP contributions per feature group
|
| 355 |
+
- 'behavioral_summary': human-readable summary text
|
| 356 |
+
- 'feature_values': dict of original feature values
|
| 357 |
+
- 'subject_id': subject identifier if provided
|
| 358 |
+
"""
|
| 359 |
+
if X.ndim != 2 or X.shape[0] != 1:
|
| 360 |
+
raise ValueError("X must be a single sample with shape (1, n_features).")
|
| 361 |
+
|
| 362 |
+
# Compute SHAP values
|
| 363 |
+
shap_values = self.compute_shap_values(X, feature_names)
|
| 364 |
+
|
| 365 |
+
# Determine verdict
|
| 366 |
+
verdict = 'deceptive' if prediction >= 0.5 else 'truthful'
|
| 367 |
+
confidence = abs(prediction - 0.5) * 2 # Scale to 0-1
|
| 368 |
+
|
| 369 |
+
# Get top indicators
|
| 370 |
+
top_indicators = self.get_ranked_indicators(X, feature_names, top_k=5)
|
| 371 |
+
|
| 372 |
+
# Compute group contributions
|
| 373 |
+
group_contributions = self.explain_group_contributions(shap_values, feature_names)
|
| 374 |
+
|
| 375 |
+
# Generate behavioral summary
|
| 376 |
+
behavioral_summary = self.generate_behavioral_summary(
|
| 377 |
+
top_indicators, prediction, group_contributions
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
# Prepare feature values dictionary
|
| 381 |
+
feature_values = {
|
| 382 |
+
feat: float(val) for feat, val in zip(feature_names, X[0])
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
return {
|
| 386 |
+
'verdict': verdict,
|
| 387 |
+
'confidence': float(confidence),
|
| 388 |
+
'deception_probability': float(prediction),
|
| 389 |
+
'top_indicators': top_indicators,
|
| 390 |
+
'group_contributions': group_contributions,
|
| 391 |
+
'behavioral_summary': behavioral_summary,
|
| 392 |
+
'feature_values': feature_values,
|
| 393 |
+
'subject_id': subject_id
|
| 394 |
+
}
|
| 395 |
+
|
| 396 |
+
def generate_behavioral_summary(
|
| 397 |
+
self,
|
| 398 |
+
top_indicators: List[Dict],
|
| 399 |
+
prediction: float,
|
| 400 |
+
group_contributions: Dict[str, float]
|
| 401 |
+
) -> str:
|
| 402 |
+
"""Generate human-readable behavioral explanation paragraph.
|
| 403 |
+
|
| 404 |
+
Args:
|
| 405 |
+
top_indicators: Top indicators from get_ranked_indicators.
|
| 406 |
+
prediction: Deception probability (0-1).
|
| 407 |
+
group_contributions: Normalized SHAP contributions per feature group.
|
| 408 |
+
|
| 409 |
+
Returns:
|
| 410 |
+
str: Human-readable summary paragraph.
|
| 411 |
+
|
| 412 |
+
Example:
|
| 413 |
+
"The subject shows elevated deception probability (0.82). Key indicators include
|
| 414 |
+
increased pause duration (SHAP: +0.12), which suggests cognitive load during
|
| 415 |
+
response fabrication, and reduced audio-visual synchrony (SHAP: +0.09),
|
| 416 |
+
indicating incongruent verbal and non-verbal behavior."
|
| 417 |
+
"""
|
| 418 |
+
# Determine deception level
|
| 419 |
+
if prediction >= 0.7:
|
| 420 |
+
level = "strongly elevated"
|
| 421 |
+
elif prediction >= 0.6:
|
| 422 |
+
level = "elevated"
|
| 423 |
+
elif prediction >= 0.5:
|
| 424 |
+
level = "slightly elevated"
|
| 425 |
+
elif prediction >= 0.4:
|
| 426 |
+
level = "slightly reduced"
|
| 427 |
+
elif prediction >= 0.3:
|
| 428 |
+
level = "reduced"
|
| 429 |
+
else:
|
| 430 |
+
level = "strongly reduced"
|
| 431 |
+
|
| 432 |
+
# Build summary opening
|
| 433 |
+
summary_parts = [
|
| 434 |
+
f"The subject shows {level} deception probability ({prediction:.2f})."
|
| 435 |
+
]
|
| 436 |
+
|
| 437 |
+
# Add top indicators
|
| 438 |
+
if top_indicators:
|
| 439 |
+
indicator_desc = []
|
| 440 |
+
for idx, ind in enumerate(top_indicators[:3]): # top 3
|
| 441 |
+
feat = ind['feature']
|
| 442 |
+
shap_val = ind['shap_value']
|
| 443 |
+
direction = ind['direction']
|
| 444 |
+
interp = self.FEATURE_INTERPRETATIONS.get(feat, '')
|
| 445 |
+
# Extract the main behavioral insight (first sentence)
|
| 446 |
+
if interp:
|
| 447 |
+
insight = interp.split('.')[0] + '.'
|
| 448 |
+
else:
|
| 449 |
+
insight = "This feature contributes to the prediction."
|
| 450 |
+
|
| 451 |
+
# Format SHAP value with sign
|
| 452 |
+
sign = '+' if shap_val >= 0 else ''
|
| 453 |
+
indicator_desc.append(
|
| 454 |
+
f"{feat.replace('_', ' ')} (SHAP: {sign}{shap_val:.2f}), "
|
| 455 |
+
f"which {insight.lower()}"
|
| 456 |
+
)
|
| 457 |
+
|
| 458 |
+
if indicator_desc:
|
| 459 |
+
summary_parts.append("Key indicators include " + ", ".join(indicator_desc) + ".")
|
| 460 |
+
|
| 461 |
+
# Add group contributions
|
| 462 |
+
top_group = max(group_contributions.items(), key=lambda x: x[1]) if group_contributions else None
|
| 463 |
+
if top_group:
|
| 464 |
+
group_name, contrib = top_group
|
| 465 |
+
summary_parts.append(
|
| 466 |
+
f"The {group_name} feature group contributed the most ({contrib:.1%}) to the prediction, "
|
| 467 |
+
f"suggesting that {group_name.lower()} played a dominant role."
|
| 468 |
+
)
|
| 469 |
+
|
| 470 |
+
# Join into a single paragraph
|
| 471 |
+
return " ".join(summary_parts)
|
| 472 |
+
|
| 473 |
+
def explain_group_contributions(
|
| 474 |
+
self, shap_values: np.ndarray, feature_names: List[str]
|
| 475 |
+
) -> Dict[str, float]:
|
| 476 |
+
"""Aggregate SHAP contribution per feature group, normalized to sum to 1.0.
|
| 477 |
+
|
| 478 |
+
Args:
|
| 479 |
+
shap_values: SHAP values array of shape (n_samples, n_features).
|
| 480 |
+
feature_names: Ordered list of feature names.
|
| 481 |
+
|
| 482 |
+
Returns:
|
| 483 |
+
Dict[str, float]: Mapping from group name to normalized contribution.
|
| 484 |
+
"""
|
| 485 |
+
# Ensure single sample (take first if multiple)
|
| 486 |
+
if shap_values.ndim == 2 and shap_values.shape[0] > 1:
|
| 487 |
+
shap_values = shap_values[0:1]
|
| 488 |
+
shap_abs = np.abs(shap_values).mean(axis=0) # average across samples (usually 1)
|
| 489 |
+
|
| 490 |
+
# Map feature to group
|
| 491 |
+
feature_to_group = {}
|
| 492 |
+
for group, features in self.FEATURE_GROUPS.items():
|
| 493 |
+
for feat in features:
|
| 494 |
+
feature_to_group[feat] = group
|
| 495 |
+
|
| 496 |
+
# Aggregate contributions
|
| 497 |
+
group_sums = {}
|
| 498 |
+
for i, feat in enumerate(feature_names):
|
| 499 |
+
if i >= len(shap_abs):
|
| 500 |
+
continue
|
| 501 |
+
group = feature_to_group.get(feat)
|
| 502 |
+
if group is None:
|
| 503 |
+
# Assign to 'Other' group
|
| 504 |
+
group = 'Other'
|
| 505 |
+
group_sums[group] = group_sums.get(group, 0.0) + shap_abs[i]
|
| 506 |
+
|
| 507 |
+
# Normalize to sum to 1.0
|
| 508 |
+
total = sum(group_sums.values())
|
| 509 |
+
if total > 0:
|
| 510 |
+
group_contributions = {k: v / total for k, v in group_sums.items()}
|
| 511 |
+
else:
|
| 512 |
+
group_contributions = {k: 0.0 for k in group_sums.keys()}
|
| 513 |
+
return group_contributions
|
src/explainability/visualizer.py
ADDED
|
@@ -0,0 +1,743 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Publication-quality explainability visualizations for deception detection."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import warnings
|
| 6 |
+
from typing import Dict, List, Optional, Tuple, Any
|
| 7 |
+
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import numpy as np
|
| 10 |
+
import pandas as pd
|
| 11 |
+
import seaborn as sns
|
| 12 |
+
from matplotlib import gridspec
|
| 13 |
+
from matplotlib.cm import ScalarMappable
|
| 14 |
+
from matplotlib.colors import Normalize
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class ExplainabilityVisualizer:
|
| 18 |
+
"""Publication-quality explainability visualizations."""
|
| 19 |
+
|
| 20 |
+
def __init__(self):
|
| 21 |
+
"""Set up matplotlib style for publication plots.
|
| 22 |
+
|
| 23 |
+
Configures seaborn style, font sizes, and color palette.
|
| 24 |
+
"""
|
| 25 |
+
# Use seaborn style with white background
|
| 26 |
+
sns.set_style("whitegrid")
|
| 27 |
+
sns.set_context("paper", font_scale=1.2)
|
| 28 |
+
|
| 29 |
+
# Set publication-quality defaults
|
| 30 |
+
plt.rcParams.update({
|
| 31 |
+
'font.size': 10,
|
| 32 |
+
'axes.titlesize': 12,
|
| 33 |
+
'axes.labelsize': 10,
|
| 34 |
+
'xtick.labelsize': 9,
|
| 35 |
+
'ytick.labelsize': 9,
|
| 36 |
+
'legend.fontsize': 9,
|
| 37 |
+
'figure.titlesize': 14,
|
| 38 |
+
'figure.dpi': 100,
|
| 39 |
+
'savefig.dpi': 300,
|
| 40 |
+
'savefig.bbox': 'tight',
|
| 41 |
+
'savefig.pad_inches': 0.1,
|
| 42 |
+
'font.family': 'sans-serif',
|
| 43 |
+
'font.sans-serif': ['Arial', 'DejaVu Sans', 'Liberation Sans'],
|
| 44 |
+
})
|
| 45 |
+
|
| 46 |
+
# Define colorblind-friendly palette
|
| 47 |
+
self.cb_palette = sns.color_palette("colorblind")
|
| 48 |
+
self.importance_color = self.cb_palette[0] # blue
|
| 49 |
+
self.shap_color = self.cb_palette[2] # red
|
| 50 |
+
self.truth_color = self.cb_palette[1] # green
|
| 51 |
+
self.deception_color = self.cb_palette[3] # orange
|
| 52 |
+
|
| 53 |
+
# Store current figure reference
|
| 54 |
+
self._current_fig = None
|
| 55 |
+
|
| 56 |
+
def plot_feature_importance(
|
| 57 |
+
self,
|
| 58 |
+
importance_df: pd.DataFrame,
|
| 59 |
+
top_k: int = 15,
|
| 60 |
+
save_path: Optional[str] = None,
|
| 61 |
+
) -> plt.Figure:
|
| 62 |
+
"""Horizontal bar chart comparing model importance vs SHAP importance.
|
| 63 |
+
|
| 64 |
+
Args:
|
| 65 |
+
importance_df: DataFrame with columns ['feature', 'importance',
|
| 66 |
+
'shap_importance', 'combined_score', 'rank', 'interpretation'].
|
| 67 |
+
top_k: Number of top features to display (default 15).
|
| 68 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 69 |
+
|
| 70 |
+
Returns:
|
| 71 |
+
matplotlib.figure.Figure: The created figure.
|
| 72 |
+
|
| 73 |
+
Raises:
|
| 74 |
+
ValueError: If required columns are missing or DataFrame is empty.
|
| 75 |
+
"""
|
| 76 |
+
if importance_df.empty:
|
| 77 |
+
raise ValueError("importance_df is empty")
|
| 78 |
+
|
| 79 |
+
required_cols = {'feature', 'importance', 'shap_importance'}
|
| 80 |
+
missing = required_cols - set(importance_df.columns)
|
| 81 |
+
if missing:
|
| 82 |
+
raise ValueError(f"Missing columns: {missing}")
|
| 83 |
+
|
| 84 |
+
# Sort by combined_score if available, else by importance
|
| 85 |
+
if 'combined_score' in importance_df.columns:
|
| 86 |
+
df = importance_df.sort_values('combined_score', ascending=False).head(top_k)
|
| 87 |
+
else:
|
| 88 |
+
df = importance_df.sort_values('importance', ascending=False).head(top_k)
|
| 89 |
+
|
| 90 |
+
# Ensure we have at least one feature
|
| 91 |
+
if df.empty:
|
| 92 |
+
warnings.warn("No features to plot after filtering")
|
| 93 |
+
fig, ax = plt.subplots(figsize=(8, 4))
|
| 94 |
+
ax.text(0.5, 0.5, "No feature importance data",
|
| 95 |
+
ha='center', va='center', transform=ax.transAxes)
|
| 96 |
+
ax.set_axis_off()
|
| 97 |
+
if save_path:
|
| 98 |
+
fig.savefig(save_path, dpi=300)
|
| 99 |
+
return fig
|
| 100 |
+
|
| 101 |
+
# Create horizontal positions
|
| 102 |
+
y_pos = np.arange(len(df))
|
| 103 |
+
bar_height = 0.35
|
| 104 |
+
|
| 105 |
+
fig, ax = plt.subplots(figsize=(10, max(6, len(df) * 0.4)))
|
| 106 |
+
|
| 107 |
+
# Plot side-by-side bars
|
| 108 |
+
bars1 = ax.barh(y_pos - bar_height/2, df['importance'],
|
| 109 |
+
height=bar_height, color=self.importance_color,
|
| 110 |
+
label='Model Importance', edgecolor='black', linewidth=0.5)
|
| 111 |
+
bars2 = ax.barh(y_pos + bar_height/2, df['shap_importance'],
|
| 112 |
+
height=bar_height, color=self.shap_color,
|
| 113 |
+
label='SHAP Importance', edgecolor='black', linewidth=0.5)
|
| 114 |
+
|
| 115 |
+
# Add feature names
|
| 116 |
+
ax.set_yticks(y_pos)
|
| 117 |
+
ax.set_yticklabels(df['feature'])
|
| 118 |
+
|
| 119 |
+
# Add value labels on bars
|
| 120 |
+
for bar in bars1:
|
| 121 |
+
width = bar.get_width()
|
| 122 |
+
if width > 0:
|
| 123 |
+
ax.text(width + 0.01, bar.get_y() + bar.get_height()/2,
|
| 124 |
+
f'{width:.3f}', va='center', ha='left', fontsize=8)
|
| 125 |
+
|
| 126 |
+
for bar in bars2:
|
| 127 |
+
width = bar.get_width()
|
| 128 |
+
if width > 0:
|
| 129 |
+
ax.text(width + 0.01, bar.get_y() + bar.get_height()/2,
|
| 130 |
+
f'{width:.3f}', va='center', ha='left', fontsize=8)
|
| 131 |
+
|
| 132 |
+
ax.set_xlabel('Importance Score')
|
| 133 |
+
ax.set_title('Feature Importance Comparison (Model vs SHAP)')
|
| 134 |
+
ax.legend(loc='lower right')
|
| 135 |
+
|
| 136 |
+
# Add interpretation as hover-like annotations (optional)
|
| 137 |
+
if 'interpretation' in df.columns:
|
| 138 |
+
# Add interpretation as text on the right side if space
|
| 139 |
+
max_x = max(df['importance'].max(), df['shap_importance'].max()) * 1.3
|
| 140 |
+
for i, (_, row) in enumerate(df.iterrows()):
|
| 141 |
+
if isinstance(row['interpretation'], str) and row['interpretation']:
|
| 142 |
+
ax.text(max_x, i, f" {row['interpretation']}",
|
| 143 |
+
va='center', fontsize=7, alpha=0.7,
|
| 144 |
+
bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.5))
|
| 145 |
+
|
| 146 |
+
ax.invert_yaxis() # Highest importance at top
|
| 147 |
+
plt.tight_layout()
|
| 148 |
+
|
| 149 |
+
if save_path:
|
| 150 |
+
fig.savefig(save_path, dpi=300)
|
| 151 |
+
|
| 152 |
+
self._current_fig = fig
|
| 153 |
+
return fig
|
| 154 |
+
|
| 155 |
+
def plot_shap_summary(
|
| 156 |
+
self,
|
| 157 |
+
shap_values: np.ndarray,
|
| 158 |
+
feature_names: List[str],
|
| 159 |
+
X: np.ndarray,
|
| 160 |
+
top_k: int = 15,
|
| 161 |
+
save_path: Optional[str] = None,
|
| 162 |
+
) -> plt.Figure:
|
| 163 |
+
"""SHAP beeswarm-style summary plot.
|
| 164 |
+
|
| 165 |
+
Each dot represents one sample, colored by feature value,
|
| 166 |
+
x-position is SHAP value.
|
| 167 |
+
|
| 168 |
+
Args:
|
| 169 |
+
shap_values: 2D array of SHAP values (n_samples, n_features).
|
| 170 |
+
feature_names: List of feature names.
|
| 171 |
+
X: 2D array of feature values (n_samples, n_features).
|
| 172 |
+
top_k: Number of top features to display (default 15).
|
| 173 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 174 |
+
|
| 175 |
+
Returns:
|
| 176 |
+
matplotlib.figure.Figure: The created figure.
|
| 177 |
+
|
| 178 |
+
Raises:
|
| 179 |
+
ValueError: If dimensions mismatch.
|
| 180 |
+
"""
|
| 181 |
+
if shap_values.shape[1] != len(feature_names):
|
| 182 |
+
raise ValueError("shap_values columns must match feature_names length")
|
| 183 |
+
if X.shape[1] != len(feature_names):
|
| 184 |
+
raise ValueError("X columns must match feature_names length")
|
| 185 |
+
|
| 186 |
+
n_samples, n_features = shap_values.shape
|
| 187 |
+
if n_samples == 0 or n_features == 0:
|
| 188 |
+
warnings.warn("No data to plot")
|
| 189 |
+
fig, ax = plt.subplots(figsize=(8, 4))
|
| 190 |
+
ax.text(0.5, 0.5, "No SHAP data",
|
| 191 |
+
ha='center', va='center', transform=ax.transAxes)
|
| 192 |
+
ax.set_axis_off()
|
| 193 |
+
if save_path:
|
| 194 |
+
fig.savefig(save_path, dpi=300)
|
| 195 |
+
return fig
|
| 196 |
+
|
| 197 |
+
# Compute mean absolute SHAP to rank features
|
| 198 |
+
mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
|
| 199 |
+
top_indices = np.argsort(mean_abs_shap)[-top_k:][::-1]
|
| 200 |
+
|
| 201 |
+
# Prepare data for plotting
|
| 202 |
+
top_shap = shap_values[:, top_indices]
|
| 203 |
+
top_X = X[:, top_indices]
|
| 204 |
+
top_names = [feature_names[i] for i in top_indices]
|
| 205 |
+
|
| 206 |
+
# Create figure
|
| 207 |
+
fig, ax = plt.subplots(figsize=(10, max(6, top_k * 0.5)))
|
| 208 |
+
|
| 209 |
+
# Create colormap for feature values
|
| 210 |
+
cmap = plt.cm.coolwarm
|
| 211 |
+
norm = Normalize(vmin=np.nanmin(top_X), vmax=np.nanmax(top_X))
|
| 212 |
+
|
| 213 |
+
# Plot each feature
|
| 214 |
+
y_positions = np.arange(len(top_names))
|
| 215 |
+
|
| 216 |
+
for i, (name, shap_vals, x_vals) in enumerate(zip(top_names, top_shap.T, top_X.T)):
|
| 217 |
+
y = y_positions[i]
|
| 218 |
+
|
| 219 |
+
# Add jitter for visibility
|
| 220 |
+
jitter = np.random.normal(0, 0.05, size=len(shap_vals))
|
| 221 |
+
|
| 222 |
+
# Scatter plot
|
| 223 |
+
scatter = ax.scatter(
|
| 224 |
+
shap_vals,
|
| 225 |
+
y + jitter,
|
| 226 |
+
c=x_vals,
|
| 227 |
+
cmap=cmap,
|
| 228 |
+
norm=norm,
|
| 229 |
+
s=20,
|
| 230 |
+
alpha=0.6,
|
| 231 |
+
edgecolors='black',
|
| 232 |
+
linewidths=0.3,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
# Add colorbar
|
| 236 |
+
sm = ScalarMappable(cmap=cmap, norm=norm)
|
| 237 |
+
sm.set_array([])
|
| 238 |
+
cbar = plt.colorbar(sm, ax=ax, pad=0.01)
|
| 239 |
+
cbar.set_label('Feature Value', fontsize=9)
|
| 240 |
+
|
| 241 |
+
# Add vertical line at zero
|
| 242 |
+
ax.axvline(x=0, color='black', linestyle='--', linewidth=0.5, alpha=0.5)
|
| 243 |
+
|
| 244 |
+
# Set labels and ticks
|
| 245 |
+
ax.set_yticks(y_positions)
|
| 246 |
+
ax.set_yticklabels(top_names)
|
| 247 |
+
ax.set_xlabel('SHAP Value (impact on prediction)')
|
| 248 |
+
ax.set_ylabel('Feature')
|
| 249 |
+
ax.set_title(f'SHAP Summary Plot (Top {top_k} Features)')
|
| 250 |
+
|
| 251 |
+
# Add mean absolute SHAP annotation
|
| 252 |
+
for i, idx in enumerate(top_indices):
|
| 253 |
+
ax.text(0.02, y_positions[i], f'mean|SHAP|={mean_abs_shap[idx]:.3f}',
|
| 254 |
+
transform=ax.get_yaxis_transform(),
|
| 255 |
+
fontsize=7, va='center', bbox=dict(boxstyle='round,pad=0.2',
|
| 256 |
+
facecolor='white', alpha=0.7))
|
| 257 |
+
|
| 258 |
+
plt.tight_layout()
|
| 259 |
+
|
| 260 |
+
if save_path:
|
| 261 |
+
fig.savefig(save_path, dpi=300)
|
| 262 |
+
|
| 263 |
+
self._current_fig = fig
|
| 264 |
+
return fig
|
| 265 |
+
|
| 266 |
+
def plot_shap_waterfall(
|
| 267 |
+
self,
|
| 268 |
+
shap_values: np.ndarray,
|
| 269 |
+
feature_names: List[str],
|
| 270 |
+
base_value: float,
|
| 271 |
+
top_k: int = 10,
|
| 272 |
+
save_path: Optional[str] = None,
|
| 273 |
+
) -> plt.Figure:
|
| 274 |
+
"""Waterfall plot for a single prediction's SHAP explanation.
|
| 275 |
+
|
| 276 |
+
Red bars = features pushing toward deception (positive SHAP)
|
| 277 |
+
Blue bars = features pushing toward truth (negative SHAP)
|
| 278 |
+
|
| 279 |
+
Args:
|
| 280 |
+
shap_values: 1D array of SHAP values for a single sample.
|
| 281 |
+
feature_names: List of feature names.
|
| 282 |
+
base_value: Base (expected) value from SHAP explainer.
|
| 283 |
+
top_k: Number of top features to display (default 10).
|
| 284 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 285 |
+
|
| 286 |
+
Returns:
|
| 287 |
+
matplotlib.figure.Figure: The created figure.
|
| 288 |
+
"""
|
| 289 |
+
if shap_values.ndim != 1:
|
| 290 |
+
if shap_values.ndim == 2 and shap_values.shape[0] == 1:
|
| 291 |
+
shap_values = shap_values[0]
|
| 292 |
+
else:
|
| 293 |
+
raise ValueError("shap_values must be 1D or 2D with single sample")
|
| 294 |
+
|
| 295 |
+
if len(shap_values) != len(feature_names):
|
| 296 |
+
raise ValueError("shap_values length must match feature_names")
|
| 297 |
+
|
| 298 |
+
# Select top K features by absolute SHAP
|
| 299 |
+
abs_shap = np.abs(shap_values)
|
| 300 |
+
top_indices = np.argsort(abs_shap)[-top_k:][::-1]
|
| 301 |
+
|
| 302 |
+
# Sort by SHAP value (deception to truth)
|
| 303 |
+
sorted_indices = top_indices[np.argsort(shap_values[top_indices])[::-1]]
|
| 304 |
+
|
| 305 |
+
top_shap = shap_values[sorted_indices]
|
| 306 |
+
top_names = [feature_names[i] for i in sorted_indices]
|
| 307 |
+
|
| 308 |
+
# Compute cumulative values for waterfall
|
| 309 |
+
cumulative = base_value + np.cumsum(top_shap)
|
| 310 |
+
cumulative = np.insert(cumulative, 0, base_value)
|
| 311 |
+
|
| 312 |
+
# Create figure
|
| 313 |
+
fig, ax = plt.subplots(figsize=(10, max(6, top_k * 0.6)))
|
| 314 |
+
|
| 315 |
+
# Plot bars
|
| 316 |
+
for i, (shap_val, name) in enumerate(zip(top_shap, top_names)):
|
| 317 |
+
color = self.deception_color if shap_val > 0 else self.truth_color
|
| 318 |
+
ax.barh(i, shap_val, left=cumulative[i], color=color,
|
| 319 |
+
edgecolor='black', linewidth=0.5, height=0.8)
|
| 320 |
+
|
| 321 |
+
# Add connecting lines
|
| 322 |
+
for i in range(len(cumulative) - 1):
|
| 323 |
+
ax.plot([cumulative[i], cumulative[i+1]], [i, i],
|
| 324 |
+
color='gray', linewidth=0.5, linestyle='--', alpha=0.5)
|
| 325 |
+
|
| 326 |
+
# Add base value and final prediction
|
| 327 |
+
final_value = cumulative[-1]
|
| 328 |
+
ax.axvline(x=base_value, color='black', linestyle=':', linewidth=1,
|
| 329 |
+
label=f'Base Value: {base_value:.3f}')
|
| 330 |
+
ax.axvline(x=final_value, color='red', linestyle=':', linewidth=1,
|
| 331 |
+
label=f'Final Prediction: {final_value:.3f}')
|
| 332 |
+
|
| 333 |
+
# Add feature labels
|
| 334 |
+
ax.set_yticks(np.arange(len(top_names)))
|
| 335 |
+
ax.set_yticklabels(top_names)
|
| 336 |
+
|
| 337 |
+
# Add value annotations
|
| 338 |
+
for i, (shap_val, cum_val) in enumerate(zip(top_shap, cumulative[1:])):
|
| 339 |
+
ax.text(cum_val, i, f'{shap_val:+.3f}',
|
| 340 |
+
va='center', ha='left' if shap_val > 0 else 'right',
|
| 341 |
+
fontsize=8, fontweight='bold',
|
| 342 |
+
bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.8))
|
| 343 |
+
|
| 344 |
+
ax.set_xlabel('Model Output (Deception Probability)')
|
| 345 |
+
ax.set_title('SHAP Waterfall Explanation')
|
| 346 |
+
ax.legend(loc='upper right')
|
| 347 |
+
|
| 348 |
+
# Add interpretation
|
| 349 |
+
ax.text(0.02, 0.98, f'Base: {base_value:.3f} → Final: {final_value:.3f}',
|
| 350 |
+
transform=ax.transAxes, fontsize=9, va='top',
|
| 351 |
+
bbox=dict(boxstyle='round,pad=0.3', facecolor='white', alpha=0.9))
|
| 352 |
+
|
| 353 |
+
plt.tight_layout()
|
| 354 |
+
|
| 355 |
+
if save_path:
|
| 356 |
+
fig.savefig(save_path, dpi=300)
|
| 357 |
+
|
| 358 |
+
self._current_fig = fig
|
| 359 |
+
return fig
|
| 360 |
+
|
| 361 |
+
def plot_group_contributions(
|
| 362 |
+
self,
|
| 363 |
+
group_contributions: Dict[str, float],
|
| 364 |
+
save_path: Optional[str] = None,
|
| 365 |
+
) -> plt.Figure:
|
| 366 |
+
"""Donut chart of feature group contributions.
|
| 367 |
+
|
| 368 |
+
Args:
|
| 369 |
+
group_contributions: Maps group name to normalized contribution (sums to 1.0).
|
| 370 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 371 |
+
|
| 372 |
+
Returns:
|
| 373 |
+
matplotlib.figure.Figure: The created figure.
|
| 374 |
+
"""
|
| 375 |
+
if not group_contributions:
|
| 376 |
+
warnings.warn("Empty group_contributions dictionary")
|
| 377 |
+
fig, ax = plt.subplots(figsize=(6, 6))
|
| 378 |
+
ax.text(0.5, 0.5, "No group contribution data",
|
| 379 |
+
ha='center', va='center', transform=ax.transAxes)
|
| 380 |
+
ax.set_axis_off()
|
| 381 |
+
if save_path:
|
| 382 |
+
fig.savefig(save_path, dpi=300)
|
| 383 |
+
return fig
|
| 384 |
+
|
| 385 |
+
# Normalize contributions to sum to 1.0
|
| 386 |
+
total = sum(group_contributions.values())
|
| 387 |
+
if total == 0:
|
| 388 |
+
# Equal distribution if all zero
|
| 389 |
+
groups = list(group_contributions.keys())
|
| 390 |
+
contributions = [1.0 / len(groups)] * len(groups)
|
| 391 |
+
else:
|
| 392 |
+
groups = list(group_contributions.keys())
|
| 393 |
+
contributions = [group_contributions[g] / total for g in groups]
|
| 394 |
+
|
| 395 |
+
# Sort by contribution
|
| 396 |
+
sorted_indices = np.argsort(contributions)[::-1]
|
| 397 |
+
groups = [groups[i] for i in sorted_indices]
|
| 398 |
+
contributions = [contributions[i] for i in sorted_indices]
|
| 399 |
+
|
| 400 |
+
# Create donut chart
|
| 401 |
+
fig, ax = plt.subplots(figsize=(8, 8))
|
| 402 |
+
|
| 403 |
+
# Use colorblind-friendly palette
|
| 404 |
+
colors = sns.color_palette("colorblind", len(groups))
|
| 405 |
+
|
| 406 |
+
# Outer pie
|
| 407 |
+
wedges, texts, autotexts = ax.pie(
|
| 408 |
+
contributions,
|
| 409 |
+
labels=groups,
|
| 410 |
+
colors=colors,
|
| 411 |
+
autopct='%1.1f%%',
|
| 412 |
+
startangle=90,
|
| 413 |
+
pctdistance=0.85,
|
| 414 |
+
textprops={'fontsize': 9},
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
# Draw circle for donut effect
|
| 418 |
+
centre_circle = plt.Circle((0, 0), 0.70, fc='white')
|
| 419 |
+
ax.add_artist(centre_circle)
|
| 420 |
+
|
| 421 |
+
# Equal aspect ratio
|
| 422 |
+
ax.axis('equal')
|
| 423 |
+
|
| 424 |
+
# Add title
|
| 425 |
+
ax.set_title('Feature Group Contributions', fontsize=12)
|
| 426 |
+
|
| 427 |
+
plt.tight_layout()
|
| 428 |
+
|
| 429 |
+
if save_path:
|
| 430 |
+
fig.savefig(save_path, dpi=300)
|
| 431 |
+
|
| 432 |
+
self._current_fig = fig
|
| 433 |
+
return fig
|
| 434 |
+
|
| 435 |
+
def plot_shap_summary(
|
| 436 |
+
self,
|
| 437 |
+
shap_values: np.ndarray,
|
| 438 |
+
feature_names: List[str],
|
| 439 |
+
plot_type: str = 'bar',
|
| 440 |
+
top_k: int = 15,
|
| 441 |
+
save_path: Optional[str] = None,
|
| 442 |
+
) -> plt.Figure:
|
| 443 |
+
"""SHAP summary plot (bar or beeswarm).
|
| 444 |
+
|
| 445 |
+
Args:
|
| 446 |
+
shap_values: 2D array of SHAP values (n_samples, n_features).
|
| 447 |
+
feature_names: List of feature names.
|
| 448 |
+
plot_type: 'bar' for bar chart of mean absolute SHAP,
|
| 449 |
+
'beeswarm' for beeswarm plot (requires X).
|
| 450 |
+
top_k: Number of top features to display (default 15).
|
| 451 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 452 |
+
|
| 453 |
+
Returns:
|
| 454 |
+
matplotlib.figure.Figure: The created figure.
|
| 455 |
+
|
| 456 |
+
Raises:
|
| 457 |
+
ValueError: If dimensions mismatch or unsupported plot_type.
|
| 458 |
+
"""
|
| 459 |
+
if shap_values.shape[1] != len(feature_names):
|
| 460 |
+
raise ValueError("shap_values columns must match feature_names length")
|
| 461 |
+
|
| 462 |
+
n_samples, n_features = shap_values.shape
|
| 463 |
+
if n_samples == 0 or n_features == 0:
|
| 464 |
+
warnings.warn("No data to plot")
|
| 465 |
+
fig, ax = plt.subplots(figsize=(8, 4))
|
| 466 |
+
ax.text(0.5, 0.5, "No SHAP data",
|
| 467 |
+
ha='center', va='center', transform=ax.transAxes)
|
| 468 |
+
ax.set_axis_off()
|
| 469 |
+
if save_path:
|
| 470 |
+
fig.savefig(save_path, dpi=300)
|
| 471 |
+
return fig
|
| 472 |
+
|
| 473 |
+
# Compute mean absolute SHAP to rank features
|
| 474 |
+
mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
|
| 475 |
+
top_indices = np.argsort(mean_abs_shap)[-top_k:][::-1]
|
| 476 |
+
top_means = mean_abs_shap[top_indices]
|
| 477 |
+
top_names = [feature_names[i] for i in top_indices]
|
| 478 |
+
|
| 479 |
+
if plot_type == 'bar':
|
| 480 |
+
fig, ax = plt.subplots(figsize=(10, max(6, top_k * 0.5)))
|
| 481 |
+
y_pos = np.arange(len(top_names))
|
| 482 |
+
ax.barh(y_pos, top_means, color=self.shap_color, edgecolor='black', linewidth=0.5)
|
| 483 |
+
ax.set_yticks(y_pos)
|
| 484 |
+
ax.set_yticklabels(top_names)
|
| 485 |
+
ax.set_xlabel('Mean |SHAP|')
|
| 486 |
+
ax.set_title(f'SHAP Feature Importance (Top {top_k})')
|
| 487 |
+
ax.invert_yaxis()
|
| 488 |
+
plt.tight_layout()
|
| 489 |
+
elif plot_type == 'beeswarm':
|
| 490 |
+
# Requires X (feature values) which we don't have; fallback to bar
|
| 491 |
+
warnings.warn("Beeswarm plot requires X matrix; falling back to bar plot.")
|
| 492 |
+
return self.plot_shap_summary(shap_values, feature_names, plot_type='bar', top_k=top_k, save_path=save_path)
|
| 493 |
+
else:
|
| 494 |
+
raise ValueError(f"Unsupported plot_type: {plot_type}")
|
| 495 |
+
|
| 496 |
+
if save_path:
|
| 497 |
+
fig.savefig(save_path, dpi=300)
|
| 498 |
+
|
| 499 |
+
self._current_fig = fig
|
| 500 |
+
return fig
|
| 501 |
+
|
| 502 |
+
def plot_feature_groups(
|
| 503 |
+
self,
|
| 504 |
+
shap_values: np.ndarray,
|
| 505 |
+
feature_names: List[str],
|
| 506 |
+
feature_groups: Dict[str, List[str]],
|
| 507 |
+
save_path: Optional[str] = None,
|
| 508 |
+
) -> plt.Figure:
|
| 509 |
+
"""Grouped feature contribution plot.
|
| 510 |
+
|
| 511 |
+
Args:
|
| 512 |
+
shap_values: 2D array of SHAP values (n_samples, n_features).
|
| 513 |
+
feature_names: List of feature names.
|
| 514 |
+
feature_groups: Mapping from group name to list of feature names.
|
| 515 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 516 |
+
|
| 517 |
+
Returns:
|
| 518 |
+
matplotlib.figure.Figure: The created figure.
|
| 519 |
+
"""
|
| 520 |
+
# Compute mean absolute SHAP per feature
|
| 521 |
+
mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
|
| 522 |
+
# Aggregate per group
|
| 523 |
+
group_sums = {}
|
| 524 |
+
for group, features in feature_groups.items():
|
| 525 |
+
total = 0.0
|
| 526 |
+
for feat in features:
|
| 527 |
+
if feat in feature_names:
|
| 528 |
+
idx = feature_names.index(feat)
|
| 529 |
+
total += mean_abs_shap[idx]
|
| 530 |
+
group_sums[group] = total
|
| 531 |
+
|
| 532 |
+
# Normalize to percentages
|
| 533 |
+
total = sum(group_sums.values())
|
| 534 |
+
if total == 0:
|
| 535 |
+
group_percents = {k: 0.0 for k in group_sums.keys()}
|
| 536 |
+
else:
|
| 537 |
+
group_percents = {k: v / total * 100 for k, v in group_sums.items()}
|
| 538 |
+
|
| 539 |
+
# Sort groups by contribution
|
| 540 |
+
sorted_groups = sorted(group_percents.items(), key=lambda x: x[1], reverse=True)
|
| 541 |
+
groups = [g for g, _ in sorted_groups]
|
| 542 |
+
percents = [p for _, p in sorted_groups]
|
| 543 |
+
|
| 544 |
+
fig, ax = plt.subplots(figsize=(10, max(6, len(groups) * 0.5)))
|
| 545 |
+
y_pos = np.arange(len(groups))
|
| 546 |
+
colors = sns.color_palette("colorblind", len(groups))
|
| 547 |
+
ax.barh(y_pos, percents, color=colors, edgecolor='black', linewidth=0.5)
|
| 548 |
+
ax.set_yticks(y_pos)
|
| 549 |
+
ax.set_yticklabels(groups)
|
| 550 |
+
ax.set_xlabel('Contribution (%)')
|
| 551 |
+
ax.set_title('Feature Group Contributions (SHAP)')
|
| 552 |
+
# Add percentage labels
|
| 553 |
+
for i, p in enumerate(percents):
|
| 554 |
+
ax.text(p + 0.5, i, f'{p:.1f}%', va='center', fontsize=9)
|
| 555 |
+
|
| 556 |
+
ax.invert_yaxis()
|
| 557 |
+
plt.tight_layout()
|
| 558 |
+
|
| 559 |
+
if save_path:
|
| 560 |
+
fig.savefig(save_path, dpi=300)
|
| 561 |
+
|
| 562 |
+
self._current_fig = fig
|
| 563 |
+
return fig
|
| 564 |
+
|
| 565 |
+
def plot_behavioral_interpretation(
|
| 566 |
+
self,
|
| 567 |
+
behavioral_explanations: List[Dict[str, Any]],
|
| 568 |
+
save_path: Optional[str] = None,
|
| 569 |
+
) -> plt.Figure:
|
| 570 |
+
"""Human-readable behavioral interpretation visualization.
|
| 571 |
+
|
| 572 |
+
Args:
|
| 573 |
+
behavioral_explanations: List of dicts with keys 'feature', 'value',
|
| 574 |
+
'shap_value', 'direction', 'interpretation'.
|
| 575 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 576 |
+
|
| 577 |
+
Returns:
|
| 578 |
+
matplotlib.figure.Figure: The created figure.
|
| 579 |
+
"""
|
| 580 |
+
if not behavioral_explanations:
|
| 581 |
+
warnings.warn("Empty behavioral_explanations list")
|
| 582 |
+
fig, ax = plt.subplots(figsize=(8, 4))
|
| 583 |
+
ax.text(0.5, 0.5, "No behavioral explanation data",
|
| 584 |
+
ha='center', va='center', transform=ax.transAxes)
|
| 585 |
+
ax.set_axis_off()
|
| 586 |
+
if save_path:
|
| 587 |
+
fig.savefig(save_path, dpi=300)
|
| 588 |
+
return fig
|
| 589 |
+
|
| 590 |
+
# Limit to top 10 for readability
|
| 591 |
+
top_explanations = behavioral_explanations[:10]
|
| 592 |
+
n = len(top_explanations)
|
| 593 |
+
|
| 594 |
+
fig, ax = plt.subplots(figsize=(12, max(6, n * 0.8)))
|
| 595 |
+
ax.axis('off')
|
| 596 |
+
|
| 597 |
+
# Create a table-like visualization
|
| 598 |
+
y_pos = np.arange(n) * 1.2
|
| 599 |
+
for i, exp in enumerate(top_explanations):
|
| 600 |
+
feature = exp.get('feature', 'Unknown')
|
| 601 |
+
value = exp.get('value', 0.0)
|
| 602 |
+
shap_val = exp.get('shap_value', 0.0)
|
| 603 |
+
direction = exp.get('direction', 'neutral')
|
| 604 |
+
interpretation = exp.get('interpretation', '')
|
| 605 |
+
|
| 606 |
+
# Determine color based on direction
|
| 607 |
+
if direction == 'toward_deception':
|
| 608 |
+
color = self.deception_color
|
| 609 |
+
dir_text = '→ Deception'
|
| 610 |
+
elif direction == 'toward_truth':
|
| 611 |
+
color = self.truth_color
|
| 612 |
+
dir_text = '→ Truth'
|
| 613 |
+
else:
|
| 614 |
+
color = 'gray'
|
| 615 |
+
dir_text = 'Neutral'
|
| 616 |
+
|
| 617 |
+
# Feature name and value
|
| 618 |
+
ax.text(0.1, y_pos[i], f'{feature}', fontsize=10, fontweight='bold',
|
| 619 |
+
verticalalignment='center')
|
| 620 |
+
ax.text(0.3, y_pos[i], f'{value:.3f}', fontsize=9,
|
| 621 |
+
verticalalignment='center')
|
| 622 |
+
# SHAP value with sign
|
| 623 |
+
shap_sign = '+' if shap_val >= 0 else ''
|
| 624 |
+
ax.text(0.45, y_pos[i], f'{shap_sign}{shap_val:.3f}', fontsize=9,
|
| 625 |
+
color=color, verticalalignment='center')
|
| 626 |
+
# Direction arrow
|
| 627 |
+
ax.text(0.6, y_pos[i], dir_text, fontsize=9, color=color,
|
| 628 |
+
verticalalignment='center')
|
| 629 |
+
# Interpretation (wrap)
|
| 630 |
+
ax.text(0.75, y_pos[i], interpretation, fontsize=8, wrap=True,
|
| 631 |
+
verticalalignment='center', style='italic')
|
| 632 |
+
|
| 633 |
+
# Add column headers
|
| 634 |
+
ax.text(0.1, y_pos[-1] + 1.5, 'Feature', fontsize=10, fontweight='bold')
|
| 635 |
+
ax.text(0.3, y_pos[-1] + 1.5, 'Value', fontsize=10, fontweight='bold')
|
| 636 |
+
ax.text(0.45, y_pos[-1] + 1.5, 'SHAP', fontsize=10, fontweight='bold')
|
| 637 |
+
ax.text(0.6, y_pos[-1] + 1.5, 'Direction', fontsize=10, fontweight='bold')
|
| 638 |
+
ax.text(0.75, y_pos[-1] + 1.5, 'Interpretation', fontsize=10, fontweight='bold')
|
| 639 |
+
|
| 640 |
+
ax.set_xlim(0, 1)
|
| 641 |
+
ax.set_ylim(-1, y_pos[-1] + 3)
|
| 642 |
+
ax.set_title('Behavioral Interpretation', fontsize=14, pad=20)
|
| 643 |
+
|
| 644 |
+
plt.tight_layout()
|
| 645 |
+
|
| 646 |
+
if save_path:
|
| 647 |
+
fig.savefig(save_path, dpi=300)
|
| 648 |
+
|
| 649 |
+
self._current_fig = fig
|
| 650 |
+
return fig
|
| 651 |
+
|
| 652 |
+
def plot_explanation_dashboard(
|
| 653 |
+
self,
|
| 654 |
+
shap_values: np.ndarray,
|
| 655 |
+
feature_names: List[str],
|
| 656 |
+
feature_groups: Dict[str, List[str]],
|
| 657 |
+
behavioral_explanations: List[Dict[str, Any]],
|
| 658 |
+
save_path: Optional[str] = None,
|
| 659 |
+
) -> plt.Figure:
|
| 660 |
+
"""Combined dashboard with all plots.
|
| 661 |
+
|
| 662 |
+
Args:
|
| 663 |
+
shap_values: 2D array of SHAP values (n_samples, n_features).
|
| 664 |
+
feature_names: List of feature names.
|
| 665 |
+
feature_groups: Mapping from group name to list of feature names.
|
| 666 |
+
behavioral_explanations: List of behavioral explanation dicts.
|
| 667 |
+
save_path: Optional path to save figure at 300 DPI.
|
| 668 |
+
|
| 669 |
+
Returns:
|
| 670 |
+
matplotlib.figure.Figure: The created figure.
|
| 671 |
+
"""
|
| 672 |
+
fig = plt.figure(figsize=(16, 12))
|
| 673 |
+
gs = gridspec.GridSpec(2, 2, figure=fig, hspace=0.3, wspace=0.3)
|
| 674 |
+
|
| 675 |
+
# 1. SHAP summary bar plot
|
| 676 |
+
ax1 = fig.add_subplot(gs[0, 0])
|
| 677 |
+
mean_abs_shap = np.mean(np.abs(shap_values), axis=0)
|
| 678 |
+
top_k = 10
|
| 679 |
+
top_indices = np.argsort(mean_abs_shap)[-top_k:][::-1]
|
| 680 |
+
top_means = mean_abs_shap[top_indices]
|
| 681 |
+
top_names = [feature_names[i] for i in top_indices]
|
| 682 |
+
y_pos = np.arange(len(top_names))
|
| 683 |
+
ax1.barh(y_pos, top_means, color=self.shap_color, edgecolor='black', linewidth=0.5)
|
| 684 |
+
ax1.set_yticks(y_pos)
|
| 685 |
+
ax1.set_yticklabels(top_names)
|
| 686 |
+
ax1.set_xlabel('Mean |SHAP|')
|
| 687 |
+
ax1.set_title('SHAP Feature Importance (Top 10)')
|
| 688 |
+
ax1.invert_yaxis()
|
| 689 |
+
|
| 690 |
+
# 2. Feature group contributions
|
| 691 |
+
ax2 = fig.add_subplot(gs[0, 1])
|
| 692 |
+
group_sums = {}
|
| 693 |
+
for group, features in feature_groups.items():
|
| 694 |
+
total = 0.0
|
| 695 |
+
for feat in features:
|
| 696 |
+
if feat in feature_names:
|
| 697 |
+
idx = feature_names.index(feat)
|
| 698 |
+
total += mean_abs_shap[idx]
|
| 699 |
+
group_sums[group] = total
|
| 700 |
+
total = sum(group_sums.values())
|
| 701 |
+
if total == 0:
|
| 702 |
+
group_percents = {k: 0.0 for k in group_sums.keys()}
|
| 703 |
+
else:
|
| 704 |
+
group_percents = {k: v / total * 100 for k, v in group_sums.items()}
|
| 705 |
+
sorted_groups = sorted(group_percents.items(), key=lambda x: x[1], reverse=True)
|
| 706 |
+
groups = [g for g, _ in sorted_groups]
|
| 707 |
+
percents = [p for _, p in sorted_groups]
|
| 708 |
+
colors = sns.color_palette("colorblind", len(groups))
|
| 709 |
+
ax2.barh(np.arange(len(groups)), percents, color=colors, edgecolor='black', linewidth=0.5)
|
| 710 |
+
ax2.set_yticks(np.arange(len(groups)))
|
| 711 |
+
ax2.set_yticklabels(groups)
|
| 712 |
+
ax2.set_xlabel('Contribution (%)')
|
| 713 |
+
ax2.set_title('Feature Group Contributions')
|
| 714 |
+
ax2.invert_yaxis()
|
| 715 |
+
|
| 716 |
+
# 3. Behavioral interpretation table
|
| 717 |
+
ax3 = fig.add_subplot(gs[1, :])
|
| 718 |
+
ax3.axis('off')
|
| 719 |
+
if behavioral_explanations:
|
| 720 |
+
top_explanations = behavioral_explanations[:5]
|
| 721 |
+
y_pos = np.arange(len(top_explanations)) * 1.5
|
| 722 |
+
for i, exp in enumerate(top_explanations):
|
| 723 |
+
feature = exp.get('feature', 'Unknown')
|
| 724 |
+
shap_val = exp.get('shap_value', 0.0)
|
| 725 |
+
direction = exp.get('direction', 'neutral')
|
| 726 |
+
interpretation = exp.get('interpretation', '')
|
| 727 |
+
color = self.deception_color if direction == 'toward_deception' else self.truth_color
|
| 728 |
+
ax3.text(0.1, y_pos[i], f'{feature}', fontsize=10, fontweight='bold')
|
| 729 |
+
ax3.text(0.3, y_pos[i], f'{shap_val:+.3f}', fontsize=9, color=color)
|
| 730 |
+
ax3.text(0.5, y_pos[i], direction, fontsize=9, color=color)
|
| 731 |
+
ax3.text(0.7, y_pos[i], interpretation, fontsize=8, wrap=True, style='italic')
|
| 732 |
+
ax3.set_title('Top Behavioral Indicators', fontsize=12, pad=10)
|
| 733 |
+
else:
|
| 734 |
+
ax3.text(0.5, 0.5, 'No behavioral explanations', ha='center', va='center', fontsize=12)
|
| 735 |
+
|
| 736 |
+
fig.suptitle('Explainability Dashboard', fontsize=16, y=0.98)
|
| 737 |
+
plt.tight_layout(rect=[0, 0, 1, 0.96])
|
| 738 |
+
|
| 739 |
+
if save_path:
|
| 740 |
+
fig.savefig(save_path, dpi=300)
|
| 741 |
+
|
| 742 |
+
self._current_fig = fig
|
| 743 |
+
return fig
|
src/facial/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Facial analysis module for micro-expression detection."""
|
| 2 |
+
|
| 3 |
+
from .extractor import FacialFeatureExtractor
|
| 4 |
+
from .visualizer import FacialVisualizer
|
| 5 |
+
|
| 6 |
+
__all__ = ['FacialFeatureExtractor', 'FacialVisualizer']
|
src/facial/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (337 Bytes). View file
|
|
|
src/facial/extractor.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Facial micro-expression feature extraction using MediaPipe Face Mesh."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Dict, List, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
import cv2
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pandas as pd
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
|
| 12 |
+
from src.utils.config import FeatureConfig, get_default_config
|
| 13 |
+
from src.utils.io import load_video
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class FacialFeatureExtractor:
|
| 17 |
+
"""Extracts micro-expression features from video using MediaPipe Face Mesh."""
|
| 18 |
+
|
| 19 |
+
def _get_mp_solutions(self):
|
| 20 |
+
"""Lazily load mediapipe to avoid global import errors."""
|
| 21 |
+
try:
|
| 22 |
+
import mediapipe as mp
|
| 23 |
+
return mp.solutions
|
| 24 |
+
except Exception as e:
|
| 25 |
+
raise ImportError(
|
| 26 |
+
"MediaPipe 'solutions' module not found. Original error: " + str(e)
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
_LEFT_EYE = [33, 160, 158, 133, 153, 144]
|
| 30 |
+
_RIGHT_EYE = [362, 385, 387, 263, 373, 380]
|
| 31 |
+
_LEFT_BROW = [70, 63, 105, 66, 107]
|
| 32 |
+
_RIGHT_BROW = [336, 296, 334, 293, 300]
|
| 33 |
+
_NOSE = [1, 168, 98, 327]
|
| 34 |
+
_HEAD_POINTS = [1, 33, 263, 61, 291, 199]
|
| 35 |
+
_FEATURE_NAMES = [
|
| 36 |
+
"eye_left_openness",
|
| 37 |
+
"eye_right_openness",
|
| 38 |
+
"mouth_openness",
|
| 39 |
+
"mouth_width",
|
| 40 |
+
"mouth_asymmetry",
|
| 41 |
+
"left_brow_height",
|
| 42 |
+
"right_brow_height",
|
| 43 |
+
"brow_asymmetry",
|
| 44 |
+
"brow_tension",
|
| 45 |
+
"nose_wrinkle",
|
| 46 |
+
"nostril_flare",
|
| 47 |
+
"nose_asymmetry",
|
| 48 |
+
"head_pitch",
|
| 49 |
+
"head_yaw",
|
| 50 |
+
"head_roll",
|
| 51 |
+
"head_displacement",
|
| 52 |
+
]
|
| 53 |
+
|
| 54 |
+
def __init__(self, config: Optional[FeatureConfig] = None):
|
| 55 |
+
"""Initialize the MediaPipe Face Mesh extractor.
|
| 56 |
+
|
| 57 |
+
Gracefully handles missing MediaPipe dependencies by setting an 'available' flag.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
self.config = config or get_default_config().features
|
| 61 |
+
self.available = False
|
| 62 |
+
self.face_mesh = None
|
| 63 |
+
|
| 64 |
+
try:
|
| 65 |
+
solutions = self._get_mp_solutions()
|
| 66 |
+
self.mp_face_mesh = solutions.face_mesh
|
| 67 |
+
self.face_mesh = self.mp_face_mesh.FaceMesh(
|
| 68 |
+
static_image_mode=False,
|
| 69 |
+
max_num_faces=1,
|
| 70 |
+
refine_landmarks=True,
|
| 71 |
+
min_detection_confidence=0.5,
|
| 72 |
+
min_tracking_confidence=0.5,
|
| 73 |
+
)
|
| 74 |
+
self.available = True
|
| 75 |
+
except (ImportError, AttributeError) as e:
|
| 76 |
+
print(f"\u26a0\ufe0f WARNING: Facial features disabled. MediaPipe error: {e}")
|
| 77 |
+
|
| 78 |
+
@staticmethod
|
| 79 |
+
def _distance(point_a: np.ndarray, point_b: np.ndarray) -> float:
|
| 80 |
+
"""Return Euclidean distance between two landmark points."""
|
| 81 |
+
return float(np.linalg.norm(point_a[:3] - point_b[:3]))
|
| 82 |
+
|
| 83 |
+
@staticmethod
|
| 84 |
+
def _face_diagonal(landmarks: np.ndarray) -> float:
|
| 85 |
+
"""Return face bounding-box diagonal for scale normalization."""
|
| 86 |
+
if landmarks.size == 0:
|
| 87 |
+
return 1.0
|
| 88 |
+
mins = np.nanmin(landmarks[:, :2], axis=0)
|
| 89 |
+
maxs = np.nanmax(landmarks[:, :2], axis=0)
|
| 90 |
+
diagonal = float(np.linalg.norm(maxs - mins))
|
| 91 |
+
return diagonal if diagonal > 1e-6 else 1.0
|
| 92 |
+
|
| 93 |
+
@staticmethod
|
| 94 |
+
def _safe_ratio(numerator: float, denominator: float) -> float:
|
| 95 |
+
"""Return a bounded ratio while avoiding division by zero."""
|
| 96 |
+
if denominator <= 1e-8 or not np.isfinite(denominator):
|
| 97 |
+
return 0.0
|
| 98 |
+
value = numerator / denominator
|
| 99 |
+
return float(np.clip(value, 0.0, 1.0))
|
| 100 |
+
|
| 101 |
+
def _extract_landmarks(self, frame: np.ndarray) -> Optional[np.ndarray]:
|
| 102 |
+
"""Run Face Mesh on a BGR frame and return normalized landmarks."""
|
| 103 |
+
|
| 104 |
+
if not self.available or self.face_mesh is None or frame is None or frame.size == 0:
|
| 105 |
+
return None
|
| 106 |
+
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
| 107 |
+
if float(cv2.Laplacian(gray, cv2.CV_64F).var()) < 5.0:
|
| 108 |
+
# Extremely blurry frames can cause unstable micro-expression measurements.
|
| 109 |
+
return None
|
| 110 |
+
|
| 111 |
+
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 112 |
+
results = self.face_mesh.process(rgb_frame)
|
| 113 |
+
if not results.multi_face_landmarks:
|
| 114 |
+
return None
|
| 115 |
+
|
| 116 |
+
height, width = frame.shape[:2]
|
| 117 |
+
landmarks = np.array(
|
| 118 |
+
[[point.x * width, point.y * height, point.z * width] for point in results.multi_face_landmarks[0].landmark],
|
| 119 |
+
dtype=np.float32,
|
| 120 |
+
)
|
| 121 |
+
if self._face_diagonal(landmarks) < 25.0:
|
| 122 |
+
# Very small faces do not provide reliable deception-relevant movements.
|
| 123 |
+
return None
|
| 124 |
+
return landmarks
|
| 125 |
+
|
| 126 |
+
def extract_eye_openness(self, landmarks: np.ndarray) -> Tuple[float, float]:
|
| 127 |
+
"""Calculate left and right eye openness ratios.
|
| 128 |
+
|
| 129 |
+
Args:
|
| 130 |
+
landmarks: Face Mesh landmarks with shape ``(468, 3)``.
|
| 131 |
+
|
| 132 |
+
Returns:
|
| 133 |
+
Tuple containing left and right Eye Aspect Ratio values.
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
def ear(indices: List[int]) -> float:
|
| 137 |
+
p1, p2, p3, p4, p5, p6 = landmarks[indices]
|
| 138 |
+
vertical = self._distance(p2, p6) + self._distance(p3, p5)
|
| 139 |
+
horizontal = 2.0 * self._distance(p1, p4)
|
| 140 |
+
# Blink rate and eye narrowing are behavioral cues often reviewed for stress or concealment.
|
| 141 |
+
return self._safe_ratio(vertical, horizontal)
|
| 142 |
+
|
| 143 |
+
return ear(self._LEFT_EYE), ear(self._RIGHT_EYE)
|
| 144 |
+
|
| 145 |
+
def extract_mouth_movement(self, landmarks: np.ndarray) -> Dict[str, float]:
|
| 146 |
+
"""Calculate scale-normalized mouth movement features.
|
| 147 |
+
|
| 148 |
+
Args:
|
| 149 |
+
landmarks: Face Mesh landmarks with shape ``(468, 3)``.
|
| 150 |
+
|
| 151 |
+
Returns:
|
| 152 |
+
Dictionary containing mouth openness, width, and asymmetry.
|
| 153 |
+
"""
|
| 154 |
+
|
| 155 |
+
diagonal = self._face_diagonal(landmarks)
|
| 156 |
+
upper_lip, lower_lip = landmarks[13], landmarks[14]
|
| 157 |
+
left_corner, right_corner = landmarks[61], landmarks[291]
|
| 158 |
+
center = (upper_lip + lower_lip) / 2.0
|
| 159 |
+
left_distance = self._distance(center, left_corner)
|
| 160 |
+
right_distance = self._distance(center, right_corner)
|
| 161 |
+
# Mouth tension, dry-mouth behavior, and articulation changes can accompany cognitive load.
|
| 162 |
+
return {
|
| 163 |
+
"mouth_openness": self._distance(upper_lip, lower_lip) / diagonal,
|
| 164 |
+
"mouth_width": self._distance(left_corner, right_corner) / diagonal,
|
| 165 |
+
"mouth_asymmetry": abs(left_distance - right_distance) / diagonal,
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
def extract_eyebrow_tension(self, landmarks: np.ndarray) -> Dict[str, float]:
|
| 169 |
+
"""Calculate eyebrow raise, asymmetry, and squeeze features.
|
| 170 |
+
|
| 171 |
+
Args:
|
| 172 |
+
landmarks: Face Mesh landmarks with shape ``(468, 3)``.
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
Dictionary containing brow height, asymmetry, and tension.
|
| 176 |
+
"""
|
| 177 |
+
|
| 178 |
+
diagonal = self._face_diagonal(landmarks)
|
| 179 |
+
nose_bridge = landmarks[168]
|
| 180 |
+
left_brow = landmarks[self._LEFT_BROW]
|
| 181 |
+
right_brow = landmarks[self._RIGHT_BROW]
|
| 182 |
+
left_height = float(np.mean(nose_bridge[1] - left_brow[:, 1]) / diagonal)
|
| 183 |
+
right_height = float(np.mean(nose_bridge[1] - right_brow[:, 1]) / diagonal)
|
| 184 |
+
inner_distance = self._distance(landmarks[107], landmarks[336]) / diagonal
|
| 185 |
+
# Brow squeeze and asymmetry are micro-expression proxies for stress, concern, or suppression.
|
| 186 |
+
return {
|
| 187 |
+
"left_brow_height": left_height,
|
| 188 |
+
"right_brow_height": right_height,
|
| 189 |
+
"brow_asymmetry": abs(left_height - right_height),
|
| 190 |
+
"brow_tension": max(0.0, 1.0 - inner_distance),
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
def extract_nose_movement(self, landmarks: np.ndarray) -> Dict[str, float]:
|
| 194 |
+
"""Calculate nose wrinkle, nostril flare, and asymmetry features.
|
| 195 |
+
|
| 196 |
+
Args:
|
| 197 |
+
landmarks: Face Mesh landmarks with shape ``(468, 3)``.
|
| 198 |
+
|
| 199 |
+
Returns:
|
| 200 |
+
Dictionary containing nose movement measurements.
|
| 201 |
+
"""
|
| 202 |
+
|
| 203 |
+
diagonal = self._face_diagonal(landmarks)
|
| 204 |
+
nose_tip, nose_bridge = landmarks[1], landmarks[168]
|
| 205 |
+
left_nostril, right_nostril = landmarks[98], landmarks[327]
|
| 206 |
+
center = (left_nostril + right_nostril) / 2.0
|
| 207 |
+
# Nose wrinkling or nostril flare can indicate disgust, arousal, or regulated emotion.
|
| 208 |
+
return {
|
| 209 |
+
"nose_wrinkle": abs(float(nose_tip[1] - nose_bridge[1])) / diagonal,
|
| 210 |
+
"nostril_flare": self._distance(left_nostril, right_nostril) / diagonal,
|
| 211 |
+
"nose_asymmetry": abs(self._distance(center, left_nostril) - self._distance(center, right_nostril)) / diagonal,
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
def extract_head_dynamics(
|
| 215 |
+
self, landmarks: np.ndarray, prev_landmarks: Optional[np.ndarray] = None
|
| 216 |
+
) -> Dict[str, float]:
|
| 217 |
+
"""Calculate approximate head pose and inter-frame movement.
|
| 218 |
+
|
| 219 |
+
Args:
|
| 220 |
+
landmarks: Current Face Mesh landmarks with shape ``(468, 3)``.
|
| 221 |
+
prev_landmarks: Optional previous frame landmarks for displacement.
|
| 222 |
+
|
| 223 |
+
Returns:
|
| 224 |
+
Dictionary containing pitch, yaw, roll, and displacement.
|
| 225 |
+
"""
|
| 226 |
+
|
| 227 |
+
diagonal = self._face_diagonal(landmarks)
|
| 228 |
+
left_eye = landmarks[33]
|
| 229 |
+
right_eye = landmarks[263]
|
| 230 |
+
nose_tip = landmarks[1]
|
| 231 |
+
chin = landmarks[199]
|
| 232 |
+
mouth_left = landmarks[61]
|
| 233 |
+
mouth_right = landmarks[291]
|
| 234 |
+
|
| 235 |
+
eye_center = (left_eye + right_eye) / 2.0
|
| 236 |
+
mouth_center = (mouth_left + mouth_right) / 2.0
|
| 237 |
+
eye_vector = right_eye - left_eye
|
| 238 |
+
face_height = max(self._distance(eye_center, chin), 1e-6)
|
| 239 |
+
face_width = max(self._distance(left_eye, right_eye), 1e-6)
|
| 240 |
+
|
| 241 |
+
roll = float(np.degrees(np.arctan2(eye_vector[1], eye_vector[0])))
|
| 242 |
+
yaw = float(((nose_tip[0] - eye_center[0]) / face_width) * 45.0)
|
| 243 |
+
pitch = float(((nose_tip[1] - mouth_center[1]) / face_height) * 45.0)
|
| 244 |
+
displacement = 0.0
|
| 245 |
+
if prev_landmarks is not None and prev_landmarks.shape == landmarks.shape:
|
| 246 |
+
# Sudden head movement or freezing can be relevant when paired with verbal hesitation.
|
| 247 |
+
displacement = float(np.mean(np.linalg.norm(landmarks[self._HEAD_POINTS] - prev_landmarks[self._HEAD_POINTS], axis=1)) / diagonal)
|
| 248 |
+
|
| 249 |
+
return {
|
| 250 |
+
"head_pitch": pitch,
|
| 251 |
+
"head_yaw": yaw,
|
| 252 |
+
"head_roll": roll,
|
| 253 |
+
"head_displacement": displacement,
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
def process_frame(
|
| 257 |
+
self, frame: np.ndarray, prev_landmarks: Optional[np.ndarray] = None
|
| 258 |
+
) -> Optional[Dict[str, float]]:
|
| 259 |
+
"""Process a single BGR frame and extract all facial features.
|
| 260 |
+
|
| 261 |
+
Args:
|
| 262 |
+
frame: BGR video frame.
|
| 263 |
+
prev_landmarks: Optional previous frame landmarks for dynamics.
|
| 264 |
+
|
| 265 |
+
Returns:
|
| 266 |
+
Combined feature dictionary, or ``None`` when no reliable face is detected.
|
| 267 |
+
"""
|
| 268 |
+
|
| 269 |
+
landmarks = self._extract_landmarks(frame)
|
| 270 |
+
if landmarks is None:
|
| 271 |
+
return None
|
| 272 |
+
|
| 273 |
+
left_eye, right_eye = self.extract_eye_openness(landmarks)
|
| 274 |
+
features: Dict[str, float] = {
|
| 275 |
+
"eye_left_openness": left_eye,
|
| 276 |
+
"eye_right_openness": right_eye,
|
| 277 |
+
}
|
| 278 |
+
features.update(self.extract_mouth_movement(landmarks))
|
| 279 |
+
features.update(self.extract_eyebrow_tension(landmarks))
|
| 280 |
+
features.update(self.extract_nose_movement(landmarks))
|
| 281 |
+
features.update(self.extract_head_dynamics(landmarks, prev_landmarks))
|
| 282 |
+
features["_landmarks"] = landmarks # Internal handoff for process_video only.
|
| 283 |
+
return features
|
| 284 |
+
|
| 285 |
+
def process_video(self, video_path: str, max_frames: Optional[int] = None) -> pd.DataFrame:
|
| 286 |
+
"""Process a video and extract frame-level facial features.
|
| 287 |
+
|
| 288 |
+
Args:
|
| 289 |
+
video_path: Path to a video file.
|
| 290 |
+
max_frames: Optional maximum number of frames to process.
|
| 291 |
+
|
| 292 |
+
Returns:
|
| 293 |
+
DataFrame with frame index, timestamp, and feature columns. Missing
|
| 294 |
+
detections are represented with ``NaN`` feature values.
|
| 295 |
+
"""
|
| 296 |
+
|
| 297 |
+
capture, metadata = load_video(video_path)
|
| 298 |
+
fps = float(metadata.get("fps") or 30.0)
|
| 299 |
+
total_frames = int(metadata.get("frame_count") or 0)
|
| 300 |
+
limit = min(total_frames, max_frames) if max_frames is not None and total_frames else max_frames or total_frames
|
| 301 |
+
records: List[Dict[str, float]] = []
|
| 302 |
+
prev_landmarks: Optional[np.ndarray] = None
|
| 303 |
+
|
| 304 |
+
try:
|
| 305 |
+
progress = tqdm(total=limit if limit else None, desc="Extracting facial features")
|
| 306 |
+
frame_idx = 0
|
| 307 |
+
while True:
|
| 308 |
+
if max_frames is not None and frame_idx >= max_frames:
|
| 309 |
+
break
|
| 310 |
+
ok, frame = capture.read()
|
| 311 |
+
if not ok:
|
| 312 |
+
break
|
| 313 |
+
record: Dict[str, float] = {"frame_idx": float(frame_idx), "timestamp": frame_idx / fps}
|
| 314 |
+
features = self.process_frame(frame, prev_landmarks)
|
| 315 |
+
if features is None:
|
| 316 |
+
record.update({name: np.nan for name in self.get_feature_names()})
|
| 317 |
+
prev_landmarks = None
|
| 318 |
+
else:
|
| 319 |
+
landmarks = features.pop("_landmarks")
|
| 320 |
+
record.update(features)
|
| 321 |
+
prev_landmarks = landmarks
|
| 322 |
+
records.append(record)
|
| 323 |
+
frame_idx += 1
|
| 324 |
+
progress.update(1)
|
| 325 |
+
progress.close()
|
| 326 |
+
finally:
|
| 327 |
+
capture.release()
|
| 328 |
+
|
| 329 |
+
columns = ["frame_idx", "timestamp"] + self.get_feature_names()
|
| 330 |
+
return pd.DataFrame(records, columns=columns)
|
| 331 |
+
|
| 332 |
+
def get_feature_names(self) -> List[str]:
|
| 333 |
+
"""Return ordered names of all produced facial features."""
|
| 334 |
+
|
| 335 |
+
return list(self._FEATURE_NAMES)
|
| 336 |
+
|
| 337 |
+
def close(self) -> None:
|
| 338 |
+
"""Release MediaPipe resources."""
|
| 339 |
+
|
| 340 |
+
self.face_mesh.close()
|
| 341 |
+
|
src/facial/visualizer.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Visualization utilities for facial landmarks and feature timelines."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Dict, Optional
|
| 6 |
+
|
| 7 |
+
import cv2
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import numpy as np
|
| 10 |
+
import pandas as pd
|
| 11 |
+
import seaborn as sns
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class FacialVisualizer:
|
| 15 |
+
"""Visualizes facial landmarks and feature values on video frames."""
|
| 16 |
+
|
| 17 |
+
def __init__(self):
|
| 18 |
+
"""Define landmark groups and display thresholds for overlays."""
|
| 19 |
+
|
| 20 |
+
self.eye_landmarks = [33, 160, 158, 133, 153, 144, 362, 385, 387, 263, 373, 380]
|
| 21 |
+
self.mouth_landmarks = [13, 14, 61, 291]
|
| 22 |
+
self.eyebrow_landmarks = [70, 63, 105, 66, 107, 336, 296, 334, 293, 300]
|
| 23 |
+
self.eye_threshold = 0.22
|
| 24 |
+
self.mouth_threshold = 0.35
|
| 25 |
+
self.brow_threshold = 0.18
|
| 26 |
+
|
| 27 |
+
@staticmethod
|
| 28 |
+
def _as_array(landmarks) -> np.ndarray:
|
| 29 |
+
"""Convert supported landmark representations to an array."""
|
| 30 |
+
|
| 31 |
+
if landmarks is None:
|
| 32 |
+
return np.empty((0, 3), dtype=np.float32)
|
| 33 |
+
if isinstance(landmarks, np.ndarray):
|
| 34 |
+
return landmarks
|
| 35 |
+
if hasattr(landmarks, "landmark"):
|
| 36 |
+
return np.array([[point.x, point.y, point.z] for point in landmarks.landmark], dtype=np.float32)
|
| 37 |
+
return np.asarray(landmarks, dtype=np.float32)
|
| 38 |
+
|
| 39 |
+
def draw_landmarks(self, frame: np.ndarray, landmarks, feature_values: Dict[str, float]) -> np.ndarray:
|
| 40 |
+
"""Draw face mesh landmarks and highlight active feature groups.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
frame: BGR image frame to annotate.
|
| 44 |
+
landmarks: Landmark array or MediaPipe landmark list.
|
| 45 |
+
feature_values: Dictionary of extracted facial feature values.
|
| 46 |
+
|
| 47 |
+
Returns:
|
| 48 |
+
Annotated frame copy.
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
output = frame.copy()
|
| 52 |
+
points = self._as_array(landmarks)
|
| 53 |
+
if points.size == 0:
|
| 54 |
+
cv2.putText(output, "No face detected", (20, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2)
|
| 55 |
+
return output
|
| 56 |
+
|
| 57 |
+
height, width = output.shape[:2]
|
| 58 |
+
if np.nanmax(points[:, :2]) <= 1.5:
|
| 59 |
+
draw_points = np.column_stack((points[:, 0] * width, points[:, 1] * height)).astype(int)
|
| 60 |
+
else:
|
| 61 |
+
draw_points = points[:, :2].astype(int)
|
| 62 |
+
|
| 63 |
+
for x_coord, y_coord in draw_points:
|
| 64 |
+
cv2.circle(output, (int(x_coord), int(y_coord)), 1, (0, 180, 0), -1)
|
| 65 |
+
|
| 66 |
+
eye_open = min(
|
| 67 |
+
float(feature_values.get("eye_left_openness", 1.0)),
|
| 68 |
+
float(feature_values.get("eye_right_openness", 1.0)),
|
| 69 |
+
)
|
| 70 |
+
if eye_open < self.eye_threshold:
|
| 71 |
+
# Eye closure highlights possible blinking or avoidance-related stress markers.
|
| 72 |
+
for idx in self.eye_landmarks:
|
| 73 |
+
cv2.circle(output, tuple(draw_points[idx]), 3, (255, 0, 0), -1)
|
| 74 |
+
|
| 75 |
+
if float(feature_values.get("mouth_openness", 0.0)) > self.mouth_threshold:
|
| 76 |
+
# Mouth movement highlights speech articulation shifts and possible hesitation.
|
| 77 |
+
for idx in self.mouth_landmarks:
|
| 78 |
+
cv2.circle(output, tuple(draw_points[idx]), 4, (0, 0, 255), -1)
|
| 79 |
+
|
| 80 |
+
if float(feature_values.get("brow_tension", 0.0)) > self.brow_threshold:
|
| 81 |
+
# Brow tension highlights facial stress or emotion-suppression cues.
|
| 82 |
+
for idx in self.eyebrow_landmarks:
|
| 83 |
+
cv2.circle(output, tuple(draw_points[idx]), 3, (0, 255, 255), -1)
|
| 84 |
+
|
| 85 |
+
overlay_items = [
|
| 86 |
+
("Eye", eye_open),
|
| 87 |
+
("Mouth", float(feature_values.get("mouth_openness", np.nan))),
|
| 88 |
+
("Brow", float(feature_values.get("brow_tension", np.nan))),
|
| 89 |
+
("Yaw", float(feature_values.get("head_yaw", np.nan))),
|
| 90 |
+
]
|
| 91 |
+
for row, (name, value) in enumerate(overlay_items):
|
| 92 |
+
cv2.putText(output, f"{name}: {value:.3f}", (20, 30 + row * 25), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 255), 2)
|
| 93 |
+
return output
|
| 94 |
+
|
| 95 |
+
def create_feature_timeline(self, features_df: pd.DataFrame, save_path: Optional[str] = None) -> plt.Figure:
|
| 96 |
+
"""Create a multi-panel timeline plot of facial features.
|
| 97 |
+
|
| 98 |
+
Args:
|
| 99 |
+
features_df: DataFrame containing frame-level facial features.
|
| 100 |
+
save_path: Optional path where the figure should be saved.
|
| 101 |
+
|
| 102 |
+
Returns:
|
| 103 |
+
Matplotlib figure containing four synchronized subplots.
|
| 104 |
+
"""
|
| 105 |
+
|
| 106 |
+
sns.set_theme(style="whitegrid")
|
| 107 |
+
time = features_df["timestamp"] if "timestamp" in features_df else features_df.index
|
| 108 |
+
fig, axes = plt.subplots(4, 1, figsize=(12, 10), sharex=True)
|
| 109 |
+
|
| 110 |
+
axes[0].plot(time, features_df.get("eye_left_openness"), label="Left eye")
|
| 111 |
+
axes[0].plot(time, features_df.get("eye_right_openness"), label="Right eye")
|
| 112 |
+
axes[0].set_ylabel("Eye openness")
|
| 113 |
+
axes[0].legend(loc="best")
|
| 114 |
+
|
| 115 |
+
axes[1].plot(time, features_df.get("mouth_openness"), label="Openness")
|
| 116 |
+
axes[1].plot(time, features_df.get("mouth_width"), label="Width")
|
| 117 |
+
axes[1].set_ylabel("Mouth movement")
|
| 118 |
+
axes[1].legend(loc="best")
|
| 119 |
+
|
| 120 |
+
axes[2].plot(time, features_df.get("brow_tension"), label="Brow tension", color="tab:orange")
|
| 121 |
+
axes[2].set_ylabel("Eyebrow tension")
|
| 122 |
+
axes[2].legend(loc="best")
|
| 123 |
+
|
| 124 |
+
for column in ["head_pitch", "head_yaw", "head_roll"]:
|
| 125 |
+
axes[3].plot(time, features_df.get(column), label=column.replace("head_", "").title())
|
| 126 |
+
axes[3].set_ylabel("Degrees")
|
| 127 |
+
axes[3].set_xlabel("Time (s)")
|
| 128 |
+
axes[3].legend(loc="best")
|
| 129 |
+
|
| 130 |
+
fig.suptitle("Facial Feature Timeline")
|
| 131 |
+
fig.tight_layout()
|
| 132 |
+
if save_path:
|
| 133 |
+
fig.savefig(save_path, dpi=300, bbox_inches="tight")
|
| 134 |
+
return fig
|
| 135 |
+
|
src/model/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model module for deception classification and calibration."""
|
| 2 |
+
|
| 3 |
+
from .classifier import DeceptionClassifier
|
| 4 |
+
from .calibration import SubjectCalibrator
|
| 5 |
+
|
| 6 |
+
__all__ = ['DeceptionClassifier', 'SubjectCalibrator']
|
src/model/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (340 Bytes). View file
|
|
|
src/model/calibration.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Subject-adaptive calibration utilities for deception detection.
|
| 2 |
+
|
| 3 |
+
This module implements personal calibration to account for inter-subject
|
| 4 |
+
behavioral variability, a major confound in affective/deception modeling.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
from dataclasses import asdict
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Dict, List, Optional
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
from sklearn.base import clone
|
| 17 |
+
|
| 18 |
+
from src.model.classifier import DeceptionClassifier
|
| 19 |
+
from src.utils.config import AppConfig, get_default_config
|
| 20 |
+
from src.utils.io import ensure_dir
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class SubjectCalibrator:
|
| 24 |
+
"""Subject-specific calibration for deception detection.
|
| 25 |
+
|
| 26 |
+
Adapts a global model to individual subjects by learning personal baselines
|
| 27 |
+
and applying feature-space plus score-space adjustments.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(self, base_classifier: DeceptionClassifier, config: Optional[AppConfig] = None):
|
| 31 |
+
"""Initialize calibrator.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
base_classifier: Trained global classifier.
|
| 35 |
+
config: Optional app-level configuration.
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
self.base_classifier = base_classifier
|
| 39 |
+
self.config: AppConfig = config or get_default_config()
|
| 40 |
+
self.subject_profiles: Dict[str, Dict[str, float]] = {}
|
| 41 |
+
|
| 42 |
+
def compute_baseline(self, subject_features: pd.DataFrame, subject_id: str) -> Dict[str, float]:
|
| 43 |
+
"""Compute a subject's baseline behavior from truthful samples.
|
| 44 |
+
|
| 45 |
+
Args:
|
| 46 |
+
subject_features: DataFrame containing subject feature rows.
|
| 47 |
+
subject_id: Subject identifier.
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
Dict[str, float]: Baseline profile with mean/std per feature.
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
if subject_features is None or subject_features.empty:
|
| 54 |
+
return {"subject_id": subject_id, "n_samples": 0.0}
|
| 55 |
+
|
| 56 |
+
X, feature_names = self.base_classifier.prepare_features(subject_features)
|
| 57 |
+
if X.size == 0 or len(feature_names) == 0:
|
| 58 |
+
return {"subject_id": subject_id, "n_samples": float(len(subject_features))}
|
| 59 |
+
|
| 60 |
+
means = np.nanmean(X, axis=0)
|
| 61 |
+
stds = np.nanstd(X, axis=0)
|
| 62 |
+
|
| 63 |
+
profile: Dict[str, float] = {
|
| 64 |
+
"subject_id": subject_id, # type: ignore[assignment]
|
| 65 |
+
"n_samples": float(X.shape[0]),
|
| 66 |
+
}
|
| 67 |
+
for i, name in enumerate(feature_names):
|
| 68 |
+
profile[f"{name}_mean"] = float(means[i])
|
| 69 |
+
profile[f"{name}_std"] = float(max(stds[i], 1e-8))
|
| 70 |
+
|
| 71 |
+
# Baseline deception tendency is used for threshold adaptation.
|
| 72 |
+
if self.base_classifier.is_fitted_:
|
| 73 |
+
scores = self.base_classifier.predict_deception_score(X)
|
| 74 |
+
profile["baseline_deception_mean"] = float(np.mean(scores))
|
| 75 |
+
profile["baseline_deception_std"] = float(max(np.std(scores), 1e-8))
|
| 76 |
+
else:
|
| 77 |
+
profile["baseline_deception_mean"] = 0.5
|
| 78 |
+
profile["baseline_deception_std"] = 0.1
|
| 79 |
+
|
| 80 |
+
return profile
|
| 81 |
+
|
| 82 |
+
def calibrate_features(
|
| 83 |
+
self,
|
| 84 |
+
features: np.ndarray,
|
| 85 |
+
subject_id: str,
|
| 86 |
+
feature_names: List[str],
|
| 87 |
+
) -> np.ndarray:
|
| 88 |
+
"""Apply subject-specific z-score normalization.
|
| 89 |
+
|
| 90 |
+
Args:
|
| 91 |
+
features: Raw feature array.
|
| 92 |
+
subject_id: Subject identifier.
|
| 93 |
+
feature_names: Ordered feature names.
|
| 94 |
+
|
| 95 |
+
Returns:
|
| 96 |
+
np.ndarray: Calibrated feature array.
|
| 97 |
+
"""
|
| 98 |
+
|
| 99 |
+
X = np.asarray(features, dtype=np.float64)
|
| 100 |
+
if X.size == 0:
|
| 101 |
+
return X
|
| 102 |
+
|
| 103 |
+
profile = self.subject_profiles.get(subject_id)
|
| 104 |
+
if profile is None:
|
| 105 |
+
return X
|
| 106 |
+
|
| 107 |
+
calibrated = X.copy()
|
| 108 |
+
for i, name in enumerate(feature_names):
|
| 109 |
+
mean_key = f"{name}_mean"
|
| 110 |
+
std_key = f"{name}_std"
|
| 111 |
+
if mean_key in profile and std_key in profile:
|
| 112 |
+
mu = float(profile[mean_key])
|
| 113 |
+
sigma = float(profile[std_key])
|
| 114 |
+
calibrated[:, i] = (calibrated[:, i] - mu) / (sigma + 1e-8)
|
| 115 |
+
|
| 116 |
+
return calibrated
|
| 117 |
+
|
| 118 |
+
def calibrate_prediction(self, raw_proba: float, subject_id: str) -> float:
|
| 119 |
+
"""Apply subject-specific logistic threshold adjustment.
|
| 120 |
+
|
| 121 |
+
Args:
|
| 122 |
+
raw_proba: Raw deception probability from global model.
|
| 123 |
+
subject_id: Subject identifier.
|
| 124 |
+
|
| 125 |
+
Returns:
|
| 126 |
+
float: Calibrated deception probability in [0, 1].
|
| 127 |
+
"""
|
| 128 |
+
|
| 129 |
+
p = float(np.clip(raw_proba, 1e-6, 1.0 - 1e-6))
|
| 130 |
+
profile = self.subject_profiles.get(subject_id)
|
| 131 |
+
if profile is None:
|
| 132 |
+
return p
|
| 133 |
+
|
| 134 |
+
baseline_mean = float(profile.get("baseline_deception_mean", 0.5))
|
| 135 |
+
baseline_std = float(profile.get("baseline_deception_std", 0.1))
|
| 136 |
+
n_samples = float(profile.get("n_samples", 0.0))
|
| 137 |
+
|
| 138 |
+
# Research rationale: move the decision surface in logit space, where
|
| 139 |
+
# additive shifts correspond to multiplicative odds adjustment.
|
| 140 |
+
confidence = float(np.clip(np.log1p(n_samples) / np.log1p(200.0), 0.0, 1.0))
|
| 141 |
+
shift_strength = confidence * np.clip((baseline_mean - 0.5) / (baseline_std + 1e-8), -2.0, 2.0)
|
| 142 |
+
shift_strength *= 0.15
|
| 143 |
+
|
| 144 |
+
logit = np.log(p / (1.0 - p))
|
| 145 |
+
adjusted = logit - shift_strength
|
| 146 |
+
calibrated = 1.0 / (1.0 + np.exp(-adjusted))
|
| 147 |
+
return float(np.clip(calibrated, 0.0, 1.0))
|
| 148 |
+
|
| 149 |
+
def add_subject_profile(self, subject_id: str, truthful_features: pd.DataFrame) -> None:
|
| 150 |
+
"""Register a subject profile using truthful behavior samples.
|
| 151 |
+
|
| 152 |
+
Args:
|
| 153 |
+
subject_id: Subject identifier.
|
| 154 |
+
truthful_features: Truthful-only feature dataframe for the subject.
|
| 155 |
+
"""
|
| 156 |
+
|
| 157 |
+
profile = self.compute_baseline(truthful_features, subject_id)
|
| 158 |
+
self.subject_profiles[subject_id] = profile
|
| 159 |
+
|
| 160 |
+
def save_profiles(self, path: str) -> None:
|
| 161 |
+
"""Save all subject profiles to a JSON file.
|
| 162 |
+
|
| 163 |
+
Args:
|
| 164 |
+
path: Output JSON path.
|
| 165 |
+
"""
|
| 166 |
+
|
| 167 |
+
out_path = Path(path)
|
| 168 |
+
ensure_dir(out_path.parent)
|
| 169 |
+
with out_path.open("w", encoding="utf-8") as f:
|
| 170 |
+
json.dump(self.subject_profiles, f, indent=2, sort_keys=True)
|
| 171 |
+
|
| 172 |
+
def load_profiles(self, path: str) -> None:
|
| 173 |
+
"""Load subject profiles from JSON file.
|
| 174 |
+
|
| 175 |
+
Args:
|
| 176 |
+
path: Input JSON path.
|
| 177 |
+
"""
|
| 178 |
+
|
| 179 |
+
in_path = Path(path)
|
| 180 |
+
if not in_path.exists():
|
| 181 |
+
raise FileNotFoundError(f"Subject profile file not found: {in_path}")
|
| 182 |
+
with in_path.open("r", encoding="utf-8") as f:
|
| 183 |
+
data = json.load(f)
|
| 184 |
+
if not isinstance(data, dict):
|
| 185 |
+
raise ValueError("Invalid subject profile format. Expected JSON object.")
|
| 186 |
+
self.subject_profiles = data
|
| 187 |
+
|
| 188 |
+
def get_subject_ids(self) -> List[str]:
|
| 189 |
+
"""Return registered subject identifiers."""
|
| 190 |
+
|
| 191 |
+
return sorted(self.subject_profiles.keys())
|
| 192 |
+
|
| 193 |
+
def calibrate_subject_model(
|
| 194 |
+
self,
|
| 195 |
+
subject_id: str,
|
| 196 |
+
subject_features: np.ndarray,
|
| 197 |
+
subject_labels: np.ndarray,
|
| 198 |
+
feature_names: List[str],
|
| 199 |
+
) -> DeceptionClassifier:
|
| 200 |
+
"""Fine-tune a personal model for a specific subject.
|
| 201 |
+
|
| 202 |
+
Hard examples (where base model disagrees) are upweighted to specialize
|
| 203 |
+
the model for subject-specific decision boundaries.
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
subject_id: Subject identifier.
|
| 207 |
+
subject_features: Subject feature matrix.
|
| 208 |
+
subject_labels: Subject labels.
|
| 209 |
+
feature_names: Feature names used by the model.
|
| 210 |
+
|
| 211 |
+
Returns:
|
| 212 |
+
DeceptionClassifier: Subject-adapted classifier.
|
| 213 |
+
"""
|
| 214 |
+
|
| 215 |
+
X = np.asarray(subject_features, dtype=np.float64)
|
| 216 |
+
y = np.asarray(subject_labels).astype(int)
|
| 217 |
+
if X.ndim != 2 or y.ndim != 1 or len(X) != len(y):
|
| 218 |
+
raise ValueError("Invalid subject data shapes for calibration.")
|
| 219 |
+
if len(y) == 0:
|
| 220 |
+
raise ValueError("Cannot calibrate subject model with zero samples.")
|
| 221 |
+
|
| 222 |
+
personal = DeceptionClassifier(config=self.base_classifier.config)
|
| 223 |
+
|
| 224 |
+
# Transfer model state when available to keep inductive bias from global data.
|
| 225 |
+
if self.base_classifier.is_fitted_:
|
| 226 |
+
try:
|
| 227 |
+
personal.scaler = clone(self.base_classifier.scaler)
|
| 228 |
+
personal.scaler.mean_ = np.copy(self.base_classifier.scaler.mean_)
|
| 229 |
+
personal.scaler.scale_ = np.copy(self.base_classifier.scaler.scale_)
|
| 230 |
+
personal.scaler.var_ = np.copy(self.base_classifier.scaler.var_)
|
| 231 |
+
personal.scaler.n_features_in_ = self.base_classifier.scaler.n_features_in_
|
| 232 |
+
if hasattr(self.base_classifier.scaler, "n_samples_seen_"):
|
| 233 |
+
personal.scaler.n_samples_seen_ = self.base_classifier.scaler.n_samples_seen_
|
| 234 |
+
except Exception:
|
| 235 |
+
# Fallback to refitting scaler below.
|
| 236 |
+
pass
|
| 237 |
+
|
| 238 |
+
# Subject-space calibration (if profile exists) before fine-tuning.
|
| 239 |
+
X_cal = self.calibrate_features(X, subject_id, feature_names)
|
| 240 |
+
|
| 241 |
+
# Derive sample weights emphasizing hard examples.
|
| 242 |
+
weights = np.ones(len(y), dtype=np.float64)
|
| 243 |
+
if self.base_classifier.is_fitted_:
|
| 244 |
+
base_pred = self.base_classifier.predict(X)
|
| 245 |
+
disagreement = (base_pred != y).astype(np.float64)
|
| 246 |
+
weights += 2.0 * disagreement
|
| 247 |
+
|
| 248 |
+
X_scaled = personal.scaler.fit_transform(X_cal)
|
| 249 |
+
personal.model.fit(X_scaled, y, sample_weight=weights)
|
| 250 |
+
|
| 251 |
+
# Calibrator requires both classes and enough support.
|
| 252 |
+
unique = np.unique(y)
|
| 253 |
+
if len(unique) > 1 and len(y) >= 10:
|
| 254 |
+
personal.calibrated_model.fit(X_scaled, y, sample_weight=weights)
|
| 255 |
+
|
| 256 |
+
personal.feature_names_ = list(feature_names)
|
| 257 |
+
personal.is_fitted_ = True
|
| 258 |
+
personal.training_metadata_ = {
|
| 259 |
+
"subject_id": subject_id,
|
| 260 |
+
"n_samples": float(len(y)),
|
| 261 |
+
"class_distribution": {int(c): int((y == c).sum()) for c in unique},
|
| 262 |
+
"config": asdict(personal.config),
|
| 263 |
+
}
|
| 264 |
+
return personal
|
| 265 |
+
|
src/model/classifier.py
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Core Gradient Boosting classifier for deception detection.
|
| 2 |
+
|
| 3 |
+
This module implements a robust, production-oriented classifier wrapper that
|
| 4 |
+
standardizes features, handles missing values, performs probability calibration,
|
| 5 |
+
and persists all metadata required for reproducible inference.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from dataclasses import asdict
|
| 11 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import pandas as pd
|
| 15 |
+
from sklearn.calibration import CalibratedClassifierCV
|
| 16 |
+
from sklearn.ensemble import GradientBoostingClassifier
|
| 17 |
+
from sklearn.metrics import accuracy_score, log_loss, roc_auc_score
|
| 18 |
+
from sklearn.model_selection import cross_val_score
|
| 19 |
+
from sklearn.pipeline import make_pipeline
|
| 20 |
+
from sklearn.preprocessing import StandardScaler
|
| 21 |
+
|
| 22 |
+
from src.utils.config import ModelConfig
|
| 23 |
+
from src.utils.io import load_model as io_load_model
|
| 24 |
+
from src.utils.io import save_model as io_save_model
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class DeceptionClassifier:
|
| 28 |
+
"""Gradient Boosting-based deception classifier with probability calibration.
|
| 29 |
+
|
| 30 |
+
Uses `sklearn` GradientBoostingClassifier as the base estimator and wraps it
|
| 31 |
+
with calibration for better probabilistic interpretation.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
def __init__(self, config: Optional[ModelConfig] = None):
|
| 35 |
+
"""Initialize classifier with model config parameters.
|
| 36 |
+
|
| 37 |
+
Args:
|
| 38 |
+
config: Optional model configuration. If not provided, defaults are used.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
self.config: ModelConfig = config or ModelConfig()
|
| 42 |
+
self.model = GradientBoostingClassifier(
|
| 43 |
+
n_estimators=self.config.n_estimators,
|
| 44 |
+
learning_rate=self.config.learning_rate,
|
| 45 |
+
max_depth=self.config.max_depth,
|
| 46 |
+
min_samples_split=self.config.min_samples_split,
|
| 47 |
+
min_samples_leaf=self.config.min_samples_leaf,
|
| 48 |
+
subsample=self.config.subsample,
|
| 49 |
+
random_state=self.config.random_state,
|
| 50 |
+
)
|
| 51 |
+
self.scaler = StandardScaler()
|
| 52 |
+
|
| 53 |
+
calibration_method = getattr(self.config, "calibration_method", "sigmoid")
|
| 54 |
+
self.calibrated_model = CalibratedClassifierCV(
|
| 55 |
+
estimator=GradientBoostingClassifier(
|
| 56 |
+
n_estimators=self.config.n_estimators,
|
| 57 |
+
learning_rate=self.config.learning_rate,
|
| 58 |
+
max_depth=self.config.max_depth,
|
| 59 |
+
min_samples_split=self.config.min_samples_split,
|
| 60 |
+
min_samples_leaf=self.config.min_samples_leaf,
|
| 61 |
+
subsample=self.config.subsample,
|
| 62 |
+
random_state=self.config.random_state,
|
| 63 |
+
),
|
| 64 |
+
method=calibration_method,
|
| 65 |
+
cv=min(max(2, getattr(self.config, "cross_validation_folds", 5)), 5),
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
self.feature_names_: Optional[List[str]] = None
|
| 69 |
+
self.is_fitted_: bool = False
|
| 70 |
+
self.training_metadata_: Dict[str, Any] = {}
|
| 71 |
+
|
| 72 |
+
def prepare_features(
|
| 73 |
+
self,
|
| 74 |
+
features_df: pd.DataFrame,
|
| 75 |
+
feature_columns: Optional[List[str]] = None,
|
| 76 |
+
) -> Tuple[np.ndarray, List[str]]:
|
| 77 |
+
"""Prepare feature matrix from a feature dataframe.
|
| 78 |
+
|
| 79 |
+
Args:
|
| 80 |
+
features_df: DataFrame with extracted features.
|
| 81 |
+
feature_columns: Optional explicit feature column list.
|
| 82 |
+
|
| 83 |
+
Returns:
|
| 84 |
+
Tuple[np.ndarray, List[str]]: Feature matrix and ordered feature names.
|
| 85 |
+
"""
|
| 86 |
+
|
| 87 |
+
if features_df is None or features_df.empty:
|
| 88 |
+
return np.empty((0, 0), dtype=np.float64), []
|
| 89 |
+
|
| 90 |
+
excluded_columns = {
|
| 91 |
+
"frame_idx",
|
| 92 |
+
"timestamp",
|
| 93 |
+
"segment_idx",
|
| 94 |
+
"start_time",
|
| 95 |
+
"end_time",
|
| 96 |
+
"label",
|
| 97 |
+
"subject_id",
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
if feature_columns is None:
|
| 101 |
+
if self.is_fitted_ and self.feature_names_ is not None:
|
| 102 |
+
feature_columns = self.feature_names_
|
| 103 |
+
else:
|
| 104 |
+
candidate_cols = [c for c in features_df.columns if c not in excluded_columns]
|
| 105 |
+
# Research rationale: constraining to numeric-only columns prevents accidental
|
| 106 |
+
# leakage of identifiers and ensures stable scaling behavior.
|
| 107 |
+
feature_columns = [
|
| 108 |
+
c for c in candidate_cols if pd.api.types.is_numeric_dtype(features_df[c])
|
| 109 |
+
]
|
| 110 |
+
|
| 111 |
+
if not feature_columns:
|
| 112 |
+
return np.empty((len(features_df), 0), dtype=np.float64), []
|
| 113 |
+
|
| 114 |
+
missing_cols = set(feature_columns) - set(features_df.columns)
|
| 115 |
+
if missing_cols:
|
| 116 |
+
for col in missing_cols:
|
| 117 |
+
features_df[col] = 0.0
|
| 118 |
+
|
| 119 |
+
selected = features_df[feature_columns].copy()
|
| 120 |
+
for col in selected.columns:
|
| 121 |
+
selected[col] = pd.to_numeric(selected[col], errors="coerce")
|
| 122 |
+
|
| 123 |
+
selected = selected.replace([np.inf, -np.inf], np.nan)
|
| 124 |
+
medians = selected.median(numeric_only=True)
|
| 125 |
+
selected = selected.fillna(medians)
|
| 126 |
+
selected = selected.fillna(0.0)
|
| 127 |
+
|
| 128 |
+
# Clip to robust quantiles to reduce effect of extreme outliers.
|
| 129 |
+
lower = selected.quantile(0.01)
|
| 130 |
+
upper = selected.quantile(0.99)
|
| 131 |
+
selected = selected.clip(lower=lower, upper=upper, axis=1)
|
| 132 |
+
|
| 133 |
+
return selected.to_numpy(dtype=np.float64), list(selected.columns)
|
| 134 |
+
|
| 135 |
+
def train(self, X: np.ndarray, y: np.ndarray, feature_names: List[str]) -> Dict[str, float]:
|
| 136 |
+
"""Train the gradient boosting classifier.
|
| 137 |
+
|
| 138 |
+
Args:
|
| 139 |
+
X: Feature matrix of shape (n_samples, n_features).
|
| 140 |
+
y: Binary labels (0=truthful, 1=deceptive).
|
| 141 |
+
feature_names: Ordered feature names.
|
| 142 |
+
|
| 143 |
+
Returns:
|
| 144 |
+
Dict[str, float]: Training metrics and dataset summary.
|
| 145 |
+
"""
|
| 146 |
+
|
| 147 |
+
X = np.asarray(X, dtype=np.float64)
|
| 148 |
+
y = np.asarray(y).astype(int)
|
| 149 |
+
|
| 150 |
+
if X.ndim != 2:
|
| 151 |
+
raise ValueError("X must be a 2D array.")
|
| 152 |
+
if y.ndim != 1:
|
| 153 |
+
raise ValueError("y must be a 1D array.")
|
| 154 |
+
if len(X) != len(y):
|
| 155 |
+
raise ValueError("X and y must have the same number of samples.")
|
| 156 |
+
if len(y) == 0:
|
| 157 |
+
raise ValueError("Cannot train with zero samples.")
|
| 158 |
+
|
| 159 |
+
self.feature_names_ = list(feature_names)
|
| 160 |
+
|
| 161 |
+
X_scaled = self.scaler.fit_transform(X)
|
| 162 |
+
|
| 163 |
+
unique_classes, counts = np.unique(y, return_counts=True)
|
| 164 |
+
class_distribution = {int(k): int(v) for k, v in zip(unique_classes, counts)}
|
| 165 |
+
|
| 166 |
+
# Research rationale: inverse-frequency weighting mitigates skewed priors and
|
| 167 |
+
# helps gradient boosting focus on under-represented deceptive/truthful samples.
|
| 168 |
+
class_weights = {
|
| 169 |
+
cls: len(y) / (len(unique_classes) * count)
|
| 170 |
+
for cls, count in zip(unique_classes, counts)
|
| 171 |
+
if count > 0
|
| 172 |
+
}
|
| 173 |
+
sample_weight = np.array([class_weights.get(label, 1.0) for label in y], dtype=np.float64)
|
| 174 |
+
|
| 175 |
+
self.model.fit(X_scaled, y, sample_weight=sample_weight)
|
| 176 |
+
|
| 177 |
+
can_calibrate = len(unique_classes) > 1 and len(y) >= max(10, len(unique_classes) * 3)
|
| 178 |
+
if can_calibrate:
|
| 179 |
+
self.calibrated_model.fit(X_scaled, y, sample_weight=sample_weight)
|
| 180 |
+
|
| 181 |
+
# Research rationale: setting is_fitted_ prior to metric computation allows
|
| 182 |
+
# reuse of internal predict/predict_proba methods for training performance tracking.
|
| 183 |
+
self.is_fitted_ = True
|
| 184 |
+
|
| 185 |
+
train_preds = self.predict(X, check_fitted=False)
|
| 186 |
+
train_probs = self.predict_proba(X, check_fitted=False)
|
| 187 |
+
|
| 188 |
+
metrics: Dict[str, float] = {
|
| 189 |
+
"train_accuracy": float(accuracy_score(y, train_preds)),
|
| 190 |
+
"n_samples": float(len(y)),
|
| 191 |
+
"n_features": float(X.shape[1]),
|
| 192 |
+
"class_distribution": class_distribution, # type: ignore[assignment]
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
try:
|
| 196 |
+
metrics["train_log_loss"] = float(log_loss(y, train_probs, labels=[0, 1]))
|
| 197 |
+
except ValueError:
|
| 198 |
+
metrics["train_log_loss"] = float("nan")
|
| 199 |
+
|
| 200 |
+
if len(unique_classes) > 1:
|
| 201 |
+
try:
|
| 202 |
+
metrics["train_auc"] = float(roc_auc_score(y, train_probs[:, 1]))
|
| 203 |
+
except ValueError:
|
| 204 |
+
metrics["train_auc"] = float("nan")
|
| 205 |
+
else:
|
| 206 |
+
metrics["train_auc"] = float("nan")
|
| 207 |
+
|
| 208 |
+
cv_folds = min(getattr(self.config, "cross_validation_folds", 5), len(y))
|
| 209 |
+
if cv_folds >= 2 and len(unique_classes) > 1:
|
| 210 |
+
pipeline = make_pipeline(
|
| 211 |
+
StandardScaler(),
|
| 212 |
+
GradientBoostingClassifier(
|
| 213 |
+
n_estimators=self.config.n_estimators,
|
| 214 |
+
learning_rate=self.config.learning_rate,
|
| 215 |
+
max_depth=self.config.max_depth,
|
| 216 |
+
min_samples_split=self.config.min_samples_split,
|
| 217 |
+
min_samples_leaf=self.config.min_samples_leaf,
|
| 218 |
+
subsample=self.config.subsample,
|
| 219 |
+
random_state=self.config.random_state,
|
| 220 |
+
),
|
| 221 |
+
)
|
| 222 |
+
cv_scores = cross_val_score(pipeline, X, y, cv=cv_folds, scoring="accuracy")
|
| 223 |
+
metrics["cv_accuracy_mean"] = float(np.mean(cv_scores))
|
| 224 |
+
metrics["cv_accuracy_std"] = float(np.std(cv_scores))
|
| 225 |
+
else:
|
| 226 |
+
metrics["cv_accuracy_mean"] = float("nan")
|
| 227 |
+
metrics["cv_accuracy_std"] = float("nan")
|
| 228 |
+
|
| 229 |
+
self.training_metadata_ = metrics
|
| 230 |
+
return metrics
|
| 231 |
+
|
| 232 |
+
def predict(self, X: np.ndarray, check_fitted: bool = True) -> np.ndarray:
|
| 233 |
+
"""Predict binary deception labels for input features.
|
| 234 |
+
|
| 235 |
+
Args:
|
| 236 |
+
X: Input feature matrix.
|
| 237 |
+
check_fitted: Whether to verify that the model is fitted.
|
| 238 |
+
"""
|
| 239 |
+
|
| 240 |
+
if check_fitted and not self.is_fitted_:
|
| 241 |
+
raise ValueError("Model is not trained. Call train() first.")
|
| 242 |
+
|
| 243 |
+
X_scaled = self.scaler.transform(np.asarray(X, dtype=np.float64))
|
| 244 |
+
if hasattr(self.calibrated_model, "calibrated_classifiers_"):
|
| 245 |
+
return self.calibrated_model.predict(X_scaled)
|
| 246 |
+
return self.model.predict(X_scaled)
|
| 247 |
+
|
| 248 |
+
def predict_proba(self, X: np.ndarray, check_fitted: bool = True) -> np.ndarray:
|
| 249 |
+
"""Predict deception probabilities for input features.
|
| 250 |
+
|
| 251 |
+
Args:
|
| 252 |
+
X: Input feature matrix.
|
| 253 |
+
check_fitted: Whether to verify that the model is fitted.
|
| 254 |
+
|
| 255 |
+
Returns:
|
| 256 |
+
np.ndarray: Probability matrix [truthful_prob, deceptive_prob].
|
| 257 |
+
"""
|
| 258 |
+
|
| 259 |
+
if check_fitted and not self.is_fitted_:
|
| 260 |
+
raise ValueError("Model is not trained. Call train() first.")
|
| 261 |
+
|
| 262 |
+
X_scaled = self.scaler.transform(np.asarray(X, dtype=np.float64))
|
| 263 |
+
if hasattr(self.calibrated_model, "calibrated_classifiers_"):
|
| 264 |
+
return self.calibrated_model.predict_proba(X_scaled)
|
| 265 |
+
return self.model.predict_proba(X_scaled)
|
| 266 |
+
|
| 267 |
+
def predict_deception_score(self, X: np.ndarray, check_fitted: bool = True) -> np.ndarray:
|
| 268 |
+
"""Predict scalar deception score in [0, 1] for each sample."""
|
| 269 |
+
|
| 270 |
+
proba = self.predict_proba(X, check_fitted=check_fitted)
|
| 271 |
+
return proba[:, 1]
|
| 272 |
+
|
| 273 |
+
def get_feature_importance(self) -> pd.DataFrame:
|
| 274 |
+
"""Get sorted feature importance values from the trained base model."""
|
| 275 |
+
|
| 276 |
+
if not self.is_fitted_:
|
| 277 |
+
raise ValueError("Model is not trained. Call train() first.")
|
| 278 |
+
|
| 279 |
+
if self.feature_names_ is None:
|
| 280 |
+
raise ValueError("Feature names are unavailable.")
|
| 281 |
+
|
| 282 |
+
importances = getattr(self.model, "feature_importances_", None)
|
| 283 |
+
if importances is None:
|
| 284 |
+
raise ValueError("Model does not expose feature importances.")
|
| 285 |
+
|
| 286 |
+
importance_df = pd.DataFrame(
|
| 287 |
+
{"feature": self.feature_names_, "importance": np.asarray(importances, dtype=np.float64)}
|
| 288 |
+
)
|
| 289 |
+
return importance_df.sort_values("importance", ascending=False).reset_index(drop=True)
|
| 290 |
+
|
| 291 |
+
def save(self, path: str) -> None:
|
| 292 |
+
"""Save model, scaler, calibration wrapper, and metadata using joblib."""
|
| 293 |
+
|
| 294 |
+
payload = {
|
| 295 |
+
"model": self.model,
|
| 296 |
+
"calibrated_model": self.calibrated_model,
|
| 297 |
+
"scaler": self.scaler,
|
| 298 |
+
"feature_names": self.feature_names_,
|
| 299 |
+
"is_fitted": self.is_fitted_,
|
| 300 |
+
"config": asdict(self.config),
|
| 301 |
+
"training_metadata": self.training_metadata_,
|
| 302 |
+
}
|
| 303 |
+
io_save_model(payload, path)
|
| 304 |
+
|
| 305 |
+
def load(self, path: str) -> None:
|
| 306 |
+
"""Load model, scaler, calibration wrapper, and metadata from file."""
|
| 307 |
+
|
| 308 |
+
payload = io_load_model(path)
|
| 309 |
+
self.model = payload["model"]
|
| 310 |
+
self.calibrated_model = payload.get("calibrated_model", self.calibrated_model)
|
| 311 |
+
self.scaler = payload["scaler"]
|
| 312 |
+
self.feature_names_ = payload.get("feature_names")
|
| 313 |
+
self.is_fitted_ = bool(payload.get("is_fitted", True))
|
| 314 |
+
self.training_metadata_ = dict(payload.get("training_metadata", {}))
|
| 315 |
+
|
| 316 |
+
config_dict = payload.get("config", {})
|
| 317 |
+
if isinstance(config_dict, dict):
|
| 318 |
+
self.config = ModelConfig(**config_dict)
|
| 319 |
+
|
| 320 |
+
def get_model_info(self) -> Dict[str, Any]:
|
| 321 |
+
"""Return model metadata and configuration details."""
|
| 322 |
+
|
| 323 |
+
return {
|
| 324 |
+
"is_fitted": self.is_fitted_,
|
| 325 |
+
"n_features": 0 if self.feature_names_ is None else len(self.feature_names_),
|
| 326 |
+
"feature_names": self.feature_names_ or [],
|
| 327 |
+
"config": asdict(self.config),
|
| 328 |
+
"training_metadata": self.training_metadata_,
|
| 329 |
+
}
|
| 330 |
+
|
src/pipeline.py
ADDED
|
@@ -0,0 +1,524 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Main pipeline for multimodal deception analysis."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
import tempfile
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Dict, List, Optional, Union
|
| 9 |
+
|
| 10 |
+
import pandas as pd
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
|
| 13 |
+
from src.audio.extractor import AudioFeatureExtractor
|
| 14 |
+
from src.audio.fusion import MultimodalFusion
|
| 15 |
+
from src.explainability.explainer import DeceptionExplainer
|
| 16 |
+
from src.explainability.visualizer import ExplainabilityVisualizer
|
| 17 |
+
from src.facial.extractor import FacialFeatureExtractor
|
| 18 |
+
from src.facial.visualizer import FacialVisualizer
|
| 19 |
+
from src.model.calibration import SubjectCalibrator
|
| 20 |
+
from src.model.classifier import DeceptionClassifier
|
| 21 |
+
from src.temporal.analyzer import TemporalAnalyzer
|
| 22 |
+
from src.temporal.visualizer import TemporalVisualizer
|
| 23 |
+
from src.utils.config import AppConfig, get_default_config
|
| 24 |
+
from src.utils.io import extract_audio_from_video, save_results
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class DeceptionPipeline:
|
| 28 |
+
"""Main pipeline for multimodal deception analysis."""
|
| 29 |
+
|
| 30 |
+
def __init__(self, config: Optional[AppConfig] = None):
|
| 31 |
+
"""Initialize pipeline with all sub-modules.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
config: Optional application configuration. If None, default config is used.
|
| 35 |
+
"""
|
| 36 |
+
self.config = config or get_default_config()
|
| 37 |
+
self.logger = logging.getLogger(__name__)
|
| 38 |
+
|
| 39 |
+
# Initialize all components with config
|
| 40 |
+
self.facial_extractor = FacialFeatureExtractor(self.config.features)
|
| 41 |
+
self.audio_extractor = AudioFeatureExtractor(self.config.audio)
|
| 42 |
+
self.fusion = MultimodalFusion(self.config)
|
| 43 |
+
self.classifier = DeceptionClassifier(self.config.model)
|
| 44 |
+
self.calibrator = SubjectCalibrator(self.classifier, self.config)
|
| 45 |
+
self.temporal_analyzer = TemporalAnalyzer(self.config.temporal)
|
| 46 |
+
self.explainer = DeceptionExplainer(self.classifier)
|
| 47 |
+
|
| 48 |
+
# Visualizers (no config needed)
|
| 49 |
+
self.facial_visualizer = FacialVisualizer()
|
| 50 |
+
self.temporal_visualizer = TemporalVisualizer()
|
| 51 |
+
self.explainability_visualizer = ExplainabilityVisualizer()
|
| 52 |
+
|
| 53 |
+
self.logger.info("DeceptionPipeline initialized with default config.")
|
| 54 |
+
|
| 55 |
+
def analyze_video(
|
| 56 |
+
self,
|
| 57 |
+
video_path: str,
|
| 58 |
+
subject_id: Optional[str] = None,
|
| 59 |
+
generate_visualizations: bool = True,
|
| 60 |
+
output_dir: Optional[str] = None,
|
| 61 |
+
) -> dict:
|
| 62 |
+
"""Full analysis pipeline for a single video.
|
| 63 |
+
|
| 64 |
+
Steps:
|
| 65 |
+
1. Extract facial features from video
|
| 66 |
+
2. Extract audio from video (using io utilities), then extract audio features
|
| 67 |
+
3. Fuse facial and audio features (align + cross-modal)
|
| 68 |
+
4. Run classifier prediction
|
| 69 |
+
5. Apply subject calibration if subject_id provided
|
| 70 |
+
6. Run temporal analysis on probability trajectory
|
| 71 |
+
7. Generate explanations using SHAP
|
| 72 |
+
8. Generate visualizations if requested
|
| 73 |
+
9. Compile results into a comprehensive report dict
|
| 74 |
+
|
| 75 |
+
Args:
|
| 76 |
+
video_path: Path to the video file.
|
| 77 |
+
subject_id: Optional subject identifier for calibration.
|
| 78 |
+
generate_visualizations: Whether to generate visualization plots.
|
| 79 |
+
output_dir: Directory to save visualizations and intermediate results.
|
| 80 |
+
If None, a temporary directory is used.
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
dict with keys:
|
| 84 |
+
- 'deception_probability': float
|
| 85 |
+
- 'prediction': str ('truthful' or 'deceptive')
|
| 86 |
+
- 'confidence': float
|
| 87 |
+
- 'facial_features': DataFrame
|
| 88 |
+
- 'audio_features': DataFrame
|
| 89 |
+
- 'fused_features': DataFrame
|
| 90 |
+
- 'temporal_analysis': dict
|
| 91 |
+
- 'explanation': dict
|
| 92 |
+
- 'calibrated': bool
|
| 93 |
+
- 'visualizations': dict of file paths (if generated)
|
| 94 |
+
- 'metadata': dict
|
| 95 |
+
"""
|
| 96 |
+
self.logger.info(f"Starting analysis of video: {video_path}")
|
| 97 |
+
result: Dict[str, Union[float, str, pd.DataFrame, dict, bool, list]] = {}
|
| 98 |
+
visualizations = {}
|
| 99 |
+
metadata = {"video_path": video_path, "subject_id": subject_id}
|
| 100 |
+
|
| 101 |
+
# Determine output directory
|
| 102 |
+
if output_dir is None:
|
| 103 |
+
output_dir = tempfile.mkdtemp(prefix="deception_analysis_")
|
| 104 |
+
output_path = Path(output_dir)
|
| 105 |
+
output_path.mkdir(parents=True, exist_ok=True)
|
| 106 |
+
|
| 107 |
+
try:
|
| 108 |
+
# 1. Extract facial features
|
| 109 |
+
self.logger.info("Extracting facial features...")
|
| 110 |
+
facial_df = self.facial_extractor.process_video(video_path)
|
| 111 |
+
result["facial_features"] = facial_df
|
| 112 |
+
metadata["facial_frames"] = len(facial_df)
|
| 113 |
+
self.logger.info(f"Extracted {len(facial_df)} facial frames.")
|
| 114 |
+
|
| 115 |
+
# 2. Extract audio features
|
| 116 |
+
self.logger.info("Extracting audio features...")
|
| 117 |
+
# Extract audio from video
|
| 118 |
+
audio_temp = output_path / "audio.wav"
|
| 119 |
+
try:
|
| 120 |
+
audio_path = extract_audio_from_video(video_path, audio_temp)
|
| 121 |
+
except Exception as e:
|
| 122 |
+
self.logger.warning(f"Audio extraction failed: {e}. Using empty audio features.")
|
| 123 |
+
audio_df = pd.DataFrame()
|
| 124 |
+
else:
|
| 125 |
+
# Extract audio features
|
| 126 |
+
audio_df = self.audio_extractor.process_audio_segments(
|
| 127 |
+
str(audio_path), segment_duration=2.0
|
| 128 |
+
)
|
| 129 |
+
result["audio_features"] = audio_df
|
| 130 |
+
metadata["audio_segments"] = len(audio_df)
|
| 131 |
+
|
| 132 |
+
# 3. Fuse features
|
| 133 |
+
self.logger.info("Fusing facial and audio features...")
|
| 134 |
+
aligned_df = self.fusion.align_features(
|
| 135 |
+
facial_df, audio_df, video_fps=self.config.features.fps
|
| 136 |
+
)
|
| 137 |
+
fused_df = self.fusion.extract_fusion_features(aligned_df)
|
| 138 |
+
result["fused_features"] = fused_df
|
| 139 |
+
|
| 140 |
+
# 4. Run classifier prediction
|
| 141 |
+
self.logger.info("Running classifier prediction...")
|
| 142 |
+
if not self.classifier.is_fitted_:
|
| 143 |
+
self.logger.warning("Classifier not trained. Using default probability 0.5.")
|
| 144 |
+
proba = 0.5
|
| 145 |
+
prediction = "truthful"
|
| 146 |
+
confidence = 0.5
|
| 147 |
+
per_frame_probs = pd.Series([0.5] * len(fused_df))
|
| 148 |
+
else:
|
| 149 |
+
# Prepare features for classification
|
| 150 |
+
X, feature_names = self.classifier.prepare_features(fused_df)
|
| 151 |
+
if X.size == 0:
|
| 152 |
+
self.logger.warning("No features for classification.")
|
| 153 |
+
proba = 0.5
|
| 154 |
+
prediction = "truthful"
|
| 155 |
+
confidence = 0.5
|
| 156 |
+
per_frame_probs = pd.Series([0.5] * len(fused_df))
|
| 157 |
+
else:
|
| 158 |
+
# Overall prediction (average across frames)
|
| 159 |
+
proba = float(self.classifier.predict_deception_score(X).mean())
|
| 160 |
+
prediction = "deceptive" if proba >= self.config.model.decision_threshold else "truthful"
|
| 161 |
+
confidence = abs(proba - 0.5) * 2 # map 0.5->0, 0/1->1
|
| 162 |
+
# Per-frame probabilities for temporal analysis
|
| 163 |
+
per_frame_probs = pd.Series(self.classifier.predict_deception_score(X))
|
| 164 |
+
|
| 165 |
+
result["deception_probability"] = proba
|
| 166 |
+
result["prediction"] = prediction
|
| 167 |
+
result["confidence"] = confidence
|
| 168 |
+
metadata["decision_threshold"] = self.config.model.decision_threshold
|
| 169 |
+
|
| 170 |
+
# 5. Subject calibration
|
| 171 |
+
calibrated = False
|
| 172 |
+
if subject_id is not None and self.classifier.is_fitted_:
|
| 173 |
+
self.logger.info(f"Applying subject calibration for {subject_id}...")
|
| 174 |
+
try:
|
| 175 |
+
# Update baseline with current features
|
| 176 |
+
self.calibrator.add_subject_profile(subject_id, fused_df)
|
| 177 |
+
# Calibrate predictions
|
| 178 |
+
import numpy as np
|
| 179 |
+
calibrated_preds = np.array([
|
| 180 |
+
self.calibrator.calibrate_prediction(p, subject_id)
|
| 181 |
+
for p in per_frame_probs.values
|
| 182 |
+
])
|
| 183 |
+
if calibrated_preds is not None:
|
| 184 |
+
calibrated_proba = float(calibrated_preds.mean())
|
| 185 |
+
calibrated_pred = (
|
| 186 |
+
"deceptive"
|
| 187 |
+
if calibrated_proba >= self.config.model.decision_threshold
|
| 188 |
+
else "truthful"
|
| 189 |
+
)
|
| 190 |
+
result["deception_probability"] = calibrated_proba
|
| 191 |
+
result["prediction"] = calibrated_pred
|
| 192 |
+
result["confidence"] = abs(calibrated_proba - 0.5) * 2
|
| 193 |
+
calibrated = True
|
| 194 |
+
per_frame_probs = pd.Series(calibrated_preds)
|
| 195 |
+
except Exception as e:
|
| 196 |
+
self.logger.error(f"Subject calibration failed: {e}")
|
| 197 |
+
result["calibrated"] = calibrated
|
| 198 |
+
|
| 199 |
+
# 6. Temporal analysis
|
| 200 |
+
self.logger.info("Performing temporal analysis...")
|
| 201 |
+
timestamps = fused_df.get("timestamp", pd.Series(range(len(fused_df))))
|
| 202 |
+
temporal_result = self.temporal_analyzer.analyze(
|
| 203 |
+
per_frame_probs.values, timestamps.values
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
# Map temporal results for API
|
| 207 |
+
if isinstance(timestamps, pd.Series):
|
| 208 |
+
temporal_result["timestamps"] = timestamps.tolist()
|
| 209 |
+
else:
|
| 210 |
+
temporal_result["timestamps"] = list(timestamps)
|
| 211 |
+
|
| 212 |
+
if "smoothed_probabilities" in temporal_result:
|
| 213 |
+
import numpy as np
|
| 214 |
+
if isinstance(temporal_result["smoothed_probabilities"], np.ndarray):
|
| 215 |
+
temporal_result["probability_trajectory"] = temporal_result["smoothed_probabilities"].tolist()
|
| 216 |
+
else:
|
| 217 |
+
temporal_result["probability_trajectory"] = list(temporal_result["smoothed_probabilities"])
|
| 218 |
+
|
| 219 |
+
result["temporal_analysis"] = temporal_result
|
| 220 |
+
|
| 221 |
+
# 7. Generate explanations
|
| 222 |
+
self.logger.info("Generating explanations...")
|
| 223 |
+
if self.classifier.is_fitted_ and X.size > 0:
|
| 224 |
+
try:
|
| 225 |
+
# Explainer expects a single sample (1, n_features).
|
| 226 |
+
# We use the mean of all frames to explain the overall video prediction.
|
| 227 |
+
import numpy as np
|
| 228 |
+
X_mean = np.nanmean(X, axis=0, keepdims=True)
|
| 229 |
+
explanation = self.explainer.explain_prediction(
|
| 230 |
+
X=X_mean,
|
| 231 |
+
feature_names=feature_names,
|
| 232 |
+
prediction=proba,
|
| 233 |
+
subject_id=subject_id
|
| 234 |
+
)
|
| 235 |
+
result["explanation"] = explanation
|
| 236 |
+
except Exception as e:
|
| 237 |
+
self.logger.warning(f"Explanation generation failed: {e}")
|
| 238 |
+
result["explanation"] = {"error": str(e)}
|
| 239 |
+
else:
|
| 240 |
+
result["explanation"] = {"info": "Classifier not trained or no features."}
|
| 241 |
+
|
| 242 |
+
# 8. Generate visualizations
|
| 243 |
+
if generate_visualizations:
|
| 244 |
+
self.logger.info("Generating visualizations...")
|
| 245 |
+
try:
|
| 246 |
+
# Facial landmarks visualization
|
| 247 |
+
facial_viz_path = output_path / "facial_landmarks.png"
|
| 248 |
+
self.facial_visualizer.plot_landmarks(facial_df, str(facial_viz_path))
|
| 249 |
+
visualizations["facial_landmarks"] = str(facial_viz_path)
|
| 250 |
+
except Exception as e:
|
| 251 |
+
self.logger.warning(f"Facial visualization failed: {e}")
|
| 252 |
+
|
| 253 |
+
try:
|
| 254 |
+
# Temporal probability trajectory
|
| 255 |
+
temporal_viz_path = output_path / "temporal_trajectory.png"
|
| 256 |
+
self.temporal_visualizer.plot_trajectory(
|
| 257 |
+
per_frame_probs.values,
|
| 258 |
+
timestamps.values,
|
| 259 |
+
str(temporal_viz_path),
|
| 260 |
+
)
|
| 261 |
+
visualizations["temporal_trajectory"] = str(temporal_viz_path)
|
| 262 |
+
except Exception as e:
|
| 263 |
+
self.logger.warning(f"Temporal visualization failed: {e}")
|
| 264 |
+
|
| 265 |
+
if self.classifier.is_fitted_ and X.size > 0:
|
| 266 |
+
try:
|
| 267 |
+
# SHAP summary plot
|
| 268 |
+
shap_viz_path = output_path / "shap_summary.png"
|
| 269 |
+
self.explainability_visualizer.plot_shap_summary(
|
| 270 |
+
self.explainer, X, str(shap_viz_path)
|
| 271 |
+
)
|
| 272 |
+
visualizations["shap_summary"] = str(shap_viz_path)
|
| 273 |
+
except Exception as e:
|
| 274 |
+
self.logger.warning(f"SHAP visualization failed: {e}")
|
| 275 |
+
|
| 276 |
+
result["visualizations"] = visualizations
|
| 277 |
+
|
| 278 |
+
# 9. Metadata
|
| 279 |
+
metadata.update(
|
| 280 |
+
{
|
| 281 |
+
"video_fps": self.config.features.fps,
|
| 282 |
+
"audio_sample_rate": self.config.audio.sample_rate,
|
| 283 |
+
"pipeline_version": "1.0",
|
| 284 |
+
}
|
| 285 |
+
)
|
| 286 |
+
result["metadata"] = metadata
|
| 287 |
+
|
| 288 |
+
self.logger.info(f"Analysis completed. Prediction: {prediction} (prob={proba:.3f})")
|
| 289 |
+
return result
|
| 290 |
+
|
| 291 |
+
except Exception as e:
|
| 292 |
+
self.logger.error(f"Pipeline analysis failed: {e}")
|
| 293 |
+
# Return partial results with error flag
|
| 294 |
+
error_result = {
|
| 295 |
+
"error": str(e),
|
| 296 |
+
"deception_probability": 0.5,
|
| 297 |
+
"prediction": "error",
|
| 298 |
+
"confidence": 0.0,
|
| 299 |
+
"facial_features": pd.DataFrame(),
|
| 300 |
+
"audio_features": pd.DataFrame(),
|
| 301 |
+
"fused_features": pd.DataFrame(),
|
| 302 |
+
"temporal_analysis": {},
|
| 303 |
+
"explanation": {},
|
| 304 |
+
"calibrated": False,
|
| 305 |
+
"visualizations": {},
|
| 306 |
+
"metadata": metadata,
|
| 307 |
+
}
|
| 308 |
+
return error_result
|
| 309 |
+
|
| 310 |
+
def analyze_batch(
|
| 311 |
+
self,
|
| 312 |
+
video_paths: List[str],
|
| 313 |
+
subject_ids: Optional[List[str]] = None,
|
| 314 |
+
generate_visualizations: bool = False,
|
| 315 |
+
output_dir: Optional[str] = None,
|
| 316 |
+
) -> List[dict]:
|
| 317 |
+
"""Analyze multiple videos. Wraps analyze_video with progress tracking.
|
| 318 |
+
|
| 319 |
+
Args:
|
| 320 |
+
video_paths: List of video file paths.
|
| 321 |
+
subject_ids: Optional list of subject IDs (same length as video_paths).
|
| 322 |
+
generate_visualizations: Whether to generate visualizations per video.
|
| 323 |
+
output_dir: Base output directory. Subdirectories will be created per video.
|
| 324 |
+
|
| 325 |
+
Returns:
|
| 326 |
+
List of analysis result dicts, one per video.
|
| 327 |
+
"""
|
| 328 |
+
if subject_ids is not None and len(subject_ids) != len(video_paths):
|
| 329 |
+
raise ValueError("subject_ids must have same length as video_paths")
|
| 330 |
+
|
| 331 |
+
results = []
|
| 332 |
+
for idx, video_path in enumerate(tqdm(video_paths, desc="Analyzing videos")):
|
| 333 |
+
subject_id = subject_ids[idx] if subject_ids else None
|
| 334 |
+
video_output_dir = None
|
| 335 |
+
if output_dir is not None:
|
| 336 |
+
video_name = Path(video_path).stem
|
| 337 |
+
video_output_dir = str(Path(output_dir) / video_name)
|
| 338 |
+
Path(video_output_dir).mkdir(parents=True, exist_ok=True)
|
| 339 |
+
|
| 340 |
+
result = self.analyze_video(
|
| 341 |
+
video_path,
|
| 342 |
+
subject_id=subject_id,
|
| 343 |
+
generate_visualizations=generate_visualizations,
|
| 344 |
+
output_dir=video_output_dir,
|
| 345 |
+
)
|
| 346 |
+
results.append(result)
|
| 347 |
+
|
| 348 |
+
return results
|
| 349 |
+
|
| 350 |
+
def train(
|
| 351 |
+
self,
|
| 352 |
+
video_paths: List[str],
|
| 353 |
+
labels: List[int],
|
| 354 |
+
subject_ids: Optional[List[str]] = None,
|
| 355 |
+
output_dir: Optional[str] = None,
|
| 356 |
+
) -> dict:
|
| 357 |
+
"""Train the classifier on labeled video data.
|
| 358 |
+
|
| 359 |
+
Steps:
|
| 360 |
+
1. Extract features from all videos
|
| 361 |
+
2. Fuse features
|
| 362 |
+
3. Train classifier
|
| 363 |
+
4. Compute training metrics
|
| 364 |
+
5. Save model if output_dir provided
|
| 365 |
+
|
| 366 |
+
Args:
|
| 367 |
+
video_paths: List of video file paths.
|
| 368 |
+
labels: Binary labels (0=truthful, 1=deceptive).
|
| 369 |
+
subject_ids: Optional list of subject IDs for calibration baseline.
|
| 370 |
+
output_dir: Directory to save trained model and training results.
|
| 371 |
+
|
| 372 |
+
Returns:
|
| 373 |
+
dict with training metrics and model info.
|
| 374 |
+
"""
|
| 375 |
+
if len(video_paths) != len(labels):
|
| 376 |
+
raise ValueError("video_paths and labels must have same length")
|
| 377 |
+
if subject_ids is not None and len(subject_ids) != len(video_paths):
|
| 378 |
+
raise ValueError("subject_ids must have same length as video_paths")
|
| 379 |
+
|
| 380 |
+
self.logger.info(f"Starting training on {len(video_paths)} videos.")
|
| 381 |
+
|
| 382 |
+
# 1. Extract features from all videos
|
| 383 |
+
all_fused_dfs = []
|
| 384 |
+
for idx, video_path in enumerate(tqdm(video_paths, desc="Extracting features")):
|
| 385 |
+
try:
|
| 386 |
+
# Extract facial features
|
| 387 |
+
facial_df = self.facial_extractor.process_video(video_path)
|
| 388 |
+
|
| 389 |
+
# Extract audio features
|
| 390 |
+
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
| 391 |
+
audio_path = extract_audio_from_video(video_path, tmp.name)
|
| 392 |
+
audio_df = self.audio_extractor.process_audio_segments(str(audio_path), segment_duration=2.0)
|
| 393 |
+
|
| 394 |
+
# Fuse
|
| 395 |
+
aligned_df = self.fusion.align_features(
|
| 396 |
+
facial_df, audio_df, video_fps=self.config.features.fps
|
| 397 |
+
)
|
| 398 |
+
fused_df = self.fusion.extract_fusion_features(aligned_df)
|
| 399 |
+
fused_df["label"] = labels[idx]
|
| 400 |
+
if subject_ids is not None:
|
| 401 |
+
fused_df["subject_id"] = subject_ids[idx]
|
| 402 |
+
all_fused_dfs.append(fused_df)
|
| 403 |
+
except Exception as e:
|
| 404 |
+
self.logger.warning(f"Failed to extract features from {video_path}: {e}")
|
| 405 |
+
continue
|
| 406 |
+
|
| 407 |
+
if not all_fused_dfs:
|
| 408 |
+
raise ValueError("No features extracted from any video.")
|
| 409 |
+
|
| 410 |
+
# Concatenate all features
|
| 411 |
+
full_df = pd.concat(all_fused_dfs, ignore_index=True)
|
| 412 |
+
|
| 413 |
+
# 2. Prepare feature matrix and labels
|
| 414 |
+
X, feature_names = self.classifier.prepare_features(full_df)
|
| 415 |
+
y = full_df["label"].values
|
| 416 |
+
|
| 417 |
+
if X.size == 0:
|
| 418 |
+
raise ValueError("No valid features after preparation.")
|
| 419 |
+
|
| 420 |
+
# 3. Train classifier
|
| 421 |
+
self.logger.info(f"Training classifier with {X.shape[0]} samples, {X.shape[1]} features.")
|
| 422 |
+
metrics = self.classifier.train(X, y, feature_names)
|
| 423 |
+
|
| 424 |
+
# 4. Update calibrator with subject baselines if subject_ids provided
|
| 425 |
+
if subject_ids is not None and self.classifier.is_fitted_:
|
| 426 |
+
self.logger.info("Updating subject baselines...")
|
| 427 |
+
for subject_id in set(subject_ids):
|
| 428 |
+
subject_mask = full_df["subject_id"] == subject_id
|
| 429 |
+
if subject_mask.any():
|
| 430 |
+
subject_features = full_df[subject_mask]
|
| 431 |
+
try:
|
| 432 |
+
self.calibrator.add_subject_profile(subject_id, subject_features)
|
| 433 |
+
except Exception as e:
|
| 434 |
+
self.logger.warning(f"Failed to update baseline for {subject_id}: {e}")
|
| 435 |
+
|
| 436 |
+
# 5. Save model if output_dir provided
|
| 437 |
+
if output_dir is not None:
|
| 438 |
+
model_path = Path(output_dir) / "deception_classifier.joblib"
|
| 439 |
+
self.classifier.save(str(model_path))
|
| 440 |
+
metrics["model_path"] = str(model_path)
|
| 441 |
+
self.logger.info(f"Model saved to {model_path}")
|
| 442 |
+
|
| 443 |
+
# Save training results
|
| 444 |
+
results_path = Path(output_dir) / "training_results.json"
|
| 445 |
+
save_results(metrics, str(results_path))
|
| 446 |
+
|
| 447 |
+
self.logger.info("Training completed.")
|
| 448 |
+
return metrics
|
| 449 |
+
|
| 450 |
+
def generate_report(self, analysis_result: dict, output_path: str) -> str:
|
| 451 |
+
"""Generate a comprehensive text/JSON report from analysis results.
|
| 452 |
+
|
| 453 |
+
Creates a formatted report including:
|
| 454 |
+
- Overall deception assessment
|
| 455 |
+
- Key behavioral indicators
|
| 456 |
+
- Temporal analysis summary
|
| 457 |
+
- Feature importance highlights
|
| 458 |
+
- Suspicious intervals if detected
|
| 459 |
+
|
| 460 |
+
Args:
|
| 461 |
+
analysis_result: Result dict from analyze_video.
|
| 462 |
+
output_path: Path to save the report (JSON).
|
| 463 |
+
|
| 464 |
+
Returns:
|
| 465 |
+
Path to the saved report file.
|
| 466 |
+
"""
|
| 467 |
+
from src.utils.io import save_results
|
| 468 |
+
|
| 469 |
+
# Build report structure
|
| 470 |
+
report = {
|
| 471 |
+
"summary": {
|
| 472 |
+
"prediction": analysis_result.get("prediction", "unknown"),
|
| 473 |
+
"deception_probability": analysis_result.get("deception_probability", 0.0),
|
| 474 |
+
"confidence": analysis_result.get("confidence", 0.0),
|
| 475 |
+
"calibrated": analysis_result.get("calibrated", False),
|
| 476 |
+
},
|
| 477 |
+
"temporal_analysis": analysis_result.get("temporal_analysis", {}),
|
| 478 |
+
"explanation": analysis_result.get("explanation", {}),
|
| 479 |
+
"metadata": analysis_result.get("metadata", {}),
|
| 480 |
+
}
|
| 481 |
+
|
| 482 |
+
# Add feature importance if available
|
| 483 |
+
if self.classifier.is_fitted_:
|
| 484 |
+
try:
|
| 485 |
+
importance_df = self.classifier.get_feature_importance()
|
| 486 |
+
top_features = importance_df.head(10).to_dict(orient="records")
|
| 487 |
+
report["feature_importance"] = top_features
|
| 488 |
+
except Exception as e:
|
| 489 |
+
self.logger.warning(f"Could not compute feature importance: {e}")
|
| 490 |
+
report["feature_importance"] = []
|
| 491 |
+
|
| 492 |
+
# Add suspicious intervals from temporal analysis
|
| 493 |
+
temporal = analysis_result.get("temporal_analysis", {})
|
| 494 |
+
if "suspicious_intervals" in temporal:
|
| 495 |
+
report["suspicious_intervals"] = temporal["suspicious_intervals"]
|
| 496 |
+
if "spikes" in temporal:
|
| 497 |
+
report["spikes"] = temporal["spikes"]
|
| 498 |
+
|
| 499 |
+
# Save report as JSON
|
| 500 |
+
save_results(report, output_path)
|
| 501 |
+
self.logger.info(f"Report saved to {output_path}")
|
| 502 |
+
return output_path
|
| 503 |
+
|
| 504 |
+
def load_model(self, model_path: str) -> None:
|
| 505 |
+
"""Load a pre-trained classifier model.
|
| 506 |
+
|
| 507 |
+
Args:
|
| 508 |
+
model_path: Path to the saved model file.
|
| 509 |
+
"""
|
| 510 |
+
self.classifier.load(model_path)
|
| 511 |
+
# Re-initialize explainer with the loaded classifier
|
| 512 |
+
self.explainer = DeceptionExplainer(self.classifier)
|
| 513 |
+
# Re-initialize calibrator with the loaded classifier
|
| 514 |
+
self.calibrator = SubjectCalibrator(self.classifier, self.config)
|
| 515 |
+
self.logger.info(f"Model loaded from {model_path}")
|
| 516 |
+
|
| 517 |
+
def save_model(self, model_path: str) -> None:
|
| 518 |
+
"""Save the current classifier model.
|
| 519 |
+
|
| 520 |
+
Args:
|
| 521 |
+
model_path: Path where the model should be saved.
|
| 522 |
+
"""
|
| 523 |
+
self.classifier.save(model_path)
|
| 524 |
+
self.logger.info(f"Model saved to {model_path}")
|
src/temporal/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Temporal analysis module for deception trajectory tracking."""
|
| 2 |
+
|
| 3 |
+
from .analyzer import TemporalAnalyzer
|
| 4 |
+
from .visualizer import TemporalVisualizer
|
| 5 |
+
|
| 6 |
+
__all__ = ['TemporalAnalyzer', 'TemporalVisualizer']
|
src/temporal/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (339 Bytes). View file
|
|
|
src/temporal/analyzer.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Temporal deception probability analysis utilities.
|
| 2 |
+
|
| 3 |
+
This module is model-agnostic and operates on probability trajectories to detect
|
| 4 |
+
spikes, contiguous suspicious regions, and aggregate temporal statistics.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import Dict, List, Optional
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
from src.utils.config import TemporalConfig
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class TemporalAnalyzer:
|
| 17 |
+
"""Frame-by-frame temporal deception analysis with spike detection."""
|
| 18 |
+
|
| 19 |
+
def __init__(self, config: Optional[TemporalConfig] = None):
|
| 20 |
+
"""Initialize temporal analyzer.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
config: Optional temporal configuration.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
self.config = config or TemporalConfig()
|
| 27 |
+
self.window_size = int(getattr(self.config, "window_size", 30))
|
| 28 |
+
self.spike_threshold = float(getattr(self.config, "spike_threshold", 0.7))
|
| 29 |
+
self.min_suspicious_duration = int(getattr(self.config, "min_suspicious_duration", 10))
|
| 30 |
+
self.smoothing_alpha = float(getattr(self.config, "smoothing_alpha", 0.3))
|
| 31 |
+
|
| 32 |
+
def smooth_probabilities(self, probabilities: np.ndarray) -> np.ndarray:
|
| 33 |
+
"""Apply exponential moving average smoothing.
|
| 34 |
+
|
| 35 |
+
Args:
|
| 36 |
+
probabilities: Raw deception probabilities.
|
| 37 |
+
|
| 38 |
+
Returns:
|
| 39 |
+
np.ndarray: Smoothed probabilities.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
probs = np.asarray(probabilities, dtype=np.float64)
|
| 43 |
+
if probs.size == 0:
|
| 44 |
+
return np.array([], dtype=np.float64)
|
| 45 |
+
if probs.size == 1:
|
| 46 |
+
return np.clip(probs.copy(), 0.0, 1.0)
|
| 47 |
+
|
| 48 |
+
probs = np.nan_to_num(probs, nan=0.0, posinf=1.0, neginf=0.0)
|
| 49 |
+
probs = np.clip(probs, 0.0, 1.0)
|
| 50 |
+
|
| 51 |
+
alpha = float(np.clip(self.smoothing_alpha, 1e-3, 1.0))
|
| 52 |
+
smoothed = np.empty_like(probs)
|
| 53 |
+
smoothed[0] = probs[0]
|
| 54 |
+
for i in range(1, len(probs)):
|
| 55 |
+
# EMA is used because it denoises while preserving abrupt changes,
|
| 56 |
+
# which are behaviorally meaningful in deception dynamics.
|
| 57 |
+
smoothed[i] = alpha * probs[i] + (1.0 - alpha) * smoothed[i - 1]
|
| 58 |
+
|
| 59 |
+
return smoothed
|
| 60 |
+
|
| 61 |
+
def detect_spikes(self, probabilities: np.ndarray, timestamps: np.ndarray) -> List[Dict]:
|
| 62 |
+
"""Detect high-confidence temporal spikes.
|
| 63 |
+
|
| 64 |
+
Args:
|
| 65 |
+
probabilities: Raw deception probability sequence.
|
| 66 |
+
timestamps: Timestamps aligned with probabilities.
|
| 67 |
+
|
| 68 |
+
Returns:
|
| 69 |
+
List[Dict]: Spike descriptors.
|
| 70 |
+
"""
|
| 71 |
+
|
| 72 |
+
probs = np.asarray(probabilities, dtype=np.float64)
|
| 73 |
+
times = np.asarray(timestamps, dtype=np.float64)
|
| 74 |
+
n = min(len(probs), len(times))
|
| 75 |
+
if n == 0:
|
| 76 |
+
return []
|
| 77 |
+
|
| 78 |
+
probs = probs[:n]
|
| 79 |
+
times = times[:n]
|
| 80 |
+
smoothed = self.smooth_probabilities(probs)
|
| 81 |
+
|
| 82 |
+
if n == 1:
|
| 83 |
+
if smoothed[0] >= self.spike_threshold:
|
| 84 |
+
return [
|
| 85 |
+
{
|
| 86 |
+
"frame_idx": 0,
|
| 87 |
+
"timestamp": float(times[0]),
|
| 88 |
+
"probability": float(smoothed[0]),
|
| 89 |
+
"magnitude": float(smoothed[0] - self.spike_threshold),
|
| 90 |
+
"rate_of_change": 0.0,
|
| 91 |
+
}
|
| 92 |
+
]
|
| 93 |
+
return []
|
| 94 |
+
|
| 95 |
+
deriv = np.diff(smoothed, prepend=smoothed[0])
|
| 96 |
+
deriv_std = float(np.std(deriv))
|
| 97 |
+
deriv_threshold = 2.0 * deriv_std
|
| 98 |
+
|
| 99 |
+
spikes: List[Dict] = []
|
| 100 |
+
for i in range(n):
|
| 101 |
+
prob_high = smoothed[i] >= self.spike_threshold
|
| 102 |
+
roc_high = abs(deriv[i]) >= deriv_threshold if deriv_std > 0 else False
|
| 103 |
+
if prob_high or roc_high:
|
| 104 |
+
spikes.append(
|
| 105 |
+
{
|
| 106 |
+
"frame_idx": int(i),
|
| 107 |
+
"timestamp": float(times[i]),
|
| 108 |
+
"probability": float(smoothed[i]),
|
| 109 |
+
"magnitude": float(max(0.0, smoothed[i] - self.spike_threshold)),
|
| 110 |
+
"rate_of_change": float(deriv[i]),
|
| 111 |
+
}
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
return spikes
|
| 115 |
+
|
| 116 |
+
def localize_suspicious_intervals(
|
| 117 |
+
self,
|
| 118 |
+
probabilities: np.ndarray,
|
| 119 |
+
timestamps: np.ndarray,
|
| 120 |
+
) -> List[Dict]:
|
| 121 |
+
"""Identify contiguous suspicious intervals.
|
| 122 |
+
|
| 123 |
+
Args:
|
| 124 |
+
probabilities: Raw deception probability sequence.
|
| 125 |
+
timestamps: Timestamps aligned with probabilities.
|
| 126 |
+
|
| 127 |
+
Returns:
|
| 128 |
+
List[Dict]: Interval dictionaries.
|
| 129 |
+
"""
|
| 130 |
+
|
| 131 |
+
probs = np.asarray(probabilities, dtype=np.float64)
|
| 132 |
+
times = np.asarray(timestamps, dtype=np.float64)
|
| 133 |
+
n = min(len(probs), len(times))
|
| 134 |
+
if n == 0:
|
| 135 |
+
return []
|
| 136 |
+
|
| 137 |
+
probs = probs[:n]
|
| 138 |
+
times = times[:n]
|
| 139 |
+
smoothed = self.smooth_probabilities(probs)
|
| 140 |
+
mask = smoothed >= self.spike_threshold
|
| 141 |
+
|
| 142 |
+
if not np.any(mask):
|
| 143 |
+
return []
|
| 144 |
+
|
| 145 |
+
padded = np.pad(mask.astype(np.int8), (1, 1), mode="constant")
|
| 146 |
+
transitions = np.diff(padded)
|
| 147 |
+
starts = np.where(transitions == 1)[0]
|
| 148 |
+
ends = np.where(transitions == -1)[0] - 1
|
| 149 |
+
|
| 150 |
+
intervals: List[Dict] = []
|
| 151 |
+
for start, end in zip(starts, ends):
|
| 152 |
+
duration_frames = int(end - start + 1)
|
| 153 |
+
if duration_frames < self.min_suspicious_duration:
|
| 154 |
+
continue
|
| 155 |
+
|
| 156 |
+
segment = smoothed[start : end + 1]
|
| 157 |
+
avg_p = float(np.mean(segment))
|
| 158 |
+
max_p = float(np.max(segment))
|
| 159 |
+
if max_p < 0.8:
|
| 160 |
+
severity = "low"
|
| 161 |
+
elif max_p < 0.9:
|
| 162 |
+
severity = "medium"
|
| 163 |
+
else:
|
| 164 |
+
severity = "high"
|
| 165 |
+
|
| 166 |
+
start_t = float(times[start])
|
| 167 |
+
end_t = float(times[end])
|
| 168 |
+
duration_s = float(max(0.0, end_t - start_t))
|
| 169 |
+
|
| 170 |
+
intervals.append(
|
| 171 |
+
{
|
| 172 |
+
"start_frame": int(start),
|
| 173 |
+
"end_frame": int(end),
|
| 174 |
+
"start_time": start_t,
|
| 175 |
+
"end_time": end_t,
|
| 176 |
+
"duration_frames": duration_frames,
|
| 177 |
+
"duration_seconds": duration_s,
|
| 178 |
+
"avg_probability": avg_p,
|
| 179 |
+
"max_probability": max_p,
|
| 180 |
+
"severity": severity,
|
| 181 |
+
}
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
return intervals
|
| 185 |
+
|
| 186 |
+
def compute_temporal_features(
|
| 187 |
+
self,
|
| 188 |
+
probabilities: np.ndarray,
|
| 189 |
+
timestamps: np.ndarray,
|
| 190 |
+
) -> Dict[str, float]:
|
| 191 |
+
"""Compute aggregate temporal statistics.
|
| 192 |
+
|
| 193 |
+
Args:
|
| 194 |
+
probabilities: Raw deception probability sequence.
|
| 195 |
+
timestamps: Timestamps aligned with probabilities.
|
| 196 |
+
|
| 197 |
+
Returns:
|
| 198 |
+
Dict[str, float]: Aggregate temporal metrics.
|
| 199 |
+
"""
|
| 200 |
+
|
| 201 |
+
probs = np.asarray(probabilities, dtype=np.float64)
|
| 202 |
+
times = np.asarray(timestamps, dtype=np.float64)
|
| 203 |
+
n = min(len(probs), len(times))
|
| 204 |
+
if n == 0:
|
| 205 |
+
return {
|
| 206 |
+
"mean_probability": 0.0,
|
| 207 |
+
"max_probability": 0.0,
|
| 208 |
+
"std_probability": 0.0,
|
| 209 |
+
"spike_count": 0.0,
|
| 210 |
+
"suspicious_time_ratio": 0.0,
|
| 211 |
+
"longest_suspicious_duration": 0.0,
|
| 212 |
+
"transition_count": 0.0,
|
| 213 |
+
"temporal_entropy": 0.0,
|
| 214 |
+
"trend_slope": 0.0,
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
probs = probs[:n]
|
| 218 |
+
times = times[:n]
|
| 219 |
+
smoothed = self.smooth_probabilities(probs)
|
| 220 |
+
|
| 221 |
+
spikes = self.detect_spikes(probs, times)
|
| 222 |
+
intervals = self.localize_suspicious_intervals(probs, times)
|
| 223 |
+
|
| 224 |
+
total_duration = float(max(1e-8, times[-1] - times[0])) if n > 1 else 1e-8
|
| 225 |
+
suspicious_duration = float(sum(item["duration_seconds"] for item in intervals))
|
| 226 |
+
longest_duration = float(max([item["duration_seconds"] for item in intervals], default=0.0))
|
| 227 |
+
|
| 228 |
+
binary = (smoothed >= 0.5).astype(np.int8)
|
| 229 |
+
transition_count = float(np.sum(np.abs(np.diff(binary)))) if n > 1 else 0.0
|
| 230 |
+
|
| 231 |
+
hist, _ = np.histogram(smoothed, bins=10, range=(0.0, 1.0), density=True)
|
| 232 |
+
hist = hist[hist > 0]
|
| 233 |
+
entropy = float(-np.sum(hist * np.log2(hist + 1e-12))) if hist.size else 0.0
|
| 234 |
+
|
| 235 |
+
if n > 1 and np.std(times) > 0:
|
| 236 |
+
slope = float(np.polyfit(times, smoothed, deg=1)[0])
|
| 237 |
+
else:
|
| 238 |
+
slope = 0.0
|
| 239 |
+
|
| 240 |
+
return {
|
| 241 |
+
"mean_probability": float(np.mean(smoothed)),
|
| 242 |
+
"max_probability": float(np.max(smoothed)),
|
| 243 |
+
"std_probability": float(np.std(smoothed)),
|
| 244 |
+
"spike_count": float(len(spikes)),
|
| 245 |
+
"suspicious_time_ratio": float(np.clip(suspicious_duration / total_duration, 0.0, 1.0)),
|
| 246 |
+
"longest_suspicious_duration": longest_duration,
|
| 247 |
+
"transition_count": transition_count,
|
| 248 |
+
"temporal_entropy": entropy,
|
| 249 |
+
"trend_slope": slope,
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
def analyze(self, probabilities: np.ndarray, timestamps: np.ndarray, fps: float = 30.0) -> Dict:
|
| 253 |
+
"""Perform complete temporal analysis.
|
| 254 |
+
|
| 255 |
+
Args:
|
| 256 |
+
probabilities: Raw deception probability sequence.
|
| 257 |
+
timestamps: Optional timestamp sequence. If empty, generated using fps.
|
| 258 |
+
fps: Frame rate for synthetic timestamps.
|
| 259 |
+
|
| 260 |
+
Returns:
|
| 261 |
+
Dict: Full temporal analysis output.
|
| 262 |
+
"""
|
| 263 |
+
|
| 264 |
+
probs = np.asarray(probabilities, dtype=np.float64)
|
| 265 |
+
times = np.asarray(timestamps, dtype=np.float64)
|
| 266 |
+
|
| 267 |
+
if probs.size == 0:
|
| 268 |
+
return {
|
| 269 |
+
"smoothed_probabilities": np.array([], dtype=np.float64),
|
| 270 |
+
"spikes": [],
|
| 271 |
+
"suspicious_intervals": [],
|
| 272 |
+
"temporal_features": self.compute_temporal_features(np.array([]), np.array([])),
|
| 273 |
+
"verdict": "truthful",
|
| 274 |
+
"confidence": 0.0,
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
if times.size == 0 or len(times) != len(probs):
|
| 278 |
+
safe_fps = fps if fps > 0 else 30.0
|
| 279 |
+
times = np.arange(len(probs), dtype=np.float64) / float(safe_fps)
|
| 280 |
+
|
| 281 |
+
smoothed = self.smooth_probabilities(probs)
|
| 282 |
+
spikes = self.detect_spikes(probs, times)
|
| 283 |
+
intervals = self.localize_suspicious_intervals(probs, times)
|
| 284 |
+
features = self.compute_temporal_features(probs, times)
|
| 285 |
+
|
| 286 |
+
mean_prob = float(features.get("mean_probability", 0.0))
|
| 287 |
+
verdict = "deceptive" if mean_prob > 0.5 else "truthful"
|
| 288 |
+
confidence = float(np.clip(abs(mean_prob - 0.5) * 2.0, 0.0, 1.0))
|
| 289 |
+
|
| 290 |
+
return {
|
| 291 |
+
"smoothed_probabilities": smoothed,
|
| 292 |
+
"spikes": spikes,
|
| 293 |
+
"suspicious_intervals": intervals,
|
| 294 |
+
"temporal_features": features,
|
| 295 |
+
"verdict": verdict,
|
| 296 |
+
"confidence": confidence,
|
| 297 |
+
}
|
| 298 |
+
|
src/temporal/visualizer.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Visualization helpers for temporal deception analysis outputs."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Dict, List, Optional
|
| 6 |
+
|
| 7 |
+
import matplotlib.pyplot as plt
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
from src.utils.config import TemporalConfig
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class TemporalVisualizer:
|
| 14 |
+
"""Publication-quality temporal analysis visualizations."""
|
| 15 |
+
|
| 16 |
+
def __init__(self):
|
| 17 |
+
"""Initialize plotting style for publication-quality figures."""
|
| 18 |
+
|
| 19 |
+
plt.style.use("seaborn-v0_8-whitegrid")
|
| 20 |
+
plt.rcParams.update(
|
| 21 |
+
{
|
| 22 |
+
"figure.dpi": 300,
|
| 23 |
+
"savefig.dpi": 300,
|
| 24 |
+
"axes.titlesize": 14,
|
| 25 |
+
"axes.labelsize": 12,
|
| 26 |
+
"legend.fontsize": 10,
|
| 27 |
+
"xtick.labelsize": 10,
|
| 28 |
+
"ytick.labelsize": 10,
|
| 29 |
+
"font.size": 11,
|
| 30 |
+
}
|
| 31 |
+
)
|
| 32 |
+
self._temporal_cfg = TemporalConfig()
|
| 33 |
+
|
| 34 |
+
def plot_deception_timeline(
|
| 35 |
+
self,
|
| 36 |
+
probabilities: np.ndarray,
|
| 37 |
+
timestamps: np.ndarray,
|
| 38 |
+
smoothed: np.ndarray,
|
| 39 |
+
suspicious_intervals: List[Dict],
|
| 40 |
+
spikes: List[Dict],
|
| 41 |
+
save_path: Optional[str] = None,
|
| 42 |
+
) -> plt.Figure:
|
| 43 |
+
"""Create the main deception timeline plot."""
|
| 44 |
+
|
| 45 |
+
probs = np.asarray(probabilities, dtype=np.float64)
|
| 46 |
+
times = np.asarray(timestamps, dtype=np.float64)
|
| 47 |
+
smooth = np.asarray(smoothed, dtype=np.float64)
|
| 48 |
+
|
| 49 |
+
n = min(len(probs), len(times), len(smooth))
|
| 50 |
+
probs = probs[:n]
|
| 51 |
+
times = times[:n]
|
| 52 |
+
smooth = smooth[:n]
|
| 53 |
+
|
| 54 |
+
fig, ax = plt.subplots(figsize=(12, 5))
|
| 55 |
+
|
| 56 |
+
if n > 0:
|
| 57 |
+
ax.plot(times, probs, color="#93c5fd", linewidth=1.2, alpha=0.8, label="Raw probability")
|
| 58 |
+
ax.plot(times, smooth, color="#1e3a8a", linewidth=2.0, label="Smoothed probability")
|
| 59 |
+
|
| 60 |
+
threshold = float(getattr(self._temporal_cfg, "spike_threshold", 0.7))
|
| 61 |
+
ax.axhline(threshold, color="#dc2626", linestyle="--", linewidth=1.4, label="Spike threshold")
|
| 62 |
+
ax.axhline(0.5, color="#6b7280", linestyle="--", linewidth=1.2, label="Decision boundary")
|
| 63 |
+
|
| 64 |
+
for i, interval in enumerate(suspicious_intervals):
|
| 65 |
+
start_t = float(interval.get("start_time", 0.0))
|
| 66 |
+
end_t = float(interval.get("end_time", start_t))
|
| 67 |
+
ax.axvspan(
|
| 68 |
+
start_t,
|
| 69 |
+
end_t,
|
| 70 |
+
color="#fecaca",
|
| 71 |
+
alpha=0.35,
|
| 72 |
+
label="Suspicious interval" if i == 0 else None,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
if spikes:
|
| 76 |
+
spike_x = [float(item.get("timestamp", 0.0)) for item in spikes]
|
| 77 |
+
spike_y = [float(item.get("probability", 0.0)) for item in spikes]
|
| 78 |
+
ax.scatter(
|
| 79 |
+
spike_x,
|
| 80 |
+
spike_y,
|
| 81 |
+
marker="^",
|
| 82 |
+
color="#ef4444",
|
| 83 |
+
edgecolors="black",
|
| 84 |
+
linewidths=0.3,
|
| 85 |
+
s=40,
|
| 86 |
+
zorder=5,
|
| 87 |
+
label="Spikes",
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
ax.set_title("Temporal Deception Analysis")
|
| 91 |
+
ax.set_xlabel("Time (seconds)")
|
| 92 |
+
ax.set_ylabel("Deception Probability")
|
| 93 |
+
ax.set_ylim(0.0, 1.02)
|
| 94 |
+
ax.legend(loc="best")
|
| 95 |
+
fig.tight_layout()
|
| 96 |
+
|
| 97 |
+
if save_path:
|
| 98 |
+
fig.savefig(save_path, bbox_inches="tight")
|
| 99 |
+
|
| 100 |
+
return fig
|
| 101 |
+
|
| 102 |
+
def plot_suspicious_regions(
|
| 103 |
+
self,
|
| 104 |
+
suspicious_intervals: List[Dict],
|
| 105 |
+
total_duration: float,
|
| 106 |
+
save_path: Optional[str] = None,
|
| 107 |
+
) -> plt.Figure:
|
| 108 |
+
"""Create a Gantt-style suspicious interval chart."""
|
| 109 |
+
|
| 110 |
+
fig, ax = plt.subplots(figsize=(11, 4))
|
| 111 |
+
|
| 112 |
+
severity_color = {
|
| 113 |
+
"low": "#facc15",
|
| 114 |
+
"medium": "#fb923c",
|
| 115 |
+
"high": "#ef4444",
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
if not suspicious_intervals:
|
| 119 |
+
ax.barh([0], [max(total_duration, 1.0)], left=[0.0], color="#e5e7eb", alpha=0.4)
|
| 120 |
+
ax.text(0.5 * max(total_duration, 1.0), 0, "No suspicious intervals", ha="center", va="center")
|
| 121 |
+
ax.set_yticks([0])
|
| 122 |
+
ax.set_yticklabels(["N/A"])
|
| 123 |
+
else:
|
| 124 |
+
for idx, interval in enumerate(suspicious_intervals):
|
| 125 |
+
start = float(interval.get("start_time", 0.0))
|
| 126 |
+
end = float(interval.get("end_time", start))
|
| 127 |
+
width = max(0.0, end - start)
|
| 128 |
+
severity = str(interval.get("severity", "low")).lower()
|
| 129 |
+
color = severity_color.get(severity, "#facc15")
|
| 130 |
+
ax.barh(idx, width, left=start, color=color, edgecolor="black", alpha=0.85)
|
| 131 |
+
|
| 132 |
+
ax.set_yticks(np.arange(len(suspicious_intervals)))
|
| 133 |
+
ax.set_yticklabels([f"Interval {i + 1}" for i in range(len(suspicious_intervals))])
|
| 134 |
+
|
| 135 |
+
ax.set_xlim(0.0, max(float(total_duration), 1.0))
|
| 136 |
+
ax.set_xlabel("Time (seconds)")
|
| 137 |
+
ax.set_ylabel("Suspicious Intervals")
|
| 138 |
+
ax.set_title("Suspicious Regions Timeline")
|
| 139 |
+
fig.tight_layout()
|
| 140 |
+
|
| 141 |
+
if save_path:
|
| 142 |
+
fig.savefig(save_path, bbox_inches="tight")
|
| 143 |
+
|
| 144 |
+
return fig
|
| 145 |
+
|
| 146 |
+
def plot_temporal_summary(
|
| 147 |
+
self,
|
| 148 |
+
temporal_features: Dict[str, float],
|
| 149 |
+
save_path: Optional[str] = None,
|
| 150 |
+
) -> plt.Figure:
|
| 151 |
+
"""Create a multi-panel summary dashboard of temporal statistics."""
|
| 152 |
+
|
| 153 |
+
fig, axes = plt.subplots(1, 3, figsize=(15, 4.5))
|
| 154 |
+
|
| 155 |
+
# Panel 1: synthetic distribution from mean/std when raw sequence is unavailable.
|
| 156 |
+
mean_p = float(temporal_features.get("mean_probability", 0.0))
|
| 157 |
+
std_p = float(max(temporal_features.get("std_probability", 0.0), 1e-3))
|
| 158 |
+
synthetic = np.clip(np.random.normal(loc=mean_p, scale=std_p, size=800), 0.0, 1.0)
|
| 159 |
+
axes[0].hist(synthetic, bins=20, color="#60a5fa", edgecolor="black", alpha=0.85)
|
| 160 |
+
axes[0].axvline(mean_p, color="#1d4ed8", linestyle="--", linewidth=2)
|
| 161 |
+
axes[0].set_title("Probability Distribution")
|
| 162 |
+
axes[0].set_xlabel("Deception Probability")
|
| 163 |
+
axes[0].set_ylabel("Count")
|
| 164 |
+
|
| 165 |
+
# Panel 2: key metrics.
|
| 166 |
+
key_names = [
|
| 167 |
+
"mean_probability",
|
| 168 |
+
"max_probability",
|
| 169 |
+
"spike_count",
|
| 170 |
+
"suspicious_time_ratio",
|
| 171 |
+
"longest_suspicious_duration",
|
| 172 |
+
]
|
| 173 |
+
key_vals = [float(temporal_features.get(name, 0.0)) for name in key_names]
|
| 174 |
+
axes[1].barh(np.arange(len(key_names)), key_vals, color="#34d399", edgecolor="black", alpha=0.85)
|
| 175 |
+
axes[1].set_yticks(np.arange(len(key_names)))
|
| 176 |
+
axes[1].set_yticklabels([name.replace("_", " ") for name in key_names])
|
| 177 |
+
axes[1].set_title("Key Temporal Metrics")
|
| 178 |
+
axes[1].set_xlabel("Value")
|
| 179 |
+
|
| 180 |
+
# Panel 3: severity pie chart proxy from max probability.
|
| 181 |
+
max_p = float(temporal_features.get("max_probability", 0.0))
|
| 182 |
+
if max_p < 0.7:
|
| 183 |
+
sev = [1.0, 0.0, 0.0]
|
| 184 |
+
elif max_p < 0.85:
|
| 185 |
+
sev = [0.4, 0.6, 0.0]
|
| 186 |
+
else:
|
| 187 |
+
sev = [0.2, 0.3, 0.5]
|
| 188 |
+
axes[2].pie(
|
| 189 |
+
sev,
|
| 190 |
+
labels=["Low", "Medium", "High"],
|
| 191 |
+
colors=["#facc15", "#fb923c", "#ef4444"],
|
| 192 |
+
autopct="%1.0f%%",
|
| 193 |
+
startangle=90,
|
| 194 |
+
)
|
| 195 |
+
axes[2].set_title("Severity Breakdown")
|
| 196 |
+
|
| 197 |
+
fig.tight_layout()
|
| 198 |
+
if save_path:
|
| 199 |
+
fig.savefig(save_path, bbox_inches="tight")
|
| 200 |
+
return fig
|
| 201 |
+
|
src/utils/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Utility modules for configuration and I/O operations."""
|
| 2 |
+
|
| 3 |
+
from .config import AppConfig, ProjectConfig, FeatureConfig, AudioConfig, ModelConfig, TemporalConfig
|
| 4 |
+
from .io import (
|
| 5 |
+
load_video as VideoLoader,
|
| 6 |
+
extract_audio_from_video as AudioExtractor,
|
| 7 |
+
save_model as ModelSerializer,
|
| 8 |
+
save_results as ResultsSaver,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
__all__ = [
|
| 12 |
+
'AppConfig', 'ProjectConfig', 'FeatureConfig', 'AudioConfig',
|
| 13 |
+
'ModelConfig', 'TemporalConfig',
|
| 14 |
+
'VideoLoader', 'AudioExtractor', 'ModelSerializer', 'ResultsSaver'
|
| 15 |
+
]
|
src/utils/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (609 Bytes). View file
|
|
|
src/utils/__pycache__/config.cpython-314.pyc
ADDED
|
Binary file (8.43 kB). View file
|
|
|
src/utils/__pycache__/io.cpython-314.pyc
ADDED
|
Binary file (6.94 kB). View file
|
|
|