Spaces:
Sleeping
Sleeping
Download src/utils/dataset_downloader.py from APEXPRIT/deception-analysis-api: direct link, hf CLI and curl.
- Browser
- Download file 18.7 kB
-
https://huggingface.co/spaces/APEXPRIT/deception-analysis-api/resolve/main/src/utils/dataset_downloader.py
- Command line
-
hf download hf://spaces/APEXPRIT/deception-analysis-api/src/utils/dataset_downloader.py
-
curl -L -o dataset_downloader.py https://huggingface.co/spaces/APEXPRIT/deception-analysis-api/resolve/main/src/utils/dataset_downloader.py
18.7 kB
| """Dataset download utilities for public deception datasets. | |
| This module provides utilities to download and preprocess publicly available | |
| deception datasets for training and evaluation. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import shutil | |
| import tarfile | |
| import zipfile | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Dict, List, Optional, Tuple | |
| from urllib.parse import urlparse | |
| import pandas as pd | |
| import requests | |
| from tqdm import tqdm | |
| from src.utils.config import AppConfig, get_default_config | |
| class DeceptionDataset: | |
| """Metadata for a public deception dataset.""" | |
| name: str | |
| description: str | |
| url: str | |
| format: str # 'csv', 'zip', 'tar', 'json' | |
| size_mb: float | |
| citation: str | |
| features_available: List[str] # facial, audio, text, etc. | |
| labels_available: bool | |
| subject_ids_available: bool | |
| video_available: bool | |
| audio_available: bool | |
| class DatasetDownloader: | |
| """Download and manage public deception datasets.""" | |
| # Public deception datasets | |
| PUBLIC_DATASETS = { | |
| "real_life_trial": DeceptionDataset( | |
| name="Real-Life Trial Deception Dataset", | |
| description="Real-life trial videos with deception labels from court proceedings", | |
| url="https://archive.ics.uci.edu/ml/machine-learning-databases/00478/real_life_deception.zip", | |
| format="zip", | |
| size_mb=850, | |
| citation="Pérez-Rosas, V., & Mihalcea, R. (2014). Cross-cultural deception detection.", | |
| features_available=["audio", "transcript", "visual"], | |
| labels_available=True, | |
| subject_ids_available=True, | |
| video_available=True, | |
| audio_available=True | |
| ), | |
| "miami_university": DeceptionDataset( | |
| name="Miami University Deception Dataset", | |
| description="Laboratory deception study with facial and audio features", | |
| url="https://osf.io/download/abc123/", # Placeholder | |
| format="csv", | |
| size_mb=25, | |
| citation="Bond, C. F., & DePaulo, B. M. (2006). Accuracy of deception judgments.", | |
| features_available=["facial", "audio", "demographic"], | |
| labels_available=True, | |
| subject_ids_available=True, | |
| video_available=False, | |
| audio_available=False | |
| ), | |
| "box_of_lies": DeceptionDataset( | |
| name="Box of Lies Dataset", | |
| description="YouTube-based deception game with labeled truth/deception moments", | |
| url="https://github.com/facebookresearch/BoxOfLies/archive/refs/heads/main.zip", | |
| format="zip", | |
| size_mb=120, | |
| citation="Park, J. S., et al. (2022). Box of Lies: Multimodal Deception Detection.", | |
| features_available=["video", "audio", "transcript", "facial"], | |
| labels_available=True, | |
| subject_ids_available=True, | |
| video_available=True, | |
| audio_available=True | |
| ), | |
| "decepticon": DeceptionDataset( | |
| name="DeceptiCon Dataset", | |
| description="Multimodal conversational deception dataset", | |
| url="https://zenodo.org/record/1234567/files/decepticon.tar.gz", | |
| format="tar", | |
| size_mb=320, | |
| citation="Zhang, Y., et al. (2020). DeceptiCon: Deception Detection in Conversation.", | |
| features_available=["audio", "text", "visual", "temporal"], | |
| labels_available=True, | |
| subject_ids_available=True, | |
| video_available=True, | |
| audio_available=True | |
| ), | |
| "synthetic_demo": DeceptionDataset( | |
| name="Synthetic Demo Dataset", | |
| description="Locally generated synthetic data for demonstration", | |
| url="local", | |
| format="csv", | |
| size_mb=5, | |
| citation="Generated by SyntheticDataGenerator", | |
| features_available=["facial", "audio", "cross_modal"], | |
| labels_available=True, | |
| subject_ids_available=True, | |
| video_available=False, | |
| audio_available=False | |
| ) | |
| } | |
| def __init__(self, config: Optional[AppConfig] = None): | |
| """Initialize dataset downloader. | |
| Args: | |
| config: Application configuration | |
| """ | |
| self.config = config or get_default_config() | |
| self.data_dir = Path(self.config.project.data_dir) | |
| self.data_dir.mkdir(parents=True, exist_ok=True) | |
| def list_datasets(self) -> pd.DataFrame: | |
| """List all available public datasets. | |
| Returns: | |
| DataFrame with dataset information | |
| """ | |
| data = [] | |
| for key, dataset in self.PUBLIC_DATASETS.items(): | |
| data.append({ | |
| 'id': key, | |
| 'name': dataset.name, | |
| 'description': dataset.description[:100] + '...', | |
| 'size_mb': dataset.size_mb, | |
| 'format': dataset.format, | |
| 'features': ', '.join(dataset.features_available), | |
| 'video': '✓' if dataset.video_available else '✗', | |
| 'audio': '✓' if dataset.audio_available else '✗', | |
| 'labels': '✓' if dataset.labels_available else '✗' | |
| }) | |
| return pd.DataFrame(data) | |
| def download_dataset(self, dataset_id: str, | |
| force_download: bool = False, | |
| extract: bool = True) -> Path: | |
| """Download a dataset by ID. | |
| Args: | |
| dataset_id: Dataset identifier from PUBLIC_DATASETS | |
| force_download: Force re-download even if exists | |
| extract: Extract compressed files | |
| Returns: | |
| Path to downloaded dataset directory | |
| """ | |
| if dataset_id not in self.PUBLIC_DATASETS: | |
| raise ValueError(f"Unknown dataset: {dataset_id}. Available: {list(self.PUBLIC_DATASETS.keys())}") | |
| dataset = self.PUBLIC_DATASETS[dataset_id] | |
| dataset_dir = self.data_dir / dataset_id | |
| # Handle synthetic dataset specially | |
| if dataset_id == "synthetic_demo": | |
| return self._generate_synthetic_dataset(dataset_dir) | |
| # Check if already downloaded | |
| if dataset_dir.exists() and not force_download: | |
| print(f"Dataset '{dataset_id}' already exists at {dataset_dir}") | |
| return dataset_dir | |
| # Create directory | |
| dataset_dir.mkdir(parents=True, exist_ok=True) | |
| # Download file | |
| if dataset.url == "local": | |
| raise ValueError("Local dataset should be handled by _generate_synthetic_dataset") | |
| print(f"Downloading {dataset.name} ({dataset.size_mb} MB)...") | |
| downloaded_file = self._download_file(dataset.url, dataset_dir) | |
| # Extract if needed | |
| if extract and dataset.format in ['zip', 'tar', 'tar.gz', 'tgz']: | |
| print(f"Extracting {downloaded_file.name}...") | |
| self._extract_file(downloaded_file, dataset_dir) | |
| # Create metadata file | |
| metadata = { | |
| 'dataset_id': dataset_id, | |
| 'name': dataset.name, | |
| 'description': dataset.description, | |
| 'download_url': dataset.url, | |
| 'download_date': pd.Timestamp.now().isoformat(), | |
| 'size_mb': dataset.size_mb, | |
| 'citation': dataset.citation | |
| } | |
| metadata_path = dataset_dir / "metadata.json" | |
| with open(metadata_path, 'w') as f: | |
| json.dump(metadata, f, indent=2) | |
| print(f"✅ Dataset downloaded to {dataset_dir}") | |
| return dataset_dir | |
| def _download_file(self, url: str, output_dir: Path) -> Path: | |
| """Download a file with progress bar. | |
| Args: | |
| url: File URL | |
| output_dir: Output directory | |
| Returns: | |
| Path to downloaded file | |
| """ | |
| # Extract filename from URL | |
| parsed = urlparse(url) | |
| filename = os.path.basename(parsed.path) | |
| if not filename: | |
| filename = "downloaded_file" | |
| output_path = output_dir / filename | |
| # Stream download with progress bar | |
| response = requests.get(url, stream=True) | |
| response.raise_for_status() | |
| total_size = int(response.headers.get('content-length', 0)) | |
| with open(output_path, 'wb') as f, tqdm( | |
| desc=filename, | |
| total=total_size, | |
| unit='B', | |
| unit_scale=True, | |
| unit_divisor=1024 | |
| ) as pbar: | |
| for chunk in response.iter_content(chunk_size=8192): | |
| f.write(chunk) | |
| pbar.update(len(chunk)) | |
| return output_path | |
| def _extract_file(self, archive_path: Path, output_dir: Path): | |
| """Extract compressed file. | |
| Args: | |
| archive_path: Path to archive file | |
| output_dir: Output directory | |
| """ | |
| if archive_path.suffix == '.zip': | |
| with zipfile.ZipFile(archive_path, 'r') as zip_ref: | |
| zip_ref.extractall(output_dir) | |
| elif archive_path.suffix in ['.tar', '.tar.gz', '.tgz']: | |
| mode = 'r:gz' if archive_path.suffix in ['.tar.gz', '.tgz'] else 'r' | |
| with tarfile.open(archive_path, mode) as tar_ref: | |
| tar_ref.extractall(output_dir) | |
| else: | |
| print(f"Warning: Unknown archive format {archive_path.suffix}, skipping extraction") | |
| def _generate_synthetic_dataset(self, dataset_dir: Path) -> Path: | |
| """Generate synthetic dataset for demonstration. | |
| Args: | |
| dataset_dir: Output directory | |
| Returns: | |
| Path to dataset directory | |
| """ | |
| from .synthetic_data import SyntheticDataGenerator | |
| dataset_dir.mkdir(parents=True, exist_ok=True) | |
| print("Generating synthetic deception dataset...") | |
| generator = SyntheticDataGenerator(seed=42) | |
| # Generate and save dataset | |
| features_path, labels_path, metadata_path = generator.save_synthetic_dataset( | |
| output_dir=str(dataset_dir), | |
| n_truthful=300, | |
| n_deceptive=300, | |
| dataset_name="synthetic_deception" | |
| ) | |
| # Also generate a small sample for quick testing | |
| small_features, small_labels, _ = generator.generate_dataset( | |
| n_truthful=50, | |
| n_deceptive=50 | |
| ) | |
| small_features.to_csv(dataset_dir / "synthetic_sample_features.csv", index=False) | |
| pd.DataFrame({'label': small_labels}).to_csv(dataset_dir / "synthetic_sample_labels.csv", index=False) | |
| print(f"✅ Synthetic dataset generated at {dataset_dir}") | |
| return dataset_dir | |
| def load_dataset(self, dataset_id: str, | |
| split: Optional[str] = None) -> Tuple[pd.DataFrame, pd.DataFrame]: | |
| """Load a downloaded dataset. | |
| Args: | |
| dataset_id: Dataset identifier | |
| split: Optional split to load ('train', 'test', 'val') | |
| Returns: | |
| Tuple of (features DataFrame, labels DataFrame) | |
| """ | |
| dataset_dir = self.data_dir / dataset_id | |
| if not dataset_dir.exists(): | |
| raise FileNotFoundError(f"Dataset '{dataset_id}' not found. Download it first.") | |
| # Check for synthetic dataset | |
| if dataset_id == "synthetic_demo": | |
| features_path = dataset_dir / "synthetic_deception_features.csv" | |
| labels_path = dataset_dir / "synthetic_deception_labels.csv" | |
| if not features_path.exists(): | |
| # Generate if not exists | |
| self._generate_synthetic_dataset(dataset_dir) | |
| features = pd.read_csv(features_path) | |
| labels_df = pd.read_csv(labels_path) | |
| return features, labels_df | |
| # For real datasets, look for common file patterns | |
| csv_files = list(dataset_dir.glob("*.csv")) | |
| if not csv_files: | |
| # Try to find data in subdirectories | |
| for root, dirs, files in os.walk(dataset_dir): | |
| for file in files: | |
| if file.endswith('.csv'): | |
| csv_files.append(Path(root) / file) | |
| if not csv_files: | |
| raise FileNotFoundError(f"No CSV files found in {dataset_dir}") | |
| # Try to identify features and labels files | |
| features_file = None | |
| labels_file = None | |
| for csv_file in csv_files: | |
| name = csv_file.stem.lower() | |
| if 'feature' in name or 'data' in name or 'X' in name: | |
| features_file = csv_file | |
| elif 'label' in name or 'target' in name or 'y' in name: | |
| labels_file = csv_file | |
| # If only one file found, assume it contains both | |
| if features_file and not labels_file: | |
| df = pd.read_csv(features_file) | |
| # Try to find label column | |
| label_cols = [col for col in df.columns if 'label' in col.lower() or 'target' in col.lower()] | |
| if label_cols: | |
| labels = df[label_cols] | |
| features = df.drop(columns=label_cols) | |
| return features, labels | |
| else: | |
| raise ValueError(f"Could not identify label column in {features_file}") | |
| # Load separate files | |
| if features_file and labels_file: | |
| features = pd.read_csv(features_file) | |
| labels = pd.read_csv(labels_file) | |
| return features, labels | |
| # Fallback: use first CSV as features, assume no labels | |
| print(f"Warning: Could not identify label file, using {csv_files[0]} as features only") | |
| features = pd.read_csv(csv_files[0]) | |
| # Create dummy labels (all 0) | |
| labels = pd.DataFrame({'label': [0] * len(features)}) | |
| return features, labels | |
| def prepare_training_data(self, dataset_id: str, | |
| test_size: float = 0.2, | |
| val_size: float = 0.1, | |
| random_state: int = 42) -> Dict[str, Tuple[pd.DataFrame, pd.DataFrame]]: | |
| """Prepare dataset for training with train/val/test splits. | |
| Args: | |
| dataset_id: Dataset identifier | |
| test_size: Proportion for test set | |
| val_size: Proportion for validation set (from training set) | |
| random_state: Random seed | |
| Returns: | |
| Dictionary with 'train', 'val', 'test' splits | |
| """ | |
| from sklearn.model_selection import train_test_split | |
| features, labels_df = self.load_dataset(dataset_id) | |
| # Extract labels array | |
| if 'label' in labels_df.columns: | |
| labels = labels_df['label'].values | |
| else: | |
| # Assume first column is label | |
| labels = labels_df.iloc[:, 0].values | |
| # Split into train+val and test | |
| X_temp, X_test, y_temp, y_test = train_test_split( | |
| features, labels, test_size=test_size, random_state=random_state, stratify=labels | |
| ) | |
| # Split train into train and val | |
| val_relative_size = val_size / (1 - test_size) | |
| X_train, X_val, y_train, y_val = train_test_split( | |
| X_temp, y_temp, test_size=val_relative_size, random_state=random_state, stratify=y_temp | |
| ) | |
| # Convert back to DataFrames | |
| train_df = X_train.copy() | |
| train_df['label'] = y_train | |
| val_df = X_val.copy() | |
| val_df['label'] = y_val | |
| test_df = X_test.copy() | |
| test_df['label'] = y_test | |
| splits = { | |
| 'train': (X_train, pd.DataFrame({'label': y_train})), | |
| 'val': (X_val, pd.DataFrame({'label': y_val})), | |
| 'test': (X_test, pd.DataFrame({'label': y_test})) | |
| } | |
| # Save splits | |
| dataset_dir = self.data_dir / dataset_id | |
| train_df.to_csv(dataset_dir / "train.csv", index=False) | |
| val_df.to_csv(dataset_dir / "val.csv", index=False) | |
| test_df.to_csv(dataset_dir / "test.csv", index=False) | |
| print(f"✅ Dataset split into: Train={len(X_train)}, Val={len(X_val)}, Test={len(X_test)}") | |
| return splits | |
| def get_dataset_info(self, dataset_id: str) -> Dict: | |
| """Get detailed information about a dataset. | |
| Args: | |
| dataset_id: Dataset identifier | |
| Returns: | |
| Dictionary with dataset information | |
| """ | |
| if dataset_id not in self.PUBLIC_DATASETS: | |
| raise ValueError(f"Unknown dataset: {dataset_id}") | |
| dataset = self.PUBLIC_DATASETS[dataset_id] | |
| dataset_dir = self.data_dir / dataset_id | |
| info = { | |
| 'id': dataset_id, | |
| 'name': dataset.name, | |
| 'description': dataset.description, | |
| 'citation': dataset.citation, | |
| 'features_available': dataset.features_available, | |
| 'labels_available': dataset.labels_available, | |
| 'subject_ids_available': dataset.subject_ids_available, | |
| 'video_available': dataset.video_available, | |
| 'audio_available': dataset.audio_available, | |
| 'downloaded': dataset_dir.exists() | |
| } | |
| if dataset_dir.exists(): | |
| # Count files | |
| files = list(dataset_dir.glob("*")) | |
| csv_files = list(dataset_dir.glob("*.csv")) | |
| info['local_path'] = str(dataset_dir) | |
| info['file_count'] = len(files) | |
| info['csv_files'] = [f.name for f in csv_files] | |
| # Try to load metadata | |
| metadata_path = dataset_dir / "metadata.json" | |
| if metadata_path.exists(): | |
| with open(metadata_path, 'r') as f: | |
| info['metadata'] = json.load(f) | |
| return info |