mindscope-Model / src /dataset /collate.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
2.79 kB
from typing import Any, Dict, List
import torch
def _detect_dim(batch: List[Dict[str, Any]], key: str) -> int:
for item in batch:
value = item.get(key)
if value is not None and value.ndim == 2:
return value.shape[1]
return 0
def multimodal_collate_fn(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
batch_size = len(batch)
audio_lengths = [item["audio"].shape[0] if item["audio"] is not None else 0 for item in batch]
video_lengths = [item["video"].shape[0] if item["video"] is not None else 0 for item in batch]
text_lengths = [item["text"].shape[0] if item["text"] is not None else 0 for item in batch]
max_audio = max(audio_lengths) if audio_lengths else 0
max_video = max(video_lengths) if video_lengths else 0
max_text = max(text_lengths) if text_lengths else 0
audio_dim = _detect_dim(batch, "audio")
video_dim = _detect_dim(batch, "video")
text_dim = _detect_dim(batch, "text")
audio_tensor = torch.zeros(batch_size, max_audio, audio_dim, dtype=torch.float32)
video_tensor = torch.zeros(batch_size, max_video, video_dim, dtype=torch.float32)
text_tensor = torch.zeros(batch_size, max_text, text_dim, dtype=torch.float32)
audio_mask = torch.zeros(batch_size, max_audio, dtype=torch.bool)
video_mask = torch.zeros(batch_size, max_video, dtype=torch.bool)
text_mask = torch.zeros(batch_size, max_text, dtype=torch.bool)
phq_totals = torch.zeros(batch_size, dtype=torch.float32)
phq_questions = torch.zeros(batch_size, 8, dtype=torch.float32)
classifications = torch.zeros(batch_size, dtype=torch.float32)
participant_ids = []
for i, item in enumerate(batch):
if item["audio"] is not None:
length = item["audio"].shape[0]
audio_tensor[i, :length] = item["audio"]
audio_mask[i, :length] = True
if item["video"] is not None:
length = item["video"].shape[0]
video_tensor[i, :length] = item["video"]
video_mask[i, :length] = True
if item["text"] is not None:
length = item["text"].shape[0]
text_tensor[i, :length] = item["text"]
text_mask[i, :length] = True
phq_totals[i] = item["phq_total"]
phq_questions[i] = item["phq_questions"]
classifications[i] = item["classification"]
participant_ids.append(item["participant_id"])
return {
"audio": audio_tensor,
"video": video_tensor,
"text": text_tensor,
"audio_mask": audio_mask,
"video_mask": video_mask,
"text_mask": text_mask,
"phq_total": phq_totals,
"phq_questions": phq_questions,
"classification": classifications,
"participant_ids": participant_ids,
}