mindscope-Model / src /models /temporal_modules.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
9.66 kB
"""Improved temporal modeling architectures optimized for small datasets.
These architectures prioritize:
- Generalization with limited data
- Stable convergence
- Parameter efficiency
- Temporal coherence
"""
from __future__ import annotations
import torch
import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
class LightweightTemporalCNN(nn.Module):
"""Lightweight temporal CNN for small datasets.
Uses depthwise-separable convolutions to reduce parameters.
"""
def __init__(
self,
input_dim: int,
hidden_dim: int,
num_layers: int = 2,
kernel_size: int = 3,
dropout: float = 0.2,
):
super().__init__()
self.input_norm = nn.LayerNorm(input_dim)
layers = []
in_channels = input_dim
out_channels = hidden_dim
for i in range(num_layers):
# Depthwise-separable convolution
layers.append(nn.Sequential(
# Depthwise: apply 1 filter per input channel
nn.Conv1d(
in_channels,
in_channels,
kernel_size=kernel_size,
padding=kernel_size // 2,
groups=in_channels,
bias=False,
),
# Pointwise: 1x1 convolution
nn.Conv1d(in_channels, out_channels, kernel_size=1),
nn.GELU(),
nn.Dropout(dropout),
))
in_channels = out_channels
self.conv_stack = nn.Sequential(*layers)
self.output_proj = nn.Linear(hidden_dim, hidden_dim)
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Process temporal features.
Args:
x: [batch_size, time_steps, input_dim]
mask: [batch_size, time_steps]
Returns:
[batch_size, time_steps, hidden_dim]
"""
x = self.input_norm(x)
x = x.transpose(1, 2) # [batch, input_dim, time]
x = self.conv_stack(x)
x = x.transpose(1, 2) # [batch, time, hidden_dim]
x = self.output_proj(x)
return x * mask.unsqueeze(-1).float()
class StableGRUEncoder(nn.Module):
"""Stable GRU encoder for small datasets.
Uses layer normalization and careful initialization for better convergence.
"""
def __init__(
self,
input_dim: int,
hidden_dim: int,
num_layers: int = 1,
dropout: float = 0.2,
bidirectional: bool = True,
):
super().__init__()
self.input_norm = nn.LayerNorm(input_dim)
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
self.gru = nn.GRU(
input_dim,
hidden_dim,
num_layers=num_layers,
batch_first=True,
bidirectional=bidirectional,
dropout=dropout if num_layers > 1 else 0.0,
)
# Projection for bidirectional output
if bidirectional:
self.output_proj = nn.Sequential(
nn.Linear(hidden_dim * 2, hidden_dim),
nn.LayerNorm(hidden_dim),
)
else:
self.output_proj = nn.Identity()
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Process temporal features.
Args:
x: [batch_size, time_steps, input_dim]
mask: [batch_size, time_steps]
Returns:
[batch_size, time_steps, hidden_dim]
"""
x = self.input_norm(x)
lengths = mask.sum(dim=1).cpu()
lengths_clamped = lengths.clamp(min=1)
# Pack for efficient RNN processing
packed = pack_padded_sequence(x, lengths_clamped, batch_first=True, enforce_sorted=False)
packed_out, _ = self.gru(packed)
out, _ = pad_packed_sequence(packed_out, batch_first=True, total_length=x.size(1))
# Project if bidirectional
out = self.output_proj(out)
# Apply mask
return out * mask.unsqueeze(-1).float()
class HybridTemporalModule(nn.Module):
"""Hybrid CNN+GRU for optimal temporal modeling.
Combines CNN for local pattern detection with GRU for long-range dependencies.
Optimized for small datasets.
"""
def __init__(
self,
input_dim: int,
hidden_dim: int,
dropout: float = 0.2,
):
super().__init__()
self.input_norm = nn.LayerNorm(input_dim)
# Light CNN for local patterns
self.conv = nn.Sequential(
nn.Conv1d(input_dim, hidden_dim, kernel_size=3, padding=1),
nn.GELU(),
nn.Dropout(dropout),
)
# GRU for temporal context
self.gru = nn.GRU(
hidden_dim,
hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True,
dropout=0.0,
)
# Output projection
self.proj = nn.Sequential(
nn.Linear(hidden_dim * 2, hidden_dim),
nn.LayerNorm(hidden_dim),
)
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Process temporal features.
Args:
x: [batch_size, time_steps, input_dim]
mask: [batch_size, time_steps]
Returns:
[batch_size, time_steps, hidden_dim]
"""
x = self.input_norm(x)
# CNN: local pattern extraction
x_t = x.transpose(1, 2) # [batch, input_dim, time]
x_conv = self.conv(x_t).transpose(1, 2) # [batch, time, hidden_dim]
# GRU: temporal context
lengths = mask.sum(dim=1).cpu()
lengths_clamped = lengths.clamp(min=1)
packed = pack_padded_sequence(x_conv, lengths_clamped, batch_first=True, enforce_sorted=False)
packed_out, _ = self.gru(packed)
x_gru, _ = pad_packed_sequence(packed_out, batch_first=True, total_length=x.size(1))
# Project and mask
out = self.proj(x_gru)
return out * mask.unsqueeze(-1).float()
class AttentionPooling(nn.Module):
"""Learnable attention pooling for temporal aggregation.
Better than mean pooling for capturing important temporal elements.
"""
def __init__(self, feature_dim: int, dropout: float = 0.1):
super().__init__()
self.query = nn.Parameter(torch.randn(feature_dim))
self.proj = nn.Sequential(
nn.Linear(feature_dim, feature_dim),
nn.GELU(),
nn.Dropout(dropout),
)
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Apply attention pooling.
Args:
x: [batch_size, time_steps, feature_dim]
mask: [batch_size, time_steps]
Returns:
Pooled output [batch_size, feature_dim]
"""
# Compute attention scores
attn = self.proj(x) # [batch, time, feature_dim]
scores = torch.matmul(attn, self.query) # [batch, time]
# Apply mask
scores = scores.masked_fill(~mask, float("-1e9"))
# Softmax
weights = torch.softmax(scores, dim=1) # [batch, time]
# Weighted sum
return (x * weights.unsqueeze(-1)).sum(dim=1) # [batch, feature_dim]
class StatisticsPooling(nn.Module):
"""Statistics-based temporal pooling (mean + std).
Preserves information about feature variation which can be important
for behavioral signals.
"""
def __init__(self, feature_dim: int):
super().__init__()
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Compute statistics over time.
Args:
x: [batch_size, time_steps, feature_dim]
mask: [batch_size, time_steps]
Returns:
Concatenated mean and std [batch_size, 2 * feature_dim]
"""
# Apply mask
x_masked = x * mask.unsqueeze(-1).float()
# Compute mean
lengths = mask.sum(dim=1, keepdim=True).clamp(min=1)
mean = x_masked.sum(dim=1) / lengths # [batch, feature_dim]
# Compute std
x_centered = (x - mean.unsqueeze(1)) * mask.unsqueeze(-1).float()
std = torch.sqrt((x_centered ** 2).sum(dim=1) / lengths + 1e-8) # [batch, feature_dim]
# Concatenate
return torch.cat([mean, std], dim=-1) # [batch, 2 * feature_dim]
class TemporalAttentionAggregation(nn.Module):
"""Multi-head attention for temporal aggregation.
More expressive than single-head attention but still parameter-efficient.
"""
def __init__(
self,
feature_dim: int,
num_heads: int = 4,
dropout: float = 0.1,
):
super().__init__()
self.attention = nn.MultiheadAttention(
embed_dim=feature_dim,
num_heads=num_heads,
dropout=dropout,
batch_first=True,
)
self.norm = nn.LayerNorm(feature_dim)
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Apply temporal self-attention.
Args:
x: [batch_size, time_steps, feature_dim]
mask: [batch_size, time_steps]
Returns:
Aggregated output [batch_size, feature_dim]
"""
# Self-attention
src_key_padding_mask = ~mask
attn_out, _ = self.attention(x, x, x, key_padding_mask=src_key_padding_mask)
attn_out = self.norm(attn_out + x)
# Mean pooling
return (attn_out * mask.unsqueeze(-1).float()).sum(dim=1) / mask.sum(dim=1, keepdim=True).clamp(min=1)