EmoSphere-PP-16kHz / modeling_emotion2vec.py
RetentionLabs's picture
Release GPTQ calibrated NVFP4 weights and processor
2f05e3a verified
Raw History Blame Contribute Delete
15.4 kB
# Copyright 2026 The emotion2vec Authors, The FunASR Authors, TICTAP-11 and b-re-w. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch emotion2vec model."""
import math
import torch
import torch.nn.functional as F
from torch import nn
from transformers import initialization as init
from transformers.modeling_outputs import SequenceClassifierOutput, Wav2Vec2BaseModelOutput
from transformers.models.data2vec.modeling_data2vec_audio import (
Data2VecAudioEncoder,
Data2VecAudioFeatureEncoder,
Data2VecAudioFeatureProjection,
Data2VecAudioPreTrainedModel,
)
from .configuration_emotion2vec import Emotion2vecConfig
__all__ = [
"Emotion2vecEncoder",
"Emotion2vecForSequenceClassification",
"Emotion2vecModel",
"Emotion2vecPreTrainedModel",
"get_alibi_slopes",
]
def get_alibi_slopes(num_heads: int) -> list[float]:
r"""
Computes the ALiBi slope of every head, as `get_alibi` of FunASR (`models/emotion2vec/base.py`) does. For $n$ a
power of two the slopes are $s_h = 2^{-8(h+1)/n}$. Otherwise the slopes of the largest power of two $m < n$ are
followed by every other slope of $2m$ heads.
Args:
num_heads (`int`):
Number of attention heads.
Returns:
`list[float]`: One slope per head.
"""
def slopes_power_of_2(n: int) -> list[float]:
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
return [start * start**index for index in range(n)]
if math.log2(num_heads).is_integer():
return slopes_power_of_2(num_heads)
closest = 2 ** math.floor(math.log2(num_heads))
return slopes_power_of_2(closest) + get_alibi_slopes(2 * closest)[0::2][: num_heads - closest]
class Emotion2vecEncoder(Data2VecAudioEncoder):
r"""
Constructs the transformer of emotion2vec. It is the [`~transformers.Data2VecAudioEncoder`] (positional
convolution added to the frames, layer norm, post norm layers) with the two parts data2vec 2.0 adds:
- `extra_tokens`, `num_extra_tokens` learned vectors put in front of the frames before the layer norm and dropped
from the output (`ModalitySpecificEncoder.contextualized_features` and `Data2VecMultiModel.forward` with
`remove_extra_tokens=True`).
- An ALiBi bias added to every attention score: head $h$ adds $-\max(a_h, 0)\,s_h\,\lvert i-j\rvert$ between
frames $i$ and $j$, with the slopes $s_h$ of [`get_alibi_slopes`] and a learned scale $a_h$ (`alibi_scale`),
and adds 0 on every pair that involves an extra token.
FunASR projects query, key and value with one fused `qkv` layer (`AltAttention`). The three projections are split
into the `q_proj`, `k_proj` and `v_proj` of [`~transformers.Data2VecAudioAttention`] by the weight conversion,
which gives the same scores. The 4 prenet layers and the 8 layers after them have the same form and are one stack.
Args:
config ([`Emotion2vecConfig`]):
Model configuration.
"""
def __init__(self, config: Emotion2vecConfig):
super().__init__(config)
self.extra_tokens = nn.Parameter(torch.zeros(1, config.num_extra_tokens, config.hidden_size))
self.alibi_scale = nn.Parameter(torch.full((1, config.num_attention_heads, 1, 1), float(config.alibi_scale)))
def get_attention_bias(self, batch_size: int, num_frames: int, dtype: torch.dtype) -> torch.Tensor:
r"""
Builds the ALiBi bias over the extra tokens and the frames.
Args:
batch_size (`int`):
Batch size.
num_frames (`int`):
Frames per clip, without the extra tokens.
dtype (`torch.dtype`):
Precision of the returned bias.
Returns:
`torch.Tensor` of shape `(batch_size, num_heads, num_extra_tokens + num_frames, num_extra_tokens +
num_frames)`: The bias, zero on every row and column of an extra token.
"""
device = self.alibi_scale.device
positions = torch.arange(num_frames, device=device)
distance = -(positions.unsqueeze(0) - positions.unsqueeze(1)).abs().float()
slopes = torch.tensor(get_alibi_slopes(self.config.num_attention_heads), device=device)
# FunASR builds the bias in float32 and multiplies it by the clamped scale, in that order.
bias = slopes.view(-1, 1, 1) * distance.unsqueeze(0)
bias = bias.unsqueeze(0) * self.alibi_scale.clamp_min(0).float()
num_extra = self.extra_tokens.shape[1]
bias = F.pad(bias, (num_extra, 0, num_extra, 0))
return bias.expand(batch_size, -1, -1, -1).to(dtype)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
output_hidden_states: bool = False,
**kwargs,
) -> tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]:
r"""
Args:
hidden_states (`torch.Tensor` of shape `(batch_size, num_frames, hidden_size)`):
Projected convolutional features.
attention_mask (`torch.BoolTensor` of shape `(batch_size, num_frames)`, *optional*):
True on frames, False on padding. Padded frames are masked as keys, as FunASR's `padding_mask` does.
output_hidden_states (`bool`, *optional*, defaults to `False`):
Whether to return the input of every layer and the last output, without the extra tokens.
Returns:
`tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]`: The output frames without the extra tokens, and
the hidden states when asked for.
"""
batch_size, num_frames, _ = hidden_states.shape
num_extra = self.extra_tokens.shape[1]
hidden_states = hidden_states + self.pos_conv_embed(hidden_states)
hidden_states = torch.cat([self.extra_tokens.expand(batch_size, -1, -1), hidden_states], dim=1)
hidden_states = self.layer_norm(hidden_states)
hidden_states = self.dropout(hidden_states)
bias = self.get_attention_bias(batch_size, num_frames, hidden_states.dtype)
if attention_mask is not None and not attention_mask.all():
key_mask = F.pad(attention_mask.bool(), (num_extra, 0), value=True)
bias = bias.masked_fill(~key_mask[:, None, None, :], torch.finfo(bias.dtype).min)
all_hidden_states = () if output_hidden_states else None
for layer in self.layers:
if output_hidden_states:
all_hidden_states += (hidden_states[:, num_extra:],)
if self.training and torch.rand([]) < self.config.layerdrop:
continue
hidden_states = layer(hidden_states, attention_mask=bias, **kwargs)
hidden_states = hidden_states[:, num_extra:]
if output_hidden_states:
all_hidden_states += (hidden_states,)
return hidden_states, all_hidden_states
class Emotion2vecPreTrainedModel(Data2VecAudioPreTrainedModel):
config: Emotion2vecConfig
base_model_prefix = "emotion2vec"
# The ALiBi bias is an additive float mask, which flash and flex attention do not take.
_supports_flash_attn = False
_supports_flex_attn = False
@torch.no_grad()
def _init_weights(self, module: nn.Module) -> None:
super()._init_weights(module)
if isinstance(module, Emotion2vecEncoder):
init.zeros_(module.extra_tokens)
init.constant_(module.alibi_scale, self.config.alibi_scale)
class Emotion2vecModel(Emotion2vecPreTrainedModel):
r"""
Constructs the emotion2vec encoder (Ma et al., *emotion2vec: Self-Supervised Pre-Training for Speech Emotion
Representation*, arXiv:2312.15185) as FunASR runs it (`funasr/models/emotion2vec/model.py`, `Emotion2vec`). Its
output frames are the `feats` FunASR returns with `granularity="frame"`, 50 per second of 16 kHz audio.
The waveform is layer normalized over the clip ([`~Emotion2vecModel.normalize_input`]), then goes through the
convolutional feature encoder and feature projection of data2vec audio
([`~transformers.models.data2vec.modeling_data2vec_audio.Data2VecAudioFeatureEncoder`] and
[`~transformers.models.data2vec.modeling_data2vec_audio.Data2VecAudioFeatureProjection`], which are FunASR's
`ConvFeatureExtractionModel` in `layer_norm` mode and `project_features`), and through [`Emotion2vecEncoder`].
Args:
config ([`Emotion2vecConfig`]):
Model configuration.
"""
def __init__(self, config: Emotion2vecConfig):
super().__init__(config)
self.feature_extractor = Data2VecAudioFeatureEncoder(config)
self.feature_projection = Data2VecAudioFeatureProjection(config)
self.encoder = Emotion2vecEncoder(config)
self.post_init()
def freeze_feature_encoder(self) -> None:
"""Disables the gradient of the convolutional feature encoder."""
self.feature_extractor._freeze_parameters()
def normalize_input(self, input_values: torch.Tensor, attention_mask: torch.Tensor | None = None) -> torch.Tensor:
r"""
Layer normalizes each waveform over its own samples, $(x-\mu)/\sqrt{\sigma^2+10^{-5}}$ with the biased
variance, as `F.layer_norm(source, source.shape)` in `Emotion2vec.inference` does. Padding is left out of the
statistics and set to zero.
Args:
input_values (`torch.Tensor` of shape `(batch_size, sequence_length)`):
16 kHz waveforms.
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
1 on samples, 0 on padding.
Returns:
`torch.Tensor`: The normalized waveforms.
"""
if attention_mask is None:
return F.layer_norm(input_values, input_values.shape[-1:])
mask = attention_mask.to(input_values.dtype)
count = mask.sum(-1, keepdim=True)
mean = (input_values * mask).sum(-1, keepdim=True) / count
variance = (((input_values - mean) * mask) ** 2).sum(-1, keepdim=True) / count
return (input_values - mean) / torch.sqrt(variance + 1e-5) * mask
def forward(
self,
input_values: torch.Tensor,
attention_mask: torch.Tensor | None = None,
output_hidden_states: bool = False,
**kwargs,
) -> Wav2Vec2BaseModelOutput:
r"""
Args:
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
16 kHz waveforms in $[-1, 1]$, not normalized.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1 on samples, 0 on padding. FunASR runs one clip at a time without it.
output_hidden_states (`bool`, *optional*, defaults to `False`):
Whether to return the hidden states of every layer.
Returns:
[`~transformers.modeling_outputs.Wav2Vec2BaseModelOutput`]: `last_hidden_state` holds the frame
features, `extract_features` the convolutional features before the projection.
"""
if self.config.normalize_input:
input_values = self.normalize_input(input_values, attention_mask)
extract_features = self.feature_extractor(input_values).transpose(1, 2)
frame_mask = None
if attention_mask is not None:
frame_mask = self._get_feature_vector_attention_mask(
extract_features.shape[1], attention_mask, add_adapter=False
)
hidden_states, extract_features = self.feature_projection(extract_features)
hidden_states, all_hidden_states = self.encoder(
hidden_states, attention_mask=frame_mask, output_hidden_states=output_hidden_states, **kwargs
)
return Wav2Vec2BaseModelOutput(
last_hidden_state=hidden_states,
extract_features=extract_features,
hidden_states=all_hidden_states,
)
class Emotion2vecForSequenceClassification(Emotion2vecPreTrainedModel):
r"""
Constructs emotion2vec with the utterance emotion classifier of the fine-tuned checkpoints: the frame features are
averaged over time and projected to `num_labels` logits (`proj` in FunASR's `Emotion2vec`). The 9 labels of
`emotion2vec_base_finetuned` are angry, disgusted, fearful, happy, neutral, other, sad, surprised and `<unk>`.
Args:
config ([`Emotion2vecConfig`]):
Model configuration.
"""
def __init__(self, config: Emotion2vecConfig):
super().__init__(config)
self.emotion2vec = Emotion2vecModel(config)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.post_init()
def freeze_feature_encoder(self) -> None:
"""Disables the gradient of the convolutional feature encoder."""
self.emotion2vec.freeze_feature_encoder()
def forward(
self,
input_values: torch.Tensor,
attention_mask: torch.Tensor | None = None,
labels: torch.Tensor | None = None,
output_hidden_states: bool = False,
**kwargs,
) -> SequenceClassifierOutput:
r"""
Args:
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
16 kHz waveforms in $[-1, 1]$, not normalized.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1 on samples, 0 on padding.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Emotion class indices. When given, the cross entropy of the logits is returned.
output_hidden_states (`bool`, *optional*, defaults to `False`):
Whether to return the hidden states of every layer.
Returns:
[`~transformers.modeling_outputs.SequenceClassifierOutput`]: The logits, FunASR's `scores` before the
softmax, and the loss when `labels` is given.
"""
outputs = self.emotion2vec(
input_values, attention_mask=attention_mask, output_hidden_states=output_hidden_states, **kwargs
)
hidden_states = outputs.last_hidden_state
if attention_mask is None:
pooled_output = hidden_states.mean(dim=1)
else:
frame_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask)
weights = frame_mask.unsqueeze(-1).to(hidden_states.dtype)
pooled_output = (hidden_states * weights).sum(dim=1) / weights.sum(dim=1)
logits = self.classifier(pooled_output)
loss = None
if labels is not None:
loss = F.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1).to(logits.device))
return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states)