# Copyright 2026 The emotion2vec Authors, The FunASR Authors, TICTAP-11 and b-re-w. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """PyTorch emotion2vec model.""" import math import torch import torch.nn.functional as F from torch import nn from transformers import initialization as init from transformers.modeling_outputs import SequenceClassifierOutput, Wav2Vec2BaseModelOutput from transformers.models.data2vec.modeling_data2vec_audio import ( Data2VecAudioEncoder, Data2VecAudioFeatureEncoder, Data2VecAudioFeatureProjection, Data2VecAudioPreTrainedModel, ) from .configuration_emotion2vec import Emotion2vecConfig __all__ = [ "Emotion2vecEncoder", "Emotion2vecForSequenceClassification", "Emotion2vecModel", "Emotion2vecPreTrainedModel", "get_alibi_slopes", ] def get_alibi_slopes(num_heads: int) -> list[float]: r""" Computes the ALiBi slope of every head, as `get_alibi` of FunASR (`models/emotion2vec/base.py`) does. For $n$ a power of two the slopes are $s_h = 2^{-8(h+1)/n}$. Otherwise the slopes of the largest power of two $m < n$ are followed by every other slope of $2m$ heads. Args: num_heads (`int`): Number of attention heads. Returns: `list[float]`: One slope per head. """ def slopes_power_of_2(n: int) -> list[float]: start = 2 ** (-(2 ** -(math.log2(n) - 3))) return [start * start**index for index in range(n)] if math.log2(num_heads).is_integer(): return slopes_power_of_2(num_heads) closest = 2 ** math.floor(math.log2(num_heads)) return slopes_power_of_2(closest) + get_alibi_slopes(2 * closest)[0::2][: num_heads - closest] class Emotion2vecEncoder(Data2VecAudioEncoder): r""" Constructs the transformer of emotion2vec. It is the [`~transformers.Data2VecAudioEncoder`] (positional convolution added to the frames, layer norm, post norm layers) with the two parts data2vec 2.0 adds: - `extra_tokens`, `num_extra_tokens` learned vectors put in front of the frames before the layer norm and dropped from the output (`ModalitySpecificEncoder.contextualized_features` and `Data2VecMultiModel.forward` with `remove_extra_tokens=True`). - An ALiBi bias added to every attention score: head $h$ adds $-\max(a_h, 0)\,s_h\,\lvert i-j\rvert$ between frames $i$ and $j$, with the slopes $s_h$ of [`get_alibi_slopes`] and a learned scale $a_h$ (`alibi_scale`), and adds 0 on every pair that involves an extra token. FunASR projects query, key and value with one fused `qkv` layer (`AltAttention`). The three projections are split into the `q_proj`, `k_proj` and `v_proj` of [`~transformers.Data2VecAudioAttention`] by the weight conversion, which gives the same scores. The 4 prenet layers and the 8 layers after them have the same form and are one stack. Args: config ([`Emotion2vecConfig`]): Model configuration. """ def __init__(self, config: Emotion2vecConfig): super().__init__(config) self.extra_tokens = nn.Parameter(torch.zeros(1, config.num_extra_tokens, config.hidden_size)) self.alibi_scale = nn.Parameter(torch.full((1, config.num_attention_heads, 1, 1), float(config.alibi_scale))) def get_attention_bias(self, batch_size: int, num_frames: int, dtype: torch.dtype) -> torch.Tensor: r""" Builds the ALiBi bias over the extra tokens and the frames. Args: batch_size (`int`): Batch size. num_frames (`int`): Frames per clip, without the extra tokens. dtype (`torch.dtype`): Precision of the returned bias. Returns: `torch.Tensor` of shape `(batch_size, num_heads, num_extra_tokens + num_frames, num_extra_tokens + num_frames)`: The bias, zero on every row and column of an extra token. """ device = self.alibi_scale.device positions = torch.arange(num_frames, device=device) distance = -(positions.unsqueeze(0) - positions.unsqueeze(1)).abs().float() slopes = torch.tensor(get_alibi_slopes(self.config.num_attention_heads), device=device) # FunASR builds the bias in float32 and multiplies it by the clamped scale, in that order. bias = slopes.view(-1, 1, 1) * distance.unsqueeze(0) bias = bias.unsqueeze(0) * self.alibi_scale.clamp_min(0).float() num_extra = self.extra_tokens.shape[1] bias = F.pad(bias, (num_extra, 0, num_extra, 0)) return bias.expand(batch_size, -1, -1, -1).to(dtype) def forward( self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, output_hidden_states: bool = False, **kwargs, ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]: r""" Args: hidden_states (`torch.Tensor` of shape `(batch_size, num_frames, hidden_size)`): Projected convolutional features. attention_mask (`torch.BoolTensor` of shape `(batch_size, num_frames)`, *optional*): True on frames, False on padding. Padded frames are masked as keys, as FunASR's `padding_mask` does. output_hidden_states (`bool`, *optional*, defaults to `False`): Whether to return the input of every layer and the last output, without the extra tokens. Returns: `tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]`: The output frames without the extra tokens, and the hidden states when asked for. """ batch_size, num_frames, _ = hidden_states.shape num_extra = self.extra_tokens.shape[1] hidden_states = hidden_states + self.pos_conv_embed(hidden_states) hidden_states = torch.cat([self.extra_tokens.expand(batch_size, -1, -1), hidden_states], dim=1) hidden_states = self.layer_norm(hidden_states) hidden_states = self.dropout(hidden_states) bias = self.get_attention_bias(batch_size, num_frames, hidden_states.dtype) if attention_mask is not None and not attention_mask.all(): key_mask = F.pad(attention_mask.bool(), (num_extra, 0), value=True) bias = bias.masked_fill(~key_mask[:, None, None, :], torch.finfo(bias.dtype).min) all_hidden_states = () if output_hidden_states else None for layer in self.layers: if output_hidden_states: all_hidden_states += (hidden_states[:, num_extra:],) if self.training and torch.rand([]) < self.config.layerdrop: continue hidden_states = layer(hidden_states, attention_mask=bias, **kwargs) hidden_states = hidden_states[:, num_extra:] if output_hidden_states: all_hidden_states += (hidden_states,) return hidden_states, all_hidden_states class Emotion2vecPreTrainedModel(Data2VecAudioPreTrainedModel): config: Emotion2vecConfig base_model_prefix = "emotion2vec" # The ALiBi bias is an additive float mask, which flash and flex attention do not take. _supports_flash_attn = False _supports_flex_attn = False @torch.no_grad() def _init_weights(self, module: nn.Module) -> None: super()._init_weights(module) if isinstance(module, Emotion2vecEncoder): init.zeros_(module.extra_tokens) init.constant_(module.alibi_scale, self.config.alibi_scale) class Emotion2vecModel(Emotion2vecPreTrainedModel): r""" Constructs the emotion2vec encoder (Ma et al., *emotion2vec: Self-Supervised Pre-Training for Speech Emotion Representation*, arXiv:2312.15185) as FunASR runs it (`funasr/models/emotion2vec/model.py`, `Emotion2vec`). Its output frames are the `feats` FunASR returns with `granularity="frame"`, 50 per second of 16 kHz audio. The waveform is layer normalized over the clip ([`~Emotion2vecModel.normalize_input`]), then goes through the convolutional feature encoder and feature projection of data2vec audio ([`~transformers.models.data2vec.modeling_data2vec_audio.Data2VecAudioFeatureEncoder`] and [`~transformers.models.data2vec.modeling_data2vec_audio.Data2VecAudioFeatureProjection`], which are FunASR's `ConvFeatureExtractionModel` in `layer_norm` mode and `project_features`), and through [`Emotion2vecEncoder`]. Args: config ([`Emotion2vecConfig`]): Model configuration. """ def __init__(self, config: Emotion2vecConfig): super().__init__(config) self.feature_extractor = Data2VecAudioFeatureEncoder(config) self.feature_projection = Data2VecAudioFeatureProjection(config) self.encoder = Emotion2vecEncoder(config) self.post_init() def freeze_feature_encoder(self) -> None: """Disables the gradient of the convolutional feature encoder.""" self.feature_extractor._freeze_parameters() def normalize_input(self, input_values: torch.Tensor, attention_mask: torch.Tensor | None = None) -> torch.Tensor: r""" Layer normalizes each waveform over its own samples, $(x-\mu)/\sqrt{\sigma^2+10^{-5}}$ with the biased variance, as `F.layer_norm(source, source.shape)` in `Emotion2vec.inference` does. Padding is left out of the statistics and set to zero. Args: input_values (`torch.Tensor` of shape `(batch_size, sequence_length)`): 16 kHz waveforms. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): 1 on samples, 0 on padding. Returns: `torch.Tensor`: The normalized waveforms. """ if attention_mask is None: return F.layer_norm(input_values, input_values.shape[-1:]) mask = attention_mask.to(input_values.dtype) count = mask.sum(-1, keepdim=True) mean = (input_values * mask).sum(-1, keepdim=True) / count variance = (((input_values - mean) * mask) ** 2).sum(-1, keepdim=True) / count return (input_values - mean) / torch.sqrt(variance + 1e-5) * mask def forward( self, input_values: torch.Tensor, attention_mask: torch.Tensor | None = None, output_hidden_states: bool = False, **kwargs, ) -> Wav2Vec2BaseModelOutput: r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): 16 kHz waveforms in $[-1, 1]$, not normalized. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): 1 on samples, 0 on padding. FunASR runs one clip at a time without it. output_hidden_states (`bool`, *optional*, defaults to `False`): Whether to return the hidden states of every layer. Returns: [`~transformers.modeling_outputs.Wav2Vec2BaseModelOutput`]: `last_hidden_state` holds the frame features, `extract_features` the convolutional features before the projection. """ if self.config.normalize_input: input_values = self.normalize_input(input_values, attention_mask) extract_features = self.feature_extractor(input_values).transpose(1, 2) frame_mask = None if attention_mask is not None: frame_mask = self._get_feature_vector_attention_mask( extract_features.shape[1], attention_mask, add_adapter=False ) hidden_states, extract_features = self.feature_projection(extract_features) hidden_states, all_hidden_states = self.encoder( hidden_states, attention_mask=frame_mask, output_hidden_states=output_hidden_states, **kwargs ) return Wav2Vec2BaseModelOutput( last_hidden_state=hidden_states, extract_features=extract_features, hidden_states=all_hidden_states, ) class Emotion2vecForSequenceClassification(Emotion2vecPreTrainedModel): r""" Constructs emotion2vec with the utterance emotion classifier of the fine-tuned checkpoints: the frame features are averaged over time and projected to `num_labels` logits (`proj` in FunASR's `Emotion2vec`). The 9 labels of `emotion2vec_base_finetuned` are angry, disgusted, fearful, happy, neutral, other, sad, surprised and ``. Args: config ([`Emotion2vecConfig`]): Model configuration. """ def __init__(self, config: Emotion2vecConfig): super().__init__(config) self.emotion2vec = Emotion2vecModel(config) self.classifier = nn.Linear(config.hidden_size, config.num_labels) self.post_init() def freeze_feature_encoder(self) -> None: """Disables the gradient of the convolutional feature encoder.""" self.emotion2vec.freeze_feature_encoder() def forward( self, input_values: torch.Tensor, attention_mask: torch.Tensor | None = None, labels: torch.Tensor | None = None, output_hidden_states: bool = False, **kwargs, ) -> SequenceClassifierOutput: r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): 16 kHz waveforms in $[-1, 1]$, not normalized. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): 1 on samples, 0 on padding. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Emotion class indices. When given, the cross entropy of the logits is returned. output_hidden_states (`bool`, *optional*, defaults to `False`): Whether to return the hidden states of every layer. Returns: [`~transformers.modeling_outputs.SequenceClassifierOutput`]: The logits, FunASR's `scores` before the softmax, and the loss when `labels` is given. """ outputs = self.emotion2vec( input_values, attention_mask=attention_mask, output_hidden_states=output_hidden_states, **kwargs ) hidden_states = outputs.last_hidden_state if attention_mask is None: pooled_output = hidden_states.mean(dim=1) else: frame_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask) weights = frame_mask.unsqueeze(-1).to(hidden_states.dtype) pooled_output = (hidden_states * weights).sum(dim=1) / weights.sum(dim=1) logits = self.classifier(pooled_output) loss = None if labels is not None: loss = F.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1).to(logits.device)) return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states)