Text-to-Speech
Transformers
Safetensors
English
emosphere_pp
feature-extraction
emotional-tts
zero-shot-tts
flow-matching
emosphere
custom_code
Instructions to use RetentionLabs/EmoSphere-PP-16kHz with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use RetentionLabs/EmoSphere-PP-16kHz with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-to-speech", model="RetentionLabs/EmoSphere-PP-16kHz", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("RetentionLabs/EmoSphere-PP-16kHz", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download modeling_emotion2vec.py from RetentionLabs/EmoSphere-PP-16kHz: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/RetentionLabs/EmoSphere-PP-16kHz/resolve/main/modeling_emotion2vec.py
- Command line
-
hf download hf://RetentionLabs/EmoSphere-PP-16kHz/modeling_emotion2vec.py
-
curl -L -o modeling_emotion2vec.py https://huggingface.co/RetentionLabs/EmoSphere-PP-16kHz/resolve/main/modeling_emotion2vec.py
15.4 kB
| # Copyright 2026 The emotion2vec Authors, The FunASR Authors, TICTAP-11 and b-re-w. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """PyTorch emotion2vec model.""" | |
| import math | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from transformers import initialization as init | |
| from transformers.modeling_outputs import SequenceClassifierOutput, Wav2Vec2BaseModelOutput | |
| from transformers.models.data2vec.modeling_data2vec_audio import ( | |
| Data2VecAudioEncoder, | |
| Data2VecAudioFeatureEncoder, | |
| Data2VecAudioFeatureProjection, | |
| Data2VecAudioPreTrainedModel, | |
| ) | |
| from .configuration_emotion2vec import Emotion2vecConfig | |
| __all__ = [ | |
| "Emotion2vecEncoder", | |
| "Emotion2vecForSequenceClassification", | |
| "Emotion2vecModel", | |
| "Emotion2vecPreTrainedModel", | |
| "get_alibi_slopes", | |
| ] | |
| def get_alibi_slopes(num_heads: int) -> list[float]: | |
| r""" | |
| Computes the ALiBi slope of every head, as `get_alibi` of FunASR (`models/emotion2vec/base.py`) does. For $n$ a | |
| power of two the slopes are $s_h = 2^{-8(h+1)/n}$. Otherwise the slopes of the largest power of two $m < n$ are | |
| followed by every other slope of $2m$ heads. | |
| Args: | |
| num_heads (`int`): | |
| Number of attention heads. | |
| Returns: | |
| `list[float]`: One slope per head. | |
| """ | |
| def slopes_power_of_2(n: int) -> list[float]: | |
| start = 2 ** (-(2 ** -(math.log2(n) - 3))) | |
| return [start * start**index for index in range(n)] | |
| if math.log2(num_heads).is_integer(): | |
| return slopes_power_of_2(num_heads) | |
| closest = 2 ** math.floor(math.log2(num_heads)) | |
| return slopes_power_of_2(closest) + get_alibi_slopes(2 * closest)[0::2][: num_heads - closest] | |
| class Emotion2vecEncoder(Data2VecAudioEncoder): | |
| r""" | |
| Constructs the transformer of emotion2vec. It is the [`~transformers.Data2VecAudioEncoder`] (positional | |
| convolution added to the frames, layer norm, post norm layers) with the two parts data2vec 2.0 adds: | |
| - `extra_tokens`, `num_extra_tokens` learned vectors put in front of the frames before the layer norm and dropped | |
| from the output (`ModalitySpecificEncoder.contextualized_features` and `Data2VecMultiModel.forward` with | |
| `remove_extra_tokens=True`). | |
| - An ALiBi bias added to every attention score: head $h$ adds $-\max(a_h, 0)\,s_h\,\lvert i-j\rvert$ between | |
| frames $i$ and $j$, with the slopes $s_h$ of [`get_alibi_slopes`] and a learned scale $a_h$ (`alibi_scale`), | |
| and adds 0 on every pair that involves an extra token. | |
| FunASR projects query, key and value with one fused `qkv` layer (`AltAttention`). The three projections are split | |
| into the `q_proj`, `k_proj` and `v_proj` of [`~transformers.Data2VecAudioAttention`] by the weight conversion, | |
| which gives the same scores. The 4 prenet layers and the 8 layers after them have the same form and are one stack. | |
| Args: | |
| config ([`Emotion2vecConfig`]): | |
| Model configuration. | |
| """ | |
| def __init__(self, config: Emotion2vecConfig): | |
| super().__init__(config) | |
| self.extra_tokens = nn.Parameter(torch.zeros(1, config.num_extra_tokens, config.hidden_size)) | |
| self.alibi_scale = nn.Parameter(torch.full((1, config.num_attention_heads, 1, 1), float(config.alibi_scale))) | |
| def get_attention_bias(self, batch_size: int, num_frames: int, dtype: torch.dtype) -> torch.Tensor: | |
| r""" | |
| Builds the ALiBi bias over the extra tokens and the frames. | |
| Args: | |
| batch_size (`int`): | |
| Batch size. | |
| num_frames (`int`): | |
| Frames per clip, without the extra tokens. | |
| dtype (`torch.dtype`): | |
| Precision of the returned bias. | |
| Returns: | |
| `torch.Tensor` of shape `(batch_size, num_heads, num_extra_tokens + num_frames, num_extra_tokens + | |
| num_frames)`: The bias, zero on every row and column of an extra token. | |
| """ | |
| device = self.alibi_scale.device | |
| positions = torch.arange(num_frames, device=device) | |
| distance = -(positions.unsqueeze(0) - positions.unsqueeze(1)).abs().float() | |
| slopes = torch.tensor(get_alibi_slopes(self.config.num_attention_heads), device=device) | |
| # FunASR builds the bias in float32 and multiplies it by the clamped scale, in that order. | |
| bias = slopes.view(-1, 1, 1) * distance.unsqueeze(0) | |
| bias = bias.unsqueeze(0) * self.alibi_scale.clamp_min(0).float() | |
| num_extra = self.extra_tokens.shape[1] | |
| bias = F.pad(bias, (num_extra, 0, num_extra, 0)) | |
| return bias.expand(batch_size, -1, -1, -1).to(dtype) | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| output_hidden_states: bool = False, | |
| **kwargs, | |
| ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]: | |
| r""" | |
| Args: | |
| hidden_states (`torch.Tensor` of shape `(batch_size, num_frames, hidden_size)`): | |
| Projected convolutional features. | |
| attention_mask (`torch.BoolTensor` of shape `(batch_size, num_frames)`, *optional*): | |
| True on frames, False on padding. Padded frames are masked as keys, as FunASR's `padding_mask` does. | |
| output_hidden_states (`bool`, *optional*, defaults to `False`): | |
| Whether to return the input of every layer and the last output, without the extra tokens. | |
| Returns: | |
| `tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]`: The output frames without the extra tokens, and | |
| the hidden states when asked for. | |
| """ | |
| batch_size, num_frames, _ = hidden_states.shape | |
| num_extra = self.extra_tokens.shape[1] | |
| hidden_states = hidden_states + self.pos_conv_embed(hidden_states) | |
| hidden_states = torch.cat([self.extra_tokens.expand(batch_size, -1, -1), hidden_states], dim=1) | |
| hidden_states = self.layer_norm(hidden_states) | |
| hidden_states = self.dropout(hidden_states) | |
| bias = self.get_attention_bias(batch_size, num_frames, hidden_states.dtype) | |
| if attention_mask is not None and not attention_mask.all(): | |
| key_mask = F.pad(attention_mask.bool(), (num_extra, 0), value=True) | |
| bias = bias.masked_fill(~key_mask[:, None, None, :], torch.finfo(bias.dtype).min) | |
| all_hidden_states = () if output_hidden_states else None | |
| for layer in self.layers: | |
| if output_hidden_states: | |
| all_hidden_states += (hidden_states[:, num_extra:],) | |
| if self.training and torch.rand([]) < self.config.layerdrop: | |
| continue | |
| hidden_states = layer(hidden_states, attention_mask=bias, **kwargs) | |
| hidden_states = hidden_states[:, num_extra:] | |
| if output_hidden_states: | |
| all_hidden_states += (hidden_states,) | |
| return hidden_states, all_hidden_states | |
| class Emotion2vecPreTrainedModel(Data2VecAudioPreTrainedModel): | |
| config: Emotion2vecConfig | |
| base_model_prefix = "emotion2vec" | |
| # The ALiBi bias is an additive float mask, which flash and flex attention do not take. | |
| _supports_flash_attn = False | |
| _supports_flex_attn = False | |
| def _init_weights(self, module: nn.Module) -> None: | |
| super()._init_weights(module) | |
| if isinstance(module, Emotion2vecEncoder): | |
| init.zeros_(module.extra_tokens) | |
| init.constant_(module.alibi_scale, self.config.alibi_scale) | |
| class Emotion2vecModel(Emotion2vecPreTrainedModel): | |
| r""" | |
| Constructs the emotion2vec encoder (Ma et al., *emotion2vec: Self-Supervised Pre-Training for Speech Emotion | |
| Representation*, arXiv:2312.15185) as FunASR runs it (`funasr/models/emotion2vec/model.py`, `Emotion2vec`). Its | |
| output frames are the `feats` FunASR returns with `granularity="frame"`, 50 per second of 16 kHz audio. | |
| The waveform is layer normalized over the clip ([`~Emotion2vecModel.normalize_input`]), then goes through the | |
| convolutional feature encoder and feature projection of data2vec audio | |
| ([`~transformers.models.data2vec.modeling_data2vec_audio.Data2VecAudioFeatureEncoder`] and | |
| [`~transformers.models.data2vec.modeling_data2vec_audio.Data2VecAudioFeatureProjection`], which are FunASR's | |
| `ConvFeatureExtractionModel` in `layer_norm` mode and `project_features`), and through [`Emotion2vecEncoder`]. | |
| Args: | |
| config ([`Emotion2vecConfig`]): | |
| Model configuration. | |
| """ | |
| def __init__(self, config: Emotion2vecConfig): | |
| super().__init__(config) | |
| self.feature_extractor = Data2VecAudioFeatureEncoder(config) | |
| self.feature_projection = Data2VecAudioFeatureProjection(config) | |
| self.encoder = Emotion2vecEncoder(config) | |
| self.post_init() | |
| def freeze_feature_encoder(self) -> None: | |
| """Disables the gradient of the convolutional feature encoder.""" | |
| self.feature_extractor._freeze_parameters() | |
| def normalize_input(self, input_values: torch.Tensor, attention_mask: torch.Tensor | None = None) -> torch.Tensor: | |
| r""" | |
| Layer normalizes each waveform over its own samples, $(x-\mu)/\sqrt{\sigma^2+10^{-5}}$ with the biased | |
| variance, as `F.layer_norm(source, source.shape)` in `Emotion2vec.inference` does. Padding is left out of the | |
| statistics and set to zero. | |
| Args: | |
| input_values (`torch.Tensor` of shape `(batch_size, sequence_length)`): | |
| 16 kHz waveforms. | |
| attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): | |
| 1 on samples, 0 on padding. | |
| Returns: | |
| `torch.Tensor`: The normalized waveforms. | |
| """ | |
| if attention_mask is None: | |
| return F.layer_norm(input_values, input_values.shape[-1:]) | |
| mask = attention_mask.to(input_values.dtype) | |
| count = mask.sum(-1, keepdim=True) | |
| mean = (input_values * mask).sum(-1, keepdim=True) / count | |
| variance = (((input_values - mean) * mask) ** 2).sum(-1, keepdim=True) / count | |
| return (input_values - mean) / torch.sqrt(variance + 1e-5) * mask | |
| def forward( | |
| self, | |
| input_values: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| output_hidden_states: bool = False, | |
| **kwargs, | |
| ) -> Wav2Vec2BaseModelOutput: | |
| r""" | |
| Args: | |
| input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): | |
| 16 kHz waveforms in $[-1, 1]$, not normalized. | |
| attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): | |
| 1 on samples, 0 on padding. FunASR runs one clip at a time without it. | |
| output_hidden_states (`bool`, *optional*, defaults to `False`): | |
| Whether to return the hidden states of every layer. | |
| Returns: | |
| [`~transformers.modeling_outputs.Wav2Vec2BaseModelOutput`]: `last_hidden_state` holds the frame | |
| features, `extract_features` the convolutional features before the projection. | |
| """ | |
| if self.config.normalize_input: | |
| input_values = self.normalize_input(input_values, attention_mask) | |
| extract_features = self.feature_extractor(input_values).transpose(1, 2) | |
| frame_mask = None | |
| if attention_mask is not None: | |
| frame_mask = self._get_feature_vector_attention_mask( | |
| extract_features.shape[1], attention_mask, add_adapter=False | |
| ) | |
| hidden_states, extract_features = self.feature_projection(extract_features) | |
| hidden_states, all_hidden_states = self.encoder( | |
| hidden_states, attention_mask=frame_mask, output_hidden_states=output_hidden_states, **kwargs | |
| ) | |
| return Wav2Vec2BaseModelOutput( | |
| last_hidden_state=hidden_states, | |
| extract_features=extract_features, | |
| hidden_states=all_hidden_states, | |
| ) | |
| class Emotion2vecForSequenceClassification(Emotion2vecPreTrainedModel): | |
| r""" | |
| Constructs emotion2vec with the utterance emotion classifier of the fine-tuned checkpoints: the frame features are | |
| averaged over time and projected to `num_labels` logits (`proj` in FunASR's `Emotion2vec`). The 9 labels of | |
| `emotion2vec_base_finetuned` are angry, disgusted, fearful, happy, neutral, other, sad, surprised and `<unk>`. | |
| Args: | |
| config ([`Emotion2vecConfig`]): | |
| Model configuration. | |
| """ | |
| def __init__(self, config: Emotion2vecConfig): | |
| super().__init__(config) | |
| self.emotion2vec = Emotion2vecModel(config) | |
| self.classifier = nn.Linear(config.hidden_size, config.num_labels) | |
| self.post_init() | |
| def freeze_feature_encoder(self) -> None: | |
| """Disables the gradient of the convolutional feature encoder.""" | |
| self.emotion2vec.freeze_feature_encoder() | |
| def forward( | |
| self, | |
| input_values: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| labels: torch.Tensor | None = None, | |
| output_hidden_states: bool = False, | |
| **kwargs, | |
| ) -> SequenceClassifierOutput: | |
| r""" | |
| Args: | |
| input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): | |
| 16 kHz waveforms in $[-1, 1]$, not normalized. | |
| attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): | |
| 1 on samples, 0 on padding. | |
| labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): | |
| Emotion class indices. When given, the cross entropy of the logits is returned. | |
| output_hidden_states (`bool`, *optional*, defaults to `False`): | |
| Whether to return the hidden states of every layer. | |
| Returns: | |
| [`~transformers.modeling_outputs.SequenceClassifierOutput`]: The logits, FunASR's `scores` before the | |
| softmax, and the loss when `labels` is given. | |
| """ | |
| outputs = self.emotion2vec( | |
| input_values, attention_mask=attention_mask, output_hidden_states=output_hidden_states, **kwargs | |
| ) | |
| hidden_states = outputs.last_hidden_state | |
| if attention_mask is None: | |
| pooled_output = hidden_states.mean(dim=1) | |
| else: | |
| frame_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask) | |
| weights = frame_mask.unsqueeze(-1).to(hidden_states.dtype) | |
| pooled_output = (hidden_states * weights).sum(dim=1) / weights.sum(dim=1) | |
| logits = self.classifier(pooled_output) | |
| loss = None | |
| if labels is not None: | |
| loss = F.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1).to(logits.device)) | |
| return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) | |