nida-tts / src /nida /data /workers /emotion_e2v.py
Bilal7Dev's picture
Nida v0.5 (research preview): weights, 16 voices, code, model card
c0bce59 verified
Raw History Blame Contribute Delete
7.13 kB
"""Utterance-level categorical emotion with emotion2vec+ large (stage S6).
Model: ``emotion2vec/emotion2vec_plus_large`` (Hugging Face; ModelScope id
``iic/emotion2vec_plus_large``) at a pinned revision, loaded through FunASR.
License: FunASR Model License 1.1 (see ``configs/sources.yaml``). Turkish performance is
unmeasured.
Documented call (official README, https://github.com/ddlBoJack/emotion2vec, and the model
card, https://huggingface.co/emotion2vec/emotion2vec_plus_large)::
model = AutoModel(model="iic/emotion2vec_plus_large")
res = model.generate(wav_file, output_dir="./outputs", granularity="utterance",
extract_embedding=False)
``res`` is a list with one dict per input; with ``extract_embedding=False`` it holds ``key``,
``labels`` and ``scores`` (9 classes). FunASR source (``funasr/models/emotion2vec/model.py``,
https://github.com/modelscope/FunASR/blob/main/funasr/models/emotion2vec/model.py) shows
``scores`` are a softmax over the labels listed in the repo's ``tokens.txt``. That file
(revision ``6c303ba987b86b93193de93e34bb2b077a6bedc4``) reads, in order:
``生气/angry``, ``厌恶/disgusted``, ``恐惧/fearful``, ``开心/happy``, ``中立/neutral``,
``其他/other``, ``难过/sad``, ``吃惊/surprised`` and ``<unk>``. The last token has no
bilingual form; the README names class 8 ``unknown``, so :func:`parse_emotion_label`
maps exactly ``<unk>`` to ``unknown`` (an explicit rule, not a fallback).
Loading: the pinned snapshot (``config.yaml``, ``configuration.json``, ``model.pt``,
``tokens.txt``) is downloaded with ``huggingface_hub`` and passed to ``AutoModel`` as a
local directory, which FunASR's ``download_model`` accepts without any network access
(``funasr/download/download_model_from_hub.py``). ``disable_update=True`` skips FunASR's
online version check. FunASR silently falls back to CPU when CUDA is unavailable; we
compare the resolved device with the requested one and raise instead.
Output per request (no arrays, so no ``.npy`` file): ``probabilities`` {label: p} for the 9
labels in :data:`EMOTION_LABELS`, ``top_label`` and ``top_probability``. Input: 16 kHz mono.
"""
from __future__ import annotations
import importlib
import math
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from nida.bench.protocol import WorkRequest
from nida.bench.workers.common import snapshot
from nida.bench.workers.runner import Adapter, ModelRef, main_for
from nida.data.workers import require_16k_mono
__all__ = [
"EMOTION_LABELS",
"SNAPSHOT_FILES",
"EmotionAdapter",
"parse_emotion_label",
"parse_emotion_result",
]
#: Class order of the README (indices 0-8); also the key order of the payload.
EMOTION_LABELS: tuple[str, ...] = (
"angry",
"disgusted",
"fearful",
"happy",
"neutral",
"other",
"sad",
"surprised",
"unknown",
)
SNAPSHOT_FILES: tuple[str, ...] = ("config.yaml", "configuration.json", "model.pt", "tokens.txt")
#: Softmax outputs must sum to 1 within this absolute error.
SUM_TOLERANCE = 1e-3
_UNKNOWN_TOKEN = "<unk>"
def parse_emotion_label(raw: str) -> str:
"""Map a FunASR label (``"生气/angry"``) to its English name (``"angry"``).
The English part is the text after the last ``/``. The bare token ``<unk>`` maps to
``"unknown"``. Matching is exact (no case folding).
Raises:
ValueError: if the result is not one of :data:`EMOTION_LABELS`.
"""
if raw == _UNKNOWN_TOKEN:
return "unknown"
english = raw.rsplit("/", 1)[1].strip() if "/" in raw else None
if english is None or english not in EMOTION_LABELS:
raise ValueError(f"unknown emotion label {raw!r}; expected one of {EMOTION_LABELS}")
return english
def parse_emotion_result(result: Mapping[str, Any]) -> dict[str, float]:
"""Turn one FunASR result dict into ``{english_label: probability}``.
Raises:
ValueError: on missing keys, a labels/scores length mismatch, an unknown, repeated or
missing label, a non-finite or out-of-range score, or scores not summing to 1.
"""
labels = result.get("labels")
scores = result.get("scores")
if not isinstance(labels, list) or not isinstance(scores, list):
raise ValueError(f"emotion result lacks 'labels'/'scores' lists: keys {sorted(result)}")
if len(labels) != len(scores):
raise ValueError(f"{len(labels)} labels but {len(scores)} scores")
probabilities: dict[str, float] = {}
for raw_label, raw_score in zip(labels, scores, strict=True):
label = parse_emotion_label(str(raw_label))
if label in probabilities:
raise ValueError(f"emotion label {label!r} occurs twice")
score = float(raw_score)
if not math.isfinite(score) or not 0.0 <= score <= 1.0:
raise ValueError(f"emotion score for {label!r} is {score}, expected a probability")
probabilities[label] = score
missing = [label for label in EMOTION_LABELS if label not in probabilities]
if missing:
raise ValueError(f"emotion result lacks labels {missing}")
total = sum(probabilities.values())
if abs(total - 1.0) > SUM_TOLERANCE:
raise ValueError(f"emotion scores sum to {total}, expected 1")
return {label: probabilities[label] for label in EMOTION_LABELS}
class EmotionAdapter(Adapter):
"""emotion2vec+ large via FunASR; see the module docstring."""
name = "emotion_e2v"
distributions = ("torch", "funasr", "modelscope", "huggingface-hub", "soundfile", "numpy")
def __init__(self) -> None:
self._model: Any = None # Any: funasr.AutoModel (untyped)
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
funasr = importlib.import_module("funasr")
local = Path(snapshot(model.model_id, model.revision, SNAPSHOT_FILES))
auto = funasr.AutoModel(
model=str(local), hub="hf", device=device, disable_update=True, disable_pbar=True
)
resolved = str(auto.kwargs["device"])
if resolved != device:
raise RuntimeError(f"FunASR placed the model on {resolved!r}, not on {device!r}")
self._model = auto
def _classify(self, request: WorkRequest) -> Mapping[str, object]:
require_16k_mono(request)
results = self._model.generate(
request.audio_path, granularity="utterance", extract_embedding=False
)
if not isinstance(results, list) or len(results) != 1:
raise ValueError(f"{request.clip_id}: expected one emotion result, got {results!r}")
probabilities = parse_emotion_result(results[0])
top_label = max(EMOTION_LABELS, key=lambda label: probabilities[label])
return {
"probabilities": probabilities,
"top_label": top_label,
"top_probability": probabilities[top_label],
}
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
return [self._classify(request) for request in batch]
if __name__ == "__main__":
raise SystemExit(main_for(EmotionAdapter()))