alibidaran/Emotional_Orpheus_16bit

A text-to-speech (TTS) model that supports multiple emotions and multiple characters. Choose a speaker and an emotion, provide text, and the model generates expressive 24 kHz speech.

Audio Demos

Character	Emotion	Audio
Leda	whisper
Leda	gentle
Leda	anxious

Supported Characters

Charon, Achird, Fenrir, Orus, Leda, Zephyr, Puck, Aoede, Gacrux, Enceladus, Kore, Sulafat

Supported Emotions

cheerful, gentle, serious, neutral, energetic, excited, firm, playful, kind, soothing, angry, happy, calm, surprised, whispering, slow, authoritative, commanding, enthusiastic, formal, anxious, cry, fearful, mellow, bright, animated

Prompt Format

{Character}: <{emotion}> {text}

Example: Leda: <anxious> When you eat very quickly, your tummy can fill up rapidly.

Direct Usage

Installation

pip install torch transformers snac pydub numpy ipython

pydub needs ffmpeg installed on your system to export MP3 files.

Example

import tempfile

import numpy as np
import torch
from IPython.display import display, Audio
from pydub import AudioSegment
from snac import SNAC
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("alibidaran/Emotional_Orpheus_16bit")
model = AutoModelForCausalLM.from_pretrained(
    "alibidaran/Emotional_Orpheus_16bit", device_map="auto"
)
snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").to("cpu")

characters = [
    "Charon", "Achird", "Fenrir", "Orus", "Leda", "Zephyr",
    "Puck", "Aoede", "Gacrux", "Enceladus", "Kore", "Sulafat",
]

emotions = [
    "cheerful", "gentle", "serious", "neutral", "energetic", "excited",
    "firm", "playful", "kind", "soothing", "angry", "happy", "calm",
    "surprised", "whispering", "slow", "authoritative", "commanding",
    "enthusiastic", "formal", "anxious", "cry", "fearful", "mellow",
    "bright", "animated",
]


def generate_tokens(prompts_):
    all_input_ids = []
    for prompt in prompts_:
        input_ids = tokenizer(prompt, return_tensors="pt").input_ids
        all_input_ids.append(input_ids)

    start_token = torch.tensor([[128259]], dtype=torch.int64)       # Start of human
    end_tokens = torch.tensor([[128009, 128260]], dtype=torch.int64)  # End of text, End of human

    all_modified_input_ids = []
    for input_ids in all_input_ids:
        modified_input_ids = torch.cat([start_token, input_ids, end_tokens], dim=1)
        all_modified_input_ids.append(modified_input_ids)

    all_padded_tensors = []
    all_attention_masks = []
    max_length = max(ids.shape[1] for ids in all_modified_input_ids)
    for modified_input_ids in all_modified_input_ids:
        padding = max_length - modified_input_ids.shape[1]
        padded_tensor = torch.cat(
            [torch.full((1, padding), 128263, dtype=torch.int64), modified_input_ids], dim=1
        )
        attention_mask = torch.cat(
            [
                torch.zeros((1, padding), dtype=torch.int64),
                torch.ones((1, modified_input_ids.shape[1]), dtype=torch.int64),
            ],
            dim=1,
        )
        all_padded_tensors.append(padded_tensor)
        all_attention_masks.append(attention_mask)

    all_padded_tensors = torch.cat(all_padded_tensors, dim=0)
    all_attention_masks = torch.cat(all_attention_masks, dim=0)

    input_ids = all_padded_tensors.to("cuda")
    attention_mask = all_attention_masks.to("cuda")

    generated_ids = model.generate(
        input_ids=input_ids,
        attention_mask=attention_mask,
        max_new_tokens=1200,
        do_sample=True,
        temperature=0.6,
        top_p=0.95,
        repetition_penalty=1.1,
        num_return_sequences=1,
        eos_token_id=128258,
        use_cache=True,
    )

    token_to_find = 128257
    token_to_remove = 128258

    token_indices = (generated_ids == token_to_find).nonzero(as_tuple=True)
    if len(token_indices[1]) > 0:
        last_occurrence_idx = token_indices[1][-1].item()
        cropped_tensor = generated_ids[:, last_occurrence_idx + 1:]
    else:
        cropped_tensor = generated_ids

    processed_rows = []
    for row in cropped_tensor:
        masked_row = row[row != token_to_remove]
        processed_rows.append(masked_row)

    code_lists = []
    for row in processed_rows:
        row_length = row.size(0)
        new_length = (row_length // 7) * 7
        trimmed_row = row[:new_length]
        trimmed_row = [t - 128266 for t in trimmed_row]
        code_lists.append(trimmed_row)
    return code_lists


def redistribute_codes(code_list):
    layer_1, layer_2, layer_3 = [], [], []
    for i in range((len(code_list) + 1) // 7):
        layer_1.append(code_list[7 * i])
        layer_2.append(code_list[7 * i + 1] - 4096)
        layer_3.append(code_list[7 * i + 2] - (2 * 4096))
        layer_3.append(code_list[7 * i + 3] - (3 * 4096))
        layer_2.append(code_list[7 * i + 4] - (4 * 4096))
        layer_3.append(code_list[7 * i + 5] - (5 * 4096))
        layer_3.append(code_list[7 * i + 6] - (6 * 4096))

    codes = [
        torch.tensor(layer_1).unsqueeze(0),
        torch.tensor(layer_2).unsqueeze(0),
        torch.tensor(layer_3).unsqueeze(0),
    ]
    return snac_model.decode(codes)


def numpy_to_mp3_file(audio_array, sampling_rate):
    audio_array = np.asarray(audio_array).squeeze()

    # Normalize floating-point signals to [-1.0, 1.0] and convert to 16-bit PCM
    if np.issubdtype(audio_array.dtype, np.floating):
        max_val = np.max(np.abs(audio_array))
        if max_val > 0:
            audio_array = audio_array / max_val
        audio_array = (audio_array * 32767.0).astype(np.int16)
    elif audio_array.dtype != np.int16:
        audio_array = audio_array.astype(np.int16)

    audio_segment = AudioSegment(
        audio_array.tobytes(),
        frame_rate=sampling_rate,
        sample_width=2,  # int16 -> 2 bytes
        channels=1,
    )

    tmp = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
    audio_segment.export(tmp.name, format="mp3", bitrate="320k")
    tmp.close()
    return tmp.name


def tts_interface(text, character, emotion):
    formatted_prompt = f"<{emotion}> {text}"
    prompt_input = f"{character}: {formatted_prompt}" if character else formatted_prompt

    code_lists = generate_tokens([prompt_input])
    samples = redistribute_codes(code_lists[0])
    audio_data = samples.detach().squeeze().to("cpu").numpy()
    return numpy_to_mp3_file(audio_data, 24000)  # returns the path to an .mp3 file


# Synthesize speech
temp_audio_path = tts_interface(
    text="When you eat very quickly, your tummy can fill up rapidly with food and extra swallowed air",
    character="Leda",
    emotion="anxious",
)

# Play in a notebook
if temp_audio_path:
    display(Audio(temp_audio_path, autoplay=True))
else:
    print("Generation failed or returned empty audio.")

Notes

• Audio is generated at a 24 kHz sampling rate and decoded with hubertsiuzdak/snac_24khz.
• The example above runs generation on a CUDA GPU.
• Output varies between runs because sampling is enabled (temperature=0.6, top_p=0.95).