Spaces:
Build error
Build error
初始化项目
Browse files- .gitignore +163 -0
- ChatTTS/__init__.py +1 -0
- ChatTTS/core.py +200 -0
- ChatTTS/experimental/llm.py +40 -0
- ChatTTS/infer/api.py +125 -0
- ChatTTS/model/dvae.py +155 -0
- ChatTTS/model/gpt.py +265 -0
- ChatTTS/utils/gpu_utils.py +23 -0
- ChatTTS/utils/infer_utils.py +141 -0
- ChatTTS/utils/io_utils.py +14 -0
- LICENSE +407 -0
- README_CN.md +136 -0
- example.ipynb +0 -0
- requirements.txt +8 -0
- webui.py +113 -0
.gitignore
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
*.ckpt
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
*.pt
|
| 9 |
+
|
| 10 |
+
# Distribution / packaging
|
| 11 |
+
.Python
|
| 12 |
+
outputs/
|
| 13 |
+
build/
|
| 14 |
+
develop-eggs/
|
| 15 |
+
dist/
|
| 16 |
+
downloads/
|
| 17 |
+
eggs/
|
| 18 |
+
.eggs/
|
| 19 |
+
lib/
|
| 20 |
+
lib64/
|
| 21 |
+
parts/
|
| 22 |
+
sdist/
|
| 23 |
+
var/
|
| 24 |
+
wheels/
|
| 25 |
+
share/python-wheels/
|
| 26 |
+
*.egg-info/
|
| 27 |
+
asset/*
|
| 28 |
+
.installed.cfg
|
| 29 |
+
*.egg
|
| 30 |
+
MANIFEST
|
| 31 |
+
|
| 32 |
+
# PyInstaller
|
| 33 |
+
# Usually these files are written by a python script from a template
|
| 34 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 35 |
+
*.manifest
|
| 36 |
+
*.spec
|
| 37 |
+
|
| 38 |
+
# Installer logs
|
| 39 |
+
pip-log.txt
|
| 40 |
+
pip-delete-this-directory.txt
|
| 41 |
+
|
| 42 |
+
# Unit test / coverage reports
|
| 43 |
+
htmlcov/
|
| 44 |
+
.tox/
|
| 45 |
+
.nox/
|
| 46 |
+
.coverage
|
| 47 |
+
.coverage.*
|
| 48 |
+
.cache
|
| 49 |
+
nosetests.xml
|
| 50 |
+
coverage.xml
|
| 51 |
+
*.cover
|
| 52 |
+
*.py,cover
|
| 53 |
+
.hypothesis/
|
| 54 |
+
.pytest_cache/
|
| 55 |
+
cover/
|
| 56 |
+
|
| 57 |
+
# Translations
|
| 58 |
+
*.mo
|
| 59 |
+
*.pot
|
| 60 |
+
|
| 61 |
+
# Django stuff:
|
| 62 |
+
*.log
|
| 63 |
+
local_settings.py
|
| 64 |
+
db.sqlite3
|
| 65 |
+
db.sqlite3-journal
|
| 66 |
+
|
| 67 |
+
# Flask stuff:
|
| 68 |
+
instance/
|
| 69 |
+
.webassets-cache
|
| 70 |
+
|
| 71 |
+
# Scrapy stuff:
|
| 72 |
+
.scrapy
|
| 73 |
+
|
| 74 |
+
# Sphinx documentation
|
| 75 |
+
docs/_build/
|
| 76 |
+
|
| 77 |
+
# PyBuilder
|
| 78 |
+
.pybuilder/
|
| 79 |
+
target/
|
| 80 |
+
|
| 81 |
+
# Jupyter Notebook
|
| 82 |
+
.ipynb_checkpoints
|
| 83 |
+
|
| 84 |
+
# IPython
|
| 85 |
+
profile_default/
|
| 86 |
+
ipython_config.py
|
| 87 |
+
|
| 88 |
+
# pyenv
|
| 89 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 90 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 91 |
+
# .python-version
|
| 92 |
+
|
| 93 |
+
# pipenv
|
| 94 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 95 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 96 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 97 |
+
# install all needed dependencies.
|
| 98 |
+
#Pipfile.lock
|
| 99 |
+
|
| 100 |
+
# poetry
|
| 101 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 102 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 103 |
+
# commonly ignored for libraries.
|
| 104 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 105 |
+
#poetry.lock
|
| 106 |
+
|
| 107 |
+
# pdm
|
| 108 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 109 |
+
#pdm.lock
|
| 110 |
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
| 111 |
+
# in version control.
|
| 112 |
+
# https://pdm.fming.dev/#use-with-ide
|
| 113 |
+
.pdm.toml
|
| 114 |
+
|
| 115 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 116 |
+
__pypackages__/
|
| 117 |
+
|
| 118 |
+
# Celery stuff
|
| 119 |
+
celerybeat-schedule
|
| 120 |
+
celerybeat.pid
|
| 121 |
+
|
| 122 |
+
# SageMath parsed files
|
| 123 |
+
*.sage.py
|
| 124 |
+
|
| 125 |
+
# Environments
|
| 126 |
+
.env
|
| 127 |
+
.venv
|
| 128 |
+
env/
|
| 129 |
+
venv/
|
| 130 |
+
ENV/
|
| 131 |
+
env.bak/
|
| 132 |
+
venv.bak/
|
| 133 |
+
|
| 134 |
+
# Spyder project settings
|
| 135 |
+
.spyderproject
|
| 136 |
+
.spyproject
|
| 137 |
+
|
| 138 |
+
# Rope project settings
|
| 139 |
+
.ropeproject
|
| 140 |
+
|
| 141 |
+
# mkdocs documentation
|
| 142 |
+
/site
|
| 143 |
+
|
| 144 |
+
# mypy
|
| 145 |
+
.mypy_cache/
|
| 146 |
+
.dmypy.json
|
| 147 |
+
dmypy.json
|
| 148 |
+
|
| 149 |
+
# Pyre type checker
|
| 150 |
+
.pyre/
|
| 151 |
+
|
| 152 |
+
# pytype static type analyzer
|
| 153 |
+
.pytype/
|
| 154 |
+
|
| 155 |
+
# Cython debug symbols
|
| 156 |
+
cython_debug/
|
| 157 |
+
|
| 158 |
+
# PyCharm
|
| 159 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 160 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 161 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 162 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 163 |
+
#.idea/
|
ChatTTS/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from .core import Chat
|
ChatTTS/core.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import os
|
| 3 |
+
import logging
|
| 4 |
+
from functools import partial
|
| 5 |
+
from omegaconf import OmegaConf
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from vocos import Vocos
|
| 9 |
+
from .model.dvae import DVAE
|
| 10 |
+
from .model.gpt import GPT_warpper
|
| 11 |
+
from .utils.gpu_utils import select_device
|
| 12 |
+
from .utils.infer_utils import count_invalid_characters, detect_language, apply_character_map, apply_half2full_map
|
| 13 |
+
from .utils.io_utils import get_latest_modified_file
|
| 14 |
+
from .infer.api import refine_text, infer_code
|
| 15 |
+
|
| 16 |
+
from huggingface_hub import snapshot_download
|
| 17 |
+
|
| 18 |
+
logging.basicConfig(level = logging.INFO)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Chat:
|
| 22 |
+
def __init__(self, ):
|
| 23 |
+
self.pretrain_models = {}
|
| 24 |
+
self.normalizer = {}
|
| 25 |
+
self.logger = logging.getLogger(__name__)
|
| 26 |
+
|
| 27 |
+
def check_model(self, level = logging.INFO, use_decoder = False):
|
| 28 |
+
not_finish = False
|
| 29 |
+
check_list = ['vocos', 'gpt', 'tokenizer']
|
| 30 |
+
|
| 31 |
+
if use_decoder:
|
| 32 |
+
check_list.append('decoder')
|
| 33 |
+
else:
|
| 34 |
+
check_list.append('dvae')
|
| 35 |
+
|
| 36 |
+
for module in check_list:
|
| 37 |
+
if module not in self.pretrain_models:
|
| 38 |
+
self.logger.log(logging.WARNING, f'{module} not initialized.')
|
| 39 |
+
not_finish = True
|
| 40 |
+
|
| 41 |
+
if not not_finish:
|
| 42 |
+
self.logger.log(level, f'All initialized.')
|
| 43 |
+
|
| 44 |
+
return not not_finish
|
| 45 |
+
|
| 46 |
+
def load_models(self, source='huggingface', force_redownload=False, local_path='<LOCAL_PATH>', **kwargs):
|
| 47 |
+
if source == 'huggingface':
|
| 48 |
+
hf_home = os.getenv('HF_HOME', os.path.expanduser("~/.cache/huggingface"))
|
| 49 |
+
try:
|
| 50 |
+
download_path = get_latest_modified_file(os.path.join(hf_home, 'hub/models--2Noise--ChatTTS/snapshots'))
|
| 51 |
+
except:
|
| 52 |
+
download_path = None
|
| 53 |
+
if download_path is None or force_redownload:
|
| 54 |
+
self.logger.log(logging.INFO, f'Download from HF: https://huggingface.co/2Noise/ChatTTS')
|
| 55 |
+
download_path = snapshot_download(repo_id="2Noise/ChatTTS", allow_patterns=["*.pt", "*.yaml"])
|
| 56 |
+
else:
|
| 57 |
+
self.logger.log(logging.INFO, f'Load from cache: {download_path}')
|
| 58 |
+
elif source == 'local':
|
| 59 |
+
self.logger.log(logging.INFO, f'Load from local: {local_path}')
|
| 60 |
+
download_path = local_path
|
| 61 |
+
|
| 62 |
+
self._load(**{k: os.path.join(download_path, v) for k, v in OmegaConf.load(os.path.join(download_path, 'config', 'path.yaml')).items()}, **kwargs)
|
| 63 |
+
|
| 64 |
+
def _load(
|
| 65 |
+
self,
|
| 66 |
+
vocos_config_path: str = None,
|
| 67 |
+
vocos_ckpt_path: str = None,
|
| 68 |
+
dvae_config_path: str = None,
|
| 69 |
+
dvae_ckpt_path: str = None,
|
| 70 |
+
gpt_config_path: str = None,
|
| 71 |
+
gpt_ckpt_path: str = None,
|
| 72 |
+
decoder_config_path: str = None,
|
| 73 |
+
decoder_ckpt_path: str = None,
|
| 74 |
+
tokenizer_path: str = None,
|
| 75 |
+
device: str = None,
|
| 76 |
+
compile: bool = True,
|
| 77 |
+
):
|
| 78 |
+
if not device:
|
| 79 |
+
device = select_device(4096)
|
| 80 |
+
self.logger.log(logging.INFO, f'use {device}')
|
| 81 |
+
|
| 82 |
+
if vocos_config_path:
|
| 83 |
+
vocos = Vocos.from_hparams(vocos_config_path).to(device).eval()
|
| 84 |
+
assert vocos_ckpt_path, 'vocos_ckpt_path should not be None'
|
| 85 |
+
vocos.load_state_dict(torch.load(vocos_ckpt_path))
|
| 86 |
+
self.pretrain_models['vocos'] = vocos
|
| 87 |
+
self.logger.log(logging.INFO, 'vocos loaded.')
|
| 88 |
+
|
| 89 |
+
if dvae_config_path:
|
| 90 |
+
cfg = OmegaConf.load(dvae_config_path)
|
| 91 |
+
dvae = DVAE(**cfg).to(device).eval()
|
| 92 |
+
assert dvae_ckpt_path, 'dvae_ckpt_path should not be None'
|
| 93 |
+
dvae.load_state_dict(torch.load(dvae_ckpt_path, map_location='cpu'))
|
| 94 |
+
self.pretrain_models['dvae'] = dvae
|
| 95 |
+
self.logger.log(logging.INFO, 'dvae loaded.')
|
| 96 |
+
|
| 97 |
+
if gpt_config_path:
|
| 98 |
+
cfg = OmegaConf.load(gpt_config_path)
|
| 99 |
+
gpt = GPT_warpper(**cfg).to(device).eval()
|
| 100 |
+
assert gpt_ckpt_path, 'gpt_ckpt_path should not be None'
|
| 101 |
+
gpt.load_state_dict(torch.load(gpt_ckpt_path, map_location='cpu'))
|
| 102 |
+
if compile and 'cuda' in str(device):
|
| 103 |
+
gpt.gpt.forward = torch.compile(gpt.gpt.forward, backend='inductor', dynamic=True)
|
| 104 |
+
self.pretrain_models['gpt'] = gpt
|
| 105 |
+
spk_stat_path = os.path.join(os.path.dirname(gpt_ckpt_path), 'spk_stat.pt')
|
| 106 |
+
assert os.path.exists(spk_stat_path), f'Missing spk_stat.pt: {spk_stat_path}'
|
| 107 |
+
self.pretrain_models['spk_stat'] = torch.load(spk_stat_path).to(device)
|
| 108 |
+
self.logger.log(logging.INFO, 'gpt loaded.')
|
| 109 |
+
|
| 110 |
+
if decoder_config_path:
|
| 111 |
+
cfg = OmegaConf.load(decoder_config_path)
|
| 112 |
+
decoder = DVAE(**cfg).to(device).eval()
|
| 113 |
+
assert decoder_ckpt_path, 'decoder_ckpt_path should not be None'
|
| 114 |
+
decoder.load_state_dict(torch.load(decoder_ckpt_path, map_location='cpu'))
|
| 115 |
+
self.pretrain_models['decoder'] = decoder
|
| 116 |
+
self.logger.log(logging.INFO, 'decoder loaded.')
|
| 117 |
+
|
| 118 |
+
if tokenizer_path:
|
| 119 |
+
tokenizer = torch.load(tokenizer_path, map_location='cpu')
|
| 120 |
+
tokenizer.padding_side = 'left'
|
| 121 |
+
self.pretrain_models['tokenizer'] = tokenizer
|
| 122 |
+
self.logger.log(logging.INFO, 'tokenizer loaded.')
|
| 123 |
+
|
| 124 |
+
self.check_model()
|
| 125 |
+
|
| 126 |
+
def infer(
|
| 127 |
+
self,
|
| 128 |
+
text,
|
| 129 |
+
skip_refine_text=False,
|
| 130 |
+
refine_text_only=False,
|
| 131 |
+
params_refine_text={},
|
| 132 |
+
params_infer_code={'prompt':'[speed_5]'},
|
| 133 |
+
use_decoder=True,
|
| 134 |
+
do_text_normalization=True,
|
| 135 |
+
lang=None,
|
| 136 |
+
):
|
| 137 |
+
|
| 138 |
+
assert self.check_model(use_decoder=use_decoder)
|
| 139 |
+
|
| 140 |
+
if not isinstance(text, list):
|
| 141 |
+
text = [text]
|
| 142 |
+
|
| 143 |
+
# if do_text_normalization:
|
| 144 |
+
# for i, t in enumerate(text):
|
| 145 |
+
# _lang = detect_language(t) if lang is None else lang
|
| 146 |
+
# self.init_normalizer(_lang)
|
| 147 |
+
# text[i] = self.normalizer[_lang](t)
|
| 148 |
+
# if _lang == 'zh':
|
| 149 |
+
# text[i] = apply_half2full_map(text[i])
|
| 150 |
+
|
| 151 |
+
for i, t in enumerate(text):
|
| 152 |
+
invalid_characters = count_invalid_characters(t)
|
| 153 |
+
if len(invalid_characters):
|
| 154 |
+
self.logger.log(logging.WARNING, f'Invalid characters found! : {invalid_characters}')
|
| 155 |
+
text[i] = apply_character_map(t)
|
| 156 |
+
|
| 157 |
+
if not skip_refine_text:
|
| 158 |
+
text_tokens = refine_text(self.pretrain_models, text, **params_refine_text)['ids']
|
| 159 |
+
text_tokens = [i[i < self.pretrain_models['tokenizer'].convert_tokens_to_ids('[break_0]')] for i in text_tokens]
|
| 160 |
+
text = self.pretrain_models['tokenizer'].batch_decode(text_tokens)
|
| 161 |
+
if refine_text_only:
|
| 162 |
+
return text
|
| 163 |
+
|
| 164 |
+
text = [params_infer_code.get('prompt', '') + i for i in text]
|
| 165 |
+
params_infer_code.pop('prompt', '')
|
| 166 |
+
result = infer_code(self.pretrain_models, text, **params_infer_code, return_hidden=use_decoder)
|
| 167 |
+
|
| 168 |
+
if use_decoder:
|
| 169 |
+
mel_spec = [self.pretrain_models['decoder'](i[None].permute(0,2,1)) for i in result['hiddens']]
|
| 170 |
+
else:
|
| 171 |
+
mel_spec = [self.pretrain_models['dvae'](i[None].permute(0,2,1)) for i in result['ids']]
|
| 172 |
+
|
| 173 |
+
wav = [self.pretrain_models['vocos'].decode(i).cpu().numpy() for i in mel_spec]
|
| 174 |
+
|
| 175 |
+
return wav
|
| 176 |
+
|
| 177 |
+
def sample_random_speaker(self, ):
|
| 178 |
+
|
| 179 |
+
dim = self.pretrain_models['gpt'].gpt.layers[0].mlp.gate_proj.in_features
|
| 180 |
+
std, mean = self.pretrain_models['spk_stat'].chunk(2)
|
| 181 |
+
return torch.randn(dim, device=std.device) * std + mean
|
| 182 |
+
|
| 183 |
+
def init_normalizer(self, lang):
|
| 184 |
+
|
| 185 |
+
if lang not in self.normalizer:
|
| 186 |
+
if lang == 'zh':
|
| 187 |
+
try:
|
| 188 |
+
from tn.chinese.normalizer import Normalizer
|
| 189 |
+
except:
|
| 190 |
+
self.logger.log(logging.WARNING, f'Package WeTextProcessing not found! \
|
| 191 |
+
Run: conda install -c conda-forge pynini=2.1.5 && pip install WeTextProcessing')
|
| 192 |
+
self.normalizer[lang] = Normalizer().normalize
|
| 193 |
+
else:
|
| 194 |
+
try:
|
| 195 |
+
from nemo_text_processing.text_normalization.normalize import Normalizer
|
| 196 |
+
except:
|
| 197 |
+
self.logger.log(logging.WARNING, f'Package nemo_text_processing not found! \
|
| 198 |
+
Run: conda install -c conda-forge pynini=2.1.5 && pip install nemo_text_processing')
|
| 199 |
+
self.normalizer[lang] = partial(Normalizer(input_case='cased', lang=lang).normalize, verbose=False, punct_post_process=True)
|
| 200 |
+
|
ChatTTS/experimental/llm.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
from openai import OpenAI
|
| 3 |
+
|
| 4 |
+
prompt_dict = {
|
| 5 |
+
'kimi': [ {"role": "system", "content": "你是 Kimi,由 Moonshot AI 提供的人工智能助手,你更擅长中文和英文的对话。"},
|
| 6 |
+
{"role": "user", "content": "你好,请注意你现在生成的文字要按照人日常生活的口吻,你的回复将会后续用TTS模型转为语音,并且请把回答控制在100字以内。并且标点符号仅包含逗号和句号,将数字等转为文字回答。"},
|
| 7 |
+
{"role": "assistant", "content": "好的,我现在生成的文字将按照人日常生活的口吻, 并且我会把回答控制在一百字以内, 标点符号仅包含逗号和句号,将阿拉伯数字等转为中文文字回答。下面请开始对话。"},],
|
| 8 |
+
'deepseek': [
|
| 9 |
+
{"role": "system", "content": "You are a helpful assistant"},
|
| 10 |
+
{"role": "user", "content": "你好,请注意你现在生成的文字要按照人日常生活的口吻,你的回复将会后续用TTS模型转为语音,并且请把回答控制在100字以内。并且标点符号仅包含逗号和句号,将数字等转为文字回答。"},
|
| 11 |
+
{"role": "assistant", "content": "好的,我现在生成的文字将按照人日常生活的口吻, 并且我会把回答控制在一百字以内, 标点符号仅包含逗号和句号,将阿拉伯数字等转为中文文字回答。下面请开始对话。"},],
|
| 12 |
+
'deepseek_TN': [
|
| 13 |
+
{"role": "system", "content": "You are a helpful assistant"},
|
| 14 |
+
{"role": "user", "content": "你好,现在我们在处理TTS的文本输入,下面将会给你输入一段文本,请你将其中的阿拉伯数字等等转为文字表达,并且输出的文本里仅包含逗号和句号这两个标点符号"},
|
| 15 |
+
{"role": "assistant", "content": "好的,我现在对TTS的文本输入进行处理。这一般叫做text normalization。下面请输入"},
|
| 16 |
+
{"role": "user", "content": "We paid $123 for this desk."},
|
| 17 |
+
{"role": "assistant", "content": "We paid one hundred and twenty three dollars for this desk."},
|
| 18 |
+
{"role": "user", "content": "详询请拨打010-724654"},
|
| 19 |
+
{"role": "assistant", "content": "详询请拨打零幺零,七二四六五四"},
|
| 20 |
+
{"role": "user", "content": "罗森宣布将于7月24日退市,在华门店超6000家!"},
|
| 21 |
+
{"role": "assistant", "content": "罗森宣布将于七月二十四日退市,在华门店超过六千家。"},
|
| 22 |
+
],
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
class llm_api:
|
| 26 |
+
def __init__(self, api_key, base_url, model):
|
| 27 |
+
self.client = OpenAI(
|
| 28 |
+
api_key = api_key,
|
| 29 |
+
base_url = base_url,
|
| 30 |
+
)
|
| 31 |
+
self.model = model
|
| 32 |
+
def call(self, user_question, temperature = 0.3, prompt_version='kimi', **kwargs):
|
| 33 |
+
|
| 34 |
+
completion = self.client.chat.completions.create(
|
| 35 |
+
model = self.model,
|
| 36 |
+
messages = prompt_dict[prompt_version]+[{"role": "user", "content": user_question},],
|
| 37 |
+
temperature = temperature,
|
| 38 |
+
**kwargs
|
| 39 |
+
)
|
| 40 |
+
return completion.choices[0].message.content
|
ChatTTS/infer/api.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from transformers.generation import TopKLogitsWarper, TopPLogitsWarper
|
| 5 |
+
from ..utils.infer_utils import CustomRepetitionPenaltyLogitsProcessorRepeat
|
| 6 |
+
|
| 7 |
+
def infer_code(
|
| 8 |
+
models,
|
| 9 |
+
text,
|
| 10 |
+
spk_emb = None,
|
| 11 |
+
top_P = 0.7,
|
| 12 |
+
top_K = 20,
|
| 13 |
+
temperature = 0.3,
|
| 14 |
+
repetition_penalty = 1.05,
|
| 15 |
+
max_new_token = 2048,
|
| 16 |
+
**kwargs
|
| 17 |
+
):
|
| 18 |
+
|
| 19 |
+
device = next(models['gpt'].parameters()).device
|
| 20 |
+
|
| 21 |
+
if not isinstance(text, list):
|
| 22 |
+
text = [text]
|
| 23 |
+
|
| 24 |
+
if not isinstance(temperature, list):
|
| 25 |
+
temperature = [temperature] * models['gpt'].num_vq
|
| 26 |
+
|
| 27 |
+
if spk_emb is not None:
|
| 28 |
+
text = [f'[Stts][spk_emb]{i}[Ptts]' for i in text]
|
| 29 |
+
else:
|
| 30 |
+
text = [f'[Stts][empty_spk]{i}[Ptts]' for i in text]
|
| 31 |
+
|
| 32 |
+
text_token = models['tokenizer'](text, return_tensors='pt', add_special_tokens=False, padding=True).to(device)
|
| 33 |
+
input_ids = text_token['input_ids'][...,None].expand(-1, -1, models['gpt'].num_vq)
|
| 34 |
+
text_mask = torch.ones(text_token['input_ids'].shape, dtype=bool, device=device)
|
| 35 |
+
|
| 36 |
+
inputs = {
|
| 37 |
+
'input_ids': input_ids,
|
| 38 |
+
'text_mask': text_mask,
|
| 39 |
+
'attention_mask': text_token['attention_mask'],
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
emb = models['gpt'].get_emb(**inputs)
|
| 43 |
+
if spk_emb is not None:
|
| 44 |
+
emb[inputs['input_ids'][..., 0] == models['tokenizer'].convert_tokens_to_ids('[spk_emb]')] = \
|
| 45 |
+
F.normalize(spk_emb.to(device).to(emb.dtype)[None].expand(len(text), -1), p=2.0, dim=1, eps=1e-12)
|
| 46 |
+
|
| 47 |
+
num_code = models['gpt'].emb_code[0].num_embeddings - 1
|
| 48 |
+
|
| 49 |
+
LogitsWarpers = []
|
| 50 |
+
if top_P is not None:
|
| 51 |
+
LogitsWarpers.append(TopPLogitsWarper(top_P, min_tokens_to_keep=3))
|
| 52 |
+
if top_K is not None:
|
| 53 |
+
LogitsWarpers.append(TopKLogitsWarper(top_K, min_tokens_to_keep=3))
|
| 54 |
+
|
| 55 |
+
LogitsProcessors = []
|
| 56 |
+
if repetition_penalty is not None and repetition_penalty != 1:
|
| 57 |
+
LogitsProcessors.append(CustomRepetitionPenaltyLogitsProcessorRepeat(\
|
| 58 |
+
repetition_penalty, num_code, 16))
|
| 59 |
+
|
| 60 |
+
result = models['gpt'].generate(
|
| 61 |
+
emb, inputs['input_ids'],
|
| 62 |
+
temperature = torch.tensor(temperature, device=device),
|
| 63 |
+
attention_mask = inputs['attention_mask'],
|
| 64 |
+
LogitsWarpers = LogitsWarpers,
|
| 65 |
+
LogitsProcessors = LogitsProcessors,
|
| 66 |
+
eos_token = num_code,
|
| 67 |
+
max_new_token = max_new_token,
|
| 68 |
+
infer_text = False,
|
| 69 |
+
**kwargs
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
return result
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def refine_text(
|
| 76 |
+
models,
|
| 77 |
+
text,
|
| 78 |
+
top_P = 0.7,
|
| 79 |
+
top_K = 20,
|
| 80 |
+
temperature = 0.7,
|
| 81 |
+
repetition_penalty = 1.0,
|
| 82 |
+
max_new_token = 384,
|
| 83 |
+
prompt = '',
|
| 84 |
+
**kwargs
|
| 85 |
+
):
|
| 86 |
+
|
| 87 |
+
device = next(models['gpt'].parameters()).device
|
| 88 |
+
|
| 89 |
+
if not isinstance(text, list):
|
| 90 |
+
text = [text]
|
| 91 |
+
|
| 92 |
+
assert len(text), 'text should not be empty'
|
| 93 |
+
|
| 94 |
+
text = [f"[Sbreak]{i}[Pbreak]{prompt}" for i in text]
|
| 95 |
+
text_token = models['tokenizer'](text, return_tensors='pt', add_special_tokens=False, padding=True).to(device)
|
| 96 |
+
text_mask = torch.ones(text_token['input_ids'].shape, dtype=bool, device=device)
|
| 97 |
+
|
| 98 |
+
inputs = {
|
| 99 |
+
'input_ids': text_token['input_ids'][...,None].expand(-1, -1, models['gpt'].num_vq),
|
| 100 |
+
'text_mask': text_mask,
|
| 101 |
+
'attention_mask': text_token['attention_mask'],
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
LogitsWarpers = []
|
| 105 |
+
if top_P is not None:
|
| 106 |
+
LogitsWarpers.append(TopPLogitsWarper(top_P, min_tokens_to_keep=3))
|
| 107 |
+
if top_K is not None:
|
| 108 |
+
LogitsWarpers.append(TopKLogitsWarper(top_K, min_tokens_to_keep=3))
|
| 109 |
+
|
| 110 |
+
LogitsProcessors = []
|
| 111 |
+
if repetition_penalty is not None and repetition_penalty != 1:
|
| 112 |
+
LogitsProcessors.append(CustomRepetitionPenaltyLogitsProcessorRepeat(repetition_penalty, len(models['tokenizer']), 16))
|
| 113 |
+
|
| 114 |
+
result = models['gpt'].generate(
|
| 115 |
+
models['gpt'].get_emb(**inputs), inputs['input_ids'],
|
| 116 |
+
temperature = torch.tensor([temperature,], device=device),
|
| 117 |
+
attention_mask = inputs['attention_mask'],
|
| 118 |
+
LogitsWarpers = LogitsWarpers,
|
| 119 |
+
LogitsProcessors = LogitsProcessors,
|
| 120 |
+
eos_token = torch.tensor(models['tokenizer'].convert_tokens_to_ids('[Ebreak]'), device=device)[None],
|
| 121 |
+
max_new_token = max_new_token,
|
| 122 |
+
infer_text = True,
|
| 123 |
+
**kwargs
|
| 124 |
+
)
|
| 125 |
+
return result
|
ChatTTS/model/dvae.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
from einops import rearrange
|
| 3 |
+
from vector_quantize_pytorch import GroupedResidualFSQ
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
|
| 9 |
+
class ConvNeXtBlock(nn.Module):
|
| 10 |
+
def __init__(
|
| 11 |
+
self,
|
| 12 |
+
dim: int,
|
| 13 |
+
intermediate_dim: int,
|
| 14 |
+
kernel, dilation,
|
| 15 |
+
layer_scale_init_value: float = 1e-6,
|
| 16 |
+
):
|
| 17 |
+
# ConvNeXt Block copied from Vocos.
|
| 18 |
+
super().__init__()
|
| 19 |
+
self.dwconv = nn.Conv1d(dim, dim,
|
| 20 |
+
kernel_size=kernel, padding=dilation*(kernel//2),
|
| 21 |
+
dilation=dilation, groups=dim
|
| 22 |
+
) # depthwise conv
|
| 23 |
+
|
| 24 |
+
self.norm = nn.LayerNorm(dim, eps=1e-6)
|
| 25 |
+
self.pwconv1 = nn.Linear(dim, intermediate_dim) # pointwise/1x1 convs, implemented with linear layers
|
| 26 |
+
self.act = nn.GELU()
|
| 27 |
+
self.pwconv2 = nn.Linear(intermediate_dim, dim)
|
| 28 |
+
self.gamma = (
|
| 29 |
+
nn.Parameter(layer_scale_init_value * torch.ones(dim), requires_grad=True)
|
| 30 |
+
if layer_scale_init_value > 0
|
| 31 |
+
else None
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
def forward(self, x: torch.Tensor, cond = None) -> torch.Tensor:
|
| 35 |
+
residual = x
|
| 36 |
+
x = self.dwconv(x)
|
| 37 |
+
x = x.transpose(1, 2) # (B, C, T) -> (B, T, C)
|
| 38 |
+
x = self.norm(x)
|
| 39 |
+
x = self.pwconv1(x)
|
| 40 |
+
x = self.act(x)
|
| 41 |
+
x = self.pwconv2(x)
|
| 42 |
+
if self.gamma is not None:
|
| 43 |
+
x = self.gamma * x
|
| 44 |
+
x = x.transpose(1, 2) # (B, T, C) -> (B, C, T)
|
| 45 |
+
|
| 46 |
+
x = residual + x
|
| 47 |
+
return x
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class GFSQ(nn.Module):
|
| 52 |
+
|
| 53 |
+
def __init__(self,
|
| 54 |
+
dim, levels, G, R, eps=1e-5, transpose = True
|
| 55 |
+
):
|
| 56 |
+
super(GFSQ, self).__init__()
|
| 57 |
+
self.quantizer = GroupedResidualFSQ(
|
| 58 |
+
dim=dim,
|
| 59 |
+
levels=levels,
|
| 60 |
+
num_quantizers=R,
|
| 61 |
+
groups=G,
|
| 62 |
+
)
|
| 63 |
+
self.n_ind = math.prod(levels)
|
| 64 |
+
self.eps = eps
|
| 65 |
+
self.transpose = transpose
|
| 66 |
+
self.G = G
|
| 67 |
+
self.R = R
|
| 68 |
+
|
| 69 |
+
def _embed(self, x):
|
| 70 |
+
if self.transpose:
|
| 71 |
+
x = x.transpose(1,2)
|
| 72 |
+
x = rearrange(
|
| 73 |
+
x, "b t (g r) -> g b t r", g = self.G, r = self.R,
|
| 74 |
+
)
|
| 75 |
+
feat = self.quantizer.get_output_from_indices(x)
|
| 76 |
+
return feat.transpose(1,2) if self.transpose else feat
|
| 77 |
+
|
| 78 |
+
def forward(self, x,):
|
| 79 |
+
if self.transpose:
|
| 80 |
+
x = x.transpose(1,2)
|
| 81 |
+
feat, ind = self.quantizer(x)
|
| 82 |
+
ind = rearrange(
|
| 83 |
+
ind, "g b t r ->b t (g r)",
|
| 84 |
+
)
|
| 85 |
+
embed_onehot = F.one_hot(ind.long(), self.n_ind).to(x.dtype)
|
| 86 |
+
e_mean = torch.mean(embed_onehot, dim=[0,1])
|
| 87 |
+
e_mean = e_mean / (e_mean.sum(dim=1) + self.eps).unsqueeze(1)
|
| 88 |
+
perplexity = torch.exp(-torch.sum(e_mean * torch.log(e_mean + self.eps), dim=1))
|
| 89 |
+
|
| 90 |
+
return (
|
| 91 |
+
torch.zeros(perplexity.shape, dtype=x.dtype, device=x.device),
|
| 92 |
+
feat.transpose(1,2) if self.transpose else feat,
|
| 93 |
+
perplexity,
|
| 94 |
+
None,
|
| 95 |
+
ind.transpose(1,2) if self.transpose else ind,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
class DVAEDecoder(nn.Module):
|
| 99 |
+
def __init__(self, idim, odim,
|
| 100 |
+
n_layer = 12, bn_dim = 64, hidden = 256,
|
| 101 |
+
kernel = 7, dilation = 2, up = False
|
| 102 |
+
):
|
| 103 |
+
super().__init__()
|
| 104 |
+
self.up = up
|
| 105 |
+
self.conv_in = nn.Sequential(
|
| 106 |
+
nn.Conv1d(idim, bn_dim, 3, 1, 1), nn.GELU(),
|
| 107 |
+
nn.Conv1d(bn_dim, hidden, 3, 1, 1)
|
| 108 |
+
)
|
| 109 |
+
self.decoder_block = nn.ModuleList([
|
| 110 |
+
ConvNeXtBlock(hidden, hidden* 4, kernel, dilation,)
|
| 111 |
+
for _ in range(n_layer)])
|
| 112 |
+
self.conv_out = nn.Conv1d(hidden, odim, kernel_size=1, bias=False)
|
| 113 |
+
|
| 114 |
+
def forward(self, input, conditioning=None):
|
| 115 |
+
# B, T, C
|
| 116 |
+
x = input.transpose(1, 2)
|
| 117 |
+
x = self.conv_in(x)
|
| 118 |
+
for f in self.decoder_block:
|
| 119 |
+
x = f(x, conditioning)
|
| 120 |
+
|
| 121 |
+
x = self.conv_out(x)
|
| 122 |
+
return x.transpose(1, 2)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class DVAE(nn.Module):
|
| 126 |
+
def __init__(
|
| 127 |
+
self, decoder_config, vq_config, dim=512
|
| 128 |
+
):
|
| 129 |
+
super().__init__()
|
| 130 |
+
self.register_buffer('coef', torch.randn(1, 100, 1))
|
| 131 |
+
|
| 132 |
+
self.decoder = DVAEDecoder(**decoder_config)
|
| 133 |
+
self.out_conv = nn.Conv1d(dim, 100, 3, 1, 1, bias=False)
|
| 134 |
+
if vq_config is not None:
|
| 135 |
+
self.vq_layer = GFSQ(**vq_config)
|
| 136 |
+
else:
|
| 137 |
+
self.vq_layer = None
|
| 138 |
+
|
| 139 |
+
def forward(self, inp):
|
| 140 |
+
|
| 141 |
+
if self.vq_layer is not None:
|
| 142 |
+
vq_feats = self.vq_layer._embed(inp)
|
| 143 |
+
else:
|
| 144 |
+
vq_feats = inp.detach().clone()
|
| 145 |
+
|
| 146 |
+
temp = torch.chunk(vq_feats, 2, dim=1) # flatten trick :)
|
| 147 |
+
temp = torch.stack(temp, -1)
|
| 148 |
+
vq_feats = temp.reshape(*temp.shape[:2], -1)
|
| 149 |
+
|
| 150 |
+
vq_feats = vq_feats.transpose(1, 2)
|
| 151 |
+
dec_out = self.decoder(input=vq_feats)
|
| 152 |
+
dec_out = self.out_conv(dec_out.transpose(1, 2))
|
| 153 |
+
mel = dec_out * self.coef
|
| 154 |
+
|
| 155 |
+
return mel
|
ChatTTS/model/gpt.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
| 3 |
+
|
| 4 |
+
import logging
|
| 5 |
+
from tqdm import tqdm
|
| 6 |
+
from einops import rearrange
|
| 7 |
+
from transformers.cache_utils import Cache
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
import torch.nn.utils.parametrize as P
|
| 13 |
+
from torch.nn.utils.parametrizations import weight_norm
|
| 14 |
+
from transformers import LlamaModel, LlamaConfig
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class LlamaMLP(nn.Module):
|
| 18 |
+
def __init__(self, hidden_size, intermediate_size):
|
| 19 |
+
super().__init__()
|
| 20 |
+
self.hidden_size = hidden_size
|
| 21 |
+
self.intermediate_size = intermediate_size
|
| 22 |
+
self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 23 |
+
self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 24 |
+
self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
|
| 25 |
+
self.act_fn = F.silu
|
| 26 |
+
|
| 27 |
+
def forward(self, x):
|
| 28 |
+
down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 29 |
+
return down_proj
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class GPT_warpper(nn.Module):
|
| 33 |
+
def __init__(
|
| 34 |
+
self,
|
| 35 |
+
gpt_config,
|
| 36 |
+
num_audio_tokens,
|
| 37 |
+
num_text_tokens,
|
| 38 |
+
num_vq=4,
|
| 39 |
+
**kwargs,
|
| 40 |
+
):
|
| 41 |
+
super().__init__()
|
| 42 |
+
|
| 43 |
+
self.logger = logging.getLogger(__name__)
|
| 44 |
+
self.gpt = self.build_model(gpt_config)
|
| 45 |
+
self.model_dim = self.gpt.config.hidden_size
|
| 46 |
+
|
| 47 |
+
self.num_vq = num_vq
|
| 48 |
+
self.emb_code = nn.ModuleList([nn.Embedding(num_audio_tokens, self.model_dim) for i in range(self.num_vq)])
|
| 49 |
+
self.emb_text = nn.Embedding(num_text_tokens, self.model_dim)
|
| 50 |
+
self.head_text = weight_norm(nn.Linear(self.model_dim, num_text_tokens, bias=False), name='weight')
|
| 51 |
+
self.head_code = nn.ModuleList([weight_norm(nn.Linear(self.model_dim, num_audio_tokens, bias=False), name='weight') for i in range(self.num_vq)])
|
| 52 |
+
|
| 53 |
+
def build_model(self, config):
|
| 54 |
+
|
| 55 |
+
configuration = LlamaConfig(**config)
|
| 56 |
+
model = LlamaModel(configuration)
|
| 57 |
+
del model.embed_tokens
|
| 58 |
+
|
| 59 |
+
return model
|
| 60 |
+
|
| 61 |
+
def get_emb(self, input_ids, text_mask, **kwargs):
|
| 62 |
+
|
| 63 |
+
emb_text = self.emb_text(input_ids[text_mask][:, 0])
|
| 64 |
+
|
| 65 |
+
emb_code = [self.emb_code[i](input_ids[~text_mask][:, i]) for i in range(self.num_vq)]
|
| 66 |
+
emb_code = torch.stack(emb_code, 2).sum(2)
|
| 67 |
+
|
| 68 |
+
emb = torch.zeros((input_ids.shape[:-1])+(emb_text.shape[-1],), device=emb_text.device, dtype=emb_text.dtype)
|
| 69 |
+
emb[text_mask] = emb_text
|
| 70 |
+
emb[~text_mask] = emb_code.to(emb.dtype)
|
| 71 |
+
|
| 72 |
+
return emb
|
| 73 |
+
|
| 74 |
+
def prepare_inputs_for_generation(
|
| 75 |
+
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, cache_position=None, **kwargs
|
| 76 |
+
):
|
| 77 |
+
# With static cache, the `past_key_values` is None
|
| 78 |
+
# TODO joao: standardize interface for the different Cache classes and remove of this if
|
| 79 |
+
has_static_cache = False
|
| 80 |
+
if past_key_values is None:
|
| 81 |
+
past_key_values = getattr(self.gpt.layers[0].self_attn, "past_key_value", None)
|
| 82 |
+
has_static_cache = past_key_values is not None
|
| 83 |
+
|
| 84 |
+
past_length = 0
|
| 85 |
+
if past_key_values is not None:
|
| 86 |
+
if isinstance(past_key_values, Cache):
|
| 87 |
+
past_length = cache_position[0] if cache_position is not None else past_key_values.get_seq_length()
|
| 88 |
+
max_cache_length = (
|
| 89 |
+
torch.tensor(past_key_values.get_max_length(), device=input_ids.device)
|
| 90 |
+
if past_key_values.get_max_length() is not None
|
| 91 |
+
else None
|
| 92 |
+
)
|
| 93 |
+
cache_length = past_length if max_cache_length is None else torch.min(max_cache_length, past_length)
|
| 94 |
+
# TODO joao: remove this `else` after `generate` prioritizes `Cache` objects
|
| 95 |
+
else:
|
| 96 |
+
cache_length = past_length = past_key_values[0][0].shape[2]
|
| 97 |
+
max_cache_length = None
|
| 98 |
+
|
| 99 |
+
# Keep only the unprocessed tokens:
|
| 100 |
+
# 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
|
| 101 |
+
# some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
|
| 102 |
+
# input)
|
| 103 |
+
if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
|
| 104 |
+
input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
|
| 105 |
+
# 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
|
| 106 |
+
# input_ids based on the past_length.
|
| 107 |
+
elif past_length < input_ids.shape[1]:
|
| 108 |
+
input_ids = input_ids[:, past_length:]
|
| 109 |
+
# 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
|
| 110 |
+
|
| 111 |
+
# If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
|
| 112 |
+
if (
|
| 113 |
+
max_cache_length is not None
|
| 114 |
+
and attention_mask is not None
|
| 115 |
+
and cache_length + input_ids.shape[1] > max_cache_length
|
| 116 |
+
):
|
| 117 |
+
attention_mask = attention_mask[:, -max_cache_length:]
|
| 118 |
+
|
| 119 |
+
position_ids = kwargs.get("position_ids", None)
|
| 120 |
+
if attention_mask is not None and position_ids is None:
|
| 121 |
+
# create position_ids on the fly for batch generation
|
| 122 |
+
position_ids = attention_mask.long().cumsum(-1) - 1
|
| 123 |
+
position_ids.masked_fill_(attention_mask == 0, 1)
|
| 124 |
+
if past_key_values:
|
| 125 |
+
position_ids = position_ids[:, -input_ids.shape[1] :]
|
| 126 |
+
|
| 127 |
+
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
|
| 128 |
+
if inputs_embeds is not None and past_key_values is None:
|
| 129 |
+
model_inputs = {"inputs_embeds": inputs_embeds}
|
| 130 |
+
else:
|
| 131 |
+
# The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise
|
| 132 |
+
# recompiles graphs as the stride of the inputs is a guard. Ref: https://github.com/huggingface/transformers/pull/29114
|
| 133 |
+
# TODO: use `next_tokens` directly instead.
|
| 134 |
+
model_inputs = {"input_ids": input_ids.contiguous()}
|
| 135 |
+
|
| 136 |
+
input_length = position_ids.shape[-1] if position_ids is not None else input_ids.shape[-1]
|
| 137 |
+
if cache_position is None:
|
| 138 |
+
cache_position = torch.arange(past_length, past_length + input_length, device=input_ids.device)
|
| 139 |
+
else:
|
| 140 |
+
cache_position = cache_position[-input_length:]
|
| 141 |
+
|
| 142 |
+
if has_static_cache:
|
| 143 |
+
past_key_values = None
|
| 144 |
+
|
| 145 |
+
model_inputs.update(
|
| 146 |
+
{
|
| 147 |
+
"position_ids": position_ids,
|
| 148 |
+
"cache_position": cache_position,
|
| 149 |
+
"past_key_values": past_key_values,
|
| 150 |
+
"use_cache": kwargs.get("use_cache"),
|
| 151 |
+
"attention_mask": attention_mask,
|
| 152 |
+
}
|
| 153 |
+
)
|
| 154 |
+
return model_inputs
|
| 155 |
+
|
| 156 |
+
def generate(
|
| 157 |
+
self,
|
| 158 |
+
emb,
|
| 159 |
+
inputs_ids,
|
| 160 |
+
temperature,
|
| 161 |
+
eos_token,
|
| 162 |
+
attention_mask = None,
|
| 163 |
+
max_new_token = 2048,
|
| 164 |
+
min_new_token = 0,
|
| 165 |
+
LogitsWarpers = [],
|
| 166 |
+
LogitsProcessors = [],
|
| 167 |
+
infer_text=False,
|
| 168 |
+
return_attn=False,
|
| 169 |
+
return_hidden=False,
|
| 170 |
+
):
|
| 171 |
+
|
| 172 |
+
with torch.no_grad():
|
| 173 |
+
|
| 174 |
+
attentions = []
|
| 175 |
+
hiddens = []
|
| 176 |
+
|
| 177 |
+
start_idx, end_idx = inputs_ids.shape[1], torch.zeros(inputs_ids.shape[0], device=inputs_ids.device, dtype=torch.long)
|
| 178 |
+
finish = torch.zeros(inputs_ids.shape[0], device=inputs_ids.device).bool()
|
| 179 |
+
|
| 180 |
+
temperature = temperature[None].expand(inputs_ids.shape[0], -1)
|
| 181 |
+
temperature = rearrange(temperature, "b n -> (b n) 1")
|
| 182 |
+
|
| 183 |
+
attention_mask_cache = torch.ones((inputs_ids.shape[0], inputs_ids.shape[1]+max_new_token,), dtype=torch.bool, device=inputs_ids.device)
|
| 184 |
+
if attention_mask is not None:
|
| 185 |
+
attention_mask_cache[:, :attention_mask.shape[1]] = attention_mask
|
| 186 |
+
|
| 187 |
+
for i in tqdm(range(max_new_token)):
|
| 188 |
+
|
| 189 |
+
model_input = self.prepare_inputs_for_generation(inputs_ids,
|
| 190 |
+
outputs.past_key_values if i!=0 else None,
|
| 191 |
+
attention_mask_cache[:, :inputs_ids.shape[1]], use_cache=True)
|
| 192 |
+
|
| 193 |
+
if i == 0:
|
| 194 |
+
model_input['inputs_embeds'] = emb
|
| 195 |
+
else:
|
| 196 |
+
if infer_text:
|
| 197 |
+
model_input['inputs_embeds'] = self.emb_text(model_input['input_ids'][:,:,0])
|
| 198 |
+
else:
|
| 199 |
+
code_emb = [self.emb_code[i](model_input['input_ids'][:,:,i]) for i in range(self.num_vq)]
|
| 200 |
+
model_input['inputs_embeds'] = torch.stack(code_emb, 3).sum(3)
|
| 201 |
+
|
| 202 |
+
model_input['input_ids'] = None
|
| 203 |
+
outputs = self.gpt.forward(**model_input, output_attentions=return_attn)
|
| 204 |
+
attentions.append(outputs.attentions)
|
| 205 |
+
hidden_states = outputs[0] # 🐻
|
| 206 |
+
if return_hidden:
|
| 207 |
+
hiddens.append(hidden_states[:, -1])
|
| 208 |
+
|
| 209 |
+
with P.cached():
|
| 210 |
+
if infer_text:
|
| 211 |
+
logits = self.head_text(hidden_states)
|
| 212 |
+
else:
|
| 213 |
+
logits = torch.stack([self.head_code[i](hidden_states) for i in range(self.num_vq)], 3)
|
| 214 |
+
|
| 215 |
+
logits = logits[:, -1].float()
|
| 216 |
+
|
| 217 |
+
if not infer_text:
|
| 218 |
+
logits = rearrange(logits, "b c n -> (b n) c")
|
| 219 |
+
logits_token = rearrange(inputs_ids[:, start_idx:], "b c n -> (b n) c")
|
| 220 |
+
else:
|
| 221 |
+
logits_token = inputs_ids[:, start_idx:, 0]
|
| 222 |
+
|
| 223 |
+
logits = logits / temperature
|
| 224 |
+
|
| 225 |
+
for logitsProcessors in LogitsProcessors:
|
| 226 |
+
logits = logitsProcessors(logits_token, logits)
|
| 227 |
+
|
| 228 |
+
for logitsWarpers in LogitsWarpers:
|
| 229 |
+
logits = logitsWarpers(logits_token, logits)
|
| 230 |
+
|
| 231 |
+
if i < min_new_token:
|
| 232 |
+
logits[:, eos_token] = -torch.inf
|
| 233 |
+
|
| 234 |
+
scores = F.softmax(logits, dim=-1)
|
| 235 |
+
|
| 236 |
+
idx_next = torch.multinomial(scores, num_samples=1)
|
| 237 |
+
|
| 238 |
+
if not infer_text:
|
| 239 |
+
idx_next = rearrange(idx_next, "(b n) 1 -> b n", n=self.num_vq)
|
| 240 |
+
finish = finish | (idx_next == eos_token).any(1)
|
| 241 |
+
inputs_ids = torch.cat([inputs_ids, idx_next.unsqueeze(1)], 1)
|
| 242 |
+
else:
|
| 243 |
+
finish = finish | (idx_next == eos_token).any(1)
|
| 244 |
+
inputs_ids = torch.cat([inputs_ids, idx_next.unsqueeze(-1).expand(-1, -1, self.num_vq)], 1)
|
| 245 |
+
|
| 246 |
+
end_idx = end_idx + (~finish).int()
|
| 247 |
+
|
| 248 |
+
if finish.all():
|
| 249 |
+
break
|
| 250 |
+
|
| 251 |
+
inputs_ids = [inputs_ids[idx, start_idx: start_idx+i] for idx, i in enumerate(end_idx.int())]
|
| 252 |
+
inputs_ids = [i[:, 0] for i in inputs_ids] if infer_text else inputs_ids
|
| 253 |
+
|
| 254 |
+
if return_hidden:
|
| 255 |
+
hiddens = torch.stack(hiddens, 1)
|
| 256 |
+
hiddens = [hiddens[idx, :i] for idx, i in enumerate(end_idx.int())]
|
| 257 |
+
|
| 258 |
+
if not finish.all():
|
| 259 |
+
self.logger.warn(f'Incomplete result. hit max_new_token: {max_new_token}')
|
| 260 |
+
|
| 261 |
+
return {
|
| 262 |
+
'ids': inputs_ids,
|
| 263 |
+
'attentions': attentions,
|
| 264 |
+
'hiddens':hiddens,
|
| 265 |
+
}
|
ChatTTS/utils/gpu_utils.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import torch
|
| 3 |
+
import logging
|
| 4 |
+
|
| 5 |
+
def select_device(min_memory = 2048):
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
if torch.cuda.is_available():
|
| 8 |
+
available_gpus = []
|
| 9 |
+
for i in range(torch.cuda.device_count()):
|
| 10 |
+
props = torch.cuda.get_device_properties(i)
|
| 11 |
+
free_memory = props.total_memory - torch.cuda.memory_reserved(i)
|
| 12 |
+
available_gpus.append((i, free_memory))
|
| 13 |
+
selected_gpu, max_free_memory = max(available_gpus, key=lambda x: x[1])
|
| 14 |
+
device = torch.device(f'cuda:{selected_gpu}')
|
| 15 |
+
free_memory_mb = max_free_memory / (1024 * 1024)
|
| 16 |
+
if free_memory_mb < min_memory:
|
| 17 |
+
logger.log(logging.WARNING, f'GPU {selected_gpu} has {round(free_memory_mb, 2)} MB memory left.')
|
| 18 |
+
device = torch.device('cpu')
|
| 19 |
+
else:
|
| 20 |
+
logger.log(logging.WARNING, f'No GPU found, use CPU instead')
|
| 21 |
+
device = torch.device('cpu')
|
| 22 |
+
|
| 23 |
+
return device
|
ChatTTS/utils/infer_utils.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import re
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class CustomRepetitionPenaltyLogitsProcessorRepeat():
|
| 8 |
+
|
| 9 |
+
def __init__(self, penalty: float, max_input_ids, past_window):
|
| 10 |
+
if not isinstance(penalty, float) or not (penalty > 0):
|
| 11 |
+
raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")
|
| 12 |
+
|
| 13 |
+
self.penalty = penalty
|
| 14 |
+
self.max_input_ids = max_input_ids
|
| 15 |
+
self.past_window = past_window
|
| 16 |
+
|
| 17 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
|
| 18 |
+
|
| 19 |
+
input_ids = input_ids[:, -self.past_window:]
|
| 20 |
+
freq = F.one_hot(input_ids, scores.size(1)).sum(1)
|
| 21 |
+
freq[self.max_input_ids:] = 0
|
| 22 |
+
alpha = self.penalty**freq
|
| 23 |
+
scores = torch.where(scores < 0, scores*alpha, scores/alpha)
|
| 24 |
+
|
| 25 |
+
return scores
|
| 26 |
+
|
| 27 |
+
class CustomRepetitionPenaltyLogitsProcessor():
|
| 28 |
+
|
| 29 |
+
def __init__(self, penalty: float, max_input_ids, past_window):
|
| 30 |
+
if not isinstance(penalty, float) or not (penalty > 0):
|
| 31 |
+
raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")
|
| 32 |
+
|
| 33 |
+
self.penalty = penalty
|
| 34 |
+
self.max_input_ids = max_input_ids
|
| 35 |
+
self.past_window = past_window
|
| 36 |
+
|
| 37 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
|
| 38 |
+
|
| 39 |
+
input_ids = input_ids[:, -self.past_window:]
|
| 40 |
+
score = torch.gather(scores, 1, input_ids)
|
| 41 |
+
_score = score.detach().clone()
|
| 42 |
+
score = torch.where(score < 0, score * self.penalty, score / self.penalty)
|
| 43 |
+
score[input_ids>=self.max_input_ids] = _score[input_ids>=self.max_input_ids]
|
| 44 |
+
scores.scatter_(1, input_ids, score)
|
| 45 |
+
|
| 46 |
+
return scores
|
| 47 |
+
|
| 48 |
+
def count_invalid_characters(s):
|
| 49 |
+
|
| 50 |
+
s = re.sub(r'\[uv_break\]|\[laugh\]|\[lbreak\]', '', s)
|
| 51 |
+
pattern = re.compile(r'[^\u4e00-\u9fffA-Za-z,。、,\. ]')
|
| 52 |
+
non_alphabetic_chinese_chars = pattern.findall(s)
|
| 53 |
+
return set(non_alphabetic_chinese_chars)
|
| 54 |
+
|
| 55 |
+
def detect_language(sentence):
|
| 56 |
+
|
| 57 |
+
chinese_char_pattern = re.compile(r'[\u4e00-\u9fff]')
|
| 58 |
+
english_word_pattern = re.compile(r'\b[A-Za-z]+\b')
|
| 59 |
+
|
| 60 |
+
chinese_chars = chinese_char_pattern.findall(sentence)
|
| 61 |
+
english_words = english_word_pattern.findall(sentence)
|
| 62 |
+
|
| 63 |
+
if len(chinese_chars) > len(english_words):
|
| 64 |
+
return "zh"
|
| 65 |
+
else:
|
| 66 |
+
return "en"
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
character_map = {
|
| 70 |
+
':': ',',
|
| 71 |
+
';': ',',
|
| 72 |
+
'!': '。',
|
| 73 |
+
'(': ',',
|
| 74 |
+
')': ',',
|
| 75 |
+
'【': ',',
|
| 76 |
+
'】': ',',
|
| 77 |
+
'『': ',',
|
| 78 |
+
'』': ',',
|
| 79 |
+
'「': ',',
|
| 80 |
+
'」': ',',
|
| 81 |
+
'《': ',',
|
| 82 |
+
'》': ',',
|
| 83 |
+
'-': ',',
|
| 84 |
+
'‘': '',
|
| 85 |
+
'“': '',
|
| 86 |
+
'’': '',
|
| 87 |
+
'”': '',
|
| 88 |
+
':': ',',
|
| 89 |
+
';': ',',
|
| 90 |
+
'!': '.',
|
| 91 |
+
'(': ',',
|
| 92 |
+
')': ',',
|
| 93 |
+
'[': ',',
|
| 94 |
+
']': ',',
|
| 95 |
+
'>': ',',
|
| 96 |
+
'<': ',',
|
| 97 |
+
'-': ',',
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
halfwidth_2_fullwidth_map = {
|
| 101 |
+
'!': '!',
|
| 102 |
+
'"': '“',
|
| 103 |
+
"'": '‘',
|
| 104 |
+
'#': '#',
|
| 105 |
+
'$': '$',
|
| 106 |
+
'%': '%',
|
| 107 |
+
'&': '&',
|
| 108 |
+
'(': '(',
|
| 109 |
+
')': ')',
|
| 110 |
+
',': ',',
|
| 111 |
+
'-': '-',
|
| 112 |
+
'*': '*',
|
| 113 |
+
'+': '+',
|
| 114 |
+
'.': '。',
|
| 115 |
+
'/': '/',
|
| 116 |
+
':': ':',
|
| 117 |
+
';': ';',
|
| 118 |
+
'<': '<',
|
| 119 |
+
'=': '=',
|
| 120 |
+
'>': '>',
|
| 121 |
+
'?': '?',
|
| 122 |
+
'@': '@',
|
| 123 |
+
# '[': '[',
|
| 124 |
+
'\\': '\',
|
| 125 |
+
# ']': ']',
|
| 126 |
+
'^': '^',
|
| 127 |
+
# '_': '_',
|
| 128 |
+
'`': '`',
|
| 129 |
+
'{': '{',
|
| 130 |
+
'|': '|',
|
| 131 |
+
'}': '}',
|
| 132 |
+
'~': '~'
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
def apply_half2full_map(text):
|
| 136 |
+
translation_table = str.maketrans(halfwidth_2_fullwidth_map)
|
| 137 |
+
return text.translate(translation_table)
|
| 138 |
+
|
| 139 |
+
def apply_character_map(text):
|
| 140 |
+
translation_table = str.maketrans(character_map)
|
| 141 |
+
return text.translate(translation_table)
|
ChatTTS/utils/io_utils.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import os
|
| 3 |
+
import logging
|
| 4 |
+
|
| 5 |
+
def get_latest_modified_file(directory):
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
files = [os.path.join(directory, f) for f in os.listdir(directory)]
|
| 9 |
+
if not files:
|
| 10 |
+
logger.log(logging.WARNING, f'No files found in the directory: {directory}')
|
| 11 |
+
return None
|
| 12 |
+
latest_file = max(files, key=os.path.getmtime)
|
| 13 |
+
|
| 14 |
+
return latest_file
|
LICENSE
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution-NonCommercial 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution-NonCommercial 4.0 International Public
|
| 58 |
+
License
|
| 59 |
+
|
| 60 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 61 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 62 |
+
Attribution-NonCommercial 4.0 International Public License ("Public
|
| 63 |
+
License"). To the extent this Public License may be interpreted as a
|
| 64 |
+
contract, You are granted the Licensed Rights in consideration of Your
|
| 65 |
+
acceptance of these terms and conditions, and the Licensor grants You
|
| 66 |
+
such rights in consideration of benefits the Licensor receives from
|
| 67 |
+
making the Licensed Material available under these terms and
|
| 68 |
+
conditions.
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Section 1 -- Definitions.
|
| 72 |
+
|
| 73 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 74 |
+
Rights that is derived from or based upon the Licensed Material
|
| 75 |
+
and in which the Licensed Material is translated, altered,
|
| 76 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 77 |
+
permission under the Copyright and Similar Rights held by the
|
| 78 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 79 |
+
Material is a musical work, performance, or sound recording,
|
| 80 |
+
Adapted Material is always produced where the Licensed Material is
|
| 81 |
+
synched in timed relation with a moving image.
|
| 82 |
+
|
| 83 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 84 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 85 |
+
accordance with the terms and conditions of this Public License.
|
| 86 |
+
|
| 87 |
+
c. Copyright and Similar Rights means copyright and/or similar rights
|
| 88 |
+
closely related to copyright including, without limitation,
|
| 89 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 90 |
+
Rights, without regard to how the rights are labeled or
|
| 91 |
+
categorized. For purposes of this Public License, the rights
|
| 92 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 93 |
+
Rights.
|
| 94 |
+
d. Effective Technological Measures means those measures that, in the
|
| 95 |
+
absence of proper authority, may not be circumvented under laws
|
| 96 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 97 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 98 |
+
agreements.
|
| 99 |
+
|
| 100 |
+
e. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 101 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 102 |
+
that applies to Your use of the Licensed Material.
|
| 103 |
+
|
| 104 |
+
f. Licensed Material means the artistic or literary work, database,
|
| 105 |
+
or other material to which the Licensor applied this Public
|
| 106 |
+
License.
|
| 107 |
+
|
| 108 |
+
g. Licensed Rights means the rights granted to You subject to the
|
| 109 |
+
terms and conditions of this Public License, which are limited to
|
| 110 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 111 |
+
Licensed Material and that the Licensor has authority to license.
|
| 112 |
+
|
| 113 |
+
h. Licensor means the individual(s) or entity(ies) granting rights
|
| 114 |
+
under this Public License.
|
| 115 |
+
|
| 116 |
+
i. NonCommercial means not primarily intended for or directed towards
|
| 117 |
+
commercial advantage or monetary compensation. For purposes of
|
| 118 |
+
this Public License, the exchange of the Licensed Material for
|
| 119 |
+
other material subject to Copyright and Similar Rights by digital
|
| 120 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 121 |
+
no payment of monetary compensation in connection with the
|
| 122 |
+
exchange.
|
| 123 |
+
|
| 124 |
+
j. Share means to provide material to the public by any means or
|
| 125 |
+
process that requires permission under the Licensed Rights, such
|
| 126 |
+
as reproduction, public display, public performance, distribution,
|
| 127 |
+
dissemination, communication, or importation, and to make material
|
| 128 |
+
available to the public including in ways that members of the
|
| 129 |
+
public may access the material from a place and at a time
|
| 130 |
+
individually chosen by them.
|
| 131 |
+
|
| 132 |
+
k. Sui Generis Database Rights means rights other than copyright
|
| 133 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 134 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 135 |
+
as amended and/or succeeded, as well as other essentially
|
| 136 |
+
equivalent rights anywhere in the world.
|
| 137 |
+
|
| 138 |
+
l. You means the individual or entity exercising the Licensed Rights
|
| 139 |
+
under this Public License. Your has a corresponding meaning.
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
Section 2 -- Scope.
|
| 143 |
+
|
| 144 |
+
a. License grant.
|
| 145 |
+
|
| 146 |
+
1. Subject to the terms and conditions of this Public License,
|
| 147 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 148 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 149 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 150 |
+
|
| 151 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 152 |
+
in part, for NonCommercial purposes only; and
|
| 153 |
+
|
| 154 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 155 |
+
NonCommercial purposes only.
|
| 156 |
+
|
| 157 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 158 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 159 |
+
License does not apply, and You do not need to comply with
|
| 160 |
+
its terms and conditions.
|
| 161 |
+
|
| 162 |
+
3. Term. The term of this Public License is specified in Section
|
| 163 |
+
6(a).
|
| 164 |
+
|
| 165 |
+
4. Media and formats; technical modifications allowed. The
|
| 166 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 167 |
+
all media and formats whether now known or hereafter created,
|
| 168 |
+
and to make technical modifications necessary to do so. The
|
| 169 |
+
Licensor waives and/or agrees not to assert any right or
|
| 170 |
+
authority to forbid You from making technical modifications
|
| 171 |
+
necessary to exercise the Licensed Rights, including
|
| 172 |
+
technical modifications necessary to circumvent Effective
|
| 173 |
+
Technological Measures. For purposes of this Public License,
|
| 174 |
+
simply making modifications authorized by this Section 2(a)
|
| 175 |
+
(4) never produces Adapted Material.
|
| 176 |
+
|
| 177 |
+
5. Downstream recipients.
|
| 178 |
+
|
| 179 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 180 |
+
recipient of the Licensed Material automatically
|
| 181 |
+
receives an offer from the Licensor to exercise the
|
| 182 |
+
Licensed Rights under the terms and conditions of this
|
| 183 |
+
Public License.
|
| 184 |
+
|
| 185 |
+
b. No downstream restrictions. You may not offer or impose
|
| 186 |
+
any additional or different terms or conditions on, or
|
| 187 |
+
apply any Effective Technological Measures to, the
|
| 188 |
+
Licensed Material if doing so restricts exercise of the
|
| 189 |
+
Licensed Rights by any recipient of the Licensed
|
| 190 |
+
Material.
|
| 191 |
+
|
| 192 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 193 |
+
may be construed as permission to assert or imply that You
|
| 194 |
+
are, or that Your use of the Licensed Material is, connected
|
| 195 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 196 |
+
the Licensor or others designated to receive attribution as
|
| 197 |
+
provided in Section 3(a)(1)(A)(i).
|
| 198 |
+
|
| 199 |
+
b. Other rights.
|
| 200 |
+
|
| 201 |
+
1. Moral rights, such as the right of integrity, are not
|
| 202 |
+
licensed under this Public License, nor are publicity,
|
| 203 |
+
privacy, and/or other similar personality rights; however, to
|
| 204 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 205 |
+
assert any such rights held by the Licensor to the limited
|
| 206 |
+
extent necessary to allow You to exercise the Licensed
|
| 207 |
+
Rights, but not otherwise.
|
| 208 |
+
|
| 209 |
+
2. Patent and trademark rights are not licensed under this
|
| 210 |
+
Public License.
|
| 211 |
+
|
| 212 |
+
3. To the extent possible, the Licensor waives any right to
|
| 213 |
+
collect royalties from You for the exercise of the Licensed
|
| 214 |
+
Rights, whether directly or through a collecting society
|
| 215 |
+
under any voluntary or waivable statutory or compulsory
|
| 216 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 217 |
+
reserves any right to collect such royalties, including when
|
| 218 |
+
the Licensed Material is used other than for NonCommercial
|
| 219 |
+
purposes.
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
Section 3 -- License Conditions.
|
| 223 |
+
|
| 224 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 225 |
+
following conditions.
|
| 226 |
+
|
| 227 |
+
a. Attribution.
|
| 228 |
+
|
| 229 |
+
1. If You Share the Licensed Material (including in modified
|
| 230 |
+
form), You must:
|
| 231 |
+
|
| 232 |
+
a. retain the following if it is supplied by the Licensor
|
| 233 |
+
with the Licensed Material:
|
| 234 |
+
|
| 235 |
+
i. identification of the creator(s) of the Licensed
|
| 236 |
+
Material and any others designated to receive
|
| 237 |
+
attribution, in any reasonable manner requested by
|
| 238 |
+
the Licensor (including by pseudonym if
|
| 239 |
+
designated);
|
| 240 |
+
|
| 241 |
+
ii. a copyright notice;
|
| 242 |
+
|
| 243 |
+
iii. a notice that refers to this Public License;
|
| 244 |
+
|
| 245 |
+
iv. a notice that refers to the disclaimer of
|
| 246 |
+
warranties;
|
| 247 |
+
|
| 248 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 249 |
+
extent reasonably practicable;
|
| 250 |
+
|
| 251 |
+
b. indicate if You modified the Licensed Material and
|
| 252 |
+
retain an indication of any previous modifications; and
|
| 253 |
+
|
| 254 |
+
c. indicate the Licensed Material is licensed under this
|
| 255 |
+
Public License, and include the text of, or the URI or
|
| 256 |
+
hyperlink to, this Public License.
|
| 257 |
+
|
| 258 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 259 |
+
reasonable manner based on the medium, means, and context in
|
| 260 |
+
which You Share the Licensed Material. For example, it may be
|
| 261 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 262 |
+
hyperlink to a resource that includes the required
|
| 263 |
+
information.
|
| 264 |
+
|
| 265 |
+
3. If requested by the Licensor, You must remove any of the
|
| 266 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 267 |
+
reasonably practicable.
|
| 268 |
+
|
| 269 |
+
4. If You Share Adapted Material You produce, the Adapter's
|
| 270 |
+
License You apply must not prevent recipients of the Adapted
|
| 271 |
+
Material from complying with this Public License.
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
Section 4 -- Sui Generis Database Rights.
|
| 275 |
+
|
| 276 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 277 |
+
apply to Your use of the Licensed Material:
|
| 278 |
+
|
| 279 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 280 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 281 |
+
portion of the contents of the database for NonCommercial purposes
|
| 282 |
+
only;
|
| 283 |
+
|
| 284 |
+
b. if You include all or a substantial portion of the database
|
| 285 |
+
contents in a database in which You have Sui Generis Database
|
| 286 |
+
Rights, then the database in which You have Sui Generis Database
|
| 287 |
+
Rights (but not its individual contents) is Adapted Material; and
|
| 288 |
+
|
| 289 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 290 |
+
all or a substantial portion of the contents of the database.
|
| 291 |
+
|
| 292 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 293 |
+
replace Your obligations under this Public License where the Licensed
|
| 294 |
+
Rights include other Copyright and Similar Rights.
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 298 |
+
|
| 299 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 300 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 301 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 302 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 303 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 304 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 305 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 306 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 307 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 308 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 309 |
+
|
| 310 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 311 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 312 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 313 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 314 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 315 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 316 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 317 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 318 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 319 |
+
|
| 320 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 321 |
+
above shall be interpreted in a manner that, to the extent
|
| 322 |
+
possible, most closely approximates an absolute disclaimer and
|
| 323 |
+
waiver of all liability.
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
Section 6 -- Term and Termination.
|
| 327 |
+
|
| 328 |
+
a. This Public License applies for the term of the Copyright and
|
| 329 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 330 |
+
this Public License, then Your rights under this Public License
|
| 331 |
+
terminate automatically.
|
| 332 |
+
|
| 333 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 334 |
+
Section 6(a), it reinstates:
|
| 335 |
+
|
| 336 |
+
1. automatically as of the date the violation is cured, provided
|
| 337 |
+
it is cured within 30 days of Your discovery of the
|
| 338 |
+
violation; or
|
| 339 |
+
|
| 340 |
+
2. upon express reinstatement by the Licensor.
|
| 341 |
+
|
| 342 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 343 |
+
right the Licensor may have to seek remedies for Your violations
|
| 344 |
+
of this Public License.
|
| 345 |
+
|
| 346 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 347 |
+
Licensed Material under separate terms or conditions or stop
|
| 348 |
+
distributing the Licensed Material at any time; however, doing so
|
| 349 |
+
will not terminate this Public License.
|
| 350 |
+
|
| 351 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 352 |
+
License.
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
Section 7 -- Other Terms and Conditions.
|
| 356 |
+
|
| 357 |
+
a. The Licensor shall not be bound by any additional or different
|
| 358 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 359 |
+
|
| 360 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 361 |
+
Licensed Material not stated herein are separate from and
|
| 362 |
+
independent of the terms and conditions of this Public License.
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
Section 8 -- Interpretation.
|
| 366 |
+
|
| 367 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 368 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 369 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 370 |
+
be made without permission under this Public License.
|
| 371 |
+
|
| 372 |
+
b. To the extent possible, if any provision of this Public License is
|
| 373 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 374 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 375 |
+
cannot be reformed, it shall be severed from this Public License
|
| 376 |
+
without affecting the enforceability of the remaining terms and
|
| 377 |
+
conditions.
|
| 378 |
+
|
| 379 |
+
c. No term or condition of this Public License will be waived and no
|
| 380 |
+
failure to comply consented to unless expressly agreed to by the
|
| 381 |
+
Licensor.
|
| 382 |
+
|
| 383 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 384 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 385 |
+
that apply to the Licensor or You, including from the legal
|
| 386 |
+
processes of any jurisdiction or authority.
|
| 387 |
+
|
| 388 |
+
=======================================================================
|
| 389 |
+
|
| 390 |
+
Creative Commons is not a party to its public
|
| 391 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 392 |
+
its public licenses to material it publishes and in those instances
|
| 393 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 394 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 395 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 396 |
+
material is shared under a Creative Commons public license or as
|
| 397 |
+
otherwise permitted by the Creative Commons policies published at
|
| 398 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 399 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 400 |
+
of Creative Commons without its prior written consent including,
|
| 401 |
+
without limitation, in connection with any unauthorized modifications
|
| 402 |
+
to any of its public licenses or any other arrangements,
|
| 403 |
+
understandings, or agreements concerning use of licensed material. For
|
| 404 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 405 |
+
public licenses.
|
| 406 |
+
|
| 407 |
+
Creative Commons may be contacted at creativecommons.org.
|
README_CN.md
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ChatTTS
|
| 2 |
+
[**English**](./README.md) | [**中文简体**](./README_CN.md)
|
| 3 |
+
|
| 4 |
+
ChatTTS是专门为对话场景设计的文本转语音模型,例如LLM助手对话任务。它支持英文和中文两种语言。最大的模型使用了10万小时以上的中英文数据进行训练。在HuggingFace中开源的版本为4万小时训练且未SFT的版本.
|
| 5 |
+
|
| 6 |
+
如需就模型进行正式商业咨询,请发送邮件至 **open-source@2noise.com**。对于中文用户,您可以加入我们的QQ群:~~808364215 (已满)~~ 230696694 (二群) 进行讨论。同时欢迎在GitHub上提出问题。如果遇到无法使用 **[HuggingFace](https://huggingface.co/2Noise/ChatTTS)** 的情况,可以在 [modelscope](https://www.modelscope.cn/models/pzc163/chatTTS) 上进行下载.
|
| 7 |
+
|
| 8 |
+
---
|
| 9 |
+
## 亮点
|
| 10 |
+
1. **对话式 TTS**: ChatTTS针对对话式任务进行了优化,实现了自然流畅的语音合成,同时支持多说话人。
|
| 11 |
+
2. **细粒度控制**: 该模型能够预测和控制细粒度的韵律特征,包括笑声、停顿和插入词等。
|
| 12 |
+
3. **更好的韵律**: ChatTTS在韵律方面超越了大部分开源TTS模型。同时提供预训练模型,支持进一步的研究。
|
| 13 |
+
|
| 14 |
+
对于模型的具体介绍, 可以参考B站的 **[宣传视频](https://www.bilibili.com/video/BV1zn4y1o7iV)**
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## 免责声明
|
| 19 |
+
本文件中的信息仅供学术交流使用。其目的是用于教育和研究,不得用于任何商业或法律目的。作者不保证信息的准确性、完整性或可靠性。本文件中使用的信息和数据,仅用于学术研究目的。这些数据来自公开可用的来源,作者不对数据的所有权或版权提出任何主张。
|
| 20 |
+
|
| 21 |
+
ChatTTS是一个强大的文本转语音系统。然而,负责任地和符合伦理地利用这项技术是非常重要的。为了限制ChatTTS的使用,我们在4w小时模型的训练过程中添加了少量额外的高频噪音,并用mp3格式尽可能压低了音质,以防不法分子用于潜在的犯罪可能。同时我们在内部训练了检测模型,并计划在未来开放。
|
| 22 |
+
|
| 23 |
+
---
|
| 24 |
+
## 用法
|
| 25 |
+
|
| 26 |
+
<h4>基本用法</h4>
|
| 27 |
+
|
| 28 |
+
```python
|
| 29 |
+
import ChatTTS
|
| 30 |
+
from IPython.display import Audio
|
| 31 |
+
|
| 32 |
+
chat = ChatTTS.Chat()
|
| 33 |
+
chat.load_models(compile=False) # 设置为True以获得更快速度
|
| 34 |
+
|
| 35 |
+
texts = ["在这里输入你的文本",]
|
| 36 |
+
|
| 37 |
+
wavs = chat.infer(texts, use_decoder=True)
|
| 38 |
+
|
| 39 |
+
torchaudio.save("output1.wav", torch.from_numpy(wavs[0]), 24000)
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
<h4>进阶用法</h4>
|
| 43 |
+
|
| 44 |
+
```python
|
| 45 |
+
###################################
|
| 46 |
+
# Sample a speaker from Gaussian.
|
| 47 |
+
|
| 48 |
+
rand_spk = chat.sample_random_speaker()
|
| 49 |
+
|
| 50 |
+
params_infer_code = {
|
| 51 |
+
'spk_emb': rand_spk, # add sampled speaker
|
| 52 |
+
'temperature': .3, # using custom temperature
|
| 53 |
+
'top_P': 0.7, # top P decode
|
| 54 |
+
'top_K': 20, # top K decode
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
###################################
|
| 58 |
+
# For sentence level manual control.
|
| 59 |
+
|
| 60 |
+
# use oral_(0-9), laugh_(0-2), break_(0-7)
|
| 61 |
+
# to generate special token in text to synthesize.
|
| 62 |
+
params_refine_text = {
|
| 63 |
+
'prompt': '[oral_2][laugh_0][break_6]'
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
wav = chat.infer(texts, params_refine_text=params_refine_text, params_infer_code=params_infer_code)
|
| 67 |
+
|
| 68 |
+
###################################
|
| 69 |
+
# For word level manual control.
|
| 70 |
+
# use_decoder=False to infer faster with a bit worse quality
|
| 71 |
+
text = 'What is [uv_break]your favorite english food?[laugh][lbreak]'
|
| 72 |
+
wav = chat.infer(text, skip_refine_text=True, params_infer_code=params_infer_code, use_decoder=False)
|
| 73 |
+
|
| 74 |
+
torchaudio.save("output2.wav", torch.from_numpy(wavs[0]), 24000)
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
<details open>
|
| 78 |
+
<summary><h4>自我介绍样例</h4></summary>
|
| 79 |
+
|
| 80 |
+
```python
|
| 81 |
+
inputs_cn = """
|
| 82 |
+
chat T T S 是一款强大的对话式文本转语音模型。它有中英混读和多说话人的能力。
|
| 83 |
+
chat T T S 不仅能够生成自然流畅的语音,还能控制[laugh]笑声啊[laugh],
|
| 84 |
+
停顿啊[uv_break]语气词啊等副语言现象[uv_break]。这个韵律超越了许多开源模型[uv_break]。
|
| 85 |
+
请注意,chat T T S 的使用应遵守法律和伦理准则,避免滥用的安全风险。[uv_break]'
|
| 86 |
+
""".replace('\n', '')
|
| 87 |
+
|
| 88 |
+
params_refine_text = {
|
| 89 |
+
'prompt': '[oral_2][laugh_0][break_4]'
|
| 90 |
+
}
|
| 91 |
+
audio_array_cn = chat.infer(inputs_cn, params_refine_text=params_refine_text)
|
| 92 |
+
# audio_array_en = chat.infer(inputs_en, params_refine_text=params_refine_text)
|
| 93 |
+
|
| 94 |
+
torchaudio.save("output3.wav", torch.from_numpy(audio_array_cn[0]), 24000)
|
| 95 |
+
```
|
| 96 |
+
[男说话人](https://github.com/2noise/ChatTTS/assets/130631963/bbfa3b83-2b67-4bb6-9315-64c992b63788)
|
| 97 |
+
|
| 98 |
+
[女说话人](https://github.com/2noise/ChatTTS/assets/130631963/e061f230-0e05-45e6-8e4e-0189f2d260c4)
|
| 99 |
+
</details>
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
---
|
| 103 |
+
## 计划路线
|
| 104 |
+
- [x] 开源4w小时基础模型和spk_stats文件
|
| 105 |
+
- [ ] 开源VQ encoder和Lora 训练代码
|
| 106 |
+
- [ ] 在非refine text情况下, 流式生成音频*
|
| 107 |
+
- [ ] 开源多情感可控的4w小时版本
|
| 108 |
+
- [ ] ChatTTS.cpp maybe? (欢迎社区PR或独立的新repo)
|
| 109 |
+
|
| 110 |
+
---
|
| 111 |
+
## 常见问题
|
| 112 |
+
|
| 113 |
+
##### 连不上HuggingFace
|
| 114 |
+
请使用[modelscope](https://www.modelscope.cn/models/pzc163/chatTTS)的版本. 并设置cache的位置:
|
| 115 |
+
```python
|
| 116 |
+
chat.load_models(source='local', local_path='你的下载位置')
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
##### 我要多少显存? Infer的速度是怎么样的?
|
| 120 |
+
对于30s的音频, 至少需要4G的显存. 对于4090, 1s生成约7个字所对应的音频. RTF约0.3.
|
| 121 |
+
|
| 122 |
+
##### 模型稳定性似乎不够好, 会出现其他说话人或音质很差的现象.
|
| 123 |
+
这是自回归模型通常都会出现的问题. 说话人可能会在中间变化, 可能会采样到音质非常差的结果, 这通常难以避免. 可以多采样几次来找到合适的结果.
|
| 124 |
+
|
| 125 |
+
##### 除了笑声还能控制什么吗? 还能控制其他情感吗?
|
| 126 |
+
在现在放出的模型版本中, 只有[laugh]和[uv_break], [lbreak]作为字级别的控制单元. 在未来的版本中我们可能会开源其他情感控制的版本.
|
| 127 |
+
|
| 128 |
+
---
|
| 129 |
+
## 致谢
|
| 130 |
+
- [bark](https://github.com/suno-ai/bark),[XTTSv2](https://github.com/coqui-ai/TTS)和[valle](https://arxiv.org/abs/2301.02111)展示了自回归任务用于TTS任务的可能性.
|
| 131 |
+
- [fish-speech](https://github.com/fishaudio/fish-speech)一个优秀的自回归TTS模型, 揭示了GVQ用于LLM任务的可能性.
|
| 132 |
+
- [vocos](https://github.com/gemelo-ai/vocos)作为模型中的vocoder.
|
| 133 |
+
|
| 134 |
+
---
|
| 135 |
+
## 特别致谢
|
| 136 |
+
- [wlu-audio lab](https://audio.westlake.edu.cn/)为我们提供了早期算法试验的支持.
|
example.ipynb
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
omegaconf~=2.3.0
|
| 2 |
+
torch~=2.1.0
|
| 3 |
+
tqdm
|
| 4 |
+
einops
|
| 5 |
+
vector_quantize_pytorch
|
| 6 |
+
transformers~=4.41.1
|
| 7 |
+
vocos
|
| 8 |
+
IPython
|
webui.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import random
|
| 3 |
+
import argparse
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import gradio as gr
|
| 7 |
+
import numpy as np
|
| 8 |
+
|
| 9 |
+
import ChatTTS
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def generate_seed():
|
| 13 |
+
new_seed = random.randint(1, 100000000)
|
| 14 |
+
return {
|
| 15 |
+
"__type__": "update",
|
| 16 |
+
"value": new_seed
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def generate_audio(text, temperature, top_P, top_K, audio_seed_input, text_seed_input, refine_text_flag):
|
| 21 |
+
|
| 22 |
+
torch.manual_seed(audio_seed_input)
|
| 23 |
+
rand_spk = chat.sample_random_speaker()
|
| 24 |
+
params_infer_code = {
|
| 25 |
+
'spk_emb': rand_spk,
|
| 26 |
+
'temperature': temperature,
|
| 27 |
+
'top_P': top_P,
|
| 28 |
+
'top_K': top_K,
|
| 29 |
+
}
|
| 30 |
+
params_refine_text = {'prompt': '[oral_2][laugh_0][break_6]'}
|
| 31 |
+
|
| 32 |
+
torch.manual_seed(text_seed_input)
|
| 33 |
+
|
| 34 |
+
if refine_text_flag:
|
| 35 |
+
text = chat.infer(text,
|
| 36 |
+
skip_refine_text=False,
|
| 37 |
+
refine_text_only=True,
|
| 38 |
+
params_refine_text=params_refine_text,
|
| 39 |
+
params_infer_code=params_infer_code
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
wav = chat.infer(text,
|
| 43 |
+
skip_refine_text=True,
|
| 44 |
+
params_refine_text=params_refine_text,
|
| 45 |
+
params_infer_code=params_infer_code
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
audio_data = np.array(wav[0]).flatten()
|
| 49 |
+
sample_rate = 24000
|
| 50 |
+
text_data = text[0] if isinstance(text, list) else text
|
| 51 |
+
|
| 52 |
+
return [(sample_rate, audio_data), text_data]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def main():
|
| 56 |
+
|
| 57 |
+
with gr.Blocks() as demo:
|
| 58 |
+
gr.Markdown("# ChatTTS Webui")
|
| 59 |
+
gr.Markdown("ChatTTS Model: [2noise/ChatTTS](https://github.com/2noise/ChatTTS)")
|
| 60 |
+
|
| 61 |
+
default_text = "四川美食确实以辣闻名,但也有不辣的选择。比如甜水面、赖汤圆、蛋烘糕、叶儿粑等,这些小吃口味温和,甜而不腻,也很受欢迎。"
|
| 62 |
+
text_input = gr.Textbox(label="Input Text", lines=4, placeholder="Please Input Text...", value=default_text)
|
| 63 |
+
|
| 64 |
+
with gr.Row():
|
| 65 |
+
refine_text_checkbox = gr.Checkbox(label="Refine text", value=True)
|
| 66 |
+
temperature_slider = gr.Slider(minimum=0.00001, maximum=1.0, step=0.00001, value=0.3, label="Audio temperature")
|
| 67 |
+
top_p_slider = gr.Slider(minimum=0.1, maximum=0.9, step=0.05, value=0.7, label="top_P")
|
| 68 |
+
top_k_slider = gr.Slider(minimum=1, maximum=20, step=1, value=20, label="top_K")
|
| 69 |
+
|
| 70 |
+
with gr.Row():
|
| 71 |
+
audio_seed_input = gr.Number(value=2, label="Audio Seed")
|
| 72 |
+
generate_audio_seed = gr.Button("\U0001F3B2")
|
| 73 |
+
text_seed_input = gr.Number(value=42, label="Text Seed")
|
| 74 |
+
generate_text_seed = gr.Button("\U0001F3B2")
|
| 75 |
+
|
| 76 |
+
generate_button = gr.Button("Generate")
|
| 77 |
+
|
| 78 |
+
text_output = gr.Textbox(label="Output Text", interactive=False)
|
| 79 |
+
audio_output = gr.Audio(label="Output Audio")
|
| 80 |
+
|
| 81 |
+
generate_audio_seed.click(generate_seed,
|
| 82 |
+
inputs=[],
|
| 83 |
+
outputs=audio_seed_input)
|
| 84 |
+
|
| 85 |
+
generate_text_seed.click(generate_seed,
|
| 86 |
+
inputs=[],
|
| 87 |
+
outputs=text_seed_input)
|
| 88 |
+
|
| 89 |
+
generate_button.click(generate_audio,
|
| 90 |
+
inputs=[text_input, temperature_slider, top_p_slider, top_k_slider, audio_seed_input, text_seed_input, refine_text_checkbox],
|
| 91 |
+
outputs=[audio_output, text_output])
|
| 92 |
+
|
| 93 |
+
parser = argparse.ArgumentParser(description='ChatTTS demo Launch')
|
| 94 |
+
parser.add_argument('--server_name', type=str, default='0.0.0.0', help='Server name')
|
| 95 |
+
parser.add_argument('--server_port', type=int, default=8080, help='Server port')
|
| 96 |
+
parser.add_argument('--local_path', type=str, default=None, help='the local_path if need')
|
| 97 |
+
args = parser.parse_args()
|
| 98 |
+
|
| 99 |
+
print("loading ChatTTS model...")
|
| 100 |
+
global chat
|
| 101 |
+
chat = ChatTTS.Chat()
|
| 102 |
+
|
| 103 |
+
if args.local_path == None:
|
| 104 |
+
chat.load_models()
|
| 105 |
+
else:
|
| 106 |
+
print('local model path:', args.local_path)
|
| 107 |
+
chat.load_models('local', local_path=args.local_path)
|
| 108 |
+
|
| 109 |
+
demo.launch(server_name=args.server_name, server_port=args.server_port, inbrowser=True)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
if __name__ == '__main__':
|
| 113 |
+
main()
|