Alpa v0.1 early preview: model, inference code, model card, samples
Browse files- .gitattributes +5 -0
- LICENSES/Apache-2.0.txt +202 -0
- README.md +131 -0
- THIRD_PARTY_NOTICES.md +23 -0
- alpa-v0.1-preview.pt +3 -0
- alpa/__init__.py +3 -0
- alpa/config.py +96 -0
- alpa/infer.py +63 -0
- alpa/model.py +377 -0
- alpa/text.py +275 -0
- samples/1_gita_1_1.wav +3 -0
- samples/2_retroflex_and_dental.wav +3 -0
- samples/3_everyday.wav +3 -0
- samples/4_aspirates.wav +3 -0
- samples/5_subhashita.wav +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
samples/1_gita_1_1.wav filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
samples/2_retroflex_and_dental.wav filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
samples/3_everyday.wav filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
samples/4_aspirates.wav filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
samples/5_subhashita.wav filter=lfs diff=lfs merge=lfs -text
|
LICENSES/Apache-2.0.txt
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- sa
|
| 5 |
+
pipeline_tag: text-to-speech
|
| 6 |
+
base_model: n4ze3m/tiny-voca
|
| 7 |
+
base_model_relation: finetune
|
| 8 |
+
datasets:
|
| 9 |
+
- ai4bharat/Rasa
|
| 10 |
+
tags:
|
| 11 |
+
- sanskrit
|
| 12 |
+
- text-to-speech
|
| 13 |
+
- tts
|
| 14 |
+
- tiny
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# Alpa (अल्प) — a very small Sanskrit text-to-speech model
|
| 18 |
+
|
| 19 |
+
**Early preview, v0.1.**
|
| 20 |
+
|
| 21 |
+
Alpa turns Sanskrit text into speech with a model of **4.5 million parameters** (an 18 MB file). On a four-core CPU it speaks about a hundred times faster than real time.
|
| 22 |
+
|
| 23 |
+
> **This is an early preview, not a finished model.** It has one trained voice, has learned from 3.5 hours of speech, and has been checked by one listener and one speech recogniser. Please read [the limits](#limits) before relying on it.
|
| 24 |
+
|
| 25 |
+
## Listen
|
| 26 |
+
|
| 27 |
+
| Text | Audio |
|
| 28 |
+
|---|---|
|
| 29 |
+
| धर्मक्षेत्रे कुरुक्षेत्रे समवेता युयुत्सवः। मामकाः पाण्डवाश्चैव किमकुर्वत सञ्जय॥ | <audio controls src="https://huggingface.co/77ethers/alpa/resolve/main/samples/1_gita_1_1.wav"></audio> |
|
| 30 |
+
| नटः पटं धरति, बालः पत्रं पठति। कटिः कति? दण्डः दन्तः च। | <audio controls src="https://huggingface.co/77ethers/alpa/resolve/main/samples/2_retroflex_and_dental.wav"></audio> |
|
| 31 |
+
| घटे जलं नास्ति। कोटिः जनाः नाटकं पश्यन्ति। गुडः मधुरः, पीडा कठिना। | <audio controls src="https://huggingface.co/77ethers/alpa/resolve/main/samples/3_everyday.wav"></audio> |
|
| 32 |
+
| यथा धर्मः तथा अर्थः। बुद्धिमान् भीमः सुखेन कथां कथयति। | <audio controls src="https://huggingface.co/77ethers/alpa/resolve/main/samples/4_aspirates.wav"></audio> |
|
| 33 |
+
| विद्या ददाति विनयं, विनयाद् याति पात्रताम्। | <audio controls src="https://huggingface.co/77ethers/alpa/resolve/main/samples/5_subhashita.wav"></audio> |
|
| 34 |
+
|
| 35 |
+
Each sample was generated once; nothing was re-rolled or selected. Verse is read aloud as text; Alpa does not chant.
|
| 36 |
+
|
| 37 |
+
## At a glance
|
| 38 |
+
|
| 39 |
+
| | |
|
| 40 |
+
|---|---|
|
| 41 |
+
| Parameters | 4,531,723 |
|
| 42 |
+
| File | `alpa-v0.1-preview.pt`, 18 MB |
|
| 43 |
+
| Audio | 24 kHz, mono |
|
| 44 |
+
| Speed | About 100 times faster than real time on four CPU threads |
|
| 45 |
+
| Input | Sanskrit in Devanagari |
|
| 46 |
+
| Voices | `male` is trained. `female` is present in the file but was never trained on Sanskrit |
|
| 47 |
+
| Licence | Apache-2.0 |
|
| 48 |
+
|
| 49 |
+
## Use it
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
pip install "torch>=2.4" numpy soundfile huggingface_hub
|
| 53 |
+
hf download 77ethers/alpa --local-dir alpa-model
|
| 54 |
+
cd alpa-model
|
| 55 |
+
python -m alpa.infer --checkpoint alpa-v0.1-preview.pt --voice male \
|
| 56 |
+
--text "सत्यमेव जयते नानृतम्।" --output out.wav
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
From Python, inside the same folder:
|
| 60 |
+
|
| 61 |
+
```python
|
| 62 |
+
import soundfile as sf
|
| 63 |
+
from alpa.infer import load_model, speak
|
| 64 |
+
|
| 65 |
+
model = load_model("alpa-v0.1-preview.pt")
|
| 66 |
+
audio, phonemes, _ = speak(model, "सत्यमेव जयते नानृतम्।", "male")
|
| 67 |
+
sf.write("out.wav", audio.numpy(), 24000)
|
| 68 |
+
```
|
| 69 |
+
|
| 70 |
+
`speak` also takes `speed`, `pitch_shift` and `pitch_range`. Text in Kannada, Telugu, Bengali or Malayalam script is read as well if you `pip install indic-transliteration`. The model file holds plain tensors and settings only and is loaded with `weights_only=True`.
|
| 71 |
+
|
| 72 |
+
## What the text reader does
|
| 73 |
+
|
| 74 |
+
- Every written vowel is spoken, including the final `a`: राम is *rāma*.
|
| 75 |
+
- An aspirated stop is spoken as the stop followed by ह.
|
| 76 |
+
- Anusvāra takes the class of the stop that follows it inside a word.
|
| 77 |
+
- Comma, daṇḍa, double daṇḍa, question mark and exclamation mark become pauses.
|
| 78 |
+
- Each syllable is marked laghu or guru, and the model is told which.
|
| 79 |
+
- The text is read as written. Sandhi is neither applied nor undone.
|
| 80 |
+
|
| 81 |
+
## How it was made
|
| 82 |
+
|
| 83 |
+
1. **Design and starting weights** come from [tiny-voca](https://huggingface.co/n4ze3m/tiny-voca), an English model of the same size by Muhammed Nazeem.
|
| 84 |
+
2. **Two small tables were rewritten by hand.** Each Sanskrit sound starts from its nearest English sound. ट ड ण ष start as 70% of their dental partner and 30% of English "r". Laghu and guru take the place of English stress.
|
| 85 |
+
3. **It was then trained for 3,000 steps** on 3.5 hours (1,396 clips) of the male voice of AI4Bharat's Rasa Sanskrit recordings, neutral styles only. This took about 100 minutes on a four-core CPU; no GPU was used.
|
| 86 |
+
|
| 87 |
+
Only the spectral losses of tiny-voca's recipe were used. Its adversarial stage, which adds much of the naturalness, has not been run on this model.
|
| 88 |
+
|
| 89 |
+
## How well does it pronounce?
|
| 90 |
+
|
| 91 |
+
A Sanskrit speech recogniser, [Su-śrotā](https://huggingface.co/prathoshap/sushrota-sanskrit-asr), listened to 148 sentences that were held out of training, spoken once by the model from text alone and once by the original voice artist.
|
| 92 |
+
|
| 93 |
+
| | Alpa, `male` | The real recordings |
|
| 94 |
+
|---|---:|---:|
|
| 95 |
+
| Phonemes it transcribed wrongly | 3.1% | 3.3% |
|
| 96 |
+
| ट heard as ट | 93% | 97% |
|
| 97 |
+
| ड heard as ड | 89% | 94% |
|
| 98 |
+
| ण heard as ण | 98% | 99% |
|
| 99 |
+
| त, द, न heard correctly | 98–100% | 98–100% |
|
| 100 |
+
|
| 101 |
+
Two cautions about these figures. The recogniser knows Sanskrit words and partly guesses from context, so they flatter both columns; counting only consonants between two vowels, it heard 18 of 21 ट and 17 of 21 ड from the model, against 21 and 19 from the recordings. And a recogniser is not an ear: one human listener has checked a handful of lines.
|
| 102 |
+
|
| 103 |
+
## Limits
|
| 104 |
+
|
| 105 |
+
- **One trained voice.** With `female`, ट and ड are noticeably weaker.
|
| 106 |
+
- **Little data and a short training run.** The voice can sound flatter or buzzier than larger systems.
|
| 107 |
+
- **ण and न** are not yet fully distinct to a human listener.
|
| 108 |
+
- **Word-final visarga is sometimes weak.** The recogniser missed it in three of the five samples above.
|
| 109 |
+
- **It speaks; it does not chant.** Vedic accent marks are ignored.
|
| 110 |
+
- **Untested:** single words on their own, very long compounds, long passages. One call takes up to 512 symbols; split longer text at sentence marks.
|
| 111 |
+
- **Digits and Latin letters are not spoken.**
|
| 112 |
+
|
| 113 |
+
## Credits and licence
|
| 114 |
+
|
| 115 |
+
Alpa is released under the Apache License 2.0. It stands on other people's work:
|
| 116 |
+
|
| 117 |
+
- **tiny-voca** by Muhammed Nazeem (Apache-2.0): the model design, the training recipe and the starting weights. Alpa's weights are derived from tiny-voca's. See `THIRD_PARTY_NOTICES.md`.
|
| 118 |
+
- **Rasa** by AI4Bharat (CC-BY-4.0): the recordings it learned from.
|
| 119 |
+
- **Vāgdhenu** and **Su-śrotā** by Prathosh A P, Indian Institute of Science: the pronunciation and scansion rules follow Vāgdhenu's, and Su-śrotā was used to measure the result. Neither is included here.
|
| 120 |
+
- **wav2vec2 phoneme recogniser** (`facebook/wav2vec2-lv-60-espeak-cv-ft`): used to time the sounds in the training recordings.
|
| 121 |
+
|
| 122 |
+
This is an independent project. None of the people or groups above has reviewed or endorsed it.
|
| 123 |
+
|
| 124 |
+
```bibtex
|
| 125 |
+
@inproceedings{ai4bharat2024rasa,
|
| 126 |
+
author={Praveen Srinivasa Varadhan and Ashwin Sankar and Giri Raju and Mitesh M. Khapra},
|
| 127 |
+
title={{Rasa: Building Expressive Speech Synthesis Systems for Indian Languages in Low-resource Settings}},
|
| 128 |
+
year=2024,
|
| 129 |
+
booktitle={Proc. INTERSPEECH 2024},
|
| 130 |
+
}
|
| 131 |
+
```
|
THIRD_PARTY_NOTICES.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Third-party notices
|
| 2 |
+
|
| 3 |
+
## tiny-voca
|
| 4 |
+
|
| 5 |
+
The model in this folder is derived from **tiny-voca** by Muhammed Nazeem
|
| 6 |
+
(https://github.com/n4ze3m/tiny-voca, commit `28e0722`; weights at https://huggingface.co/n4ze3m/tiny-voca),
|
| 7 |
+
licensed under the Apache License, Version 2.0. A copy of that licence is in `LICENSES/Apache-2.0.txt`.
|
| 8 |
+
|
| 9 |
+
- **Weights.** `alpa-v0.1-preview.pt` was trained starting from tiny-voca's published weights. Its phoneme
|
| 10 |
+
table and its stress table were rewritten for Sanskrit before training; every other layer started as a copy.
|
| 11 |
+
- **Code.** `alpa/model.py`, `alpa/config.py` and `alpa/infer.py` are adapted from tiny-voca's files of the
|
| 12 |
+
same names. Each states its changes at its top. `alpa/text.py` is original to Alpa.
|
| 13 |
+
|
| 14 |
+
## Vāgdhenu
|
| 15 |
+
|
| 16 |
+
`alpa/text.py` follows the pronunciation and scansion rules of **Vāgdhenu** by Prathosh A P
|
| 17 |
+
(https://github.com/prathoshap/vagdhenu), licensed under the Apache License, Version 2.0. It is a separate
|
| 18 |
+
implementation and includes no code from that project.
|
| 19 |
+
|
| 20 |
+
## Rasa
|
| 21 |
+
|
| 22 |
+
The model learned from recordings in **Rasa** by AI4Bharat (https://huggingface.co/datasets/ai4bharat/Rasa),
|
| 23 |
+
licensed under CC-BY-4.0. No recordings are included here.
|
alpa-v0.1-preview.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae052622c2b08c90fc979c16b1e7464834252facbea928eb5af18ae8509e073d
|
| 3 |
+
size 18186565
|
alpa/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Alpa: a small Sanskrit text-to-speech model."""
|
| 2 |
+
|
| 3 |
+
__version__ = "0.1.0"
|
alpa/config.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from tiny-voca by Muhammed Nazeem, commit 28e0722 (Apache-2.0; see LICENSES/Apache-2.0.txt
|
| 2 |
+
# and THIRD_PARTY_NOTICES.md). File: src/tiny_voca/config.py. Changes for Alpa: the voices and vocabulary size are
|
| 3 |
+
# Alpa's and versions start at 1; the fingerprint of the earlier byte-token features is dropped.
|
| 4 |
+
|
| 5 |
+
from dataclasses import asdict, dataclass
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
|
| 9 |
+
CHECKPOINT_FORMAT = 1
|
| 10 |
+
ANNOTATION_VERSION = 1
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _digest(value):
|
| 14 |
+
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass(frozen=True)
|
| 18 |
+
class Config:
|
| 19 |
+
# Acoustic features.
|
| 20 |
+
sample_rate: int = 24000
|
| 21 |
+
hop: int = 256
|
| 22 |
+
n_fft: int = 1024
|
| 23 |
+
n_mels: int = 80
|
| 24 |
+
f0_min: float = 50.0
|
| 25 |
+
f0_max: float = 600.0
|
| 26 |
+
speakers: tuple[str, ...] = ("female", "male")
|
| 27 |
+
# Phoneme tokens from alpa.text.SYMBOLS.
|
| 28 |
+
vocab_size: int = 47
|
| 29 |
+
# Text encoder and frame-level transformer.
|
| 30 |
+
channels: int = 192
|
| 31 |
+
heads: int = 4
|
| 32 |
+
feedforward: int = 640
|
| 33 |
+
kernel: int = 7
|
| 34 |
+
encoder_layers: int = 4
|
| 35 |
+
prosody_layers: int = 2
|
| 36 |
+
acoustic_layers: int = 5
|
| 37 |
+
dropout: float = 0.1
|
| 38 |
+
predictor_channels: int = 128
|
| 39 |
+
# Source-filter waveform head: harmonic branch and noise branch STFT sizes,
|
| 40 |
+
# causal/anticausal complex-cepstrum orders.
|
| 41 |
+
synth_fft: int = 512
|
| 42 |
+
synth_hop: int = 128
|
| 43 |
+
noise_fft: int = 256
|
| 44 |
+
noise_hop: int = 64
|
| 45 |
+
cep_causal: int = 80
|
| 46 |
+
cep_anti: int = 16
|
| 47 |
+
noise_cep: int = 32
|
| 48 |
+
harmonic_limit: float = 0.47
|
| 49 |
+
max_tokens: int = 512
|
| 50 |
+
max_frames: int = 3000
|
| 51 |
+
mel_mean: float = -5.0
|
| 52 |
+
mel_scale: float = 5.0
|
| 53 |
+
energy_mean: float = -4.0
|
| 54 |
+
energy_scale: float = 2.0
|
| 55 |
+
pitch_reference_hz: float = 200.0
|
| 56 |
+
|
| 57 |
+
def __post_init__(self):
|
| 58 |
+
from .text import SYMBOLS
|
| 59 |
+
if self.vocab_size != len(SYMBOLS):
|
| 60 |
+
raise ValueError(f"vocab_size must equal the phoneme inventory size {len(SYMBOLS)}")
|
| 61 |
+
if self.channels % self.heads or (self.channels // self.heads) % 2:
|
| 62 |
+
raise ValueError("channels must be divisible by heads with an even head width")
|
| 63 |
+
if not self.speakers or len(set(self.speakers)) != len(self.speakers):
|
| 64 |
+
raise ValueError("speaker names must be unique and nonempty")
|
| 65 |
+
for fft, hop in ((self.synth_fft, self.synth_hop), (self.noise_fft, self.noise_hop)):
|
| 66 |
+
if fft != 4 * hop or self.hop % hop:
|
| 67 |
+
raise ValueError("synthesis windows need 4x overlap and a hop dividing the acoustic hop")
|
| 68 |
+
if self.cep_causal + self.cep_anti >= self.synth_fft // 2 or self.noise_cep >= self.noise_fft // 2:
|
| 69 |
+
raise ValueError("cepstrum orders must be well below half the synthesis FFT size")
|
| 70 |
+
if min(self.encoder_layers, self.prosody_layers, self.acoustic_layers, self.kernel) < 1 or not self.kernel % 2:
|
| 71 |
+
raise ValueError("layer counts must be positive and the kernel odd")
|
| 72 |
+
if min(self.mel_scale, self.energy_scale) <= 0 or not 0 < self.harmonic_limit < 0.5:
|
| 73 |
+
raise ValueError("invalid normalization scale or harmonic limit")
|
| 74 |
+
|
| 75 |
+
def to_dict(self):
|
| 76 |
+
return asdict(self)
|
| 77 |
+
|
| 78 |
+
@classmethod
|
| 79 |
+
def from_dict(cls, value):
|
| 80 |
+
return cls(**{**value, "speakers": tuple(value["speakers"])})
|
| 81 |
+
|
| 82 |
+
@property
|
| 83 |
+
def fingerprint(self):
|
| 84 |
+
return _digest(self.to_dict())
|
| 85 |
+
|
| 86 |
+
@property
|
| 87 |
+
def acoustic_config(self):
|
| 88 |
+
"""Settings that determine stored wave/mel/prosody tensors. Tokens and
|
| 89 |
+
durations are versioned separately by the annotation layer."""
|
| 90 |
+
fields = ("sample_rate", "hop", "n_fft", "n_mels", "f0_min", "f0_max")
|
| 91 |
+
return {**{key: getattr(self, key) for key in fields}, "feature_version": 1,
|
| 92 |
+
"pitch_reference_hz": 200, "log_floor": 1e-5}
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def acoustic_fingerprint(self):
|
| 96 |
+
return _digest(self.acoustic_config)
|
alpa/infer.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from tiny-voca by Muhammed Nazeem, commit 28e0722 (Apache-2.0; see LICENSES/Apache-2.0.txt
|
| 2 |
+
# and THIRD_PARTY_NOTICES.md). File: src/tiny_voca/infer.py. Changes for Alpa: Alpa's model and syllable weight; one
|
| 3 |
+
# front end, so the options for choosing between two are removed; the voice is checked against the checkpoint's own list.
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
import torch
|
| 8 |
+
from .config import CHECKPOINT_FORMAT, Config
|
| 9 |
+
from .model import Alpa
|
| 10 |
+
from .text import frontend_fingerprint, phonemize
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def load_model(path, device="cpu"):
|
| 14 |
+
state = torch.load(path, map_location="cpu", weights_only=True)
|
| 15 |
+
if state.get("format_version") != CHECKPOINT_FORMAT:
|
| 16 |
+
raise ValueError("unsupported checkpoint format")
|
| 17 |
+
if state["frontend"]["fingerprint"] != frontend_fingerprint():
|
| 18 |
+
raise ValueError("checkpoint was trained with a different phoneme inventory/front end")
|
| 19 |
+
model = Alpa(Config.from_dict(state["config"]))
|
| 20 |
+
model.load_state_dict(state["model"], strict=True)
|
| 21 |
+
return model.to(device).eval()
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def speak(model, text, voice, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0):
|
| 25 |
+
"""Text -> (float waveform tensor, phonemized text). Same front end as training."""
|
| 26 |
+
device = next(model.parameters()).device
|
| 27 |
+
phones = phonemize(text)
|
| 28 |
+
if len(phones) > model.config.max_tokens:
|
| 29 |
+
raise ValueError("text is too long for one utterance; split it into sentences")
|
| 30 |
+
tokens = torch.tensor([phones.ids], device=device)
|
| 31 |
+
weight = torch.tensor([phones.weight], device=device)
|
| 32 |
+
if voice not in model.config.speakers:
|
| 33 |
+
raise ValueError(f"unknown voice {voice!r}; this model has {model.config.speakers}")
|
| 34 |
+
speaker = torch.tensor([model.config.speakers.index(voice)], device=device)
|
| 35 |
+
audio, lengths, details = model.synthesize(tokens, weight, torch.tensor([len(phones)], device=device), speaker, speed, pitch_shift, seed, pitch_range)
|
| 36 |
+
if not torch.isfinite(audio).all():
|
| 37 |
+
raise RuntimeError("model produced non-finite audio")
|
| 38 |
+
return audio[0, :int(lengths[0])].cpu(), phones, details
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def main():
|
| 42 |
+
parser = argparse.ArgumentParser(description="Generate Sanskrit speech from a trained Alpa inference checkpoint")
|
| 43 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 44 |
+
parser.add_argument("--text", required=True)
|
| 45 |
+
parser.add_argument("--voice", help="One of the model's voices (default: its first)")
|
| 46 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 47 |
+
parser.add_argument("--speed", type=float, default=1)
|
| 48 |
+
parser.add_argument("--pitch-shift", type=float, default=0, help="Added to log F0 (0.058 is about one semitone)")
|
| 49 |
+
parser.add_argument("--pitch-range", type=float, default=1, help="Scale pitch movement around the utterance mean (1 = as predicted)")
|
| 50 |
+
parser.add_argument("--device", default="cpu")
|
| 51 |
+
args = parser.parse_args()
|
| 52 |
+
import soundfile as sf
|
| 53 |
+
model = load_model(args.checkpoint, args.device)
|
| 54 |
+
wave, phones, _ = speak(model, args.text, args.voice or model.config.speakers[0], args.speed, args.pitch_shift, pitch_range=args.pitch_range)
|
| 55 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 56 |
+
# Floating WAV preserves output for inspection; no silent clipping or normalization.
|
| 57 |
+
sf.write(args.output, wave.numpy(), model.config.sample_rate, subtype="FLOAT")
|
| 58 |
+
print(" ".join(phones.symbols))
|
| 59 |
+
print(f"Saved {len(wave)/model.config.sample_rate:.2f}s; peak={float(wave.abs().max()):.4f}")
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
if __name__ == "__main__":
|
| 63 |
+
main()
|
alpa/model.py
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from tiny-voca by Muhammed Nazeem, commit 28e0722 (Apache-2.0; see LICENSES/Apache-2.0.txt
|
| 2 |
+
# and THIRD_PARTY_NOTICES.md). File: src/tiny_voca/model.py. Changes for Alpa: the model is named Alpa; the per-token
|
| 3 |
+
# stress input is the syllable weight (0 none, 1 laghu, 2 guru); the docstring and one comment are reworded.
|
| 4 |
+
|
| 5 |
+
"""Alpa: feed-forward transformer with a pitch-driven source-filter head.
|
| 6 |
+
|
| 7 |
+
All weights start random. Tensor conventions: hidden [B,T,C], frame features
|
| 8 |
+
[B,T], waveform [B,S]. Acoustic frame i is centred on sample ``i * hop``.
|
| 9 |
+
|
| 10 |
+
phoneme + syllable weight + voice -> transformer encoder -> duration predictor
|
| 11 |
+
-> repeat tokens by duration (supervised durations while training)
|
| 12 |
+
-> frame transformer A -> pitch / voicing / energy
|
| 13 |
+
-> frame transformer B -> filter parameters
|
| 14 |
+
-> harmonic pulse train (from pitch) and noise, each shaped by its
|
| 15 |
+
time-varying filter in the STFT domain -> 24 kHz waveform
|
| 16 |
+
|
| 17 |
+
The waveform head has no neural upsampling network. The oscillator supplies the
|
| 18 |
+
periodicity that a tiny decoder cannot render reliably on its own; the network
|
| 19 |
+
only has to predict slowly varying spectral envelopes (complex cepstra), which
|
| 20 |
+
is what keeps the voice from sounding buzzy or unstable at this size.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
import math
|
| 24 |
+
import torch
|
| 25 |
+
from torch import nn
|
| 26 |
+
from torch.nn import functional as F
|
| 27 |
+
from .config import Config
|
| 28 |
+
from .text import PHONE_IDS, SYMBOL_TO_ID, VOWELS
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def length_mask(lengths, size=None):
|
| 32 |
+
size = int(lengths.max()) if size is None else size
|
| 33 |
+
return torch.arange(size, device=lengths.device)[None, :] < lengths[:, None]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def rotary_table(size, width, device):
|
| 37 |
+
inverse = 1.0 / (10000 ** (torch.arange(0, width, 2, device=device, dtype=torch.float32) / width))
|
| 38 |
+
angle = torch.arange(size, device=device, dtype=torch.float32)[:, None] * inverse[None]
|
| 39 |
+
return angle.cos(), angle.sin()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def rotate(x, table):
|
| 43 |
+
cos, sin = table[0].to(x.dtype), table[1].to(x.dtype)
|
| 44 |
+
a, b = x[..., 0::2], x[..., 1::2]
|
| 45 |
+
return torch.stack((a * cos - b * sin, a * sin + b * cos), dim=-1).flatten(-2)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class Attention(nn.Module):
|
| 49 |
+
"""Self-attention with rotary (relative) positions: no absolute position
|
| 50 |
+
table, so utterances longer than those seen in training stay in range."""
|
| 51 |
+
|
| 52 |
+
def __init__(self, channels, heads, dropout):
|
| 53 |
+
super().__init__()
|
| 54 |
+
self.heads, self.dropout = heads, dropout
|
| 55 |
+
self.qkv = nn.Linear(channels, 3 * channels)
|
| 56 |
+
self.out = nn.Linear(channels, channels)
|
| 57 |
+
|
| 58 |
+
def forward(self, x, mask, table):
|
| 59 |
+
b, t, c = x.shape
|
| 60 |
+
q, k, v = self.qkv(x).view(b, t, 3, self.heads, c // self.heads).permute(2, 0, 3, 1, 4)
|
| 61 |
+
y = F.scaled_dot_product_attention(rotate(q, table), rotate(k, table), v, attn_mask=mask[:, None, None, :],
|
| 62 |
+
dropout_p=self.dropout if self.training else 0.0)
|
| 63 |
+
return self.out(y.transpose(1, 2).reshape(b, t, c))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class Block(nn.Module):
|
| 67 |
+
def __init__(self, c):
|
| 68 |
+
super().__init__()
|
| 69 |
+
self.norm1 = nn.LayerNorm(c.channels)
|
| 70 |
+
self.attention = Attention(c.channels, c.heads, c.dropout)
|
| 71 |
+
self.norm2 = nn.LayerNorm(c.channels)
|
| 72 |
+
self.depthwise = nn.Conv1d(c.channels, c.channels, c.kernel, padding=c.kernel // 2, groups=c.channels)
|
| 73 |
+
self.expand = nn.Linear(c.channels, c.feedforward)
|
| 74 |
+
self.project = nn.Linear(c.feedforward, c.channels)
|
| 75 |
+
self.dropout = nn.Dropout(c.dropout)
|
| 76 |
+
|
| 77 |
+
def forward(self, x, mask, table):
|
| 78 |
+
keep = mask[..., None]
|
| 79 |
+
x = (x + self.dropout(self.attention(self.norm1(x), mask, table))) * keep
|
| 80 |
+
# Mask before the convolution so padded positions cannot leak back in.
|
| 81 |
+
y = self.depthwise((self.norm2(x) * keep).transpose(1, 2)).transpose(1, 2)
|
| 82 |
+
y = self.project(self.dropout(F.gelu(self.expand(y))))
|
| 83 |
+
return (x + self.dropout(y)) * keep
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class Stack(nn.Module):
|
| 87 |
+
def __init__(self, c, layers):
|
| 88 |
+
super().__init__()
|
| 89 |
+
self.blocks = nn.ModuleList(Block(c) for _ in range(layers))
|
| 90 |
+
self.norm = nn.LayerNorm(c.channels)
|
| 91 |
+
self.width = c.channels // c.heads
|
| 92 |
+
|
| 93 |
+
def forward(self, x, mask):
|
| 94 |
+
table = rotary_table(x.shape[1], self.width, x.device)
|
| 95 |
+
for block in self.blocks:
|
| 96 |
+
x = block(x, mask, table)
|
| 97 |
+
return self.norm(x) * mask[..., None]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class Predictor(nn.Module):
|
| 101 |
+
def __init__(self, c, outputs):
|
| 102 |
+
super().__init__()
|
| 103 |
+
p = c.predictor_channels
|
| 104 |
+
self.conv1 = nn.Conv1d(c.channels, p, 3, padding=1)
|
| 105 |
+
self.conv2 = nn.Conv1d(p, p, 3, padding=1)
|
| 106 |
+
self.norm1, self.norm2 = nn.LayerNorm(p), nn.LayerNorm(p)
|
| 107 |
+
self.drop = nn.Dropout(c.dropout)
|
| 108 |
+
self.out = nn.Linear(p, outputs)
|
| 109 |
+
|
| 110 |
+
def forward(self, x, mask):
|
| 111 |
+
x = self.conv1((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)
|
| 112 |
+
x = self.drop(self.norm1(F.gelu(x))) * mask[..., None]
|
| 113 |
+
x = self.conv2(x.transpose(1, 2)).transpose(1, 2)
|
| 114 |
+
x = self.drop(self.norm2(F.gelu(x)))
|
| 115 |
+
return self.out(x) * mask[..., None]
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def pulse_train(f0, sample_rate, hop, limit=0.47, phase=None):
|
| 119 |
+
"""Band-limited pulse train [B, T*hop] with unit power for f0 [B, T] in Hz.
|
| 120 |
+
|
| 121 |
+
Frame values sit at samples i*hop and are interpolated in the log domain.
|
| 122 |
+
The sum of all harmonics below ``limit * sample_rate`` has the closed form
|
| 123 |
+
sin((K+1/2)x) / (2 sin(x/2)) - 1/2; the top harmonic fades in and out so the
|
| 124 |
+
signal stays continuous while the pitch glides. Float64 keeps the phase exact.
|
| 125 |
+
"""
|
| 126 |
+
frames = f0.shape[1]
|
| 127 |
+
knots = torch.log(f0.double().clamp_min(1.0))
|
| 128 |
+
knots = torch.cat((knots, knots[:, -1:]), dim=1)
|
| 129 |
+
position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
|
| 130 |
+
low = position.floor().long()
|
| 131 |
+
weight = position - low
|
| 132 |
+
frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
|
| 133 |
+
theta = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
|
| 134 |
+
if phase is not None:
|
| 135 |
+
theta = theta + phase.double()[:, None]
|
| 136 |
+
theta = torch.remainder(theta, 2 * math.pi)
|
| 137 |
+
harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
|
| 138 |
+
whole = torch.floor(harmonics)
|
| 139 |
+
denominator = 2 * torch.sin(theta / 2)
|
| 140 |
+
tiny = denominator.abs() < 1e-7
|
| 141 |
+
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
|
| 142 |
+
wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta)
|
| 143 |
+
return (wave * torch.sqrt(2.0 / harmonics)).float()
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def filtered(source, response, fft, hop, window):
|
| 147 |
+
"""Time-varying filtering: response [B, frames, bins] multiplies the STFT."""
|
| 148 |
+
spectrum = torch.stft(source, fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
|
| 149 |
+
if spectrum.shape[-1] != response.shape[1]:
|
| 150 |
+
raise ValueError("filter frames do not match the source")
|
| 151 |
+
return torch.istft(spectrum * response.transpose(1, 2), fft, hop, window=window, center=True, length=source.shape[-1])
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def cepstrum_response(causal, anticausal, fft):
|
| 155 |
+
"""Complex frequency response exp(FFT(cepstrum)). Causal coefficients give a
|
| 156 |
+
minimum-phase filter; anticausal ones add a maximum-phase part (glottal pulse)."""
|
| 157 |
+
buffer = causal.new_zeros(*causal.shape[:-1], fft)
|
| 158 |
+
buffer[..., :causal.shape[-1]] = causal
|
| 159 |
+
if anticausal is not None:
|
| 160 |
+
buffer[..., fft - anticausal.shape[-1]:] = anticausal.flip(-1)
|
| 161 |
+
spectrum = torch.fft.rfft(buffer, dim=-1)
|
| 162 |
+
return torch.polar(torch.exp(spectrum.real.clamp(-14, 6)), spectrum.imag)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class SourceFilter(nn.Module):
|
| 166 |
+
"""Harmonic-plus-noise synthesis controlled per frame by two linear heads."""
|
| 167 |
+
|
| 168 |
+
def __init__(self, c):
|
| 169 |
+
super().__init__()
|
| 170 |
+
self.config = c
|
| 171 |
+
self.harmonic_steps, self.noise_steps = c.hop // c.synth_hop, c.hop // c.noise_hop
|
| 172 |
+
self.harmonic = nn.Linear(c.channels, c.cep_causal + c.cep_anti + self.harmonic_steps)
|
| 173 |
+
self.noise = nn.Linear(c.channels, c.noise_cep + self.noise_steps)
|
| 174 |
+
for head, level in ((self.harmonic, -1.5), (self.noise, -2.5)): # start quiet and spectrally flat
|
| 175 |
+
nn.init.zeros_(head.weight)
|
| 176 |
+
nn.init.zeros_(head.bias)
|
| 177 |
+
head.bias.data[0] = level
|
| 178 |
+
|
| 179 |
+
def lifter(size, first):
|
| 180 |
+
scale = 1.0 / (1.0 + torch.arange(first, first + size, dtype=torch.float32) / 4)
|
| 181 |
+
if first == 0:
|
| 182 |
+
scale[0] = 2.0 # coefficient 0 is the log gain and needs range
|
| 183 |
+
return scale
|
| 184 |
+
self.register_buffer("causal_lifter", lifter(c.cep_causal, 0), persistent=False)
|
| 185 |
+
self.register_buffer("anti_lifter", lifter(c.cep_anti, 1), persistent=False)
|
| 186 |
+
self.register_buffer("noise_lifter", lifter(c.noise_cep, 0), persistent=False)
|
| 187 |
+
self.register_buffer("synth_window", torch.hann_window(c.synth_fft), persistent=False)
|
| 188 |
+
self.register_buffer("noise_window", torch.hann_window(c.noise_fft), persistent=False)
|
| 189 |
+
|
| 190 |
+
@staticmethod
|
| 191 |
+
def _take(values, index):
|
| 192 |
+
return values.gather(1, index[..., None].expand(-1, -1, values.shape[-1]))
|
| 193 |
+
|
| 194 |
+
def _subframes(self, frames, lengths, steps):
|
| 195 |
+
"""Sub-frame j uses frame j // steps; indices past an utterance's end
|
| 196 |
+
repeat its last sub-frame so padding never changes real audio. Two extra
|
| 197 |
+
sub-frames cover the half window of zeros appended to the source."""
|
| 198 |
+
sub = torch.arange(steps * frames + 3, device=lengths.device)[None].expand(len(lengths), -1)
|
| 199 |
+
return torch.minimum(sub, (steps * lengths - 1).clamp_min(0)[:, None])
|
| 200 |
+
|
| 201 |
+
@staticmethod
|
| 202 |
+
def _shape(source, response, fft, hop, window):
|
| 203 |
+
# Half a window of trailing zeros: the frames around an utterance's end are
|
| 204 |
+
# then the same whether it stands alone or is padded inside a batch.
|
| 205 |
+
return filtered(F.pad(source, (0, fft // 2)), response, fft, hop, window)[:, :source.shape[-1]]
|
| 206 |
+
|
| 207 |
+
def forward(self, hidden, f0, lengths, noise=None, phase=None):
|
| 208 |
+
c = self.config
|
| 209 |
+
frames = hidden.shape[1]
|
| 210 |
+
samples = frames * c.hop
|
| 211 |
+
with torch.autocast(device_type=hidden.device.type, enabled=False):
|
| 212 |
+
hidden = hidden.float()
|
| 213 |
+
audible = length_mask(lengths * c.hop, samples)
|
| 214 |
+
last = (lengths - 1).clamp_min(0)[:, None]
|
| 215 |
+
# Harmonic branch: envelope interpolated between frames, one gain per sub-frame.
|
| 216 |
+
h = self.harmonic(hidden)
|
| 217 |
+
split = c.cep_causal + c.cep_anti
|
| 218 |
+
sub = self._subframes(frames, lengths, self.harmonic_steps)
|
| 219 |
+
here, following = sub // self.harmonic_steps, torch.minimum((sub + 1) // self.harmonic_steps, last)
|
| 220 |
+
causal = 0.5 * (self._take(h[..., :c.cep_causal], here) + self._take(h[..., :c.cep_causal], following)) * self.causal_lifter
|
| 221 |
+
anti = 0.5 * (self._take(h[..., c.cep_causal:split], here) + self._take(h[..., c.cep_causal:split], following)) * self.anti_lifter
|
| 222 |
+
causal = torch.cat((causal[..., :1] + h[..., split:].flatten(1).gather(1, sub)[..., None], causal[..., 1:]), dim=-1)
|
| 223 |
+
harmonic_response = cepstrum_response(causal, anti, c.synth_fft)
|
| 224 |
+
# Noise branch: one envelope per frame, four gains per frame for sharp consonants.
|
| 225 |
+
n = self.noise(hidden)
|
| 226 |
+
sub = self._subframes(frames, lengths, self.noise_steps)
|
| 227 |
+
shape = self._take(n[..., :c.noise_cep], sub // self.noise_steps) * self.noise_lifter
|
| 228 |
+
shape = torch.cat((shape[..., :1] + n[..., c.noise_cep:].flatten(1).gather(1, sub)[..., None], shape[..., 1:]), dim=-1)
|
| 229 |
+
noise_response = cepstrum_response(shape, None, c.noise_fft)
|
| 230 |
+
# Hold the last real pitch value through padding: the pitch glide inside an
|
| 231 |
+
# utterance's final frame must not depend on what the batch was padded with.
|
| 232 |
+
f0 = f0.float().gather(1, torch.minimum(torch.arange(frames, device=f0.device)[None], last))
|
| 233 |
+
pulses = pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, phase) * audible
|
| 234 |
+
if noise is None:
|
| 235 |
+
noise = torch.randn(hidden.shape[0], samples, device=hidden.device)
|
| 236 |
+
wave = (self._shape(pulses, harmonic_response, c.synth_fft, c.synth_hop, self.synth_window)
|
| 237 |
+
+ self._shape(noise * audible, noise_response, c.noise_fft, c.noise_hop, self.noise_window))
|
| 238 |
+
return wave * audible
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
class Alpa(nn.Module):
|
| 242 |
+
def __init__(self, config=Config()):
|
| 243 |
+
super().__init__()
|
| 244 |
+
self.config = c = config
|
| 245 |
+
self.tokens = nn.Embedding(c.vocab_size, c.channels, padding_idx=0)
|
| 246 |
+
self.syllable_weight = nn.Embedding(3, c.channels)
|
| 247 |
+
self.voices = nn.Embedding(len(c.speakers), c.channels)
|
| 248 |
+
self.encoder = Stack(c, c.encoder_layers)
|
| 249 |
+
# Outputs per token: log(1 + frames) and, for pause-capable tokens, a
|
| 250 |
+
# "has frames at all" logit. A pause is absent more often than present, so
|
| 251 |
+
# one regression would average both cases into a pause that is too short.
|
| 252 |
+
self.duration = Predictor(c, 2)
|
| 253 |
+
self.frame_voices = nn.Embedding(len(c.speakers), c.channels)
|
| 254 |
+
self.frame_position = nn.Linear(3, c.channels)
|
| 255 |
+
self.prosody_stack = Stack(c, c.prosody_layers)
|
| 256 |
+
# Outputs: continuous log(F0/reference), voiced logit, normalized log RMS energy.
|
| 257 |
+
self.prosody_head = nn.Linear(c.channels, 3)
|
| 258 |
+
self.prosody_projection = nn.Linear(3, c.channels)
|
| 259 |
+
self.acoustic_stack = Stack(c, c.acoustic_layers)
|
| 260 |
+
self.vocoder = SourceFilter(c)
|
| 261 |
+
phones = torch.zeros(c.vocab_size, dtype=torch.bool)
|
| 262 |
+
phones[sorted(PHONE_IDS)] = True
|
| 263 |
+
self.register_buffer("phone_tokens", phones, persistent=False)
|
| 264 |
+
vowels = torch.zeros(c.vocab_size, dtype=torch.bool)
|
| 265 |
+
vowels[[SYMBOL_TO_ID[v] for v in VOWELS]] = True
|
| 266 |
+
self.register_buffer("vowel_tokens", vowels, persistent=False)
|
| 267 |
+
|
| 268 |
+
def encode(self, tokens, weight, lengths, speakers):
|
| 269 |
+
if tokens.ndim != 2 or weight.shape != tokens.shape or lengths.shape != (tokens.shape[0],) or speakers.shape != lengths.shape:
|
| 270 |
+
raise ValueError("expected tokens[B,N], weight[B,N], lengths[B], speakers[B]")
|
| 271 |
+
if tokens.shape[1] > self.config.max_tokens or bool((lengths < 1).any()) or bool((lengths > tokens.shape[1]).any()):
|
| 272 |
+
raise ValueError("invalid token length")
|
| 273 |
+
mask = length_mask(lengths, tokens.shape[1])
|
| 274 |
+
x = (self.tokens(tokens) + self.syllable_weight(weight) + self.voices(speakers)[:, None]) * mask[..., None]
|
| 275 |
+
return self.encoder(x, mask), mask
|
| 276 |
+
|
| 277 |
+
def expand(self, hidden, durations, token_mask, speakers):
|
| 278 |
+
"""Repeat token states by integer durations (zero is allowed for pause
|
| 279 |
+
tokens) and add each frame's position inside its token."""
|
| 280 |
+
if durations.shape != token_mask.shape or bool((durations < 0).any()) or bool((durations[~token_mask] != 0).any()):
|
| 281 |
+
raise ValueError("invalid durations")
|
| 282 |
+
lengths = durations.sum(1)
|
| 283 |
+
if bool((lengths < 1).any()) or bool((lengths > self.config.max_frames).any()):
|
| 284 |
+
raise ValueError("utterance has no frames or exceeds max_frames; split the input")
|
| 285 |
+
size = int(lengths.max())
|
| 286 |
+
ends = durations.cumsum(1)
|
| 287 |
+
frame = torch.arange(size, device=hidden.device)[None].expand(hidden.shape[0], size).contiguous()
|
| 288 |
+
index = torch.searchsorted(ends, frame, right=True).clamp(max=durations.shape[1] - 1)
|
| 289 |
+
owned = durations.gather(1, index).clamp_min(1)
|
| 290 |
+
offset = frame - (ends.gather(1, index) - durations.gather(1, index))
|
| 291 |
+
relative = (offset.to(hidden.dtype) + 0.5) / owned
|
| 292 |
+
features = torch.stack((relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1)
|
| 293 |
+
mask = frame < lengths[:, None]
|
| 294 |
+
expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1]))
|
| 295 |
+
expanded = expanded + self.frame_position(features) + self.frame_voices(speakers)[:, None]
|
| 296 |
+
return expanded * mask[..., None], mask, lengths
|
| 297 |
+
|
| 298 |
+
def predict_prosody(self, frames, frame_mask):
|
| 299 |
+
hidden = self.prosody_stack(frames, frame_mask)
|
| 300 |
+
return hidden, self.prosody_head(hidden) * frame_mask[..., None]
|
| 301 |
+
|
| 302 |
+
@staticmethod
|
| 303 |
+
def hard_prosody(prediction):
|
| 304 |
+
"""What the decoder is conditioned on when it is not teacher-forced."""
|
| 305 |
+
return torch.stack((prediction[..., 0].clamp(-2.0, 1.5), (prediction[..., 1] > 0).to(prediction.dtype),
|
| 306 |
+
prediction[..., 2].clamp(-4.5, 3.0)), dim=-1)
|
| 307 |
+
|
| 308 |
+
def decode(self, prosody_hidden, condition, frame_mask):
|
| 309 |
+
return self.acoustic_stack((prosody_hidden + self.prosody_projection(condition)) * frame_mask[..., None], frame_mask)
|
| 310 |
+
|
| 311 |
+
def frequency(self, log_f0):
|
| 312 |
+
c = self.config
|
| 313 |
+
return (c.pitch_reference_hz * torch.exp(log_f0.float())).clamp(c.f0_min, c.f0_max)
|
| 314 |
+
|
| 315 |
+
def predict_durations(self, hidden, mask, tokens, speed=1.0):
|
| 316 |
+
output = self.duration(hidden, mask).float()
|
| 317 |
+
frames = torch.round(torch.expm1(output[..., 0].clamp(0, math.log1p(400))) / speed).long().clamp_min(1)
|
| 318 |
+
# Every phoneme is audible for at least one frame. A pause-capable token
|
| 319 |
+
# (edge, punctuation, word boundary) lasts its predicted length when the
|
| 320 |
+
# presence logit says there is a pause, and vanishes otherwise.
|
| 321 |
+
present = self.phone_tokens[tokens] | (output[..., 1] > 0)
|
| 322 |
+
return torch.where(present, frames, torch.zeros_like(frames)) * mask
|
| 323 |
+
|
| 324 |
+
@staticmethod
|
| 325 |
+
def widen_pitch(prosody, frame_mask, pitch_range):
|
| 326 |
+
"""Scale the predicted pitch movement around each utterance's voiced mean.
|
| 327 |
+
Regression predicts a flatter contour than the recordings; values above 1
|
| 328 |
+
restore range (1.0 leaves the prediction untouched)."""
|
| 329 |
+
if pitch_range == 1.0:
|
| 330 |
+
return prosody
|
| 331 |
+
voiced = ((prosody[..., 1] > 0) & frame_mask).to(prosody.dtype)
|
| 332 |
+
mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(1, keepdim=True).clamp_min(1)
|
| 333 |
+
pitch = mean + (prosody[..., 0] - mean) * pitch_range
|
| 334 |
+
return torch.cat((pitch[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
|
| 335 |
+
|
| 336 |
+
@staticmethod
|
| 337 |
+
def pitch_wobble(shape, semitones, seed, device):
|
| 338 |
+
"""Small random pitch movement for the oscillator only (log-F0 units).
|
| 339 |
+
Recorded pitch has cycle-to-cycle irregularity that a predicted contour
|
| 340 |
+
lacks; two thirds of the wobble is slow drift, one third frame-level."""
|
| 341 |
+
generator = torch.Generator().manual_seed(seed + 7919)
|
| 342 |
+
fast = torch.randn(shape, generator=generator)
|
| 343 |
+
slow = F.avg_pool1d(torch.randn(shape[0], 1, shape[1] + 16, generator=generator), 17, stride=1)[:, 0]
|
| 344 |
+
slow = slow / slow.std().clamp_min(1e-6)
|
| 345 |
+
return ((2 * slow + fast) / 5 ** 0.5 * semitones * math.log(2) / 12).to(device)
|
| 346 |
+
|
| 347 |
+
@torch.inference_mode()
|
| 348 |
+
def synthesize(self, tokens, weight, lengths, speakers, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, pitch_jitter=0.0,
|
| 349 |
+
voiced_vowels=True):
|
| 350 |
+
"""Free-running synthesis: predicted durations, pitch, voicing and energy."""
|
| 351 |
+
if not math.isfinite(speed) or not 0.25 <= speed <= 4 or not math.isfinite(pitch_shift) or not 0 <= pitch_range <= 3:
|
| 352 |
+
raise ValueError("speed must be between 0.25 and 4, pitch_shift finite and pitch_range between 0 and 3")
|
| 353 |
+
hidden, mask = self.encode(tokens, weight, lengths, speakers)
|
| 354 |
+
durations = self.predict_durations(hidden, mask, tokens, speed)
|
| 355 |
+
frames, frame_mask, frame_lengths = self.expand(hidden, durations, mask, speakers)
|
| 356 |
+
prosody_hidden, prosody = self.predict_prosody(frames, frame_mask)
|
| 357 |
+
if voiced_vowels:
|
| 358 |
+
# A pitch tracker can miss creaky vowels in the recordings, and the
|
| 359 |
+
# voicing predictor then learns to drop whole vowels (heard as a whispered
|
| 360 |
+
# word). A vowel is voiced by definition, so its frames are kept voiced.
|
| 361 |
+
ends = durations.cumsum(1)
|
| 362 |
+
frame = torch.arange(frames.shape[1], device=tokens.device)[None].expand(len(lengths), -1).contiguous()
|
| 363 |
+
owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
|
| 364 |
+
vowel = self.vowel_tokens[tokens].gather(1, owner) & frame_mask
|
| 365 |
+
prosody = torch.cat((prosody[..., :1], torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])[..., None],
|
| 366 |
+
prosody[..., 2:]), dim=-1)
|
| 367 |
+
prosody = self.widen_pitch(prosody, frame_mask, pitch_range)
|
| 368 |
+
acoustic = self.decode(prosody_hidden, self.hard_prosody(prosody), frame_mask)
|
| 369 |
+
# One noise stream per row, drawn at the row's own length, so batching or
|
| 370 |
+
# padding never changes a row's audio.
|
| 371 |
+
noise = nn.utils.rnn.pad_sequence([torch.randn(int(n) * self.config.hop, generator=torch.Generator().manual_seed(seed + row))
|
| 372 |
+
for row, n in enumerate(frame_lengths)], batch_first=True)
|
| 373 |
+
pitch = prosody[..., 0] + pitch_shift
|
| 374 |
+
if pitch_jitter > 0:
|
| 375 |
+
pitch = pitch + self.pitch_wobble(pitch.shape, pitch_jitter, seed, pitch.device)
|
| 376 |
+
wave = self.vocoder(acoustic, self.frequency(pitch), frame_lengths, noise=noise.to(hidden.device))
|
| 377 |
+
return wave, frame_lengths * self.config.hop, {"durations": durations, "prosody": prosody, "frame_mask": frame_mask}
|
alpa/text.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sanskrit front end: text -> phoneme tokens with a syllable-weight feature.
|
| 2 |
+
|
| 3 |
+
The same code runs during data preparation, training and inference, so tokens
|
| 4 |
+
can never drift between them.
|
| 5 |
+
|
| 6 |
+
* A token is one Sanskrit phoneme in SLP1 notation (``k``, ``A``, ``w``, ``z``,
|
| 7 |
+
...), so every retroflex and every long vowel is a single token.
|
| 8 |
+
* Syllable weight is a parallel feature carried by each syllable's vowel
|
| 9 |
+
(0 none, 1 laghu, 2 guru), not a token, so it never has to be given audio
|
| 10 |
+
frames. It takes the place of lexical stress in an English front end.
|
| 11 |
+
* Utterance edges, word boundaries and punctuation are explicit tokens that may
|
| 12 |
+
last zero frames. They carry pauses when the recording has them.
|
| 13 |
+
|
| 14 |
+
Devanagari is read directly: a consonant letter carries the inherent ``a``
|
| 15 |
+
unless a vowel sign or a virama follows, and that vowel is never dropped (no
|
| 16 |
+
Hindi-style schwa deletion). Other Brahmic scripts are first converted to
|
| 17 |
+
Devanagari with the optional ``indic-transliteration`` package.
|
| 18 |
+
|
| 19 |
+
Pronunciation rules applied here:
|
| 20 |
+
|
| 21 |
+
* Anusvara inside a word becomes the nasal of the following stop's class
|
| 22 |
+
(``aMga`` and ``aNga`` give the same tokens); before a pause it is ``m``.
|
| 23 |
+
Elsewhere it stays ``M`` and the model learns its sound from the recordings,
|
| 24 |
+
as it does for visarga ``H``.
|
| 25 |
+
* An aspirated stop is written as the plain stop followed by ``h``: ``K`` is
|
| 26 |
+
``k h``, ``D`` is ``d h``. The breath gets its own duration, and a model that
|
| 27 |
+
starts from English weights aspirates at once. For syllable weight the pair
|
| 28 |
+
still counts as one consonant.
|
| 29 |
+
* The rare vowels are written with frequent ones so they are never untrained:
|
| 30 |
+
long vocalic ``F`` is ``f`` with guru weight, vocalic ``x`` is ``l f``.
|
| 31 |
+
Retroflex ``L`` is ``l``. Candrabindu is read as anusvara.
|
| 32 |
+
* A colon typed after a letter that ends in a vowel is a visarga; ``(...)``
|
| 33 |
+
and verse numbers between dandas are not recited and are removed.
|
| 34 |
+
|
| 35 |
+
Letters outside Sanskrit (nukta consonants, candra vowels, digits, Latin) are
|
| 36 |
+
reported in ``Phonemized.unknown`` so that data preparation can leave such
|
| 37 |
+
recordings out. A nukta consonant is read as its base consonant; anything else
|
| 38 |
+
becomes ``<unk>``.
|
| 39 |
+
|
| 40 |
+
The token layout follows tiny-voca's English front end (Muhammed Nazeem); the
|
| 41 |
+
pronunciation and scansion rules follow Vagdhenu's Sanskrit front end
|
| 42 |
+
(Prathosh A P). Both are Apache-2.0; this file is a separate implementation.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
from dataclasses import dataclass
|
| 46 |
+
import hashlib
|
| 47 |
+
import json
|
| 48 |
+
import re
|
| 49 |
+
import unicodedata
|
| 50 |
+
|
| 51 |
+
FRONTEND_VERSION = 2 # 2: aspirated stops are written as stop + h
|
| 52 |
+
|
| 53 |
+
PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>"
|
| 54 |
+
SPECIALS = (PAD, BOS, EOS, SP, UNK)
|
| 55 |
+
DANDA, DOUBLE_DANDA, YATI = "|", "||", "<yati>"
|
| 56 |
+
PUNCTUATION = (",", "?", "!", DANDA, DOUBLE_DANDA, YATI)
|
| 57 |
+
VOWELS = ("a", "A", "i", "I", "u", "U", "f", "e", "E", "o", "O")
|
| 58 |
+
CONSONANTS = ("k", "g", "N", "c", "j", "Y", "w", "q", "R", "t", "d", "n", "p", "b", "m", "y", "r", "l", "v", "S", "z", "s", "h")
|
| 59 |
+
MARKS = ("M", "H")
|
| 60 |
+
PHONES = VOWELS + CONSONANTS + MARKS
|
| 61 |
+
# Append-only: changing the order or removing a symbol invalidates checkpoints.
|
| 62 |
+
SYMBOLS = SPECIALS + PUNCTUATION + PHONES
|
| 63 |
+
SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)}
|
| 64 |
+
PHONE_SET = frozenset(PHONES)
|
| 65 |
+
PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]}
|
| 66 |
+
VOWEL_SET = frozenset(VOWELS)
|
| 67 |
+
VOICELESS = frozenset("kcwtpSzsH")
|
| 68 |
+
WEIGHT_NONE, LAGHU, GURU = 0, 1, 2
|
| 69 |
+
|
| 70 |
+
_INDEPENDENT = dict(zip("अआइईउऊऋॠऌॡएऐओऔ", "aAiIuUfFxXeEoO"))
|
| 71 |
+
_SIGNS = dict(zip("ािीुूृॄॢॣेैोौ", "AiIuUfFxXeEoO"))
|
| 72 |
+
_CONSONANTS = dict(zip("कखगघङचछजझञटठडढणतथदधनपफबभमयरलवशषसहळ", "kKgGNcCjJYwWqQRtTdDnpPbBmyrlvSzshl"))
|
| 73 |
+
_VIRAMA, _NUKTA, _AVAGRAHA, _OM = "्", "़", "ऽ", "ॐ"
|
| 74 |
+
_NASALISED = frozenset("ंँ") # anusvara, candrabindu
|
| 75 |
+
_VISARGAS = frozenset(("ः", chr(0x1CF5), chr(0x1CF6))) # visarga, jihvamuliya, upadhmaniya
|
| 76 |
+
# Vedic accent marks (udatta, anudatta) and the zero-width space, non-joiner and joiner.
|
| 77 |
+
_UNSPOKEN = frozenset(map(chr, (0x0951, 0x0952, 0x200B, 0x200C, 0x200D)))
|
| 78 |
+
_READABLE = (frozenset(_INDEPENDENT) | frozenset(_SIGNS) | frozenset(_CONSONANTS) | _NASALISED | _VISARGAS
|
| 79 |
+
| {_VIRAMA, _NUKTA, _AVAGRAHA, _OM, " "})
|
| 80 |
+
# Letters that end in a vowel: a colon typed after one of them stands for visarga.
|
| 81 |
+
_VOWEL_FINAL = "".join((*_INDEPENDENT, *_SIGNS, *_CONSONANTS))
|
| 82 |
+
# Vowels as parsed, before the rare ones are rewritten.
|
| 83 |
+
_RAW_VOWELS = frozenset("aAiIuUfFxXeEoO")
|
| 84 |
+
_LONG = frozenset("AIUFXeEoO")
|
| 85 |
+
# Phonemes as parsed that are written with other tokens: the rare vowels, and each aspirated stop.
|
| 86 |
+
_WRITTEN_AS = {"F": ("f",), "x": ("l", "f"), "X": ("l", "f"),
|
| 87 |
+
**{aspirated: (plain, "h") for aspirated, plain in zip("KGCJWQTDPB", "kgcjwqtdpb")}}
|
| 88 |
+
_CLASS_NASAL = {**dict.fromkeys("kKgGN", "N"), **dict.fromkeys("cCjJY", "Y"), **dict.fromkeys("wWqQR", "R"),
|
| 89 |
+
**dict.fromkeys("tTdDn", "n"), **dict.fromkeys("pPbBm", "m")}
|
| 90 |
+
|
| 91 |
+
_MARKS_TO_TOKEN = {"।": DANDA, "|": DANDA, ".": DANDA, "॥": DOUBLE_DANDA, ",": ",", ";": ",", "…": ",",
|
| 92 |
+
"–": ",", "—": ",", "?": "?", "!": "!"}
|
| 93 |
+
_QUOTES = dict.fromkeys(map(ord, "'\"`´‘’‚‛“”„‟«»‹›"))
|
| 94 |
+
_DIGITS = "0-9०-९"
|
| 95 |
+
_VERSE_NUMBER = re.compile(rf"([।॥])\s*[{_DIGITS}]+(?:\s*[/.\-]\s*[{_DIGITS}]+)*\s*(?=[।॥]|$)")
|
| 96 |
+
_PIECE = re.compile(r"\s+|([" + re.escape("".join(_MARKS_TO_TOKEN)) + r"])")
|
| 97 |
+
# Scripts that are converted to Devanagari before reading: code-point block and scheme name.
|
| 98 |
+
_SCRIPTS = ((0x0980, 0x09FF, "BENGALI"), (0x0A00, 0x0A7F, "GURMUKHI"), (0x0A80, 0x0AFF, "GUJARATI"),
|
| 99 |
+
(0x0B00, 0x0B7F, "ORIYA"), (0x0B80, 0x0BFF, "TAMIL"), (0x0C00, 0x0C7F, "TELUGU"),
|
| 100 |
+
(0x0C80, 0x0CFF, "KANNADA"), (0x0D00, 0x0D7F, "MALAYALAM"), (0x11300, 0x1137F, "GRANTHA"))
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@dataclass(frozen=True)
|
| 104 |
+
class Phonemized:
|
| 105 |
+
symbols: tuple
|
| 106 |
+
ids: tuple
|
| 107 |
+
weight: tuple
|
| 108 |
+
unknown: tuple # characters outside the Sanskrit inventory, approximated or replaced by <unk>
|
| 109 |
+
|
| 110 |
+
def __len__(self):
|
| 111 |
+
return len(self.ids)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def frontend_identity():
|
| 115 |
+
"""What decides the tokens: the rules version and the symbol inventory."""
|
| 116 |
+
return {"frontend": "alpa", "frontend_version": FRONTEND_VERSION, "symbols": list(SYMBOLS)}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def frontend_fingerprint():
|
| 120 |
+
return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def to_devanagari(text):
|
| 124 |
+
"""Convert text in another Brahmic script to Devanagari. The first letter
|
| 125 |
+
from a supported script decides; Devanagari and unrecognised text pass through."""
|
| 126 |
+
for char in text:
|
| 127 |
+
code = ord(char)
|
| 128 |
+
for low, high, scheme in _SCRIPTS:
|
| 129 |
+
if low <= code <= high:
|
| 130 |
+
try:
|
| 131 |
+
from indic_transliteration import sanscript
|
| 132 |
+
except ImportError as error:
|
| 133 |
+
raise RuntimeError("reading scripts other than Devanagari needs the indic-transliteration "
|
| 134 |
+
'package: pip install "alpa[scripts]"') from error
|
| 135 |
+
return sanscript.transliterate(text, getattr(sanscript, scheme), sanscript.DEVANAGARI)
|
| 136 |
+
return text
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def normalize_text(text):
|
| 140 |
+
"""Canonical text before tokenisation: only spoken letters and the
|
| 141 |
+
punctuation marks that become tokens remain."""
|
| 142 |
+
text = unicodedata.normalize("NFC", to_devanagari(text))
|
| 143 |
+
text = re.sub(rf"(?<=[{_VOWEL_FINAL}]):-?", "ः", text) # colon typed for visarga
|
| 144 |
+
text = text.replace("।।", "॥").replace("||", "॥")
|
| 145 |
+
text = re.sub(r"\([^)]*\)", " ", text) # editorial insertions are not recited
|
| 146 |
+
text = _VERSE_NUMBER.sub(r"\1", text)
|
| 147 |
+
text = re.sub(r"\s-+\s", " , ", text).replace("-", "") # a spaced hyphen pauses, a plain one joins
|
| 148 |
+
text = re.sub(r"[:()]", " , ", text)
|
| 149 |
+
return re.sub(r"\s+", " ", text.translate(_QUOTES)).strip()
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def word_phones(word):
|
| 153 |
+
"""One written word -> phonemes as written (SLP1, before any pronunciation
|
| 154 |
+
rule) and the characters that are not Sanskrit letters."""
|
| 155 |
+
phones, unknown = [], []
|
| 156 |
+
open_consonant = False # the last consonant still owes its inherent a
|
| 157 |
+
stray = "" # run of characters with no reading, given one <unk>
|
| 158 |
+
for char in word + " ":
|
| 159 |
+
if char in _UNSPOKEN:
|
| 160 |
+
continue
|
| 161 |
+
if char not in _READABLE:
|
| 162 |
+
if open_consonant:
|
| 163 |
+
phones.append("a")
|
| 164 |
+
open_consonant = False
|
| 165 |
+
if not stray:
|
| 166 |
+
phones.append(UNK)
|
| 167 |
+
stray += char
|
| 168 |
+
continue
|
| 169 |
+
if stray:
|
| 170 |
+
unknown.append(stray)
|
| 171 |
+
stray = ""
|
| 172 |
+
if char == _VIRAMA:
|
| 173 |
+
open_consonant = False
|
| 174 |
+
elif char == _NUKTA:
|
| 175 |
+
unknown.append(char)
|
| 176 |
+
elif char in _SIGNS:
|
| 177 |
+
phones.append(_SIGNS[char])
|
| 178 |
+
open_consonant = False
|
| 179 |
+
else:
|
| 180 |
+
if open_consonant:
|
| 181 |
+
phones.append("a")
|
| 182 |
+
open_consonant = char in _CONSONANTS
|
| 183 |
+
if open_consonant:
|
| 184 |
+
phones.append(_CONSONANTS[char])
|
| 185 |
+
elif char in _INDEPENDENT:
|
| 186 |
+
phones.append(_INDEPENDENT[char])
|
| 187 |
+
elif char in _NASALISED:
|
| 188 |
+
phones.append("M")
|
| 189 |
+
elif char in _VISARGAS:
|
| 190 |
+
phones.append("H")
|
| 191 |
+
elif char == _OM:
|
| 192 |
+
phones.extend(("o", "m"))
|
| 193 |
+
return phones, unknown
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _pronounce(pieces):
|
| 197 |
+
"""Anusvara rule over words in order. ``pieces`` holds a phoneme list per
|
| 198 |
+
word and a punctuation token (str) per mark; lists are changed in place."""
|
| 199 |
+
for index, phones in enumerate(pieces):
|
| 200 |
+
if isinstance(phones, str):
|
| 201 |
+
continue
|
| 202 |
+
for i, phone in enumerate(phones):
|
| 203 |
+
if phone != "M":
|
| 204 |
+
continue
|
| 205 |
+
if i + 1 < len(phones):
|
| 206 |
+
phones[i] = _CLASS_NASAL.get(phones[i + 1], "M")
|
| 207 |
+
elif index + 1 == len(pieces) or isinstance(pieces[index + 1], str):
|
| 208 |
+
phones[i] = "m"
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _weigh(phones):
|
| 212 |
+
"""Weights for one stretch of speech between pauses. A syllable is guru
|
| 213 |
+
when its vowel is long, or is followed by anusvara or visarga, or by two or
|
| 214 |
+
more consonants before the next vowel (counted across word boundaries)."""
|
| 215 |
+
weights = [WEIGHT_NONE] * len(phones)
|
| 216 |
+
vowels = [i for i, phone in enumerate(phones) if phone in _RAW_VOWELS]
|
| 217 |
+
for n, i in enumerate(vowels):
|
| 218 |
+
between = phones[i + 1:vowels[n + 1] if n + 1 < len(vowels) else len(phones)]
|
| 219 |
+
heavy = phones[i] in _LONG or len(between) >= 2 or "M" in between or "H" in between
|
| 220 |
+
weights[i] = GURU if heavy else LAGHU
|
| 221 |
+
return weights
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def tokenize(pieces):
|
| 225 |
+
"""Build the token sequence from pieces in order: a list of written
|
| 226 |
+
phonemes for a word, a punctuation token (str) for a mark. Pure function."""
|
| 227 |
+
pieces = [list(piece) if not isinstance(piece, str) else piece for piece in pieces]
|
| 228 |
+
_pronounce(pieces)
|
| 229 |
+
symbols, weight = [BOS], [WEIGHT_NONE]
|
| 230 |
+
start = 0
|
| 231 |
+
while start < len(pieces):
|
| 232 |
+
if isinstance(pieces[start], str):
|
| 233 |
+
if symbols[-1] != pieces[start]: # collapse repeats such as "!!"
|
| 234 |
+
symbols.append(pieces[start])
|
| 235 |
+
weight.append(WEIGHT_NONE)
|
| 236 |
+
start += 1
|
| 237 |
+
continue
|
| 238 |
+
end = start
|
| 239 |
+
while end < len(pieces) and not isinstance(pieces[end], str):
|
| 240 |
+
end += 1
|
| 241 |
+
weights = iter(_weigh([phone for word in pieces[start:end] for phone in word]))
|
| 242 |
+
for word in pieces[start:end]:
|
| 243 |
+
if symbols[-1] in PHONE_SET or symbols[-1] == UNK:
|
| 244 |
+
symbols.append(SP)
|
| 245 |
+
weight.append(WEIGHT_NONE)
|
| 246 |
+
for phone in word:
|
| 247 |
+
level = next(weights)
|
| 248 |
+
mapped = _WRITTEN_AS.get(phone, (phone,))
|
| 249 |
+
for item in mapped:
|
| 250 |
+
symbols.append(item)
|
| 251 |
+
weight.append(level if item in VOWEL_SET else WEIGHT_NONE)
|
| 252 |
+
start = end
|
| 253 |
+
symbols.append(EOS)
|
| 254 |
+
weight.append(WEIGHT_NONE)
|
| 255 |
+
if not any(s in PHONE_SET or s == UNK for s in symbols):
|
| 256 |
+
raise ValueError("the text has nothing to pronounce")
|
| 257 |
+
return symbols, weight
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def phonemize(text):
|
| 261 |
+
if not isinstance(text, str) or not text.strip():
|
| 262 |
+
raise ValueError("text must not be empty")
|
| 263 |
+
pieces, unknown = [], []
|
| 264 |
+
for piece in _PIECE.split(normalize_text(text)):
|
| 265 |
+
if not piece:
|
| 266 |
+
continue
|
| 267 |
+
if piece in _MARKS_TO_TOKEN:
|
| 268 |
+
pieces.append(_MARKS_TO_TOKEN[piece])
|
| 269 |
+
continue
|
| 270 |
+
phones, odd = word_phones(piece)
|
| 271 |
+
unknown.extend(odd)
|
| 272 |
+
if phones:
|
| 273 |
+
pieces.append(phones)
|
| 274 |
+
symbols, weight = tokenize(pieces)
|
| 275 |
+
return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(weight), tuple(unknown))
|
samples/1_gita_1_1.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cd23c66fcdf3a9aea44bef08f2b9f362007e0ed0e01003d1e09e28c51f864f9b
|
| 3 |
+
size 375852
|
samples/2_retroflex_and_dental.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:37eac779074966caf3cb610b8a854236b3914dce69ca49c592b1809ec5aa396a
|
| 3 |
+
size 307756
|
samples/3_everyday.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8a29f8671224c32e000b2de03fe3d813bf7f2d5c442ad4dd435b3d77cdc7b9e0
|
| 3 |
+
size 343084
|
samples/4_aspirates.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e1c5e465dc8dac0dbd0e89ee959c67655f93deb51cb64ab35b202a0a7b7f3f6
|
| 3 |
+
size 297516
|
samples/5_subhashita.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6228c2b25903d184e411d79a5cc082f1a3470e0a1b48fecb35b9b76767ac6035
|
| 3 |
+
size 200236
|