pitput try 1 (step 104,000)
Browse files- .gitattributes +10 -0
- LICENSE +202 -0
- LICENSE.tiny-voca +202 -0
- NOTICE +22 -0
- README.md +175 -0
- config.json +46 -0
- inference.pt +3 -0
- pitput/__init__.py +8 -0
- pitput/__pycache__/__init__.cpython-311.pyc +0 -0
- pitput/__pycache__/g2p.cpython-311.pyc +0 -0
- pitput/__pycache__/labels.cpython-311.pyc +0 -0
- pitput/__pycache__/text.cpython-311.pyc +0 -0
- pitput/config.py +98 -0
- pitput/g2p.py +179 -0
- pitput/infer.py +184 -0
- pitput/labels.py +72 -0
- pitput/model.py +382 -0
- pitput/text.py +201 -0
- recipe.json +41 -0
- requirements.txt +6 -0
- samples/01-default.wav +3 -0
- samples/02-default.wav +3 -0
- samples/03-default.wav +3 -0
- samples/04-excited.wav +3 -0
- samples/05-sad.wav +3 -0
- samples/06-angry.wav +3 -0
- samples/07-fearful.wav +3 -0
- samples/08-surprised.wav +3 -0
- samples/09-calm.wav +3 -0
- samples/10-whisper.wav +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
samples/01-default.wav filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
samples/02-default.wav filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
samples/03-default.wav filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
samples/04-excited.wav filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
samples/05-sad.wav filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
samples/06-angry.wav filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
samples/07-fearful.wav filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
samples/08-surprised.wav filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
samples/09-calm.wav filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
samples/10-whisper.wav filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
LICENSE.tiny-voca
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
This package is a fork of fiato by cfahlgren1
|
| 2 |
+
(https://huggingface.co/cfahlgren1/fiato), used under the Apache License 2.0.
|
| 3 |
+
|
| 4 |
+
fiato is a fork of tiny-voca by Muhammed Nazeem
|
| 5 |
+
(https://github.com/n4ze3m/tiny-voca, commit 62e2ade), used under the
|
| 6 |
+
Apache License 2.0. The original license is in LICENSE.tiny-voca.
|
| 7 |
+
|
| 8 |
+
Changes in fiato:
|
| 9 |
+
- Expression tokens (<laugh>, <sigh>, ...) in the text front end, the aligner
|
| 10 |
+
and the annotation checks.
|
| 11 |
+
- Two new voices, F1 and M1.
|
| 12 |
+
- Feature preparation runs on several processes.
|
| 13 |
+
- Support for tiny-voca's earlier byte-token feature files is removed.
|
| 14 |
+
|
| 15 |
+
Changes in this fork (pitput):
|
| 16 |
+
- Hebrew front end: Phonikud phoneme labels instead of eSpeak/dizione.
|
| 17 |
+
- Eight delivery styles in place of the voices; whispered styles are not
|
| 18 |
+
forced voiced.
|
| 19 |
+
- Eight expression tags.
|
| 20 |
+
- Features stored as 16-bit waveform and float16 mel; training can stream
|
| 21 |
+
them from disk with background prefetching.
|
| 22 |
+
- Inference reads Hebrew text through Phonikud.
|
README.md
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- he
|
| 4 |
+
license: other
|
| 5 |
+
license_name: private-research-only
|
| 6 |
+
license_link: https://huggingface.co/useless-ai/pitput/blob/main/README.md#license
|
| 7 |
+
pipeline_tag: text-to-speech
|
| 8 |
+
library_name: pytorch
|
| 9 |
+
tags:
|
| 10 |
+
- text-to-speech
|
| 11 |
+
- tts
|
| 12 |
+
- hebrew
|
| 13 |
+
- fastspeech2
|
| 14 |
+
- ddsp
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# pitput
|
| 18 |
+
|
| 19 |
+
**pitput** (פִּטְפּוּט) is a Hebrew take on [tiny-voca](https://github.com/n4ze3m/tiny-voca): a tiny TTS model
|
| 20 |
+
built to run on a CPU. The whole model, vocoder included, is **only 18 MB** (4.5M parameters). It outputs 24 kHz
|
| 21 |
+
mono and runs faster than real time on a laptop CPU. It reads plain Hebrew without niqqud, in one voice with
|
| 22 |
+
**8 delivery styles** and **8 expression tags** (`<laugh>`, `<sigh>`, `<short pause>` …).
|
| 23 |
+
|
| 24 |
+
It is a personal learning project, trained from scratch on synthetic speech. This is **try 1** (step 104,000,
|
| 25 |
+
November 18, 2026). **Private research use only; see [License](#license).**
|
| 26 |
+
|
| 27 |
+
## Samples
|
| 28 |
+
|
| 29 |
+
The 10 listening texts that were synthesized after every epoch, here from the final weights.
|
| 30 |
+
|
| 31 |
+
| Style | Text | Audio |
|
| 32 |
+
|---|---|---|
|
| 33 |
+
| default | לפי נתוני הלשכה המרכזית לסטטיסטיקה, אוכלוסיית ישראל גדלה בשנה האחרונה בכמאה ושמונים אלף איש. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/01-default.wav"></audio> |
|
| 34 |
+
| default, addressee f | מה שלומך? שמעתי שלא הרגשת טוב בשבוע שעבר. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/02-default.wav"></audio> |
|
| 35 |
+
| default | הרכבת לחיפה יוצאת בשבע ועשרים ומגיעה בשמונה וחצי, `<chuckle>` אם היא בכלל תגיע. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/03-default.wav"></audio> |
|
| 36 |
+
| excited | `<laugh>` אני לא מאמינה, קיבלנו את הדירה! | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/04-excited.wav"></audio> |
|
| 37 |
+
| sad | `<sigh>` חיכיתי לו כל הערב, והוא פשוט לא הגיע. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/05-sad.wav"></audio> |
|
| 38 |
+
| angry, addressee m | כמה פעמים אמרתי לך לסגור את הדלת?! | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/06-angry.wav"></audio> |
|
| 39 |
+
| fearful | שמעת את זה? `<gasp>` יש מישהו בחוץ. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/07-fearful.wav"></audio> |
|
| 40 |
+
| surprised | רגע, `<short pause>` הוא באמת התפטר? אחרי עשרים שנה? | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/08-surprised.wav"></audio> |
|
| 41 |
+
| calm | `<breath>` קחו נשימה עמוקה, `<long pause>` ותנו לגוף להירגע לאט לאט. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/09-calm.wav"></audio> |
|
| 42 |
+
| whisper | בוא נדבר בשקט, כולם כבר ישנים. | <audio controls src="https://huggingface.co/useless-ai/pitput/resolve/main/samples/10-whisper.wav"></audio> |
|
| 43 |
+
|
| 44 |
+
## Usage
|
| 45 |
+
|
| 46 |
+
```bash
|
| 47 |
+
huggingface-cli login # the repository is private
|
| 48 |
+
git clone https://huggingface.co/useless-ai/pitput
|
| 49 |
+
cd pitput
|
| 50 |
+
pip install -r requirements.txt # PyTorch, Phonikud, onnxruntime, soundfile, huggingface_hub
|
| 51 |
+
|
| 52 |
+
python -m pitput.infer --style excited --text "<laugh> אני לא מאמינה, קיבלנו את הדירה!" --output out.wav
|
| 53 |
+
python -m pitput.infer --style default --addressee f --text "מה שלומך? שמעתי שלא הרגשת טוב." --output hello.wav
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
On first use the Phonikud model (~300 MB) is downloaded to the Hugging Face cache. Options:
|
| 57 |
+
|
| 58 |
+
| Option | Meaning |
|
| 59 |
+
|---|---|
|
| 60 |
+
| `--style` | `default`, `excited`, `sad`, `angry`, `fearful`, `surprised`, `calm`, `whisper` |
|
| 61 |
+
| `--addressee m` / `f` | Gender of the person spoken to (לך, אותך, אמרת …). Without it Phonikud reads the masculine form |
|
| 62 |
+
| `--text-file` | Read the text from a UTF-8 file instead of `--text` |
|
| 63 |
+
| `--speed` | 1.0 = as trained; 0.9 is 10% slower |
|
| 64 |
+
| `--pitch-shift` | Added to log F0 (0.058 ≈ one semitone) |
|
| 65 |
+
| `--pitch-range` | Scales the melody around each sentence's mean pitch (1.0 = as predicted) |
|
| 66 |
+
| `--device` | `cpu` (default) or `cuda` |
|
| 67 |
+
|
| 68 |
+
From Python:
|
| 69 |
+
|
| 70 |
+
```python
|
| 71 |
+
import soundfile as sf
|
| 72 |
+
from pitput.infer import CHECKPOINT, load_model, speak_text
|
| 73 |
+
|
| 74 |
+
model = load_model(CHECKPOINT) # inference.pt next to the package
|
| 75 |
+
wave, utterances = speak_text(model, "שלום, <sigh> מה שלומך?", "calm", addressee="f")
|
| 76 |
+
sf.write("out.wav", wave.numpy(), model.config.sample_rate)
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
### What the input must look like
|
| 80 |
+
|
| 81 |
+
- **Plain Hebrew in spoken form.** There is no text normalizer: write numbers, dates, times, acronyms and
|
| 82 |
+
foreign words out in Hebrew words (`בשבע ועשרים`, not `7:20`). Characters it can't read are skipped.
|
| 83 |
+
- **No niqqud needed.** [Phonikud](https://github.com/thewh1teagle/phonikud) vocalizes each sentence in context
|
| 84 |
+
and adds stress; the model reads its phonemes.
|
| 85 |
+
- **Tags** go where the sound should be, in English inside the Hebrew: `<laugh>`, `<chuckle>`, `<sigh>`,
|
| 86 |
+
`<breath>`, `<gasp>`, `<throat-clearing>`, `<short pause>`, `<long pause>`.
|
| 87 |
+
- **Punctuation** shapes the delivery (`, . ? ! …`). Long texts are read sentence by sentence.
|
| 88 |
+
|
| 89 |
+
## Model
|
| 90 |
+
|
| 91 |
+
A fork of [fiato](https://huggingface.co/cfahlgren1/fiato), itself a fork of
|
| 92 |
+
[tiny-voca](https://github.com/n4ze3m/tiny-voca): a FastSpeech2-style model with a built-in
|
| 93 |
+
differentiable source-filter vocoder, so there is no separate neural vocoder.
|
| 94 |
+
|
| 95 |
+
- **Input:** 47 tokens: 29 Phonikud phonemes, 5 punctuation marks, word boundary, utterance edges and the 8
|
| 96 |
+
expression tags. Stress is a parallel feature, not a token.
|
| 97 |
+
- **Text encoder:** 4 transformer layers, 192 channels, with the style embedding added to every token.
|
| 98 |
+
- **Duration predictor:** frames per token, plus a "pause present" switch for punctuation, word boundaries and tags.
|
| 99 |
+
- **Frame level:** the style embedding is added again; 2 layers predict pitch (log F0), voicing and energy, and 5
|
| 100 |
+
layers predict the acoustic features.
|
| 101 |
+
- **Vocoder:** harmonic pulse train (from the predicted pitch) plus noise, each shaped by a complex cepstrum per frame.
|
| 102 |
+
- **Size:** 4,534,027 parameters at inference (18 MB checkpoint). Training also uses 2.7M parameters that are not
|
| 103 |
+
in the checkpoint (discriminators and an auxiliary mel head).
|
| 104 |
+
|
| 105 |
+
## Training data
|
| 106 |
+
|
| 107 |
+
The dataset is **synthetic only**, with no recorded human speech: 65,722 clips (139.3 h) of one synthetic voice,
|
| 108 |
+
of which 63,745 training clips (136.4 h) passed forced alignment. 160 held-out sentences (20 per style) were kept
|
| 109 |
+
for validation and 160 for test.
|
| 110 |
+
|
| 111 |
+
- **Text:** 65,728 Hebrew sentences from news (archive, HeSum, RSS), HeDC4 web text and stories, Hebrew Wikipedia,
|
| 112 |
+
ivrit.ai transcripts and LLM-written scenes (dialogue for the expressive styles).
|
| 113 |
+
- **Spoken form:** an LLM rewrote numbers, dates, money, acronyms and Latin script as Hebrew words, and every answer
|
| 114 |
+
was checked by code.
|
| 115 |
+
- **Addressee gender:** an LLM decided the gender of ambiguous second-person words (לך, אותך, אמרת) in context.
|
| 116 |
+
The audio and the training phonemes follow the same decision.
|
| 117 |
+
- **Styles:** an LLM picked a style for each dialogue sentence and inserted the tags; calm and whisper were assigned
|
| 118 |
+
to plain statements. Each style was rendered with one fixed delivery instruction; default is a bright, friendly
|
| 119 |
+
delivery.
|
| 120 |
+
|
| 121 |
+
| Style | Clips synthesized | Rejected at alignment |
|
| 122 |
+
|---|---|---|
|
| 123 |
+
| default | 52,193 | 1% |
|
| 124 |
+
| calm | 4,900 | 1% |
|
| 125 |
+
| whisper | 4,900 | 1% |
|
| 126 |
+
| fearful | 1,104 | 42% |
|
| 127 |
+
| angry | 1,027 | 12% |
|
| 128 |
+
| sad | 685 | 19% |
|
| 129 |
+
| excited | 470 | 18% |
|
| 130 |
+
| surprised | 449 | 14% |
|
| 131 |
+
|
| 132 |
+
4,582 clips contain tags (at most 2 per clip); `<laugh>` appears in only about 90 of them.
|
| 133 |
+
- **Labels:** Phonikud phonemes with the addressee decisions applied, phonemized word by word with the tags at
|
| 134 |
+
their positions. Durations come from forced alignment with
|
| 135 |
+
[facebook/wav2vec2-lv-60-espeak-cv-ft](https://huggingface.co/facebook/wav2vec2-lv-60-espeak-cv-ft), which is used
|
| 136 |
+
only for labeling and is not part of the model. A tag owns the sound between the silences around it.
|
| 137 |
+
|
| 138 |
+
## Limitations
|
| 139 |
+
|
| 140 |
+
- **Laughs and other non-speech tags are weak.** The model gives each tag its time, but fills it with a quiet,
|
| 141 |
+
mostly unvoiced sound: a laugh comes out about 15 dB quieter than in the recordings and is heard as a soft breath.
|
| 142 |
+
Chuckle, throat-clearing and gasp suffer the same way. Pauses and breaths work better.
|
| 143 |
+
- **Intonation can sound a little robotic.** Pitch range is close to the recordings for most styles, but fearful
|
| 144 |
+
and surprised have only 70–75% of it. The model speaks 3–17% faster than the recordings, with shorter pauses.
|
| 145 |
+
- **Fearful is the weakest style:** it has the fewest usable training clips and the most pronunciation errors.
|
| 146 |
+
- **Pronunciation follows Phonikud,** including its mistakes (e.g. וואו read as "vaʔu").
|
| 147 |
+
- **Second person defaults to masculine** unless `--addressee f` is given; there is no per-word decision at
|
| 148 |
+
inference.
|
| 149 |
+
- One voice only. Sentences are synthesized independently, with no prosody carried between them.
|
| 150 |
+
|
| 151 |
+
## License
|
| 152 |
+
|
| 153 |
+
- **Code** (`pitput/`): Apache License 2.0. It is a fork of fiato by cfahlgren1 and tiny-voca by Muhammed Nazeem,
|
| 154 |
+
both Apache-2.0; see `NOTICE`, `LICENSE` and `LICENSE.tiny-voca`.
|
| 155 |
+
- **Weights and audio samples: private research use only, not licensed for redistribution or any commercial use.**
|
| 156 |
+
Keep this repository private.
|
| 157 |
+
- **Phonikud** ([thewh1teagle/phonikud-onnx](https://huggingface.co/thewh1teagle/phonikud-onnx), MIT) is downloaded
|
| 158 |
+
at first use and is not included here.
|
| 159 |
+
|
| 160 |
+
## Files
|
| 161 |
+
|
| 162 |
+
| File | Contents |
|
| 163 |
+
|---|---|
|
| 164 |
+
| `inference.pt` | Weights, model configuration and front-end fingerprint (loaded with `weights_only=True`) |
|
| 165 |
+
| `pitput/` | Inference code: `infer.py` (command line and `speak_text`), `model.py`, `text.py` (phoneme tokens), `g2p.py` (Phonikud and addressee gender), `labels.py`, `config.py` |
|
| 166 |
+
| `samples/` | The audio above |
|
| 167 |
+
| `config.json`, `recipe.json` | Model and training settings of this run |
|
| 168 |
+
| `requirements.txt` | Python dependencies |
|
| 169 |
+
| `NOTICE`, `LICENSE`, `LICENSE.tiny-voca` | Attribution and license of the code |
|
| 170 |
+
|
| 171 |
+
## Acknowledgements
|
| 172 |
+
|
| 173 |
+
[tiny-voca](https://github.com/n4ze3m/tiny-voca) for the architecture and training recipe,
|
| 174 |
+
[fiato](https://huggingface.co/cfahlgren1/fiato) for expression tags on it, [Phonikud](https://github.com/thewh1teagle/phonikud)
|
| 175 |
+
for Hebrew vocalization and phonemes, and the Hebrew text sources: HeDC4, HeSum, Hebrew Wikipedia and ivrit.ai.
|
config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"sample_rate": 24000,
|
| 3 |
+
"hop": 256,
|
| 4 |
+
"n_fft": 1024,
|
| 5 |
+
"n_mels": 80,
|
| 6 |
+
"f0_min": 50.0,
|
| 7 |
+
"f0_max": 600.0,
|
| 8 |
+
"speakers": [
|
| 9 |
+
"default",
|
| 10 |
+
"excited",
|
| 11 |
+
"sad",
|
| 12 |
+
"angry",
|
| 13 |
+
"fearful",
|
| 14 |
+
"surprised",
|
| 15 |
+
"calm",
|
| 16 |
+
"whisper"
|
| 17 |
+
],
|
| 18 |
+
"unvoiced_speakers": [
|
| 19 |
+
"whisper"
|
| 20 |
+
],
|
| 21 |
+
"vocab_size": 47,
|
| 22 |
+
"channels": 192,
|
| 23 |
+
"heads": 4,
|
| 24 |
+
"feedforward": 640,
|
| 25 |
+
"kernel": 7,
|
| 26 |
+
"encoder_layers": 4,
|
| 27 |
+
"prosody_layers": 2,
|
| 28 |
+
"acoustic_layers": 5,
|
| 29 |
+
"dropout": 0.1,
|
| 30 |
+
"predictor_channels": 128,
|
| 31 |
+
"synth_fft": 512,
|
| 32 |
+
"synth_hop": 128,
|
| 33 |
+
"noise_fft": 256,
|
| 34 |
+
"noise_hop": 64,
|
| 35 |
+
"cep_causal": 80,
|
| 36 |
+
"cep_anti": 16,
|
| 37 |
+
"noise_cep": 32,
|
| 38 |
+
"harmonic_limit": 0.47,
|
| 39 |
+
"max_tokens": 512,
|
| 40 |
+
"max_frames": 3000,
|
| 41 |
+
"mel_mean": -5.0,
|
| 42 |
+
"mel_scale": 5.0,
|
| 43 |
+
"energy_mean": -4.0,
|
| 44 |
+
"energy_scale": 2.0,
|
| 45 |
+
"pitch_reference_hz": 200.0
|
| 46 |
+
}
|
inference.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8c9b32940c4854be0e7a299fbaa218a8440df6ba8a4b51a731286ff356c51b94
|
| 3 |
+
size 18196101
|
pitput/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""pitput: a small Hebrew text-to-speech model, a fork of fiato (itself a fork of tiny-voca). See NOTICE.
|
| 2 |
+
|
| 3 |
+
One voice in 8 delivery styles, which take the place of fiato's voices, and 8
|
| 4 |
+
expression tags. Text goes through Phonikud (``pitput.g2p``) and is phonemized
|
| 5 |
+
word by word (``pitput.labels``) exactly as for the training clips.
|
| 6 |
+
|
| 7 |
+
Nothing is imported here, so ``pitput.text`` works without PyTorch.
|
| 8 |
+
"""
|
pitput/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (577 Bytes). View file
|
|
|
pitput/__pycache__/g2p.cpython-311.pyc
ADDED
|
Binary file (15.2 kB). View file
|
|
|
pitput/__pycache__/labels.cpython-311.pyc
ADDED
|
Binary file (6.11 kB). View file
|
|
|
pitput/__pycache__/text.cpython-311.pyc
ADDED
|
Binary file (14.4 kB). View file
|
|
|
pitput/config.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import asdict, dataclass
|
| 2 |
+
import hashlib
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
CHECKPOINT_FORMAT = 4 # 2-3: byte tokens + learned aligner (rejected explicitly).
|
| 6 |
+
ANNOTATION_VERSION = 3
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _digest(value):
|
| 10 |
+
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass(frozen=True)
|
| 14 |
+
class Config:
|
| 15 |
+
# Acoustic features.
|
| 16 |
+
sample_rate: int = 24000
|
| 17 |
+
hop: int = 256
|
| 18 |
+
n_fft: int = 1024
|
| 19 |
+
n_mels: int = 80
|
| 20 |
+
f0_min: float = 50.0
|
| 21 |
+
f0_max: float = 600.0
|
| 22 |
+
# One "voice" per delivery style, all spoken by one voice.
|
| 23 |
+
speakers: tuple[str, ...] = ("default", "excited", "sad", "angry", "fearful", "surprised", "calm", "whisper")
|
| 24 |
+
# Styles whose vowels are not voiced: no pitch, and vowels are not forced voiced in training or synthesis.
|
| 25 |
+
unvoiced_speakers: tuple[str, ...] = ("whisper",)
|
| 26 |
+
# Phoneme tokens from pitput.text.SYMBOLS (front-end version 1).
|
| 27 |
+
vocab_size: int = 47
|
| 28 |
+
# Text encoder and frame-level transformer.
|
| 29 |
+
channels: int = 192
|
| 30 |
+
heads: int = 4
|
| 31 |
+
feedforward: int = 640
|
| 32 |
+
kernel: int = 7
|
| 33 |
+
encoder_layers: int = 4
|
| 34 |
+
prosody_layers: int = 2
|
| 35 |
+
acoustic_layers: int = 5
|
| 36 |
+
dropout: float = 0.1
|
| 37 |
+
predictor_channels: int = 128
|
| 38 |
+
# Source-filter waveform head: harmonic branch and noise branch STFT sizes,
|
| 39 |
+
# causal/anticausal complex-cepstrum orders.
|
| 40 |
+
synth_fft: int = 512
|
| 41 |
+
synth_hop: int = 128
|
| 42 |
+
noise_fft: int = 256
|
| 43 |
+
noise_hop: int = 64
|
| 44 |
+
cep_causal: int = 80
|
| 45 |
+
cep_anti: int = 16
|
| 46 |
+
noise_cep: int = 32
|
| 47 |
+
harmonic_limit: float = 0.47
|
| 48 |
+
max_tokens: int = 512
|
| 49 |
+
max_frames: int = 3000
|
| 50 |
+
mel_mean: float = -5.0
|
| 51 |
+
mel_scale: float = 5.0
|
| 52 |
+
energy_mean: float = -4.0
|
| 53 |
+
energy_scale: float = 2.0
|
| 54 |
+
pitch_reference_hz: float = 200.0
|
| 55 |
+
|
| 56 |
+
def __post_init__(self):
|
| 57 |
+
from .text import SYMBOLS
|
| 58 |
+
if self.vocab_size != len(SYMBOLS):
|
| 59 |
+
raise ValueError(f"vocab_size must equal the phoneme inventory size {len(SYMBOLS)}")
|
| 60 |
+
if self.channels % self.heads or (self.channels // self.heads) % 2:
|
| 61 |
+
raise ValueError("channels must be divisible by heads with an even head width")
|
| 62 |
+
if not self.speakers or len(set(self.speakers)) != len(self.speakers):
|
| 63 |
+
raise ValueError("speaker names must be unique and nonempty")
|
| 64 |
+
if not set(self.unvoiced_speakers) <= set(self.speakers):
|
| 65 |
+
raise ValueError("unvoiced speakers must be speakers")
|
| 66 |
+
for fft, hop in ((self.synth_fft, self.synth_hop), (self.noise_fft, self.noise_hop)):
|
| 67 |
+
if fft != 4 * hop or self.hop % hop:
|
| 68 |
+
raise ValueError("synthesis windows need 4x overlap and a hop dividing the acoustic hop")
|
| 69 |
+
if self.cep_causal + self.cep_anti >= self.synth_fft // 2 or self.noise_cep >= self.noise_fft // 2:
|
| 70 |
+
raise ValueError("cepstrum orders must be well below half the synthesis FFT size")
|
| 71 |
+
if min(self.encoder_layers, self.prosody_layers, self.acoustic_layers, self.kernel) < 1 or not self.kernel % 2:
|
| 72 |
+
raise ValueError("layer counts must be positive and the kernel odd")
|
| 73 |
+
if min(self.mel_scale, self.energy_scale) <= 0 or not 0 < self.harmonic_limit < 0.5:
|
| 74 |
+
raise ValueError("invalid normalization scale or harmonic limit")
|
| 75 |
+
|
| 76 |
+
def to_dict(self):
|
| 77 |
+
return asdict(self)
|
| 78 |
+
|
| 79 |
+
@classmethod
|
| 80 |
+
def from_dict(cls, value):
|
| 81 |
+
return cls(**{**value, "speakers": tuple(value["speakers"]),
|
| 82 |
+
"unvoiced_speakers": tuple(value.get("unvoiced_speakers", ()))})
|
| 83 |
+
|
| 84 |
+
@property
|
| 85 |
+
def fingerprint(self):
|
| 86 |
+
return _digest(self.to_dict())
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def acoustic_config(self):
|
| 90 |
+
"""Settings that determine stored wave/mel/prosody tensors. Tokens and
|
| 91 |
+
durations are versioned separately by the annotation layer."""
|
| 92 |
+
fields = ("sample_rate", "hop", "n_fft", "n_mels", "f0_min", "f0_max")
|
| 93 |
+
return {**{key: getattr(self, key) for key in fields}, "feature_version": 3,
|
| 94 |
+
"pitch_reference_hz": 200, "log_floor": 1e-5, "storage": {"wave": "int16", "mel": "float16"}}
|
| 95 |
+
|
| 96 |
+
@property
|
| 97 |
+
def acoustic_fingerprint(self):
|
| 98 |
+
return _digest(self.acoustic_config)
|
pitput/g2p.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hebrew text -> vocalized text with Phonikud, plus control over second-person gender.
|
| 2 |
+
|
| 3 |
+
Phonikud reads plain Hebrew but defaults to the masculine reading of words like אותך / לך / אמרת.
|
| 4 |
+
`addressee="f"/"m"` gives every recognizable second-person singular form in the text that gender
|
| 5 |
+
(for apps that know who they are talking to), through `set_gender`.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import re
|
| 10 |
+
|
| 11 |
+
MODEL_REPO = "thewh1teagle/phonikud-onnx"
|
| 12 |
+
MODEL_FILE = "phonikud-1.0.onnx"
|
| 13 |
+
|
| 14 |
+
SHEVA, HIRIQ, TSERE, SEGOL, PATAH, QAMATS = "\u05b0", "\u05b4", "\u05b5", "\u05b6", "\u05b7", "\u05b8"
|
| 15 |
+
DAGESH, METEG, SHIN_DOT, SIN_DOT, OLE = "\u05bc", "\u05bd", "\u05c1", "\u05c2", "\u05ab"
|
| 16 |
+
VOWELS = set("\u05b0\u05b1\u05b2\u05b3\u05b4\u05b5\u05b6\u05b7\u05b8\u05b9\u05ba\u05bb\u05c7")
|
| 17 |
+
PREFIXES = "והשבלמכ"
|
| 18 |
+
|
| 19 |
+
# Stems (without the final ך) whose feminine form takes qamats: לָךְ, בָּךְ, אוֹתָךְ, אִתָּךְ, שֶׁלָּךְ.
|
| 20 |
+
QAMATS_STEMS = {"ל", "ב", "אות", "את", "אית", "של"}
|
| 21 |
+
CLOSED_STEMS = QAMATS_STEMS | {"ממ"}
|
| 22 |
+
# Words ending in ך that are never second person.
|
| 23 |
+
NOT_SECOND_PERSON = {
|
| 24 |
+
"כך", "דרך", "מלך", "ערך", "אך", "תוך", "משך", "סך", "הלך", "ארוך", "חינוך", "נמוך", "צריך", "איך",
|
| 25 |
+
"שפך", "חשך", "עשך", "נסך", "בירך", "פרך", "רך", "סמך", "מסמך", "תהליך", "תאריך", "שליך", "כרך",
|
| 26 |
+
"חך", "דך", "ברך", "פלך", "מהלך", "תמך", "נמשך", "המשך", "ממשך", "סרך", "שדך", "חיוך", "מבוך",
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
_TOKEN = re.compile(r"^([^\u05d0-\u05ea]*)(.*?)([^\u05d0-\u05ea\u0591-\u05c7|]*)$")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _clusters(stem: str) -> list[list[str]]:
|
| 33 |
+
out: list[list[str]] = []
|
| 34 |
+
for ch in stem:
|
| 35 |
+
if "\u05d0" <= ch <= "\u05ea" or not out:
|
| 36 |
+
out.append([ch, ""])
|
| 37 |
+
else:
|
| 38 |
+
out[-1][1] += ch
|
| 39 |
+
return out
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _bare(text: str) -> str:
|
| 43 |
+
return "".join(ch for ch in text if "\u05d0" <= ch <= "\u05ea")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _set_vowel(cluster: list[str], vowel: str, stress: bool | None = None, vocal_shva: bool = False) -> None:
|
| 47 |
+
marks = [m for m in cluster[1] if m not in VOWELS and m != METEG]
|
| 48 |
+
points = "".join(m for m in marks if m in (DAGESH, SHIN_DOT, SIN_DOT))
|
| 49 |
+
ole = OLE in marks if stress is None else stress
|
| 50 |
+
cluster[1] = points + vowel + (METEG if vocal_shva else "") + (OLE if ole else "")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _unprefixed(bare: str) -> list[str]:
|
| 54 |
+
"""The word and its forms with up to two prefix letters removed."""
|
| 55 |
+
forms = [bare]
|
| 56 |
+
for cut in (1, 2):
|
| 57 |
+
if len(bare) > cut + 1 and all(ch in PREFIXES for ch in bare[:cut]):
|
| 58 |
+
forms.append(bare[cut:])
|
| 59 |
+
return forms
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _kind(stem: str, force: bool) -> tuple[str, str | None] | None:
|
| 63 |
+
"""Classify a vocalized word: ("sing"|"plural"|"kamokh"|"past", current gender or None)."""
|
| 64 |
+
cl = _clusters(stem)
|
| 65 |
+
bare = _bare(stem)
|
| 66 |
+
if len(cl) < 2 or len(cl) != len(bare):
|
| 67 |
+
return None
|
| 68 |
+
forms = _unprefixed(bare)
|
| 69 |
+
last = cl[-1][1]
|
| 70 |
+
if bare.endswith("ך"):
|
| 71 |
+
if bare.endswith("ייך") or (not force and any(f in NOT_SECOND_PERSON for f in forms)):
|
| 72 |
+
return None
|
| 73 |
+
if any(f == "כמוך" for f in forms):
|
| 74 |
+
return "kamokh", "m" if QAMATS in last else "f"
|
| 75 |
+
if len(bare) >= 3 and bare[-2] == "י":
|
| 76 |
+
if QAMATS in last and SEGOL in cl[-3][1]:
|
| 77 |
+
return "plural", "m"
|
| 78 |
+
if PATAH in cl[-3][1] and HIRIQ in cl[-2][1]:
|
| 79 |
+
return "plural", "f"
|
| 80 |
+
return ("plural", None) if force else None
|
| 81 |
+
if QAMATS in last:
|
| 82 |
+
return "sing", "m"
|
| 83 |
+
if any(f[:-1] in CLOSED_STEMS for f in forms):
|
| 84 |
+
return "sing", "f" if (QAMATS in cl[-2][1] or TSERE in cl[-2][1]) and SHEVA in last else None
|
| 85 |
+
return ("sing", None) if force else None
|
| 86 |
+
if bare.endswith("ת") and len(bare) >= 3 and not any(f == "את" for f in forms):
|
| 87 |
+
if QAMATS in last:
|
| 88 |
+
return "past", "m"
|
| 89 |
+
if SHEVA in last:
|
| 90 |
+
return "past", "f"
|
| 91 |
+
return ("past", None) if force else None
|
| 92 |
+
return None
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _stem_letters(stem: str) -> list[str]:
|
| 96 |
+
return [f[:-1] for f in _unprefixed(_bare(stem))]
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def set_gender(word: str, gender: str, force: bool = False) -> str:
|
| 100 |
+
"""Rewrite a vocalized second-person singular word (Phonikud output) to gender "m" or "f".
|
| 101 |
+
|
| 102 |
+
Words that are not recognized as second person are returned unchanged. With force=True the caller
|
| 103 |
+
vouches that the word is second person, so less certain shapes are rewritten too.
|
| 104 |
+
"""
|
| 105 |
+
lead, core, trail = _TOKEN.match(word).groups()
|
| 106 |
+
prefix, _, stem = core.rpartition("|")
|
| 107 |
+
kind = _kind(stem, force)
|
| 108 |
+
if kind is None or kind[1] == gender:
|
| 109 |
+
return word
|
| 110 |
+
cl = _clusters(stem)
|
| 111 |
+
if kind[0] == "sing":
|
| 112 |
+
if gender == "m":
|
| 113 |
+
vocal = (len(cl) == 2 and not prefix) or (len(cl) > 2 and SHEVA in cl[-3][1])
|
| 114 |
+
_set_vowel(cl[-2], SHEVA, stress=False, vocal_shva=vocal)
|
| 115 |
+
_set_vowel(cl[-1], QAMATS)
|
| 116 |
+
else:
|
| 117 |
+
qamats = any(s in QAMATS_STEMS for s in _stem_letters(stem))
|
| 118 |
+
_set_vowel(cl[-2], QAMATS if qamats else TSERE, stress=False)
|
| 119 |
+
_set_vowel(cl[-1], SHEVA)
|
| 120 |
+
elif kind[0] == "plural":
|
| 121 |
+
_set_vowel(cl[-3], SEGOL if gender == "m" else PATAH, stress=True)
|
| 122 |
+
_set_vowel(cl[-2], "" if gender == "m" else HIRIQ)
|
| 123 |
+
_set_vowel(cl[-1], QAMATS if gender == "m" else SHEVA)
|
| 124 |
+
elif kind[0] == "kamokh":
|
| 125 |
+
_set_vowel(cl[-1], QAMATS if gender == "m" else SHEVA)
|
| 126 |
+
elif kind[0] == "past":
|
| 127 |
+
if gender == "m":
|
| 128 |
+
_set_vowel(cl[-1], QAMATS)
|
| 129 |
+
else:
|
| 130 |
+
closed = SHEVA in cl[-2][1] or DAGESH in cl[-1][1]
|
| 131 |
+
_set_vowel(cl[-1], SHEVA if closed else "")
|
| 132 |
+
stem = "".join(letter + marks for letter, marks in cl)
|
| 133 |
+
return lead + (prefix + "|" if prefix else "") + stem + trail
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def apply_hints(vocalized: str, hints: dict) -> str:
|
| 137 |
+
"""Phonikud output with the hinted tokens ({token_index: "m"/"f"}) switched to that gender."""
|
| 138 |
+
tokens = vocalized.split(" ")
|
| 139 |
+
for index, gender in hints.items():
|
| 140 |
+
index = int(index)
|
| 141 |
+
if gender in ("m", "f") and 0 <= index < len(tokens):
|
| 142 |
+
tokens[index] = set_gender(tokens[index], gender, force=True)
|
| 143 |
+
return " ".join(tokens)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
class G2P:
|
| 147 |
+
def __init__(self, model_path: str | None = None):
|
| 148 |
+
import onnxruntime
|
| 149 |
+
import phonikud_onnx
|
| 150 |
+
|
| 151 |
+
if model_path is None:
|
| 152 |
+
from huggingface_hub import hf_hub_download
|
| 153 |
+
|
| 154 |
+
model_path = hf_hub_download(MODEL_REPO, MODEL_FILE)
|
| 155 |
+
# Uses the GPU when onnxruntime-gpu and CUDA are installed, otherwise the CPU.
|
| 156 |
+
providers = [p for p in ("CUDAExecutionProvider", "CPUExecutionProvider")
|
| 157 |
+
if p in onnxruntime.get_available_providers()]
|
| 158 |
+
if "CUDAExecutionProvider" in providers:
|
| 159 |
+
# CUDA/cuDNN from the nvidia-* pip packages are not on the Windows DLL search path.
|
| 160 |
+
onnxruntime.preload_dlls()
|
| 161 |
+
session = onnxruntime.InferenceSession(model_path, providers=providers)
|
| 162 |
+
self.device = session.get_providers()[0]
|
| 163 |
+
self.model = phonikud_onnx.Phonikud.__new__(phonikud_onnx.Phonikud)
|
| 164 |
+
self.model.model = phonikud_onnx.OnnxModel(model_path, session=session)
|
| 165 |
+
|
| 166 |
+
def vocalize(self, text: str, *, context: str | None = None, addressee: str | None = None,
|
| 167 |
+
hints: dict[int, str] | None = None) -> str:
|
| 168 |
+
"""Add niqqud, stress and vocal-shva marks. `context` = preceding text the model may look at."""
|
| 169 |
+
if context:
|
| 170 |
+
joined = self.model.add_diacritics(context.strip() + " " + text)
|
| 171 |
+
tokens = joined.split(" ")[-len(text.split(" ")):]
|
| 172 |
+
else:
|
| 173 |
+
tokens = self.model.add_diacritics(text).split(" ")
|
| 174 |
+
if addressee:
|
| 175 |
+
tokens = [set_gender(t, addressee) for t in tokens]
|
| 176 |
+
for index, gender in (hints or {}).items():
|
| 177 |
+
if gender in ("m", "f") and 0 <= index < len(tokens):
|
| 178 |
+
tokens[index] = set_gender(tokens[index], gender, force=True)
|
| 179 |
+
return " ".join(tokens)
|
pitput/infer.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hebrew text -> speech with a trained checkpoint.
|
| 2 |
+
|
| 3 |
+
python -m pitput.infer --style default --text "שלום, <sigh> מה שלומך?" --output out.wav
|
| 4 |
+
|
| 5 |
+
The text must be in spoken form (numbers and acronyms written out as words);
|
| 6 |
+
expression tags (``<sigh>``, ``<laugh>`` ...) go where the sound should be.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import re
|
| 12 |
+
import torch
|
| 13 |
+
from .config import CHECKPOINT_FORMAT, Config
|
| 14 |
+
from .model import Fiato, UtteranceTooLong
|
| 15 |
+
from .text import NothingToPronounce, frontend_fingerprint, hebrew_label, phonemize
|
| 16 |
+
|
| 17 |
+
CHECKPOINT = Path(__file__).resolve().parent.parent / "inference.pt"
|
| 18 |
+
# The loudest sample of the audio that ``speak_text`` and the command return.
|
| 19 |
+
PEAK = 0.99
|
| 20 |
+
NOTHING = "nothing in the text could be read (Hebrew words only)"
|
| 21 |
+
_SPACE = re.compile("[\\s" + chr(0xFEFF) + "]+")
|
| 22 |
+
# A tag after the mark stays with its sentence: "היה מצחיק! <laugh> טוב." -> "היה מצחיק! <laugh>", "טוב."
|
| 23 |
+
_SENTENCE_END = re.compile(r"(?<=[.!?…])[\"'”’)\]]*\s+(?=[^\s<])")
|
| 24 |
+
_CLAUSE_END = re.compile(r"(?<=[,;:—])\s+")
|
| 25 |
+
_TEXT_END = re.compile("[\\s" + chr(0xFEFF) + "]+$")
|
| 26 |
+
_TRAILING_TAGS = re.compile(r"(?:\s*<[^<>]+>)+$")
|
| 27 |
+
_CLOSING = "\"'”’)]»" # these may stand after the last mark: היא אמרה "לכי!"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def load_model(path, device="cpu"):
|
| 31 |
+
state = torch.load(path, map_location="cpu", weights_only=True)
|
| 32 |
+
if state.get("format_version") != CHECKPOINT_FORMAT:
|
| 33 |
+
raise ValueError("unsupported checkpoint format")
|
| 34 |
+
if state["frontend"]["fingerprint"] != frontend_fingerprint():
|
| 35 |
+
raise ValueError("checkpoint was trained with a different phoneme inventory/front end")
|
| 36 |
+
model = Fiato(Config.from_dict(state["config"]))
|
| 37 |
+
model.load_state_dict(state["model"], strict=True)
|
| 38 |
+
return model.to(device).eval()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def sentences(text):
|
| 42 |
+
"""The sentences of ``text``: a break after . ! ? or … followed by a space."""
|
| 43 |
+
parts = _SENTENCE_END.split(_SPACE.sub(" ", text).strip())
|
| 44 |
+
return [part for part in parts if any(char.isalnum() for char in part)]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def finished(text):
|
| 48 |
+
"""``text`` with a full stop after its last word when that word has no mark,
|
| 49 |
+
before any tags that follow it, as in the training labels ("היה מצחיק. <laugh>").
|
| 50 |
+
Before a final mark the voice lengthens the last syllable and lets the sound
|
| 51 |
+
fall; without one the audio breaks off."""
|
| 52 |
+
body = _TEXT_END.sub("", text)
|
| 53 |
+
tags = _TRAILING_TAGS.search(body)
|
| 54 |
+
head, tail = (body[:tags.start()], body[tags.start():]) if tags else (body, "")
|
| 55 |
+
if not head.strip():
|
| 56 |
+
return text
|
| 57 |
+
last = _SPACE.split(head.strip())[-1]
|
| 58 |
+
if any(char.isalnum() for char in last):
|
| 59 |
+
marked = last.rstrip(_CLOSING)[-1:] in tuple(".!?…,;:")
|
| 60 |
+
else:
|
| 61 |
+
marked = any(char in ".!?…" for char in last) or last in (",", ";", ":")
|
| 62 |
+
return text if marked else head.rstrip() + "." + tail
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def shorter(text):
|
| 66 |
+
"""``text`` in smaller pieces: cut after commas and the like if it has
|
| 67 |
+
any, else in two between words. ``[]`` for a single word."""
|
| 68 |
+
pieces = _CLAUSE_END.split(text)
|
| 69 |
+
if len(pieces) > 1:
|
| 70 |
+
return pieces
|
| 71 |
+
words = text.split(" ")
|
| 72 |
+
if len(words) < 2:
|
| 73 |
+
return []
|
| 74 |
+
half = (len(words) + 1) // 2
|
| 75 |
+
return [" ".join(words[:half]), " ".join(words[half:])]
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def limit_peak(wave, peak=PEAK):
|
| 79 |
+
"""``wave`` scaled down as a whole when its loudest sample is above
|
| 80 |
+
``peak``; quieter audio is returned as it is. The model sometimes peaks
|
| 81 |
+
above 1.0, and that clips when the audio is saved as 16-bit samples."""
|
| 82 |
+
loudest = float(wave.abs().max()) if wave.numel() else 0.0
|
| 83 |
+
return wave * (peak / loudest) if loudest > peak else wave
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _voice(model, phones, style, speed, pitch_shift, seed, pitch_range):
|
| 87 |
+
"""Phonemes -> (float waveform tensor, details): one call of the model."""
|
| 88 |
+
device = next(model.parameters()).device
|
| 89 |
+
tokens = torch.tensor([phones.ids], device=device)
|
| 90 |
+
stress = torch.tensor([phones.stress], device=device)
|
| 91 |
+
speaker = torch.tensor([model.config.speakers.index(style)], device=device)
|
| 92 |
+
audio, lengths, details = model.synthesize(tokens, stress, torch.tensor([len(phones)], device=device), speaker, speed, pitch_shift, seed, pitch_range)
|
| 93 |
+
if not torch.isfinite(audio).all():
|
| 94 |
+
raise RuntimeError("model produced non-finite audio")
|
| 95 |
+
return audio[0, :int(lengths[0])].cpu(), details
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def speak(model, label, style, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0):
|
| 99 |
+
"""One utterance from a label string (``pitput.labels`` format): -> (float
|
| 100 |
+
waveform tensor, phonemized text, details). ``speak_text`` reads Hebrew text."""
|
| 101 |
+
phones = phonemize(label)
|
| 102 |
+
if len(phones) > model.config.max_tokens:
|
| 103 |
+
raise ValueError("text is too long for one utterance; speak_text splits it into sentences")
|
| 104 |
+
wave, details = _voice(model, phones, style, speed, pitch_shift, seed, pitch_range)
|
| 105 |
+
return wave, phones, details
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def speak_text(model, text, style, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, peak=PEAK, addressee=None):
|
| 109 |
+
"""Hebrew text of any length -> (float waveform tensor, phonemized utterances).
|
| 110 |
+
|
| 111 |
+
The text is read sentence by sentence and the audio is joined. A text that
|
| 112 |
+
ends without a mark gets a full stop (``finished``). A sentence too long for
|
| 113 |
+
one call of the model is cut after commas, then between words. ``addressee``
|
| 114 |
+
("m" or "f") sets the gender of second-person words. The audio is scaled
|
| 115 |
+
down as a whole when it peaks above ``peak``; ``peak=None`` returns the
|
| 116 |
+
model's output as it is."""
|
| 117 |
+
limit = model.config.max_tokens - 12
|
| 118 |
+
waves, spoken = [], []
|
| 119 |
+
|
| 120 |
+
def read(piece):
|
| 121 |
+
try:
|
| 122 |
+
phones = phonemize(hebrew_label(piece, addressee))
|
| 123 |
+
except NothingToPronounce:
|
| 124 |
+
return
|
| 125 |
+
if len(phones) <= limit:
|
| 126 |
+
try:
|
| 127 |
+
wave, _ = _voice(model, phones, style, speed, pitch_shift, seed + len(waves), pitch_range)
|
| 128 |
+
except UtteranceTooLong:
|
| 129 |
+
pass
|
| 130 |
+
else:
|
| 131 |
+
waves.append(wave)
|
| 132 |
+
spoken.append(phones)
|
| 133 |
+
return
|
| 134 |
+
pieces = shorter(piece)
|
| 135 |
+
if not pieces:
|
| 136 |
+
raise ValueError(f"one word is too long to speak: {piece[:40]!r}")
|
| 137 |
+
for part in pieces:
|
| 138 |
+
read(part)
|
| 139 |
+
|
| 140 |
+
for sentence in sentences(finished(text)):
|
| 141 |
+
read(sentence)
|
| 142 |
+
if not waves:
|
| 143 |
+
raise NothingToPronounce(NOTHING)
|
| 144 |
+
wave = torch.cat(waves)
|
| 145 |
+
return (wave if peak is None else limit_peak(wave, peak)), spoken
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def main():
|
| 149 |
+
parser = argparse.ArgumentParser(description="Speak Hebrew text with a trained inference checkpoint")
|
| 150 |
+
parser.add_argument("--checkpoint", type=Path, default=CHECKPOINT)
|
| 151 |
+
source = parser.add_mutually_exclusive_group(required=True)
|
| 152 |
+
source.add_argument("--text", help="Hebrew text of any length; it is read sentence by sentence")
|
| 153 |
+
source.add_argument("--text-file", type=Path, help="Read the text from a file (UTF-8)")
|
| 154 |
+
parser.add_argument("--style", default="default")
|
| 155 |
+
parser.add_argument("--addressee", choices=("m", "f"), help="Gender of the person spoken to")
|
| 156 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 157 |
+
parser.add_argument("--speed", type=float, default=1)
|
| 158 |
+
parser.add_argument("--pitch-shift", type=float, default=0, help="Added to log F0 (0.058 is about one semitone)")
|
| 159 |
+
parser.add_argument("--pitch-range", type=float, default=1, help="Scale pitch movement around the utterance mean (1 = as predicted)")
|
| 160 |
+
parser.add_argument("--device", default="cpu")
|
| 161 |
+
args = parser.parse_args()
|
| 162 |
+
import soundfile as sf
|
| 163 |
+
model = load_model(args.checkpoint, args.device)
|
| 164 |
+
if args.style not in model.config.speakers:
|
| 165 |
+
parser.error(f"--style must be one of {model.config.speakers}")
|
| 166 |
+
text = args.text if args.text is not None else args.text_file.read_text(encoding="utf-8")
|
| 167 |
+
try:
|
| 168 |
+
raw, spoken = speak_text(model, text, args.style, args.speed, args.pitch_shift, pitch_range=args.pitch_range, peak=None,
|
| 169 |
+
addressee=args.addressee)
|
| 170 |
+
except NothingToPronounce:
|
| 171 |
+
parser.exit(1, f"{NOTHING}, so no file was written.\n")
|
| 172 |
+
wave = limit_peak(raw)
|
| 173 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 174 |
+
# Float samples, never above PEAK: a later conversion to 16-bit does not clip.
|
| 175 |
+
sf.write(args.output, wave.numpy(), model.config.sample_rate, subtype="FLOAT")
|
| 176 |
+
for phones in spoken:
|
| 177 |
+
print(" ".join(phones.symbols))
|
| 178 |
+
loudest = float(raw.abs().max())
|
| 179 |
+
scaled = f", scaled down to {PEAK}" if wave is not raw else ""
|
| 180 |
+
print(f"Saved {len(wave)/model.config.sample_rate:.2f}s in {len(spoken)} utterance{'s' if len(spoken) != 1 else ''}; peak={loudest:.4f}{scaled}")
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
if __name__ == "__main__":
|
| 184 |
+
main()
|
pitput/labels.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Phoneme labels: what the model reads.
|
| 2 |
+
|
| 3 |
+
The Phonikud-vocalized sentence, phonemized word by word so expression tags go back at their token index, with the
|
| 4 |
+
punctuation that shapes delivery kept. Words are separated by spaces, tags are <name> tokens, and punctuation is glued
|
| 5 |
+
to the word before it, e.g. "ʔatˈa <sigh> bˈa?". Kept: , . ! ? and … ; colons, semicolons and dashes become ",";
|
| 6 |
+
quotes and brackets are dropped.
|
| 7 |
+
|
| 8 |
+
Why word by word: phonikud.phonemize on a whole sentence splits hyphenated words and turns a standalone dash into an
|
| 9 |
+
empty token, which shifts tag positions, and it drops : ; and dashes.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import functools
|
| 14 |
+
import re
|
| 15 |
+
|
| 16 |
+
import phonikud
|
| 17 |
+
|
| 18 |
+
from .g2p import apply_hints
|
| 19 |
+
|
| 20 |
+
PUNCT = {".": ".", ",": ",", "!": "!", "?": "?", "…": "…",
|
| 21 |
+
";": ",", ":": ",", "-": ",", "\u2013": ",", "\u2014": ",", "\u05be": ","}
|
| 22 |
+
_PUNCT_ONLY = re.compile("[" + re.escape("".join(set(PUNCT.values()))) + "]+")
|
| 23 |
+
_WORD_CHAR = re.compile("[\u05d0-\u05ea\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u05f3\u05f4|]")
|
| 24 |
+
_LETTER = re.compile("[\u05d0-\u05ea][\u0591-\u05c7]*")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def fix_meteg(word: str) -> str:
|
| 28 |
+
"""Drop Phonikud's vocal-shva mark from letters without a shva: phonemize turns it into an extra "e"."""
|
| 29 |
+
return _LETTER.sub(lambda m: m.group() if "\u05b0" in m.group() else m.group().replace("\u05bd", ""), word)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def punctuation(text: str) -> str:
|
| 33 |
+
out = ""
|
| 34 |
+
for ch in text.replace("...", "…"):
|
| 35 |
+
symbol = PUNCT.get(ch)
|
| 36 |
+
if symbol and not out.endswith(symbol):
|
| 37 |
+
out += symbol
|
| 38 |
+
return out
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@functools.lru_cache(maxsize=100_000)
|
| 42 |
+
def _phonemize_word(core: str) -> str:
|
| 43 |
+
return phonikud.phonemize(fix_meteg(core)).strip()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def phonemize_token(token: str) -> str:
|
| 47 |
+
"""One space-separated token: the word's phonemes plus its trailing punctuation (leading punctuation is dropped)."""
|
| 48 |
+
letters = [k for k, ch in enumerate(token) if "\u05d0" <= ch <= "\u05ea"]
|
| 49 |
+
if not letters:
|
| 50 |
+
return punctuation(token)
|
| 51 |
+
end = max(k for k, ch in enumerate(token) if _WORD_CHAR.match(ch)) + 1
|
| 52 |
+
return _phonemize_word(token[letters[0]:end]) + punctuation(token[end:])
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def label(vocalized: str, hints: dict, tags: list[dict]) -> str:
|
| 56 |
+
"""Vocalized text, per-word gender hints ({token_index: "m"/"f"}) and tags ([{i, tag}]) -> a label string."""
|
| 57 |
+
tokens = apply_hints(vocalized, hints).split(" ")
|
| 58 |
+
pieces = []
|
| 59 |
+
for i in range(len(tokens) + 1):
|
| 60 |
+
pieces.extend(f"<{t['tag']}>" for t in tags if t["i"] == i)
|
| 61 |
+
if i < len(tokens):
|
| 62 |
+
pieces.append(phonemize_token(tokens[i]))
|
| 63 |
+
out = []
|
| 64 |
+
for piece in pieces:
|
| 65 |
+
if not piece:
|
| 66 |
+
continue
|
| 67 |
+
if _PUNCT_ONLY.fullmatch(piece):
|
| 68 |
+
if out and not out[-1].startswith("<") and not out[-1].endswith(piece):
|
| 69 |
+
out[-1] += piece
|
| 70 |
+
continue
|
| 71 |
+
out.append(piece)
|
| 72 |
+
return " ".join(out)
|
pitput/model.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""fiato v2: feed-forward transformer with a pitch-driven source-filter head.
|
| 2 |
+
|
| 3 |
+
All weights start random. Tensor conventions: hidden [B,T,C], frame features
|
| 4 |
+
[B,T], waveform [B,S]. Acoustic frame i is centred on sample ``i * hop``.
|
| 5 |
+
|
| 6 |
+
phoneme + stress + voice -> transformer encoder -> duration predictor
|
| 7 |
+
-> repeat tokens by duration (supervised durations while training)
|
| 8 |
+
-> frame transformer A -> pitch / voicing / energy
|
| 9 |
+
-> frame transformer B -> filter parameters
|
| 10 |
+
-> harmonic pulse train (from pitch) and noise, each shaped by its
|
| 11 |
+
time-varying filter in the STFT domain -> 24 kHz waveform
|
| 12 |
+
|
| 13 |
+
The waveform head has no neural upsampling network. The oscillator supplies the
|
| 14 |
+
periodicity that a tiny decoder cannot render reliably on its own; the network
|
| 15 |
+
only has to predict slowly varying spectral envelopes (complex cepstra), which
|
| 16 |
+
is what keeps the voice from sounding buzzy or unstable at this size.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import math
|
| 20 |
+
import torch
|
| 21 |
+
from torch import nn
|
| 22 |
+
from torch.nn import functional as F
|
| 23 |
+
from .config import Config
|
| 24 |
+
from .text import PHONE_IDS, SYMBOL_TO_ID, VOWELS
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class UtteranceTooLong(ValueError):
|
| 28 |
+
"""The utterance needs more frames than ``max_frames``: split the input."""
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def length_mask(lengths, size=None):
|
| 32 |
+
size = int(lengths.max()) if size is None else size
|
| 33 |
+
return torch.arange(size, device=lengths.device)[None, :] < lengths[:, None]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def rotary_table(size, width, device):
|
| 37 |
+
inverse = 1.0 / (10000 ** (torch.arange(0, width, 2, device=device, dtype=torch.float32) / width))
|
| 38 |
+
angle = torch.arange(size, device=device, dtype=torch.float32)[:, None] * inverse[None]
|
| 39 |
+
return angle.cos(), angle.sin()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def rotate(x, table):
|
| 43 |
+
cos, sin = table[0].to(x.dtype), table[1].to(x.dtype)
|
| 44 |
+
a, b = x[..., 0::2], x[..., 1::2]
|
| 45 |
+
return torch.stack((a * cos - b * sin, a * sin + b * cos), dim=-1).flatten(-2)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class Attention(nn.Module):
|
| 49 |
+
"""Self-attention with rotary (relative) positions: no absolute position
|
| 50 |
+
table, so utterances longer than those seen in training stay in range."""
|
| 51 |
+
|
| 52 |
+
def __init__(self, channels, heads, dropout):
|
| 53 |
+
super().__init__()
|
| 54 |
+
self.heads, self.dropout = heads, dropout
|
| 55 |
+
self.qkv = nn.Linear(channels, 3 * channels)
|
| 56 |
+
self.out = nn.Linear(channels, channels)
|
| 57 |
+
|
| 58 |
+
def forward(self, x, mask, table):
|
| 59 |
+
b, t, c = x.shape
|
| 60 |
+
q, k, v = self.qkv(x).view(b, t, 3, self.heads, c // self.heads).permute(2, 0, 3, 1, 4)
|
| 61 |
+
y = F.scaled_dot_product_attention(rotate(q, table), rotate(k, table), v, attn_mask=mask[:, None, None, :],
|
| 62 |
+
dropout_p=self.dropout if self.training else 0.0)
|
| 63 |
+
return self.out(y.transpose(1, 2).reshape(b, t, c))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class Block(nn.Module):
|
| 67 |
+
def __init__(self, c):
|
| 68 |
+
super().__init__()
|
| 69 |
+
self.norm1 = nn.LayerNorm(c.channels)
|
| 70 |
+
self.attention = Attention(c.channels, c.heads, c.dropout)
|
| 71 |
+
self.norm2 = nn.LayerNorm(c.channels)
|
| 72 |
+
self.depthwise = nn.Conv1d(c.channels, c.channels, c.kernel, padding=c.kernel // 2, groups=c.channels)
|
| 73 |
+
self.expand = nn.Linear(c.channels, c.feedforward)
|
| 74 |
+
self.project = nn.Linear(c.feedforward, c.channels)
|
| 75 |
+
self.dropout = nn.Dropout(c.dropout)
|
| 76 |
+
|
| 77 |
+
def forward(self, x, mask, table):
|
| 78 |
+
keep = mask[..., None]
|
| 79 |
+
x = (x + self.dropout(self.attention(self.norm1(x), mask, table))) * keep
|
| 80 |
+
# Mask before the convolution so padded positions cannot leak back in.
|
| 81 |
+
y = self.depthwise((self.norm2(x) * keep).transpose(1, 2)).transpose(1, 2)
|
| 82 |
+
y = self.project(self.dropout(F.gelu(self.expand(y))))
|
| 83 |
+
return (x + self.dropout(y)) * keep
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class Stack(nn.Module):
|
| 87 |
+
def __init__(self, c, layers):
|
| 88 |
+
super().__init__()
|
| 89 |
+
self.blocks = nn.ModuleList(Block(c) for _ in range(layers))
|
| 90 |
+
self.norm = nn.LayerNorm(c.channels)
|
| 91 |
+
self.width = c.channels // c.heads
|
| 92 |
+
|
| 93 |
+
def forward(self, x, mask):
|
| 94 |
+
table = rotary_table(x.shape[1], self.width, x.device)
|
| 95 |
+
for block in self.blocks:
|
| 96 |
+
x = block(x, mask, table)
|
| 97 |
+
return self.norm(x) * mask[..., None]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class Predictor(nn.Module):
|
| 101 |
+
def __init__(self, c, outputs):
|
| 102 |
+
super().__init__()
|
| 103 |
+
p = c.predictor_channels
|
| 104 |
+
self.conv1 = nn.Conv1d(c.channels, p, 3, padding=1)
|
| 105 |
+
self.conv2 = nn.Conv1d(p, p, 3, padding=1)
|
| 106 |
+
self.norm1, self.norm2 = nn.LayerNorm(p), nn.LayerNorm(p)
|
| 107 |
+
self.drop = nn.Dropout(c.dropout)
|
| 108 |
+
self.out = nn.Linear(p, outputs)
|
| 109 |
+
|
| 110 |
+
def forward(self, x, mask):
|
| 111 |
+
x = self.conv1((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)
|
| 112 |
+
x = self.drop(self.norm1(F.gelu(x))) * mask[..., None]
|
| 113 |
+
x = self.conv2(x.transpose(1, 2)).transpose(1, 2)
|
| 114 |
+
x = self.drop(self.norm2(F.gelu(x)))
|
| 115 |
+
return self.out(x) * mask[..., None]
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def pulse_train(f0, sample_rate, hop, limit=0.47, phase=None):
|
| 119 |
+
"""Band-limited pulse train [B, T*hop] with unit power for f0 [B, T] in Hz.
|
| 120 |
+
|
| 121 |
+
Frame values sit at samples i*hop and are interpolated in the log domain.
|
| 122 |
+
The sum of all harmonics below ``limit * sample_rate`` has the closed form
|
| 123 |
+
sin((K+1/2)x) / (2 sin(x/2)) - 1/2; the top harmonic fades in and out so the
|
| 124 |
+
signal stays continuous while the pitch glides. Float64 keeps the phase exact.
|
| 125 |
+
"""
|
| 126 |
+
frames = f0.shape[1]
|
| 127 |
+
knots = torch.log(f0.double().clamp_min(1.0))
|
| 128 |
+
knots = torch.cat((knots, knots[:, -1:]), dim=1)
|
| 129 |
+
position = torch.arange(frames * hop, device=f0.device, dtype=torch.float64) / hop
|
| 130 |
+
low = position.floor().long()
|
| 131 |
+
weight = position - low
|
| 132 |
+
frequency = torch.exp(knots[:, low] * (1 - weight) + knots[:, low + 1] * weight)
|
| 133 |
+
theta = 2 * math.pi * torch.cumsum(frequency / sample_rate, dim=1)
|
| 134 |
+
if phase is not None:
|
| 135 |
+
theta = theta + phase.double()[:, None]
|
| 136 |
+
theta = torch.remainder(theta, 2 * math.pi)
|
| 137 |
+
harmonics = (limit * sample_rate / frequency).clamp_min(1.0)
|
| 138 |
+
whole = torch.floor(harmonics)
|
| 139 |
+
denominator = 2 * torch.sin(theta / 2)
|
| 140 |
+
tiny = denominator.abs() < 1e-7
|
| 141 |
+
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / torch.where(tiny, torch.ones_like(denominator), denominator) - 0.5)
|
| 142 |
+
wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta)
|
| 143 |
+
return (wave * torch.sqrt(2.0 / harmonics)).float()
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def filtered(source, response, fft, hop, window):
|
| 147 |
+
"""Time-varying filtering: response [B, frames, bins] multiplies the STFT."""
|
| 148 |
+
spectrum = torch.stft(source, fft, hop, window=window, center=True, pad_mode="constant", return_complex=True)
|
| 149 |
+
if spectrum.shape[-1] != response.shape[1]:
|
| 150 |
+
raise ValueError("filter frames do not match the source")
|
| 151 |
+
return torch.istft(spectrum * response.transpose(1, 2), fft, hop, window=window, center=True, length=source.shape[-1])
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def cepstrum_response(causal, anticausal, fft):
|
| 155 |
+
"""Complex frequency response exp(FFT(cepstrum)). Causal coefficients give a
|
| 156 |
+
minimum-phase filter; anticausal ones add a maximum-phase part (glottal pulse)."""
|
| 157 |
+
buffer = causal.new_zeros(*causal.shape[:-1], fft)
|
| 158 |
+
buffer[..., :causal.shape[-1]] = causal
|
| 159 |
+
if anticausal is not None:
|
| 160 |
+
buffer[..., fft - anticausal.shape[-1]:] = anticausal.flip(-1)
|
| 161 |
+
spectrum = torch.fft.rfft(buffer, dim=-1)
|
| 162 |
+
return torch.polar(torch.exp(spectrum.real.clamp(-14, 6)), spectrum.imag)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class SourceFilter(nn.Module):
|
| 166 |
+
"""Harmonic-plus-noise synthesis controlled per frame by two linear heads."""
|
| 167 |
+
|
| 168 |
+
def __init__(self, c):
|
| 169 |
+
super().__init__()
|
| 170 |
+
self.config = c
|
| 171 |
+
self.harmonic_steps, self.noise_steps = c.hop // c.synth_hop, c.hop // c.noise_hop
|
| 172 |
+
self.harmonic = nn.Linear(c.channels, c.cep_causal + c.cep_anti + self.harmonic_steps)
|
| 173 |
+
self.noise = nn.Linear(c.channels, c.noise_cep + self.noise_steps)
|
| 174 |
+
for head, level in ((self.harmonic, -1.5), (self.noise, -2.5)): # start quiet and spectrally flat
|
| 175 |
+
nn.init.zeros_(head.weight)
|
| 176 |
+
nn.init.zeros_(head.bias)
|
| 177 |
+
head.bias.data[0] = level
|
| 178 |
+
|
| 179 |
+
def lifter(size, first):
|
| 180 |
+
scale = 1.0 / (1.0 + torch.arange(first, first + size, dtype=torch.float32) / 4)
|
| 181 |
+
if first == 0:
|
| 182 |
+
scale[0] = 2.0 # coefficient 0 is the log gain and needs range
|
| 183 |
+
return scale
|
| 184 |
+
self.register_buffer("causal_lifter", lifter(c.cep_causal, 0), persistent=False)
|
| 185 |
+
self.register_buffer("anti_lifter", lifter(c.cep_anti, 1), persistent=False)
|
| 186 |
+
self.register_buffer("noise_lifter", lifter(c.noise_cep, 0), persistent=False)
|
| 187 |
+
self.register_buffer("synth_window", torch.hann_window(c.synth_fft), persistent=False)
|
| 188 |
+
self.register_buffer("noise_window", torch.hann_window(c.noise_fft), persistent=False)
|
| 189 |
+
|
| 190 |
+
@staticmethod
|
| 191 |
+
def _take(values, index):
|
| 192 |
+
return values.gather(1, index[..., None].expand(-1, -1, values.shape[-1]))
|
| 193 |
+
|
| 194 |
+
def _subframes(self, frames, lengths, steps):
|
| 195 |
+
"""Sub-frame j uses frame j // steps; indices past an utterance's end
|
| 196 |
+
repeat its last sub-frame so padding never changes real audio. Two extra
|
| 197 |
+
sub-frames cover the half window of zeros appended to the source."""
|
| 198 |
+
sub = torch.arange(steps * frames + 3, device=lengths.device)[None].expand(len(lengths), -1)
|
| 199 |
+
return torch.minimum(sub, (steps * lengths - 1).clamp_min(0)[:, None])
|
| 200 |
+
|
| 201 |
+
@staticmethod
|
| 202 |
+
def _shape(source, response, fft, hop, window):
|
| 203 |
+
# Half a window of trailing zeros: the frames around an utterance's end are
|
| 204 |
+
# then the same whether it stands alone or is padded inside a batch.
|
| 205 |
+
return filtered(F.pad(source, (0, fft // 2)), response, fft, hop, window)[:, :source.shape[-1]]
|
| 206 |
+
|
| 207 |
+
def forward(self, hidden, f0, lengths, noise=None, phase=None):
|
| 208 |
+
c = self.config
|
| 209 |
+
frames = hidden.shape[1]
|
| 210 |
+
samples = frames * c.hop
|
| 211 |
+
with torch.autocast(device_type=hidden.device.type, enabled=False):
|
| 212 |
+
hidden = hidden.float()
|
| 213 |
+
audible = length_mask(lengths * c.hop, samples)
|
| 214 |
+
last = (lengths - 1).clamp_min(0)[:, None]
|
| 215 |
+
# Harmonic branch: envelope interpolated between frames, one gain per sub-frame.
|
| 216 |
+
h = self.harmonic(hidden)
|
| 217 |
+
split = c.cep_causal + c.cep_anti
|
| 218 |
+
sub = self._subframes(frames, lengths, self.harmonic_steps)
|
| 219 |
+
here, following = sub // self.harmonic_steps, torch.minimum((sub + 1) // self.harmonic_steps, last)
|
| 220 |
+
causal = 0.5 * (self._take(h[..., :c.cep_causal], here) + self._take(h[..., :c.cep_causal], following)) * self.causal_lifter
|
| 221 |
+
anti = 0.5 * (self._take(h[..., c.cep_causal:split], here) + self._take(h[..., c.cep_causal:split], following)) * self.anti_lifter
|
| 222 |
+
causal = torch.cat((causal[..., :1] + h[..., split:].flatten(1).gather(1, sub)[..., None], causal[..., 1:]), dim=-1)
|
| 223 |
+
harmonic_response = cepstrum_response(causal, anti, c.synth_fft)
|
| 224 |
+
# Noise branch: one envelope per frame, four gains per frame for sharp consonants.
|
| 225 |
+
n = self.noise(hidden)
|
| 226 |
+
sub = self._subframes(frames, lengths, self.noise_steps)
|
| 227 |
+
shape = self._take(n[..., :c.noise_cep], sub // self.noise_steps) * self.noise_lifter
|
| 228 |
+
shape = torch.cat((shape[..., :1] + n[..., c.noise_cep:].flatten(1).gather(1, sub)[..., None], shape[..., 1:]), dim=-1)
|
| 229 |
+
noise_response = cepstrum_response(shape, None, c.noise_fft)
|
| 230 |
+
# Hold the last real pitch value through padding: the pitch glide inside an
|
| 231 |
+
# utterance's final frame must not depend on what the batch was padded with.
|
| 232 |
+
f0 = f0.float().gather(1, torch.minimum(torch.arange(frames, device=f0.device)[None], last))
|
| 233 |
+
pulses = pulse_train(f0, c.sample_rate, c.hop, c.harmonic_limit, phase) * audible
|
| 234 |
+
if noise is None:
|
| 235 |
+
noise = torch.randn(hidden.shape[0], samples, device=hidden.device)
|
| 236 |
+
wave = (self._shape(pulses, harmonic_response, c.synth_fft, c.synth_hop, self.synth_window)
|
| 237 |
+
+ self._shape(noise * audible, noise_response, c.noise_fft, c.noise_hop, self.noise_window))
|
| 238 |
+
return wave * audible
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
class Fiato(nn.Module):
|
| 242 |
+
def __init__(self, config=Config()):
|
| 243 |
+
super().__init__()
|
| 244 |
+
self.config = c = config
|
| 245 |
+
self.tokens = nn.Embedding(c.vocab_size, c.channels, padding_idx=0)
|
| 246 |
+
self.stress = nn.Embedding(3, c.channels)
|
| 247 |
+
self.voices = nn.Embedding(len(c.speakers), c.channels)
|
| 248 |
+
self.encoder = Stack(c, c.encoder_layers)
|
| 249 |
+
# Outputs per token: log(1 + frames) and, for pause-capable tokens, a
|
| 250 |
+
# "has frames at all" logit. A pause is absent more often than present, so
|
| 251 |
+
# one regression would average both cases into a pause that is too short.
|
| 252 |
+
self.duration = Predictor(c, 2)
|
| 253 |
+
self.frame_voices = nn.Embedding(len(c.speakers), c.channels)
|
| 254 |
+
self.frame_position = nn.Linear(3, c.channels)
|
| 255 |
+
self.prosody_stack = Stack(c, c.prosody_layers)
|
| 256 |
+
# Outputs: continuous log(F0/reference), voiced logit, normalized log RMS energy.
|
| 257 |
+
self.prosody_head = nn.Linear(c.channels, 3)
|
| 258 |
+
self.prosody_projection = nn.Linear(3, c.channels)
|
| 259 |
+
self.acoustic_stack = Stack(c, c.acoustic_layers)
|
| 260 |
+
self.vocoder = SourceFilter(c)
|
| 261 |
+
phones = torch.zeros(c.vocab_size, dtype=torch.bool)
|
| 262 |
+
phones[sorted(PHONE_IDS)] = True
|
| 263 |
+
self.register_buffer("phone_tokens", phones, persistent=False)
|
| 264 |
+
vowels = torch.zeros(c.vocab_size, dtype=torch.bool)
|
| 265 |
+
vowels[[SYMBOL_TO_ID[v] for v in VOWELS]] = True
|
| 266 |
+
self.register_buffer("vowel_tokens", vowels, persistent=False)
|
| 267 |
+
unvoiced = torch.tensor([name in c.unvoiced_speakers for name in c.speakers], dtype=torch.bool)
|
| 268 |
+
self.register_buffer("unvoiced_speakers", unvoiced, persistent=False)
|
| 269 |
+
|
| 270 |
+
def encode(self, tokens, stress, lengths, speakers):
|
| 271 |
+
if tokens.ndim != 2 or stress.shape != tokens.shape or lengths.shape != (tokens.shape[0],) or speakers.shape != lengths.shape:
|
| 272 |
+
raise ValueError("expected tokens[B,N], stress[B,N], lengths[B], speakers[B]")
|
| 273 |
+
if tokens.shape[1] > self.config.max_tokens or bool((lengths < 1).any()) or bool((lengths > tokens.shape[1]).any()):
|
| 274 |
+
raise ValueError("invalid token length")
|
| 275 |
+
mask = length_mask(lengths, tokens.shape[1])
|
| 276 |
+
x = (self.tokens(tokens) + self.stress(stress) + self.voices(speakers)[:, None]) * mask[..., None]
|
| 277 |
+
return self.encoder(x, mask), mask
|
| 278 |
+
|
| 279 |
+
def expand(self, hidden, durations, token_mask, speakers):
|
| 280 |
+
"""Repeat token states by integer durations (zero is allowed for pause
|
| 281 |
+
tokens) and add each frame's position inside its token."""
|
| 282 |
+
if durations.shape != token_mask.shape or bool((durations < 0).any()) or bool((durations[~token_mask] != 0).any()):
|
| 283 |
+
raise ValueError("invalid durations")
|
| 284 |
+
lengths = durations.sum(1)
|
| 285 |
+
if bool((lengths < 1).any()):
|
| 286 |
+
raise ValueError("utterance has no frames")
|
| 287 |
+
if bool((lengths > self.config.max_frames).any()):
|
| 288 |
+
raise UtteranceTooLong("utterance exceeds max_frames; split the input")
|
| 289 |
+
size = int(lengths.max())
|
| 290 |
+
ends = durations.cumsum(1)
|
| 291 |
+
frame = torch.arange(size, device=hidden.device)[None].expand(hidden.shape[0], size).contiguous()
|
| 292 |
+
index = torch.searchsorted(ends, frame, right=True).clamp(max=durations.shape[1] - 1)
|
| 293 |
+
owned = durations.gather(1, index).clamp_min(1)
|
| 294 |
+
offset = frame - (ends.gather(1, index) - durations.gather(1, index))
|
| 295 |
+
relative = (offset.to(hidden.dtype) + 0.5) / owned
|
| 296 |
+
features = torch.stack((relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1)
|
| 297 |
+
mask = frame < lengths[:, None]
|
| 298 |
+
expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1]))
|
| 299 |
+
expanded = expanded + self.frame_position(features) + self.frame_voices(speakers)[:, None]
|
| 300 |
+
return expanded * mask[..., None], mask, lengths
|
| 301 |
+
|
| 302 |
+
def predict_prosody(self, frames, frame_mask):
|
| 303 |
+
hidden = self.prosody_stack(frames, frame_mask)
|
| 304 |
+
return hidden, self.prosody_head(hidden) * frame_mask[..., None]
|
| 305 |
+
|
| 306 |
+
@staticmethod
|
| 307 |
+
def hard_prosody(prediction):
|
| 308 |
+
"""What the decoder is conditioned on when it is not teacher-forced."""
|
| 309 |
+
return torch.stack((prediction[..., 0].clamp(-2.0, 1.5), (prediction[..., 1] > 0).to(prediction.dtype),
|
| 310 |
+
prediction[..., 2].clamp(-4.5, 3.0)), dim=-1)
|
| 311 |
+
|
| 312 |
+
def decode(self, prosody_hidden, condition, frame_mask):
|
| 313 |
+
return self.acoustic_stack((prosody_hidden + self.prosody_projection(condition)) * frame_mask[..., None], frame_mask)
|
| 314 |
+
|
| 315 |
+
def frequency(self, log_f0):
|
| 316 |
+
c = self.config
|
| 317 |
+
return (c.pitch_reference_hz * torch.exp(log_f0.float())).clamp(c.f0_min, c.f0_max)
|
| 318 |
+
|
| 319 |
+
def predict_durations(self, hidden, mask, tokens, speed=1.0):
|
| 320 |
+
output = self.duration(hidden, mask).float()
|
| 321 |
+
frames = torch.round(torch.expm1(output[..., 0].clamp(0, math.log1p(400))) / speed).long().clamp_min(1)
|
| 322 |
+
# Every phoneme is audible for at least one frame. A pause-capable token
|
| 323 |
+
# (edge, punctuation, word boundary) lasts its predicted length when the
|
| 324 |
+
# presence logit says there is a pause, and vanishes otherwise.
|
| 325 |
+
present = self.phone_tokens[tokens] | (output[..., 1] > 0)
|
| 326 |
+
return torch.where(present, frames, torch.zeros_like(frames)) * mask
|
| 327 |
+
|
| 328 |
+
@staticmethod
|
| 329 |
+
def widen_pitch(prosody, frame_mask, pitch_range):
|
| 330 |
+
"""Scale the predicted pitch movement around each utterance's voiced mean.
|
| 331 |
+
Regression predicts a flatter contour than the recordings; values above 1
|
| 332 |
+
restore range (1.0 leaves the prediction untouched)."""
|
| 333 |
+
if pitch_range == 1.0:
|
| 334 |
+
return prosody
|
| 335 |
+
voiced = ((prosody[..., 1] > 0) & frame_mask).to(prosody.dtype)
|
| 336 |
+
mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(1, keepdim=True).clamp_min(1)
|
| 337 |
+
pitch = mean + (prosody[..., 0] - mean) * pitch_range
|
| 338 |
+
return torch.cat((pitch[..., None], prosody[..., 1:]), dim=-1) * frame_mask[..., None]
|
| 339 |
+
|
| 340 |
+
@staticmethod
|
| 341 |
+
def pitch_wobble(shape, semitones, seed, device):
|
| 342 |
+
"""Small random pitch movement for the oscillator only (log-F0 units).
|
| 343 |
+
Recorded pitch has cycle-to-cycle irregularity that a predicted contour
|
| 344 |
+
lacks; two thirds of the wobble is slow drift, one third frame-level."""
|
| 345 |
+
generator = torch.Generator().manual_seed(seed + 7919)
|
| 346 |
+
fast = torch.randn(shape, generator=generator)
|
| 347 |
+
slow = F.avg_pool1d(torch.randn(shape[0], 1, shape[1] + 16, generator=generator), 17, stride=1)[:, 0]
|
| 348 |
+
slow = slow / slow.std().clamp_min(1e-6)
|
| 349 |
+
return ((2 * slow + fast) / 5 ** 0.5 * semitones * math.log(2) / 12).to(device)
|
| 350 |
+
|
| 351 |
+
@torch.inference_mode()
|
| 352 |
+
def synthesize(self, tokens, stress, lengths, speakers, speed=1.0, pitch_shift=0.0, seed=0, pitch_range=1.0, pitch_jitter=0.0,
|
| 353 |
+
voiced_vowels=True):
|
| 354 |
+
"""Free-running synthesis: predicted durations, pitch, voicing and energy."""
|
| 355 |
+
if not math.isfinite(speed) or not 0.25 <= speed <= 4 or not math.isfinite(pitch_shift) or not 0 <= pitch_range <= 3:
|
| 356 |
+
raise ValueError("speed must be between 0.25 and 4, pitch_shift finite and pitch_range between 0 and 3")
|
| 357 |
+
hidden, mask = self.encode(tokens, stress, lengths, speakers)
|
| 358 |
+
durations = self.predict_durations(hidden, mask, tokens, speed)
|
| 359 |
+
frames, frame_mask, frame_lengths = self.expand(hidden, durations, mask, speakers)
|
| 360 |
+
prosody_hidden, prosody = self.predict_prosody(frames, frame_mask)
|
| 361 |
+
if voiced_vowels:
|
| 362 |
+
# The pitch tracker missed creaky vowels in the recordings, and the
|
| 363 |
+
# voicing predictor learned to drop whole vowels (heard as a whispered
|
| 364 |
+
# word). A vowel is voiced by definition, so its frames are kept voiced,
|
| 365 |
+
# except in a whispering style.
|
| 366 |
+
ends = durations.cumsum(1)
|
| 367 |
+
frame = torch.arange(frames.shape[1], device=tokens.device)[None].expand(len(lengths), -1).contiguous()
|
| 368 |
+
owner = torch.searchsorted(ends, frame, right=True).clamp(max=tokens.shape[1] - 1)
|
| 369 |
+
vowel = self.vowel_tokens[tokens].gather(1, owner) & frame_mask & ~self.unvoiced_speakers[speakers][:, None]
|
| 370 |
+
prosody = torch.cat((prosody[..., :1], torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])[..., None],
|
| 371 |
+
prosody[..., 2:]), dim=-1)
|
| 372 |
+
prosody = self.widen_pitch(prosody, frame_mask, pitch_range)
|
| 373 |
+
acoustic = self.decode(prosody_hidden, self.hard_prosody(prosody), frame_mask)
|
| 374 |
+
# One noise stream per row, drawn at the row's own length, so batching or
|
| 375 |
+
# padding never changes a row's audio.
|
| 376 |
+
noise = nn.utils.rnn.pad_sequence([torch.randn(int(n) * self.config.hop, generator=torch.Generator().manual_seed(seed + row))
|
| 377 |
+
for row, n in enumerate(frame_lengths)], batch_first=True)
|
| 378 |
+
pitch = prosody[..., 0] + pitch_shift
|
| 379 |
+
if pitch_jitter > 0:
|
| 380 |
+
pitch = pitch + self.pitch_wobble(pitch.shape, pitch_jitter, seed, pitch.device)
|
| 381 |
+
wave = self.vocoder(acoustic, self.frequency(pitch), frame_lengths, noise=noise.to(hidden.device))
|
| 382 |
+
return wave, frame_lengths * self.config.hop, {"durations": durations, "prosody": prosody, "frame_mask": frame_mask}
|
pitput/text.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hebrew front end: phoneme label strings -> phoneme tokens with a stress feature.
|
| 2 |
+
|
| 3 |
+
The same code runs during annotation, training and inference, so tokens can
|
| 4 |
+
never drift between them.
|
| 5 |
+
|
| 6 |
+
* Input is a label string from ``pitput.labels`` (Phonikud phonemes, word by
|
| 7 |
+
word, with the addressee decisions applied), e.g. ``"ʔatˈa <sigh> bˈa?"``.
|
| 8 |
+
``hebrew_label`` makes one from plain Hebrew at inference time.
|
| 9 |
+
* A token is one Phonikud phoneme. ``ts``, ``tʃ`` and ``dʒ`` are single
|
| 10 |
+
tokens (read greedily). Every phone is also a label of the phoneme recogniser
|
| 11 |
+
used for forced alignment, so one aligned label is exactly one token.
|
| 12 |
+
* Stress (Phonikud's ``ˈ`` before the stressed vowel) is a parallel feature
|
| 13 |
+
(0 none, 1 primary), not a token, so it never has to be given audio frames.
|
| 14 |
+
* Punctuation, word boundaries and the utterance edges are explicit tokens that
|
| 15 |
+
may last zero frames. They carry pauses when the recording has them.
|
| 16 |
+
* The expression tags are tokens too; the aligner gives each one the sound
|
| 17 |
+
between the silences around it.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from dataclasses import dataclass
|
| 21 |
+
import hashlib
|
| 22 |
+
import json
|
| 23 |
+
import re
|
| 24 |
+
|
| 25 |
+
FRONTEND_VERSION = 1
|
| 26 |
+
|
| 27 |
+
PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>"
|
| 28 |
+
SPECIALS = (PAD, BOS, EOS, SP, UNK)
|
| 29 |
+
PUNCTUATION = (",", ".", "?", "!", "…")
|
| 30 |
+
CONSONANTS = ("b", "v", "ɡ", "d", "h", "w", "z", "ʒ", "χ", "t", "j", "k", "l", "m", "n", "s", "ʔ", "f", "p",
|
| 31 |
+
"ts", "tʃ", "dʒ", "ʁ", "ʃ")
|
| 32 |
+
VOWELS = ("a", "e", "i", "o", "u")
|
| 33 |
+
PHONES = CONSONANTS + VOWELS
|
| 34 |
+
# The expression tags as written in labels and input text ("<short pause>").
|
| 35 |
+
TAGS = ("laugh", "chuckle", "sigh", "breath", "gasp", "throat-clearing", "short pause", "long pause")
|
| 36 |
+
# Their tokens. A space becomes "_" so that a token sequence joined by spaces splits back into the same tokens.
|
| 37 |
+
EXPRESSIONS = tuple(f"<{tag.replace(' ', '_')}>" for tag in TAGS)
|
| 38 |
+
# Append-only: changing the order or removing a symbol invalidates checkpoints.
|
| 39 |
+
SYMBOLS = SPECIALS + PUNCTUATION + PHONES + EXPRESSIONS
|
| 40 |
+
SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)}
|
| 41 |
+
PHONE_SET = frozenset(PHONES)
|
| 42 |
+
PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]}
|
| 43 |
+
VOICELESS = frozenset(("p", "t", "k", "f", "s", "ʃ", "χ", "h", "ts", "tʃ", "ʔ"))
|
| 44 |
+
VOWEL_SET = frozenset(VOWELS)
|
| 45 |
+
STRESS = "ˈ"
|
| 46 |
+
|
| 47 |
+
# Phonikud symbols outside the inventory (its non-modern schema, Latin leftovers).
|
| 48 |
+
FALLBACK = {"r": ("ʁ",), "x": ("χ",), "g": ("ɡ",)}
|
| 49 |
+
|
| 50 |
+
_MULTI = sorted((p for p in PHONES if len(p) > 1), key=len, reverse=True)
|
| 51 |
+
_WRITTEN = re.compile("<(" + "|".join("[ _]".join(map(re.escape, tag.split(" "))) for tag in TAGS) + ")>")
|
| 52 |
+
_EXPRESSION = re.compile("(" + "|".join(re.escape(e) for e in EXPRESSIONS) + ")")
|
| 53 |
+
_TAG = re.compile(r"<[^<>]+>")
|
| 54 |
+
_PUNCTUATION_CHARS = frozenset(PUNCTUATION)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class NothingToPronounce(ValueError):
|
| 58 |
+
"""The text has no word to say: only marks, tags and the like."""
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True)
|
| 62 |
+
class Phonemized:
|
| 63 |
+
symbols: tuple
|
| 64 |
+
ids: tuple
|
| 65 |
+
stress: tuple
|
| 66 |
+
unknown: tuple # characters that had no inventory mapping
|
| 67 |
+
|
| 68 |
+
def __len__(self):
|
| 69 |
+
return len(self.ids)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def frontend_identity():
|
| 73 |
+
return {"frontend_version": FRONTEND_VERSION, "phonemizer": "phonikud", "symbols": list(SYMBOLS),
|
| 74 |
+
"fallback": {key: list(value) for key, value in sorted(FALLBACK.items())}}
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def frontend_runtime():
|
| 78 |
+
"""The Phonikud build that made the labels, for the annotation record."""
|
| 79 |
+
from importlib.metadata import PackageNotFoundError, version
|
| 80 |
+
try:
|
| 81 |
+
return {"phonikud": version("phonikud")}
|
| 82 |
+
except PackageNotFoundError:
|
| 83 |
+
return {"phonikud": None}
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def frontend_fingerprint():
|
| 87 |
+
return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def parse_word(word):
|
| 91 |
+
"""One word of a label string -> ``(phones, unknown)`` with ``phones`` as ``(phone, stress)`` pairs."""
|
| 92 |
+
units, unknown, stress, i = [], [], 0, 0
|
| 93 |
+
while i < len(word):
|
| 94 |
+
if word[i] == STRESS:
|
| 95 |
+
stress, i = 1, i + 1
|
| 96 |
+
continue
|
| 97 |
+
phone = next((m for m in _MULTI if word.startswith(m, i)), word[i])
|
| 98 |
+
i += len(phone)
|
| 99 |
+
if phone in PHONE_SET:
|
| 100 |
+
mapped = (phone,)
|
| 101 |
+
elif phone in FALLBACK:
|
| 102 |
+
mapped = FALLBACK[phone]
|
| 103 |
+
else:
|
| 104 |
+
unknown.append(phone)
|
| 105 |
+
mapped = (UNK,)
|
| 106 |
+
for item in mapped:
|
| 107 |
+
level = stress if item in VOWEL_SET else 0
|
| 108 |
+
stress = 0 if level else stress
|
| 109 |
+
units.append((item, level))
|
| 110 |
+
return units, unknown
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def expression_tokens(text):
|
| 114 |
+
"""``text`` with every written tag as its token: ``<short pause>`` -> ``<short_pause>``."""
|
| 115 |
+
return _WRITTEN.sub(lambda match: "<" + match.group(1).replace(" ", "_") + ">", text)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def pieces(label):
|
| 119 |
+
"""``(True, mark)`` for punctuation and expressions, ``(False, [word])`` for each word, in order."""
|
| 120 |
+
out = []
|
| 121 |
+
for part in _EXPRESSION.split(expression_tokens(label)):
|
| 122 |
+
if part in EXPRESSIONS:
|
| 123 |
+
out.append((True, part))
|
| 124 |
+
continue
|
| 125 |
+
for chunk in _TAG.sub(" ", part).split():
|
| 126 |
+
word = ""
|
| 127 |
+
for char in chunk:
|
| 128 |
+
if char in _PUNCTUATION_CHARS:
|
| 129 |
+
if word:
|
| 130 |
+
out.append((False, [word]))
|
| 131 |
+
word = ""
|
| 132 |
+
out.append((True, char))
|
| 133 |
+
else:
|
| 134 |
+
word += char
|
| 135 |
+
if word:
|
| 136 |
+
out.append((False, [word]))
|
| 137 |
+
return out
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def tokenize(items):
|
| 141 |
+
"""Token sequence from ``pieces``. Pure function (testable without a phonemizer)."""
|
| 142 |
+
symbols, stress, unknown = [BOS], [0], []
|
| 143 |
+
for punctuation, value in items:
|
| 144 |
+
if punctuation:
|
| 145 |
+
if symbols[-1] != value: # collapse repeats such as "!!"
|
| 146 |
+
symbols.append(value)
|
| 147 |
+
stress.append(0)
|
| 148 |
+
continue
|
| 149 |
+
for word in value:
|
| 150 |
+
phones, missing = parse_word(word)
|
| 151 |
+
unknown.extend(missing)
|
| 152 |
+
if not phones:
|
| 153 |
+
continue
|
| 154 |
+
if symbols[-1] in PHONE_SET or symbols[-1] == UNK:
|
| 155 |
+
symbols.append(SP)
|
| 156 |
+
stress.append(0)
|
| 157 |
+
for phone, level in phones:
|
| 158 |
+
symbols.append(phone)
|
| 159 |
+
stress.append(level)
|
| 160 |
+
symbols.append(EOS)
|
| 161 |
+
stress.append(0)
|
| 162 |
+
if not any(s in PHONE_SET or s == UNK for s in symbols):
|
| 163 |
+
raise NothingToPronounce("the text has nothing to pronounce")
|
| 164 |
+
return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), tuple(unknown))
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def phonemize(label):
|
| 168 |
+
"""A label string (``pitput.labels`` format) -> tokens."""
|
| 169 |
+
if not isinstance(label, str) or not label.strip():
|
| 170 |
+
raise ValueError("text must not be empty")
|
| 171 |
+
return tokenize(pieces(label))
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
_g2p = None
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def hebrew_label(text, addressee=None):
|
| 178 |
+
"""Plain Hebrew (optionally with expression tags such as ``<sigh>``) -> a label string.
|
| 179 |
+
|
| 180 |
+
Phonikud vocalizes the sentence as a whole (context), then ``pitput.labels``
|
| 181 |
+
phonemizes it word by word exactly as for the training clips. The text must
|
| 182 |
+
already be in spoken form (numbers and acronyms written out)."""
|
| 183 |
+
global _g2p
|
| 184 |
+
from .g2p import G2P
|
| 185 |
+
from .labels import label
|
| 186 |
+
tags, words = [], []
|
| 187 |
+
for part in _EXPRESSION.split(expression_tokens(" ".join(text.split()))):
|
| 188 |
+
if part in EXPRESSIONS:
|
| 189 |
+
tags.append({"i": len(words), "tag": part[1:-1].replace("_", " ")})
|
| 190 |
+
else:
|
| 191 |
+
words.extend(part.split())
|
| 192 |
+
if not words:
|
| 193 |
+
raise NothingToPronounce("the text has nothing to pronounce")
|
| 194 |
+
if _g2p is None:
|
| 195 |
+
_g2p = G2P()
|
| 196 |
+
return label(_g2p.vocalize(" ".join(words), addressee=addressee), {}, tags)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def is_phone(token_ids):
|
| 200 |
+
"""Boolean list/tensor helper: tokens that must be realised as audio."""
|
| 201 |
+
return [int(i) in PHONE_IDS for i in token_ids]
|
recipe.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"batch_frames": 9000,
|
| 3 |
+
"max_batch": 32,
|
| 4 |
+
"learning_rate": 0.0003,
|
| 5 |
+
"warmup": 1000,
|
| 6 |
+
"lr_half_life": 100000,
|
| 7 |
+
"grad_clip": 10.0,
|
| 8 |
+
"gan_start": 8000,
|
| 9 |
+
"gan_ratio": 0.7,
|
| 10 |
+
"gan_ramp": 3000,
|
| 11 |
+
"feature_matching": 30.0,
|
| 12 |
+
"gan_scale_max": 2000.0,
|
| 13 |
+
"crop_samples": 8192,
|
| 14 |
+
"crops": 2,
|
| 15 |
+
"teacher_hold": 3000,
|
| 16 |
+
"teacher_ramp": 7000,
|
| 17 |
+
"teacher_final": 0.0,
|
| 18 |
+
"subset": null,
|
| 19 |
+
"validation_size": 32,
|
| 20 |
+
"seed": 20261113,
|
| 21 |
+
"voiced_vowel_targets": true,
|
| 22 |
+
"weights": {
|
| 23 |
+
"duration": 1.0,
|
| 24 |
+
"pause": 1.0,
|
| 25 |
+
"pitch": 2.0,
|
| 26 |
+
"pitch_spread": 2.0,
|
| 27 |
+
"voicing": 1.0,
|
| 28 |
+
"energy": 1.0,
|
| 29 |
+
"mel_head": 2.0,
|
| 30 |
+
"stft": 2.0,
|
| 31 |
+
"mel": 10.0
|
| 32 |
+
},
|
| 33 |
+
"branched_from": null,
|
| 34 |
+
"annotation": "5f09905d8382ad59e0c83526febdd145ff337040811263b54d5221ab8a0810f0",
|
| 35 |
+
"frontend": "d78567bd1892b2dec5a33d791fa537c1e0b0223742141cd5420335f73142fc13",
|
| 36 |
+
"train_utterances": 63745,
|
| 37 |
+
"datasets": [
|
| 38 |
+
"1b55242ba5c671049f2577325716a7e8af9412d68372c9edd7826c7317f47804",
|
| 39 |
+
"61ad138e1322519f380c069850099c64fc48274c94616a8b0d2d781da38e6a3c"
|
| 40 |
+
]
|
| 41 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.5
|
| 2 |
+
soundfile
|
| 3 |
+
phonikud>=0.4.1
|
| 4 |
+
phonikud-onnx>=1.0.6
|
| 5 |
+
onnxruntime
|
| 6 |
+
huggingface_hub
|
samples/01-default.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:52109d229fea8c60b75bb6d259857eedcfe30f2cd4f550e7e05c5079d1a69ee3
|
| 3 |
+
size 365612
|
samples/02-default.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a2353c8bab1919a6f03016a809f7d1bdb5a462632d5eda9b94e7f22a0115c652
|
| 3 |
+
size 192556
|
samples/03-default.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:22c6fef49155373c3ca70798910e837eb19edfd8842898c0729ab59cfd493b16
|
| 3 |
+
size 305196
|
samples/04-excited.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:89a4079b82ae4e072af77b9fc0903c1e308f20cf32246a054d09e1583880b06b
|
| 3 |
+
size 177708
|
samples/05-sad.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2fb569a4860514baf903175a8afbdcecda5ecbf873a09883612c7a04725634ff
|
| 3 |
+
size 241196
|
samples/06-angry.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9401cff709db81bb68861cbb0f1c4ef5616fe1347f883e81bffb9d003edcffc0
|
| 3 |
+
size 149548
|
samples/07-fearful.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5e71cad46009022eeb380f69b74405e55ec0cd3e96e07ebe6c469eab5c389325
|
| 3 |
+
size 133164
|
samples/08-surprised.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6626b9e9ba3ce2a23eff741f03ca2e680c0296b2c2dfd4c3426964460cc3466a
|
| 3 |
+
size 199724
|
samples/09-calm.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e45226dcb17334cdc56478c2511697bf906ea531e206c6585e75b738dc7f2ecc
|
| 3 |
+
size 270380
|
samples/10-whisper.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e74a9b0c3b189e23de77a3b0c78d01b3a2e6da1683b3eb18ecf892fe5b0d57f9
|
| 3 |
+
size 175660
|