Nida v0.5 (research preview): weights, 16 voices, code, model card
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +16 -0
- LICENSE +202 -0
- README.md +129 -0
- nida-v05-ema.pt +3 -0
- pyproject.toml +68 -0
- requirements.txt +11 -0
- src/nida/__init__.py +7 -0
- src/nida/bench/__init__.py +1 -0
- src/nida/bench/audiometrics.py +570 -0
- src/nida/bench/b1.py +622 -0
- src/nida/bench/b2.py +684 -0
- src/nida/bench/clips.py +510 -0
- src/nida/bench/envs.py +351 -0
- src/nida/bench/expresso.py +301 -0
- src/nida/bench/fetch.py +704 -0
- src/nida/bench/panel.py +817 -0
- src/nida/bench/phase0.py +358 -0
- src/nida/bench/prepare.py +450 -0
- src/nida/bench/protocol.py +325 -0
- src/nida/bench/registry.py +166 -0
- src/nida/bench/report.py +276 -0
- src/nida/bench/textmetrics.py +464 -0
- src/nida/bench/workers/__init__.py +7 -0
- src/nida/bench/workers/asr_hf.py +103 -0
- src/nida/bench/workers/asr_omni.py +42 -0
- src/nida/bench/workers/asr_qwen.py +55 -0
- src/nida/bench/workers/common.py +74 -0
- src/nida/bench/workers/quality_audiobox.py +46 -0
- src/nida/bench/workers/runner.py +340 -0
- src/nida/bench/workers/vae_voxcpm.py +146 -0
- src/nida/cli.py +929 -0
- src/nida/data/__init__.py +5 -0
- src/nida/data/acoustic.py +1002 -0
- src/nida/data/align.py +261 -0
- src/nida/data/asr_filter.py +283 -0
- src/nida/data/ears.py +1307 -0
- src/nida/data/enhance.py +593 -0
- src/nida/data/envs.py +229 -0
- src/nida/data/events.py +183 -0
- src/nida/data/fillers_ctc.py +227 -0
- src/nida/data/fusion.py +842 -0
- src/nida/data/importer/__init__.py +118 -0
- src/nida/data/importer/archives.py +296 -0
- src/nida/data/importer/audio.py +146 -0
- src/nida/data/importer/config.py +474 -0
- src/nida/data/importer/corpora.py +801 -0
- src/nida/data/importer/hub.py +150 -0
- src/nida/data/importer/pipeline.py +1326 -0
- src/nida/data/importer/sources.py +776 -0
- src/nida/data/importer/text.py +110 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
voices/ada.wav filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
voices/asli.wav filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
voices/can.wav filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
voices/defne.wav filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
voices/duru.wav filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
voices/efe.wav filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
voices/elif.wav filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
voices/emre.wav filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
voices/haluk.wav filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
voices/leyla.wav filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
voices/mert.wav filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
voices/nermin.wav filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
voices/nihat.wav filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
voices/orhan.wav filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
voices/selim.wav filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
voices/su.wav filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-4.0
|
| 3 |
+
language:
|
| 4 |
+
- tr
|
| 5 |
+
pipeline_tag: text-to-speech
|
| 6 |
+
tags:
|
| 7 |
+
- text-to-speech
|
| 8 |
+
- turkish
|
| 9 |
+
- flow-matching
|
| 10 |
+
- zero-shot
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
# Nida v0.5 — Turkish text-to-speech (research preview)
|
| 14 |
+
|
| 15 |
+
**Nida** (*nida*: an exclamation, a call) is an open, non-commercial Turkish text-to-speech model
|
| 16 |
+
trained **from scratch**. v0.5 is an early research preview: it reads Turkish clearly and stably,
|
| 17 |
+
including numbers, dates and abbreviations, with sixteen bundled voices. Its prosody is still flat
|
| 18 |
+
("reads rather than speaks"). Expressive speech (emotion, laughter, sighs) and better voice
|
| 19 |
+
cloning are the goals of later versions.
|
| 20 |
+
|
| 21 |
+
> Türkçe özet: Nida, sıfırdan eğitilmiş, açık ve ticari olmayan bir Türkçe metinden-sese modelidir.
|
| 22 |
+
> v0.5 bir araştırma önizlemesidir: sayı, tarih ve kısaltmalar dahil Türkçeyi net ve kararlı okur,
|
| 23 |
+
> 16 hazır sesle gelir. Tonlaması henüz düzdür. Duygu, sözsüz sesler ve daha iyi ses klonlama
|
| 24 |
+
> sonraki sürümlerin hedefidir.
|
| 25 |
+
|
| 26 |
+
## What it is
|
| 27 |
+
|
| 28 |
+
| | |
|
| 29 |
+
|---|---|
|
| 30 |
+
| Architecture | Diffusion transformer (DiT), flow matching on audio latents, explicit per-character durations with windowed Gaussian alignment |
|
| 31 |
+
| Parameters | 201 M |
|
| 32 |
+
| Audio latents | VoxCPM2 AudioVAE V2 (25 Hz × 64), decoded to 48 kHz |
|
| 33 |
+
| Training data | ~380 h: ~210 h Turkish YouTube speech (YODAS, CC-BY-3.0) + ~170 h synthetic multi-voice Turkish (alania, CC-BY-4.0) |
|
| 34 |
+
| Training | 200k steps, single GPU |
|
| 35 |
+
| Voices | sixteen bundled voices; a custom 3–8 s reference clip also works but is **experimental** |
|
| 36 |
+
|
| 37 |
+
## Measured quality (v0.5)
|
| 38 |
+
|
| 39 |
+
- Intelligibility: character error rate of an ASR read-back (Qwen3-ASR) **0.1–1.2 %** on narration
|
| 40 |
+
and long-form text, **4.5–7 %** on short assistant sentences (numbers, proper names).
|
| 41 |
+
- Speaker similarity with a custom reference clip (experimental; ECAPA-TDNN cosine, generated vs.
|
| 42 |
+
reference): **~0.55** (the same real speaker scores ~0.8, different speakers ≤ 0.36; the audio
|
| 43 |
+
codec alone keeps 0.955). The voice is recognisably close but not a faithful copy.
|
| 44 |
+
- Long-form: stable over 2-minute paragraphs (no drift between sentences in our tests).
|
| 45 |
+
- **TR-Zor** (155 hard Turkish sentences: numbers, dates, times, money, units, abbreviations,
|
| 46 |
+
acronyms, ordinals; [ahenk-turkish-tts](https://github.com/ozguradmin/ahenk-turkish-tts),
|
| 47 |
+
Apache-2.0), judge Whisper large-v3-turbo, word error rate with the set's spoken form as input:
|
| 48 |
+
**2.36 %** (voice *deniz*, an earlier voice not in the final set; 141/155 sentences perfect),
|
| 49 |
+
**3.15 %** (voice *emre*, 133/155). For
|
| 50 |
+
reference, the same set reports Ahenk 2.54 % and Trendyol-TTS 3.98 % (both 2B VoxCPM2 fine-tunes,
|
| 51 |
+
measured by their authors with their own text normaliser; our evaluation normaliser and Whisper
|
| 52 |
+
backend differ, so treat the comparison as indicative).
|
| 53 |
+
|
| 54 |
+
## Known limitations
|
| 55 |
+
|
| 56 |
+
- Flat, read-like prosody; little emotion even when requested.
|
| 57 |
+
- Custom voices (your own reference clip) are experimental: similarity is moderate and depends a lot
|
| 58 |
+
on the clip. The bundled voices are the supported way to use v0.5.
|
| 59 |
+
- Input text is normalised automatically (numbers, ordinals, dates, clock times, money, units,
|
| 60 |
+
common abbreviations, acronyms spelled with Turkish letter names, phone/ID numbers). Ambiguous
|
| 61 |
+
cases (e.g. `8.30` without context, unknown all-caps words) are reported; dangerous ones stop
|
| 62 |
+
generation instead of producing wrong speech. `--no-normalize` turns it off.
|
| 63 |
+
- Each sentence plus the reference must fit 20 seconds; sentences longer than ~96 spoken
|
| 64 |
+
characters are split at a comma, semicolon or colon (never inside quotes or tags).
|
| 65 |
+
- Trained mostly on Turkish; other languages are not supported.
|
| 66 |
+
|
| 67 |
+
## Bundled voices
|
| 68 |
+
|
| 69 |
+
Sixteen synthetic voices ship in `voices/`: `voices.tsv` lists all of them (`<wav>\t<transcript>`),
|
| 70 |
+
and `voices/<name>.tsv` selects one. None of them is a recording of a real person.
|
| 71 |
+
|
| 72 |
+
| Voice | | Notes |
|
| 73 |
+
|---|---|---|
|
| 74 |
+
| leyla, duru | female | the cleanest voices; a good default |
|
| 75 |
+
| ada, defne, elif, nermin, su, aslı (`asli`) | female | |
|
| 76 |
+
| nihat | male | warm and calm |
|
| 77 |
+
| emre | male | lively |
|
| 78 |
+
| mert | male | sounds like a phone line; suits an assistant |
|
| 79 |
+
| can, haluk, selim, orhan, efe | male | |
|
| 80 |
+
|
| 81 |
+
can, elif, emre, haluk, nermin, nihat, selim, leyla, defne and ada come from the alania dataset
|
| 82 |
+
(`voice_type: designed-identity`, generated with VoxCPM2 from text descriptions). su, aslı, duru,
|
| 83 |
+
mert, orhan and efe were designed for this release the same way, with VoxCPM2 voice design from
|
| 84 |
+
a text description (no reference audio).
|
| 85 |
+
|
| 86 |
+
## Usage
|
| 87 |
+
|
| 88 |
+
```bash
|
| 89 |
+
pip install -r requirements.txt # torch (CUDA build recommended), transformers, soundfile, ...
|
| 90 |
+
pip install -e . # the nida package
|
| 91 |
+
nida say --checkpoint nida-v05-ema.pt --prompt-list voices/leyla.tsv \
|
| 92 |
+
--text "Merhaba! Toplantı saat 14:30'da, 3. katta." --out out --single-env --no-asr
|
| 93 |
+
```
|
| 94 |
+
|
| 95 |
+
Run it from the release folder: the voice lists use paths relative to the working directory.
|
| 96 |
+
The audio is written to `out/ext-*.wav`, and `out/say.html` plays it. `--prompt-list
|
| 97 |
+
voices/voices.tsv` reads the text with all sixteen voices; `--text-file` reads a text file.
|
| 98 |
+
|
| 99 |
+
Script syntax inside `--text` / `--text-file`:
|
| 100 |
+
|
| 101 |
+
- Sentences are split on `. ! ? …`; long ones also at `, ; :`.
|
| 102 |
+
- `[duraksama:0.3]` inserts exactly 0.3 s of silence; `[kısa duraksama]` / `[duraksama]` let the
|
| 103 |
+
model pause.
|
| 104 |
+
|
| 105 |
+
The first run downloads the VoxCPM2 AudioVAE and a Turkish CTC aligner from Hugging Face. Later
|
| 106 |
+
runs can skip the Hub's network checks with `HF_HUB_OFFLINE=1` (useful on slow or IPv6-broken
|
| 107 |
+
networks). Each `nida say` call loads the models again (about a minute on an RTX 5070); the
|
| 108 |
+
synthesis itself takes a few seconds.
|
| 109 |
+
|
| 110 |
+
## Related work
|
| 111 |
+
|
| 112 |
+
[FreyaTTS](https://arxiv.org/abs/2607.09530) (2026) is a Turkish text-to-speech model with a
|
| 113 |
+
similar recipe: a non-autoregressive flow-matching DiT on frozen AudioVAE2 latents with a Turkish
|
| 114 |
+
character vocabulary, 183 M parameters. Its numbers are on a different test set and are not
|
| 115 |
+
directly comparable with ours.
|
| 116 |
+
|
| 117 |
+
## Responsible use
|
| 118 |
+
|
| 119 |
+
**Do not use Nida to imitate a real person's voice without their explicit consent**, to deceive,
|
| 120 |
+
or to impersonate anyone (including public figures). Generated audio must not be presented as a
|
| 121 |
+
real recording of a person. By using the model you agree to these terms in addition to the
|
| 122 |
+
CC-BY-NC-4.0 license (non-commercial use only).
|
| 123 |
+
|
| 124 |
+
## Licenses and attribution
|
| 125 |
+
|
| 126 |
+
- Model weights: CC-BY-NC-4.0. Code: Apache-2.0.
|
| 127 |
+
- YODAS (espnet/yodas, CC-BY-3.0), alania synthetic speech (cloud0day3/alania-synthetic-speech-tr,
|
| 128 |
+
CC-BY-4.0), VoxCPM2 AudioVAE and voice design (openbmb/VoxCPM2, Apache-2.0), Turkish CTC aligner
|
| 129 |
+
(mpoyraz/wav2vec2-xls-r-300m-cv7-turkish, CC-BY-4.0).
|
nida-v05-ema.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bd7ff9d7166f99b7df64a5e4ac30c43bce72868138d80bb97cca4e8ca3b939a3
|
| 3 |
+
size 805276081
|
pyproject.toml
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["hatchling>=1.24"]
|
| 3 |
+
build-backend = "hatchling.build"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "nida"
|
| 7 |
+
version = "0.5.0"
|
| 8 |
+
description = "Nida: expressive Turkish text-to-speech (research, non-commercial weights)"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.11"
|
| 11 |
+
# Core carries only the text normaliser base (turkificate, MIT, pure Python). torch is not a
|
| 12 |
+
# core dependency: Colab ships its own CUDA build; the `model` extra (ADR-0005) is for local CPU tests only.
|
| 13 |
+
dependencies = ["turkificate==0.2.0"]
|
| 14 |
+
|
| 15 |
+
[project.optional-dependencies]
|
| 16 |
+
# Phase 0 benchmarks (nida.bench); model libraries live only in isolated worker envs.
|
| 17 |
+
bench = ["numpy>=2.0", "scipy>=1.13", "soundfile>=0.12", "pyyaml>=6.0"]
|
| 18 |
+
# Phase 1 data pipeline (nida.data): Parquet stage manifests.
|
| 19 |
+
data = ["nida[bench]", "pyarrow>=17"]
|
| 20 |
+
# Phase 2 model code, local CPU tests (ADR-0005): same torch as Colab's runtime (2.11.0).
|
| 21 |
+
model = ["torch==2.11.0"]
|
| 22 |
+
|
| 23 |
+
[dependency-groups]
|
| 24 |
+
# PEP 735 group: uv installs it by default for `uv sync` / `uv run`.
|
| 25 |
+
dev = [
|
| 26 |
+
"nida[data,model]",
|
| 27 |
+
"pytest>=8.0",
|
| 28 |
+
"hypothesis>=6.100",
|
| 29 |
+
"ruff>=0.6",
|
| 30 |
+
"mypy>=1.10",
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
[project.scripts]
|
| 34 |
+
nida = "nida.cli:main"
|
| 35 |
+
|
| 36 |
+
[tool.hatch.build.targets.wheel]
|
| 37 |
+
packages = ["src/nida"]
|
| 38 |
+
|
| 39 |
+
[tool.ruff]
|
| 40 |
+
line-length = 100
|
| 41 |
+
target-version = "py311"
|
| 42 |
+
src = ["src", "tests", "lab", "lab/akan"]
|
| 43 |
+
|
| 44 |
+
[tool.ruff.lint]
|
| 45 |
+
select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF", "PTH", "BLE"]
|
| 46 |
+
# Turkish dotless "ı" is a real letter here, not a confusable.
|
| 47 |
+
allowed-confusables = ["ı", "–", "−", "×"]
|
| 48 |
+
|
| 49 |
+
[tool.mypy]
|
| 50 |
+
python_version = "3.11"
|
| 51 |
+
strict = true
|
| 52 |
+
files = ["src/nida", "tests", "lab/akan"]
|
| 53 |
+
# lab/akan holds the `akan` package (lab/akan/akan); `lab` is listed for later lab experiments.
|
| 54 |
+
mypy_path = ["src", "lab/akan", "lab"]
|
| 55 |
+
|
| 56 |
+
[[tool.mypy.overrides]]
|
| 57 |
+
# turkificate (text normaliser base, MIT) ships no py.typed marker; nida.text.normalize wraps it.
|
| 58 |
+
module = ["turkificate", "turkificate.*"]
|
| 59 |
+
ignore_missing_imports = true
|
| 60 |
+
|
| 61 |
+
[tool.pytest.ini_options]
|
| 62 |
+
testpaths = ["tests"]
|
| 63 |
+
pythonpath = ["src", "lab/akan", "lab"]
|
| 64 |
+
addopts = "-q --strict-markers -m 'not gpu'"
|
| 65 |
+
markers = [
|
| 66 |
+
"gpu: requires a CUDA GPU (deselected by default; run with -m gpu)",
|
| 67 |
+
"slow: long-running test",
|
| 68 |
+
]
|
requirements.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Nida v0.5 inference (`nida say --single-env`). Install torch + torchaudio first, a CUDA
|
| 2 |
+
# build for your GPU (https://pytorch.org/get-started/locally/), e.g.:
|
| 3 |
+
# pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
|
| 4 |
+
# then: pip install -r requirements.txt && pip install -e .
|
| 5 |
+
transformers==4.57.6
|
| 6 |
+
huggingface-hub
|
| 7 |
+
numpy
|
| 8 |
+
scipy
|
| 9 |
+
soundfile
|
| 10 |
+
pydantic
|
| 11 |
+
pyyaml
|
src/nida/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Nida: expressive Turkish text-to-speech."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
__all__ = ["__version__"]
|
| 6 |
+
|
| 7 |
+
__version__ = "0.0.1"
|
src/nida/bench/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Phase 0 benchmarks (B1 ASR comparison, B2 VAE reconstruction) and listening panels."""
|
src/nida/bench/audiometrics.py
ADDED
|
@@ -0,0 +1,570 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Objective audio metrics for comparing an original waveform with a reconstruction (B2).
|
| 2 |
+
|
| 3 |
+
All functions are pure and CPU-only (numpy + scipy). Conventions:
|
| 4 |
+
|
| 5 |
+
* Waveforms are 1-D float arrays of shape ``(num_samples,)``. Any floating dtype is accepted
|
| 6 |
+
and converted to ``float64``; integer and complex dtypes are rejected.
|
| 7 |
+
* Sample rates are integers in Hz (``sample_rate_hz``), frequencies are in Hz (``*_hz``),
|
| 8 |
+
lengths are in samples (``*_samples``).
|
| 9 |
+
* Spectrograms are laid out as ``(frames, bins)``.
|
| 10 |
+
* Inputs are validated at the module boundary and violations raise :class:`ValueError`
|
| 11 |
+
with the offending argument named.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import math
|
| 17 |
+
from typing import TypeAlias
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
import numpy.typing as npt
|
| 21 |
+
|
| 22 |
+
# scipy ships no type stubs; the ignore is scoped to this single import.
|
| 23 |
+
from scipy import signal # type: ignore[import-untyped]
|
| 24 |
+
|
| 25 |
+
__all__ = [
|
| 26 |
+
"FloatArray",
|
| 27 |
+
"align",
|
| 28 |
+
"band_energy_ratio",
|
| 29 |
+
"band_log_spectral_distance",
|
| 30 |
+
"band_power_db",
|
| 31 |
+
"estimate_delay_samples",
|
| 32 |
+
"log_mel_l1",
|
| 33 |
+
"mel_filterbank",
|
| 34 |
+
"resample",
|
| 35 |
+
"stft_magnitude",
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
FloatArray: TypeAlias = npt.NDArray[np.float64]
|
| 39 |
+
|
| 40 |
+
# Relative tolerance under which two cross-correlation values count as tied in
|
| 41 |
+
# ``estimate_delay_samples`` (FFT round-off makes exact float ties unreliable).
|
| 42 |
+
_TIE_RTOL = 1e-9
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# --------------------------------------------------------------------------------------
|
| 46 |
+
# Validation helpers
|
| 47 |
+
# --------------------------------------------------------------------------------------
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _as_waveform(x: npt.ArrayLike, name: str) -> FloatArray:
|
| 51 |
+
"""Validate ``x`` as a non-empty, finite 1-D float waveform and return it as float64."""
|
| 52 |
+
arr = np.asarray(x)
|
| 53 |
+
if arr.dtype.kind != "f":
|
| 54 |
+
raise ValueError(f"{name} must have a floating dtype, got {arr.dtype}")
|
| 55 |
+
if arr.ndim != 1:
|
| 56 |
+
raise ValueError(f"{name} must be 1-D, got shape {arr.shape}")
|
| 57 |
+
if arr.size == 0:
|
| 58 |
+
raise ValueError(f"{name} must not be empty")
|
| 59 |
+
out = arr.astype(np.float64)
|
| 60 |
+
if not np.isfinite(out).all():
|
| 61 |
+
raise ValueError(f"{name} contains non-finite values (NaN or inf)")
|
| 62 |
+
return out
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _check_int(value: int, name: str) -> int:
|
| 66 |
+
if isinstance(value, bool) or not isinstance(value, int | np.integer):
|
| 67 |
+
raise ValueError(f"{name} must be an int, got {value!r}")
|
| 68 |
+
return int(value)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _check_positive_int(value: int, name: str) -> int:
|
| 72 |
+
ivalue = _check_int(value, name)
|
| 73 |
+
if ivalue <= 0:
|
| 74 |
+
raise ValueError(f"{name} must be > 0, got {ivalue}")
|
| 75 |
+
return ivalue
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _check_positive_float(value: float, name: str) -> float:
|
| 79 |
+
fvalue = float(value)
|
| 80 |
+
if not (math.isfinite(fvalue) and fvalue > 0.0):
|
| 81 |
+
raise ValueError(f"{name} must be a finite number > 0, got {value!r}")
|
| 82 |
+
return fvalue
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _check_band(sample_rate_hz: int, low_hz: float, high_hz: float) -> None:
|
| 86 |
+
nyquist_hz = sample_rate_hz / 2
|
| 87 |
+
# Written as a negated chain so NaN bounds are rejected too.
|
| 88 |
+
if not (0.0 <= low_hz < high_hz <= nyquist_hz):
|
| 89 |
+
raise ValueError(
|
| 90 |
+
f"frequency band must satisfy 0 <= low_hz < high_hz <= {nyquist_hz} "
|
| 91 |
+
f"(sample_rate_hz/2), got low_hz={low_hz}, high_hz={high_hz}"
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _trim_to_common_length(ref: FloatArray, est: FloatArray) -> tuple[FloatArray, FloatArray]:
|
| 96 |
+
num_samples = min(ref.shape[0], est.shape[0])
|
| 97 |
+
return ref[:num_samples], est[:num_samples]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# --------------------------------------------------------------------------------------
|
| 101 |
+
# Resampling
|
| 102 |
+
# --------------------------------------------------------------------------------------
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def resample(x: npt.ArrayLike, sample_rate_hz: int, target_rate_hz: int) -> FloatArray:
|
| 106 |
+
"""Resample a waveform with a polyphase filter.
|
| 107 |
+
|
| 108 |
+
Uses :func:`scipy.signal.resample_poly` with the up/down factors reduced by their gcd.
|
| 109 |
+
When the rates are equal a copy of the input is returned.
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
x: Waveform, shape ``(num_samples,)``, any float dtype.
|
| 113 |
+
sample_rate_hz: Source sample rate in Hz (> 0).
|
| 114 |
+
target_rate_hz: Target sample rate in Hz (> 0).
|
| 115 |
+
|
| 116 |
+
Returns:
|
| 117 |
+
``float64`` array of shape ``(ceil(num_samples * target_rate_hz / sample_rate_hz),)``.
|
| 118 |
+
|
| 119 |
+
Raises:
|
| 120 |
+
ValueError: If ``x`` is not a finite non-empty 1-D float array or a rate is <= 0.
|
| 121 |
+
"""
|
| 122 |
+
wave = _as_waveform(x, "x")
|
| 123 |
+
source_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
|
| 124 |
+
target_hz = _check_positive_int(target_rate_hz, "target_rate_hz")
|
| 125 |
+
if source_hz == target_hz:
|
| 126 |
+
return wave.copy()
|
| 127 |
+
divisor = math.gcd(source_hz, target_hz)
|
| 128 |
+
up = target_hz // divisor
|
| 129 |
+
down = source_hz // divisor
|
| 130 |
+
out = np.asarray(signal.resample_poly(wave, up, down), dtype=np.float64)
|
| 131 |
+
return out
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
# --------------------------------------------------------------------------------------
|
| 135 |
+
# Spectral front end
|
| 136 |
+
# --------------------------------------------------------------------------------------
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _hz_to_mel(freq_hz: FloatArray) -> FloatArray:
|
| 140 |
+
return np.asarray(2595.0 * np.log10(1.0 + freq_hz / 700.0), dtype=np.float64)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _mel_to_hz(mel: FloatArray) -> FloatArray:
|
| 144 |
+
return np.asarray(700.0 * (10.0 ** (mel / 2595.0) - 1.0), dtype=np.float64)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _bin_frequencies_hz(sample_rate_hz: int, n_fft: int) -> FloatArray:
|
| 148 |
+
return np.arange(n_fft // 2 + 1, dtype=np.float64) * sample_rate_hz / n_fft
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def mel_filterbank(
|
| 152 |
+
sample_rate_hz: int,
|
| 153 |
+
n_fft: int,
|
| 154 |
+
n_mels: int,
|
| 155 |
+
fmin_hz: float = 0.0,
|
| 156 |
+
fmax_hz: float | None = None,
|
| 157 |
+
) -> FloatArray:
|
| 158 |
+
"""Build a triangular HTK mel filterbank.
|
| 159 |
+
|
| 160 |
+
The mel scale is ``2595 * log10(1 + f / 700)``. ``n_mels + 2`` points equally spaced on the
|
| 161 |
+
mel axis between ``fmin_hz`` and ``fmax_hz`` define the filters: filter ``i`` rises linearly
|
| 162 |
+
from point ``i`` to point ``i + 1`` and falls linearly to point ``i + 2``. Filters are *not*
|
| 163 |
+
area-normalised, so the continuous peak is 1.0 (a sampled filter reaches 1.0 only when an
|
| 164 |
+
FFT bin sits exactly on its centre).
|
| 165 |
+
|
| 166 |
+
Args:
|
| 167 |
+
sample_rate_hz: Sample rate in Hz (> 0).
|
| 168 |
+
n_fft: FFT size in samples (> 0).
|
| 169 |
+
n_mels: Number of mel filters (> 0).
|
| 170 |
+
fmin_hz: Lowest filter edge in Hz, ``0 <= fmin_hz < fmax_hz``.
|
| 171 |
+
fmax_hz: Highest filter edge in Hz, ``<= sample_rate_hz / 2``. Defaults to
|
| 172 |
+
``sample_rate_hz / 2``.
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
Array of shape ``(n_mels, n_fft // 2 + 1)``, non-negative, ``float64``.
|
| 176 |
+
|
| 177 |
+
Raises:
|
| 178 |
+
ValueError: On invalid arguments, or if some filter covers no FFT bin (``n_mels`` is
|
| 179 |
+
too large for ``n_fft``), since such a filter would silently output zeros.
|
| 180 |
+
"""
|
| 181 |
+
rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
|
| 182 |
+
fft_size = _check_positive_int(n_fft, "n_fft")
|
| 183 |
+
num_mels = _check_positive_int(n_mels, "n_mels")
|
| 184 |
+
top_hz = rate_hz / 2 if fmax_hz is None else float(fmax_hz)
|
| 185 |
+
_check_band(rate_hz, float(fmin_hz), top_hz)
|
| 186 |
+
|
| 187 |
+
mel_points = np.linspace(
|
| 188 |
+
_hz_to_mel(np.asarray(float(fmin_hz))),
|
| 189 |
+
_hz_to_mel(np.asarray(top_hz)),
|
| 190 |
+
num_mels + 2,
|
| 191 |
+
dtype=np.float64,
|
| 192 |
+
)
|
| 193 |
+
edges_hz = _mel_to_hz(mel_points)
|
| 194 |
+
left_hz = edges_hz[:-2, np.newaxis]
|
| 195 |
+
centre_hz = edges_hz[1:-1, np.newaxis]
|
| 196 |
+
right_hz = edges_hz[2:, np.newaxis]
|
| 197 |
+
freqs_hz = _bin_frequencies_hz(rate_hz, fft_size)[np.newaxis, :]
|
| 198 |
+
|
| 199 |
+
rising = (freqs_hz - left_hz) / (centre_hz - left_hz)
|
| 200 |
+
falling = (right_hz - freqs_hz) / (right_hz - centre_hz)
|
| 201 |
+
bank = np.maximum(0.0, np.minimum(rising, falling)).astype(np.float64)
|
| 202 |
+
|
| 203 |
+
empty = np.flatnonzero(bank.max(axis=1) <= 0.0)
|
| 204 |
+
if empty.size > 0:
|
| 205 |
+
raise ValueError(
|
| 206 |
+
f"{empty.size} of {num_mels} mel filters cover no FFT bin "
|
| 207 |
+
f"(first empty index {int(empty[0])}); n_fft={fft_size} is too small for "
|
| 208 |
+
f"n_mels={num_mels} at sample_rate_hz={rate_hz}"
|
| 209 |
+
)
|
| 210 |
+
return np.asarray(bank, dtype=np.float64)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def stft_magnitude(x: npt.ArrayLike, n_fft: int, hop_samples: int) -> FloatArray:
|
| 214 |
+
"""Magnitude STFT with a periodic Hann window and no centering.
|
| 215 |
+
|
| 216 |
+
Frame ``t`` covers samples ``[t * hop_samples, t * hop_samples + n_fft)``. The tail is
|
| 217 |
+
zero-padded so the last partial frame is included, and inputs shorter than ``n_fft`` are
|
| 218 |
+
zero-padded to a single frame.
|
| 219 |
+
|
| 220 |
+
Args:
|
| 221 |
+
x: Waveform, shape ``(num_samples,)``, any float dtype.
|
| 222 |
+
n_fft: FFT and window size in samples (> 0).
|
| 223 |
+
hop_samples: Hop between frames in samples (> 0).
|
| 224 |
+
|
| 225 |
+
Returns:
|
| 226 |
+
Array of shape ``(frames, n_fft // 2 + 1)`` with
|
| 227 |
+
``frames = 1 + ceil(max(0, num_samples - n_fft) / hop_samples)``.
|
| 228 |
+
|
| 229 |
+
Raises:
|
| 230 |
+
ValueError: If ``x`` is invalid or ``n_fft`` / ``hop_samples`` is <= 0.
|
| 231 |
+
"""
|
| 232 |
+
wave = _as_waveform(x, "x")
|
| 233 |
+
fft_size = _check_positive_int(n_fft, "n_fft")
|
| 234 |
+
hop = _check_positive_int(hop_samples, "hop_samples")
|
| 235 |
+
|
| 236 |
+
num_samples = wave.shape[0]
|
| 237 |
+
num_frames = 1 + -(-max(0, num_samples - fft_size) // hop)
|
| 238 |
+
padded_samples = (num_frames - 1) * hop + fft_size
|
| 239 |
+
padded = np.zeros(padded_samples, dtype=np.float64)
|
| 240 |
+
padded[:num_samples] = wave
|
| 241 |
+
|
| 242 |
+
window = np.asarray(signal.get_window("hann", fft_size, fftbins=True), dtype=np.float64)
|
| 243 |
+
frames = np.lib.stride_tricks.sliding_window_view(padded, fft_size)[::hop]
|
| 244 |
+
spectrum = np.fft.rfft(frames * window, axis=1)
|
| 245 |
+
return np.abs(spectrum).astype(np.float64)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
# --------------------------------------------------------------------------------------
|
| 249 |
+
# Metrics
|
| 250 |
+
# --------------------------------------------------------------------------------------
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def log_mel_l1(
|
| 254 |
+
ref: npt.ArrayLike,
|
| 255 |
+
est: npt.ArrayLike,
|
| 256 |
+
sample_rate_hz: int,
|
| 257 |
+
n_fft: int,
|
| 258 |
+
hop_samples: int,
|
| 259 |
+
n_mels: int,
|
| 260 |
+
eps: float = 1e-5,
|
| 261 |
+
) -> float:
|
| 262 |
+
"""Log-mel L1 distance between a reference and an estimate.
|
| 263 |
+
|
| 264 |
+
Both waveforms are trimmed to the shorter length, converted to magnitude STFTs, projected
|
| 265 |
+
through the mel filterbank (magnitude, not power), floored at ``eps`` and compared in
|
| 266 |
+
``log10``. The result is the mean absolute difference over all frames and mel bands.
|
| 267 |
+
|
| 268 |
+
Args:
|
| 269 |
+
ref: Reference waveform, shape ``(num_samples,)``.
|
| 270 |
+
est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``.
|
| 271 |
+
sample_rate_hz: Sample rate of both signals in Hz.
|
| 272 |
+
n_fft: FFT size in samples.
|
| 273 |
+
hop_samples: STFT hop in samples.
|
| 274 |
+
n_mels: Number of mel bands.
|
| 275 |
+
eps: Floor applied to the mel magnitudes before ``log10`` (> 0).
|
| 276 |
+
|
| 277 |
+
Returns:
|
| 278 |
+
Mean absolute difference in log10 mel magnitude (dimensionless, >= 0).
|
| 279 |
+
|
| 280 |
+
Raises:
|
| 281 |
+
ValueError: On invalid waveforms or arguments (see :func:`mel_filterbank` and
|
| 282 |
+
:func:`stft_magnitude`).
|
| 283 |
+
"""
|
| 284 |
+
ref_wave = _as_waveform(ref, "ref")
|
| 285 |
+
est_wave = _as_waveform(est, "est")
|
| 286 |
+
floor = _check_positive_float(eps, "eps")
|
| 287 |
+
bank = mel_filterbank(sample_rate_hz, n_fft, n_mels)
|
| 288 |
+
ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave)
|
| 289 |
+
|
| 290 |
+
ref_mel = stft_magnitude(ref_wave, n_fft, hop_samples) @ bank.T
|
| 291 |
+
est_mel = stft_magnitude(est_wave, n_fft, hop_samples) @ bank.T
|
| 292 |
+
diff = np.log10(np.maximum(ref_mel, floor)) - np.log10(np.maximum(est_mel, floor))
|
| 293 |
+
return float(np.mean(np.abs(diff)))
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _band_mask(
|
| 297 |
+
sample_rate_hz: int, n_fft: int, low_hz: float, high_hz: float
|
| 298 |
+
) -> npt.NDArray[np.bool_]:
|
| 299 |
+
"""Boolean mask over FFT bins belonging to the band (see :func:`band_power_db`)."""
|
| 300 |
+
freqs_hz = _bin_frequencies_hz(sample_rate_hz, n_fft)
|
| 301 |
+
upper = freqs_hz <= high_hz if high_hz == sample_rate_hz / 2 else freqs_hz < high_hz
|
| 302 |
+
mask: npt.NDArray[np.bool_] = (freqs_hz >= low_hz) & upper
|
| 303 |
+
if not mask.any():
|
| 304 |
+
raise ValueError(
|
| 305 |
+
f"no FFT bin falls in [{low_hz}, {high_hz}) Hz with n_fft={n_fft} at "
|
| 306 |
+
f"sample_rate_hz={sample_rate_hz} (bin spacing {sample_rate_hz / n_fft} Hz)"
|
| 307 |
+
)
|
| 308 |
+
return mask
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def _band_power(
|
| 312 |
+
x: FloatArray, sample_rate_hz: int, n_fft: int, hop_samples: int, mask: npt.NDArray[np.bool_]
|
| 313 |
+
) -> FloatArray:
|
| 314 |
+
"""Power spectrogram restricted to in-band bins, shape ``(frames, num_band_bins)``."""
|
| 315 |
+
power = stft_magnitude(x, n_fft, hop_samples) ** 2
|
| 316 |
+
return np.asarray(power[:, mask], dtype=np.float64)
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def band_power_db(
|
| 320 |
+
x: npt.ArrayLike,
|
| 321 |
+
sample_rate_hz: int,
|
| 322 |
+
low_hz: float,
|
| 323 |
+
high_hz: float,
|
| 324 |
+
n_fft: int = 2048,
|
| 325 |
+
hop_samples: int = 512,
|
| 326 |
+
eps: float = 1e-10,
|
| 327 |
+
) -> FloatArray:
|
| 328 |
+
"""Per-frame mean power in a frequency band, in dB.
|
| 329 |
+
|
| 330 |
+
The band contains the FFT bins ``k`` with centre frequency
|
| 331 |
+
``f_k = k * sample_rate_hz / n_fft`` satisfying ``low_hz <= f_k < high_hz``. When
|
| 332 |
+
``high_hz == sample_rate_hz / 2`` the upper edge is inclusive so the Nyquist bin belongs
|
| 333 |
+
to the band. For each frame the result is ``10 * log10(mean_k(|X_k|^2) + eps)`` where
|
| 334 |
+
``X`` is the Hann-windowed STFT from :func:`stft_magnitude` (unnormalised, so absolute
|
| 335 |
+
values depend on ``n_fft``; compare like with like).
|
| 336 |
+
|
| 337 |
+
Args:
|
| 338 |
+
x: Waveform, shape ``(num_samples,)``.
|
| 339 |
+
sample_rate_hz: Sample rate in Hz.
|
| 340 |
+
low_hz: Lower band edge in Hz, ``0 <= low_hz < high_hz``.
|
| 341 |
+
high_hz: Upper band edge in Hz, ``<= sample_rate_hz / 2``.
|
| 342 |
+
n_fft: FFT size in samples.
|
| 343 |
+
hop_samples: STFT hop in samples.
|
| 344 |
+
eps: Power floor added before the logarithm (> 0).
|
| 345 |
+
|
| 346 |
+
Returns:
|
| 347 |
+
Array of shape ``(frames,)`` in dB.
|
| 348 |
+
|
| 349 |
+
Raises:
|
| 350 |
+
ValueError: On invalid arguments, or if no FFT bin falls in the band.
|
| 351 |
+
"""
|
| 352 |
+
wave = _as_waveform(x, "x")
|
| 353 |
+
rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
|
| 354 |
+
fft_size = _check_positive_int(n_fft, "n_fft")
|
| 355 |
+
hop = _check_positive_int(hop_samples, "hop_samples")
|
| 356 |
+
floor = _check_positive_float(eps, "eps")
|
| 357 |
+
_check_band(rate_hz, float(low_hz), float(high_hz))
|
| 358 |
+
mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz))
|
| 359 |
+
|
| 360 |
+
in_band = _band_power(wave, rate_hz, fft_size, hop, mask)
|
| 361 |
+
return np.asarray(10.0 * np.log10(in_band.mean(axis=1) + floor), dtype=np.float64)
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def band_log_spectral_distance(
|
| 365 |
+
ref: npt.ArrayLike,
|
| 366 |
+
est: npt.ArrayLike,
|
| 367 |
+
sample_rate_hz: int,
|
| 368 |
+
low_hz: float,
|
| 369 |
+
high_hz: float,
|
| 370 |
+
n_fft: int = 2048,
|
| 371 |
+
hop_samples: int = 512,
|
| 372 |
+
eps: float = 1e-10,
|
| 373 |
+
active_within_db: float | None = None,
|
| 374 |
+
) -> float:
|
| 375 |
+
"""Log-spectral distance (RMS over in-band bins, mean over frames) in dB.
|
| 376 |
+
|
| 377 |
+
With ``active_within_db`` only *active* frames are averaged: frames whose full-band
|
| 378 |
+
reference power is within that many dB of the loudest reference frame. This keeps
|
| 379 |
+
silence and noise-floor frames (where ``eps`` dominates) from swamping the average.
|
| 380 |
+
|
| 381 |
+
Both waveforms are trimmed to the shorter length. Per frame the distance is
|
| 382 |
+
``sqrt(mean_k((10 log10(P_ref[k] + eps) - 10 log10(P_est[k] + eps))^2))`` over the in-band
|
| 383 |
+
bins ``k`` (band definition: see :func:`band_power_db`); the result is the mean over frames.
|
| 384 |
+
|
| 385 |
+
Args:
|
| 386 |
+
ref: Reference waveform, shape ``(num_samples,)``.
|
| 387 |
+
est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``.
|
| 388 |
+
sample_rate_hz: Sample rate of both signals in Hz.
|
| 389 |
+
low_hz: Lower band edge in Hz.
|
| 390 |
+
high_hz: Upper band edge in Hz.
|
| 391 |
+
n_fft: FFT size in samples.
|
| 392 |
+
hop_samples: STFT hop in samples.
|
| 393 |
+
eps: Power floor added before the logarithm (> 0).
|
| 394 |
+
active_within_db: If set (> 0), average only over active frames (see above).
|
| 395 |
+
|
| 396 |
+
Returns:
|
| 397 |
+
Distance in dB (>= 0; 0 for identical signals).
|
| 398 |
+
|
| 399 |
+
Raises:
|
| 400 |
+
ValueError: On invalid arguments, or if no FFT bin falls in the band.
|
| 401 |
+
"""
|
| 402 |
+
ref_wave = _as_waveform(ref, "ref")
|
| 403 |
+
est_wave = _as_waveform(est, "est")
|
| 404 |
+
rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
|
| 405 |
+
fft_size = _check_positive_int(n_fft, "n_fft")
|
| 406 |
+
hop = _check_positive_int(hop_samples, "hop_samples")
|
| 407 |
+
floor = _check_positive_float(eps, "eps")
|
| 408 |
+
_check_band(rate_hz, float(low_hz), float(high_hz))
|
| 409 |
+
mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz))
|
| 410 |
+
ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave)
|
| 411 |
+
|
| 412 |
+
ref_power = _band_power(ref_wave, rate_hz, fft_size, hop, mask)
|
| 413 |
+
est_power = _band_power(est_wave, rate_hz, fft_size, hop, mask)
|
| 414 |
+
diff_db = 10.0 * np.log10(ref_power + floor) - 10.0 * np.log10(est_power + floor)
|
| 415 |
+
per_frame = np.sqrt(np.mean(diff_db**2, axis=1))
|
| 416 |
+
if active_within_db is not None:
|
| 417 |
+
window_db = _check_positive_float(active_within_db, "active_within_db")
|
| 418 |
+
full_power = np.sum(stft_magnitude(ref_wave, fft_size, hop) ** 2, axis=1)
|
| 419 |
+
full_db = 10.0 * np.log10(full_power + floor)
|
| 420 |
+
active = full_db >= float(np.max(full_db)) - window_db
|
| 421 |
+
per_frame = per_frame[active]
|
| 422 |
+
return float(np.mean(per_frame))
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def band_energy_ratio(
|
| 426 |
+
ref: npt.ArrayLike,
|
| 427 |
+
est: npt.ArrayLike,
|
| 428 |
+
sample_rate_hz: int,
|
| 429 |
+
low_hz: float,
|
| 430 |
+
high_hz: float,
|
| 431 |
+
n_fft: int = 2048,
|
| 432 |
+
hop_samples: int = 512,
|
| 433 |
+
) -> float:
|
| 434 |
+
"""Ratio of in-band energy of the estimate to that of the reference.
|
| 435 |
+
|
| 436 |
+
Both waveforms are trimmed to the shorter length. The energy of a signal is the sum of
|
| 437 |
+
``|X_k|^2`` over all frames and all in-band bins ``k`` (band definition: see
|
| 438 |
+
:func:`band_power_db`).
|
| 439 |
+
|
| 440 |
+
Args:
|
| 441 |
+
ref: Reference waveform, shape ``(num_samples,)``.
|
| 442 |
+
est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``.
|
| 443 |
+
sample_rate_hz: Sample rate of both signals in Hz.
|
| 444 |
+
low_hz: Lower band edge in Hz.
|
| 445 |
+
high_hz: Upper band edge in Hz.
|
| 446 |
+
n_fft: FFT size in samples.
|
| 447 |
+
hop_samples: STFT hop in samples.
|
| 448 |
+
|
| 449 |
+
Returns:
|
| 450 |
+
``E_est / E_ref`` (dimensionless, >= 0; 1 for identical signals).
|
| 451 |
+
|
| 452 |
+
Raises:
|
| 453 |
+
ValueError: On invalid arguments, if no FFT bin falls in the band, or if the reference
|
| 454 |
+
has zero in-band energy (the ratio would be undefined).
|
| 455 |
+
"""
|
| 456 |
+
ref_wave = _as_waveform(ref, "ref")
|
| 457 |
+
est_wave = _as_waveform(est, "est")
|
| 458 |
+
rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
|
| 459 |
+
fft_size = _check_positive_int(n_fft, "n_fft")
|
| 460 |
+
hop = _check_positive_int(hop_samples, "hop_samples")
|
| 461 |
+
_check_band(rate_hz, float(low_hz), float(high_hz))
|
| 462 |
+
mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz))
|
| 463 |
+
ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave)
|
| 464 |
+
|
| 465 |
+
ref_energy = float(np.sum(_band_power(ref_wave, rate_hz, fft_size, hop, mask)))
|
| 466 |
+
est_energy = float(np.sum(_band_power(est_wave, rate_hz, fft_size, hop, mask)))
|
| 467 |
+
if ref_energy == 0.0:
|
| 468 |
+
raise ValueError(
|
| 469 |
+
f"reference has zero energy in [{low_hz}, {high_hz}) Hz; energy ratio is undefined"
|
| 470 |
+
)
|
| 471 |
+
return est_energy / ref_energy
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
# --------------------------------------------------------------------------------------
|
| 475 |
+
# Alignment
|
| 476 |
+
# --------------------------------------------------------------------------------------
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def estimate_delay_samples(ref: npt.ArrayLike, est: npt.ArrayLike, max_lag_samples: int) -> int:
|
| 480 |
+
"""Estimate the delay of ``est`` relative to ``ref`` by cross-correlation.
|
| 481 |
+
|
| 482 |
+
The cross-correlation ``c[lag] = sum_n est[n] * ref[n - lag]`` is computed with FFTs
|
| 483 |
+
(zero-padded, so there is no circular wrap-around) and maximised over
|
| 484 |
+
``-max_lag_samples <= lag <= +max_lag_samples``, further limited to lags for which the
|
| 485 |
+
signals overlap. A positive result means ``est`` is delayed: ``est[n] ~ ref[n - lag]``.
|
| 486 |
+
|
| 487 |
+
Ties (values within a relative tolerance of ``1e-9`` of the best) resolve to the smallest
|
| 488 |
+
``|lag|``; between ``+k`` and ``-k`` the positive lag wins. Two all-zero signals yield 0.
|
| 489 |
+
|
| 490 |
+
Args:
|
| 491 |
+
ref: Reference waveform, shape ``(num_ref_samples,)``.
|
| 492 |
+
est: Estimated waveform, shape ``(num_est_samples,)``.
|
| 493 |
+
max_lag_samples: Largest absolute lag to consider (>= 0).
|
| 494 |
+
|
| 495 |
+
Returns:
|
| 496 |
+
Delay in samples (``int``) in ``[-max_lag_samples, +max_lag_samples]``.
|
| 497 |
+
|
| 498 |
+
Raises:
|
| 499 |
+
ValueError: On invalid waveforms or negative / non-integer ``max_lag_samples``.
|
| 500 |
+
"""
|
| 501 |
+
ref_wave = _as_waveform(ref, "ref")
|
| 502 |
+
est_wave = _as_waveform(est, "est")
|
| 503 |
+
max_lag = _check_int(max_lag_samples, "max_lag_samples")
|
| 504 |
+
if max_lag < 0:
|
| 505 |
+
raise ValueError(f"max_lag_samples must be >= 0, got {max_lag}")
|
| 506 |
+
|
| 507 |
+
num_ref = ref_wave.shape[0]
|
| 508 |
+
num_est = est_wave.shape[0]
|
| 509 |
+
max_positive = min(max_lag, num_est - 1) # est[n] needs n = lag + m with m >= 0
|
| 510 |
+
max_negative = min(max_lag, num_ref - 1)
|
| 511 |
+
|
| 512 |
+
fft_size = 1 << (num_ref + num_est - 1).bit_length()
|
| 513 |
+
est_spectrum = np.fft.rfft(est_wave, fft_size)
|
| 514 |
+
ref_spectrum = np.fft.rfft(ref_wave, fft_size)
|
| 515 |
+
corr = np.fft.irfft(est_spectrum * np.conj(ref_spectrum), fft_size)
|
| 516 |
+
|
| 517 |
+
# Candidate lags ordered by |lag| (0, +1, -1, +2, -2, ...), so the first maximal entry
|
| 518 |
+
# is the smallest-|lag| tie winner.
|
| 519 |
+
lags: list[int] = [0]
|
| 520 |
+
for k in range(1, max(max_positive, max_negative) + 1):
|
| 521 |
+
if k <= max_positive:
|
| 522 |
+
lags.append(k)
|
| 523 |
+
if k <= max_negative:
|
| 524 |
+
lags.append(-k)
|
| 525 |
+
lag_arr = np.asarray(lags, dtype=np.int64)
|
| 526 |
+
values = corr[lag_arr % fft_size] # negative lags wrap to the end of the circular buffer
|
| 527 |
+
|
| 528 |
+
peak = float(values.max())
|
| 529 |
+
tolerance = _TIE_RTOL * float(np.abs(values).max())
|
| 530 |
+
best = int(np.argmax(values >= peak - tolerance))
|
| 531 |
+
return int(lag_arr[best])
|
| 532 |
+
|
| 533 |
+
|
| 534 |
+
def align(
|
| 535 |
+
ref: npt.ArrayLike, est: npt.ArrayLike, delay_samples: int
|
| 536 |
+
) -> tuple[FloatArray, FloatArray]:
|
| 537 |
+
"""Compensate a known delay of ``est`` and trim both signals to a common length.
|
| 538 |
+
|
| 539 |
+
With ``delay_samples = d > 0`` (``est`` lags ``ref``) the first ``d`` samples of ``est``
|
| 540 |
+
are dropped; with ``d < 0`` (``est`` leads) the first ``|d|`` samples of ``ref`` are
|
| 541 |
+
dropped. Both outputs are then cut to the shorter of the two remaining lengths.
|
| 542 |
+
|
| 543 |
+
Args:
|
| 544 |
+
ref: Reference waveform, shape ``(num_ref_samples,)``.
|
| 545 |
+
est: Estimated waveform, shape ``(num_est_samples,)``.
|
| 546 |
+
delay_samples: Delay of ``est`` relative to ``ref``, as returned by
|
| 547 |
+
:func:`estimate_delay_samples`.
|
| 548 |
+
|
| 549 |
+
Returns:
|
| 550 |
+
``(ref_aligned, est_aligned)``, two ``float64`` copies of identical length >= 1.
|
| 551 |
+
|
| 552 |
+
Raises:
|
| 553 |
+
ValueError: On invalid waveforms, non-integer ``delay_samples``, or if no samples
|
| 554 |
+
remain after compensating the delay.
|
| 555 |
+
"""
|
| 556 |
+
ref_wave = _as_waveform(ref, "ref")
|
| 557 |
+
est_wave = _as_waveform(est, "est")
|
| 558 |
+
delay = _check_int(delay_samples, "delay_samples")
|
| 559 |
+
|
| 560 |
+
if delay >= 0:
|
| 561 |
+
ref_part, est_part = ref_wave, est_wave[delay:]
|
| 562 |
+
else:
|
| 563 |
+
ref_part, est_part = ref_wave[-delay:], est_wave
|
| 564 |
+
ref_part, est_part = _trim_to_common_length(ref_part, est_part)
|
| 565 |
+
if ref_part.shape[0] < 1:
|
| 566 |
+
raise ValueError(
|
| 567 |
+
f"no samples left after compensating delay_samples={delay} "
|
| 568 |
+
f"(ref has {ref_wave.shape[0]}, est has {est_wave.shape[0]} samples)"
|
| 569 |
+
)
|
| 570 |
+
return ref_part.copy(), est_part.copy()
|
src/nida/bench/b1.py
ADDED
|
@@ -0,0 +1,622 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Benchmark B1: Turkish ASR comparison (docs/benchmarks/B1-B2-tasarim.md §1).
|
| 2 |
+
|
| 3 |
+
Stages (each idempotent and resumable):
|
| 4 |
+
|
| 5 |
+
1. ``prepare`` — :func:`nida.bench.prepare.prepare_b1` (layers A, B and the C pool).
|
| 6 |
+
2. ``asr`` — every system transcribes every clip in its own environment.
|
| 7 |
+
3. ``panel`` — choose the final layer-C clips (filler-rich, once) and build the
|
| 8 |
+
transcription panel; the user's corrected transcripts become C references.
|
| 9 |
+
4. ``score`` — metrics, paired bootstrap and the pre-registered decision rule.
|
| 10 |
+
|
| 11 |
+
Failed worker items are scored as empty hypotheses (all deletions) and counted, so a
|
| 12 |
+
system can never look better by failing on hard clips.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import itertools
|
| 18 |
+
import json
|
| 19 |
+
import logging
|
| 20 |
+
import subprocess
|
| 21 |
+
from collections.abc import Callable, Mapping, Sequence
|
| 22 |
+
from contextvars import ContextVar
|
| 23 |
+
from dataclasses import dataclass, field
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
from nida.bench.clips import Clip, select_yodas_final, text_has_filler
|
| 27 |
+
from nida.bench.panel import (
|
| 28 |
+
AudioRef,
|
| 29 |
+
PanelItem,
|
| 30 |
+
PanelResult,
|
| 31 |
+
PanelSpec,
|
| 32 |
+
Question,
|
| 33 |
+
write_panel,
|
| 34 |
+
)
|
| 35 |
+
from nida.bench.prepare import Workspace, clip_audio_path
|
| 36 |
+
from nida.bench.protocol import (
|
| 37 |
+
WorkerSummary,
|
| 38 |
+
WorkRequest,
|
| 39 |
+
WorkResult,
|
| 40 |
+
load_results,
|
| 41 |
+
read_summary,
|
| 42 |
+
write_requests,
|
| 43 |
+
)
|
| 44 |
+
from nida.bench.registry import System
|
| 45 |
+
from nida.bench.textmetrics import (
|
| 46 |
+
FILLERS,
|
| 47 |
+
EditCounts,
|
| 48 |
+
cer_counts,
|
| 49 |
+
corpus_rate,
|
| 50 |
+
filler_score,
|
| 51 |
+
has_digits,
|
| 52 |
+
is_hallucination,
|
| 53 |
+
normalize_for_scoring,
|
| 54 |
+
paired_bootstrap,
|
| 55 |
+
strip_fillers,
|
| 56 |
+
wer_counts,
|
| 57 |
+
)
|
| 58 |
+
from nida.utils.io import atomic_write_json
|
| 59 |
+
|
| 60 |
+
__all__ = [
|
| 61 |
+
"C_PANEL_QUESTIONS",
|
| 62 |
+
"DecisionRule",
|
| 63 |
+
"FillerProvenance",
|
| 64 |
+
"build_c_panel",
|
| 65 |
+
"c_references_from_panel",
|
| 66 |
+
"choose_systems",
|
| 67 |
+
"hypotheses",
|
| 68 |
+
"load_c_selection",
|
| 69 |
+
"run_system",
|
| 70 |
+
"score_b1",
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
log = logging.getLogger(__name__)
|
| 74 |
+
|
| 75 |
+
MAX_ERROR_FRACTION = 0.05
|
| 76 |
+
N_C_FINAL = 50
|
| 77 |
+
# Pre-registered 25; lowered to 20 before any score was seen (pool had 23 filler clips,
|
| 78 |
+
# run 20261008-122110-bench-phase0-b301; docs/benchmarks/B1-B2-tasarim.md §1.1).
|
| 79 |
+
MIN_C_FILLER_CLIPS = 20
|
| 80 |
+
C_REVIEWED_ANSWERS = ("tamam", "emin değilim")
|
| 81 |
+
C_SKIP_ANSWER = "konuşma değil — atla"
|
| 82 |
+
|
| 83 |
+
C_PANEL_QUESTIONS = (
|
| 84 |
+
Question(
|
| 85 |
+
"durum",
|
| 86 |
+
"Bu klip için durum",
|
| 87 |
+
"choice",
|
| 88 |
+
(*C_REVIEWED_ANSWERS, C_SKIP_ANSWER),
|
| 89 |
+
required=True,
|
| 90 |
+
),
|
| 91 |
+
)
|
| 92 |
+
C_PANEL_INSTRUCTIONS = (
|
| 93 |
+
"Her klibi dinleyin ve metin kutusundaki taslağı duyduğunuz şekilde, kelimesi kelimesine "
|
| 94 |
+
"düzeltin. Taslak YouTube altyazısıdır, hatalı olabilir. Dolgu seslerini şu yazımlarla "
|
| 95 |
+
"ekleyin: ıı, ee, hmm, ımm, aa (ve of, uf, ah, oh, vay, hıı). Sayıları yazıyla yazın. "
|
| 96 |
+
"Noktalama önemli değil. Her klip için bir durum seçin: düzelttiyseniz 'tamam', "
|
| 97 |
+
"anlaşılmayan yer kaldıysa 'emin değilim', konuşma yoksa ya da müzik baskınsa "
|
| 98 |
+
"'konuşma değil — atla'. Durumu seçilmeyen klipler kullanılmaz."
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
Runner = Callable[[Sequence[str]], int]
|
| 102 |
+
|
| 103 |
+
#: Tag put in front of every line :func:`_stream_runner` logs. Callers that run several
|
| 104 |
+
#: workers at the same time set it (per thread) to the job name so the interleaved lines
|
| 105 |
+
#: stay attributable; empty (the default) logs exactly as before.
|
| 106 |
+
LOG_TAG: ContextVar[str] = ContextVar("nida_worker_log_tag", default="")
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _stream_runner(cmd: Sequence[str]) -> int:
|
| 110 |
+
tag = LOG_TAG.get()
|
| 111 |
+
prefix = f"[{tag}] " if tag else ""
|
| 112 |
+
log.info("%s$ %s", prefix, " ".join(cmd))
|
| 113 |
+
with subprocess.Popen(
|
| 114 |
+
list(cmd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
|
| 115 |
+
) as proc:
|
| 116 |
+
assert proc.stdout is not None # stdout=PIPE guarantees a stream
|
| 117 |
+
for line in proc.stdout:
|
| 118 |
+
log.info("%s | %s", prefix, line.rstrip())
|
| 119 |
+
return proc.wait()
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _worker_dir(workspace: Workspace, system: System, variant: str) -> Path:
|
| 123 |
+
name = system.system_id if not variant else f"{system.system_id}__{variant}"
|
| 124 |
+
return workspace.workers / name
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def run_system(
|
| 128 |
+
workspace: Workspace,
|
| 129 |
+
clips: Sequence[Clip],
|
| 130 |
+
system: System,
|
| 131 |
+
python_exe: Path,
|
| 132 |
+
*,
|
| 133 |
+
variant: str = "",
|
| 134 |
+
context: str = "",
|
| 135 |
+
retry_failed: bool = False,
|
| 136 |
+
runner: Runner = _stream_runner,
|
| 137 |
+
) -> WorkerSummary:
|
| 138 |
+
"""Run one ASR system over ``clips`` (resumable) and return its summary.
|
| 139 |
+
|
| 140 |
+
Raises:
|
| 141 |
+
RuntimeError: if the worker exits non-zero or more than 5% of items failed.
|
| 142 |
+
"""
|
| 143 |
+
out_dir = _worker_dir(workspace, system, variant)
|
| 144 |
+
requests = [
|
| 145 |
+
WorkRequest(
|
| 146 |
+
clip.clip_id,
|
| 147 |
+
str(clip_audio_path(workspace, clip)),
|
| 148 |
+
{"language": clip.language, **({"context": context} if context else {})},
|
| 149 |
+
)
|
| 150 |
+
for clip in clips
|
| 151 |
+
]
|
| 152 |
+
write_requests(out_dir / "requests.jsonl", requests)
|
| 153 |
+
cmd = [
|
| 154 |
+
str(python_exe),
|
| 155 |
+
"-m",
|
| 156 |
+
f"nida.bench.workers.{system.worker}",
|
| 157 |
+
"--requests",
|
| 158 |
+
str(out_dir / "requests.jsonl"),
|
| 159 |
+
"--results",
|
| 160 |
+
str(out_dir / "results.jsonl"),
|
| 161 |
+
"--model-id",
|
| 162 |
+
system.model_id,
|
| 163 |
+
"--revision",
|
| 164 |
+
system.revision,
|
| 165 |
+
"--batch-size",
|
| 166 |
+
str(system.batch_size),
|
| 167 |
+
"--device",
|
| 168 |
+
"cuda",
|
| 169 |
+
*(["--retry-failed"] if retry_failed else []),
|
| 170 |
+
]
|
| 171 |
+
code = runner(cmd)
|
| 172 |
+
if code != 0:
|
| 173 |
+
raise RuntimeError(f"{system.system_id}: worker exited with code {code}")
|
| 174 |
+
summary = read_summary(out_dir / "results.summary.json")
|
| 175 |
+
total = summary.n_ok + summary.n_error
|
| 176 |
+
if total != len(clips):
|
| 177 |
+
raise RuntimeError(f"{system.system_id}: {total} results for {len(clips)} clips")
|
| 178 |
+
if summary.n_error > MAX_ERROR_FRACTION * total:
|
| 179 |
+
raise RuntimeError(
|
| 180 |
+
f"{system.system_id}: {summary.n_error}/{total} items failed (> 5%); "
|
| 181 |
+
"rerun with --retry-failed after fixing the cause"
|
| 182 |
+
)
|
| 183 |
+
return summary
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def hypotheses(workspace: Workspace, system: System, variant: str = "") -> dict[str, WorkResult]:
|
| 187 |
+
return load_results(_worker_dir(workspace, system, variant) / "results.jsonl")
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _hyp_text(result: WorkResult | None) -> str:
|
| 191 |
+
if result is None or result.output is None:
|
| 192 |
+
return ""
|
| 193 |
+
text = result.output.get("text", "")
|
| 194 |
+
return text if isinstance(text, str) else ""
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
# --------------------------------------------------------------------------- C panel
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
@dataclass(frozen=True)
|
| 201 |
+
class FillerProvenance:
|
| 202 |
+
"""Why a layer-C clip may have been selected (used to de-bias filler recall).
|
| 203 |
+
|
| 204 |
+
Attributes:
|
| 205 |
+
caption: The YouTube caption contains a filler.
|
| 206 |
+
systems: Systems whose hypothesis contains a filler.
|
| 207 |
+
"""
|
| 208 |
+
|
| 209 |
+
caption: bool
|
| 210 |
+
systems: frozenset[str] = field(default_factory=frozenset)
|
| 211 |
+
|
| 212 |
+
def counts_for(self, system_id: str) -> bool:
|
| 213 |
+
"""True if the clip's selection does not depend solely on ``system_id``'s output.
|
| 214 |
+
|
| 215 |
+
Filler-signal clips are favoured during selection. If the only signal was this
|
| 216 |
+
system's own hypothesis, scoring its filler recall on the clip would reward it for
|
| 217 |
+
having caused the selection, so such clips are left out of its recall.
|
| 218 |
+
"""
|
| 219 |
+
return self.caption or not self.systems or bool(self.systems - {system_id})
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _filler_provenance(
|
| 223 |
+
clips: Sequence[Clip], all_hypotheses: Mapping[str, Mapping[str, WorkResult]]
|
| 224 |
+
) -> dict[str, FillerProvenance]:
|
| 225 |
+
out: dict[str, FillerProvenance] = {}
|
| 226 |
+
for clip in clips:
|
| 227 |
+
systems = frozenset(
|
| 228 |
+
system_id
|
| 229 |
+
for system_id, results in all_hypotheses.items()
|
| 230 |
+
if text_has_filler(_hyp_text(results.get(clip.clip_id)))
|
| 231 |
+
)
|
| 232 |
+
out[clip.clip_id] = FillerProvenance(text_has_filler(clip.meta.get("caption", "")), systems)
|
| 233 |
+
return out
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _selection_path(workspace: Workspace) -> Path:
|
| 237 |
+
return workspace.manifests / "b1_c_final.json"
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def load_c_selection(workspace: Workspace, seed: int | None = None) -> dict[str, FillerProvenance]:
|
| 241 |
+
"""Read the stored layer-C selection: ``{clip_id: provenance}`` in selection order.
|
| 242 |
+
|
| 243 |
+
Raises:
|
| 244 |
+
ValueError: if ``seed`` is given and differs from the seed the selection used.
|
| 245 |
+
"""
|
| 246 |
+
obj = json.loads(_selection_path(workspace).read_text(encoding="utf-8"))
|
| 247 |
+
if seed is not None and obj.get("seed") != seed:
|
| 248 |
+
raise ValueError(f"stored C selection used seed {obj.get('seed')}, not {seed}")
|
| 249 |
+
return {
|
| 250 |
+
row["clip_id"]: FillerProvenance(bool(row["caption_filler"]), frozenset(row["systems"]))
|
| 251 |
+
for row in obj["clips"]
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def build_c_panel(
|
| 256 |
+
workspace: Workspace,
|
| 257 |
+
pool: Sequence[Clip],
|
| 258 |
+
all_hypotheses: Mapping[str, Mapping[str, WorkResult]],
|
| 259 |
+
seed: int,
|
| 260 |
+
) -> tuple[list[Clip], Path]:
|
| 261 |
+
"""Select the final layer-C clips (once) and write their transcription panel.
|
| 262 |
+
|
| 263 |
+
The selection is stored once in ``manifests/b1_c_final.json`` together with each
|
| 264 |
+
clip's filler provenance and is never re-drawn: a rerun rebuilds the panel for the same
|
| 265 |
+
clips. Before the first selection every system must have a result for every pool clip.
|
| 266 |
+
The draft text is the YouTube caption, never a candidate system's output, so
|
| 267 |
+
references are not anchored to one of the systems under test.
|
| 268 |
+
|
| 269 |
+
Raises:
|
| 270 |
+
RuntimeError: if some system has not transcribed the whole pool, or the stored
|
| 271 |
+
selection does not belong to this pool.
|
| 272 |
+
"""
|
| 273 |
+
import soundfile # type: ignore[import-untyped] # bench extra
|
| 274 |
+
|
| 275 |
+
by_id = {c.clip_id: c for c in pool}
|
| 276 |
+
if _selection_path(workspace).exists():
|
| 277 |
+
selection = load_c_selection(workspace, seed)
|
| 278 |
+
stray = [cid for cid in selection if cid not in by_id]
|
| 279 |
+
if stray:
|
| 280 |
+
raise RuntimeError(f"stored C selection has clips outside the pool: {stray[:3]}")
|
| 281 |
+
final = [by_id[cid] for cid in selection]
|
| 282 |
+
else:
|
| 283 |
+
incomplete = sorted(
|
| 284 |
+
s for s, results in all_hypotheses.items() if not set(by_id) <= set(results)
|
| 285 |
+
)
|
| 286 |
+
if incomplete:
|
| 287 |
+
raise RuntimeError(f"systems have not transcribed the whole C pool: {incomplete}")
|
| 288 |
+
texts: dict[str, list[str]] = {}
|
| 289 |
+
for results in all_hypotheses.values():
|
| 290 |
+
for clip_id, result in results.items():
|
| 291 |
+
texts.setdefault(clip_id, []).append(_hyp_text(result))
|
| 292 |
+
final = list(
|
| 293 |
+
select_yodas_final(pool, texts, N_C_FINAL, seed, min_filler_clips=MIN_C_FILLER_CLIPS)
|
| 294 |
+
)
|
| 295 |
+
provenance = _filler_provenance(final, all_hypotheses)
|
| 296 |
+
atomic_write_json(
|
| 297 |
+
_selection_path(workspace),
|
| 298 |
+
{
|
| 299 |
+
"seed": seed,
|
| 300 |
+
"clips": [
|
| 301 |
+
{
|
| 302 |
+
"clip_id": c.clip_id,
|
| 303 |
+
"caption_filler": provenance[c.clip_id].caption,
|
| 304 |
+
"systems": sorted(provenance[c.clip_id].systems),
|
| 305 |
+
}
|
| 306 |
+
for c in final
|
| 307 |
+
],
|
| 308 |
+
},
|
| 309 |
+
)
|
| 310 |
+
panel_id = f"b1c-{seed}"
|
| 311 |
+
audio_dir = workspace.cache / "panel_audio" / panel_id
|
| 312 |
+
items: list[PanelItem] = []
|
| 313 |
+
for clip in final:
|
| 314 |
+
flac = audio_dir / f"{clip.clip_id}.flac"
|
| 315 |
+
if not flac.exists():
|
| 316 |
+
audio, rate_hz = soundfile.read(str(clip_audio_path(workspace, clip)), dtype="int16")
|
| 317 |
+
flac.parent.mkdir(parents=True, exist_ok=True)
|
| 318 |
+
soundfile.write(str(flac), audio, rate_hz, subtype="PCM_16")
|
| 319 |
+
items.append(
|
| 320 |
+
PanelItem(
|
| 321 |
+
item_id=clip.clip_id,
|
| 322 |
+
audios=(AudioRef("Klip", flac),),
|
| 323 |
+
text=clip.meta.get("caption", ""),
|
| 324 |
+
editable_text=True,
|
| 325 |
+
info=f"YODAS · {clip.duration_ms / 1000:.1f} s",
|
| 326 |
+
)
|
| 327 |
+
)
|
| 328 |
+
spec = PanelSpec(
|
| 329 |
+
panel_id=panel_id,
|
| 330 |
+
title="B1 — Doğal konuşma transkripsiyonu",
|
| 331 |
+
instructions=C_PANEL_INSTRUCTIONS,
|
| 332 |
+
questions=C_PANEL_QUESTIONS,
|
| 333 |
+
items=tuple(items),
|
| 334 |
+
)
|
| 335 |
+
return final, write_panel(spec, workspace.panels / panel_id)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def c_references_from_panel(results: Sequence[PanelResult]) -> dict[str, str]:
|
| 339 |
+
"""Usable C references: items explicitly reviewed (``tamam`` / ``emin değilim``).
|
| 340 |
+
|
| 341 |
+
Unreviewed items are excluded even though their CSV row carries the caption draft: an
|
| 342 |
+
untouched caption is not a reference (glued words, fillers almost never written).
|
| 343 |
+
"""
|
| 344 |
+
refs: dict[str, str] = {}
|
| 345 |
+
for result in results:
|
| 346 |
+
if result.answers.get("durum") not in C_REVIEWED_ANSWERS:
|
| 347 |
+
continue
|
| 348 |
+
text = (result.text or "").strip()
|
| 349 |
+
if text:
|
| 350 |
+
refs[result.item_id] = text
|
| 351 |
+
return refs
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
# --------------------------------------------------------------------------- scoring
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
@dataclass(frozen=True)
|
| 358 |
+
class DecisionRule:
|
| 359 |
+
"""Pre-registered rule (design note §1.4), fixed before results are seen.
|
| 360 |
+
|
| 361 |
+
Attributes:
|
| 362 |
+
min_filler_recall: Primary must reach this C filler recall (leave-one-out).
|
| 363 |
+
min_reviewed_c: With fewer reviewed C references the rule is not applied.
|
| 364 |
+
min_filler_ref_tokens: With fewer reference fillers in C the rule is not applied.
|
| 365 |
+
min_loo_ref_tokens: Per-system minimum of reference fillers left after the
|
| 366 |
+
leave-one-out exclusion (the denominator of the recall the rule uses).
|
| 367 |
+
max_failed_ab: Most failed worker items allowed per system on A+B; failed items are
|
| 368 |
+
scored as empty hypotheses, so any more would distort the CER ranking.
|
| 369 |
+
max_failed_c: Same for layer C, where a failed item turns its reference fillers into
|
| 370 |
+
misses and would distort the filler recall the rule depends on.
|
| 371 |
+
"""
|
| 372 |
+
|
| 373 |
+
min_filler_recall: float = 0.6
|
| 374 |
+
min_reviewed_c: int = 35
|
| 375 |
+
min_filler_ref_tokens: int = 30
|
| 376 |
+
min_loo_ref_tokens: int = 20
|
| 377 |
+
max_failed_ab: int = 0
|
| 378 |
+
max_failed_c: int = 0
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def _layer_scores(
|
| 382 |
+
clips: Sequence[Clip],
|
| 383 |
+
refs: Mapping[str, str],
|
| 384 |
+
hyps: Mapping[str, WorkResult],
|
| 385 |
+
eligible_for_fillers: frozenset[str] | None = None,
|
| 386 |
+
) -> tuple[dict[str, object], list[EditCounts]]:
|
| 387 |
+
content_cer: list[EditCounts] = []
|
| 388 |
+
content_wer: list[EditCounts] = []
|
| 389 |
+
full_cer: list[EditCounts] = []
|
| 390 |
+
full_wer: list[EditCounts] = []
|
| 391 |
+
pairs: list[tuple[list[str], list[str]]] = []
|
| 392 |
+
loo_pairs: list[tuple[list[str], list[str]]] = []
|
| 393 |
+
n_failed = n_halluc = n_digits = 0
|
| 394 |
+
infer_s = audio_s = 0.0
|
| 395 |
+
for clip in clips:
|
| 396 |
+
ref_text = refs[clip.clip_id]
|
| 397 |
+
result = hyps.get(clip.clip_id)
|
| 398 |
+
hyp_text = _hyp_text(result)
|
| 399 |
+
if result is None or not result.ok:
|
| 400 |
+
n_failed += 1
|
| 401 |
+
else:
|
| 402 |
+
infer_s += result.infer_seconds
|
| 403 |
+
audio_s += clip.duration_ms / 1000
|
| 404 |
+
n_halluc += is_hallucination(ref_text, hyp_text)
|
| 405 |
+
n_digits += has_digits(hyp_text)
|
| 406 |
+
ref_tokens = normalize_for_scoring(ref_text)
|
| 407 |
+
hyp_tokens = normalize_for_scoring(hyp_text)
|
| 408 |
+
content_cer.append(cer_counts(strip_fillers(ref_tokens), strip_fillers(hyp_tokens)))
|
| 409 |
+
content_wer.append(wer_counts(strip_fillers(ref_tokens), strip_fillers(hyp_tokens)))
|
| 410 |
+
full_cer.append(cer_counts(ref_tokens, hyp_tokens))
|
| 411 |
+
full_wer.append(wer_counts(ref_tokens, hyp_tokens))
|
| 412 |
+
pairs.append((ref_tokens, hyp_tokens))
|
| 413 |
+
if eligible_for_fillers is None or clip.clip_id in eligible_for_fillers:
|
| 414 |
+
loo_pairs.append((ref_tokens, hyp_tokens))
|
| 415 |
+
fillers = filler_score(pairs)
|
| 416 |
+
loo = filler_score(loo_pairs)
|
| 417 |
+
n_ok = len(clips) - n_failed
|
| 418 |
+
scores: dict[str, object] = {
|
| 419 |
+
"n": len(clips),
|
| 420 |
+
"n_failed": n_failed,
|
| 421 |
+
"content_cer": corpus_rate(content_cer),
|
| 422 |
+
"content_wer": corpus_rate(content_wer),
|
| 423 |
+
"full_cer": corpus_rate(full_cer),
|
| 424 |
+
"full_wer": corpus_rate(full_wer),
|
| 425 |
+
"hallucination_rate": n_halluc / n_ok if n_ok else None,
|
| 426 |
+
"hyp_with_digits": n_digits,
|
| 427 |
+
"rtf": infer_s / audio_s if audio_s else None,
|
| 428 |
+
"filler_ref_total": fillers.ref_total,
|
| 429 |
+
"filler_hyp_total": fillers.hyp_total,
|
| 430 |
+
"filler_recall": fillers.recall(),
|
| 431 |
+
"filler_precision": fillers.precision(),
|
| 432 |
+
"filler_recall_loo": loo.recall(),
|
| 433 |
+
"filler_loo_ref_total": loo.ref_total,
|
| 434 |
+
"filler_recall_by_type": {
|
| 435 |
+
kind: filler_score(pairs, kind).recall()
|
| 436 |
+
for kind in sorted(FILLERS)
|
| 437 |
+
if filler_score(pairs, kind).ref_total
|
| 438 |
+
},
|
| 439 |
+
}
|
| 440 |
+
return scores, content_cer
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def score_b1(
|
| 444 |
+
clips: Sequence[Clip],
|
| 445 |
+
systems: Sequence[System],
|
| 446 |
+
all_hypotheses: Mapping[str, Mapping[str, WorkResult]],
|
| 447 |
+
c_references: Mapping[str, str] | None,
|
| 448 |
+
*,
|
| 449 |
+
seed: int,
|
| 450 |
+
summaries: Mapping[str, WorkerSummary] | None = None,
|
| 451 |
+
c_provenance: Mapping[str, FillerProvenance] | None = None,
|
| 452 |
+
) -> dict[str, object]:
|
| 453 |
+
"""Compute B1 metrics per system and layer, plus paired bootstrap on content CER.
|
| 454 |
+
|
| 455 |
+
Layers: ``A``, ``B``, ``AB`` (A+B together) and ``C`` when ``c_references`` is given
|
| 456 |
+
(only reviewed clips present in it are scored). With ``c_provenance`` the C layer's
|
| 457 |
+
``filler_recall_loo`` excludes, per system, clips selected only because of that
|
| 458 |
+
system's own hypothesis (see :class:`FillerProvenance`).
|
| 459 |
+
"""
|
| 460 |
+
refs: dict[str, str] = {}
|
| 461 |
+
for clip in clips:
|
| 462 |
+
if clip.layer in ("A", "B") and clip.reference is not None:
|
| 463 |
+
refs[clip.clip_id] = clip.reference
|
| 464 |
+
if c_references:
|
| 465 |
+
refs.update(c_references)
|
| 466 |
+
layers: dict[str, list[Clip]] = {
|
| 467 |
+
"A": [c for c in clips if c.layer == "A"],
|
| 468 |
+
"B": [c for c in clips if c.layer == "B"],
|
| 469 |
+
}
|
| 470 |
+
layers["AB"] = layers["A"] + layers["B"]
|
| 471 |
+
if c_references:
|
| 472 |
+
layers["C"] = [c for c in clips if c.layer == "C" and c.clip_id in c_references]
|
| 473 |
+
|
| 474 |
+
per_system: dict[str, dict[str, dict[str, object]]] = {}
|
| 475 |
+
counts_for_bootstrap: dict[str, dict[str, list[EditCounts]]] = {}
|
| 476 |
+
for system in systems:
|
| 477 |
+
hyps = all_hypotheses[system.system_id]
|
| 478 |
+
per_layer: dict[str, dict[str, object]] = {}
|
| 479 |
+
for layer, layer_clips in layers.items():
|
| 480 |
+
eligible: frozenset[str] | None = None
|
| 481 |
+
if layer == "C" and c_provenance is not None:
|
| 482 |
+
eligible = frozenset(
|
| 483 |
+
cid for cid, prov in c_provenance.items() if prov.counts_for(system.system_id)
|
| 484 |
+
)
|
| 485 |
+
scores, counts = _layer_scores(layer_clips, refs, hyps, eligible)
|
| 486 |
+
counts_for_bootstrap.setdefault(layer, {})[system.system_id] = counts
|
| 487 |
+
per_layer[layer] = scores
|
| 488 |
+
worker: dict[str, object] = {"batch_size": system.batch_size, "worker": system.worker}
|
| 489 |
+
if summaries and system.system_id in summaries:
|
| 490 |
+
summary = summaries[system.system_id]
|
| 491 |
+
worker |= {
|
| 492 |
+
"peak_vram_gb": summary.peak_vram_gb,
|
| 493 |
+
"load_seconds": summary.load_seconds,
|
| 494 |
+
"n_error": summary.n_error,
|
| 495 |
+
"packages": dict(summary.packages),
|
| 496 |
+
}
|
| 497 |
+
per_layer["_worker"] = worker
|
| 498 |
+
per_system[system.system_id] = per_layer
|
| 499 |
+
|
| 500 |
+
bootstrap: dict[str, list[dict[str, object]]] = {}
|
| 501 |
+
for layer in ("AB", "B", "C"):
|
| 502 |
+
if layer not in counts_for_bootstrap or not layers[layer]:
|
| 503 |
+
continue
|
| 504 |
+
rows: list[dict[str, object]] = []
|
| 505 |
+
for a, b in itertools.combinations(sorted(counts_for_bootstrap[layer]), 2):
|
| 506 |
+
result = paired_bootstrap(
|
| 507 |
+
counts_for_bootstrap[layer][a], counts_for_bootstrap[layer][b], seed=seed
|
| 508 |
+
)
|
| 509 |
+
rows.append(
|
| 510 |
+
{"a": a, "b": b, "delta": result.delta, "low": result.low, "high": result.high}
|
| 511 |
+
)
|
| 512 |
+
bootstrap[layer] = rows
|
| 513 |
+
return {
|
| 514 |
+
"systems": per_system,
|
| 515 |
+
"bootstrap_content_cer": bootstrap,
|
| 516 |
+
"n_c_reviewed": len(layers.get("C", [])),
|
| 517 |
+
}
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
def _as_float(value: object, missing: float) -> float:
|
| 521 |
+
return float(value) if isinstance(value, int | float) else missing
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def _decide(
|
| 525 |
+
systems: Mapping[str, Mapping[str, Mapping[str, object]]],
|
| 526 |
+
families: Mapping[str, str],
|
| 527 |
+
rule: DecisionRule,
|
| 528 |
+
cer_layer: str,
|
| 529 |
+
) -> dict[str, object]:
|
| 530 |
+
def cer(system_id: str) -> float:
|
| 531 |
+
return _as_float(systems[system_id][cer_layer]["content_cer"], float("inf"))
|
| 532 |
+
|
| 533 |
+
def recall(system_id: str) -> float:
|
| 534 |
+
c_layer = systems[system_id]["C"]
|
| 535 |
+
return _as_float(c_layer.get("filler_recall_loo", c_layer["filler_recall"]), -1.0)
|
| 536 |
+
|
| 537 |
+
ranked = sorted(systems, key=cer)
|
| 538 |
+
qualified = [s for s in ranked if recall(s) >= rule.min_filler_recall]
|
| 539 |
+
primary = qualified[0] if qualified else ranked[0]
|
| 540 |
+
others = [s for s in ranked if families[s] != families[primary]]
|
| 541 |
+
return {
|
| 542 |
+
"cer_layer": cer_layer,
|
| 543 |
+
"primary": primary,
|
| 544 |
+
"secondary": others[0] if others else None,
|
| 545 |
+
"needs_filler_method": not qualified,
|
| 546 |
+
"qualified_by_filler_recall": qualified,
|
| 547 |
+
}
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
def choose_systems(
|
| 551 |
+
scores: Mapping[str, object],
|
| 552 |
+
families: Mapping[str, str],
|
| 553 |
+
rule: DecisionRule | None = None,
|
| 554 |
+
) -> dict[str, object]:
|
| 555 |
+
"""Apply the pre-registered rule; returns the suggestion and its reasoning.
|
| 556 |
+
|
| 557 |
+
Primary: among systems whose C filler recall (leave-one-out when available) is at least
|
| 558 |
+
``rule.min_filler_recall``, the lowest A+B content CER. If none qualifies: the lowest
|
| 559 |
+
A+B content CER and ``needs_filler_method``. Secondary: the lowest A+B content CER among
|
| 560 |
+
systems of a *different family*. The same rule is also evaluated on layer B alone
|
| 561 |
+
(``without_fleurs``) because one candidate (BuzzASR) was trained on FLEURS; the result
|
| 562 |
+
says whether the pick changes. Not applied (``ready=False``) without enough reviewed C
|
| 563 |
+
references, reference fillers, a system whose leave-one-out filler denominator is below
|
| 564 |
+
``rule.min_loo_ref_tokens``, or a system with failed items beyond
|
| 565 |
+
``rule.max_failed_ab`` (A+B) / ``rule.max_failed_c`` (C) (the reason names them).
|
| 566 |
+
"""
|
| 567 |
+
rule = rule if rule is not None else DecisionRule()
|
| 568 |
+
systems = scores["systems"]
|
| 569 |
+
assert isinstance(systems, dict)
|
| 570 |
+
if not all("C" in layers for layers in systems.values()):
|
| 571 |
+
return {"ready": False, "reason": "C references (user panel) not available yet"}
|
| 572 |
+
n_reviewed = scores.get("n_c_reviewed", 0)
|
| 573 |
+
if not isinstance(n_reviewed, int) or n_reviewed < rule.min_reviewed_c:
|
| 574 |
+
return {
|
| 575 |
+
"ready": False,
|
| 576 |
+
"reason": f"only {n_reviewed} reviewed C clips (< {rule.min_reviewed_c})",
|
| 577 |
+
}
|
| 578 |
+
ref_fillers = min(int(_as_float(v["C"]["filler_ref_total"], 0)) for v in systems.values())
|
| 579 |
+
if ref_fillers < rule.min_filler_ref_tokens:
|
| 580 |
+
return {
|
| 581 |
+
"ready": False,
|
| 582 |
+
"reason": f"only {ref_fillers} reference fillers in C (< {rule.min_filler_ref_tokens})",
|
| 583 |
+
}
|
| 584 |
+
failed = {
|
| 585 |
+
f"{system_id}/{layer}": int(_as_float(v[layer].get("n_failed"), 0))
|
| 586 |
+
for system_id, v in systems.items()
|
| 587 |
+
for layer, limit in (("AB", rule.max_failed_ab), ("C", rule.max_failed_c))
|
| 588 |
+
if _as_float(v[layer].get("n_failed"), 0) > limit
|
| 589 |
+
}
|
| 590 |
+
if failed:
|
| 591 |
+
return {
|
| 592 |
+
"ready": False,
|
| 593 |
+
"reason": f"failed worker items (rerun with --retry-failed): {failed}",
|
| 594 |
+
}
|
| 595 |
+
thin = {
|
| 596 |
+
system_id: int(_as_float(v["C"].get("filler_loo_ref_total"), 0))
|
| 597 |
+
for system_id, v in systems.items()
|
| 598 |
+
if "filler_loo_ref_total" in v["C"]
|
| 599 |
+
and _as_float(v["C"]["filler_loo_ref_total"], 0) < rule.min_loo_ref_tokens
|
| 600 |
+
}
|
| 601 |
+
if thin:
|
| 602 |
+
return {
|
| 603 |
+
"ready": False,
|
| 604 |
+
"reason": f"leave-one-out filler denominators below {rule.min_loo_ref_tokens}: {thin}",
|
| 605 |
+
}
|
| 606 |
+
main = _decide(systems, families, rule, "AB")
|
| 607 |
+
check = _decide(systems, families, rule, "B")
|
| 608 |
+
return {
|
| 609 |
+
"ready": True,
|
| 610 |
+
**main,
|
| 611 |
+
"without_fleurs": check,
|
| 612 |
+
"pick_changes_without_fleurs": (main["primary"], main["secondary"])
|
| 613 |
+
!= (check["primary"], check["secondary"]),
|
| 614 |
+
"rule": {
|
| 615 |
+
"min_filler_recall": rule.min_filler_recall,
|
| 616 |
+
"min_reviewed_c": rule.min_reviewed_c,
|
| 617 |
+
"min_filler_ref_tokens": rule.min_filler_ref_tokens,
|
| 618 |
+
"min_loo_ref_tokens": rule.min_loo_ref_tokens,
|
| 619 |
+
"max_failed_ab": rule.max_failed_ab,
|
| 620 |
+
"max_failed_c": rule.max_failed_c,
|
| 621 |
+
},
|
| 622 |
+
}
|
src/nida/bench/b2.py
ADDED
|
@@ -0,0 +1,684 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Benchmark B2: audio-VAE reconstruction (docs/benchmarks/B1-B2-tasarim.md §2).
|
| 2 |
+
|
| 3 |
+
Stages (idempotent, resumable):
|
| 4 |
+
|
| 5 |
+
1. ``prepare`` — Turkish subsets from the B1 manifest + Expresso subsets streamed from
|
| 6 |
+
the official 36 GB tar; originals written to ``<workspace>/b2/orig`` (16 kHz Turkish,
|
| 7 |
+
48 kHz Expresso, 24-bit). They live in the durable workspace (Drive), not the session
|
| 8 |
+
cache, so a new Colab session resumes without re-streaming 36 GB and without results
|
| 9 |
+
that point at vanished files.
|
| 10 |
+
2. ``vae`` — VoxCPM2 AudioVAE V2 round trip (worker) → ``<workspace>/b2/vae`` (48 kHz).
|
| 11 |
+
3. ``asr`` / ``quality`` — Whisper on 16 kHz copies of original and reconstruction
|
| 12 |
+
(clips with references), Audiobox Aesthetics on both.
|
| 13 |
+
4. ``score`` — objective metrics per subset, incl. the "band-16k" baseline
|
| 14 |
+
(48 kHz → 16 kHz → 48 kHz) that isolates what the VAE's super-resolution adds.
|
| 15 |
+
5. ``panel`` — 40 *blind* A/B pairs (random order per item; key stored outside the
|
| 16 |
+
HTML in ``manifests/b2_panel_key-<seed>.json``) and the scoring of the user's answers.
|
| 17 |
+
|
| 18 |
+
Audiobox resamples to 16 kHz internally, so its scores say nothing about the band above
|
| 19 |
+
8 kHz; the high-band LSD / energy ratio cover that.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import json
|
| 25 |
+
import logging
|
| 26 |
+
import random
|
| 27 |
+
from collections.abc import Mapping, Sequence
|
| 28 |
+
from dataclasses import asdict, dataclass
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
from typing import Any
|
| 31 |
+
|
| 32 |
+
from nida.bench import fetch
|
| 33 |
+
from nida.bench.audiometrics import (
|
| 34 |
+
align,
|
| 35 |
+
band_energy_ratio,
|
| 36 |
+
band_log_spectral_distance,
|
| 37 |
+
estimate_delay_samples,
|
| 38 |
+
log_mel_l1,
|
| 39 |
+
resample,
|
| 40 |
+
)
|
| 41 |
+
from nida.bench.clips import Clip, clip_from_obj, clip_to_obj
|
| 42 |
+
from nida.bench.expresso import (
|
| 43 |
+
is_wanted_member,
|
| 44 |
+
parse_member,
|
| 45 |
+
parse_read_transcriptions,
|
| 46 |
+
parse_vad_segments,
|
| 47 |
+
select_expresso,
|
| 48 |
+
)
|
| 49 |
+
from nida.bench.panel import AudioRef, PanelItem, PanelResult, PanelSpec, Question, write_panel
|
| 50 |
+
from nida.bench.prepare import Workspace
|
| 51 |
+
from nida.bench.protocol import WorkRequest, WorkResult
|
| 52 |
+
from nida.bench.registry import DATA_PINS
|
| 53 |
+
from nida.bench.textmetrics import (
|
| 54 |
+
cer_counts,
|
| 55 |
+
corpus_rate,
|
| 56 |
+
normalize_english,
|
| 57 |
+
normalize_for_scoring,
|
| 58 |
+
strip_fillers,
|
| 59 |
+
)
|
| 60 |
+
from nida.utils.io import atomic_write_json, atomic_write_text
|
| 61 |
+
|
| 62 |
+
__all__ = [
|
| 63 |
+
"B2Plan",
|
| 64 |
+
"asr_requests",
|
| 65 |
+
"b2_paths",
|
| 66 |
+
"build_b2_panel",
|
| 67 |
+
"clip_metrics",
|
| 68 |
+
"load_b2_manifest",
|
| 69 |
+
"missing_vae_outputs",
|
| 70 |
+
"prepare_b2",
|
| 71 |
+
"quality_requests",
|
| 72 |
+
"score_b2",
|
| 73 |
+
"score_b2_panel",
|
| 74 |
+
"vae_requests",
|
| 75 |
+
]
|
| 76 |
+
|
| 77 |
+
log = logging.getLogger(__name__)
|
| 78 |
+
|
| 79 |
+
TR_SUBSETS = {"A": "tr-temiz", "B": "tr-medya"}
|
| 80 |
+
HIGH_BAND_HZ = (8000.0, 24000.0)
|
| 81 |
+
MAX_DELAY_MS = 50
|
| 82 |
+
#: High-band LSD is averaged over frames within this many dB of the loudest frame.
|
| 83 |
+
ACTIVE_WITHIN_DB = 40.0
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@dataclass(frozen=True)
|
| 87 |
+
class B2Plan:
|
| 88 |
+
seed: int = 20261008
|
| 89 |
+
n_tr_per_layer: int = 30
|
| 90 |
+
n_whisper: int = 25
|
| 91 |
+
n_laughing: int = 25
|
| 92 |
+
n_projected: int = 25
|
| 93 |
+
n_nonverbal: int = 15
|
| 94 |
+
n_default: int = 10
|
| 95 |
+
n_panel_per_subset: int = 8
|
| 96 |
+
expresso_url: str = DATA_PINS["expresso_url"]
|
| 97 |
+
|
| 98 |
+
def expresso_counts(self) -> dict[str, int]:
|
| 99 |
+
return {
|
| 100 |
+
"whisper": self.n_whisper,
|
| 101 |
+
"laughing": self.n_laughing,
|
| 102 |
+
"projected": self.n_projected,
|
| 103 |
+
"nonverbal": self.n_nonverbal,
|
| 104 |
+
"default": self.n_default,
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@dataclass(frozen=True)
|
| 109 |
+
class B2Paths:
|
| 110 |
+
orig: Path
|
| 111 |
+
vae: Path
|
| 112 |
+
asr16k: Path
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def b2_paths(workspace: Workspace) -> B2Paths:
|
| 116 |
+
base = workspace.root / "b2"
|
| 117 |
+
return B2Paths(base / "orig", base / "vae", base / "asr16k")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _orig_path(workspace: Workspace, clip: Clip) -> Path:
|
| 121 |
+
return b2_paths(workspace).orig / f"{clip.clip_id}.wav"
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _vae_path(workspace: Workspace, clip: Clip) -> Path:
|
| 125 |
+
return b2_paths(workspace).vae / f"{clip.clip_id}.wav"
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _write_expresso_original(source: Path, clip: Clip, dest: Path) -> None:
|
| 129 |
+
import soundfile # type: ignore[import-untyped] # bench extra
|
| 130 |
+
|
| 131 |
+
if dest.exists():
|
| 132 |
+
return
|
| 133 |
+
audio: Any
|
| 134 |
+
audio, rate_hz = soundfile.read(str(source), dtype="float64", always_2d=True)
|
| 135 |
+
if rate_hz != 48000:
|
| 136 |
+
raise ValueError(f"{source}: expected 48 kHz, got {rate_hz}")
|
| 137 |
+
channel_text = clip.meta.get("channel", "0")
|
| 138 |
+
if channel_text not in ("0", "1", "2"):
|
| 139 |
+
raise ValueError(f"{clip.clip_id}: unknown channel {channel_text!r} (expected 0, 1 or 2)")
|
| 140 |
+
channel = int(channel_text)
|
| 141 |
+
column = 0 if channel in (0, 1) else 1
|
| 142 |
+
if channel == 0 and audio.shape[1] != 1:
|
| 143 |
+
raise ValueError(f"{source}: read speech should be mono, got {audio.shape[1]} channels")
|
| 144 |
+
if channel in (1, 2) and audio.shape[1] != 2:
|
| 145 |
+
raise ValueError(f"{source}: conversational audio should be stereo")
|
| 146 |
+
start = round(clip.start_ms * 48)
|
| 147 |
+
stop = round(clip.end_ms * 48)
|
| 148 |
+
if not 0 <= start < stop <= audio.shape[0]:
|
| 149 |
+
raise ValueError(f"{source}: span {clip.start_ms}..{clip.end_ms} ms outside audio")
|
| 150 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 151 |
+
tmp = dest.with_name(f".{dest.name}.tmp.wav")
|
| 152 |
+
soundfile.write(str(tmp), audio[start:stop, column], 48000, subtype="PCM_24")
|
| 153 |
+
tmp.replace(dest)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def prepare_b2(
|
| 157 |
+
workspace: Workspace,
|
| 158 |
+
b1_clips: Sequence[Clip],
|
| 159 |
+
plan: B2Plan | None = None,
|
| 160 |
+
*,
|
| 161 |
+
opener: Any = None, # Any: urllib-compatible opener (see nida.bench.fetch)
|
| 162 |
+
) -> list[Clip]:
|
| 163 |
+
"""Select B2 clips and write originals; the manifest is written once and reused.
|
| 164 |
+
|
| 165 |
+
Plan and clips are stored together in one atomically written ``b2_manifest.json``
|
| 166 |
+
after everything succeeded, so a stored plan and a stored clip list can never
|
| 167 |
+
disagree, and a failed run leaves no manifest behind.
|
| 168 |
+
"""
|
| 169 |
+
import shutil
|
| 170 |
+
|
| 171 |
+
import soundfile
|
| 172 |
+
|
| 173 |
+
plan = plan if plan is not None else B2Plan()
|
| 174 |
+
manifest = _manifest_path(workspace)
|
| 175 |
+
stored_clips: list[Clip] | None = None
|
| 176 |
+
if manifest.exists():
|
| 177 |
+
stored_plan, stored_clips = load_b2_manifest(workspace)
|
| 178 |
+
if stored_plan != plan:
|
| 179 |
+
raise ValueError(f"existing B2 manifest was built with {stored_plan}, not {plan}")
|
| 180 |
+
rng = random.Random(plan.seed)
|
| 181 |
+
tr_clips: list[Clip] = []
|
| 182 |
+
for layer, subset in TR_SUBSETS.items():
|
| 183 |
+
pool = sorted((c for c in b1_clips if c.layer == layer), key=lambda c: c.clip_id)
|
| 184 |
+
if len(pool) < plan.n_tr_per_layer:
|
| 185 |
+
raise ValueError(f"B1 layer {layer} has {len(pool)} clips, need {plan.n_tr_per_layer}")
|
| 186 |
+
for clip in rng.sample(pool, plan.n_tr_per_layer):
|
| 187 |
+
tr_clips.append(
|
| 188 |
+
Clip.build(
|
| 189 |
+
source_id=clip.source_id,
|
| 190 |
+
layer=subset,
|
| 191 |
+
file_id=clip.file_id,
|
| 192 |
+
start_ms=clip.start_ms,
|
| 193 |
+
end_ms=clip.end_ms,
|
| 194 |
+
reference=clip.reference,
|
| 195 |
+
language="tr",
|
| 196 |
+
meta={**clip.meta, "b1_clip_id": clip.clip_id},
|
| 197 |
+
)
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
if stored_clips is not None and all(_orig_path(workspace, c).exists() for c in stored_clips):
|
| 201 |
+
return stored_clips
|
| 202 |
+
|
| 203 |
+
exp_dir = workspace.cache / "expresso"
|
| 204 |
+
extracted = fetch.stream_extract(
|
| 205 |
+
plan.expresso_url,
|
| 206 |
+
exp_dir,
|
| 207 |
+
is_wanted_member,
|
| 208 |
+
opener=opener if opener is not None else fetch.default_opener,
|
| 209 |
+
)
|
| 210 |
+
by_name = {p.relative_to(exp_dir).as_posix(): p for p in extracted}
|
| 211 |
+
vad_file = next((p for n, p in by_name.items() if n.endswith("VAD_segments.txt")), None)
|
| 212 |
+
tr_file = next((p for n, p in by_name.items() if n.endswith("read_transcriptions.txt")), None)
|
| 213 |
+
if vad_file is None or tr_file is None:
|
| 214 |
+
raise FileNotFoundError(
|
| 215 |
+
"Expresso archive lacks VAD_segments.txt or read_transcriptions.txt"
|
| 216 |
+
)
|
| 217 |
+
files: list[tuple[Any, int]] = []
|
| 218 |
+
for name, path in sorted(by_name.items()):
|
| 219 |
+
parsed = parse_member(name)
|
| 220 |
+
if parsed is not None:
|
| 221 |
+
info = soundfile.info(str(path))
|
| 222 |
+
# Floor, never round up: a length past the last sample makes spans invalid.
|
| 223 |
+
files.append((parsed, (1000 * info.frames) // info.samplerate))
|
| 224 |
+
selection = select_expresso(
|
| 225 |
+
files,
|
| 226 |
+
parse_vad_segments(vad_file.read_text(encoding="utf-8")),
|
| 227 |
+
parse_read_transcriptions(tr_file.read_text(encoding="utf-8")),
|
| 228 |
+
plan.expresso_counts(),
|
| 229 |
+
plan.seed,
|
| 230 |
+
)
|
| 231 |
+
for clip in selection.clips:
|
| 232 |
+
member = clip.file_id.split("#", 1)[0]
|
| 233 |
+
_write_expresso_original(by_name[member], clip, _orig_path(workspace, clip))
|
| 234 |
+
for clip in tr_clips:
|
| 235 |
+
dest = _orig_path(workspace, clip)
|
| 236 |
+
if not dest.exists():
|
| 237 |
+
source_clip_id = clip.meta["b1_clip_id"]
|
| 238 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 239 |
+
shutil.copyfile(workspace.clips16k / f"{source_clip_id}.wav", dest)
|
| 240 |
+
clips = sorted([*tr_clips, *selection.clips], key=lambda c: (c.layer, c.clip_id))
|
| 241 |
+
if stored_clips is not None:
|
| 242 |
+
if [c.clip_id for c in stored_clips] != [c.clip_id for c in clips]:
|
| 243 |
+
raise ValueError("re-derived B2 clips differ from the stored manifest")
|
| 244 |
+
return stored_clips
|
| 245 |
+
atomic_write_text(
|
| 246 |
+
workspace.manifests / "b2_quarantine.jsonl",
|
| 247 |
+
"".join(json.dumps(asdict(q), ensure_ascii=False) + "\n" for q in selection.quarantined),
|
| 248 |
+
)
|
| 249 |
+
atomic_write_json(manifest, {"plan": asdict(plan), "clips": [clip_to_obj(c) for c in clips]})
|
| 250 |
+
return clips
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _manifest_path(workspace: Workspace) -> Path:
|
| 254 |
+
return workspace.manifests / "b2_manifest.json"
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def load_b2_manifest(workspace: Workspace) -> tuple[B2Plan, list[Clip]]:
|
| 258 |
+
"""Read ``manifests/b2_manifest.json`` (plan and clips, written together)."""
|
| 259 |
+
obj = json.loads(_manifest_path(workspace).read_text(encoding="utf-8"))
|
| 260 |
+
return B2Plan(**obj["plan"]), [clip_from_obj(c) for c in obj["clips"]]
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
# ------------------------------------------------------------------------ worker inputs
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def vae_requests(workspace: Workspace, clips: Sequence[Clip]) -> list[WorkRequest]:
|
| 267 |
+
return [
|
| 268 |
+
WorkRequest(
|
| 269 |
+
c.clip_id, str(_orig_path(workspace, c)), {"out_path": str(_vae_path(workspace, c))}
|
| 270 |
+
)
|
| 271 |
+
for c in clips
|
| 272 |
+
]
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def missing_vae_outputs(workspace: Workspace, results: Mapping[str, WorkResult]) -> list[str]:
|
| 276 |
+
"""Ids of successful VAE results whose output file no longer exists (to be redone)."""
|
| 277 |
+
missing: list[str] = []
|
| 278 |
+
for clip_id, result in results.items():
|
| 279 |
+
out = result.output.get("out_path") if result.output is not None else None
|
| 280 |
+
if result.ok and (not isinstance(out, str) or not Path(out).exists()):
|
| 281 |
+
missing.append(clip_id)
|
| 282 |
+
return missing
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def _write_16k(source: Path, dest: Path) -> None:
|
| 286 |
+
import soundfile
|
| 287 |
+
|
| 288 |
+
if dest.exists():
|
| 289 |
+
return
|
| 290 |
+
audio: Any
|
| 291 |
+
audio, rate_hz = soundfile.read(str(source), dtype="float64")
|
| 292 |
+
if rate_hz != 16000:
|
| 293 |
+
audio = resample(audio, int(rate_hz), 16000)
|
| 294 |
+
peak = float(abs(audio).max())
|
| 295 |
+
if peak > 1.0:
|
| 296 |
+
audio = audio / peak
|
| 297 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 298 |
+
tmp = dest.with_name(f".{dest.name}.tmp.wav")
|
| 299 |
+
soundfile.write(str(tmp), audio, 16000, subtype="PCM_16")
|
| 300 |
+
tmp.replace(dest)
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def asr_requests(
|
| 304 |
+
workspace: Workspace,
|
| 305 |
+
clips: Sequence[Clip],
|
| 306 |
+
language: str,
|
| 307 |
+
vae_results: Mapping[str, WorkResult] | None = None,
|
| 308 |
+
) -> list[WorkRequest]:
|
| 309 |
+
"""Whisper requests for originals and reconstructions of clips with references.
|
| 310 |
+
|
| 311 |
+
Clips without a usable reconstruction (failed VAE item or missing file) get no
|
| 312 |
+
requests at all: the CER comparison is paired, and scoring counts them as skipped.
|
| 313 |
+
"""
|
| 314 |
+
paths = b2_paths(workspace)
|
| 315 |
+
requests: list[WorkRequest] = []
|
| 316 |
+
for clip in clips:
|
| 317 |
+
if clip.language != language or not clip.reference:
|
| 318 |
+
continue
|
| 319 |
+
if not _usable_reconstruction(workspace, clip, vae_results):
|
| 320 |
+
continue
|
| 321 |
+
for kind, source in (
|
| 322 |
+
("orig", _orig_path(workspace, clip)),
|
| 323 |
+
("vae", _vae_path(workspace, clip)),
|
| 324 |
+
):
|
| 325 |
+
dest = paths.asr16k / f"{clip.clip_id}__{kind}.wav"
|
| 326 |
+
_write_16k(source, dest)
|
| 327 |
+
requests.append(
|
| 328 |
+
WorkRequest(f"{clip.clip_id}__{kind}", str(dest), {"language": language})
|
| 329 |
+
)
|
| 330 |
+
return requests
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def quality_requests(
|
| 334 |
+
workspace: Workspace,
|
| 335 |
+
clips: Sequence[Clip],
|
| 336 |
+
vae_results: Mapping[str, WorkResult] | None = None,
|
| 337 |
+
) -> list[WorkRequest]:
|
| 338 |
+
"""Audiobox requests for both members of every clip with a usable reconstruction."""
|
| 339 |
+
requests: list[WorkRequest] = []
|
| 340 |
+
for clip in clips:
|
| 341 |
+
if not _usable_reconstruction(workspace, clip, vae_results):
|
| 342 |
+
continue
|
| 343 |
+
requests.append(WorkRequest(f"{clip.clip_id}__orig", str(_orig_path(workspace, clip))))
|
| 344 |
+
requests.append(WorkRequest(f"{clip.clip_id}__vae", str(_vae_path(workspace, clip))))
|
| 345 |
+
return requests
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
# ------------------------------------------------------------------------------ scoring
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def clip_metrics(orig: Any, orig_rate_hz: int, recon: Any, recon_rate_hz: int) -> dict[str, float]:
|
| 352 |
+
"""Objective metrics for one original / reconstruction pair.
|
| 353 |
+
|
| 354 |
+
``orig`` and ``recon`` are 1-D float arrays (Any: numpy arrays, numpy is a bench extra).
|
| 355 |
+
Both are compared at 16 kHz (mel @16k); when the original is 48 kHz they are also
|
| 356 |
+
compared at 48 kHz (mel @48k, high-band LSD and energy ratio). The reconstruction
|
| 357 |
+
is delay-aligned to the original first (cross-correlation at 16 kHz, ±50 ms).
|
| 358 |
+
"""
|
| 359 |
+
o16 = resample(orig, orig_rate_hz, 16000)
|
| 360 |
+
r16 = resample(recon, recon_rate_hz, 16000)
|
| 361 |
+
delay = estimate_delay_samples(o16, r16, MAX_DELAY_MS * 16)
|
| 362 |
+
a16, b16 = align(o16, r16, delay)
|
| 363 |
+
out: dict[str, float] = {
|
| 364 |
+
"delay_ms": delay / 16,
|
| 365 |
+
"mel16_l1": log_mel_l1(a16, b16, 16000, 1024, 256, 80),
|
| 366 |
+
}
|
| 367 |
+
if orig_rate_hz == 48000:
|
| 368 |
+
r48 = resample(recon, recon_rate_hz, 48000)
|
| 369 |
+
a48, b48 = align(orig, r48, delay * 3)
|
| 370 |
+
out["mel48_l1"] = log_mel_l1(a48, b48, 48000, 2048, 480, 128)
|
| 371 |
+
out["hb_lsd_db"] = band_log_spectral_distance(
|
| 372 |
+
a48, b48, 48000, *HIGH_BAND_HZ, active_within_db=ACTIVE_WITHIN_DB
|
| 373 |
+
)
|
| 374 |
+
out["hb_energy_ratio"] = band_energy_ratio(a48, b48, 48000, *HIGH_BAND_HZ)
|
| 375 |
+
return out
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def _mean(values: Sequence[float]) -> float | None:
|
| 379 |
+
return sum(values) / len(values) if values else None
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def _result_text(result: WorkResult | None) -> str | None:
|
| 383 |
+
if result is None or result.output is None:
|
| 384 |
+
return None
|
| 385 |
+
raw = result.output.get("text")
|
| 386 |
+
return raw if isinstance(raw, str) else None
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _scoring_tokens(text: str, language: str) -> list[str]:
|
| 390 |
+
"""Content tokens: Turkish pipeline without fillers, or the English normaliser."""
|
| 391 |
+
if language == "tr":
|
| 392 |
+
return strip_fillers(normalize_for_scoring(text))
|
| 393 |
+
return normalize_english(text)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def _usable_reconstruction(
|
| 397 |
+
workspace: Workspace, clip: Clip, vae_results: Mapping[str, WorkResult] | None
|
| 398 |
+
) -> bool:
|
| 399 |
+
"""A reconstruction counts only if its worker result is ok *and* the file exists."""
|
| 400 |
+
if vae_results is not None:
|
| 401 |
+
result = vae_results.get(clip.clip_id)
|
| 402 |
+
if result is None or not result.ok:
|
| 403 |
+
return False
|
| 404 |
+
return _vae_path(workspace, clip).exists()
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def score_b2(
|
| 408 |
+
workspace: Workspace,
|
| 409 |
+
clips: Sequence[Clip],
|
| 410 |
+
asr_results: Mapping[str, WorkResult],
|
| 411 |
+
quality_results: Mapping[str, WorkResult],
|
| 412 |
+
vae_results: Mapping[str, WorkResult] | None = None,
|
| 413 |
+
) -> dict[str, object]:
|
| 414 |
+
"""Aggregate per-subset metrics for the VAE and the band-16k baseline.
|
| 415 |
+
|
| 416 |
+
Clips without a usable reconstruction (failed worker result or missing file) are
|
| 417 |
+
skipped and counted (``skipped``); every subset still gets a row (``n_scored`` may be 0).
|
| 418 |
+
Quality deltas are averaged over *paired* clips (both members scored); CER compares
|
| 419 |
+
original vs reconstruction only on clips where both transcriptions exist.
|
| 420 |
+
"""
|
| 421 |
+
import soundfile
|
| 422 |
+
|
| 423 |
+
per_subset: dict[str, dict[str, list[float]]] = {}
|
| 424 |
+
cer_pairs: dict[str, dict[str, list[Any]]] = {}
|
| 425 |
+
skipped: dict[str, int] = {}
|
| 426 |
+
for clip in clips:
|
| 427 |
+
vae_file = _vae_path(workspace, clip)
|
| 428 |
+
if not _usable_reconstruction(workspace, clip, vae_results):
|
| 429 |
+
skipped["missing_reconstruction"] = skipped.get("missing_reconstruction", 0) + 1
|
| 430 |
+
continue
|
| 431 |
+
bucket = per_subset.setdefault(clip.layer, {})
|
| 432 |
+
bucket.setdefault("_n_scored", []).append(1.0)
|
| 433 |
+
orig, orig_rate = soundfile.read(str(_orig_path(workspace, clip)), dtype="float64")
|
| 434 |
+
recon, recon_rate = soundfile.read(str(vae_file), dtype="float64")
|
| 435 |
+
for key, value in clip_metrics(orig, int(orig_rate), recon, int(recon_rate)).items():
|
| 436 |
+
bucket.setdefault(f"vae_{key}", []).append(value)
|
| 437 |
+
if int(orig_rate) == 48000:
|
| 438 |
+
band = resample(resample(orig, 48000, 16000), 16000, 48000)
|
| 439 |
+
for key, value in clip_metrics(orig, 48000, band, 48000).items():
|
| 440 |
+
bucket.setdefault(f"band16k_{key}", []).append(value)
|
| 441 |
+
pair = [quality_results.get(f"{clip.clip_id}__{kind}") for kind in ("orig", "vae")]
|
| 442 |
+
if all(q is not None and q.output is not None for q in pair):
|
| 443 |
+
non_numeric = False
|
| 444 |
+
for axis in ("PQ", "CE"):
|
| 445 |
+
scored = [q.output.get(axis) for q in pair if q is not None and q.output]
|
| 446 |
+
numeric = [
|
| 447 |
+
v for v in scored if isinstance(v, int | float) and not isinstance(v, bool)
|
| 448 |
+
]
|
| 449 |
+
if len(numeric) != 2:
|
| 450 |
+
non_numeric = True
|
| 451 |
+
else:
|
| 452 |
+
orig_score, vae_score = float(numeric[0]), float(numeric[1])
|
| 453 |
+
bucket.setdefault(f"orig_{axis}", []).append(orig_score)
|
| 454 |
+
bucket.setdefault(f"vae_{axis}", []).append(vae_score)
|
| 455 |
+
bucket.setdefault(f"delta_{axis}", []).append(vae_score - orig_score)
|
| 456 |
+
if non_numeric: # counted once per clip, not per axis
|
| 457 |
+
skipped["non_numeric_quality"] = skipped.get("non_numeric_quality", 0) + 1
|
| 458 |
+
else:
|
| 459 |
+
skipped["missing_quality_pair"] = skipped.get("missing_quality_pair", 0) + 1
|
| 460 |
+
if clip.reference:
|
| 461 |
+
orig_text = _result_text(asr_results.get(f"{clip.clip_id}__orig"))
|
| 462 |
+
vae_text = _result_text(asr_results.get(f"{clip.clip_id}__vae"))
|
| 463 |
+
if orig_text is None or vae_text is None:
|
| 464 |
+
skipped["missing_asr_pair"] = skipped.get("missing_asr_pair", 0) + 1
|
| 465 |
+
else:
|
| 466 |
+
ref_tokens = _scoring_tokens(clip.reference, clip.language)
|
| 467 |
+
for kind, text in (("orig", orig_text), ("vae", vae_text)):
|
| 468 |
+
cer_pairs.setdefault(clip.layer, {}).setdefault(kind, []).append(
|
| 469 |
+
cer_counts(ref_tokens, _scoring_tokens(text, clip.language))
|
| 470 |
+
)
|
| 471 |
+
for clip in clips: # subsets with nothing usable still get an (empty) row
|
| 472 |
+
per_subset.setdefault(clip.layer, {})
|
| 473 |
+
summary: dict[str, dict[str, float | int | None]] = {}
|
| 474 |
+
for subset, values in sorted(per_subset.items()):
|
| 475 |
+
row: dict[str, float | int | None] = {
|
| 476 |
+
"n": sum(1 for c in clips if c.layer == subset),
|
| 477 |
+
"n_scored": len(values.pop("_n_scored", [])),
|
| 478 |
+
}
|
| 479 |
+
row.update({key: _mean(vals) for key, vals in sorted(values.items())})
|
| 480 |
+
pairs = cer_pairs.get(subset)
|
| 481 |
+
if pairs:
|
| 482 |
+
row["cer_orig"] = corpus_rate(pairs["orig"])
|
| 483 |
+
row["cer_vae"] = corpus_rate(pairs["vae"])
|
| 484 |
+
row["n_cer_pairs"] = len(pairs["orig"])
|
| 485 |
+
summary[subset] = row
|
| 486 |
+
return {"subsets": summary, "skipped": skipped}
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
# -------------------------------------------------------------------------------- panel
|
| 490 |
+
|
| 491 |
+
|
| 492 |
+
B2_QUESTIONS = (
|
| 493 |
+
Question(
|
| 494 |
+
"ayni_ifade",
|
| 495 |
+
"A ve B aynı ifadeyi (fısıltı / gülme / bağırma / sözsüz ses / ton) taşıyor mu?",
|
| 496 |
+
"choice",
|
| 497 |
+
("evet", "kısmen", "hayır"),
|
| 498 |
+
),
|
| 499 |
+
Question(
|
| 500 |
+
"kalite_a", "A'nın ses kalitesi (1 kötü, 5 kusursuz)", "scale", ("1", "2", "3", "4", "5")
|
| 501 |
+
),
|
| 502 |
+
Question(
|
| 503 |
+
"kalite_b", "B'nin ses kalitesi (1 kötü, 5 kusursuz)", "scale", ("1", "2", "3", "4", "5")
|
| 504 |
+
),
|
| 505 |
+
)
|
| 506 |
+
#: Pre-registered acceptance (design §2.4): per expressive subset, the share of "evet"
|
| 507 |
+
#: ("kısmen" counts as *not* preserved) and the mean quality gap original − VAE.
|
| 508 |
+
MIN_PRESERVED_SHARE = 0.8
|
| 509 |
+
MAX_QUALITY_GAP = 1.0
|
| 510 |
+
PANEL_SUBSETS = ("tr", "whisper", "laughing", "projected", "nonverbal")
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def _panel_subset(clip: Clip) -> str:
|
| 514 |
+
return "tr" if clip.layer.startswith("tr-") else clip.layer
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def _key_path(workspace: Workspace, plan: B2Plan) -> Path:
|
| 518 |
+
return workspace.manifests / f"b2_panel_key-{plan.seed}.json"
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def build_b2_panel(
|
| 522 |
+
workspace: Workspace,
|
| 523 |
+
clips: Sequence[Clip],
|
| 524 |
+
plan: B2Plan,
|
| 525 |
+
vae_results: Mapping[str, WorkResult] | None = None,
|
| 526 |
+
) -> Path:
|
| 527 |
+
"""Write the blind 40-pair listening panel and its answer key.
|
| 528 |
+
|
| 529 |
+
Per item the order of original and reconstruction is random and both are labelled
|
| 530 |
+
neutrally ("A", "B"); the key lives in ``manifests/`` and never in the HTML. When
|
| 531 |
+
either member peaks above full scale, the *same* gain is applied to both so clipping
|
| 532 |
+
does not bias the comparison; the gain is recorded in the key.
|
| 533 |
+
"""
|
| 534 |
+
import numpy as np
|
| 535 |
+
import soundfile
|
| 536 |
+
|
| 537 |
+
rng = random.Random(plan.seed + 2)
|
| 538 |
+
chosen: list[Clip] = []
|
| 539 |
+
excluded: dict[str, int] = {}
|
| 540 |
+
for subset in PANEL_SUBSETS:
|
| 541 |
+
everything = sorted(
|
| 542 |
+
(c for c in clips if _panel_subset(c) == subset), key=lambda c: c.clip_id
|
| 543 |
+
)
|
| 544 |
+
pool = [c for c in everything if _usable_reconstruction(workspace, c, vae_results)]
|
| 545 |
+
if len(pool) < len(everything):
|
| 546 |
+
excluded[subset] = len(everything) - len(pool)
|
| 547 |
+
log.warning(
|
| 548 |
+
"B2 panel: %d %s clips lack a usable reconstruction", excluded[subset], subset
|
| 549 |
+
)
|
| 550 |
+
chosen.extend(rng.sample(pool, min(plan.n_panel_per_subset, len(pool))))
|
| 551 |
+
rng.shuffle(chosen)
|
| 552 |
+
panel_id = f"b2-{plan.seed}"
|
| 553 |
+
audio_dir = workspace.cache / "panel_audio" / panel_id
|
| 554 |
+
items: list[PanelItem] = []
|
| 555 |
+
key: dict[str, dict[str, object]] = {}
|
| 556 |
+
for clip in chosen:
|
| 557 |
+
orig, orig_rate = soundfile.read(str(_orig_path(workspace, clip)), dtype="float64")
|
| 558 |
+
recon, recon_rate = soundfile.read(str(_vae_path(workspace, clip)), dtype="float64")
|
| 559 |
+
peak = max(float(np.max(np.abs(orig))), float(np.max(np.abs(recon))), 1e-12)
|
| 560 |
+
gain = 1.0 / peak if peak > 1.0 else 1.0
|
| 561 |
+
first_is_orig = rng.random() < 0.5
|
| 562 |
+
members = [("orig", orig, orig_rate), ("vae", recon, recon_rate)]
|
| 563 |
+
if not first_is_orig:
|
| 564 |
+
members.reverse()
|
| 565 |
+
refs: list[AudioRef] = []
|
| 566 |
+
for label, (_kind, audio, rate_hz) in zip(("A", "B"), members, strict=True):
|
| 567 |
+
# Always rewritten: a cached A/B file could disagree with a re-drawn key.
|
| 568 |
+
dest = audio_dir / f"{clip.clip_id}__{label}.flac"
|
| 569 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 570 |
+
soundfile.write(str(dest), audio * gain, int(rate_hz), subtype="PCM_16")
|
| 571 |
+
refs.append(AudioRef(label, dest))
|
| 572 |
+
key[clip.clip_id] = {
|
| 573 |
+
"subset": _panel_subset(clip),
|
| 574 |
+
"a_is": "orig" if first_is_orig else "vae",
|
| 575 |
+
"gain": gain,
|
| 576 |
+
}
|
| 577 |
+
items.append(
|
| 578 |
+
PanelItem(
|
| 579 |
+
item_id=clip.clip_id,
|
| 580 |
+
audios=tuple(refs),
|
| 581 |
+
text=clip.reference or "",
|
| 582 |
+
info=f"{clip.layer} · {clip.duration_ms / 1000:.1f} s",
|
| 583 |
+
)
|
| 584 |
+
)
|
| 585 |
+
spec = PanelSpec(
|
| 586 |
+
panel_id=panel_id,
|
| 587 |
+
title="B2 — Ses kodlayıcı dinleme testi",
|
| 588 |
+
instructions=(
|
| 589 |
+
"Her satırda A ve B'yi dinleyin. Biri orijinal kayıt, diğeri modelden geçirilmiş "
|
| 590 |
+
"hâli; hangisinin hangisi olduğu rastgele ve gizli. İkisinin aynı ifadeyi "
|
| 591 |
+
"(fısıltı, gülme, bağırma, iç çekme vb.) taşıyıp taşımadığını ve her birinin "
|
| 592 |
+
"kalitesini işaretleyin. Kulaklıkla dinlemeniz önerilir."
|
| 593 |
+
),
|
| 594 |
+
questions=B2_QUESTIONS,
|
| 595 |
+
items=tuple(items),
|
| 596 |
+
)
|
| 597 |
+
atomic_write_json(
|
| 598 |
+
_key_path(workspace, plan),
|
| 599 |
+
{
|
| 600 |
+
"panel_id": panel_id,
|
| 601 |
+
"items": key,
|
| 602 |
+
"excluded_missing_reconstruction": excluded,
|
| 603 |
+
"min_per_subset": plan.n_panel_per_subset,
|
| 604 |
+
},
|
| 605 |
+
)
|
| 606 |
+
return write_panel(spec, workspace.panels / panel_id)
|
| 607 |
+
|
| 608 |
+
|
| 609 |
+
def score_b2_panel(
|
| 610 |
+
results: Sequence[PanelResult],
|
| 611 |
+
key: Mapping[str, Mapping[str, object]],
|
| 612 |
+
*,
|
| 613 |
+
min_per_subset: int,
|
| 614 |
+
) -> dict[str, object]:
|
| 615 |
+
"""Un-blind the panel answers and apply the pre-registered acceptance rule.
|
| 616 |
+
|
| 617 |
+
Returns per subset: answered items, share of "evet" (preserved), share of "kısmen",
|
| 618 |
+
mean quality of original and reconstruction, the mean gap (orig − vae) and whether the
|
| 619 |
+
subset passes (``preserved_share >= 0.8`` and ``quality_gap <= 1``; the ``tr`` subset is
|
| 620 |
+
reported but carries no expressive-preservation requirement). ``all_pass`` requires all
|
| 621 |
+
four expressive subsets to be present, *every* key item of every subset to be fully
|
| 622 |
+
answered, at least ``min_per_subset`` items per expressive subset (pre-registered:
|
| 623 |
+
the panel size per subset, 8), and every subset to pass — thin or partial evidence
|
| 624 |
+
never passes.
|
| 625 |
+
|
| 626 |
+
Raises:
|
| 627 |
+
ValueError: if an answered item is missing from the key.
|
| 628 |
+
"""
|
| 629 |
+
rows: dict[str, dict[str, list[float]]] = {}
|
| 630 |
+
for result in results:
|
| 631 |
+
if result.item_id not in key:
|
| 632 |
+
raise ValueError(f"panel item {result.item_id} not in the answer key")
|
| 633 |
+
entry = key[result.item_id]
|
| 634 |
+
same = result.answers.get("ayni_ifade", "")
|
| 635 |
+
qa, qb = result.answers.get("kalite_a", ""), result.answers.get("kalite_b", "")
|
| 636 |
+
if not same or not qa or not qb:
|
| 637 |
+
continue # unanswered rows do not count either way
|
| 638 |
+
orig_q, vae_q = (
|
| 639 |
+
(float(qa), float(qb)) if entry["a_is"] == "orig" else (float(qb), float(qa))
|
| 640 |
+
)
|
| 641 |
+
bucket = rows.setdefault(str(entry["subset"]), {})
|
| 642 |
+
bucket.setdefault("preserved", []).append(1.0 if same == "evet" else 0.0)
|
| 643 |
+
bucket.setdefault("partial", []).append(1.0 if same == "kısmen" else 0.0)
|
| 644 |
+
bucket.setdefault("orig_q", []).append(orig_q)
|
| 645 |
+
bucket.setdefault("vae_q", []).append(vae_q)
|
| 646 |
+
out: dict[str, dict[str, object]] = {}
|
| 647 |
+
for subset, bucket in sorted(rows.items()):
|
| 648 |
+
preserved = _mean(bucket["preserved"]) or 0.0
|
| 649 |
+
orig_q = _mean(bucket["orig_q"]) or 0.0
|
| 650 |
+
vae_q = _mean(bucket["vae_q"]) or 0.0
|
| 651 |
+
gap = orig_q - vae_q
|
| 652 |
+
passes = gap <= MAX_QUALITY_GAP and (subset == "tr" or preserved >= MIN_PRESERVED_SHARE)
|
| 653 |
+
out[subset] = {
|
| 654 |
+
"n_answered": len(bucket["preserved"]),
|
| 655 |
+
"preserved_share": preserved,
|
| 656 |
+
"partial_share": _mean(bucket["partial"]),
|
| 657 |
+
"quality_orig": orig_q,
|
| 658 |
+
"quality_vae": vae_q,
|
| 659 |
+
"quality_gap": gap,
|
| 660 |
+
"passes": passes,
|
| 661 |
+
}
|
| 662 |
+
in_key: dict[str, int] = {}
|
| 663 |
+
for entry in key.values():
|
| 664 |
+
in_key[str(entry["subset"])] = in_key.get(str(entry["subset"]), 0) + 1
|
| 665 |
+
unanswered = {
|
| 666 |
+
subset: count - int(str(out.get(subset, {}).get("n_answered", 0)))
|
| 667 |
+
for subset, count in sorted(in_key.items())
|
| 668 |
+
if count > int(str(out.get(subset, {}).get("n_answered", 0)))
|
| 669 |
+
}
|
| 670 |
+
expressive = [s for s in PANEL_SUBSETS if s != "tr"]
|
| 671 |
+
missing_subsets = [s for s in expressive if s not in out]
|
| 672 |
+
too_few = {s: in_key.get(s, 0) for s in expressive if in_key.get(s, 0) < min_per_subset}
|
| 673 |
+
return {
|
| 674 |
+
"subsets": out,
|
| 675 |
+
"rule": {"min_preserved_share": MIN_PRESERVED_SHARE, "max_quality_gap": MAX_QUALITY_GAP},
|
| 676 |
+
"unanswered": unanswered,
|
| 677 |
+
"missing_subsets": missing_subsets,
|
| 678 |
+
"too_few_items": too_few,
|
| 679 |
+
"min_per_subset": min_per_subset,
|
| 680 |
+
"all_pass": not missing_subsets
|
| 681 |
+
and not unanswered
|
| 682 |
+
and not too_few
|
| 683 |
+
and all(bool(v["passes"]) for v in out.values()),
|
| 684 |
+
}
|
src/nida/bench/clips.py
ADDED
|
@@ -0,0 +1,510 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Clip records and deterministic clip selection for benchmarks B1 / B2.
|
| 2 |
+
|
| 3 |
+
Selection functions are pure: they take already-parsed source metadata and a seed, and
|
| 4 |
+
return the chosen clips plus a quarantine list with a machine-readable reason for every
|
| 5 |
+
candidate that was rejected (docs/03-data-pipeline.md: nothing is dropped silently).
|
| 6 |
+
Candidates that are merely *not sampled* are not quarantined; only rule violations are.
|
| 7 |
+
|
| 8 |
+
Ids follow the pipeline convention ``sha1(source_id, file_id, start_ms, end_ms)[:16]``.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import hashlib
|
| 14 |
+
import random
|
| 15 |
+
from collections import defaultdict
|
| 16 |
+
from collections.abc import Iterable, Mapping, Sequence
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
from typing import Literal
|
| 19 |
+
|
| 20 |
+
from nida.bench.textmetrics import FILLERS, has_digits, normalize_for_scoring
|
| 21 |
+
from nida.text.turkish import nfc
|
| 22 |
+
|
| 23 |
+
__all__ = [
|
| 24 |
+
"Clip",
|
| 25 |
+
"FleursRow",
|
| 26 |
+
"MediaSpeechItem",
|
| 27 |
+
"Quarantined",
|
| 28 |
+
"Selection",
|
| 29 |
+
"YodasUtt",
|
| 30 |
+
"clip_from_obj",
|
| 31 |
+
"clip_to_obj",
|
| 32 |
+
"make_clip_id",
|
| 33 |
+
"parse_yodas_utt_id",
|
| 34 |
+
"select_fleurs",
|
| 35 |
+
"select_mediaspeech",
|
| 36 |
+
"select_yodas_final",
|
| 37 |
+
"select_yodas_pool",
|
| 38 |
+
"text_has_filler",
|
| 39 |
+
"yodas_video_id",
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
Language = Literal["tr", "en"]
|
| 43 |
+
|
| 44 |
+
#: Gender labels in the FLEURS TSV; anything else is quarantined, never guessed.
|
| 45 |
+
FLEURS_GENDERS = frozenset({"MALE", "FEMALE"})
|
| 46 |
+
#: Allowed mismatch between a YODAS duration and the span encoded in its utterance id.
|
| 47 |
+
YODAS_DURATION_TOLERANCE_MS = 50
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def make_clip_id(source_id: str, file_id: str, start_ms: int, end_ms: int) -> str:
|
| 51 |
+
"""Content-addressed clip id (first 16 hex chars of SHA-1)."""
|
| 52 |
+
if start_ms < 0 or end_ms <= start_ms:
|
| 53 |
+
raise ValueError(f"invalid span {start_ms}..{end_ms} ms for {source_id}/{file_id}")
|
| 54 |
+
key = "\x1f".join((source_id, file_id, str(start_ms), str(end_ms)))
|
| 55 |
+
return hashlib.sha1(key.encode("utf-8")).hexdigest()[:16]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass(frozen=True)
|
| 59 |
+
class Clip:
|
| 60 |
+
"""One benchmark clip.
|
| 61 |
+
|
| 62 |
+
Attributes:
|
| 63 |
+
clip_id: ``make_clip_id(source_id, file_id, start_ms, end_ms)``.
|
| 64 |
+
source_id: Id in ``configs/sources.yaml``.
|
| 65 |
+
layer: Benchmark layer / subset name (e.g. ``"A"``, ``"whisper"``).
|
| 66 |
+
file_id: File or utterance id inside the source.
|
| 67 |
+
start_ms: Start inside the source file, milliseconds.
|
| 68 |
+
end_ms: End inside the source file, milliseconds (exclusive).
|
| 69 |
+
reference: Reference transcript (raw), or ``None`` when not yet available.
|
| 70 |
+
language: ISO 639-1 code.
|
| 71 |
+
meta: Free string metadata (gender, video id, style, ...).
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
clip_id: str
|
| 75 |
+
source_id: str
|
| 76 |
+
layer: str
|
| 77 |
+
file_id: str
|
| 78 |
+
start_ms: int
|
| 79 |
+
end_ms: int
|
| 80 |
+
reference: str | None
|
| 81 |
+
language: Language
|
| 82 |
+
meta: Mapping[str, str] = field(default_factory=dict)
|
| 83 |
+
|
| 84 |
+
@property
|
| 85 |
+
def duration_ms(self) -> int:
|
| 86 |
+
return self.end_ms - self.start_ms
|
| 87 |
+
|
| 88 |
+
@staticmethod
|
| 89 |
+
def build(
|
| 90 |
+
*,
|
| 91 |
+
source_id: str,
|
| 92 |
+
layer: str,
|
| 93 |
+
file_id: str,
|
| 94 |
+
start_ms: int,
|
| 95 |
+
end_ms: int,
|
| 96 |
+
reference: str | None,
|
| 97 |
+
language: Language,
|
| 98 |
+
meta: Mapping[str, str] | None = None,
|
| 99 |
+
) -> Clip:
|
| 100 |
+
return Clip(
|
| 101 |
+
clip_id=make_clip_id(source_id, file_id, start_ms, end_ms),
|
| 102 |
+
source_id=source_id,
|
| 103 |
+
layer=layer,
|
| 104 |
+
file_id=file_id,
|
| 105 |
+
start_ms=start_ms,
|
| 106 |
+
end_ms=end_ms,
|
| 107 |
+
reference=None if reference is None else nfc(reference),
|
| 108 |
+
language=language,
|
| 109 |
+
meta=dict(meta or {}),
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@dataclass(frozen=True)
|
| 114 |
+
class Quarantined:
|
| 115 |
+
"""A rejected candidate (docs/03-data-pipeline.md quarantine record)."""
|
| 116 |
+
|
| 117 |
+
item_id: str
|
| 118 |
+
stage: str
|
| 119 |
+
reason: str
|
| 120 |
+
details: str = ""
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@dataclass(frozen=True)
|
| 124 |
+
class Selection:
|
| 125 |
+
clips: tuple[Clip, ...]
|
| 126 |
+
quarantined: tuple[Quarantined, ...]
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _check_count(n: int) -> None:
|
| 130 |
+
if n < 1:
|
| 131 |
+
raise ValueError(f"requested clip count must be >= 1, got {n}")
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _check_duration_window(min_s: float, max_s: float) -> None:
|
| 135 |
+
if not 0 < min_s < max_s:
|
| 136 |
+
raise ValueError(f"invalid duration window {min_s}..{max_s} s")
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
# ------------------------------------------------------------------------------- FLEURS
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
@dataclass(frozen=True)
|
| 143 |
+
class FleursRow:
|
| 144 |
+
"""One row of FLEURS ``<split>.tsv`` (16 kHz audio)."""
|
| 145 |
+
|
| 146 |
+
sentence_id: str
|
| 147 |
+
file_name: str
|
| 148 |
+
raw_transcription: str
|
| 149 |
+
num_samples: int
|
| 150 |
+
gender: str
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def select_fleurs(
|
| 154 |
+
rows: Iterable[FleursRow],
|
| 155 |
+
n: int,
|
| 156 |
+
seed: int,
|
| 157 |
+
*,
|
| 158 |
+
min_s: float = 2.0,
|
| 159 |
+
max_s: float = 25.0,
|
| 160 |
+
source_id: str = "fleurs_tr",
|
| 161 |
+
layer: str = "A",
|
| 162 |
+
) -> Selection:
|
| 163 |
+
"""Pick ``n`` FLEURS recordings: one per sentence id, gender-balanced.
|
| 164 |
+
|
| 165 |
+
Rules: duration in ``[min_s, max_s]``; reference non-empty and without digits (number
|
| 166 |
+
verbalisation is not decided yet, R6); at most one recording per sentence id (several
|
| 167 |
+
speakers read the same sentence and the TSV has no speaker id). Gender balance: half
|
| 168 |
+
from ``MALE`` and half from ``FEMALE`` where possible, the remainder from whatever is left.
|
| 169 |
+
|
| 170 |
+
Raises:
|
| 171 |
+
ValueError: on invalid arguments or if fewer than ``n`` eligible sentences exist.
|
| 172 |
+
"""
|
| 173 |
+
_check_count(n)
|
| 174 |
+
_check_duration_window(min_s, max_s)
|
| 175 |
+
quarantined: list[Quarantined] = []
|
| 176 |
+
by_sentence: dict[str, list[FleursRow]] = defaultdict(list)
|
| 177 |
+
for row in rows:
|
| 178 |
+
duration_s = row.num_samples / 16000
|
| 179 |
+
if row.gender not in FLEURS_GENDERS:
|
| 180 |
+
quarantined.append(
|
| 181 |
+
Quarantined(row.file_name, "select", "unexpected_gender", row.gender)
|
| 182 |
+
)
|
| 183 |
+
elif not min_s <= duration_s <= max_s:
|
| 184 |
+
quarantined.append(
|
| 185 |
+
Quarantined(row.file_name, "select", "duration_out_of_range", f"{duration_s:.2f}s")
|
| 186 |
+
)
|
| 187 |
+
elif not normalize_for_scoring(row.raw_transcription):
|
| 188 |
+
quarantined.append(Quarantined(row.file_name, "select", "empty_reference"))
|
| 189 |
+
elif has_digits(row.raw_transcription):
|
| 190 |
+
quarantined.append(Quarantined(row.file_name, "select", "reference_has_digits"))
|
| 191 |
+
else:
|
| 192 |
+
by_sentence[row.sentence_id].append(row)
|
| 193 |
+
rng = random.Random(seed)
|
| 194 |
+
one_per_sentence: list[FleursRow] = []
|
| 195 |
+
for sentence_id in sorted(by_sentence):
|
| 196 |
+
recordings = sorted(by_sentence[sentence_id], key=lambda r: r.file_name)
|
| 197 |
+
one_per_sentence.append(rng.choice(recordings))
|
| 198 |
+
if len(one_per_sentence) < n:
|
| 199 |
+
raise ValueError(f"only {len(one_per_sentence)} eligible FLEURS sentences, need {n}")
|
| 200 |
+
pools: dict[str, list[FleursRow]] = defaultdict(list)
|
| 201 |
+
for row in one_per_sentence:
|
| 202 |
+
pools[row.gender].append(row)
|
| 203 |
+
for pool in pools.values():
|
| 204 |
+
rng.shuffle(pool)
|
| 205 |
+
chosen: list[FleursRow] = []
|
| 206 |
+
half = n // 2
|
| 207 |
+
for gender in ("MALE", "FEMALE"):
|
| 208 |
+
chosen.extend(pools[gender][:half])
|
| 209 |
+
del pools[gender][:half]
|
| 210 |
+
rest = [row for gender in sorted(pools) for row in pools[gender]]
|
| 211 |
+
rng.shuffle(rest)
|
| 212 |
+
chosen.extend(rest[: n - len(chosen)])
|
| 213 |
+
clips = tuple(
|
| 214 |
+
Clip.build(
|
| 215 |
+
source_id=source_id,
|
| 216 |
+
layer=layer,
|
| 217 |
+
file_id=row.file_name,
|
| 218 |
+
start_ms=0,
|
| 219 |
+
end_ms=round(row.num_samples / 16),
|
| 220 |
+
reference=row.raw_transcription,
|
| 221 |
+
language="tr",
|
| 222 |
+
meta={"gender": row.gender, "sentence_id": row.sentence_id},
|
| 223 |
+
)
|
| 224 |
+
for row in sorted(chosen, key=lambda r: r.file_name)
|
| 225 |
+
)
|
| 226 |
+
return Selection(clips, tuple(quarantined))
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
# --------------------------------------------------------------------------- MediaSpeech
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
@dataclass(frozen=True)
|
| 233 |
+
class MediaSpeechItem:
|
| 234 |
+
file_id: str # uuid stem of TR/<uuid>.flac
|
| 235 |
+
text: str
|
| 236 |
+
duration_ms: int
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def select_mediaspeech(
|
| 240 |
+
items: Iterable[MediaSpeechItem],
|
| 241 |
+
n: int,
|
| 242 |
+
seed: int,
|
| 243 |
+
*,
|
| 244 |
+
min_s: float = 2.0,
|
| 245 |
+
max_s: float = 20.0,
|
| 246 |
+
source_id: str = "mediaspeech_tr",
|
| 247 |
+
layer: str = "B",
|
| 248 |
+
) -> Selection:
|
| 249 |
+
"""Pick ``n`` MediaSpeech segments with the same reference rules as FLEURS."""
|
| 250 |
+
_check_count(n)
|
| 251 |
+
_check_duration_window(min_s, max_s)
|
| 252 |
+
quarantined: list[Quarantined] = []
|
| 253 |
+
eligible: list[MediaSpeechItem] = []
|
| 254 |
+
for item in sorted(items, key=lambda i: i.file_id):
|
| 255 |
+
duration_s = item.duration_ms / 1000
|
| 256 |
+
if not min_s <= duration_s <= max_s:
|
| 257 |
+
quarantined.append(
|
| 258 |
+
Quarantined(item.file_id, "select", "duration_out_of_range", f"{duration_s:.2f}s")
|
| 259 |
+
)
|
| 260 |
+
elif not normalize_for_scoring(item.text):
|
| 261 |
+
quarantined.append(Quarantined(item.file_id, "select", "empty_reference"))
|
| 262 |
+
elif has_digits(item.text):
|
| 263 |
+
quarantined.append(Quarantined(item.file_id, "select", "reference_has_digits"))
|
| 264 |
+
else:
|
| 265 |
+
eligible.append(item)
|
| 266 |
+
if len(eligible) < n:
|
| 267 |
+
raise ValueError(f"only {len(eligible)} eligible MediaSpeech items, need {n}")
|
| 268 |
+
chosen = random.Random(seed).sample(eligible, n)
|
| 269 |
+
clips = tuple(
|
| 270 |
+
Clip.build(
|
| 271 |
+
source_id=source_id,
|
| 272 |
+
layer=layer,
|
| 273 |
+
file_id=item.file_id,
|
| 274 |
+
start_ms=0,
|
| 275 |
+
end_ms=item.duration_ms,
|
| 276 |
+
reference=item.text,
|
| 277 |
+
language="tr",
|
| 278 |
+
)
|
| 279 |
+
for item in sorted(chosen, key=lambda i: i.file_id)
|
| 280 |
+
)
|
| 281 |
+
return Selection(clips, tuple(quarantined))
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
# --------------------------------------------------------------------------------- YODAS
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
@dataclass(frozen=True)
|
| 288 |
+
class YodasUtt:
|
| 289 |
+
utt_id: str
|
| 290 |
+
caption: str
|
| 291 |
+
duration_ms: int
|
| 292 |
+
shard: str = "" # archive shard holding the audio ("" = unknown / single shard)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def yodas_video_id(utt_id: str) -> str:
|
| 296 |
+
"""Video id of a ``<video>-<seg>-<start_cs>-<end_cs>`` utterance id.
|
| 297 |
+
|
| 298 |
+
Only the shape is checked, not the span: a listing line with an empty span still
|
| 299 |
+
names its video (de-duplication must see it; ingest quarantines the utterance).
|
| 300 |
+
|
| 301 |
+
Raises:
|
| 302 |
+
ValueError: if the id does not have the expected shape.
|
| 303 |
+
"""
|
| 304 |
+
parts = utt_id.rsplit("-", 3)
|
| 305 |
+
if len(parts) != 4 or not all(p.isdigit() for p in parts[1:]) or not parts[0]:
|
| 306 |
+
raise ValueError(f"unexpected YODAS utterance id {utt_id!r}")
|
| 307 |
+
return parts[0]
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def parse_yodas_utt_id(utt_id: str) -> tuple[str, int, int]:
|
| 311 |
+
"""Split ``<video>-<seg>-<start_cs>-<end_cs>`` into (video_id, start_ms, end_ms).
|
| 312 |
+
|
| 313 |
+
YouTube ids may contain ``-``, so the id is split from the right. Times are in
|
| 314 |
+
centiseconds in the id.
|
| 315 |
+
|
| 316 |
+
Raises:
|
| 317 |
+
ValueError: if the id does not have the expected shape.
|
| 318 |
+
"""
|
| 319 |
+
video_id = yodas_video_id(utt_id)
|
| 320 |
+
parts = utt_id.rsplit("-", 3)
|
| 321 |
+
start_ms, end_ms = int(parts[2]) * 10, int(parts[3]) * 10
|
| 322 |
+
if end_ms <= start_ms:
|
| 323 |
+
raise ValueError(f"empty span in YODAS utterance id {utt_id!r}")
|
| 324 |
+
return video_id, start_ms, end_ms
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
def select_yodas_pool(
|
| 328 |
+
utts: Iterable[YodasUtt],
|
| 329 |
+
n: int,
|
| 330 |
+
seed: int,
|
| 331 |
+
*,
|
| 332 |
+
min_s: float = 3.0,
|
| 333 |
+
max_s: float = 15.0,
|
| 334 |
+
max_per_video: int = 3,
|
| 335 |
+
source_id: str = "yodas_tr",
|
| 336 |
+
layer: str = "C",
|
| 337 |
+
) -> Selection:
|
| 338 |
+
"""Pick a candidate pool for layer C (to be transcribed by every ASR first).
|
| 339 |
+
|
| 340 |
+
Rules: duration in range; caption non-empty and digit-free (captions become the
|
| 341 |
+
draft the user corrects, so they must not contain digits either); at most
|
| 342 |
+
``max_per_video`` clips per video (the video id stands in for the speaker).
|
| 343 |
+
``reference`` is ``None``: captions are not references; the caption is stored in
|
| 344 |
+
``meta["caption"]``.
|
| 345 |
+
"""
|
| 346 |
+
_check_count(n)
|
| 347 |
+
_check_duration_window(min_s, max_s)
|
| 348 |
+
if max_per_video < 1:
|
| 349 |
+
raise ValueError(f"max_per_video must be >= 1, got {max_per_video}")
|
| 350 |
+
quarantined: list[Quarantined] = []
|
| 351 |
+
by_video: dict[str, list[tuple[YodasUtt, int, int]]] = defaultdict(list)
|
| 352 |
+
for utt in sorted(utts, key=lambda u: u.utt_id):
|
| 353 |
+
try:
|
| 354 |
+
video_id, start_ms, end_ms = parse_yodas_utt_id(utt.utt_id)
|
| 355 |
+
except ValueError as err:
|
| 356 |
+
quarantined.append(Quarantined(utt.utt_id, "select", "bad_utt_id", str(err)))
|
| 357 |
+
continue
|
| 358 |
+
duration_s = utt.duration_ms / 1000
|
| 359 |
+
span_ms = end_ms - start_ms
|
| 360 |
+
if abs(utt.duration_ms - span_ms) > YODAS_DURATION_TOLERANCE_MS:
|
| 361 |
+
quarantined.append(
|
| 362 |
+
Quarantined(
|
| 363 |
+
utt.utt_id, "select", "duration_mismatch", f"{utt.duration_ms} vs {span_ms} ms"
|
| 364 |
+
)
|
| 365 |
+
)
|
| 366 |
+
elif not min_s <= duration_s <= max_s:
|
| 367 |
+
quarantined.append(
|
| 368 |
+
Quarantined(utt.utt_id, "select", "duration_out_of_range", f"{duration_s:.2f}s")
|
| 369 |
+
)
|
| 370 |
+
elif not normalize_for_scoring(utt.caption):
|
| 371 |
+
quarantined.append(Quarantined(utt.utt_id, "select", "empty_caption"))
|
| 372 |
+
elif has_digits(utt.caption):
|
| 373 |
+
quarantined.append(Quarantined(utt.utt_id, "select", "caption_has_digits"))
|
| 374 |
+
else:
|
| 375 |
+
by_video[video_id].append((utt, start_ms, end_ms))
|
| 376 |
+
rng = random.Random(seed)
|
| 377 |
+
eligible: list[tuple[str, YodasUtt]] = []
|
| 378 |
+
for video_id in sorted(by_video):
|
| 379 |
+
candidates = by_video[video_id]
|
| 380 |
+
picked = rng.sample(candidates, min(max_per_video, len(candidates)))
|
| 381 |
+
eligible.extend((video_id, utt) for utt, _, _ in picked)
|
| 382 |
+
if len(eligible) < n:
|
| 383 |
+
raise ValueError(f"only {len(eligible)} eligible YODAS utterances, need {n}")
|
| 384 |
+
chosen = rng.sample(eligible, n)
|
| 385 |
+
clips = tuple(
|
| 386 |
+
Clip.build(
|
| 387 |
+
source_id=source_id,
|
| 388 |
+
layer=layer,
|
| 389 |
+
file_id=utt.utt_id,
|
| 390 |
+
start_ms=0,
|
| 391 |
+
end_ms=utt.duration_ms,
|
| 392 |
+
reference=None,
|
| 393 |
+
language="tr",
|
| 394 |
+
meta={"video_id": video_id, "caption": nfc(utt.caption)}
|
| 395 |
+
| ({"shard": utt.shard} if utt.shard else {}),
|
| 396 |
+
)
|
| 397 |
+
for video_id, utt in sorted(chosen, key=lambda pair: pair[1].utt_id)
|
| 398 |
+
)
|
| 399 |
+
return Selection(clips, tuple(quarantined))
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def text_has_filler(text: str) -> bool:
|
| 403 |
+
"""True if ``text`` contains a canonical filler (tag-spec section 4).
|
| 404 |
+
|
| 405 |
+
The single definition of the layer-C "filler signal": used both to select clips and
|
| 406 |
+
to record why they were selected (``nida.bench.b1.FillerProvenance``).
|
| 407 |
+
"""
|
| 408 |
+
return any(token in FILLERS for token in normalize_for_scoring(text))
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
def select_yodas_final(
|
| 412 |
+
pool: Sequence[Clip],
|
| 413 |
+
hypotheses: Mapping[str, Sequence[str]],
|
| 414 |
+
n: int,
|
| 415 |
+
seed: int,
|
| 416 |
+
*,
|
| 417 |
+
min_filler_clips: int,
|
| 418 |
+
) -> tuple[Clip, ...]:
|
| 419 |
+
"""Choose the ``n`` layer-C clips the user will transcribe, favouring fillers.
|
| 420 |
+
|
| 421 |
+
A clip has a *filler signal* if its caption or any model hypothesis contains a
|
| 422 |
+
canonical filler (tag-spec section 4). Filler-signal clips are taken first (seeded
|
| 423 |
+
order) up to ``n``; the rest is filled with other pool clips. Hypotheses only steer
|
| 424 |
+
*which* clips are reviewed, never the reference text itself.
|
| 425 |
+
|
| 426 |
+
Args:
|
| 427 |
+
pool: Layer-C pool from :func:`select_yodas_pool`.
|
| 428 |
+
hypotheses: ``clip_id`` → raw hypothesis texts from every ASR.
|
| 429 |
+
n: Number of clips to return.
|
| 430 |
+
seed: RNG seed.
|
| 431 |
+
min_filler_clips: Required number of filler-signal clips (ValueError if the pool
|
| 432 |
+
has fewer, so a filler-poor pool is reported instead of silently accepted).
|
| 433 |
+
"""
|
| 434 |
+
_check_count(n)
|
| 435 |
+
if not 0 <= min_filler_clips <= n:
|
| 436 |
+
raise ValueError(f"min_filler_clips must be in [0, {n}], got {min_filler_clips}")
|
| 437 |
+
if len(pool) < n:
|
| 438 |
+
raise ValueError(f"pool has {len(pool)} clips, need {n}")
|
| 439 |
+
|
| 440 |
+
def has_filler_signal(clip: Clip) -> bool:
|
| 441 |
+
texts = [clip.meta.get("caption", ""), *hypotheses.get(clip.clip_id, ())]
|
| 442 |
+
return any(text_has_filler(text) for text in texts)
|
| 443 |
+
|
| 444 |
+
ordered = sorted(pool, key=lambda c: c.clip_id)
|
| 445 |
+
with_filler = [c for c in ordered if has_filler_signal(c)]
|
| 446 |
+
without = [c for c in ordered if not has_filler_signal(c)]
|
| 447 |
+
if len(with_filler) < min_filler_clips:
|
| 448 |
+
raise ValueError(
|
| 449 |
+
f"only {len(with_filler)} pool clips show a filler, need {min_filler_clips}"
|
| 450 |
+
)
|
| 451 |
+
rng = random.Random(seed)
|
| 452 |
+
rng.shuffle(with_filler)
|
| 453 |
+
rng.shuffle(without)
|
| 454 |
+
chosen = (with_filler + without)[:n]
|
| 455 |
+
return tuple(sorted(chosen, key=lambda c: c.clip_id))
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
# ------------------------------------------------------------------------- serialisation
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
def clip_to_obj(clip: Clip) -> dict[str, object]:
|
| 462 |
+
"""JSON-compatible representation of a clip."""
|
| 463 |
+
return {
|
| 464 |
+
"clip_id": clip.clip_id,
|
| 465 |
+
"source_id": clip.source_id,
|
| 466 |
+
"layer": clip.layer,
|
| 467 |
+
"file_id": clip.file_id,
|
| 468 |
+
"start_ms": clip.start_ms,
|
| 469 |
+
"end_ms": clip.end_ms,
|
| 470 |
+
"reference": clip.reference,
|
| 471 |
+
"language": clip.language,
|
| 472 |
+
"meta": dict(clip.meta),
|
| 473 |
+
}
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def clip_from_obj(obj: object) -> Clip:
|
| 477 |
+
"""Inverse of :func:`clip_to_obj`; re-derives and checks the content-addressed id.
|
| 478 |
+
|
| 479 |
+
Raises:
|
| 480 |
+
ValueError: on missing/mistyped fields or an id that does not match the content.
|
| 481 |
+
"""
|
| 482 |
+
if not isinstance(obj, dict):
|
| 483 |
+
raise ValueError("clip must be a JSON object")
|
| 484 |
+
try:
|
| 485 |
+
language = obj["language"]
|
| 486 |
+
if language not in ("tr", "en"):
|
| 487 |
+
raise ValueError(f"unsupported language {language!r}")
|
| 488 |
+
meta = obj.get("meta", {})
|
| 489 |
+
if not isinstance(meta, dict) or not all(
|
| 490 |
+
isinstance(k, str) and isinstance(v, str) for k, v in meta.items()
|
| 491 |
+
):
|
| 492 |
+
raise ValueError("meta must map strings to strings")
|
| 493 |
+
reference = obj["reference"]
|
| 494 |
+
if reference is not None and not isinstance(reference, str):
|
| 495 |
+
raise ValueError("reference must be a string or null")
|
| 496 |
+
clip = Clip.build(
|
| 497 |
+
source_id=str(obj["source_id"]),
|
| 498 |
+
layer=str(obj["layer"]),
|
| 499 |
+
file_id=str(obj["file_id"]),
|
| 500 |
+
start_ms=int(obj["start_ms"]),
|
| 501 |
+
end_ms=int(obj["end_ms"]),
|
| 502 |
+
reference=reference,
|
| 503 |
+
language=language,
|
| 504 |
+
meta=meta,
|
| 505 |
+
)
|
| 506 |
+
except KeyError as err:
|
| 507 |
+
raise ValueError(f"clip is missing field {err}") from err
|
| 508 |
+
if clip.clip_id != obj.get("clip_id"):
|
| 509 |
+
raise ValueError(f"clip_id {obj.get('clip_id')!r} does not match content ({clip.clip_id})")
|
| 510 |
+
return clip
|
src/nida/bench/envs.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Isolated ``uv`` virtual environments for the benchmark workers.
|
| 2 |
+
|
| 3 |
+
The benchmark workers have conflicting dependency pins (``omnilingual-asr`` needs
|
| 4 |
+
Python <= 3.12 and ``fairseq2n`` pins ``torch==2.8.0``; ``qwen-asr`` pins
|
| 5 |
+
``transformers==4.57.6``), so each worker family runs in its own environment built by
|
| 6 |
+
``uv``. This module only *builds and describes* those environments; it never imports the
|
| 7 |
+
heavy packages itself and uses the standard library only.
|
| 8 |
+
|
| 9 |
+
All subprocess calls go through an injectable :data:`CommandRunner` so tests never
|
| 10 |
+
execute ``uv``. An environment is described by an :class:`EnvSpec`; a marker file
|
| 11 |
+
(``.nida-env.json``) is written last so a half-built environment is always rebuilt.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import hashlib
|
| 17 |
+
import json
|
| 18 |
+
import logging
|
| 19 |
+
import re
|
| 20 |
+
import shutil
|
| 21 |
+
import subprocess
|
| 22 |
+
import sys
|
| 23 |
+
from collections.abc import Callable, Mapping, Sequence
|
| 24 |
+
from dataclasses import asdict, dataclass
|
| 25 |
+
from datetime import UTC, datetime
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
from types import MappingProxyType
|
| 28 |
+
|
| 29 |
+
from nida.utils.io import atomic_write_json, atomic_write_text
|
| 30 |
+
|
| 31 |
+
__all__ = [
|
| 32 |
+
"ENV_SPECS",
|
| 33 |
+
"MARKER_NAME",
|
| 34 |
+
"CommandRunner",
|
| 35 |
+
"EnvBuildError",
|
| 36 |
+
"EnvSpec",
|
| 37 |
+
"ensure_env",
|
| 38 |
+
"env_for_worker",
|
| 39 |
+
"find_uv",
|
| 40 |
+
"freeze_text",
|
| 41 |
+
"subprocess_runner",
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
_LOG = logging.getLogger(__name__)
|
| 45 |
+
|
| 46 |
+
MARKER_NAME = ".nida-env.json"
|
| 47 |
+
FREEZE_NAME = "freeze.txt"
|
| 48 |
+
UV_VERSION = "0.10.4"
|
| 49 |
+
_SUBPROCESS_TIMEOUT_S = 3600
|
| 50 |
+
_STDERR_TAIL_CHARS = 2000
|
| 51 |
+
_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class EnvBuildError(RuntimeError):
|
| 55 |
+
"""Raised when an environment cannot be built or a command fails."""
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
CommandRunner = Callable[[Sequence[str]], str]
|
| 59 |
+
"""Runs a command and returns its stdout; raises :class:`EnvBuildError` on failure."""
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass(frozen=True)
|
| 63 |
+
class EnvSpec:
|
| 64 |
+
"""Description of one worker environment.
|
| 65 |
+
|
| 66 |
+
Attributes:
|
| 67 |
+
name: Environment directory name (lowercase ``[a-z][a-z0-9_-]*``).
|
| 68 |
+
python: Python version passed to ``uv venv --python``.
|
| 69 |
+
torch_packages: Pinned torch packages installed first from ``torch_index_url``. Empty
|
| 70 |
+
= the packages bring their own torch (or none): the torch step and the torch
|
| 71 |
+
version guard of :func:`ensure_env` are skipped.
|
| 72 |
+
torch_index_url: Package index serving the matching CUDA wheels (unused when
|
| 73 |
+
``torch_packages`` is empty).
|
| 74 |
+
packages: Pinned packages installed from the default index after torch.
|
| 75 |
+
workers: Worker module names (under ``nida.bench.workers``) run in this env.
|
| 76 |
+
"""
|
| 77 |
+
|
| 78 |
+
name: str
|
| 79 |
+
python: str
|
| 80 |
+
torch_packages: tuple[str, ...]
|
| 81 |
+
torch_index_url: str
|
| 82 |
+
packages: tuple[str, ...]
|
| 83 |
+
workers: tuple[str, ...]
|
| 84 |
+
|
| 85 |
+
def __post_init__(self) -> None:
|
| 86 |
+
if not _NAME_RE.fullmatch(self.name):
|
| 87 |
+
raise ValueError(f"invalid environment name {self.name!r}; expected {_NAME_RE.pattern}")
|
| 88 |
+
|
| 89 |
+
def spec_hash(self) -> str:
|
| 90 |
+
"""Return the sha256 hex digest of a canonical JSON of all fields.
|
| 91 |
+
|
| 92 |
+
The digest is stable across runs and processes (sorted keys, fixed separators).
|
| 93 |
+
"""
|
| 94 |
+
canonical = json.dumps(asdict(self), sort_keys=True, separators=(",", ":"))
|
| 95 |
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
_PYTHON = "3.12"
|
| 99 |
+
_TORCH_PACKAGES = ("torch==2.8.0", "torchaudio==2.8.0")
|
| 100 |
+
_TORCH_INDEX_URL = "https://download.pytorch.org/whl/cu128"
|
| 101 |
+
|
| 102 |
+
ENV_SPECS: Mapping[str, EnvSpec] = MappingProxyType(
|
| 103 |
+
{
|
| 104 |
+
"hf": EnvSpec(
|
| 105 |
+
name="hf",
|
| 106 |
+
python=_PYTHON,
|
| 107 |
+
torch_packages=_TORCH_PACKAGES,
|
| 108 |
+
torch_index_url=_TORCH_INDEX_URL,
|
| 109 |
+
packages=("transformers==4.57.6", "huggingface-hub", "soundfile", "numpy"),
|
| 110 |
+
workers=("asr_hf",),
|
| 111 |
+
),
|
| 112 |
+
"qwen": EnvSpec(
|
| 113 |
+
name="qwen",
|
| 114 |
+
python=_PYTHON,
|
| 115 |
+
torch_packages=_TORCH_PACKAGES,
|
| 116 |
+
torch_index_url=_TORCH_INDEX_URL,
|
| 117 |
+
packages=("qwen-asr==0.0.6", "huggingface-hub"),
|
| 118 |
+
workers=("asr_qwen",),
|
| 119 |
+
),
|
| 120 |
+
"omni": EnvSpec(
|
| 121 |
+
name="omni",
|
| 122 |
+
python=_PYTHON,
|
| 123 |
+
torch_packages=_TORCH_PACKAGES,
|
| 124 |
+
torch_index_url=_TORCH_INDEX_URL,
|
| 125 |
+
# omnilingual-asr needs pyarrow>=20 but keeps NumPy 1.26, and pyarrow 26 needs
|
| 126 |
+
# NumPy 2 ("pyarrow requires NumPy 2.0 or newer, found 1.26.4", fresh env
|
| 127 |
+
# 2026-10-09). 21.0.0 resolves and imports with NumPy 1.26 (checked on Colab).
|
| 128 |
+
packages=("omnilingual-asr==0.2.0", "pyarrow==21.0.0"),
|
| 129 |
+
workers=("asr_omni",),
|
| 130 |
+
),
|
| 131 |
+
"audio": EnvSpec(
|
| 132 |
+
name="audio",
|
| 133 |
+
python=_PYTHON,
|
| 134 |
+
torch_packages=_TORCH_PACKAGES,
|
| 135 |
+
torch_index_url=_TORCH_INDEX_URL,
|
| 136 |
+
packages=(
|
| 137 |
+
"audiobox-aesthetics==0.0.4",
|
| 138 |
+
"numpy",
|
| 139 |
+
"pydantic",
|
| 140 |
+
"soundfile",
|
| 141 |
+
"huggingface-hub",
|
| 142 |
+
# audiobox_aesthetics.utils imports requests without declaring it
|
| 143 |
+
# (run 20261008-124740-bench-phase0-8c6a failed on the import).
|
| 144 |
+
"requests",
|
| 145 |
+
),
|
| 146 |
+
workers=("vae_voxcpm", "quality_audiobox"),
|
| 147 |
+
),
|
| 148 |
+
}
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def env_for_worker(worker: str) -> EnvSpec:
|
| 153 |
+
"""Return the environment spec that runs ``worker``.
|
| 154 |
+
|
| 155 |
+
Args:
|
| 156 |
+
worker: Worker module name, e.g. ``"asr_qwen"``.
|
| 157 |
+
|
| 158 |
+
Raises:
|
| 159 |
+
ValueError: if no environment lists ``worker``.
|
| 160 |
+
"""
|
| 161 |
+
for spec in ENV_SPECS.values():
|
| 162 |
+
if worker in spec.workers:
|
| 163 |
+
return spec
|
| 164 |
+
known = sorted(w for spec in ENV_SPECS.values() for w in spec.workers)
|
| 165 |
+
raise ValueError(f"unknown worker {worker!r}; known workers: {known}")
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def subprocess_runner(cmd: Sequence[str]) -> str:
|
| 169 |
+
"""Run ``cmd`` with :func:`subprocess.run` and return its stdout.
|
| 170 |
+
|
| 171 |
+
Args:
|
| 172 |
+
cmd: Command and arguments (no shell).
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
Captured stdout text.
|
| 176 |
+
|
| 177 |
+
Raises:
|
| 178 |
+
EnvBuildError: on a non-zero exit code, a timeout, or if the executable is missing.
|
| 179 |
+
The message carries the command, the exit code and the stderr tail.
|
| 180 |
+
"""
|
| 181 |
+
command = list(cmd)
|
| 182 |
+
try:
|
| 183 |
+
result = subprocess.run(
|
| 184 |
+
command,
|
| 185 |
+
capture_output=True,
|
| 186 |
+
text=True,
|
| 187 |
+
check=False,
|
| 188 |
+
timeout=_SUBPROCESS_TIMEOUT_S,
|
| 189 |
+
)
|
| 190 |
+
except subprocess.TimeoutExpired as exc:
|
| 191 |
+
raise EnvBuildError(f"command timed out after {_SUBPROCESS_TIMEOUT_S}s: {command}") from exc
|
| 192 |
+
except OSError as exc:
|
| 193 |
+
raise EnvBuildError(f"command could not be started: {command}: {exc}") from exc
|
| 194 |
+
if result.returncode != 0:
|
| 195 |
+
tail = result.stderr[-_STDERR_TAIL_CHARS:]
|
| 196 |
+
raise EnvBuildError(
|
| 197 |
+
f"command failed with exit code {result.returncode}: {command}\nstderr tail:\n{tail}"
|
| 198 |
+
)
|
| 199 |
+
return result.stdout
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def find_uv(
|
| 203 |
+
runner: CommandRunner = subprocess_runner,
|
| 204 |
+
which: Callable[[str], str | None] = shutil.which,
|
| 205 |
+
) -> str:
|
| 206 |
+
"""Locate ``uv``, installing it with pip when it is not on ``PATH``.
|
| 207 |
+
|
| 208 |
+
Args:
|
| 209 |
+
runner: Command runner used for the pip install.
|
| 210 |
+
which: Executable lookup (injectable for tests).
|
| 211 |
+
|
| 212 |
+
Returns:
|
| 213 |
+
``"uv"`` if it is already on ``PATH``, otherwise the path found after installing.
|
| 214 |
+
|
| 215 |
+
Raises:
|
| 216 |
+
EnvBuildError: if ``uv`` is still missing after the install.
|
| 217 |
+
"""
|
| 218 |
+
if which("uv"):
|
| 219 |
+
return "uv"
|
| 220 |
+
runner([sys.executable, "-m", "pip", "install", "-q", f"uv=={UV_VERSION}"])
|
| 221 |
+
found = which("uv")
|
| 222 |
+
if not found:
|
| 223 |
+
raise EnvBuildError(f"uv=={UV_VERSION} was installed with pip but is still not on PATH")
|
| 224 |
+
return found
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _python_path(env_dir: Path) -> Path:
|
| 228 |
+
return env_dir / "bin" / "python"
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def _remove_env_dir(env_dir: Path, envs_root: Path) -> None:
|
| 232 |
+
"""Delete ``env_dir`` if it exists, refusing anything not strictly inside ``envs_root``.
|
| 233 |
+
|
| 234 |
+
Raises:
|
| 235 |
+
EnvBuildError: if ``env_dir`` resolves outside (or equal to) ``envs_root``.
|
| 236 |
+
"""
|
| 237 |
+
root = envs_root.resolve()
|
| 238 |
+
target = env_dir.resolve()
|
| 239 |
+
if target == root or not target.is_relative_to(root):
|
| 240 |
+
raise EnvBuildError(f"refusing to delete {env_dir}: not strictly inside {envs_root}")
|
| 241 |
+
if env_dir.is_symlink():
|
| 242 |
+
env_dir.unlink()
|
| 243 |
+
elif env_dir.exists():
|
| 244 |
+
shutil.rmtree(env_dir)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _is_current(spec: EnvSpec, env_dir: Path) -> bool:
|
| 248 |
+
"""Return True if a complete, up-to-date environment exists at ``env_dir``."""
|
| 249 |
+
marker = env_dir / MARKER_NAME
|
| 250 |
+
if not marker.is_file() or not _python_path(env_dir).exists():
|
| 251 |
+
return False
|
| 252 |
+
try:
|
| 253 |
+
data = json.loads(marker.read_text(encoding="utf-8"))
|
| 254 |
+
except (OSError, json.JSONDecodeError) as exc:
|
| 255 |
+
_LOG.warning("unreadable env marker %s (%s); rebuilding", marker, exc)
|
| 256 |
+
return False
|
| 257 |
+
return isinstance(data, dict) and data.get("spec_hash") == spec.spec_hash()
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _expected_torch_version(spec: EnvSpec) -> str:
|
| 261 |
+
for pkg in spec.torch_packages:
|
| 262 |
+
if pkg.startswith("torch=="):
|
| 263 |
+
return pkg.removeprefix("torch==")
|
| 264 |
+
raise EnvBuildError(f"env {spec.name!r} pins no 'torch==X' in torch_packages")
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def ensure_env(
|
| 268 |
+
spec: EnvSpec,
|
| 269 |
+
envs_root: Path,
|
| 270 |
+
repo_dir: Path,
|
| 271 |
+
runner: CommandRunner = subprocess_runner,
|
| 272 |
+
uv: str = "uv",
|
| 273 |
+
) -> Path:
|
| 274 |
+
"""Build the environment for ``spec`` unless an up-to-date one already exists.
|
| 275 |
+
|
| 276 |
+
The marker ``<envs_root>/<name>/.nida-env.json`` is written last, so any failure
|
| 277 |
+
leaves no marker and the next call rebuilds from scratch.
|
| 278 |
+
|
| 279 |
+
Args:
|
| 280 |
+
spec: Environment description.
|
| 281 |
+
envs_root: Directory holding all environments.
|
| 282 |
+
repo_dir: Repository root, installed with ``--no-deps -e`` (nida has no runtime deps).
|
| 283 |
+
runner: Command runner (injectable).
|
| 284 |
+
uv: ``uv`` executable.
|
| 285 |
+
|
| 286 |
+
Returns:
|
| 287 |
+
Path of the environment's python executable.
|
| 288 |
+
|
| 289 |
+
Raises:
|
| 290 |
+
EnvBuildError: if a command fails, the torch version guard trips, or the delete
|
| 291 |
+
guard refuses the environment directory.
|
| 292 |
+
"""
|
| 293 |
+
env_dir = envs_root / spec.name
|
| 294 |
+
py = _python_path(env_dir)
|
| 295 |
+
if _is_current(spec, env_dir):
|
| 296 |
+
_LOG.info("env %s is up to date at %s", spec.name, env_dir)
|
| 297 |
+
return py
|
| 298 |
+
|
| 299 |
+
envs_root.mkdir(parents=True, exist_ok=True)
|
| 300 |
+
_remove_env_dir(env_dir, envs_root)
|
| 301 |
+
|
| 302 |
+
runner([uv, "venv", "--python", spec.python, str(env_dir)])
|
| 303 |
+
if spec.torch_packages:
|
| 304 |
+
runner(
|
| 305 |
+
[
|
| 306 |
+
uv,
|
| 307 |
+
"pip",
|
| 308 |
+
"install",
|
| 309 |
+
"--python",
|
| 310 |
+
str(py),
|
| 311 |
+
"--index-url",
|
| 312 |
+
spec.torch_index_url,
|
| 313 |
+
*spec.torch_packages,
|
| 314 |
+
]
|
| 315 |
+
)
|
| 316 |
+
runner([uv, "pip", "install", "--python", str(py), *spec.packages])
|
| 317 |
+
runner([uv, "pip", "install", "--python", str(py), "--no-deps", "-e", str(repo_dir)])
|
| 318 |
+
|
| 319 |
+
if spec.torch_packages:
|
| 320 |
+
expected = _expected_torch_version(spec)
|
| 321 |
+
torch_version = runner(
|
| 322 |
+
[str(py), "-c", "import torch, sys; print(torch.__version__)"]
|
| 323 |
+
).strip()
|
| 324 |
+
if not torch_version.startswith(expected):
|
| 325 |
+
raise EnvBuildError(
|
| 326 |
+
f"env {spec.name!r}: torch version is {torch_version!r}, expected {expected!r} "
|
| 327 |
+
"(a later install step probably replaced torch)"
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
frozen = runner([uv, "pip", "freeze", "--python", str(py)])
|
| 331 |
+
atomic_write_text(env_dir / FREEZE_NAME, frozen)
|
| 332 |
+
|
| 333 |
+
atomic_write_json(
|
| 334 |
+
env_dir / MARKER_NAME,
|
| 335 |
+
{
|
| 336 |
+
"name": spec.name,
|
| 337 |
+
"spec_hash": spec.spec_hash(),
|
| 338 |
+
"spec": asdict(spec),
|
| 339 |
+
"created_at": datetime.now(UTC).isoformat(),
|
| 340 |
+
},
|
| 341 |
+
)
|
| 342 |
+
return py
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def freeze_text(spec: EnvSpec, envs_root: Path) -> str:
|
| 346 |
+
"""Return the ``pip freeze`` output recorded when ``spec`` was built.
|
| 347 |
+
|
| 348 |
+
Raises:
|
| 349 |
+
FileNotFoundError: if the environment has not been built (no ``freeze.txt``).
|
| 350 |
+
"""
|
| 351 |
+
return (envs_root / spec.name / FREEZE_NAME).read_text(encoding="utf-8")
|
src/nida/bench/expresso.py
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Expresso layout parsing and clip selection for B2 (docs/benchmarks/B1-B2-tasarim.md §2).
|
| 2 |
+
|
| 3 |
+
Archive layout (textlesslib ``examples/expresso/dataset/README.md``)::
|
| 4 |
+
|
| 5 |
+
expresso/read_transcriptions.txt
|
| 6 |
+
expresso/VAD_segments.txt "{stem}/channel{1|2}\t(s, e) (s, e) ..."
|
| 7 |
+
(mono long-form lines: bare "{stem}", kept as
|
| 8 |
+
channel 0 and not used by B2)
|
| 9 |
+
expresso/audio_48khz/read/{spk}/{style}/{base|longform}/{spk}_{style...}_{id}.wav mono
|
| 10 |
+
expresso/audio_48khz/conversational/{spk1}-{spk2}/{style1[-style2]}/{stem}.wav stereo
|
| 11 |
+
|
| 12 |
+
In a conversational style folder without a hyphen both channels carry that style; with a
|
| 13 |
+
hyphen the left style belongs to channel 1 (left speaker) and the right one to channel 2.
|
| 14 |
+
Subsets used by B2: ``whisper``, ``laughing`` and ``default`` from read speech (they have
|
| 15 |
+
transcripts), ``projected`` and ``nonverbal`` from conversational VAD segments of a
|
| 16 |
+
channel that carries the style. Folder spellings ``non_verbal`` / ``nonverbal`` are both
|
| 17 |
+
accepted.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import hashlib
|
| 23 |
+
import random
|
| 24 |
+
import re
|
| 25 |
+
from collections import defaultdict
|
| 26 |
+
from collections.abc import Iterable, Mapping, Sequence
|
| 27 |
+
from dataclasses import dataclass
|
| 28 |
+
|
| 29 |
+
from nida.bench.clips import Clip, Quarantined, Selection
|
| 30 |
+
|
| 31 |
+
__all__ = [
|
| 32 |
+
"CONVERSATIONAL_SUBSETS",
|
| 33 |
+
"READ_SUBSETS",
|
| 34 |
+
"ExpressoFile",
|
| 35 |
+
"is_wanted_member",
|
| 36 |
+
"parse_member",
|
| 37 |
+
"parse_read_transcriptions",
|
| 38 |
+
"parse_vad_segments",
|
| 39 |
+
"select_expresso",
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
SOURCE_ID = "expresso"
|
| 43 |
+
READ_SUBSETS = ("whisper", "laughing", "default")
|
| 44 |
+
CONVERSATIONAL_SUBSETS = ("projected", "nonverbal")
|
| 45 |
+
_STYLE_ALIASES = {"non_verbal": "nonverbal"}
|
| 46 |
+
_AUDIO_ROOT = "audio_48khz/"
|
| 47 |
+
#: Keep 1 in N default-style read files (default read speech is large; B2 needs ~10).
|
| 48 |
+
DEFAULT_KEEP_ONE_IN = 8
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _style(name: str) -> str:
|
| 52 |
+
return _STYLE_ALIASES.get(name, name)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@dataclass(frozen=True)
|
| 56 |
+
class ExpressoFile:
|
| 57 |
+
"""One audio member of the archive.
|
| 58 |
+
|
| 59 |
+
Attributes:
|
| 60 |
+
member: Archive member name (as stored in the tar).
|
| 61 |
+
kind: ``"read"`` or ``"conversational"``.
|
| 62 |
+
stem: File name without extension (key of VAD segments and transcripts).
|
| 63 |
+
speakers: One speaker (read) or (channel1, channel2) speakers.
|
| 64 |
+
styles: One style (read) or (channel1, channel2) styles.
|
| 65 |
+
corpus: ``"base"`` or ``"longform"`` for read speech, ``""`` otherwise.
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
member: str
|
| 69 |
+
kind: str
|
| 70 |
+
stem: str
|
| 71 |
+
speakers: tuple[str, ...]
|
| 72 |
+
styles: tuple[str, ...]
|
| 73 |
+
corpus: str
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def parse_member(name: str) -> ExpressoFile | None:
|
| 77 |
+
"""Parse an archive member name; ``None`` for anything that is not a 48 kHz WAV."""
|
| 78 |
+
if not name.endswith(".wav") or _AUDIO_ROOT not in name:
|
| 79 |
+
return None
|
| 80 |
+
rel = name.split(_AUDIO_ROOT, 1)[1].split("/")
|
| 81 |
+
stem = rel[-1][: -len(".wav")]
|
| 82 |
+
if rel[0] == "read" and len(rel) == 5:
|
| 83 |
+
_, speaker, style, corpus, _ = rel
|
| 84 |
+
return ExpressoFile(name, "read", stem, (speaker,), (_style(style),), corpus)
|
| 85 |
+
if rel[0] == "conversational" and len(rel) == 4:
|
| 86 |
+
_, pair, style_dir, _ = rel
|
| 87 |
+
speakers = tuple(pair.split("-"))
|
| 88 |
+
styles = tuple(_style(s) for s in style_dir.split("-"))
|
| 89 |
+
if len(speakers) != 2 or len(styles) not in (1, 2):
|
| 90 |
+
return None
|
| 91 |
+
return ExpressoFile(
|
| 92 |
+
name, "conversational", stem, speakers, styles if len(styles) == 2 else styles * 2, ""
|
| 93 |
+
)
|
| 94 |
+
return None
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _stable_fraction(text: str) -> float:
|
| 98 |
+
return int(hashlib.sha1(text.encode("utf-8")).hexdigest()[:8], 16) / 0xFFFFFFFF
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def is_wanted_member(name: str) -> bool:
|
| 102 |
+
"""Archive filter for streaming extraction (metadata files + B2 subsets only)."""
|
| 103 |
+
if name.endswith(("read_transcriptions.txt", "VAD_segments.txt")):
|
| 104 |
+
return True
|
| 105 |
+
parsed = parse_member(name)
|
| 106 |
+
if parsed is None:
|
| 107 |
+
return False
|
| 108 |
+
if parsed.kind == "read":
|
| 109 |
+
if parsed.corpus != "base":
|
| 110 |
+
return False
|
| 111 |
+
style = parsed.styles[0]
|
| 112 |
+
if style == "default":
|
| 113 |
+
return _stable_fraction(parsed.stem) < 1 / DEFAULT_KEEP_ONE_IN
|
| 114 |
+
return style in ("whisper", "laughing")
|
| 115 |
+
return any(style in CONVERSATIONAL_SUBSETS for style in parsed.styles)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
_NUMBER = r"\d+(?:\.\d+)?"
|
| 119 |
+
_SEGMENT_RE = re.compile(rf"\(\s*({_NUMBER})\s*,\s*({_NUMBER})\s*\)")
|
| 120 |
+
_SEGMENTS_RE = re.compile(rf"\s*(?:\(\s*{_NUMBER}\s*,\s*{_NUMBER}\s*\)\s*)*")
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def parse_vad_segments(text: str) -> dict[tuple[str, int], list[tuple[float, float]]]:
|
| 124 |
+
"""Parse ``VAD_segments.txt`` into ``{(stem, channel): [(start_s, end_s), ...]}``.
|
| 125 |
+
|
| 126 |
+
Conversational keys are ``<stem>/channel1|2``; the official file also lists mono
|
| 127 |
+
long-form read recordings with a bare ``<stem>`` key, stored here as channel 0.
|
| 128 |
+
|
| 129 |
+
Raises:
|
| 130 |
+
ValueError: on a malformed key or a segment with end <= start.
|
| 131 |
+
"""
|
| 132 |
+
out: dict[tuple[str, int], list[tuple[float, float]]] = {}
|
| 133 |
+
for number, line in enumerate(text.splitlines(), start=1):
|
| 134 |
+
if not line.strip() or line.startswith("#"):
|
| 135 |
+
continue
|
| 136 |
+
key, _, segments = line.partition("\t")
|
| 137 |
+
stem, sep, channel = key.rpartition("/channel")
|
| 138 |
+
if not sep:
|
| 139 |
+
stem, channel = key, "0" # mono long-form recording (bare key)
|
| 140 |
+
elif channel not in ("1", "2"):
|
| 141 |
+
raise ValueError(f"VAD line {number}: bad key {key!r}")
|
| 142 |
+
if not stem or "/" in stem or any(ch.isspace() for ch in stem):
|
| 143 |
+
raise ValueError(f"VAD line {number}: bad key {key!r}")
|
| 144 |
+
if not _SEGMENTS_RE.fullmatch(segments):
|
| 145 |
+
raise ValueError(f"VAD line {number}: unparseable segments {segments[:80]!r}")
|
| 146 |
+
if (stem, int(channel)) in out:
|
| 147 |
+
raise ValueError(f"VAD line {number}: duplicate key {key!r}")
|
| 148 |
+
spans: list[tuple[float, float]] = []
|
| 149 |
+
for match in _SEGMENT_RE.finditer(segments):
|
| 150 |
+
start_s, end_s = float(match.group(1)), float(match.group(2))
|
| 151 |
+
if end_s <= start_s:
|
| 152 |
+
raise ValueError(f"VAD line {number}: empty segment ({start_s}, {end_s})")
|
| 153 |
+
spans.append((start_s, end_s))
|
| 154 |
+
out[(stem, int(channel))] = spans
|
| 155 |
+
return out
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def parse_read_transcriptions(text: str) -> dict[str, str]:
|
| 159 |
+
"""Parse ``read_transcriptions.txt`` as ``<stem><TAB or space><text>`` lines.
|
| 160 |
+
|
| 161 |
+
Raises:
|
| 162 |
+
ValueError: on duplicate stems.
|
| 163 |
+
"""
|
| 164 |
+
out: dict[str, str] = {}
|
| 165 |
+
for line in text.splitlines():
|
| 166 |
+
if not line.strip() or line.startswith("#"):
|
| 167 |
+
continue
|
| 168 |
+
sep = "\t" if "\t" in line else " "
|
| 169 |
+
stem, _, transcript = line.partition(sep)
|
| 170 |
+
stem = stem.strip()
|
| 171 |
+
if stem in out:
|
| 172 |
+
raise ValueError(f"duplicate transcript id {stem}")
|
| 173 |
+
out[stem] = transcript.strip()
|
| 174 |
+
return out
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _round_robin_by_speaker(
|
| 178 |
+
candidates: Sequence[tuple[str, Clip]], n: int, rng: random.Random
|
| 179 |
+
) -> list[Clip]:
|
| 180 |
+
by_speaker: dict[str, list[Clip]] = defaultdict(list)
|
| 181 |
+
for speaker, clip in sorted(candidates, key=lambda pair: pair[1].clip_id):
|
| 182 |
+
by_speaker[speaker].append(clip)
|
| 183 |
+
for clips in by_speaker.values():
|
| 184 |
+
rng.shuffle(clips)
|
| 185 |
+
order = sorted(by_speaker)
|
| 186 |
+
chosen: list[Clip] = []
|
| 187 |
+
while len(chosen) < n and any(by_speaker[s] for s in order):
|
| 188 |
+
for speaker in order:
|
| 189 |
+
if by_speaker[speaker] and len(chosen) < n:
|
| 190 |
+
chosen.append(by_speaker[speaker].pop())
|
| 191 |
+
return chosen
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def select_expresso(
|
| 195 |
+
files: Iterable[tuple[ExpressoFile, int]],
|
| 196 |
+
vad: Mapping[tuple[str, int], Sequence[tuple[float, float]]],
|
| 197 |
+
transcripts: Mapping[str, str],
|
| 198 |
+
counts: Mapping[str, int],
|
| 199 |
+
seed: int,
|
| 200 |
+
*,
|
| 201 |
+
min_s: float = 3.0,
|
| 202 |
+
max_s: float = 15.0,
|
| 203 |
+
long_segment_cut_s: float = 12.0,
|
| 204 |
+
) -> Selection:
|
| 205 |
+
"""Choose B2 Expresso clips per subset, balanced across speakers.
|
| 206 |
+
|
| 207 |
+
Args:
|
| 208 |
+
files: Extracted files with their length in milliseconds.
|
| 209 |
+
vad: Parsed VAD segments.
|
| 210 |
+
transcripts: Parsed read transcriptions.
|
| 211 |
+
counts: Wanted clips per subset name (keys from READ/CONVERSATIONAL_SUBSETS).
|
| 212 |
+
seed: RNG seed.
|
| 213 |
+
min_s: Minimum clip duration.
|
| 214 |
+
max_s: Maximum duration of a read file / VAD segment taken whole.
|
| 215 |
+
long_segment_cut_s: Longer VAD segments contribute their first this-many seconds.
|
| 216 |
+
|
| 217 |
+
Raises:
|
| 218 |
+
ValueError: on unknown subsets or if a subset has fewer candidates than requested.
|
| 219 |
+
"""
|
| 220 |
+
unknown = set(counts) - set(READ_SUBSETS) - set(CONVERSATIONAL_SUBSETS)
|
| 221 |
+
if unknown:
|
| 222 |
+
raise ValueError(f"unknown Expresso subsets {sorted(unknown)}")
|
| 223 |
+
if not 0 < min_s < long_segment_cut_s <= max_s:
|
| 224 |
+
raise ValueError("need 0 < min_s < long_segment_cut_s <= max_s")
|
| 225 |
+
min_ms, max_ms, cut_ms = (round(v * 1000) for v in (min_s, max_s, long_segment_cut_s))
|
| 226 |
+
quarantined: list[Quarantined] = []
|
| 227 |
+
candidates: dict[str, list[tuple[str, Clip]]] = defaultdict(list)
|
| 228 |
+
for item, length_ms in sorted(files, key=lambda pair: pair[0].member):
|
| 229 |
+
if item.kind == "read":
|
| 230 |
+
subset = item.styles[0]
|
| 231 |
+
if subset not in counts:
|
| 232 |
+
continue
|
| 233 |
+
if not min_ms <= length_ms <= max_ms:
|
| 234 |
+
quarantined.append(Quarantined(item.stem, "select", "duration_out_of_range"))
|
| 235 |
+
continue
|
| 236 |
+
reference = transcripts.get(item.stem)
|
| 237 |
+
if not reference:
|
| 238 |
+
quarantined.append(Quarantined(item.stem, "select", "no_transcript"))
|
| 239 |
+
continue
|
| 240 |
+
clip = Clip.build(
|
| 241 |
+
source_id=SOURCE_ID,
|
| 242 |
+
layer=subset,
|
| 243 |
+
file_id=item.member,
|
| 244 |
+
start_ms=0,
|
| 245 |
+
end_ms=length_ms,
|
| 246 |
+
reference=reference,
|
| 247 |
+
language="en",
|
| 248 |
+
meta={"speaker": item.speakers[0], "channel": "0"},
|
| 249 |
+
)
|
| 250 |
+
candidates[subset].append((item.speakers[0], clip))
|
| 251 |
+
continue
|
| 252 |
+
for channel in (1, 2):
|
| 253 |
+
subset = item.styles[channel - 1]
|
| 254 |
+
if subset not in counts:
|
| 255 |
+
continue
|
| 256 |
+
spans = vad.get((item.stem, channel))
|
| 257 |
+
if spans is None:
|
| 258 |
+
quarantined.append(
|
| 259 |
+
Quarantined(f"{item.stem}/channel{channel}", "select", "no_vad_segments")
|
| 260 |
+
)
|
| 261 |
+
continue
|
| 262 |
+
for start_s, end_s in spans:
|
| 263 |
+
# Integer milliseconds: float seconds would drop exact-length segments
|
| 264 |
+
# (4.02 - 1.02 < 3.0 in binary floating point).
|
| 265 |
+
start_ms, end_ms = round(start_s * 1000), round(end_s * 1000)
|
| 266 |
+
if end_ms - start_ms < min_ms:
|
| 267 |
+
continue # short VAD islands are expected; not a rule violation
|
| 268 |
+
if end_ms - start_ms > max_ms:
|
| 269 |
+
end_ms = start_ms + cut_ms
|
| 270 |
+
if end_ms > length_ms:
|
| 271 |
+
if end_ms - length_ms > 1:
|
| 272 |
+
quarantined.append(
|
| 273 |
+
Quarantined(
|
| 274 |
+
item.stem, "select", "vad_beyond_audio", f"{end_ms}>{length_ms}"
|
| 275 |
+
)
|
| 276 |
+
)
|
| 277 |
+
continue
|
| 278 |
+
end_ms = length_ms # ≤ 1 ms of rounding at the very end of the file
|
| 279 |
+
if end_ms - start_ms < min_ms:
|
| 280 |
+
continue
|
| 281 |
+
clip = Clip.build(
|
| 282 |
+
source_id=SOURCE_ID,
|
| 283 |
+
layer=subset,
|
| 284 |
+
file_id=f"{item.member}#ch{channel}",
|
| 285 |
+
start_ms=start_ms,
|
| 286 |
+
end_ms=end_ms,
|
| 287 |
+
reference=None,
|
| 288 |
+
language="en",
|
| 289 |
+
meta={"speaker": item.speakers[channel - 1], "channel": str(channel)},
|
| 290 |
+
)
|
| 291 |
+
candidates[subset].append((item.speakers[channel - 1], clip))
|
| 292 |
+
rng = random.Random(seed)
|
| 293 |
+
chosen: list[Clip] = []
|
| 294 |
+
for subset in sorted(counts):
|
| 295 |
+
wanted = counts[subset]
|
| 296 |
+
if len(candidates[subset]) < wanted:
|
| 297 |
+
raise ValueError(
|
| 298 |
+
f"Expresso subset {subset}: {len(candidates[subset])} candidates, need {wanted}"
|
| 299 |
+
)
|
| 300 |
+
chosen.extend(_round_robin_by_speaker(candidates[subset], wanted, rng))
|
| 301 |
+
return Selection(tuple(sorted(chosen, key=lambda c: c.clip_id)), tuple(quarantined))
|
src/nida/bench/fetch.py
ADDED
|
@@ -0,0 +1,704 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reliable, resumable downloads and streaming tar extraction (stdlib only).
|
| 2 |
+
|
| 3 |
+
Benchmark data lives on public hosts (Hugging Face and others) and ranges from a few
|
| 4 |
+
hundred KB to a 36 GB tar. Colab sessions drop connections, so two tools are provided:
|
| 5 |
+
|
| 6 |
+
* :func:`download` fetches one file to disk. It resumes from a ``<name>.part`` file with an
|
| 7 |
+
HTTP ``Range`` request, retries transient network failures, verifies size and SHA-256 and
|
| 8 |
+
only then atomically renames the ``.part`` file to its final name. A completed file is
|
| 9 |
+
never overwritten and a partial file never looks complete.
|
| 10 |
+
* :func:`stream_extract` reads a (possibly huge, possibly compressed) tar archive as a
|
| 11 |
+
stream and writes only the selected regular files to disk, without ever storing the
|
| 12 |
+
archive itself.
|
| 13 |
+
|
| 14 |
+
All network access goes through an injectable :data:`Opener`, so tests run offline.
|
| 15 |
+
:func:`hf_resolve_url` builds revision-pinned Hugging Face download URLs.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import hashlib
|
| 21 |
+
import http.client
|
| 22 |
+
import logging
|
| 23 |
+
import os
|
| 24 |
+
import re
|
| 25 |
+
import shutil
|
| 26 |
+
import ssl
|
| 27 |
+
import tarfile
|
| 28 |
+
import tempfile
|
| 29 |
+
import time
|
| 30 |
+
import urllib.error
|
| 31 |
+
import urllib.parse
|
| 32 |
+
import urllib.request
|
| 33 |
+
import zlib
|
| 34 |
+
from collections.abc import Callable, Iterator
|
| 35 |
+
from contextlib import contextmanager
|
| 36 |
+
from dataclasses import dataclass
|
| 37 |
+
from pathlib import Path
|
| 38 |
+
from types import TracebackType
|
| 39 |
+
from typing import IO, Literal, NoReturn, Protocol, cast
|
| 40 |
+
|
| 41 |
+
__all__ = [
|
| 42 |
+
"Downloaded",
|
| 43 |
+
"FetchError",
|
| 44 |
+
"HeadersLike",
|
| 45 |
+
"Opener",
|
| 46 |
+
"Response",
|
| 47 |
+
"default_opener",
|
| 48 |
+
"download",
|
| 49 |
+
"hf_resolve_url",
|
| 50 |
+
"stream_extract",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
logger = logging.getLogger(__name__)
|
| 54 |
+
|
| 55 |
+
_USER_AGENT = "nida-bench/0"
|
| 56 |
+
_SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
| 57 |
+
_COMMIT_SHA_RE = re.compile(r"[0-9a-f]{40}")
|
| 58 |
+
_CONTENT_RANGE_RE = re.compile(r"bytes (\d+)-(\d+)/(\d+|\*)")
|
| 59 |
+
_DRIVE_LETTER_RE = re.compile(r"[A-Za-z]:")
|
| 60 |
+
_MAX_BACKOFF_S = 30.0
|
| 61 |
+
_COPY_CHUNK_BYTES = 1 << 20
|
| 62 |
+
# HTTP 4xx codes that are worth retrying; every other 4xx is a permanent client error.
|
| 63 |
+
_RETRYABLE_4XX = frozenset({408, 429})
|
| 64 |
+
|
| 65 |
+
# Transient network failures. ``HTTPError`` is a ``URLError`` and is classified separately.
|
| 66 |
+
_NETWORK_ERRORS: tuple[type[BaseException], ...] = (
|
| 67 |
+
urllib.error.URLError,
|
| 68 |
+
ConnectionError,
|
| 69 |
+
TimeoutError,
|
| 70 |
+
http.client.HTTPException, # includes IncompleteRead
|
| 71 |
+
ssl.SSLError,
|
| 72 |
+
)
|
| 73 |
+
# Everything that can go wrong while reading an archive stream (network or corrupt data).
|
| 74 |
+
_STREAM_ERRORS: tuple[type[BaseException], ...] = (
|
| 75 |
+
*_NETWORK_ERRORS,
|
| 76 |
+
tarfile.TarError,
|
| 77 |
+
EOFError,
|
| 78 |
+
zlib.error,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class FetchError(RuntimeError):
|
| 83 |
+
"""A download or extraction failed; the message carries URL/path and byte counts."""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class _TransientError(Exception):
|
| 87 |
+
"""Internal: this attempt failed but a resumed attempt may succeed."""
|
| 88 |
+
|
| 89 |
+
def __init__(self, message: str, *, expected_bytes: int | None = None) -> None:
|
| 90 |
+
super().__init__(message)
|
| 91 |
+
self.expected_bytes = expected_bytes
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class HeadersLike(Protocol):
|
| 95 |
+
"""Response headers: only ``get`` is used (``http.client.HTTPMessage`` satisfies it)."""
|
| 96 |
+
|
| 97 |
+
def get(self, name: str, default: str | None = None, /) -> str | None:
|
| 98 |
+
"""Return the header value, or ``default`` when absent."""
|
| 99 |
+
...
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class Response(Protocol):
|
| 103 |
+
"""The slice of an HTTP response this module relies on.
|
| 104 |
+
|
| 105 |
+
``http.client.HTTPResponse`` (what ``urllib.request.urlopen`` returns) satisfies it.
|
| 106 |
+
"""
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def status(self) -> int:
|
| 110 |
+
"""HTTP status code."""
|
| 111 |
+
...
|
| 112 |
+
|
| 113 |
+
@property
|
| 114 |
+
def headers(self) -> HeadersLike:
|
| 115 |
+
"""Response headers."""
|
| 116 |
+
...
|
| 117 |
+
|
| 118 |
+
def read(self, amt: int = -1, /) -> bytes:
|
| 119 |
+
"""Read up to ``amt`` bytes; ``b""`` means end of stream."""
|
| 120 |
+
...
|
| 121 |
+
|
| 122 |
+
def __enter__(self) -> Response:
|
| 123 |
+
"""Enter the context."""
|
| 124 |
+
...
|
| 125 |
+
|
| 126 |
+
def __exit__(
|
| 127 |
+
self,
|
| 128 |
+
exc_type: type[BaseException] | None,
|
| 129 |
+
exc: BaseException | None,
|
| 130 |
+
tb: TracebackType | None,
|
| 131 |
+
/,
|
| 132 |
+
) -> object:
|
| 133 |
+
"""Close the response."""
|
| 134 |
+
...
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
Opener = Callable[[urllib.request.Request, float], Response]
|
| 138 |
+
"""Callable performing a request with a timeout in seconds and returning a response."""
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def default_opener(request: urllib.request.Request, timeout_s: float) -> Response:
|
| 142 |
+
"""Open ``request`` with :func:`urllib.request.urlopen`.
|
| 143 |
+
|
| 144 |
+
Args:
|
| 145 |
+
request: The prepared request.
|
| 146 |
+
timeout_s: Socket timeout in seconds.
|
| 147 |
+
|
| 148 |
+
Returns:
|
| 149 |
+
The open HTTP response (a context manager).
|
| 150 |
+
"""
|
| 151 |
+
response: Response = urllib.request.urlopen(request, timeout=timeout_s)
|
| 152 |
+
return response
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@dataclass(frozen=True)
|
| 156 |
+
class Downloaded:
|
| 157 |
+
"""A completed, verified download.
|
| 158 |
+
|
| 159 |
+
Attributes:
|
| 160 |
+
path: Final file path.
|
| 161 |
+
size_bytes: File size in bytes.
|
| 162 |
+
sha256: Lowercase hex SHA-256 computed over the final file.
|
| 163 |
+
url: Source URL.
|
| 164 |
+
"""
|
| 165 |
+
|
| 166 |
+
path: Path
|
| 167 |
+
size_bytes: int
|
| 168 |
+
sha256: str
|
| 169 |
+
url: str
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
# --------------------------------------------------------------------------------------
|
| 173 |
+
# download
|
| 174 |
+
# --------------------------------------------------------------------------------------
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def download(
|
| 178 |
+
url: str,
|
| 179 |
+
dest: Path,
|
| 180 |
+
*,
|
| 181 |
+
expected_sha256: str | None = None,
|
| 182 |
+
expected_size: int | None = None,
|
| 183 |
+
opener: Opener = default_opener,
|
| 184 |
+
chunk_bytes: int = 1 << 20,
|
| 185 |
+
timeout_s: float = 60.0,
|
| 186 |
+
max_attempts: int = 5,
|
| 187 |
+
sleep: Callable[[float], None] = time.sleep,
|
| 188 |
+
) -> Downloaded:
|
| 189 |
+
"""Download ``url`` to ``dest``, resuming and retrying as needed.
|
| 190 |
+
|
| 191 |
+
If ``dest`` already exists it is verified (size and/or SHA-256 when given) and returned
|
| 192 |
+
without any network access; it is never overwritten. Otherwise the data is written to
|
| 193 |
+
``<dest>.part`` and resumed with ``Range: bytes=<n>-`` on every retry. A server that
|
| 194 |
+
answers 200 instead of 206 to a ranged request makes the download restart from zero.
|
| 195 |
+
Transient failures (``URLError``, ``ConnectionError``, ``TimeoutError``, short reads,
|
| 196 |
+
HTTP 5xx/408/429) are retried with backoff of 1, 2, 4, ... seconds (capped at 30). Other
|
| 197 |
+
HTTP 4xx errors fail immediately. When all attempts are exhausted the ``.part`` file is
|
| 198 |
+
kept so a later call can resume it.
|
| 199 |
+
|
| 200 |
+
The SHA-256 is computed over the final file (re-read once at the end), so it also
|
| 201 |
+
covers bytes written by earlier, interrupted attempts.
|
| 202 |
+
|
| 203 |
+
Concurrent downloads of the same ``dest`` are not supported.
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
url: ``http://`` or ``https://`` URL.
|
| 207 |
+
dest: Final file path; parent directories are created.
|
| 208 |
+
expected_sha256: Expected lowercase/uppercase hex SHA-256 (64 chars), if known.
|
| 209 |
+
expected_size: Expected size in bytes, if known.
|
| 210 |
+
opener: Network access function (injectable for tests).
|
| 211 |
+
chunk_bytes: Read chunk size in bytes.
|
| 212 |
+
timeout_s: Socket timeout per request, seconds.
|
| 213 |
+
max_attempts: Total number of attempts (first try included), at least 1.
|
| 214 |
+
sleep: Sleep function used for backoff between attempts.
|
| 215 |
+
|
| 216 |
+
Returns:
|
| 217 |
+
The verified :class:`Downloaded` record.
|
| 218 |
+
|
| 219 |
+
Raises:
|
| 220 |
+
ValueError: On invalid arguments (bad URL scheme, malformed hash, ...).
|
| 221 |
+
FetchError: On verification failure, permanent HTTP errors or exhausted attempts.
|
| 222 |
+
OSError: On local filesystem failures (e.g. disk full).
|
| 223 |
+
"""
|
| 224 |
+
_require_http_url(url)
|
| 225 |
+
sha_expected = _normalise_sha256(expected_sha256)
|
| 226 |
+
if expected_size is not None and expected_size < 0:
|
| 227 |
+
raise ValueError(f"expected_size must be >= 0, got {expected_size}")
|
| 228 |
+
if chunk_bytes < 1:
|
| 229 |
+
raise ValueError(f"chunk_bytes must be >= 1, got {chunk_bytes}")
|
| 230 |
+
if max_attempts < 1:
|
| 231 |
+
raise ValueError(f"max_attempts must be >= 1, got {max_attempts}")
|
| 232 |
+
if timeout_s <= 0:
|
| 233 |
+
raise ValueError(f"timeout_s must be > 0, got {timeout_s}")
|
| 234 |
+
|
| 235 |
+
if dest.exists():
|
| 236 |
+
return _verify_existing(url, dest, sha_expected, expected_size)
|
| 237 |
+
|
| 238 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 239 |
+
part = dest.with_name(dest.name + ".part")
|
| 240 |
+
|
| 241 |
+
last_error: _TransientError | None = None
|
| 242 |
+
known_total = expected_size # best knowledge of the final size, for error messages
|
| 243 |
+
for attempt in range(1, max_attempts + 1):
|
| 244 |
+
try:
|
| 245 |
+
_attempt(url, part, expected_size, opener, chunk_bytes, timeout_s)
|
| 246 |
+
except _TransientError as exc:
|
| 247 |
+
last_error = exc
|
| 248 |
+
if exc.expected_bytes is not None:
|
| 249 |
+
known_total = exc.expected_bytes
|
| 250 |
+
part_bytes = part.stat().st_size if part.exists() else 0
|
| 251 |
+
logger.warning(
|
| 252 |
+
"download %s attempt %d/%d failed after %d bytes: %s",
|
| 253 |
+
url,
|
| 254 |
+
attempt,
|
| 255 |
+
max_attempts,
|
| 256 |
+
part_bytes,
|
| 257 |
+
exc,
|
| 258 |
+
)
|
| 259 |
+
if attempt < max_attempts:
|
| 260 |
+
sleep(min(2.0 ** (attempt - 1), _MAX_BACKOFF_S))
|
| 261 |
+
continue
|
| 262 |
+
last_error = None
|
| 263 |
+
break
|
| 264 |
+
|
| 265 |
+
if last_error is not None:
|
| 266 |
+
part_bytes = part.stat().st_size if part.exists() else 0
|
| 267 |
+
raise FetchError(
|
| 268 |
+
f"download of {url} failed after {max_attempts} attempts: {part_bytes} bytes "
|
| 269 |
+
f"received in {part} (expected "
|
| 270 |
+
f"{known_total if known_total is not None else 'unknown'}); last error: {last_error}"
|
| 271 |
+
) from last_error
|
| 272 |
+
|
| 273 |
+
size_bytes = part.stat().st_size
|
| 274 |
+
digest = _sha256_file(part)
|
| 275 |
+
if sha_expected is not None and digest != sha_expected:
|
| 276 |
+
part.unlink()
|
| 277 |
+
raise FetchError(
|
| 278 |
+
f"sha256 mismatch for {url}: got {digest}, expected {sha_expected} "
|
| 279 |
+
f"({size_bytes} bytes; partial file deleted)"
|
| 280 |
+
)
|
| 281 |
+
part.replace(dest)
|
| 282 |
+
logger.info("downloaded %s -> %s (%d bytes, sha256 %s)", url, dest, size_bytes, digest)
|
| 283 |
+
return Downloaded(path=dest, size_bytes=size_bytes, sha256=digest, url=url)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _require_http_url(url: str) -> None:
|
| 287 |
+
if not url.lower().startswith(("http://", "https://")):
|
| 288 |
+
raise ValueError(f"only http(s) URLs are supported, got {url!r}")
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _normalise_sha256(value: str | None) -> str | None:
|
| 292 |
+
if value is None:
|
| 293 |
+
return None
|
| 294 |
+
lowered = value.strip().lower()
|
| 295 |
+
if _SHA256_RE.fullmatch(lowered) is None:
|
| 296 |
+
raise ValueError(f"expected_sha256 must be 64 hex characters, got {value!r}")
|
| 297 |
+
return lowered
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
def _sha256_file(path: Path) -> str:
|
| 301 |
+
digest = hashlib.sha256()
|
| 302 |
+
with path.open("rb") as handle:
|
| 303 |
+
while chunk := handle.read(_COPY_CHUNK_BYTES):
|
| 304 |
+
digest.update(chunk)
|
| 305 |
+
return digest.hexdigest()
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def _verify_existing(
|
| 309 |
+
url: str, dest: Path, expected_sha256: str | None, expected_size: int | None
|
| 310 |
+
) -> Downloaded:
|
| 311 |
+
if not dest.is_file():
|
| 312 |
+
raise FetchError(f"destination {dest} exists but is not a regular file (url {url})")
|
| 313 |
+
size_bytes = dest.stat().st_size
|
| 314 |
+
if expected_size is not None and size_bytes != expected_size:
|
| 315 |
+
raise FetchError(
|
| 316 |
+
f"existing file {dest} has {size_bytes} bytes, expected {expected_size} "
|
| 317 |
+
f"(url {url}); refusing to overwrite, delete it manually to re-download"
|
| 318 |
+
)
|
| 319 |
+
digest = _sha256_file(dest)
|
| 320 |
+
if expected_sha256 is not None and digest != expected_sha256:
|
| 321 |
+
raise FetchError(
|
| 322 |
+
f"existing file {dest} has sha256 {digest}, expected {expected_sha256} "
|
| 323 |
+
f"(url {url}); refusing to overwrite, delete it manually to re-download"
|
| 324 |
+
)
|
| 325 |
+
logger.info("download skipped, %s already present (%d bytes)", dest, size_bytes)
|
| 326 |
+
return Downloaded(path=dest, size_bytes=size_bytes, sha256=digest, url=url)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _attempt(
|
| 330 |
+
url: str,
|
| 331 |
+
part: Path,
|
| 332 |
+
expected_size: int | None,
|
| 333 |
+
opener: Opener,
|
| 334 |
+
chunk_bytes: int,
|
| 335 |
+
timeout_s: float,
|
| 336 |
+
) -> None:
|
| 337 |
+
"""Run one (possibly resumed) transfer into ``part``.
|
| 338 |
+
|
| 339 |
+
Returns normally when ``part`` holds the complete body. Raises ``_TransientError`` when
|
| 340 |
+
a resumed attempt may help and ``FetchError`` for permanent problems.
|
| 341 |
+
"""
|
| 342 |
+
offset = part.stat().st_size if part.exists() else 0
|
| 343 |
+
if expected_size is not None:
|
| 344 |
+
if offset > expected_size:
|
| 345 |
+
logger.warning("%s is larger than expected size %d, restarting", part, expected_size)
|
| 346 |
+
part.unlink()
|
| 347 |
+
offset = 0
|
| 348 |
+
elif offset == expected_size:
|
| 349 |
+
return # everything is already on disk; only verification is left
|
| 350 |
+
|
| 351 |
+
headers = {"User-Agent": _USER_AGENT}
|
| 352 |
+
if offset > 0:
|
| 353 |
+
headers["Range"] = f"bytes={offset}-"
|
| 354 |
+
request = urllib.request.Request(url, headers=headers)
|
| 355 |
+
|
| 356 |
+
try:
|
| 357 |
+
response = opener(request, timeout_s)
|
| 358 |
+
except urllib.error.HTTPError as exc:
|
| 359 |
+
code = exc.code
|
| 360 |
+
exc.close()
|
| 361 |
+
_raise_for_http_status(url, part, code, offset)
|
| 362 |
+
except _NETWORK_ERRORS as exc:
|
| 363 |
+
raise _TransientError(f"connecting to {url} failed: {exc!r}") from exc
|
| 364 |
+
|
| 365 |
+
with response:
|
| 366 |
+
status = response.status
|
| 367 |
+
if status not in (200, 206):
|
| 368 |
+
raise FetchError(f"unexpected HTTP status {status} from {url}")
|
| 369 |
+
if status == 200 and offset > 0:
|
| 370 |
+
logger.warning("server ignored Range for %s, restarting from byte 0", url)
|
| 371 |
+
offset = 0
|
| 372 |
+
total = _expected_total(url, response, status, offset)
|
| 373 |
+
if total is not None and expected_size is not None and total != expected_size:
|
| 374 |
+
part.unlink(missing_ok=True)
|
| 375 |
+
raise FetchError(
|
| 376 |
+
f"{url} is {total} bytes according to the server, expected {expected_size}"
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
position = offset
|
| 380 |
+
mode = "ab" if (status == 206 and offset > 0) else "wb"
|
| 381 |
+
with part.open(mode) as handle:
|
| 382 |
+
try:
|
| 383 |
+
while chunk := response.read(chunk_bytes):
|
| 384 |
+
handle.write(chunk)
|
| 385 |
+
position += len(chunk)
|
| 386 |
+
if total is not None and position > total:
|
| 387 |
+
break
|
| 388 |
+
except _NETWORK_ERRORS as exc:
|
| 389 |
+
raise _TransientError(
|
| 390 |
+
f"reading {url} failed at byte {position}: {exc!r}",
|
| 391 |
+
expected_bytes=total,
|
| 392 |
+
) from exc
|
| 393 |
+
handle.flush()
|
| 394 |
+
os.fsync(handle.fileno())
|
| 395 |
+
|
| 396 |
+
if total is not None and position != total:
|
| 397 |
+
if position > total:
|
| 398 |
+
part.unlink()
|
| 399 |
+
raise _TransientError(
|
| 400 |
+
f"{url} sent more than its Content-Length: {position} bytes, expected {total}",
|
| 401 |
+
expected_bytes=total,
|
| 402 |
+
)
|
| 403 |
+
raise _TransientError(
|
| 404 |
+
f"incomplete body from {url}: {position} bytes, expected {total}",
|
| 405 |
+
expected_bytes=total,
|
| 406 |
+
)
|
| 407 |
+
if expected_size is not None and position != expected_size:
|
| 408 |
+
if position > expected_size:
|
| 409 |
+
part.unlink()
|
| 410 |
+
raise FetchError(
|
| 411 |
+
f"{url} delivered {position} bytes, expected {expected_size} (partial file deleted)"
|
| 412 |
+
)
|
| 413 |
+
raise _TransientError(
|
| 414 |
+
f"stream from {url} ended early: {position} bytes, expected {expected_size}",
|
| 415 |
+
expected_bytes=expected_size,
|
| 416 |
+
)
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
def _raise_for_http_status(url: str, part: Path, code: int, offset: int) -> NoReturn:
|
| 420 |
+
if code == 416 and offset > 0:
|
| 421 |
+
# The stale .part is at or beyond the end of the remote file: start over.
|
| 422 |
+
part.unlink(missing_ok=True)
|
| 423 |
+
raise _TransientError(f"HTTP 416 for ranged request at byte {offset}; restarting")
|
| 424 |
+
if 400 <= code < 500 and code not in _RETRYABLE_4XX:
|
| 425 |
+
raise FetchError(f"HTTP {code} for {url}")
|
| 426 |
+
raise _TransientError(f"HTTP {code} for {url}")
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def _parse_length(url: str, name: str, value: str) -> int:
|
| 430 |
+
try:
|
| 431 |
+
parsed = int(value)
|
| 432 |
+
except ValueError:
|
| 433 |
+
raise FetchError(f"malformed {name} header from {url}: {value!r}") from None
|
| 434 |
+
if parsed < 0:
|
| 435 |
+
raise FetchError(f"malformed {name} header from {url}: {value!r}")
|
| 436 |
+
return parsed
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def _expected_total(url: str, response: Response, status: int, offset: int) -> int | None:
|
| 440 |
+
"""Return the total file size implied by the response headers, if the server says."""
|
| 441 |
+
length_header = response.headers.get("Content-Length")
|
| 442 |
+
length = _parse_length(url, "Content-Length", length_header) if length_header else None
|
| 443 |
+
if status == 200:
|
| 444 |
+
return length
|
| 445 |
+
range_header = response.headers.get("Content-Range")
|
| 446 |
+
if range_header is None:
|
| 447 |
+
raise FetchError(f"206 response from {url} has no Content-Range header")
|
| 448 |
+
match = _CONTENT_RANGE_RE.fullmatch(range_header.strip())
|
| 449 |
+
if match is None:
|
| 450 |
+
raise FetchError(f"malformed Content-Range from {url}: {range_header!r}")
|
| 451 |
+
start = int(match.group(1))
|
| 452 |
+
if start != offset:
|
| 453 |
+
raise FetchError(
|
| 454 |
+
f"server returned bytes starting at {start} from {url}, requested {offset}"
|
| 455 |
+
)
|
| 456 |
+
if match.group(3) != "*":
|
| 457 |
+
return int(match.group(3))
|
| 458 |
+
return offset + length if length is not None else None
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
# --------------------------------------------------------------------------------------
|
| 462 |
+
# stream_extract
|
| 463 |
+
# --------------------------------------------------------------------------------------
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
def stream_extract(
|
| 467 |
+
url_or_path: str | Path,
|
| 468 |
+
dest_dir: Path,
|
| 469 |
+
select: Callable[[str], bool],
|
| 470 |
+
*,
|
| 471 |
+
opener: Opener = default_opener,
|
| 472 |
+
timeout_s: float = 60.0,
|
| 473 |
+
) -> list[Path]:
|
| 474 |
+
"""Extract selected regular files from a tar archive while streaming it.
|
| 475 |
+
|
| 476 |
+
The archive (plain, gzip, bz2 or xz; auto-detected) is read sequentially from a URL or a
|
| 477 |
+
local path and is never stored. A stream cannot be resumed mid-archive: if reading fails
|
| 478 |
+
the call raises :class:`FetchError` and must simply be run again. Files extracted before
|
| 479 |
+
the failure are kept and skipped quickly on the rerun (the archive is read again from
|
| 480 |
+
the start, so the whole transfer is repeated).
|
| 481 |
+
|
| 482 |
+
For every regular-file member the name is normalised (leading ``./`` and empty or ``.``
|
| 483 |
+
components removed) and validated. Unsafe names (empty, absolute, containing ``..``
|
| 484 |
+
components, backslashes, NUL bytes or a Windows drive prefix) raise
|
| 485 |
+
:class:`FetchError` even when ``select`` would reject them: an archive with such members
|
| 486 |
+
is treated as hostile. Selected members are written to ``dest_dir / name`` through a
|
| 487 |
+
temporary file in the same directory and an atomic rename. A target that already exists
|
| 488 |
+
with the member's size is left untouched. Non-regular members (directories, symlinks,
|
| 489 |
+
hard links, devices) are never written; those that ``select`` accepts are counted in the
|
| 490 |
+
log message. If a name occurs twice in the archive the last occurrence wins, as with
|
| 491 |
+
``tar``.
|
| 492 |
+
|
| 493 |
+
Args:
|
| 494 |
+
url_or_path: ``http(s)://`` URL (fetched with ``opener``) or a local file path.
|
| 495 |
+
dest_dir: Output directory; created if missing.
|
| 496 |
+
select: Predicate on the normalised member name (``/``-separated, relative).
|
| 497 |
+
opener: Network access function (injectable for tests); unused for local paths.
|
| 498 |
+
timeout_s: Socket timeout per request, seconds.
|
| 499 |
+
|
| 500 |
+
Returns:
|
| 501 |
+
Paths of the extracted or already present files, in archive order, without
|
| 502 |
+
duplicates.
|
| 503 |
+
|
| 504 |
+
Raises:
|
| 505 |
+
FetchError: On unsafe member names, unreadable/corrupt/truncated archives and network
|
| 506 |
+
failures.
|
| 507 |
+
OSError: On local filesystem failures (e.g. disk full).
|
| 508 |
+
"""
|
| 509 |
+
if timeout_s <= 0:
|
| 510 |
+
raise ValueError(f"timeout_s must be > 0, got {timeout_s}")
|
| 511 |
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
| 512 |
+
root = dest_dir.resolve()
|
| 513 |
+
label = str(url_or_path)
|
| 514 |
+
|
| 515 |
+
result: dict[str, Path] = {} # normalised name -> path; insertion order == archive order
|
| 516 |
+
written_this_run: set[str] = set()
|
| 517 |
+
scanned = 0
|
| 518 |
+
extracted = 0
|
| 519 |
+
already_present = 0
|
| 520 |
+
ignored_non_regular = 0
|
| 521 |
+
|
| 522 |
+
with _open_source(url_or_path, opener, timeout_s) as stream:
|
| 523 |
+
try:
|
| 524 |
+
# tarfile in "r|*" mode only ever calls ``stream.read``; urllib responses are not
|
| 525 |
+
# nominally IO[bytes], hence the cast.
|
| 526 |
+
with tarfile.open(fileobj=cast(IO[bytes], stream), mode="r|*") as archive:
|
| 527 |
+
for member in archive:
|
| 528 |
+
scanned += 1
|
| 529 |
+
if not member.isreg():
|
| 530 |
+
if select(_collapse_name(member.name)):
|
| 531 |
+
ignored_non_regular += 1
|
| 532 |
+
_forget_members(archive)
|
| 533 |
+
continue
|
| 534 |
+
name = _safe_member_name(member.name)
|
| 535 |
+
if select(name):
|
| 536 |
+
target = _resolve_target(root, dest_dir, name)
|
| 537 |
+
if (
|
| 538 |
+
name not in written_this_run
|
| 539 |
+
and target.is_file()
|
| 540 |
+
and target.stat().st_size == member.size
|
| 541 |
+
):
|
| 542 |
+
already_present += 1
|
| 543 |
+
else:
|
| 544 |
+
source = archive.extractfile(member)
|
| 545 |
+
if source is None:
|
| 546 |
+
raise FetchError(f"cannot read member {name!r} of archive {label}")
|
| 547 |
+
_write_member_atomically(source, target, member.size, name, label)
|
| 548 |
+
written_this_run.add(name)
|
| 549 |
+
extracted += 1
|
| 550 |
+
result.setdefault(name, target)
|
| 551 |
+
# Streaming mode would otherwise keep every TarInfo alive.
|
| 552 |
+
_forget_members(archive)
|
| 553 |
+
except _STREAM_ERRORS as exc:
|
| 554 |
+
raise FetchError(
|
| 555 |
+
f"reading archive {label} failed after {scanned} members "
|
| 556 |
+
f"({extracted} extracted, {already_present} already present): {exc!r}; "
|
| 557 |
+
"re-run to continue (archives cannot be resumed mid-stream)"
|
| 558 |
+
) from exc
|
| 559 |
+
|
| 560 |
+
logger.info(
|
| 561 |
+
"stream_extract %s: scanned %d members, extracted %d, already present %d, "
|
| 562 |
+
"ignored %d selected non-regular members (directories/links/devices)",
|
| 563 |
+
label,
|
| 564 |
+
scanned,
|
| 565 |
+
extracted,
|
| 566 |
+
already_present,
|
| 567 |
+
ignored_non_regular,
|
| 568 |
+
)
|
| 569 |
+
return list(result.values())
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def _forget_members(archive: tarfile.TarFile) -> None:
|
| 573 |
+
"""Drop ``TarFile``'s per-member cache; streaming a huge archive would otherwise grow it."""
|
| 574 |
+
# ``members`` is a runtime attribute that typeshed does not declare.
|
| 575 |
+
archive.members = [] # type: ignore[attr-defined]
|
| 576 |
+
|
| 577 |
+
|
| 578 |
+
@contextmanager
|
| 579 |
+
def _open_source(
|
| 580 |
+
url_or_path: str | Path, opener: Opener, timeout_s: float
|
| 581 |
+
) -> Iterator[Response | IO[bytes]]:
|
| 582 |
+
if isinstance(url_or_path, str) and url_or_path.lower().startswith(("http://", "https://")):
|
| 583 |
+
request = urllib.request.Request(url_or_path, headers={"User-Agent": _USER_AGENT})
|
| 584 |
+
try:
|
| 585 |
+
response = opener(request, timeout_s)
|
| 586 |
+
except _NETWORK_ERRORS as exc: # includes HTTPError
|
| 587 |
+
raise FetchError(f"opening {url_or_path} failed: {exc!r}") from exc
|
| 588 |
+
with response:
|
| 589 |
+
if response.status != 200:
|
| 590 |
+
raise FetchError(f"unexpected HTTP status {response.status} from {url_or_path}")
|
| 591 |
+
yield response
|
| 592 |
+
else:
|
| 593 |
+
path = Path(url_or_path)
|
| 594 |
+
if not path.is_file():
|
| 595 |
+
raise FetchError(f"archive {path} does not exist or is not a file")
|
| 596 |
+
with path.open("rb") as handle:
|
| 597 |
+
yield handle
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
def _collapse_name(name: str) -> str:
|
| 601 |
+
"""Drop leading ``./`` and empty/``.`` components; performs no safety checks."""
|
| 602 |
+
return "/".join(part for part in name.split("/") if part not in ("", "."))
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
def _safe_member_name(raw_name: str) -> str:
|
| 606 |
+
"""Normalise a regular-file member name or raise ``FetchError`` if it is unsafe."""
|
| 607 |
+
reason: str | None = None
|
| 608 |
+
if "\x00" in raw_name:
|
| 609 |
+
reason = "contains a NUL byte"
|
| 610 |
+
elif "\\" in raw_name:
|
| 611 |
+
reason = "contains a backslash"
|
| 612 |
+
elif raw_name.startswith("/"):
|
| 613 |
+
reason = "is an absolute path"
|
| 614 |
+
elif _DRIVE_LETTER_RE.match(raw_name) is not None:
|
| 615 |
+
reason = "starts with a drive letter"
|
| 616 |
+
elif ".." in raw_name.split("/"):
|
| 617 |
+
reason = "contains a '..' component"
|
| 618 |
+
else:
|
| 619 |
+
name = _collapse_name(raw_name)
|
| 620 |
+
if name:
|
| 621 |
+
return name
|
| 622 |
+
reason = "is empty"
|
| 623 |
+
raise FetchError(f"unsafe tar member name {raw_name!r}: {reason}")
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def _resolve_target(root: Path, dest_dir: Path, name: str) -> Path:
|
| 627 |
+
"""Return ``dest_dir / name`` after checking that it stays inside ``dest_dir``."""
|
| 628 |
+
target = dest_dir.joinpath(*name.split("/"))
|
| 629 |
+
try:
|
| 630 |
+
target.resolve().relative_to(root)
|
| 631 |
+
except ValueError:
|
| 632 |
+
raise FetchError(f"tar member {name!r} would be written outside {dest_dir}") from None
|
| 633 |
+
return target
|
| 634 |
+
|
| 635 |
+
|
| 636 |
+
def _write_member_atomically(
|
| 637 |
+
source: IO[bytes], target: Path, expected_bytes: int, name: str, label: str
|
| 638 |
+
) -> None:
|
| 639 |
+
"""Copy ``source`` to ``target`` via a temp file in the same directory and a rename."""
|
| 640 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 641 |
+
fd, tmp_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".tmp", dir=target.parent)
|
| 642 |
+
tmp_path = Path(tmp_name)
|
| 643 |
+
try:
|
| 644 |
+
with os.fdopen(fd, "wb") as handle:
|
| 645 |
+
shutil.copyfileobj(source, handle, _COPY_CHUNK_BYTES)
|
| 646 |
+
written = handle.tell()
|
| 647 |
+
if written != expected_bytes:
|
| 648 |
+
raise FetchError(
|
| 649 |
+
f"member {name!r} of {label}: wrote {written} bytes, expected {expected_bytes}"
|
| 650 |
+
)
|
| 651 |
+
handle.flush()
|
| 652 |
+
os.fsync(handle.fileno())
|
| 653 |
+
tmp_path.replace(target)
|
| 654 |
+
except BaseException:
|
| 655 |
+
tmp_path.unlink(missing_ok=True)
|
| 656 |
+
raise
|
| 657 |
+
|
| 658 |
+
|
| 659 |
+
# --------------------------------------------------------------------------------------
|
| 660 |
+
# Hugging Face URLs
|
| 661 |
+
# --------------------------------------------------------------------------------------
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
def hf_resolve_url(
|
| 665 |
+
repo_id: str,
|
| 666 |
+
revision: str,
|
| 667 |
+
path_in_repo: str,
|
| 668 |
+
repo_type: Literal["model", "dataset"],
|
| 669 |
+
) -> str:
|
| 670 |
+
"""Build a revision-pinned Hugging Face ``resolve`` URL.
|
| 671 |
+
|
| 672 |
+
Args:
|
| 673 |
+
repo_id: Repository id such as ``"google/fleurs"``.
|
| 674 |
+
revision: Full 40-character lowercase hex commit SHA (branches and tags are refused
|
| 675 |
+
so the downloaded data is pinned).
|
| 676 |
+
path_in_repo: File path inside the repository (``/``-separated, relative).
|
| 677 |
+
repo_type: ``"dataset"`` or ``"model"``.
|
| 678 |
+
|
| 679 |
+
Returns:
|
| 680 |
+
``https://huggingface.co/[datasets/]<repo_id>/resolve/<revision>/<path>`` with every
|
| 681 |
+
path component percent-encoded (``/`` kept).
|
| 682 |
+
|
| 683 |
+
Raises:
|
| 684 |
+
ValueError: On a malformed revision, repo id or path, or an unknown repo type.
|
| 685 |
+
"""
|
| 686 |
+
if repo_type not in ("model", "dataset"):
|
| 687 |
+
raise ValueError(f"repo_type must be 'model' or 'dataset', got {repo_type!r}")
|
| 688 |
+
if _COMMIT_SHA_RE.fullmatch(revision) is None:
|
| 689 |
+
raise ValueError(f"revision must be a 40-character lowercase hex commit SHA: {revision!r}")
|
| 690 |
+
_require_clean_path("repo_id", repo_id)
|
| 691 |
+
_require_clean_path("path_in_repo", path_in_repo)
|
| 692 |
+
prefix = "datasets/" if repo_type == "dataset" else ""
|
| 693 |
+
quoted_repo = urllib.parse.quote(repo_id, safe="/")
|
| 694 |
+
quoted_path = urllib.parse.quote(path_in_repo, safe="/")
|
| 695 |
+
return f"https://huggingface.co/{prefix}{quoted_repo}/resolve/{revision}/{quoted_path}"
|
| 696 |
+
|
| 697 |
+
|
| 698 |
+
def _require_clean_path(what: str, value: str) -> None:
|
| 699 |
+
parts = value.split("/")
|
| 700 |
+
if not value or any(part in ("", ".", "..") for part in parts) or "\\" in value:
|
| 701 |
+
raise ValueError(
|
| 702 |
+
f"{what} must be a relative '/'-separated path without empty, '.' or '..' "
|
| 703 |
+
f"components: {value!r}"
|
| 704 |
+
)
|
src/nida/bench/panel.py
ADDED
|
@@ -0,0 +1,817 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Self-contained HTML listening panels and a strict parser for their CSV export.
|
| 2 |
+
|
| 3 |
+
A panel is a single HTML file that a human opens locally in a browser. It embeds the audio
|
| 4 |
+
(base64 data URIs), shows one card per item with per-item questions, keeps answers in
|
| 5 |
+
``localStorage`` and exports a CSV through a "CSV indir" button. The panel makes no network
|
| 6 |
+
requests of any kind.
|
| 7 |
+
|
| 8 |
+
The CSV layout is a contract between the in-page JavaScript (``buildCsv``) and
|
| 9 |
+
:func:`read_panel_results`::
|
| 10 |
+
|
| 11 |
+
panel_id,item_id,<question ids in spec order>,[text,]note
|
| 12 |
+
|
| 13 |
+
``text`` is present only if at least one item has ``editable_text``. The file is UTF-8 with
|
| 14 |
+
a BOM, fields are quoted per RFC 4180 and lines end with CRLF. Unanswered questions are
|
| 15 |
+
empty fields.
|
| 16 |
+
|
| 17 |
+
User-visible UI strings are Turkish; everything else is English.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import base64
|
| 23 |
+
import csv
|
| 24 |
+
import html
|
| 25 |
+
import io
|
| 26 |
+
import json
|
| 27 |
+
import re
|
| 28 |
+
from collections.abc import Mapping
|
| 29 |
+
from dataclasses import dataclass
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
from typing import Literal
|
| 32 |
+
|
| 33 |
+
from nida.text.turkish import nfc
|
| 34 |
+
from nida.utils.io import atomic_write_json, atomic_write_text
|
| 35 |
+
|
| 36 |
+
__all__ = [
|
| 37 |
+
"AudioRef",
|
| 38 |
+
"PanelItem",
|
| 39 |
+
"PanelResult",
|
| 40 |
+
"PanelSpec",
|
| 41 |
+
"Question",
|
| 42 |
+
"build_panel_html",
|
| 43 |
+
"csv_header",
|
| 44 |
+
"has_text_column",
|
| 45 |
+
"load_panel_spec",
|
| 46 |
+
"missing_required",
|
| 47 |
+
"read_panel_results",
|
| 48 |
+
"write_panel",
|
| 49 |
+
]
|
| 50 |
+
|
| 51 |
+
QuestionKind = Literal["choice", "scale", "text"]
|
| 52 |
+
|
| 53 |
+
_QUESTION_KINDS: tuple[str, ...] = ("choice", "scale", "text")
|
| 54 |
+
_QUESTION_ID_RE = re.compile(r"[a-z][a-z0-9_]{0,31}")
|
| 55 |
+
_ITEM_ID_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}")
|
| 56 |
+
_RESERVED_QUESTION_IDS = frozenset({"panel_id", "item_id", "text", "note"})
|
| 57 |
+
_AUDIO_MIME_BY_SUFFIX: Mapping[str, str] = {
|
| 58 |
+
".wav": "audio/wav",
|
| 59 |
+
".flac": "audio/flac",
|
| 60 |
+
".mp3": "audio/mpeg",
|
| 61 |
+
".ogg": "audio/ogg",
|
| 62 |
+
".opus": "audio/ogg",
|
| 63 |
+
}
|
| 64 |
+
_DEFAULT_MAX_BYTES = 60_000_000
|
| 65 |
+
_LINE_SEPARATOR = chr(0x2028)
|
| 66 |
+
_PARAGRAPH_SEPARATOR = chr(0x2029)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def audio_mime(path: Path) -> str:
|
| 70 |
+
"""Return the MIME type for an audio file, chosen by (case-insensitive) suffix.
|
| 71 |
+
|
| 72 |
+
Raises:
|
| 73 |
+
ValueError: if the suffix is not one of .wav, .flac, .mp3, .ogg, .opus.
|
| 74 |
+
"""
|
| 75 |
+
mime = _AUDIO_MIME_BY_SUFFIX.get(path.suffix.lower())
|
| 76 |
+
if mime is None:
|
| 77 |
+
supported = ", ".join(sorted(_AUDIO_MIME_BY_SUFFIX))
|
| 78 |
+
raise ValueError(
|
| 79 |
+
f"unsupported audio suffix {path.suffix!r} for {path} (supported: {supported})"
|
| 80 |
+
)
|
| 81 |
+
return mime
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@dataclass(frozen=True)
|
| 85 |
+
class AudioRef:
|
| 86 |
+
"""One audio clip shown on a card, with the label the listener sees."""
|
| 87 |
+
|
| 88 |
+
label: str
|
| 89 |
+
path: Path
|
| 90 |
+
|
| 91 |
+
def __post_init__(self) -> None:
|
| 92 |
+
if not self.label.strip():
|
| 93 |
+
raise ValueError(f"AudioRef for {self.path} has an empty label")
|
| 94 |
+
audio_mime(self.path) # validates the suffix
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
@dataclass(frozen=True)
|
| 98 |
+
class Question:
|
| 99 |
+
"""A per-item question.
|
| 100 |
+
|
| 101 |
+
``choice`` renders radio buttons from ``options``; ``scale`` does the same with the
|
| 102 |
+
options being the ordered allowed values (e.g. ``("1", "2", "3", "4", "5")``); ``text``
|
| 103 |
+
is a free-text box and takes no options.
|
| 104 |
+
"""
|
| 105 |
+
|
| 106 |
+
id: str
|
| 107 |
+
prompt: str
|
| 108 |
+
kind: QuestionKind
|
| 109 |
+
options: tuple[str, ...] = ()
|
| 110 |
+
required: bool = True
|
| 111 |
+
|
| 112 |
+
def __post_init__(self) -> None:
|
| 113 |
+
if _QUESTION_ID_RE.fullmatch(self.id) is None:
|
| 114 |
+
raise ValueError(f"question id {self.id!r} does not match {_QUESTION_ID_RE.pattern}")
|
| 115 |
+
if self.id in _RESERVED_QUESTION_IDS:
|
| 116 |
+
raise ValueError(f"question id {self.id!r} is reserved")
|
| 117 |
+
if not self.prompt.strip():
|
| 118 |
+
raise ValueError(f"question {self.id!r} has an empty prompt")
|
| 119 |
+
if self.kind not in _QUESTION_KINDS:
|
| 120 |
+
raise ValueError(f"question {self.id!r} has unknown kind {self.kind!r}")
|
| 121 |
+
if self.kind == "text":
|
| 122 |
+
if self.options:
|
| 123 |
+
raise ValueError(f"text question {self.id!r} must not have options")
|
| 124 |
+
return
|
| 125 |
+
if len(self.options) < 2:
|
| 126 |
+
raise ValueError(f"{self.kind} question {self.id!r} needs at least 2 options")
|
| 127 |
+
if any(option == "" for option in self.options):
|
| 128 |
+
raise ValueError(f"{self.kind} question {self.id!r} has an empty option")
|
| 129 |
+
if len(set(self.options)) != len(self.options):
|
| 130 |
+
raise ValueError(f"{self.kind} question {self.id!r} has duplicate options")
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@dataclass(frozen=True)
|
| 134 |
+
class PanelItem:
|
| 135 |
+
"""One card: audio clips plus optional (editable) text and a caption."""
|
| 136 |
+
|
| 137 |
+
item_id: str
|
| 138 |
+
audios: tuple[AudioRef, ...]
|
| 139 |
+
text: str = ""
|
| 140 |
+
editable_text: bool = False
|
| 141 |
+
info: str = ""
|
| 142 |
+
|
| 143 |
+
def __post_init__(self) -> None:
|
| 144 |
+
if _ITEM_ID_RE.fullmatch(self.item_id) is None:
|
| 145 |
+
raise ValueError(f"item id {self.item_id!r} does not match {_ITEM_ID_RE.pattern}")
|
| 146 |
+
if not self.audios:
|
| 147 |
+
raise ValueError(f"item {self.item_id!r} has no audio")
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
@dataclass(frozen=True)
|
| 151 |
+
class PanelSpec:
|
| 152 |
+
"""A whole panel: title, instructions, questions and items."""
|
| 153 |
+
|
| 154 |
+
panel_id: str
|
| 155 |
+
title: str
|
| 156 |
+
instructions: str
|
| 157 |
+
questions: tuple[Question, ...]
|
| 158 |
+
items: tuple[PanelItem, ...]
|
| 159 |
+
|
| 160 |
+
def __post_init__(self) -> None:
|
| 161 |
+
if _ITEM_ID_RE.fullmatch(self.panel_id) is None:
|
| 162 |
+
raise ValueError(f"panel id {self.panel_id!r} does not match {_ITEM_ID_RE.pattern}")
|
| 163 |
+
if not self.title.strip():
|
| 164 |
+
raise ValueError(f"panel {self.panel_id!r} has an empty title")
|
| 165 |
+
if not self.items:
|
| 166 |
+
raise ValueError(f"panel {self.panel_id!r} has no items")
|
| 167 |
+
_raise_on_duplicates([q.id for q in self.questions], "question id", self.panel_id)
|
| 168 |
+
_raise_on_duplicates([i.item_id for i in self.items], "item id", self.panel_id)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _raise_on_duplicates(values: list[str], what: str, panel_id: str) -> None:
|
| 172 |
+
seen: set[str] = set()
|
| 173 |
+
duplicates: list[str] = []
|
| 174 |
+
for value in values:
|
| 175 |
+
if value in seen and value not in duplicates:
|
| 176 |
+
duplicates.append(value)
|
| 177 |
+
seen.add(value)
|
| 178 |
+
if duplicates:
|
| 179 |
+
raise ValueError(f"panel {panel_id!r} has duplicate {what}s: {duplicates}")
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def has_text_column(spec: PanelSpec) -> bool:
|
| 183 |
+
"""Whether the CSV has a ``text`` column (any item has ``editable_text``)."""
|
| 184 |
+
return any(item.editable_text for item in spec.items)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def csv_header(spec: PanelSpec) -> list[str]:
|
| 188 |
+
"""The exact CSV header row produced by the panel for ``spec``."""
|
| 189 |
+
header = ["panel_id", "item_id", *(q.id for q in spec.questions)]
|
| 190 |
+
if has_text_column(spec):
|
| 191 |
+
header.append("text")
|
| 192 |
+
header.append("note")
|
| 193 |
+
return header
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
# --------------------------------------------------------------------------------------
|
| 197 |
+
# HTML generation
|
| 198 |
+
# --------------------------------------------------------------------------------------
|
| 199 |
+
|
| 200 |
+
_CSS = r"""
|
| 201 |
+
:root {
|
| 202 |
+
color-scheme: light dark;
|
| 203 |
+
--bg: #f6f6f3; --fg: #1b1b1a; --muted: #5d5d58; --card: #ffffff; --line: #d7d6cf;
|
| 204 |
+
--accent: #1d5fd0; --accent-fg: #ffffff; --ok: #1a7f3c; --warn: #b45309;
|
| 205 |
+
--field: #ffffff;
|
| 206 |
+
}
|
| 207 |
+
@media (prefers-color-scheme: dark) {
|
| 208 |
+
:root {
|
| 209 |
+
--bg: #151515; --fg: #ececea; --muted: #a3a39d; --card: #1f1f1e; --line: #3a3a37;
|
| 210 |
+
--accent: #6ea0ff; --accent-fg: #0b0b0b; --ok: #5fcf85; --warn: #f0b05a;
|
| 211 |
+
--field: #161616;
|
| 212 |
+
}
|
| 213 |
+
}
|
| 214 |
+
* { box-sizing: border-box; }
|
| 215 |
+
html { -webkit-text-size-adjust: 100%; }
|
| 216 |
+
body {
|
| 217 |
+
margin: 0; background: var(--bg); color: var(--fg);
|
| 218 |
+
font: 16px/1.5 system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
|
| 219 |
+
}
|
| 220 |
+
header.top { padding: 20px 16px 8px; max-width: 760px; margin: 0 auto; }
|
| 221 |
+
h1 { font-size: 1.4rem; margin: 0 0 8px; }
|
| 222 |
+
.instructions { white-space: pre-wrap; color: var(--muted); margin: 0 0 8px; }
|
| 223 |
+
.bar {
|
| 224 |
+
position: sticky; top: 0; z-index: 5; background: var(--bg);
|
| 225 |
+
border-bottom: 1px solid var(--line);
|
| 226 |
+
}
|
| 227 |
+
.bar-inner {
|
| 228 |
+
max-width: 760px; margin: 0 auto; padding: 10px 16px; display: flex; gap: 12px;
|
| 229 |
+
align-items: center; justify-content: space-between; flex-wrap: wrap;
|
| 230 |
+
}
|
| 231 |
+
#progress { font-weight: 600; }
|
| 232 |
+
button.primary {
|
| 233 |
+
min-height: 44px; padding: 0 18px; border: 0; border-radius: 10px; cursor: pointer;
|
| 234 |
+
background: var(--accent); color: var(--accent-fg); font: inherit; font-weight: 600;
|
| 235 |
+
}
|
| 236 |
+
main { max-width: 760px; margin: 0 auto; padding: 8px 16px 64px; }
|
| 237 |
+
.card {
|
| 238 |
+
background: var(--card); border: 1px solid var(--line); border-radius: 14px;
|
| 239 |
+
padding: 14px; margin: 14px 0;
|
| 240 |
+
}
|
| 241 |
+
.card.done { border-color: var(--ok); }
|
| 242 |
+
.card-head { display: flex; gap: 10px; align-items: baseline; flex-wrap: wrap; }
|
| 243 |
+
.num { font-weight: 700; font-size: 1.1rem; }
|
| 244 |
+
.item-id { color: var(--muted); font-size: 0.85rem; word-break: break-all; }
|
| 245 |
+
.info { color: var(--muted); font-size: 0.9rem; margin: 2px 0 8px; white-space: pre-wrap; }
|
| 246 |
+
.audio { margin: 10px 0; }
|
| 247 |
+
.alabel { font-size: 0.9rem; font-weight: 600; margin-bottom: 4px; }
|
| 248 |
+
audio { width: 100%; }
|
| 249 |
+
.text {
|
| 250 |
+
white-space: pre-wrap; margin: 10px 0; padding: 10px 12px; border-radius: 10px;
|
| 251 |
+
background: var(--bg); border: 1px solid var(--line);
|
| 252 |
+
}
|
| 253 |
+
fieldset.q { border: 0; margin: 14px 0 0; padding: 0; min-width: 0; }
|
| 254 |
+
legend, label.lbl { font-weight: 600; padding: 0; margin-bottom: 6px; display: block; }
|
| 255 |
+
.req { color: var(--warn); margin-left: 4px; }
|
| 256 |
+
.opts { display: flex; flex-wrap: wrap; gap: 8px; }
|
| 257 |
+
.opt { position: relative; }
|
| 258 |
+
.opt input {
|
| 259 |
+
position: absolute; opacity: 0; width: 100%; height: 100%; margin: 0; cursor: pointer;
|
| 260 |
+
}
|
| 261 |
+
.opt span {
|
| 262 |
+
display: inline-flex; align-items: center; justify-content: center; min-width: 44px;
|
| 263 |
+
min-height: 44px; padding: 0 14px; border: 1px solid var(--line); border-radius: 10px;
|
| 264 |
+
background: var(--field);
|
| 265 |
+
}
|
| 266 |
+
.opt input:checked + span {
|
| 267 |
+
background: var(--accent); color: var(--accent-fg); border-color: var(--accent);
|
| 268 |
+
}
|
| 269 |
+
.opt input:focus-visible + span { outline: 2px solid var(--accent); outline-offset: 2px; }
|
| 270 |
+
textarea {
|
| 271 |
+
width: 100%; font: inherit; color: var(--fg); background: var(--field);
|
| 272 |
+
border: 1px solid var(--line); border-radius: 10px; padding: 8px 10px; resize: vertical;
|
| 273 |
+
}
|
| 274 |
+
.field { margin-top: 14px; }
|
| 275 |
+
"""
|
| 276 |
+
|
| 277 |
+
_JS = r"""
|
| 278 |
+
(function () {
|
| 279 |
+
"use strict";
|
| 280 |
+
var spec = JSON.parse(document.getElementById("panel-data").textContent);
|
| 281 |
+
var STORAGE_KEY = "nida-panel:" + spec.panel_id;
|
| 282 |
+
|
| 283 |
+
/* BEGIN buildCsv */
|
| 284 |
+
function own(obj, key) {
|
| 285 |
+
return obj !== null && obj !== undefined &&
|
| 286 |
+
Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : undefined;
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
function buildCsv(spec, answers) {
|
| 290 |
+
function field(value) {
|
| 291 |
+
var s = value === undefined || value === null ? "" : String(value);
|
| 292 |
+
return /[",\r\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
|
| 293 |
+
}
|
| 294 |
+
var hasText = spec.items.some(function (it) { return it.editable_text; });
|
| 295 |
+
var header = ["panel_id", "item_id"];
|
| 296 |
+
spec.questions.forEach(function (q) { header.push(q.id); });
|
| 297 |
+
if (hasText) { header.push("text"); }
|
| 298 |
+
header.push("note");
|
| 299 |
+
var lines = [header.map(field).join(",")];
|
| 300 |
+
spec.items.forEach(function (it) {
|
| 301 |
+
var a = own(answers, it.item_id);
|
| 302 |
+
var row = [spec.panel_id, it.item_id];
|
| 303 |
+
spec.questions.forEach(function (q) { row.push(own(a, q.id)); });
|
| 304 |
+
if (hasText) {
|
| 305 |
+
var text = own(a, "text");
|
| 306 |
+
row.push(it.editable_text ? (text === undefined ? it.text : text) : "");
|
| 307 |
+
}
|
| 308 |
+
row.push(own(a, "note"));
|
| 309 |
+
lines.push(row.map(field).join(","));
|
| 310 |
+
});
|
| 311 |
+
return lines.join("\r\n") + "\r\n";
|
| 312 |
+
}
|
| 313 |
+
/* END buildCsv */
|
| 314 |
+
|
| 315 |
+
/* BEGIN missingRequired */
|
| 316 |
+
function missingRequired(spec, answers) {
|
| 317 |
+
var missing = [];
|
| 318 |
+
spec.items.forEach(function (it) {
|
| 319 |
+
var a = own(answers, it.item_id);
|
| 320 |
+
var absent = spec.questions.some(function (q) {
|
| 321 |
+
if (!q.required) { return false; }
|
| 322 |
+
var v = own(a, q.id);
|
| 323 |
+
return v === undefined || String(v).trim() === "";
|
| 324 |
+
});
|
| 325 |
+
if (absent) { missing.push(it.item_id); }
|
| 326 |
+
});
|
| 327 |
+
return missing;
|
| 328 |
+
}
|
| 329 |
+
/* END missingRequired */
|
| 330 |
+
|
| 331 |
+
var cards = Array.prototype.slice.call(document.querySelectorAll(".card"));
|
| 332 |
+
var hasRequired = spec.questions.some(function (q) { return q.required; });
|
| 333 |
+
|
| 334 |
+
function collect() {
|
| 335 |
+
var out = Object.create(null);
|
| 336 |
+
cards.forEach(function (card) {
|
| 337 |
+
var id = card.getAttribute("data-item");
|
| 338 |
+
var a = Object.create(null);
|
| 339 |
+
spec.questions.forEach(function (q) {
|
| 340 |
+
if (q.kind === "text") {
|
| 341 |
+
var ta = card.querySelector('textarea[data-q="' + q.id + '"]');
|
| 342 |
+
a[q.id] = ta ? ta.value : "";
|
| 343 |
+
} else {
|
| 344 |
+
var r = card.querySelector('input[data-q="' + q.id + '"]:checked');
|
| 345 |
+
a[q.id] = r ? r.value : "";
|
| 346 |
+
}
|
| 347 |
+
});
|
| 348 |
+
var t = card.querySelector('textarea[data-field="text"]');
|
| 349 |
+
if (t) { a.text = t.value; }
|
| 350 |
+
var n = card.querySelector('textarea[data-field="note"]');
|
| 351 |
+
a.note = n ? n.value : "";
|
| 352 |
+
out[id] = a;
|
| 353 |
+
});
|
| 354 |
+
return out;
|
| 355 |
+
}
|
| 356 |
+
|
| 357 |
+
function restore(saved) {
|
| 358 |
+
if (saved === null || typeof saved !== "object") { return; }
|
| 359 |
+
cards.forEach(function (card) {
|
| 360 |
+
var a = own(saved, card.getAttribute("data-item"));
|
| 361 |
+
if (a === null || typeof a !== "object") { return; }
|
| 362 |
+
spec.questions.forEach(function (q) {
|
| 363 |
+
var v = own(a, q.id);
|
| 364 |
+
if (typeof v !== "string") { return; }
|
| 365 |
+
if (q.kind === "text") {
|
| 366 |
+
var ta = card.querySelector('textarea[data-q="' + q.id + '"]');
|
| 367 |
+
if (ta) { ta.value = v; }
|
| 368 |
+
} else {
|
| 369 |
+
var radios = card.querySelectorAll('input[data-q="' + q.id + '"]');
|
| 370 |
+
Array.prototype.forEach.call(radios, function (r) { r.checked = (r.value === v); });
|
| 371 |
+
}
|
| 372 |
+
});
|
| 373 |
+
var t = card.querySelector('textarea[data-field="text"]');
|
| 374 |
+
if (t && typeof own(a, "text") === "string") { t.value = a.text; }
|
| 375 |
+
var n = card.querySelector('textarea[data-field="note"]');
|
| 376 |
+
if (n && typeof own(a, "note") === "string") { n.value = a.note; }
|
| 377 |
+
});
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
function isComplete(item, a) {
|
| 381 |
+
if (missingRequired({questions: spec.questions, items: [item]}, {[item.item_id]: a}).length) {
|
| 382 |
+
return false;
|
| 383 |
+
}
|
| 384 |
+
if (hasRequired) { return true; }
|
| 385 |
+
var touched = spec.questions.some(function (q) { return String(own(a, q.id) || "") !== ""; });
|
| 386 |
+
return touched || String(own(a, "note") || "") !== "" ||
|
| 387 |
+
(item.editable_text && own(a, "text") !== item.text);
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
function refresh() {
|
| 391 |
+
var answers = collect();
|
| 392 |
+
var done = 0;
|
| 393 |
+
spec.items.forEach(function (item, i) {
|
| 394 |
+
var ok = isComplete(item, own(answers, item.item_id));
|
| 395 |
+
cards[i].classList.toggle("done", ok);
|
| 396 |
+
if (ok) { done += 1; }
|
| 397 |
+
});
|
| 398 |
+
document.getElementById("progress").textContent =
|
| 399 |
+
done + " / " + spec.items.length + " tamamlandı";
|
| 400 |
+
try {
|
| 401 |
+
window.localStorage.setItem(STORAGE_KEY, JSON.stringify(answers));
|
| 402 |
+
} catch (e) { /* storage unavailable */ }
|
| 403 |
+
}
|
| 404 |
+
|
| 405 |
+
function download() {
|
| 406 |
+
var answers = collect();
|
| 407 |
+
var missing = missingRequired(spec, answers);
|
| 408 |
+
if (missing.length > 0) {
|
| 409 |
+
var msg = missing.length + " öğede zorunlu sorular yanıtsız. Yine de CSV indirilsin mi?";
|
| 410 |
+
if (!window.confirm(msg)) { return; }
|
| 411 |
+
}
|
| 412 |
+
var blob = new Blob(["" + buildCsv(spec, answers)], {type: "text/csv;charset=utf-8"});
|
| 413 |
+
var url = URL.createObjectURL(blob);
|
| 414 |
+
var link = document.createElement("a");
|
| 415 |
+
link.href = url;
|
| 416 |
+
link.download = spec.panel_id + "-results.csv";
|
| 417 |
+
document.body.appendChild(link);
|
| 418 |
+
link.click();
|
| 419 |
+
document.body.removeChild(link);
|
| 420 |
+
setTimeout(function () { URL.revokeObjectURL(url); }, 1000);
|
| 421 |
+
}
|
| 422 |
+
|
| 423 |
+
try {
|
| 424 |
+
var raw = window.localStorage.getItem(STORAGE_KEY);
|
| 425 |
+
if (raw) { restore(JSON.parse(raw)); }
|
| 426 |
+
} catch (e) { /* storage unavailable or corrupt: start fresh */ }
|
| 427 |
+
|
| 428 |
+
document.addEventListener("input", refresh);
|
| 429 |
+
document.addEventListener("change", refresh);
|
| 430 |
+
document.addEventListener("play", function (ev) {
|
| 431 |
+
Array.prototype.forEach.call(document.querySelectorAll("audio"), function (el) {
|
| 432 |
+
if (el !== ev.target) { el.pause(); }
|
| 433 |
+
});
|
| 434 |
+
}, true);
|
| 435 |
+
document.getElementById("download").addEventListener("click", download);
|
| 436 |
+
refresh();
|
| 437 |
+
})();
|
| 438 |
+
"""
|
| 439 |
+
|
| 440 |
+
_HTML_TEMPLATE = """<!DOCTYPE html>
|
| 441 |
+
<html lang="tr">
|
| 442 |
+
<head>
|
| 443 |
+
<meta charset="utf-8">
|
| 444 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 445 |
+
<title>@@TITLE@@</title>
|
| 446 |
+
<style>@@CSS@@</style>
|
| 447 |
+
</head>
|
| 448 |
+
<body>
|
| 449 |
+
<header class="top">
|
| 450 |
+
<h1>@@TITLE@@</h1>
|
| 451 |
+
<p class="instructions">@@INSTRUCTIONS@@</p>
|
| 452 |
+
<p class="instructions">* ile işaretli sorular zorunludur.
|
| 453 |
+
Yanıtlarınız bu tarayıcıda otomatik kaydedilir.</p>
|
| 454 |
+
</header>
|
| 455 |
+
<div class="bar"><div class="bar-inner">
|
| 456 |
+
<span id="progress" role="status">0 / @@COUNT@@ tamamlandı</span>
|
| 457 |
+
<button id="download" class="primary" type="button">CSV indir</button>
|
| 458 |
+
</div></div>
|
| 459 |
+
<main>
|
| 460 |
+
@@CARDS@@
|
| 461 |
+
</main>
|
| 462 |
+
<script type="application/json" id="panel-data">@@DATA@@</script>
|
| 463 |
+
<script>@@JS@@</script>
|
| 464 |
+
</body>
|
| 465 |
+
</html>
|
| 466 |
+
"""
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def _esc(value: str) -> str:
|
| 470 |
+
return html.escape(value, quote=True)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def _embed_json(obj: object) -> str:
|
| 474 |
+
"""Serialise ``obj`` for a ``<script type="application/json">`` block.
|
| 475 |
+
|
| 476 |
+
``<``, ``>``, ``&`` and the JS line separators are escaped so that no text can close the
|
| 477 |
+
script element or be parsed as markup; the JSON stays valid and decodes identically.
|
| 478 |
+
"""
|
| 479 |
+
text = json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
|
| 480 |
+
return (
|
| 481 |
+
text.replace("<", "\\u003c")
|
| 482 |
+
.replace(">", "\\u003e")
|
| 483 |
+
.replace("&", "\\u0026")
|
| 484 |
+
.replace(_LINE_SEPARATOR, "\\u2028")
|
| 485 |
+
.replace(_PARAGRAPH_SEPARATOR, "\\u2029")
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def _question_to_dict(question: Question) -> dict[str, object]:
|
| 490 |
+
return {
|
| 491 |
+
"id": question.id,
|
| 492 |
+
"prompt": question.prompt,
|
| 493 |
+
"kind": question.kind,
|
| 494 |
+
"options": list(question.options),
|
| 495 |
+
"required": question.required,
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def _spec_to_dict(spec: PanelSpec, *, embed: bool) -> dict[str, object]:
|
| 500 |
+
"""Machine-readable spec. ``embed=True`` is the in-page form (audio MIME types);
|
| 501 |
+
otherwise audio file names are listed, never bytes."""
|
| 502 |
+
items: list[dict[str, object]] = []
|
| 503 |
+
for item in spec.items:
|
| 504 |
+
audios: list[dict[str, str]] = [
|
| 505 |
+
{"label": a.label, "mime": audio_mime(a.path)}
|
| 506 |
+
if embed
|
| 507 |
+
else {"label": a.label, "file_name": a.path.name}
|
| 508 |
+
for a in item.audios
|
| 509 |
+
]
|
| 510 |
+
items.append(
|
| 511 |
+
{
|
| 512 |
+
"item_id": item.item_id,
|
| 513 |
+
"audios": audios,
|
| 514 |
+
"text": item.text,
|
| 515 |
+
"editable_text": item.editable_text,
|
| 516 |
+
"info": item.info,
|
| 517 |
+
}
|
| 518 |
+
)
|
| 519 |
+
return {
|
| 520 |
+
"panel_id": spec.panel_id,
|
| 521 |
+
"title": spec.title,
|
| 522 |
+
"instructions": spec.instructions,
|
| 523 |
+
"questions": [_question_to_dict(q) for q in spec.questions],
|
| 524 |
+
"items": items,
|
| 525 |
+
}
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def _audio_html(audio: AudioRef) -> str:
|
| 529 |
+
payload = base64.b64encode(audio.path.read_bytes()).decode("ascii")
|
| 530 |
+
src = f"data:{audio_mime(audio.path)};base64,{payload}"
|
| 531 |
+
return (
|
| 532 |
+
f'<div class="audio"><div class="alabel">{_esc(audio.label)}</div>'
|
| 533 |
+
f'<audio controls preload="none" src="{src}"></audio></div>'
|
| 534 |
+
)
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
def _question_html(question: Question, item_index: int) -> str:
|
| 538 |
+
star = '<span class="req" aria-hidden="true">*</span>' if question.required else ""
|
| 539 |
+
prompt = _esc(question.prompt)
|
| 540 |
+
qid = _esc(question.id)
|
| 541 |
+
if question.kind == "text":
|
| 542 |
+
field_id = f"t{item_index}_{qid}"
|
| 543 |
+
return (
|
| 544 |
+
f'<div class="field"><label class="lbl" for="{field_id}">{prompt}{star}</label>'
|
| 545 |
+
f'<textarea id="{field_id}" data-q="{qid}" rows="2"></textarea></div>'
|
| 546 |
+
)
|
| 547 |
+
name = f"r{item_index}_{qid}"
|
| 548 |
+
options = "".join(
|
| 549 |
+
f'<label class="opt"><input type="radio" name="{name}" data-q="{qid}" '
|
| 550 |
+
f'value="{_esc(option)}"><span>{_esc(option)}</span></label>'
|
| 551 |
+
for option in question.options
|
| 552 |
+
)
|
| 553 |
+
return (
|
| 554 |
+
f'<fieldset class="q"><legend>{prompt}{star}</legend>'
|
| 555 |
+
f'<div class="opts">{options}</div></fieldset>'
|
| 556 |
+
)
|
| 557 |
+
|
| 558 |
+
|
| 559 |
+
def _card_html(item: PanelItem, index: int, questions: tuple[Question, ...]) -> str:
|
| 560 |
+
parts = [
|
| 561 |
+
f'<section class="card" data-item="{_esc(item.item_id)}">',
|
| 562 |
+
f'<div class="card-head"><span class="num">{index + 1}</span>'
|
| 563 |
+
f'<span class="item-id">{_esc(item.item_id)}</span></div>',
|
| 564 |
+
]
|
| 565 |
+
if item.info:
|
| 566 |
+
parts.append(f'<div class="info">{_esc(item.info)}</div>')
|
| 567 |
+
parts.extend(_audio_html(a) for a in item.audios)
|
| 568 |
+
if item.editable_text:
|
| 569 |
+
# A single leading newline inside <textarea> is dropped by the HTML parser, so add one
|
| 570 |
+
# to keep text that itself starts with a newline intact.
|
| 571 |
+
parts.append(
|
| 572 |
+
f'<div class="field"><label class="lbl" for="e{index}">'
|
| 573 |
+
"Metin (düzenleyebilirsiniz)</label>"
|
| 574 |
+
f'<textarea id="e{index}" data-field="text" rows="3">\n{_esc(item.text)}</textarea>'
|
| 575 |
+
"</div>"
|
| 576 |
+
)
|
| 577 |
+
elif item.text:
|
| 578 |
+
parts.append(f'<div class="text">{_esc(item.text)}</div>')
|
| 579 |
+
parts.extend(_question_html(q, index) for q in questions)
|
| 580 |
+
parts.append(
|
| 581 |
+
f'<div class="field"><label class="lbl" for="n{index}">Not (isteğe bağlı)</label>'
|
| 582 |
+
f'<textarea id="n{index}" data-field="note" rows="2"></textarea></div>'
|
| 583 |
+
)
|
| 584 |
+
parts.append("</section>")
|
| 585 |
+
return "\n".join(parts)
|
| 586 |
+
|
| 587 |
+
|
| 588 |
+
def build_panel_html(spec: PanelSpec, max_bytes: int = _DEFAULT_MAX_BYTES) -> str:
|
| 589 |
+
"""Render ``spec`` as one self-contained HTML5 document.
|
| 590 |
+
|
| 591 |
+
Args:
|
| 592 |
+
spec: The panel definition.
|
| 593 |
+
max_bytes: Upper bound for the UTF-8 size of the document.
|
| 594 |
+
|
| 595 |
+
Returns:
|
| 596 |
+
The HTML text. It references no external resource; audio is embedded as data URIs.
|
| 597 |
+
|
| 598 |
+
Raises:
|
| 599 |
+
ValueError: if the document exceeds ``max_bytes`` (the message reports the size).
|
| 600 |
+
FileNotFoundError: if an audio file is missing.
|
| 601 |
+
"""
|
| 602 |
+
cards = "\n".join(_card_html(item, i, spec.questions) for i, item in enumerate(spec.items))
|
| 603 |
+
values = {
|
| 604 |
+
"TITLE": _esc(spec.title),
|
| 605 |
+
"INSTRUCTIONS": _esc(spec.instructions),
|
| 606 |
+
"COUNT": str(len(spec.items)),
|
| 607 |
+
"CSS": _CSS,
|
| 608 |
+
"CARDS": cards,
|
| 609 |
+
"DATA": _embed_json(_spec_to_dict(spec, embed=True)),
|
| 610 |
+
"JS": _JS,
|
| 611 |
+
}
|
| 612 |
+
# Single-pass substitution: inserted values are never rescanned for placeholders.
|
| 613 |
+
document = re.sub(r"@@([A-Z]+)@@", lambda m: values[m.group(1)], _HTML_TEMPLATE)
|
| 614 |
+
size_bytes = len(document.encode("utf-8"))
|
| 615 |
+
if size_bytes > max_bytes:
|
| 616 |
+
raise ValueError(
|
| 617 |
+
f"panel {spec.panel_id!r} is {size_bytes} bytes, over the limit of {max_bytes} bytes; "
|
| 618 |
+
"use fewer items or shorter clips"
|
| 619 |
+
)
|
| 620 |
+
return document
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def write_panel(spec: PanelSpec, out_dir: Path) -> Path:
|
| 624 |
+
"""Write ``panel.html`` and ``panel.json`` into ``out_dir`` and return the HTML path."""
|
| 625 |
+
document = build_panel_html(spec)
|
| 626 |
+
html_path = out_dir / "panel.html"
|
| 627 |
+
atomic_write_text(html_path, document)
|
| 628 |
+
atomic_write_json(out_dir / "panel.json", _spec_to_dict(spec, embed=False))
|
| 629 |
+
return html_path
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
# --------------------------------------------------------------------------------------
|
| 633 |
+
# CSV parsing
|
| 634 |
+
# --------------------------------------------------------------------------------------
|
| 635 |
+
|
| 636 |
+
|
| 637 |
+
@dataclass(frozen=True)
|
| 638 |
+
class PanelResult:
|
| 639 |
+
"""One parsed CSV row.
|
| 640 |
+
|
| 641 |
+
``answers`` maps every question id to its answer ("" if unanswered). ``text`` is the
|
| 642 |
+
edited text for items with ``editable_text`` and ``None`` otherwise.
|
| 643 |
+
"""
|
| 644 |
+
|
| 645 |
+
item_id: str
|
| 646 |
+
answers: Mapping[str, str]
|
| 647 |
+
text: str | None
|
| 648 |
+
note: str
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
def _describe_header_difference(expected: list[str], actual: list[str]) -> str:
|
| 652 |
+
missing = [c for c in expected if c not in actual]
|
| 653 |
+
unexpected = [c for c in actual if c not in expected]
|
| 654 |
+
parts: list[str] = []
|
| 655 |
+
if missing:
|
| 656 |
+
parts.append(f"missing columns {missing}")
|
| 657 |
+
if unexpected:
|
| 658 |
+
parts.append(f"unexpected columns {unexpected}")
|
| 659 |
+
if not parts:
|
| 660 |
+
parts.append("columns are the same but in a different order")
|
| 661 |
+
return "; ".join(parts)
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
def _decode_csv_bytes(data: bytes, csv_path: Path) -> str:
|
| 665 |
+
try:
|
| 666 |
+
text = data.decode("utf-8")
|
| 667 |
+
except UnicodeDecodeError as exc:
|
| 668 |
+
raise ValueError(f"{csv_path} is not valid UTF-8: {exc}") from exc
|
| 669 |
+
return text.removeprefix("")
|
| 670 |
+
|
| 671 |
+
|
| 672 |
+
def read_panel_results(csv_path: Path, spec: PanelSpec) -> list[PanelResult]:
|
| 673 |
+
"""Parse a CSV exported by the panel for ``spec``, strictly.
|
| 674 |
+
|
| 675 |
+
Free-text fields (``text``, ``note`` and ``text`` questions) are NFC-normalised.
|
| 676 |
+
|
| 677 |
+
Returns:
|
| 678 |
+
One :class:`PanelResult` per row, in file order.
|
| 679 |
+
|
| 680 |
+
Raises:
|
| 681 |
+
ValueError: on a malformed file, header mismatch, wrong ``panel_id``, unknown or
|
| 682 |
+
repeated item, an answer that is not one of the options, text in the ``text``
|
| 683 |
+
column of a non-editable item, or items of ``spec`` absent from the file.
|
| 684 |
+
"""
|
| 685 |
+
text = _decode_csv_bytes(csv_path.read_bytes(), csv_path)
|
| 686 |
+
reader = csv.reader(io.StringIO(text, newline=""), strict=True)
|
| 687 |
+
expected = csv_header(spec)
|
| 688 |
+
questions = {q.id: q for q in spec.questions}
|
| 689 |
+
items = {item.item_id: item for item in spec.items}
|
| 690 |
+
with_text = has_text_column(spec)
|
| 691 |
+
|
| 692 |
+
try:
|
| 693 |
+
header = next(reader, None)
|
| 694 |
+
if header is None:
|
| 695 |
+
raise ValueError(f"{csv_path} is empty; expected header {expected}")
|
| 696 |
+
if header != expected:
|
| 697 |
+
raise ValueError(
|
| 698 |
+
f"{csv_path}: header mismatch ({_describe_header_difference(expected, header)}); "
|
| 699 |
+
f"expected {expected}, got {header}"
|
| 700 |
+
)
|
| 701 |
+
results: list[PanelResult] = []
|
| 702 |
+
seen: set[str] = set()
|
| 703 |
+
for row in reader:
|
| 704 |
+
if not row: # blank line (e.g. trailing newline added by an editor)
|
| 705 |
+
continue
|
| 706 |
+
where = f"{csv_path} line {reader.line_num}"
|
| 707 |
+
if len(row) != len(expected):
|
| 708 |
+
raise ValueError(f"{where}: expected {len(expected)} fields, got {len(row)}")
|
| 709 |
+
cells = dict(zip(expected, row, strict=True))
|
| 710 |
+
if cells["panel_id"] != spec.panel_id:
|
| 711 |
+
raise ValueError(
|
| 712 |
+
f"{where}: panel_id {cells['panel_id']!r} does not match {spec.panel_id!r}"
|
| 713 |
+
)
|
| 714 |
+
item_id = cells["item_id"]
|
| 715 |
+
item = items.get(item_id)
|
| 716 |
+
if item is None:
|
| 717 |
+
raise ValueError(f"{where}: unknown item_id {item_id!r}")
|
| 718 |
+
if item_id in seen:
|
| 719 |
+
raise ValueError(f"{where}: duplicate item_id {item_id!r}")
|
| 720 |
+
seen.add(item_id)
|
| 721 |
+
answers: dict[str, str] = {}
|
| 722 |
+
for qid, question in questions.items():
|
| 723 |
+
value = cells[qid]
|
| 724 |
+
if question.kind == "text":
|
| 725 |
+
value = nfc(value)
|
| 726 |
+
elif value != "" and value not in question.options:
|
| 727 |
+
raise ValueError(
|
| 728 |
+
f"{where}: item {item_id!r} question {qid!r}: {value!r} is not one of "
|
| 729 |
+
f"{list(question.options)}"
|
| 730 |
+
)
|
| 731 |
+
answers[qid] = value
|
| 732 |
+
edited: str | None = None
|
| 733 |
+
if with_text:
|
| 734 |
+
if item.editable_text:
|
| 735 |
+
edited = nfc(cells["text"])
|
| 736 |
+
elif cells["text"] != "":
|
| 737 |
+
raise ValueError(
|
| 738 |
+
f"{where}: item {item_id!r} is not editable but has text {cells['text']!r}"
|
| 739 |
+
)
|
| 740 |
+
results.append(
|
| 741 |
+
PanelResult(item_id=item_id, answers=answers, text=edited, note=nfc(cells["note"]))
|
| 742 |
+
)
|
| 743 |
+
except csv.Error as exc:
|
| 744 |
+
raise ValueError(f"{csv_path}: malformed CSV near line {reader.line_num}: {exc}") from exc
|
| 745 |
+
|
| 746 |
+
absent = [item.item_id for item in spec.items if item.item_id not in seen]
|
| 747 |
+
if absent:
|
| 748 |
+
raise ValueError(f"{csv_path}: {len(absent)} item(s) missing from the CSV: {absent}")
|
| 749 |
+
return results
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def missing_required(results: list[PanelResult], spec: PanelSpec) -> list[str]:
|
| 753 |
+
"""Item ids (in ``results`` order) whose required questions lack a non-blank answer."""
|
| 754 |
+
required = [q.id for q in spec.questions if q.required]
|
| 755 |
+
return [
|
| 756 |
+
result.item_id
|
| 757 |
+
for result in results
|
| 758 |
+
if any(not result.answers.get(qid, "").strip() for qid in required)
|
| 759 |
+
]
|
| 760 |
+
|
| 761 |
+
|
| 762 |
+
def _bare_name(name: object) -> str:
|
| 763 |
+
"""A stored audio file name must be a plain name inside the panel directory."""
|
| 764 |
+
if (
|
| 765 |
+
not isinstance(name, str)
|
| 766 |
+
or name in ("", ".", "..")
|
| 767 |
+
or "/" in name
|
| 768 |
+
or "\\" in name
|
| 769 |
+
or Path(name).name != name
|
| 770 |
+
):
|
| 771 |
+
raise ValueError(f"panel audio file name must be a bare name, got {name!r}")
|
| 772 |
+
return name
|
| 773 |
+
|
| 774 |
+
|
| 775 |
+
def load_panel_spec(path: Path) -> PanelSpec:
|
| 776 |
+
"""Rebuild a :class:`PanelSpec` from the ``panel.json`` written by :func:`write_panel`.
|
| 777 |
+
|
| 778 |
+
Audio paths point next to ``panel.json`` (file names only are stored); they are needed
|
| 779 |
+
for validating results, not for reading audio, so they need not exist.
|
| 780 |
+
|
| 781 |
+
Raises:
|
| 782 |
+
ValueError: if the file does not have the expected structure.
|
| 783 |
+
"""
|
| 784 |
+
obj = json.loads(path.read_text(encoding="utf-8"))
|
| 785 |
+
try:
|
| 786 |
+
questions = tuple(
|
| 787 |
+
Question(
|
| 788 |
+
id=q["id"],
|
| 789 |
+
prompt=q["prompt"],
|
| 790 |
+
kind=q["kind"],
|
| 791 |
+
options=tuple(q["options"]),
|
| 792 |
+
required=bool(q["required"]),
|
| 793 |
+
)
|
| 794 |
+
for q in obj["questions"]
|
| 795 |
+
)
|
| 796 |
+
items = tuple(
|
| 797 |
+
PanelItem(
|
| 798 |
+
item_id=i["item_id"],
|
| 799 |
+
audios=tuple(
|
| 800 |
+
AudioRef(label=a["label"], path=path.parent / _bare_name(a["file_name"]))
|
| 801 |
+
for a in i["audios"]
|
| 802 |
+
),
|
| 803 |
+
text=i["text"],
|
| 804 |
+
editable_text=bool(i["editable_text"]),
|
| 805 |
+
info=i["info"],
|
| 806 |
+
)
|
| 807 |
+
for i in obj["items"]
|
| 808 |
+
)
|
| 809 |
+
return PanelSpec(
|
| 810 |
+
panel_id=obj["panel_id"],
|
| 811 |
+
title=obj["title"],
|
| 812 |
+
instructions=obj["instructions"],
|
| 813 |
+
questions=questions,
|
| 814 |
+
items=items,
|
| 815 |
+
)
|
| 816 |
+
except (KeyError, TypeError) as err:
|
| 817 |
+
raise ValueError(f"{path}: malformed panel spec ({err})") from err
|
src/nida/bench/phase0.py
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Phase 0 benchmark stage runner behind ``nida bench-phase0`` (B1 + B2 on Colab).
|
| 2 |
+
|
| 3 |
+
Every stage first checks that all sources it touches are ``approved`` in
|
| 4 |
+
``configs/sources.yaml``. Stages are idempotent and resumable; ``all`` runs the whole
|
| 5 |
+
GPU part in one session (one Drive permission), in this order:
|
| 6 |
+
|
| 7 |
+
check → prepare-b1 → asr-b1 → panel-b1c → score-b1 → prepare-b2 → vae-b2 → asr-b2
|
| 8 |
+
→ quality-b2 → score-b2 → panel-b2
|
| 9 |
+
|
| 10 |
+
After the user has answered the panels (no GPU needed):
|
| 11 |
+
|
| 12 |
+
* ``score-b1`` with ``--c-results <csv>`` adds layer C and applies the decision rule;
|
| 13 |
+
* ``score-b2-panel`` with ``--b2-panel-results <csv>`` un-blinds and scores the B2 panel.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import importlib.metadata
|
| 19 |
+
import json
|
| 20 |
+
import logging
|
| 21 |
+
import shutil
|
| 22 |
+
from collections.abc import Sequence
|
| 23 |
+
from dataclasses import dataclass, field
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
from nida.bench import b1, b2, envs
|
| 27 |
+
from nida.bench.clips import Clip
|
| 28 |
+
from nida.bench.panel import load_panel_spec, read_panel_results
|
| 29 |
+
from nida.bench.prepare import Workspace, load_manifest, prepare_b1
|
| 30 |
+
from nida.bench.protocol import (
|
| 31 |
+
WorkerSummary,
|
| 32 |
+
WorkRequest,
|
| 33 |
+
WorkResult,
|
| 34 |
+
drop_results,
|
| 35 |
+
load_results,
|
| 36 |
+
read_summary,
|
| 37 |
+
write_requests,
|
| 38 |
+
)
|
| 39 |
+
from nida.bench.registry import (
|
| 40 |
+
ASR_SYSTEMS,
|
| 41 |
+
QUALITY_SYSTEM,
|
| 42 |
+
VAE_SYSTEMS,
|
| 43 |
+
System,
|
| 44 |
+
load_sources,
|
| 45 |
+
require_approved,
|
| 46 |
+
)
|
| 47 |
+
from nida.bench.report import render_b1, render_b2, render_b2_panel
|
| 48 |
+
from nida.utils.io import atomic_write_json, atomic_write_text
|
| 49 |
+
|
| 50 |
+
__all__ = ["ALL_STAGES", "STAGES", "Phase0"]
|
| 51 |
+
|
| 52 |
+
log = logging.getLogger(__name__)
|
| 53 |
+
|
| 54 |
+
ALL_STAGES = (
|
| 55 |
+
"check",
|
| 56 |
+
"prepare-b1",
|
| 57 |
+
"asr-b1",
|
| 58 |
+
"panel-b1c",
|
| 59 |
+
"score-b1",
|
| 60 |
+
"prepare-b2",
|
| 61 |
+
"vae-b2",
|
| 62 |
+
"asr-b2",
|
| 63 |
+
"quality-b2",
|
| 64 |
+
"score-b2",
|
| 65 |
+
"panel-b2",
|
| 66 |
+
)
|
| 67 |
+
STAGES = (*ALL_STAGES, "score-b2-panel")
|
| 68 |
+
|
| 69 |
+
B1_DATA = ("fleurs_tr", "mediaspeech_tr", "yodas_tr")
|
| 70 |
+
B2_DATA = ("expresso",)
|
| 71 |
+
FAMILIES = {
|
| 72 |
+
"qwen3_asr_1_7b": "qwen",
|
| 73 |
+
"qwen3_asr_0_6b": "qwen",
|
| 74 |
+
"omniasr_llm_7b": "omni",
|
| 75 |
+
"omniasr_ctc_1b": "omni",
|
| 76 |
+
"whisper_large_v3": "whisper",
|
| 77 |
+
"buzzasr_turkish": "whisper",
|
| 78 |
+
}
|
| 79 |
+
WHISPER_FOR_B2 = ASR_SYSTEMS["whisper_large_v3"]
|
| 80 |
+
MAIN_PACKAGES = ("numpy", "scipy", "soundfile", "PyYAML", "torch")
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _main_package_versions() -> dict[str, str]:
|
| 84 |
+
versions: dict[str, str] = {}
|
| 85 |
+
for name in MAIN_PACKAGES:
|
| 86 |
+
try:
|
| 87 |
+
versions[name] = importlib.metadata.version(name)
|
| 88 |
+
except importlib.metadata.PackageNotFoundError:
|
| 89 |
+
versions[name] = "not-installed"
|
| 90 |
+
return versions
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
@dataclass
|
| 94 |
+
class Phase0:
|
| 95 |
+
workspace: Workspace
|
| 96 |
+
envs_root: Path
|
| 97 |
+
repo_dir: Path
|
| 98 |
+
sources_path: Path
|
| 99 |
+
systems: tuple[System, ...] = tuple(ASR_SYSTEMS.values())
|
| 100 |
+
seed: int = 20261008
|
| 101 |
+
uv: str | None = None # located lazily, after the approval gate
|
| 102 |
+
retry_failed: bool = False
|
| 103 |
+
_c_results: Path | None = field(default=None, init=False, repr=False)
|
| 104 |
+
_b2_panel_results: Path | None = field(default=None, init=False, repr=False)
|
| 105 |
+
|
| 106 |
+
# ------------------------------------------------------------------ helpers
|
| 107 |
+
|
| 108 |
+
def _require(self, stage: str) -> None:
|
| 109 |
+
sources = load_sources(self.sources_path)
|
| 110 |
+
needed: set[str] = set()
|
| 111 |
+
if stage.endswith("b1") or stage in ("panel-b1c", "check"):
|
| 112 |
+
needed |= set(B1_DATA) | {s.source_id for s in self.systems}
|
| 113 |
+
if "b2" in stage or stage == "check":
|
| 114 |
+
# B2 reuses the Turkish B1 clips (FLEURS, MediaSpeech) as its tr subsets.
|
| 115 |
+
needed |= set(B2_DATA) | {"fleurs_tr", "mediaspeech_tr"}
|
| 116 |
+
needed |= {s.source_id for s in VAE_SYSTEMS.values()}
|
| 117 |
+
needed |= {QUALITY_SYSTEM.source_id, WHISPER_FOR_B2.source_id}
|
| 118 |
+
require_approved(needed, sources)
|
| 119 |
+
|
| 120 |
+
def _python(self, worker: str) -> Path:
|
| 121 |
+
spec = envs.env_for_worker(worker)
|
| 122 |
+
if self.uv is None:
|
| 123 |
+
self.uv = envs.find_uv(envs.subprocess_runner)
|
| 124 |
+
python = envs.ensure_env(spec, self.envs_root, self.repo_dir, uv=self.uv)
|
| 125 |
+
freeze = self.envs_root / spec.name / "freeze.txt"
|
| 126 |
+
if freeze.exists(): # keep the exact package set next to the results (durable)
|
| 127 |
+
dest = self.workspace.root / "envs" / f"{spec.name}-freeze.txt"
|
| 128 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 129 |
+
shutil.copyfile(freeze, dest)
|
| 130 |
+
return python
|
| 131 |
+
|
| 132 |
+
def _vae_results(self) -> dict[str, WorkResult]:
|
| 133 |
+
return load_results(self.workspace.workers / "b2_vae" / "results.jsonl")
|
| 134 |
+
|
| 135 |
+
def _b2_plan(self) -> b2.B2Plan:
|
| 136 |
+
return b2.B2Plan(seed=self.seed)
|
| 137 |
+
|
| 138 |
+
def _b1_clips(self) -> list[Clip]:
|
| 139 |
+
return load_manifest(self.workspace)[1]
|
| 140 |
+
|
| 141 |
+
def _b2_clips(self) -> list[Clip]:
|
| 142 |
+
return b2.load_b2_manifest(self.workspace)[1]
|
| 143 |
+
|
| 144 |
+
def _summaries(self) -> dict[str, WorkerSummary]:
|
| 145 |
+
out: dict[str, WorkerSummary] = {}
|
| 146 |
+
for system in self.systems:
|
| 147 |
+
path = self.workspace.workers / system.system_id / "results.summary.json"
|
| 148 |
+
if path.exists():
|
| 149 |
+
out[system.system_id] = read_summary(path)
|
| 150 |
+
return out
|
| 151 |
+
|
| 152 |
+
def _run_aux(self, name: str, system: System, requests: Sequence[WorkRequest]) -> None:
|
| 153 |
+
"""Run a non-B1 worker (VAE, Whisper on B2, Audiobox) over prepared requests."""
|
| 154 |
+
if not requests:
|
| 155 |
+
# Never prune a results file down to nothing: an empty request set means an
|
| 156 |
+
# upstream input is missing (e.g. the VAE results file), not "no work".
|
| 157 |
+
raise RuntimeError(f"{name}: no requests; is an upstream stage's output missing?")
|
| 158 |
+
out_dir = self.workspace.workers / name
|
| 159 |
+
results_path = out_dir / "results.jsonl"
|
| 160 |
+
if results_path.exists():
|
| 161 |
+
# The request set is the source of truth: results for items that are no longer
|
| 162 |
+
# requested (e.g. a reconstruction that failed on redo) are removed, not trusted.
|
| 163 |
+
wanted = {r.clip_id for r in requests}
|
| 164 |
+
stray = [cid for cid in load_results(results_path, repair=True) if cid not in wanted]
|
| 165 |
+
if stray:
|
| 166 |
+
log.warning("%s: removing %d results no longer requested", name, len(stray))
|
| 167 |
+
drop_results(results_path, stray)
|
| 168 |
+
write_requests(out_dir / "requests.jsonl", requests)
|
| 169 |
+
cmd = [
|
| 170 |
+
str(self._python(system.worker)),
|
| 171 |
+
"-m",
|
| 172 |
+
f"nida.bench.workers.{system.worker}",
|
| 173 |
+
"--requests",
|
| 174 |
+
str(out_dir / "requests.jsonl"),
|
| 175 |
+
"--results",
|
| 176 |
+
str(out_dir / "results.jsonl"),
|
| 177 |
+
"--model-id",
|
| 178 |
+
system.model_id,
|
| 179 |
+
"--revision",
|
| 180 |
+
system.revision,
|
| 181 |
+
"--batch-size",
|
| 182 |
+
str(system.batch_size),
|
| 183 |
+
*(["--retry-failed"] if self.retry_failed else []),
|
| 184 |
+
]
|
| 185 |
+
code = b1._stream_runner(cmd)
|
| 186 |
+
if code != 0:
|
| 187 |
+
raise RuntimeError(f"{name}: worker exited with code {code}")
|
| 188 |
+
summary = read_summary(out_dir / "results.summary.json")
|
| 189 |
+
if summary.n_ok + summary.n_error != len(requests):
|
| 190 |
+
raise RuntimeError(
|
| 191 |
+
f"{name}: {summary.n_ok + summary.n_error} results for {len(requests)} requests"
|
| 192 |
+
)
|
| 193 |
+
if summary.n_error > b1.MAX_ERROR_FRACTION * (summary.n_ok + summary.n_error):
|
| 194 |
+
raise RuntimeError(
|
| 195 |
+
f"{name}: {summary.n_error} items failed (> 5%); rerun with --retry-failed"
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
# ------------------------------------------------------------------- stages
|
| 199 |
+
|
| 200 |
+
def run(
|
| 201 |
+
self,
|
| 202 |
+
stage: str,
|
| 203 |
+
*,
|
| 204 |
+
c_results: Path | None = None,
|
| 205 |
+
b2_panel_results: Path | None = None,
|
| 206 |
+
) -> dict[str, object]:
|
| 207 |
+
"""Run one stage (or ``all``); returns a JSON-compatible summary for metrics.json."""
|
| 208 |
+
stages = ALL_STAGES if stage == "all" else (stage,)
|
| 209 |
+
unknown = [s for s in stages if s not in STAGES]
|
| 210 |
+
if unknown:
|
| 211 |
+
raise ValueError(f"unknown stage(s) {unknown}; choose from {STAGES} or 'all'")
|
| 212 |
+
self._c_results = c_results
|
| 213 |
+
self._b2_panel_results = b2_panel_results
|
| 214 |
+
result: dict[str, object] = {"main_packages": _main_package_versions()}
|
| 215 |
+
for name in stages:
|
| 216 |
+
log.info("=== stage %s", name)
|
| 217 |
+
self._require(name)
|
| 218 |
+
output = getattr(self, "_stage_" + name.replace("-", "_"))()
|
| 219 |
+
if output is not None:
|
| 220 |
+
result[name] = output
|
| 221 |
+
return result
|
| 222 |
+
|
| 223 |
+
def _stage_check(self) -> dict[str, object]:
|
| 224 |
+
return {"approved": True}
|
| 225 |
+
|
| 226 |
+
def _stage_prepare_b1(self) -> dict[str, object]:
|
| 227 |
+
clips = prepare_b1(self.workspace)
|
| 228 |
+
counts: dict[str, int] = {}
|
| 229 |
+
for clip in clips:
|
| 230 |
+
counts[clip.layer] = counts.get(clip.layer, 0) + 1
|
| 231 |
+
return {"clips_per_layer": counts}
|
| 232 |
+
|
| 233 |
+
def _stage_asr_b1(self) -> dict[str, object]:
|
| 234 |
+
clips = self._b1_clips()
|
| 235 |
+
done: dict[str, object] = {}
|
| 236 |
+
for system in self.systems:
|
| 237 |
+
summary = b1.run_system(
|
| 238 |
+
self.workspace,
|
| 239 |
+
clips,
|
| 240 |
+
system,
|
| 241 |
+
self._python(system.worker),
|
| 242 |
+
retry_failed=self.retry_failed,
|
| 243 |
+
)
|
| 244 |
+
done[system.system_id] = {"n_ok": summary.n_ok, "n_error": summary.n_error}
|
| 245 |
+
return done
|
| 246 |
+
|
| 247 |
+
def _stage_panel_b1c(self) -> dict[str, object]:
|
| 248 |
+
clips = self._b1_clips()
|
| 249 |
+
pool = [c for c in clips if c.layer == "C"]
|
| 250 |
+
hyps = {s.system_id: b1.hypotheses(self.workspace, s) for s in self.systems}
|
| 251 |
+
final, html = b1.build_c_panel(self.workspace, pool, hyps, self.seed)
|
| 252 |
+
return {"panel": str(html), "n_items": len(final)}
|
| 253 |
+
|
| 254 |
+
def _stage_score_b1(self) -> dict[str, object]:
|
| 255 |
+
clips = self._b1_clips()
|
| 256 |
+
hyps = {s.system_id: b1.hypotheses(self.workspace, s) for s in self.systems}
|
| 257 |
+
c_refs: dict[str, str] | None = None
|
| 258 |
+
provenance = None
|
| 259 |
+
if self._c_results is not None:
|
| 260 |
+
spec = load_panel_spec(self.workspace.panels / f"b1c-{self.seed}" / "panel.json")
|
| 261 |
+
c_refs = b1.c_references_from_panel(read_panel_results(self._c_results, spec))
|
| 262 |
+
provenance = b1.load_c_selection(self.workspace, self.seed)
|
| 263 |
+
scores = b1.score_b1(
|
| 264 |
+
clips,
|
| 265 |
+
self.systems,
|
| 266 |
+
hyps,
|
| 267 |
+
c_refs,
|
| 268 |
+
seed=self.seed,
|
| 269 |
+
summaries=self._summaries(),
|
| 270 |
+
c_provenance=provenance,
|
| 271 |
+
)
|
| 272 |
+
choice = b1.choose_systems(scores, FAMILIES)
|
| 273 |
+
report_dir = self.workspace.root / "reports"
|
| 274 |
+
atomic_write_json(report_dir / "b1_metrics.json", {"scores": scores, "choice": choice})
|
| 275 |
+
atomic_write_text(report_dir / "b1_report.md", render_b1(scores, choice))
|
| 276 |
+
return {"choice": choice}
|
| 277 |
+
|
| 278 |
+
def _stage_prepare_b2(self) -> dict[str, object]:
|
| 279 |
+
clips = b2.prepare_b2(self.workspace, self._b1_clips(), self._b2_plan())
|
| 280 |
+
counts: dict[str, int] = {}
|
| 281 |
+
for clip in clips:
|
| 282 |
+
counts[clip.layer] = counts.get(clip.layer, 0) + 1
|
| 283 |
+
return {"clips_per_subset": counts}
|
| 284 |
+
|
| 285 |
+
def _stage_vae_b2(self) -> dict[str, object]:
|
| 286 |
+
system = VAE_SYSTEMS["voxcpm2_audiovae_v2"]
|
| 287 |
+
results_path = self.workspace.workers / "b2_vae" / "results.jsonl"
|
| 288 |
+
redo = b2.missing_vae_outputs(self.workspace, load_results(results_path, repair=True))
|
| 289 |
+
if redo:
|
| 290 |
+
log.warning("b2_vae: %d reconstructions missing on disk; redoing them", len(redo))
|
| 291 |
+
# Invalidate derived data FIRST and the VAE results LAST: if the process dies in
|
| 292 |
+
# between, the rerun still sees the missing reconstruction and redoes all of it.
|
| 293 |
+
stale = [f"{clip_id}__vae" for clip_id in redo]
|
| 294 |
+
for name in ("b2_asr_tr", "b2_asr_en", "b2_quality"):
|
| 295 |
+
downstream = self.workspace.workers / name / "results.jsonl"
|
| 296 |
+
if downstream.exists():
|
| 297 |
+
drop_results(downstream, stale)
|
| 298 |
+
for clip_id in redo:
|
| 299 |
+
(b2.b2_paths(self.workspace).asr16k / f"{clip_id}__vae.wav").unlink(missing_ok=True)
|
| 300 |
+
drop_results(results_path, redo)
|
| 301 |
+
self._run_aux("b2_vae", system, b2.vae_requests(self.workspace, self._b2_clips()))
|
| 302 |
+
return {"redone_missing_outputs": len(redo)}
|
| 303 |
+
|
| 304 |
+
def _stage_asr_b2(self) -> None:
|
| 305 |
+
clips = self._b2_clips()
|
| 306 |
+
for language in ("tr", "en"):
|
| 307 |
+
requests = b2.asr_requests(self.workspace, clips, language, self._vae_results())
|
| 308 |
+
if requests:
|
| 309 |
+
self._run_aux(f"b2_asr_{language}", WHISPER_FOR_B2, requests)
|
| 310 |
+
|
| 311 |
+
def _stage_quality_b2(self) -> None:
|
| 312 |
+
self._run_aux(
|
| 313 |
+
"b2_quality",
|
| 314 |
+
QUALITY_SYSTEM,
|
| 315 |
+
b2.quality_requests(self.workspace, self._b2_clips(), self._vae_results()),
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
def _stage_score_b2(self) -> dict[str, object]:
|
| 319 |
+
asr: dict[str, WorkResult] = {}
|
| 320 |
+
for language in ("tr", "en"):
|
| 321 |
+
asr.update(
|
| 322 |
+
load_results(self.workspace.workers / f"b2_asr_{language}" / "results.jsonl")
|
| 323 |
+
)
|
| 324 |
+
quality = load_results(self.workspace.workers / "b2_quality" / "results.jsonl")
|
| 325 |
+
scores = b2.score_b2(self.workspace, self._b2_clips(), asr, quality, self._vae_results())
|
| 326 |
+
report_dir = self.workspace.root / "reports"
|
| 327 |
+
atomic_write_json(report_dir / "b2_metrics.json", scores)
|
| 328 |
+
atomic_write_text(report_dir / "b2_report.md", render_b2(scores))
|
| 329 |
+
return scores
|
| 330 |
+
|
| 331 |
+
def _stage_panel_b2(self) -> dict[str, object]:
|
| 332 |
+
html = b2.build_b2_panel(
|
| 333 |
+
self.workspace, self._b2_clips(), self._b2_plan(), self._vae_results()
|
| 334 |
+
)
|
| 335 |
+
return {"panel": str(html)}
|
| 336 |
+
|
| 337 |
+
def _stage_score_b2_panel(self) -> dict[str, object]:
|
| 338 |
+
if self._b2_panel_results is None:
|
| 339 |
+
raise ValueError("score-b2-panel needs --b2-panel-results <csv>")
|
| 340 |
+
plan = self._b2_plan()
|
| 341 |
+
spec = load_panel_spec(self.workspace.panels / f"b2-{plan.seed}" / "panel.json")
|
| 342 |
+
key_obj = json.loads(
|
| 343 |
+
(self.workspace.manifests / f"b2_panel_key-{plan.seed}.json").read_text(
|
| 344 |
+
encoding="utf-8"
|
| 345 |
+
)
|
| 346 |
+
)
|
| 347 |
+
scores = b2.score_b2_panel(
|
| 348 |
+
read_panel_results(self._b2_panel_results, spec),
|
| 349 |
+
key_obj["items"],
|
| 350 |
+
min_per_subset=int(key_obj.get("min_per_subset", plan.n_panel_per_subset)),
|
| 351 |
+
)
|
| 352 |
+
scores["excluded_missing_reconstruction"] = key_obj.get(
|
| 353 |
+
"excluded_missing_reconstruction", {}
|
| 354 |
+
)
|
| 355 |
+
report_dir = self.workspace.root / "reports"
|
| 356 |
+
atomic_write_json(report_dir / "b2_panel_metrics.json", scores)
|
| 357 |
+
atomic_write_text(report_dir / "b2_panel_report.md", render_b2_panel(scores))
|
| 358 |
+
return scores
|
src/nida/bench/prepare.py
ADDED
|
@@ -0,0 +1,450 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""B1 data preparation: fetch pinned sources, select clips, write uniform 16 kHz clips.
|
| 2 |
+
|
| 3 |
+
Layout (``Workspace``):
|
| 4 |
+
|
| 5 |
+
* ``root`` — durable directory on Google Drive (manifests, worker results, panels).
|
| 6 |
+
* ``cache`` — local scratch on the Colab disk (downloads, extracted audio, clip WAVs).
|
| 7 |
+
It may disappear between sessions; everything in it is re-derived deterministically.
|
| 8 |
+
|
| 9 |
+
Every ASR clip is written as 16 kHz mono 16-bit PCM WAV at ``cache/clips16k/<clip_id>.wav``
|
| 10 |
+
so all systems see byte-identical input. Selection is deterministic for a given seed and
|
| 11 |
+
pinned source revision; the resulting manifest is written once to ``root`` and reused.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import csv
|
| 17 |
+
import json
|
| 18 |
+
import logging
|
| 19 |
+
import random
|
| 20 |
+
import re
|
| 21 |
+
from collections.abc import Callable, Iterable, Sequence
|
| 22 |
+
from dataclasses import asdict, dataclass
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
from typing import Any
|
| 25 |
+
|
| 26 |
+
from nida.bench import fetch
|
| 27 |
+
from nida.bench.audiometrics import resample
|
| 28 |
+
from nida.bench.clips import (
|
| 29 |
+
YODAS_DURATION_TOLERANCE_MS,
|
| 30 |
+
Clip,
|
| 31 |
+
FleursRow,
|
| 32 |
+
MediaSpeechItem,
|
| 33 |
+
Quarantined,
|
| 34 |
+
Selection,
|
| 35 |
+
YodasUtt,
|
| 36 |
+
clip_from_obj,
|
| 37 |
+
clip_to_obj,
|
| 38 |
+
select_fleurs,
|
| 39 |
+
select_mediaspeech,
|
| 40 |
+
select_yodas_pool,
|
| 41 |
+
)
|
| 42 |
+
from nida.bench.registry import DATA_PINS
|
| 43 |
+
from nida.utils.io import atomic_write_text
|
| 44 |
+
|
| 45 |
+
__all__ = [
|
| 46 |
+
"B1Plan",
|
| 47 |
+
"Workspace",
|
| 48 |
+
"clip_audio_path",
|
| 49 |
+
"load_manifest",
|
| 50 |
+
"parse_fleurs_tsv",
|
| 51 |
+
"parse_yodas_listing",
|
| 52 |
+
"prepare_b1",
|
| 53 |
+
"write_clip_audio",
|
| 54 |
+
]
|
| 55 |
+
|
| 56 |
+
log = logging.getLogger(__name__)
|
| 57 |
+
|
| 58 |
+
ASR_SAMPLE_RATE_HZ = 16000
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True)
|
| 62 |
+
class Workspace:
|
| 63 |
+
root: Path
|
| 64 |
+
cache: Path
|
| 65 |
+
|
| 66 |
+
@property
|
| 67 |
+
def manifests(self) -> Path:
|
| 68 |
+
return self.root / "manifests"
|
| 69 |
+
|
| 70 |
+
@property
|
| 71 |
+
def workers(self) -> Path:
|
| 72 |
+
return self.root / "workers"
|
| 73 |
+
|
| 74 |
+
@property
|
| 75 |
+
def panels(self) -> Path:
|
| 76 |
+
return self.root / "panels"
|
| 77 |
+
|
| 78 |
+
@property
|
| 79 |
+
def clips16k(self) -> Path:
|
| 80 |
+
return self.cache / "clips16k"
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
@dataclass(frozen=True)
|
| 84 |
+
class B1Plan:
|
| 85 |
+
"""Parameters that fully determine the B1 clip manifest."""
|
| 86 |
+
|
| 87 |
+
seed: int = 20261008
|
| 88 |
+
n_fleurs: int = 100
|
| 89 |
+
n_mediaspeech: int = 75
|
| 90 |
+
n_yodas_pool: int = 300
|
| 91 |
+
fleurs_revision: str = DATA_PINS["fleurs_revision"]
|
| 92 |
+
yodas_revision: str = DATA_PINS["yodas_revision"]
|
| 93 |
+
mediaspeech_url: str = DATA_PINS["mediaspeech_url"]
|
| 94 |
+
# Comma-separated tr000 shards. One shard (00000000) has only 289 eligible
|
| 95 |
+
# utterances at <= 3 per video (run 20261008-120354-bench-phase0-69dc), so two.
|
| 96 |
+
yodas_shards: str = "00000000,00000001"
|
| 97 |
+
# Extra pool candidates drawn so that clips with broken audio (YODAS has empty wavs,
|
| 98 |
+
# run 20261008-121509-bench-phase0-1347) can be quarantined without a short pool.
|
| 99 |
+
yodas_spare: int = 30
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def clip_audio_path(workspace: Workspace, clip: Clip) -> Path:
|
| 103 |
+
return workspace.clips16k / f"{clip.clip_id}.wav"
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
# ------------------------------------------------------------------------------ parsing
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def parse_fleurs_tsv(text: str) -> list[FleursRow]:
|
| 110 |
+
"""Parse FLEURS ``<split>.tsv`` (7 tab-separated columns, no header, no quoting).
|
| 111 |
+
|
| 112 |
+
Columns: sentence id, file name, raw transcription, normalized transcription,
|
| 113 |
+
characters, number of samples (16 kHz), gender.
|
| 114 |
+
|
| 115 |
+
Raises:
|
| 116 |
+
ValueError: on a row with the wrong number of columns or a non-integer sample count.
|
| 117 |
+
"""
|
| 118 |
+
rows: list[FleursRow] = []
|
| 119 |
+
reader = csv.reader(text.splitlines(), delimiter="\t", quoting=csv.QUOTE_NONE)
|
| 120 |
+
for number, cols in enumerate(reader, start=1):
|
| 121 |
+
if not cols:
|
| 122 |
+
continue
|
| 123 |
+
if len(cols) != 7:
|
| 124 |
+
raise ValueError(f"FLEURS tsv line {number}: expected 7 columns, got {len(cols)}")
|
| 125 |
+
try:
|
| 126 |
+
num_samples = int(cols[5])
|
| 127 |
+
except ValueError as err:
|
| 128 |
+
raise ValueError(f"FLEURS tsv line {number}: bad sample count {cols[5]!r}") from err
|
| 129 |
+
rows.append(FleursRow(cols[0], cols[1], cols[2], num_samples, cols[6]))
|
| 130 |
+
return rows
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def parse_yodas_listing(text_file: str, duration_file: str, shard: str = "") -> list[YodasUtt]:
|
| 134 |
+
"""Join YODAS ``text/<shard>.txt`` (``utt_id caption``) with ``duration/<shard>.txt``.
|
| 135 |
+
|
| 136 |
+
Raises:
|
| 137 |
+
ValueError: on malformed lines, duplicate ids or ids missing from either file.
|
| 138 |
+
"""
|
| 139 |
+
|
| 140 |
+
def split(content: str, what: str) -> dict[str, str]:
|
| 141 |
+
out: dict[str, str] = {}
|
| 142 |
+
for number, line in enumerate(content.splitlines(), start=1):
|
| 143 |
+
if not line.strip():
|
| 144 |
+
continue
|
| 145 |
+
parts = line.split(" ", 1)
|
| 146 |
+
utt_id = parts[0]
|
| 147 |
+
value = parts[1] if len(parts) == 2 else ""
|
| 148 |
+
if utt_id in out:
|
| 149 |
+
raise ValueError(f"YODAS {what} line {number}: duplicate id {utt_id}")
|
| 150 |
+
out[utt_id] = value
|
| 151 |
+
return out
|
| 152 |
+
|
| 153 |
+
captions = split(text_file, "text")
|
| 154 |
+
durations = split(duration_file, "duration")
|
| 155 |
+
if set(captions) != set(durations):
|
| 156 |
+
diff = sorted(set(captions) ^ set(durations))
|
| 157 |
+
raise ValueError(f"YODAS text/duration ids differ, e.g. {diff[:3]}")
|
| 158 |
+
utts: list[YodasUtt] = []
|
| 159 |
+
for utt_id in sorted(captions):
|
| 160 |
+
try:
|
| 161 |
+
seconds = float(durations[utt_id])
|
| 162 |
+
except ValueError as err:
|
| 163 |
+
raise ValueError(f"YODAS duration for {utt_id} is not a number") from err
|
| 164 |
+
utts.append(YodasUtt(utt_id, captions[utt_id].strip(), round(seconds * 1000), shard))
|
| 165 |
+
return utts
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
# -------------------------------------------------------------------------------- audio
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def write_clip_audio(source: Path, dest: Path, start_ms: int, end_ms: int | None) -> int:
|
| 172 |
+
"""Write ``source[start_ms:end_ms]`` as 16 kHz mono PCM16 WAV at ``dest``.
|
| 173 |
+
|
| 174 |
+
Multi-channel input is rejected (callers pick a channel explicitly). Returns the
|
| 175 |
+
number of samples written. Idempotent: an existing ``dest`` is left untouched.
|
| 176 |
+
|
| 177 |
+
Raises:
|
| 178 |
+
ValueError: on multi-channel audio or an empty / out-of-range span.
|
| 179 |
+
"""
|
| 180 |
+
import numpy as np
|
| 181 |
+
import soundfile # type: ignore[import-untyped] # bench extra, untyped
|
| 182 |
+
|
| 183 |
+
if dest.exists():
|
| 184 |
+
return int(soundfile.info(str(dest)).frames)
|
| 185 |
+
audio: Any
|
| 186 |
+
audio, rate_hz = soundfile.read(str(source), dtype="float64", always_2d=True)
|
| 187 |
+
if audio.shape[1] != 1:
|
| 188 |
+
raise ValueError(f"{source}: expected mono, got {audio.shape[1]} channels")
|
| 189 |
+
mono = audio[:, 0]
|
| 190 |
+
start = round(start_ms * rate_hz / 1000)
|
| 191 |
+
stop = mono.shape[0] if end_ms is None else min(mono.shape[0], round(end_ms * rate_hz / 1000))
|
| 192 |
+
if not 0 <= start < stop:
|
| 193 |
+
raise ValueError(f"{source}: empty span {start_ms}..{end_ms} ms ({mono.shape[0]} samples)")
|
| 194 |
+
segment = mono[start:stop]
|
| 195 |
+
if rate_hz != ASR_SAMPLE_RATE_HZ:
|
| 196 |
+
segment = resample(segment, int(rate_hz), ASR_SAMPLE_RATE_HZ)
|
| 197 |
+
peak = float(np.max(np.abs(segment)))
|
| 198 |
+
if peak > 1.0: # resampling overshoot; never wrap around in PCM16
|
| 199 |
+
segment = segment / peak
|
| 200 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 201 |
+
tmp = dest.with_name(f".{dest.name}.tmp.wav")
|
| 202 |
+
soundfile.write(str(tmp), segment, ASR_SAMPLE_RATE_HZ, subtype="PCM_16")
|
| 203 |
+
tmp.replace(dest)
|
| 204 |
+
return int(segment.shape[0])
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# ----------------------------------------------------------------------------- manifest
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _write_jsonl(path: Path, rows: Iterable[dict[str, object]]) -> None:
|
| 211 |
+
atomic_write_text(
|
| 212 |
+
path,
|
| 213 |
+
"".join(json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in rows),
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def load_manifest(workspace: Workspace) -> tuple[B1Plan, list[Clip]]:
|
| 218 |
+
"""Read the B1 manifest written by :func:`prepare_b1`."""
|
| 219 |
+
plan_obj = json.loads((workspace.manifests / "b1_plan.json").read_text(encoding="utf-8"))
|
| 220 |
+
clips = [
|
| 221 |
+
clip_from_obj(json.loads(line))
|
| 222 |
+
for line in (workspace.manifests / "b1_clips.jsonl")
|
| 223 |
+
.read_text(encoding="utf-8")
|
| 224 |
+
.splitlines()
|
| 225 |
+
if line.strip()
|
| 226 |
+
]
|
| 227 |
+
return B1Plan(**plan_obj), clips
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
# ---------------------------------------------------------------------------- prepare
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
Opener = Callable[..., Any] # Any: urllib-compatible opener, see nida.bench.fetch
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _require_extracted(wanted: set[str], extracted: Sequence[Path], base: Path, what: str) -> None:
|
| 237 |
+
"""Fail loudly when an archive did not contain every member we selected."""
|
| 238 |
+
got = {p.relative_to(base).as_posix() for p in extracted}
|
| 239 |
+
missing = sorted(wanted - got)
|
| 240 |
+
if missing:
|
| 241 |
+
raise FileNotFoundError(
|
| 242 |
+
f"{what}: {len(missing)} selected members missing from the archive, e.g. {missing[:3]}"
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def _fleurs(plan: B1Plan, ws: Workspace, opener: Opener) -> tuple[Selection, Path]:
|
| 247 |
+
base = ws.cache / "fleurs"
|
| 248 |
+
tsv = fetch.download(
|
| 249 |
+
fetch.hf_resolve_url(
|
| 250 |
+
"google/fleurs", plan.fleurs_revision, "data/tr_tr/test.tsv", "dataset"
|
| 251 |
+
),
|
| 252 |
+
base / "test.tsv",
|
| 253 |
+
opener=opener,
|
| 254 |
+
)
|
| 255 |
+
selection = select_fleurs(
|
| 256 |
+
parse_fleurs_tsv(tsv.path.read_text(encoding="utf-8")), plan.n_fleurs, plan.seed
|
| 257 |
+
)
|
| 258 |
+
wanted = {f"test/{c.file_id}" for c in selection.clips}
|
| 259 |
+
extracted = fetch.stream_extract(
|
| 260 |
+
fetch.hf_resolve_url(
|
| 261 |
+
"google/fleurs", plan.fleurs_revision, "data/tr_tr/audio/test.tar.gz", "dataset"
|
| 262 |
+
),
|
| 263 |
+
base,
|
| 264 |
+
wanted.__contains__,
|
| 265 |
+
opener=opener,
|
| 266 |
+
)
|
| 267 |
+
_require_extracted(wanted, extracted, base, "FLEURS")
|
| 268 |
+
return selection, base / "test"
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def _mediaspeech(plan: B1Plan, ws: Workspace, opener: Opener) -> tuple[Selection, Path]:
|
| 272 |
+
import soundfile
|
| 273 |
+
|
| 274 |
+
base = ws.cache / "mediaspeech"
|
| 275 |
+
fetch.stream_extract(
|
| 276 |
+
plan.mediaspeech_url, base, lambda name: name.startswith("TR/"), opener=opener
|
| 277 |
+
)
|
| 278 |
+
audio_dir = base / "TR"
|
| 279 |
+
items: list[MediaSpeechItem] = []
|
| 280 |
+
for txt in sorted(audio_dir.glob("*.txt")):
|
| 281 |
+
flac = txt.with_suffix(".flac")
|
| 282 |
+
if not flac.exists():
|
| 283 |
+
raise FileNotFoundError(f"MediaSpeech transcript without audio: {txt.name}")
|
| 284 |
+
info = soundfile.info(str(flac))
|
| 285 |
+
items.append(
|
| 286 |
+
MediaSpeechItem(
|
| 287 |
+
txt.stem,
|
| 288 |
+
txt.read_text(encoding="utf-8").strip(),
|
| 289 |
+
round(1000 * info.frames / info.samplerate),
|
| 290 |
+
)
|
| 291 |
+
)
|
| 292 |
+
if not items:
|
| 293 |
+
raise FileNotFoundError(f"no MediaSpeech transcripts under {audio_dir}")
|
| 294 |
+
return select_mediaspeech(items, plan.n_mediaspeech, plan.seed), audio_dir
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _yodas(plan: B1Plan, ws: Workspace, opener: Opener) -> tuple[Selection, Path]:
|
| 298 |
+
base = ws.cache / "yodas"
|
| 299 |
+
shards = yodas_shards(plan)
|
| 300 |
+
|
| 301 |
+
def get(kind: str, shard: str, ext: str) -> Path:
|
| 302 |
+
name = f"data/tr000/{kind}/{shard}.{ext}"
|
| 303 |
+
url = fetch.hf_resolve_url("espnet/yodas", plan.yodas_revision, name, "dataset")
|
| 304 |
+
return fetch.download(url, base / kind / f"{shard}.{ext}", opener=opener).path
|
| 305 |
+
|
| 306 |
+
utts: list[YodasUtt] = []
|
| 307 |
+
seen: set[str] = set()
|
| 308 |
+
for shard in shards:
|
| 309 |
+
shard_utts = parse_yodas_listing(
|
| 310 |
+
get("text", shard, "txt").read_text(encoding="utf-8"),
|
| 311 |
+
get("duration", shard, "txt").read_text(encoding="utf-8"),
|
| 312 |
+
shard,
|
| 313 |
+
)
|
| 314 |
+
duplicates = seen & {u.utt_id for u in shard_utts}
|
| 315 |
+
if duplicates:
|
| 316 |
+
raise ValueError(f"YODAS utterance ids repeat across shards: {sorted(duplicates)[:3]}")
|
| 317 |
+
seen |= {u.utt_id for u in shard_utts}
|
| 318 |
+
utts.extend(shard_utts)
|
| 319 |
+
if plan.yodas_spare < 0:
|
| 320 |
+
raise ValueError(f"yodas_spare must be >= 0, got {plan.yodas_spare}")
|
| 321 |
+
selection = select_yodas_pool(utts, plan.n_yodas_pool + plan.yodas_spare, plan.seed)
|
| 322 |
+
for shard in shards:
|
| 323 |
+
wanted = {
|
| 324 |
+
f"{shard}/{c.file_id}.wav" for c in selection.clips if c.meta.get("shard") == shard
|
| 325 |
+
}
|
| 326 |
+
if not wanted:
|
| 327 |
+
continue
|
| 328 |
+
audio_url = fetch.hf_resolve_url(
|
| 329 |
+
"espnet/yodas", plan.yodas_revision, f"data/tr000/audio/{shard}.tar.gz", "dataset"
|
| 330 |
+
)
|
| 331 |
+
extracted = fetch.stream_extract(
|
| 332 |
+
audio_url, base / "audio", wanted.__contains__, opener=opener
|
| 333 |
+
)
|
| 334 |
+
_require_extracted(wanted, extracted, base / "audio", "YODAS")
|
| 335 |
+
good: list[Clip] = []
|
| 336 |
+
bad: list[Quarantined] = []
|
| 337 |
+
for clip in selection.clips:
|
| 338 |
+
problem = yodas_audio_problem(_source_file(clip, base / "audio"), clip.end_ms)
|
| 339 |
+
if problem is None:
|
| 340 |
+
good.append(clip)
|
| 341 |
+
else:
|
| 342 |
+
bad.append(Quarantined(clip.file_id, "select", problem[0], problem[1]))
|
| 343 |
+
if len(good) < plan.n_yodas_pool:
|
| 344 |
+
raise ValueError(
|
| 345 |
+
f"only {len(good)} YODAS clips with usable audio, need {plan.n_yodas_pool} "
|
| 346 |
+
f"({len(bad)} quarantined; raise yodas_spare)"
|
| 347 |
+
)
|
| 348 |
+
chosen = random.Random(plan.seed).sample(good, plan.n_yodas_pool)
|
| 349 |
+
log.info("YODAS pool: %d usable, %d broken audio, %d chosen", len(good), len(bad), len(chosen))
|
| 350 |
+
return (
|
| 351 |
+
Selection(tuple(sorted(chosen, key=lambda c: c.file_id)), (*selection.quarantined, *bad)),
|
| 352 |
+
base / "audio",
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def yodas_audio_problem(path: Path, expected_ms: int) -> tuple[str, str] | None:
|
| 357 |
+
"""``(reason, details)`` if a YODAS wav is unreadable, empty or of the wrong length."""
|
| 358 |
+
import soundfile
|
| 359 |
+
|
| 360 |
+
try:
|
| 361 |
+
info = soundfile.info(str(path))
|
| 362 |
+
except (RuntimeError, soundfile.LibsndfileError) as err:
|
| 363 |
+
return "unreadable_audio", str(err)
|
| 364 |
+
if info.frames == 0:
|
| 365 |
+
return "empty_audio", f"0 frames at {info.samplerate} Hz"
|
| 366 |
+
actual_ms = round(1000 * info.frames / info.samplerate)
|
| 367 |
+
if abs(actual_ms - expected_ms) > YODAS_DURATION_TOLERANCE_MS:
|
| 368 |
+
return "audio_duration_mismatch", f"{actual_ms} vs {expected_ms} ms"
|
| 369 |
+
return None
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def yodas_shards(plan: B1Plan) -> list[str]:
|
| 373 |
+
"""The plan's YODAS shards; each must be an 8-digit shard name, no repeats."""
|
| 374 |
+
shards = [s.strip() for s in plan.yodas_shards.split(",")]
|
| 375 |
+
if not shards or any(re.fullmatch(r"[0-9]{8}", s) is None for s in shards):
|
| 376 |
+
raise ValueError(f"invalid yodas_shards {plan.yodas_shards!r}")
|
| 377 |
+
if len(set(shards)) != len(shards):
|
| 378 |
+
raise ValueError(f"repeated YODAS shard in {plan.yodas_shards!r}")
|
| 379 |
+
return shards
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def _source_file(clip: Clip, audio_dir: Path) -> Path:
|
| 383 |
+
if clip.source_id == "fleurs_tr":
|
| 384 |
+
return audio_dir / clip.file_id
|
| 385 |
+
if clip.source_id == "mediaspeech_tr":
|
| 386 |
+
return audio_dir / f"{clip.file_id}.flac"
|
| 387 |
+
if clip.source_id == "yodas_tr":
|
| 388 |
+
shard = clip.meta.get("shard")
|
| 389 |
+
if not shard:
|
| 390 |
+
raise ValueError(f"YODAS clip {clip.clip_id} has no shard in meta")
|
| 391 |
+
return audio_dir / shard / f"{clip.file_id}.wav"
|
| 392 |
+
raise ValueError(f"unknown B1 source {clip.source_id}")
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def prepare_b1(
|
| 396 |
+
workspace: Workspace,
|
| 397 |
+
plan: B1Plan | None = None,
|
| 398 |
+
*,
|
| 399 |
+
opener: Opener | None = None,
|
| 400 |
+
) -> list[Clip]:
|
| 401 |
+
"""Fetch, select and convert all B1 clips; write the manifest once.
|
| 402 |
+
|
| 403 |
+
If a manifest already exists it must have been produced with the same plan
|
| 404 |
+
(otherwise ``ValueError``); its clips are reused and only missing clip audio is
|
| 405 |
+
regenerated (the cache may have been wiped by a new Colab session).
|
| 406 |
+
"""
|
| 407 |
+
plan = plan if plan is not None else B1Plan()
|
| 408 |
+
plan_path = workspace.manifests / "b1_plan.json"
|
| 409 |
+
if plan_path.exists():
|
| 410 |
+
existing_plan, existing = load_manifest(workspace)
|
| 411 |
+
if existing_plan != plan:
|
| 412 |
+
raise ValueError(f"existing B1 manifest was built with {existing_plan}, not {plan}")
|
| 413 |
+
if all(clip_audio_path(workspace, c).exists() for c in existing):
|
| 414 |
+
log.info("B1 manifest and clip audio present: %d clips", len(existing))
|
| 415 |
+
return existing
|
| 416 |
+
use_opener = opener if opener is not None else fetch.default_opener
|
| 417 |
+
selections: list[tuple[Selection, Path]] = [
|
| 418 |
+
_fleurs(plan, workspace, use_opener),
|
| 419 |
+
_mediaspeech(plan, workspace, use_opener),
|
| 420 |
+
_yodas(plan, workspace, use_opener),
|
| 421 |
+
]
|
| 422 |
+
clips: list[Clip] = []
|
| 423 |
+
quarantined: list[Quarantined] = []
|
| 424 |
+
for selection, audio_dir in selections:
|
| 425 |
+
quarantined.extend(selection.quarantined)
|
| 426 |
+
for clip in selection.clips:
|
| 427 |
+
write_clip_audio(
|
| 428 |
+
_source_file(clip, audio_dir),
|
| 429 |
+
clip_audio_path(workspace, clip),
|
| 430 |
+
clip.start_ms,
|
| 431 |
+
clip.end_ms,
|
| 432 |
+
)
|
| 433 |
+
clips.append(clip)
|
| 434 |
+
if plan_path.exists():
|
| 435 |
+
_, existing = load_manifest(workspace)
|
| 436 |
+
if [c.clip_id for c in existing] != [c.clip_id for c in clips]:
|
| 437 |
+
raise ValueError("re-derived B1 clips differ from the stored manifest")
|
| 438 |
+
return existing
|
| 439 |
+
_write_jsonl(workspace.manifests / "b1_clips.jsonl", (clip_to_obj(c) for c in clips))
|
| 440 |
+
_write_jsonl(workspace.manifests / "b1_quarantine.jsonl", (asdict(q) for q in quarantined))
|
| 441 |
+
atomic_write_text(plan_path, json.dumps(asdict(plan), indent=2, sort_keys=True) + "\n")
|
| 442 |
+
log.info("B1 manifest written: %d clips, %d quarantined", len(clips), len(quarantined))
|
| 443 |
+
return clips
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
def layer_counts(clips: Sequence[Clip]) -> dict[str, int]:
|
| 447 |
+
counts: dict[str, int] = {}
|
| 448 |
+
for clip in clips:
|
| 449 |
+
counts[clip.layer] = counts.get(clip.layer, 0) + 1
|
| 450 |
+
return counts
|
src/nida/bench/protocol.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""JSONL protocol between the benchmark orchestrator and model workers.
|
| 2 |
+
|
| 3 |
+
Each model family runs in its own virtual environment (dependency pins conflict, see
|
| 4 |
+
docs/benchmarks/B1-B2-tasarim.md §1.3), so the orchestrator and the workers only share
|
| 5 |
+
files. This module is the single definition of those files and must stay stdlib-only:
|
| 6 |
+
it is imported inside every worker environment.
|
| 7 |
+
|
| 8 |
+
Layout for one worker invocation::
|
| 9 |
+
|
| 10 |
+
requests.jsonl one ``WorkRequest`` per line (written by the orchestrator)
|
| 11 |
+
results.jsonl one ``WorkResult`` per line (appended by the worker)
|
| 12 |
+
results.summary.json one ``WorkerSummary`` (written by the worker at the end)
|
| 13 |
+
|
| 14 |
+
Results are appended one line at a time and flushed + fsynced, so an interrupted worker
|
| 15 |
+
leaves a valid prefix plus at most one partial trailing line. On restart the worker
|
| 16 |
+
calls :func:`load_results` with ``repair=True``, which drops the partial line, and
|
| 17 |
+
skips every request whose ``clip_id`` already has a result. Failed items are recorded
|
| 18 |
+
with an ``error`` and are never retried silently: callers decide via ``retry_failed``.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import json
|
| 24 |
+
import os
|
| 25 |
+
from collections.abc import Iterable, Iterator, Mapping
|
| 26 |
+
from dataclasses import asdict, dataclass, field
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
from typing import IO
|
| 29 |
+
|
| 30 |
+
__all__ = [
|
| 31 |
+
"ProtocolError",
|
| 32 |
+
"WorkRequest",
|
| 33 |
+
"WorkResult",
|
| 34 |
+
"WorkerSummary",
|
| 35 |
+
"append_result",
|
| 36 |
+
"drop_results",
|
| 37 |
+
"load_requests",
|
| 38 |
+
"load_results",
|
| 39 |
+
"pending_requests",
|
| 40 |
+
"read_summary",
|
| 41 |
+
"write_requests",
|
| 42 |
+
"write_summary",
|
| 43 |
+
]
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class ProtocolError(ValueError):
|
| 47 |
+
"""A protocol file is malformed or inconsistent."""
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@dataclass(frozen=True)
|
| 51 |
+
class WorkRequest:
|
| 52 |
+
"""One unit of work for a worker.
|
| 53 |
+
|
| 54 |
+
Attributes:
|
| 55 |
+
clip_id: Content-addressed clip id; unique within a requests file.
|
| 56 |
+
audio_path: Absolute path of the input audio file.
|
| 57 |
+
params: Worker-specific string parameters (e.g. ``language``, ``context``,
|
| 58 |
+
``out_path``). Values are strings so the format stays trivially portable.
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
clip_id: str
|
| 62 |
+
audio_path: str
|
| 63 |
+
params: Mapping[str, str] = field(default_factory=dict)
|
| 64 |
+
|
| 65 |
+
def to_json(self) -> str:
|
| 66 |
+
return json.dumps(
|
| 67 |
+
{"clip_id": self.clip_id, "audio_path": self.audio_path, "params": dict(self.params)},
|
| 68 |
+
ensure_ascii=False,
|
| 69 |
+
sort_keys=True,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
@staticmethod
|
| 73 |
+
def from_obj(obj: object, where: str) -> WorkRequest:
|
| 74 |
+
if not isinstance(obj, dict):
|
| 75 |
+
raise ProtocolError(f"{where}: request must be a JSON object")
|
| 76 |
+
clip_id = obj.get("clip_id")
|
| 77 |
+
audio_path = obj.get("audio_path")
|
| 78 |
+
params = obj.get("params", {})
|
| 79 |
+
if not isinstance(clip_id, str) or not clip_id:
|
| 80 |
+
raise ProtocolError(f"{where}: clip_id must be a non-empty string")
|
| 81 |
+
if not isinstance(audio_path, str) or not audio_path:
|
| 82 |
+
raise ProtocolError(f"{where}: audio_path must be a non-empty string")
|
| 83 |
+
if not isinstance(params, dict) or not all(
|
| 84 |
+
isinstance(k, str) and isinstance(v, str) for k, v in params.items()
|
| 85 |
+
):
|
| 86 |
+
raise ProtocolError(f"{where}: params must map strings to strings")
|
| 87 |
+
return WorkRequest(clip_id=clip_id, audio_path=audio_path, params=dict(params))
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@dataclass(frozen=True)
|
| 91 |
+
class WorkResult:
|
| 92 |
+
"""Outcome for one request: exactly one of ``output`` or ``error`` is set.
|
| 93 |
+
|
| 94 |
+
Attributes:
|
| 95 |
+
clip_id: Id of the request this answers.
|
| 96 |
+
output: Worker-specific JSON-compatible payload (e.g. ``{"text": ...}``).
|
| 97 |
+
error: ``"<ExceptionType>: <message>"`` when the item failed.
|
| 98 |
+
infer_seconds: Wall-clock inference time attributed to this item (batch time
|
| 99 |
+
divided by batch size), excluding model loading.
|
| 100 |
+
"""
|
| 101 |
+
|
| 102 |
+
clip_id: str
|
| 103 |
+
output: Mapping[str, object] | None
|
| 104 |
+
error: str | None
|
| 105 |
+
infer_seconds: float
|
| 106 |
+
|
| 107 |
+
def __post_init__(self) -> None:
|
| 108 |
+
if (self.output is None) == (self.error is None):
|
| 109 |
+
raise ProtocolError(f"{self.clip_id}: exactly one of output/error must be set")
|
| 110 |
+
if self.infer_seconds < 0:
|
| 111 |
+
raise ProtocolError(f"{self.clip_id}: infer_seconds must be >= 0")
|
| 112 |
+
|
| 113 |
+
@property
|
| 114 |
+
def ok(self) -> bool:
|
| 115 |
+
return self.error is None
|
| 116 |
+
|
| 117 |
+
def to_json(self) -> str:
|
| 118 |
+
return json.dumps(
|
| 119 |
+
{
|
| 120 |
+
"clip_id": self.clip_id,
|
| 121 |
+
"output": None if self.output is None else dict(self.output),
|
| 122 |
+
"error": self.error,
|
| 123 |
+
"infer_seconds": self.infer_seconds,
|
| 124 |
+
},
|
| 125 |
+
ensure_ascii=False,
|
| 126 |
+
sort_keys=True,
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
@staticmethod
|
| 130 |
+
def from_obj(obj: object, where: str) -> WorkResult:
|
| 131 |
+
if not isinstance(obj, dict):
|
| 132 |
+
raise ProtocolError(f"{where}: result must be a JSON object")
|
| 133 |
+
clip_id = obj.get("clip_id")
|
| 134 |
+
output = obj.get("output")
|
| 135 |
+
error = obj.get("error")
|
| 136 |
+
seconds = obj.get("infer_seconds")
|
| 137 |
+
if not isinstance(clip_id, str) or not clip_id:
|
| 138 |
+
raise ProtocolError(f"{where}: clip_id must be a non-empty string")
|
| 139 |
+
if output is not None and not isinstance(output, dict):
|
| 140 |
+
raise ProtocolError(f"{where}: output must be an object or null")
|
| 141 |
+
if error is not None and not isinstance(error, str):
|
| 142 |
+
raise ProtocolError(f"{where}: error must be a string or null")
|
| 143 |
+
if isinstance(seconds, bool) or not isinstance(seconds, int | float):
|
| 144 |
+
raise ProtocolError(f"{where}: infer_seconds must be a number")
|
| 145 |
+
return WorkResult(clip_id=clip_id, output=output, error=error, infer_seconds=float(seconds))
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
@dataclass(frozen=True)
|
| 149 |
+
class WorkerSummary:
|
| 150 |
+
"""Facts about one worker invocation, written once at the end."""
|
| 151 |
+
|
| 152 |
+
worker: str
|
| 153 |
+
model_id: str
|
| 154 |
+
revision: str
|
| 155 |
+
load_seconds: float
|
| 156 |
+
peak_vram_gb: float | None
|
| 157 |
+
n_ok: int
|
| 158 |
+
n_error: int
|
| 159 |
+
started_at: str
|
| 160 |
+
ended_at: str
|
| 161 |
+
packages: Mapping[str, str] = field(default_factory=dict)
|
| 162 |
+
extra: Mapping[str, str] = field(default_factory=dict)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _fsync_line(handle: IO[str], line: str) -> None:
|
| 166 |
+
handle.write(line + "\n")
|
| 167 |
+
handle.flush()
|
| 168 |
+
os.fsync(handle.fileno())
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def write_requests(path: Path, requests: Iterable[WorkRequest]) -> int:
|
| 172 |
+
"""Write a requests file atomically and return the number of requests.
|
| 173 |
+
|
| 174 |
+
Raises:
|
| 175 |
+
ProtocolError: on duplicate clip ids.
|
| 176 |
+
"""
|
| 177 |
+
from nida.utils.io import atomic_write_text # local import: workers never call this
|
| 178 |
+
|
| 179 |
+
seen: set[str] = set()
|
| 180 |
+
lines: list[str] = []
|
| 181 |
+
for request in requests:
|
| 182 |
+
if request.clip_id in seen:
|
| 183 |
+
raise ProtocolError(f"duplicate clip_id in requests: {request.clip_id}")
|
| 184 |
+
seen.add(request.clip_id)
|
| 185 |
+
lines.append(request.to_json())
|
| 186 |
+
atomic_write_text(path, "".join(line + "\n" for line in lines))
|
| 187 |
+
return len(lines)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _iter_json_lines(path: Path) -> Iterator[tuple[int, str]]:
|
| 191 |
+
with path.open("r", encoding="utf-8") as handle:
|
| 192 |
+
yield from enumerate(handle, start=1)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def load_requests(path: Path) -> list[WorkRequest]:
|
| 196 |
+
"""Read and validate a requests file (no partial lines tolerated)."""
|
| 197 |
+
requests: list[WorkRequest] = []
|
| 198 |
+
seen: set[str] = set()
|
| 199 |
+
for number, line in _iter_json_lines(path):
|
| 200 |
+
where = f"{path}:{number}"
|
| 201 |
+
if not line.endswith("\n"):
|
| 202 |
+
raise ProtocolError(f"{where}: truncated request line")
|
| 203 |
+
try:
|
| 204 |
+
obj = json.loads(line)
|
| 205 |
+
except json.JSONDecodeError as err:
|
| 206 |
+
raise ProtocolError(f"{where}: invalid JSON ({err.msg})") from err
|
| 207 |
+
request = WorkRequest.from_obj(obj, where)
|
| 208 |
+
if request.clip_id in seen:
|
| 209 |
+
raise ProtocolError(f"{where}: duplicate clip_id {request.clip_id}")
|
| 210 |
+
seen.add(request.clip_id)
|
| 211 |
+
requests.append(request)
|
| 212 |
+
return requests
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def load_results(path: Path, *, repair: bool = False) -> dict[str, WorkResult]:
|
| 216 |
+
"""Read a results file into ``{clip_id: result}``.
|
| 217 |
+
|
| 218 |
+
A missing file yields an empty mapping. Only the *last* line may be incomplete (an
|
| 219 |
+
interrupted append): with ``repair=True`` it is removed from the file, otherwise a
|
| 220 |
+
``ProtocolError`` is raised. Any malformed line before the last one, and any
|
| 221 |
+
duplicate clip id, is always an error.
|
| 222 |
+
"""
|
| 223 |
+
if not path.exists():
|
| 224 |
+
return {}
|
| 225 |
+
raw = path.read_bytes()
|
| 226 |
+
# Split at the last newline *in bytes*: an interrupted append may end in the middle of
|
| 227 |
+
# a multi-byte UTF-8 character, so the trailing fragment must not be decoded.
|
| 228 |
+
keep = raw.rfind(b"\n") + 1 # 0 when there is no complete line
|
| 229 |
+
body, trailing = raw[:keep], raw[keep:]
|
| 230 |
+
try:
|
| 231 |
+
text = body.decode("utf-8")
|
| 232 |
+
except UnicodeDecodeError as err:
|
| 233 |
+
raise ProtocolError(f"{path}: complete lines are not valid UTF-8 ({err})") from err
|
| 234 |
+
lines = text.split("\n")[:-1] if text else []
|
| 235 |
+
results: dict[str, WorkResult] = {}
|
| 236 |
+
for number, line in enumerate(lines, start=1):
|
| 237 |
+
where = f"{path}:{number}"
|
| 238 |
+
try:
|
| 239 |
+
obj = json.loads(line)
|
| 240 |
+
except json.JSONDecodeError as err:
|
| 241 |
+
raise ProtocolError(f"{where}: invalid JSON ({err.msg})") from err
|
| 242 |
+
result = WorkResult.from_obj(obj, where)
|
| 243 |
+
if result.clip_id in results:
|
| 244 |
+
raise ProtocolError(f"{where}: duplicate result for {result.clip_id}")
|
| 245 |
+
results[result.clip_id] = result
|
| 246 |
+
if trailing:
|
| 247 |
+
if not repair:
|
| 248 |
+
raise ProtocolError(f"{path}: incomplete trailing line ({len(trailing)} bytes)")
|
| 249 |
+
with path.open("r+b") as handle:
|
| 250 |
+
handle.truncate(keep)
|
| 251 |
+
handle.flush()
|
| 252 |
+
os.fsync(handle.fileno())
|
| 253 |
+
return results
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def pending_requests(
|
| 257 |
+
requests: Iterable[WorkRequest],
|
| 258 |
+
results: Mapping[str, WorkResult],
|
| 259 |
+
*,
|
| 260 |
+
retry_failed: bool = False,
|
| 261 |
+
) -> list[WorkRequest]:
|
| 262 |
+
"""Requests that still need work, in input order.
|
| 263 |
+
|
| 264 |
+
Raises:
|
| 265 |
+
ProtocolError: if ``results`` contains clip ids that are not in ``requests``
|
| 266 |
+
(the results file belongs to a different request set).
|
| 267 |
+
"""
|
| 268 |
+
request_list = list(requests)
|
| 269 |
+
known = {r.clip_id for r in request_list}
|
| 270 |
+
stray = sorted(set(results) - known)
|
| 271 |
+
if stray:
|
| 272 |
+
raise ProtocolError(f"results contain unknown clip ids, e.g. {stray[:3]}")
|
| 273 |
+
pending: list[WorkRequest] = []
|
| 274 |
+
for request in request_list:
|
| 275 |
+
done = results.get(request.clip_id)
|
| 276 |
+
if done is None or (retry_failed and not done.ok):
|
| 277 |
+
pending.append(request)
|
| 278 |
+
return pending
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def append_result(handle: IO[str], result: WorkResult) -> None:
|
| 282 |
+
"""Append one result line and fsync, so an interruption loses at most this line."""
|
| 283 |
+
_fsync_line(handle, result.to_json())
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def write_summary(path: Path, summary: WorkerSummary) -> None:
|
| 287 |
+
"""Write the summary JSON next to the results (stdlib-only atomic replace)."""
|
| 288 |
+
tmp = path.with_name(f".{path.name}.tmp")
|
| 289 |
+
with tmp.open("w", encoding="utf-8") as handle:
|
| 290 |
+
json.dump(asdict(summary), handle, ensure_ascii=False, indent=2, sort_keys=True)
|
| 291 |
+
handle.write("\n")
|
| 292 |
+
handle.flush()
|
| 293 |
+
os.fsync(handle.fileno())
|
| 294 |
+
tmp.replace(path)
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def read_summary(path: Path) -> WorkerSummary:
|
| 298 |
+
"""Read and validate a summary file."""
|
| 299 |
+
obj = json.loads(path.read_text(encoding="utf-8"))
|
| 300 |
+
if not isinstance(obj, dict):
|
| 301 |
+
raise ProtocolError(f"{path}: summary must be a JSON object")
|
| 302 |
+
try:
|
| 303 |
+
return WorkerSummary(**obj)
|
| 304 |
+
except TypeError as err:
|
| 305 |
+
raise ProtocolError(f"{path}: invalid summary fields ({err})") from err
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def drop_results(path: Path, clip_ids: Iterable[str]) -> int:
|
| 309 |
+
"""Remove the given clip ids from a results file (atomic rewrite); returns how many.
|
| 310 |
+
|
| 311 |
+
Used when a result's side effect is gone (e.g. its output file was lost), so the
|
| 312 |
+
item becomes pending again instead of being trusted.
|
| 313 |
+
"""
|
| 314 |
+
doomed = set(clip_ids)
|
| 315 |
+
results = load_results(path, repair=True)
|
| 316 |
+
kept = [r.to_json() + "\n" for cid, r in results.items() if cid not in doomed]
|
| 317 |
+
removed = len(results) - len(kept)
|
| 318 |
+
if removed:
|
| 319 |
+
tmp = path.with_name(f".{path.name}.tmp")
|
| 320 |
+
with tmp.open("w", encoding="utf-8") as handle:
|
| 321 |
+
handle.writelines(kept)
|
| 322 |
+
handle.flush()
|
| 323 |
+
os.fsync(handle.fileno())
|
| 324 |
+
tmp.replace(path)
|
| 325 |
+
return removed
|
src/nida/bench/registry.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Benchmarked systems and their pinned revisions (single source of truth).
|
| 2 |
+
|
| 3 |
+
Every system maps to a ``configs/sources.yaml`` id; :func:`require_approved` refuses to
|
| 4 |
+
run anything whose source is not ``approved`` (golden rule 6). Revisions are the
|
| 5 |
+
Hugging Face commit SHAs checked on 2026-10-08; Omnilingual weights are addressed by
|
| 6 |
+
model card through fairseq2 (no hub revision).
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from collections.abc import Iterable, Mapping
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from types import MappingProxyType
|
| 15 |
+
from typing import Any
|
| 16 |
+
|
| 17 |
+
__all__ = [
|
| 18 |
+
"ASR_SYSTEMS",
|
| 19 |
+
"DATA_PINS",
|
| 20 |
+
"QUALITY_SYSTEM",
|
| 21 |
+
"VAE_SYSTEMS",
|
| 22 |
+
"SourceNotApprovedError",
|
| 23 |
+
"System",
|
| 24 |
+
"load_sources",
|
| 25 |
+
"require_approved",
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@dataclass(frozen=True)
|
| 30 |
+
class System:
|
| 31 |
+
"""One model under test.
|
| 32 |
+
|
| 33 |
+
Attributes:
|
| 34 |
+
system_id: Short id used in file names and reports.
|
| 35 |
+
source_id: Id in ``configs/sources.yaml``.
|
| 36 |
+
worker: Worker module name in ``nida.bench.workers``.
|
| 37 |
+
model_id: Hub id / model card passed to the worker.
|
| 38 |
+
revision: Pinned revision (``""`` when the worker fetches by model card).
|
| 39 |
+
batch_size: Worker batch size on an L4 (22 GB).
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
system_id: str
|
| 43 |
+
source_id: str
|
| 44 |
+
worker: str
|
| 45 |
+
model_id: str
|
| 46 |
+
revision: str
|
| 47 |
+
batch_size: int
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _index(systems: Iterable[System]) -> Mapping[str, System]:
|
| 51 |
+
out: dict[str, System] = {}
|
| 52 |
+
for system in systems:
|
| 53 |
+
if system.system_id in out:
|
| 54 |
+
raise ValueError(f"duplicate system id {system.system_id}")
|
| 55 |
+
out[system.system_id] = system
|
| 56 |
+
return MappingProxyType(out)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
ASR_SYSTEMS: Mapping[str, System] = _index(
|
| 60 |
+
(
|
| 61 |
+
System(
|
| 62 |
+
"qwen3_asr_1_7b",
|
| 63 |
+
"qwen3_asr_1_7b",
|
| 64 |
+
"asr_qwen",
|
| 65 |
+
"Qwen/Qwen3-ASR-1.7B",
|
| 66 |
+
"7278e1e70fe206f11671096ffdd38061171dd6e5",
|
| 67 |
+
16,
|
| 68 |
+
),
|
| 69 |
+
System(
|
| 70 |
+
"qwen3_asr_0_6b",
|
| 71 |
+
"qwen3_asr_0_6b",
|
| 72 |
+
"asr_qwen",
|
| 73 |
+
"Qwen/Qwen3-ASR-0.6B",
|
| 74 |
+
"5eb144179a02acc5e5ba31e748d22b0cf3e303b0",
|
| 75 |
+
16,
|
| 76 |
+
),
|
| 77 |
+
System("omniasr_llm_7b", "omniasr_llm_7b", "asr_omni", "omniASR_LLM_7B_v2", "", 4),
|
| 78 |
+
System("omniasr_ctc_1b", "omniasr_ctc_1b", "asr_omni", "omniASR_CTC_1B_v2", "", 16),
|
| 79 |
+
System(
|
| 80 |
+
"whisper_large_v3",
|
| 81 |
+
"whisper_large_v3",
|
| 82 |
+
"asr_hf",
|
| 83 |
+
"openai/whisper-large-v3",
|
| 84 |
+
"06f233fe06e710322aca913c1bc4249a0d71fce1",
|
| 85 |
+
8,
|
| 86 |
+
),
|
| 87 |
+
System(
|
| 88 |
+
"buzzasr_turkish",
|
| 89 |
+
"buzzasr_turkish",
|
| 90 |
+
"asr_hf",
|
| 91 |
+
"BuzzASR/turkish",
|
| 92 |
+
"928b47596e31776dab9daf1e606ecf76234b9ecd",
|
| 93 |
+
8,
|
| 94 |
+
),
|
| 95 |
+
)
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
VAE_SYSTEMS: Mapping[str, System] = _index(
|
| 99 |
+
(
|
| 100 |
+
System(
|
| 101 |
+
"voxcpm2_audiovae_v2",
|
| 102 |
+
"voxcpm2_audiovae_v2",
|
| 103 |
+
"vae_voxcpm",
|
| 104 |
+
"openbmb/VoxCPM2",
|
| 105 |
+
"32279effe8c19989596f05d353d1447f51d9e915",
|
| 106 |
+
1,
|
| 107 |
+
),
|
| 108 |
+
)
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
QUALITY_SYSTEM = System(
|
| 112 |
+
"audiobox_aesthetics",
|
| 113 |
+
"audiobox_aesthetics",
|
| 114 |
+
"quality_audiobox",
|
| 115 |
+
"facebook/audiobox-aesthetics",
|
| 116 |
+
"9b1dd8e5df9af7216e836a98974fe3b82c56ded6",
|
| 117 |
+
16,
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
#: Pinned dataset revisions / URLs (checked 2026-10-08).
|
| 121 |
+
DATA_PINS: Mapping[str, str] = MappingProxyType(
|
| 122 |
+
{
|
| 123 |
+
"fleurs_revision": "70bb2e84b976b7e960aa89f1c648e09c59f894dd",
|
| 124 |
+
"yodas_revision": "52c5a1b9730a136bfd7c4513d4962c70d4e50530",
|
| 125 |
+
"mediaspeech_url": "https://www.openslr.org/resources/108/TR.tgz",
|
| 126 |
+
"expresso_url": "https://dl.fbaipublicfiles.com/textless_nlp/expresso/data/expresso.tar",
|
| 127 |
+
}
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class SourceNotApprovedError(RuntimeError):
|
| 132 |
+
"""A source is missing from the registry or not approved by the user."""
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def load_sources(path: Path) -> dict[str, Mapping[str, Any]]:
|
| 136 |
+
"""Read ``configs/sources.yaml`` into ``{id: record}``.
|
| 137 |
+
|
| 138 |
+
Raises:
|
| 139 |
+
ValueError: on a malformed file or duplicate ids.
|
| 140 |
+
"""
|
| 141 |
+
import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
|
| 142 |
+
|
| 143 |
+
data: Any = yaml.safe_load(path.read_text(encoding="utf-8")) # Any: untyped YAML tree
|
| 144 |
+
if not isinstance(data, dict) or not isinstance(data.get("sources"), list):
|
| 145 |
+
raise ValueError(f"{path}: expected a mapping with a 'sources' list")
|
| 146 |
+
records: dict[str, Mapping[str, Any]] = {}
|
| 147 |
+
for record in data["sources"]:
|
| 148 |
+
if not isinstance(record, dict) or not isinstance(record.get("id"), str):
|
| 149 |
+
raise ValueError(f"{path}: every source needs a string 'id'")
|
| 150 |
+
if record["id"] in records:
|
| 151 |
+
raise ValueError(f"{path}: duplicate source id {record['id']}")
|
| 152 |
+
records[record["id"]] = record
|
| 153 |
+
return records
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def require_approved(source_ids: Iterable[str], sources: Mapping[str, Mapping[str, Any]]) -> None:
|
| 157 |
+
"""Raise unless every id exists in the registry with ``status: approved``."""
|
| 158 |
+
problems: list[str] = []
|
| 159 |
+
for source_id in sorted(set(source_ids)):
|
| 160 |
+
record = sources.get(source_id)
|
| 161 |
+
if record is None:
|
| 162 |
+
problems.append(f"{source_id}: not in sources.yaml")
|
| 163 |
+
elif record.get("status") != "approved":
|
| 164 |
+
problems.append(f"{source_id}: status is {record.get('status')!r}")
|
| 165 |
+
if problems:
|
| 166 |
+
raise SourceNotApprovedError("sources not approved: " + "; ".join(problems))
|
src/nida/bench/report.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Markdown rendering of B1 / B2 results (Turkish, for the user and the ADRs)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from collections.abc import Mapping, Sequence
|
| 6 |
+
|
| 7 |
+
__all__ = ["render_b1", "render_b2", "render_b2_panel"]
|
| 8 |
+
|
| 9 |
+
_B1_LAYER_TITLES = {
|
| 10 |
+
"A": "A — temiz okuma (FLEURS)",
|
| 11 |
+
"B": "B — medya (MediaSpeech)",
|
| 12 |
+
"AB": "A+B",
|
| 13 |
+
"C": "C — doğal + dolgu (YODAS, kullanıcı referansı)",
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _fmt(value: object, digits: int = 3, percent: bool = False) -> str:
|
| 18 |
+
if value is None:
|
| 19 |
+
return "—"
|
| 20 |
+
if isinstance(value, bool):
|
| 21 |
+
return "evet" if value else "hayır"
|
| 22 |
+
if isinstance(value, int):
|
| 23 |
+
return str(value)
|
| 24 |
+
if isinstance(value, float):
|
| 25 |
+
return f"{100 * value:.1f}%" if percent else f"{value:.{digits}f}"
|
| 26 |
+
return str(value)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _pct(value: object) -> str:
|
| 30 |
+
return _fmt(value, percent=True)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _row(cells: Sequence[str]) -> str:
|
| 34 |
+
return "| " + " | ".join(cells) + " |"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _header(cells: Sequence[str]) -> list[str]:
|
| 38 |
+
return [_row(cells), _row(["---"] * len(cells))]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def render_b1(scores: Mapping[str, object], choice: Mapping[str, object]) -> str:
|
| 42 |
+
systems = scores["systems"]
|
| 43 |
+
assert isinstance(systems, Mapping)
|
| 44 |
+
lines = [
|
| 45 |
+
"# B1 — Türkçe ASR karşılaştırması",
|
| 46 |
+
"",
|
| 47 |
+
"Notlar: (1) `buzzasr_turkish` FLEURS ile eğitildi; A katmanında (FLEURS test) iyimser "
|
| 48 |
+
"olabilir, bu yüzden karar kuralı ayrıca yalnız B katmanıyla da uygulanır. "
|
| 49 |
+
"(2) Omnilingual CTC dil koşulu almaz (lang yok sayılır). (3) Başarısız öğeler boş "
|
| 50 |
+
"hipotez (tamamı silme) sayılır; halüsinasyon oranına girmez. (4) RTF, sistemin kendi "
|
| 51 |
+
"toplu iş (batch) boyutuyla ölçüldü; doğrudan hız karşılaştırması değildir.",
|
| 52 |
+
"",
|
| 53 |
+
]
|
| 54 |
+
if "n_c_reviewed" in scores:
|
| 55 |
+
lines += [f"C katmanında incelenmiş (referanslı) klip: {scores['n_c_reviewed']}", ""]
|
| 56 |
+
for layer, title in _B1_LAYER_TITLES.items():
|
| 57 |
+
if not all(layer in v for v in systems.values()):
|
| 58 |
+
continue
|
| 59 |
+
lines += [f"## {title}", ""]
|
| 60 |
+
lines += _header(
|
| 61 |
+
[
|
| 62 |
+
"Sistem",
|
| 63 |
+
"n",
|
| 64 |
+
"başarısız",
|
| 65 |
+
"CER içerik",
|
| 66 |
+
"WER içerik",
|
| 67 |
+
"CER tam",
|
| 68 |
+
"WER tam",
|
| 69 |
+
"halüsinasyon",
|
| 70 |
+
"dolgu geri çağırma",
|
| 71 |
+
"dolgu geri çağırma (LOO)",
|
| 72 |
+
"LOO payda",
|
| 73 |
+
"dolgu kesinlik",
|
| 74 |
+
"rakamlı hipotez",
|
| 75 |
+
"RTF",
|
| 76 |
+
]
|
| 77 |
+
)
|
| 78 |
+
for system_id in sorted(systems):
|
| 79 |
+
r = systems[system_id][layer]
|
| 80 |
+
lines.append(
|
| 81 |
+
_row(
|
| 82 |
+
[
|
| 83 |
+
system_id,
|
| 84 |
+
_fmt(r["n"]),
|
| 85 |
+
_fmt(r["n_failed"]),
|
| 86 |
+
_pct(r["content_cer"]),
|
| 87 |
+
_pct(r["content_wer"]),
|
| 88 |
+
_pct(r["full_cer"]),
|
| 89 |
+
_pct(r["full_wer"]),
|
| 90 |
+
_pct(r["hallucination_rate"]),
|
| 91 |
+
_pct(r["filler_recall"]),
|
| 92 |
+
_pct(r.get("filler_recall_loo")),
|
| 93 |
+
_fmt(r.get("filler_loo_ref_total")),
|
| 94 |
+
_pct(r["filler_precision"]),
|
| 95 |
+
_fmt(r.get("hyp_with_digits")),
|
| 96 |
+
_fmt(r["rtf"]),
|
| 97 |
+
]
|
| 98 |
+
)
|
| 99 |
+
)
|
| 100 |
+
lines.append("")
|
| 101 |
+
workers = {k: v["_worker"] for k, v in systems.items() if "_worker" in v}
|
| 102 |
+
if workers:
|
| 103 |
+
lines += ["## Kaynak kullanımı", ""]
|
| 104 |
+
lines += _header(["Sistem", "batch", "tepe VRAM (GB)", "yükleme (s)", "hata"])
|
| 105 |
+
for system_id, w in sorted(workers.items()):
|
| 106 |
+
lines.append(
|
| 107 |
+
_row(
|
| 108 |
+
[
|
| 109 |
+
system_id,
|
| 110 |
+
_fmt(w.get("batch_size")),
|
| 111 |
+
_fmt(w.get("peak_vram_gb"), 1),
|
| 112 |
+
_fmt(w.get("load_seconds"), 0),
|
| 113 |
+
_fmt(w.get("n_error")),
|
| 114 |
+
]
|
| 115 |
+
)
|
| 116 |
+
)
|
| 117 |
+
lines.append("")
|
| 118 |
+
boot = scores.get("bootstrap_content_cer")
|
| 119 |
+
if isinstance(boot, Mapping):
|
| 120 |
+
for layer, rows in boot.items():
|
| 121 |
+
lines += [f"## Eşleştirilmiş bootstrap — içerik CER farkı ({layer}, a − b, %95 GA)", ""]
|
| 122 |
+
lines += _header(["a", "b", "fark", "alt", "üst", "anlamlı mı"])
|
| 123 |
+
for r in rows:
|
| 124 |
+
significant = r["low"] > 0 or r["high"] < 0
|
| 125 |
+
lines.append(
|
| 126 |
+
_row(
|
| 127 |
+
[
|
| 128 |
+
r["a"],
|
| 129 |
+
r["b"],
|
| 130 |
+
_pct(r["delta"]),
|
| 131 |
+
_pct(r["low"]),
|
| 132 |
+
_pct(r["high"]),
|
| 133 |
+
"evet" if significant else "hayır",
|
| 134 |
+
]
|
| 135 |
+
)
|
| 136 |
+
)
|
| 137 |
+
lines.append("")
|
| 138 |
+
lines += ["## Önceden sabitlenmiş karar kuralına göre öneri", ""]
|
| 139 |
+
if not choice.get("ready"):
|
| 140 |
+
lines.append(f"Henüz uygulanamaz: {choice.get('reason')}.")
|
| 141 |
+
else:
|
| 142 |
+
qualified = choice.get("qualified_by_filler_recall")
|
| 143 |
+
qualified_text = ", ".join(qualified) if isinstance(qualified, list) else ""
|
| 144 |
+
lines += [
|
| 145 |
+
f"- Birincil: **{choice['primary']}**",
|
| 146 |
+
f"- İkincil (farklı aile): **{choice['secondary'] or 'yok (başka aile yok)'}**",
|
| 147 |
+
f"- Dolgu için ayrı yöntem gerekir mi: {_fmt(choice['needs_filler_method'])}",
|
| 148 |
+
f"- Dolgu geri çağırma eşiğini geçenler: {qualified_text or '—'}",
|
| 149 |
+
]
|
| 150 |
+
without = choice.get("without_fleurs")
|
| 151 |
+
if isinstance(without, Mapping):
|
| 152 |
+
changed = choice.get("pick_changes_without_fleurs")
|
| 153 |
+
lines += [
|
| 154 |
+
"",
|
| 155 |
+
"Aynı kural yalnız B katmanıyla (FLEURS hariç): "
|
| 156 |
+
f"birincil **{without['primary']}**, "
|
| 157 |
+
f"ikincil **{without['secondary'] or 'yok'}** — "
|
| 158 |
+
f"seçim {'DEĞİŞİYOR' if changed else 'değişmiyor'}.",
|
| 159 |
+
]
|
| 160 |
+
lines.append("")
|
| 161 |
+
return "\n".join(lines)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def render_b2(scores: Mapping[str, object]) -> str:
|
| 165 |
+
subsets = scores["subsets"]
|
| 166 |
+
assert isinstance(subsets, Mapping)
|
| 167 |
+
lines = [
|
| 168 |
+
"# B2 — VAE yeniden kurma (VoxCPM2 AudioVAE V2)",
|
| 169 |
+
"",
|
| 170 |
+
"Mel L1: log10 mel genliği farkı (düşük = iyi). HB: 8–24 kHz bandı. "
|
| 171 |
+
"`band16k`: 48→16→48 kHz yeniden örnekleme taban çizgisi (8 kHz üstü yok).",
|
| 172 |
+
"Audiobox 16 kHz'de çalışır; yüksek bant kalitesini ölçmez. HB LSD yalnız etkin "
|
| 173 |
+
"karelerde (en yüksek karenin 40 dB altına kadar) ortalanır. ΔPQ/ΔCE eşleştirilmiş "
|
| 174 |
+
"kliplerin farklarının ortalamasıdır (VAE − orijinal).",
|
| 175 |
+
"",
|
| 176 |
+
]
|
| 177 |
+
skipped = scores.get("skipped")
|
| 178 |
+
if isinstance(skipped, Mapping) and skipped:
|
| 179 |
+
lines += ["Atlanan: " + ", ".join(f"{k}={v}" for k, v in sorted(skipped.items())), ""]
|
| 180 |
+
lines += _header(
|
| 181 |
+
[
|
| 182 |
+
"Alt küme",
|
| 183 |
+
"n",
|
| 184 |
+
"puanlanan",
|
| 185 |
+
"mel16 VAE",
|
| 186 |
+
"mel48 VAE",
|
| 187 |
+
"mel48 band16k",
|
| 188 |
+
"HB LSD VAE (dB)",
|
| 189 |
+
"HB LSD band16k (dB)",
|
| 190 |
+
"HB enerji oranı VAE",
|
| 191 |
+
"ΔPQ",
|
| 192 |
+
"ΔCE",
|
| 193 |
+
"CER orijinal",
|
| 194 |
+
"CER VAE",
|
| 195 |
+
"gecikme (ms)",
|
| 196 |
+
]
|
| 197 |
+
)
|
| 198 |
+
for subset, row in subsets.items():
|
| 199 |
+
assert isinstance(row, Mapping)
|
| 200 |
+
lines.append(
|
| 201 |
+
_row(
|
| 202 |
+
[
|
| 203 |
+
subset,
|
| 204 |
+
_fmt(row["n"]),
|
| 205 |
+
_fmt(row.get("n_scored")),
|
| 206 |
+
_fmt(row.get("vae_mel16_l1")),
|
| 207 |
+
_fmt(row.get("vae_mel48_l1")),
|
| 208 |
+
_fmt(row.get("band16k_mel48_l1")),
|
| 209 |
+
_fmt(row.get("vae_hb_lsd_db"), 1),
|
| 210 |
+
_fmt(row.get("band16k_hb_lsd_db"), 1),
|
| 211 |
+
_fmt(row.get("vae_hb_energy_ratio"), 2),
|
| 212 |
+
_fmt(row.get("delta_PQ"), 2),
|
| 213 |
+
_fmt(row.get("delta_CE"), 2),
|
| 214 |
+
_pct(row.get("cer_orig")),
|
| 215 |
+
_pct(row.get("cer_vae")),
|
| 216 |
+
_fmt(row.get("vae_delay_ms"), 1),
|
| 217 |
+
]
|
| 218 |
+
)
|
| 219 |
+
)
|
| 220 |
+
lines.append("")
|
| 221 |
+
return "\n".join(lines)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def render_b2_panel(scores: Mapping[str, object]) -> str:
|
| 225 |
+
"""Un-blinded B2 listening-panel results with the pre-registered pass/fail per subset."""
|
| 226 |
+
subsets = scores["subsets"]
|
| 227 |
+
rule = scores.get("rule")
|
| 228 |
+
assert isinstance(subsets, Mapping) and isinstance(rule, Mapping)
|
| 229 |
+
lines = [
|
| 230 |
+
"# B2 — Dinleme panosu (kör A/B, çözülmüş)",
|
| 231 |
+
"",
|
| 232 |
+
f"Kural: ifade korunma oranı ≥ {_pct(rule['min_preserved_share'])} "
|
| 233 |
+
'("kısmen" korunmamış sayılır; tr hariç) ve kalite farkı (orijinal − VAE) '
|
| 234 |
+
f"≤ {_fmt(rule['max_quality_gap'], 1)}.",
|
| 235 |
+
"",
|
| 236 |
+
]
|
| 237 |
+
lines += _header(
|
| 238 |
+
[
|
| 239 |
+
"Alt küme",
|
| 240 |
+
"yanıt",
|
| 241 |
+
"korunmuş",
|
| 242 |
+
"kısmen",
|
| 243 |
+
"kalite orijinal",
|
| 244 |
+
"kalite VAE",
|
| 245 |
+
"fark",
|
| 246 |
+
"geçti",
|
| 247 |
+
]
|
| 248 |
+
)
|
| 249 |
+
for subset, row in subsets.items():
|
| 250 |
+
assert isinstance(row, Mapping)
|
| 251 |
+
lines.append(
|
| 252 |
+
_row(
|
| 253 |
+
[
|
| 254 |
+
subset,
|
| 255 |
+
_fmt(row["n_answered"]),
|
| 256 |
+
_pct(row["preserved_share"]),
|
| 257 |
+
_pct(row["partial_share"]),
|
| 258 |
+
_fmt(row["quality_orig"], 2),
|
| 259 |
+
_fmt(row["quality_vae"], 2),
|
| 260 |
+
_fmt(row["quality_gap"], 2),
|
| 261 |
+
_fmt(row["passes"]),
|
| 262 |
+
]
|
| 263 |
+
)
|
| 264 |
+
)
|
| 265 |
+
lines += ["", f"Tümü geçti mi: **{_fmt(scores.get('all_pass'))}**", ""]
|
| 266 |
+
for key, label in (
|
| 267 |
+
("missing_subsets", "Panoda hiç olmayan ifadeli alt kümeler"),
|
| 268 |
+
("unanswered", "Yanıtlanmamış satırlar (alt küme: adet)"),
|
| 269 |
+
("too_few_items", "En az adedin altında kalan alt kümeler"),
|
| 270 |
+
("excluded_missing_reconstruction", "Yeniden kurulamadığı için panoya alınmayan"),
|
| 271 |
+
):
|
| 272 |
+
value = scores.get(key)
|
| 273 |
+
if value:
|
| 274 |
+
lines.append(f"- {label}: {value}")
|
| 275 |
+
lines.append("")
|
| 276 |
+
return "\n".join(lines)
|
src/nida/bench/textmetrics.py
ADDED
|
@@ -0,0 +1,464 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Text metrics for scoring ASR hypotheses against Turkish references (benchmark B1).
|
| 2 |
+
|
| 3 |
+
Pure standard-library implementation. The pipeline is:
|
| 4 |
+
|
| 5 |
+
1. :func:`normalize_for_scoring` turns a raw string into scoring tokens (NFC, Turkish
|
| 6 |
+
lowercasing, punctuation removal, filler canonicalisation per tag-spec section 4).
|
| 7 |
+
2. :func:`cer_counts` / :func:`wer_counts` return :class:`EditCounts` per utterance.
|
| 8 |
+
3. :func:`corpus_rate` aggregates counts over a corpus (sum of errors over sum of reference
|
| 9 |
+
lengths, *not* a mean of per-utterance rates).
|
| 10 |
+
4. :func:`filler_score` measures filler recall / precision, and :func:`paired_bootstrap`
|
| 11 |
+
gives a confidence interval for the difference between two systems.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import math
|
| 17 |
+
import random
|
| 18 |
+
import unicodedata
|
| 19 |
+
from collections import Counter
|
| 20 |
+
from collections.abc import Hashable, Iterable, Sequence
|
| 21 |
+
from dataclasses import dataclass
|
| 22 |
+
from typing import TypeVar
|
| 23 |
+
|
| 24 |
+
from nida.text.fillers import FILLERS, canonical_filler
|
| 25 |
+
from nida.text.turkish import nfc, tr_lower
|
| 26 |
+
|
| 27 |
+
__all__ = [
|
| 28 |
+
"FILLERS",
|
| 29 |
+
"BootstrapResult",
|
| 30 |
+
"EditCounts",
|
| 31 |
+
"FillerScore",
|
| 32 |
+
"canonical_filler",
|
| 33 |
+
"cer_counts",
|
| 34 |
+
"corpus_rate",
|
| 35 |
+
"edit_counts",
|
| 36 |
+
"filler_counts",
|
| 37 |
+
"filler_score",
|
| 38 |
+
"has_digits",
|
| 39 |
+
"is_hallucination",
|
| 40 |
+
"normalize_english",
|
| 41 |
+
"normalize_for_scoring",
|
| 42 |
+
"paired_bootstrap",
|
| 43 |
+
"strip_fillers",
|
| 44 |
+
"wer_counts",
|
| 45 |
+
]
|
| 46 |
+
|
| 47 |
+
T = TypeVar("T", bound=Hashable)
|
| 48 |
+
|
| 49 |
+
# Apostrophe-like characters that are deleted (not replaced by a space), so that
|
| 50 |
+
# "Ankara'da" becomes "ankarada": U+0027, U+2019 and U+02BC (a modifier letter, category Lm).
|
| 51 |
+
_APOSTROPHES: frozenset[str] = frozenset({"'", "\u2019", "\u02bc"})
|
| 52 |
+
|
| 53 |
+
# Circumflex letters are optional in modern Turkish spelling ("hala"/"hâlâ"); systems and
|
| 54 |
+
# references use them inconsistently, so scoring folds them to the plain vowel.
|
| 55 |
+
_CIRCUMFLEX_FOLD = str.maketrans({"â": "a", "î": "i", "û": "u"})
|
| 56 |
+
|
| 57 |
+
# --------------------------------------------------------------------------------------
|
| 58 |
+
# Normalisation and fillers
|
| 59 |
+
# --------------------------------------------------------------------------------------
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def normalize_for_scoring(text: str) -> list[str]:
|
| 63 |
+
"""Normalise ``text`` into scoring tokens.
|
| 64 |
+
|
| 65 |
+
Steps: NFC, Turkish lowercase, fold circumflexes (â/î/û → a/i/u), delete apostrophes
|
| 66 |
+
(U+0027, U+2019, U+02BC), turn whitespace and every other punctuation (``P*``) or
|
| 67 |
+
symbol (``S*``) character into a space, split on whitespace, and map each token
|
| 68 |
+
through :func:`canonical_filler`. Letters and digits are kept as-is. The cleaned
|
| 69 |
+
string is re-composed to NFC before splitting, because deleting a character can leave
|
| 70 |
+
a base letter next to a combining mark.
|
| 71 |
+
|
| 72 |
+
Args:
|
| 73 |
+
text: Raw reference or hypothesis text.
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
List of tokens (possibly empty).
|
| 77 |
+
|
| 78 |
+
Examples:
|
| 79 |
+
>>> normalize_for_scoring("Ankara'da, ııı... Hımm!")
|
| 80 |
+
['ankarada', 'ıı', 'hmm']
|
| 81 |
+
"""
|
| 82 |
+
chars: list[str] = []
|
| 83 |
+
for ch in tr_lower(text).translate(_CIRCUMFLEX_FOLD):
|
| 84 |
+
if ch in _APOSTROPHES:
|
| 85 |
+
continue
|
| 86 |
+
if ch.isspace():
|
| 87 |
+
chars.append(" ")
|
| 88 |
+
continue
|
| 89 |
+
# Punctuation and symbols become word boundaries, never glue: "merhaba,dünya" must
|
| 90 |
+
# score as two words, as in common ASR normalisers.
|
| 91 |
+
if unicodedata.category(ch)[0] in ("P", "S"):
|
| 92 |
+
chars.append(" ")
|
| 93 |
+
else:
|
| 94 |
+
chars.append(ch)
|
| 95 |
+
tokens = nfc("".join(chars)).split()
|
| 96 |
+
result: list[str] = []
|
| 97 |
+
for token in tokens:
|
| 98 |
+
canonical = canonical_filler(token)
|
| 99 |
+
result.append(token if canonical is None else canonical)
|
| 100 |
+
return result
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def strip_fillers(tokens: Sequence[str]) -> list[str]:
|
| 104 |
+
"""Return ``tokens`` without canonical fillers.
|
| 105 |
+
|
| 106 |
+
Tokens must come from :func:`normalize_for_scoring` (already canonical); membership in
|
| 107 |
+
:data:`FILLERS` is tested literally.
|
| 108 |
+
"""
|
| 109 |
+
return [token for token in tokens if token not in FILLERS]
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def has_digits(text: str) -> bool:
|
| 113 |
+
"""Return True if ``text`` (after NFC) contains any Unicode decimal digit."""
|
| 114 |
+
return any(ch.isdecimal() for ch in nfc(text))
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
# --------------------------------------------------------------------------------------
|
| 118 |
+
# Edit distance
|
| 119 |
+
# --------------------------------------------------------------------------------------
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@dataclass(frozen=True)
|
| 123 |
+
class EditCounts:
|
| 124 |
+
"""Substitution / deletion / insertion counts against a reference of ``ref_len`` units.
|
| 125 |
+
|
| 126 |
+
Attributes:
|
| 127 |
+
substitutions: Reference units replaced by a different hypothesis unit.
|
| 128 |
+
deletions: Reference units missing from the hypothesis.
|
| 129 |
+
insertions: Extra hypothesis units not in the reference.
|
| 130 |
+
ref_len: Number of reference units (characters or tokens).
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
substitutions: int
|
| 134 |
+
deletions: int
|
| 135 |
+
insertions: int
|
| 136 |
+
ref_len: int
|
| 137 |
+
|
| 138 |
+
def __post_init__(self) -> None:
|
| 139 |
+
for name in ("substitutions", "deletions", "insertions", "ref_len"):
|
| 140 |
+
value = getattr(self, name)
|
| 141 |
+
if value < 0:
|
| 142 |
+
raise ValueError(f"EditCounts.{name} must be >= 0, got {value}")
|
| 143 |
+
|
| 144 |
+
@property
|
| 145 |
+
def errors(self) -> int:
|
| 146 |
+
"""Total number of edits (S + D + I)."""
|
| 147 |
+
return self.substitutions + self.deletions + self.insertions
|
| 148 |
+
|
| 149 |
+
def rate(self) -> float | None:
|
| 150 |
+
"""Error rate ``errors / ref_len``; ``None`` when ``ref_len`` is 0."""
|
| 151 |
+
if self.ref_len == 0:
|
| 152 |
+
return None
|
| 153 |
+
return self.errors / self.ref_len
|
| 154 |
+
|
| 155 |
+
def __add__(self, other: object) -> EditCounts:
|
| 156 |
+
"""Component-wise sum, for corpus aggregation."""
|
| 157 |
+
if not isinstance(other, EditCounts):
|
| 158 |
+
return NotImplemented
|
| 159 |
+
return EditCounts(
|
| 160 |
+
substitutions=self.substitutions + other.substitutions,
|
| 161 |
+
deletions=self.deletions + other.deletions,
|
| 162 |
+
insertions=self.insertions + other.insertions,
|
| 163 |
+
ref_len=self.ref_len + other.ref_len,
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def edit_counts(ref: Sequence[T], hyp: Sequence[T]) -> EditCounts:
|
| 168 |
+
"""Levenshtein alignment with unit costs, returning the S/D/I split.
|
| 169 |
+
|
| 170 |
+
Time is O(len(ref) * len(hyp)); memory is O(len(hyp)) because each DP cell carries its
|
| 171 |
+
own (cost, S, D, I) tuple instead of keeping the full matrix for backtracking.
|
| 172 |
+
|
| 173 |
+
Tie-breaking (deterministic): walking back from the end of both sequences, whenever
|
| 174 |
+
several predecessors give the same minimal total cost the alignment prefers, in order,
|
| 175 |
+
a substitution/match (diagonal), then a deletion (a reference unit with no hypothesis
|
| 176 |
+
unit), then an insertion. Carrying counts forward along the per-cell choice is
|
| 177 |
+
equivalent to that backtracking, because each cell's choice depends only on that cell.
|
| 178 |
+
|
| 179 |
+
Args:
|
| 180 |
+
ref: Reference units (characters or tokens).
|
| 181 |
+
hyp: Hypothesis units.
|
| 182 |
+
|
| 183 |
+
Returns:
|
| 184 |
+
Counts with ``ref_len == len(ref)``.
|
| 185 |
+
"""
|
| 186 |
+
num_hyp = len(hyp)
|
| 187 |
+
# Each cell: (cost, substitutions, deletions, insertions).
|
| 188 |
+
prev: list[tuple[int, int, int, int]] = [(j, 0, 0, j) for j in range(num_hyp + 1)]
|
| 189 |
+
for i in range(1, len(ref) + 1):
|
| 190 |
+
ref_unit = ref[i - 1]
|
| 191 |
+
row: list[tuple[int, int, int, int]] = [(i, 0, i, 0)]
|
| 192 |
+
for j in range(1, num_hyp + 1):
|
| 193 |
+
diag = prev[j - 1]
|
| 194 |
+
up = prev[j]
|
| 195 |
+
left = row[j - 1]
|
| 196 |
+
diag_cost = diag[0] + (0 if ref_unit == hyp[j - 1] else 1)
|
| 197 |
+
up_cost = up[0] + 1
|
| 198 |
+
left_cost = left[0] + 1
|
| 199 |
+
if diag_cost <= up_cost and diag_cost <= left_cost:
|
| 200 |
+
mismatch = diag_cost - diag[0]
|
| 201 |
+
row.append((diag_cost, diag[1] + mismatch, diag[2], diag[3]))
|
| 202 |
+
elif up_cost <= left_cost:
|
| 203 |
+
row.append((up_cost, up[1], up[2] + 1, up[3]))
|
| 204 |
+
else:
|
| 205 |
+
row.append((left_cost, left[1], left[2], left[3] + 1))
|
| 206 |
+
prev = row
|
| 207 |
+
_, subs, dels, ins = prev[num_hyp]
|
| 208 |
+
return EditCounts(substitutions=subs, deletions=dels, insertions=ins, ref_len=len(ref))
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def cer_counts(ref_tokens: Sequence[str], hyp_tokens: Sequence[str]) -> EditCounts:
|
| 212 |
+
"""Character-level edit counts over ``" ".join(tokens)`` (spaces count as characters)."""
|
| 213 |
+
return edit_counts(" ".join(ref_tokens), " ".join(hyp_tokens))
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def wer_counts(ref_tokens: Sequence[str], hyp_tokens: Sequence[str]) -> EditCounts:
|
| 217 |
+
"""Token-level (word) edit counts."""
|
| 218 |
+
return edit_counts(ref_tokens, hyp_tokens)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def corpus_rate(counts: Iterable[EditCounts]) -> float | None:
|
| 222 |
+
"""Corpus-level error rate: ``sum(errors) / sum(ref_len)``.
|
| 223 |
+
|
| 224 |
+
Returns ``None`` if the total reference length is 0.
|
| 225 |
+
"""
|
| 226 |
+
total_errors = 0
|
| 227 |
+
total_ref = 0
|
| 228 |
+
for item in counts:
|
| 229 |
+
total_errors += item.errors
|
| 230 |
+
total_ref += item.ref_len
|
| 231 |
+
if total_ref == 0:
|
| 232 |
+
return None
|
| 233 |
+
return total_errors / total_ref
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
# --------------------------------------------------------------------------------------
|
| 237 |
+
# Fillers
|
| 238 |
+
# --------------------------------------------------------------------------------------
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def filler_counts(tokens: Sequence[str]) -> Counter[str]:
|
| 242 |
+
"""Count canonical fillers in ``tokens`` (tokens from :func:`normalize_for_scoring`)."""
|
| 243 |
+
return Counter(token for token in tokens if token in FILLERS)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
@dataclass(frozen=True)
|
| 247 |
+
class FillerScore:
|
| 248 |
+
"""Filler matching totals.
|
| 249 |
+
|
| 250 |
+
Attributes:
|
| 251 |
+
matched: Sum over utterances and types of ``min(n_ref, n_hyp)``.
|
| 252 |
+
ref_total: Number of reference fillers.
|
| 253 |
+
hyp_total: Number of hypothesis fillers.
|
| 254 |
+
"""
|
| 255 |
+
|
| 256 |
+
matched: int
|
| 257 |
+
ref_total: int
|
| 258 |
+
hyp_total: int
|
| 259 |
+
|
| 260 |
+
def recall(self) -> float | None:
|
| 261 |
+
"""``matched / ref_total``; ``None`` when there are no reference fillers."""
|
| 262 |
+
if self.ref_total == 0:
|
| 263 |
+
return None
|
| 264 |
+
return self.matched / self.ref_total
|
| 265 |
+
|
| 266 |
+
def precision(self) -> float | None:
|
| 267 |
+
"""``matched / hyp_total``; ``None`` when there are no hypothesis fillers."""
|
| 268 |
+
if self.hyp_total == 0:
|
| 269 |
+
return None
|
| 270 |
+
return self.matched / self.hyp_total
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def filler_score(
|
| 274 |
+
pairs: Iterable[tuple[Sequence[str], Sequence[str]]], filler: str | None = None
|
| 275 |
+
) -> FillerScore:
|
| 276 |
+
"""Aggregate filler matches over ``(ref_tokens, hyp_tokens)`` pairs.
|
| 277 |
+
|
| 278 |
+
Per utterance, ``matched = sum over types of min(n_ref, n_hyp)``.
|
| 279 |
+
|
| 280 |
+
Args:
|
| 281 |
+
pairs: Normalised (reference tokens, hypothesis tokens) per utterance.
|
| 282 |
+
filler: If given, restrict scoring to this canonical filler type.
|
| 283 |
+
|
| 284 |
+
Raises:
|
| 285 |
+
ValueError: If ``filler`` is not a member of :data:`FILLERS`.
|
| 286 |
+
"""
|
| 287 |
+
if filler is not None and filler not in FILLERS:
|
| 288 |
+
raise ValueError(f"unknown filler {filler!r}; expected one of {sorted(FILLERS)}")
|
| 289 |
+
matched = 0
|
| 290 |
+
ref_total = 0
|
| 291 |
+
hyp_total = 0
|
| 292 |
+
for ref_tokens, hyp_tokens in pairs:
|
| 293 |
+
ref_counter = filler_counts(ref_tokens)
|
| 294 |
+
hyp_counter = filler_counts(hyp_tokens)
|
| 295 |
+
types = FILLERS if filler is None else frozenset({filler})
|
| 296 |
+
for kind in types:
|
| 297 |
+
n_ref = ref_counter[kind]
|
| 298 |
+
n_hyp = hyp_counter[kind]
|
| 299 |
+
matched += min(n_ref, n_hyp)
|
| 300 |
+
ref_total += n_ref
|
| 301 |
+
hyp_total += n_hyp
|
| 302 |
+
return FillerScore(matched=matched, ref_total=ref_total, hyp_total=hyp_total)
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
# --------------------------------------------------------------------------------------
|
| 306 |
+
# Hallucination
|
| 307 |
+
# --------------------------------------------------------------------------------------
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def is_hallucination(ref_text: str, hyp_text: str, max_ratio: float = 1.5) -> bool:
|
| 311 |
+
"""Flag a hypothesis that is empty or far longer than the reference.
|
| 312 |
+
|
| 313 |
+
Lengths are character counts of ``" ".join(normalize_for_scoring(text))``.
|
| 314 |
+
|
| 315 |
+
Returns:
|
| 316 |
+
True if the hypothesis is empty while the reference is not, or if
|
| 317 |
+
``len(hyp) > max_ratio * len(ref)`` (reference non-empty). If the reference is
|
| 318 |
+
empty, True iff the hypothesis is non-empty.
|
| 319 |
+
|
| 320 |
+
Raises:
|
| 321 |
+
ValueError: If ``max_ratio`` is not > 0.
|
| 322 |
+
"""
|
| 323 |
+
if not max_ratio > 0:
|
| 324 |
+
raise ValueError(f"max_ratio must be > 0, got {max_ratio}")
|
| 325 |
+
ref_len = len(" ".join(normalize_for_scoring(ref_text)))
|
| 326 |
+
hyp_len = len(" ".join(normalize_for_scoring(hyp_text)))
|
| 327 |
+
if ref_len == 0:
|
| 328 |
+
return hyp_len > 0
|
| 329 |
+
if hyp_len == 0:
|
| 330 |
+
return True
|
| 331 |
+
return hyp_len > max_ratio * ref_len
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
# --------------------------------------------------------------------------------------
|
| 335 |
+
# Paired bootstrap
|
| 336 |
+
# --------------------------------------------------------------------------------------
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
@dataclass(frozen=True)
|
| 340 |
+
class BootstrapResult:
|
| 341 |
+
"""Result of :func:`paired_bootstrap`.
|
| 342 |
+
|
| 343 |
+
Attributes:
|
| 344 |
+
delta: ``corpus_rate(a) - corpus_rate(b)`` on the full set.
|
| 345 |
+
low: Lower percentile bound of the resampled deltas.
|
| 346 |
+
high: Upper percentile bound of the resampled deltas.
|
| 347 |
+
n_resamples: Number of resamples requested.
|
| 348 |
+
n_skipped: Resamples skipped because their total reference length was 0; the
|
| 349 |
+
interval is computed from ``n_resamples - n_skipped`` resamples.
|
| 350 |
+
"""
|
| 351 |
+
|
| 352 |
+
delta: float
|
| 353 |
+
low: float
|
| 354 |
+
high: float
|
| 355 |
+
n_resamples: int
|
| 356 |
+
n_skipped: int = 0
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def _quantile(sorted_values: Sequence[float], q: float) -> float:
|
| 360 |
+
"""Linear-interpolation quantile of an already sorted, non-empty sequence."""
|
| 361 |
+
position = q * (len(sorted_values) - 1)
|
| 362 |
+
lower = math.floor(position)
|
| 363 |
+
upper = math.ceil(position)
|
| 364 |
+
if lower == upper:
|
| 365 |
+
return sorted_values[lower]
|
| 366 |
+
fraction = position - lower
|
| 367 |
+
return sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * fraction
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def paired_bootstrap(
|
| 371 |
+
a: Sequence[EditCounts],
|
| 372 |
+
b: Sequence[EditCounts],
|
| 373 |
+
n_resamples: int = 10000,
|
| 374 |
+
seed: int = 0,
|
| 375 |
+
confidence: float = 0.95,
|
| 376 |
+
) -> BootstrapResult:
|
| 377 |
+
"""Paired bootstrap confidence interval for the corpus error-rate difference a - b.
|
| 378 |
+
|
| 379 |
+
Utterance indices are resampled with replacement (the same indices for both systems)
|
| 380 |
+
using ``random.Random(seed)``; the interval is the percentile interval of the resampled
|
| 381 |
+
``corpus_rate(a) - corpus_rate(b)``.
|
| 382 |
+
|
| 383 |
+
Args:
|
| 384 |
+
a: Per-utterance counts for system A.
|
| 385 |
+
b: Per-utterance counts for system B, scored against the same references.
|
| 386 |
+
n_resamples: Number of bootstrap resamples (>= 1).
|
| 387 |
+
seed: Seed for the resampling RNG.
|
| 388 |
+
confidence: Confidence level in (0, 1).
|
| 389 |
+
|
| 390 |
+
Raises:
|
| 391 |
+
ValueError: On length mismatch, empty input, differing per-index ``ref_len``,
|
| 392 |
+
invalid ``n_resamples`` / ``confidence``, zero total reference length, or if
|
| 393 |
+
more than half of the resamples have zero total reference length.
|
| 394 |
+
"""
|
| 395 |
+
if len(a) != len(b):
|
| 396 |
+
raise ValueError(f"a and b must have the same length, got {len(a)} and {len(b)}")
|
| 397 |
+
if len(a) == 0:
|
| 398 |
+
raise ValueError("a and b must be non-empty")
|
| 399 |
+
for index, (item_a, item_b) in enumerate(zip(a, b, strict=True)):
|
| 400 |
+
if item_a.ref_len != item_b.ref_len:
|
| 401 |
+
raise ValueError(
|
| 402 |
+
f"ref_len mismatch at index {index}: {item_a.ref_len} vs {item_b.ref_len}; "
|
| 403 |
+
"both systems must be scored against the same references"
|
| 404 |
+
)
|
| 405 |
+
if n_resamples < 1:
|
| 406 |
+
raise ValueError(f"n_resamples must be >= 1, got {n_resamples}")
|
| 407 |
+
if not 0.0 < confidence < 1.0:
|
| 408 |
+
raise ValueError(f"confidence must be in (0, 1), got {confidence}")
|
| 409 |
+
ref_lens = [item.ref_len for item in a]
|
| 410 |
+
total_ref = sum(ref_lens)
|
| 411 |
+
if total_ref == 0:
|
| 412 |
+
raise ValueError("total reference length is 0; the error rate is undefined")
|
| 413 |
+
|
| 414 |
+
errors_a = [item.errors for item in a]
|
| 415 |
+
errors_b = [item.errors for item in b]
|
| 416 |
+
rate_a = corpus_rate(a)
|
| 417 |
+
rate_b = corpus_rate(b)
|
| 418 |
+
if rate_a is None or rate_b is None: # unreachable: total_ref > 0 and refs are equal
|
| 419 |
+
raise ValueError("corpus rate undefined")
|
| 420 |
+
delta = rate_a - rate_b
|
| 421 |
+
|
| 422 |
+
num_items = len(a)
|
| 423 |
+
rng = random.Random(seed)
|
| 424 |
+
population = range(num_items)
|
| 425 |
+
deltas: list[float] = []
|
| 426 |
+
skipped = 0
|
| 427 |
+
for _ in range(n_resamples):
|
| 428 |
+
indices = rng.choices(population, k=num_items)
|
| 429 |
+
sample_ref = sum(ref_lens[i] for i in indices)
|
| 430 |
+
if sample_ref == 0:
|
| 431 |
+
skipped += 1
|
| 432 |
+
continue
|
| 433 |
+
sample_a = sum(errors_a[i] for i in indices)
|
| 434 |
+
sample_b = sum(errors_b[i] for i in indices)
|
| 435 |
+
deltas.append((sample_a - sample_b) / sample_ref)
|
| 436 |
+
if skipped * 2 > n_resamples:
|
| 437 |
+
raise ValueError(
|
| 438 |
+
f"{skipped} of {n_resamples} resamples had zero total reference length "
|
| 439 |
+
"(more than half); the data has too few non-empty references to bootstrap"
|
| 440 |
+
)
|
| 441 |
+
deltas.sort()
|
| 442 |
+
alpha = (1.0 - confidence) / 2.0
|
| 443 |
+
return BootstrapResult(
|
| 444 |
+
delta=delta,
|
| 445 |
+
low=_quantile(deltas, alpha),
|
| 446 |
+
high=_quantile(deltas, 1.0 - alpha),
|
| 447 |
+
n_resamples=n_resamples,
|
| 448 |
+
n_skipped=skipped,
|
| 449 |
+
)
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
def normalize_english(text: str) -> list[str]:
|
| 453 |
+
"""Scoring tokens for *English* text (B2 Expresso clips).
|
| 454 |
+
|
| 455 |
+
NFC, ``str.casefold`` (correct for English; never use it on Turkish text), punctuation
|
| 456 |
+
and symbols become spaces, apostrophes are deleted, split on whitespace. No filler
|
| 457 |
+
canonicalisation: Turkish filler spellings ("of", "ah", "oh") are English words here.
|
| 458 |
+
"""
|
| 459 |
+
chars: list[str] = []
|
| 460 |
+
for ch in nfc(text).casefold():
|
| 461 |
+
if ch in _APOSTROPHES:
|
| 462 |
+
continue
|
| 463 |
+
chars.append(" " if ch.isspace() or unicodedata.category(ch)[0] in ("P", "S") else ch)
|
| 464 |
+
return "".join(chars).split()
|
src/nida/bench/workers/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model workers for Phase 0 benchmarks.
|
| 2 |
+
|
| 3 |
+
Each module here runs inside its own virtual environment (see ``nida.bench.envs``) and is
|
| 4 |
+
started as ``python -m nida.bench.workers.<name> ...``. Workers depend only on the
|
| 5 |
+
standard library, ``nida.bench.protocol`` and their model library; heavy imports happen
|
| 6 |
+
inside ``Adapter.load`` so this package stays importable everywhere.
|
| 7 |
+
"""
|
src/nida/bench/workers/asr_hf.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Whisper-family ASR via the transformers pipeline (whisper-large-v3, BuzzASR/turkish).
|
| 2 |
+
|
| 3 |
+
Decoding: beam search 5, no sampling, task ``transcribe`` with the language forced.
|
| 4 |
+
|
| 5 |
+
An optional per-request ``params["prompt"]`` is passed to Whisper as a decoder prompt
|
| 6 |
+
(``generate_kwargs["prompt_ids"]``, from ``tokenizer.get_prompt_ids``). With transformers
|
| 7 |
+
4.57 the tokenizer drops the prompt tokens when it decodes (``_strip_prompt``), so the pipeline
|
| 8 |
+
output normally holds the transcript only; a literal leading copy of the prompt is stripped
|
| 9 |
+
here as a safeguard and flagged with ``"prompt_stripped": true``. Without the parameter the
|
| 10 |
+
behaviour (and the output) is exactly the unprompted decoding, so earlier results stay
|
| 11 |
+
comparable.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import importlib
|
| 17 |
+
from collections.abc import Mapping, Sequence
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
from nida.bench.protocol import WorkRequest
|
| 21 |
+
from nida.bench.workers.common import language_for, load_mono_16k, snapshot
|
| 22 |
+
from nida.bench.workers.runner import Adapter, ModelRef, main_for
|
| 23 |
+
|
| 24 |
+
NUM_BEAMS = 5
|
| 25 |
+
PROMPT_PARAM = "prompt"
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def prompt_for(batch: Sequence[WorkRequest]) -> str | None:
|
| 29 |
+
"""The prompt shared by every request of ``batch`` (``None`` = unprompted decoding).
|
| 30 |
+
|
| 31 |
+
Raises:
|
| 32 |
+
ValueError: if requests disagree (a prompt and no prompt count as different) or the
|
| 33 |
+
prompt is blank.
|
| 34 |
+
"""
|
| 35 |
+
prompts = {r.params.get(PROMPT_PARAM) for r in batch}
|
| 36 |
+
if len(prompts) != 1:
|
| 37 |
+
shown = sorted("<none>" if p is None else repr(p) for p in prompts)
|
| 38 |
+
raise ValueError(f"mixed prompts in one batch: {shown}")
|
| 39 |
+
prompt = next(iter(prompts))
|
| 40 |
+
if prompt is not None and not prompt.strip():
|
| 41 |
+
raise ValueError("params.prompt is blank; omit it for unprompted decoding")
|
| 42 |
+
return prompt
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def strip_prompt_prefix(text: str, prompt: str) -> tuple[str, bool]:
|
| 46 |
+
"""Remove a literal leading copy of ``prompt`` from ``text``.
|
| 47 |
+
|
| 48 |
+
Returns:
|
| 49 |
+
``(text without the prompt, whether a prefix was removed)``.
|
| 50 |
+
"""
|
| 51 |
+
text, prompt = text.strip(), prompt.strip()
|
| 52 |
+
if text.startswith(prompt):
|
| 53 |
+
return text[len(prompt) :].strip(), True
|
| 54 |
+
return text, False
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class WhisperAdapter(Adapter):
|
| 58 |
+
name = "asr_hf"
|
| 59 |
+
distributions = ("torch", "transformers", "huggingface-hub", "soundfile", "numpy")
|
| 60 |
+
|
| 61 |
+
def __init__(self) -> None:
|
| 62 |
+
self._pipe: Any = None # Any: transformers pipeline object (untyped here)
|
| 63 |
+
self._batch_size = 1
|
| 64 |
+
self._device = "cpu"
|
| 65 |
+
|
| 66 |
+
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
|
| 67 |
+
torch = importlib.import_module("torch")
|
| 68 |
+
transformers = importlib.import_module("transformers")
|
| 69 |
+
self._batch_size = batch_size
|
| 70 |
+
self._device = device
|
| 71 |
+
self._pipe = transformers.pipeline(
|
| 72 |
+
"automatic-speech-recognition",
|
| 73 |
+
model=snapshot(model.model_id, model.revision),
|
| 74 |
+
torch_dtype=torch.float16,
|
| 75 |
+
device=device,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 79 |
+
languages = {language_for(r, "whisper") for r in batch}
|
| 80 |
+
if len(languages) != 1:
|
| 81 |
+
raise ValueError(f"mixed languages in one batch: {sorted(languages)}")
|
| 82 |
+
prompt = prompt_for(batch)
|
| 83 |
+
generate_kwargs: dict[str, Any] = { # Any: tensors / strings handed to generate()
|
| 84 |
+
"language": languages.pop(),
|
| 85 |
+
"task": "transcribe",
|
| 86 |
+
"num_beams": NUM_BEAMS,
|
| 87 |
+
"do_sample": False,
|
| 88 |
+
}
|
| 89 |
+
if prompt is not None:
|
| 90 |
+
# rank-1 tensor (<|startofprev|> + prompt tokens); generate() repeats it per item
|
| 91 |
+
prompt_ids = self._pipe.tokenizer.get_prompt_ids(prompt, return_tensors="pt")
|
| 92 |
+
generate_kwargs["prompt_ids"] = prompt_ids.to(self._device)
|
| 93 |
+
inputs = [{"raw": load_mono_16k(r.audio_path), "sampling_rate": 16000} for r in batch]
|
| 94 |
+
outputs = self._pipe(inputs, batch_size=self._batch_size, generate_kwargs=generate_kwargs)
|
| 95 |
+
texts = [str(o["text"]).strip() for o in outputs]
|
| 96 |
+
if prompt is None:
|
| 97 |
+
return [{"text": text} for text in texts]
|
| 98 |
+
stripped = [strip_prompt_prefix(text, prompt) for text in texts]
|
| 99 |
+
return [{"text": text, "prompt_stripped": was} for text, was in stripped]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
if __name__ == "__main__":
|
| 103 |
+
raise SystemExit(main_for(WhisperAdapter()))
|
src/nida/bench/workers/asr_omni.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Meta Omnilingual ASR via its inference pipeline (CTC and LLM model cards).
|
| 2 |
+
|
| 3 |
+
``model_id`` is the model card name (e.g. ``omniASR_LLM_7B_v2``); weights come from
|
| 4 |
+
Meta's CDN through fairseq2's asset store, so ``revision`` is recorded but unused.
|
| 5 |
+
The pipeline rejects audio longer than 40 s; B1 clips are at most 25 s.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import importlib
|
| 11 |
+
from collections.abc import Mapping, Sequence
|
| 12 |
+
from typing import Any
|
| 13 |
+
|
| 14 |
+
from nida.bench.protocol import WorkRequest
|
| 15 |
+
from nida.bench.workers.common import language_for
|
| 16 |
+
from nida.bench.workers.runner import Adapter, ModelRef, main_for
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class OmniAdapter(Adapter):
|
| 20 |
+
name = "asr_omni"
|
| 21 |
+
distributions = ("torch", "omnilingual-asr", "fairseq2", "fairseq2n")
|
| 22 |
+
|
| 23 |
+
def __init__(self) -> None:
|
| 24 |
+
self._pipeline: Any = None # Any: ASRInferencePipeline (untyped)
|
| 25 |
+
self._batch_size = 1
|
| 26 |
+
|
| 27 |
+
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
|
| 28 |
+
module = importlib.import_module("omnilingual_asr.models.inference.pipeline")
|
| 29 |
+
self._batch_size = batch_size
|
| 30 |
+
self._pipeline = module.ASRInferencePipeline(model_card=model.model_id, device=device)
|
| 31 |
+
|
| 32 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 33 |
+
texts = self._pipeline.transcribe(
|
| 34 |
+
[r.audio_path for r in batch],
|
| 35 |
+
lang=[language_for(r, "omni") for r in batch],
|
| 36 |
+
batch_size=self._batch_size,
|
| 37 |
+
)
|
| 38 |
+
return [{"text": str(t).strip()} for t in texts]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
raise SystemExit(main_for(OmniAdapter()))
|
src/nida/bench/workers/asr_qwen.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Qwen3-ASR via the official ``qwen-asr`` package (transformers backend).
|
| 2 |
+
|
| 3 |
+
``params["context"]`` (optional) is passed as the system context; B1 uses no context by
|
| 4 |
+
default and a separate, explicitly labelled run for the filler-context variant.
|
| 5 |
+
``params["language"] == "auto"`` (all items of a batch) lets the model identify the
|
| 6 |
+
language itself (``language=None``, Qwen3-ASR README); the detected name is returned.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import importlib
|
| 12 |
+
from collections.abc import Mapping, Sequence
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
from nida.bench.protocol import WorkRequest
|
| 16 |
+
from nida.bench.workers.common import language_for, snapshot
|
| 17 |
+
from nida.bench.workers.runner import Adapter, ModelRef, main_for
|
| 18 |
+
|
| 19 |
+
MAX_NEW_TOKENS = 256
|
| 20 |
+
#: ``params["language"]`` value requesting automatic language identification.
|
| 21 |
+
AUTO_LANGUAGE = "auto"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class QwenAdapter(Adapter):
|
| 25 |
+
name = "asr_qwen"
|
| 26 |
+
distributions = ("torch", "qwen-asr", "transformers", "huggingface-hub")
|
| 27 |
+
|
| 28 |
+
def __init__(self) -> None:
|
| 29 |
+
self._model: Any = None # Any: qwen_asr.Qwen3ASRModel (untyped)
|
| 30 |
+
|
| 31 |
+
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
|
| 32 |
+
torch = importlib.import_module("torch")
|
| 33 |
+
qwen_asr = importlib.import_module("qwen_asr")
|
| 34 |
+
self._model = qwen_asr.Qwen3ASRModel.from_pretrained(
|
| 35 |
+
snapshot(model.model_id, model.revision),
|
| 36 |
+
dtype=torch.bfloat16,
|
| 37 |
+
device_map=device,
|
| 38 |
+
max_inference_batch_size=batch_size,
|
| 39 |
+
max_new_tokens=MAX_NEW_TOKENS,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 43 |
+
auto = [r.params.get("language") == AUTO_LANGUAGE for r in batch]
|
| 44 |
+
if any(auto) and not all(auto):
|
| 45 |
+
raise ValueError("a batch mixes automatic and forced languages")
|
| 46 |
+
results = self._model.transcribe(
|
| 47 |
+
audio=[r.audio_path for r in batch],
|
| 48 |
+
context=[r.params.get("context", "") for r in batch],
|
| 49 |
+
language=None if all(auto) else [language_for(r, "qwen") for r in batch],
|
| 50 |
+
)
|
| 51 |
+
return [{"text": str(res.text).strip(), "language": str(res.language)} for res in results]
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
if __name__ == "__main__":
|
| 55 |
+
raise SystemExit(main_for(QwenAdapter()))
|
src/nida/bench/workers/common.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small helpers shared by adapters (stdlib only)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import importlib
|
| 6 |
+
from collections.abc import Mapping, Sequence
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from nida.bench.protocol import WorkRequest
|
| 10 |
+
|
| 11 |
+
__all__ = ["HF_WEIGHT_PATTERNS", "LANGUAGES", "language_for", "load_mono_16k", "snapshot"]
|
| 12 |
+
|
| 13 |
+
#: Files needed to load a transformers checkpoint in safetensors form. Excludes duplicate
|
| 14 |
+
#: formats (flax, pytorch .bin, fp32 shards) that some repos also ship.
|
| 15 |
+
HF_WEIGHT_PATTERNS: tuple[str, ...] = (
|
| 16 |
+
"*.json",
|
| 17 |
+
"*.txt",
|
| 18 |
+
"*.model",
|
| 19 |
+
"*.tiktoken",
|
| 20 |
+
"model.safetensors",
|
| 21 |
+
"model-*.safetensors",
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
#: ISO 639-1 code → spelling each model family expects.
|
| 25 |
+
LANGUAGES: Mapping[str, Mapping[str, str]] = {
|
| 26 |
+
"tr": {"whisper": "turkish", "qwen": "Turkish", "omni": "tur_Latn"},
|
| 27 |
+
"en": {"whisper": "english", "qwen": "English", "omni": "eng_Latn"},
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def language_for(request: WorkRequest, family: str) -> str:
|
| 32 |
+
"""Model-specific language name for ``request.params["language"]`` (required).
|
| 33 |
+
|
| 34 |
+
Raises:
|
| 35 |
+
ValueError: if the language parameter is missing or unsupported.
|
| 36 |
+
"""
|
| 37 |
+
code = request.params.get("language")
|
| 38 |
+
if code is None:
|
| 39 |
+
raise ValueError(f"{request.clip_id}: missing params.language")
|
| 40 |
+
try:
|
| 41 |
+
return LANGUAGES[code][family]
|
| 42 |
+
except KeyError as err:
|
| 43 |
+
raise ValueError(f"{request.clip_id}: unsupported language {code!r} for {family}") from err
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def snapshot(
|
| 47 |
+
model_id: str, revision: str, allow_patterns: Sequence[str] = HF_WEIGHT_PATTERNS
|
| 48 |
+
) -> str:
|
| 49 |
+
"""Download (or reuse) a pinned Hugging Face snapshot and return its local path.
|
| 50 |
+
|
| 51 |
+
An empty ``revision`` is refused: benchmark runs must be reproducible. Only files
|
| 52 |
+
matching ``allow_patterns`` are fetched.
|
| 53 |
+
"""
|
| 54 |
+
if not revision:
|
| 55 |
+
raise ValueError(f"{model_id}: a pinned revision is required")
|
| 56 |
+
hub = importlib.import_module("huggingface_hub")
|
| 57 |
+
path: str = hub.snapshot_download(
|
| 58 |
+
repo_id=model_id, revision=revision, allow_patterns=list(allow_patterns)
|
| 59 |
+
)
|
| 60 |
+
return path
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# Any: numpy is only present inside worker environments and has no stub here.
|
| 64 |
+
def load_mono_16k(path: str) -> Any:
|
| 65 |
+
"""Read a 16 kHz mono file as float32 (the orchestrator guarantees this format).
|
| 66 |
+
|
| 67 |
+
Raises:
|
| 68 |
+
ValueError: if the file is not 16 kHz mono.
|
| 69 |
+
"""
|
| 70 |
+
soundfile = importlib.import_module("soundfile")
|
| 71 |
+
audio, sample_rate_hz = soundfile.read(path, dtype="float32", always_2d=False)
|
| 72 |
+
if sample_rate_hz != 16000 or audio.ndim != 1:
|
| 73 |
+
raise ValueError(f"{path}: expected 16 kHz mono, got {sample_rate_hz} Hz, {audio.ndim}-D")
|
| 74 |
+
return audio
|
src/nida/bench/workers/quality_audiobox.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Meta Audiobox Aesthetics quality scores (CE, CU, PC, PQ) for B2.
|
| 2 |
+
|
| 3 |
+
The checkpoint (``checkpoint.pt``, CC-BY-4.0) is taken from the pinned
|
| 4 |
+
``facebook/audiobox-aesthetics`` revision and passed explicitly, so the package never
|
| 5 |
+
fetches an unpinned "latest" model. The predictor resamples input to 16 kHz.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import importlib
|
| 11 |
+
from collections.abc import Mapping, Sequence
|
| 12 |
+
from typing import Any
|
| 13 |
+
|
| 14 |
+
from nida.bench.protocol import WorkRequest
|
| 15 |
+
from nida.bench.workers.common import snapshot
|
| 16 |
+
from nida.bench.workers.runner import Adapter, ModelRef, main_for
|
| 17 |
+
|
| 18 |
+
AXES = ("CE", "CU", "PC", "PQ")
|
| 19 |
+
CHECKPOINT_FILE = "checkpoint.pt"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class AudioboxAdapter(Adapter):
|
| 23 |
+
name = "quality_audiobox"
|
| 24 |
+
distributions = ("torch", "torchaudio", "audiobox-aesthetics")
|
| 25 |
+
|
| 26 |
+
def __init__(self) -> None:
|
| 27 |
+
self._predictor: Any = None # Any: audiobox predictor (untyped)
|
| 28 |
+
|
| 29 |
+
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
|
| 30 |
+
infer = importlib.import_module("audiobox_aesthetics.infer")
|
| 31 |
+
local = snapshot(model.model_id, model.revision, (CHECKPOINT_FILE,))
|
| 32 |
+
self._predictor = infer.initialize_predictor(ckpt=f"{local}/{CHECKPOINT_FILE}")
|
| 33 |
+
|
| 34 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 35 |
+
scores = self._predictor.forward([{"path": r.audio_path} for r in batch])
|
| 36 |
+
outputs: list[Mapping[str, object]] = []
|
| 37 |
+
for score in scores:
|
| 38 |
+
missing = [axis for axis in AXES if axis not in score]
|
| 39 |
+
if missing:
|
| 40 |
+
raise ValueError(f"audiobox output lacks axes {missing}: {score}")
|
| 41 |
+
outputs.append({axis: float(score[axis]) for axis in AXES})
|
| 42 |
+
return outputs
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
if __name__ == "__main__":
|
| 46 |
+
raise SystemExit(main_for(AudioboxAdapter()))
|
src/nida/bench/workers/runner.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generic worker loop shared by every model adapter.
|
| 2 |
+
|
| 3 |
+
The loop is model-agnostic: it reads the requests file, resumes from the results file,
|
| 4 |
+
calls ``Adapter.run_batch`` in fixed-size batches, and appends one fsynced result line per
|
| 5 |
+
request. A failing batch is retried item by item so a single bad clip cannot take its
|
| 6 |
+
neighbours down; every failure is recorded with its exception type and message (never
|
| 7 |
+
dropped) and counted in the summary. Model loading failures are fatal: they propagate and
|
| 8 |
+
the process exits non-zero.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import abc
|
| 14 |
+
import argparse
|
| 15 |
+
import hashlib
|
| 16 |
+
import importlib
|
| 17 |
+
import importlib.metadata
|
| 18 |
+
import json
|
| 19 |
+
import logging
|
| 20 |
+
import sys
|
| 21 |
+
import time
|
| 22 |
+
import traceback
|
| 23 |
+
from collections.abc import Mapping, Sequence
|
| 24 |
+
from dataclasses import dataclass, replace
|
| 25 |
+
from datetime import UTC, datetime
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
from typing import ClassVar
|
| 28 |
+
|
| 29 |
+
from nida.bench.protocol import (
|
| 30 |
+
ProtocolError,
|
| 31 |
+
WorkerSummary,
|
| 32 |
+
WorkRequest,
|
| 33 |
+
WorkResult,
|
| 34 |
+
append_result,
|
| 35 |
+
drop_results,
|
| 36 |
+
load_requests,
|
| 37 |
+
load_results,
|
| 38 |
+
pending_requests,
|
| 39 |
+
read_summary,
|
| 40 |
+
write_summary,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
__all__ = [
|
| 44 |
+
"MAX_CONSECUTIVE_FAILURES",
|
| 45 |
+
"Adapter",
|
| 46 |
+
"ModelRef",
|
| 47 |
+
"TooManyFailuresError",
|
| 48 |
+
"WorkerOptions",
|
| 49 |
+
"main_for",
|
| 50 |
+
"run_worker",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
log = logging.getLogger(__name__)
|
| 54 |
+
|
| 55 |
+
#: Abort when this many items in a row fail: the model/device is broken, not the clips.
|
| 56 |
+
MAX_CONSECUTIVE_FAILURES = 10
|
| 57 |
+
#: Progress is logged when this many more items are done (and at the end).
|
| 58 |
+
PROGRESS_EVERY = 200
|
| 59 |
+
#: When the *same* failure run (same clip ids) aborts more than this many times in a row,
|
| 60 |
+
#: it is evidently repeatable rather than transient and is kept as recorded failures so a
|
| 61 |
+
#: rerun can move past it. Unrelated aborts never accumulate (see ``_abort_count``).
|
| 62 |
+
MAX_ABORTS_BEFORE_KEEPING = 2
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
class TooManyFailuresError(RuntimeError):
|
| 66 |
+
"""Consecutive item failures suggest a broken model or device (e.g. CUDA context)."""
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@dataclass(frozen=True)
|
| 70 |
+
class ModelRef:
|
| 71 |
+
"""Which model to load.
|
| 72 |
+
|
| 73 |
+
Attributes:
|
| 74 |
+
model_id: Hub id, model card name or local path, interpreted by the adapter.
|
| 75 |
+
revision: Pinned revision (commit SHA) or ``""`` when the source has none.
|
| 76 |
+
"""
|
| 77 |
+
|
| 78 |
+
model_id: str
|
| 79 |
+
revision: str
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class Adapter(abc.ABC):
|
| 83 |
+
"""One model family. Subclasses keep heavy imports inside their methods."""
|
| 84 |
+
|
| 85 |
+
name: ClassVar[str]
|
| 86 |
+
#: Distribution names whose installed versions are recorded in the summary.
|
| 87 |
+
distributions: ClassVar[tuple[str, ...]] = ()
|
| 88 |
+
|
| 89 |
+
@abc.abstractmethod
|
| 90 |
+
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
|
| 91 |
+
"""Load the model onto ``device``. Exceptions here abort the worker."""
|
| 92 |
+
|
| 93 |
+
@abc.abstractmethod
|
| 94 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 95 |
+
"""Return one JSON-compatible output mapping per request, in the same order."""
|
| 96 |
+
|
| 97 |
+
def peak_vram_gb(self) -> float | None:
|
| 98 |
+
"""Peak CUDA memory allocated by this process, if torch with CUDA is present."""
|
| 99 |
+
try:
|
| 100 |
+
torch = importlib.import_module("torch")
|
| 101 |
+
except ImportError:
|
| 102 |
+
return None
|
| 103 |
+
if not torch.cuda.is_available():
|
| 104 |
+
return None
|
| 105 |
+
return round(float(torch.cuda.max_memory_allocated()) / 1024**3, 2)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@dataclass(frozen=True)
|
| 109 |
+
class WorkerOptions:
|
| 110 |
+
requests: Path
|
| 111 |
+
results: Path
|
| 112 |
+
summary: Path
|
| 113 |
+
model: ModelRef
|
| 114 |
+
device: str
|
| 115 |
+
batch_size: int
|
| 116 |
+
retry_failed: bool
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _now() -> str:
|
| 120 |
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _describe(err: BaseException) -> str:
|
| 124 |
+
return f"{type(err).__name__}: {err}"
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _package_versions(names: Sequence[str]) -> dict[str, str]:
|
| 128 |
+
versions: dict[str, str] = {}
|
| 129 |
+
for name in names:
|
| 130 |
+
try:
|
| 131 |
+
versions[name] = importlib.metadata.version(name)
|
| 132 |
+
except importlib.metadata.PackageNotFoundError:
|
| 133 |
+
versions[name] = "not-installed"
|
| 134 |
+
return versions
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _abort_counter_path(results: Path) -> Path:
|
| 138 |
+
return results.with_name(results.name + ".aborts.json")
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _abort_count(results: Path, failure_ids: Sequence[str]) -> int:
|
| 142 |
+
"""Record an abort and return how often *this* failure run has aborted in a row.
|
| 143 |
+
|
| 144 |
+
The counter stores a signature of the failing clip ids: a different failure run
|
| 145 |
+
restarts at 1, and :func:`_clear_abort_count` removes it after any run that finishes
|
| 146 |
+
without aborting, so transient incidents in different sessions never add up.
|
| 147 |
+
"""
|
| 148 |
+
signature = hashlib.sha1("\n".join(sorted(failure_ids)).encode("utf-8")).hexdigest()
|
| 149 |
+
path = _abort_counter_path(results)
|
| 150 |
+
count = 1
|
| 151 |
+
if path.exists():
|
| 152 |
+
try:
|
| 153 |
+
previous = json.loads(path.read_text(encoding="utf-8"))
|
| 154 |
+
if previous.get("signature") == signature:
|
| 155 |
+
count = int(previous.get("count", 0)) + 1
|
| 156 |
+
except (ValueError, AttributeError, TypeError) as err:
|
| 157 |
+
# A damaged counter must not wedge the worker; recount from this abort.
|
| 158 |
+
log.warning("ignoring unreadable abort counter %s (%s); restarting at 1", path, err)
|
| 159 |
+
tmp = path.with_name(f".{path.name}.tmp")
|
| 160 |
+
tmp.write_text(json.dumps({"signature": signature, "count": count}), encoding="utf-8")
|
| 161 |
+
tmp.replace(path)
|
| 162 |
+
return count
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _clear_abort_count(results: Path) -> None:
|
| 166 |
+
_abort_counter_path(results).unlink(missing_ok=True)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _drop_failed(path: Path) -> None:
|
| 170 |
+
"""Rewrite the results file without failed entries (for ``--retry-failed``)."""
|
| 171 |
+
results = load_results(path, repair=True)
|
| 172 |
+
kept = [r.to_json() + "\n" for r in results.values() if r.ok]
|
| 173 |
+
tmp = path.with_name(f".{path.name}.tmp")
|
| 174 |
+
tmp.write_text("".join(kept), encoding="utf-8")
|
| 175 |
+
tmp.replace(path)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _validated_outputs(batch: Sequence[WorkRequest], outputs: object) -> list[Mapping[str, object]]:
|
| 179 |
+
if not isinstance(outputs, list) or len(outputs) != len(batch):
|
| 180 |
+
got = len(outputs) if isinstance(outputs, list) else type(outputs).__name__
|
| 181 |
+
raise ProtocolError(f"adapter returned {got} outputs for a batch of {len(batch)}")
|
| 182 |
+
for output in outputs:
|
| 183 |
+
if not isinstance(output, Mapping):
|
| 184 |
+
raise ProtocolError(f"adapter output must be a mapping, got {type(output).__name__}")
|
| 185 |
+
return outputs
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _run_one_batch(adapter: Adapter, batch: Sequence[WorkRequest]) -> list[WorkResult]:
|
| 189 |
+
started = time.perf_counter()
|
| 190 |
+
try:
|
| 191 |
+
outputs = _validated_outputs(batch, adapter.run_batch(batch))
|
| 192 |
+
except Exception as err: # noqa: BLE001 - recorded per item below, never swallowed
|
| 193 |
+
if len(batch) == 1:
|
| 194 |
+
log.warning("item %s failed: %s", batch[0].clip_id, _describe(err))
|
| 195 |
+
return [
|
| 196 |
+
WorkResult(
|
| 197 |
+
clip_id=batch[0].clip_id,
|
| 198 |
+
output=None,
|
| 199 |
+
error=_describe(err),
|
| 200 |
+
infer_seconds=time.perf_counter() - started,
|
| 201 |
+
)
|
| 202 |
+
]
|
| 203 |
+
log.warning("batch of %d failed (%s); retrying items one by one", len(batch), err)
|
| 204 |
+
results: list[WorkResult] = []
|
| 205 |
+
for request in batch:
|
| 206 |
+
results.extend(_run_one_batch(adapter, [request]))
|
| 207 |
+
return results
|
| 208 |
+
per_item = (time.perf_counter() - started) / len(batch)
|
| 209 |
+
return [
|
| 210 |
+
WorkResult(clip_id=r.clip_id, output=dict(o), error=None, infer_seconds=per_item)
|
| 211 |
+
for r, o in zip(batch, outputs, strict=True)
|
| 212 |
+
]
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def run_worker(adapter: Adapter, options: WorkerOptions) -> WorkerSummary:
|
| 216 |
+
"""Process every pending request and write the summary. Returns the summary."""
|
| 217 |
+
if options.batch_size < 1:
|
| 218 |
+
raise ValueError(f"batch_size must be >= 1, got {options.batch_size}")
|
| 219 |
+
started_at = _now()
|
| 220 |
+
requests = load_requests(options.requests)
|
| 221 |
+
if options.retry_failed and options.results.exists():
|
| 222 |
+
_drop_failed(options.results)
|
| 223 |
+
done = load_results(options.results, repair=True)
|
| 224 |
+
pending = pending_requests(requests, done)
|
| 225 |
+
log.info(
|
| 226 |
+
"%s: %d requests, %d already done, %d pending",
|
| 227 |
+
adapter.name,
|
| 228 |
+
len(requests),
|
| 229 |
+
len(done),
|
| 230 |
+
len(pending),
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
previous = read_summary(options.summary) if options.summary.exists() else None
|
| 234 |
+
load_seconds = 0.0
|
| 235 |
+
if pending:
|
| 236 |
+
load_start = time.perf_counter()
|
| 237 |
+
adapter.load(options.model, options.device, options.batch_size)
|
| 238 |
+
load_seconds = time.perf_counter() - load_start
|
| 239 |
+
options.results.parent.mkdir(parents=True, exist_ok=True)
|
| 240 |
+
failure_run: list[WorkResult] = []
|
| 241 |
+
aborted = False
|
| 242 |
+
with options.results.open("a", encoding="utf-8") as handle:
|
| 243 |
+
for start in range(0, len(pending), options.batch_size):
|
| 244 |
+
batch = pending[start : start + options.batch_size]
|
| 245 |
+
for result in _run_one_batch(adapter, batch):
|
| 246 |
+
append_result(handle, result)
|
| 247 |
+
failure_run = [] if result.ok else [*failure_run, result]
|
| 248 |
+
if len(failure_run) >= MAX_CONSECUTIVE_FAILURES:
|
| 249 |
+
aborted = True
|
| 250 |
+
break
|
| 251 |
+
if aborted:
|
| 252 |
+
break
|
| 253 |
+
finished = start + len(batch)
|
| 254 |
+
if (
|
| 255 |
+
finished == len(pending)
|
| 256 |
+
or finished // PROGRESS_EVERY != start // PROGRESS_EVERY
|
| 257 |
+
):
|
| 258 |
+
log.info("%s: %d/%d", adapter.name, finished, len(pending))
|
| 259 |
+
if aborted:
|
| 260 |
+
aborts = _abort_count(options.results, [r.clip_id for r in failure_run])
|
| 261 |
+
if aborts <= MAX_ABORTS_BEFORE_KEEPING:
|
| 262 |
+
# Most likely infrastructure (e.g. a broken CUDA context), not bad clips: make
|
| 263 |
+
# them pending again so a plain rerun retries them instead of scoring them as
|
| 264 |
+
# empty hypotheses.
|
| 265 |
+
drop_results(options.results, [r.clip_id for r in failure_run])
|
| 266 |
+
outcome = "results removed, they will be retried"
|
| 267 |
+
else:
|
| 268 |
+
outcome = (
|
| 269 |
+
f"same failure run aborted {aborts} times: kept as recorded failures so "
|
| 270 |
+
"the next run moves on; use --retry-failed to try them again"
|
| 271 |
+
)
|
| 272 |
+
raise TooManyFailuresError(
|
| 273 |
+
f"{adapter.name}: {len(failure_run)} consecutive failures ({outcome}), "
|
| 274 |
+
f"last: {failure_run[-1].error}"
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
_clear_abort_count(options.results) # finished without aborting
|
| 278 |
+
final = load_results(options.results)
|
| 279 |
+
n_ok = sum(1 for r in final.values() if r.ok)
|
| 280 |
+
if not pending and previous is not None:
|
| 281 |
+
# No-op resume: keep the measurements of the invocation that did the work.
|
| 282 |
+
summary = replace(previous, n_ok=n_ok, n_error=len(final) - n_ok)
|
| 283 |
+
else:
|
| 284 |
+
peaks = [v for v in (adapter.peak_vram_gb(), previous and previous.peak_vram_gb) if v]
|
| 285 |
+
invocations = int(previous.extra.get("invocations", "1")) + 1 if previous else 1
|
| 286 |
+
summary = WorkerSummary(
|
| 287 |
+
worker=adapter.name,
|
| 288 |
+
model_id=options.model.model_id,
|
| 289 |
+
revision=options.model.revision,
|
| 290 |
+
load_seconds=round(load_seconds, 3),
|
| 291 |
+
peak_vram_gb=max(peaks) if peaks else None,
|
| 292 |
+
n_ok=n_ok,
|
| 293 |
+
n_error=len(final) - n_ok,
|
| 294 |
+
started_at=previous.started_at if previous else started_at,
|
| 295 |
+
ended_at=_now(),
|
| 296 |
+
packages=_package_versions(adapter.distributions),
|
| 297 |
+
extra={
|
| 298 |
+
"python": sys.version.split()[0],
|
| 299 |
+
"device": options.device,
|
| 300 |
+
"invocations": str(invocations),
|
| 301 |
+
},
|
| 302 |
+
)
|
| 303 |
+
write_summary(options.summary, summary)
|
| 304 |
+
return summary
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _parse(argv: Sequence[str] | None) -> WorkerOptions:
|
| 308 |
+
parser = argparse.ArgumentParser(description="Run a Nida benchmark worker.")
|
| 309 |
+
parser.add_argument("--requests", type=Path, required=True)
|
| 310 |
+
parser.add_argument("--results", type=Path, required=True)
|
| 311 |
+
parser.add_argument("--summary", type=Path, default=None)
|
| 312 |
+
parser.add_argument("--model-id", required=True)
|
| 313 |
+
parser.add_argument("--revision", default="")
|
| 314 |
+
parser.add_argument("--device", default="cuda")
|
| 315 |
+
parser.add_argument("--batch-size", type=int, default=8)
|
| 316 |
+
parser.add_argument("--retry-failed", action="store_true")
|
| 317 |
+
args = parser.parse_args(argv)
|
| 318 |
+
summary = args.summary or args.results.with_name(args.results.stem + ".summary.json")
|
| 319 |
+
return WorkerOptions(
|
| 320 |
+
requests=args.requests,
|
| 321 |
+
results=args.results,
|
| 322 |
+
summary=summary,
|
| 323 |
+
model=ModelRef(model_id=args.model_id, revision=args.revision),
|
| 324 |
+
device=args.device,
|
| 325 |
+
batch_size=args.batch_size,
|
| 326 |
+
retry_failed=args.retry_failed,
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def main_for(adapter: Adapter, argv: Sequence[str] | None = None) -> int:
|
| 331 |
+
"""CLI entry point used by each worker module's ``__main__`` block."""
|
| 332 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 333 |
+
options = _parse(argv)
|
| 334 |
+
try:
|
| 335 |
+
summary = run_worker(adapter, options)
|
| 336 |
+
except (ProtocolError, OSError, ValueError, TooManyFailuresError):
|
| 337 |
+
traceback.print_exc()
|
| 338 |
+
return 2
|
| 339 |
+
print(f"{adapter.name}: ok={summary.n_ok} error={summary.n_error}")
|
| 340 |
+
return 0
|
src/nida/bench/workers/vae_voxcpm.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""VoxCPM2 AudioVAE V2 encode → decode round trip (benchmark B2).
|
| 2 |
+
|
| 3 |
+
Only the VAE is used. Its model code is a single file in the Apache-2.0 VoxCPM repository
|
| 4 |
+
that needs just torch, numpy and pydantic, so it is fetched at a pinned commit and
|
| 5 |
+
verified by SHA-256 instead of installing the ``voxcpm`` package (whose dependencies
|
| 6 |
+
include gradio, funasr and modelscope). Weights and config come from the pinned
|
| 7 |
+
``openbmb/VoxCPM2`` Hugging Face revision.
|
| 8 |
+
|
| 9 |
+
Per request: read audio (mono, any rate) → resample to the VAE input rate (16 kHz, as
|
| 10 |
+
VoxCPM does with ``librosa.load(sr=16000)``) → ``encode`` (posterior mean, deterministic)
|
| 11 |
+
→ ``decode`` at the default output rate (48 kHz) → trim to ``ceil(n_16k * 48000/16000)``
|
| 12 |
+
samples → write ``params["out_path"]`` as 32-bit float WAV.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import hashlib
|
| 18 |
+
import importlib
|
| 19 |
+
import importlib.util
|
| 20 |
+
import json
|
| 21 |
+
import math
|
| 22 |
+
import urllib.request
|
| 23 |
+
from collections.abc import Mapping, Sequence
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
from typing import Any
|
| 26 |
+
|
| 27 |
+
from nida.bench.protocol import WorkRequest
|
| 28 |
+
from nida.bench.workers.common import snapshot
|
| 29 |
+
from nida.bench.workers.runner import Adapter, ModelRef, main_for
|
| 30 |
+
|
| 31 |
+
VOXCPM_COMMIT = "9d95dd132ad4c54b42ed50561ff878b18e4c75de"
|
| 32 |
+
VAE_SOURCE_URL = (
|
| 33 |
+
"https://raw.githubusercontent.com/OpenBMB/VoxCPM/"
|
| 34 |
+
f"{VOXCPM_COMMIT}/src/voxcpm/modules/audiovae/audio_vae_v2.py"
|
| 35 |
+
)
|
| 36 |
+
VAE_SOURCE_SHA256 = "2efdff1708d8ec1471624aae6f232d0f933de26b788b6901c434246847e2d3a8"
|
| 37 |
+
WEIGHTS_FILE = "audiovae.pth"
|
| 38 |
+
CONFIG_FILE = "config.json"
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _sha256(data: bytes) -> str:
|
| 42 |
+
return hashlib.sha256(data).hexdigest()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def fetch_vae_source(cache_dir: Path) -> Path:
|
| 46 |
+
"""Download the pinned ``audio_vae_v2.py`` once and verify its SHA-256.
|
| 47 |
+
|
| 48 |
+
Raises:
|
| 49 |
+
ValueError: if the downloaded or cached file does not match the pinned hash.
|
| 50 |
+
"""
|
| 51 |
+
path = cache_dir / f"audio_vae_v2-{VOXCPM_COMMIT[:12]}.py"
|
| 52 |
+
if not path.exists():
|
| 53 |
+
with urllib.request.urlopen(VAE_SOURCE_URL, timeout=60) as response:
|
| 54 |
+
data = response.read()
|
| 55 |
+
if _sha256(data) != VAE_SOURCE_SHA256:
|
| 56 |
+
raise ValueError(f"audio_vae_v2.py hash mismatch: {_sha256(data)}")
|
| 57 |
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
| 58 |
+
tmp = path.with_suffix(".tmp")
|
| 59 |
+
tmp.write_bytes(data)
|
| 60 |
+
tmp.replace(path)
|
| 61 |
+
if _sha256(path.read_bytes()) != VAE_SOURCE_SHA256:
|
| 62 |
+
raise ValueError(f"{path}: cached file does not match the pinned hash")
|
| 63 |
+
return path
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _import_file(path: Path, module_name: str) -> Any: # Any: dynamically loaded module
|
| 67 |
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
| 68 |
+
if spec is None or spec.loader is None:
|
| 69 |
+
raise ImportError(f"cannot import {path}")
|
| 70 |
+
module = importlib.util.module_from_spec(spec)
|
| 71 |
+
spec.loader.exec_module(module)
|
| 72 |
+
return module
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class VoxCPMVAEAdapter(Adapter):
|
| 76 |
+
name = "vae_voxcpm"
|
| 77 |
+
distributions = ("torch", "torchaudio", "numpy", "pydantic", "soundfile", "huggingface-hub")
|
| 78 |
+
|
| 79 |
+
def __init__(self, cache_dir: Path | None = None) -> None:
|
| 80 |
+
self._cache_dir = cache_dir or Path.home() / ".cache" / "nida"
|
| 81 |
+
self._vae: Any = None # Any: AudioVAE V2 torch module
|
| 82 |
+
self._torch: Any = None
|
| 83 |
+
self._device = "cpu"
|
| 84 |
+
|
| 85 |
+
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
|
| 86 |
+
torch = importlib.import_module("torch")
|
| 87 |
+
source = _import_file(fetch_vae_source(self._cache_dir), "nida_audio_vae_v2")
|
| 88 |
+
local = Path(snapshot(model.model_id, model.revision, (CONFIG_FILE, WEIGHTS_FILE)))
|
| 89 |
+
config = json.loads((local / CONFIG_FILE).read_text(encoding="utf-8"))
|
| 90 |
+
vae = source.AudioVAE(config=source.AudioVAEConfig(**config["audio_vae_config"]))
|
| 91 |
+
checkpoint = torch.load(local / WEIGHTS_FILE, map_location="cpu", weights_only=True)
|
| 92 |
+
state = checkpoint.get("state_dict", checkpoint)
|
| 93 |
+
incompatible = vae.load_state_dict(state, strict=False)
|
| 94 |
+
if incompatible.missing_keys or incompatible.unexpected_keys:
|
| 95 |
+
raise ValueError(
|
| 96 |
+
f"state dict mismatch: missing={incompatible.missing_keys[:5]} "
|
| 97 |
+
f"unexpected={incompatible.unexpected_keys[:5]}"
|
| 98 |
+
)
|
| 99 |
+
self._vae = vae.to(device=device, dtype=torch.float32).eval()
|
| 100 |
+
self._torch = torch
|
| 101 |
+
self._device = device
|
| 102 |
+
|
| 103 |
+
def _round_trip(self, request: WorkRequest) -> Mapping[str, object]:
|
| 104 |
+
out_path = request.params.get("out_path")
|
| 105 |
+
if not out_path:
|
| 106 |
+
raise ValueError(f"{request.clip_id}: missing params.out_path")
|
| 107 |
+
torch = self._torch
|
| 108 |
+
soundfile = importlib.import_module("soundfile")
|
| 109 |
+
functional = importlib.import_module("torchaudio.functional")
|
| 110 |
+
audio, source_rate_hz = soundfile.read(request.audio_path, dtype="float32")
|
| 111 |
+
if audio.ndim != 1:
|
| 112 |
+
raise ValueError(f"{request.clip_id}: expected mono audio, got shape {audio.shape}")
|
| 113 |
+
in_rate_hz = int(self._vae.sample_rate)
|
| 114 |
+
out_rate_hz = int(self._vae.out_sample_rate)
|
| 115 |
+
wave = torch.from_numpy(audio).unsqueeze(0)
|
| 116 |
+
if source_rate_hz != in_rate_hz:
|
| 117 |
+
wave = functional.resample(wave, source_rate_hz, in_rate_hz)
|
| 118 |
+
num_in_samples = int(wave.shape[-1])
|
| 119 |
+
with torch.inference_mode():
|
| 120 |
+
latents = self._vae.encode(wave.to(self._device), in_rate_hz) # (1, D, frames)
|
| 121 |
+
decoded = self._vae.decode(latents) # (1, 1, samples) at out_rate_hz
|
| 122 |
+
num_out_samples = math.ceil(num_in_samples * out_rate_hz / in_rate_hz)
|
| 123 |
+
out = decoded.reshape(-1)[:num_out_samples].float().cpu().numpy()
|
| 124 |
+
if out.shape[0] < num_out_samples:
|
| 125 |
+
raise ValueError(f"{request.clip_id}: decoder returned {out.shape[0]} samples")
|
| 126 |
+
target = Path(out_path)
|
| 127 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 128 |
+
tmp = target.with_name(f".{target.name}.tmp.wav") # never leave a partial output
|
| 129 |
+
soundfile.write(str(tmp), out, out_rate_hz, subtype="FLOAT")
|
| 130 |
+
tmp.replace(target)
|
| 131 |
+
return {
|
| 132 |
+
"out_path": out_path,
|
| 133 |
+
"out_sample_rate_hz": out_rate_hz,
|
| 134 |
+
"num_latent_frames": int(latents.shape[-1]),
|
| 135 |
+
"latent_dim": int(latents.shape[1]),
|
| 136 |
+
"latent_mean": float(latents.mean()),
|
| 137 |
+
"latent_std": float(latents.std()),
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 141 |
+
# Clips have different lengths and the VAE is cheap: process one at a time.
|
| 142 |
+
return [self._round_trip(r) for r in batch]
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
if __name__ == "__main__":
|
| 146 |
+
raise SystemExit(main_for(VoxCPMVAEAdapter()))
|
src/nida/cli.py
ADDED
|
@@ -0,0 +1,929 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Nida command-line interface.
|
| 2 |
+
|
| 3 |
+
Every subcommand accepts the common run flags (``--runs-root``, ``--run-id``,
|
| 4 |
+
``--git-commit``) so the Colab runner can call any command the same way.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
import logging
|
| 12 |
+
import sys
|
| 13 |
+
from collections.abc import Sequence
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
from nida.envreport import collect_env, summarize_env
|
| 17 |
+
from nida.runs import RunContext
|
| 18 |
+
from nida.utils.io import atomic_write_json, atomic_write_text
|
| 19 |
+
|
| 20 |
+
__all__ = ["main"]
|
| 21 |
+
|
| 22 |
+
# src/nida/cli.py → repository root (editable installs, as on Colab).
|
| 23 |
+
_REPO_DIR = Path(__file__).resolve().parents[2]
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _common_parser() -> argparse.ArgumentParser:
|
| 27 |
+
common = argparse.ArgumentParser(add_help=False)
|
| 28 |
+
common.add_argument(
|
| 29 |
+
"--runs-root",
|
| 30 |
+
type=Path,
|
| 31 |
+
default=None,
|
| 32 |
+
help="Directory holding run folders. If omitted, nothing is written to disk.",
|
| 33 |
+
)
|
| 34 |
+
common.add_argument("--run-id", default=None, help="Explicit run id (default: generated).")
|
| 35 |
+
common.add_argument("--git-commit", default=None, help="Commit SHA of the code being run.")
|
| 36 |
+
return common
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _positive_int(text: str) -> int:
|
| 40 |
+
"""argparse type: an integer >= 1 (clear error otherwise)."""
|
| 41 |
+
try:
|
| 42 |
+
value = int(text)
|
| 43 |
+
except ValueError:
|
| 44 |
+
raise argparse.ArgumentTypeError(f"expected an integer >= 1, got {text!r}") from None
|
| 45 |
+
if value < 1:
|
| 46 |
+
raise argparse.ArgumentTypeError(f"expected an integer >= 1, got {value}")
|
| 47 |
+
return value
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _build_parser() -> argparse.ArgumentParser:
|
| 51 |
+
parser = argparse.ArgumentParser(prog="nida", description="Nida expressive Turkish TTS.")
|
| 52 |
+
sub = parser.add_subparsers(dest="command", required=True)
|
| 53 |
+
sub.add_parser(
|
| 54 |
+
"env-report",
|
| 55 |
+
parents=[_common_parser()],
|
| 56 |
+
help="Print (and optionally record) Python/torch/GPU facts.",
|
| 57 |
+
)
|
| 58 |
+
bench = sub.add_parser(
|
| 59 |
+
"bench-phase0",
|
| 60 |
+
parents=[_common_parser()],
|
| 61 |
+
help="Phase 0 benchmarks B1 (ASR) and B2 (VAE); see docs/benchmarks/B1-B2-tasarim.md.",
|
| 62 |
+
)
|
| 63 |
+
bench.add_argument("--stage", default="all", help="A stage name or 'all'.")
|
| 64 |
+
bench.add_argument(
|
| 65 |
+
"--workspace",
|
| 66 |
+
type=Path,
|
| 67 |
+
default=None,
|
| 68 |
+
help="Durable benchmark directory (default: <runs-root>/../bench/phase0).",
|
| 69 |
+
)
|
| 70 |
+
bench.add_argument("--cache", type=Path, default=Path("/content/nida_cache"))
|
| 71 |
+
bench.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
|
| 72 |
+
bench.add_argument(
|
| 73 |
+
"--repo", type=Path, default=_REPO_DIR, help="Repository root (for configs and envs)."
|
| 74 |
+
)
|
| 75 |
+
bench.add_argument("--c-results", type=Path, default=None, help="B1 layer-C panel results CSV.")
|
| 76 |
+
bench.add_argument(
|
| 77 |
+
"--b2-panel-results", type=Path, default=None, help="B2 listening panel results CSV."
|
| 78 |
+
)
|
| 79 |
+
bench.add_argument(
|
| 80 |
+
"--retry-failed", action="store_true", help="Retry worker items that failed before."
|
| 81 |
+
)
|
| 82 |
+
pilot = sub.add_parser(
|
| 83 |
+
"data-pilot",
|
| 84 |
+
parents=[_common_parser()],
|
| 85 |
+
help="Phase 1 data pipeline pilot on YODAS (docs/plans/faz-1.md).",
|
| 86 |
+
)
|
| 87 |
+
pilot.add_argument("--stage", default="all", help="A stage name or 'all'.")
|
| 88 |
+
pilot.add_argument(
|
| 89 |
+
"--root",
|
| 90 |
+
type=Path,
|
| 91 |
+
default=None,
|
| 92 |
+
help="Durable pilot directory (default: <runs-root>/../data/pilot).",
|
| 93 |
+
)
|
| 94 |
+
pilot.add_argument("--cache", type=Path, default=Path("/content/nida_cache/pilot"))
|
| 95 |
+
pilot.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
|
| 96 |
+
pilot.add_argument("--repo", type=Path, default=_REPO_DIR)
|
| 97 |
+
pilot.add_argument(
|
| 98 |
+
"--subset",
|
| 99 |
+
choices=("tr000", "tr100"), # keep in sync with nida.data.pilot.YODAS_SUBSETS
|
| 100 |
+
default="tr000",
|
| 101 |
+
help=(
|
| 102 |
+
"YODAS Turkish subset: tr000 (manual subtitles, 64 shards) or tr100 (automatic "
|
| 103 |
+
"subtitles, 430 shards; videos that occur in tr000 are dropped)."
|
| 104 |
+
),
|
| 105 |
+
)
|
| 106 |
+
pilot.add_argument(
|
| 107 |
+
"--shards", default="00000002", help="Comma-separated shards of the chosen subset."
|
| 108 |
+
)
|
| 109 |
+
pilot.add_argument("--max-hours", type=float, default=10.0)
|
| 110 |
+
pilot.add_argument("--asr-primary", default="omniasr_llm_7b")
|
| 111 |
+
pilot.add_argument("--asr-secondary", default="qwen3_asr_1_7b")
|
| 112 |
+
pilot.add_argument("--retry-failed", action="store_true")
|
| 113 |
+
pilot.add_argument(
|
| 114 |
+
"--asr-batch-size",
|
| 115 |
+
type=int,
|
| 116 |
+
default=None,
|
| 117 |
+
help="Override the ASR workers' batch size (bench sizes fit a 22 GB L4).",
|
| 118 |
+
)
|
| 119 |
+
pilot.add_argument(
|
| 120 |
+
"--parallel-workers",
|
| 121 |
+
type=_positive_int,
|
| 122 |
+
default=1,
|
| 123 |
+
help=(
|
| 124 |
+
"Run up to N of the independent GPU worker stages (sed, diar, both ASR systems, "
|
| 125 |
+
"ctc, emotion) at the same time in 'all' (default 1 = one after another)."
|
| 126 |
+
),
|
| 127 |
+
)
|
| 128 |
+
ears = sub.add_parser(
|
| 129 |
+
"data-ears",
|
| 130 |
+
parents=[_common_parser()],
|
| 131 |
+
help="EARS -> latent dataset (ADR-0007; docs/03-data-pipeline.md).",
|
| 132 |
+
)
|
| 133 |
+
ears.add_argument(
|
| 134 |
+
"--root",
|
| 135 |
+
type=Path,
|
| 136 |
+
default=None,
|
| 137 |
+
help="Durable directory (default: <runs-root>/../data/ears); dataset is <root>/dataset.",
|
| 138 |
+
)
|
| 139 |
+
ears.add_argument(
|
| 140 |
+
"--speakers", default="all", help="Comma-separated speakers (p001,p002) or 'all'."
|
| 141 |
+
)
|
| 142 |
+
ears.add_argument("--cache", type=Path, default=Path("/content/nida_cache/ears"))
|
| 143 |
+
ears.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
|
| 144 |
+
ears.add_argument("--repo", type=Path, default=_REPO_DIR)
|
| 145 |
+
ears.add_argument("--seed", type=int, default=20261009, help="Speaker split seed.")
|
| 146 |
+
ears.add_argument("--config-dir", type=Path, default=None, help="Default: <repo>/configs/data.")
|
| 147 |
+
imp = sub.add_parser(
|
| 148 |
+
"data-import",
|
| 149 |
+
parents=[_common_parser()],
|
| 150 |
+
help="External transcribed corpus -> latent dataset (nida.data.importer).",
|
| 151 |
+
)
|
| 152 |
+
imp.add_argument(
|
| 153 |
+
"--source",
|
| 154 |
+
# keep in sync with nida.data.importer.sources.SOURCES
|
| 155 |
+
choices=("alania", "tr_full", "wavdir", "issai_tsc", "mediaspeech_tr", "common_voice_tr"),
|
| 156 |
+
required=True,
|
| 157 |
+
help=(
|
| 158 |
+
"alania (HF parquet), tr_full (HF data.jsonl + wavs), wavdir (--source-dir), "
|
| 159 |
+
"issai_tsc (HF tar.gz), mediaspeech_tr (OpenSLR TR.tgz) or common_voice_tr "
|
| 160 |
+
"(--source-dir: a Common Voice release folder or tarball you downloaded)."
|
| 161 |
+
),
|
| 162 |
+
)
|
| 163 |
+
imp.add_argument(
|
| 164 |
+
"--stage",
|
| 165 |
+
default="all",
|
| 166 |
+
help="select | audio | latents | ctc | timings | dataset | report | all.",
|
| 167 |
+
)
|
| 168 |
+
imp.add_argument(
|
| 169 |
+
"--root",
|
| 170 |
+
type=Path,
|
| 171 |
+
default=None,
|
| 172 |
+
help="Durable directory (default: <runs-root>/../data/import-<source>).",
|
| 173 |
+
)
|
| 174 |
+
imp.add_argument(
|
| 175 |
+
"--cache",
|
| 176 |
+
type=Path,
|
| 177 |
+
default=None,
|
| 178 |
+
help="Local scratch (default: /content/nida_cache/import/<root name>).",
|
| 179 |
+
)
|
| 180 |
+
imp.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
|
| 181 |
+
imp.add_argument("--repo", type=Path, default=_REPO_DIR)
|
| 182 |
+
imp.add_argument("--config-dir", type=Path, default=None, help="Default: <repo>/configs/data.")
|
| 183 |
+
imp.add_argument("--max-hours", type=float, default=10.0, help="Stop selecting at this many h.")
|
| 184 |
+
imp.add_argument(
|
| 185 |
+
"--shard-offset",
|
| 186 |
+
type=int,
|
| 187 |
+
default=0,
|
| 188 |
+
help="First work unit (parquet shard / row chunk); roots take disjoint ranges.",
|
| 189 |
+
)
|
| 190 |
+
imp.add_argument(
|
| 191 |
+
"--shards", type=_positive_int, default=None, help="Number of work units (default: rest)."
|
| 192 |
+
)
|
| 193 |
+
imp.add_argument(
|
| 194 |
+
"--source-dir",
|
| 195 |
+
type=Path,
|
| 196 |
+
default=None,
|
| 197 |
+
help="wavdir: folder with manifest.jsonl; common_voice_tr: release folder or tarball.",
|
| 198 |
+
)
|
| 199 |
+
imp.add_argument(
|
| 200 |
+
"--revision",
|
| 201 |
+
default=None,
|
| 202 |
+
help=(
|
| 203 |
+
"Hub commit sha to pin (default: current head, recorded); mediaspeech_tr: sha256 of "
|
| 204 |
+
"TR.tgz (default: the one in configs/data/import_corpora.yaml)."
|
| 205 |
+
),
|
| 206 |
+
)
|
| 207 |
+
imp.add_argument("--retry-failed", action="store_true")
|
| 208 |
+
imp.add_argument(
|
| 209 |
+
"--audio-threads",
|
| 210 |
+
type=_positive_int,
|
| 211 |
+
default=4,
|
| 212 |
+
help="Threads converting raw audio to 16 kHz wavs (does not change the outputs).",
|
| 213 |
+
)
|
| 214 |
+
train = sub.add_parser(
|
| 215 |
+
"train",
|
| 216 |
+
parents=[_common_parser()],
|
| 217 |
+
help="Phase 2 flow-matching training on a latent dataset (docs/04-architecture.md).",
|
| 218 |
+
)
|
| 219 |
+
train.add_argument("--config", type=Path, required=True, help="YAML under configs/train/.")
|
| 220 |
+
train.add_argument(
|
| 221 |
+
"--data",
|
| 222 |
+
type=Path,
|
| 223 |
+
required=True,
|
| 224 |
+
action="append",
|
| 225 |
+
help="Latent dataset directory; repeat the flag to train on the union of several.",
|
| 226 |
+
)
|
| 227 |
+
train.add_argument(
|
| 228 |
+
"--out", type=Path, required=True, help="Training workspace (checkpoints, metrics.jsonl)."
|
| 229 |
+
)
|
| 230 |
+
train.add_argument(
|
| 231 |
+
"--resume", action="store_true", help="Continue from the newest checkpoint in --out."
|
| 232 |
+
)
|
| 233 |
+
train.add_argument("--device", default=None, help="cpu | cuda | auto (default: auto).")
|
| 234 |
+
train.add_argument(
|
| 235 |
+
"--init-from",
|
| 236 |
+
type=Path,
|
| 237 |
+
default=None,
|
| 238 |
+
help="Fine-tune: start from this checkpoint's weights and latent stats (same model "
|
| 239 |
+
"config, except dur_cond none -> emotion) when --out has no checkpoint yet; "
|
| 240 |
+
"optimizer and schedule start fresh.",
|
| 241 |
+
)
|
| 242 |
+
samples = sub.add_parser(
|
| 243 |
+
"synth-samples",
|
| 244 |
+
parents=[_common_parser()],
|
| 245 |
+
help="Listening page: validation items re-spoken by a trained checkpoint.",
|
| 246 |
+
)
|
| 247 |
+
samples.add_argument(
|
| 248 |
+
"--checkpoint", type=Path, required=True, help="ckpt-*.pt or a training workspace."
|
| 249 |
+
)
|
| 250 |
+
samples.add_argument("--data", type=Path, required=True, help="Latent dataset directory.")
|
| 251 |
+
samples.add_argument("--out", type=Path, required=True, help="Output directory.")
|
| 252 |
+
samples.add_argument("--n", type=int, default=12, help="Validation items to generate.")
|
| 253 |
+
samples.add_argument("--steps", type=int, default=32, help="ODE steps.")
|
| 254 |
+
samples.add_argument("--w-text", type=float, default=2.0)
|
| 255 |
+
# Keeps 1.0 on purpose (sample CER stays comparable with earlier runs); `say` uses 2.0.
|
| 256 |
+
samples.add_argument("--w-emo", type=float, default=1.0)
|
| 257 |
+
samples.add_argument("--seed", type=int, default=0)
|
| 258 |
+
samples.add_argument("--method", choices=["euler", "midpoint"], default="euler")
|
| 259 |
+
samples.add_argument(
|
| 260 |
+
"--sway", type=float, default=-1.0, help="Sway time-step warp (0 = uniform steps)."
|
| 261 |
+
)
|
| 262 |
+
samples.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
|
| 263 |
+
samples.add_argument("--repo", type=Path, default=_REPO_DIR)
|
| 264 |
+
samples.add_argument("--device", default=None, help="cpu | cuda | auto (default: auto).")
|
| 265 |
+
samples.add_argument(
|
| 266 |
+
"--no-asr", action="store_true", help="Skip the Qwen3-ASR read-back (CER) check."
|
| 267 |
+
)
|
| 268 |
+
say = sub.add_parser(
|
| 269 |
+
"say",
|
| 270 |
+
parents=[_common_parser()],
|
| 271 |
+
help="Speak free text after a validation speaker prompt (aligned checkpoints).",
|
| 272 |
+
)
|
| 273 |
+
say.add_argument("--checkpoint", type=Path, required=True, help="ckpt-*.pt or a workspace.")
|
| 274 |
+
say.add_argument(
|
| 275 |
+
"--data",
|
| 276 |
+
type=Path,
|
| 277 |
+
default=None,
|
| 278 |
+
help="Latent dataset (prompt source); not needed with only --prompt-list prompts.",
|
| 279 |
+
)
|
| 280 |
+
say.add_argument("--out", type=Path, required=True, help="Output directory.")
|
| 281 |
+
text = say.add_mutually_exclusive_group(required=True)
|
| 282 |
+
text.add_argument(
|
| 283 |
+
"--text",
|
| 284 |
+
help=(
|
| 285 |
+
"Text to speak (sentences split on . ! ? …; long ones also at , ; :). Numbers, "
|
| 286 |
+
"%%, TL, clock times, abbreviations and acronyms need not be written out: they are "
|
| 287 |
+
"normalised to spoken Turkish first (--no-normalize disables this). "
|
| 288 |
+
"'@mutlu:0.8' sets the emotion of the text after it; '[duraksama:0.3]' inserts "
|
| 289 |
+
"exactly 0.3 s of silence (it ends the sentence); '[duraksama]' lets the model pause."
|
| 290 |
+
),
|
| 291 |
+
)
|
| 292 |
+
text.add_argument("--text-file", type=Path, help="UTF-8 file with the text to speak.")
|
| 293 |
+
say.add_argument(
|
| 294 |
+
"--prompt-id", action="append", default=None, help="Validation segment (repeatable)."
|
| 295 |
+
)
|
| 296 |
+
say.add_argument(
|
| 297 |
+
"--n-prompts", type=int, default=3, help="Prompts to pick when no --prompt-id is given."
|
| 298 |
+
)
|
| 299 |
+
say.add_argument(
|
| 300 |
+
"--prompt-list",
|
| 301 |
+
type=Path,
|
| 302 |
+
default=None,
|
| 303 |
+
help="TSV '<wav>\t<transcript>' of prompts from outside the datasets.",
|
| 304 |
+
)
|
| 305 |
+
say.add_argument(
|
| 306 |
+
"--rank-prompts",
|
| 307 |
+
type=int,
|
| 308 |
+
default=0,
|
| 309 |
+
help="Decode this many prompt candidates and keep the cleanest --n-prompts (0 = off).",
|
| 310 |
+
)
|
| 311 |
+
say.add_argument("--steps", type=int, default=32, help="ODE steps.")
|
| 312 |
+
say.add_argument("--w-text", type=float, default=3.0)
|
| 313 |
+
say.add_argument("--w-emo", type=float, default=2.0)
|
| 314 |
+
say.add_argument("--seed", type=int, default=0)
|
| 315 |
+
say.add_argument("--method", choices=["euler", "midpoint"], default="euler")
|
| 316 |
+
say.add_argument("--sway", type=float, default=-1.0)
|
| 317 |
+
say.add_argument("--speed", type=float, default=1.0, help=">1 faster, <1 slower.")
|
| 318 |
+
say.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
|
| 319 |
+
say.add_argument("--repo", type=Path, default=_REPO_DIR)
|
| 320 |
+
say.add_argument("--device", default=None, help="cpu | cuda | auto (default: auto).")
|
| 321 |
+
say.add_argument("--no-asr", action="store_true", help="Skip the ASR read-back check.")
|
| 322 |
+
say.add_argument(
|
| 323 |
+
"--no-normalize",
|
| 324 |
+
action="store_true",
|
| 325 |
+
help=(
|
| 326 |
+
"Use the text as written: no expansion of digits, symbols, abbreviations or "
|
| 327 |
+
"acronyms (they must then be spelled out, or the text is rejected)."
|
| 328 |
+
),
|
| 329 |
+
)
|
| 330 |
+
say.add_argument(
|
| 331 |
+
"--strict-normalize",
|
| 332 |
+
action="store_true",
|
| 333 |
+
help=(
|
| 334 |
+
"Stop on every text-normalisation issue (a guessed acronym, an ambiguous time or "
|
| 335 |
+
"ordinal, ...); by default only issues that would be misread stop the run."
|
| 336 |
+
),
|
| 337 |
+
)
|
| 338 |
+
say.add_argument(
|
| 339 |
+
"--single-env",
|
| 340 |
+
action="store_true",
|
| 341 |
+
help=(
|
| 342 |
+
"Run every worker (VAE, CTC, ASR) with this interpreter instead of building one "
|
| 343 |
+
"venv per worker (a local install with all packages, e.g. a release user)."
|
| 344 |
+
),
|
| 345 |
+
)
|
| 346 |
+
queue = sub.add_parser(
|
| 347 |
+
"queue",
|
| 348 |
+
parents=[_common_parser()],
|
| 349 |
+
help="Run nida.cli commands listed in a file, one after another (nida.jobqueue).",
|
| 350 |
+
)
|
| 351 |
+
queue.add_argument("--file", type=Path, required=True, help="Queue file (one job a line).")
|
| 352 |
+
queue.add_argument(
|
| 353 |
+
"--wait-minutes",
|
| 354 |
+
type=float,
|
| 355 |
+
default=0.0,
|
| 356 |
+
help="Keep polling the file this long after the last job before returning.",
|
| 357 |
+
)
|
| 358 |
+
queue.add_argument(
|
| 359 |
+
"--bundle-dir", type=Path, default=None, help="Default: <runs-root>/../code."
|
| 360 |
+
)
|
| 361 |
+
queue.add_argument("--log-dir", type=Path, default=Path("/content/nida_logs"))
|
| 362 |
+
queue.add_argument("--repo", type=Path, default=_REPO_DIR)
|
| 363 |
+
return parser
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def _env_report_md(metrics: dict[str, object]) -> str:
|
| 367 |
+
def show(value: object) -> str:
|
| 368 |
+
return "—" if value is None else str(value)
|
| 369 |
+
|
| 370 |
+
lines = ["# env-report", ""]
|
| 371 |
+
if metrics["gpu_name"] is None:
|
| 372 |
+
lines.append("- device: CPU only")
|
| 373 |
+
else:
|
| 374 |
+
lines.append(f"- device: {metrics['gpu_name']} (count {metrics['gpu_count']})")
|
| 375 |
+
for key in (
|
| 376 |
+
"gpu_memory_gb",
|
| 377 |
+
"compute_capability",
|
| 378 |
+
"bf16_supported",
|
| 379 |
+
"torch_version",
|
| 380 |
+
"cuda_version",
|
| 381 |
+
"torchaudio_version",
|
| 382 |
+
"python",
|
| 383 |
+
"cpu_count",
|
| 384 |
+
"ram_total_gb",
|
| 385 |
+
"disk_free_gb",
|
| 386 |
+
):
|
| 387 |
+
lines.append(f"- {key}: {show(metrics[key])}")
|
| 388 |
+
return "\n".join(lines) + "\n"
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def _resolve_device(choice: str | None, cuda_available: bool) -> str:
|
| 392 |
+
"""``--device``: ``None`` or ``auto`` → cuda when available, else cpu; other values as is."""
|
| 393 |
+
if choice in (None, "auto"):
|
| 394 |
+
return "cuda" if cuda_available else "cpu"
|
| 395 |
+
return str(choice)
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
def _cmd_env_report(args: argparse.Namespace) -> int:
|
| 399 |
+
env = collect_env()
|
| 400 |
+
print(json.dumps(env, ensure_ascii=False, indent=2, sort_keys=True))
|
| 401 |
+
if args.runs_root is None:
|
| 402 |
+
return 0
|
| 403 |
+
with RunContext(
|
| 404 |
+
args.runs_root, "env-report", run_id=args.run_id, git_commit=args.git_commit
|
| 405 |
+
) as run:
|
| 406 |
+
atomic_write_json(run.dir / "env.json", env)
|
| 407 |
+
metrics = summarize_env(env)
|
| 408 |
+
atomic_write_json(run.dir / "metrics.json", metrics)
|
| 409 |
+
atomic_write_text(run.dir / "report.md", _env_report_md(metrics))
|
| 410 |
+
print(f"run_id={run.run_id}")
|
| 411 |
+
return 0
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def _cmd_bench_phase0(args: argparse.Namespace) -> int:
|
| 415 |
+
from nida.bench.phase0 import Phase0
|
| 416 |
+
from nida.bench.prepare import Workspace
|
| 417 |
+
|
| 418 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 419 |
+
workspace_root = args.workspace
|
| 420 |
+
if workspace_root is None:
|
| 421 |
+
if args.runs_root is None:
|
| 422 |
+
raise SystemExit("bench-phase0 needs --workspace or --runs-root")
|
| 423 |
+
workspace_root = args.runs_root.parent / "bench" / "phase0"
|
| 424 |
+
phase0 = Phase0(
|
| 425 |
+
workspace=Workspace(root=workspace_root, cache=args.cache),
|
| 426 |
+
envs_root=args.envs,
|
| 427 |
+
repo_dir=args.repo,
|
| 428 |
+
sources_path=args.repo / "configs" / "sources.yaml",
|
| 429 |
+
retry_failed=args.retry_failed,
|
| 430 |
+
)
|
| 431 |
+
if args.runs_root is None:
|
| 432 |
+
result = phase0.run(
|
| 433 |
+
args.stage, c_results=args.c_results, b2_panel_results=args.b2_panel_results
|
| 434 |
+
)
|
| 435 |
+
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
|
| 436 |
+
return 0
|
| 437 |
+
with RunContext(
|
| 438 |
+
args.runs_root, f"bench-phase0-{args.stage}", run_id=args.run_id, git_commit=args.git_commit
|
| 439 |
+
) as run:
|
| 440 |
+
result = phase0.run(
|
| 441 |
+
args.stage, c_results=args.c_results, b2_panel_results=args.b2_panel_results
|
| 442 |
+
)
|
| 443 |
+
atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
|
| 444 |
+
reports = [
|
| 445 |
+
(workspace_root / "reports" / name)
|
| 446 |
+
for name in ("b1_report.md", "b2_report.md", "b2_panel_report.md")
|
| 447 |
+
if (workspace_root / "reports" / name).exists()
|
| 448 |
+
]
|
| 449 |
+
body = "\n\n".join(r.read_text(encoding="utf-8") for r in reports)
|
| 450 |
+
atomic_write_text(
|
| 451 |
+
run.dir / "report.md",
|
| 452 |
+
f"# bench-phase0 ({args.stage})\n\nworkspace: {workspace_root}\n\n{body}\n",
|
| 453 |
+
)
|
| 454 |
+
print(f"run_id={run.run_id}")
|
| 455 |
+
return 0
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def _cmd_data_pilot(args: argparse.Namespace) -> int:
|
| 459 |
+
from nida.data.pilot import Pilot, PilotPlan
|
| 460 |
+
|
| 461 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 462 |
+
root = args.root
|
| 463 |
+
if root is None:
|
| 464 |
+
if args.runs_root is None:
|
| 465 |
+
raise SystemExit("data-pilot needs --root or --runs-root")
|
| 466 |
+
root = args.runs_root.parent / "data" / "pilot"
|
| 467 |
+
pilot = Pilot(
|
| 468 |
+
root=root,
|
| 469 |
+
cache=args.cache,
|
| 470 |
+
envs_root=args.envs,
|
| 471 |
+
repo_dir=args.repo,
|
| 472 |
+
sources_path=args.repo / "configs" / "sources.yaml",
|
| 473 |
+
config_dir=args.repo / "configs" / "data",
|
| 474 |
+
plan=PilotPlan(
|
| 475 |
+
subset=args.subset,
|
| 476 |
+
shards=args.shards,
|
| 477 |
+
max_hours=args.max_hours,
|
| 478 |
+
asr_primary=args.asr_primary,
|
| 479 |
+
asr_secondary=args.asr_secondary,
|
| 480 |
+
),
|
| 481 |
+
retry_failed=args.retry_failed,
|
| 482 |
+
asr_batch_size=args.asr_batch_size,
|
| 483 |
+
parallel_workers=args.parallel_workers,
|
| 484 |
+
)
|
| 485 |
+
if args.runs_root is None:
|
| 486 |
+
print(json.dumps(pilot.run(args.stage), ensure_ascii=False, indent=2, default=str))
|
| 487 |
+
return 0
|
| 488 |
+
with RunContext(
|
| 489 |
+
args.runs_root, f"data-pilot-{args.stage}", run_id=args.run_id, git_commit=args.git_commit
|
| 490 |
+
) as run:
|
| 491 |
+
result = pilot.run(args.stage)
|
| 492 |
+
atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
|
| 493 |
+
report = root / "reports" / "pilot.md"
|
| 494 |
+
body = report.read_text(encoding="utf-8") if report.exists() else ""
|
| 495 |
+
result_json = json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
| 496 |
+
lines = [
|
| 497 |
+
f"# data-pilot ({args.stage})",
|
| 498 |
+
"",
|
| 499 |
+
f"root: {root}",
|
| 500 |
+
"",
|
| 501 |
+
"```json",
|
| 502 |
+
result_json,
|
| 503 |
+
"```",
|
| 504 |
+
"",
|
| 505 |
+
body,
|
| 506 |
+
]
|
| 507 |
+
atomic_write_text(run.dir / "report.md", "\n".join(lines) + "\n")
|
| 508 |
+
print(f"run_id={run.run_id}")
|
| 509 |
+
return 0
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
def _cmd_data_ears(args: argparse.Namespace) -> int:
|
| 513 |
+
from nida.bench import fetch
|
| 514 |
+
from nida.bench.b1 import _stream_runner
|
| 515 |
+
from nida.data.ears import Ears, load_settings, parse_speakers
|
| 516 |
+
|
| 517 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 518 |
+
root = args.root
|
| 519 |
+
if root is None:
|
| 520 |
+
if args.runs_root is None:
|
| 521 |
+
raise SystemExit("data-ears needs --root or --runs-root")
|
| 522 |
+
root = args.runs_root.parent / "data" / "ears"
|
| 523 |
+
config_dir = args.config_dir or args.repo / "configs" / "data"
|
| 524 |
+
speakers = parse_speakers(args.speakers, load_settings(config_dir).ears.n_speakers)
|
| 525 |
+
ears = Ears(
|
| 526 |
+
root=root,
|
| 527 |
+
cache=args.cache,
|
| 528 |
+
envs_root=args.envs,
|
| 529 |
+
repo_dir=args.repo,
|
| 530 |
+
sources_path=args.repo / "configs" / "sources.yaml",
|
| 531 |
+
config_dir=config_dir,
|
| 532 |
+
seed=args.seed,
|
| 533 |
+
runner=_stream_runner,
|
| 534 |
+
opener=fetch.default_opener,
|
| 535 |
+
)
|
| 536 |
+
if args.runs_root is None:
|
| 537 |
+
print(json.dumps(ears.run(speakers), ensure_ascii=False, indent=2, default=str))
|
| 538 |
+
return 0
|
| 539 |
+
with RunContext(
|
| 540 |
+
args.runs_root, "data-ears", run_id=args.run_id, git_commit=args.git_commit
|
| 541 |
+
) as run:
|
| 542 |
+
result = ears.run(speakers)
|
| 543 |
+
atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
|
| 544 |
+
report = root / "report.md"
|
| 545 |
+
body = report.read_text(encoding="utf-8") if report.exists() else ""
|
| 546 |
+
result_json = json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
| 547 |
+
lines = ["# data-ears", "", f"root: {root}", "", "```json", result_json, "```", "", body]
|
| 548 |
+
atomic_write_text(run.dir / "report.md", "\n".join(lines) + "\n")
|
| 549 |
+
print(f"run_id={run.run_id}")
|
| 550 |
+
return 0
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
def _cmd_data_import(args: argparse.Namespace) -> int:
|
| 554 |
+
from nida.data.importer import Importer, ImportRequest
|
| 555 |
+
|
| 556 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 557 |
+
root = args.root
|
| 558 |
+
if root is None:
|
| 559 |
+
if args.runs_root is None:
|
| 560 |
+
raise SystemExit("data-import needs --root or --runs-root")
|
| 561 |
+
root = args.runs_root.parent / "data" / f"import-{args.source}"
|
| 562 |
+
cache = args.cache or Path("/content/nida_cache/import") / root.name
|
| 563 |
+
importer = Importer(
|
| 564 |
+
root=root,
|
| 565 |
+
cache=cache,
|
| 566 |
+
envs_root=args.envs,
|
| 567 |
+
repo_dir=args.repo,
|
| 568 |
+
request=ImportRequest(
|
| 569 |
+
source=args.source,
|
| 570 |
+
max_hours=args.max_hours,
|
| 571 |
+
unit_offset=args.shard_offset,
|
| 572 |
+
unit_count=args.shards,
|
| 573 |
+
revision=args.revision,
|
| 574 |
+
),
|
| 575 |
+
config_dir=args.config_dir,
|
| 576 |
+
wavdir=args.source_dir,
|
| 577 |
+
retry_failed=args.retry_failed,
|
| 578 |
+
audio_threads=args.audio_threads,
|
| 579 |
+
)
|
| 580 |
+
if args.runs_root is None:
|
| 581 |
+
print(json.dumps(importer.run(args.stage), ensure_ascii=False, indent=2, default=str))
|
| 582 |
+
return 0
|
| 583 |
+
with RunContext(
|
| 584 |
+
args.runs_root, f"data-import-{args.stage}", run_id=args.run_id, git_commit=args.git_commit
|
| 585 |
+
) as run:
|
| 586 |
+
result = importer.run(args.stage)
|
| 587 |
+
atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
|
| 588 |
+
report = root / "report.md"
|
| 589 |
+
body = report.read_text(encoding="utf-8") if report.exists() else ""
|
| 590 |
+
result_json = json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
| 591 |
+
lines = [
|
| 592 |
+
f"# data-import {args.source} ({args.stage})",
|
| 593 |
+
"",
|
| 594 |
+
f"root: {root}",
|
| 595 |
+
"",
|
| 596 |
+
"```json",
|
| 597 |
+
result_json,
|
| 598 |
+
"```",
|
| 599 |
+
"",
|
| 600 |
+
body,
|
| 601 |
+
]
|
| 602 |
+
atomic_write_text(run.dir / "report.md", "\n".join(lines) + "\n")
|
| 603 |
+
print(f"run_id={run.run_id}")
|
| 604 |
+
return 0
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
#: Exit code of ``nida train`` when a SIGTERM stopped it after saving a checkpoint.
|
| 608 |
+
EXIT_TRAIN_INTERRUPTED = 3
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
def _cmd_train(args: argparse.Namespace) -> int:
|
| 612 |
+
from nida.train.runner import TrainOutcome, run_training
|
| 613 |
+
|
| 614 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 615 |
+
|
| 616 |
+
def execute() -> TrainOutcome:
|
| 617 |
+
return run_training(
|
| 618 |
+
args.config,
|
| 619 |
+
args.data,
|
| 620 |
+
args.out,
|
| 621 |
+
resume=args.resume,
|
| 622 |
+
device=args.device,
|
| 623 |
+
git_commit=args.git_commit,
|
| 624 |
+
repo_dir=_REPO_DIR,
|
| 625 |
+
init_from=args.init_from,
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
if args.runs_root is None:
|
| 629 |
+
outcome = execute()
|
| 630 |
+
atomic_write_json(args.out / "metrics.json", outcome.metrics)
|
| 631 |
+
atomic_write_text(args.out / "report.md", outcome.report_md)
|
| 632 |
+
else:
|
| 633 |
+
with RunContext(
|
| 634 |
+
args.runs_root, "train", run_id=args.run_id, git_commit=args.git_commit
|
| 635 |
+
) as run:
|
| 636 |
+
outcome = execute()
|
| 637 |
+
if outcome.result.stopped_by_signal:
|
| 638 |
+
run.status.extra["interrupted"] = "sigterm"
|
| 639 |
+
atomic_write_json(run.dir / "metrics.json", outcome.metrics)
|
| 640 |
+
atomic_write_text(run.dir / "report.md", outcome.report_md)
|
| 641 |
+
print(f"run_id={run.run_id}")
|
| 642 |
+
return EXIT_TRAIN_INTERRUPTED if outcome.result.stopped_by_signal else 0
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
def _cmd_synth_samples(args: argparse.Namespace) -> int:
|
| 646 |
+
import torch
|
| 647 |
+
|
| 648 |
+
from nida.bench import envs as bench_envs
|
| 649 |
+
from nida.bench.b1 import _stream_runner
|
| 650 |
+
from nida.data.envs import data_env_for_worker
|
| 651 |
+
from nida.infer.samples import SampleSettings, make_listening_page
|
| 652 |
+
|
| 653 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 654 |
+
settings = SampleSettings(
|
| 655 |
+
n_items=args.n,
|
| 656 |
+
steps=args.steps,
|
| 657 |
+
w_text=args.w_text,
|
| 658 |
+
w_emo=args.w_emo,
|
| 659 |
+
seed=args.seed,
|
| 660 |
+
method=args.method,
|
| 661 |
+
sway=args.sway,
|
| 662 |
+
)
|
| 663 |
+
device = _resolve_device(args.device, torch.cuda.is_available())
|
| 664 |
+
|
| 665 |
+
def execute() -> Path:
|
| 666 |
+
python = bench_envs.ensure_env(
|
| 667 |
+
data_env_for_worker("vae_decode"),
|
| 668 |
+
args.envs,
|
| 669 |
+
args.repo,
|
| 670 |
+
uv=bench_envs.find_uv(bench_envs.subprocess_runner),
|
| 671 |
+
)
|
| 672 |
+
asr_python = None
|
| 673 |
+
if not args.no_asr:
|
| 674 |
+
asr_python = bench_envs.ensure_env(
|
| 675 |
+
bench_envs.env_for_worker("asr_qwen"),
|
| 676 |
+
args.envs,
|
| 677 |
+
args.repo,
|
| 678 |
+
uv=bench_envs.find_uv(bench_envs.subprocess_runner),
|
| 679 |
+
)
|
| 680 |
+
return make_listening_page(
|
| 681 |
+
args.checkpoint,
|
| 682 |
+
args.data,
|
| 683 |
+
args.out,
|
| 684 |
+
settings,
|
| 685 |
+
decoder_python=python,
|
| 686 |
+
runner=_stream_runner,
|
| 687 |
+
device=torch.device(device),
|
| 688 |
+
asr_python=asr_python,
|
| 689 |
+
)
|
| 690 |
+
|
| 691 |
+
if args.runs_root is None:
|
| 692 |
+
print(execute())
|
| 693 |
+
return 0
|
| 694 |
+
with RunContext(
|
| 695 |
+
args.runs_root, "synth-samples", run_id=args.run_id, git_commit=args.git_commit
|
| 696 |
+
) as run:
|
| 697 |
+
page = execute()
|
| 698 |
+
settings_json = (args.out / "settings.json").read_text(encoding="utf-8")
|
| 699 |
+
atomic_write_json(run.dir / "metrics.json", {"page": str(page)})
|
| 700 |
+
atomic_write_text(
|
| 701 |
+
run.dir / "report.md",
|
| 702 |
+
f"# synth-samples\n\npage: {page}\n\n```json\n{settings_json}\n```\n",
|
| 703 |
+
)
|
| 704 |
+
print(f"run_id={run.run_id}")
|
| 705 |
+
return 0
|
| 706 |
+
|
| 707 |
+
|
| 708 |
+
#: Package pins of the isolated worker envs that ``say --single-env`` replaces
|
| 709 |
+
#: (``nida.data.envs`` / ``nida.bench.envs``): checked, never installed.
|
| 710 |
+
_SINGLE_ENV_PINS = {"transformers": "4.57.6"}
|
| 711 |
+
_SINGLE_ENV_ASR_PINS = {"qwen-asr": "0.0.6"}
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def _single_env_report(with_asr: bool) -> dict[str, object]:
|
| 715 |
+
"""Interpreter and package versions a ``--single-env`` run uses, with every difference
|
| 716 |
+
from the worker envs' pins logged as a warning and listed (results may then differ
|
| 717 |
+
from Colab runs)."""
|
| 718 |
+
from importlib import metadata
|
| 719 |
+
|
| 720 |
+
pins = {**_SINGLE_ENV_PINS, **(_SINGLE_ENV_ASR_PINS if with_asr else {})}
|
| 721 |
+
versions: dict[str, str | None] = {}
|
| 722 |
+
for name in ("torch", "torchaudio", *pins):
|
| 723 |
+
try:
|
| 724 |
+
versions[name] = metadata.version(name)
|
| 725 |
+
except metadata.PackageNotFoundError:
|
| 726 |
+
versions[name] = None
|
| 727 |
+
mismatches = {
|
| 728 |
+
name: {"pinned": pin, "installed": versions[name]}
|
| 729 |
+
for name, pin in pins.items()
|
| 730 |
+
if versions[name] != pin
|
| 731 |
+
}
|
| 732 |
+
for name, diff in mismatches.items():
|
| 733 |
+
logging.getLogger(__name__).warning(
|
| 734 |
+
"say --single-env: %s %s installed, worker env pins %s",
|
| 735 |
+
name,
|
| 736 |
+
diff["installed"],
|
| 737 |
+
diff["pinned"],
|
| 738 |
+
)
|
| 739 |
+
return {
|
| 740 |
+
"single_env": True,
|
| 741 |
+
"python": sys.executable,
|
| 742 |
+
"versions": versions,
|
| 743 |
+
"pin_mismatches": mismatches,
|
| 744 |
+
}
|
| 745 |
+
|
| 746 |
+
|
| 747 |
+
def _cmd_say(args: argparse.Namespace) -> int:
|
| 748 |
+
import torch
|
| 749 |
+
|
| 750 |
+
from nida.bench import envs as bench_envs
|
| 751 |
+
from nida.bench.b1 import _stream_runner
|
| 752 |
+
from nida.data.envs import data_env_for_worker
|
| 753 |
+
from nida.infer.say import (
|
| 754 |
+
SaySettings,
|
| 755 |
+
make_say_page,
|
| 756 |
+
pick_prompts,
|
| 757 |
+
prompt_candidates,
|
| 758 |
+
rank_prompts,
|
| 759 |
+
)
|
| 760 |
+
|
| 761 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 762 |
+
settings = SaySettings(
|
| 763 |
+
steps=args.steps,
|
| 764 |
+
w_text=args.w_text,
|
| 765 |
+
w_emo=args.w_emo,
|
| 766 |
+
seed=args.seed,
|
| 767 |
+
method=args.method,
|
| 768 |
+
sway=args.sway,
|
| 769 |
+
speed=args.speed,
|
| 770 |
+
normalize_text=not args.no_normalize,
|
| 771 |
+
strict_normalize=args.strict_normalize,
|
| 772 |
+
)
|
| 773 |
+
text = args.text if args.text is not None else args.text_file.read_text(encoding="utf-8")
|
| 774 |
+
device = _resolve_device(args.device, torch.cuda.is_available())
|
| 775 |
+
if args.data is None and (args.prompt_list is None or args.prompt_id or args.rank_prompts):
|
| 776 |
+
raise SystemExit("say: --data is required unless every prompt comes from --prompt-list")
|
| 777 |
+
if args.rank_prompts and (args.prompt_list is not None or args.prompt_id):
|
| 778 |
+
logging.getLogger(__name__).warning(
|
| 779 |
+
"say: --rank-prompts is ignored when --prompt-list or --prompt-id gives the prompts"
|
| 780 |
+
)
|
| 781 |
+
# make_say_page reads data_dir only for dataset prompt ids, which the guard above rules
|
| 782 |
+
# out when --data is missing.
|
| 783 |
+
data_dir: Path = args.data if args.data is not None else args.out / "no-dataset"
|
| 784 |
+
worker_device = "cuda" if torch.device(device).type == "cuda" else "cpu"
|
| 785 |
+
if args.single_env:
|
| 786 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 787 |
+
atomic_write_json(args.out / "environment.json", _single_env_report(not args.no_asr))
|
| 788 |
+
else:
|
| 789 |
+
(args.out / "environment.json").unlink(missing_ok=True) # never a stale single-env report
|
| 790 |
+
|
| 791 |
+
def env_python(spec: bench_envs.EnvSpec, uv: str | None) -> Path:
|
| 792 |
+
if args.single_env:
|
| 793 |
+
return Path(sys.executable)
|
| 794 |
+
assert uv is not None
|
| 795 |
+
return bench_envs.ensure_env(spec, args.envs, args.repo, uv=uv)
|
| 796 |
+
|
| 797 |
+
def execute() -> Path:
|
| 798 |
+
uv = None if args.single_env else bench_envs.find_uv(bench_envs.subprocess_runner)
|
| 799 |
+
python = env_python(data_env_for_worker("vae_decode"), uv)
|
| 800 |
+
external = []
|
| 801 |
+
if args.prompt_list is not None:
|
| 802 |
+
from nida.data.registry import DATA_MODELS
|
| 803 |
+
from nida.infer.external_prompt import build_external_prompts, read_prompt_list
|
| 804 |
+
|
| 805 |
+
ctc_python = env_python(data_env_for_worker(DATA_MODELS["mpoyraz_ctc_tr"].worker), uv)
|
| 806 |
+
external = build_external_prompts(
|
| 807 |
+
read_prompt_list(args.prompt_list),
|
| 808 |
+
args.out / "external",
|
| 809 |
+
vae_python=env_python(data_env_for_worker("vae_encode"), uv),
|
| 810 |
+
ctc_python=ctc_python,
|
| 811 |
+
runner=_stream_runner,
|
| 812 |
+
device=worker_device,
|
| 813 |
+
)
|
| 814 |
+
if args.prompt_id:
|
| 815 |
+
prompt_ids = args.prompt_id
|
| 816 |
+
elif external:
|
| 817 |
+
prompt_ids = []
|
| 818 |
+
elif args.rank_prompts > 0:
|
| 819 |
+
ranked = rank_prompts(
|
| 820 |
+
args.data,
|
| 821 |
+
prompt_candidates(args.data, args.rank_prompts),
|
| 822 |
+
args.out,
|
| 823 |
+
decoder_python=python,
|
| 824 |
+
runner=_stream_runner,
|
| 825 |
+
device=worker_device,
|
| 826 |
+
)
|
| 827 |
+
prompt_ids = [str(r["segment_id"]) for r in ranked[: args.n_prompts]]
|
| 828 |
+
else:
|
| 829 |
+
prompt_ids = pick_prompts(args.data, args.n_prompts)
|
| 830 |
+
asr_python = None
|
| 831 |
+
if not args.no_asr:
|
| 832 |
+
asr_python = env_python(bench_envs.env_for_worker("asr_qwen"), uv)
|
| 833 |
+
return make_say_page(
|
| 834 |
+
args.checkpoint,
|
| 835 |
+
data_dir,
|
| 836 |
+
prompt_ids,
|
| 837 |
+
text,
|
| 838 |
+
args.out,
|
| 839 |
+
settings,
|
| 840 |
+
decoder_python=python,
|
| 841 |
+
runner=_stream_runner,
|
| 842 |
+
device=torch.device(device),
|
| 843 |
+
asr_python=asr_python,
|
| 844 |
+
external=external,
|
| 845 |
+
)
|
| 846 |
+
|
| 847 |
+
if args.runs_root is None:
|
| 848 |
+
print(execute())
|
| 849 |
+
return 0
|
| 850 |
+
with RunContext(args.runs_root, "say", run_id=args.run_id, git_commit=args.git_commit) as run:
|
| 851 |
+
page = execute()
|
| 852 |
+
settings_json = (args.out / "settings.json").read_text(encoding="utf-8")
|
| 853 |
+
atomic_write_json(run.dir / "metrics.json", {"page": str(page)})
|
| 854 |
+
atomic_write_text(
|
| 855 |
+
run.dir / "report.md", f"# say\n\npage: {page}\n\n```json\n{settings_json}\n```\n"
|
| 856 |
+
)
|
| 857 |
+
print(f"run_id={run.run_id}")
|
| 858 |
+
return 0
|
| 859 |
+
|
| 860 |
+
|
| 861 |
+
def _cmd_queue(args: argparse.Namespace) -> int:
|
| 862 |
+
from nida.jobqueue import (
|
| 863 |
+
run_git,
|
| 864 |
+
run_queue,
|
| 865 |
+
state_path_for,
|
| 866 |
+
stream_to_log,
|
| 867 |
+
summary_markdown,
|
| 868 |
+
)
|
| 869 |
+
|
| 870 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 871 |
+
if args.runs_root is None or args.git_commit is None:
|
| 872 |
+
print("queue needs --runs-root and --git-commit (the Colab runner passes both)")
|
| 873 |
+
return 2
|
| 874 |
+
bundle_dir = args.bundle_dir or args.runs_root.parent / "code"
|
| 875 |
+
with RunContext(args.runs_root, "queue", run_id=args.run_id, git_commit=args.git_commit) as run:
|
| 876 |
+
summary = run_queue(
|
| 877 |
+
args.file,
|
| 878 |
+
runs_root=args.runs_root,
|
| 879 |
+
git_commit=args.git_commit,
|
| 880 |
+
repo_dir=args.repo,
|
| 881 |
+
bundle_dir=bundle_dir,
|
| 882 |
+
log_dir=args.log_dir,
|
| 883 |
+
runner=stream_to_log,
|
| 884 |
+
git=run_git,
|
| 885 |
+
wait_minutes=args.wait_minutes,
|
| 886 |
+
)
|
| 887 |
+
state_file = state_path_for(args.file)
|
| 888 |
+
state = json.loads(state_file.read_text(encoding="utf-8")) if state_file.exists() else {}
|
| 889 |
+
atomic_write_json(
|
| 890 |
+
run.dir / "metrics.json",
|
| 891 |
+
{
|
| 892 |
+
"jobs": len(summary.outcomes),
|
| 893 |
+
"failed": len(summary.failed),
|
| 894 |
+
"interrupted": summary.interrupted,
|
| 895 |
+
},
|
| 896 |
+
)
|
| 897 |
+
atomic_write_text(run.dir / "report.md", summary_markdown(summary, state))
|
| 898 |
+
if summary.interrupted:
|
| 899 |
+
run.status.extra["interrupted"] = str(summary.interrupted_key)
|
| 900 |
+
print(f"run_id={run.run_id}")
|
| 901 |
+
return EXIT_TRAIN_INTERRUPTED if summary.interrupted else 0
|
| 902 |
+
|
| 903 |
+
|
| 904 |
+
def main(argv: Sequence[str] | None = None) -> int:
|
| 905 |
+
"""Entry point. Returns a process exit code."""
|
| 906 |
+
args = _build_parser().parse_args(argv)
|
| 907 |
+
if args.command == "env-report":
|
| 908 |
+
return _cmd_env_report(args)
|
| 909 |
+
if args.command == "bench-phase0":
|
| 910 |
+
return _cmd_bench_phase0(args)
|
| 911 |
+
if args.command == "data-pilot":
|
| 912 |
+
return _cmd_data_pilot(args)
|
| 913 |
+
if args.command == "data-ears":
|
| 914 |
+
return _cmd_data_ears(args)
|
| 915 |
+
if args.command == "data-import":
|
| 916 |
+
return _cmd_data_import(args)
|
| 917 |
+
if args.command == "train":
|
| 918 |
+
return _cmd_train(args)
|
| 919 |
+
if args.command == "synth-samples":
|
| 920 |
+
return _cmd_synth_samples(args)
|
| 921 |
+
if args.command == "say":
|
| 922 |
+
return _cmd_say(args)
|
| 923 |
+
if args.command == "queue":
|
| 924 |
+
return _cmd_queue(args)
|
| 925 |
+
raise AssertionError(f"unhandled command {args.command!r}") # argparse prevents this
|
| 926 |
+
|
| 927 |
+
|
| 928 |
+
if __name__ == "__main__":
|
| 929 |
+
raise SystemExit(main())
|
src/nida/data/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Phase 1 data pipeline (docs/03-data-pipeline.md).
|
| 2 |
+
|
| 3 |
+
Stage logic lives here as pure, CPU-testable functions over the records in
|
| 4 |
+
:mod:`nida.data.types`; model inference runs in isolated worker environments.
|
| 5 |
+
"""
|
src/nida/data/acoustic.py
ADDED
|
@@ -0,0 +1,1002 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Acoustic measurements for data-pipeline stages S1, S2 and S6 (docs/03-data-pipeline.md).
|
| 2 |
+
|
| 3 |
+
Everything here is pure, CPU-only numpy/scipy over mono waveforms:
|
| 4 |
+
|
| 5 |
+
* **S1** :func:`estimate_bandwidth_hz` - effective bandwidth of a file (detects content that
|
| 6 |
+
was low-passed or upsampled from a lower rate).
|
| 7 |
+
* **S2** :func:`clipping_ratio`, :func:`silence_ratio`, :func:`loudness_dbfs`.
|
| 8 |
+
* **S6** :func:`track_pitch` (numpy YIN), :func:`word_features` (per-word voicing, F0, HNR,
|
| 9 |
+
RMS, spectral tilt), :func:`speaker_baseline`, :func:`whisper_scores`, :func:`shout_scores`.
|
| 10 |
+
|
| 11 |
+
Conventions: waveforms are 1-D floating arrays (float32 or float64) with samples in
|
| 12 |
+
``[-1, 1]`` (``1.0`` = full scale); ``sample_rate_hz`` is an int; times are integer
|
| 13 |
+
milliseconds (``*_ms``) relative to the start of the waveform. All decision parameters come
|
| 14 |
+
from :class:`AcousticConfig` (``configs/data/acoustic.yaml``); the few literals left in
|
| 15 |
+
this file are numerical guards (division/log floors), not thresholds.
|
| 16 |
+
|
| 17 |
+
Invalid input (non-float dtype, not 1-D, empty, NaN/inf, bad rate) raises
|
| 18 |
+
:class:`ValueError` naming the argument; nothing is silently repaired or dropped.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import math
|
| 24 |
+
import typing
|
| 25 |
+
from collections.abc import Mapping, Sequence
|
| 26 |
+
from dataclasses import dataclass
|
| 27 |
+
from itertools import pairwise
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
from typing import Any, TypeAlias
|
| 30 |
+
|
| 31 |
+
import numpy as np
|
| 32 |
+
import numpy.typing as npt
|
| 33 |
+
|
| 34 |
+
# scipy ships no type stubs; the ignore is scoped to this single import.
|
| 35 |
+
from scipy import signal # type: ignore[import-untyped]
|
| 36 |
+
|
| 37 |
+
from nida.bench.audiometrics import stft_magnitude
|
| 38 |
+
from nida.data.types import TimedWord, WordStyleScore
|
| 39 |
+
|
| 40 |
+
__all__ = [
|
| 41 |
+
"AcousticConfig",
|
| 42 |
+
"BandwidthParams",
|
| 43 |
+
"BaselineError",
|
| 44 |
+
"FloatArray",
|
| 45 |
+
"PitchTrack",
|
| 46 |
+
"SilentAudioError",
|
| 47 |
+
"SpeakerBaseline",
|
| 48 |
+
"WordFeatures",
|
| 49 |
+
"clipping_ratio",
|
| 50 |
+
"estimate_bandwidth_hz",
|
| 51 |
+
"load_acoustic_config",
|
| 52 |
+
"loudness_dbfs",
|
| 53 |
+
"phrase_support",
|
| 54 |
+
"shout_scores",
|
| 55 |
+
"silence_ratio",
|
| 56 |
+
"speaker_baseline",
|
| 57 |
+
"track_pitch",
|
| 58 |
+
"whisper_scores",
|
| 59 |
+
"word_features",
|
| 60 |
+
]
|
| 61 |
+
|
| 62 |
+
FloatArray: TypeAlias = npt.NDArray[np.float64]
|
| 63 |
+
BoolArray: TypeAlias = npt.NDArray[np.bool_]
|
| 64 |
+
IntArray: TypeAlias = npt.NDArray[np.int64]
|
| 65 |
+
|
| 66 |
+
WHISPER_SOURCE = "acoustic.whisper.v0"
|
| 67 |
+
SHOUT_SOURCE = "acoustic.shout.v0"
|
| 68 |
+
|
| 69 |
+
# Numerical guards (not thresholds): floor under power before log10 / before division.
|
| 70 |
+
_POWER_FLOOR = 1e-24 # -240 dB; makes digital silence a finite number
|
| 71 |
+
_REL_EPS = 1e-9 # relative floor for "this energy is effectively zero"
|
| 72 |
+
_LOGISTIC_CLIP = 60.0 # |z| beyond this saturates to 0/1 and would overflow exp()
|
| 73 |
+
# Frames are processed in chunks to bound the memory of the batched FFTs (an
|
| 74 |
+
# implementation detail with no effect on results).
|
| 75 |
+
_CHUNK_FRAMES = 256
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
class SilentAudioError(ValueError):
|
| 79 |
+
"""The waveform carries no energy, so the requested measurement is undefined."""
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class BaselineError(ValueError):
|
| 83 |
+
"""Too few voiced words to estimate a speaker baseline (shout not measurable)."""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# --------------------------------------------------------------------------------------
|
| 87 |
+
# Config
|
| 88 |
+
# --------------------------------------------------------------------------------------
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _check_finite(value: float, name: str) -> None:
|
| 92 |
+
if not math.isfinite(value):
|
| 93 |
+
raise ValueError(f"{name} must be finite, got {value!r}")
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _check_pos(value: float, name: str) -> None:
|
| 97 |
+
if not (math.isfinite(value) and value > 0.0):
|
| 98 |
+
raise ValueError(f"{name} must be a finite number > 0, got {value!r}")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
@dataclass(frozen=True)
|
| 102 |
+
class BandwidthParams:
|
| 103 |
+
"""Analysis parameters of :func:`estimate_bandwidth_hz` (S1).
|
| 104 |
+
|
| 105 |
+
The field defaults are used only when a caller passes no config; they equal the
|
| 106 |
+
values in ``configs/data/acoustic.yaml`` at the time of writing.
|
| 107 |
+
|
| 108 |
+
Attributes:
|
| 109 |
+
n_fft_samples: FFT size of the long-term average spectrum (LTAS).
|
| 110 |
+
active_within_db: A frame is *active* when its energy is within this many dB of the
|
| 111 |
+
loudest frame; only active frames enter the LTAS.
|
| 112 |
+
ref_low_hz: Lower edge of the reference band.
|
| 113 |
+
ref_high_hz: Upper edge of the reference band (clipped to Nyquist at run time).
|
| 114 |
+
The reference level is the median smoothed LTAS level inside this band.
|
| 115 |
+
smooth_hz: Width of the median filter run over the LTAS; rejects isolated
|
| 116 |
+
narrow-band spikes (hum, pilot tones) above the true cut-off. A tone is
|
| 117 |
+
rejected when its spectral footprint (Hann main lobe plus side lobes above the
|
| 118 |
+
``floor_db`` threshold) is narrower than half this width; a tone far louder
|
| 119 |
+
than the in-band level leaks wider than that and counts as bandwidth.
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
n_fft_samples: int = 2048
|
| 123 |
+
active_within_db: float = 30.0
|
| 124 |
+
ref_low_hz: float = 300.0
|
| 125 |
+
ref_high_hz: float = 3400.0
|
| 126 |
+
smooth_hz: float = 600.0
|
| 127 |
+
|
| 128 |
+
def __post_init__(self) -> None:
|
| 129 |
+
if self.n_fft_samples < 16:
|
| 130 |
+
raise ValueError(f"n_fft_samples must be >= 16, got {self.n_fft_samples}")
|
| 131 |
+
_check_pos(self.active_within_db, "active_within_db")
|
| 132 |
+
_check_pos(self.ref_low_hz, "ref_low_hz")
|
| 133 |
+
_check_pos(self.ref_high_hz, "ref_high_hz")
|
| 134 |
+
_check_pos(self.smooth_hz, "smooth_hz")
|
| 135 |
+
if self.ref_low_hz >= self.ref_high_hz:
|
| 136 |
+
raise ValueError(
|
| 137 |
+
f"need ref_low_hz < ref_high_hz, got {self.ref_low_hz} >= {self.ref_high_hz}"
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
@classmethod
|
| 141 |
+
def from_mapping(cls, raw: object) -> BandwidthParams:
|
| 142 |
+
"""Build from a mapping; unknown or missing keys raise :class:`ValueError`."""
|
| 143 |
+
return cls(**_coerce_fields(cls, raw, "bandwidth")) # type: ignore[arg-type]
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
@dataclass(frozen=True)
|
| 147 |
+
class AcousticConfig:
|
| 148 |
+
"""All decision parameters of this module. Values: ``configs/data/acoustic.yaml``.
|
| 149 |
+
|
| 150 |
+
S1 / S2 measurement parameters (consumed by the stage code, which passes them to the
|
| 151 |
+
functions below):
|
| 152 |
+
bandwidth_floor_db: ``floor_db`` for :func:`estimate_bandwidth_hz`.
|
| 153 |
+
bandwidth: Other bandwidth-estimation parameters (:class:`BandwidthParams`).
|
| 154 |
+
clipping_threshold: ``threshold`` for :func:`clipping_ratio` (in (0, 1]).
|
| 155 |
+
silence_frame_ms: ``frame_ms`` for :func:`silence_ratio`.
|
| 156 |
+
silence_dbfs: ``silence_dbfs`` for :func:`silence_ratio`.
|
| 157 |
+
|
| 158 |
+
S6 word features:
|
| 159 |
+
min_word_ms: Words shorter than this get ``None``-marked features.
|
| 160 |
+
word_end_tolerance_ms: A word may overshoot the end of the audio by at most this
|
| 161 |
+
much (alignment rounding); it is clipped. More raises ``ValueError``.
|
| 162 |
+
f0_min_hz, f0_max_hz: Search range of the pitch tracker.
|
| 163 |
+
f0_frame_ms, f0_hop_ms: YIN analysis window and hop. The window must span at least
|
| 164 |
+
two periods of ``f0_min_hz``.
|
| 165 |
+
yin_threshold: Absolute threshold on the cumulative-mean-normalised difference
|
| 166 |
+
function below which a frame counts as voiced.
|
| 167 |
+
f0_energy_floor_dbfs: Frames quieter than this (RMS dBFS) are never voiced.
|
| 168 |
+
hnr_max_db: Per-frame HNR is clipped to ``[-hnr_max_db, +hnr_max_db]``.
|
| 169 |
+
tilt_low_hz, tilt_high_hz: Frequency range of the spectral-tilt regression (the
|
| 170 |
+
upper edge is clipped to Nyquist at run time).
|
| 171 |
+
tilt_num_bands: Number of log-spaced bands the regression is fitted on (>= 3).
|
| 172 |
+
tilt_n_fft_samples: FFT size for the tilt spectrum.
|
| 173 |
+
|
| 174 |
+
Whisper score (soft AND of three logistic factors, see :func:`whisper_scores`):
|
| 175 |
+
whisper_unvoiced_center, whisper_unvoiced_scale: on ``1 - voicing_ratio``.
|
| 176 |
+
whisper_hnr_center_db, whisper_hnr_scale_db: on ``-hnr_db`` (low HNR -> whisper).
|
| 177 |
+
whisper_energy_center_dbfs, whisper_energy_scale_db: on ``rms_dbfs`` (energy must
|
| 178 |
+
be present, otherwise silence would look like a whisper).
|
| 179 |
+
|
| 180 |
+
Shout score (see :func:`shout_scores`):
|
| 181 |
+
baseline_min_voicing_ratio: A word is *voiced* for the speaker baseline when its
|
| 182 |
+
``voicing_ratio`` is at least this.
|
| 183 |
+
baseline_min_voiced_words: Fewer voiced words than this -> no baseline.
|
| 184 |
+
shout_weight_db, shout_weight_f0_semitones, shout_weight_tilt: Weights of the
|
| 185 |
+
deviations from the speaker baseline in the linear combination.
|
| 186 |
+
shout_center, shout_scale: Logistic centre/scale on that combination.
|
| 187 |
+
"""
|
| 188 |
+
|
| 189 |
+
bandwidth_floor_db: float
|
| 190 |
+
bandwidth: BandwidthParams
|
| 191 |
+
clipping_threshold: float
|
| 192 |
+
silence_frame_ms: float
|
| 193 |
+
silence_dbfs: float
|
| 194 |
+
|
| 195 |
+
min_word_ms: int
|
| 196 |
+
word_end_tolerance_ms: int
|
| 197 |
+
f0_min_hz: float
|
| 198 |
+
f0_max_hz: float
|
| 199 |
+
f0_frame_ms: float
|
| 200 |
+
f0_hop_ms: float
|
| 201 |
+
yin_threshold: float
|
| 202 |
+
f0_energy_floor_dbfs: float
|
| 203 |
+
hnr_max_db: float
|
| 204 |
+
tilt_low_hz: float
|
| 205 |
+
tilt_high_hz: float
|
| 206 |
+
tilt_num_bands: int
|
| 207 |
+
tilt_n_fft_samples: int
|
| 208 |
+
|
| 209 |
+
whisper_unvoiced_center: float
|
| 210 |
+
whisper_unvoiced_scale: float
|
| 211 |
+
whisper_hnr_center_db: float
|
| 212 |
+
whisper_hnr_scale_db: float
|
| 213 |
+
whisper_energy_center_dbfs: float
|
| 214 |
+
whisper_energy_scale_db: float
|
| 215 |
+
|
| 216 |
+
baseline_min_voicing_ratio: float
|
| 217 |
+
baseline_min_voiced_words: int
|
| 218 |
+
shout_weight_db: float
|
| 219 |
+
shout_weight_f0_semitones: float
|
| 220 |
+
shout_weight_tilt: float
|
| 221 |
+
shout_center: float
|
| 222 |
+
shout_scale: float
|
| 223 |
+
|
| 224 |
+
def __post_init__(self) -> None:
|
| 225 |
+
_check_pos(self.bandwidth_floor_db, "bandwidth_floor_db")
|
| 226 |
+
if not (math.isfinite(self.clipping_threshold) and 0.0 < self.clipping_threshold <= 1.0):
|
| 227 |
+
raise ValueError(f"clipping_threshold must be in (0, 1], got {self.clipping_threshold}")
|
| 228 |
+
_check_pos(self.silence_frame_ms, "silence_frame_ms")
|
| 229 |
+
_check_finite(self.silence_dbfs, "silence_dbfs")
|
| 230 |
+
|
| 231 |
+
if self.min_word_ms <= 0:
|
| 232 |
+
raise ValueError(f"min_word_ms must be > 0, got {self.min_word_ms}")
|
| 233 |
+
if self.word_end_tolerance_ms < 0:
|
| 234 |
+
raise ValueError(
|
| 235 |
+
f"word_end_tolerance_ms must be >= 0, got {self.word_end_tolerance_ms}"
|
| 236 |
+
)
|
| 237 |
+
_check_pos(self.f0_min_hz, "f0_min_hz")
|
| 238 |
+
_check_pos(self.f0_max_hz, "f0_max_hz")
|
| 239 |
+
if self.f0_min_hz >= self.f0_max_hz:
|
| 240 |
+
raise ValueError(f"need f0_min_hz < f0_max_hz, got {self.f0_min_hz}..{self.f0_max_hz}")
|
| 241 |
+
_check_pos(self.f0_frame_ms, "f0_frame_ms")
|
| 242 |
+
_check_pos(self.f0_hop_ms, "f0_hop_ms")
|
| 243 |
+
if not (math.isfinite(self.yin_threshold) and 0.0 < self.yin_threshold < 1.0):
|
| 244 |
+
raise ValueError(f"yin_threshold must be in (0, 1), got {self.yin_threshold}")
|
| 245 |
+
_check_finite(self.f0_energy_floor_dbfs, "f0_energy_floor_dbfs")
|
| 246 |
+
_check_pos(self.hnr_max_db, "hnr_max_db")
|
| 247 |
+
_check_pos(self.tilt_low_hz, "tilt_low_hz")
|
| 248 |
+
_check_pos(self.tilt_high_hz, "tilt_high_hz")
|
| 249 |
+
if self.tilt_low_hz >= self.tilt_high_hz:
|
| 250 |
+
raise ValueError(
|
| 251 |
+
f"need tilt_low_hz < tilt_high_hz, got {self.tilt_low_hz}..{self.tilt_high_hz}"
|
| 252 |
+
)
|
| 253 |
+
if self.tilt_num_bands < 3:
|
| 254 |
+
raise ValueError(f"tilt_num_bands must be >= 3, got {self.tilt_num_bands}")
|
| 255 |
+
if self.tilt_n_fft_samples < 16:
|
| 256 |
+
raise ValueError(f"tilt_n_fft_samples must be >= 16, got {self.tilt_n_fft_samples}")
|
| 257 |
+
if self.min_word_ms < self.f0_hop_ms:
|
| 258 |
+
raise ValueError(
|
| 259 |
+
f"min_word_ms ({self.min_word_ms}) must be >= f0_hop_ms ({self.f0_hop_ms}) so "
|
| 260 |
+
"that every measurable word contains at least one pitch frame"
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
for name in (
|
| 264 |
+
"whisper_unvoiced_scale",
|
| 265 |
+
"whisper_hnr_scale_db",
|
| 266 |
+
"whisper_energy_scale_db",
|
| 267 |
+
"shout_scale",
|
| 268 |
+
):
|
| 269 |
+
_check_pos(getattr(self, name), name)
|
| 270 |
+
for name in (
|
| 271 |
+
"whisper_unvoiced_center",
|
| 272 |
+
"whisper_hnr_center_db",
|
| 273 |
+
"whisper_energy_center_dbfs",
|
| 274 |
+
"shout_weight_db",
|
| 275 |
+
"shout_weight_f0_semitones",
|
| 276 |
+
"shout_weight_tilt",
|
| 277 |
+
"shout_center",
|
| 278 |
+
):
|
| 279 |
+
_check_finite(getattr(self, name), name)
|
| 280 |
+
if not (
|
| 281 |
+
math.isfinite(self.baseline_min_voicing_ratio)
|
| 282 |
+
and 0.0 < self.baseline_min_voicing_ratio <= 1.0
|
| 283 |
+
):
|
| 284 |
+
raise ValueError(
|
| 285 |
+
"baseline_min_voicing_ratio must be in (0, 1], "
|
| 286 |
+
f"got {self.baseline_min_voicing_ratio}"
|
| 287 |
+
)
|
| 288 |
+
if self.baseline_min_voiced_words < 1:
|
| 289 |
+
raise ValueError(
|
| 290 |
+
f"baseline_min_voiced_words must be >= 1, got {self.baseline_min_voiced_words}"
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
@classmethod
|
| 294 |
+
def from_mapping(cls, raw: object) -> AcousticConfig:
|
| 295 |
+
"""Build from a mapping (e.g. parsed YAML); unknown or missing keys are rejected."""
|
| 296 |
+
return cls(**_coerce_fields(cls, raw, "acoustic config")) # type: ignore[arg-type]
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _coerce_fields(cls: type, raw: object, where: str) -> dict[str, object]:
|
| 300 |
+
"""Check ``raw`` against the dataclass ``cls`` and coerce each value to its field type.
|
| 301 |
+
|
| 302 |
+
Rejects non-mappings, non-string keys, unknown keys, missing keys, bools in numeric
|
| 303 |
+
fields and non-integral values in int fields. Nested dataclass fields are built
|
| 304 |
+
recursively.
|
| 305 |
+
"""
|
| 306 |
+
if not isinstance(raw, Mapping):
|
| 307 |
+
raise ValueError(f"{where}: expected a mapping, got {type(raw).__name__}")
|
| 308 |
+
hints = typing.get_type_hints(cls)
|
| 309 |
+
bad_keys = [k for k in raw if not isinstance(k, str)]
|
| 310 |
+
if bad_keys:
|
| 311 |
+
raise ValueError(f"{where}: non-string keys {bad_keys!r}")
|
| 312 |
+
unknown = sorted(set(raw) - set(hints))
|
| 313 |
+
missing = sorted(set(hints) - set(raw))
|
| 314 |
+
if unknown or missing:
|
| 315 |
+
raise ValueError(f"{where}: unknown keys {unknown}, missing keys {missing}")
|
| 316 |
+
|
| 317 |
+
out: dict[str, object] = {}
|
| 318 |
+
for name, hint in hints.items():
|
| 319 |
+
value = raw[name]
|
| 320 |
+
path = f"{where}.{name}"
|
| 321 |
+
if hint is float:
|
| 322 |
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
| 323 |
+
raise ValueError(f"{path}: expected a number, got {value!r}")
|
| 324 |
+
out[name] = float(value)
|
| 325 |
+
elif hint is int:
|
| 326 |
+
if isinstance(value, bool) or not isinstance(value, int):
|
| 327 |
+
raise ValueError(f"{path}: expected an integer, got {value!r}")
|
| 328 |
+
out[name] = int(value)
|
| 329 |
+
elif hint is BandwidthParams:
|
| 330 |
+
out[name] = BandwidthParams(**_coerce_fields(BandwidthParams, value, path)) # type: ignore[arg-type]
|
| 331 |
+
else: # pragma: no cover - guards future field types
|
| 332 |
+
raise TypeError(f"{path}: unsupported field type {hint!r}")
|
| 333 |
+
return out
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def load_acoustic_config(path: Path) -> AcousticConfig:
|
| 337 |
+
"""Read an :class:`AcousticConfig` from a YAML file (``configs/data/acoustic.yaml``)."""
|
| 338 |
+
import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
|
| 339 |
+
|
| 340 |
+
data: Any = yaml.safe_load(path.read_text(encoding="utf-8")) # Any: untyped YAML tree
|
| 341 |
+
try:
|
| 342 |
+
return AcousticConfig.from_mapping(data)
|
| 343 |
+
except ValueError as exc:
|
| 344 |
+
raise ValueError(f"{path}: {exc}") from exc
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
# --------------------------------------------------------------------------------------
|
| 348 |
+
# Input validation
|
| 349 |
+
# --------------------------------------------------------------------------------------
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def _as_audio(audio: npt.ArrayLike, name: str = "audio") -> FloatArray:
|
| 353 |
+
"""Validate a mono float waveform and return it as float64."""
|
| 354 |
+
arr = np.asarray(audio)
|
| 355 |
+
if arr.dtype.kind != "f":
|
| 356 |
+
raise ValueError(f"{name} must have a floating dtype, got {arr.dtype}")
|
| 357 |
+
if arr.ndim != 1:
|
| 358 |
+
raise ValueError(f"{name} must be 1-D mono, got shape {arr.shape}")
|
| 359 |
+
if arr.size == 0:
|
| 360 |
+
raise ValueError(f"{name} must not be empty")
|
| 361 |
+
out = arr.astype(np.float64)
|
| 362 |
+
if not np.isfinite(out).all():
|
| 363 |
+
raise ValueError(f"{name} contains non-finite values (NaN or inf)")
|
| 364 |
+
return out
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def _as_rate(sample_rate_hz: int) -> int:
|
| 368 |
+
if isinstance(sample_rate_hz, bool) or not isinstance(sample_rate_hz, int | np.integer):
|
| 369 |
+
raise ValueError(f"sample_rate_hz must be an int, got {sample_rate_hz!r}")
|
| 370 |
+
rate = int(sample_rate_hz)
|
| 371 |
+
if rate <= 0:
|
| 372 |
+
raise ValueError(f"sample_rate_hz must be > 0, got {rate}")
|
| 373 |
+
return rate
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
def _db_power(power: float) -> float:
|
| 377 |
+
return 10.0 * math.log10(max(power, _POWER_FLOOR))
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
# --------------------------------------------------------------------------------------
|
| 381 |
+
# S1: bandwidth
|
| 382 |
+
# --------------------------------------------------------------------------------------
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def estimate_bandwidth_hz(
|
| 386 |
+
audio: npt.ArrayLike,
|
| 387 |
+
sample_rate_hz: int,
|
| 388 |
+
*,
|
| 389 |
+
floor_db: float,
|
| 390 |
+
params: BandwidthParams | None = None,
|
| 391 |
+
) -> float:
|
| 392 |
+
"""Effective bandwidth: the highest frequency still within ``floor_db`` of the in-band level.
|
| 393 |
+
|
| 394 |
+
Method: Hann STFT (hop = ``n_fft/2``); keep *active* frames (within
|
| 395 |
+
``params.active_within_db`` of the loudest); average their power into a long-term
|
| 396 |
+
average spectrum (LTAS) in dB; median-filter it over ``params.smooth_hz`` so isolated
|
| 397 |
+
narrow spikes cannot masquerade as bandwidth; take the reference level as the median of
|
| 398 |
+
the smoothed LTAS over ``[ref_low_hz, min(ref_high_hz, Nyquist)]``; return the centre
|
| 399 |
+
frequency of the highest bin whose smoothed level is ``>= reference - floor_db``.
|
| 400 |
+
|
| 401 |
+
The result is quantised to the FFT bin spacing ``sample_rate_hz / n_fft_samples``.
|
| 402 |
+
Audio upsampled from 16 kHz to 48 kHz reports about 8 kHz *provided* ``floor_db`` is
|
| 403 |
+
smaller than the resampler's stop-band attenuation (choose it per resampler/ source).
|
| 404 |
+
|
| 405 |
+
Args:
|
| 406 |
+
audio: Mono waveform, shape ``(num_samples,)``.
|
| 407 |
+
sample_rate_hz: Sample rate in Hz.
|
| 408 |
+
floor_db: How far below the reference level still counts as "content" (> 0).
|
| 409 |
+
params: Analysis parameters; defaults to :class:`BandwidthParams` defaults.
|
| 410 |
+
|
| 411 |
+
Returns:
|
| 412 |
+
Bandwidth in Hz, in ``(0, sample_rate_hz / 2]``.
|
| 413 |
+
|
| 414 |
+
Raises:
|
| 415 |
+
SilentAudioError: if the waveform is digitally silent.
|
| 416 |
+
ValueError: on invalid input or if the reference band lies above Nyquist.
|
| 417 |
+
"""
|
| 418 |
+
x = _as_audio(audio)
|
| 419 |
+
rate = _as_rate(sample_rate_hz)
|
| 420 |
+
_check_pos(floor_db, "floor_db")
|
| 421 |
+
p = BandwidthParams() if params is None else params
|
| 422 |
+
|
| 423 |
+
n_fft = p.n_fft_samples
|
| 424 |
+
nyquist_hz = rate / 2
|
| 425 |
+
bin_hz = rate / n_fft
|
| 426 |
+
if p.ref_low_hz >= nyquist_hz:
|
| 427 |
+
raise ValueError(f"ref_low_hz {p.ref_low_hz} is not below Nyquist {nyquist_hz} Hz")
|
| 428 |
+
|
| 429 |
+
power = stft_magnitude(x, n_fft, n_fft // 2) ** 2 # (frames, bins)
|
| 430 |
+
frame_energy = power.sum(axis=1)
|
| 431 |
+
loudest = float(np.max(frame_energy))
|
| 432 |
+
if loudest <= 0.0:
|
| 433 |
+
raise SilentAudioError("audio is digitally silent; bandwidth is undefined")
|
| 434 |
+
active = frame_energy >= loudest * 10.0 ** (-p.active_within_db / 10.0)
|
| 435 |
+
ltas_db = 10.0 * np.log10(power[active].mean(axis=0) + _POWER_FLOOR)
|
| 436 |
+
|
| 437 |
+
kernel = max(1, round(p.smooth_hz / bin_hz)) | 1 # medfilt needs an odd length
|
| 438 |
+
half = kernel // 2
|
| 439 |
+
padded = np.pad(ltas_db, half, mode="edge") # medfilt zero-pads; edge-pad instead
|
| 440 |
+
smooth = np.asarray(signal.medfilt(padded, kernel_size=kernel), dtype=np.float64)[
|
| 441 |
+
half : half + ltas_db.shape[0]
|
| 442 |
+
]
|
| 443 |
+
|
| 444 |
+
freqs_hz = np.arange(ltas_db.shape[0], dtype=np.float64) * bin_hz
|
| 445 |
+
ref_band = (freqs_hz >= p.ref_low_hz) & (freqs_hz <= min(p.ref_high_hz, nyquist_hz))
|
| 446 |
+
if not ref_band.any():
|
| 447 |
+
raise ValueError(
|
| 448 |
+
f"no FFT bin in the reference band [{p.ref_low_hz}, {p.ref_high_hz}] Hz "
|
| 449 |
+
f"(n_fft_samples={n_fft}, sample_rate_hz={rate})"
|
| 450 |
+
)
|
| 451 |
+
reference_db = float(np.median(smooth[ref_band]))
|
| 452 |
+
passing = np.flatnonzero(smooth >= reference_db - floor_db)
|
| 453 |
+
return float(freqs_hz[int(passing[-1])])
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
# --------------------------------------------------------------------------------------
|
| 457 |
+
# S2: clipping, silence, loudness
|
| 458 |
+
# --------------------------------------------------------------------------------------
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
def clipping_ratio(audio: npt.ArrayLike, *, threshold: float) -> float:
|
| 462 |
+
"""Fraction of samples with ``|x| >= threshold``.
|
| 463 |
+
|
| 464 |
+
Args:
|
| 465 |
+
audio: Mono waveform.
|
| 466 |
+
threshold: Magnitude counted as clipped, in ``(0, 1]``.
|
| 467 |
+
|
| 468 |
+
Returns:
|
| 469 |
+
Ratio in ``[0, 1]``.
|
| 470 |
+
"""
|
| 471 |
+
x = _as_audio(audio)
|
| 472 |
+
if not (math.isfinite(threshold) and 0.0 < threshold <= 1.0):
|
| 473 |
+
raise ValueError(f"threshold must be in (0, 1], got {threshold!r}")
|
| 474 |
+
return float(np.mean(np.abs(x) >= threshold))
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def silence_ratio(
|
| 478 |
+
audio: npt.ArrayLike, sample_rate_hz: int, *, frame_ms: float, silence_dbfs: float
|
| 479 |
+
) -> float:
|
| 480 |
+
"""Fraction of non-overlapping frames whose RMS is below ``silence_dbfs``.
|
| 481 |
+
|
| 482 |
+
The last, possibly shorter frame is included with its own RMS; audio shorter than one
|
| 483 |
+
frame forms a single frame.
|
| 484 |
+
|
| 485 |
+
Args:
|
| 486 |
+
audio: Mono waveform.
|
| 487 |
+
sample_rate_hz: Sample rate in Hz.
|
| 488 |
+
frame_ms: Frame length in ms (> 0).
|
| 489 |
+
silence_dbfs: RMS level (dBFS) below which a frame is silent.
|
| 490 |
+
|
| 491 |
+
Returns:
|
| 492 |
+
Ratio in ``[0, 1]``.
|
| 493 |
+
"""
|
| 494 |
+
x = _as_audio(audio)
|
| 495 |
+
rate = _as_rate(sample_rate_hz)
|
| 496 |
+
_check_pos(frame_ms, "frame_ms")
|
| 497 |
+
_check_finite(silence_dbfs, "silence_dbfs")
|
| 498 |
+
frame_len = max(1, round(rate * frame_ms / 1000.0))
|
| 499 |
+
starts = np.arange(0, x.shape[0], frame_len)
|
| 500 |
+
sums = np.add.reduceat(x * x, starts)
|
| 501 |
+
counts = np.diff(np.append(starts, x.shape[0]))
|
| 502 |
+
frame_db = 10.0 * np.log10(sums / counts + _POWER_FLOOR)
|
| 503 |
+
return float(np.mean(frame_db < silence_dbfs))
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def loudness_dbfs(audio: npt.ArrayLike) -> float:
|
| 507 |
+
"""Whole-signal RMS level in dBFS (``20 log10(rms)``, full-scale amplitude 1.0).
|
| 508 |
+
|
| 509 |
+
This is an unweighted RMS level, *not* LUFS (no K-weighting, no gating). Digital
|
| 510 |
+
silence returns -240.0 instead of ``-inf``.
|
| 511 |
+
"""
|
| 512 |
+
x = _as_audio(audio)
|
| 513 |
+
return _db_power(float(np.mean(x * x)))
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
# --------------------------------------------------------------------------------------
|
| 517 |
+
# S6: pitch tracking (YIN)
|
| 518 |
+
# --------------------------------------------------------------------------------------
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
@dataclass(frozen=True)
|
| 522 |
+
class PitchTrack:
|
| 523 |
+
"""Frame-wise pitch track on a regular grid.
|
| 524 |
+
|
| 525 |
+
Frame ``t`` is centred on sample ``center_samples[t] = t * hop_samples`` (the signal is
|
| 526 |
+
zero-padded by half a window at both ends so that the file edges are covered).
|
| 527 |
+
|
| 528 |
+
Attributes:
|
| 529 |
+
center_samples: Centre sample of each frame, shape ``(num_frames,)``.
|
| 530 |
+
voiced: Whether F0 was detected, shape ``(num_frames,)``.
|
| 531 |
+
f0_hz: F0 in Hz, ``nan`` where unvoiced.
|
| 532 |
+
hnr_db: Harmonics-to-noise ratio in dB (clipped to +-``hnr_max_db``), ``nan``
|
| 533 |
+
where unvoiced.
|
| 534 |
+
"""
|
| 535 |
+
|
| 536 |
+
center_samples: IntArray
|
| 537 |
+
voiced: BoolArray
|
| 538 |
+
f0_hz: FloatArray
|
| 539 |
+
hnr_db: FloatArray
|
| 540 |
+
|
| 541 |
+
|
| 542 |
+
def _yin_chunk(
|
| 543 |
+
frames: FloatArray,
|
| 544 |
+
*,
|
| 545 |
+
tau_min: int,
|
| 546 |
+
tau_max: int,
|
| 547 |
+
threshold: float,
|
| 548 |
+
energy_floor_dbfs: float,
|
| 549 |
+
hnr_max_db: float,
|
| 550 |
+
) -> tuple[BoolArray, FloatArray, FloatArray]:
|
| 551 |
+
"""YIN on a batch of frames ``(F, W)``; returns ``(voiced, f0_samples, hnr_db)``.
|
| 552 |
+
|
| 553 |
+
``f0_samples`` is the (parabolically refined) period in samples, ``nan`` if unvoiced.
|
| 554 |
+
"""
|
| 555 |
+
num_frames, width = frames.shape
|
| 556 |
+
n_lags = tau_max + 2 # lags 0..tau_max+1 (tau_max+1 is needed for the interpolation)
|
| 557 |
+
integ = width - n_lags + 1 # integration length so that j + tau stays inside the frame
|
| 558 |
+
nfft = 1 << (integ + width - 1).bit_length()
|
| 559 |
+
|
| 560 |
+
head = frames[:, :integ]
|
| 561 |
+
corr = np.fft.irfft(
|
| 562 |
+
np.conj(np.fft.rfft(head, nfft, axis=1)) * np.fft.rfft(frames, nfft, axis=1),
|
| 563 |
+
nfft,
|
| 564 |
+
axis=1,
|
| 565 |
+
)[:, :n_lags]
|
| 566 |
+
csum = np.concatenate(
|
| 567 |
+
[np.zeros((num_frames, 1)), np.cumsum(frames * frames, axis=1)], axis=1
|
| 568 |
+
) # (F, W + 1)
|
| 569 |
+
energy0 = csum[:, integ : integ + 1] # (F, 1)
|
| 570 |
+
taus = np.arange(n_lags)
|
| 571 |
+
energy1 = csum[:, integ + taus] - csum[:, taus] # (F, n_lags)
|
| 572 |
+
diff = np.maximum(energy0 + energy1 - 2.0 * corr, 0.0)
|
| 573 |
+
diff[:, 0] = 0.0
|
| 574 |
+
|
| 575 |
+
# Cumulative-mean-normalised difference function (YIN step 3).
|
| 576 |
+
cum = np.cumsum(diff[:, 1:], axis=1)
|
| 577 |
+
cmnd = np.ones((num_frames, n_lags))
|
| 578 |
+
np.divide(diff[:, 1:] * taus[1:], cum, out=cmnd[:, 1:], where=cum > _REL_EPS * energy0)
|
| 579 |
+
|
| 580 |
+
# Absolute threshold (step 4): first lag below it, then walk down to the local minimum.
|
| 581 |
+
below = cmnd[:, tau_min : tau_max + 1] < threshold
|
| 582 |
+
has_dip = below.any(axis=1)
|
| 583 |
+
idx = np.argmax(below, axis=1) + tau_min
|
| 584 |
+
rows = np.arange(num_frames)
|
| 585 |
+
while True:
|
| 586 |
+
advance = has_dip & (idx < tau_max) & (cmnd[rows, idx + 1] < cmnd[rows, idx])
|
| 587 |
+
if not advance.any():
|
| 588 |
+
break
|
| 589 |
+
idx = idx + advance
|
| 590 |
+
|
| 591 |
+
# Parabolic interpolation (step 5).
|
| 592 |
+
c_prev, c_mid, c_next = cmnd[rows, idx - 1], cmnd[rows, idx], cmnd[rows, idx + 1]
|
| 593 |
+
curvature = c_prev - 2.0 * c_mid + c_next
|
| 594 |
+
shift = np.zeros(num_frames)
|
| 595 |
+
np.divide(0.5 * (c_prev - c_next), curvature, out=shift, where=np.abs(curvature) > 1e-12)
|
| 596 |
+
period = idx + np.clip(shift, -1.0, 1.0)
|
| 597 |
+
|
| 598 |
+
# HNR from the normalised autocorrelation at the period (best of idx-1, idx, idx+1).
|
| 599 |
+
best_r = np.zeros(num_frames)
|
| 600 |
+
for offset in (-1, 0, 1):
|
| 601 |
+
lag = idx + offset
|
| 602 |
+
denom = np.sqrt(energy0[:, 0] * energy1[rows, lag])
|
| 603 |
+
r_norm = np.zeros(num_frames)
|
| 604 |
+
np.divide(corr[rows, lag], denom, out=r_norm, where=denom > 0.0)
|
| 605 |
+
best_r = np.maximum(best_r, r_norm)
|
| 606 |
+
best_r = np.clip(best_r, _REL_EPS, 1.0 - _REL_EPS)
|
| 607 |
+
hnr = np.clip(10.0 * np.log10(best_r / (1.0 - best_r)), -hnr_max_db, hnr_max_db)
|
| 608 |
+
|
| 609 |
+
frame_db = 10.0 * np.log10(np.mean(frames * frames, axis=1) + _POWER_FLOOR)
|
| 610 |
+
voiced = has_dip & (frame_db >= energy_floor_dbfs)
|
| 611 |
+
return (
|
| 612 |
+
voiced,
|
| 613 |
+
np.where(voiced, period, np.nan),
|
| 614 |
+
np.where(voiced, hnr, np.nan),
|
| 615 |
+
)
|
| 616 |
+
|
| 617 |
+
|
| 618 |
+
def track_pitch(audio: npt.ArrayLike, sample_rate_hz: int, cfg: AcousticConfig) -> PitchTrack:
|
| 619 |
+
"""Track F0 and HNR with a numpy YIN implementation.
|
| 620 |
+
|
| 621 |
+
Per frame (``cfg.f0_frame_ms`` window, ``cfg.f0_hop_ms`` hop): difference function via
|
| 622 |
+
FFT cross-correlation, cumulative-mean normalisation, first dip below
|
| 623 |
+
``cfg.yin_threshold`` within the lag range of ``[f0_min_hz, f0_max_hz]``, descent to
|
| 624 |
+
the local minimum, parabolic refinement. A frame is voiced when a dip exists *and* its
|
| 625 |
+
RMS is at least ``cfg.f0_energy_floor_dbfs``. HNR is
|
| 626 |
+
``10 log10(r / (1 - r))`` with ``r`` the normalised autocorrelation at the period,
|
| 627 |
+
clipped to ``+-cfg.hnr_max_db``.
|
| 628 |
+
|
| 629 |
+
Args:
|
| 630 |
+
audio: Mono waveform.
|
| 631 |
+
sample_rate_hz: Sample rate in Hz; must satisfy ``sample_rate_hz >= 4 * f0_max_hz``.
|
| 632 |
+
cfg: Configuration.
|
| 633 |
+
|
| 634 |
+
Raises:
|
| 635 |
+
ValueError: on invalid input, or if the window is shorter than two periods of
|
| 636 |
+
``f0_min_hz`` at this sample rate, or the rate is too low for ``f0_max_hz``.
|
| 637 |
+
"""
|
| 638 |
+
x = _as_audio(audio)
|
| 639 |
+
rate = _as_rate(sample_rate_hz)
|
| 640 |
+
window = round(rate * cfg.f0_frame_ms / 1000.0)
|
| 641 |
+
hop = round(rate * cfg.f0_hop_ms / 1000.0)
|
| 642 |
+
tau_max = math.ceil(rate / cfg.f0_min_hz)
|
| 643 |
+
tau_min = math.ceil(rate / cfg.f0_max_hz)
|
| 644 |
+
if rate < 4 * cfg.f0_max_hz:
|
| 645 |
+
raise ValueError(
|
| 646 |
+
f"sample_rate_hz={rate} is too low for f0_max_hz={cfg.f0_max_hz} "
|
| 647 |
+
"(need sample_rate_hz >= 4 * f0_max_hz)"
|
| 648 |
+
)
|
| 649 |
+
if hop < 1 or window < 2 * tau_max + 2:
|
| 650 |
+
raise ValueError(
|
| 651 |
+
f"f0_frame_ms={cfg.f0_frame_ms} is too short for f0_min_hz={cfg.f0_min_hz} at "
|
| 652 |
+
f"sample_rate_hz={rate}: need a window of >= {2 * tau_max + 2} samples, "
|
| 653 |
+
f"got {window}"
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
num_samples = x.shape[0]
|
| 657 |
+
num_frames = (num_samples - 1) // hop + 1
|
| 658 |
+
pad = window // 2
|
| 659 |
+
padded = np.zeros(num_samples + 2 * pad + window)
|
| 660 |
+
padded[pad : pad + num_samples] = x
|
| 661 |
+
windows = np.lib.stride_tricks.sliding_window_view(padded, window)
|
| 662 |
+
|
| 663 |
+
voiced = np.zeros(num_frames, dtype=np.bool_)
|
| 664 |
+
f0 = np.full(num_frames, np.nan)
|
| 665 |
+
hnr = np.full(num_frames, np.nan)
|
| 666 |
+
for first in range(0, num_frames, _CHUNK_FRAMES):
|
| 667 |
+
stop = min(first + _CHUNK_FRAMES, num_frames)
|
| 668 |
+
# Frame t spans padded[t*hop : t*hop + window], i.e. is centred on sample t*hop.
|
| 669 |
+
chunk = np.ascontiguousarray(windows[first * hop : stop * hop : hop])
|
| 670 |
+
v, period, h = _yin_chunk(
|
| 671 |
+
chunk,
|
| 672 |
+
tau_min=tau_min,
|
| 673 |
+
tau_max=tau_max,
|
| 674 |
+
threshold=cfg.yin_threshold,
|
| 675 |
+
energy_floor_dbfs=cfg.f0_energy_floor_dbfs,
|
| 676 |
+
hnr_max_db=cfg.hnr_max_db,
|
| 677 |
+
)
|
| 678 |
+
voiced[first:stop] = v
|
| 679 |
+
f0[first:stop] = rate / period
|
| 680 |
+
hnr[first:stop] = h
|
| 681 |
+
centers = np.arange(num_frames, dtype=np.int64) * hop
|
| 682 |
+
return PitchTrack(center_samples=centers, voiced=voiced, f0_hz=f0, hnr_db=hnr)
|
| 683 |
+
|
| 684 |
+
|
| 685 |
+
# --------------------------------------------------------------------------------------
|
| 686 |
+
# S6: per-word features
|
| 687 |
+
# --------------------------------------------------------------------------------------
|
| 688 |
+
|
| 689 |
+
|
| 690 |
+
@dataclass(frozen=True)
|
| 691 |
+
class WordFeatures:
|
| 692 |
+
"""Acoustic features of one word. ``None`` means *not measurable*, never zero.
|
| 693 |
+
|
| 694 |
+
* Word shorter than ``cfg.min_word_ms``: every field except ``duration_ms`` is ``None``.
|
| 695 |
+
* Word measured but without a single voiced frame: ``voicing_ratio`` is ``0.0`` and
|
| 696 |
+
``median_f0_hz`` / ``hnr_db`` are ``None``.
|
| 697 |
+
|
| 698 |
+
Attributes:
|
| 699 |
+
duration_ms: Analysed duration (after clipping to the audio end).
|
| 700 |
+
voicing_ratio: Fraction of the pitch frames centred inside the word that are voiced.
|
| 701 |
+
median_f0_hz: Median F0 over voiced frames.
|
| 702 |
+
hnr_db: Median per-frame HNR over voiced frames.
|
| 703 |
+
rms_dbfs: RMS level of the word's samples.
|
| 704 |
+
spectral_tilt_db_per_octave: Slope of log power versus log2 frequency (negative =
|
| 705 |
+
energy falls with frequency; shouted speech is flatter, i.e. less negative).
|
| 706 |
+
"""
|
| 707 |
+
|
| 708 |
+
duration_ms: int
|
| 709 |
+
voicing_ratio: float | None
|
| 710 |
+
median_f0_hz: float | None
|
| 711 |
+
hnr_db: float | None
|
| 712 |
+
rms_dbfs: float | None
|
| 713 |
+
spectral_tilt_db_per_octave: float | None
|
| 714 |
+
|
| 715 |
+
@property
|
| 716 |
+
def measurable(self) -> bool:
|
| 717 |
+
"""False when the word was too short to measure."""
|
| 718 |
+
return self.voicing_ratio is not None
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
def _spectral_tilt(word: FloatArray, rate: int, cfg: AcousticConfig) -> float | None:
|
| 722 |
+
"""Slope (dB per octave) of band power versus log2 frequency, or ``None`` if undefined.
|
| 723 |
+
|
| 724 |
+
The Hann-windowed power spectrum (averaged over ``tilt_n_fft_samples``-long frames for
|
| 725 |
+
long words; a word shorter than the FFT size gets one window of its own length,
|
| 726 |
+
zero-padded) is averaged into ``cfg.tilt_num_bands`` bands equally spaced in log2
|
| 727 |
+
frequency; a straight line is fitted to band dB versus band-centre log2 frequency.
|
| 728 |
+
Binning in log-frequency stops the dense high-frequency FFT bins from dominating.
|
| 729 |
+
"""
|
| 730 |
+
n_fft = cfg.tilt_n_fft_samples
|
| 731 |
+
if word.shape[0] <= n_fft:
|
| 732 |
+
frames = (word * np.hanning(word.shape[0]))[np.newaxis, :]
|
| 733 |
+
else:
|
| 734 |
+
win = np.hanning(n_fft)
|
| 735 |
+
frames = np.lib.stride_tricks.sliding_window_view(word, n_fft)[:: n_fft // 2] * win
|
| 736 |
+
power = np.mean(np.abs(np.fft.rfft(frames, n_fft, axis=1)) ** 2, axis=0)
|
| 737 |
+
if float(power.sum()) <= 0.0:
|
| 738 |
+
return None
|
| 739 |
+
|
| 740 |
+
freqs_hz = np.arange(power.shape[0]) * rate / n_fft
|
| 741 |
+
high_hz = min(cfg.tilt_high_hz, rate / 2)
|
| 742 |
+
if cfg.tilt_low_hz >= high_hz:
|
| 743 |
+
return None
|
| 744 |
+
edges = np.linspace(math.log2(cfg.tilt_low_hz), math.log2(high_hz), cfg.tilt_num_bands + 1)
|
| 745 |
+
centers: list[float] = []
|
| 746 |
+
levels_db: list[float] = []
|
| 747 |
+
for lo, hi in pairwise(edges):
|
| 748 |
+
mask = (freqs_hz >= 2.0**lo) & (freqs_hz < 2.0**hi)
|
| 749 |
+
if mask.any():
|
| 750 |
+
centers.append(float(lo + hi) / 2.0)
|
| 751 |
+
levels_db.append(_db_power(float(power[mask].mean())))
|
| 752 |
+
if len(centers) < 3:
|
| 753 |
+
return None
|
| 754 |
+
slope = np.polyfit(np.asarray(centers), np.asarray(levels_db), 1)[0]
|
| 755 |
+
return float(slope)
|
| 756 |
+
|
| 757 |
+
|
| 758 |
+
def word_features(
|
| 759 |
+
audio: npt.ArrayLike,
|
| 760 |
+
sample_rate_hz: int,
|
| 761 |
+
words: Sequence[TimedWord],
|
| 762 |
+
cfg: AcousticConfig,
|
| 763 |
+
) -> list[WordFeatures]:
|
| 764 |
+
"""Per-word voicing, F0, HNR, RMS and spectral tilt (S6).
|
| 765 |
+
|
| 766 |
+
The pitch track is computed once over the whole waveform; a word owns the frames whose
|
| 767 |
+
centre lies in ``[start, end)`` (so frames at a boundary see a little context from the
|
| 768 |
+
neighbouring word). RMS and tilt use the word's own samples only.
|
| 769 |
+
|
| 770 |
+
Args:
|
| 771 |
+
audio: Mono waveform of the segment.
|
| 772 |
+
sample_rate_hz: Sample rate in Hz.
|
| 773 |
+
words: Words with segment-relative ``start_ms`` / ``end_ms``; need not be sorted.
|
| 774 |
+
cfg: Configuration.
|
| 775 |
+
|
| 776 |
+
Returns:
|
| 777 |
+
One :class:`WordFeatures` per word, in input order; ``[]`` for no words.
|
| 778 |
+
|
| 779 |
+
Raises:
|
| 780 |
+
ValueError: on invalid audio; if a word starts at or after the end of the audio;
|
| 781 |
+
or if a word ends more than ``cfg.word_end_tolerance_ms`` after it (smaller
|
| 782 |
+
overshoots are clipped to the audio end).
|
| 783 |
+
"""
|
| 784 |
+
x = _as_audio(audio)
|
| 785 |
+
rate = _as_rate(sample_rate_hz)
|
| 786 |
+
if not words:
|
| 787 |
+
return []
|
| 788 |
+
|
| 789 |
+
num_samples = x.shape[0]
|
| 790 |
+
audio_ms = num_samples * 1000.0 / rate
|
| 791 |
+
track = track_pitch(x, rate, cfg)
|
| 792 |
+
|
| 793 |
+
out: list[WordFeatures] = []
|
| 794 |
+
for word in words:
|
| 795 |
+
if word.start_ms >= audio_ms:
|
| 796 |
+
raise ValueError(
|
| 797 |
+
f"word {word.text!r} starts at {word.start_ms} ms, at or after the end of the "
|
| 798 |
+
f"audio ({audio_ms:.1f} ms)"
|
| 799 |
+
)
|
| 800 |
+
if word.end_ms > audio_ms + cfg.word_end_tolerance_ms:
|
| 801 |
+
raise ValueError(
|
| 802 |
+
f"word {word.text!r} ends at {word.end_ms} ms, more than "
|
| 803 |
+
f"{cfg.word_end_tolerance_ms} ms after the end of the audio ({audio_ms:.1f} ms)"
|
| 804 |
+
)
|
| 805 |
+
end_ms = min(word.end_ms, math.floor(audio_ms))
|
| 806 |
+
duration_ms = max(0, end_ms - word.start_ms)
|
| 807 |
+
if duration_ms < cfg.min_word_ms:
|
| 808 |
+
out.append(WordFeatures(duration_ms, None, None, None, None, None))
|
| 809 |
+
continue
|
| 810 |
+
|
| 811 |
+
start = round(word.start_ms * rate / 1000.0)
|
| 812 |
+
stop = min(round(end_ms * rate / 1000.0), num_samples)
|
| 813 |
+
samples = x[start:stop]
|
| 814 |
+
in_word = (track.center_samples >= start) & (track.center_samples < stop)
|
| 815 |
+
num_frames = int(in_word.sum())
|
| 816 |
+
if samples.shape[0] == 0 or num_frames == 0:
|
| 817 |
+
out.append(WordFeatures(duration_ms, None, None, None, None, None))
|
| 818 |
+
continue
|
| 819 |
+
|
| 820 |
+
voiced = in_word & track.voiced
|
| 821 |
+
num_voiced = int(voiced.sum())
|
| 822 |
+
out.append(
|
| 823 |
+
WordFeatures(
|
| 824 |
+
duration_ms=duration_ms,
|
| 825 |
+
voicing_ratio=num_voiced / num_frames,
|
| 826 |
+
median_f0_hz=float(np.median(track.f0_hz[voiced])) if num_voiced else None,
|
| 827 |
+
hnr_db=float(np.median(track.hnr_db[voiced])) if num_voiced else None,
|
| 828 |
+
rms_dbfs=_db_power(float(np.mean(samples * samples))),
|
| 829 |
+
spectral_tilt_db_per_octave=_spectral_tilt(samples, rate, cfg),
|
| 830 |
+
)
|
| 831 |
+
)
|
| 832 |
+
return out
|
| 833 |
+
|
| 834 |
+
|
| 835 |
+
# --------------------------------------------------------------------------------------
|
| 836 |
+
# S6: speaker-relative style scores
|
| 837 |
+
# --------------------------------------------------------------------------------------
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
def _logistic(value: float, center: float, scale: float) -> float:
|
| 841 |
+
"""``sigmoid((value - center) / scale)``, numerically safe."""
|
| 842 |
+
z = max(-_LOGISTIC_CLIP, min(_LOGISTIC_CLIP, (value - center) / scale))
|
| 843 |
+
return 1.0 / (1.0 + math.exp(-z))
|
| 844 |
+
|
| 845 |
+
|
| 846 |
+
def whisper_scores(features: Sequence[WordFeatures], cfg: AcousticConfig) -> WordStyleScore:
|
| 847 |
+
"""Per-word whisper probability: unvoiced, noise-like, but with energy present.
|
| 848 |
+
|
| 849 |
+
``score = f_unvoiced * f_noise * f_energy`` (a soft AND), each factor a logistic
|
| 850 |
+
``sigmoid((x - center) / scale)`` with centre/scale from ``cfg``:
|
| 851 |
+
|
| 852 |
+
* ``f_unvoiced`` on ``x = 1 - voicing_ratio`` (``whisper_unvoiced_*``);
|
| 853 |
+
* ``f_noise`` on ``x = -hnr_db`` with centre ``-whisper_hnr_center_db`` (low HNR is
|
| 854 |
+
whisper-like); words with no voiced frame have no HNR and get factor 1.0;
|
| 855 |
+
* ``f_energy`` on ``x = rms_dbfs`` (``whisper_energy_*``), so silence is not a whisper.
|
| 856 |
+
|
| 857 |
+
Words that are not measurable (too short) score ``None``.
|
| 858 |
+
"""
|
| 859 |
+
scores: list[float | None] = []
|
| 860 |
+
for feat in features:
|
| 861 |
+
if feat.voicing_ratio is None or feat.rms_dbfs is None:
|
| 862 |
+
scores.append(None)
|
| 863 |
+
continue
|
| 864 |
+
f_unvoiced = _logistic(
|
| 865 |
+
1.0 - feat.voicing_ratio, cfg.whisper_unvoiced_center, cfg.whisper_unvoiced_scale
|
| 866 |
+
)
|
| 867 |
+
f_noise = (
|
| 868 |
+
1.0
|
| 869 |
+
if feat.hnr_db is None
|
| 870 |
+
else _logistic(-feat.hnr_db, -cfg.whisper_hnr_center_db, cfg.whisper_hnr_scale_db)
|
| 871 |
+
)
|
| 872 |
+
f_energy = _logistic(
|
| 873 |
+
feat.rms_dbfs, cfg.whisper_energy_center_dbfs, cfg.whisper_energy_scale_db
|
| 874 |
+
)
|
| 875 |
+
scores.append(f_unvoiced * f_noise * f_energy)
|
| 876 |
+
return WordStyleScore(style="whisper", scores=tuple(scores), source=WHISPER_SOURCE)
|
| 877 |
+
|
| 878 |
+
|
| 879 |
+
@dataclass(frozen=True)
|
| 880 |
+
class SpeakerBaseline:
|
| 881 |
+
"""A speaker's own reference levels (medians over their voiced words).
|
| 882 |
+
|
| 883 |
+
Attributes:
|
| 884 |
+
median_rms_dbfs: Median word RMS level.
|
| 885 |
+
median_f0_hz: Median of the words' median F0.
|
| 886 |
+
median_tilt_db_per_octave: Median spectral tilt.
|
| 887 |
+
num_voiced_words: How many words the medians are based on.
|
| 888 |
+
"""
|
| 889 |
+
|
| 890 |
+
median_rms_dbfs: float
|
| 891 |
+
median_f0_hz: float
|
| 892 |
+
median_tilt_db_per_octave: float
|
| 893 |
+
num_voiced_words: int
|
| 894 |
+
|
| 895 |
+
|
| 896 |
+
def speaker_baseline(features: Sequence[WordFeatures], cfg: AcousticConfig) -> SpeakerBaseline:
|
| 897 |
+
"""Medians over the speaker's *voiced* words (``voicing_ratio >= baseline_min_voicing_ratio``).
|
| 898 |
+
|
| 899 |
+
Pass the features of **all** of one speaker's words (many segments), not of a single
|
| 900 |
+
segment: the median is only a baseline if shouted words are a minority.
|
| 901 |
+
|
| 902 |
+
Raises:
|
| 903 |
+
ValueError: if fewer than ``cfg.baseline_min_voiced_words`` voiced words exist.
|
| 904 |
+
"""
|
| 905 |
+
voiced = [
|
| 906 |
+
f
|
| 907 |
+
for f in features
|
| 908 |
+
if f.voicing_ratio is not None
|
| 909 |
+
and f.voicing_ratio >= cfg.baseline_min_voicing_ratio
|
| 910 |
+
and f.median_f0_hz is not None
|
| 911 |
+
and f.rms_dbfs is not None
|
| 912 |
+
and f.spectral_tilt_db_per_octave is not None
|
| 913 |
+
]
|
| 914 |
+
if len(voiced) < cfg.baseline_min_voiced_words:
|
| 915 |
+
raise BaselineError(
|
| 916 |
+
f"speaker baseline needs >= {cfg.baseline_min_voiced_words} voiced words, "
|
| 917 |
+
f"got {len(voiced)} of {len(features)}"
|
| 918 |
+
)
|
| 919 |
+
|
| 920 |
+
def median_of(values: list[float]) -> float:
|
| 921 |
+
return float(np.median(np.asarray(values)))
|
| 922 |
+
|
| 923 |
+
return SpeakerBaseline(
|
| 924 |
+
median_rms_dbfs=median_of([f.rms_dbfs for f in voiced if f.rms_dbfs is not None]),
|
| 925 |
+
median_f0_hz=median_of([f.median_f0_hz for f in voiced if f.median_f0_hz is not None]),
|
| 926 |
+
median_tilt_db_per_octave=median_of(
|
| 927 |
+
[
|
| 928 |
+
f.spectral_tilt_db_per_octave
|
| 929 |
+
for f in voiced
|
| 930 |
+
if f.spectral_tilt_db_per_octave is not None
|
| 931 |
+
]
|
| 932 |
+
),
|
| 933 |
+
num_voiced_words=len(voiced),
|
| 934 |
+
)
|
| 935 |
+
|
| 936 |
+
|
| 937 |
+
def shout_scores(
|
| 938 |
+
features: Sequence[WordFeatures],
|
| 939 |
+
speaker_baseline: SpeakerBaseline,
|
| 940 |
+
cfg: AcousticConfig,
|
| 941 |
+
) -> WordStyleScore:
|
| 942 |
+
"""Per-word shout probability relative to the speaker's own baseline (no absolute levels).
|
| 943 |
+
|
| 944 |
+
With deviations from the baseline
|
| 945 |
+
|
| 946 |
+
* ``delta_db = rms_dbfs - baseline.median_rms_dbfs``,
|
| 947 |
+
* ``f0_st = 12 * log2(median_f0_hz / baseline.median_f0_hz)`` (F0 ratio in semitones),
|
| 948 |
+
* ``delta_tilt = tilt - baseline.median_tilt`` (dB/octave; positive = flatter),
|
| 949 |
+
|
| 950 |
+
the score is ``sigmoid((z - shout_center) / shout_scale)`` with
|
| 951 |
+
``z = w_db * delta_db + w_f0 * f0_st + w_tilt * delta_tilt``.
|
| 952 |
+
|
| 953 |
+
A word that is too short, or has no F0 (unvoiced), cannot be judged and scores ``None``
|
| 954 |
+
("not marked"), not 0.
|
| 955 |
+
"""
|
| 956 |
+
scores: list[float | None] = []
|
| 957 |
+
for feat in features:
|
| 958 |
+
if (
|
| 959 |
+
feat.median_f0_hz is None
|
| 960 |
+
or feat.rms_dbfs is None
|
| 961 |
+
or feat.spectral_tilt_db_per_octave is None
|
| 962 |
+
):
|
| 963 |
+
scores.append(None)
|
| 964 |
+
continue
|
| 965 |
+
delta_db = feat.rms_dbfs - speaker_baseline.median_rms_dbfs
|
| 966 |
+
f0_st = 12.0 * math.log2(feat.median_f0_hz / speaker_baseline.median_f0_hz)
|
| 967 |
+
delta_tilt = feat.spectral_tilt_db_per_octave - speaker_baseline.median_tilt_db_per_octave
|
| 968 |
+
z = (
|
| 969 |
+
cfg.shout_weight_db * delta_db
|
| 970 |
+
+ cfg.shout_weight_f0_semitones * f0_st
|
| 971 |
+
+ cfg.shout_weight_tilt * delta_tilt
|
| 972 |
+
)
|
| 973 |
+
scores.append(_logistic(z, cfg.shout_center, cfg.shout_scale))
|
| 974 |
+
return WordStyleScore(style="shout", scores=tuple(scores), source=SHOUT_SOURCE)
|
| 975 |
+
|
| 976 |
+
|
| 977 |
+
def phrase_support(style: WordStyleScore) -> WordStyleScore:
|
| 978 |
+
"""Cap each word's style score by its strongest measured neighbour.
|
| 979 |
+
|
| 980 |
+
Whispering and shouting are properties of a phrase, not of one word. A lone short word
|
| 981 |
+
("de", "ve", "bir") scores high easily: its aligned window is mostly a voiceless
|
| 982 |
+
consonant, or it carries ordinary lexical stress. The pilot smoke run (2026-10-08) lost
|
| 983 |
+
42 of 258 segments that way (``short_style_run`` / ``ambiguous_style`` on single words).
|
| 984 |
+
So ``s'[i] = min(s[i], max(s[i-1], s[i+1]))`` over the neighbours that have a score.
|
| 985 |
+
A word with no measured neighbour gets ``None`` (not judged). A real style spanning two
|
| 986 |
+
or more words keeps its scores; two adjacent middle-band words stay ambiguous.
|
| 987 |
+
|
| 988 |
+
Returns:
|
| 989 |
+
Same style, scores capped, source suffixed with ``+phrase``.
|
| 990 |
+
"""
|
| 991 |
+
raw = style.scores
|
| 992 |
+
out: list[float | None] = []
|
| 993 |
+
for index, score in enumerate(raw):
|
| 994 |
+
neighbours = [
|
| 995 |
+
raw[j] for j in (index - 1, index + 1) if 0 <= j < len(raw) and raw[j] is not None
|
| 996 |
+
]
|
| 997 |
+
if score is None or not neighbours:
|
| 998 |
+
out.append(None)
|
| 999 |
+
continue
|
| 1000 |
+
support = max(n for n in neighbours if n is not None)
|
| 1001 |
+
out.append(min(score, support))
|
| 1002 |
+
return WordStyleScore(style=style.style, scores=tuple(out), source=style.source + "+phrase")
|
src/nida/data/align.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""S5 — CTC forced alignment of a transcript to character CTC emissions (Viterbi).
|
| 2 |
+
|
| 3 |
+
Given log-softmax emissions ``[num_frames, vocab]`` from a character CTC model (e.g.
|
| 4 |
+
``mpoyraz_ctc_tr``) and the primary transcript, find the single most likely CTC path that
|
| 5 |
+
emits exactly the transcript's characters (words separated by the model's word-delimiter
|
| 6 |
+
token) and read word intervals off that path. Plain numpy, no torchaudio dependency
|
| 7 |
+
(``torchaudio.functional.forced_align`` is deprecated in the 2.8 line, R8).
|
| 8 |
+
|
| 9 |
+
Text handling: each transcript token is NFC + ``tr_lower``-ed, circumflexes are folded
|
| 10 |
+
(â/î/û → a/i/u: the model has no such letters) and characters outside the vocabulary that
|
| 11 |
+
are punctuation are dropped for alignment only. A token with no alignable character
|
| 12 |
+
(``…``, ``-``) is glued to the previous word (or the next one at the start), so the
|
| 13 |
+
returned words keep their original spelling and punctuation. Any other character
|
| 14 |
+
outside the vocabulary (digits, foreign letters) makes the segment unalignable.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import itertools
|
| 20 |
+
import math
|
| 21 |
+
import unicodedata
|
| 22 |
+
from collections.abc import Mapping, Sequence
|
| 23 |
+
from dataclasses import dataclass, field
|
| 24 |
+
from typing import TYPE_CHECKING
|
| 25 |
+
|
| 26 |
+
from nida.data.types import TimedWord
|
| 27 |
+
from nida.text.turkish import nfc, tr_lower
|
| 28 |
+
|
| 29 |
+
if TYPE_CHECKING:
|
| 30 |
+
import numpy as np
|
| 31 |
+
from numpy.typing import NDArray
|
| 32 |
+
|
| 33 |
+
__all__ = ["AlignmentResult", "align_words", "alignment_units"]
|
| 34 |
+
|
| 35 |
+
_CIRCUMFLEX_FOLD = str.maketrans({"â": "a", "î": "i", "û": "u"})
|
| 36 |
+
# Typographic apostrophes (U+2019, U+02BC) are the same letter-joining apostrophe as U+0027.
|
| 37 |
+
_APOSTROPHES = str.maketrans({"\u2019": "'", "\u02bc": "'"})
|
| 38 |
+
#: Allowed difference between ``num_frames * hop`` and the segment duration, in frames
|
| 39 |
+
#: (wav2vec2 yields ``floor((samples - 400) / 320) + 1`` frames, i.e. up to ~1 frame short).
|
| 40 |
+
FRAME_SLACK = 2
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@dataclass(frozen=True)
|
| 44 |
+
class AlignmentResult:
|
| 45 |
+
"""Either ``words`` (aligned) or ``failure`` (a machine-readable reason) is set."""
|
| 46 |
+
|
| 47 |
+
words: tuple[TimedWord, ...] | None
|
| 48 |
+
failure: str | None
|
| 49 |
+
details: Mapping[str, object] = field(default_factory=dict)
|
| 50 |
+
|
| 51 |
+
def __post_init__(self) -> None:
|
| 52 |
+
if (self.words is None) == (self.failure is None):
|
| 53 |
+
raise ValueError("AlignmentResult needs exactly one of words / failure")
|
| 54 |
+
if self.words is not None and self.details:
|
| 55 |
+
raise ValueError("an aligned result carries no failure details")
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def alignment_units(
|
| 59 |
+
tokens: Sequence[str], vocab: Mapping[str, int]
|
| 60 |
+
) -> tuple[list[str], list[str]] | str:
|
| 61 |
+
"""Group transcript tokens into alignable words.
|
| 62 |
+
|
| 63 |
+
Returns:
|
| 64 |
+
``(display_words, alignment_strings)`` of equal length, or a failure string
|
| 65 |
+
``"unalignable_chars:<chars>"`` / ``"no_alignable_text"``. Apostrophes join
|
| 66 |
+
letters only: leading/trailing ones are not aligned, and a token made only of
|
| 67 |
+
punctuation (``…``, ``—``, ``'``) is glued to the previous word (or the next one at
|
| 68 |
+
the start) *without* the space between them (``a — b`` → ``a—``, ``b``), because a
|
| 69 |
+
returned word cannot contain whitespace.
|
| 70 |
+
"""
|
| 71 |
+
display: list[str] = []
|
| 72 |
+
targets: list[str] = []
|
| 73 |
+
pending_prefix = ""
|
| 74 |
+
bad: set[str] = set()
|
| 75 |
+
for raw in tokens:
|
| 76 |
+
token = nfc(raw)
|
| 77 |
+
if not token or any(ch.isspace() for ch in token):
|
| 78 |
+
raise ValueError(f"invalid transcript token {raw!r}")
|
| 79 |
+
chars: list[str] = []
|
| 80 |
+
for ch in tr_lower(token).translate(_CIRCUMFLEX_FOLD).translate(_APOSTROPHES):
|
| 81 |
+
if ch in vocab:
|
| 82 |
+
chars.append(ch)
|
| 83 |
+
elif unicodedata.category(ch).startswith("P"):
|
| 84 |
+
continue
|
| 85 |
+
else:
|
| 86 |
+
bad.add(ch)
|
| 87 |
+
chars = list("".join(chars).strip("'"))
|
| 88 |
+
if not chars:
|
| 89 |
+
if display:
|
| 90 |
+
display[-1] += token
|
| 91 |
+
else:
|
| 92 |
+
pending_prefix += token
|
| 93 |
+
continue
|
| 94 |
+
display.append(pending_prefix + token)
|
| 95 |
+
pending_prefix = ""
|
| 96 |
+
targets.append("".join(chars))
|
| 97 |
+
if bad:
|
| 98 |
+
return "unalignable_chars:" + "".join(sorted(bad))
|
| 99 |
+
if not targets:
|
| 100 |
+
return "no_alignable_text"
|
| 101 |
+
return display, targets
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _viterbi(log_probs: NDArray[np.float64], labels: Sequence[int], blank: int) -> list[int] | None:
|
| 105 |
+
"""Best CTC state path; ``None`` if the label sequence cannot fit in the frames.
|
| 106 |
+
|
| 107 |
+
States are the blank-extended sequence ``b l1 b l2 b … lN b`` (index ``2k+1`` is
|
| 108 |
+
label ``k``). Returns the state index for every frame.
|
| 109 |
+
"""
|
| 110 |
+
import numpy as np
|
| 111 |
+
|
| 112 |
+
num_frames = log_probs.shape[0]
|
| 113 |
+
ext = np.full(2 * len(labels) + 1, blank, dtype=np.int64)
|
| 114 |
+
ext[1::2] = labels
|
| 115 |
+
num_states = ext.shape[0]
|
| 116 |
+
# A skip s-2 -> s is allowed into a label that differs from the label two back.
|
| 117 |
+
can_skip = np.zeros(num_states, dtype=bool)
|
| 118 |
+
can_skip[3::2] = ext[3::2] != ext[1:-2:2]
|
| 119 |
+
neg_inf = -np.inf
|
| 120 |
+
score = np.full(num_states, neg_inf)
|
| 121 |
+
score[0] = log_probs[0, ext[0]]
|
| 122 |
+
if num_states > 1:
|
| 123 |
+
score[1] = log_probs[0, ext[1]]
|
| 124 |
+
back = np.zeros((num_frames, num_states), dtype=np.int8) # 0 stay, 1 step, 2 skip
|
| 125 |
+
for t in range(1, num_frames):
|
| 126 |
+
stay = score
|
| 127 |
+
step = np.concatenate(([neg_inf], score[:-1]))
|
| 128 |
+
skip = np.concatenate(([neg_inf, neg_inf], score[:-2]))
|
| 129 |
+
skip = np.where(can_skip, skip, neg_inf)
|
| 130 |
+
stacked = np.stack([stay, step, skip])
|
| 131 |
+
choice = np.argmax(stacked, axis=0)
|
| 132 |
+
best = np.take_along_axis(stacked, choice[None, :], axis=0)[0]
|
| 133 |
+
score = best + log_probs[t, ext]
|
| 134 |
+
back[t] = choice
|
| 135 |
+
final_candidates = [num_states - 1] + ([num_states - 2] if num_states > 1 else [])
|
| 136 |
+
end = max(final_candidates, key=lambda s: score[s])
|
| 137 |
+
if not math.isfinite(float(score[end])):
|
| 138 |
+
return None
|
| 139 |
+
path = [0] * num_frames
|
| 140 |
+
state = end
|
| 141 |
+
for t in range(num_frames - 1, -1, -1):
|
| 142 |
+
path[t] = state
|
| 143 |
+
state -= int(back[t, state])
|
| 144 |
+
return path
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def align_words(
|
| 148 |
+
tokens: Sequence[str],
|
| 149 |
+
log_probs: NDArray[np.float32],
|
| 150 |
+
vocab: Mapping[str, int],
|
| 151 |
+
*,
|
| 152 |
+
blank_token: str,
|
| 153 |
+
word_delimiter: str,
|
| 154 |
+
frame_hop_ms: float,
|
| 155 |
+
duration_ms: int,
|
| 156 |
+
) -> AlignmentResult:
|
| 157 |
+
"""Align transcript ``tokens`` (whitespace-split words) to CTC ``log_probs``.
|
| 158 |
+
|
| 159 |
+
Args:
|
| 160 |
+
tokens: Transcript words in order.
|
| 161 |
+
log_probs: ``[num_frames, vocab_size]`` emissions. Log-softmax is re-applied
|
| 162 |
+
here (idempotent on log-probabilities), so raw logits cannot inflate
|
| 163 |
+
``align_score``. ``-inf`` entries are allowed; NaN and ``+inf`` are not.
|
| 164 |
+
vocab: Token → id (characters, ``blank_token``, ``word_delimiter``).
|
| 165 |
+
blank_token: CTC blank (``[PAD]`` for wav2vec2 HF models).
|
| 166 |
+
word_delimiter: Word separator token (``|``).
|
| 167 |
+
frame_hop_ms: Frame stride in ms (20 for wav2vec2).
|
| 168 |
+
duration_ms: Segment length. ``num_frames * frame_hop_ms`` must be within
|
| 169 |
+
:data:`FRAME_SLACK` frames of it (catches unresampled audio); word ends are
|
| 170 |
+
clamped to it.
|
| 171 |
+
|
| 172 |
+
Returns:
|
| 173 |
+
:class:`AlignmentResult` with words carrying ``align_score`` = mean emission
|
| 174 |
+
probability of the word's character frames, or a failure reason
|
| 175 |
+
(``unalignable_chars``, ``no_alignable_text``, ``too_few_frames``, ``no_path``,
|
| 176 |
+
``words_beyond_audio``).
|
| 177 |
+
|
| 178 |
+
Raises:
|
| 179 |
+
ValueError: malformed inputs (shape, vocab ids not ``0..V-1``, NaN/+inf, bad hop,
|
| 180 |
+
frame count inconsistent with the duration).
|
| 181 |
+
"""
|
| 182 |
+
import numpy as np
|
| 183 |
+
|
| 184 |
+
if log_probs.ndim != 2 or log_probs.shape[1] != len(vocab):
|
| 185 |
+
raise ValueError(f"log_probs shape {log_probs.shape} does not match vocab {len(vocab)}")
|
| 186 |
+
if sorted(vocab.values()) != list(range(len(vocab))):
|
| 187 |
+
raise ValueError("vocab ids must be exactly 0..V-1")
|
| 188 |
+
if np.any(np.isnan(log_probs)) or np.any(np.isposinf(log_probs)):
|
| 189 |
+
raise ValueError("log_probs contain NaN or +inf")
|
| 190 |
+
if blank_token not in vocab or word_delimiter not in vocab:
|
| 191 |
+
raise ValueError("blank or word delimiter missing from vocab")
|
| 192 |
+
if not (math.isfinite(frame_hop_ms) and frame_hop_ms >= 1):
|
| 193 |
+
raise ValueError(f"frame_hop_ms must be a finite number >= 1 ms, got {frame_hop_ms!r}")
|
| 194 |
+
if type(duration_ms) is not int or duration_ms <= 0:
|
| 195 |
+
raise ValueError(f"invalid duration_ms {duration_ms!r}")
|
| 196 |
+
if abs(log_probs.shape[0] * frame_hop_ms - duration_ms) > FRAME_SLACK * frame_hop_ms:
|
| 197 |
+
raise ValueError(
|
| 198 |
+
f"{log_probs.shape[0]} frames x {frame_hop_ms} ms do not match {duration_ms} ms"
|
| 199 |
+
)
|
| 200 |
+
with np.errstate(invalid="ignore"):
|
| 201 |
+
lp = log_probs.astype(np.float64)
|
| 202 |
+
row_max = np.max(lp, axis=1, keepdims=True)
|
| 203 |
+
if not np.all(np.isfinite(row_max)):
|
| 204 |
+
raise ValueError("a frame has no finite log-probability")
|
| 205 |
+
lp = lp - (row_max + np.log(np.sum(np.exp(lp - row_max), axis=1, keepdims=True)))
|
| 206 |
+
special = {blank_token, word_delimiter}
|
| 207 |
+
char_vocab = {tok: idx for tok, idx in vocab.items() if len(tok) == 1 and tok not in special}
|
| 208 |
+
units = alignment_units(tokens, char_vocab)
|
| 209 |
+
if isinstance(units, str):
|
| 210 |
+
reason, _, chars = units.partition(":")
|
| 211 |
+
return AlignmentResult(None, reason, {"chars": chars} if chars else {})
|
| 212 |
+
display, targets = units
|
| 213 |
+
|
| 214 |
+
labels: list[int] = []
|
| 215 |
+
owner: list[int] = [] # word index for each label, -1 for delimiters
|
| 216 |
+
for index, target in enumerate(targets):
|
| 217 |
+
if index:
|
| 218 |
+
labels.append(vocab[word_delimiter])
|
| 219 |
+
owner.append(-1)
|
| 220 |
+
for ch in target:
|
| 221 |
+
labels.append(char_vocab[ch])
|
| 222 |
+
owner.append(index)
|
| 223 |
+
repeats = sum(1 for a, b in itertools.pairwise(labels) if a == b)
|
| 224 |
+
if log_probs.shape[0] < len(labels) + repeats:
|
| 225 |
+
return AlignmentResult(
|
| 226 |
+
None,
|
| 227 |
+
"too_few_frames",
|
| 228 |
+
{"frames": int(log_probs.shape[0]), "needed": len(labels) + repeats},
|
| 229 |
+
)
|
| 230 |
+
path = _viterbi(lp, labels, vocab[blank_token])
|
| 231 |
+
if path is None:
|
| 232 |
+
return AlignmentResult(None, "no_path")
|
| 233 |
+
|
| 234 |
+
first: list[int | None] = [None] * len(targets)
|
| 235 |
+
last: list[int | None] = [None] * len(targets)
|
| 236 |
+
probs: list[list[float]] = [[] for _ in targets]
|
| 237 |
+
for frame, state in enumerate(path):
|
| 238 |
+
if state % 2 == 0:
|
| 239 |
+
continue
|
| 240 |
+
word = owner[state // 2]
|
| 241 |
+
if word < 0:
|
| 242 |
+
continue
|
| 243 |
+
if first[word] is None:
|
| 244 |
+
first[word] = frame
|
| 245 |
+
last[word] = frame
|
| 246 |
+
probs[word].append(math.exp(float(lp[frame, labels[state // 2]])))
|
| 247 |
+
words: list[TimedWord] = []
|
| 248 |
+
for index, text in enumerate(display):
|
| 249 |
+
start_frame, end_frame = first[index], last[index]
|
| 250 |
+
if start_frame is None or end_frame is None:
|
| 251 |
+
return AlignmentResult(None, "no_path", {"word_index": index})
|
| 252 |
+
start_ms = min(round(start_frame * frame_hop_ms), duration_ms - 1)
|
| 253 |
+
end_ms = min(round((end_frame + 1) * frame_hop_ms), duration_ms)
|
| 254 |
+
end_ms = max(end_ms, start_ms + 1)
|
| 255 |
+
if words and start_ms < words[-1].end_ms:
|
| 256 |
+
# Only reachable through the end clamp (emissions slightly longer than the
|
| 257 |
+
# audio): words squeezed past the end cannot be timed, never let them overlap.
|
| 258 |
+
return AlignmentResult(None, "words_beyond_audio", {"word_index": index})
|
| 259 |
+
score = sum(probs[index]) / len(probs[index])
|
| 260 |
+
words.append(TimedWord(text, start_ms, end_ms, score))
|
| 261 |
+
return AlignmentResult(tuple(words), None)
|
src/nida/data/asr_filter.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""S4 / S5 filters: dual-ASR agreement and forced-alignment sanity checks.
|
| 2 |
+
|
| 3 |
+
Pure functions; every threshold comes from :class:`AsrFilterConfig` (``configs/data/
|
| 4 |
+
asr_filter.yaml``). A check returns ``None`` when the segment passes and otherwise a
|
| 5 |
+
machine-readable reason string from a fixed set, which the caller writes to quarantine.
|
| 6 |
+
|
| 7 |
+
The agreement metric is computed on transcripts **with filler words removed**
|
| 8 |
+
(docs/03-data-pipeline.md S4): the two ASR models legitimately disagree on fillers, and
|
| 9 |
+
filtering on them would throw away exactly the expressive segments we want.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import math
|
| 15 |
+
from collections.abc import Mapping, Sequence
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
from itertools import pairwise
|
| 18 |
+
from typing import Literal, get_args
|
| 19 |
+
|
| 20 |
+
from nida.bench.textmetrics import cer_counts, normalize_for_scoring, strip_fillers
|
| 21 |
+
from nida.data.types import TimedWord
|
| 22 |
+
|
| 23 |
+
__all__ = [
|
| 24 |
+
"ALIGNMENT_REASONS",
|
| 25 |
+
"TRANSCRIPT_REASONS",
|
| 26 |
+
"AlignmentReason",
|
| 27 |
+
"AsrFilterConfig",
|
| 28 |
+
"TranscriptReason",
|
| 29 |
+
"asr_agreement",
|
| 30 |
+
"check_alignment",
|
| 31 |
+
"check_transcripts",
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
TranscriptReason = Literal["language", "asr_disagreement", "empty_transcript"]
|
| 35 |
+
AlignmentReason = Literal[
|
| 36 |
+
"align_score",
|
| 37 |
+
"chars_per_s_low",
|
| 38 |
+
"chars_per_s_high",
|
| 39 |
+
"word_outside_segment",
|
| 40 |
+
"words_not_monotonic",
|
| 41 |
+
]
|
| 42 |
+
TRANSCRIPT_REASONS: tuple[str, ...] = get_args(TranscriptReason)
|
| 43 |
+
ALIGNMENT_REASONS: tuple[str, ...] = get_args(AlignmentReason)
|
| 44 |
+
|
| 45 |
+
_MS_PER_S = 1000 # unit conversion, not a threshold
|
| 46 |
+
|
| 47 |
+
# ASCII-only case folding for language codes. Built-in case mapping is avoided: "ID" must
|
| 48 |
+
# not be mapped through Turkish rules ("ID" -> "ıd"), nor through locale-dependent ones.
|
| 49 |
+
_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
| 50 |
+
|
| 51 |
+
_CONFIG_FIELDS: tuple[str, ...] = (
|
| 52 |
+
"max_cer",
|
| 53 |
+
"allowed_languages",
|
| 54 |
+
"min_chars_per_s",
|
| 55 |
+
"max_chars_per_s",
|
| 56 |
+
"min_align_score",
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _norm_language(code: str) -> str:
|
| 61 |
+
return code.strip().translate(_ASCII_LOWER)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _real_number(mapping: Mapping[str, object], name: str) -> float:
|
| 65 |
+
value = mapping[name]
|
| 66 |
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
| 67 |
+
raise ValueError(f"asr filter config: {name} must be a number, got {value!r}")
|
| 68 |
+
number = float(value)
|
| 69 |
+
if not math.isfinite(number):
|
| 70 |
+
raise ValueError(f"asr filter config: {name} must be finite, got {value!r}")
|
| 71 |
+
return number
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@dataclass(frozen=True)
|
| 75 |
+
class AsrFilterConfig:
|
| 76 |
+
"""Thresholds of the S4 / S5 filters; load with :meth:`from_mapping`.
|
| 77 |
+
|
| 78 |
+
Attributes:
|
| 79 |
+
max_cer: Largest accepted CER between the filler-stripped transcripts, in [0, 1].
|
| 80 |
+
A CER exactly equal to it passes.
|
| 81 |
+
allowed_languages: Accepted detected-language codes (ASCII, stored lowercase).
|
| 82 |
+
min_chars_per_s: Smallest accepted letters+digits per second of speech span.
|
| 83 |
+
max_chars_per_s: Largest accepted letters+digits per second of speech span.
|
| 84 |
+
min_align_score: Smallest accepted mean per-word alignment score; the scale depends
|
| 85 |
+
on the aligner.
|
| 86 |
+
"""
|
| 87 |
+
|
| 88 |
+
max_cer: float
|
| 89 |
+
allowed_languages: tuple[str, ...]
|
| 90 |
+
min_chars_per_s: float
|
| 91 |
+
max_chars_per_s: float
|
| 92 |
+
min_align_score: float
|
| 93 |
+
|
| 94 |
+
def __post_init__(self) -> None:
|
| 95 |
+
for name in ("max_cer", "min_chars_per_s", "max_chars_per_s", "min_align_score"):
|
| 96 |
+
if not math.isfinite(getattr(self, name)):
|
| 97 |
+
raise ValueError(f"{name} must be finite, got {getattr(self, name)!r}")
|
| 98 |
+
if not 0.0 <= self.max_cer <= 1.0:
|
| 99 |
+
raise ValueError(f"max_cer must be in [0, 1], got {self.max_cer}")
|
| 100 |
+
if self.min_chars_per_s < 0.0:
|
| 101 |
+
raise ValueError(f"min_chars_per_s must be >= 0, got {self.min_chars_per_s}")
|
| 102 |
+
if self.max_chars_per_s <= 0.0 or self.max_chars_per_s < self.min_chars_per_s:
|
| 103 |
+
raise ValueError(
|
| 104 |
+
f"max_chars_per_s ({self.max_chars_per_s}) must be > 0 and >= "
|
| 105 |
+
f"min_chars_per_s ({self.min_chars_per_s})"
|
| 106 |
+
)
|
| 107 |
+
if not self.allowed_languages:
|
| 108 |
+
raise ValueError("allowed_languages must not be empty")
|
| 109 |
+
for code in self.allowed_languages:
|
| 110 |
+
if not code or not code.isascii() or not code.replace("-", "").isalpha():
|
| 111 |
+
raise ValueError(f"invalid language code {code!r}: expected ASCII letters/'-'")
|
| 112 |
+
if code != _norm_language(code):
|
| 113 |
+
raise ValueError(f"language code {code!r} must be lowercase")
|
| 114 |
+
if len(set(self.allowed_languages)) != len(self.allowed_languages):
|
| 115 |
+
raise ValueError(f"duplicate entries in allowed_languages {self.allowed_languages}")
|
| 116 |
+
|
| 117 |
+
@classmethod
|
| 118 |
+
def from_mapping(cls, mapping: Mapping[str, object]) -> AsrFilterConfig:
|
| 119 |
+
"""Build a config from a parsed YAML mapping.
|
| 120 |
+
|
| 121 |
+
Language codes are stripped and lowercased (ASCII only).
|
| 122 |
+
|
| 123 |
+
Raises:
|
| 124 |
+
ValueError: If ``mapping`` is not a mapping, has unknown or missing keys, a
|
| 125 |
+
number is not a finite int/float (bools are rejected), or a bound or the
|
| 126 |
+
language list is invalid.
|
| 127 |
+
"""
|
| 128 |
+
if not isinstance(mapping, Mapping):
|
| 129 |
+
raise ValueError(f"asr filter config must be a mapping, got {type(mapping).__name__}")
|
| 130 |
+
keys = set(mapping)
|
| 131 |
+
unknown = sorted(str(key) for key in keys - set(_CONFIG_FIELDS))
|
| 132 |
+
missing = sorted(set(_CONFIG_FIELDS) - keys)
|
| 133 |
+
if unknown or missing:
|
| 134 |
+
raise ValueError(f"asr filter config: unknown keys {unknown}, missing keys {missing}")
|
| 135 |
+
raw_languages = mapping["allowed_languages"]
|
| 136 |
+
if isinstance(raw_languages, str) or not isinstance(raw_languages, Sequence):
|
| 137 |
+
raise ValueError(
|
| 138 |
+
f"asr filter config: allowed_languages must be a list, got {raw_languages!r}"
|
| 139 |
+
)
|
| 140 |
+
languages: list[str] = []
|
| 141 |
+
for code in raw_languages:
|
| 142 |
+
if not isinstance(code, str):
|
| 143 |
+
raise ValueError(f"asr filter config: language code {code!r} is not a string")
|
| 144 |
+
languages.append(_norm_language(code))
|
| 145 |
+
return cls(
|
| 146 |
+
max_cer=_real_number(mapping, "max_cer"),
|
| 147 |
+
allowed_languages=tuple(languages),
|
| 148 |
+
min_chars_per_s=_real_number(mapping, "min_chars_per_s"),
|
| 149 |
+
max_chars_per_s=_real_number(mapping, "max_chars_per_s"),
|
| 150 |
+
min_align_score=_real_number(mapping, "min_align_score"),
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
# --------------------------------------------------------------------------------------
|
| 155 |
+
# S4: transcripts
|
| 156 |
+
# --------------------------------------------------------------------------------------
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def asr_agreement(primary: str, secondary: str) -> float:
|
| 160 |
+
"""Character error rate between two transcripts, ignoring fillers.
|
| 161 |
+
|
| 162 |
+
Both texts go through :func:`~nida.bench.textmetrics.normalize_for_scoring` (NFC,
|
| 163 |
+
Turkish lowercase, punctuation removed, fillers canonicalised), fillers are removed,
|
| 164 |
+
and the CER is taken over the space-joined tokens with ``primary`` as the reference.
|
| 165 |
+
|
| 166 |
+
Returns:
|
| 167 |
+
0.0 if both are empty after filler removal (e.g. ``"ıı"`` vs ``""``), 1.0 if
|
| 168 |
+
exactly one is empty, otherwise the CER capped at 1.0 (lower = better agreement).
|
| 169 |
+
"""
|
| 170 |
+
ref = strip_fillers(normalize_for_scoring(primary))
|
| 171 |
+
hyp = strip_fillers(normalize_for_scoring(secondary))
|
| 172 |
+
if not ref and not hyp:
|
| 173 |
+
return 0.0
|
| 174 |
+
if not ref or not hyp:
|
| 175 |
+
return 1.0
|
| 176 |
+
rate = cer_counts(ref, hyp).rate()
|
| 177 |
+
if rate is None: # unreachable: ref is non-empty
|
| 178 |
+
raise ValueError("CER undefined for a non-empty reference")
|
| 179 |
+
return min(rate, 1.0)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def check_transcripts(
|
| 183 |
+
primary_text: str,
|
| 184 |
+
secondary_text: str,
|
| 185 |
+
language: str | None,
|
| 186 |
+
cfg: AsrFilterConfig,
|
| 187 |
+
) -> TranscriptReason | None:
|
| 188 |
+
"""S4 filter: empty transcript, language and dual-ASR agreement.
|
| 189 |
+
|
| 190 |
+
Checks run in this order and the first failure is returned:
|
| 191 |
+
|
| 192 |
+
1. ``"empty_transcript"``: the primary transcript has no token at all after
|
| 193 |
+
normalisation. A fillers-only primary (``"hmm"``) is *not* empty: it is a valid
|
| 194 |
+
sound-only segment.
|
| 195 |
+
2. ``"language"``: ``language`` is ``None`` or not in ``cfg.allowed_languages``
|
| 196 |
+
(case-insensitive exact match on the code).
|
| 197 |
+
3. ``"asr_disagreement"``: :func:`asr_agreement` is greater than ``cfg.max_cer``
|
| 198 |
+
(equal passes).
|
| 199 |
+
|
| 200 |
+
Args:
|
| 201 |
+
primary_text: Primary ASR transcript (the label source).
|
| 202 |
+
secondary_text: Secondary ASR transcript.
|
| 203 |
+
language: Detected language code of the segment, or ``None`` if unknown.
|
| 204 |
+
cfg: Thresholds.
|
| 205 |
+
|
| 206 |
+
Returns:
|
| 207 |
+
``None`` if the segment passes, else the reason.
|
| 208 |
+
"""
|
| 209 |
+
if not normalize_for_scoring(primary_text):
|
| 210 |
+
return "empty_transcript"
|
| 211 |
+
if language is None or _norm_language(language) not in cfg.allowed_languages:
|
| 212 |
+
return "language"
|
| 213 |
+
if asr_agreement(primary_text, secondary_text) > cfg.max_cer:
|
| 214 |
+
return "asr_disagreement"
|
| 215 |
+
return None
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
# --------------------------------------------------------------------------------------
|
| 219 |
+
# S5: alignment
|
| 220 |
+
# --------------------------------------------------------------------------------------
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _num_chars(text: str) -> int:
|
| 224 |
+
"""Letters and digits of the normalised text (fillers included, spaces excluded)."""
|
| 225 |
+
return sum(1 for token in normalize_for_scoring(text) for ch in token if ch.isalnum())
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def check_alignment(
|
| 229 |
+
words: Sequence[TimedWord], duration_ms: int, cfg: AsrFilterConfig
|
| 230 |
+
) -> AlignmentReason | None:
|
| 231 |
+
"""S5 filter: sanity of the forced-alignment output of one segment.
|
| 232 |
+
|
| 233 |
+
Word times are relative to the segment start. Checks run in this order and the first
|
| 234 |
+
failure is returned:
|
| 235 |
+
|
| 236 |
+
1. ``"words_not_monotonic"``: a word starts before the previous one ends (touching is
|
| 237 |
+
fine).
|
| 238 |
+
2. ``"word_outside_segment"``: a word ends after ``duration_ms``.
|
| 239 |
+
3. ``"align_score"``: the mean word alignment score is below ``cfg.min_align_score``,
|
| 240 |
+
or any word has no (or a non-finite) score: a segment whose alignment cannot be
|
| 241 |
+
verified is not accepted. The mean, not the minimum, is used so that a single hard-to-align
|
| 242 |
+
filler or laugh does not discard an expressive segment.
|
| 243 |
+
4. ``"chars_per_s_low"`` / ``"chars_per_s_high"``: letters+digits of the normalised
|
| 244 |
+
words (fillers included) per second of speech span (first word start to last word
|
| 245 |
+
end) is below ``cfg.min_chars_per_s`` / above ``cfg.max_chars_per_s``. An empty word
|
| 246 |
+
list (or words without any letter or digit) has rate 0 and is reported as
|
| 247 |
+
``"chars_per_s_low"`` rather than crashing the pipeline.
|
| 248 |
+
|
| 249 |
+
Thresholds are inclusive: values exactly equal to a limit pass.
|
| 250 |
+
|
| 251 |
+
Args:
|
| 252 |
+
words: Aligned words of the primary transcript, in order.
|
| 253 |
+
duration_ms: Segment duration in milliseconds, must be > 0.
|
| 254 |
+
cfg: Thresholds.
|
| 255 |
+
|
| 256 |
+
Returns:
|
| 257 |
+
``None`` if the alignment looks sane, else the reason.
|
| 258 |
+
|
| 259 |
+
Raises:
|
| 260 |
+
ValueError: If ``duration_ms`` is not a positive int.
|
| 261 |
+
"""
|
| 262 |
+
if isinstance(duration_ms, bool) or not isinstance(duration_ms, int) or duration_ms <= 0:
|
| 263 |
+
raise ValueError(f"duration_ms must be a positive int, got {duration_ms!r}")
|
| 264 |
+
if not words:
|
| 265 |
+
return "chars_per_s_low"
|
| 266 |
+
for previous, word in pairwise(words):
|
| 267 |
+
if word.start_ms < previous.end_ms:
|
| 268 |
+
return "words_not_monotonic"
|
| 269 |
+
if any(word.end_ms > duration_ms for word in words):
|
| 270 |
+
return "word_outside_segment"
|
| 271 |
+
scores = [word.align_score for word in words]
|
| 272 |
+
known = [score for score in scores if score is not None]
|
| 273 |
+
if len(known) != len(scores) or not all(math.isfinite(score) for score in known):
|
| 274 |
+
return "align_score"
|
| 275 |
+
if math.fsum(known) / len(known) < cfg.min_align_score:
|
| 276 |
+
return "align_score"
|
| 277 |
+
span_ms = words[-1].end_ms - words[0].start_ms
|
| 278 |
+
chars_per_s = sum(_num_chars(word.text) for word in words) * _MS_PER_S / span_ms
|
| 279 |
+
if chars_per_s < cfg.min_chars_per_s:
|
| 280 |
+
return "chars_per_s_low"
|
| 281 |
+
if chars_per_s > cfg.max_chars_per_s:
|
| 282 |
+
return "chars_per_s_high"
|
| 283 |
+
return None
|
src/nida/data/ears.py
ADDED
|
@@ -0,0 +1,1307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""EARS -> latent dataset builder (ADR-0007, ``nida data-ears``).
|
| 2 |
+
|
| 3 |
+
EARS (Expressive Anechoic Recordings of Speech, CC BY-NC 4.0) provides English emotional,
|
| 4 |
+
whispered and non-verbal speech. This module turns the selected files into a *latent dataset
|
| 5 |
+
directory* in exactly the format of :mod:`nida.train.data` (records carry the optional key
|
| 6 |
+
``"language": "en"``), so it can be trained together with Turkish datasets
|
| 7 |
+
(``train --data A --data B``).
|
| 8 |
+
|
| 9 |
+
What is used (rules and mappings live in ``configs/data/ears.yaml``):
|
| 10 |
+
|
| 11 |
+
* ``emo_<emotion>_sentences``: reading transcript, emotion mapped (unmapped -> ``null``);
|
| 12 |
+
* ``rainbow_NN_<style>`` / ``sentences_NN_<style>``: ``whisper`` -> the whole text wrapped in a
|
| 13 |
+
whisper span (rendered by :mod:`nida.text.tags`), the plain styles -> plain text;
|
| 14 |
+
* laughter/crying/screaming/coughing/throat-clearing files: just the event tag;
|
| 15 |
+
* everything else is skipped and counted by reason.
|
| 16 |
+
|
| 17 |
+
Per speaker (one remote zip of ~564 MB, read with HTTP Range requests, see
|
| 18 |
+
:mod:`nida.data.remote_zip`): list the archive -> decide on every member (skip / text
|
| 19 |
+
quarantine, before any download) -> extract the selected members, convert 48 kHz -> 16 kHz mono
|
| 20 |
+
-> measure. The speakers are handled in *waves* of ``speakers_per_wave``: one AudioVAE run
|
| 21 |
+
(the same ``vae_encode`` worker and environment as the pilot, so the model is loaded once per
|
| 22 |
+
wave, not once per speaker) encodes the wave's audio, then every speaker of the wave gets its
|
| 23 |
+
float16 shard, index records and ``done.json``. Layout::
|
| 24 |
+
|
| 25 |
+
<root>/plan.json seed, config digest, VAE pin, transcripts SHA-256
|
| 26 |
+
<root>/source/transcripts.json pinned reading transcripts
|
| 27 |
+
<root>/speakers/pNNN/records.jsonl index rows of the speaker
|
| 28 |
+
<root>/speakers/pNNN/quarantine.jsonl every quarantined item with its reason
|
| 29 |
+
<root>/speakers/pNNN/done.json written last: the speaker is finished
|
| 30 |
+
<root>/workers/vae/ worker requests/results of the current wave
|
| 31 |
+
<root>/dataset/ latent dataset: index.jsonl, meta.json, shard-pNNN.npy
|
| 32 |
+
<root>/report.md, <root>/stats.json
|
| 33 |
+
<cache>/... raw members, 16 kHz wavs and latent arrays (local disk)
|
| 34 |
+
|
| 35 |
+
Resumable and idempotent: a speaker with ``done.json`` is skipped without any network access, an
|
| 36 |
+
interrupted wave continues (worker results are kept, already converted audio is reused), and
|
| 37 |
+
the dataset directory is reassembled from all finished speakers at the end of every run, so
|
| 38 |
+
it is usable after any run. A run on a root made with another seed, config digest or VAE pin,
|
| 39 |
+
or with different transcripts, is refused.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
from __future__ import annotations
|
| 43 |
+
|
| 44 |
+
import hashlib
|
| 45 |
+
import io
|
| 46 |
+
import json
|
| 47 |
+
import logging
|
| 48 |
+
import math
|
| 49 |
+
import re
|
| 50 |
+
import shutil
|
| 51 |
+
import string
|
| 52 |
+
import time
|
| 53 |
+
from bisect import bisect_right
|
| 54 |
+
from collections import Counter
|
| 55 |
+
from collections.abc import Callable, Mapping, Sequence
|
| 56 |
+
from dataclasses import asdict, dataclass, field
|
| 57 |
+
from pathlib import Path
|
| 58 |
+
from typing import Literal
|
| 59 |
+
|
| 60 |
+
import numpy as np
|
| 61 |
+
import numpy.typing as npt
|
| 62 |
+
|
| 63 |
+
# scipy ships no type stubs; the ignore is scoped to this single import (as in acoustic.py).
|
| 64 |
+
from scipy import signal # type: ignore[import-untyped]
|
| 65 |
+
|
| 66 |
+
from nida.bench import envs as bench_envs
|
| 67 |
+
from nida.bench import fetch
|
| 68 |
+
from nida.bench.b1 import MAX_ERROR_FRACTION, _stream_runner
|
| 69 |
+
from nida.bench.protocol import (
|
| 70 |
+
WorkRequest,
|
| 71 |
+
WorkResult,
|
| 72 |
+
drop_results,
|
| 73 |
+
load_results,
|
| 74 |
+
read_summary,
|
| 75 |
+
write_requests,
|
| 76 |
+
)
|
| 77 |
+
from nida.bench.registry import load_sources, require_approved
|
| 78 |
+
from nida.data import acoustic, envs
|
| 79 |
+
from nida.data.ingest import IngestConfig
|
| 80 |
+
from nida.data.pilot import DatasetConfig
|
| 81 |
+
from nida.data.quarantine import QuarantineLog
|
| 82 |
+
from nida.data.registry import DATA_MODELS
|
| 83 |
+
from nida.data.remote_zip import RemoteZip, ZipMember
|
| 84 |
+
from nida.data.splits import assign_splits
|
| 85 |
+
from nida.data.types import QuarantineRecord, segment_id
|
| 86 |
+
from nida.model.vocab import encode_text
|
| 87 |
+
from nida.text import tags
|
| 88 |
+
from nida.text.turkish import nfc
|
| 89 |
+
from nida.train.data import (
|
| 90 |
+
EMOTIONS,
|
| 91 |
+
REJECT_TEXT_TOO_LONG,
|
| 92 |
+
SPLITS,
|
| 93 |
+
DataFormatError,
|
| 94 |
+
load_latent_dataset,
|
| 95 |
+
)
|
| 96 |
+
from nida.utils.io import atomic_write_bytes, atomic_write_json, atomic_write_text
|
| 97 |
+
|
| 98 |
+
__all__ = [
|
| 99 |
+
"SOURCE_ID",
|
| 100 |
+
"AudioFacts",
|
| 101 |
+
"Ears",
|
| 102 |
+
"EarsConfig",
|
| 103 |
+
"EarsSettings",
|
| 104 |
+
"ItemRejected",
|
| 105 |
+
"PlannedItem",
|
| 106 |
+
"Selection",
|
| 107 |
+
"SelectionPlan",
|
| 108 |
+
"Skip",
|
| 109 |
+
"TextRejected",
|
| 110 |
+
"bucket_of",
|
| 111 |
+
"build_text",
|
| 112 |
+
"classify_stem",
|
| 113 |
+
"convert_to_target_rate",
|
| 114 |
+
"en_lower",
|
| 115 |
+
"load_settings",
|
| 116 |
+
"measure_audio",
|
| 117 |
+
"normalise_english",
|
| 118 |
+
"parse_speakers",
|
| 119 |
+
"rate_chars_per_s",
|
| 120 |
+
"resample",
|
| 121 |
+
"select_members",
|
| 122 |
+
"speaker_split",
|
| 123 |
+
]
|
| 124 |
+
|
| 125 |
+
log = logging.getLogger(__name__)
|
| 126 |
+
|
| 127 |
+
SOURCE_ID = "ears"
|
| 128 |
+
STAGE = "ears"
|
| 129 |
+
LANGUAGE = "en"
|
| 130 |
+
VAE_KEY = "voxcpm2_audiovae_v2"
|
| 131 |
+
DEFAULT_SEED = 20261009
|
| 132 |
+
FLOAT16_MAX_ABS = 6.0e4 # latents beyond this would overflow the float16 shards (as in pilot.py)
|
| 133 |
+
CONFIG_FILE = "ears.yaml"
|
| 134 |
+
INGEST_FILE = "ingest.yaml" # duration limits shared with the Turkish pilot
|
| 135 |
+
DATASET_FILE = "dataset.yaml" # fixed condition-bucket edges
|
| 136 |
+
_ASCII_LOWER = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)
|
| 137 |
+
_SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
| 138 |
+
_SPEAKER_RE = re.compile(r"p(\d{3})")
|
| 139 |
+
_WORD_RE = re.compile(r"[a-z]+")
|
| 140 |
+
_STEM_RE = re.compile(r"[a-z0-9_]+")
|
| 141 |
+
_EMO_RE = re.compile(r"emo_([a-z]+)_(sentences|freeform)")
|
| 142 |
+
_READ_RE = re.compile(r"([a-z]+)_(\d+)_([a-z]+)")
|
| 143 |
+
_TAG_CHARS = frozenset("[]<>")
|
| 144 |
+
|
| 145 |
+
Kind = Literal["emotion", "read", "whisper", "event"]
|
| 146 |
+
FloatArray = npt.NDArray[np.float64]
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
# --------------------------------------------------------------------------------------
|
| 150 |
+
# Config
|
| 151 |
+
# --------------------------------------------------------------------------------------
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _load_yaml(path: Path) -> Mapping[str, object]:
|
| 155 |
+
import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
|
| 156 |
+
|
| 157 |
+
obj = yaml.safe_load(path.read_text(encoding="utf-8"))
|
| 158 |
+
if not isinstance(obj, dict):
|
| 159 |
+
raise ValueError(f"{path}: expected a mapping")
|
| 160 |
+
return obj
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def _non_negative_int(raw: Mapping[str, object], key: str) -> int:
|
| 164 |
+
value = raw[key]
|
| 165 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
| 166 |
+
raise ValueError(f"{key} must be a non-negative int, got {value!r}")
|
| 167 |
+
return value
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _positive_int(raw: Mapping[str, object], key: str) -> int:
|
| 171 |
+
value = raw[key]
|
| 172 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
| 173 |
+
raise ValueError(f"{key} must be a positive int, got {value!r}")
|
| 174 |
+
return value
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _fraction(raw: Mapping[str, object], key: str) -> float:
|
| 178 |
+
value = raw[key]
|
| 179 |
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
| 180 |
+
raise ValueError(f"{key} must be a number, got {value!r}")
|
| 181 |
+
if not (math.isfinite(value) and 0.0 <= value < 1.0):
|
| 182 |
+
raise ValueError(f"{key} must be in [0, 1), got {value!r}")
|
| 183 |
+
return float(value)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _words(raw: Mapping[str, object], key: str) -> tuple[str, ...]:
|
| 187 |
+
value = raw[key]
|
| 188 |
+
if (
|
| 189 |
+
not isinstance(value, list)
|
| 190 |
+
or not value
|
| 191 |
+
or not all(isinstance(x, str) and _WORD_RE.fullmatch(x) for x in value)
|
| 192 |
+
or len(set(value)) != len(value)
|
| 193 |
+
):
|
| 194 |
+
raise ValueError(f"{key} must be a non-empty list of distinct lowercase words")
|
| 195 |
+
return tuple(value)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _str_map(raw: Mapping[str, object], key: str) -> dict[str, str]:
|
| 199 |
+
value = raw[key]
|
| 200 |
+
if not isinstance(value, dict) or not all(
|
| 201 |
+
isinstance(k, str) and isinstance(v, str) for k, v in value.items()
|
| 202 |
+
):
|
| 203 |
+
raise ValueError(f"{key} must be a mapping of strings")
|
| 204 |
+
return dict(value)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
@dataclass(frozen=True)
|
| 208 |
+
class EarsConfig:
|
| 209 |
+
"""``configs/data/ears.yaml`` (all keys required; see the file for the meaning)."""
|
| 210 |
+
|
| 211 |
+
transcripts_url: str
|
| 212 |
+
transcripts_sha256: str | None
|
| 213 |
+
archive_url_template: str
|
| 214 |
+
n_speakers: int
|
| 215 |
+
expected_sample_rate_hz: int
|
| 216 |
+
target_sample_rate_hz: int
|
| 217 |
+
val_speaker_fraction: float
|
| 218 |
+
test_speaker_fraction: float
|
| 219 |
+
speakers_per_wave: int
|
| 220 |
+
read_prefixes: tuple[str, ...]
|
| 221 |
+
whisper_styles: tuple[str, ...]
|
| 222 |
+
plain_styles: tuple[str, ...]
|
| 223 |
+
max_read_per_speaker: int
|
| 224 |
+
emotion_map: dict[str, str] # plain dicts (not frozen views): ``asdict`` feeds the digest
|
| 225 |
+
event_map: dict[str, str]
|
| 226 |
+
char_map: dict[str, str]
|
| 227 |
+
|
| 228 |
+
def __post_init__(self) -> None:
|
| 229 |
+
for name in ("transcripts_url", "archive_url_template"):
|
| 230 |
+
if not getattr(self, name).lower().startswith(("http://", "https://")):
|
| 231 |
+
raise ValueError(f"{name} must be an http(s) URL")
|
| 232 |
+
if "{speaker}" not in self.archive_url_template:
|
| 233 |
+
raise ValueError("archive_url_template must contain {speaker}")
|
| 234 |
+
if self.transcripts_sha256 is not None and not _SHA256_RE.fullmatch(
|
| 235 |
+
self.transcripts_sha256
|
| 236 |
+
):
|
| 237 |
+
raise ValueError("transcripts_sha256 must be null or 64 lowercase hex characters")
|
| 238 |
+
if self.val_speaker_fraction + self.test_speaker_fraction >= 1.0:
|
| 239 |
+
raise ValueError("val and test speaker fractions must leave room for train")
|
| 240 |
+
if set(self.whisper_styles) & set(self.plain_styles):
|
| 241 |
+
raise ValueError("a style cannot be both whisper and plain")
|
| 242 |
+
for emotion, mapped in self.emotion_map.items():
|
| 243 |
+
if not _WORD_RE.fullmatch(emotion) or mapped not in EMOTIONS:
|
| 244 |
+
raise ValueError(f"emotion_map {emotion!r}: {mapped!r} is not one of {EMOTIONS}")
|
| 245 |
+
for stem, event in self.event_map.items():
|
| 246 |
+
if not _STEM_RE.fullmatch(stem) or event not in tags.EVENTS:
|
| 247 |
+
raise ValueError(f"event_map {stem!r}: {event!r} is not a tag-spec event id")
|
| 248 |
+
for char, replacement in self.char_map.items():
|
| 249 |
+
if len(char) != 1 or not replacement:
|
| 250 |
+
raise ValueError(f"char_map entries map one character to a string: {char!r}")
|
| 251 |
+
|
| 252 |
+
@classmethod
|
| 253 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> EarsConfig:
|
| 254 |
+
expected = set(cls.__dataclass_fields__)
|
| 255 |
+
if set(raw) != expected:
|
| 256 |
+
raise ValueError(
|
| 257 |
+
f"ears config keys: missing={sorted(expected - set(raw))} "
|
| 258 |
+
f"unknown={sorted(set(raw) - expected)}"
|
| 259 |
+
)
|
| 260 |
+
for key in ("transcripts_url", "archive_url_template"):
|
| 261 |
+
if not isinstance(raw[key], str):
|
| 262 |
+
raise ValueError(f"{key} must be a string")
|
| 263 |
+
sha = raw["transcripts_sha256"]
|
| 264 |
+
if sha is not None and not isinstance(sha, str):
|
| 265 |
+
raise ValueError("transcripts_sha256 must be null or a string")
|
| 266 |
+
return cls(
|
| 267 |
+
transcripts_url=str(raw["transcripts_url"]),
|
| 268 |
+
transcripts_sha256=sha,
|
| 269 |
+
archive_url_template=str(raw["archive_url_template"]),
|
| 270 |
+
n_speakers=_positive_int(raw, "n_speakers"),
|
| 271 |
+
expected_sample_rate_hz=_positive_int(raw, "expected_sample_rate_hz"),
|
| 272 |
+
target_sample_rate_hz=_positive_int(raw, "target_sample_rate_hz"),
|
| 273 |
+
val_speaker_fraction=_fraction(raw, "val_speaker_fraction"),
|
| 274 |
+
test_speaker_fraction=_fraction(raw, "test_speaker_fraction"),
|
| 275 |
+
speakers_per_wave=_positive_int(raw, "speakers_per_wave"),
|
| 276 |
+
read_prefixes=_words(raw, "read_prefixes"),
|
| 277 |
+
whisper_styles=_words(raw, "whisper_styles"),
|
| 278 |
+
plain_styles=_words(raw, "plain_styles"),
|
| 279 |
+
max_read_per_speaker=_non_negative_int(raw, "max_read_per_speaker"),
|
| 280 |
+
emotion_map=_str_map(raw, "emotion_map"),
|
| 281 |
+
event_map=_str_map(raw, "event_map"),
|
| 282 |
+
char_map=_str_map(raw, "char_map"),
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
def archive_url(self, speaker: str) -> str:
|
| 286 |
+
return self.archive_url_template.replace("{speaker}", speaker)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
@dataclass(frozen=True)
|
| 290 |
+
class EarsSettings:
|
| 291 |
+
"""Everything the builder reads from ``configs/data``."""
|
| 292 |
+
|
| 293 |
+
ears: EarsConfig
|
| 294 |
+
min_duration_ms: int
|
| 295 |
+
max_duration_ms: int
|
| 296 |
+
dataset: DatasetConfig
|
| 297 |
+
digest: str
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
def load_settings(config_dir: Path) -> EarsSettings:
|
| 301 |
+
"""Load ``ears.yaml`` plus the duration limits (``ingest.yaml``) and bucket edges
|
| 302 |
+
(``dataset.yaml``) it shares with the Turkish pilot.
|
| 303 |
+
|
| 304 |
+
``digest`` names everything a finished speaker depends on; it ignores
|
| 305 |
+
``transcripts_sha256`` so that pinning the hash in the config later is allowed.
|
| 306 |
+
"""
|
| 307 |
+
ears = EarsConfig.from_mapping(_load_yaml(config_dir / CONFIG_FILE))
|
| 308 |
+
ingest_cfg = IngestConfig.from_mapping(_load_yaml(config_dir / INGEST_FILE))
|
| 309 |
+
dataset = DatasetConfig.from_mapping(_load_yaml(config_dir / DATASET_FILE))
|
| 310 |
+
body = asdict(ears)
|
| 311 |
+
del body["transcripts_sha256"]
|
| 312 |
+
del body["speakers_per_wave"] # operational: does not change any output
|
| 313 |
+
canonical = json.dumps(
|
| 314 |
+
{
|
| 315 |
+
"ears": body,
|
| 316 |
+
"min_duration_ms": ingest_cfg.min_duration_ms,
|
| 317 |
+
"max_duration_ms": ingest_cfg.max_duration_ms,
|
| 318 |
+
"dataset": asdict(dataset),
|
| 319 |
+
},
|
| 320 |
+
sort_keys=True,
|
| 321 |
+
ensure_ascii=False,
|
| 322 |
+
)
|
| 323 |
+
return EarsSettings(
|
| 324 |
+
ears=ears,
|
| 325 |
+
min_duration_ms=ingest_cfg.min_duration_ms,
|
| 326 |
+
max_duration_ms=ingest_cfg.max_duration_ms,
|
| 327 |
+
dataset=dataset,
|
| 328 |
+
digest=hashlib.sha1(canonical.encode("utf-8")).hexdigest()[:12],
|
| 329 |
+
)
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def parse_speakers(spec: str, n_speakers: int) -> list[str]:
|
| 333 |
+
"""``"all"`` or ``"p001,p007"`` -> speaker ids.
|
| 334 |
+
|
| 335 |
+
Raises:
|
| 336 |
+
ValueError: on an empty list, a malformed or out-of-range id, or a repeated id.
|
| 337 |
+
"""
|
| 338 |
+
if spec.strip() == "all":
|
| 339 |
+
return [f"p{i:03d}" for i in range(1, n_speakers + 1)]
|
| 340 |
+
ids = [part.strip() for part in spec.split(",")]
|
| 341 |
+
for speaker in ids:
|
| 342 |
+
match = _SPEAKER_RE.fullmatch(speaker)
|
| 343 |
+
if match is None or not 1 <= int(match.group(1)) <= n_speakers:
|
| 344 |
+
raise ValueError(f"invalid speaker {speaker!r}: use p001..p{n_speakers:03d} or 'all'")
|
| 345 |
+
if len(set(ids)) != len(ids):
|
| 346 |
+
raise ValueError(f"repeated speaker in {spec!r}")
|
| 347 |
+
return ids
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
# --------------------------------------------------------------------------------------
|
| 351 |
+
# Text
|
| 352 |
+
# --------------------------------------------------------------------------------------
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
class TextRejected(ValueError):
|
| 356 |
+
"""The text of an item cannot be used; ``reason`` is the quarantine reason."""
|
| 357 |
+
|
| 358 |
+
def __init__(self, reason: str, detail: str) -> None:
|
| 359 |
+
super().__init__(f"{reason}: {detail}")
|
| 360 |
+
self.reason = reason
|
| 361 |
+
self.detail = detail
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def en_lower(text: str) -> str:
|
| 365 |
+
"""Lower-case ``A``-``Z`` only (English). Never use ``tr_lower`` (``I`` -> ``ı``) or
|
| 366 |
+
``str.lower`` (``İ`` -> ``i`` + combining dot) on English text: every other character is
|
| 367 |
+
returned unchanged."""
|
| 368 |
+
return text.translate(_ASCII_LOWER)
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def normalise_english(text: str, char_map: Mapping[str, str]) -> str:
|
| 372 |
+
"""NFC, ``char_map`` replacements, ASCII lower-case and collapsed whitespace.
|
| 373 |
+
|
| 374 |
+
Returns ``""`` for text without any non-space character.
|
| 375 |
+
"""
|
| 376 |
+
mapped = nfc(text).translate({ord(k): v for k, v in char_map.items()})
|
| 377 |
+
return " ".join(en_lower(mapped).split())
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
@dataclass(frozen=True)
|
| 381 |
+
class BuiltText:
|
| 382 |
+
"""Model text (tag-spec surface) and the plain words the speaker says (may be empty)."""
|
| 383 |
+
|
| 384 |
+
tagged: str
|
| 385 |
+
plain: str
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
def build_text(selection: Selection, transcript: str | None, cfg: EarsConfig) -> BuiltText:
|
| 389 |
+
"""Build the training text of one selected file.
|
| 390 |
+
|
| 391 |
+
Raises:
|
| 392 |
+
TextRejected: with reason ``no_transcript``, ``empty_text``, ``digits``,
|
| 393 |
+
``tag_characters``, ``non_ascii_letter`` or ``text_encoding`` (the model
|
| 394 |
+
vocabulary cannot encode the result).
|
| 395 |
+
"""
|
| 396 |
+
if selection.kind == "event":
|
| 397 |
+
assert selection.event is not None # classify_stem always sets it for events
|
| 398 |
+
tagged = tags.render([tags.Event(selection.event)])
|
| 399 |
+
plain = ""
|
| 400 |
+
else:
|
| 401 |
+
if transcript is None:
|
| 402 |
+
raise TextRejected("no_transcript", f"no entry for {selection.stem!r}")
|
| 403 |
+
plain = normalise_english(transcript, cfg.char_map)
|
| 404 |
+
if not plain:
|
| 405 |
+
raise TextRejected("empty_text", "transcript has no characters")
|
| 406 |
+
if any(ch.isdigit() for ch in plain):
|
| 407 |
+
raise TextRejected("digits", plain[:80])
|
| 408 |
+
if any(ch in _TAG_CHARS for ch in plain):
|
| 409 |
+
raise TextRejected("tag_characters", plain[:80])
|
| 410 |
+
if any(ch.isalpha() and not ch.isascii() for ch in plain):
|
| 411 |
+
raise TextRejected("non_ascii_letter", plain[:80])
|
| 412 |
+
if selection.kind == "whisper":
|
| 413 |
+
try:
|
| 414 |
+
span = tags.Span("whisper", tuple(tags.Word(w) for w in plain.split()))
|
| 415 |
+
except ValueError as err:
|
| 416 |
+
raise TextRejected("text_encoding", str(err)) from err
|
| 417 |
+
tagged = tags.render([span])
|
| 418 |
+
else:
|
| 419 |
+
tagged = plain
|
| 420 |
+
try:
|
| 421 |
+
ids = encode_text(tagged)
|
| 422 |
+
except ValueError as err: # unknown character, bad tag
|
| 423 |
+
raise TextRejected("text_encoding", str(err)) from err
|
| 424 |
+
if not ids:
|
| 425 |
+
raise TextRejected("empty_text", "text encodes to no tokens")
|
| 426 |
+
return BuiltText(tagged=tagged, plain=plain)
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# --------------------------------------------------------------------------------------
|
| 430 |
+
# Selection
|
| 431 |
+
# --------------------------------------------------------------------------------------
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
@dataclass(frozen=True)
|
| 435 |
+
class Selection:
|
| 436 |
+
"""A file that is used. ``emotion`` is set for ``emotion`` items (may be ``None``),
|
| 437 |
+
``event`` for ``event`` items."""
|
| 438 |
+
|
| 439 |
+
stem: str
|
| 440 |
+
kind: Kind
|
| 441 |
+
emotion: str | None = None
|
| 442 |
+
event: str | None = None
|
| 443 |
+
|
| 444 |
+
@property
|
| 445 |
+
def label(self) -> str:
|
| 446 |
+
"""Report category: ``emotion``, ``read``, ``whisper`` or ``event:<id>``."""
|
| 447 |
+
return f"event:{self.event}" if self.kind == "event" else self.kind
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
@dataclass(frozen=True)
|
| 451 |
+
class Skip:
|
| 452 |
+
"""A file that is not used, with the counted reason."""
|
| 453 |
+
|
| 454 |
+
stem: str
|
| 455 |
+
reason: str
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def classify_stem(stem: str, cfg: EarsConfig) -> Selection | Skip:
|
| 459 |
+
"""Decide on one file stem (ADR-0007 table). Pure; thresholds and maps come from ``cfg``.
|
| 460 |
+
|
| 461 |
+
Skip reasons: ``freeform``, ``interjection``, ``melodic``, ``event_unmapped``,
|
| 462 |
+
``style_unsupported``, ``unknown_stem``.
|
| 463 |
+
"""
|
| 464 |
+
emo = _EMO_RE.fullmatch(stem)
|
| 465 |
+
if emo is not None:
|
| 466 |
+
if emo.group(2) != "sentences":
|
| 467 |
+
return Skip(stem, "freeform")
|
| 468 |
+
return Selection(stem, "emotion", emotion=cfg.emotion_map.get(emo.group(1)))
|
| 469 |
+
event = cfg.event_map.get(stem)
|
| 470 |
+
if event is not None:
|
| 471 |
+
return Selection(stem, "event", event=event)
|
| 472 |
+
read = _READ_RE.fullmatch(stem)
|
| 473 |
+
if read is not None and read.group(1) in cfg.read_prefixes:
|
| 474 |
+
style = read.group(3)
|
| 475 |
+
if style in cfg.whisper_styles:
|
| 476 |
+
return Selection(stem, "whisper")
|
| 477 |
+
if style in cfg.plain_styles:
|
| 478 |
+
return Selection(stem, "read")
|
| 479 |
+
return Skip(stem, "style_unsupported")
|
| 480 |
+
if stem.startswith("freeform_speech"):
|
| 481 |
+
return Skip(stem, "freeform")
|
| 482 |
+
for prefix, reason in (
|
| 483 |
+
("interjection_", "interjection"),
|
| 484 |
+
("melodic_", "melodic"),
|
| 485 |
+
("nonverbal_", "event_unmapped"),
|
| 486 |
+
("vegetative_", "event_unmapped"),
|
| 487 |
+
):
|
| 488 |
+
if stem.startswith(prefix):
|
| 489 |
+
return Skip(stem, reason)
|
| 490 |
+
return Skip(stem, "unknown_stem")
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
@dataclass(frozen=True)
|
| 494 |
+
class PlannedItem:
|
| 495 |
+
"""A selected archive member with its final text."""
|
| 496 |
+
|
| 497 |
+
stem: str
|
| 498 |
+
member: ZipMember
|
| 499 |
+
selection: Selection
|
| 500 |
+
text: BuiltText
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
@dataclass
|
| 504 |
+
class SelectionPlan:
|
| 505 |
+
"""Result of :func:`select_members`: nothing is lost, every member is in one of three places."""
|
| 506 |
+
|
| 507 |
+
items: list[PlannedItem] = field(default_factory=list)
|
| 508 |
+
skipped: Counter[str] = field(default_factory=Counter)
|
| 509 |
+
quarantine: list[QuarantineRecord] = field(default_factory=list)
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
def select_members(
|
| 513 |
+
speaker: str,
|
| 514 |
+
members: Sequence[ZipMember],
|
| 515 |
+
transcripts: Mapping[str, str],
|
| 516 |
+
cfg: EarsConfig,
|
| 517 |
+
) -> SelectionPlan:
|
| 518 |
+
"""Decide on every archive member before anything is downloaded.
|
| 519 |
+
|
| 520 |
+
``items`` come back in archive order (ascending ``header_offset``) so extraction profits
|
| 521 |
+
from the read-ahead window. Skipped members are counted by reason; text problems become
|
| 522 |
+
quarantine records (stage ``ears``, item id ``<speaker>/<stem>``).
|
| 523 |
+
"""
|
| 524 |
+
plan = SelectionPlan()
|
| 525 |
+
seen: set[str] = set()
|
| 526 |
+
for member in sorted(members, key=lambda m: m.header_offset):
|
| 527 |
+
if member.is_dir:
|
| 528 |
+
plan.skipped["directory"] += 1
|
| 529 |
+
continue
|
| 530 |
+
parts = member.name.split("/")
|
| 531 |
+
if len(parts) != 2 or parts[0] != speaker:
|
| 532 |
+
plan.skipped["outside_speaker_folder"] += 1
|
| 533 |
+
continue
|
| 534 |
+
if not parts[1].endswith(".wav"):
|
| 535 |
+
plan.skipped["not_wav"] += 1
|
| 536 |
+
continue
|
| 537 |
+
stem = parts[1][: -len(".wav")]
|
| 538 |
+
decision = classify_stem(stem, cfg)
|
| 539 |
+
if isinstance(decision, Skip):
|
| 540 |
+
plan.skipped[decision.reason] += 1
|
| 541 |
+
continue
|
| 542 |
+
item_id = f"{speaker}/{stem}"
|
| 543 |
+
if stem in seen:
|
| 544 |
+
plan.quarantine.append(
|
| 545 |
+
QuarantineRecord(item_id, STAGE, "duplicate_member", {"name": member.name})
|
| 546 |
+
)
|
| 547 |
+
continue
|
| 548 |
+
seen.add(stem)
|
| 549 |
+
try:
|
| 550 |
+
text = build_text(decision, transcripts.get(stem), cfg)
|
| 551 |
+
except TextRejected as err:
|
| 552 |
+
plan.quarantine.append(
|
| 553 |
+
QuarantineRecord(item_id, STAGE, err.reason, {"detail": err.detail})
|
| 554 |
+
)
|
| 555 |
+
continue
|
| 556 |
+
plan.items.append(PlannedItem(stem, member, decision, text))
|
| 557 |
+
_cap_reads(plan, speaker, cfg.max_read_per_speaker)
|
| 558 |
+
return plan
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def _cap_reads(plan: SelectionPlan, speaker: str, cap: int) -> None:
|
| 562 |
+
"""Keep at most ``cap`` plain reading items per speaker (0 = no limit; ADR-0007: plain
|
| 563 |
+
reading is a low share). The kept ones are a deterministic hash choice of the stems;
|
| 564 |
+
the rest are counted as skipped ``read_cap``."""
|
| 565 |
+
reads = [item for item in plan.items if item.selection.kind == "read"]
|
| 566 |
+
if cap == 0 or len(reads) <= cap:
|
| 567 |
+
return
|
| 568 |
+
ranked = sorted(reads, key=lambda it: hashlib.sha1(f"{speaker}/{it.stem}".encode()).hexdigest())
|
| 569 |
+
dropped = {id(item) for item in ranked[cap:]}
|
| 570 |
+
plan.items = [item for item in plan.items if id(item) not in dropped]
|
| 571 |
+
plan.skipped["read_cap"] += len(dropped)
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
# --------------------------------------------------------------------------------------
|
| 575 |
+
# Audio
|
| 576 |
+
# --------------------------------------------------------------------------------------
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
class ItemRejected(Exception):
|
| 580 |
+
"""An audio file cannot be used; ``reason`` is the quarantine reason."""
|
| 581 |
+
|
| 582 |
+
def __init__(self, reason: str, **details: object) -> None:
|
| 583 |
+
super().__init__(f"{reason}: {details}")
|
| 584 |
+
self.reason = reason
|
| 585 |
+
self.details = details
|
| 586 |
+
|
| 587 |
+
|
| 588 |
+
@dataclass(frozen=True)
|
| 589 |
+
class AudioFacts:
|
| 590 |
+
"""Measurements of a converted file: ``duration_ms`` of the 16 kHz audio, RMS level."""
|
| 591 |
+
|
| 592 |
+
duration_ms: int
|
| 593 |
+
loudness_dbfs: float
|
| 594 |
+
|
| 595 |
+
|
| 596 |
+
def resample(audio: FloatArray, rate_hz: int, target_hz: int) -> FloatArray:
|
| 597 |
+
"""Polyphase resampling of a mono signal (anti-aliased) from ``rate_hz`` to ``target_hz``.
|
| 598 |
+
|
| 599 |
+
Args:
|
| 600 |
+
audio: ``(num_samples,)`` float signal.
|
| 601 |
+
rate_hz: Source rate.
|
| 602 |
+
target_hz: Target rate.
|
| 603 |
+
|
| 604 |
+
Returns:
|
| 605 |
+
``(ceil(num_samples * target_hz / rate_hz),)`` float64 signal.
|
| 606 |
+
"""
|
| 607 |
+
if rate_hz == target_hz:
|
| 608 |
+
return np.asarray(audio, dtype=np.float64)
|
| 609 |
+
divisor = math.gcd(rate_hz, target_hz)
|
| 610 |
+
out = signal.resample_poly(audio, target_hz // divisor, rate_hz // divisor)
|
| 611 |
+
return np.asarray(out, dtype=np.float64)
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def convert_to_target_rate(raw: Path, dest: Path, cfg: EarsConfig) -> None:
|
| 615 |
+
"""Read a raw EARS wav, check it, resample to ``target_sample_rate_hz`` mono and write a
|
| 616 |
+
32-bit float wav atomically to ``dest``.
|
| 617 |
+
|
| 618 |
+
Raises:
|
| 619 |
+
ItemRejected: ``unreadable_audio``, ``not_mono``, ``unexpected_sample_rate``,
|
| 620 |
+
``empty_audio`` or ``non_finite_audio``.
|
| 621 |
+
"""
|
| 622 |
+
import soundfile # type: ignore[import-untyped] # bench extra, untyped
|
| 623 |
+
|
| 624 |
+
try:
|
| 625 |
+
audio, rate_hz = soundfile.read(str(raw), dtype="float64", always_2d=True)
|
| 626 |
+
except (RuntimeError, soundfile.LibsndfileError) as err:
|
| 627 |
+
raise ItemRejected("unreadable_audio", error=str(err)) from err
|
| 628 |
+
if audio.shape[1] != 1:
|
| 629 |
+
raise ItemRejected("not_mono", channels=int(audio.shape[1]))
|
| 630 |
+
if rate_hz != cfg.expected_sample_rate_hz:
|
| 631 |
+
raise ItemRejected("unexpected_sample_rate", sample_rate_hz=int(rate_hz))
|
| 632 |
+
mono = np.ascontiguousarray(audio[:, 0])
|
| 633 |
+
if mono.shape[0] == 0:
|
| 634 |
+
raise ItemRejected("empty_audio")
|
| 635 |
+
if not np.all(np.isfinite(mono)):
|
| 636 |
+
raise ItemRejected("non_finite_audio")
|
| 637 |
+
converted = resample(mono, int(rate_hz), cfg.target_sample_rate_hz)
|
| 638 |
+
buffer = io.BytesIO()
|
| 639 |
+
soundfile.write(buffer, converted, cfg.target_sample_rate_hz, format="WAV", subtype="FLOAT")
|
| 640 |
+
atomic_write_bytes(dest, buffer.getvalue())
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
def measure_audio(
|
| 644 |
+
path: Path, cfg: EarsConfig, min_duration_ms: int, max_duration_ms: int
|
| 645 |
+
) -> AudioFacts:
|
| 646 |
+
"""Duration and loudness of a converted wav; enforces the duration limits.
|
| 647 |
+
|
| 648 |
+
The loudness is :func:`nida.data.acoustic.loudness_dbfs`, the measure the pilot ingest
|
| 649 |
+
stores, here on the 16 kHz signal the VAE hears.
|
| 650 |
+
|
| 651 |
+
Raises:
|
| 652 |
+
ItemRejected: ``unreadable_audio``, ``too_short``, ``too_long`` or ``silent``.
|
| 653 |
+
ValueError: if the file is not at ``target_sample_rate_hz`` (the cache is inconsistent
|
| 654 |
+
with the config).
|
| 655 |
+
"""
|
| 656 |
+
import soundfile
|
| 657 |
+
|
| 658 |
+
try:
|
| 659 |
+
audio, rate_hz = soundfile.read(str(path), dtype="float64", always_2d=True)
|
| 660 |
+
except (RuntimeError, soundfile.LibsndfileError) as err:
|
| 661 |
+
raise ItemRejected("unreadable_audio", error=str(err)) from err
|
| 662 |
+
if rate_hz != cfg.target_sample_rate_hz or audio.shape[1] != 1:
|
| 663 |
+
raise ValueError(
|
| 664 |
+
f"{path}: expected {cfg.target_sample_rate_hz} Hz mono, got {rate_hz} Hz x "
|
| 665 |
+
f"{audio.shape[1]}; delete the cache folder"
|
| 666 |
+
)
|
| 667 |
+
mono = np.ascontiguousarray(audio[:, 0])
|
| 668 |
+
duration_ms = round(1000 * mono.shape[0] / rate_hz)
|
| 669 |
+
if duration_ms < min_duration_ms:
|
| 670 |
+
raise ItemRejected("too_short", duration_ms=duration_ms)
|
| 671 |
+
if duration_ms > max_duration_ms:
|
| 672 |
+
raise ItemRejected("too_long", duration_ms=duration_ms)
|
| 673 |
+
if float(np.max(np.abs(mono))) == 0.0:
|
| 674 |
+
raise ItemRejected("silent")
|
| 675 |
+
return AudioFacts(duration_ms=duration_ms, loudness_dbfs=float(acoustic.loudness_dbfs(mono)))
|
| 676 |
+
|
| 677 |
+
|
| 678 |
+
# --------------------------------------------------------------------------------------
|
| 679 |
+
# Buckets and splits
|
| 680 |
+
# --------------------------------------------------------------------------------------
|
| 681 |
+
|
| 682 |
+
|
| 683 |
+
def bucket_of(value: float, edges: Sequence[float]) -> int:
|
| 684 |
+
"""Bucket index of ``value``: the number of ``edges`` that are ``<= value``
|
| 685 |
+
(``numpy.searchsorted(edges, value, side="right")``, as the pilot dataset stage)."""
|
| 686 |
+
return bisect_right(edges, value)
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
def rate_chars_per_s(plain_text: str, duration_ms: int) -> float:
|
| 690 |
+
"""Alphanumeric characters of the plain text per second of audio (pilot formula)."""
|
| 691 |
+
return sum(ch.isalnum() for ch in plain_text) / (duration_ms / 1000)
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
def speaker_split(group: str, val_fraction: float, test_fraction: float, seed: int) -> str:
|
| 695 |
+
"""Split of a speaker group: a pure function of ``(seed, group)`` (:mod:`nida.data.splits`)."""
|
| 696 |
+
fractions = {
|
| 697 |
+
"train": 1.0 - val_fraction - test_fraction,
|
| 698 |
+
"val": val_fraction,
|
| 699 |
+
"test": test_fraction,
|
| 700 |
+
}
|
| 701 |
+
return assign_splits({"speaker": group}, fractions, str(seed))["speaker"]
|
| 702 |
+
|
| 703 |
+
|
| 704 |
+
# --------------------------------------------------------------------------------------
|
| 705 |
+
# The builder
|
| 706 |
+
# --------------------------------------------------------------------------------------
|
| 707 |
+
|
| 708 |
+
|
| 709 |
+
@dataclass(frozen=True)
|
| 710 |
+
class _Measured:
|
| 711 |
+
item: PlannedItem
|
| 712 |
+
segment_id: str
|
| 713 |
+
wav: Path
|
| 714 |
+
facts: AudioFacts
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
@dataclass
|
| 718 |
+
class _Prepared:
|
| 719 |
+
"""A speaker whose audio is ready for the VAE: nothing is written for it yet."""
|
| 720 |
+
|
| 721 |
+
plan: SelectionPlan
|
| 722 |
+
quarantine: list[QuarantineRecord]
|
| 723 |
+
measured: list[_Measured]
|
| 724 |
+
|
| 725 |
+
|
| 726 |
+
@dataclass
|
| 727 |
+
class Ears:
|
| 728 |
+
"""EARS -> latent dataset builder (see the module docstring).
|
| 729 |
+
|
| 730 |
+
Attributes:
|
| 731 |
+
root: Durable directory (Drive on Colab).
|
| 732 |
+
cache: Local scratch for raw members, 16 kHz audio and latent arrays.
|
| 733 |
+
envs_root: Where the worker virtual environments live.
|
| 734 |
+
repo_dir: Repository root (environment builds need it).
|
| 735 |
+
sources_path: ``configs/sources.yaml``.
|
| 736 |
+
config_dir: ``configs/data``.
|
| 737 |
+
seed: Split seed (recorded in ``plan.json``).
|
| 738 |
+
runner: Worker command runner (injectable).
|
| 739 |
+
opener: Network access (injectable).
|
| 740 |
+
sleep: Backoff sleep for retries (injectable).
|
| 741 |
+
"""
|
| 742 |
+
|
| 743 |
+
root: Path
|
| 744 |
+
cache: Path
|
| 745 |
+
envs_root: Path
|
| 746 |
+
repo_dir: Path
|
| 747 |
+
sources_path: Path
|
| 748 |
+
config_dir: Path
|
| 749 |
+
seed: int = DEFAULT_SEED
|
| 750 |
+
uv: str | None = None
|
| 751 |
+
runner: Callable[[Sequence[str]], int] = _stream_runner
|
| 752 |
+
opener: fetch.Opener = fetch.default_opener
|
| 753 |
+
sleep: Callable[[float], None] = time.sleep
|
| 754 |
+
_python: Path | None = field(default=None, init=False, repr=False)
|
| 755 |
+
|
| 756 |
+
# ---------------------------------------------------------------- paths
|
| 757 |
+
|
| 758 |
+
@property
|
| 759 |
+
def dataset_dir(self) -> Path:
|
| 760 |
+
return self.root / "dataset"
|
| 761 |
+
|
| 762 |
+
def _speaker_dir(self, speaker: str) -> Path:
|
| 763 |
+
return self.root / "speakers" / speaker
|
| 764 |
+
|
| 765 |
+
def _npy(self, segment: str) -> Path:
|
| 766 |
+
return self.cache / "arrays" / "vae" / f"{segment}.npy"
|
| 767 |
+
|
| 768 |
+
# ---------------------------------------------------------------- run
|
| 769 |
+
|
| 770 |
+
def run(self, speakers: Sequence[str]) -> dict[str, object]:
|
| 771 |
+
"""Build (or continue) the dataset for ``speakers``.
|
| 772 |
+
|
| 773 |
+
Finished speakers are skipped. The dataset directory, ``report.md`` and
|
| 774 |
+
``stats.json`` are rewritten from *all* finished speakers of the root at the end.
|
| 775 |
+
|
| 776 |
+
Returns:
|
| 777 |
+
Summary with ``dataset``, ``speakers_done``, ``segments``, ``hours``,
|
| 778 |
+
``skipped`` and ``quarantined`` counts, and ``failed_speakers``.
|
| 779 |
+
|
| 780 |
+
Raises:
|
| 781 |
+
SourceNotApprovedError: before any download, if a source is not approved.
|
| 782 |
+
ValueError: on a plan/config/transcripts mismatch with the root.
|
| 783 |
+
RuntimeError: if a worker fails, a speaker could not be fetched (after all other
|
| 784 |
+
speakers were processed and the dataset was assembled), or nothing was built.
|
| 785 |
+
"""
|
| 786 |
+
settings = load_settings(self.config_dir)
|
| 787 |
+
model = DATA_MODELS[VAE_KEY]
|
| 788 |
+
require_approved({SOURCE_ID, model.source_id}, load_sources(self.sources_path))
|
| 789 |
+
wanted = list(speakers)
|
| 790 |
+
for speaker in wanted:
|
| 791 |
+
parse_speakers(speaker, settings.ears.n_speakers)
|
| 792 |
+
transcripts, transcripts_sha = self._prepare_plan(settings)
|
| 793 |
+
failed: dict[str, str] = {}
|
| 794 |
+
pending: list[str] = []
|
| 795 |
+
for speaker in dict.fromkeys(wanted):
|
| 796 |
+
if (self._speaker_dir(speaker) / "done.json").is_file():
|
| 797 |
+
log.info("EARS %s already done", speaker)
|
| 798 |
+
else:
|
| 799 |
+
pending.append(speaker)
|
| 800 |
+
wave_size = settings.ears.speakers_per_wave
|
| 801 |
+
for start in range(0, len(pending), wave_size):
|
| 802 |
+
wave = pending[start : start + wave_size]
|
| 803 |
+
prepared: dict[str, _Prepared] = {}
|
| 804 |
+
for speaker in wave:
|
| 805 |
+
log.info("EARS %s (%d/%d)", speaker, pending.index(speaker) + 1, len(pending))
|
| 806 |
+
try:
|
| 807 |
+
prepared[speaker] = self._prepare_speaker(speaker, settings, transcripts)
|
| 808 |
+
except fetch.FetchError as err: # network or archive problem: others still run
|
| 809 |
+
log.error("EARS %s failed: %s", speaker, err)
|
| 810 |
+
failed[speaker] = str(err)
|
| 811 |
+
results = self._encode([m for entry in prepared.values() for m in entry.measured])
|
| 812 |
+
for speaker, entry in prepared.items():
|
| 813 |
+
self._finish_speaker(speaker, settings, entry, results)
|
| 814 |
+
summary = self._assemble(settings, transcripts_sha, failed)
|
| 815 |
+
if failed:
|
| 816 |
+
raise RuntimeError(
|
| 817 |
+
f"{len(failed)} speaker(s) failed and can be retried by running again: "
|
| 818 |
+
+ "; ".join(f"{s}: {e[:200]}" for s, e in sorted(failed.items()))
|
| 819 |
+
)
|
| 820 |
+
return summary
|
| 821 |
+
|
| 822 |
+
# ---------------------------------------------------------------- plan and transcripts
|
| 823 |
+
|
| 824 |
+
def _prepare_plan(self, settings: EarsSettings) -> tuple[dict[str, str], str]:
|
| 825 |
+
"""Check/write ``plan.json`` and fetch the pinned transcripts."""
|
| 826 |
+
model = DATA_MODELS[VAE_KEY]
|
| 827 |
+
core: dict[str, object] = {
|
| 828 |
+
"seed": self.seed,
|
| 829 |
+
"config_digest": settings.digest,
|
| 830 |
+
"archive_url_template": settings.ears.archive_url_template,
|
| 831 |
+
"transcripts_url": settings.ears.transcripts_url,
|
| 832 |
+
"vae": [model.ref.model_id, model.ref.revision],
|
| 833 |
+
}
|
| 834 |
+
plan_path = self.root / "plan.json"
|
| 835 |
+
previous: dict[str, object] | None = None
|
| 836 |
+
if plan_path.is_file():
|
| 837 |
+
loaded = json.loads(plan_path.read_text(encoding="utf-8"))
|
| 838 |
+
previous = dict(loaded)
|
| 839 |
+
recorded_sha = previous.pop("transcripts_sha256", None)
|
| 840 |
+
if previous != core:
|
| 841 |
+
raise ValueError(
|
| 842 |
+
f"{plan_path} was written for a different plan: {previous} != {core}"
|
| 843 |
+
)
|
| 844 |
+
if (
|
| 845 |
+
settings.ears.transcripts_sha256 is not None
|
| 846 |
+
and recorded_sha is not None
|
| 847 |
+
and settings.ears.transcripts_sha256 != recorded_sha
|
| 848 |
+
):
|
| 849 |
+
raise ValueError(
|
| 850 |
+
f"config pins transcripts {settings.ears.transcripts_sha256} but the root "
|
| 851 |
+
f"was built with {recorded_sha}"
|
| 852 |
+
)
|
| 853 |
+
expected = recorded_sha if isinstance(recorded_sha, str) else None
|
| 854 |
+
else:
|
| 855 |
+
expected = None
|
| 856 |
+
expected = settings.ears.transcripts_sha256 or expected
|
| 857 |
+
downloaded = fetch.download(
|
| 858 |
+
settings.ears.transcripts_url,
|
| 859 |
+
self.root / "source" / "transcripts.json",
|
| 860 |
+
expected_sha256=expected,
|
| 861 |
+
opener=self.opener,
|
| 862 |
+
sleep=self.sleep,
|
| 863 |
+
)
|
| 864 |
+
if previous is None:
|
| 865 |
+
atomic_write_json(plan_path, {**core, "transcripts_sha256": downloaded.sha256})
|
| 866 |
+
return _parse_transcripts(downloaded.path), downloaded.sha256
|
| 867 |
+
|
| 868 |
+
# ---------------------------------------------------------------- speaker
|
| 869 |
+
|
| 870 |
+
def _python_exe(self) -> Path:
|
| 871 |
+
if self._python is None:
|
| 872 |
+
if self.uv is None:
|
| 873 |
+
self.uv = bench_envs.find_uv(bench_envs.subprocess_runner)
|
| 874 |
+
spec = envs.data_env_for_worker(DATA_MODELS[VAE_KEY].worker)
|
| 875 |
+
self._python = bench_envs.ensure_env(spec, self.envs_root, self.repo_dir, uv=self.uv)
|
| 876 |
+
return self._python
|
| 877 |
+
|
| 878 |
+
def _prepare_speaker(
|
| 879 |
+
self, speaker: str, settings: EarsSettings, transcripts: Mapping[str, str]
|
| 880 |
+
) -> _Prepared:
|
| 881 |
+
"""List the archive, decide on every member, fetch + convert + measure the selected."""
|
| 882 |
+
cfg = settings.ears
|
| 883 |
+
QuarantineLog(self._speaker_dir(speaker)).path.unlink(missing_ok=True) # rebuilt below
|
| 884 |
+
archive = RemoteZip(cfg.archive_url(speaker), opener=self.opener, sleep=self.sleep)
|
| 885 |
+
members = archive.members()
|
| 886 |
+
plan = select_members(speaker, members, transcripts, cfg)
|
| 887 |
+
quarantine = list(plan.quarantine)
|
| 888 |
+
log.info(
|
| 889 |
+
"EARS %s: %d members, %d selected, %d quarantined, skipped %s",
|
| 890 |
+
speaker,
|
| 891 |
+
len(members),
|
| 892 |
+
len(plan.items),
|
| 893 |
+
len(quarantine),
|
| 894 |
+
dict(plan.skipped),
|
| 895 |
+
)
|
| 896 |
+
measured: list[_Measured] = []
|
| 897 |
+
for item in plan.items: # archive order: one read-ahead window serves neighbours
|
| 898 |
+
item_id = f"{speaker}/{item.stem}"
|
| 899 |
+
wav = self.cache / "audio16k" / speaker / f"{item.stem}.wav"
|
| 900 |
+
try:
|
| 901 |
+
if not wav.is_file():
|
| 902 |
+
raw = archive.extract(
|
| 903 |
+
item.member, self.cache / "raw" / speaker / f"{item.stem}.wav"
|
| 904 |
+
)
|
| 905 |
+
convert_to_target_rate(raw, wav, cfg)
|
| 906 |
+
raw.unlink()
|
| 907 |
+
facts = measure_audio(wav, cfg, settings.min_duration_ms, settings.max_duration_ms)
|
| 908 |
+
except ItemRejected as err:
|
| 909 |
+
quarantine.append(QuarantineRecord(item_id, STAGE, err.reason, dict(err.details)))
|
| 910 |
+
continue
|
| 911 |
+
sid = segment_id(SOURCE_ID, item_id, 0, facts.duration_ms)
|
| 912 |
+
measured.append(_Measured(item, sid, wav, facts))
|
| 913 |
+
return _Prepared(plan, quarantine, measured)
|
| 914 |
+
|
| 915 |
+
def _finish_speaker(
|
| 916 |
+
self,
|
| 917 |
+
speaker: str,
|
| 918 |
+
settings: EarsSettings,
|
| 919 |
+
prepared: _Prepared,
|
| 920 |
+
results: Mapping[str, WorkResult],
|
| 921 |
+
) -> None:
|
| 922 |
+
"""Write the speaker's outputs, then drop its scratch files."""
|
| 923 |
+
self._write_speaker(
|
| 924 |
+
speaker, settings, prepared.measured, results, prepared.plan, prepared.quarantine
|
| 925 |
+
)
|
| 926 |
+
for sub in ("raw", "audio16k"):
|
| 927 |
+
shutil.rmtree(self.cache / sub / speaker, ignore_errors=True)
|
| 928 |
+
for entry in prepared.measured:
|
| 929 |
+
self._npy(entry.segment_id).unlink(missing_ok=True)
|
| 930 |
+
|
| 931 |
+
def _encode(self, measured: Sequence[_Measured]) -> dict[str, WorkResult]:
|
| 932 |
+
"""Run the pilot's ``vae_encode`` worker once over the audio of a wave (resumable)."""
|
| 933 |
+
if not measured:
|
| 934 |
+
return {}
|
| 935 |
+
model = DATA_MODELS[VAE_KEY]
|
| 936 |
+
python = self._python_exe()
|
| 937 |
+
out_dir = self.root / "workers" / "vae"
|
| 938 |
+
results_path = out_dir / "results.jsonl"
|
| 939 |
+
wanted = {m.segment_id for m in measured}
|
| 940 |
+
summary_path = out_dir / "results.summary.json"
|
| 941 |
+
if results_path.exists() and summary_path.exists():
|
| 942 |
+
previous = read_summary(summary_path)
|
| 943 |
+
if (previous.model_id, previous.revision) != (model.ref.model_id, model.ref.revision):
|
| 944 |
+
raise RuntimeError(
|
| 945 |
+
f"existing latents come from {previous.model_id}@"
|
| 946 |
+
f"{previous.revision}, not {model.ref.model_id}@{model.ref.revision}; "
|
| 947 |
+
"move them away first"
|
| 948 |
+
)
|
| 949 |
+
if results_path.exists():
|
| 950 |
+
existing = load_results(results_path, repair=True)
|
| 951 |
+
stale = [
|
| 952 |
+
cid
|
| 953 |
+
for cid, res in existing.items()
|
| 954 |
+
if cid not in wanted
|
| 955 |
+
or res.output is None # failed before: try again (a stuck item fails again)
|
| 956 |
+
or not Path(str(res.output.get("out_path", ""))).is_file()
|
| 957 |
+
]
|
| 958 |
+
if stale:
|
| 959 |
+
log.info("dropping %d stray, failed or lost worker results", len(stale))
|
| 960 |
+
drop_results(results_path, stale)
|
| 961 |
+
requests = [
|
| 962 |
+
WorkRequest(m.segment_id, str(m.wav), {"out_path": str(self._npy(m.segment_id))})
|
| 963 |
+
for m in measured
|
| 964 |
+
]
|
| 965 |
+
write_requests(out_dir / "requests.jsonl", requests)
|
| 966 |
+
command = [
|
| 967 |
+
str(python),
|
| 968 |
+
"-m",
|
| 969 |
+
envs.worker_module_name(model.worker),
|
| 970 |
+
"--requests",
|
| 971 |
+
str(out_dir / "requests.jsonl"),
|
| 972 |
+
"--results",
|
| 973 |
+
str(results_path),
|
| 974 |
+
"--model-id",
|
| 975 |
+
model.ref.model_id,
|
| 976 |
+
"--revision",
|
| 977 |
+
model.ref.revision,
|
| 978 |
+
"--batch-size",
|
| 979 |
+
str(model.batch_size),
|
| 980 |
+
"--device",
|
| 981 |
+
"cuda",
|
| 982 |
+
]
|
| 983 |
+
code = self.runner(command)
|
| 984 |
+
if code != 0:
|
| 985 |
+
raise RuntimeError(f"vae_encode worker exited with code {code}")
|
| 986 |
+
summary = read_summary(summary_path)
|
| 987 |
+
total = summary.n_ok + summary.n_error
|
| 988 |
+
if total != len(requests):
|
| 989 |
+
raise RuntimeError(f"{total} latent results for {len(requests)} requests")
|
| 990 |
+
if summary.n_error > MAX_ERROR_FRACTION * total:
|
| 991 |
+
raise RuntimeError(f"{summary.n_error}/{total} items failed in the worker")
|
| 992 |
+
return load_results(results_path)
|
| 993 |
+
|
| 994 |
+
def _write_speaker(
|
| 995 |
+
self,
|
| 996 |
+
speaker: str,
|
| 997 |
+
settings: EarsSettings,
|
| 998 |
+
measured: Sequence[_Measured],
|
| 999 |
+
results: Mapping[str, WorkResult],
|
| 1000 |
+
plan: SelectionPlan,
|
| 1001 |
+
quarantine: list[QuarantineRecord],
|
| 1002 |
+
) -> dict[str, object]:
|
| 1003 |
+
"""Shard, records, quarantine log and finally the done marker of one speaker."""
|
| 1004 |
+
cfg = settings.ears
|
| 1005 |
+
group = f"ears-{speaker}"
|
| 1006 |
+
split = speaker_split(group, cfg.val_speaker_fraction, cfg.test_speaker_fraction, self.seed)
|
| 1007 |
+
rate_edges = list(settings.dataset.rate_bucket_edges_chars_per_s)
|
| 1008 |
+
loud_edges = list(settings.dataset.loudness_bucket_edges_dbfs)
|
| 1009 |
+
shard_name = f"shard-{speaker}.npy"
|
| 1010 |
+
rows: list[dict[str, object]] = []
|
| 1011 |
+
arrays: list[npt.NDArray[np.float32]] = []
|
| 1012 |
+
kinds: dict[str, dict[str, int]] = {}
|
| 1013 |
+
latent_dim: int | None = None
|
| 1014 |
+
frame_hop_ms: float | None = None
|
| 1015 |
+
frames = 0
|
| 1016 |
+
for entry in sorted(measured, key=lambda m: m.item.stem):
|
| 1017 |
+
item_id = f"{speaker}/{entry.item.stem}"
|
| 1018 |
+
result = results.get(entry.segment_id)
|
| 1019 |
+
if result is None:
|
| 1020 |
+
raise RuntimeError(f"{speaker}: the worker returned no result for {item_id}")
|
| 1021 |
+
if result.output is None:
|
| 1022 |
+
quarantine.append(
|
| 1023 |
+
QuarantineRecord(item_id, STAGE, "vae_error", {"error": str(result.error)})
|
| 1024 |
+
)
|
| 1025 |
+
continue
|
| 1026 |
+
array = np.load(str(result.output["out_path"]), allow_pickle=False)
|
| 1027 |
+
hop = float(str(result.output["frame_hop_ms"]))
|
| 1028 |
+
if array.ndim != 2 or array.shape[0] < 1:
|
| 1029 |
+
raise ValueError(f"{item_id}: latents must be 2-D with frames, got {array.shape}")
|
| 1030 |
+
if frame_hop_ms is not None and hop != frame_hop_ms:
|
| 1031 |
+
raise ValueError(f"{item_id}: frame hop {hop} != {frame_hop_ms}")
|
| 1032 |
+
if latent_dim is not None and int(array.shape[1]) != latent_dim:
|
| 1033 |
+
raise ValueError(f"{item_id}: latent_dim {array.shape[1]} != {latent_dim}")
|
| 1034 |
+
frame_hop_ms, latent_dim = hop, int(array.shape[1])
|
| 1035 |
+
if not np.all(np.isfinite(array)) or float(np.max(np.abs(array))) > FLOAT16_MAX_ABS:
|
| 1036 |
+
quarantine.append(QuarantineRecord(item_id, STAGE, "latents_not_float16", {}))
|
| 1037 |
+
continue
|
| 1038 |
+
num_frames = int(array.shape[0])
|
| 1039 |
+
n_tokens = len(encode_text(entry.item.text.tagged))
|
| 1040 |
+
if n_tokens > num_frames:
|
| 1041 |
+
quarantine.append(
|
| 1042 |
+
QuarantineRecord(
|
| 1043 |
+
item_id,
|
| 1044 |
+
STAGE,
|
| 1045 |
+
REJECT_TEXT_TOO_LONG,
|
| 1046 |
+
{"text_tokens": n_tokens, "num_frames": num_frames},
|
| 1047 |
+
)
|
| 1048 |
+
)
|
| 1049 |
+
continue
|
| 1050 |
+
plain = entry.item.text.plain
|
| 1051 |
+
rows.append(
|
| 1052 |
+
{
|
| 1053 |
+
"segment_id": entry.segment_id,
|
| 1054 |
+
"shard": shard_name,
|
| 1055 |
+
"offset": frames,
|
| 1056 |
+
"num_frames": num_frames,
|
| 1057 |
+
"text": entry.item.text.tagged,
|
| 1058 |
+
"emotion": entry.item.selection.emotion,
|
| 1059 |
+
"intensity": None,
|
| 1060 |
+
"rate_bucket": (
|
| 1061 |
+
bucket_of(rate_chars_per_s(plain, entry.facts.duration_ms), rate_edges)
|
| 1062 |
+
if plain
|
| 1063 |
+
else None
|
| 1064 |
+
),
|
| 1065 |
+
"loudness_bucket": bucket_of(entry.facts.loudness_dbfs, loud_edges),
|
| 1066 |
+
"group": group,
|
| 1067 |
+
"split": split,
|
| 1068 |
+
"language": LANGUAGE,
|
| 1069 |
+
}
|
| 1070 |
+
)
|
| 1071 |
+
arrays.append(array.astype(np.float32))
|
| 1072 |
+
frames += num_frames
|
| 1073 |
+
stat = kinds.setdefault(entry.item.selection.label, {"segments": 0, "audio_ms": 0})
|
| 1074 |
+
stat["segments"] += 1
|
| 1075 |
+
stat["audio_ms"] += entry.facts.duration_ms
|
| 1076 |
+
|
| 1077 |
+
spk_dir = self._speaker_dir(speaker)
|
| 1078 |
+
if arrays:
|
| 1079 |
+
buffer = io.BytesIO()
|
| 1080 |
+
np.save(buffer, np.concatenate(arrays).astype(np.float16), allow_pickle=False)
|
| 1081 |
+
atomic_write_bytes(self.dataset_dir / shard_name, buffer.getvalue())
|
| 1082 |
+
atomic_write_text(
|
| 1083 |
+
spk_dir / "records.jsonl",
|
| 1084 |
+
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
|
| 1085 |
+
)
|
| 1086 |
+
QuarantineLog(spk_dir).append(quarantine)
|
| 1087 |
+
marker: dict[str, object] = {
|
| 1088 |
+
"speaker": speaker,
|
| 1089 |
+
"group": group,
|
| 1090 |
+
"split": split,
|
| 1091 |
+
"segments": len(rows),
|
| 1092 |
+
"frames": frames,
|
| 1093 |
+
"audio_ms": sum(s["audio_ms"] for s in kinds.values()),
|
| 1094 |
+
"latent_dim": latent_dim if rows else None,
|
| 1095 |
+
"frame_hop_ms": frame_hop_ms if rows else None,
|
| 1096 |
+
"shard": shard_name if rows else None,
|
| 1097 |
+
"kinds": kinds,
|
| 1098 |
+
"skipped": dict(sorted(plan.skipped.items())),
|
| 1099 |
+
"quarantined": dict(sorted(Counter(q.reason for q in quarantine).items())),
|
| 1100 |
+
}
|
| 1101 |
+
atomic_write_json(spk_dir / "done.json", marker) # written last: the speaker is done
|
| 1102 |
+
return marker
|
| 1103 |
+
|
| 1104 |
+
# ---------------------------------------------------------------- assembly
|
| 1105 |
+
|
| 1106 |
+
def _assemble(
|
| 1107 |
+
self, settings: EarsSettings, transcripts_sha: str, failed: Mapping[str, str]
|
| 1108 |
+
) -> dict[str, object]:
|
| 1109 |
+
"""Index, meta, report and stats from every finished speaker of the root."""
|
| 1110 |
+
markers: list[dict[str, object]] = []
|
| 1111 |
+
speakers_dir = self.root / "speakers"
|
| 1112 |
+
for done_path in sorted(speakers_dir.glob("p*/done.json")) if speakers_dir.is_dir() else []:
|
| 1113 |
+
markers.append(json.loads(done_path.read_text(encoding="utf-8")))
|
| 1114 |
+
with_data = [m for m in markers if m["segments"]]
|
| 1115 |
+
if not with_data:
|
| 1116 |
+
raise RuntimeError(
|
| 1117 |
+
f"no EARS segment built (speakers done: {len(markers)}, failed: {sorted(failed)})"
|
| 1118 |
+
)
|
| 1119 |
+
hops = {float(str(m["frame_hop_ms"])) for m in with_data}
|
| 1120 |
+
dims = {int(str(m["latent_dim"])) for m in with_data}
|
| 1121 |
+
if len(hops) != 1 or len(dims) != 1:
|
| 1122 |
+
raise ValueError(
|
| 1123 |
+
f"speakers disagree on frame hop {sorted(hops)} or latent_dim {sorted(dims)}"
|
| 1124 |
+
)
|
| 1125 |
+
index_lines: list[str] = []
|
| 1126 |
+
for marker in with_data:
|
| 1127 |
+
lines = (
|
| 1128 |
+
(self._speaker_dir(str(marker["speaker"])) / "records.jsonl")
|
| 1129 |
+
.read_text(encoding="utf-8")
|
| 1130 |
+
.splitlines(keepends=True)
|
| 1131 |
+
)
|
| 1132 |
+
if len(lines) != marker["segments"]:
|
| 1133 |
+
raise ValueError(f"{marker['speaker']}: records.jsonl and done.json disagree")
|
| 1134 |
+
if not (self.dataset_dir / str(marker["shard"])).is_file():
|
| 1135 |
+
raise ValueError(f"{marker['speaker']}: shard {marker['shard']} is missing")
|
| 1136 |
+
index_lines.extend(lines)
|
| 1137 |
+
total_segments = len(index_lines)
|
| 1138 |
+
total_frames = sum(int(str(m["frames"])) for m in with_data)
|
| 1139 |
+
atomic_write_text(self.dataset_dir / "index.jsonl", "".join(index_lines))
|
| 1140 |
+
source = load_sources(self.sources_path)[SOURCE_ID]
|
| 1141 |
+
model = DATA_MODELS[VAE_KEY]
|
| 1142 |
+
meta: dict[str, object] = {
|
| 1143 |
+
"segments": total_segments,
|
| 1144 |
+
"frames": total_frames,
|
| 1145 |
+
"latent_dim": dims.pop(),
|
| 1146 |
+
"frame_hop_ms": hops.pop(),
|
| 1147 |
+
"rate_bucket_edges_chars_per_s": list(settings.dataset.rate_bucket_edges_chars_per_s),
|
| 1148 |
+
"loudness_bucket_edges_dbfs": list(settings.dataset.loudness_bucket_edges_dbfs),
|
| 1149 |
+
"source": SOURCE_ID,
|
| 1150 |
+
"license": source.get("license"),
|
| 1151 |
+
"language": LANGUAGE,
|
| 1152 |
+
"transcripts_sha256": transcripts_sha,
|
| 1153 |
+
"transcripts_url": settings.ears.transcripts_url,
|
| 1154 |
+
"config_digest": settings.digest,
|
| 1155 |
+
"seed": self.seed,
|
| 1156 |
+
"vae": [model.ref.model_id, model.ref.revision],
|
| 1157 |
+
"speakers": {
|
| 1158 |
+
str(m["speaker"]): {"split": m["split"], "segments": m["segments"]} for m in markers
|
| 1159 |
+
},
|
| 1160 |
+
}
|
| 1161 |
+
atomic_write_json(self.dataset_dir / "meta.json", meta)
|
| 1162 |
+
|
| 1163 |
+
loader = self._check_with_loader()
|
| 1164 |
+
stats = _stats(markers, loader, settings, failed, meta)
|
| 1165 |
+
atomic_write_json(self.root / "stats.json", stats)
|
| 1166 |
+
atomic_write_text(self.root / "report.md", _render_report(stats))
|
| 1167 |
+
rejected = {s: c for s, c in loader.items() if c["rejected"]}
|
| 1168 |
+
if rejected:
|
| 1169 |
+
raise RuntimeError(f"nida.train.data rejects records the builder accepted: {rejected}")
|
| 1170 |
+
return {
|
| 1171 |
+
"dataset": str(self.dataset_dir),
|
| 1172 |
+
"speakers_done": len(markers),
|
| 1173 |
+
"segments": total_segments,
|
| 1174 |
+
"hours": stats["hours"],
|
| 1175 |
+
"skipped": stats["skipped"],
|
| 1176 |
+
"quarantined": stats["quarantined"],
|
| 1177 |
+
"failed_speakers": dict(failed),
|
| 1178 |
+
}
|
| 1179 |
+
|
| 1180 |
+
def _check_with_loader(self) -> dict[str, dict[str, object]]:
|
| 1181 |
+
"""Load every split with the training loader; counts per split (must not reject)."""
|
| 1182 |
+
out: dict[str, dict[str, object]] = {}
|
| 1183 |
+
for split in SPLITS:
|
| 1184 |
+
try:
|
| 1185 |
+
dataset, report = load_latent_dataset(self.dataset_dir, split)
|
| 1186 |
+
except DataFormatError as err:
|
| 1187 |
+
raise RuntimeError(f"the assembled dataset is invalid: {err}") from err
|
| 1188 |
+
dataset.close()
|
| 1189 |
+
out[split] = {
|
| 1190 |
+
"records": report.num_in_split,
|
| 1191 |
+
"accepted": report.num_accepted,
|
| 1192 |
+
"rejected": report.num_rejected,
|
| 1193 |
+
"rejection_counts": dict(report.rejection_counts),
|
| 1194 |
+
}
|
| 1195 |
+
return out
|
| 1196 |
+
|
| 1197 |
+
|
| 1198 |
+
# --------------------------------------------------------------------------------------
|
| 1199 |
+
# Reporting
|
| 1200 |
+
# --------------------------------------------------------------------------------------
|
| 1201 |
+
|
| 1202 |
+
|
| 1203 |
+
def _parse_transcripts(path: Path) -> dict[str, str]:
|
| 1204 |
+
obj = json.loads(path.read_text(encoding="utf-8"))
|
| 1205 |
+
if not isinstance(obj, dict) or not obj:
|
| 1206 |
+
raise ValueError(f"{path}: expected a non-empty mapping stem -> text")
|
| 1207 |
+
bad = [k for k, v in obj.items() if not isinstance(k, str) or not isinstance(v, str)]
|
| 1208 |
+
if bad:
|
| 1209 |
+
raise ValueError(f"{path}: entries must map strings to strings, e.g. {bad[:3]}")
|
| 1210 |
+
return dict(obj)
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
def _sum_counts(markers: Sequence[Mapping[str, object]], key: str) -> dict[str, int]:
|
| 1214 |
+
total: Counter[str] = Counter()
|
| 1215 |
+
for marker in markers:
|
| 1216 |
+
value = marker[key]
|
| 1217 |
+
assert isinstance(value, dict)
|
| 1218 |
+
total.update({str(k): int(v) for k, v in value.items()})
|
| 1219 |
+
return dict(sorted(total.items(), key=lambda kv: (-kv[1], kv[0])))
|
| 1220 |
+
|
| 1221 |
+
|
| 1222 |
+
def _stats(
|
| 1223 |
+
markers: Sequence[Mapping[str, object]],
|
| 1224 |
+
loader: Mapping[str, Mapping[str, object]],
|
| 1225 |
+
settings: EarsSettings,
|
| 1226 |
+
failed: Mapping[str, str],
|
| 1227 |
+
meta: Mapping[str, object],
|
| 1228 |
+
) -> dict[str, object]:
|
| 1229 |
+
kinds: dict[str, dict[str, int]] = {}
|
| 1230 |
+
for marker in markers:
|
| 1231 |
+
value = marker["kinds"]
|
| 1232 |
+
assert isinstance(value, dict)
|
| 1233 |
+
for label, stat in value.items():
|
| 1234 |
+
into = kinds.setdefault(label, {"segments": 0, "audio_ms": 0})
|
| 1235 |
+
into["segments"] += int(stat["segments"])
|
| 1236 |
+
into["audio_ms"] += int(stat["audio_ms"])
|
| 1237 |
+
splits: dict[str, dict[str, int]] = {s: {"speakers": 0, "segments": 0} for s in SPLITS}
|
| 1238 |
+
for marker in markers:
|
| 1239 |
+
entry = splits[str(marker["split"])]
|
| 1240 |
+
entry["speakers"] += 1
|
| 1241 |
+
entry["segments"] += int(str(marker["segments"]))
|
| 1242 |
+
audio_ms = sum(int(str(m["audio_ms"])) for m in markers)
|
| 1243 |
+
return {
|
| 1244 |
+
"speakers_done": len(markers),
|
| 1245 |
+
"speakers_total": settings.ears.n_speakers,
|
| 1246 |
+
"segments": meta["segments"],
|
| 1247 |
+
"frames": meta["frames"],
|
| 1248 |
+
"hours": round(audio_ms / 3_600_000, 3),
|
| 1249 |
+
"kinds": dict(sorted(kinds.items())),
|
| 1250 |
+
"splits": splits,
|
| 1251 |
+
"skipped": _sum_counts(markers, "skipped"),
|
| 1252 |
+
"quarantined": _sum_counts(markers, "quarantined"),
|
| 1253 |
+
"loader": {k: dict(v) for k, v in loader.items()},
|
| 1254 |
+
"failed_speakers": dict(failed),
|
| 1255 |
+
"transcripts_sha256": meta["transcripts_sha256"],
|
| 1256 |
+
"config_digest": settings.digest,
|
| 1257 |
+
}
|
| 1258 |
+
|
| 1259 |
+
|
| 1260 |
+
def _render_report(stats: Mapping[str, object]) -> str:
|
| 1261 |
+
kinds = stats["kinds"]
|
| 1262 |
+
splits = stats["splits"]
|
| 1263 |
+
loader = stats["loader"]
|
| 1264 |
+
assert isinstance(kinds, dict) and isinstance(splits, dict) and isinstance(loader, dict)
|
| 1265 |
+
lines = [
|
| 1266 |
+
"# EARS latent veri seti raporu",
|
| 1267 |
+
"",
|
| 1268 |
+
f"- Konuşmacı: {stats['speakers_done']}/{stats['speakers_total']} tamamlandı",
|
| 1269 |
+
f"- Segment: {stats['segments']}, süre: {stats['hours']} sa, kare: {stats['frames']}",
|
| 1270 |
+
f"- Yapılandırma özeti: `{stats['config_digest']}`; transkript SHA-256: "
|
| 1271 |
+
f"`{stats['transcripts_sha256']}`",
|
| 1272 |
+
"",
|
| 1273 |
+
"## Seçilen türler",
|
| 1274 |
+
"",
|
| 1275 |
+
*(
|
| 1276 |
+
f"- {label}: {v['segments']} segment, {v['audio_ms'] / 3_600_000:.3f} sa"
|
| 1277 |
+
for label, v in kinds.items()
|
| 1278 |
+
),
|
| 1279 |
+
"",
|
| 1280 |
+
"## Bölmeler (konuşmacı-ayrık)",
|
| 1281 |
+
"",
|
| 1282 |
+
*(
|
| 1283 |
+
f"- {name}: {v['speakers']} konuşmacı, {v['segments']} segment"
|
| 1284 |
+
for name, v in splits.items()
|
| 1285 |
+
),
|
| 1286 |
+
"",
|
| 1287 |
+
"## Atlananlar (neden: sayı)",
|
| 1288 |
+
"",
|
| 1289 |
+
json.dumps(stats["skipped"], ensure_ascii=False),
|
| 1290 |
+
"",
|
| 1291 |
+
"## Karantina (neden: sayı)",
|
| 1292 |
+
"",
|
| 1293 |
+
json.dumps(stats["quarantined"], ensure_ascii=False),
|
| 1294 |
+
"",
|
| 1295 |
+
"## Eğitim yükleyicisi doğrulaması (`nida.train.data`)",
|
| 1296 |
+
"",
|
| 1297 |
+
*(
|
| 1298 |
+
f"- {name}: {v['accepted']}/{v['records']} kabul, ret: {v['rejection_counts']}"
|
| 1299 |
+
for name, v in loader.items()
|
| 1300 |
+
),
|
| 1301 |
+
]
|
| 1302 |
+
failed = stats["failed_speakers"]
|
| 1303 |
+
if failed:
|
| 1304 |
+
lines += ["", "## Başarısız konuşmacılar (yeniden çalıştırın)", ""]
|
| 1305 |
+
assert isinstance(failed, dict)
|
| 1306 |
+
lines += [f"- {speaker}: {error[:300]}" for speaker, error in failed.items()]
|
| 1307 |
+
return "\n".join(lines) + "\n"
|
src/nida/data/enhance.py
ADDED
|
@@ -0,0 +1,593 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Optional speech-enhancement step of the data pilot (docs/03-data-pipeline.md).
|
| 2 |
+
|
| 3 |
+
Everything about enhancement that does not depend on one arm lives here:
|
| 4 |
+
|
| 5 |
+
* :class:`EnhanceConfig` (``configs/data/enhance.yaml``): which arm (``none`` | ``uvr`` |
|
| 6 |
+
``sidon`` | ``dfn``) and how much length difference between the input and the enhanced
|
| 7 |
+
segment is tolerated;
|
| 8 |
+
* naming of the arm-specific audio root / dataset suffix;
|
| 9 |
+
* the pure audio functions shared by the three arm workers (:func:`fit_to_length`,
|
| 10 |
+
:func:`finish_enhanced_wav`) and the :class:`EnhanceAdapter` template they plug into;
|
| 11 |
+
* the small checks the arms share (device resolution, pinned-file SHA-256).
|
| 12 |
+
|
| 13 |
+
The arms themselves are in ``nida.data.workers.enhance_<arm>``; each runs in its own isolated
|
| 14 |
+
environment (:data:`nida.data.envs.ENHANCE_ENV_SPECS`), because their dependency pins
|
| 15 |
+
conflict (numpy 1 vs 2, Python 3.11 vs 3.12). The worker modules and this module are imported
|
| 16 |
+
inside those environments, so this module imports only the standard library and the stdlib-only
|
| 17 |
+
protocol/worker helpers at import time; numpy, scipy and soundfile are imported inside the
|
| 18 |
+
functions that need them (all three arm environments install them).
|
| 19 |
+
|
| 20 |
+
Contract of a worker request: ``audio_path`` is a 16 kHz mono wav; ``params["out_path"]`` is
|
| 21 |
+
the ``.wav`` to write; ``params["max_length_diff_ms"]`` is the largest length difference the
|
| 22 |
+
worker accepts at all. The output is a 16 kHz mono PCM16 wav with **exactly** as many samples as
|
| 23 |
+
the input. The difference between the arm's output length (after resampling to 16 kHz) and the
|
| 24 |
+
input length is recorded as ``length_diff_samples`` before it is fixed by zero-padding or
|
| 25 |
+
trimming; the payload also carries ``peak`` (before clipping) and ``loudness_dbfs`` (RMS level of
|
| 26 |
+
the PCM16 samples actually written, the same measure as the ingest ``loudness_dbfs``).
|
| 27 |
+
|
| 28 |
+
Who judges the length: the worker only refuses an absurd difference (above
|
| 29 |
+
``max_length_diff_ms``: a misaligned or truncated output, :class:`EnhanceLengthError`, a failed
|
| 30 |
+
item). Everything between the tolerance and that limit is written and recorded, and the pilot
|
| 31 |
+
alone quarantines it by ``length_tolerance_ms`` (:mod:`nida.data.pilot`), so the tolerance can be
|
| 32 |
+
changed without recomputing and tolerated-but-large differences do not count toward the worker
|
| 33 |
+
error budget.
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
from __future__ import annotations
|
| 37 |
+
|
| 38 |
+
import abc
|
| 39 |
+
import hashlib
|
| 40 |
+
import io
|
| 41 |
+
import math
|
| 42 |
+
import os
|
| 43 |
+
from collections.abc import Mapping, Sequence
|
| 44 |
+
from dataclasses import dataclass
|
| 45 |
+
from pathlib import Path
|
| 46 |
+
from typing import TYPE_CHECKING
|
| 47 |
+
|
| 48 |
+
from nida.bench.protocol import WorkRequest
|
| 49 |
+
from nida.bench.workers.runner import Adapter
|
| 50 |
+
from nida.data.workers import REQUIRED_SAMPLE_RATE_HZ, require_16k_mono
|
| 51 |
+
from nida.utils.io import atomic_write_bytes
|
| 52 |
+
|
| 53 |
+
if TYPE_CHECKING:
|
| 54 |
+
import numpy as np
|
| 55 |
+
import numpy.typing as npt
|
| 56 |
+
|
| 57 |
+
Waveform = npt.NDArray[np.float32]
|
| 58 |
+
|
| 59 |
+
__all__ = [
|
| 60 |
+
"ARMS",
|
| 61 |
+
"ARM_NONE",
|
| 62 |
+
"CONFIG_FILENAME",
|
| 63 |
+
"ENHANCED_ARMS",
|
| 64 |
+
"NO_ENHANCE",
|
| 65 |
+
"PAYLOAD_KEYS",
|
| 66 |
+
"REASON_CLIPPED",
|
| 67 |
+
"REASON_FAILED",
|
| 68 |
+
"REASON_LENGTH",
|
| 69 |
+
"ArmUnavailableError",
|
| 70 |
+
"EnhanceAdapter",
|
| 71 |
+
"EnhanceConfig",
|
| 72 |
+
"EnhanceError",
|
| 73 |
+
"EnhanceLengthError",
|
| 74 |
+
"FittedAudio",
|
| 75 |
+
"arm_suffix",
|
| 76 |
+
"audio_dir_name",
|
| 77 |
+
"clipping_threshold_param",
|
| 78 |
+
"encode_wav_pcm16",
|
| 79 |
+
"finish_enhanced_wav",
|
| 80 |
+
"fit_to_length",
|
| 81 |
+
"is_length_error",
|
| 82 |
+
"max_length_diff_param",
|
| 83 |
+
"resample_mono",
|
| 84 |
+
"resolve_device",
|
| 85 |
+
"sha256_file",
|
| 86 |
+
"verify_pinned_sha256",
|
| 87 |
+
"wav_loudness_dbfs",
|
| 88 |
+
"wav_out_path_param",
|
| 89 |
+
"wav_stats",
|
| 90 |
+
]
|
| 91 |
+
|
| 92 |
+
ARM_NONE = "none"
|
| 93 |
+
#: Selectable arms, ``none`` first. The other three have a worker, an environment and a pin.
|
| 94 |
+
ARMS = (ARM_NONE, "uvr", "sidon", "dfn")
|
| 95 |
+
ENHANCED_ARMS = ARMS[1:]
|
| 96 |
+
#: Name of the optional config file in ``configs/data/``; absent = no enhancement.
|
| 97 |
+
CONFIG_FILENAME = "enhance.yaml"
|
| 98 |
+
#: Prefix of the local-cache folder (sibling of ``audio``) holding one arm's enhanced wavs.
|
| 99 |
+
AUDIO_DIR_PREFIX = "audio-enh"
|
| 100 |
+
#: Quarantine reasons of the pilot ``enhance`` stage.
|
| 101 |
+
REASON_LENGTH = "enhance_length_mismatch"
|
| 102 |
+
REASON_FAILED = "enhance_failed"
|
| 103 |
+
REASON_CLIPPED = "enhance_clipped"
|
| 104 |
+
#: Prefix of a pinned file hash in ``ModelRef.revision`` (``sha256:<64 hex digits>``).
|
| 105 |
+
SHA256_REVISION_PREFIX = "sha256:"
|
| 106 |
+
#: Keys of every successful worker payload (:func:`finish_enhanced_wav`); anything else in a
|
| 107 |
+
#: payload came from :meth:`EnhanceAdapter.payload_extras`.
|
| 108 |
+
PAYLOAD_KEYS = frozenset(
|
| 109 |
+
{
|
| 110 |
+
"out_path",
|
| 111 |
+
"sample_rate_hz",
|
| 112 |
+
"num_samples",
|
| 113 |
+
"input_num_samples",
|
| 114 |
+
"length_diff_samples",
|
| 115 |
+
"peak",
|
| 116 |
+
"loudness_dbfs",
|
| 117 |
+
"clipping_ratio",
|
| 118 |
+
"clipping_threshold",
|
| 119 |
+
}
|
| 120 |
+
)
|
| 121 |
+
_SHA256_HEX_LENGTH = 64
|
| 122 |
+
_HASH_BLOCK_BYTES = 1 << 20
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class EnhanceError(ValueError):
|
| 126 |
+
"""An arm returned unusable audio (not 1-D mono, empty or non-finite)."""
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class EnhanceLengthError(EnhanceError):
|
| 130 |
+
"""The arm's output length differs from the input by more than the tolerance."""
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class ArmUnavailableError(RuntimeError):
|
| 134 |
+
"""The arm cannot start at all (missing library, no GPU, silent CPU fallback)."""
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def is_length_error(error: str | None) -> bool:
|
| 138 |
+
"""True if a worker result's ``error`` string (``"<Type>: <message>"``) is a length error."""
|
| 139 |
+
return error is not None and error.startswith(f"{EnhanceLengthError.__name__}:")
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
@dataclass(frozen=True)
|
| 143 |
+
class EnhanceConfig:
|
| 144 |
+
"""``configs/data/enhance.yaml``.
|
| 145 |
+
|
| 146 |
+
Attributes:
|
| 147 |
+
arm: ``none`` (no enhancement, the baseline), ``uvr``, ``sidon`` or ``dfn``.
|
| 148 |
+
length_tolerance_ms: Largest tolerated difference, in milliseconds, between the
|
| 149 |
+
enhanced and the input segment length (checked in 16 kHz samples). Larger
|
| 150 |
+
differences make the segment an ``enhance_length_mismatch`` quarantine record
|
| 151 |
+
(decided by the pilot from the recorded ``length_diff_samples``).
|
| 152 |
+
max_length_diff_ms: Differences above this make the *worker* fail the item (also
|
| 153 |
+
``enhance_length_mismatch``); between the two limits the wav is still written.
|
| 154 |
+
Must be >= ``length_tolerance_ms``.
|
| 155 |
+
max_enhanced_clipping_ratio: Largest fraction of samples of the written (PCM16) wav
|
| 156 |
+
that may sit at ``|x| >= acoustic.clipping_threshold`` (the S2 definition; the
|
| 157 |
+
threshold is the one of ``acoustic.yaml``); more makes the segment an
|
| 158 |
+
``enhance_clipped`` quarantine record. In ``[0, 1]``.
|
| 159 |
+
"""
|
| 160 |
+
|
| 161 |
+
arm: str
|
| 162 |
+
length_tolerance_ms: int
|
| 163 |
+
max_length_diff_ms: int
|
| 164 |
+
max_enhanced_clipping_ratio: float
|
| 165 |
+
|
| 166 |
+
def __post_init__(self) -> None:
|
| 167 |
+
if self.arm not in ARMS:
|
| 168 |
+
raise ValueError(f"enhance arm must be one of {ARMS}, got {self.arm!r}")
|
| 169 |
+
for name in ("length_tolerance_ms", "max_length_diff_ms"):
|
| 170 |
+
value = getattr(self, name)
|
| 171 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
| 172 |
+
raise ValueError(f"{name} must be an integer >= 0, got {value!r}")
|
| 173 |
+
if self.max_length_diff_ms < self.length_tolerance_ms:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
f"max_length_diff_ms ({self.max_length_diff_ms}) must be >= "
|
| 176 |
+
f"length_tolerance_ms ({self.length_tolerance_ms})"
|
| 177 |
+
)
|
| 178 |
+
ratio = self.max_enhanced_clipping_ratio
|
| 179 |
+
if isinstance(ratio, bool) or not isinstance(ratio, int | float):
|
| 180 |
+
raise ValueError(f"max_enhanced_clipping_ratio must be a number, got {ratio!r}")
|
| 181 |
+
if not (math.isfinite(ratio) and 0.0 <= ratio <= 1.0):
|
| 182 |
+
raise ValueError(f"max_enhanced_clipping_ratio must be in [0, 1], got {ratio!r}")
|
| 183 |
+
|
| 184 |
+
@property
|
| 185 |
+
def enabled(self) -> bool:
|
| 186 |
+
return self.arm != ARM_NONE
|
| 187 |
+
|
| 188 |
+
@property
|
| 189 |
+
def length_tolerance_samples(self) -> int:
|
| 190 |
+
"""The tolerance in 16 kHz samples (exact: 16 samples per millisecond)."""
|
| 191 |
+
return self.length_tolerance_ms * REQUIRED_SAMPLE_RATE_HZ // 1000
|
| 192 |
+
|
| 193 |
+
@property
|
| 194 |
+
def max_length_diff_samples(self) -> int:
|
| 195 |
+
"""The worker's refusal limit in 16 kHz samples."""
|
| 196 |
+
return self.max_length_diff_ms * REQUIRED_SAMPLE_RATE_HZ // 1000
|
| 197 |
+
|
| 198 |
+
@classmethod
|
| 199 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> EnhanceConfig:
|
| 200 |
+
expected = {
|
| 201 |
+
"arm",
|
| 202 |
+
"length_tolerance_ms",
|
| 203 |
+
"max_length_diff_ms",
|
| 204 |
+
"max_enhanced_clipping_ratio",
|
| 205 |
+
}
|
| 206 |
+
if set(raw) != expected:
|
| 207 |
+
raise ValueError(
|
| 208 |
+
f"enhance config keys: missing={sorted(expected - set(raw))} "
|
| 209 |
+
f"unknown={sorted(set(raw) - expected)}"
|
| 210 |
+
)
|
| 211 |
+
arm = raw["arm"]
|
| 212 |
+
if not isinstance(arm, str):
|
| 213 |
+
raise ValueError(f"enhance arm must be a string, got {arm!r}")
|
| 214 |
+
limits: dict[str, int] = {}
|
| 215 |
+
for name in ("length_tolerance_ms", "max_length_diff_ms"):
|
| 216 |
+
value = raw[name]
|
| 217 |
+
if not isinstance(value, int): # bool is rejected by __post_init__
|
| 218 |
+
raise ValueError(f"{name} must be an integer, got {value!r}")
|
| 219 |
+
limits[name] = value
|
| 220 |
+
ratio = raw["max_enhanced_clipping_ratio"]
|
| 221 |
+
if isinstance(ratio, bool) or not isinstance(ratio, int | float):
|
| 222 |
+
raise ValueError(f"max_enhanced_clipping_ratio must be a number, got {ratio!r}")
|
| 223 |
+
return cls(arm=arm, max_enhanced_clipping_ratio=float(ratio), **limits)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
#: What a config directory without ``enhance.yaml`` means: the baseline pipeline. The limits are
|
| 227 |
+
#: never read while the arm is ``none``.
|
| 228 |
+
NO_ENHANCE = EnhanceConfig(
|
| 229 |
+
arm=ARM_NONE, length_tolerance_ms=0, max_length_diff_ms=0, max_enhanced_clipping_ratio=0.0
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def audio_dir_name(arm: str) -> str:
|
| 234 |
+
"""Local-cache folder name of ``arm``'s enhanced wavs (``audio-enh-<arm>``).
|
| 235 |
+
|
| 236 |
+
Raises:
|
| 237 |
+
ValueError: if ``arm`` is ``none`` or unknown.
|
| 238 |
+
"""
|
| 239 |
+
if arm not in ENHANCED_ARMS:
|
| 240 |
+
raise ValueError(f"no enhanced audio for arm {arm!r}; arms: {ENHANCED_ARMS}")
|
| 241 |
+
return f"{AUDIO_DIR_PREFIX}-{arm}"
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def arm_suffix(arm: str) -> str:
|
| 245 |
+
"""Suffix of the arm-specific names (dataset directory, latent results, arrays)."""
|
| 246 |
+
audio_dir_name(arm) # validates
|
| 247 |
+
return f"-enh-{arm}"
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
# ------------------------------------------------------------------------ audio functions
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
@dataclass(frozen=True)
|
| 254 |
+
class FittedAudio:
|
| 255 |
+
"""An arm output forced to the input length.
|
| 256 |
+
|
| 257 |
+
Attributes:
|
| 258 |
+
samples: float32 mono at 16 kHz, exactly the expected number of samples.
|
| 259 |
+
length_diff_samples: ``len(output after resampling) - len(input)`` *before* the
|
| 260 |
+
pad/trim (positive = the arm returned more samples than the input had).
|
| 261 |
+
peak: Largest absolute sample value before clipping to ``[-1, 1]``.
|
| 262 |
+
"""
|
| 263 |
+
|
| 264 |
+
samples: Waveform
|
| 265 |
+
length_diff_samples: int
|
| 266 |
+
peak: float
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def resample_mono(samples: Waveform, src_hz: int, dst_hz: int) -> Waveform:
|
| 270 |
+
"""Polyphase resample (scipy); the input itself when the rates match.
|
| 271 |
+
|
| 272 |
+
Raises:
|
| 273 |
+
ValueError: on a non-positive rate.
|
| 274 |
+
"""
|
| 275 |
+
import numpy as np
|
| 276 |
+
from scipy.signal import resample_poly # type: ignore[import-untyped] # no stubs
|
| 277 |
+
|
| 278 |
+
if src_hz <= 0 or dst_hz <= 0:
|
| 279 |
+
raise ValueError(f"sample rates must be positive, got {src_hz} -> {dst_hz}")
|
| 280 |
+
if src_hz == dst_hz:
|
| 281 |
+
return samples
|
| 282 |
+
divisor = math.gcd(src_hz, dst_hz)
|
| 283 |
+
out = resample_poly(samples.astype(np.float64), dst_hz // divisor, src_hz // divisor)
|
| 284 |
+
return np.ascontiguousarray(out, dtype=np.float32)
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def fit_to_length(
|
| 288 |
+
samples: npt.ArrayLike,
|
| 289 |
+
*,
|
| 290 |
+
source_rate_hz: int,
|
| 291 |
+
expected_num_samples: int,
|
| 292 |
+
max_diff_samples: int,
|
| 293 |
+
what: str,
|
| 294 |
+
) -> FittedAudio:
|
| 295 |
+
"""Resample an arm output to 16 kHz and force it to ``expected_num_samples``.
|
| 296 |
+
|
| 297 |
+
Raises:
|
| 298 |
+
EnhanceError: for non-1-D, empty or non-finite output.
|
| 299 |
+
EnhanceLengthError: if the resampled length differs from ``expected_num_samples`` by
|
| 300 |
+
more than ``max_diff_samples`` (nothing is padded or trimmed in that case).
|
| 301 |
+
ValueError: on a non-positive ``expected_num_samples``, negative limit or bad rate.
|
| 302 |
+
"""
|
| 303 |
+
import numpy as np
|
| 304 |
+
|
| 305 |
+
if expected_num_samples < 1 or max_diff_samples < 0:
|
| 306 |
+
raise ValueError(
|
| 307 |
+
f"{what}: invalid expected length / limit: {expected_num_samples} / {max_diff_samples}"
|
| 308 |
+
)
|
| 309 |
+
array = np.asarray(samples)
|
| 310 |
+
if array.ndim != 1:
|
| 311 |
+
raise EnhanceError(f"{what}: enhancer returned shape {array.shape}, expected 1-D mono")
|
| 312 |
+
if array.size == 0:
|
| 313 |
+
raise EnhanceError(f"{what}: enhancer returned no samples")
|
| 314 |
+
if not np.isfinite(array).all():
|
| 315 |
+
raise EnhanceError(f"{what}: enhancer returned non-finite samples")
|
| 316 |
+
resampled = resample_mono(
|
| 317 |
+
array.astype(np.float32, copy=False), source_rate_hz, REQUIRED_SAMPLE_RATE_HZ
|
| 318 |
+
)
|
| 319 |
+
diff = int(resampled.shape[0]) - expected_num_samples
|
| 320 |
+
if abs(diff) > max_diff_samples:
|
| 321 |
+
raise EnhanceLengthError(
|
| 322 |
+
f"{what}: output has {resampled.shape[0]} samples at {REQUIRED_SAMPLE_RATE_HZ} Hz, "
|
| 323 |
+
f"expected {expected_num_samples} (difference {diff} samples = "
|
| 324 |
+
f"{1000.0 * diff / REQUIRED_SAMPLE_RATE_HZ:.1f} ms, limit {max_diff_samples} "
|
| 325 |
+
"samples)"
|
| 326 |
+
)
|
| 327 |
+
if diff >= 0:
|
| 328 |
+
fitted = np.ascontiguousarray(resampled[:expected_num_samples])
|
| 329 |
+
else:
|
| 330 |
+
fitted = np.pad(resampled, (0, -diff))
|
| 331 |
+
return FittedAudio(fitted, diff, float(np.abs(fitted).max()))
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def encode_wav_pcm16(samples: Waveform) -> bytes:
|
| 335 |
+
"""The bytes of a 16 kHz mono PCM16 wav; ``samples`` are clipped to ``[-1, 1]`` first."""
|
| 336 |
+
import numpy as np
|
| 337 |
+
import soundfile # type: ignore[import-untyped] # untyped
|
| 338 |
+
|
| 339 |
+
buffer = io.BytesIO()
|
| 340 |
+
soundfile.write(
|
| 341 |
+
buffer,
|
| 342 |
+
np.clip(samples, -1.0, 1.0),
|
| 343 |
+
REQUIRED_SAMPLE_RATE_HZ,
|
| 344 |
+
subtype="PCM_16",
|
| 345 |
+
format="WAV",
|
| 346 |
+
)
|
| 347 |
+
return buffer.getvalue()
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def wav_stats(data: bytes, *, clipping_threshold: float) -> tuple[float, float]:
|
| 351 |
+
"""``(loudness_dbfs, clipping_ratio)`` of the samples a wav file really holds.
|
| 352 |
+
|
| 353 |
+
Decoded from the bytes, so the numbers describe what was written (PCM16, clipped to
|
| 354 |
+
``[-1, 1]``). Both are the ingest (S2) measures, :func:`nida.data.acoustic.loudness_dbfs`
|
| 355 |
+
and :func:`nida.data.acoustic.clipping_ratio` on the float64 samples, so enhanced and
|
| 356 |
+
original audio are comparable.
|
| 357 |
+
"""
|
| 358 |
+
import soundfile
|
| 359 |
+
|
| 360 |
+
# numpy + scipy; kept out of the import time of this module
|
| 361 |
+
from nida.data.acoustic import clipping_ratio, loudness_dbfs
|
| 362 |
+
|
| 363 |
+
decoded, _ = soundfile.read(io.BytesIO(data), dtype="float64", always_2d=False)
|
| 364 |
+
return (
|
| 365 |
+
float(loudness_dbfs(decoded)),
|
| 366 |
+
float(clipping_ratio(decoded, threshold=clipping_threshold)),
|
| 367 |
+
)
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def wav_loudness_dbfs(data: bytes) -> float:
|
| 371 |
+
"""RMS level in dBFS of the samples a wav file really holds (see :func:`wav_stats`)."""
|
| 372 |
+
return wav_stats(data, clipping_threshold=1.0)[0]
|
| 373 |
+
|
| 374 |
+
|
| 375 |
+
def finish_enhanced_wav(
|
| 376 |
+
*,
|
| 377 |
+
out_path: Path,
|
| 378 |
+
samples: npt.ArrayLike,
|
| 379 |
+
source_rate_hz: int,
|
| 380 |
+
input_num_samples: int,
|
| 381 |
+
max_diff_samples: int,
|
| 382 |
+
clipping_threshold: float,
|
| 383 |
+
what: str,
|
| 384 |
+
) -> dict[str, object]:
|
| 385 |
+
"""Fit an arm output to the input length, write it atomically, describe it.
|
| 386 |
+
|
| 387 |
+
The checks run before the write, so a failing segment leaves no output file.
|
| 388 |
+
|
| 389 |
+
Returns:
|
| 390 |
+
Payload: ``out_path``, ``sample_rate_hz``, ``num_samples`` (== ``input_num_samples``),
|
| 391 |
+
``input_num_samples``, ``length_diff_samples`` (before the fix), ``peak`` (before
|
| 392 |
+
clipping to the PCM16 range), ``loudness_dbfs`` and ``clipping_ratio`` (fraction of the
|
| 393 |
+
PCM16 samples written at ``|x| >= clipping_threshold``, which is echoed back).
|
| 394 |
+
|
| 395 |
+
Raises:
|
| 396 |
+
EnhanceError, EnhanceLengthError, ValueError: see :func:`fit_to_length`.
|
| 397 |
+
"""
|
| 398 |
+
fitted = fit_to_length(
|
| 399 |
+
samples,
|
| 400 |
+
source_rate_hz=source_rate_hz,
|
| 401 |
+
expected_num_samples=input_num_samples,
|
| 402 |
+
max_diff_samples=max_diff_samples,
|
| 403 |
+
what=what,
|
| 404 |
+
)
|
| 405 |
+
data = encode_wav_pcm16(fitted.samples)
|
| 406 |
+
atomic_write_bytes(out_path, data)
|
| 407 |
+
loudness, clipping = wav_stats(data, clipping_threshold=clipping_threshold)
|
| 408 |
+
return {
|
| 409 |
+
"out_path": str(out_path),
|
| 410 |
+
"sample_rate_hz": REQUIRED_SAMPLE_RATE_HZ,
|
| 411 |
+
"num_samples": int(fitted.samples.shape[0]),
|
| 412 |
+
"input_num_samples": input_num_samples,
|
| 413 |
+
"length_diff_samples": fitted.length_diff_samples,
|
| 414 |
+
"peak": round(fitted.peak, 6),
|
| 415 |
+
"loudness_dbfs": loudness,
|
| 416 |
+
"clipping_ratio": clipping,
|
| 417 |
+
"clipping_threshold": clipping_threshold,
|
| 418 |
+
}
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
def wav_out_path_param(request: WorkRequest) -> Path:
|
| 422 |
+
"""The ``.wav`` output path named by ``request.params["out_path"]``.
|
| 423 |
+
|
| 424 |
+
Raises:
|
| 425 |
+
ValueError: if the parameter is missing or does not end in ``.wav``.
|
| 426 |
+
"""
|
| 427 |
+
raw = request.params.get("out_path")
|
| 428 |
+
if not raw:
|
| 429 |
+
raise ValueError(f"{request.clip_id}: missing params.out_path")
|
| 430 |
+
path = Path(raw)
|
| 431 |
+
if path.suffix != ".wav":
|
| 432 |
+
raise ValueError(f"{request.clip_id}: params.out_path must end in .wav, got {raw!r}")
|
| 433 |
+
return path
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def max_length_diff_param(request: WorkRequest) -> int:
|
| 437 |
+
"""``params["max_length_diff_ms"]`` converted to 16 kHz samples.
|
| 438 |
+
|
| 439 |
+
Raises:
|
| 440 |
+
ValueError: if the parameter is missing, not a non-negative integer, or unreasonable.
|
| 441 |
+
"""
|
| 442 |
+
raw = request.params.get("max_length_diff_ms")
|
| 443 |
+
if raw is None:
|
| 444 |
+
raise ValueError(f"{request.clip_id}: missing params.max_length_diff_ms")
|
| 445 |
+
try:
|
| 446 |
+
milliseconds = int(raw)
|
| 447 |
+
except ValueError:
|
| 448 |
+
raise ValueError(
|
| 449 |
+
f"{request.clip_id}: params.max_length_diff_ms must be an integer, got {raw!r}"
|
| 450 |
+
) from None
|
| 451 |
+
if milliseconds < 0:
|
| 452 |
+
raise ValueError(f"{request.clip_id}: params.max_length_diff_ms must be >= 0")
|
| 453 |
+
return milliseconds * REQUIRED_SAMPLE_RATE_HZ // 1000
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def clipping_threshold_param(request: WorkRequest) -> float:
|
| 457 |
+
"""``params["clipping_threshold"]``: the magnitude counted as clipped, in ``(0, 1]``.
|
| 458 |
+
|
| 459 |
+
Raises:
|
| 460 |
+
ValueError: if the parameter is missing or not a number in ``(0, 1]``.
|
| 461 |
+
"""
|
| 462 |
+
raw = request.params.get("clipping_threshold")
|
| 463 |
+
if raw is None:
|
| 464 |
+
raise ValueError(f"{request.clip_id}: missing params.clipping_threshold")
|
| 465 |
+
try:
|
| 466 |
+
value = float(raw)
|
| 467 |
+
except ValueError:
|
| 468 |
+
raise ValueError(
|
| 469 |
+
f"{request.clip_id}: params.clipping_threshold must be a number, got {raw!r}"
|
| 470 |
+
) from None
|
| 471 |
+
if not (math.isfinite(value) and 0.0 < value <= 1.0):
|
| 472 |
+
raise ValueError(f"{request.clip_id}: params.clipping_threshold must be in (0, 1]")
|
| 473 |
+
return value
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
# ----------------------------------------------------------------------- worker template
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
class EnhanceAdapter(Adapter):
|
| 480 |
+
"""Template of the three arm workers: subclasses load a model and implement :meth:`enhance`.
|
| 481 |
+
|
| 482 |
+
Clips are processed one at a time (the arms are not batched), so a failing clip is an
|
| 483 |
+
error of that clip only: the runner records it and carries on.
|
| 484 |
+
"""
|
| 485 |
+
|
| 486 |
+
@abc.abstractmethod
|
| 487 |
+
def enhance(self, request: WorkRequest, wav: Waveform) -> tuple[Waveform, int]:
|
| 488 |
+
"""Enhance one clip.
|
| 489 |
+
|
| 490 |
+
Args:
|
| 491 |
+
request: The request (``audio_path`` is the same file ``wav`` was read from).
|
| 492 |
+
wav: float32 mono samples at 16 kHz, shape ``(num_samples,)``.
|
| 493 |
+
|
| 494 |
+
Returns:
|
| 495 |
+
``(samples, sample_rate_hz)``: a 1-D float waveform at any rate; the caller
|
| 496 |
+
resamples it to 16 kHz and checks its length.
|
| 497 |
+
"""
|
| 498 |
+
|
| 499 |
+
def payload_extras(self) -> Mapping[str, object]:
|
| 500 |
+
"""Facts about the loaded model/runtime added to every payload (default: none).
|
| 501 |
+
|
| 502 |
+
They let the pilot record, e.g., the execution provider that really ran or the digests
|
| 503 |
+
of model files that are recorded rather than pinned. Keys must not clash with the
|
| 504 |
+
standard payload keys.
|
| 505 |
+
"""
|
| 506 |
+
return {}
|
| 507 |
+
|
| 508 |
+
def _one(self, request: WorkRequest) -> Mapping[str, object]:
|
| 509 |
+
import soundfile
|
| 510 |
+
|
| 511 |
+
info = require_16k_mono(request)
|
| 512 |
+
out_path = wav_out_path_param(request)
|
| 513 |
+
max_diff_samples = max_length_diff_param(request)
|
| 514 |
+
clipping_threshold = clipping_threshold_param(request)
|
| 515 |
+
wav, _ = soundfile.read(request.audio_path, dtype="float32", always_2d=False)
|
| 516 |
+
if wav.ndim != 1 or int(wav.shape[0]) != info.num_samples:
|
| 517 |
+
raise ValueError(f"{request.clip_id}: read {wav.shape}, header says {info.num_samples}")
|
| 518 |
+
samples, rate_hz = self.enhance(request, wav)
|
| 519 |
+
extras = self.payload_extras()
|
| 520 |
+
clash = sorted(set(extras) & PAYLOAD_KEYS)
|
| 521 |
+
if clash:
|
| 522 |
+
raise ValueError(f"{request.clip_id}: payload extras clash with {clash}")
|
| 523 |
+
payload = finish_enhanced_wav(
|
| 524 |
+
out_path=out_path,
|
| 525 |
+
samples=samples,
|
| 526 |
+
source_rate_hz=rate_hz,
|
| 527 |
+
input_num_samples=info.num_samples,
|
| 528 |
+
max_diff_samples=max_diff_samples,
|
| 529 |
+
clipping_threshold=clipping_threshold,
|
| 530 |
+
what=request.clip_id,
|
| 531 |
+
)
|
| 532 |
+
return {**payload, **extras}
|
| 533 |
+
|
| 534 |
+
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
|
| 535 |
+
return [self._one(request) for request in batch]
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
# ----------------------------------------------------------------------- shared arm checks
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def resolve_device(device: str) -> str:
|
| 542 |
+
"""Validate ``device`` for this process; ``cpu`` hides CUDA before any library loads.
|
| 543 |
+
|
| 544 |
+
Raises:
|
| 545 |
+
ArmUnavailableError: for CUDA requested but unavailable, torch missing, or a device
|
| 546 |
+
string other than ``cpu`` / ``cuda*``.
|
| 547 |
+
"""
|
| 548 |
+
if device == "cpu":
|
| 549 |
+
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
| 550 |
+
return "cpu"
|
| 551 |
+
if device.startswith("cuda"):
|
| 552 |
+
try:
|
| 553 |
+
import torch
|
| 554 |
+
except ImportError as exc:
|
| 555 |
+
raise ArmUnavailableError("torch is not installed in this environment") from exc
|
| 556 |
+
if not torch.cuda.is_available():
|
| 557 |
+
raise ArmUnavailableError(f"device {device!r} requested but CUDA is not available here")
|
| 558 |
+
return device
|
| 559 |
+
raise ArmUnavailableError(f"unsupported device {device!r} (use cpu or cuda)")
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
def sha256_file(path: Path) -> str:
|
| 563 |
+
"""Hex SHA-256 of a file, read in 1 MiB blocks."""
|
| 564 |
+
digest = hashlib.sha256()
|
| 565 |
+
with path.open("rb") as handle:
|
| 566 |
+
for block in iter(lambda: handle.read(_HASH_BLOCK_BYTES), b""):
|
| 567 |
+
digest.update(block)
|
| 568 |
+
return digest.hexdigest()
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
def verify_pinned_sha256(path: Path, revision: str, *, what: str) -> str:
|
| 572 |
+
"""Check that ``path`` hashes to the digest pinned in ``revision`` (``sha256:<hex>``).
|
| 573 |
+
|
| 574 |
+
Returns:
|
| 575 |
+
The verified digest.
|
| 576 |
+
|
| 577 |
+
Raises:
|
| 578 |
+
ValueError: if ``revision`` is not a ``sha256:`` pin, the file is missing, or the
|
| 579 |
+
digest differs (the model file is not the one the pipeline was validated with).
|
| 580 |
+
"""
|
| 581 |
+
if not revision.startswith(SHA256_REVISION_PREFIX):
|
| 582 |
+
raise ValueError(
|
| 583 |
+
f"{what}: revision must be '{SHA256_REVISION_PREFIX}<hex>', got {revision!r}"
|
| 584 |
+
)
|
| 585 |
+
expected = revision.removeprefix(SHA256_REVISION_PREFIX)
|
| 586 |
+
if len(expected) != _SHA256_HEX_LENGTH or set(expected) - set("0123456789abcdef"):
|
| 587 |
+
raise ValueError(f"{what}: {revision!r} is not a lowercase SHA-256 hex digest")
|
| 588 |
+
if not path.is_file():
|
| 589 |
+
raise ValueError(f"{what}: expected model file {path} does not exist")
|
| 590 |
+
actual = sha256_file(path)
|
| 591 |
+
if actual != expected:
|
| 592 |
+
raise ValueError(f"{what}: {path.name} has sha256 {actual}, the pinned value is {expected}")
|
| 593 |
+
return actual
|
src/nida/data/envs.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Isolated ``uv`` environments for the Phase 1 data-pipeline model workers.
|
| 2 |
+
|
| 3 |
+
Same mechanism as :mod:`nida.bench.envs` (this module only *describes* environments; build
|
| 4 |
+
them with :func:`nida.bench.envs.ensure_env`, which writes the freeze file and the marker):
|
| 5 |
+
one environment per model family because their dependency stacks conflict (NeMo,
|
| 6 |
+
FunASR/ModelScope, transformers, panns-inference/librosa). Python and the torch pins are
|
| 7 |
+
taken from the Phase 0 ``hf`` environment, so every worker environment runs the same
|
| 8 |
+
``torch==2.8.0`` (cu128) build.
|
| 9 |
+
|
| 10 |
+
``EnvSpec.workers`` here are module names under ``nida.data.workers`` (see
|
| 11 |
+
:func:`worker_module_name`), not under ``nida.bench.workers``.
|
| 12 |
+
|
| 13 |
+
Version pins (PyPI, looked up 2026-10-08):
|
| 14 |
+
|
| 15 |
+
* ``panns-inference==0.1.1`` (latest, 2023-03-26) with ``torchlibrosa==0.1.0`` (latest).
|
| 16 |
+
* ``nemo_toolkit[asr]==3.0.0`` (latest, 2026-08-07; ``torch>=2.6.0`` so the torch 2.8.0 wheel
|
| 17 |
+
is kept; the build's torch guard would catch a replacement). The model card asks for
|
| 18 |
+
``Cython`` and ``packaging`` to be installed first. System packages ``libsndfile1`` and
|
| 19 |
+
``ffmpeg`` must exist (present on Colab).
|
| 20 |
+
* ``funasr==1.4.16`` (latest, 2026-09-18).
|
| 21 |
+
* ``speechbrain==1.1.1`` (2026-08-27; the latest, 1.1.2, was published on 2026-10-14 and is not
|
| 22 |
+
pinned yet: one day old). It requires ``torch>=2.1.0`` (the 2.8.0 wheel is kept), and its
|
| 23 |
+
dependency ``hyperpyyaml==1.2.3`` (pinned explicitly) caps ``ruamel.yaml<0.19`` (0.19.x breaks
|
| 24 |
+
HyperPyYAML with ``'Loader' object has no attribute 'max_depth'``), which the resolver
|
| 25 |
+
respects. Importing ``speechbrain`` needs ``torchaudio`` (in the torch pins). Checked locally
|
| 26 |
+
on CPU with torch and torchaudio 2.11.0 (not with the cu128 2.8.0 pair: unverified).
|
| 27 |
+
* ``transformers==4.57.6``: **not** the latest (5.19.0, 2026-10-06). That is a new major
|
| 28 |
+
version never exercised here; 4.57.6 is what the Phase 0 ``hf`` environment already ran
|
| 29 |
+
with torch 2.8.0.
|
| 30 |
+
|
| 31 |
+
Unpinned helpers (numpy, soundfile, huggingface-hub, librosa, matplotlib) follow the bench
|
| 32 |
+
environments; the exact resolved versions are recorded in each environment's ``freeze.txt``.
|
| 33 |
+
|
| 34 |
+
The speech-enhancement arms (:mod:`nida.data.enhance`) have their own mapping,
|
| 35 |
+
:data:`ENHANCE_ENV_SPECS`, because their pins deliberately differ from the shared stack above
|
| 36 |
+
(Python 3.11 for ``dfn``, numpy 1 vs 2, no torch pin for ``uvr``). They are the pins of
|
| 37 |
+
``scripts/enhance_envs.sh`` (exercised for the A/B listening test, 2026-10-13), copied
|
| 38 |
+
unchanged; ``tests/test_data_enhance.py`` compares both. Same ``EnvSpec`` mechanism, so
|
| 39 |
+
``ensure_env`` builds them; :func:`data_env_for_worker` looks in both mappings.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
from __future__ import annotations
|
| 43 |
+
|
| 44 |
+
from collections.abc import Mapping
|
| 45 |
+
from types import MappingProxyType
|
| 46 |
+
|
| 47 |
+
from nida.bench.envs import ENV_SPECS as BENCH_ENV_SPECS
|
| 48 |
+
from nida.bench.envs import EnvSpec
|
| 49 |
+
|
| 50 |
+
__all__ = [
|
| 51 |
+
"DATA_ENV_SPECS",
|
| 52 |
+
"DATA_WORKER_PACKAGE",
|
| 53 |
+
"ENHANCE_ENV_SPECS",
|
| 54 |
+
"data_env_for_worker",
|
| 55 |
+
"worker_module_name",
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
#: Package whose modules are the Phase 1 workers.
|
| 59 |
+
DATA_WORKER_PACKAGE = "nida.data.workers"
|
| 60 |
+
|
| 61 |
+
_BASE = BENCH_ENV_SPECS["hf"] # python + torch pins shared with Phase 0
|
| 62 |
+
|
| 63 |
+
DATA_ENV_SPECS: Mapping[str, EnvSpec] = MappingProxyType(
|
| 64 |
+
{
|
| 65 |
+
"data_sed": EnvSpec(
|
| 66 |
+
name="data_sed",
|
| 67 |
+
python=_BASE.python,
|
| 68 |
+
torch_packages=_BASE.torch_packages,
|
| 69 |
+
torch_index_url=_BASE.torch_index_url,
|
| 70 |
+
packages=(
|
| 71 |
+
"panns-inference==0.1.1",
|
| 72 |
+
"torchlibrosa==0.1.0",
|
| 73 |
+
"librosa",
|
| 74 |
+
"matplotlib",
|
| 75 |
+
"numpy",
|
| 76 |
+
"soundfile",
|
| 77 |
+
),
|
| 78 |
+
workers=("sed_panns",),
|
| 79 |
+
),
|
| 80 |
+
"data_diar": EnvSpec(
|
| 81 |
+
name="data_diar",
|
| 82 |
+
python=_BASE.python,
|
| 83 |
+
torch_packages=_BASE.torch_packages,
|
| 84 |
+
torch_index_url=_BASE.torch_index_url,
|
| 85 |
+
packages=(
|
| 86 |
+
"Cython",
|
| 87 |
+
"packaging",
|
| 88 |
+
"nemo_toolkit[asr]==3.0.0",
|
| 89 |
+
# Unpinned, the resolver picks transformers 4.12 + tokenizers 0.10 (no wheel
|
| 90 |
+
# for py3.12, Rust build fails on Colab); pin the Phase 0 transformers.
|
| 91 |
+
"transformers==4.57.6",
|
| 92 |
+
"huggingface-hub<1.0",
|
| 93 |
+
"numpy",
|
| 94 |
+
"soundfile",
|
| 95 |
+
),
|
| 96 |
+
workers=("diar_sortformer",),
|
| 97 |
+
),
|
| 98 |
+
"data_ctc": EnvSpec(
|
| 99 |
+
name="data_ctc",
|
| 100 |
+
python=_BASE.python,
|
| 101 |
+
torch_packages=_BASE.torch_packages,
|
| 102 |
+
torch_index_url=_BASE.torch_index_url,
|
| 103 |
+
packages=("transformers==4.57.6", "huggingface-hub", "numpy", "soundfile"),
|
| 104 |
+
workers=("ctc_emissions",),
|
| 105 |
+
),
|
| 106 |
+
"data_emotion": EnvSpec(
|
| 107 |
+
name="data_emotion",
|
| 108 |
+
python=_BASE.python,
|
| 109 |
+
torch_packages=_BASE.torch_packages,
|
| 110 |
+
torch_index_url=_BASE.torch_index_url,
|
| 111 |
+
packages=(
|
| 112 |
+
"funasr==1.4.16",
|
| 113 |
+
"transformers==4.57.6", # same resolver trap as data_diar
|
| 114 |
+
"huggingface-hub<1.0",
|
| 115 |
+
"numpy",
|
| 116 |
+
"soundfile",
|
| 117 |
+
),
|
| 118 |
+
workers=("emotion_e2v",),
|
| 119 |
+
),
|
| 120 |
+
# Speaker similarity (evaluation, scripts/speaker_sim.py): SpeechBrain ECAPA-TDNN.
|
| 121 |
+
# scipy: SpeechBrain depends on it and the script resamples with it.
|
| 122 |
+
"data_spk": EnvSpec(
|
| 123 |
+
name="data_spk",
|
| 124 |
+
python=_BASE.python,
|
| 125 |
+
torch_packages=_BASE.torch_packages,
|
| 126 |
+
torch_index_url=_BASE.torch_index_url,
|
| 127 |
+
packages=(
|
| 128 |
+
"speechbrain==1.1.1",
|
| 129 |
+
# The version SpeechBrain 1.1.1 was checked with; it caps ruamel.yaml<0.19.
|
| 130 |
+
"hyperpyyaml==1.2.3",
|
| 131 |
+
"huggingface-hub",
|
| 132 |
+
"numpy",
|
| 133 |
+
"scipy",
|
| 134 |
+
"soundfile",
|
| 135 |
+
),
|
| 136 |
+
workers=("spk_ecapa",),
|
| 137 |
+
),
|
| 138 |
+
# S9 latents: the B2 VAE stack (torch + torchaudio for resampling), without the
|
| 139 |
+
# Audiobox quality model.
|
| 140 |
+
"data_vae": EnvSpec(
|
| 141 |
+
name="data_vae",
|
| 142 |
+
python=_BASE.python,
|
| 143 |
+
torch_packages=BENCH_ENV_SPECS["audio"].torch_packages,
|
| 144 |
+
torch_index_url=_BASE.torch_index_url,
|
| 145 |
+
packages=("numpy", "pydantic", "soundfile", "huggingface-hub"),
|
| 146 |
+
workers=("vae_encode", "vae_decode"),
|
| 147 |
+
),
|
| 148 |
+
}
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
#: scripts/enhance_envs.sh, copied: ``torch==2.8.0`` + ``torchaudio==2.8.0`` is the pair of the
|
| 153 |
+
#: sidon/dfn venvs (PyPI's Linux 2.8.0 wheel is the CUDA 12.8 build, as is the cu128 index
|
| 154 |
+
#: wheel installed here). ``uvr`` pins no torch: ``audio-separator[gpu]==0.47.0`` resolves its
|
| 155 |
+
#: own (torch 2.14.1 and onnxruntime-gpu 1.31.0, CUDA 13 builds, NVIDIA driver >= 580; this
|
| 156 |
+
#: combination was NOT exercised on a GPU: unverified, and to be pinned (torch, onnxruntime-gpu)
|
| 157 |
+
#: from the first GPU run's freeze file) and ``ensure_env`` skips its torch step for an empty
|
| 158 |
+
#: ``torch_packages``. ``[gpu]`` is the extra the script picks when ``nvidia-smi`` exists; if
|
| 159 |
+
#: onnxruntime cannot activate the CUDA provider the worker aborts (it never runs on the CPU
|
| 160 |
+
#: silently) and the A/B script's ``ENH_UVR_EXTRA=cpu`` variant is the fallback.
|
| 161 |
+
ENHANCE_ENV_SPECS: Mapping[str, EnvSpec] = MappingProxyType(
|
| 162 |
+
{
|
| 163 |
+
"data_enh_uvr": EnvSpec(
|
| 164 |
+
name="data_enh_uvr",
|
| 165 |
+
python="3.12",
|
| 166 |
+
torch_packages=(),
|
| 167 |
+
torch_index_url="",
|
| 168 |
+
packages=(
|
| 169 |
+
"audio-separator[gpu]==0.47.0",
|
| 170 |
+
"audioread==3.1.0",
|
| 171 |
+
"numpy==2.5.4",
|
| 172 |
+
"scipy==1.18.1",
|
| 173 |
+
"soundfile==0.14.0",
|
| 174 |
+
),
|
| 175 |
+
workers=("enhance_uvr",),
|
| 176 |
+
),
|
| 177 |
+
"data_enh_sidon": EnvSpec(
|
| 178 |
+
name="data_enh_sidon",
|
| 179 |
+
python="3.12",
|
| 180 |
+
torch_packages=_BASE.torch_packages,
|
| 181 |
+
torch_index_url=_BASE.torch_index_url,
|
| 182 |
+
packages=(
|
| 183 |
+
"transformers==5.19.0",
|
| 184 |
+
"huggingface_hub==2.2.0",
|
| 185 |
+
"numpy==2.5.4",
|
| 186 |
+
"scipy==1.18.1",
|
| 187 |
+
"soundfile==0.14.0",
|
| 188 |
+
),
|
| 189 |
+
workers=("enhance_sidon",),
|
| 190 |
+
),
|
| 191 |
+
"data_enh_dfn": EnvSpec(
|
| 192 |
+
name="data_enh_dfn",
|
| 193 |
+
python="3.11", # deepfilterlib 0.5.6 has wheels for cp38..cp311 only
|
| 194 |
+
torch_packages=_BASE.torch_packages,
|
| 195 |
+
torch_index_url=_BASE.torch_index_url,
|
| 196 |
+
packages=(
|
| 197 |
+
"deepfilternet==0.5.6",
|
| 198 |
+
"numpy==1.26.4",
|
| 199 |
+
"scipy==1.17.1",
|
| 200 |
+
"soundfile==0.14.0",
|
| 201 |
+
),
|
| 202 |
+
workers=("enhance_dfn",),
|
| 203 |
+
),
|
| 204 |
+
}
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def data_env_for_worker(worker: str) -> EnvSpec:
|
| 209 |
+
"""Return the environment spec that runs the Phase 1 ``worker``.
|
| 210 |
+
|
| 211 |
+
Raises:
|
| 212 |
+
ValueError: if no data (or enhancement) environment lists ``worker``.
|
| 213 |
+
"""
|
| 214 |
+
specs = (*DATA_ENV_SPECS.values(), *ENHANCE_ENV_SPECS.values())
|
| 215 |
+
for spec in specs:
|
| 216 |
+
if worker in spec.workers:
|
| 217 |
+
return spec
|
| 218 |
+
known = sorted(w for spec in specs for w in spec.workers)
|
| 219 |
+
raise ValueError(f"unknown data worker {worker!r}; known workers: {known}")
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def worker_module_name(worker: str) -> str:
|
| 223 |
+
"""Dotted module to run with ``python -m`` for ``worker``.
|
| 224 |
+
|
| 225 |
+
Raises:
|
| 226 |
+
ValueError: if ``worker`` is not a known data worker.
|
| 227 |
+
"""
|
| 228 |
+
data_env_for_worker(worker) # validates the name
|
| 229 |
+
return f"{DATA_WORKER_PACKAGE}.{worker}"
|
src/nida/data/events.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""S6 — frame-level sound-event probabilities → :class:`EventDetection` intervals.
|
| 2 |
+
|
| 3 |
+
The tagger (PANNs ``Cnn14_DecisionLevelMax``) gives per-frame probabilities for AudioSet
|
| 4 |
+
classes. Each Nida event kind maps to one or more AudioSet classes (tag-spec §2, R3); the
|
| 5 |
+
kind's frame probability is the maximum over its classes. A detection is a maximal run of
|
| 6 |
+
frames at or above ``onset`` (gaps up to ``merge_gap_ms`` are bridged); its confidence is
|
| 7 |
+
the maximum frame probability inside the run. Fusion (S7) then applies the per-kind
|
| 8 |
+
confidence bands, so ``onset`` must not exceed any band's ``low`` — otherwise a
|
| 9 |
+
middle-band event could vanish here instead of dropping its segment. Runs shorter than
|
| 10 |
+
the kind's ``min_ms`` are not detections; for each kind the highest confidence among such
|
| 11 |
+
blips is returned (``too_short``), never silently discarded — the caller drops a segment
|
| 12 |
+
whose blip reaches the fusion band ``low`` (precision before recall, tag-spec §7).
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
from collections.abc import Mapping, Sequence
|
| 19 |
+
from dataclasses import dataclass
|
| 20 |
+
from typing import TYPE_CHECKING
|
| 21 |
+
|
| 22 |
+
from nida.data.types import EventDetection
|
| 23 |
+
from nida.text.tags import EVENTS
|
| 24 |
+
|
| 25 |
+
if TYPE_CHECKING:
|
| 26 |
+
import numpy as np
|
| 27 |
+
from numpy.typing import NDArray
|
| 28 |
+
|
| 29 |
+
__all__ = ["EventConfig", "FrameDetections", "detect_events"]
|
| 30 |
+
|
| 31 |
+
_NON_DETECTABLE = frozenset({"pause_short", "pause_long"})
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass(frozen=True)
|
| 35 |
+
class EventConfig:
|
| 36 |
+
"""configs/data/events.yaml. ``classes[kind]`` lists AudioSet display names."""
|
| 37 |
+
|
| 38 |
+
classes: Mapping[str, tuple[str, ...]]
|
| 39 |
+
onset: float
|
| 40 |
+
merge_gap_ms: int
|
| 41 |
+
min_ms: Mapping[str, int]
|
| 42 |
+
|
| 43 |
+
def __post_init__(self) -> None:
|
| 44 |
+
kinds = set(EVENTS) - _NON_DETECTABLE
|
| 45 |
+
if set(self.classes) != kinds:
|
| 46 |
+
raise ValueError(
|
| 47 |
+
f"classes must map exactly {sorted(kinds)}; "
|
| 48 |
+
f"missing={sorted(kinds - set(self.classes))} "
|
| 49 |
+
f"extra={sorted(set(self.classes) - kinds)}"
|
| 50 |
+
)
|
| 51 |
+
if set(self.min_ms) != kinds:
|
| 52 |
+
raise ValueError("min_ms must have exactly one entry per detectable event kind")
|
| 53 |
+
seen: dict[str, str] = {}
|
| 54 |
+
for kind, names in self.classes.items():
|
| 55 |
+
if not names:
|
| 56 |
+
raise ValueError(f"event kind {kind!r} maps to no AudioSet class")
|
| 57 |
+
for name in names:
|
| 58 |
+
if name in seen:
|
| 59 |
+
raise ValueError(
|
| 60 |
+
f"AudioSet class {name!r} mapped to {seen[name]!r} and {kind!r}"
|
| 61 |
+
)
|
| 62 |
+
seen[name] = kind
|
| 63 |
+
if not (math.isfinite(self.onset) and 0.0 < self.onset <= 1.0):
|
| 64 |
+
raise ValueError(f"onset must be in (0, 1], got {self.onset!r}")
|
| 65 |
+
for name, value in [("merge_gap_ms", self.merge_gap_ms), *self.min_ms.items()]:
|
| 66 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
| 67 |
+
raise ValueError(f"{name} must be a non-negative int, got {value!r}")
|
| 68 |
+
|
| 69 |
+
@classmethod
|
| 70 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> EventConfig:
|
| 71 |
+
expected = {"classes", "onset", "merge_gap_ms", "min_ms"}
|
| 72 |
+
if set(raw) != expected:
|
| 73 |
+
raise ValueError(
|
| 74 |
+
f"event config keys: missing={sorted(expected - set(raw))} "
|
| 75 |
+
f"unknown={sorted(set(raw) - expected)}"
|
| 76 |
+
)
|
| 77 |
+
classes, min_ms, onset, gap = (
|
| 78 |
+
raw["classes"],
|
| 79 |
+
raw["min_ms"],
|
| 80 |
+
raw["onset"],
|
| 81 |
+
raw["merge_gap_ms"],
|
| 82 |
+
)
|
| 83 |
+
if not isinstance(classes, Mapping) or not all(
|
| 84 |
+
isinstance(v, list) and all(isinstance(n, str) for n in v) for v in classes.values()
|
| 85 |
+
):
|
| 86 |
+
raise ValueError("classes must map kinds to lists of class names")
|
| 87 |
+
if not isinstance(min_ms, Mapping):
|
| 88 |
+
raise ValueError("min_ms must be a mapping")
|
| 89 |
+
if isinstance(onset, bool) or not isinstance(onset, int | float):
|
| 90 |
+
raise ValueError("onset must be a number")
|
| 91 |
+
if isinstance(gap, bool) or not isinstance(gap, int):
|
| 92 |
+
raise ValueError("merge_gap_ms must be an int")
|
| 93 |
+
return cls(
|
| 94 |
+
classes={str(k): tuple(v) for k, v in classes.items()},
|
| 95 |
+
onset=float(onset),
|
| 96 |
+
merge_gap_ms=gap,
|
| 97 |
+
min_ms={str(k): v for k, v in min_ms.items()}, # validated in __post_init__
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
def check_onset_against(self, band_lows: Mapping[str, float]) -> None:
|
| 101 |
+
"""Raise if ``onset`` exceeds a fusion band's ``low`` for some event kind."""
|
| 102 |
+
missing = sorted(set(self.classes) - set(band_lows))
|
| 103 |
+
if missing:
|
| 104 |
+
raise ValueError(f"no fusion band for event kinds {missing}")
|
| 105 |
+
too_high = sorted(k for k in self.classes if band_lows[k] < self.onset)
|
| 106 |
+
if too_high:
|
| 107 |
+
raise ValueError(f"onset {self.onset} above the fusion band low of {too_high}")
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
@dataclass(frozen=True)
|
| 111 |
+
class FrameDetections:
|
| 112 |
+
detections: tuple[EventDetection, ...]
|
| 113 |
+
too_short: Mapping[str, float] # kind -> max confidence among too-short runs
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def detect_events(
|
| 117 |
+
probs: NDArray[np.float32],
|
| 118 |
+
class_names: Sequence[str],
|
| 119 |
+
*,
|
| 120 |
+
frame_hop_ms: float,
|
| 121 |
+
duration_ms: int,
|
| 122 |
+
cfg: EventConfig,
|
| 123 |
+
source: str,
|
| 124 |
+
) -> FrameDetections:
|
| 125 |
+
"""Turn framewise class probabilities ``[num_frames, len(class_names)]`` into detections.
|
| 126 |
+
|
| 127 |
+
Frame ``t`` covers ``[t * hop, (t + 1) * hop)`` ms, clamped to ``duration_ms``.
|
| 128 |
+
|
| 129 |
+
Raises:
|
| 130 |
+
ValueError: shape mismatch, values outside [0, 1] or non-finite, a configured
|
| 131 |
+
class missing from ``class_names``, a bad hop or duration, or a frame count
|
| 132 |
+
inconsistent with ``duration_ms`` (more than one frame of slack).
|
| 133 |
+
"""
|
| 134 |
+
import numpy as np
|
| 135 |
+
|
| 136 |
+
if probs.ndim != 2 or probs.shape[1] != len(class_names):
|
| 137 |
+
raise ValueError(f"probs shape {probs.shape} does not match {len(class_names)} classes")
|
| 138 |
+
if len(set(class_names)) != len(class_names):
|
| 139 |
+
raise ValueError("duplicate class names")
|
| 140 |
+
in_range = probs.size == 0 or (float(np.min(probs)) >= 0.0 and float(np.max(probs)) <= 1.0)
|
| 141 |
+
if not np.all(np.isfinite(probs)) or not in_range:
|
| 142 |
+
raise ValueError("probabilities must be finite and in [0, 1]")
|
| 143 |
+
if not (math.isfinite(frame_hop_ms) and frame_hop_ms >= 1):
|
| 144 |
+
raise ValueError(f"frame_hop_ms must be a finite number >= 1 ms, got {frame_hop_ms!r}")
|
| 145 |
+
if type(duration_ms) is not int or duration_ms <= 0:
|
| 146 |
+
raise ValueError(f"invalid duration_ms {duration_ms!r}")
|
| 147 |
+
expected_frames = duration_ms / frame_hop_ms
|
| 148 |
+
if abs(probs.shape[0] - expected_frames) > 1 + 1e-9:
|
| 149 |
+
raise ValueError(
|
| 150 |
+
f"{probs.shape[0]} frames for {duration_ms} ms at {frame_hop_ms} ms hop "
|
| 151 |
+
f"(expected about {expected_frames:.1f})"
|
| 152 |
+
)
|
| 153 |
+
column = {name: i for i, name in enumerate(class_names)}
|
| 154 |
+
missing = sorted({n for names in cfg.classes.values() for n in names} - set(column))
|
| 155 |
+
if missing:
|
| 156 |
+
raise ValueError(f"tagger output lacks configured classes {missing}")
|
| 157 |
+
gap_frames = math.floor(cfg.merge_gap_ms / frame_hop_ms)
|
| 158 |
+
|
| 159 |
+
detections: list[EventDetection] = []
|
| 160 |
+
too_short: dict[str, float] = {}
|
| 161 |
+
for kind in sorted(cfg.classes):
|
| 162 |
+
kind_probs = probs[:, [column[n] for n in cfg.classes[kind]]].max(axis=1)
|
| 163 |
+
active = np.flatnonzero(kind_probs >= cfg.onset)
|
| 164 |
+
if active.size == 0:
|
| 165 |
+
continue
|
| 166 |
+
runs: list[tuple[int, int]] = [] # inclusive frame ranges
|
| 167 |
+
start = prev = int(active[0])
|
| 168 |
+
for frame in active[1:].tolist():
|
| 169 |
+
if frame - prev - 1 > gap_frames:
|
| 170 |
+
runs.append((start, prev))
|
| 171 |
+
start = frame
|
| 172 |
+
prev = frame
|
| 173 |
+
runs.append((start, prev))
|
| 174 |
+
for first, last in runs:
|
| 175 |
+
start_ms = min(round(first * frame_hop_ms), duration_ms - 1)
|
| 176 |
+
end_ms = min(round((last + 1) * frame_hop_ms), duration_ms)
|
| 177 |
+
conf = float(np.max(kind_probs[first : last + 1]))
|
| 178 |
+
if end_ms - start_ms < cfg.min_ms[kind]:
|
| 179 |
+
too_short[kind] = max(too_short.get(kind, 0.0), conf)
|
| 180 |
+
continue
|
| 181 |
+
detections.append(EventDetection(kind, start_ms, end_ms, conf, source))
|
| 182 |
+
detections.sort(key=lambda d: (d.start_ms, d.end_ms, d.kind))
|
| 183 |
+
return FrameDetections(tuple(detections), too_short)
|
src/nida/data/fillers_ctc.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Filler candidates from CTC emissions in the gaps between aligned words (prototype).
|
| 2 |
+
|
| 3 |
+
docs/research/dolgu-tespiti-oneri.md, option A. Neither the primary nor the secondary ASR
|
| 4 |
+
writes fillers (B1-C), but the character CTC model may still emit their letters where the
|
| 5 |
+
speaker hesitates. For every silent-in-the-transcript gap (segment start → first word,
|
| 6 |
+
between words, last word → end) of at least ``min_gap_ms``, the emissions inside the gap
|
| 7 |
+
are greedily decoded (argmax, collapse repeats, drop blanks and word delimiters). If the
|
| 8 |
+
decoded string is a filler spelling, the gap yields a candidate.
|
| 9 |
+
|
| 10 |
+
Option A was accepted by the user (2026-10-09). :func:`insert_fillers` turns accepted
|
| 11 |
+
candidates into extra :class:`TimedWord` items before fusion (S7); whether that happens is
|
| 12 |
+
``configs/data/fillers.yaml`` ``enabled`` (off until the filler listening panel shows the
|
| 13 |
+
precision, docs/research/dolgu-tespiti-oneri.md).
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import math
|
| 19 |
+
from collections.abc import Mapping, Sequence
|
| 20 |
+
from dataclasses import dataclass
|
| 21 |
+
from itertools import pairwise
|
| 22 |
+
from typing import TYPE_CHECKING
|
| 23 |
+
|
| 24 |
+
from nida.data.types import TimedWord
|
| 25 |
+
from nida.text.fillers import canonical_filler
|
| 26 |
+
|
| 27 |
+
if TYPE_CHECKING:
|
| 28 |
+
import numpy as np
|
| 29 |
+
from numpy.typing import NDArray
|
| 30 |
+
|
| 31 |
+
__all__ = [
|
| 32 |
+
"CTC_ONLY_FILLERS",
|
| 33 |
+
"FillerCandidate",
|
| 34 |
+
"FillerConfig",
|
| 35 |
+
"gap_filler_candidates",
|
| 36 |
+
"insert_fillers",
|
| 37 |
+
]
|
| 38 |
+
|
| 39 |
+
#: Greedy CTC collapses doubled letters without a blank between them, so a drawn-out
|
| 40 |
+
#: "ıı" or "mm" often decodes as a single letter. These single-letter forms are accepted
|
| 41 |
+
#: *only* inside a gap (never in running text, where "e" is a conjunction).
|
| 42 |
+
CTC_ONLY_FILLERS: Mapping[str, str] = {"e": "ee", "m": "hmm", "a": "aa"}
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class FillerCandidate:
|
| 47 |
+
"""A proposed filler: canonical text, time span (ms) and evidence.
|
| 48 |
+
|
| 49 |
+
``start_ms``/``end_ms`` is the examined gap; ``speech_start_ms``/``speech_end_ms`` the
|
| 50 |
+
part of it from the first to the last non-blank frame (where the filler is voiced).
|
| 51 |
+
``conf`` is the mean argmax probability over the gap's non-blank frames; ``decoded`` is
|
| 52 |
+
the raw greedy string.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
text: str
|
| 56 |
+
start_ms: int
|
| 57 |
+
end_ms: int
|
| 58 |
+
conf: float
|
| 59 |
+
decoded: str
|
| 60 |
+
speech_start_ms: int
|
| 61 |
+
speech_end_ms: int
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@dataclass(frozen=True)
|
| 65 |
+
class FillerConfig:
|
| 66 |
+
"""``configs/data/fillers.yaml``. All keys required."""
|
| 67 |
+
|
| 68 |
+
enabled: bool
|
| 69 |
+
min_gap_ms: int
|
| 70 |
+
min_conf: float
|
| 71 |
+
fillers: frozenset[str]
|
| 72 |
+
|
| 73 |
+
def __post_init__(self) -> None:
|
| 74 |
+
if not isinstance(self.enabled, bool):
|
| 75 |
+
raise ValueError("enabled must be a bool")
|
| 76 |
+
if isinstance(self.min_gap_ms, bool) or not isinstance(self.min_gap_ms, int):
|
| 77 |
+
raise ValueError("min_gap_ms must be an int")
|
| 78 |
+
if self.min_gap_ms < 1:
|
| 79 |
+
raise ValueError("min_gap_ms must be >= 1")
|
| 80 |
+
if isinstance(self.min_conf, bool) or not isinstance(self.min_conf, int | float):
|
| 81 |
+
raise ValueError("min_conf must be a number")
|
| 82 |
+
if not (math.isfinite(self.min_conf) and 0.0 <= self.min_conf <= 1.0):
|
| 83 |
+
raise ValueError("min_conf must be in [0, 1]")
|
| 84 |
+
unknown = sorted(f for f in self.fillers if canonical_filler(f) != f)
|
| 85 |
+
if unknown or not self.fillers:
|
| 86 |
+
raise ValueError(f"fillers must be non-empty canonical filler forms, bad: {unknown}")
|
| 87 |
+
|
| 88 |
+
@classmethod
|
| 89 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> FillerConfig:
|
| 90 |
+
expected = {"enabled", "min_gap_ms", "min_conf", "fillers"}
|
| 91 |
+
if set(raw) != expected:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
f"fillers config keys: missing={sorted(expected - set(raw))} "
|
| 94 |
+
f"unknown={sorted(set(raw) - expected)}"
|
| 95 |
+
)
|
| 96 |
+
fillers = raw["fillers"]
|
| 97 |
+
if not isinstance(fillers, list) or not all(isinstance(f, str) for f in fillers):
|
| 98 |
+
raise ValueError("fillers must be a list of strings")
|
| 99 |
+
return cls(
|
| 100 |
+
enabled=raw["enabled"], # type: ignore[arg-type] # validated in __post_init__
|
| 101 |
+
min_gap_ms=raw["min_gap_ms"], # type: ignore[arg-type]
|
| 102 |
+
min_conf=raw["min_conf"], # type: ignore[arg-type]
|
| 103 |
+
fillers=frozenset(fillers),
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _greedy(
|
| 108 |
+
log_probs: NDArray[np.float64], skip: set[int], id_to_token: Sequence[str]
|
| 109 |
+
) -> tuple[str, float, int, int]:
|
| 110 |
+
"""Collapsed argmax string of ``log_probs``, mean prob of its non-skipped frames, and
|
| 111 |
+
the first / one-past-last non-skipped frame (``0, 0`` when there is none)."""
|
| 112 |
+
import numpy as np
|
| 113 |
+
|
| 114 |
+
best = np.argmax(log_probs, axis=1)
|
| 115 |
+
chars: list[str] = []
|
| 116 |
+
probs: list[float] = []
|
| 117 |
+
frames: list[int] = []
|
| 118 |
+
previous = -1
|
| 119 |
+
for frame, token in enumerate(best.tolist()):
|
| 120 |
+
if token not in skip:
|
| 121 |
+
probs.append(math.exp(float(log_probs[frame, token])))
|
| 122 |
+
frames.append(frame)
|
| 123 |
+
if token != previous:
|
| 124 |
+
chars.append(id_to_token[token])
|
| 125 |
+
previous = token
|
| 126 |
+
if not frames:
|
| 127 |
+
return "", 0.0, 0, 0
|
| 128 |
+
return "".join(chars), sum(probs) / len(probs), frames[0], frames[-1] + 1
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def gap_filler_candidates(
|
| 132 |
+
words: Sequence[TimedWord],
|
| 133 |
+
log_probs: NDArray[np.float32],
|
| 134 |
+
vocab: Mapping[str, int],
|
| 135 |
+
*,
|
| 136 |
+
blank_token: str,
|
| 137 |
+
word_delimiter: str,
|
| 138 |
+
frame_hop_ms: float,
|
| 139 |
+
duration_ms: int,
|
| 140 |
+
min_gap_ms: int = 150,
|
| 141 |
+
) -> list[FillerCandidate]:
|
| 142 |
+
"""Filler candidates in the gaps of an aligned segment.
|
| 143 |
+
|
| 144 |
+
Args:
|
| 145 |
+
words: Aligned words (relative ms, in order, non-overlapping), possibly empty.
|
| 146 |
+
log_probs: ``[num_frames, vocab_size]`` CTC emissions (log-softmax re-applied here).
|
| 147 |
+
vocab: Token → id, ids exactly ``0..V-1``.
|
| 148 |
+
blank_token, word_delimiter: Special tokens (``[PAD]``, ``|`` for wav2vec2 HF).
|
| 149 |
+
frame_hop_ms: Frame stride (20 ms for wav2vec2).
|
| 150 |
+
duration_ms: Segment length.
|
| 151 |
+
min_gap_ms: Shorter gaps are not examined.
|
| 152 |
+
|
| 153 |
+
Returns:
|
| 154 |
+
Candidates in time order.
|
| 155 |
+
|
| 156 |
+
Raises:
|
| 157 |
+
ValueError: malformed inputs.
|
| 158 |
+
"""
|
| 159 |
+
import numpy as np
|
| 160 |
+
|
| 161 |
+
if log_probs.ndim != 2 or log_probs.shape[1] != len(vocab):
|
| 162 |
+
raise ValueError(f"log_probs shape {log_probs.shape} does not match vocab {len(vocab)}")
|
| 163 |
+
if sorted(vocab.values()) != list(range(len(vocab))):
|
| 164 |
+
raise ValueError("vocab ids must be exactly 0..V-1")
|
| 165 |
+
if blank_token not in vocab or word_delimiter not in vocab:
|
| 166 |
+
raise ValueError("blank or word delimiter missing from vocab")
|
| 167 |
+
if not (math.isfinite(frame_hop_ms) and frame_hop_ms >= 1):
|
| 168 |
+
raise ValueError(f"frame_hop_ms must be finite and >= 1, got {frame_hop_ms!r}")
|
| 169 |
+
if min_gap_ms < 1 or duration_ms <= 0:
|
| 170 |
+
raise ValueError("min_gap_ms and duration_ms must be positive")
|
| 171 |
+
lp = log_probs.astype(np.float64)
|
| 172 |
+
row_max = np.max(lp, axis=1, keepdims=True)
|
| 173 |
+
if not np.all(np.isfinite(row_max)):
|
| 174 |
+
raise ValueError("a frame has no finite log-probability")
|
| 175 |
+
lp = lp - (row_max + np.log(np.sum(np.exp(lp - row_max), axis=1, keepdims=True)))
|
| 176 |
+
id_to_token = [""] * len(vocab)
|
| 177 |
+
for token, index in vocab.items():
|
| 178 |
+
id_to_token[index] = token
|
| 179 |
+
skip = {vocab[blank_token], vocab[word_delimiter]}
|
| 180 |
+
|
| 181 |
+
edges = [0, *[x for w in words for x in (w.start_ms, w.end_ms)], duration_ms]
|
| 182 |
+
candidates: list[FillerCandidate] = []
|
| 183 |
+
for start_ms, end_ms in zip(edges[0::2], edges[1::2], strict=True):
|
| 184 |
+
if end_ms - start_ms < min_gap_ms:
|
| 185 |
+
continue
|
| 186 |
+
first = math.ceil(start_ms / frame_hop_ms)
|
| 187 |
+
last = min(math.floor(end_ms / frame_hop_ms), lp.shape[0])
|
| 188 |
+
if last <= first:
|
| 189 |
+
continue
|
| 190 |
+
decoded, conf, voiced_first, voiced_last = _greedy(lp[first:last], skip, id_to_token)
|
| 191 |
+
if not decoded:
|
| 192 |
+
continue
|
| 193 |
+
text = canonical_filler(decoded) or CTC_ONLY_FILLERS.get(decoded)
|
| 194 |
+
if text is not None:
|
| 195 |
+
speech_start = max(start_ms, round((first + voiced_first) * frame_hop_ms))
|
| 196 |
+
speech_end = min(end_ms, round((first + voiced_last) * frame_hop_ms))
|
| 197 |
+
if speech_end <= speech_start: # rounding at a gap edge: keep one ms
|
| 198 |
+
speech_start = min(speech_start, end_ms - 1)
|
| 199 |
+
speech_end = speech_start + 1
|
| 200 |
+
candidates.append(
|
| 201 |
+
FillerCandidate(text, start_ms, end_ms, conf, decoded, speech_start, speech_end)
|
| 202 |
+
)
|
| 203 |
+
return candidates
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def insert_fillers(
|
| 207 |
+
words: Sequence[TimedWord], candidates: Sequence[FillerCandidate], cfg: FillerConfig
|
| 208 |
+
) -> tuple[tuple[TimedWord, ...], list[FillerCandidate]]:
|
| 209 |
+
"""Words with the accepted candidates added as filler words, in time order.
|
| 210 |
+
|
| 211 |
+
A candidate is accepted when its text is in ``cfg.fillers`` and ``conf >= min_conf``;
|
| 212 |
+
it becomes ``TimedWord(text, speech_start_ms, speech_end_ms)`` (no alignment score).
|
| 213 |
+
Candidates come from the gaps of ``words``, so they never overlap a word.
|
| 214 |
+
|
| 215 |
+
Returns:
|
| 216 |
+
``(words_with_fillers, accepted_candidates)``.
|
| 217 |
+
|
| 218 |
+
Raises:
|
| 219 |
+
ValueError: a candidate overlaps a word or another candidate (a caller bug).
|
| 220 |
+
"""
|
| 221 |
+
accepted = [c for c in candidates if c.text in cfg.fillers and c.conf >= cfg.min_conf]
|
| 222 |
+
added = [TimedWord(c.text, c.speech_start_ms, c.speech_end_ms) for c in accepted]
|
| 223 |
+
merged = sorted([*words, *added], key=lambda w: (w.start_ms, w.end_ms))
|
| 224 |
+
for a, b in pairwise(merged):
|
| 225 |
+
if b.start_ms < a.end_ms:
|
| 226 |
+
raise ValueError(f"filler overlaps a word: {a} / {b}")
|
| 227 |
+
return tuple(merged), accepted
|
src/nida/data/fusion.py
ADDED
|
@@ -0,0 +1,842 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""S7 — deterministic fusion of transcript, alignment and paralinguistic detections.
|
| 2 |
+
|
| 3 |
+
Turns one segment's aligned words (S5), event detections and per-word style scores (S6)
|
| 4 |
+
into tagged text following docs/02-tag-spec.md, or into a quarantine record with a
|
| 5 |
+
machine-readable reason. No LLM, no randomness: the same inputs always give the same
|
| 6 |
+
output. Rules (docs/03-data-pipeline.md S7, tag-spec §7):
|
| 7 |
+
|
| 8 |
+
1. Bands: a detection below ``low`` is ignored (and counted); at or above ``high`` it is
|
| 9 |
+
used; in the middle band ``[low, high)`` it drops the whole segment
|
| 10 |
+
(``ambiguous_event`` / ``ambiguous_style``) — unless every millisecond of it is
|
| 11 |
+
covered by confident detections of the same family (then the event is there anyway).
|
| 12 |
+
2. Families: ``laugh`` and ``laugh_long`` are one family, every other kind its own.
|
| 13 |
+
Confident detections of one family that overlap or are at most ``merge_gap_ms`` apart
|
| 14 |
+
are merged.
|
| 15 |
+
3. Speech overlap: a laugh/cry detection covering more than ``style_word_overlap`` of a
|
| 16 |
+
word's duration marks the word ``laughing`` / ``crying``. If its longest non-speech
|
| 17 |
+
part is at least ``min_residual_event_ms`` it is *also* placed as an event. Any other
|
| 18 |
+
detection whose speech overlap exceeds ``max_event_speech_overlap`` of its length
|
| 19 |
+
drops the segment (``event_overlaps_speech``: false positive or overlapped audio).
|
| 20 |
+
4. Placement: an event is placed at the word gap nearest the middle of its longest
|
| 21 |
+
non-speech part. A laugh becomes ``laugh_long`` if a detector said so or that placed
|
| 22 |
+
part lasts at least ``laugh_long_min_ms``. Placed events of different kinds whose
|
| 23 |
+
parts overlap by more than ``max_cross_kind_overlap`` of the shorter one drop the
|
| 24 |
+
segment (``conflicting_events``): taggers often fire related classes on one sound.
|
| 25 |
+
5. Spans: consecutive words marked with the same style form a span if they are at least
|
| 26 |
+
``min_span_words`` words or ``min_span_ms`` long; a shorter run drops the segment
|
| 27 |
+
(``short_style_run``: audible but untaggable). A word with two styles drops the
|
| 28 |
+
segment (``conflicting_styles``). Per-word score tracks of an ``ignored_styles``
|
| 29 |
+
style are discarded before any of this: the detector is known to be unreliable, so
|
| 30 |
+
it neither tags nor drops anything (unlike ``disabled_tags``, which drop the segment
|
| 31 |
+
whenever the detector fires at or above ``low``).
|
| 32 |
+
6. Pauses: measured on the longest part of an inter-word gap not covered by a confident
|
| 33 |
+
detection. Longer than ``pause_long_ms`` → ``pause_long``; at least ``pause_short_ms``
|
| 34 |
+
→ ``pause_short`` unless the previous word ends in punctuation. None at segment edges.
|
| 35 |
+
7. Text: words are NFC; standalone fillers are canonicalised to their lowercase canonical
|
| 36 |
+
form (tag-spec §4); words with digits drop the segment (``unnormalised_number``) until
|
| 37 |
+
number normalisation exists; ``[ ] < >`` in a word drops it
|
| 38 |
+
(``reserved_char_in_text``). The output is validated by round-tripping through
|
| 39 |
+
:func:`nida.text.tags.parse` (``invalid_output``).
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
from __future__ import annotations
|
| 43 |
+
|
| 44 |
+
import math
|
| 45 |
+
import unicodedata
|
| 46 |
+
from collections.abc import Mapping, Sequence
|
| 47 |
+
from dataclasses import dataclass, field
|
| 48 |
+
|
| 49 |
+
from nida.data.types import EventDetection, QuarantineRecord, TimedWord, WordStyleScore
|
| 50 |
+
from nida.text.fillers import canonical_filler
|
| 51 |
+
from nida.text.tags import EVENTS, SPANS, Event, Item, Span, TagSyntaxError, Word, parse, render
|
| 52 |
+
from nida.text.turkish import nfc, tr_lower
|
| 53 |
+
|
| 54 |
+
__all__ = [
|
| 55 |
+
"EMOTION_LABELS",
|
| 56 |
+
"STAGE",
|
| 57 |
+
"STYLE_FROM_EVENT",
|
| 58 |
+
"Band",
|
| 59 |
+
"EmotionCondition",
|
| 60 |
+
"FusionConfig",
|
| 61 |
+
"FusionResult",
|
| 62 |
+
"TagRecord",
|
| 63 |
+
"emotion_condition",
|
| 64 |
+
"fuse_segment",
|
| 65 |
+
"normalise_word",
|
| 66 |
+
]
|
| 67 |
+
|
| 68 |
+
STAGE = "fusion"
|
| 69 |
+
|
| 70 |
+
#: Event families that, when they overlap words, mean the words carry a style (rule 3).
|
| 71 |
+
STYLE_FROM_EVENT: Mapping[str, str] = {"laugh": "laughing", "cry": "crying"}
|
| 72 |
+
|
| 73 |
+
#: Output classes of the emotion model (emotion2vec+); ``other``/``unknown`` map to null.
|
| 74 |
+
EMOTION_LABELS: frozenset[str] = frozenset(
|
| 75 |
+
{"angry", "disgusted", "fearful", "happy", "neutral", "other", "sad", "surprised", "unknown"}
|
| 76 |
+
)
|
| 77 |
+
_NULL_EMOTIONS = frozenset({"other", "unknown"})
|
| 78 |
+
_PAUSE_EVENTS = frozenset({"pause_short", "pause_long"})
|
| 79 |
+
_RESERVED_CHARS = frozenset("[]<>")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _family(kind: str) -> str:
|
| 83 |
+
return "laugh" if kind in {"laugh", "laugh_long"} else kind
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@dataclass(frozen=True)
|
| 87 |
+
class Band:
|
| 88 |
+
"""Confidence band: ``< low`` ignored, ``[low, high)`` ambiguous, ``>= high`` used."""
|
| 89 |
+
|
| 90 |
+
low: float
|
| 91 |
+
high: float
|
| 92 |
+
|
| 93 |
+
def __post_init__(self) -> None:
|
| 94 |
+
if not (0.0 <= self.low <= self.high <= 1.0):
|
| 95 |
+
raise ValueError(f"invalid band low={self.low} high={self.high}")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
_INT_FIELDS = (
|
| 99 |
+
"min_span_words",
|
| 100 |
+
"min_span_ms",
|
| 101 |
+
"min_residual_event_ms",
|
| 102 |
+
"laugh_long_min_ms",
|
| 103 |
+
"merge_gap_ms",
|
| 104 |
+
"pause_short_ms",
|
| 105 |
+
"pause_long_ms",
|
| 106 |
+
)
|
| 107 |
+
_FRACTION_FIELDS = (
|
| 108 |
+
"style_word_overlap",
|
| 109 |
+
"max_event_speech_overlap",
|
| 110 |
+
"max_cross_kind_overlap",
|
| 111 |
+
"emotion_min_conf",
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
@dataclass(frozen=True)
|
| 116 |
+
class FusionConfig:
|
| 117 |
+
"""Fusion thresholds (configs/data/fusion.yaml). All fields are required."""
|
| 118 |
+
|
| 119 |
+
bands: Mapping[str, Band]
|
| 120 |
+
disabled_tags: frozenset[str]
|
| 121 |
+
ignored_styles: frozenset[str]
|
| 122 |
+
min_span_words: int
|
| 123 |
+
min_span_ms: int
|
| 124 |
+
style_word_overlap: float
|
| 125 |
+
max_event_speech_overlap: float
|
| 126 |
+
max_cross_kind_overlap: float
|
| 127 |
+
min_residual_event_ms: int
|
| 128 |
+
laugh_long_min_ms: int
|
| 129 |
+
merge_gap_ms: int
|
| 130 |
+
pause_short_ms: int
|
| 131 |
+
pause_long_ms: int
|
| 132 |
+
emotion_min_conf: float
|
| 133 |
+
|
| 134 |
+
def __post_init__(self) -> None:
|
| 135 |
+
required = (set(EVENTS) - _PAUSE_EVENTS) | set(SPANS)
|
| 136 |
+
if set(self.bands) != required:
|
| 137 |
+
missing = sorted(required - set(self.bands))
|
| 138 |
+
extra = sorted(set(self.bands) - required)
|
| 139 |
+
raise ValueError(f"bands must cover exactly the detectable tags: {missing=} {extra=}")
|
| 140 |
+
unknown = self.disabled_tags - (set(EVENTS) | set(SPANS))
|
| 141 |
+
if unknown:
|
| 142 |
+
raise ValueError(f"unknown disabled tags {sorted(unknown)}")
|
| 143 |
+
# Only styles that come as per-word score tracks can be ignored; laughing/crying
|
| 144 |
+
# come from events (STYLE_FROM_EVENT) and have no track to discard.
|
| 145 |
+
scored = set(SPANS) - set(STYLE_FROM_EVENT.values())
|
| 146 |
+
not_styles = self.ignored_styles - scored
|
| 147 |
+
if not_styles:
|
| 148 |
+
raise ValueError(
|
| 149 |
+
f"ignored_styles must be span styles with score tracks {sorted(scored)}, "
|
| 150 |
+
f"got {sorted(not_styles)}"
|
| 151 |
+
)
|
| 152 |
+
both = self.ignored_styles & self.disabled_tags
|
| 153 |
+
if both:
|
| 154 |
+
raise ValueError(f"tags both disabled and ignored: {sorted(both)}")
|
| 155 |
+
for name in _INT_FIELDS:
|
| 156 |
+
value = getattr(self, name)
|
| 157 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
| 158 |
+
raise ValueError(f"{name} must be a non-negative int, got {value!r}")
|
| 159 |
+
if self.min_span_words < 1:
|
| 160 |
+
raise ValueError("min_span_words must be >= 1")
|
| 161 |
+
if self.min_residual_event_ms < 1:
|
| 162 |
+
raise ValueError("min_residual_event_ms must be >= 1")
|
| 163 |
+
for name in _FRACTION_FIELDS:
|
| 164 |
+
value = getattr(self, name)
|
| 165 |
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
| 166 |
+
raise ValueError(f"{name} must be a number, got {value!r}")
|
| 167 |
+
if not (math.isfinite(value) and 0.0 <= value <= 1.0):
|
| 168 |
+
raise ValueError(f"{name} must be in [0, 1], got {value!r}")
|
| 169 |
+
if not 0 < self.pause_short_ms <= self.pause_long_ms:
|
| 170 |
+
raise ValueError("need 0 < pause_short_ms <= pause_long_ms")
|
| 171 |
+
|
| 172 |
+
@classmethod
|
| 173 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> FusionConfig:
|
| 174 |
+
"""Build from a parsed YAML mapping; unknown or missing keys are errors."""
|
| 175 |
+
expected = {f.name for f in cls.__dataclass_fields__.values()}
|
| 176 |
+
if set(raw) != expected:
|
| 177 |
+
raise ValueError(
|
| 178 |
+
f"fusion config keys: missing={sorted(expected - set(raw))} "
|
| 179 |
+
f"unknown={sorted(set(raw) - expected)}"
|
| 180 |
+
)
|
| 181 |
+
bands_raw = raw["bands"]
|
| 182 |
+
if not isinstance(bands_raw, Mapping):
|
| 183 |
+
raise ValueError("bands must be a mapping")
|
| 184 |
+
bands: dict[str, Band] = {}
|
| 185 |
+
for kind, band in bands_raw.items():
|
| 186 |
+
if not isinstance(band, Mapping) or set(band) != {"low", "high"}:
|
| 187 |
+
raise ValueError(f"band {kind!r} must be a mapping with exactly low/high")
|
| 188 |
+
low, high = band["low"], band["high"]
|
| 189 |
+
if not all(isinstance(v, int | float) and not isinstance(v, bool) for v in (low, high)):
|
| 190 |
+
raise ValueError(f"band {kind!r} values must be numbers")
|
| 191 |
+
bands[str(kind)] = Band(float(low), float(high))
|
| 192 |
+
disabled = raw["disabled_tags"]
|
| 193 |
+
if not isinstance(disabled, list) or not all(isinstance(t, str) for t in disabled):
|
| 194 |
+
raise ValueError("disabled_tags must be a list of tag ids")
|
| 195 |
+
ignored = raw["ignored_styles"]
|
| 196 |
+
if not isinstance(ignored, list) or not all(isinstance(t, str) for t in ignored):
|
| 197 |
+
raise ValueError("ignored_styles must be a list of style ids")
|
| 198 |
+
lists = {"bands", "disabled_tags", "ignored_styles"}
|
| 199 |
+
values = {k: v for k, v in raw.items() if k not in lists}
|
| 200 |
+
# The remaining values are typed `object` here; __post_init__ validates each one.
|
| 201 |
+
return cls(
|
| 202 |
+
bands=bands,
|
| 203 |
+
disabled_tags=frozenset(disabled),
|
| 204 |
+
ignored_styles=frozenset(ignored),
|
| 205 |
+
**values, # type: ignore[arg-type]
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
@dataclass(frozen=True)
|
| 210 |
+
class TagRecord:
|
| 211 |
+
"""Provenance of one emitted tag (tag-spec §7.3): ``{id, start, end, conf, source}``.
|
| 212 |
+
|
| 213 |
+
``start_ms``/``end_ms`` is what the tag stands for (the placed non-speech part of an
|
| 214 |
+
event, the words of a span, the silence of a pause); ``det_start_ms``/``det_end_ms``
|
| 215 |
+
is the merged detector interval it came from, so it can be checked against the
|
| 216 |
+
detector output.
|
| 217 |
+
"""
|
| 218 |
+
|
| 219 |
+
tag: str
|
| 220 |
+
start_ms: int
|
| 221 |
+
end_ms: int
|
| 222 |
+
conf: float
|
| 223 |
+
source: str
|
| 224 |
+
det_start_ms: int
|
| 225 |
+
det_end_ms: int
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
@dataclass(frozen=True)
|
| 229 |
+
class FusionResult:
|
| 230 |
+
"""Either ``items``/``text`` (accepted) or ``quarantine`` (dropped), never both."""
|
| 231 |
+
|
| 232 |
+
segment_id: str
|
| 233 |
+
items: tuple[Item, ...] | None
|
| 234 |
+
text: str | None
|
| 235 |
+
tags: tuple[TagRecord, ...]
|
| 236 |
+
quarantine: QuarantineRecord | None
|
| 237 |
+
ignored_detections: int = 0
|
| 238 |
+
#: The below-band detections themselves (``len == ignored_detections``), so a caller can
|
| 239 |
+
#: record what was heard but left untagged (ADR-0010: uncertain breaths keep the segment).
|
| 240 |
+
ignored_events: tuple[EventDetection, ...] = ()
|
| 241 |
+
|
| 242 |
+
def __post_init__(self) -> None:
|
| 243 |
+
accepted = self.items is not None and self.text is not None
|
| 244 |
+
dropped = self.items is None and self.text is None and not self.tags
|
| 245 |
+
if self.quarantine is None and not accepted:
|
| 246 |
+
raise ValueError("accepted result needs items and text")
|
| 247 |
+
if self.quarantine is not None and not dropped:
|
| 248 |
+
raise ValueError("quarantined result must not carry items, text or tags")
|
| 249 |
+
|
| 250 |
+
@property
|
| 251 |
+
def accepted(self) -> bool:
|
| 252 |
+
return self.quarantine is None
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
@dataclass(frozen=True)
|
| 256 |
+
class EmotionCondition:
|
| 257 |
+
emotion: str | None
|
| 258 |
+
intensity: float | None
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
@dataclass
|
| 262 |
+
class _Merged:
|
| 263 |
+
family: str
|
| 264 |
+
kinds: set[str]
|
| 265 |
+
start_ms: int
|
| 266 |
+
end_ms: int
|
| 267 |
+
conf: float
|
| 268 |
+
sources: set[str] = field(default_factory=set)
|
| 269 |
+
|
| 270 |
+
@property
|
| 271 |
+
def source(self) -> str:
|
| 272 |
+
return "+".join(sorted(self.sources))
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
@dataclass(frozen=True)
|
| 276 |
+
class _Placed:
|
| 277 |
+
gap: int
|
| 278 |
+
kind: str
|
| 279 |
+
start_ms: int
|
| 280 |
+
end_ms: int
|
| 281 |
+
event: _Merged
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
class _Drop(Exception):
|
| 285 |
+
"""Internal: the segment is quarantined with ``reason``."""
|
| 286 |
+
|
| 287 |
+
def __init__(self, reason: str, **details: object) -> None:
|
| 288 |
+
super().__init__(reason)
|
| 289 |
+
self.reason = reason
|
| 290 |
+
self.details = details
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _overlap_ms(a_start: int, a_end: int, b_start: int, b_end: int) -> int:
|
| 294 |
+
return max(0, min(a_end, b_end) - max(a_start, b_start))
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _union(intervals: Sequence[tuple[int, int]]) -> list[tuple[int, int]]:
|
| 298 |
+
out: list[tuple[int, int]] = []
|
| 299 |
+
for start, end in sorted(intervals):
|
| 300 |
+
if out and start <= out[-1][1]:
|
| 301 |
+
out[-1] = (out[-1][0], max(out[-1][1], end))
|
| 302 |
+
else:
|
| 303 |
+
out.append((start, end))
|
| 304 |
+
return out
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _validate_inputs(
|
| 308 |
+
words: Sequence[TimedWord],
|
| 309 |
+
duration_ms: int,
|
| 310 |
+
events: Sequence[EventDetection],
|
| 311 |
+
styles: Sequence[WordStyleScore],
|
| 312 |
+
) -> None:
|
| 313 |
+
if type(duration_ms) is not int or duration_ms <= 0:
|
| 314 |
+
raise ValueError(f"duration_ms must be a positive int, got {duration_ms!r}")
|
| 315 |
+
previous_end = 0
|
| 316 |
+
for word in words:
|
| 317 |
+
if word.start_ms < previous_end:
|
| 318 |
+
raise ValueError(f"words overlap or are unsorted at {word.text!r} ({word.start_ms} ms)")
|
| 319 |
+
if word.end_ms > duration_ms:
|
| 320 |
+
raise ValueError(f"word {word.text!r} ends after the segment ({word.end_ms} ms)")
|
| 321 |
+
previous_end = word.end_ms
|
| 322 |
+
for event in events:
|
| 323 |
+
if event.kind not in EVENTS or event.kind in _PAUSE_EVENTS:
|
| 324 |
+
raise ValueError(f"not a detectable event kind: {event.kind!r}")
|
| 325 |
+
if event.end_ms > duration_ms:
|
| 326 |
+
raise ValueError(f"event {event.kind!r} ends after the segment")
|
| 327 |
+
seen: set[str] = set()
|
| 328 |
+
for style in styles:
|
| 329 |
+
if style.style not in SPANS:
|
| 330 |
+
raise ValueError(f"unknown style {style.style!r}")
|
| 331 |
+
if style.style in seen:
|
| 332 |
+
raise ValueError(f"duplicate style scores for {style.style!r}")
|
| 333 |
+
seen.add(style.style)
|
| 334 |
+
if len(style.scores) != len(words):
|
| 335 |
+
raise ValueError(
|
| 336 |
+
f"style {style.style!r} has {len(style.scores)} scores for {len(words)} words"
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def _split_by_band(
|
| 341 |
+
events: Sequence[EventDetection], cfg: FusionConfig
|
| 342 |
+
) -> tuple[list[EventDetection], list[EventDetection], list[EventDetection]]:
|
| 343 |
+
"""(confident, middle band, ignored = below ``low``)."""
|
| 344 |
+
confident: list[EventDetection] = []
|
| 345 |
+
middle: list[EventDetection] = []
|
| 346 |
+
ignored: list[EventDetection] = []
|
| 347 |
+
for event in events:
|
| 348 |
+
band = cfg.bands[event.kind]
|
| 349 |
+
if event.conf < band.low:
|
| 350 |
+
ignored.append(event)
|
| 351 |
+
elif event.conf < band.high:
|
| 352 |
+
middle.append(event)
|
| 353 |
+
else:
|
| 354 |
+
confident.append(event)
|
| 355 |
+
return confident, middle, ignored
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def _merge_confident(
|
| 359 |
+
confident: Sequence[EventDetection], middle: Sequence[EventDetection], cfg: FusionConfig
|
| 360 |
+
) -> list[_Merged]:
|
| 361 |
+
"""Rules 1–2: ambiguity check against confident cover, then per-family merging."""
|
| 362 |
+
cover: dict[str, list[tuple[int, int]]] = {}
|
| 363 |
+
for event in confident:
|
| 364 |
+
cover.setdefault(_family(event.kind), []).append((event.start_ms, event.end_ms))
|
| 365 |
+
union = {family: _union(spans) for family, spans in cover.items()}
|
| 366 |
+
for event in sorted(middle, key=lambda e: (e.start_ms, e.end_ms, e.kind, e.source, e.conf)):
|
| 367 |
+
if event.kind in cfg.disabled_tags:
|
| 368 |
+
raise _Drop("disabled_tag", tag=event.kind, start_ms=event.start_ms, conf=event.conf)
|
| 369 |
+
covered = sum(
|
| 370 |
+
_overlap_ms(event.start_ms, event.end_ms, a, b)
|
| 371 |
+
for a, b in union.get(_family(event.kind), [])
|
| 372 |
+
)
|
| 373 |
+
if covered < event.duration_ms:
|
| 374 |
+
raise _Drop(
|
| 375 |
+
"ambiguous_event",
|
| 376 |
+
kind=event.kind,
|
| 377 |
+
conf=event.conf,
|
| 378 |
+
start_ms=event.start_ms,
|
| 379 |
+
end_ms=event.end_ms,
|
| 380 |
+
source=event.source,
|
| 381 |
+
)
|
| 382 |
+
for event in sorted(confident, key=lambda e: (e.start_ms, e.end_ms, e.kind, e.source)):
|
| 383 |
+
if event.kind in cfg.disabled_tags:
|
| 384 |
+
# Checked on the raw kind too: a disabled `laugh` must not survive by being
|
| 385 |
+
# relabelled `laugh_long` after merging.
|
| 386 |
+
raise _Drop("disabled_tag", tag=event.kind, start_ms=event.start_ms, conf=event.conf)
|
| 387 |
+
merged: list[_Merged] = []
|
| 388 |
+
ordered = sorted(
|
| 389 |
+
confident, key=lambda e: (_family(e.kind), e.start_ms, e.end_ms, e.kind, e.source)
|
| 390 |
+
)
|
| 391 |
+
for event in ordered:
|
| 392 |
+
family = _family(event.kind)
|
| 393 |
+
last = merged[-1] if merged else None
|
| 394 |
+
if (
|
| 395 |
+
last is not None
|
| 396 |
+
and last.family == family
|
| 397 |
+
and event.start_ms <= last.end_ms + cfg.merge_gap_ms
|
| 398 |
+
):
|
| 399 |
+
last.end_ms = max(last.end_ms, event.end_ms)
|
| 400 |
+
last.conf = max(last.conf, event.conf)
|
| 401 |
+
last.kinds.add(event.kind)
|
| 402 |
+
last.sources.add(event.source)
|
| 403 |
+
else:
|
| 404 |
+
merged.append(
|
| 405 |
+
_Merged(
|
| 406 |
+
family, {event.kind}, event.start_ms, event.end_ms, event.conf, {event.source}
|
| 407 |
+
)
|
| 408 |
+
)
|
| 409 |
+
return sorted(merged, key=lambda m: (m.start_ms, m.end_ms, m.family))
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
def _non_speech_pieces(
|
| 413 |
+
start_ms: int, end_ms: int, words: Sequence[TimedWord]
|
| 414 |
+
) -> list[tuple[int, int]]:
|
| 415 |
+
"""Parts of ``[start_ms, end_ms)`` not covered by any word (words are sorted, disjoint)."""
|
| 416 |
+
pieces: list[tuple[int, int]] = []
|
| 417 |
+
cursor = start_ms
|
| 418 |
+
for word in words:
|
| 419 |
+
if word.end_ms <= cursor:
|
| 420 |
+
continue
|
| 421 |
+
if word.start_ms >= end_ms:
|
| 422 |
+
break
|
| 423 |
+
if word.start_ms > cursor:
|
| 424 |
+
pieces.append((cursor, word.start_ms))
|
| 425 |
+
cursor = max(cursor, word.end_ms)
|
| 426 |
+
if cursor < end_ms:
|
| 427 |
+
pieces.append((cursor, end_ms))
|
| 428 |
+
return pieces
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def _nearest_gap(time_ms: float, words: Sequence[TimedWord]) -> int:
|
| 432 |
+
"""Gap ``g`` lies before word ``g`` (``g == len(words)``: after the last word)."""
|
| 433 |
+
for index, word in enumerate(words):
|
| 434 |
+
if time_ms <= word.start_ms:
|
| 435 |
+
return index
|
| 436 |
+
if time_ms < word.end_ms:
|
| 437 |
+
# Inside a word (defensive: callers pass the middle of a non-speech piece, which
|
| 438 |
+
# never lies inside a word): the closer boundary wins; a tie goes after it.
|
| 439 |
+
return index if time_ms - word.start_ms < word.end_ms - time_ms else index + 1
|
| 440 |
+
return len(words)
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def _event_kind(event: _Merged, placed_ms: int, cfg: FusionConfig) -> str:
|
| 444 |
+
if event.family != "laugh":
|
| 445 |
+
return event.family
|
| 446 |
+
if "laugh_long" in event.kinds or placed_ms >= cfg.laugh_long_min_ms:
|
| 447 |
+
return "laugh_long"
|
| 448 |
+
return "laugh"
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
def _place_events(
|
| 452 |
+
merged: Sequence[_Merged], words: Sequence[TimedWord], cfg: FusionConfig
|
| 453 |
+
) -> tuple[list[_Placed], list[tuple[str, list[int], _Merged]]]:
|
| 454 |
+
"""Rules 3–4. Returns (placed events, style evidence ``(style, word indices, event)``)."""
|
| 455 |
+
placed: list[_Placed] = []
|
| 456 |
+
style_events: list[tuple[str, list[int], _Merged]] = []
|
| 457 |
+
for event in merged:
|
| 458 |
+
length = event.end_ms - event.start_ms
|
| 459 |
+
style_id = STYLE_FROM_EVENT.get(event.family)
|
| 460 |
+
styled_words = [
|
| 461 |
+
i
|
| 462 |
+
for i, w in enumerate(words)
|
| 463 |
+
if style_id is not None
|
| 464 |
+
and _overlap_ms(event.start_ms, event.end_ms, w.start_ms, w.end_ms)
|
| 465 |
+
> cfg.style_word_overlap * w.duration_ms
|
| 466 |
+
]
|
| 467 |
+
pieces = _non_speech_pieces(event.start_ms, event.end_ms, words)
|
| 468 |
+
longest = max(pieces, key=lambda p: (p[1] - p[0], -p[0])) if pieces else None
|
| 469 |
+
longest_ms = longest[1] - longest[0] if longest is not None else 0
|
| 470 |
+
if styled_words:
|
| 471 |
+
assert style_id is not None
|
| 472 |
+
# The detection may also cover words it did not mark (overlap <= the style
|
| 473 |
+
# threshold); that speech is laughed/cried over untagged, so bound it like any
|
| 474 |
+
# other event's speech overlap.
|
| 475 |
+
unmarked_ms = sum(
|
| 476 |
+
_overlap_ms(event.start_ms, event.end_ms, w.start_ms, w.end_ms)
|
| 477 |
+
for i, w in enumerate(words)
|
| 478 |
+
if i not in styled_words
|
| 479 |
+
)
|
| 480 |
+
if unmarked_ms > cfg.max_event_speech_overlap * length:
|
| 481 |
+
raise _Drop(
|
| 482 |
+
"event_overlaps_speech",
|
| 483 |
+
kind=sorted(event.kinds),
|
| 484 |
+
start_ms=event.start_ms,
|
| 485 |
+
end_ms=event.end_ms,
|
| 486 |
+
speech_ms=unmarked_ms,
|
| 487 |
+
)
|
| 488 |
+
style_events.append((style_id, styled_words, event))
|
| 489 |
+
if longest_ms < cfg.min_residual_event_ms:
|
| 490 |
+
continue
|
| 491 |
+
else:
|
| 492 |
+
speech_ms = length - sum(b - a for a, b in pieces)
|
| 493 |
+
if longest is None or speech_ms > cfg.max_event_speech_overlap * length:
|
| 494 |
+
raise _Drop(
|
| 495 |
+
"event_overlaps_speech",
|
| 496 |
+
kind=sorted(event.kinds),
|
| 497 |
+
start_ms=event.start_ms,
|
| 498 |
+
end_ms=event.end_ms,
|
| 499 |
+
speech_ms=speech_ms,
|
| 500 |
+
)
|
| 501 |
+
assert longest is not None
|
| 502 |
+
kind = _event_kind(event, longest_ms, cfg)
|
| 503 |
+
if kind in cfg.disabled_tags:
|
| 504 |
+
raise _Drop("disabled_tag", tag=kind, start_ms=event.start_ms, conf=event.conf)
|
| 505 |
+
placed.append(
|
| 506 |
+
_Placed(_nearest_gap((longest[0] + longest[1]) / 2, words), kind, *longest, event)
|
| 507 |
+
)
|
| 508 |
+
for i, first in enumerate(placed):
|
| 509 |
+
for second in placed[i + 1 :]:
|
| 510 |
+
if first.kind == second.kind:
|
| 511 |
+
continue
|
| 512 |
+
shorter = min(first.end_ms - first.start_ms, second.end_ms - second.start_ms)
|
| 513 |
+
shared = _overlap_ms(first.start_ms, first.end_ms, second.start_ms, second.end_ms)
|
| 514 |
+
if shared > cfg.max_cross_kind_overlap * shorter:
|
| 515 |
+
raise _Drop(
|
| 516 |
+
"conflicting_events",
|
| 517 |
+
kinds=sorted({first.kind, second.kind}),
|
| 518 |
+
start_ms=max(first.start_ms, second.start_ms),
|
| 519 |
+
shared_ms=shared,
|
| 520 |
+
)
|
| 521 |
+
return placed, style_events
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def _style_marks(
|
| 525 |
+
words: Sequence[TimedWord],
|
| 526 |
+
styles: Sequence[WordStyleScore],
|
| 527 |
+
style_events: Sequence[tuple[str, list[int], _Merged]],
|
| 528 |
+
cfg: FusionConfig,
|
| 529 |
+
) -> tuple[list[str | None], dict[int, tuple[float, str, int, int]]]:
|
| 530 |
+
"""Per-word style (or None) and, per marked word, its evidence
|
| 531 |
+
``(conf, source, det_start_ms, det_end_ms)`` (the word itself for acoustic scores)."""
|
| 532 |
+
marks: list[set[str]] = [set() for _ in words]
|
| 533 |
+
evidence: list[dict[str, tuple[float, str, int, int]]] = [{} for _ in words]
|
| 534 |
+
for style in sorted(styles, key=lambda s: s.style):
|
| 535 |
+
if style.style in cfg.ignored_styles:
|
| 536 |
+
continue
|
| 537 |
+
band = cfg.bands[style.style]
|
| 538 |
+
for index, score in enumerate(style.scores):
|
| 539 |
+
if score is None or score < band.low:
|
| 540 |
+
continue
|
| 541 |
+
if score < band.high:
|
| 542 |
+
raise _Drop(
|
| 543 |
+
"ambiguous_style",
|
| 544 |
+
style=style.style,
|
| 545 |
+
word_index=index,
|
| 546 |
+
word=words[index].text,
|
| 547 |
+
score=score,
|
| 548 |
+
source=style.source,
|
| 549 |
+
)
|
| 550 |
+
if style.style in cfg.disabled_tags:
|
| 551 |
+
raise _Drop("disabled_tag", tag=style.style, word_index=index, score=score)
|
| 552 |
+
marks[index].add(style.style)
|
| 553 |
+
word = words[index]
|
| 554 |
+
evidence[index][style.style] = (score, style.source, word.start_ms, word.end_ms)
|
| 555 |
+
for style_id, indices, event in style_events:
|
| 556 |
+
if style_id in cfg.disabled_tags:
|
| 557 |
+
raise _Drop("disabled_tag", tag=style_id, start_ms=event.start_ms, conf=event.conf)
|
| 558 |
+
for index in indices:
|
| 559 |
+
marks[index].add(style_id)
|
| 560 |
+
previous = evidence[index].get(style_id)
|
| 561 |
+
if previous is None or event.conf > previous[0]:
|
| 562 |
+
evidence[index][style_id] = (
|
| 563 |
+
event.conf,
|
| 564 |
+
event.source,
|
| 565 |
+
event.start_ms,
|
| 566 |
+
event.end_ms,
|
| 567 |
+
)
|
| 568 |
+
resolved: list[str | None] = []
|
| 569 |
+
for index, word_marks in enumerate(marks):
|
| 570 |
+
if len(word_marks) > 1:
|
| 571 |
+
raise _Drop(
|
| 572 |
+
"conflicting_styles",
|
| 573 |
+
word_index=index,
|
| 574 |
+
word=words[index].text,
|
| 575 |
+
styles=sorted(word_marks),
|
| 576 |
+
)
|
| 577 |
+
resolved.append(next(iter(word_marks)) if word_marks else None)
|
| 578 |
+
chosen = {
|
| 579 |
+
index: evidence[index][style_id]
|
| 580 |
+
for index, style_id in enumerate(resolved)
|
| 581 |
+
if style_id is not None
|
| 582 |
+
}
|
| 583 |
+
return resolved, chosen
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _style_runs(
|
| 587 |
+
words: Sequence[TimedWord], marks: Sequence[str | None], cfg: FusionConfig
|
| 588 |
+
) -> list[tuple[str, int, int]]:
|
| 589 |
+
"""Rule 5: maximal runs ``(style, first, last)`` of equally marked consecutive words."""
|
| 590 |
+
runs: list[tuple[str, int, int]] = []
|
| 591 |
+
index = 0
|
| 592 |
+
while index < len(marks):
|
| 593 |
+
style_id = marks[index]
|
| 594 |
+
if style_id is None:
|
| 595 |
+
index += 1
|
| 596 |
+
continue
|
| 597 |
+
last = index
|
| 598 |
+
while last + 1 < len(marks) and marks[last + 1] == style_id:
|
| 599 |
+
last += 1
|
| 600 |
+
count = last - index + 1
|
| 601 |
+
span_ms = words[last].end_ms - words[index].start_ms
|
| 602 |
+
if count < cfg.min_span_words and span_ms < cfg.min_span_ms:
|
| 603 |
+
raise _Drop(
|
| 604 |
+
"short_style_run",
|
| 605 |
+
style=style_id,
|
| 606 |
+
words=[w.text for w in words[index : last + 1]],
|
| 607 |
+
span_ms=span_ms,
|
| 608 |
+
)
|
| 609 |
+
runs.append((style_id, index, last))
|
| 610 |
+
index = last + 1
|
| 611 |
+
return runs
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def _normalise_word(text: str) -> str:
|
| 615 |
+
"""Rule 7 for one word."""
|
| 616 |
+
text = nfc(text)
|
| 617 |
+
if any(ch in _RESERVED_CHARS for ch in text):
|
| 618 |
+
raise _Drop("reserved_char_in_text", word=text)
|
| 619 |
+
if any(unicodedata.category(ch).startswith("N") for ch in text): # 3, ², ½, Ⅳ
|
| 620 |
+
raise _Drop("unnormalised_number", word=text)
|
| 621 |
+
start, end = 0, len(text)
|
| 622 |
+
while start < end and unicodedata.category(text[start]).startswith("P"):
|
| 623 |
+
start += 1
|
| 624 |
+
while end > start and unicodedata.category(text[end - 1]).startswith("P"):
|
| 625 |
+
end -= 1
|
| 626 |
+
core = text[start:end]
|
| 627 |
+
if not core:
|
| 628 |
+
return text
|
| 629 |
+
canonical = canonical_filler(tr_lower(core))
|
| 630 |
+
return text if canonical is None else text[:start] + canonical + text[end:]
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
def normalise_word(text: str) -> str:
|
| 634 |
+
"""Rule 7 for one word, as :func:`fuse_segment` applies it to every aligned word.
|
| 635 |
+
|
| 636 |
+
Lets later stages (``timings``) compare the words of a stored tagged text with the
|
| 637 |
+
aligned words they came from.
|
| 638 |
+
|
| 639 |
+
Raises:
|
| 640 |
+
ValueError: if fusion would drop the segment for this word (reserved tag character
|
| 641 |
+
or a digit-like character), with fusion's reason in the message.
|
| 642 |
+
"""
|
| 643 |
+
try:
|
| 644 |
+
return _normalise_word(text)
|
| 645 |
+
except _Drop as drop:
|
| 646 |
+
raise ValueError(f"{drop.reason}: {drop.details}") from drop
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def _ends_with_punctuation(text: str) -> bool:
|
| 650 |
+
return unicodedata.category(text[-1]).startswith("P")
|
| 651 |
+
|
| 652 |
+
|
| 653 |
+
def _pauses(
|
| 654 |
+
words: Sequence[TimedWord],
|
| 655 |
+
texts: Sequence[str],
|
| 656 |
+
merged: Sequence[_Merged],
|
| 657 |
+
cfg: FusionConfig,
|
| 658 |
+
) -> list[_Placed]:
|
| 659 |
+
"""Rule 6, on the longest part of each gap that no confident detection covers.
|
| 660 |
+
|
| 661 |
+
A short breath inside a 5 s silence must not hide the pause; the pause tag record
|
| 662 |
+
spans that uncovered part, and tags in one gap are later ordered by time.
|
| 663 |
+
"""
|
| 664 |
+
out: list[_Placed] = []
|
| 665 |
+
for gap in range(1, len(words)):
|
| 666 |
+
gap_start, gap_end = words[gap - 1].end_ms, words[gap].start_ms
|
| 667 |
+
covered = _union(
|
| 668 |
+
[
|
| 669 |
+
(max(gap_start, e.start_ms), min(gap_end, e.end_ms))
|
| 670 |
+
for e in merged
|
| 671 |
+
if _overlap_ms(gap_start, gap_end, e.start_ms, e.end_ms) > 0
|
| 672 |
+
]
|
| 673 |
+
)
|
| 674 |
+
free: list[tuple[int, int]] = []
|
| 675 |
+
cursor = gap_start
|
| 676 |
+
for a, b in covered:
|
| 677 |
+
if a > cursor:
|
| 678 |
+
free.append((cursor, a))
|
| 679 |
+
cursor = max(cursor, b)
|
| 680 |
+
if cursor < gap_end:
|
| 681 |
+
free.append((cursor, gap_end))
|
| 682 |
+
if not free:
|
| 683 |
+
continue
|
| 684 |
+
start, end = max(free, key=lambda p: (p[1] - p[0], -p[0]))
|
| 685 |
+
silence = end - start
|
| 686 |
+
kind: str | None = None
|
| 687 |
+
if silence > cfg.pause_long_ms:
|
| 688 |
+
kind = "pause_long"
|
| 689 |
+
elif silence >= cfg.pause_short_ms and not _ends_with_punctuation(texts[gap - 1]):
|
| 690 |
+
kind = "pause_short"
|
| 691 |
+
if kind is not None:
|
| 692 |
+
if kind in cfg.disabled_tags:
|
| 693 |
+
raise _Drop("disabled_tag", tag=kind, start_ms=start)
|
| 694 |
+
silence_event = _Merged(kind, {kind}, start, end, 1.0, {"alignment"})
|
| 695 |
+
out.append(_Placed(gap, kind, start, end, silence_event))
|
| 696 |
+
return out
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def _build_items(
|
| 700 |
+
texts: Sequence[str],
|
| 701 |
+
events_by_gap: Mapping[int, Sequence[_Placed]],
|
| 702 |
+
runs: Sequence[tuple[str, int, int]],
|
| 703 |
+
) -> tuple[Item, ...]:
|
| 704 |
+
run_at = {first: (style_id, last) for style_id, first, last in runs}
|
| 705 |
+
items: list[Item] = []
|
| 706 |
+
index = 0
|
| 707 |
+
while True:
|
| 708 |
+
items.extend(Event(p.kind) for p in events_by_gap.get(index, ()))
|
| 709 |
+
if index == len(texts):
|
| 710 |
+
break
|
| 711 |
+
if index in run_at:
|
| 712 |
+
style_id, last = run_at[index]
|
| 713 |
+
children: list[Word | Event] = []
|
| 714 |
+
for inner in range(index, last + 1):
|
| 715 |
+
if inner > index:
|
| 716 |
+
children.extend(Event(p.kind) for p in events_by_gap.get(inner, ()))
|
| 717 |
+
children.append(Word(texts[inner]))
|
| 718 |
+
items.append(Span(style_id, tuple(children)))
|
| 719 |
+
index = last + 1
|
| 720 |
+
else:
|
| 721 |
+
items.append(Word(texts[index]))
|
| 722 |
+
index += 1
|
| 723 |
+
return tuple(items)
|
| 724 |
+
|
| 725 |
+
|
| 726 |
+
def fuse_segment(
|
| 727 |
+
segment_id: str,
|
| 728 |
+
words: Sequence[TimedWord],
|
| 729 |
+
duration_ms: int,
|
| 730 |
+
events: Sequence[EventDetection],
|
| 731 |
+
styles: Sequence[WordStyleScore],
|
| 732 |
+
cfg: FusionConfig,
|
| 733 |
+
) -> FusionResult:
|
| 734 |
+
"""Fuse one segment's annotations into tagged text (see module docstring).
|
| 735 |
+
|
| 736 |
+
Args:
|
| 737 |
+
segment_id: Content-addressed id (:func:`nida.data.types.segment_id`).
|
| 738 |
+
words: Aligned words, sorted and non-overlapping, segment-relative ms.
|
| 739 |
+
duration_ms: Segment length in ms.
|
| 740 |
+
events: Raw event detections (any confidence; bands are applied here).
|
| 741 |
+
styles: At most one per-word score track per span style.
|
| 742 |
+
cfg: Thresholds.
|
| 743 |
+
|
| 744 |
+
Returns:
|
| 745 |
+
A :class:`FusionResult`; when ``quarantine`` is set, ``items``/``text`` are None.
|
| 746 |
+
``ignored_detections`` counts below-band detections in both cases and
|
| 747 |
+
``ignored_events`` holds them.
|
| 748 |
+
|
| 749 |
+
Raises:
|
| 750 |
+
ValueError: on inconsistent inputs (a caller bug, not a data property).
|
| 751 |
+
"""
|
| 752 |
+
_validate_inputs(words, duration_ms, events, styles)
|
| 753 |
+
confident, middle, ignored = _split_by_band(events, cfg)
|
| 754 |
+
try:
|
| 755 |
+
if not words:
|
| 756 |
+
raise _Drop("no_words")
|
| 757 |
+
texts = [_normalise_word(w.text) for w in words]
|
| 758 |
+
merged = _merge_confident(confident, middle, cfg)
|
| 759 |
+
placed, style_events = _place_events(merged, words, cfg)
|
| 760 |
+
marks, evidence = _style_marks(words, styles, style_events, cfg)
|
| 761 |
+
runs = _style_runs(words, marks, cfg)
|
| 762 |
+
placed += _pauses(words, texts, merged, cfg)
|
| 763 |
+
|
| 764 |
+
events_by_gap: dict[int, list[_Placed]] = {}
|
| 765 |
+
tags: list[TagRecord] = []
|
| 766 |
+
for p in sorted(placed, key=lambda p: (p.start_ms, p.end_ms, p.kind)):
|
| 767 |
+
events_by_gap.setdefault(p.gap, []).append(p)
|
| 768 |
+
tags.append(
|
| 769 |
+
TagRecord(
|
| 770 |
+
p.kind,
|
| 771 |
+
p.start_ms,
|
| 772 |
+
p.end_ms,
|
| 773 |
+
p.event.conf,
|
| 774 |
+
p.event.source,
|
| 775 |
+
p.event.start_ms,
|
| 776 |
+
p.event.end_ms,
|
| 777 |
+
)
|
| 778 |
+
)
|
| 779 |
+
for style_id, first, last in runs:
|
| 780 |
+
run_evidence = [evidence[i] for i in range(first, last + 1)]
|
| 781 |
+
tags.append(
|
| 782 |
+
TagRecord(
|
| 783 |
+
style_id,
|
| 784 |
+
words[first].start_ms,
|
| 785 |
+
words[last].end_ms,
|
| 786 |
+
min(e[0] for e in run_evidence),
|
| 787 |
+
"+".join(sorted({e[1] for e in run_evidence})),
|
| 788 |
+
min(e[2] for e in run_evidence),
|
| 789 |
+
max(e[3] for e in run_evidence),
|
| 790 |
+
)
|
| 791 |
+
)
|
| 792 |
+
try:
|
| 793 |
+
items = _build_items(texts, events_by_gap, runs)
|
| 794 |
+
text = render(items)
|
| 795 |
+
if parse(text) != items:
|
| 796 |
+
raise _Drop("invalid_output", text=text, detail="round-trip mismatch")
|
| 797 |
+
except (TagSyntaxError, ValueError) as err:
|
| 798 |
+
raise _Drop("invalid_output", error=str(err)) from err
|
| 799 |
+
except _Drop as drop:
|
| 800 |
+
return FusionResult(
|
| 801 |
+
segment_id=segment_id,
|
| 802 |
+
items=None,
|
| 803 |
+
text=None,
|
| 804 |
+
tags=(),
|
| 805 |
+
quarantine=QuarantineRecord(segment_id, STAGE, drop.reason, dict(drop.details)),
|
| 806 |
+
ignored_detections=len(ignored),
|
| 807 |
+
ignored_events=tuple(ignored),
|
| 808 |
+
)
|
| 809 |
+
return FusionResult(
|
| 810 |
+
segment_id=segment_id,
|
| 811 |
+
items=items,
|
| 812 |
+
text=text,
|
| 813 |
+
tags=tuple(sorted(tags, key=lambda t: (t.start_ms, t.end_ms, t.tag))),
|
| 814 |
+
quarantine=None,
|
| 815 |
+
ignored_detections=len(ignored),
|
| 816 |
+
ignored_events=tuple(ignored),
|
| 817 |
+
)
|
| 818 |
+
|
| 819 |
+
|
| 820 |
+
def emotion_condition(scores: Mapping[str, float], cfg: FusionConfig) -> EmotionCondition:
|
| 821 |
+
"""Utterance-level emotion condition from emotion-model class probabilities.
|
| 822 |
+
|
| 823 |
+
The top class becomes the condition with its probability as intensity. ``other`` /
|
| 824 |
+
``unknown`` as top class, or a top probability below ``emotion_min_conf``, give a
|
| 825 |
+
null condition (not "neutral"; tag-spec §5, ADR-0003). Ties resolve to the
|
| 826 |
+
alphabetically first label so the result is deterministic.
|
| 827 |
+
|
| 828 |
+
Raises:
|
| 829 |
+
ValueError: unknown labels, empty scores, or probabilities outside [0, 1].
|
| 830 |
+
"""
|
| 831 |
+
if not scores:
|
| 832 |
+
raise ValueError("empty emotion scores")
|
| 833 |
+
unknown = set(scores) - EMOTION_LABELS
|
| 834 |
+
if unknown:
|
| 835 |
+
raise ValueError(f"unknown emotion labels {sorted(unknown)}")
|
| 836 |
+
for label, prob in scores.items():
|
| 837 |
+
if not (math.isfinite(prob) and 0.0 <= prob <= 1.0):
|
| 838 |
+
raise ValueError(f"emotion {label!r} probability {prob!r} not in [0, 1]")
|
| 839 |
+
label, prob = min(scores.items(), key=lambda kv: (-kv[1], kv[0]))
|
| 840 |
+
if label in _NULL_EMOTIONS or prob < cfg.emotion_min_conf:
|
| 841 |
+
return EmotionCondition(None, None)
|
| 842 |
+
return EmotionCondition(label, prob)
|
src/nida/data/importer/__init__.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Import of an already transcribed external Turkish speech corpus (``nida data-import``).
|
| 2 |
+
|
| 3 |
+
The result is a *latent dataset directory* in exactly the format of :mod:`nida.train.data`
|
| 4 |
+
(``index.jsonl``, ``shard-NNNNN.npy`` float16, ``meta.json``) plus ``timings.jsonl``
|
| 5 |
+
(:mod:`nida.data.timings`), so ``load_latent_dataset(dir, split, timings=True)`` accepts it and
|
| 6 |
+
``train --data A --data B ...`` can mix it with the YODAS pilot datasets. There is no ASR, no
|
| 7 |
+
fusion and no tag detection: the transcript is trusted, aligned and encoded.
|
| 8 |
+
|
| 9 |
+
Sources (adapters, :mod:`nida.data.importer.sources`; all must be ``approved`` in
|
| 10 |
+
``configs/sources.yaml``)::
|
| 11 |
+
|
| 12 |
+
alania HF cloud0day3/alania-synthetic-speech-tr, config cc-by only (parquet shards,
|
| 13 |
+
48 kHz FLAC), QC-filtered, text = ``text_spoken``, group = voice id
|
| 14 |
+
tr_full HF Taklaxbr/tr-full-dataset (``data.jsonl`` + wavs), language tr only, emotion
|
| 15 |
+
neutral/angry/sad/happy, group = speaker id
|
| 16 |
+
wavdir a folder with ``manifest.jsonl`` {file, text, group, source, emotion?}: our own
|
| 17 |
+
synthetic speech (e.g. ema-lightning); each line's ``source`` must be approved
|
| 18 |
+
issai_tsc HF issai/Turkish_Speech_Corpus: one 21 GB ``.tar.gz``
|
| 19 |
+
(``ISSAI_TSC_218/{Train,Dev,Test}/<n>.wav|.txt``, 16 kHz); no speaker in the
|
| 20 |
+
corpus, so group = block of running utterance numbers (see import_corpora.yaml)
|
| 21 |
+
mediaspeech_tr
|
| 22 |
+
OpenSLR SLR108 ``TR.tgz`` (``TR/<uuid>.flac|.txt``), pinned by the sha256 of the
|
| 23 |
+
archive (recorded as the plan's revision); the whole corpus is one group
|
| 24 |
+
common_voice_tr
|
| 25 |
+
a Common Voice release the user downloaded (``--source-dir``: release folder,
|
| 26 |
+
its ``tr`` folder, or the tarball): ``validated.tsv`` rows with
|
| 27 |
+
``down_votes < up_votes``, MP3 clips, group = hash of ``client_id``
|
| 28 |
+
|
| 29 |
+
A corpus is cut into deterministic *work units* (a parquet shard, a chunk of rows). A root
|
| 30 |
+
(``--root``) is bound to one plan (source, pinned Hub commit sha / archive sha256 / sha1 of the
|
| 31 |
+
local manifest, ``--max-hours``, ``--shard-offset``/``--shards`` unit range, config digest)
|
| 32 |
+
recorded in ``plan.json``; several roots
|
| 33 |
+
with disjoint unit ranges select disjoint data, and the split of a group is a pure function of
|
| 34 |
+
the group id, so groups stay disjoint across roots.
|
| 35 |
+
|
| 36 |
+
Stages (each idempotent and resumable; ``all`` runs them in order)::
|
| 37 |
+
|
| 38 |
+
select -> audio -> latents -> ctc -> timings -> dataset -> report
|
| 39 |
+
|
| 40 |
+
* ``select`` (CPU, network): per unit, metadata filters and transcript normalisation
|
| 41 |
+
(:mod:`nida.data.importer.text`: NFC, text with tag characters rejected, digits spelled out,
|
| 42 |
+
every character must be in the model vocabulary; fillers are not canonicalised); the raw
|
| 43 |
+
audio of the selected rows is put in the local cache; stops at ``--max-hours``. Finished
|
| 44 |
+
units are stored, so an interruption loses at most one unit.
|
| 45 |
+
* ``audio`` (CPU): 16 kHz mono PCM wav per item in the cache, duration and loudness (dBFS)
|
| 46 |
+
measured, duration/silence checks. Lost cache files are rebuilt (the unit is fetched again).
|
| 47 |
+
* ``latents`` / ``ctc`` (GPU): the pilot's ``vae_encode`` and ``ctc_emissions`` workers
|
| 48 |
+
(same isolated environments, same resumable results protocol,
|
| 49 |
+
:func:`nida.data.pilot.run_worker_job`).
|
| 50 |
+
* ``timings`` (CPU): forced alignment of ``text.split()``, the pilot's alignment sanity check
|
| 51 |
+
(:func:`nida.data.asr_filter.check_alignment`, thresholds in ``import.yaml``) and the unit
|
| 52 |
+
timeline (:func:`nida.data.timings.build_timings`, no event tags).
|
| 53 |
+
* ``dataset`` (CPU): ``dataset-<config digest>/`` with only the segments that got a timeline;
|
| 54 |
+
group-disjoint deterministic splits (a pure function of seed and group id: ~3 % of the
|
| 55 |
+
groups val, ~2 % test; the same group is in the same split in every root); rate/loudness
|
| 56 |
+
buckets with the fixed edges of ``configs/data/dataset.yaml``, exactly like the pilot;
|
| 57 |
+
``language`` ``tr``; ``segment_id = sha1(source id, item id)[:16]``. Verified with the
|
| 58 |
+
training loader. ``meta.json`` is written last.
|
| 59 |
+
* ``report``: ``report.md`` and ``metrics.json``.
|
| 60 |
+
|
| 61 |
+
Every item a stage drops is written to ``<root>/<stage>/quarantine.jsonl`` with a
|
| 62 |
+
machine-readable reason (:mod:`nida.data.quarantine`); counts are in the stage results and in
|
| 63 |
+
the report. Layout::
|
| 64 |
+
|
| 65 |
+
<root>/plan.json
|
| 66 |
+
<root>/select/{manifest.jsonl, quarantine.jsonl, stage.json, units/<digest>.json}
|
| 67 |
+
<root>/{audio,latents,ctc}/{manifest.jsonl, quarantine.jsonl, stage.json}
|
| 68 |
+
<root>/timings/{timings.jsonl, quarantine.jsonl, stage.json}
|
| 69 |
+
<root>/workers/{voxcpm2_audiovae_v2,mpoyraz_ctc_tr}/ worker requests/results/summary
|
| 70 |
+
<root>/dataset-<digest>/{index.jsonl, shard-NNNNN.npy, timings.jsonl, meta.json}
|
| 71 |
+
<root>/{report.md, metrics.json}
|
| 72 |
+
<cache>/hub, raw, audio16k, arrays/ bulky intermediates on local disk, rebuilt when lost
|
| 73 |
+
|
| 74 |
+
Durable outputs live under ``root`` (Drive), bulky intermediates under ``cache`` (local).
|
| 75 |
+
Thresholds and mappings are in ``configs/data/import.yaml`` (shared, and the three first
|
| 76 |
+
sources) and ``configs/data/import_corpora.yaml`` (the three corpora above; its sections have
|
| 77 |
+
their own config digest, so adding them did not move the digest of the first three sources).
|
| 78 |
+
"""
|
| 79 |
+
|
| 80 |
+
from __future__ import annotations
|
| 81 |
+
|
| 82 |
+
from nida.data.importer.config import ImportConfig, load_import_config
|
| 83 |
+
from nida.data.importer.corpora import CommonVoiceAdapter, IssaiTscAdapter, MediaSpeechAdapter
|
| 84 |
+
from nida.data.importer.hub import HubClient, HuggingFaceHub
|
| 85 |
+
from nida.data.importer.pipeline import STAGES, Importer, ImportPlan, ImportRequest
|
| 86 |
+
from nida.data.importer.sources import (
|
| 87 |
+
AlaniaAdapter,
|
| 88 |
+
Item,
|
| 89 |
+
SourceAdapter,
|
| 90 |
+
TrFullAdapter,
|
| 91 |
+
WavdirAdapter,
|
| 92 |
+
item_segment_id,
|
| 93 |
+
make_adapter,
|
| 94 |
+
)
|
| 95 |
+
from nida.data.importer.text import TextRejected, normalise_transcript
|
| 96 |
+
|
| 97 |
+
__all__ = [
|
| 98 |
+
"STAGES",
|
| 99 |
+
"AlaniaAdapter",
|
| 100 |
+
"CommonVoiceAdapter",
|
| 101 |
+
"HubClient",
|
| 102 |
+
"HuggingFaceHub",
|
| 103 |
+
"ImportConfig",
|
| 104 |
+
"ImportPlan",
|
| 105 |
+
"ImportRequest",
|
| 106 |
+
"Importer",
|
| 107 |
+
"IssaiTscAdapter",
|
| 108 |
+
"Item",
|
| 109 |
+
"MediaSpeechAdapter",
|
| 110 |
+
"SourceAdapter",
|
| 111 |
+
"TextRejected",
|
| 112 |
+
"TrFullAdapter",
|
| 113 |
+
"WavdirAdapter",
|
| 114 |
+
"item_segment_id",
|
| 115 |
+
"load_import_config",
|
| 116 |
+
"make_adapter",
|
| 117 |
+
"normalise_transcript",
|
| 118 |
+
]
|
src/nida/data/importer/archives.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One streaming pass over a tar archive, for the importer sources that ship as one.
|
| 2 |
+
|
| 3 |
+
Three sources arrive as an archive (the ISSAI corpus: a 21 GB ``.tar.gz``; MediaSpeech:
|
| 4 |
+
``TR.tgz``; a Common Voice release tarball). A stream cannot be searched, and the audio and
|
| 5 |
+
the transcript of an utterance are far apart in it, so the adapters work in *passes*:
|
| 6 |
+
|
| 7 |
+
* an *index* pass reads the small transcript members into memory (and notes which audio
|
| 8 |
+
members exist);
|
| 9 |
+
* an *extract* pass writes the audio members a work unit needs to the cache.
|
| 10 |
+
|
| 11 |
+
Both are :func:`scan_archive`. A member is never written to a path derived from its name: a
|
| 12 |
+
caller-supplied classifier maps a (normalised) member name to a ``(key, kind)`` pair or rejects
|
| 13 |
+
it, and audio goes to the path the caller registered for that key. Anything the classifier
|
| 14 |
+
does not know is an error, not a silent skip (the archives are pinned, so a surprise means
|
| 15 |
+
the format changed).
|
| 16 |
+
|
| 17 |
+
A pass that does not stop early must see the archive *end*: the decompressor is a file object
|
| 18 |
+
that raises on a truncated stream (gzip/bz2/xz trailer), and the tar stream must finish with
|
| 19 |
+
its zero blocks. A cut inside a header, or exactly between members, is an :class:`ArchiveError`,
|
| 20 |
+
never a smaller corpus. (``tarfile`` alone ends silently in both cases and does not check the
|
| 21 |
+
gzip trailer in stream mode.)
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import bz2
|
| 27 |
+
import gzip
|
| 28 |
+
import hashlib
|
| 29 |
+
import json
|
| 30 |
+
import lzma
|
| 31 |
+
import tarfile
|
| 32 |
+
import zlib
|
| 33 |
+
from collections.abc import Callable, Mapping
|
| 34 |
+
from dataclasses import dataclass, field
|
| 35 |
+
from pathlib import Path
|
| 36 |
+
from typing import IO, Literal, Protocol, cast
|
| 37 |
+
|
| 38 |
+
from nida.utils.io import atomic_write_bytes, atomic_write_json
|
| 39 |
+
|
| 40 |
+
__all__ = [
|
| 41 |
+
"MAX_AUDIO_BYTES",
|
| 42 |
+
"MAX_TEXT_BYTES",
|
| 43 |
+
"ArchiveError",
|
| 44 |
+
"ArchiveScan",
|
| 45 |
+
"Classifier",
|
| 46 |
+
"Kind",
|
| 47 |
+
"scan_archive",
|
| 48 |
+
"verify_sha256_once",
|
| 49 |
+
]
|
| 50 |
+
|
| 51 |
+
#: A transcript member larger than this is not a transcript (the biggest real text member is
|
| 52 |
+
#: a Common Voice ``validated.tsv``, tens of MB).
|
| 53 |
+
MAX_TEXT_BYTES = 512 * 1024 * 1024
|
| 54 |
+
#: An audio member larger than this is refused instead of being read into memory.
|
| 55 |
+
MAX_AUDIO_BYTES = 512 * 1024 * 1024
|
| 56 |
+
#: Names of unexpected members listed in the error message.
|
| 57 |
+
_SHOWN_UNEXPECTED = 5
|
| 58 |
+
#: A tar archive ends with (at least) two zero blocks of this many bytes.
|
| 59 |
+
_BLOCK_BYTES = 512
|
| 60 |
+
_END_BYTES = 2 * _BLOCK_BYTES
|
| 61 |
+
_DRAIN_BYTES = 1 << 20
|
| 62 |
+
_HASH_CHUNK_BYTES = 1 << 20
|
| 63 |
+
|
| 64 |
+
Kind = Literal["text", "audio", "ignore"]
|
| 65 |
+
#: Maps a normalised member name (``/``-separated, no leading ``./``) to ``(key, kind)``, or
|
| 66 |
+
#: ``None`` for a member that must not exist in the archive.
|
| 67 |
+
Classifier = Callable[[str], tuple[str, Kind] | None]
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class ArchiveError(RuntimeError):
|
| 71 |
+
"""The archive is unreadable, truncated, corrupt or not laid out as expected."""
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@dataclass
|
| 75 |
+
class ArchiveScan:
|
| 76 |
+
"""What one pass found.
|
| 77 |
+
|
| 78 |
+
Attributes:
|
| 79 |
+
texts: ``key -> bytes`` of the ``text`` members (empty unless ``read_text``).
|
| 80 |
+
audio_heads: ``key -> first bytes`` of the ``audio`` members (see ``audio_head``).
|
| 81 |
+
audio_keys: Keys of every ``audio`` member seen (an early stop leaves it partial).
|
| 82 |
+
extracted: Keys of the audio members written to their registered paths.
|
| 83 |
+
members: Regular-file members visited.
|
| 84 |
+
complete: ``True`` if the whole archive was read, up to its end blocks (no early stop).
|
| 85 |
+
"""
|
| 86 |
+
|
| 87 |
+
texts: dict[str, bytes] = field(default_factory=dict)
|
| 88 |
+
audio_heads: dict[str, bytes] = field(default_factory=dict)
|
| 89 |
+
audio_keys: set[str] = field(default_factory=set)
|
| 90 |
+
extracted: set[str] = field(default_factory=set)
|
| 91 |
+
members: int = 0
|
| 92 |
+
complete: bool = True
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _normalise(name: str) -> str:
|
| 96 |
+
while name.startswith("./"):
|
| 97 |
+
name = name[2:]
|
| 98 |
+
return name.rstrip("/")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class _ByteStream(Protocol):
|
| 102 |
+
def read(self, size: int = -1, /) -> bytes: ...
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class _Reader:
|
| 106 |
+
"""The decompressed byte stream ``tarfile`` reads.
|
| 107 |
+
|
| 108 |
+
Counts the bytes handed out and remembers where the last non-zero byte ended (to check the
|
| 109 |
+
tar end blocks), and turns every error of the decompressor into :class:`ArchiveError`.
|
| 110 |
+
Errors of *writing* extracted files are not touched: they happen elsewhere.
|
| 111 |
+
"""
|
| 112 |
+
|
| 113 |
+
def __init__(self, stream: _ByteStream, archive: Path) -> None:
|
| 114 |
+
self._stream = stream
|
| 115 |
+
self._archive = archive
|
| 116 |
+
self.pos = 0
|
| 117 |
+
self.last_nonzero_end = 0
|
| 118 |
+
|
| 119 |
+
def read(self, size: int = -1) -> bytes:
|
| 120 |
+
try:
|
| 121 |
+
data = self._stream.read(size)
|
| 122 |
+
except (OSError, EOFError, zlib.error, lzma.LZMAError) as err:
|
| 123 |
+
raise ArchiveError(
|
| 124 |
+
f"{self._archive}: cannot be read (truncated or corrupt? after "
|
| 125 |
+
f"{self.pos} bytes): {err!r}"
|
| 126 |
+
) from err
|
| 127 |
+
if data:
|
| 128 |
+
stripped = data.rstrip(b"\x00")
|
| 129 |
+
if stripped:
|
| 130 |
+
self.last_nonzero_end = self.pos + len(stripped)
|
| 131 |
+
self.pos += len(data)
|
| 132 |
+
return data
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _decompressed(raw: IO[bytes]) -> _ByteStream:
|
| 136 |
+
"""``raw`` wrapped in the file object of its compression (by magic number), or itself."""
|
| 137 |
+
magic = raw.read(6)
|
| 138 |
+
raw.seek(0)
|
| 139 |
+
if magic.startswith(b"\x1f\x8b"):
|
| 140 |
+
return gzip.GzipFile(fileobj=raw)
|
| 141 |
+
if magic.startswith(b"BZh"):
|
| 142 |
+
return bz2.BZ2File(raw)
|
| 143 |
+
if magic.startswith(b"\xfd7zXZ\x00"):
|
| 144 |
+
return lzma.LZMAFile(raw)
|
| 145 |
+
return raw
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def scan_archive(
|
| 149 |
+
archive: Path,
|
| 150 |
+
classify: Classifier,
|
| 151 |
+
*,
|
| 152 |
+
read_text: bool,
|
| 153 |
+
extract: Mapping[str, Path] | None = None,
|
| 154 |
+
audio_head: int = 0,
|
| 155 |
+
) -> ArchiveScan:
|
| 156 |
+
"""Read ``archive`` once (plain, gzip, bz2 or xz; auto-detected).
|
| 157 |
+
|
| 158 |
+
Args:
|
| 159 |
+
archive: Local tar file.
|
| 160 |
+
classify: Member classifier (see :data:`Classifier`).
|
| 161 |
+
read_text: Keep the bytes of every ``text`` member.
|
| 162 |
+
extract: ``key -> destination`` of the ``audio`` members to write (atomically). When
|
| 163 |
+
``read_text`` is false the pass stops as soon as all of them are written.
|
| 164 |
+
audio_head: Keep the first this many bytes of every ``audio`` member (e.g. a FLAC
|
| 165 |
+
header, to learn the duration without reading the audio).
|
| 166 |
+
|
| 167 |
+
Raises:
|
| 168 |
+
ArchiveError: unreadable/truncated/corrupt archive (including one that ends without
|
| 169 |
+
its end blocks, or whose compressed stream has no trailer); a member that is
|
| 170 |
+
neither a regular file nor a directory, or that ``classify`` rejects; a key seen
|
| 171 |
+
twice; an oversized member.
|
| 172 |
+
FileNotFoundError: ``archive`` does not exist.
|
| 173 |
+
OSError: a destination cannot be written (disk full ...).
|
| 174 |
+
"""
|
| 175 |
+
scan = ArchiveScan()
|
| 176 |
+
unexpected: list[str] = []
|
| 177 |
+
seen: set[str] = set()
|
| 178 |
+
wanted = extract or {}
|
| 179 |
+
try:
|
| 180 |
+
with archive.open("rb") as raw:
|
| 181 |
+
reader = _Reader(_decompressed(raw), archive)
|
| 182 |
+
with tarfile.open(
|
| 183 |
+
fileobj=cast(IO[bytes], reader), # stream mode only calls ``read``
|
| 184 |
+
mode="r|",
|
| 185 |
+
) as tar:
|
| 186 |
+
for member in tar:
|
| 187 |
+
# Streaming mode keeps every TarInfo alive otherwise (hundreds of thousands).
|
| 188 |
+
tar.members = [] # type: ignore[attr-defined] # runtime attribute, not typed
|
| 189 |
+
if member.isdir():
|
| 190 |
+
continue
|
| 191 |
+
name = _normalise(member.name)
|
| 192 |
+
classified = classify(name) if member.isfile() else None
|
| 193 |
+
if classified is None:
|
| 194 |
+
unexpected.append(name)
|
| 195 |
+
continue
|
| 196 |
+
key, kind = classified
|
| 197 |
+
if kind == "ignore":
|
| 198 |
+
continue
|
| 199 |
+
if key in seen:
|
| 200 |
+
raise ArchiveError(f"{archive}: member {name!r} occurs twice")
|
| 201 |
+
seen.add(key)
|
| 202 |
+
scan.members += 1
|
| 203 |
+
if kind == "text":
|
| 204 |
+
_check_size(archive, name, member.size, MAX_TEXT_BYTES)
|
| 205 |
+
if read_text:
|
| 206 |
+
scan.texts[key] = _read(tar, member, archive)
|
| 207 |
+
elif key in wanted:
|
| 208 |
+
_write(tar, member, archive, wanted[key])
|
| 209 |
+
scan.extracted.add(key)
|
| 210 |
+
else:
|
| 211 |
+
scan.audio_keys.add(key)
|
| 212 |
+
if audio_head:
|
| 213 |
+
scan.audio_heads[key] = _read(tar, member, archive, audio_head)
|
| 214 |
+
if key in wanted:
|
| 215 |
+
_check_size(archive, name, member.size, MAX_AUDIO_BYTES)
|
| 216 |
+
_write(tar, member, archive, wanted[key])
|
| 217 |
+
scan.extracted.add(key)
|
| 218 |
+
if not read_text and wanted and len(scan.extracted) == len(wanted):
|
| 219 |
+
scan.complete = False
|
| 220 |
+
break
|
| 221 |
+
if scan.complete:
|
| 222 |
+
_require_end_blocks(archive, reader, tar.offset)
|
| 223 |
+
except (tarfile.TarError, EOFError, zlib.error, gzip.BadGzipFile, lzma.LZMAError) as err:
|
| 224 |
+
raise ArchiveError(f"{archive}: cannot be read as a tar archive: {err!r}") from err
|
| 225 |
+
if unexpected and scan.complete:
|
| 226 |
+
shown = ", ".join(repr(n) for n in unexpected[:_SHOWN_UNEXPECTED])
|
| 227 |
+
raise ArchiveError(
|
| 228 |
+
f"{archive}: {len(unexpected)} unexpected member(s) (not a directory or a known "
|
| 229 |
+
f"file; the format changed?): {shown}"
|
| 230 |
+
)
|
| 231 |
+
return scan
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _require_end_blocks(archive: Path, reader: _Reader, end_offset: int) -> None:
|
| 235 |
+
"""The member loop ended: make sure it ended at the archive's end blocks and read the
|
| 236 |
+
stream to its true end (that is where gzip/bz2/xz check their trailer).
|
| 237 |
+
|
| 238 |
+
``end_offset`` is where ``tarfile`` stopped looking for headers. From there on the stream
|
| 239 |
+
must be zeros only, at least two blocks of them; a cut inside a header leaves non-zero
|
| 240 |
+
bytes or too few zeros, a cut between members leaves nothing.
|
| 241 |
+
"""
|
| 242 |
+
while reader.read(_DRAIN_BYTES):
|
| 243 |
+
pass
|
| 244 |
+
if reader.last_nonzero_end > end_offset or reader.pos - end_offset < _END_BYTES:
|
| 245 |
+
raise ArchiveError(
|
| 246 |
+
f"{archive}: ends without the tar end-of-archive blocks (truncated?): "
|
| 247 |
+
f"{reader.pos - end_offset} bytes after the last member at {end_offset}, "
|
| 248 |
+
f"non-zero data up to {reader.last_nonzero_end}"
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def verify_sha256_once(path: Path, expected: str) -> None:
|
| 253 |
+
"""Check that the file at ``path`` has the SHA-256 ``expected``, hashing it only once.
|
| 254 |
+
|
| 255 |
+
A marker next to the file (``<name>.verified.json``) records the size, mtime and digest of
|
| 256 |
+
a verified file; while those still match, the (21 GB) file is not read again.
|
| 257 |
+
|
| 258 |
+
Raises:
|
| 259 |
+
ArchiveError: the digest differs (the file is incomplete or not the pinned one).
|
| 260 |
+
"""
|
| 261 |
+
marker = path.with_name(path.name + ".verified.json")
|
| 262 |
+
stat = path.stat()
|
| 263 |
+
key = {"size": stat.st_size, "mtime_ns": stat.st_mtime_ns, "sha256": expected}
|
| 264 |
+
if marker.is_file():
|
| 265 |
+
try:
|
| 266 |
+
if json.loads(marker.read_text(encoding="utf-8")) == key:
|
| 267 |
+
return
|
| 268 |
+
except json.JSONDecodeError:
|
| 269 |
+
pass # a damaged marker is only a missing marker: hash again
|
| 270 |
+
digest = hashlib.sha256()
|
| 271 |
+
with path.open("rb") as handle:
|
| 272 |
+
while chunk := handle.read(_HASH_CHUNK_BYTES):
|
| 273 |
+
digest.update(chunk)
|
| 274 |
+
if digest.hexdigest() != expected:
|
| 275 |
+
marker.unlink(missing_ok=True)
|
| 276 |
+
raise ArchiveError(
|
| 277 |
+
f"{path}: sha256 {digest.hexdigest()} (size {stat.st_size}) is not the pinned "
|
| 278 |
+
f"{expected}; the download is incomplete or corrupt - delete it and run again"
|
| 279 |
+
)
|
| 280 |
+
atomic_write_json(marker, key)
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def _check_size(archive: Path, name: str, size: int, limit: int) -> None:
|
| 284 |
+
if size > limit:
|
| 285 |
+
raise ArchiveError(f"{archive}: member {name!r} has {size} bytes (limit {limit})")
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def _read(tar: tarfile.TarFile, member: tarfile.TarInfo, archive: Path, size: int = -1) -> bytes:
|
| 289 |
+
handle = tar.extractfile(member)
|
| 290 |
+
if handle is None:
|
| 291 |
+
raise ArchiveError(f"{archive}: cannot read member {member.name!r}")
|
| 292 |
+
return handle.read(size)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _write(tar: tarfile.TarFile, member: tarfile.TarInfo, archive: Path, dest: Path) -> None:
|
| 296 |
+
atomic_write_bytes(dest, _read(tar, member, archive))
|
src/nida/data/importer/audio.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio helpers of the import stages: raw file -> 16 kHz mono PCM wav, plus measurements.
|
| 2 |
+
|
| 3 |
+
The wav is what both model workers read (the CTC model requires 16 kHz mono; the VAE encoder
|
| 4 |
+
resamples to its own rate itself, from 16 kHz exactly like the pilot's YODAS audio). Channels
|
| 5 |
+
are averaged, the signal is resampled with a polyphase filter
|
| 6 |
+
(:func:`nida.data.ears.resample`) and written as 16-bit PCM atomically.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import io
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import numpy.typing as npt
|
| 17 |
+
|
| 18 |
+
from nida.data import acoustic
|
| 19 |
+
from nida.data.ears import resample
|
| 20 |
+
from nida.utils.io import atomic_write_bytes
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"TARGET_RATE_HZ",
|
| 24 |
+
"AudioFacts",
|
| 25 |
+
"AudioRejected",
|
| 26 |
+
"convert_to_wav16k",
|
| 27 |
+
"measure_wav16k",
|
| 28 |
+
"raw_duration_ms",
|
| 29 |
+
"require_audio_format",
|
| 30 |
+
]
|
| 31 |
+
|
| 32 |
+
#: Rate of the working copy (CTC requirement).
|
| 33 |
+
TARGET_RATE_HZ = 16000
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class AudioRejected(Exception):
|
| 37 |
+
"""An audio file that cannot be used; ``reason`` is the quarantine reason."""
|
| 38 |
+
|
| 39 |
+
def __init__(self, reason: str, **details: object) -> None:
|
| 40 |
+
super().__init__(f"{reason}: {details}")
|
| 41 |
+
self.reason = reason
|
| 42 |
+
self.details = dict(details)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class AudioFacts:
|
| 47 |
+
"""Measurements of a 16 kHz working copy.
|
| 48 |
+
|
| 49 |
+
Attributes:
|
| 50 |
+
duration_ms: ``round(1000 * samples / 16000)``.
|
| 51 |
+
loudness_dbfs: Whole-signal RMS level (:func:`nida.data.acoustic.loudness_dbfs`).
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
duration_ms: int
|
| 55 |
+
loudness_dbfs: float
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def require_audio_format(name: str) -> None:
|
| 59 |
+
"""Raise unless the installed libsndfile reads the format ``name`` (e.g. ``"MP3"``,
|
| 60 |
+
which needs libsndfile >= 1.1, bundled with ``soundfile`` >= 0.12).
|
| 61 |
+
|
| 62 |
+
Raises:
|
| 63 |
+
RuntimeError: the format is not available (installing a newer ``soundfile`` fixes it).
|
| 64 |
+
"""
|
| 65 |
+
import soundfile # type: ignore[import-untyped] # bench extra, untyped
|
| 66 |
+
|
| 67 |
+
if name not in soundfile.available_formats():
|
| 68 |
+
raise RuntimeError(
|
| 69 |
+
f"this soundfile/libsndfile ({soundfile.__libsndfile_version__}) cannot read "
|
| 70 |
+
f"{name}; upgrade soundfile (>= 0.12 bundles libsndfile 1.1+)"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def raw_duration_ms(path: Path) -> int:
|
| 75 |
+
"""Duration of an audio file from its header (no decoding).
|
| 76 |
+
|
| 77 |
+
Raises:
|
| 78 |
+
AudioRejected: ``unreadable_audio`` (not an audio file) or ``empty_audio``.
|
| 79 |
+
"""
|
| 80 |
+
import soundfile
|
| 81 |
+
|
| 82 |
+
try:
|
| 83 |
+
info = soundfile.info(str(path))
|
| 84 |
+
except (RuntimeError, OSError) as err:
|
| 85 |
+
raise AudioRejected("unreadable_audio", error=str(err)) from err
|
| 86 |
+
if int(info.frames) <= 0 or int(info.samplerate) <= 0:
|
| 87 |
+
raise AudioRejected("empty_audio")
|
| 88 |
+
duration_ms: int = round(1000 * int(info.frames) / int(info.samplerate))
|
| 89 |
+
return duration_ms
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _read_mono(path: Path) -> tuple[npt.NDArray[np.float64], int]:
|
| 93 |
+
"""``(samples, rate_hz)`` with channels averaged; rejects unreadable/empty/non-finite."""
|
| 94 |
+
import soundfile
|
| 95 |
+
|
| 96 |
+
try:
|
| 97 |
+
audio, rate_hz = soundfile.read(str(path), dtype="float64", always_2d=True)
|
| 98 |
+
except (RuntimeError, OSError) as err:
|
| 99 |
+
raise AudioRejected("unreadable_audio", error=str(err)) from err
|
| 100 |
+
if audio.shape[0] == 0:
|
| 101 |
+
raise AudioRejected("empty_audio")
|
| 102 |
+
mono = np.ascontiguousarray(audio.mean(axis=1))
|
| 103 |
+
if not np.all(np.isfinite(mono)):
|
| 104 |
+
raise AudioRejected("non_finite_audio")
|
| 105 |
+
return mono, int(rate_hz)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def convert_to_wav16k(raw: Path, wav: Path) -> None:
|
| 109 |
+
"""Write the 16 kHz mono 16-bit PCM copy of ``raw`` atomically to ``wav``.
|
| 110 |
+
|
| 111 |
+
Raises:
|
| 112 |
+
AudioRejected: ``unreadable_audio``, ``empty_audio`` or ``non_finite_audio``.
|
| 113 |
+
"""
|
| 114 |
+
import soundfile
|
| 115 |
+
|
| 116 |
+
mono, rate_hz = _read_mono(raw)
|
| 117 |
+
converted = resample(mono, rate_hz, TARGET_RATE_HZ)
|
| 118 |
+
buffer = io.BytesIO()
|
| 119 |
+
soundfile.write(
|
| 120 |
+
buffer, np.clip(converted, -1.0, 1.0), TARGET_RATE_HZ, format="WAV", subtype="PCM_16"
|
| 121 |
+
)
|
| 122 |
+
atomic_write_bytes(wav, buffer.getvalue())
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def measure_wav16k(wav: Path) -> AudioFacts:
|
| 126 |
+
"""Duration and loudness of a working copy.
|
| 127 |
+
|
| 128 |
+
Raises:
|
| 129 |
+
AudioRejected: the file is unreadable/empty, or is not a 16 kHz mono file (a stale
|
| 130 |
+
or foreign cache entry; the caller reconverts it).
|
| 131 |
+
"""
|
| 132 |
+
import soundfile
|
| 133 |
+
|
| 134 |
+
try:
|
| 135 |
+
info = soundfile.info(str(wav))
|
| 136 |
+
except (RuntimeError, OSError) as err:
|
| 137 |
+
raise AudioRejected("unreadable_audio", error=str(err)) from err
|
| 138 |
+
if int(info.samplerate) != TARGET_RATE_HZ or int(info.channels) != 1:
|
| 139 |
+
raise AudioRejected(
|
| 140 |
+
"bad_working_copy", sample_rate_hz=int(info.samplerate), channels=int(info.channels)
|
| 141 |
+
)
|
| 142 |
+
mono, _ = _read_mono(wav)
|
| 143 |
+
return AudioFacts(
|
| 144 |
+
duration_ms=round(1000 * mono.shape[0] / TARGET_RATE_HZ),
|
| 145 |
+
loudness_dbfs=float(acoustic.loudness_dbfs(mono)),
|
| 146 |
+
)
|
src/nida/data/importer/config.py
ADDED
|
@@ -0,0 +1,474 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""``configs/data/import.yaml`` (+ ``dataset.yaml``) as validated, frozen records."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import re
|
| 9 |
+
from collections.abc import Mapping
|
| 10 |
+
from dataclasses import asdict, dataclass
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
from nida.data.asr_filter import AsrFilterConfig
|
| 14 |
+
from nida.data.ears import en_lower
|
| 15 |
+
from nida.data.pilot import DatasetConfig
|
| 16 |
+
from nida.data.registry import DATA_MODELS
|
| 17 |
+
from nida.train.data import EMOTIONS, SPLITS
|
| 18 |
+
|
| 19 |
+
__all__ = [
|
| 20 |
+
"CORPUS_SOURCES",
|
| 21 |
+
"LEGACY_SOURCES",
|
| 22 |
+
"AlaniaConfig",
|
| 23 |
+
"CommonVoiceConfig",
|
| 24 |
+
"ImportConfig",
|
| 25 |
+
"IssaiTscConfig",
|
| 26 |
+
"MediaSpeechConfig",
|
| 27 |
+
"TrFullConfig",
|
| 28 |
+
"WavdirConfig",
|
| 29 |
+
"load_import_config",
|
| 30 |
+
]
|
| 31 |
+
|
| 32 |
+
IMPORT_CONFIG_FILE = "import.yaml"
|
| 33 |
+
DATASET_CONFIG_FILE = "dataset.yaml"
|
| 34 |
+
#: Sections of the corpora added after the first three sources (see :func:`load_import_config`).
|
| 35 |
+
CORPORA_CONFIG_FILE = "import_corpora.yaml"
|
| 36 |
+
#: Importer sources whose config lives in ``import.yaml``; their digest is the original one.
|
| 37 |
+
LEGACY_SOURCES = ("alania", "tr_full", "wavdir")
|
| 38 |
+
#: Importer sources configured in ``import_corpora.yaml`` (a digest scoped per source).
|
| 39 |
+
CORPUS_SOURCES = ("issai_tsc", "mediaspeech_tr", "common_voice_tr")
|
| 40 |
+
_SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _load_yaml(path: Path) -> Mapping[str, object]:
|
| 44 |
+
import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
|
| 45 |
+
|
| 46 |
+
obj = yaml.safe_load(path.read_text(encoding="utf-8"))
|
| 47 |
+
if not isinstance(obj, dict):
|
| 48 |
+
raise ValueError(f"{path}: expected a mapping")
|
| 49 |
+
return obj
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _keys(raw: Mapping[str, object], expected: set[str], where: str) -> None:
|
| 53 |
+
if set(raw) != expected:
|
| 54 |
+
raise ValueError(
|
| 55 |
+
f"{where}: keys differ: missing={sorted(expected - set(raw))} "
|
| 56 |
+
f"unknown={sorted(set(raw) - expected)}"
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _mapping(raw: Mapping[str, object], key: str) -> Mapping[str, object]:
|
| 61 |
+
value = raw[key]
|
| 62 |
+
if not isinstance(value, dict) or not all(isinstance(k, str) for k in value):
|
| 63 |
+
raise ValueError(f"{key} must be a mapping")
|
| 64 |
+
return value
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _text(raw: Mapping[str, object], key: str, *, allow_empty: bool = False) -> str:
|
| 68 |
+
value = raw[key]
|
| 69 |
+
if not isinstance(value, str) or not (value or allow_empty):
|
| 70 |
+
raise ValueError(f"{key} must be a non-empty string, got {value!r}")
|
| 71 |
+
return value
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _int(raw: Mapping[str, object], key: str, *, minimum: int) -> int:
|
| 75 |
+
value = raw[key]
|
| 76 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
|
| 77 |
+
raise ValueError(f"{key} must be an int >= {minimum}, got {value!r}")
|
| 78 |
+
return value
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _bool(raw: Mapping[str, object], key: str) -> bool:
|
| 82 |
+
value = raw[key]
|
| 83 |
+
if not isinstance(value, bool):
|
| 84 |
+
raise ValueError(f"{key} must be true or false, got {value!r}")
|
| 85 |
+
return value
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _float(raw: Mapping[str, object], key: str, *, lo: float, hi: float) -> float:
|
| 89 |
+
value = raw[key]
|
| 90 |
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
| 91 |
+
raise ValueError(f"{key} must be a number, got {value!r}")
|
| 92 |
+
if not (math.isfinite(value) and lo <= value <= hi):
|
| 93 |
+
raise ValueError(f"{key} must be in [{lo}, {hi}], got {value!r}")
|
| 94 |
+
return float(value)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
@dataclass(frozen=True)
|
| 98 |
+
class AlaniaConfig:
|
| 99 |
+
"""``alania:`` block (source ``alania_synth_tr``)."""
|
| 100 |
+
|
| 101 |
+
repo_id: str
|
| 102 |
+
parquet_glob: str
|
| 103 |
+
min_utmos: float
|
| 104 |
+
max_cer: float
|
| 105 |
+
license_prefix: str
|
| 106 |
+
batch_rows: int
|
| 107 |
+
group_prefix: str
|
| 108 |
+
|
| 109 |
+
@classmethod
|
| 110 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> AlaniaConfig:
|
| 111 |
+
_keys(raw, set(cls.__dataclass_fields__), "alania")
|
| 112 |
+
return cls(
|
| 113 |
+
repo_id=_text(raw, "repo_id"),
|
| 114 |
+
parquet_glob=_text(raw, "parquet_glob"),
|
| 115 |
+
min_utmos=_float(raw, "min_utmos", lo=0.0, hi=5.0),
|
| 116 |
+
max_cer=_float(raw, "max_cer", lo=0.0, hi=1.0),
|
| 117 |
+
license_prefix=_text(raw, "license_prefix"),
|
| 118 |
+
batch_rows=_int(raw, "batch_rows", minimum=1),
|
| 119 |
+
group_prefix=_text(raw, "group_prefix", allow_empty=True),
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@dataclass(frozen=True)
|
| 124 |
+
class TrFullConfig:
|
| 125 |
+
"""``tr_full:`` block (source ``tr_full_dataset``)."""
|
| 126 |
+
|
| 127 |
+
repo_id: str
|
| 128 |
+
data_file: str
|
| 129 |
+
language: str
|
| 130 |
+
unit_rows: int
|
| 131 |
+
group_prefix: str
|
| 132 |
+
|
| 133 |
+
@classmethod
|
| 134 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> TrFullConfig:
|
| 135 |
+
_keys(raw, set(cls.__dataclass_fields__), "tr_full")
|
| 136 |
+
return cls(
|
| 137 |
+
repo_id=_text(raw, "repo_id"),
|
| 138 |
+
data_file=_text(raw, "data_file"),
|
| 139 |
+
language=_text(raw, "language"),
|
| 140 |
+
unit_rows=_int(raw, "unit_rows", minimum=1),
|
| 141 |
+
group_prefix=_text(raw, "group_prefix", allow_empty=True),
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
@dataclass(frozen=True)
|
| 146 |
+
class WavdirConfig:
|
| 147 |
+
"""``wavdir:`` block (a folder with ``manifest.jsonl``)."""
|
| 148 |
+
|
| 149 |
+
unit_rows: int
|
| 150 |
+
group_prefix: str
|
| 151 |
+
|
| 152 |
+
@classmethod
|
| 153 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> WavdirConfig:
|
| 154 |
+
_keys(raw, set(cls.__dataclass_fields__), "wavdir")
|
| 155 |
+
return cls(
|
| 156 |
+
unit_rows=_int(raw, "unit_rows", minimum=1),
|
| 157 |
+
group_prefix=_text(raw, "group_prefix", allow_empty=True),
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def _str_list(raw: Mapping[str, object], key: str) -> tuple[str, ...]:
|
| 162 |
+
value = raw[key]
|
| 163 |
+
if (
|
| 164 |
+
not isinstance(value, list)
|
| 165 |
+
or not value
|
| 166 |
+
or not all(isinstance(v, str) and v for v in value)
|
| 167 |
+
or len(set(value)) != len(value)
|
| 168 |
+
):
|
| 169 |
+
raise ValueError(f"{key} must be a non-empty list of distinct non-empty strings")
|
| 170 |
+
return tuple(value)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _plain_name(raw: Mapping[str, object], key: str) -> str:
|
| 174 |
+
"""A single path component (no separators, not ``.``/``..``)."""
|
| 175 |
+
value = _text(raw, key)
|
| 176 |
+
if "/" in value or "\\" in value or "\x00" in value or value in (".", ".."):
|
| 177 |
+
raise ValueError(f"{key} must be one plain file or folder name, got {value!r}")
|
| 178 |
+
return value
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _sha256(raw: Mapping[str, object], key: str) -> str:
|
| 182 |
+
value = _text(raw, key)
|
| 183 |
+
if _SHA256_RE.fullmatch(value) is None:
|
| 184 |
+
raise ValueError(f"{key} must be 64 lower-case hex digits, got {value!r}")
|
| 185 |
+
return value
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _force_split(raw: Mapping[str, object]) -> str | None:
|
| 189 |
+
"""``force_split``: ``null`` (split by group hash) or the one split every row goes to."""
|
| 190 |
+
value = raw["force_split"]
|
| 191 |
+
if value is not None and value not in SPLITS:
|
| 192 |
+
raise ValueError(f"force_split must be null or one of {SPLITS}, got {value!r}")
|
| 193 |
+
return value
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@dataclass(frozen=True)
|
| 197 |
+
class IssaiTscConfig:
|
| 198 |
+
"""``issai_tsc:`` block of ``import_corpora.yaml`` (source ``issai_tsc``).
|
| 199 |
+
|
| 200 |
+
``group_block_ids``: the corpus names no speaker or session, only a running utterance
|
| 201 |
+
number that follows the recording order, so ``numeric id // group_block_ids`` is the
|
| 202 |
+
group (reporting only while ``force_split`` is set). ``archive_sha256`` and
|
| 203 |
+
``expected_pairs`` (utterance stems, wav and/or txt) guard against a truncated or changed
|
| 204 |
+
archive. ``force_split``: every row goes to this split instead of the group hash (the
|
| 205 |
+
corpus' recordings are contiguous and no group can be pure).
|
| 206 |
+
"""
|
| 207 |
+
|
| 208 |
+
repo_id: str
|
| 209 |
+
archive_file: str
|
| 210 |
+
archive_sha256: str
|
| 211 |
+
expected_pairs: int
|
| 212 |
+
archive_root: str
|
| 213 |
+
splits: tuple[str, ...]
|
| 214 |
+
unit_rows: int
|
| 215 |
+
group_block_ids: int
|
| 216 |
+
group_prefix: str
|
| 217 |
+
force_split: str | None
|
| 218 |
+
|
| 219 |
+
@classmethod
|
| 220 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> IssaiTscConfig:
|
| 221 |
+
_keys(raw, set(cls.__dataclass_fields__), "issai_tsc")
|
| 222 |
+
return cls(
|
| 223 |
+
repo_id=_text(raw, "repo_id"),
|
| 224 |
+
archive_file=_plain_name(raw, "archive_file"),
|
| 225 |
+
archive_sha256=_sha256(raw, "archive_sha256"),
|
| 226 |
+
expected_pairs=_int(raw, "expected_pairs", minimum=1),
|
| 227 |
+
archive_root=_plain_name(raw, "archive_root"),
|
| 228 |
+
splits=_str_list(raw, "splits"),
|
| 229 |
+
unit_rows=_int(raw, "unit_rows", minimum=1),
|
| 230 |
+
group_block_ids=_int(raw, "group_block_ids", minimum=1),
|
| 231 |
+
group_prefix=_text(raw, "group_prefix", allow_empty=True),
|
| 232 |
+
force_split=_force_split(raw),
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
@dataclass(frozen=True)
|
| 237 |
+
class MediaSpeechConfig:
|
| 238 |
+
"""``mediaspeech_tr:`` block of ``import_corpora.yaml`` (OpenSLR SLR108 ``TR.tgz``).
|
| 239 |
+
|
| 240 |
+
``sha256`` pins the archive (a different file is refused); ``expected_pairs`` is the
|
| 241 |
+
number of utterance stems it holds; ``group`` is the one group of the whole corpus (file
|
| 242 |
+
names are random UUIDs: no speaker, channel or video); ``force_split``: see
|
| 243 |
+
:class:`IssaiTscConfig`; ``reserve_bench_clips``: quarantine the clips the phase-0 B1
|
| 244 |
+
benchmark drew from this corpus (``bench_reserved``), so they stay unseen by training.
|
| 245 |
+
"""
|
| 246 |
+
|
| 247 |
+
url: str
|
| 248 |
+
sha256: str
|
| 249 |
+
expected_pairs: int
|
| 250 |
+
archive_root: str
|
| 251 |
+
unit_rows: int
|
| 252 |
+
group: str
|
| 253 |
+
force_split: str | None
|
| 254 |
+
reserve_bench_clips: bool
|
| 255 |
+
|
| 256 |
+
@classmethod
|
| 257 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> MediaSpeechConfig:
|
| 258 |
+
_keys(raw, set(cls.__dataclass_fields__), "mediaspeech_tr")
|
| 259 |
+
url = _text(raw, "url")
|
| 260 |
+
if not url.startswith("https://"):
|
| 261 |
+
raise ValueError(f"url must be an https:// URL, got {url!r}")
|
| 262 |
+
return cls(
|
| 263 |
+
url=url,
|
| 264 |
+
sha256=_sha256(raw, "sha256"),
|
| 265 |
+
expected_pairs=_int(raw, "expected_pairs", minimum=1),
|
| 266 |
+
archive_root=_plain_name(raw, "archive_root"),
|
| 267 |
+
unit_rows=_int(raw, "unit_rows", minimum=1),
|
| 268 |
+
group=_text(raw, "group"),
|
| 269 |
+
force_split=_force_split(raw),
|
| 270 |
+
reserve_bench_clips=_bool(raw, "reserve_bench_clips"),
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
@dataclass(frozen=True)
|
| 275 |
+
class CommonVoiceConfig:
|
| 276 |
+
"""``common_voice_tr:`` block of ``import_corpora.yaml`` (a local Common Voice release).
|
| 277 |
+
|
| 278 |
+
``min_net_votes``: a clip is used when ``up_votes - down_votes >= min_net_votes`` (1 means
|
| 279 |
+
``down_votes < up_votes``). The group is a hash of ``client_id`` behind ``group_prefix``.
|
| 280 |
+
"""
|
| 281 |
+
|
| 282 |
+
locale: str
|
| 283 |
+
validated_file: str
|
| 284 |
+
clips_dir: str
|
| 285 |
+
min_net_votes: int
|
| 286 |
+
unit_rows: int
|
| 287 |
+
group_prefix: str
|
| 288 |
+
|
| 289 |
+
@classmethod
|
| 290 |
+
def from_mapping(cls, raw: Mapping[str, object]) -> CommonVoiceConfig:
|
| 291 |
+
_keys(raw, set(cls.__dataclass_fields__), "common_voice_tr")
|
| 292 |
+
return cls(
|
| 293 |
+
locale=_text(raw, "locale"),
|
| 294 |
+
validated_file=_plain_name(raw, "validated_file"),
|
| 295 |
+
clips_dir=_plain_name(raw, "clips_dir"),
|
| 296 |
+
min_net_votes=_int(raw, "min_net_votes", minimum=1),
|
| 297 |
+
unit_rows=_int(raw, "unit_rows", minimum=1),
|
| 298 |
+
group_prefix=_text(raw, "group_prefix", allow_empty=True),
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
@dataclass(frozen=True)
|
| 303 |
+
class ImportConfig:
|
| 304 |
+
"""Every threshold and mapping of the import stages.
|
| 305 |
+
|
| 306 |
+
Attributes:
|
| 307 |
+
seed: Split-hash seed (``str(seed)`` is passed to :func:`nida.data.splits.assign_splits`).
|
| 308 |
+
val_fraction: Share of groups in the ``val`` split.
|
| 309 |
+
test_fraction: Share of groups in the ``test`` split.
|
| 310 |
+
min_duration_ms: Shortest accepted segment.
|
| 311 |
+
max_duration_ms: Longest accepted segment.
|
| 312 |
+
min_loudness_dbfs: Segments quieter than this (RMS, dBFS) are ``silent``.
|
| 313 |
+
alignment: Thresholds of the forced-alignment sanity check
|
| 314 |
+
(:func:`nida.data.asr_filter.check_alignment`; ``max_cer`` and the languages of
|
| 315 |
+
that record are unused here).
|
| 316 |
+
char_map: Single-character replacements applied to transcripts.
|
| 317 |
+
emotion_map: Source emotion label -> :data:`nida.train.data.EMOTIONS` id.
|
| 318 |
+
issai_tsc: ``import_corpora.yaml`` section of the ISSAI Turkish Speech Corpus.
|
| 319 |
+
mediaspeech_tr: ``import_corpora.yaml`` section of MediaSpeech (SLR108).
|
| 320 |
+
common_voice_tr: ``import_corpora.yaml`` section of Common Voice.
|
| 321 |
+
dataset: Fixed rate/loudness bucket edges (``dataset.yaml``, shared with the pilot).
|
| 322 |
+
digest: 12 hex digits over the config files and the VAE/CTC model pins; names the
|
| 323 |
+
dataset directory and is recorded in ``plan.json``. For ``alania``, ``tr_full``,
|
| 324 |
+
``wavdir`` (and when no source is given) it covers ``import.yaml`` and
|
| 325 |
+
``dataset.yaml`` only - the value those roots were made with. For a source of
|
| 326 |
+
``import_corpora.yaml`` it is derived from that value and the source's own section.
|
| 327 |
+
"""
|
| 328 |
+
|
| 329 |
+
seed: int
|
| 330 |
+
val_fraction: float
|
| 331 |
+
test_fraction: float
|
| 332 |
+
min_duration_ms: int
|
| 333 |
+
max_duration_ms: int
|
| 334 |
+
min_loudness_dbfs: float
|
| 335 |
+
alignment: AsrFilterConfig
|
| 336 |
+
char_map: Mapping[str, str]
|
| 337 |
+
emotion_map: Mapping[str, str]
|
| 338 |
+
alania: AlaniaConfig
|
| 339 |
+
tr_full: TrFullConfig
|
| 340 |
+
wavdir: WavdirConfig
|
| 341 |
+
issai_tsc: IssaiTscConfig
|
| 342 |
+
mediaspeech_tr: MediaSpeechConfig
|
| 343 |
+
common_voice_tr: CommonVoiceConfig
|
| 344 |
+
dataset: DatasetConfig
|
| 345 |
+
digest: str
|
| 346 |
+
|
| 347 |
+
def forced_split(self, source: str) -> str | None:
|
| 348 |
+
"""The split every row of ``source`` goes to (``force_split``), or ``None``."""
|
| 349 |
+
if source == "issai_tsc":
|
| 350 |
+
return self.issai_tsc.force_split
|
| 351 |
+
if source == "mediaspeech_tr":
|
| 352 |
+
return self.mediaspeech_tr.force_split
|
| 353 |
+
return None
|
| 354 |
+
|
| 355 |
+
@property
|
| 356 |
+
def split_fractions(self) -> dict[str, float]:
|
| 357 |
+
"""``{"train": 1 - val - test, "val": ..., "test": ...}``."""
|
| 358 |
+
return {
|
| 359 |
+
"train": 1.0 - self.val_fraction - self.test_fraction,
|
| 360 |
+
"val": self.val_fraction,
|
| 361 |
+
"test": self.test_fraction,
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def load_import_config(config_dir: Path, source: str | None = None) -> ImportConfig:
|
| 366 |
+
"""Read and validate ``import.yaml``, ``dataset.yaml`` and ``import_corpora.yaml``.
|
| 367 |
+
|
| 368 |
+
Args:
|
| 369 |
+
config_dir: Folder with the three files.
|
| 370 |
+
source: The importer source the config is for; it scopes :attr:`ImportConfig.digest`
|
| 371 |
+
(``None`` or a legacy source: the digest of ``import.yaml`` + ``dataset.yaml``,
|
| 372 |
+
unchanged by ``import_corpora.yaml``; a corpus source: that digest combined with
|
| 373 |
+
the source's own section, so editing one corpus never moves another's roots).
|
| 374 |
+
|
| 375 |
+
Raises:
|
| 376 |
+
ValueError: on unknown/missing keys, out-of-range numbers, multi-character
|
| 377 |
+
``char_map`` keys, an ``emotion_map`` target that is not an emotion id, or an
|
| 378 |
+
unknown ``source``.
|
| 379 |
+
FileNotFoundError: a config file is missing.
|
| 380 |
+
"""
|
| 381 |
+
if source is not None and source not in (*LEGACY_SOURCES, *CORPUS_SOURCES):
|
| 382 |
+
known = (*LEGACY_SOURCES, *CORPUS_SOURCES)
|
| 383 |
+
raise ValueError(f"unknown source {source!r}; known: {known}")
|
| 384 |
+
import_path = config_dir / IMPORT_CONFIG_FILE
|
| 385 |
+
dataset_path = config_dir / DATASET_CONFIG_FILE
|
| 386 |
+
corpora = _load_yaml(config_dir / CORPORA_CONFIG_FILE)
|
| 387 |
+
_keys(corpora, set(CORPUS_SOURCES), CORPORA_CONFIG_FILE)
|
| 388 |
+
raw = _load_yaml(import_path)
|
| 389 |
+
_keys(
|
| 390 |
+
raw,
|
| 391 |
+
{
|
| 392 |
+
"seed",
|
| 393 |
+
"val_fraction",
|
| 394 |
+
"test_fraction",
|
| 395 |
+
"min_duration_ms",
|
| 396 |
+
"max_duration_ms",
|
| 397 |
+
"min_loudness_dbfs",
|
| 398 |
+
"alignment",
|
| 399 |
+
"char_map",
|
| 400 |
+
"emotion_map",
|
| 401 |
+
"alania",
|
| 402 |
+
"tr_full",
|
| 403 |
+
"wavdir",
|
| 404 |
+
},
|
| 405 |
+
IMPORT_CONFIG_FILE,
|
| 406 |
+
)
|
| 407 |
+
char_map = _mapping(raw, "char_map")
|
| 408 |
+
for key, value in char_map.items():
|
| 409 |
+
if len(key) != 1 or not isinstance(value, str):
|
| 410 |
+
raise ValueError(f"char_map: {key!r} must be one character mapped to a string")
|
| 411 |
+
emotion_map = _mapping(raw, "emotion_map")
|
| 412 |
+
for label, emotion in emotion_map.items():
|
| 413 |
+
if not label or not label.isascii() or label != en_lower(label) or label != label.strip():
|
| 414 |
+
raise ValueError(f"emotion_map: label {label!r} must be ASCII lower-case")
|
| 415 |
+
if emotion not in EMOTIONS:
|
| 416 |
+
raise ValueError(f"emotion_map: {label!r} -> {emotion!r} is not one of {EMOTIONS}")
|
| 417 |
+
val_fraction = _float(raw, "val_fraction", lo=0.0, hi=1.0)
|
| 418 |
+
test_fraction = _float(raw, "test_fraction", lo=0.0, hi=1.0)
|
| 419 |
+
if val_fraction + test_fraction >= 1.0:
|
| 420 |
+
raise ValueError("val_fraction + test_fraction must be < 1")
|
| 421 |
+
min_duration_ms = _int(raw, "min_duration_ms", minimum=1)
|
| 422 |
+
max_duration_ms = _int(raw, "max_duration_ms", minimum=1)
|
| 423 |
+
if max_duration_ms <= min_duration_ms:
|
| 424 |
+
raise ValueError("max_duration_ms must exceed min_duration_ms")
|
| 425 |
+
align_raw = _mapping(raw, "alignment")
|
| 426 |
+
_keys(align_raw, {"min_chars_per_s", "max_chars_per_s", "min_align_score"}, "alignment")
|
| 427 |
+
alignment = AsrFilterConfig(
|
| 428 |
+
max_cer=0.0, # S4 only; not used by the importer
|
| 429 |
+
allowed_languages=("tr",), # S4 only; not used by the importer
|
| 430 |
+
min_chars_per_s=_float(align_raw, "min_chars_per_s", lo=0.0, hi=1000.0),
|
| 431 |
+
max_chars_per_s=_float(align_raw, "max_chars_per_s", lo=0.0, hi=1000.0),
|
| 432 |
+
min_align_score=_float(align_raw, "min_align_score", lo=0.0, hi=1.0),
|
| 433 |
+
)
|
| 434 |
+
digest = hashlib.sha1()
|
| 435 |
+
for path in (import_path, dataset_path):
|
| 436 |
+
digest.update(path.name.encode("utf-8") + b"\x00" + path.read_bytes())
|
| 437 |
+
refs = {
|
| 438 |
+
key: [DATA_MODELS[key].ref.model_id, DATA_MODELS[key].ref.revision]
|
| 439 |
+
for key in ("voxcpm2_audiovae_v2", "mpoyraz_ctc_tr")
|
| 440 |
+
}
|
| 441 |
+
digest.update(json.dumps(refs, sort_keys=True).encode("utf-8"))
|
| 442 |
+
issai_tsc = IssaiTscConfig.from_mapping(_mapping(corpora, "issai_tsc"))
|
| 443 |
+
mediaspeech_tr = MediaSpeechConfig.from_mapping(_mapping(corpora, "mediaspeech_tr"))
|
| 444 |
+
common_voice_tr = CommonVoiceConfig.from_mapping(_mapping(corpora, "common_voice_tr"))
|
| 445 |
+
digest_hex = digest.hexdigest()
|
| 446 |
+
sections: dict[str, IssaiTscConfig | MediaSpeechConfig | CommonVoiceConfig] = {
|
| 447 |
+
"issai_tsc": issai_tsc,
|
| 448 |
+
"mediaspeech_tr": mediaspeech_tr,
|
| 449 |
+
"common_voice_tr": common_voice_tr,
|
| 450 |
+
}
|
| 451 |
+
if source is not None and source in sections:
|
| 452 |
+
scoped = hashlib.sha1(digest_hex.encode("ascii") + b"\x00" + source.encode("ascii"))
|
| 453 |
+
section_json = json.dumps(asdict(sections[source]), sort_keys=True)
|
| 454 |
+
scoped.update(b"\x00" + section_json.encode("utf-8"))
|
| 455 |
+
digest_hex = scoped.hexdigest()
|
| 456 |
+
return ImportConfig(
|
| 457 |
+
seed=_int(raw, "seed", minimum=0),
|
| 458 |
+
val_fraction=val_fraction,
|
| 459 |
+
test_fraction=test_fraction,
|
| 460 |
+
min_duration_ms=min_duration_ms,
|
| 461 |
+
max_duration_ms=max_duration_ms,
|
| 462 |
+
min_loudness_dbfs=_float(raw, "min_loudness_dbfs", lo=-240.0, hi=0.0),
|
| 463 |
+
alignment=alignment,
|
| 464 |
+
char_map={str(k): str(v) for k, v in char_map.items()},
|
| 465 |
+
emotion_map={str(k): str(v) for k, v in emotion_map.items()},
|
| 466 |
+
alania=AlaniaConfig.from_mapping(_mapping(raw, "alania")),
|
| 467 |
+
tr_full=TrFullConfig.from_mapping(_mapping(raw, "tr_full")),
|
| 468 |
+
wavdir=WavdirConfig.from_mapping(_mapping(raw, "wavdir")),
|
| 469 |
+
issai_tsc=issai_tsc,
|
| 470 |
+
mediaspeech_tr=mediaspeech_tr,
|
| 471 |
+
common_voice_tr=common_voice_tr,
|
| 472 |
+
dataset=DatasetConfig.from_mapping(_load_yaml(dataset_path)),
|
| 473 |
+
digest=digest_hex[:12],
|
| 474 |
+
)
|
src/nida/data/importer/corpora.py
ADDED
|
@@ -0,0 +1,801 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Source adapters for the corpora that ship as an archive or as a Common Voice release.
|
| 2 |
+
|
| 3 |
+
* :class:`IssaiTscAdapter` - ``issai/Turkish_Speech_Corpus`` (HF; one 21 GB ``.tar.gz``).
|
| 4 |
+
* :class:`MediaSpeechAdapter` - OpenSLR SLR108 ``TR.tgz`` (pinned by sha256).
|
| 5 |
+
* :class:`CommonVoiceAdapter` - a Common Voice release directory or tarball the user supplied.
|
| 6 |
+
|
| 7 |
+
The first two are *archive corpora*. Transcript and audio of an utterance are far apart in a
|
| 8 |
+
tar stream, so an *index pass* (:func:`nida.data.importer.archives.scan_archive`) reads all
|
| 9 |
+
transcripts once and notes which audio members exist; the rows are put in a deterministic
|
| 10 |
+
order and cut into work units. A unit then costs one *extract pass* that writes the audio of
|
| 11 |
+
its selected rows to ``<cache>/raw/<source>/`` (the audio stage deletes each file after
|
| 12 |
+
converting it, :attr:`ephemeral_raw`; a lost file is extracted again).
|
| 13 |
+
|
| 14 |
+
Every row the adapter cannot turn into an item is quarantined with a reason: ``audio_missing``
|
| 15 |
+
(no audio member for a transcript), ``text_missing`` (no transcript for an audio member),
|
| 16 |
+
``bad_encoding`` (transcript is not UTF-8), the text reasons of
|
| 17 |
+
:func:`~nida.data.importer.text.normalise_transcript`, and per source the ones listed on the
|
| 18 |
+
class.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import csv
|
| 24 |
+
import hashlib
|
| 25 |
+
import json
|
| 26 |
+
import re
|
| 27 |
+
from collections.abc import Generator, Mapping, Set
|
| 28 |
+
from dataclasses import dataclass
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
|
| 31 |
+
from nida.bench import fetch
|
| 32 |
+
from nida.bench.clips import MediaSpeechItem, select_mediaspeech
|
| 33 |
+
from nida.bench.prepare import B1Plan
|
| 34 |
+
from nida.data.importer.archives import (
|
| 35 |
+
ArchiveError,
|
| 36 |
+
ArchiveScan,
|
| 37 |
+
Kind,
|
| 38 |
+
scan_archive,
|
| 39 |
+
verify_sha256_once,
|
| 40 |
+
)
|
| 41 |
+
from nida.data.importer.audio import require_audio_format
|
| 42 |
+
from nida.data.importer.config import (
|
| 43 |
+
CommonVoiceConfig,
|
| 44 |
+
ImportConfig,
|
| 45 |
+
IssaiTscConfig,
|
| 46 |
+
MediaSpeechConfig,
|
| 47 |
+
)
|
| 48 |
+
from nida.data.importer.hub import HubClient, HubError
|
| 49 |
+
from nida.data.importer.sources import (
|
| 50 |
+
Item,
|
| 51 |
+
Scalar,
|
| 52 |
+
UnitListing,
|
| 53 |
+
_bad_row,
|
| 54 |
+
_chunk_ids,
|
| 55 |
+
_chunk_slice,
|
| 56 |
+
_drop,
|
| 57 |
+
_first_seen,
|
| 58 |
+
_text_drop,
|
| 59 |
+
item_segment_id,
|
| 60 |
+
safe_relative_path,
|
| 61 |
+
)
|
| 62 |
+
from nida.data.importer.text import TextRejected, normalise_transcript
|
| 63 |
+
from nida.data.types import QuarantineRecord
|
| 64 |
+
from nida.text.turkish import nfc
|
| 65 |
+
from nida.utils.io import atomic_write_text
|
| 66 |
+
|
| 67 |
+
__all__ = [
|
| 68 |
+
"COMMON_VOICE_SOURCE_ID",
|
| 69 |
+
"ISSAI_TSC_SOURCE_ID",
|
| 70 |
+
"MEDIASPEECH_SOURCE_ID",
|
| 71 |
+
"CommonVoiceAdapter",
|
| 72 |
+
"IssaiTscAdapter",
|
| 73 |
+
"MediaSpeechAdapter",
|
| 74 |
+
"find_common_voice_root",
|
| 75 |
+
]
|
| 76 |
+
|
| 77 |
+
ISSAI_TSC_SOURCE_ID = "issai_tsc"
|
| 78 |
+
MEDIASPEECH_SOURCE_ID = "mediaspeech_tr"
|
| 79 |
+
COMMON_VOICE_SOURCE_ID = "common_voice_tr"
|
| 80 |
+
|
| 81 |
+
STATE_OK = "ok"
|
| 82 |
+
STATE_NO_AUDIO = "no_audio"
|
| 83 |
+
STATE_NO_TEXT = "no_text"
|
| 84 |
+
STATE_BAD_ENCODING = "bad_encoding"
|
| 85 |
+
STATE_BENCH_RESERVED = "bench_reserved"
|
| 86 |
+
_STATES = (STATE_OK, STATE_NO_AUDIO, STATE_NO_TEXT, STATE_BAD_ENCODING, STATE_BENCH_RESERVED)
|
| 87 |
+
#: FLAC: ``fLaC`` + metadata block header (4 bytes) + STREAMINFO; its bytes 10-17 hold the
|
| 88 |
+
#: sample rate (20 bits), channels, bit depth and the total sample count (36 bits).
|
| 89 |
+
_FLAC_HEAD_BYTES = 26
|
| 90 |
+
_FLAC_STREAMINFO_AT = 18
|
| 91 |
+
|
| 92 |
+
_ISSAI_MEMBER_RE = re.compile(r"([^/]+)/([^/]+)/([0-9]+)\.(wav|txt)")
|
| 93 |
+
_MEDIASPEECH_MEMBER_RE = re.compile(r"([^/]+)/([A-Za-z0-9_-]+)\.(flac|txt)")
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
# ------------------------------------------------------------------------------ rows
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
@dataclass(frozen=True)
|
| 100 |
+
class _ArchiveRow:
|
| 101 |
+
"""One utterance of an archive corpus.
|
| 102 |
+
|
| 103 |
+
Attributes:
|
| 104 |
+
stem: Name of the utterance without extension (``Train/123``, a UUID).
|
| 105 |
+
text: Decoded transcript (``None`` unless ``state`` is ``ok``).
|
| 106 |
+
state: ``ok``, ``no_audio``, ``no_text``, ``bad_encoding`` or ``bench_reserved``.
|
| 107 |
+
"""
|
| 108 |
+
|
| 109 |
+
stem: str
|
| 110 |
+
text: str | None
|
| 111 |
+
state: str
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _pair_rows(
|
| 115 |
+
texts: Mapping[str, bytes], audio_keys: Set[str], audio_ext: str, text_ext: str
|
| 116 |
+
) -> list[_ArchiveRow]:
|
| 117 |
+
"""Pair transcripts with audio members by stem (every stem of either kind is a row)."""
|
| 118 |
+
stems = {k[: -len(audio_ext)] for k in audio_keys} | {k[: -len(text_ext)] for k in texts}
|
| 119 |
+
rows: list[_ArchiveRow] = []
|
| 120 |
+
for stem in stems:
|
| 121 |
+
raw = texts.get(stem + text_ext)
|
| 122 |
+
if stem + audio_ext not in audio_keys:
|
| 123 |
+
rows.append(_ArchiveRow(stem, None, STATE_NO_AUDIO))
|
| 124 |
+
elif raw is None:
|
| 125 |
+
rows.append(_ArchiveRow(stem, None, STATE_NO_TEXT))
|
| 126 |
+
else:
|
| 127 |
+
try:
|
| 128 |
+
rows.append(_ArchiveRow(stem, raw.decode("utf-8"), STATE_OK))
|
| 129 |
+
except UnicodeDecodeError:
|
| 130 |
+
rows.append(_ArchiveRow(stem, None, STATE_BAD_ENCODING))
|
| 131 |
+
return rows
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _order_rows(rows: list[_ArchiveRow]) -> list[_ArchiveRow]:
|
| 135 |
+
"""Deterministic order that does not depend on the order inside the archive: by the sha1
|
| 136 |
+
of the stem, so a prefix of the corpus (``--max-hours``) is a spread over all of it rather
|
| 137 |
+
than its first recordings."""
|
| 138 |
+
return sorted(rows, key=lambda r: (hashlib.sha1(r.stem.encode("utf-8")).hexdigest(), r.stem))
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _bad_state_record(sid: str, item_id: str, row: _ArchiveRow) -> QuarantineRecord | None:
|
| 142 |
+
if row.state == STATE_NO_AUDIO:
|
| 143 |
+
return _drop(sid, item_id, "audio_missing", where="archive")
|
| 144 |
+
if row.state == STATE_NO_TEXT:
|
| 145 |
+
return _drop(sid, item_id, "text_missing")
|
| 146 |
+
if row.state == STATE_BAD_ENCODING:
|
| 147 |
+
return _drop(sid, item_id, "bad_encoding")
|
| 148 |
+
if row.state == STATE_BENCH_RESERVED:
|
| 149 |
+
return _drop(sid, item_id, "bench_reserved")
|
| 150 |
+
return None
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class _ArchiveCorpusAdapter:
|
| 154 |
+
"""Shared machinery of the archive corpora (see the module docstring).
|
| 155 |
+
|
| 156 |
+
Subclasses provide the archive, the classifier, the rows and the item mapping.
|
| 157 |
+
"""
|
| 158 |
+
|
| 159 |
+
ephemeral_raw = True
|
| 160 |
+
source_id = ""
|
| 161 |
+
audio_ext = ""
|
| 162 |
+
text_ext = ""
|
| 163 |
+
|
| 164 |
+
def __init__(self, cfg: ImportConfig, cache: Path) -> None:
|
| 165 |
+
self._cfg = cfg
|
| 166 |
+
self._cache = cache
|
| 167 |
+
self._rows_cache: list[_ArchiveRow] | None = None
|
| 168 |
+
|
| 169 |
+
# -- provided by the subclass
|
| 170 |
+
def _archive(self) -> Path:
|
| 171 |
+
raise NotImplementedError
|
| 172 |
+
|
| 173 |
+
def _classify(self, name: str) -> tuple[str, Kind] | None:
|
| 174 |
+
raise NotImplementedError
|
| 175 |
+
|
| 176 |
+
def _load_rows(self) -> list[_ArchiveRow]:
|
| 177 |
+
raise NotImplementedError
|
| 178 |
+
|
| 179 |
+
def _unit_rows(self) -> int:
|
| 180 |
+
raise NotImplementedError
|
| 181 |
+
|
| 182 |
+
def _expected_pairs(self) -> int:
|
| 183 |
+
raise NotImplementedError
|
| 184 |
+
|
| 185 |
+
def _group(self, row: _ArchiveRow) -> str:
|
| 186 |
+
raise NotImplementedError
|
| 187 |
+
|
| 188 |
+
def _extra(self, row: _ArchiveRow) -> dict[str, Scalar]:
|
| 189 |
+
return {}
|
| 190 |
+
|
| 191 |
+
# -- shared
|
| 192 |
+
@property
|
| 193 |
+
def source_ids(self) -> frozenset[str]:
|
| 194 |
+
return frozenset({self.source_id})
|
| 195 |
+
|
| 196 |
+
def _rows(self) -> list[_ArchiveRow]:
|
| 197 |
+
if self._rows_cache is None:
|
| 198 |
+
rows = self._load_rows()
|
| 199 |
+
if len(rows) != self._expected_pairs():
|
| 200 |
+
raise ArchiveError(
|
| 201 |
+
f"{self._archive()}: {len(rows)} utterances, expected "
|
| 202 |
+
f"{self._expected_pairs()} (configured): a truncated, changed or wrong "
|
| 203 |
+
"archive"
|
| 204 |
+
)
|
| 205 |
+
self._rows_cache = _order_rows(rows)
|
| 206 |
+
return self._rows_cache
|
| 207 |
+
|
| 208 |
+
def _item_id(self, row: _ArchiveRow) -> str:
|
| 209 |
+
return row.stem + self.audio_ext
|
| 210 |
+
|
| 211 |
+
def raw_path(self, item: Item) -> Path:
|
| 212 |
+
return self._cache / "raw" / self.source_id / item.item_id
|
| 213 |
+
|
| 214 |
+
def _units(self) -> tuple[str, ...]:
|
| 215 |
+
return _chunk_ids(len(self._rows()), self._unit_rows())
|
| 216 |
+
|
| 217 |
+
def iter_unit(
|
| 218 |
+
self,
|
| 219 |
+
unit_id: str,
|
| 220 |
+
only: Set[str] | None = None,
|
| 221 |
+
exclude: Mapping[str, str] | None = None,
|
| 222 |
+
) -> Generator[Item | QuarantineRecord, None, None]:
|
| 223 |
+
window = _chunk_slice(unit_id, self._unit_rows())
|
| 224 |
+
first = dict(exclude or {})
|
| 225 |
+
candidates: list[Item] = []
|
| 226 |
+
for row in self._rows()[window]:
|
| 227 |
+
made = self._row(unit_id, row, only, first)
|
| 228 |
+
if isinstance(made, QuarantineRecord):
|
| 229 |
+
yield made
|
| 230 |
+
elif made is not None:
|
| 231 |
+
candidates.append(made)
|
| 232 |
+
# The key of an audio member is its item id (see the classifiers).
|
| 233 |
+
wanted = {c.item_id: self.raw_path(c) for c in candidates if not self.raw_path(c).is_file()}
|
| 234 |
+
yielded = 0
|
| 235 |
+
try:
|
| 236 |
+
if wanted:
|
| 237 |
+
scan_archive(self._archive(), self._classify, read_text=False, extract=wanted)
|
| 238 |
+
for number, item in enumerate(candidates):
|
| 239 |
+
yielded = number + 1
|
| 240 |
+
if self.raw_path(item).is_file():
|
| 241 |
+
yield item
|
| 242 |
+
else:
|
| 243 |
+
yield _drop(item.segment_id, item.item_id, "audio_missing")
|
| 244 |
+
finally:
|
| 245 |
+
# Raw files of rows nobody took (budget reached, or an error): nothing else would
|
| 246 |
+
# delete them.
|
| 247 |
+
for item in candidates[yielded:]:
|
| 248 |
+
self.raw_path(item).unlink(missing_ok=True)
|
| 249 |
+
|
| 250 |
+
def _row(
|
| 251 |
+
self,
|
| 252 |
+
unit_id: str,
|
| 253 |
+
row: _ArchiveRow,
|
| 254 |
+
only: Set[str] | None,
|
| 255 |
+
first: dict[str, str],
|
| 256 |
+
) -> Item | QuarantineRecord | None:
|
| 257 |
+
item_id = self._item_id(row)
|
| 258 |
+
sid = item_segment_id(self.source_id, item_id)
|
| 259 |
+
if only is not None and sid not in only:
|
| 260 |
+
return None
|
| 261 |
+
bad = _bad_state_record(sid, item_id, row)
|
| 262 |
+
if bad is not None:
|
| 263 |
+
return bad
|
| 264 |
+
assert row.text is not None
|
| 265 |
+
try:
|
| 266 |
+
text = normalise_transcript(row.text, self._cfg.char_map)
|
| 267 |
+
except TextRejected as err:
|
| 268 |
+
return _text_drop(sid, item_id, err)
|
| 269 |
+
duplicate = _first_seen(first, sid, item_id, unit_id)
|
| 270 |
+
if duplicate is not None:
|
| 271 |
+
return duplicate
|
| 272 |
+
return Item(
|
| 273 |
+
source_id=self.source_id,
|
| 274 |
+
item_id=item_id,
|
| 275 |
+
segment_id=sid,
|
| 276 |
+
unit_id=unit_id,
|
| 277 |
+
text=text,
|
| 278 |
+
group=nfc(self._group(row)),
|
| 279 |
+
emotion=None,
|
| 280 |
+
extra=self._extra(row),
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
# ------------------------------------------------------------------------------ ISSAI TSC
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
class IssaiTscAdapter(_ArchiveCorpusAdapter):
|
| 288 |
+
"""``issai/Turkish_Speech_Corpus``: ``ISSAI_TSC_218.tar.gz`` with
|
| 289 |
+
``ISSAI_TSC_218/{Train,Dev,Test}/<number>.wav|.txt`` (layout and numbers: the comments of
|
| 290 |
+
``configs/data/import_corpora.yaml``).
|
| 291 |
+
|
| 292 |
+
The archive is downloaded once to ``<cache>/hub/issai_tsc/`` (pinned by the Hub commit,
|
| 293 |
+
which fixes the LFS object), indexed once (the index is kept next to it), and read again
|
| 294 |
+
for every work unit. Rows are ordered by the sha1 of ``<folder>/<number>``; the group is
|
| 295 |
+
``issai_tsc.group_prefix + number // group_block_ids`` because the corpus names no speaker.
|
| 296 |
+
``extra`` keeps the corpus folder (``corpus_split``) and the number.
|
| 297 |
+
|
| 298 |
+
Reasons of its own: none beyond the shared ones.
|
| 299 |
+
"""
|
| 300 |
+
|
| 301 |
+
source_id = ISSAI_TSC_SOURCE_ID
|
| 302 |
+
audio_ext = ".wav"
|
| 303 |
+
text_ext = ".txt"
|
| 304 |
+
|
| 305 |
+
def __init__(
|
| 306 |
+
self, cfg: ImportConfig, cache: Path, hub: HubClient, revision: str | None
|
| 307 |
+
) -> None:
|
| 308 |
+
super().__init__(cfg, cache)
|
| 309 |
+
self._hub = hub
|
| 310 |
+
self.revision = revision
|
| 311 |
+
self._archive_path: Path | None = None
|
| 312 |
+
|
| 313 |
+
@property
|
| 314 |
+
def _c(self) -> IssaiTscConfig:
|
| 315 |
+
return self._cfg.issai_tsc
|
| 316 |
+
|
| 317 |
+
def _revision(self) -> str:
|
| 318 |
+
if self.revision is None:
|
| 319 |
+
raise HubError("issai_tsc: the revision is not pinned; call list_units() first")
|
| 320 |
+
return self.revision
|
| 321 |
+
|
| 322 |
+
def _archive(self) -> Path:
|
| 323 |
+
if self._archive_path is None or not self._archive_path.is_file():
|
| 324 |
+
path = self._hub.download(
|
| 325 |
+
self._c.repo_id,
|
| 326 |
+
self._revision(),
|
| 327 |
+
self._c.archive_file,
|
| 328 |
+
self._cache / "hub" / "issai_tsc",
|
| 329 |
+
)
|
| 330 |
+
verify_sha256_once(path, self._c.archive_sha256) # raises on a partial download
|
| 331 |
+
self._archive_path = path
|
| 332 |
+
return self._archive_path
|
| 333 |
+
|
| 334 |
+
def _classify(self, name: str) -> tuple[str, Kind] | None:
|
| 335 |
+
match = _ISSAI_MEMBER_RE.fullmatch(name)
|
| 336 |
+
if match is None or match.group(1) != self._c.archive_root:
|
| 337 |
+
return None
|
| 338 |
+
_, split, number, ext = match.groups()
|
| 339 |
+
if split not in self._c.splits:
|
| 340 |
+
return None
|
| 341 |
+
return f"{split}/{number}.{ext}", ("audio" if ext == "wav" else "text")
|
| 342 |
+
|
| 343 |
+
def _index_path(self) -> Path:
|
| 344 |
+
"""The index depends on the revision and on how the archive is read (root, folders)."""
|
| 345 |
+
layout = json.dumps(
|
| 346 |
+
[self._c.archive_file, self._c.archive_sha256, self._c.archive_root, self._c.splits]
|
| 347 |
+
)
|
| 348 |
+
key = hashlib.sha1(layout.encode("utf-8")).hexdigest()[:8]
|
| 349 |
+
name = f"index-{self._revision()[:12]}-{key}.jsonl"
|
| 350 |
+
return self._cache / "hub" / "issai_tsc" / name
|
| 351 |
+
|
| 352 |
+
def _load_rows(self) -> list[_ArchiveRow]:
|
| 353 |
+
index = self._index_path()
|
| 354 |
+
if index.is_file():
|
| 355 |
+
return _read_index(index)
|
| 356 |
+
scan = scan_archive(self._archive(), self._classify, read_text=True)
|
| 357 |
+
rows = _pair_rows(scan.texts, scan.audio_keys, self.audio_ext, self.text_ext)
|
| 358 |
+
_write_index(index, rows)
|
| 359 |
+
return rows
|
| 360 |
+
|
| 361 |
+
def _unit_rows(self) -> int:
|
| 362 |
+
return self._c.unit_rows
|
| 363 |
+
|
| 364 |
+
def _expected_pairs(self) -> int:
|
| 365 |
+
return self._c.expected_pairs
|
| 366 |
+
|
| 367 |
+
def _group(self, row: _ArchiveRow) -> str:
|
| 368 |
+
number = int(row.stem.split("/", 1)[1])
|
| 369 |
+
return f"{self._c.group_prefix}{number // self._c.group_block_ids}"
|
| 370 |
+
|
| 371 |
+
def _extra(self, row: _ArchiveRow) -> dict[str, Scalar]:
|
| 372 |
+
split, number = row.stem.split("/", 1)
|
| 373 |
+
return {"corpus_split": split, "utterance_number": int(number)}
|
| 374 |
+
|
| 375 |
+
def list_units(self) -> UnitListing:
|
| 376 |
+
repo = self._c.repo_id
|
| 377 |
+
if self.revision is None:
|
| 378 |
+
self.revision = self._hub.resolve_revision(repo)
|
| 379 |
+
files = self._hub.list_files(repo, self.revision)
|
| 380 |
+
if self._c.archive_file not in files:
|
| 381 |
+
raise HubError(
|
| 382 |
+
f"{repo}@{self.revision}: no file {self._c.archive_file!r}; "
|
| 383 |
+
f"first files: {sorted(files)[:5]}"
|
| 384 |
+
)
|
| 385 |
+
return UnitListing(self.revision, self._units())
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
def _write_index(path: Path, rows: list[_ArchiveRow]) -> None:
|
| 389 |
+
lines = [
|
| 390 |
+
json.dumps({"stem": r.stem, "text": r.text, "state": r.state}, ensure_ascii=False)
|
| 391 |
+
for r in sorted(rows, key=lambda r: r.stem)
|
| 392 |
+
]
|
| 393 |
+
atomic_write_text(path, "\n".join(lines) + "\n")
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def _read_index(path: Path) -> list[_ArchiveRow]:
|
| 397 |
+
rows: list[_ArchiveRow] = []
|
| 398 |
+
for number, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
|
| 399 |
+
if not line:
|
| 400 |
+
continue
|
| 401 |
+
try:
|
| 402 |
+
obj = json.loads(line)
|
| 403 |
+
except json.JSONDecodeError as err:
|
| 404 |
+
raise ValueError(f"{path}:{number}: invalid JSON ({err.msg}); delete the file") from err
|
| 405 |
+
ok = (
|
| 406 |
+
isinstance(obj, dict)
|
| 407 |
+
and set(obj) == {"stem", "text", "state"}
|
| 408 |
+
and isinstance(obj["stem"], str)
|
| 409 |
+
and obj["state"] in _STATES
|
| 410 |
+
and (obj["text"] is None or isinstance(obj["text"], str))
|
| 411 |
+
)
|
| 412 |
+
if not ok:
|
| 413 |
+
raise ValueError(f"{path}:{number}: not an index row; delete the file")
|
| 414 |
+
rows.append(_ArchiveRow(obj["stem"], obj["text"], obj["state"]))
|
| 415 |
+
return rows
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
# ------------------------------------------------------------------------------ MediaSpeech
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
class MediaSpeechAdapter(_ArchiveCorpusAdapter):
|
| 422 |
+
"""OpenSLR SLR108 MediaSpeech Turkish: ``TR.tgz`` with ``TR/<uuid>.flac`` + ``<uuid>.txt``.
|
| 423 |
+
|
| 424 |
+
The archive is downloaded from the primary OpenSLR URL to ``<cache>/hub/mediaspeech_tr/``
|
| 425 |
+
(resumable) and must have the sha256 of the pin: ``revision`` (``--revision``, or the one
|
| 426 |
+
recorded in ``plan.json``) or ``mediaspeech_tr.sha256``. A different file is an error and
|
| 427 |
+
is not used. The corpus is one group (``mediaspeech_tr.group``).
|
| 428 |
+
"""
|
| 429 |
+
|
| 430 |
+
source_id = MEDIASPEECH_SOURCE_ID
|
| 431 |
+
audio_ext = ".flac"
|
| 432 |
+
text_ext = ".txt"
|
| 433 |
+
|
| 434 |
+
def __init__(
|
| 435 |
+
self,
|
| 436 |
+
cfg: ImportConfig,
|
| 437 |
+
cache: Path,
|
| 438 |
+
opener: fetch.Opener | None,
|
| 439 |
+
revision: str | None,
|
| 440 |
+
) -> None:
|
| 441 |
+
super().__init__(cfg, cache)
|
| 442 |
+
self._opener = opener or fetch.default_opener
|
| 443 |
+
self.revision = revision
|
| 444 |
+
self._download: fetch.Downloaded | None = None
|
| 445 |
+
|
| 446 |
+
@property
|
| 447 |
+
def _c(self) -> MediaSpeechConfig:
|
| 448 |
+
return self._cfg.mediaspeech_tr
|
| 449 |
+
|
| 450 |
+
def _archive(self) -> Path:
|
| 451 |
+
if self._download is None or not self._download.path.is_file():
|
| 452 |
+
self._download = fetch.download(
|
| 453 |
+
self._c.url,
|
| 454 |
+
self._cache / "hub" / "mediaspeech_tr" / "TR.tgz",
|
| 455 |
+
expected_sha256=self.revision or self._c.sha256,
|
| 456 |
+
opener=self._opener,
|
| 457 |
+
)
|
| 458 |
+
return self._download.path
|
| 459 |
+
|
| 460 |
+
def _classify(self, name: str) -> tuple[str, Kind] | None:
|
| 461 |
+
match = _MEDIASPEECH_MEMBER_RE.fullmatch(name)
|
| 462 |
+
if match is None or match.group(1) != self._c.archive_root:
|
| 463 |
+
return None
|
| 464 |
+
_, stem, ext = match.groups()
|
| 465 |
+
return f"{stem}.{ext}", ("audio" if ext == "flac" else "text")
|
| 466 |
+
|
| 467 |
+
def _load_rows(self) -> list[_ArchiveRow]:
|
| 468 |
+
head = _FLAC_HEAD_BYTES if self._c.reserve_bench_clips else 0
|
| 469 |
+
scan = scan_archive(self._archive(), self._classify, read_text=True, audio_head=head)
|
| 470 |
+
rows = _pair_rows(scan.texts, scan.audio_keys, self.audio_ext, self.text_ext)
|
| 471 |
+
if not self._c.reserve_bench_clips:
|
| 472 |
+
return rows
|
| 473 |
+
reserved = _bench_reserved_stems(scan)
|
| 474 |
+
return [
|
| 475 |
+
_ArchiveRow(r.stem, None, STATE_BENCH_RESERVED)
|
| 476 |
+
if r.state == STATE_OK and r.stem in reserved
|
| 477 |
+
else r
|
| 478 |
+
for r in rows
|
| 479 |
+
]
|
| 480 |
+
|
| 481 |
+
def _unit_rows(self) -> int:
|
| 482 |
+
return self._c.unit_rows
|
| 483 |
+
|
| 484 |
+
def _expected_pairs(self) -> int:
|
| 485 |
+
return self._c.expected_pairs
|
| 486 |
+
|
| 487 |
+
def _group(self, row: _ArchiveRow) -> str:
|
| 488 |
+
return self._c.group
|
| 489 |
+
|
| 490 |
+
def list_units(self) -> UnitListing:
|
| 491 |
+
units = self._units() # downloads and verifies the archive
|
| 492 |
+
assert self._download is not None
|
| 493 |
+
self.revision = self._download.sha256
|
| 494 |
+
return UnitListing(self.revision, units)
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def flac_duration_ms(head: bytes) -> int:
|
| 498 |
+
"""Duration of a FLAC file from its first 26 bytes (the STREAMINFO block).
|
| 499 |
+
|
| 500 |
+
The rounding is the benchmark's (``round(1000 * frames / sample_rate)``).
|
| 501 |
+
|
| 502 |
+
Raises:
|
| 503 |
+
ArchiveError: not FLAC, no STREAMINFO first, or a zero sample rate.
|
| 504 |
+
"""
|
| 505 |
+
if len(head) < _FLAC_HEAD_BYTES or head[:4] != b"fLaC" or head[4] & 0x7F != 0:
|
| 506 |
+
raise ArchiveError(f"not a FLAC header with STREAMINFO first: {head[:8]!r}")
|
| 507 |
+
packed = int.from_bytes(head[_FLAC_STREAMINFO_AT : _FLAC_STREAMINFO_AT + 8], "big")
|
| 508 |
+
rate_hz, frames = packed >> 44, packed & ((1 << 36) - 1)
|
| 509 |
+
if rate_hz == 0:
|
| 510 |
+
raise ArchiveError("FLAC STREAMINFO with sample rate 0")
|
| 511 |
+
return round(1000 * frames / rate_hz)
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def _bench_reserved_stems(scan: ArchiveScan) -> set[str]:
|
| 515 |
+
"""Stems of the clips the phase-0 B1 benchmark drew from MediaSpeech.
|
| 516 |
+
|
| 517 |
+
The benchmark's selection (:func:`nida.bench.clips.select_mediaspeech` with the default
|
| 518 |
+
:class:`~nida.bench.prepare.B1Plan`) is recomputed from the archive, exactly as
|
| 519 |
+
:mod:`nida.bench.prepare` feeds it: every pair, transcript stripped, FLAC duration.
|
| 520 |
+
"""
|
| 521 |
+
plan = B1Plan()
|
| 522 |
+
items: list[MediaSpeechItem] = []
|
| 523 |
+
for key, raw in scan.texts.items():
|
| 524 |
+
stem = key[: -len(".txt")]
|
| 525 |
+
head = scan.audio_heads.get(stem + ".flac")
|
| 526 |
+
if head is None:
|
| 527 |
+
continue
|
| 528 |
+
try:
|
| 529 |
+
text = raw.decode("utf-8").strip()
|
| 530 |
+
except UnicodeDecodeError:
|
| 531 |
+
continue
|
| 532 |
+
items.append(MediaSpeechItem(stem, text, flac_duration_ms(head)))
|
| 533 |
+
try:
|
| 534 |
+
selection = select_mediaspeech(items, plan.n_mediaspeech, plan.seed)
|
| 535 |
+
except ValueError as err:
|
| 536 |
+
raise ArchiveError(f"cannot reproduce the B1 benchmark selection: {err}") from err
|
| 537 |
+
return {clip.file_id for clip in selection.clips}
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
# ------------------------------------------------------------------------------ Common Voice
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
def find_common_voice_root(source: Path, cfg: CommonVoiceConfig) -> Path:
|
| 544 |
+
"""The folder with ``validated.tsv`` of a Common Voice release directory.
|
| 545 |
+
|
| 546 |
+
Tries ``source`` itself (the language folder), ``source/<locale>`` and
|
| 547 |
+
``source/<release>/<locale>`` (``source`` is a folder of releases).
|
| 548 |
+
|
| 549 |
+
Raises:
|
| 550 |
+
FileNotFoundError: none of them has the validated file.
|
| 551 |
+
"""
|
| 552 |
+
tried = [source, source / cfg.locale]
|
| 553 |
+
if source.is_dir():
|
| 554 |
+
tried += [d / cfg.locale for d in sorted(source.iterdir()) if d.is_dir()]
|
| 555 |
+
for folder in tried:
|
| 556 |
+
if (folder / cfg.validated_file).is_file():
|
| 557 |
+
return folder
|
| 558 |
+
raise FileNotFoundError(
|
| 559 |
+
f"no {cfg.validated_file} in {source} (tried {[str(t) for t in tried[:4]]}...); "
|
| 560 |
+
f"--source-dir is the Common Voice release folder or its {cfg.locale!r} folder, "
|
| 561 |
+
"or the release tarball"
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
@dataclass(frozen=True)
|
| 566 |
+
class _CvRow:
|
| 567 |
+
"""A row of ``validated.tsv`` (only the columns the importer uses).
|
| 568 |
+
|
| 569 |
+
``problem`` is set (and the other fields are empty) for a row that does not match the
|
| 570 |
+
header; ``raw`` is then a description of its shape for the quarantine record (never its
|
| 571 |
+
text: the client id would end up in the quarantine log).
|
| 572 |
+
"""
|
| 573 |
+
|
| 574 |
+
path: str
|
| 575 |
+
sentence: str
|
| 576 |
+
client_id: str
|
| 577 |
+
up_votes: str
|
| 578 |
+
down_votes: str
|
| 579 |
+
locale: str
|
| 580 |
+
problem: str = ""
|
| 581 |
+
raw: str = ""
|
| 582 |
+
|
| 583 |
+
|
| 584 |
+
_CV_REQUIRED = ("client_id", "path", "sentence", "up_votes", "down_votes")
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def _read_cv_tsv(path: Path) -> list[_CvRow]:
|
| 588 |
+
"""Rows of a Common Voice TSV (tab separated, no quoting).
|
| 589 |
+
|
| 590 |
+
Raises:
|
| 591 |
+
ValueError: the header lacks a required column or repeats a column.
|
| 592 |
+
"""
|
| 593 |
+
rows: list[_CvRow] = []
|
| 594 |
+
with path.open(encoding="utf-8", newline="") as handle:
|
| 595 |
+
reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE)
|
| 596 |
+
header = next(reader, None)
|
| 597 |
+
if header is None:
|
| 598 |
+
raise ValueError(f"{path}: empty file (no header)")
|
| 599 |
+
header = [h.strip() for h in header]
|
| 600 |
+
if len(set(header)) != len(header):
|
| 601 |
+
raise ValueError(f"{path}: duplicate columns in the header {header}")
|
| 602 |
+
missing = [c for c in _CV_REQUIRED if c not in header]
|
| 603 |
+
if missing:
|
| 604 |
+
raise ValueError(f"{path}: columns missing: {missing}; header is {header}")
|
| 605 |
+
at = {name: header.index(name) for name in (*_CV_REQUIRED, "locale") if name in header}
|
| 606 |
+
for fields in reader:
|
| 607 |
+
if not fields:
|
| 608 |
+
continue
|
| 609 |
+
if len(fields) != len(header):
|
| 610 |
+
# Only the shape of the row is kept: its text may hold the client id.
|
| 611 |
+
shape = f"{len(fields)} fields, the header has {len(header)}"
|
| 612 |
+
rows.append(_CvRow("", "", "", "", "", "", "columns", shape))
|
| 613 |
+
continue
|
| 614 |
+
rows.append(
|
| 615 |
+
_CvRow(
|
| 616 |
+
path=fields[at["path"]],
|
| 617 |
+
sentence=fields[at["sentence"]],
|
| 618 |
+
client_id=fields[at["client_id"]],
|
| 619 |
+
up_votes=fields[at["up_votes"]],
|
| 620 |
+
down_votes=fields[at["down_votes"]],
|
| 621 |
+
locale=fields[at["locale"]] if "locale" in at else "",
|
| 622 |
+
)
|
| 623 |
+
)
|
| 624 |
+
return rows
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
class CommonVoiceAdapter:
|
| 628 |
+
"""A Common Voice release the user downloaded (``--source-dir``): a folder or a tarball.
|
| 629 |
+
|
| 630 |
+
Rows are the lines of ``validated.tsv``; a clip is used when ``up_votes - down_votes >=
|
| 631 |
+
common_voice_tr.min_net_votes`` and its ``locale`` is empty or ``common_voice_tr.locale``.
|
| 632 |
+
The group is ``group_prefix`` + the first 16 hex digits of sha1(``client_id``): the raw
|
| 633 |
+
client id (a pseudonymous speaker id the MDC terms forbid trying to resolve) never reaches
|
| 634 |
+
a manifest or dataset. Votes are kept in ``extra``; nothing demographic is.
|
| 635 |
+
|
| 636 |
+
From a folder the MP3 clips are read in place and never deleted. From a tarball only
|
| 637 |
+
``validated.tsv`` is extracted up front (to ``<cache>/raw/common_voice_tr/<id>/``); the
|
| 638 |
+
clips of a work unit are extracted per unit and deleted by the audio stage like the other
|
| 639 |
+
archive corpora.
|
| 640 |
+
|
| 641 |
+
Reasons of its own: ``language`` (another non-empty locale), ``low_votes``, ``no_group``,
|
| 642 |
+
``bad_row`` (wrong column count, bad path, non-integer votes).
|
| 643 |
+
"""
|
| 644 |
+
|
| 645 |
+
source_id = COMMON_VOICE_SOURCE_ID
|
| 646 |
+
|
| 647 |
+
def __init__(self, cfg: ImportConfig, cache: Path, source: Path) -> None:
|
| 648 |
+
if not source.exists():
|
| 649 |
+
raise FileNotFoundError(f"{source} does not exist (--source-dir)")
|
| 650 |
+
require_audio_format("MP3")
|
| 651 |
+
self._cfg = cfg
|
| 652 |
+
self._cache = cache
|
| 653 |
+
self._source = source
|
| 654 |
+
self._tarball = source.is_file()
|
| 655 |
+
self.ephemeral_raw = self._tarball
|
| 656 |
+
self._layout_cache: tuple[Path, Path] | None = None
|
| 657 |
+
self._rows_cache: list[_CvRow] | None = None
|
| 658 |
+
|
| 659 |
+
@property
|
| 660 |
+
def _c(self) -> CommonVoiceConfig:
|
| 661 |
+
return self._cfg.common_voice_tr
|
| 662 |
+
|
| 663 |
+
@property
|
| 664 |
+
def source_ids(self) -> frozenset[str]:
|
| 665 |
+
return frozenset({self.source_id})
|
| 666 |
+
|
| 667 |
+
def _classify(self, name: str) -> tuple[str, Kind] | None:
|
| 668 |
+
parts = name.split("/")
|
| 669 |
+
in_locale = len(parts) < 3 or parts[-3] == self._c.locale
|
| 670 |
+
if parts[-1] == self._c.validated_file and (len(parts) == 1 or parts[-2] == self._c.locale):
|
| 671 |
+
return "validated", "text"
|
| 672 |
+
if len(parts) >= 2 and parts[-2] == self._c.clips_dir and in_locale:
|
| 673 |
+
return f"{self._c.clips_dir}/{parts[-1]}", "audio"
|
| 674 |
+
return name, "ignore"
|
| 675 |
+
|
| 676 |
+
def _layout(self) -> tuple[Path, Path]:
|
| 677 |
+
"""``(validated.tsv, clips folder)`` on disk (a tarball is partly extracted)."""
|
| 678 |
+
if self._layout_cache is None:
|
| 679 |
+
if not self._tarball:
|
| 680 |
+
root = find_common_voice_root(self._source, self._c)
|
| 681 |
+
else:
|
| 682 |
+
stat = self._source.stat()
|
| 683 |
+
identity = f"{self._source.name}\x1f{stat.st_size}\x1f{stat.st_mtime_ns}"
|
| 684 |
+
ident = hashlib.sha1(identity.encode()).hexdigest()
|
| 685 |
+
root = self._cache / "raw" / self.source_id / ident[:12]
|
| 686 |
+
tsv = root / self._c.validated_file
|
| 687 |
+
if not tsv.is_file():
|
| 688 |
+
done = scan_archive(
|
| 689 |
+
self._source, self._classify, read_text=False, extract={"validated": tsv}
|
| 690 |
+
)
|
| 691 |
+
if "validated" not in done.extracted:
|
| 692 |
+
raise ArchiveError(
|
| 693 |
+
f"{self._source}: no {self._c.locale}/{self._c.validated_file}"
|
| 694 |
+
)
|
| 695 |
+
self._layout_cache = (root / self._c.validated_file, root / self._c.clips_dir)
|
| 696 |
+
return self._layout_cache
|
| 697 |
+
|
| 698 |
+
def _rows(self) -> list[_CvRow]:
|
| 699 |
+
if self._rows_cache is None:
|
| 700 |
+
self._rows_cache = _read_cv_tsv(self._layout()[0])
|
| 701 |
+
return self._rows_cache
|
| 702 |
+
|
| 703 |
+
def list_units(self) -> UnitListing:
|
| 704 |
+
tsv = self._layout()[0]
|
| 705 |
+
digest = hashlib.sha1(tsv.read_bytes()).hexdigest()
|
| 706 |
+
rows = self._rows()
|
| 707 |
+
if not rows:
|
| 708 |
+
raise ValueError(f"{tsv}: no row")
|
| 709 |
+
return UnitListing(None, _chunk_ids(len(rows), self._c.unit_rows), digest)
|
| 710 |
+
|
| 711 |
+
def raw_path(self, item: Item) -> Path:
|
| 712 |
+
return self._clip(item.item_id)
|
| 713 |
+
|
| 714 |
+
def iter_unit(
|
| 715 |
+
self,
|
| 716 |
+
unit_id: str,
|
| 717 |
+
only: Set[str] | None = None,
|
| 718 |
+
exclude: Mapping[str, str] | None = None,
|
| 719 |
+
) -> Generator[Item | QuarantineRecord, None, None]:
|
| 720 |
+
window = _chunk_slice(unit_id, self._c.unit_rows)
|
| 721 |
+
first = dict(exclude or {})
|
| 722 |
+
candidates: list[Item] = []
|
| 723 |
+
for offset, row in enumerate(self._rows()[window]):
|
| 724 |
+
made = self._row(unit_id, window.start + offset, row, only, first)
|
| 725 |
+
if isinstance(made, QuarantineRecord):
|
| 726 |
+
yield made
|
| 727 |
+
elif made is not None:
|
| 728 |
+
candidates.append(made)
|
| 729 |
+
yielded = 0
|
| 730 |
+
try:
|
| 731 |
+
if self._tarball:
|
| 732 |
+
wanted = {
|
| 733 |
+
f"{self._c.clips_dir}/{c.item_id}": self.raw_path(c)
|
| 734 |
+
for c in candidates
|
| 735 |
+
if not self.raw_path(c).is_file()
|
| 736 |
+
}
|
| 737 |
+
if wanted:
|
| 738 |
+
scan_archive(self._source, self._classify, read_text=False, extract=wanted)
|
| 739 |
+
for number, item in enumerate(candidates):
|
| 740 |
+
yielded = number + 1
|
| 741 |
+
if self.raw_path(item).is_file():
|
| 742 |
+
yield item
|
| 743 |
+
else:
|
| 744 |
+
yield _drop(item.segment_id, item.item_id, "audio_missing")
|
| 745 |
+
finally:
|
| 746 |
+
if self._tarball: # a cache copy; the user's own clips are never touched
|
| 747 |
+
for item in candidates[yielded:]:
|
| 748 |
+
self.raw_path(item).unlink(missing_ok=True)
|
| 749 |
+
|
| 750 |
+
def _row(
|
| 751 |
+
self,
|
| 752 |
+
unit_id: str,
|
| 753 |
+
position: int,
|
| 754 |
+
row: _CvRow,
|
| 755 |
+
only: Set[str] | None,
|
| 756 |
+
first: dict[str, str],
|
| 757 |
+
) -> Item | QuarantineRecord | None:
|
| 758 |
+
cfg = self._cfg
|
| 759 |
+
if row.problem:
|
| 760 |
+
return _bad_row(unit_id, position, row.raw, "bad_row", problem=row.problem)
|
| 761 |
+
rel = safe_relative_path(row.path)
|
| 762 |
+
if rel is None or "\x1f" in rel:
|
| 763 |
+
return _bad_row(unit_id, position, row.path, "bad_row", problem="path")
|
| 764 |
+
sid = item_segment_id(self.source_id, rel)
|
| 765 |
+
if only is not None and sid not in only:
|
| 766 |
+
return None
|
| 767 |
+
locale = row.locale.strip()
|
| 768 |
+
if locale and locale != self._c.locale:
|
| 769 |
+
return _drop(sid, rel, "language", locale=locale)
|
| 770 |
+
try:
|
| 771 |
+
up, down = int(row.up_votes), int(row.down_votes)
|
| 772 |
+
except ValueError:
|
| 773 |
+
return _drop(sid, rel, "bad_row", problem="votes", up_votes=row.up_votes[:20])
|
| 774 |
+
if up - down < self._c.min_net_votes:
|
| 775 |
+
return _drop(sid, rel, "low_votes", up_votes=up, down_votes=down)
|
| 776 |
+
client = nfc(row.client_id).strip()
|
| 777 |
+
if not client:
|
| 778 |
+
return _drop(sid, rel, "no_group")
|
| 779 |
+
try:
|
| 780 |
+
text = normalise_transcript(row.sentence, cfg.char_map)
|
| 781 |
+
except TextRejected as err:
|
| 782 |
+
return _text_drop(sid, rel, err)
|
| 783 |
+
if not self._tarball and not self._clip(rel).is_file():
|
| 784 |
+
return _drop(sid, rel, "audio_missing")
|
| 785 |
+
duplicate = _first_seen(first, sid, rel, unit_id)
|
| 786 |
+
if duplicate is not None:
|
| 787 |
+
return duplicate
|
| 788 |
+
who = hashlib.sha1(f"common-voice-client\x1f{client}".encode()).hexdigest()[:16]
|
| 789 |
+
return Item(
|
| 790 |
+
source_id=self.source_id,
|
| 791 |
+
item_id=rel,
|
| 792 |
+
segment_id=sid,
|
| 793 |
+
unit_id=unit_id,
|
| 794 |
+
text=text,
|
| 795 |
+
group=nfc(self._c.group_prefix + who),
|
| 796 |
+
emotion=None,
|
| 797 |
+
extra={"up_votes": up, "down_votes": down},
|
| 798 |
+
)
|
| 799 |
+
|
| 800 |
+
def _clip(self, item_id: str) -> Path:
|
| 801 |
+
return self._layout()[1] / item_id
|
src/nida/data/importer/hub.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hugging Face Hub access of the import adapters, behind a small injectable protocol.
|
| 2 |
+
|
| 3 |
+
The production implementation uses ``huggingface_hub`` (imported lazily; Colab ships it, it is
|
| 4 |
+
not a declared dependency of Nida). Tests inject a fake that serves local files. Every
|
| 5 |
+
download is pinned to a full commit sha: :meth:`HubClient.resolve_revision` turns the current
|
| 6 |
+
head of a dataset into one, once, and the sha is recorded in ``plan.json``.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import importlib
|
| 12 |
+
import re
|
| 13 |
+
from collections.abc import Sequence
|
| 14 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Any, Protocol
|
| 17 |
+
|
| 18 |
+
__all__ = ["HubClient", "HubError", "HuggingFaceHub", "require_commit_sha", "require_sha256"]
|
| 19 |
+
|
| 20 |
+
_SHA_RE = re.compile(r"[0-9a-f]{40}")
|
| 21 |
+
_SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
| 22 |
+
#: Parallel file downloads of :meth:`HuggingFaceHub.download_many`.
|
| 23 |
+
DOWNLOAD_THREADS = 8
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class HubError(RuntimeError):
|
| 27 |
+
"""The Hub answered with something the importer cannot use."""
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def require_commit_sha(revision: str) -> str:
|
| 31 |
+
"""Return ``revision`` if it is a full 40-character lowercase hex commit sha.
|
| 32 |
+
|
| 33 |
+
Raises:
|
| 34 |
+
ValueError: otherwise (branch and tag names are not pins).
|
| 35 |
+
"""
|
| 36 |
+
if _SHA_RE.fullmatch(revision) is None:
|
| 37 |
+
raise ValueError(f"revision must be a 40-character lowercase hex commit sha: {revision!r}")
|
| 38 |
+
return revision
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def require_sha256(value: str) -> str:
|
| 42 |
+
"""Return ``value`` if it is a 64-character lowercase hex SHA-256 digest (the pin of an
|
| 43 |
+
archive that is not a Hub commit).
|
| 44 |
+
|
| 45 |
+
Raises:
|
| 46 |
+
ValueError: otherwise.
|
| 47 |
+
"""
|
| 48 |
+
if _SHA256_RE.fullmatch(value) is None:
|
| 49 |
+
raise ValueError(f"expected a 64-character lowercase hex sha256 digest: {value!r}")
|
| 50 |
+
return value
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class HubClient(Protocol):
|
| 54 |
+
"""What the adapters need from the Hub (dataset repositories only)."""
|
| 55 |
+
|
| 56 |
+
def resolve_revision(self, repo_id: str) -> str:
|
| 57 |
+
"""Commit sha of the current head of ``repo_id``."""
|
| 58 |
+
...
|
| 59 |
+
|
| 60 |
+
def list_files(self, repo_id: str, revision: str) -> list[str]:
|
| 61 |
+
"""All file paths of the repository at ``revision`` (unsorted is fine)."""
|
| 62 |
+
...
|
| 63 |
+
|
| 64 |
+
def download(self, repo_id: str, revision: str, filename: str, local_dir: Path) -> Path:
|
| 65 |
+
"""Download one file to ``local_dir / filename``; returns that path."""
|
| 66 |
+
...
|
| 67 |
+
|
| 68 |
+
def download_many(
|
| 69 |
+
self, repo_id: str, revision: str, filenames: Sequence[str], local_dir: Path
|
| 70 |
+
) -> None:
|
| 71 |
+
"""Download several files to ``local_dir / <name>``; files that do not exist in the
|
| 72 |
+
repository are skipped silently (the caller checks which ones arrived)."""
|
| 73 |
+
...
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class HuggingFaceHub:
|
| 77 |
+
""":class:`HubClient` over ``huggingface_hub`` (public datasets, no token)."""
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def _module() -> Any: # Any: huggingface_hub is imported lazily and ships no stubs here
|
| 81 |
+
try:
|
| 82 |
+
return importlib.import_module("huggingface_hub")
|
| 83 |
+
except ImportError as err:
|
| 84 |
+
raise HubError(
|
| 85 |
+
"huggingface_hub is required for Hub sources (pip install huggingface_hub)"
|
| 86 |
+
) from err
|
| 87 |
+
|
| 88 |
+
def resolve_revision(self, repo_id: str) -> str:
|
| 89 |
+
info = self._module().HfApi().dataset_info(repo_id)
|
| 90 |
+
sha = getattr(info, "sha", None)
|
| 91 |
+
if not isinstance(sha, str):
|
| 92 |
+
raise HubError(f"{repo_id}: the Hub returned no commit sha")
|
| 93 |
+
return require_commit_sha(sha)
|
| 94 |
+
|
| 95 |
+
def list_files(self, repo_id: str, revision: str) -> list[str]:
|
| 96 |
+
files = (
|
| 97 |
+
self._module()
|
| 98 |
+
.HfApi()
|
| 99 |
+
.list_repo_files(repo_id, repo_type="dataset", revision=require_commit_sha(revision))
|
| 100 |
+
)
|
| 101 |
+
return [str(name) for name in files]
|
| 102 |
+
|
| 103 |
+
def download(self, repo_id: str, revision: str, filename: str, local_dir: Path) -> Path:
|
| 104 |
+
path = self._module().hf_hub_download(
|
| 105 |
+
repo_id=repo_id,
|
| 106 |
+
filename=filename,
|
| 107 |
+
repo_type="dataset",
|
| 108 |
+
revision=require_commit_sha(revision),
|
| 109 |
+
local_dir=str(local_dir),
|
| 110 |
+
)
|
| 111 |
+
return Path(str(path))
|
| 112 |
+
|
| 113 |
+
@classmethod
|
| 114 |
+
def _not_found_error(cls) -> type[BaseException]:
|
| 115 |
+
"""``EntryNotFoundError`` of the installed ``huggingface_hub`` (its module moved)."""
|
| 116 |
+
for name in ("huggingface_hub.errors", "huggingface_hub.utils"):
|
| 117 |
+
try:
|
| 118 |
+
error = getattr(importlib.import_module(name), "EntryNotFoundError") # noqa: B009
|
| 119 |
+
except (ImportError, AttributeError):
|
| 120 |
+
continue
|
| 121 |
+
if isinstance(error, type) and issubclass(error, BaseException):
|
| 122 |
+
return error
|
| 123 |
+
raise HubError("huggingface_hub has no EntryNotFoundError")
|
| 124 |
+
|
| 125 |
+
def download_many(
|
| 126 |
+
self, repo_id: str, revision: str, filenames: Sequence[str], local_dir: Path
|
| 127 |
+
) -> None:
|
| 128 |
+
"""One ``hf_hub_download`` per file on a small thread pool; a file that is not in
|
| 129 |
+
the repository is skipped, any other error is raised."""
|
| 130 |
+
if not filenames:
|
| 131 |
+
return
|
| 132 |
+
module = self._module()
|
| 133 |
+
pinned = require_commit_sha(revision)
|
| 134 |
+
missing = self._not_found_error()
|
| 135 |
+
|
| 136 |
+
def fetch_one(name: str) -> None:
|
| 137 |
+
try:
|
| 138 |
+
module.hf_hub_download(
|
| 139 |
+
repo_id=repo_id,
|
| 140 |
+
filename=name,
|
| 141 |
+
repo_type="dataset",
|
| 142 |
+
revision=pinned,
|
| 143 |
+
local_dir=str(local_dir),
|
| 144 |
+
)
|
| 145 |
+
except missing:
|
| 146 |
+
return
|
| 147 |
+
|
| 148 |
+
with ThreadPoolExecutor(max_workers=DOWNLOAD_THREADS) as pool:
|
| 149 |
+
for _ in pool.map(fetch_one, filenames):
|
| 150 |
+
pass
|
src/nida/data/importer/pipeline.py
ADDED
|
@@ -0,0 +1,1326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The importer: stages, plan and resumability (layout: see :mod:`nida.data.importer`)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import io
|
| 7 |
+
import json
|
| 8 |
+
import logging
|
| 9 |
+
import math
|
| 10 |
+
from collections import Counter
|
| 11 |
+
from collections.abc import Callable, Mapping, Sequence
|
| 12 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 13 |
+
from dataclasses import dataclass, field
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from typing import Any
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
from nida.bench import envs as bench_envs
|
| 20 |
+
from nida.bench import fetch
|
| 21 |
+
from nida.bench.b1 import _stream_runner
|
| 22 |
+
from nida.bench.protocol import WorkRequest, WorkResult, load_results
|
| 23 |
+
from nida.bench.registry import load_sources, require_approved
|
| 24 |
+
from nida.data import envs
|
| 25 |
+
from nida.data.align import align_words
|
| 26 |
+
from nida.data.asr_filter import check_alignment
|
| 27 |
+
from nida.data.ears import bucket_of, rate_chars_per_s
|
| 28 |
+
from nida.data.importer.audio import (
|
| 29 |
+
AudioRejected,
|
| 30 |
+
convert_to_wav16k,
|
| 31 |
+
measure_wav16k,
|
| 32 |
+
raw_duration_ms,
|
| 33 |
+
)
|
| 34 |
+
from nida.data.importer.config import ImportConfig, load_import_config
|
| 35 |
+
from nida.data.importer.hub import (
|
| 36 |
+
HubClient,
|
| 37 |
+
HuggingFaceHub,
|
| 38 |
+
require_commit_sha,
|
| 39 |
+
require_sha256,
|
| 40 |
+
)
|
| 41 |
+
from nida.data.importer.sources import (
|
| 42 |
+
HUB_SOURCES,
|
| 43 |
+
LOCAL_SOURCES,
|
| 44 |
+
SOURCES,
|
| 45 |
+
Item,
|
| 46 |
+
SourceAdapter,
|
| 47 |
+
make_adapter,
|
| 48 |
+
)
|
| 49 |
+
from nida.data.pilot import DATASET_SHARD_FRAMES, FLOAT16_MAX_ABS, run_worker_job
|
| 50 |
+
from nida.data.quarantine import QuarantineLog
|
| 51 |
+
from nida.data.registry import DATA_MODELS
|
| 52 |
+
from nida.data.splits import assign_splits
|
| 53 |
+
from nida.data.timings import SegmentTimings, build_timings, write_timings
|
| 54 |
+
from nida.data.types import QuarantineRecord
|
| 55 |
+
from nida.model.vocab import encode_text
|
| 56 |
+
from nida.train.data import SPLITS, DataFormatError, load_latent_dataset
|
| 57 |
+
from nida.utils.io import atomic_write_bytes, atomic_write_json, atomic_write_text
|
| 58 |
+
|
| 59 |
+
__all__ = ["STAGES", "ImportPlan", "ImportRequest", "Importer", "assign_group_splits"]
|
| 60 |
+
|
| 61 |
+
log = logging.getLogger(__name__)
|
| 62 |
+
|
| 63 |
+
#: Stages in run order; ``all`` runs them one after another.
|
| 64 |
+
STAGES = ("select", "audio", "latents", "ctc", "timings", "dataset", "report")
|
| 65 |
+
VAE_KEY = "voxcpm2_audiovae_v2"
|
| 66 |
+
CTC_KEY = "mpoyraz_ctc_tr"
|
| 67 |
+
#: Language written to every index record (the importer reads Turkish corpora only).
|
| 68 |
+
LANGUAGE = "tr"
|
| 69 |
+
#: Items between two progress lines.
|
| 70 |
+
PROGRESS_EVERY = 1000
|
| 71 |
+
#: Default number of audio conversion threads (decoding and resampling release the GIL).
|
| 72 |
+
DEFAULT_AUDIO_THREADS = 4
|
| 73 |
+
SECONDS_PER_HOUR = 3600.0
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@dataclass(frozen=True)
|
| 77 |
+
class ImportRequest:
|
| 78 |
+
"""What the caller asks for; part of the plan of a root.
|
| 79 |
+
|
| 80 |
+
Attributes:
|
| 81 |
+
source: One of :data:`nida.data.importer.sources.SOURCES` (``alania``, ``tr_full``,
|
| 82 |
+
``wavdir``, ``issai_tsc``, ``mediaspeech_tr``, ``common_voice_tr``).
|
| 83 |
+
max_hours: Stop selecting once this many hours of audio are selected.
|
| 84 |
+
unit_offset: Index of the first work unit (parquet shard / row chunk) of this root.
|
| 85 |
+
unit_count: Number of work units from ``unit_offset`` (``None`` = all the rest).
|
| 86 |
+
Roots with disjoint ranges select disjoint data.
|
| 87 |
+
revision: Pin (``None`` = resolve it once and record it): a Hub commit sha, or for
|
| 88 |
+
``mediaspeech_tr`` the sha256 of the ``TR.tgz`` archive (default: the one in
|
| 89 |
+
``import_corpora.yaml``). Refused for the local sources (``wavdir``,
|
| 90 |
+
``common_voice_tr``), which pin the sha1 of their manifest / validated file.
|
| 91 |
+
"""
|
| 92 |
+
|
| 93 |
+
source: str
|
| 94 |
+
max_hours: float = 10.0
|
| 95 |
+
unit_offset: int = 0
|
| 96 |
+
unit_count: int | None = None
|
| 97 |
+
revision: str | None = None
|
| 98 |
+
|
| 99 |
+
def __post_init__(self) -> None:
|
| 100 |
+
if self.source not in SOURCES:
|
| 101 |
+
raise ValueError(f"unknown source {self.source!r}; known: {SOURCES}")
|
| 102 |
+
if not (math.isfinite(self.max_hours) and self.max_hours > 0):
|
| 103 |
+
raise ValueError(f"max_hours must be a positive finite number, got {self.max_hours!r}")
|
| 104 |
+
if isinstance(self.unit_offset, bool) or self.unit_offset < 0:
|
| 105 |
+
raise ValueError(f"unit_offset must be an integer >= 0, got {self.unit_offset!r}")
|
| 106 |
+
if self.unit_count is not None and (
|
| 107 |
+
isinstance(self.unit_count, bool) or self.unit_count < 1
|
| 108 |
+
):
|
| 109 |
+
raise ValueError(f"unit_count must be an integer >= 1, got {self.unit_count!r}")
|
| 110 |
+
if self.revision is not None and self.source in LOCAL_SOURCES:
|
| 111 |
+
raise ValueError(
|
| 112 |
+
f"--revision is meaningless for source {self.source!r} (a local folder has no "
|
| 113 |
+
"revision; the plan pins the sha1 of its manifest / validated file)"
|
| 114 |
+
)
|
| 115 |
+
if self.revision is not None:
|
| 116 |
+
if self.source == "mediaspeech_tr":
|
| 117 |
+
require_sha256(self.revision)
|
| 118 |
+
else:
|
| 119 |
+
require_commit_sha(self.revision)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@dataclass(frozen=True)
|
| 123 |
+
class ImportPlan:
|
| 124 |
+
"""The resolved request of a root (``plan.json``): fixes which units are processed.
|
| 125 |
+
|
| 126 |
+
A root is bound to its plan; asking for something else raises (use a new root).
|
| 127 |
+
"""
|
| 128 |
+
|
| 129 |
+
source: str
|
| 130 |
+
revision: str | None
|
| 131 |
+
max_hours: float
|
| 132 |
+
unit_offset: int
|
| 133 |
+
unit_count: int | None
|
| 134 |
+
unit_ids: tuple[str, ...]
|
| 135 |
+
config_digest: str
|
| 136 |
+
content_sha1: str | None
|
| 137 |
+
|
| 138 |
+
def to_obj(self) -> dict[str, object]:
|
| 139 |
+
return {
|
| 140 |
+
"source": self.source,
|
| 141 |
+
"revision": self.revision,
|
| 142 |
+
"max_hours": self.max_hours,
|
| 143 |
+
"unit_offset": self.unit_offset,
|
| 144 |
+
"unit_count": self.unit_count,
|
| 145 |
+
"unit_ids": list(self.unit_ids),
|
| 146 |
+
"config_digest": self.config_digest,
|
| 147 |
+
"content_sha1": self.content_sha1,
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
@classmethod
|
| 151 |
+
def from_obj(cls, obj: object, where: Path) -> ImportPlan:
|
| 152 |
+
keys = {
|
| 153 |
+
"source",
|
| 154 |
+
"revision",
|
| 155 |
+
"max_hours",
|
| 156 |
+
"unit_offset",
|
| 157 |
+
"unit_count",
|
| 158 |
+
"unit_ids",
|
| 159 |
+
"config_digest",
|
| 160 |
+
"content_sha1",
|
| 161 |
+
}
|
| 162 |
+
if not isinstance(obj, dict) or set(obj) != keys or not isinstance(obj["unit_ids"], list):
|
| 163 |
+
raise ValueError(f"{where}: not a plan.json written by the importer")
|
| 164 |
+
return cls(
|
| 165 |
+
source=str(obj["source"]),
|
| 166 |
+
revision=None if obj["revision"] is None else str(obj["revision"]),
|
| 167 |
+
max_hours=float(obj["max_hours"]),
|
| 168 |
+
unit_offset=int(obj["unit_offset"]),
|
| 169 |
+
unit_count=None if obj["unit_count"] is None else int(obj["unit_count"]),
|
| 170 |
+
unit_ids=tuple(str(u) for u in obj["unit_ids"]),
|
| 171 |
+
config_digest=str(obj["config_digest"]),
|
| 172 |
+
content_sha1=None if obj["content_sha1"] is None else str(obj["content_sha1"]),
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
def check(self, request: ImportRequest, config_digest: str, content_sha1: str | None) -> None:
|
| 176 |
+
"""Raise unless this plan is what ``request`` (and the current config/input) mean."""
|
| 177 |
+
wanted: dict[str, object] = {
|
| 178 |
+
"source": request.source,
|
| 179 |
+
"max_hours": request.max_hours,
|
| 180 |
+
"unit_offset": request.unit_offset,
|
| 181 |
+
"unit_count": request.unit_count,
|
| 182 |
+
"config_digest": config_digest,
|
| 183 |
+
"content_sha1": content_sha1,
|
| 184 |
+
}
|
| 185 |
+
actual = self.to_obj()
|
| 186 |
+
if request.revision is not None:
|
| 187 |
+
wanted["revision"] = request.revision
|
| 188 |
+
differing = {k: (actual[k], v) for k, v in wanted.items() if actual[k] != v}
|
| 189 |
+
if differing:
|
| 190 |
+
raise ValueError(
|
| 191 |
+
"this root was made for another plan (recorded, requested): "
|
| 192 |
+
f"{differing}; use a new --root"
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@dataclass(frozen=True)
|
| 197 |
+
class _UnitResult:
|
| 198 |
+
"""Outcome of one work unit, stored as ``select/units/<digest>.json``."""
|
| 199 |
+
|
| 200 |
+
unit_id: str
|
| 201 |
+
truncated: bool
|
| 202 |
+
items: tuple[Item, ...]
|
| 203 |
+
quarantine: tuple[QuarantineRecord, ...]
|
| 204 |
+
|
| 205 |
+
def to_obj(self) -> dict[str, object]:
|
| 206 |
+
return {
|
| 207 |
+
"unit_id": self.unit_id,
|
| 208 |
+
"truncated": self.truncated,
|
| 209 |
+
"items": [item.to_obj() for item in self.items],
|
| 210 |
+
"quarantine": [
|
| 211 |
+
{"item_id": q.item_id, "stage": q.stage, "reason": q.reason, "details": q.details}
|
| 212 |
+
for q in self.quarantine
|
| 213 |
+
],
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
@classmethod
|
| 217 |
+
def load(cls, path: Path) -> _UnitResult:
|
| 218 |
+
obj = json.loads(path.read_text(encoding="utf-8"))
|
| 219 |
+
if not isinstance(obj, dict) or set(obj) != {"unit_id", "truncated", "items", "quarantine"}:
|
| 220 |
+
raise ValueError(f"{path}: not a unit result")
|
| 221 |
+
return cls(
|
| 222 |
+
unit_id=str(obj["unit_id"]),
|
| 223 |
+
truncated=bool(obj["truncated"]),
|
| 224 |
+
items=tuple(Item.from_obj(i, f"{path}: item") for i in obj["items"]),
|
| 225 |
+
quarantine=tuple(
|
| 226 |
+
QuarantineRecord(q["item_id"], q["stage"], q["reason"], q["details"])
|
| 227 |
+
for q in obj["quarantine"]
|
| 228 |
+
),
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
@dataclass(frozen=True)
|
| 233 |
+
class _Seg:
|
| 234 |
+
"""A segment on its way through the stages.
|
| 235 |
+
|
| 236 |
+
``duration_ms`` / ``loudness_dbfs`` come from the audio stage; ``num_frames`` and
|
| 237 |
+
``frame_hop_ms`` from the latents stage (``0`` before it ran).
|
| 238 |
+
"""
|
| 239 |
+
|
| 240 |
+
item: Item
|
| 241 |
+
duration_ms: int
|
| 242 |
+
loudness_dbfs: float
|
| 243 |
+
num_frames: int = 0
|
| 244 |
+
frame_hop_ms: float = 0.0
|
| 245 |
+
latent_dim: int = 0
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def sid(self) -> str:
|
| 249 |
+
return self.item.segment_id
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
@dataclass(frozen=True)
|
| 253 |
+
class _TimelineOutcome:
|
| 254 |
+
"""A built timeline (with its mean word alignment score) or the quarantine record."""
|
| 255 |
+
|
| 256 |
+
timings: SegmentTimings | None
|
| 257 |
+
quarantine: QuarantineRecord | None
|
| 258 |
+
clamped_units: int = 0
|
| 259 |
+
mean_align_score: float = 0.0
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def _digest(text: str) -> str:
|
| 263 |
+
return hashlib.sha1(text.encode("utf-8")).hexdigest()[:16]
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _file_sha1(path: Path) -> str:
|
| 267 |
+
return hashlib.sha1(path.read_bytes()).hexdigest()
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def _write_jsonl(path: Path, rows: Sequence[Mapping[str, object]]) -> None:
|
| 271 |
+
text = "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows)
|
| 272 |
+
atomic_write_text(path, text)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def _read_jsonl(path: Path) -> list[dict[str, object]]:
|
| 276 |
+
rows: list[dict[str, object]] = []
|
| 277 |
+
for number, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
|
| 278 |
+
if not line:
|
| 279 |
+
continue
|
| 280 |
+
try:
|
| 281 |
+
obj = json.loads(line)
|
| 282 |
+
except json.JSONDecodeError as err:
|
| 283 |
+
raise ValueError(f"{path}:{number}: invalid JSON ({err.msg})") from err
|
| 284 |
+
if not isinstance(obj, dict):
|
| 285 |
+
raise ValueError(f"{path}:{number}: expected a JSON object")
|
| 286 |
+
rows.append(obj)
|
| 287 |
+
return rows
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def _hours(duration_ms: int) -> float:
|
| 291 |
+
return round(duration_ms / 1000 / SECONDS_PER_HOUR, 3)
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
@dataclass
|
| 295 |
+
class Importer:
|
| 296 |
+
"""Imports one transcribed external corpus into a latent dataset (module docstring of
|
| 297 |
+
:mod:`nida.data.importer`).
|
| 298 |
+
|
| 299 |
+
Attributes:
|
| 300 |
+
root: Durable directory (Drive): plan, manifests, quarantine, worker results, dataset.
|
| 301 |
+
cache: Local scratch: raw audio, 16 kHz wavs and ``.npy`` arrays (rebuilt when lost).
|
| 302 |
+
envs_root: Where the worker virtual environments live.
|
| 303 |
+
repo_dir: Repository root (environments are built from it).
|
| 304 |
+
request: What to import.
|
| 305 |
+
config_dir: ``configs/data`` (default ``repo_dir/configs/data``).
|
| 306 |
+
sources_path: ``configs/sources.yaml`` (default ``repo_dir/configs/sources.yaml``).
|
| 307 |
+
wavdir: ``--source-dir``: the folder of a ``wavdir`` source, or the Common Voice release
|
| 308 |
+
folder / tarball of ``common_voice_tr``.
|
| 309 |
+
hub: Hub client (default: ``huggingface_hub``).
|
| 310 |
+
retry_failed: Pass ``--retry-failed`` to the workers.
|
| 311 |
+
runner: Runs a worker command line and returns its exit code.
|
| 312 |
+
opener: HTTP access of the ``mediaspeech_tr`` download (default: ``urllib``).
|
| 313 |
+
uv: Path of the ``uv`` executable (default: found/installed on demand).
|
| 314 |
+
audio_threads: Threads converting raw audio to 16 kHz wavs (operational: does not
|
| 315 |
+
change the outputs).
|
| 316 |
+
"""
|
| 317 |
+
|
| 318 |
+
root: Path
|
| 319 |
+
cache: Path
|
| 320 |
+
envs_root: Path
|
| 321 |
+
repo_dir: Path
|
| 322 |
+
request: ImportRequest
|
| 323 |
+
config_dir: Path | None = None
|
| 324 |
+
sources_path: Path | None = None
|
| 325 |
+
wavdir: Path | None = None
|
| 326 |
+
hub: HubClient | None = None
|
| 327 |
+
retry_failed: bool = False
|
| 328 |
+
runner: Callable[[Sequence[str]], int] = _stream_runner
|
| 329 |
+
opener: fetch.Opener | None = None
|
| 330 |
+
uv: str | None = None
|
| 331 |
+
audio_threads: int = DEFAULT_AUDIO_THREADS
|
| 332 |
+
_ready: bool = field(default=False, init=False, repr=False)
|
| 333 |
+
_cfg: ImportConfig | None = field(default=None, init=False, repr=False)
|
| 334 |
+
_plan: ImportPlan | None = field(default=None, init=False, repr=False)
|
| 335 |
+
_adapter: SourceAdapter | None = field(default=None, init=False, repr=False)
|
| 336 |
+
_pythons: dict[str, Path] = field(default_factory=dict, init=False, repr=False)
|
| 337 |
+
|
| 338 |
+
def __post_init__(self) -> None:
|
| 339 |
+
if isinstance(self.audio_threads, bool) or self.audio_threads < 1:
|
| 340 |
+
raise ValueError(f"audio_threads must be an integer >= 1, got {self.audio_threads!r}")
|
| 341 |
+
|
| 342 |
+
# ------------------------------------------------------------------ setup
|
| 343 |
+
|
| 344 |
+
@property
|
| 345 |
+
def cfg(self) -> ImportConfig:
|
| 346 |
+
if self._cfg is None:
|
| 347 |
+
raise RuntimeError("Importer is not prepared; call run()")
|
| 348 |
+
return self._cfg
|
| 349 |
+
|
| 350 |
+
@property
|
| 351 |
+
def plan(self) -> ImportPlan:
|
| 352 |
+
if self._plan is None:
|
| 353 |
+
raise RuntimeError("Importer is not prepared; call run()")
|
| 354 |
+
return self._plan
|
| 355 |
+
|
| 356 |
+
@property
|
| 357 |
+
def adapter(self) -> SourceAdapter:
|
| 358 |
+
if self._adapter is None:
|
| 359 |
+
raise RuntimeError("Importer is not prepared; call run()")
|
| 360 |
+
return self._adapter
|
| 361 |
+
|
| 362 |
+
def dataset_dir(self) -> Path:
|
| 363 |
+
"""``<root>/dataset-<config digest>/``."""
|
| 364 |
+
return self.root / f"dataset-{self.cfg.digest}"
|
| 365 |
+
|
| 366 |
+
def _prepare(self) -> None:
|
| 367 |
+
"""Load the config, check the licence gate, load or create the plan (once per run)."""
|
| 368 |
+
if self._ready:
|
| 369 |
+
return
|
| 370 |
+
config_dir = self.config_dir or self.repo_dir / "configs" / "data"
|
| 371 |
+
sources_path = self.sources_path or self.repo_dir / "configs" / "sources.yaml"
|
| 372 |
+
cfg = load_import_config(config_dir, self.request.source)
|
| 373 |
+
plan_path = self.root / "plan.json"
|
| 374 |
+
existing: ImportPlan | None = None
|
| 375 |
+
if plan_path.is_file():
|
| 376 |
+
existing = ImportPlan.from_obj(json.loads(plan_path.read_text("utf-8")), plan_path)
|
| 377 |
+
hub = self.hub
|
| 378 |
+
if hub is None and self.request.source in HUB_SOURCES:
|
| 379 |
+
hub = HuggingFaceHub()
|
| 380 |
+
adapter = make_adapter(
|
| 381 |
+
self.request.source,
|
| 382 |
+
cfg,
|
| 383 |
+
self.cache,
|
| 384 |
+
hub=hub,
|
| 385 |
+
revision=existing.revision if existing else self.request.revision,
|
| 386 |
+
wavdir=self.wavdir,
|
| 387 |
+
opener=self.opener,
|
| 388 |
+
)
|
| 389 |
+
needed = set(adapter.source_ids) | {DATA_MODELS[k].source_id for k in (VAE_KEY, CTC_KEY)}
|
| 390 |
+
require_approved(needed, load_sources(sources_path))
|
| 391 |
+
if existing is not None:
|
| 392 |
+
local = self.request.source in LOCAL_SOURCES
|
| 393 |
+
content = adapter.list_units().content_sha1 if local else None
|
| 394 |
+
existing.check(self.request, cfg.digest, content)
|
| 395 |
+
plan = existing
|
| 396 |
+
else:
|
| 397 |
+
listing = adapter.list_units()
|
| 398 |
+
stop = (
|
| 399 |
+
None
|
| 400 |
+
if self.request.unit_count is None
|
| 401 |
+
else (self.request.unit_offset + self.request.unit_count)
|
| 402 |
+
)
|
| 403 |
+
unit_ids = listing.unit_ids[self.request.unit_offset : stop]
|
| 404 |
+
if not unit_ids:
|
| 405 |
+
raise ValueError(
|
| 406 |
+
f"no work unit in range: offset {self.request.unit_offset}, count "
|
| 407 |
+
f"{self.request.unit_count}, {len(listing.unit_ids)} units exist"
|
| 408 |
+
)
|
| 409 |
+
plan = ImportPlan(
|
| 410 |
+
source=self.request.source,
|
| 411 |
+
revision=listing.revision,
|
| 412 |
+
max_hours=self.request.max_hours,
|
| 413 |
+
unit_offset=self.request.unit_offset,
|
| 414 |
+
unit_count=self.request.unit_count,
|
| 415 |
+
unit_ids=unit_ids,
|
| 416 |
+
config_digest=cfg.digest,
|
| 417 |
+
content_sha1=listing.content_sha1,
|
| 418 |
+
)
|
| 419 |
+
atomic_write_json(plan_path, plan.to_obj())
|
| 420 |
+
log.info(
|
| 421 |
+
"plan: %s revision %s, %d of %d units",
|
| 422 |
+
plan.source,
|
| 423 |
+
plan.revision,
|
| 424 |
+
len(unit_ids),
|
| 425 |
+
len(listing.unit_ids),
|
| 426 |
+
)
|
| 427 |
+
self._cfg, self._plan, self._adapter, self._ready = cfg, plan, adapter, True
|
| 428 |
+
|
| 429 |
+
# -------------------------------------------------------------- stage state
|
| 430 |
+
|
| 431 |
+
def _marker(self, stage: str) -> Path:
|
| 432 |
+
return self.root / stage / "stage.json"
|
| 433 |
+
|
| 434 |
+
def _reuse(self, stage: str, input_sha1: str) -> dict[str, object] | None:
|
| 435 |
+
"""The recorded result of a finished stage (``reused: True``), else ``None``."""
|
| 436 |
+
marker = self._marker(stage)
|
| 437 |
+
if not marker.is_file():
|
| 438 |
+
return None
|
| 439 |
+
recorded = json.loads(marker.read_text(encoding="utf-8"))
|
| 440 |
+
if recorded.get("input_sha1") != input_sha1:
|
| 441 |
+
raise RuntimeError(
|
| 442 |
+
f"{stage}: its input changed after it ran; move {marker.parent} away to recompute"
|
| 443 |
+
)
|
| 444 |
+
result: dict[str, object] = dict(recorded["result"])
|
| 445 |
+
result["reused"] = True
|
| 446 |
+
return result
|
| 447 |
+
|
| 448 |
+
def _finish(self, stage: str, input_sha1: str, result: dict[str, object]) -> dict[str, object]:
|
| 449 |
+
atomic_write_json(
|
| 450 |
+
self._marker(stage), {"stage": stage, "input_sha1": input_sha1, "result": result}
|
| 451 |
+
)
|
| 452 |
+
return {**result, "reused": False}
|
| 453 |
+
|
| 454 |
+
def _require_done(self, stage: str) -> None:
|
| 455 |
+
if not self._marker(stage).is_file():
|
| 456 |
+
raise RuntimeError(f"stage {stage!r} has not finished; run it first")
|
| 457 |
+
|
| 458 |
+
def _write_quarantine(self, stage: str, records: Sequence[QuarantineRecord]) -> dict[str, int]:
|
| 459 |
+
"""Replace the stage's quarantine log by ``records`` (a stage rerun rebuilds it)."""
|
| 460 |
+
qlog = QuarantineLog(self.root / stage)
|
| 461 |
+
qlog.path.unlink(missing_ok=True)
|
| 462 |
+
qlog.append(records)
|
| 463 |
+
return qlog.counts_by_reason()
|
| 464 |
+
|
| 465 |
+
def _drop(self, stage: str, item: Item, reason: str, **details: object) -> QuarantineRecord:
|
| 466 |
+
return QuarantineRecord(
|
| 467 |
+
item.segment_id, stage, reason, {"item_id": item.item_id, **details}
|
| 468 |
+
)
|
| 469 |
+
|
| 470 |
+
# --------------------------------------------------------------------- run
|
| 471 |
+
|
| 472 |
+
def run(self, stage: str) -> dict[str, object]:
|
| 473 |
+
"""Run one stage (or ``all``); returns the stage result(s) as plain JSON data.
|
| 474 |
+
|
| 475 |
+
Raises:
|
| 476 |
+
ValueError: unknown stage, or a root made for another plan/config.
|
| 477 |
+
nida.bench.registry.SourceNotApprovedError: a source or model is not approved.
|
| 478 |
+
"""
|
| 479 |
+
if stage == "all":
|
| 480 |
+
return {name: self.run(name) for name in STAGES}
|
| 481 |
+
if stage not in STAGES:
|
| 482 |
+
raise ValueError(f"unknown import stage {stage!r}; known: {', '.join(STAGES)}")
|
| 483 |
+
self._prepare()
|
| 484 |
+
log.info("=== import stage %s", stage)
|
| 485 |
+
handler: Callable[[], dict[str, object]] = getattr(self, "_stage_" + stage)
|
| 486 |
+
return handler()
|
| 487 |
+
|
| 488 |
+
# ------------------------------------------------------------------ select
|
| 489 |
+
|
| 490 |
+
def _stage_select(self) -> dict[str, object]:
|
| 491 |
+
"""Walk the planned units in order, stop at ``max_hours``; write the item manifest.
|
| 492 |
+
|
| 493 |
+
Each finished unit is stored (``select/units/<digest>.json``) so an interrupted run
|
| 494 |
+
continues with the next unit. The last unit is cut at the first item that would exceed
|
| 495 |
+
the budget (``truncated``). Duplicate segment ids across units are quarantined.
|
| 496 |
+
"""
|
| 497 |
+
cfg, plan = self.cfg, self.plan
|
| 498 |
+
plan_sha = _digest(json.dumps(plan.to_obj(), sort_keys=True))
|
| 499 |
+
reused = self._reuse("select", plan_sha)
|
| 500 |
+
if reused is not None:
|
| 501 |
+
return reused
|
| 502 |
+
units_dir = self.root / "select" / "units"
|
| 503 |
+
budget_ms = round(plan.max_hours * 3_600_000)
|
| 504 |
+
total_ms = 0
|
| 505 |
+
done: list[_UnitResult] = []
|
| 506 |
+
selected_in: dict[str, str] = {} # segment id -> unit that selected it
|
| 507 |
+
stopped = False
|
| 508 |
+
for unit_id in plan.unit_ids:
|
| 509 |
+
path = units_dir / f"{_digest(unit_id)}.json"
|
| 510 |
+
if path.is_file():
|
| 511 |
+
result = _UnitResult.load(path)
|
| 512 |
+
if result.unit_id != unit_id:
|
| 513 |
+
raise RuntimeError(
|
| 514 |
+
f"{path} belongs to unit {result.unit_id!r}, not {unit_id!r}"
|
| 515 |
+
)
|
| 516 |
+
else:
|
| 517 |
+
if budget_ms - total_ms < cfg.min_duration_ms:
|
| 518 |
+
stopped = True
|
| 519 |
+
break
|
| 520 |
+
result = self._select_unit(unit_id, budget_ms - total_ms, selected_in)
|
| 521 |
+
atomic_write_json(path, result.to_obj())
|
| 522 |
+
log.info(
|
| 523 |
+
"select: unit %s -> %d items, %d quarantined",
|
| 524 |
+
unit_id,
|
| 525 |
+
len(result.items),
|
| 526 |
+
len(result.quarantine),
|
| 527 |
+
)
|
| 528 |
+
done.append(result)
|
| 529 |
+
for item in result.items:
|
| 530 |
+
selected_in.setdefault(item.segment_id, result.unit_id)
|
| 531 |
+
total_ms += sum(item.raw_duration_ms or 0 for item in result.items)
|
| 532 |
+
if result.truncated:
|
| 533 |
+
stopped = True
|
| 534 |
+
break
|
| 535 |
+
items: list[Item] = []
|
| 536 |
+
records: list[QuarantineRecord] = []
|
| 537 |
+
first_unit: dict[str, str] = {}
|
| 538 |
+
for result in done:
|
| 539 |
+
records.extend(result.quarantine)
|
| 540 |
+
for item in result.items:
|
| 541 |
+
if item.segment_id in first_unit:
|
| 542 |
+
records.append(
|
| 543 |
+
self._drop(
|
| 544 |
+
"select",
|
| 545 |
+
item,
|
| 546 |
+
"duplicate_item",
|
| 547 |
+
unit_id=item.unit_id,
|
| 548 |
+
first_unit_id=first_unit[item.segment_id],
|
| 549 |
+
)
|
| 550 |
+
)
|
| 551 |
+
continue
|
| 552 |
+
first_unit[item.segment_id] = item.unit_id
|
| 553 |
+
items.append(item)
|
| 554 |
+
counts = self._write_quarantine("select", records)
|
| 555 |
+
if not items:
|
| 556 |
+
raise RuntimeError(f"select kept no item from {len(done)} units; quarantined: {counts}")
|
| 557 |
+
_write_jsonl(self.root / "select" / "manifest.jsonl", [i.to_obj() for i in items])
|
| 558 |
+
selected_ms = sum(item.raw_duration_ms or 0 for item in items)
|
| 559 |
+
return self._finish(
|
| 560 |
+
"select",
|
| 561 |
+
plan_sha,
|
| 562 |
+
{
|
| 563 |
+
"units_planned": len(plan.unit_ids),
|
| 564 |
+
"units_processed": len(done),
|
| 565 |
+
"stopped_by_max_hours": stopped,
|
| 566 |
+
"selected": len(items),
|
| 567 |
+
"selected_hours": _hours(selected_ms),
|
| 568 |
+
"quarantined": counts,
|
| 569 |
+
},
|
| 570 |
+
)
|
| 571 |
+
|
| 572 |
+
def _select_unit(
|
| 573 |
+
self, unit_id: str, remaining_ms: int, selected_in: Mapping[str, str]
|
| 574 |
+
) -> _UnitResult:
|
| 575 |
+
"""One unit: items within ``remaining_ms``; ids in ``selected_in`` (earlier units) are
|
| 576 |
+
quarantined as duplicates by the adapter before their audio is written."""
|
| 577 |
+
cfg, adapter = self.cfg, self.adapter
|
| 578 |
+
items: list[Item] = []
|
| 579 |
+
records: list[QuarantineRecord] = []
|
| 580 |
+
used_ms = 0
|
| 581 |
+
truncated = False
|
| 582 |
+
events = adapter.iter_unit(unit_id, exclude=selected_in)
|
| 583 |
+
try:
|
| 584 |
+
for event in events:
|
| 585 |
+
if isinstance(event, QuarantineRecord):
|
| 586 |
+
records.append(event)
|
| 587 |
+
continue
|
| 588 |
+
raw = adapter.raw_path(event)
|
| 589 |
+
reason: str | None = None
|
| 590 |
+
details: dict[str, object] = {}
|
| 591 |
+
duration_ms = 0
|
| 592 |
+
try:
|
| 593 |
+
duration_ms = raw_duration_ms(raw)
|
| 594 |
+
except AudioRejected as err:
|
| 595 |
+
reason, details = err.reason, err.details
|
| 596 |
+
else:
|
| 597 |
+
if duration_ms < cfg.min_duration_ms:
|
| 598 |
+
reason, details = "too_short", {"duration_ms": duration_ms}
|
| 599 |
+
elif duration_ms > cfg.max_duration_ms:
|
| 600 |
+
reason, details = "too_long", {"duration_ms": duration_ms}
|
| 601 |
+
if reason is not None:
|
| 602 |
+
records.append(self._drop("select", event, reason, **details))
|
| 603 |
+
elif used_ms + duration_ms > remaining_ms:
|
| 604 |
+
truncated = True
|
| 605 |
+
else:
|
| 606 |
+
items.append(_with_duration(event, duration_ms))
|
| 607 |
+
used_ms += duration_ms
|
| 608 |
+
if reason is not None or truncated:
|
| 609 |
+
if adapter.ephemeral_raw:
|
| 610 |
+
raw.unlink(missing_ok=True)
|
| 611 |
+
if truncated:
|
| 612 |
+
break
|
| 613 |
+
finally:
|
| 614 |
+
events.close()
|
| 615 |
+
return _UnitResult(unit_id, truncated, tuple(items), tuple(records))
|
| 616 |
+
|
| 617 |
+
def _select_items(self) -> list[Item]:
|
| 618 |
+
self._require_done("select")
|
| 619 |
+
return [
|
| 620 |
+
Item.from_obj(row, f"select/manifest.jsonl:{number}")
|
| 621 |
+
for number, row in enumerate(_read_jsonl(self.root / "select" / "manifest.jsonl"), 1)
|
| 622 |
+
]
|
| 623 |
+
|
| 624 |
+
# ------------------------------------------------------------------- audio
|
| 625 |
+
|
| 626 |
+
def _wav(self, sid: str) -> Path:
|
| 627 |
+
return self.cache / "audio16k" / f"{sid}.wav"
|
| 628 |
+
|
| 629 |
+
def _npy(self, model_key: str, sid: str) -> Path:
|
| 630 |
+
return self.cache / "arrays" / model_key / f"{sid}.npy"
|
| 631 |
+
|
| 632 |
+
def _refetch(self, items: Sequence[Item]) -> None:
|
| 633 |
+
"""Put the raw audio of ``items`` back into the cache (one pass per unit)."""
|
| 634 |
+
by_unit: dict[str, set[str]] = {}
|
| 635 |
+
for item in items:
|
| 636 |
+
by_unit.setdefault(item.unit_id, set()).add(item.segment_id)
|
| 637 |
+
for unit_id, sids in by_unit.items():
|
| 638 |
+
log.info("refetching raw audio of %d items of unit %s", len(sids), unit_id)
|
| 639 |
+
events = self.adapter.iter_unit(unit_id, only=sids)
|
| 640 |
+
try:
|
| 641 |
+
for _ in events:
|
| 642 |
+
pass
|
| 643 |
+
finally:
|
| 644 |
+
events.close()
|
| 645 |
+
|
| 646 |
+
def _ensure_wavs(self, items: Sequence[Item]) -> dict[str, AudioRejected]:
|
| 647 |
+
"""Make sure the 16 kHz wav of every item is in the cache.
|
| 648 |
+
|
| 649 |
+
Missing wavs are converted from the raw audio, which is fetched again first if it is
|
| 650 |
+
gone too (ephemeral sources). Converted ephemeral raw files are deleted.
|
| 651 |
+
|
| 652 |
+
Returns:
|
| 653 |
+
``{segment_id: reason}`` for the items that could not be given a wav.
|
| 654 |
+
"""
|
| 655 |
+
adapter = self.adapter
|
| 656 |
+
todo = [i for i in items if not self._wav(i.segment_id).is_file()]
|
| 657 |
+
if not todo:
|
| 658 |
+
return {}
|
| 659 |
+
if adapter.ephemeral_raw:
|
| 660 |
+
self._refetch([i for i in todo if not adapter.raw_path(i).is_file()])
|
| 661 |
+
failures: dict[str, AudioRejected] = {}
|
| 662 |
+
with ThreadPoolExecutor(max_workers=self.audio_threads) as pool:
|
| 663 |
+
outcomes = pool.map(self._convert_one, todo)
|
| 664 |
+
for number, (item, error) in enumerate(zip(todo, outcomes, strict=True), start=1):
|
| 665 |
+
if error is not None:
|
| 666 |
+
failures[item.segment_id] = error
|
| 667 |
+
if number % PROGRESS_EVERY == 0:
|
| 668 |
+
log.info("audio: converted %d/%d", number, len(todo))
|
| 669 |
+
return failures
|
| 670 |
+
|
| 671 |
+
def _convert_one(self, item: Item) -> AudioRejected | None:
|
| 672 |
+
"""Raw audio -> working wav for one item (thread-safe: touches only this item's files)."""
|
| 673 |
+
adapter = self.adapter
|
| 674 |
+
raw = adapter.raw_path(item)
|
| 675 |
+
if not raw.is_file():
|
| 676 |
+
return AudioRejected("raw_audio_missing", path=str(raw))
|
| 677 |
+
error: AudioRejected | None = None
|
| 678 |
+
try:
|
| 679 |
+
convert_to_wav16k(raw, self._wav(item.segment_id))
|
| 680 |
+
except AudioRejected as err:
|
| 681 |
+
error = err
|
| 682 |
+
if adapter.ephemeral_raw:
|
| 683 |
+
raw.unlink(missing_ok=True)
|
| 684 |
+
return error
|
| 685 |
+
|
| 686 |
+
def _measure(self, item: Item) -> tuple[int, float]:
|
| 687 |
+
"""``(duration_ms, loudness_dbfs)`` of the item's wav; a bad cached wav is rebuilt once."""
|
| 688 |
+
wav = self._wav(item.segment_id)
|
| 689 |
+
for attempt in (1, 2):
|
| 690 |
+
try:
|
| 691 |
+
facts = measure_wav16k(wav)
|
| 692 |
+
except AudioRejected:
|
| 693 |
+
if attempt == 2:
|
| 694 |
+
raise
|
| 695 |
+
wav.unlink(missing_ok=True)
|
| 696 |
+
failed = self._ensure_wavs([item])
|
| 697 |
+
if item.segment_id in failed:
|
| 698 |
+
raise failed[item.segment_id] from None
|
| 699 |
+
else:
|
| 700 |
+
return facts.duration_ms, facts.loudness_dbfs
|
| 701 |
+
raise AssertionError("unreachable") # pragma: no cover
|
| 702 |
+
|
| 703 |
+
def _stage_audio(self) -> dict[str, object]:
|
| 704 |
+
"""16 kHz mono wav of every selected item, measured; unusable audio is quarantined."""
|
| 705 |
+
cfg = self.cfg
|
| 706 |
+
items = self._select_items()
|
| 707 |
+
input_sha = _file_sha1(self.root / "select" / "manifest.jsonl")
|
| 708 |
+
reused = self._reuse("audio", input_sha)
|
| 709 |
+
if reused is not None:
|
| 710 |
+
return reused
|
| 711 |
+
failures = self._ensure_wavs(items)
|
| 712 |
+
rows: list[dict[str, object]] = []
|
| 713 |
+
records: list[QuarantineRecord] = []
|
| 714 |
+
kept_ms = 0
|
| 715 |
+
for number, item in enumerate(items, start=1):
|
| 716 |
+
error = failures.get(item.segment_id)
|
| 717 |
+
reason: str | None = None
|
| 718 |
+
details: dict[str, object] = {}
|
| 719 |
+
duration_ms, loudness = 0, 0.0
|
| 720 |
+
if error is not None:
|
| 721 |
+
reason, details = error.reason, error.details
|
| 722 |
+
else:
|
| 723 |
+
try:
|
| 724 |
+
duration_ms, loudness = self._measure(item)
|
| 725 |
+
except AudioRejected as err:
|
| 726 |
+
reason, details = err.reason, err.details
|
| 727 |
+
else:
|
| 728 |
+
if duration_ms < cfg.min_duration_ms:
|
| 729 |
+
reason, details = "too_short", {"duration_ms": duration_ms}
|
| 730 |
+
elif duration_ms > cfg.max_duration_ms:
|
| 731 |
+
reason, details = "too_long", {"duration_ms": duration_ms}
|
| 732 |
+
elif loudness < cfg.min_loudness_dbfs:
|
| 733 |
+
reason, details = "silent", {"loudness_dbfs": round(loudness, 2)}
|
| 734 |
+
if reason is not None:
|
| 735 |
+
records.append(self._drop("audio", item, reason, **details))
|
| 736 |
+
else:
|
| 737 |
+
rows.append(
|
| 738 |
+
{
|
| 739 |
+
"segment_id": item.segment_id,
|
| 740 |
+
"duration_ms": duration_ms,
|
| 741 |
+
"loudness_dbfs": loudness,
|
| 742 |
+
}
|
| 743 |
+
)
|
| 744 |
+
kept_ms += duration_ms
|
| 745 |
+
if self.adapter.ephemeral_raw:
|
| 746 |
+
self.adapter.raw_path(item).unlink(missing_ok=True)
|
| 747 |
+
if number % PROGRESS_EVERY == 0:
|
| 748 |
+
log.info("audio: measured %d/%d", number, len(items))
|
| 749 |
+
counts = self._write_quarantine("audio", records)
|
| 750 |
+
if not rows:
|
| 751 |
+
raise RuntimeError(f"audio kept no item of {len(items)}; quarantined: {counts}")
|
| 752 |
+
_write_jsonl(self.root / "audio" / "manifest.jsonl", rows)
|
| 753 |
+
return self._finish(
|
| 754 |
+
"audio",
|
| 755 |
+
input_sha,
|
| 756 |
+
{
|
| 757 |
+
"input": len(items),
|
| 758 |
+
"kept": len(rows),
|
| 759 |
+
"kept_hours": _hours(kept_ms),
|
| 760 |
+
"quarantined": counts,
|
| 761 |
+
},
|
| 762 |
+
)
|
| 763 |
+
|
| 764 |
+
# ------------------------------------------------------- shared stage tables
|
| 765 |
+
|
| 766 |
+
def _segs(self, upto: str) -> list[_Seg]:
|
| 767 |
+
"""Segments that survived every stage up to and including ``upto``
|
| 768 |
+
(``audio`` | ``latents`` | ``ctc``), in select order."""
|
| 769 |
+
order = ("audio", "latents", "ctc")
|
| 770 |
+
if upto not in order:
|
| 771 |
+
raise ValueError(f"unknown stage {upto!r}")
|
| 772 |
+
for stage in order[: order.index(upto) + 1]:
|
| 773 |
+
self._require_done(stage)
|
| 774 |
+
audio = {
|
| 775 |
+
str(r["segment_id"]): r for r in _read_jsonl(self.root / "audio" / "manifest.jsonl")
|
| 776 |
+
}
|
| 777 |
+
latents: dict[str, dict[str, object]] = {}
|
| 778 |
+
if upto != "audio":
|
| 779 |
+
latents = {
|
| 780 |
+
str(r["segment_id"]): r
|
| 781 |
+
for r in _read_jsonl(self.root / "latents" / "manifest.jsonl")
|
| 782 |
+
}
|
| 783 |
+
ctc: set[str] | None = None
|
| 784 |
+
if upto == "ctc":
|
| 785 |
+
ctc = {str(r["segment_id"]) for r in _read_jsonl(self.root / "ctc" / "manifest.jsonl")}
|
| 786 |
+
segs: list[_Seg] = []
|
| 787 |
+
for item in self._select_items():
|
| 788 |
+
row = audio.get(item.segment_id)
|
| 789 |
+
if row is None:
|
| 790 |
+
continue
|
| 791 |
+
frames, hop, dim = 0, 0.0, 0
|
| 792 |
+
if upto != "audio":
|
| 793 |
+
facts = latents.get(item.segment_id)
|
| 794 |
+
if facts is None:
|
| 795 |
+
continue
|
| 796 |
+
frames = int(str(facts["num_frames"]))
|
| 797 |
+
hop = float(str(facts["frame_hop_ms"]))
|
| 798 |
+
dim = int(str(facts["latent_dim"]))
|
| 799 |
+
if ctc is not None and item.segment_id not in ctc:
|
| 800 |
+
continue
|
| 801 |
+
segs.append(
|
| 802 |
+
_Seg(
|
| 803 |
+
item,
|
| 804 |
+
int(str(row["duration_ms"])),
|
| 805 |
+
float(str(row["loudness_dbfs"])),
|
| 806 |
+
frames,
|
| 807 |
+
hop,
|
| 808 |
+
dim,
|
| 809 |
+
)
|
| 810 |
+
)
|
| 811 |
+
return segs
|
| 812 |
+
|
| 813 |
+
# ----------------------------------------------------------------- workers
|
| 814 |
+
|
| 815 |
+
def _python(self, key: str) -> Path:
|
| 816 |
+
if key not in self._pythons:
|
| 817 |
+
if self.uv is None:
|
| 818 |
+
self.uv = bench_envs.find_uv(bench_envs.subprocess_runner)
|
| 819 |
+
spec = envs.data_env_for_worker(DATA_MODELS[key].worker)
|
| 820 |
+
self._pythons[key] = bench_envs.ensure_env(
|
| 821 |
+
spec, self.envs_root, self.repo_dir, uv=self.uv
|
| 822 |
+
)
|
| 823 |
+
return self._pythons[key]
|
| 824 |
+
|
| 825 |
+
def _pending(self, key: str, sids: Sequence[str]) -> set[str]:
|
| 826 |
+
"""Segments whose worker result is missing, failed (with ``retry_failed``) or whose
|
| 827 |
+
array left the cache: these need their wav."""
|
| 828 |
+
path = self.root / "workers" / key / "results.jsonl"
|
| 829 |
+
existing = load_results(path, repair=True) if path.exists() else {}
|
| 830 |
+
pending: set[str] = set()
|
| 831 |
+
for sid in sids:
|
| 832 |
+
result = existing.get(sid)
|
| 833 |
+
if result is None:
|
| 834 |
+
pending.add(sid)
|
| 835 |
+
elif result.output is None:
|
| 836 |
+
if self.retry_failed:
|
| 837 |
+
pending.add(sid)
|
| 838 |
+
elif not Path(str(result.output.get("out_path", ""))).is_file():
|
| 839 |
+
pending.add(sid)
|
| 840 |
+
return pending
|
| 841 |
+
|
| 842 |
+
def _run_model(self, key: str, segs: Sequence[_Seg]) -> dict[str, WorkResult]:
|
| 843 |
+
"""Run the data worker ``key`` over ``segs`` (resumable; no work when complete)."""
|
| 844 |
+
model = DATA_MODELS[key]
|
| 845 |
+
pending = self._pending(key, [s.sid for s in segs])
|
| 846 |
+
if pending:
|
| 847 |
+
failed = self._ensure_wavs([s.item for s in segs if s.sid in pending])
|
| 848 |
+
if failed:
|
| 849 |
+
sample = {sid: err.reason for sid, err in list(failed.items())[:3]}
|
| 850 |
+
raise RuntimeError(f"{key}: {len(failed)} audio files cannot be rebuilt: {sample}")
|
| 851 |
+
requests = tuple(
|
| 852 |
+
WorkRequest(s.sid, str(self._wav(s.sid)), {"out_path": str(self._npy(key, s.sid))})
|
| 853 |
+
for s in segs
|
| 854 |
+
)
|
| 855 |
+
return run_worker_job(
|
| 856 |
+
self.root / "workers" / key,
|
| 857 |
+
key,
|
| 858 |
+
lambda: self._python(key),
|
| 859 |
+
envs.worker_module_name(model.worker),
|
| 860 |
+
model.ref.model_id,
|
| 861 |
+
model.ref.revision,
|
| 862 |
+
model.batch_size,
|
| 863 |
+
requests,
|
| 864 |
+
array_outputs=True,
|
| 865 |
+
runner=self.runner,
|
| 866 |
+
retry_failed=self.retry_failed,
|
| 867 |
+
skip_if_complete=True,
|
| 868 |
+
)
|
| 869 |
+
|
| 870 |
+
# ----------------------------------------------------------------- latents
|
| 871 |
+
|
| 872 |
+
def _stage_latents(self) -> dict[str, object]:
|
| 873 |
+
"""VAE latents (``vae_encode`` worker) of the segments that passed the audio stage."""
|
| 874 |
+
segs = self._segs("audio")
|
| 875 |
+
input_sha = _file_sha1(self.root / "audio" / "manifest.jsonl")
|
| 876 |
+
reused = self._reuse("latents", input_sha)
|
| 877 |
+
if reused is not None:
|
| 878 |
+
return reused
|
| 879 |
+
results = self._run_model(VAE_KEY, segs)
|
| 880 |
+
rows: list[dict[str, object]] = []
|
| 881 |
+
records: list[QuarantineRecord] = []
|
| 882 |
+
hops: set[float] = set()
|
| 883 |
+
dims: set[int] = set()
|
| 884 |
+
for seg in segs:
|
| 885 |
+
result = results.get(seg.sid)
|
| 886 |
+
if result is None:
|
| 887 |
+
raise RuntimeError(f"{VAE_KEY}: no result for {seg.sid}")
|
| 888 |
+
if result.output is None:
|
| 889 |
+
records.append(
|
| 890 |
+
self._drop("latents", seg.item, "latents_failed", error=result.error)
|
| 891 |
+
)
|
| 892 |
+
continue
|
| 893 |
+
array = np.load(str(result.output["out_path"]), allow_pickle=False)
|
| 894 |
+
problem = _latent_problem(array)
|
| 895 |
+
if problem is not None:
|
| 896 |
+
records.append(self._drop("latents", seg.item, problem[0], **problem[1]))
|
| 897 |
+
continue
|
| 898 |
+
hop = float(str(result.output["frame_hop_ms"]))
|
| 899 |
+
hops.add(hop)
|
| 900 |
+
dims.add(int(array.shape[1]))
|
| 901 |
+
rows.append(
|
| 902 |
+
{
|
| 903 |
+
"segment_id": seg.sid,
|
| 904 |
+
"num_frames": int(array.shape[0]),
|
| 905 |
+
"latent_dim": int(array.shape[1]),
|
| 906 |
+
"frame_hop_ms": hop,
|
| 907 |
+
}
|
| 908 |
+
)
|
| 909 |
+
if len(hops) > 1 or len(dims) > 1:
|
| 910 |
+
raise ValueError(f"latents disagree on frame hop {sorted(hops)} or dim {sorted(dims)}")
|
| 911 |
+
counts = self._write_quarantine("latents", records)
|
| 912 |
+
if not rows:
|
| 913 |
+
raise RuntimeError(f"latents kept no segment of {len(segs)}; quarantined: {counts}")
|
| 914 |
+
_write_jsonl(self.root / "latents" / "manifest.jsonl", rows)
|
| 915 |
+
return self._finish(
|
| 916 |
+
"latents",
|
| 917 |
+
input_sha,
|
| 918 |
+
{
|
| 919 |
+
"input": len(segs),
|
| 920 |
+
"kept": len(rows),
|
| 921 |
+
"latent_dim": dims.pop(),
|
| 922 |
+
"frame_hop_ms": hops.pop(),
|
| 923 |
+
"quarantined": counts,
|
| 924 |
+
},
|
| 925 |
+
)
|
| 926 |
+
|
| 927 |
+
# --------------------------------------------------------------------- ctc
|
| 928 |
+
|
| 929 |
+
def _stage_ctc(self) -> dict[str, object]:
|
| 930 |
+
"""CTC emissions (``ctc_emissions`` worker) of the segments that have latents."""
|
| 931 |
+
segs = self._segs("latents")
|
| 932 |
+
input_sha = _file_sha1(self.root / "latents" / "manifest.jsonl")
|
| 933 |
+
reused = self._reuse("ctc", input_sha)
|
| 934 |
+
if reused is not None:
|
| 935 |
+
return reused
|
| 936 |
+
results = self._run_model(CTC_KEY, segs)
|
| 937 |
+
rows: list[dict[str, object]] = []
|
| 938 |
+
records: list[QuarantineRecord] = []
|
| 939 |
+
for seg in segs:
|
| 940 |
+
result = results.get(seg.sid)
|
| 941 |
+
if result is None:
|
| 942 |
+
raise RuntimeError(f"{CTC_KEY}: no result for {seg.sid}")
|
| 943 |
+
if result.output is None:
|
| 944 |
+
records.append(self._drop("ctc", seg.item, "ctc_failed", error=result.error))
|
| 945 |
+
else:
|
| 946 |
+
rows.append({"segment_id": seg.sid})
|
| 947 |
+
counts = self._write_quarantine("ctc", records)
|
| 948 |
+
if not rows:
|
| 949 |
+
raise RuntimeError(f"ctc kept no segment of {len(segs)}; quarantined: {counts}")
|
| 950 |
+
_write_jsonl(self.root / "ctc" / "manifest.jsonl", rows)
|
| 951 |
+
return self._finish(
|
| 952 |
+
"ctc", input_sha, {"input": len(segs), "kept": len(rows), "quarantined": counts}
|
| 953 |
+
)
|
| 954 |
+
|
| 955 |
+
# ----------------------------------------------------------------- timings
|
| 956 |
+
|
| 957 |
+
def _stage_timings(self) -> dict[str, object]:
|
| 958 |
+
"""Unit timeline of every segment (CTC forced alignment of the transcript), as the
|
| 959 |
+
pilot's ``timings`` stage does, with no event tags (``tags_json`` is ``[]``)."""
|
| 960 |
+
segs = self._segs("ctc")
|
| 961 |
+
input_sha = _file_sha1(self.root / "ctc" / "manifest.jsonl")
|
| 962 |
+
reused = self._reuse("timings", input_sha)
|
| 963 |
+
if reused is not None:
|
| 964 |
+
return reused
|
| 965 |
+
# The request set of the ctc stage (not just the survivors), so no result is pruned.
|
| 966 |
+
results = self._run_model(CTC_KEY, self._segs("latents"))
|
| 967 |
+
built: list[SegmentTimings] = []
|
| 968 |
+
records: list[QuarantineRecord] = []
|
| 969 |
+
clamped = 0
|
| 970 |
+
scores: list[float] = []
|
| 971 |
+
for number, seg in enumerate(segs, start=1):
|
| 972 |
+
output = results[seg.sid].output
|
| 973 |
+
if output is None:
|
| 974 |
+
raise RuntimeError(f"{CTC_KEY}: result of {seg.sid} failed after the ctc stage")
|
| 975 |
+
outcome = self._timeline(seg, output)
|
| 976 |
+
if outcome.timings is not None:
|
| 977 |
+
built.append(outcome.timings)
|
| 978 |
+
clamped += outcome.clamped_units
|
| 979 |
+
scores.append(outcome.mean_align_score)
|
| 980 |
+
else:
|
| 981 |
+
assert outcome.quarantine is not None
|
| 982 |
+
records.append(outcome.quarantine)
|
| 983 |
+
if number % PROGRESS_EVERY == 0:
|
| 984 |
+
log.info("timings: %d/%d", number, len(segs))
|
| 985 |
+
counts = self._write_quarantine("timings", records)
|
| 986 |
+
if not built:
|
| 987 |
+
raise RuntimeError(f"timings built no timeline of {len(segs)}; quarantined: {counts}")
|
| 988 |
+
write_timings(self.root / "timings" / "timings.jsonl", built)
|
| 989 |
+
return self._finish(
|
| 990 |
+
"timings",
|
| 991 |
+
input_sha,
|
| 992 |
+
{
|
| 993 |
+
"input": len(segs),
|
| 994 |
+
"written": len(built),
|
| 995 |
+
"units": sum(len(t.units) for t in built),
|
| 996 |
+
"zero_length_units": sum(t.zero_length_units for t in built),
|
| 997 |
+
"clamped_units": clamped,
|
| 998 |
+
"mean_align_score": round(math.fsum(scores) / len(scores), 4),
|
| 999 |
+
"min_segment_align_score": round(min(scores), 4),
|
| 1000 |
+
"quarantined": counts,
|
| 1001 |
+
},
|
| 1002 |
+
)
|
| 1003 |
+
|
| 1004 |
+
def _timeline(self, seg: _Seg, ctc: Mapping[str, object]) -> _TimelineOutcome:
|
| 1005 |
+
"""Align the transcript, check the alignment, build the timeline (or say why not)."""
|
| 1006 |
+
n_tokens = len(encode_text(seg.item.text))
|
| 1007 |
+
if n_tokens > seg.num_frames:
|
| 1008 |
+
record = self._drop(
|
| 1009 |
+
"timings",
|
| 1010 |
+
seg.item,
|
| 1011 |
+
"text_longer_than_frames",
|
| 1012 |
+
tokens=n_tokens,
|
| 1013 |
+
frames=seg.num_frames,
|
| 1014 |
+
)
|
| 1015 |
+
return _TimelineOutcome(None, record)
|
| 1016 |
+
vocab_list = [str(t) for t in ctc["vocab"]] # type: ignore[attr-defined]
|
| 1017 |
+
vocab = {token: index for index, token in enumerate(vocab_list)}
|
| 1018 |
+
emissions = np.load(str(ctc["out_path"]), allow_pickle=False)
|
| 1019 |
+
try:
|
| 1020 |
+
aligned = align_words(
|
| 1021 |
+
seg.item.text.split(),
|
| 1022 |
+
emissions,
|
| 1023 |
+
vocab,
|
| 1024 |
+
blank_token=vocab_list[int(str(ctc["blank_id"]))],
|
| 1025 |
+
word_delimiter="|",
|
| 1026 |
+
frame_hop_ms=float(str(ctc["frame_hop_ms"])),
|
| 1027 |
+
duration_ms=seg.duration_ms,
|
| 1028 |
+
)
|
| 1029 |
+
except ValueError as err:
|
| 1030 |
+
record = self._drop("timings", seg.item, "timings_align_error", error=str(err))
|
| 1031 |
+
return _TimelineOutcome(None, record)
|
| 1032 |
+
if aligned.words is None:
|
| 1033 |
+
assert aligned.failure is not None
|
| 1034 |
+
reason = "timings_align_" + aligned.failure.split(":", 1)[0]
|
| 1035 |
+
record = self._drop(
|
| 1036 |
+
"timings", seg.item, reason, failure=aligned.failure, **dict(aligned.details)
|
| 1037 |
+
)
|
| 1038 |
+
return _TimelineOutcome(None, record)
|
| 1039 |
+
scores = [w.align_score for w in aligned.words if w.align_score is not None]
|
| 1040 |
+
mean_score = math.fsum(scores) / len(scores) if scores else 0.0
|
| 1041 |
+
problem = check_alignment(aligned.words, seg.duration_ms, self.cfg.alignment)
|
| 1042 |
+
if problem is not None:
|
| 1043 |
+
record = self._drop(
|
| 1044 |
+
"timings",
|
| 1045 |
+
seg.item,
|
| 1046 |
+
problem,
|
| 1047 |
+
mean_align_score=round(mean_score, 4),
|
| 1048 |
+
min_align_score=self.cfg.alignment.min_align_score,
|
| 1049 |
+
)
|
| 1050 |
+
return _TimelineOutcome(None, record)
|
| 1051 |
+
built = build_timings(
|
| 1052 |
+
seg.sid,
|
| 1053 |
+
seg.item.text,
|
| 1054 |
+
json.dumps([]),
|
| 1055 |
+
aligned.words,
|
| 1056 |
+
num_frames=seg.num_frames,
|
| 1057 |
+
frame_hop_ms=seg.frame_hop_ms,
|
| 1058 |
+
)
|
| 1059 |
+
if built.timings is None:
|
| 1060 |
+
assert built.quarantine is not None
|
| 1061 |
+
q = built.quarantine
|
| 1062 |
+
record = self._drop("timings", seg.item, q.reason, **q.details)
|
| 1063 |
+
return _TimelineOutcome(None, record)
|
| 1064 |
+
return _TimelineOutcome(built.timings, None, built.clamped_units, mean_score)
|
| 1065 |
+
|
| 1066 |
+
# ----------------------------------------------------------------- dataset
|
| 1067 |
+
|
| 1068 |
+
def _stage_dataset(self) -> dict[str, object]:
|
| 1069 |
+
"""Write the latent dataset directory (format: :mod:`nida.train.data`) + timings.jsonl."""
|
| 1070 |
+
cfg = self.cfg
|
| 1071 |
+
dataset_dir = self.dataset_dir()
|
| 1072 |
+
meta_path = dataset_dir / "meta.json"
|
| 1073 |
+
timings_path = self.root / "timings" / "timings.jsonl"
|
| 1074 |
+
self._require_done("timings")
|
| 1075 |
+
timings_sha = _file_sha1(timings_path)
|
| 1076 |
+
if meta_path.is_file():
|
| 1077 |
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
| 1078 |
+
if meta.get("timings_sha1") != timings_sha:
|
| 1079 |
+
raise RuntimeError(
|
| 1080 |
+
f"{meta_path} was built from other timings; move {dataset_dir} away"
|
| 1081 |
+
)
|
| 1082 |
+
return {"dataset": str(dataset_dir), "segments": meta["segments"], "reused": True}
|
| 1083 |
+
timelines = {
|
| 1084 |
+
t.segment_id: t
|
| 1085 |
+
for t in (
|
| 1086 |
+
SegmentTimings.from_obj(row, f"{timings_path}:{n}")
|
| 1087 |
+
for n, row in enumerate(_read_jsonl(timings_path), 1)
|
| 1088 |
+
)
|
| 1089 |
+
}
|
| 1090 |
+
segs = [s for s in self._segs("ctc") if s.sid in timelines]
|
| 1091 |
+
if len(segs) != len(timelines):
|
| 1092 |
+
raise RuntimeError("timings.jsonl names segments that did not survive the stages")
|
| 1093 |
+
splits = assign_group_splits({s.sid: s.item.group for s in segs}, cfg)
|
| 1094 |
+
forced = cfg.forced_split(self.plan.source)
|
| 1095 |
+
if forced is not None: # no group of this source can be pure: all rows are one split
|
| 1096 |
+
splits = {sid: forced for sid in splits}
|
| 1097 |
+
# The request set of the latents stage; recomputes arrays lost with the cache.
|
| 1098 |
+
latents = self._run_model(VAE_KEY, self._segs("audio"))
|
| 1099 |
+
index: list[dict[str, object]] = []
|
| 1100 |
+
shard: list[np.ndarray[Any, np.dtype[np.float16]]] = []
|
| 1101 |
+
shard_frames = 0
|
| 1102 |
+
shard_no = 0
|
| 1103 |
+
latent_dim: int | None = None
|
| 1104 |
+
frame_hop_ms: float | None = None
|
| 1105 |
+
|
| 1106 |
+
def flush() -> None:
|
| 1107 |
+
nonlocal shard, shard_frames, shard_no
|
| 1108 |
+
if not shard:
|
| 1109 |
+
return
|
| 1110 |
+
buffer = io.BytesIO()
|
| 1111 |
+
np.save(buffer, np.concatenate(shard), allow_pickle=False)
|
| 1112 |
+
atomic_write_bytes(dataset_dir / f"shard-{shard_no:05d}.npy", buffer.getvalue())
|
| 1113 |
+
shard, shard_frames, shard_no = [], 0, shard_no + 1
|
| 1114 |
+
|
| 1115 |
+
for seg in sorted(segs, key=lambda s: s.sid):
|
| 1116 |
+
output = latents[seg.sid].output
|
| 1117 |
+
if output is None:
|
| 1118 |
+
raise RuntimeError(f"{VAE_KEY}: result of {seg.sid} failed after the latents stage")
|
| 1119 |
+
array = np.load(str(output["out_path"]), allow_pickle=False)
|
| 1120 |
+
if _latent_problem(array) is not None or int(array.shape[0]) != seg.num_frames:
|
| 1121 |
+
raise RuntimeError(f"{seg.sid}: cached latents differ from the latents stage")
|
| 1122 |
+
latent_dim, frame_hop_ms = seg.latent_dim, seg.frame_hop_ms
|
| 1123 |
+
index.append(
|
| 1124 |
+
{
|
| 1125 |
+
"segment_id": seg.sid,
|
| 1126 |
+
"shard": f"shard-{shard_no:05d}.npy",
|
| 1127 |
+
"offset": shard_frames,
|
| 1128 |
+
"num_frames": seg.num_frames,
|
| 1129 |
+
"text": seg.item.text,
|
| 1130 |
+
"emotion": seg.item.emotion,
|
| 1131 |
+
"intensity": None,
|
| 1132 |
+
"rate_bucket": bucket_of(
|
| 1133 |
+
rate_chars_per_s(seg.item.text, seg.duration_ms),
|
| 1134 |
+
cfg.dataset.rate_bucket_edges_chars_per_s,
|
| 1135 |
+
),
|
| 1136 |
+
"loudness_bucket": bucket_of(
|
| 1137 |
+
seg.loudness_dbfs, cfg.dataset.loudness_bucket_edges_dbfs
|
| 1138 |
+
),
|
| 1139 |
+
"group": seg.item.group,
|
| 1140 |
+
"split": splits[seg.sid],
|
| 1141 |
+
"language": LANGUAGE,
|
| 1142 |
+
}
|
| 1143 |
+
)
|
| 1144 |
+
shard.append(array.astype(np.float16))
|
| 1145 |
+
shard_frames += seg.num_frames
|
| 1146 |
+
if shard_frames >= DATASET_SHARD_FRAMES:
|
| 1147 |
+
flush()
|
| 1148 |
+
flush()
|
| 1149 |
+
_write_jsonl(dataset_dir / "index.jsonl", index)
|
| 1150 |
+
write_timings(dataset_dir / "timings.jsonl", [timelines[s.sid] for s in segs])
|
| 1151 |
+
new_meta: dict[str, object] = {
|
| 1152 |
+
"segments": len(index),
|
| 1153 |
+
"frames": sum(int(str(i["num_frames"])) for i in index),
|
| 1154 |
+
"latent_dim": latent_dim,
|
| 1155 |
+
"frame_hop_ms": frame_hop_ms,
|
| 1156 |
+
"rate_bucket_edges_chars_per_s": list(cfg.dataset.rate_bucket_edges_chars_per_s),
|
| 1157 |
+
"loudness_bucket_edges_dbfs": list(cfg.dataset.loudness_bucket_edges_dbfs),
|
| 1158 |
+
"language": LANGUAGE,
|
| 1159 |
+
"source": self.plan.source,
|
| 1160 |
+
"source_ids": sorted({s.item.source_id for s in segs}),
|
| 1161 |
+
"plan": self.plan.to_obj(),
|
| 1162 |
+
"config_digest": cfg.digest,
|
| 1163 |
+
"seed": cfg.seed,
|
| 1164 |
+
"timings_sha1": timings_sha,
|
| 1165 |
+
"vae": [DATA_MODELS[VAE_KEY].ref.model_id, DATA_MODELS[VAE_KEY].ref.revision],
|
| 1166 |
+
"ctc": [DATA_MODELS[CTC_KEY].ref.model_id, DATA_MODELS[CTC_KEY].ref.revision],
|
| 1167 |
+
"splits": dict(sorted(Counter(splits.values()).items())),
|
| 1168 |
+
}
|
| 1169 |
+
if forced is not None:
|
| 1170 |
+
new_meta["forced_split"] = forced
|
| 1171 |
+
atomic_write_json(meta_path, new_meta) # written last: its presence marks the stage done
|
| 1172 |
+
try:
|
| 1173 |
+
self._check_with_loader(dataset_dir)
|
| 1174 |
+
except (DataFormatError, RuntimeError):
|
| 1175 |
+
meta_path.unlink(missing_ok=True)
|
| 1176 |
+
raise
|
| 1177 |
+
return {
|
| 1178 |
+
"dataset": str(dataset_dir),
|
| 1179 |
+
"segments": len(index),
|
| 1180 |
+
"frames": new_meta["frames"],
|
| 1181 |
+
"splits": new_meta["splits"],
|
| 1182 |
+
"reused": False,
|
| 1183 |
+
}
|
| 1184 |
+
|
| 1185 |
+
@staticmethod
|
| 1186 |
+
def _check_with_loader(dataset_dir: Path) -> None:
|
| 1187 |
+
"""Every split must load with timelines and no rejection (else the importer is wrong)."""
|
| 1188 |
+
for split in SPLITS:
|
| 1189 |
+
dataset, report = load_latent_dataset(dataset_dir, split, timings=True)
|
| 1190 |
+
dataset.close()
|
| 1191 |
+
if report.num_rejected:
|
| 1192 |
+
raise RuntimeError(
|
| 1193 |
+
f"nida.train.data rejects records of the imported {split} split: "
|
| 1194 |
+
f"{report.rejection_counts}"
|
| 1195 |
+
)
|
| 1196 |
+
|
| 1197 |
+
# ------------------------------------------------------------------ report
|
| 1198 |
+
|
| 1199 |
+
def _stage_report(self) -> dict[str, object]:
|
| 1200 |
+
"""``report.md`` and ``metrics.json`` of the root; returns the metrics."""
|
| 1201 |
+
dataset_dir = self.dataset_dir()
|
| 1202 |
+
meta_path = dataset_dir / "meta.json"
|
| 1203 |
+
if not meta_path.is_file():
|
| 1204 |
+
raise RuntimeError(f"{meta_path} is missing: run the dataset stage first")
|
| 1205 |
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
| 1206 |
+
index = _read_jsonl(dataset_dir / "index.jsonl")
|
| 1207 |
+
kept_ids = {str(r["segment_id"]) for r in index}
|
| 1208 |
+
segs = {s.sid: s for s in self._segs("ctc") if s.sid in kept_ids}
|
| 1209 |
+
selected = self._select_items()
|
| 1210 |
+
selected_ms = sum(i.raw_duration_ms or 0 for i in selected)
|
| 1211 |
+
quarantine = {
|
| 1212 |
+
stage: QuarantineLog(self.root / stage).counts_by_reason()
|
| 1213 |
+
for stage in ("select", "audio", "latents", "ctc", "timings")
|
| 1214 |
+
if (self.root / stage).is_dir()
|
| 1215 |
+
}
|
| 1216 |
+
splits: dict[str, dict[str, object]] = {}
|
| 1217 |
+
for split in SPLITS:
|
| 1218 |
+
members = [segs[str(r["segment_id"])] for r in index if r["split"] == split]
|
| 1219 |
+
splits[split] = {
|
| 1220 |
+
"segments": len(members),
|
| 1221 |
+
"hours": _hours(sum(s.duration_ms for s in members)),
|
| 1222 |
+
"groups": len({s.item.group for s in members}),
|
| 1223 |
+
}
|
| 1224 |
+
emotions = Counter("null" if r["emotion"] is None else str(r["emotion"]) for r in index)
|
| 1225 |
+
unmapped = Counter(
|
| 1226 |
+
str(s.item.extra.get("emotion_label"))
|
| 1227 |
+
for s in segs.values()
|
| 1228 |
+
if s.item.emotion is None and s.item.extra.get("emotion_label") is not None
|
| 1229 |
+
)
|
| 1230 |
+
timings_result = json.loads(self._marker("timings").read_text(encoding="utf-8"))["result"]
|
| 1231 |
+
metrics: dict[str, object] = {
|
| 1232 |
+
"alignment": {
|
| 1233 |
+
key: timings_result[key] for key in ("mean_align_score", "min_segment_align_score")
|
| 1234 |
+
},
|
| 1235 |
+
"source": self.plan.source,
|
| 1236 |
+
"source_ids": meta["source_ids"],
|
| 1237 |
+
"revision": self.plan.revision,
|
| 1238 |
+
"config_digest": self.cfg.digest,
|
| 1239 |
+
"max_hours": self.plan.max_hours,
|
| 1240 |
+
"units_planned": len(self.plan.unit_ids),
|
| 1241 |
+
"selected": {"segments": len(selected), "hours": _hours(selected_ms)},
|
| 1242 |
+
"kept": {
|
| 1243 |
+
"segments": len(index),
|
| 1244 |
+
"hours": _hours(sum(s.duration_ms for s in segs.values())),
|
| 1245 |
+
},
|
| 1246 |
+
"quarantine": quarantine,
|
| 1247 |
+
"emotions": dict(sorted(emotions.items())),
|
| 1248 |
+
"unmapped_emotion_labels": dict(sorted(unmapped.items())),
|
| 1249 |
+
"groups": len({s.item.group for s in segs.values()}),
|
| 1250 |
+
"splits": splits,
|
| 1251 |
+
"dataset": str(dataset_dir),
|
| 1252 |
+
}
|
| 1253 |
+
atomic_write_json(self.root / "metrics.json", metrics)
|
| 1254 |
+
atomic_write_text(self.root / "report.md", _render_report(metrics))
|
| 1255 |
+
return metrics
|
| 1256 |
+
|
| 1257 |
+
|
| 1258 |
+
def assign_group_splits(groups: Mapping[str, str], cfg: ImportConfig) -> dict[str, str]:
|
| 1259 |
+
"""Group-disjoint, deterministic ``train``/``val``/``test`` split of ``{segment_id: group}``.
|
| 1260 |
+
|
| 1261 |
+
The split of a group is a pure function of ``(cfg.seed, group)``
|
| 1262 |
+
(:func:`nida.data.splits.assign_splits`; ``val_fraction`` / ``test_fraction`` of the groups),
|
| 1263 |
+
independent of which other groups the root holds: the same group lands in the same split in
|
| 1264 |
+
every root, so datasets of different roots can be merged without a group crossing splits.
|
| 1265 |
+
A root with a single group is therefore wholly one split, not necessarily ``train``.
|
| 1266 |
+
"""
|
| 1267 |
+
return assign_splits(groups, cfg.split_fractions, str(cfg.seed))
|
| 1268 |
+
|
| 1269 |
+
|
| 1270 |
+
def _with_duration(item: Item, duration_ms: int) -> Item:
|
| 1271 |
+
return Item(
|
| 1272 |
+
source_id=item.source_id,
|
| 1273 |
+
item_id=item.item_id,
|
| 1274 |
+
segment_id=item.segment_id,
|
| 1275 |
+
unit_id=item.unit_id,
|
| 1276 |
+
text=item.text,
|
| 1277 |
+
group=item.group,
|
| 1278 |
+
emotion=item.emotion,
|
| 1279 |
+
raw_duration_ms=duration_ms,
|
| 1280 |
+
extra=item.extra,
|
| 1281 |
+
)
|
| 1282 |
+
|
| 1283 |
+
|
| 1284 |
+
def _latent_problem(array: np.ndarray[Any, Any]) -> tuple[str, dict[str, object]] | None:
|
| 1285 |
+
"""A reason (and details) why latents cannot go into a float16 shard, else ``None``."""
|
| 1286 |
+
if array.ndim != 2 or array.shape[0] < 1 or array.shape[1] < 1:
|
| 1287 |
+
return "latents_bad_shape", {"shape": list(array.shape)}
|
| 1288 |
+
if not np.all(np.isfinite(array)) or float(np.max(np.abs(array))) > FLOAT16_MAX_ABS:
|
| 1289 |
+
return "latents_not_float16", {}
|
| 1290 |
+
return None
|
| 1291 |
+
|
| 1292 |
+
|
| 1293 |
+
def _render_report(m: Mapping[str, object]) -> str:
|
| 1294 |
+
selected, kept = m["selected"], m["kept"]
|
| 1295 |
+
assert isinstance(selected, dict) and isinstance(kept, dict)
|
| 1296 |
+
assert isinstance(m["splits"], dict) and isinstance(m["quarantine"], dict)
|
| 1297 |
+
lines = [
|
| 1298 |
+
f"# data-import: {m['source']}",
|
| 1299 |
+
"",
|
| 1300 |
+
f"- source ids: {m['source_ids']}",
|
| 1301 |
+
f"- revision: `{m['revision']}`",
|
| 1302 |
+
f"- config digest: `{m['config_digest']}`",
|
| 1303 |
+
f"- max hours: {m['max_hours']}; work units planned: {m['units_planned']}",
|
| 1304 |
+
f"- selected: {selected['segments']} segments, {selected['hours']} h",
|
| 1305 |
+
f"- kept (in the dataset): {kept['segments']} segments, {kept['hours']} h",
|
| 1306 |
+
f"- groups: {m['groups']}",
|
| 1307 |
+
f"- alignment (timings): {json.dumps(m['alignment'])}",
|
| 1308 |
+
f"- dataset: `{m['dataset']}`",
|
| 1309 |
+
"",
|
| 1310 |
+
"## Splits",
|
| 1311 |
+
"",
|
| 1312 |
+
*(f"- {name}: {json.dumps(info)}" for name, info in m["splits"].items()),
|
| 1313 |
+
"",
|
| 1314 |
+
"## Emotions",
|
| 1315 |
+
"",
|
| 1316 |
+
f"- {json.dumps(m['emotions'], ensure_ascii=False)}",
|
| 1317 |
+
f"- unmapped source labels (-> null): {json.dumps(m['unmapped_emotion_labels'])}",
|
| 1318 |
+
"",
|
| 1319 |
+
"## Quarantine by stage and reason",
|
| 1320 |
+
"",
|
| 1321 |
+
*(
|
| 1322 |
+
f"- {stage}: {json.dumps(counts, ensure_ascii=False)}"
|
| 1323 |
+
for stage, counts in m["quarantine"].items()
|
| 1324 |
+
),
|
| 1325 |
+
]
|
| 1326 |
+
return "\n".join(lines) + "\n"
|
src/nida/data/importer/sources.py
ADDED
|
@@ -0,0 +1,776 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Source adapters of the importer: which corpus items exist, which are selected, where the
|
| 2 |
+
raw audio is.
|
| 3 |
+
|
| 4 |
+
An adapter splits its corpus into *work units* (a parquet shard, a chunk of manifest rows) in a
|
| 5 |
+
deterministic order and, per unit, yields the selected :class:`Item` objects (metadata filters
|
| 6 |
+
and transcript normalisation applied, raw audio placed in the local cache) and a
|
| 7 |
+
:class:`~nida.data.types.QuarantineRecord` for every row it drops, with a machine-readable
|
| 8 |
+
reason. It does not look at audio samples; the stages do.
|
| 9 |
+
|
| 10 |
+
* :class:`AlaniaAdapter` - HF parquet shards (``alania_synth_tr``), filtered by the QC columns.
|
| 11 |
+
* :class:`TrFullAdapter` - HF ``data.jsonl`` + wavs (``tr_full_dataset``).
|
| 12 |
+
* :class:`WavdirAdapter` - a folder with ``manifest.jsonl`` (our own synthetic speech).
|
| 13 |
+
|
| 14 |
+
The archive and Common Voice corpora (``issai_tsc``, ``mediaspeech_tr``, ``common_voice_tr``)
|
| 15 |
+
are in :mod:`nida.data.importer.corpora`; :func:`make_adapter` builds all six.
|
| 16 |
+
|
| 17 |
+
Item ids: ``segment_id = sha1(source_id U+001F item_id)[:16]`` (:func:`item_segment_id`);
|
| 18 |
+
quarantine records use the segment id as ``item_id`` and keep the source's own id in
|
| 19 |
+
``details["item_id"]``.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import fnmatch
|
| 25 |
+
import hashlib
|
| 26 |
+
import json
|
| 27 |
+
import math
|
| 28 |
+
import re
|
| 29 |
+
from collections.abc import Generator, Mapping, Set
|
| 30 |
+
from dataclasses import dataclass, field
|
| 31 |
+
from pathlib import Path, PurePosixPath
|
| 32 |
+
from typing import TYPE_CHECKING, Protocol
|
| 33 |
+
from urllib.parse import unquote
|
| 34 |
+
|
| 35 |
+
from nida.bench import fetch
|
| 36 |
+
from nida.data.importer.config import CORPUS_SOURCES, LEGACY_SOURCES, ImportConfig
|
| 37 |
+
from nida.data.importer.hub import HubClient, HubError
|
| 38 |
+
from nida.data.importer.text import (
|
| 39 |
+
DETAIL_TEXT_CHARS,
|
| 40 |
+
TextRejected,
|
| 41 |
+
map_emotion,
|
| 42 |
+
normalise_transcript,
|
| 43 |
+
)
|
| 44 |
+
from nida.data.types import QuarantineRecord
|
| 45 |
+
from nida.text.turkish import nfc
|
| 46 |
+
from nida.utils.io import atomic_write_bytes
|
| 47 |
+
|
| 48 |
+
if TYPE_CHECKING:
|
| 49 |
+
import pyarrow as pa # type: ignore[import-untyped] # no stubs
|
| 50 |
+
|
| 51 |
+
__all__ = [
|
| 52 |
+
"ALANIA_SOURCE_ID",
|
| 53 |
+
"HUB_SOURCES",
|
| 54 |
+
"LOCAL_SOURCES",
|
| 55 |
+
"STAGE",
|
| 56 |
+
"TR_FULL_SOURCE_ID",
|
| 57 |
+
"AlaniaAdapter",
|
| 58 |
+
"Item",
|
| 59 |
+
"SourceAdapter",
|
| 60 |
+
"TrFullAdapter",
|
| 61 |
+
"UnitListing",
|
| 62 |
+
"WavdirAdapter",
|
| 63 |
+
"item_segment_id",
|
| 64 |
+
"make_adapter",
|
| 65 |
+
"safe_relative_path",
|
| 66 |
+
]
|
| 67 |
+
|
| 68 |
+
STAGE = "select"
|
| 69 |
+
ALANIA_SOURCE_ID = "alania_synth_tr"
|
| 70 |
+
TR_FULL_SOURCE_ID = "tr_full_dataset"
|
| 71 |
+
SOURCES = (*LEGACY_SOURCES, *CORPUS_SOURCES)
|
| 72 |
+
#: Sources read from a Hugging Face dataset repository (they need a ``HubClient``).
|
| 73 |
+
HUB_SOURCES = ("alania", "tr_full", "issai_tsc")
|
| 74 |
+
#: Sources read from a local folder or tarball given as ``--source-dir`` (no Hub revision;
|
| 75 |
+
#: ``plan.json`` pins the sha1 of their manifest / validated file).
|
| 76 |
+
LOCAL_SOURCES = ("wavdir", "common_voice_tr")
|
| 77 |
+
|
| 78 |
+
#: Parquet columns the alania adapter reads.
|
| 79 |
+
ALANIA_COLUMNS = (
|
| 80 |
+
"id",
|
| 81 |
+
"audio",
|
| 82 |
+
"text_spoken",
|
| 83 |
+
"duration",
|
| 84 |
+
"voice_id",
|
| 85 |
+
"gender",
|
| 86 |
+
"channel_preset",
|
| 87 |
+
"qc_cer",
|
| 88 |
+
"qc_utmos",
|
| 89 |
+
"license",
|
| 90 |
+
)
|
| 91 |
+
_CHUNK_RE = re.compile(r"chunk-(\d{5})")
|
| 92 |
+
_DRIVE_RE = re.compile(r"[A-Za-z]:")
|
| 93 |
+
|
| 94 |
+
Scalar = str | int | float | None
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def item_segment_id(source_id: str, item_id: str) -> str:
|
| 98 |
+
"""Content-addressed id of an item: ``sha1(nfc(source_id) U+001F nfc(item_id))[:16]``.
|
| 99 |
+
|
| 100 |
+
Raises:
|
| 101 |
+
ValueError: an empty id, or one containing U+001F (the separator).
|
| 102 |
+
"""
|
| 103 |
+
source_id, item_id = nfc(source_id), nfc(item_id)
|
| 104 |
+
for name, value in (("source_id", source_id), ("item_id", item_id)):
|
| 105 |
+
if not value or "\x1f" in value:
|
| 106 |
+
raise ValueError(f"invalid {name} {value!r}")
|
| 107 |
+
return hashlib.sha1(f"{source_id}\x1f{item_id}".encode()).hexdigest()[:16]
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def safe_relative_path(value: object) -> str | None:
|
| 111 |
+
"""``value`` if it is a plain relative ``/``-separated path inside a folder, else ``None``
|
| 112 |
+
(not a string, empty, absolute, drive letter, backslash, ``..`` component)."""
|
| 113 |
+
if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
|
| 114 |
+
return None
|
| 115 |
+
path = PurePosixPath(value)
|
| 116 |
+
if path.is_absolute() or _DRIVE_RE.match(value) or ".." in path.parts:
|
| 117 |
+
return None
|
| 118 |
+
return nfc(value)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
_HUB_RESOLVE_RE = re.compile(r"^https://huggingface\.co/datasets/[^/]+/[^/]+/resolve/[^/]+/(.+)$")
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def hub_url_to_path(value: object) -> object:
|
| 125 |
+
"""A Hub ``.../datasets/<owner>/<name>/resolve/<rev>/<path>`` URL -> ``<path>``
|
| 126 |
+
(URL-decoded); anything else is returned unchanged.
|
| 127 |
+
|
| 128 |
+
tr_full's ``data.jsonl`` names its audio by full resolve URLs of the dataset it was
|
| 129 |
+
copied from (Codyfederer/tr-full-dataset); the same relative paths exist in the pinned
|
| 130 |
+
repository, so only the path part is used.
|
| 131 |
+
"""
|
| 132 |
+
if isinstance(value, str):
|
| 133 |
+
match = _HUB_RESOLVE_RE.match(value)
|
| 134 |
+
if match:
|
| 135 |
+
return unquote(match.group(1))
|
| 136 |
+
return value
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@dataclass(frozen=True)
|
| 140 |
+
class Item:
|
| 141 |
+
"""One selected corpus item (a line of ``select/manifest.jsonl``).
|
| 142 |
+
|
| 143 |
+
Attributes:
|
| 144 |
+
source_id: ``configs/sources.yaml`` id of the item's source.
|
| 145 |
+
item_id: The source's own id (alania ``id``, tr_full wav path, wavdir ``file``).
|
| 146 |
+
segment_id: :func:`item_segment_id` of the two.
|
| 147 |
+
unit_id: Work unit the item came from (needed to fetch its raw audio again).
|
| 148 |
+
text: Model text (:func:`~nida.data.importer.text.normalise_transcript`).
|
| 149 |
+
group: Split group (speaker / voice), prefixed per source in the config.
|
| 150 |
+
emotion: Emotion id or ``None``.
|
| 151 |
+
raw_duration_ms: Duration of the raw audio from its header; ``None`` until the
|
| 152 |
+
select stage has measured it.
|
| 153 |
+
extra: JSON scalars worth keeping (QC scores, raw emotion label, ...).
|
| 154 |
+
"""
|
| 155 |
+
|
| 156 |
+
source_id: str
|
| 157 |
+
item_id: str
|
| 158 |
+
segment_id: str
|
| 159 |
+
unit_id: str
|
| 160 |
+
text: str
|
| 161 |
+
group: str
|
| 162 |
+
emotion: str | None
|
| 163 |
+
raw_duration_ms: int | None = None
|
| 164 |
+
extra: Mapping[str, Scalar] = field(default_factory=dict)
|
| 165 |
+
|
| 166 |
+
def to_obj(self) -> dict[str, object]:
|
| 167 |
+
return {
|
| 168 |
+
"source_id": self.source_id,
|
| 169 |
+
"item_id": self.item_id,
|
| 170 |
+
"segment_id": self.segment_id,
|
| 171 |
+
"unit_id": self.unit_id,
|
| 172 |
+
"text": self.text,
|
| 173 |
+
"group": self.group,
|
| 174 |
+
"emotion": self.emotion,
|
| 175 |
+
"raw_duration_ms": self.raw_duration_ms,
|
| 176 |
+
"extra": dict(self.extra),
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
@classmethod
|
| 180 |
+
def from_obj(cls, obj: object, where: str) -> Item:
|
| 181 |
+
"""Inverse of :meth:`to_obj`.
|
| 182 |
+
|
| 183 |
+
Raises:
|
| 184 |
+
ValueError: on a missing/extra key or a wrong type (the message names ``where``).
|
| 185 |
+
"""
|
| 186 |
+
keys = {
|
| 187 |
+
"source_id",
|
| 188 |
+
"item_id",
|
| 189 |
+
"segment_id",
|
| 190 |
+
"unit_id",
|
| 191 |
+
"text",
|
| 192 |
+
"group",
|
| 193 |
+
"emotion",
|
| 194 |
+
"raw_duration_ms",
|
| 195 |
+
"extra",
|
| 196 |
+
}
|
| 197 |
+
if not isinstance(obj, dict) or set(obj) != keys:
|
| 198 |
+
raise ValueError(f"{where}: an item needs exactly the keys {sorted(keys)}")
|
| 199 |
+
for key in ("source_id", "item_id", "segment_id", "unit_id", "text", "group"):
|
| 200 |
+
if not isinstance(obj[key], str) or not obj[key]:
|
| 201 |
+
raise ValueError(f"{where}: {key} must be a non-empty string")
|
| 202 |
+
emotion, duration, extra = obj["emotion"], obj["raw_duration_ms"], obj["extra"]
|
| 203 |
+
if emotion is not None and not isinstance(emotion, str):
|
| 204 |
+
raise ValueError(f"{where}: emotion must be a string or null")
|
| 205 |
+
if duration is not None and (isinstance(duration, bool) or not isinstance(duration, int)):
|
| 206 |
+
raise ValueError(f"{where}: raw_duration_ms must be an int or null")
|
| 207 |
+
if not isinstance(extra, dict) or not all(
|
| 208 |
+
isinstance(k, str) and (v is None or isinstance(v, str | int | float))
|
| 209 |
+
for k, v in extra.items()
|
| 210 |
+
):
|
| 211 |
+
raise ValueError(f"{where}: extra must map strings to JSON scalars")
|
| 212 |
+
return cls(
|
| 213 |
+
source_id=obj["source_id"],
|
| 214 |
+
item_id=obj["item_id"],
|
| 215 |
+
segment_id=obj["segment_id"],
|
| 216 |
+
unit_id=obj["unit_id"],
|
| 217 |
+
text=obj["text"],
|
| 218 |
+
group=obj["group"],
|
| 219 |
+
emotion=emotion,
|
| 220 |
+
raw_duration_ms=duration,
|
| 221 |
+
extra=extra,
|
| 222 |
+
)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
@dataclass(frozen=True)
|
| 226 |
+
class UnitListing:
|
| 227 |
+
"""All work units of a corpus at one pinned state.
|
| 228 |
+
|
| 229 |
+
Attributes:
|
| 230 |
+
revision: Pinned commit sha (Hub sources) or ``None`` (wavdir).
|
| 231 |
+
unit_ids: Every unit id in processing order.
|
| 232 |
+
content_sha1: sha1 of the wavdir ``manifest.jsonl`` (``None`` for Hub sources).
|
| 233 |
+
"""
|
| 234 |
+
|
| 235 |
+
revision: str | None
|
| 236 |
+
unit_ids: tuple[str, ...]
|
| 237 |
+
content_sha1: str | None = None
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class SourceAdapter(Protocol):
|
| 241 |
+
"""What the select stage needs from a corpus (see the module docstring)."""
|
| 242 |
+
|
| 243 |
+
#: ``True`` if the raw audio is a cache copy the audio stage may delete after conversion.
|
| 244 |
+
ephemeral_raw: bool
|
| 245 |
+
|
| 246 |
+
@property
|
| 247 |
+
def source_ids(self) -> frozenset[str]:
|
| 248 |
+
"""``sources.yaml`` ids this run reads (must all be ``approved``)."""
|
| 249 |
+
...
|
| 250 |
+
|
| 251 |
+
def list_units(self) -> UnitListing:
|
| 252 |
+
"""Pin the corpus state (network access for Hub sources) and list its units."""
|
| 253 |
+
...
|
| 254 |
+
|
| 255 |
+
def iter_unit(
|
| 256 |
+
self,
|
| 257 |
+
unit_id: str,
|
| 258 |
+
only: Set[str] | None = None,
|
| 259 |
+
exclude: Mapping[str, str] | None = None,
|
| 260 |
+
) -> Generator[Item | QuarantineRecord, None, None]:
|
| 261 |
+
"""Selected items and dropped rows of one unit; raw audio is placed in the cache before
|
| 262 |
+
an item is yielded. With ``only`` (segment ids) just those items are yielded again
|
| 263 |
+
(to restore lost raw audio); everything else is skipped without a record.
|
| 264 |
+
|
| 265 |
+
``exclude`` maps the segment ids selected by earlier units to their unit id. A row
|
| 266 |
+
whose segment id is in it, or was already yielded by this unit, is quarantined as
|
| 267 |
+
``duplicate_item`` *before* any raw audio is written, so the first item's audio is
|
| 268 |
+
never overwritten."""
|
| 269 |
+
...
|
| 270 |
+
|
| 271 |
+
def raw_path(self, item: Item) -> Path:
|
| 272 |
+
"""Where the raw audio of ``item`` is (or will be put by :meth:`iter_unit`)."""
|
| 273 |
+
...
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _drop(segment_id: str, item_id: str, reason: str, **details: object) -> QuarantineRecord:
|
| 277 |
+
return QuarantineRecord(segment_id, STAGE, reason, {"item_id": item_id, **details})
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _bad_row(
|
| 281 |
+
unit_id: str, position: int, raw: str, reason: str, **details: object
|
| 282 |
+
) -> QuarantineRecord:
|
| 283 |
+
"""Quarantine record of a row that has no usable id: the key is derived from the unit id
|
| 284 |
+
and the row's position in the (pinned, immutable) unit."""
|
| 285 |
+
key = hashlib.sha1(f"{unit_id}\x1f{position}".encode()).hexdigest()[:16]
|
| 286 |
+
return _drop(key, f"{unit_id}#{position}", reason, row=raw[:DETAIL_TEXT_CHARS], **details)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _first_seen(
|
| 290 |
+
first: dict[str, str], sid: str, item_id: str, unit_id: str
|
| 291 |
+
) -> QuarantineRecord | None:
|
| 292 |
+
"""Record ``sid`` as selected by ``unit_id``, or return the ``duplicate_item`` record if
|
| 293 |
+
an earlier row (of this or an earlier unit) already claimed it."""
|
| 294 |
+
earlier = first.get(sid)
|
| 295 |
+
if earlier is not None:
|
| 296 |
+
return _drop(sid, item_id, "duplicate_item", unit_id=unit_id, first_unit_id=earlier)
|
| 297 |
+
first[sid] = unit_id
|
| 298 |
+
return None
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def _finite(value: object) -> float | None:
|
| 302 |
+
"""``value`` as a finite float, else ``None`` (missing, non-numeric, NaN, inf)."""
|
| 303 |
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
| 304 |
+
return None
|
| 305 |
+
return float(value) if math.isfinite(value) else None
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def _text_drop(segment_id: str, item_id: str, err: TextRejected) -> QuarantineRecord:
|
| 309 |
+
return _drop(segment_id, item_id, err.reason, **err.details)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
# ------------------------------------------------------------------------------ alania
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
class AlaniaAdapter:
|
| 316 |
+
"""``cloud0day3/alania-synthetic-speech-tr``, config ``cc-by`` only (parquet shards).
|
| 317 |
+
|
| 318 |
+
Units are the parquet files matching ``alania.parquet_glob``, sorted by name; rows are
|
| 319 |
+
read in file order. One shard at a time is downloaded, the raw FLAC bytes of the selected
|
| 320 |
+
rows are written to ``<cache>/raw/alania/<segment_id>.flac`` and the parquet file is deleted
|
| 321 |
+
when the unit is finished or abandoned.
|
| 322 |
+
"""
|
| 323 |
+
|
| 324 |
+
ephemeral_raw = True
|
| 325 |
+
|
| 326 |
+
def __init__(
|
| 327 |
+
self, cfg: ImportConfig, cache: Path, hub: HubClient, revision: str | None
|
| 328 |
+
) -> None:
|
| 329 |
+
self._cfg = cfg
|
| 330 |
+
self._cache = cache
|
| 331 |
+
self._hub = hub
|
| 332 |
+
self.revision = revision
|
| 333 |
+
|
| 334 |
+
@property
|
| 335 |
+
def source_ids(self) -> frozenset[str]:
|
| 336 |
+
return frozenset({ALANIA_SOURCE_ID})
|
| 337 |
+
|
| 338 |
+
def list_units(self) -> UnitListing:
|
| 339 |
+
repo = self._cfg.alania.repo_id
|
| 340 |
+
if self.revision is None:
|
| 341 |
+
self.revision = self._hub.resolve_revision(repo)
|
| 342 |
+
files = self._hub.list_files(repo, self.revision)
|
| 343 |
+
units = sorted(f for f in files if fnmatch.fnmatchcase(f, self._cfg.alania.parquet_glob))
|
| 344 |
+
if not units:
|
| 345 |
+
raise HubError(
|
| 346 |
+
f"{repo}@{self.revision}: no file matches {self._cfg.alania.parquet_glob!r}; "
|
| 347 |
+
f"first files: {sorted(files)[:5]}"
|
| 348 |
+
)
|
| 349 |
+
return UnitListing(self.revision, tuple(units))
|
| 350 |
+
|
| 351 |
+
def raw_path(self, item: Item) -> Path:
|
| 352 |
+
return self._cache / "raw" / "alania" / f"{item.segment_id}.flac"
|
| 353 |
+
|
| 354 |
+
def iter_unit(
|
| 355 |
+
self,
|
| 356 |
+
unit_id: str,
|
| 357 |
+
only: Set[str] | None = None,
|
| 358 |
+
exclude: Mapping[str, str] | None = None,
|
| 359 |
+
) -> Generator[Item | QuarantineRecord, None, None]:
|
| 360 |
+
import pyarrow.parquet as pq # type: ignore[import-untyped] # no stubs
|
| 361 |
+
|
| 362 |
+
if self.revision is None:
|
| 363 |
+
raise HubError("alania: the revision is not pinned; call list_units() first")
|
| 364 |
+
path = self._hub.download(
|
| 365 |
+
self._cfg.alania.repo_id, self.revision, unit_id, self._cache / "hub" / "alania"
|
| 366 |
+
)
|
| 367 |
+
try:
|
| 368 |
+
with pq.ParquetFile(path) as shard:
|
| 369 |
+
missing = [c for c in ALANIA_COLUMNS if c not in shard.schema_arrow.names]
|
| 370 |
+
if missing:
|
| 371 |
+
raise ValueError(f"{unit_id}: parquet columns missing: {missing}")
|
| 372 |
+
position = 0
|
| 373 |
+
first = dict(exclude or {})
|
| 374 |
+
for batch in shard.iter_batches(
|
| 375 |
+
batch_size=self._cfg.alania.batch_rows, columns=list(ALANIA_COLUMNS)
|
| 376 |
+
):
|
| 377 |
+
for row in _rows(batch):
|
| 378 |
+
event = self._row(unit_id, position, row, only, first)
|
| 379 |
+
position += 1
|
| 380 |
+
if event is not None:
|
| 381 |
+
yield event
|
| 382 |
+
finally:
|
| 383 |
+
path.unlink(missing_ok=True)
|
| 384 |
+
|
| 385 |
+
def _row(
|
| 386 |
+
self,
|
| 387 |
+
unit_id: str,
|
| 388 |
+
position: int,
|
| 389 |
+
row: Mapping[str, object],
|
| 390 |
+
only: Set[str] | None,
|
| 391 |
+
first: dict[str, str],
|
| 392 |
+
) -> Item | QuarantineRecord | None:
|
| 393 |
+
cfg = self._cfg
|
| 394 |
+
alania = cfg.alania
|
| 395 |
+
item_id = row["id"]
|
| 396 |
+
if not isinstance(item_id, str) or not item_id.strip() or "\x1f" in item_id:
|
| 397 |
+
return _bad_row(unit_id, position, repr(item_id), "bad_row", problem="id")
|
| 398 |
+
sid = item_segment_id(ALANIA_SOURCE_ID, item_id)
|
| 399 |
+
if only is not None and sid not in only:
|
| 400 |
+
return None
|
| 401 |
+
license_ = row["license"]
|
| 402 |
+
if not isinstance(license_, str) or not license_.startswith(alania.license_prefix):
|
| 403 |
+
return _drop(sid, item_id, "license", license=license_)
|
| 404 |
+
utmos, cer = _finite(row["qc_utmos"]), _finite(row["qc_cer"])
|
| 405 |
+
if utmos is None or cer is None:
|
| 406 |
+
return _drop(sid, item_id, "missing_qc", qc_utmos=utmos, qc_cer=cer)
|
| 407 |
+
if utmos < alania.min_utmos:
|
| 408 |
+
return _drop(sid, item_id, "low_utmos", qc_utmos=utmos)
|
| 409 |
+
if cer > alania.max_cer:
|
| 410 |
+
return _drop(sid, item_id, "high_cer", qc_cer=cer)
|
| 411 |
+
seconds = _finite(row["duration"])
|
| 412 |
+
if seconds is None:
|
| 413 |
+
return _drop(sid, item_id, "bad_duration", duration=repr(row["duration"]))
|
| 414 |
+
if seconds * 1000 < cfg.min_duration_ms:
|
| 415 |
+
return _drop(sid, item_id, "too_short", duration_s=seconds)
|
| 416 |
+
if seconds * 1000 > cfg.max_duration_ms:
|
| 417 |
+
return _drop(sid, item_id, "too_long", duration_s=seconds)
|
| 418 |
+
voice = row["voice_id"]
|
| 419 |
+
if not isinstance(voice, str | int) or isinstance(voice, bool) or not str(voice).strip():
|
| 420 |
+
return _drop(sid, item_id, "no_group", voice_id=repr(voice))
|
| 421 |
+
spoken = row["text_spoken"]
|
| 422 |
+
try:
|
| 423 |
+
text = normalise_transcript(spoken if isinstance(spoken, str) else "", cfg.char_map)
|
| 424 |
+
except TextRejected as err:
|
| 425 |
+
return _text_drop(sid, item_id, err)
|
| 426 |
+
audio = row["audio"]
|
| 427 |
+
data = audio.get("bytes") if isinstance(audio, dict) else None
|
| 428 |
+
if not isinstance(data, bytes) or not data:
|
| 429 |
+
return _drop(sid, item_id, "audio_missing")
|
| 430 |
+
item = Item(
|
| 431 |
+
source_id=ALANIA_SOURCE_ID,
|
| 432 |
+
item_id=item_id,
|
| 433 |
+
segment_id=sid,
|
| 434 |
+
unit_id=unit_id,
|
| 435 |
+
text=text,
|
| 436 |
+
group=nfc(alania.group_prefix + str(voice).strip()),
|
| 437 |
+
emotion=None,
|
| 438 |
+
extra={
|
| 439 |
+
"voice_id": str(voice),
|
| 440 |
+
"gender": _scalar(row["gender"]),
|
| 441 |
+
"channel_preset": _scalar(row["channel_preset"]),
|
| 442 |
+
"qc_utmos": utmos,
|
| 443 |
+
"qc_cer": cer,
|
| 444 |
+
},
|
| 445 |
+
)
|
| 446 |
+
duplicate = _first_seen(first, sid, item_id, unit_id)
|
| 447 |
+
if duplicate is not None:
|
| 448 |
+
return duplicate
|
| 449 |
+
atomic_write_bytes(self.raw_path(item), data)
|
| 450 |
+
return item
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def _scalar(value: object) -> Scalar:
|
| 454 |
+
"""``value`` if it is a JSON scalar, else its ``str`` (QC metadata only)."""
|
| 455 |
+
if value is None or (isinstance(value, str | int | float) and not isinstance(value, bool)):
|
| 456 |
+
return value
|
| 457 |
+
return str(value)
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
def _rows(batch: pa.RecordBatch) -> list[dict[str, object]]:
|
| 461 |
+
rows: list[dict[str, object]] = batch.to_pylist()
|
| 462 |
+
return rows
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
# ------------------------------------------------------------------------------ chunks
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def _chunk_ids(num_rows: int, unit_rows: int) -> tuple[str, ...]:
|
| 469 |
+
count = -(-num_rows // unit_rows)
|
| 470 |
+
return tuple(f"chunk-{k:05d}" for k in range(count))
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def _chunk_slice(unit_id: str, unit_rows: int) -> slice:
|
| 474 |
+
match = _CHUNK_RE.fullmatch(unit_id)
|
| 475 |
+
if match is None:
|
| 476 |
+
raise ValueError(f"not a chunk unit id: {unit_id!r}")
|
| 477 |
+
start = int(match.group(1)) * unit_rows
|
| 478 |
+
return slice(start, start + unit_rows)
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def _json_lines(path: Path) -> list[str]:
|
| 482 |
+
"""Non-empty lines of a JSONL file (split on ``\\n`` only: U+2028 stays inside strings)."""
|
| 483 |
+
return [line for line in path.read_text(encoding="utf-8").split("\n") if line.strip()]
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
def _parse_object(line: str) -> dict[str, object] | None:
|
| 487 |
+
try:
|
| 488 |
+
obj = json.loads(line)
|
| 489 |
+
except json.JSONDecodeError:
|
| 490 |
+
return None
|
| 491 |
+
return obj if isinstance(obj, dict) else None
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
# ------------------------------------------------------------------------------ tr_full
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
class TrFullAdapter:
|
| 498 |
+
"""``Taklaxbr/tr-full-dataset``: ``data.jsonl`` rows and the wavs they name.
|
| 499 |
+
|
| 500 |
+
Units are chunks of ``tr_full.unit_rows`` rows in file order. A row whose ``language`` is
|
| 501 |
+
not ``tr_full.language`` is quarantined (``language``) before its wav is fetched;
|
| 502 |
+
the wavs of the remaining rows of a unit are downloaded one by one (threaded) into
|
| 503 |
+
``<cache>/raw/tr_full/``.
|
| 504 |
+
"""
|
| 505 |
+
|
| 506 |
+
ephemeral_raw = True
|
| 507 |
+
|
| 508 |
+
def __init__(
|
| 509 |
+
self, cfg: ImportConfig, cache: Path, hub: HubClient, revision: str | None
|
| 510 |
+
) -> None:
|
| 511 |
+
self._cfg = cfg
|
| 512 |
+
self._cache = cache
|
| 513 |
+
self._hub = hub
|
| 514 |
+
self.revision = revision
|
| 515 |
+
self._lines_cache: list[str] | None = None
|
| 516 |
+
|
| 517 |
+
@property
|
| 518 |
+
def source_ids(self) -> frozenset[str]:
|
| 519 |
+
return frozenset({TR_FULL_SOURCE_ID})
|
| 520 |
+
|
| 521 |
+
def _data_lines(self) -> list[str]:
|
| 522 |
+
"""The non-empty lines of ``data.jsonl`` (downloaded and read once per adapter)."""
|
| 523 |
+
if self._lines_cache is None:
|
| 524 |
+
path = self._hub.download(
|
| 525 |
+
self._cfg.tr_full.repo_id,
|
| 526 |
+
self._revision(),
|
| 527 |
+
self._cfg.tr_full.data_file,
|
| 528 |
+
self._cache / "hub" / "tr_full",
|
| 529 |
+
)
|
| 530 |
+
self._lines_cache = _json_lines(path)
|
| 531 |
+
return self._lines_cache
|
| 532 |
+
|
| 533 |
+
def list_units(self) -> UnitListing:
|
| 534 |
+
if self.revision is None:
|
| 535 |
+
self.revision = self._hub.resolve_revision(self._cfg.tr_full.repo_id)
|
| 536 |
+
lines = self._data_lines()
|
| 537 |
+
return UnitListing(self.revision, _chunk_ids(len(lines), self._cfg.tr_full.unit_rows))
|
| 538 |
+
|
| 539 |
+
def raw_path(self, item: Item) -> Path:
|
| 540 |
+
return self._cache / "raw" / "tr_full" / item.item_id
|
| 541 |
+
|
| 542 |
+
def iter_unit(
|
| 543 |
+
self,
|
| 544 |
+
unit_id: str,
|
| 545 |
+
only: Set[str] | None = None,
|
| 546 |
+
exclude: Mapping[str, str] | None = None,
|
| 547 |
+
) -> Generator[Item | QuarantineRecord, None, None]:
|
| 548 |
+
cfg = self._cfg
|
| 549 |
+
window = _chunk_slice(unit_id, cfg.tr_full.unit_rows)
|
| 550 |
+
lines = self._data_lines()[window]
|
| 551 |
+
base = window.start
|
| 552 |
+
candidates: list[Item] = []
|
| 553 |
+
first = dict(exclude or {})
|
| 554 |
+
for offset, line in enumerate(lines):
|
| 555 |
+
position = base + offset
|
| 556 |
+
row = _parse_object(line)
|
| 557 |
+
if row is None:
|
| 558 |
+
yield _bad_row(unit_id, position, line, "bad_row", problem="not a JSON object")
|
| 559 |
+
continue
|
| 560 |
+
made = self._row(unit_id, position, line, row, only, first)
|
| 561 |
+
if isinstance(made, QuarantineRecord):
|
| 562 |
+
yield made
|
| 563 |
+
elif made is not None:
|
| 564 |
+
candidates.append(made)
|
| 565 |
+
wanted = [c.item_id for c in candidates if not self.raw_path(c).is_file()]
|
| 566 |
+
self._hub.download_many(
|
| 567 |
+
cfg.tr_full.repo_id,
|
| 568 |
+
self._revision(),
|
| 569 |
+
wanted,
|
| 570 |
+
self._cache / "raw" / "tr_full",
|
| 571 |
+
)
|
| 572 |
+
for item in candidates:
|
| 573 |
+
if self.raw_path(item).is_file():
|
| 574 |
+
yield item
|
| 575 |
+
else:
|
| 576 |
+
yield _drop(item.segment_id, item.item_id, "audio_missing")
|
| 577 |
+
|
| 578 |
+
def _revision(self) -> str:
|
| 579 |
+
if self.revision is None:
|
| 580 |
+
raise HubError("tr_full: the revision is not pinned; call list_units() first")
|
| 581 |
+
return self.revision
|
| 582 |
+
|
| 583 |
+
def _row(
|
| 584 |
+
self,
|
| 585 |
+
unit_id: str,
|
| 586 |
+
position: int,
|
| 587 |
+
line: str,
|
| 588 |
+
row: Mapping[str, object],
|
| 589 |
+
only: Set[str] | None,
|
| 590 |
+
first: dict[str, str],
|
| 591 |
+
) -> Item | QuarantineRecord | None:
|
| 592 |
+
cfg = self._cfg
|
| 593 |
+
rel = safe_relative_path(hub_url_to_path(row.get("audio")))
|
| 594 |
+
if rel is None:
|
| 595 |
+
return _bad_row(unit_id, position, line, "bad_row", problem="audio path")
|
| 596 |
+
sid = item_segment_id(TR_FULL_SOURCE_ID, rel)
|
| 597 |
+
if only is not None and sid not in only:
|
| 598 |
+
return None
|
| 599 |
+
language = row.get("language")
|
| 600 |
+
if not isinstance(language, str) or language.strip() != cfg.tr_full.language:
|
| 601 |
+
return _drop(sid, rel, "language", language=language)
|
| 602 |
+
speaker, text_raw = row.get("speaker_id"), row.get("text")
|
| 603 |
+
if (
|
| 604 |
+
not isinstance(speaker, str | int)
|
| 605 |
+
or isinstance(speaker, bool)
|
| 606 |
+
or not str(speaker).strip()
|
| 607 |
+
):
|
| 608 |
+
return _drop(sid, rel, "no_group", speaker_id=repr(speaker))
|
| 609 |
+
try:
|
| 610 |
+
text = normalise_transcript(text_raw if isinstance(text_raw, str) else "", cfg.char_map)
|
| 611 |
+
except TextRejected as err:
|
| 612 |
+
return _text_drop(sid, rel, err)
|
| 613 |
+
label = row.get("emotion")
|
| 614 |
+
emotion = map_emotion(label, cfg.emotion_map)
|
| 615 |
+
duplicate = _first_seen(first, sid, rel, unit_id)
|
| 616 |
+
if duplicate is not None:
|
| 617 |
+
return duplicate
|
| 618 |
+
return Item(
|
| 619 |
+
source_id=TR_FULL_SOURCE_ID,
|
| 620 |
+
item_id=rel,
|
| 621 |
+
segment_id=sid,
|
| 622 |
+
unit_id=unit_id,
|
| 623 |
+
text=text,
|
| 624 |
+
group=nfc(cfg.tr_full.group_prefix + str(speaker).strip()),
|
| 625 |
+
emotion=emotion,
|
| 626 |
+
extra={"speaker_id": str(speaker), "emotion_label": _scalar(label)},
|
| 627 |
+
)
|
| 628 |
+
|
| 629 |
+
|
| 630 |
+
# ------------------------------------------------------------------------------ wavdir
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
class WavdirAdapter:
|
| 634 |
+
"""A folder with ``manifest.jsonl`` ``{file, text, group, source, emotion?}`` lines.
|
| 635 |
+
|
| 636 |
+
``file`` is relative to the folder; ``source`` is the ``sources.yaml`` id of whatever
|
| 637 |
+
produced the audio (it must be ``approved``; :attr:`source_ids` lists every one named in
|
| 638 |
+
the manifest). Units are chunks of ``wavdir.unit_rows`` lines. The wavs stay where they
|
| 639 |
+
are: they are read in place and never deleted.
|
| 640 |
+
"""
|
| 641 |
+
|
| 642 |
+
ephemeral_raw = False
|
| 643 |
+
|
| 644 |
+
def __init__(self, cfg: ImportConfig, folder: Path) -> None:
|
| 645 |
+
self._cfg = cfg
|
| 646 |
+
self._folder = folder
|
| 647 |
+
self._manifest = folder / "manifest.jsonl"
|
| 648 |
+
self._lines_cache: list[str] | None = None
|
| 649 |
+
|
| 650 |
+
def _lines(self) -> list[str]:
|
| 651 |
+
if self._lines_cache is None:
|
| 652 |
+
if not self._manifest.is_file():
|
| 653 |
+
raise FileNotFoundError(f"{self._manifest} does not exist")
|
| 654 |
+
self._lines_cache = _json_lines(self._manifest)
|
| 655 |
+
return self._lines_cache
|
| 656 |
+
|
| 657 |
+
@property
|
| 658 |
+
def source_ids(self) -> frozenset[str]:
|
| 659 |
+
found: set[str] = set()
|
| 660 |
+
for line in self._lines():
|
| 661 |
+
row = _parse_object(line)
|
| 662 |
+
source = row.get("source") if row is not None else None
|
| 663 |
+
if isinstance(source, str) and source.strip():
|
| 664 |
+
found.add(source.strip())
|
| 665 |
+
return frozenset(found)
|
| 666 |
+
|
| 667 |
+
def list_units(self) -> UnitListing:
|
| 668 |
+
digest = hashlib.sha1(self._manifest.read_bytes()).hexdigest()
|
| 669 |
+
return UnitListing(None, _chunk_ids(len(self._lines()), self._cfg.wavdir.unit_rows), digest)
|
| 670 |
+
|
| 671 |
+
def raw_path(self, item: Item) -> Path:
|
| 672 |
+
return self._folder / item.item_id
|
| 673 |
+
|
| 674 |
+
def iter_unit(
|
| 675 |
+
self,
|
| 676 |
+
unit_id: str,
|
| 677 |
+
only: Set[str] | None = None,
|
| 678 |
+
exclude: Mapping[str, str] | None = None,
|
| 679 |
+
) -> Generator[Item | QuarantineRecord, None, None]:
|
| 680 |
+
window = _chunk_slice(unit_id, self._cfg.wavdir.unit_rows)
|
| 681 |
+
first = dict(exclude or {})
|
| 682 |
+
for offset, line in enumerate(self._lines()[window]):
|
| 683 |
+
event = self._row(unit_id, window.start + offset, line, only, first)
|
| 684 |
+
if event is not None:
|
| 685 |
+
yield event
|
| 686 |
+
|
| 687 |
+
def _row(
|
| 688 |
+
self,
|
| 689 |
+
unit_id: str,
|
| 690 |
+
position: int,
|
| 691 |
+
line: str,
|
| 692 |
+
only: Set[str] | None,
|
| 693 |
+
first: dict[str, str],
|
| 694 |
+
) -> Item | QuarantineRecord | None:
|
| 695 |
+
cfg = self._cfg
|
| 696 |
+
row = _parse_object(line)
|
| 697 |
+
if row is None:
|
| 698 |
+
return _bad_row(unit_id, position, line, "bad_row", problem="not a JSON object")
|
| 699 |
+
rel = safe_relative_path(row.get("file"))
|
| 700 |
+
source, group, text_raw = row.get("source"), row.get("group"), row.get("text")
|
| 701 |
+
if rel is None or not isinstance(source, str) or not source.strip() or "\x1f" in source:
|
| 702 |
+
return _bad_row(unit_id, position, line, "bad_row", problem="file or source")
|
| 703 |
+
source = source.strip()
|
| 704 |
+
sid = item_segment_id(source, rel)
|
| 705 |
+
if only is not None and sid not in only:
|
| 706 |
+
return None
|
| 707 |
+
if not isinstance(group, str) or not group.strip():
|
| 708 |
+
return _drop(sid, rel, "no_group")
|
| 709 |
+
try:
|
| 710 |
+
text = normalise_transcript(text_raw if isinstance(text_raw, str) else "", cfg.char_map)
|
| 711 |
+
except TextRejected as err:
|
| 712 |
+
return _text_drop(sid, rel, err)
|
| 713 |
+
if not (self._folder / rel).is_file():
|
| 714 |
+
return _drop(sid, rel, "audio_missing")
|
| 715 |
+
duplicate = _first_seen(first, sid, rel, unit_id)
|
| 716 |
+
if duplicate is not None:
|
| 717 |
+
return duplicate
|
| 718 |
+
label = row.get("emotion")
|
| 719 |
+
return Item(
|
| 720 |
+
source_id=source,
|
| 721 |
+
item_id=rel,
|
| 722 |
+
segment_id=sid,
|
| 723 |
+
unit_id=unit_id,
|
| 724 |
+
text=text,
|
| 725 |
+
group=nfc(cfg.wavdir.group_prefix + group.strip()),
|
| 726 |
+
emotion=map_emotion(label, cfg.emotion_map),
|
| 727 |
+
extra={"emotion_label": _scalar(label)},
|
| 728 |
+
)
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def make_adapter(
|
| 732 |
+
source: str,
|
| 733 |
+
cfg: ImportConfig,
|
| 734 |
+
cache: Path,
|
| 735 |
+
*,
|
| 736 |
+
hub: HubClient | None,
|
| 737 |
+
revision: str | None,
|
| 738 |
+
wavdir: Path | None,
|
| 739 |
+
opener: fetch.Opener | None = None,
|
| 740 |
+
) -> SourceAdapter:
|
| 741 |
+
"""The adapter of ``source`` (one of :data:`SOURCES`).
|
| 742 |
+
|
| 743 |
+
Args:
|
| 744 |
+
source: Importer source name.
|
| 745 |
+
cfg: Import config (loaded for that source).
|
| 746 |
+
cache: Local scratch directory.
|
| 747 |
+
hub: Hub client (``alania``, ``tr_full``, ``issai_tsc``).
|
| 748 |
+
revision: Pin: a Hub commit sha, or the sha256 of the ``mediaspeech_tr`` archive.
|
| 749 |
+
wavdir: ``--source-dir`` (``wavdir``: its folder; ``common_voice_tr``: a Common Voice
|
| 750 |
+
release folder or tarball).
|
| 751 |
+
opener: HTTP access of ``mediaspeech_tr`` (default: ``urllib``); tests inject one.
|
| 752 |
+
|
| 753 |
+
Raises:
|
| 754 |
+
ValueError: unknown source, a Hub source without a ``hub``, a local source without
|
| 755 |
+
``wavdir``.
|
| 756 |
+
"""
|
| 757 |
+
if source in HUB_SOURCES and hub is None:
|
| 758 |
+
raise ValueError(f"source {source!r} needs a Hub client")
|
| 759 |
+
if source in LOCAL_SOURCES and wavdir is None:
|
| 760 |
+
raise ValueError(f"source {source!r} needs --source-dir")
|
| 761 |
+
if source == "alania" and hub is not None:
|
| 762 |
+
return AlaniaAdapter(cfg, cache, hub, revision)
|
| 763 |
+
if source == "tr_full" and hub is not None:
|
| 764 |
+
return TrFullAdapter(cfg, cache, hub, revision)
|
| 765 |
+
if source == "wavdir" and wavdir is not None:
|
| 766 |
+
return WavdirAdapter(cfg, wavdir)
|
| 767 |
+
if source in CORPUS_SOURCES:
|
| 768 |
+
from nida.data.importer import corpora # imports this module: not at module level
|
| 769 |
+
|
| 770 |
+
if source == "issai_tsc" and hub is not None:
|
| 771 |
+
return corpora.IssaiTscAdapter(cfg, cache, hub, revision)
|
| 772 |
+
if source == "mediaspeech_tr":
|
| 773 |
+
return corpora.MediaSpeechAdapter(cfg, cache, opener, revision)
|
| 774 |
+
if source == "common_voice_tr" and wavdir is not None:
|
| 775 |
+
return corpora.CommonVoiceAdapter(cfg, cache, wavdir)
|
| 776 |
+
raise ValueError(f"unknown source {source!r}; known: {SOURCES}")
|
src/nida/data/importer/text.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Transcript and label normalisation for imported corpora.
|
| 2 |
+
|
| 3 |
+
A transcript becomes *model text*: what :func:`nida.model.vocab.encode_text` encodes and what
|
| 4 |
+
the CTC aligner is given. Rules, in order:
|
| 5 |
+
|
| 6 |
+
1. NFC (:func:`nida.data.pilot.clean_asr_text`, which also undoes ``i`` + U+0307);
|
| 7 |
+
2. ``char_map`` replacements (typographic quotes and apostrophes);
|
| 8 |
+
3. text that contains a tag character ``[ ] < >`` is *rejected* (``tag_chars``): the characters
|
| 9 |
+
are reserved for the tag grammar (docs/02-tag-spec.md), and stripping them would turn an
|
| 10 |
+
annotation such as ``[laugh]`` into the spoken word "laugh". Whitespace is collapsed to
|
| 11 |
+
single spaces;
|
| 12 |
+
4. digits are written out with :func:`nida.text.numbers.verbalize_digits`;
|
| 13 |
+
5. every remaining character must be in the model vocabulary.
|
| 14 |
+
|
| 15 |
+
Case is kept (the vocabulary lower-cases with ``tr_lower`` itself). Filler sounds are NOT
|
| 16 |
+
canonicalised (tag-spec §4: ``eee`` -> ``ee`` ...): the transcript is used as the corpus
|
| 17 |
+
writes it, a known deviation for imported text.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from collections.abc import Mapping
|
| 23 |
+
|
| 24 |
+
from nida.data.ears import en_lower
|
| 25 |
+
from nida.data.pilot import clean_asr_text
|
| 26 |
+
from nida.model.vocab import VOCAB, encode_text
|
| 27 |
+
from nida.text.numbers import verbalize_digits
|
| 28 |
+
from nida.text.turkish import tr_lower
|
| 29 |
+
|
| 30 |
+
__all__ = [
|
| 31 |
+
"DETAIL_TEXT_CHARS",
|
| 32 |
+
"REASON_EMPTY",
|
| 33 |
+
"REASON_TAG_CHARS",
|
| 34 |
+
"TAG_CHARS",
|
| 35 |
+
"TextRejected",
|
| 36 |
+
"map_emotion",
|
| 37 |
+
"normalise_transcript",
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
#: Characters reserved for the tag grammar; removed from raw corpus text.
|
| 41 |
+
TAG_CHARS = "[]<>"
|
| 42 |
+
REASON_EMPTY = "empty_text"
|
| 43 |
+
REASON_TAG_CHARS = "tag_chars"
|
| 44 |
+
REASON_NUMBER = "unverbalizable_number"
|
| 45 |
+
REASON_UNTOKENIZABLE = "untokenizable"
|
| 46 |
+
|
| 47 |
+
#: Characters of a transcript / row kept as an excerpt in a quarantine record.
|
| 48 |
+
DETAIL_TEXT_CHARS = 120
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class TextRejected(ValueError):
|
| 52 |
+
"""A transcript that cannot become model text; ``reason`` is the quarantine reason."""
|
| 53 |
+
|
| 54 |
+
def __init__(self, reason: str, details: Mapping[str, object]) -> None:
|
| 55 |
+
super().__init__(f"{reason}: {dict(details)}")
|
| 56 |
+
self.reason = reason
|
| 57 |
+
self.details = dict(details)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def normalise_transcript(raw: str, char_map: Mapping[str, str]) -> str:
|
| 61 |
+
"""Model text of a raw transcript (rules in the module docstring).
|
| 62 |
+
|
| 63 |
+
Args:
|
| 64 |
+
raw: Transcript as stored in the corpus.
|
| 65 |
+
char_map: Single-character replacements (``configs/data/import.yaml``).
|
| 66 |
+
|
| 67 |
+
Raises:
|
| 68 |
+
TextRejected: ``tag_chars`` (``details["chars"]`` lists them); ``empty_text``;
|
| 69 |
+
``unverbalizable_number`` (a digit token that :func:`verbalize_digits` cannot
|
| 70 |
+
spell); ``untokenizable`` (``details["chars"]`` lists the characters outside the
|
| 71 |
+
vocabulary).
|
| 72 |
+
"""
|
| 73 |
+
text = clean_asr_text(raw).translate({ord(k): v for k, v in char_map.items()})
|
| 74 |
+
tags = sorted({ch for ch in text if ch in TAG_CHARS})
|
| 75 |
+
if tags:
|
| 76 |
+
raise TextRejected(
|
| 77 |
+
REASON_TAG_CHARS, {"chars": "".join(tags), "text": text[:DETAIL_TEXT_CHARS]}
|
| 78 |
+
)
|
| 79 |
+
text = " ".join(text.split())
|
| 80 |
+
if not text:
|
| 81 |
+
raise TextRejected(REASON_EMPTY, {})
|
| 82 |
+
if any(ch.isdigit() for ch in text):
|
| 83 |
+
spoken = verbalize_digits(text)
|
| 84 |
+
if spoken is None:
|
| 85 |
+
raise TextRejected(REASON_NUMBER, {"text": text[:DETAIL_TEXT_CHARS]})
|
| 86 |
+
text = " ".join(spoken.split())
|
| 87 |
+
offending = sorted({ch for ch in text if tr_lower(ch) not in VOCAB})
|
| 88 |
+
if offending:
|
| 89 |
+
raise TextRejected(
|
| 90 |
+
REASON_UNTOKENIZABLE,
|
| 91 |
+
{"chars": "".join(offending), "text": text[:DETAIL_TEXT_CHARS]},
|
| 92 |
+
)
|
| 93 |
+
try:
|
| 94 |
+
ids = encode_text(text)
|
| 95 |
+
except ValueError as err: # defence in depth: something other than a character
|
| 96 |
+
raise TextRejected(REASON_UNTOKENIZABLE, {"error": str(err)}) from err
|
| 97 |
+
if not ids:
|
| 98 |
+
raise TextRejected(REASON_EMPTY, {})
|
| 99 |
+
return text
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def map_emotion(label: object, emotion_map: Mapping[str, str]) -> str | None:
|
| 103 |
+
"""Emotion id of a source label, or ``None`` when absent or not in ``emotion_map``.
|
| 104 |
+
|
| 105 |
+
Labels are English: matched after stripping and ASCII lower-casing (never ``tr_lower``,
|
| 106 |
+
which would turn ``I`` into ``ı``).
|
| 107 |
+
"""
|
| 108 |
+
if not isinstance(label, str):
|
| 109 |
+
return None
|
| 110 |
+
return emotion_map.get(en_lower(label.strip()))
|