Bilal7Dev commited on
Commit
c0bce59
·
verified ·
1 Parent(s): 1ab6bc3

Nida v0.5 (research preview): weights, 16 voices, code, model card

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +16 -0
  2. LICENSE +202 -0
  3. README.md +129 -0
  4. nida-v05-ema.pt +3 -0
  5. pyproject.toml +68 -0
  6. requirements.txt +11 -0
  7. src/nida/__init__.py +7 -0
  8. src/nida/bench/__init__.py +1 -0
  9. src/nida/bench/audiometrics.py +570 -0
  10. src/nida/bench/b1.py +622 -0
  11. src/nida/bench/b2.py +684 -0
  12. src/nida/bench/clips.py +510 -0
  13. src/nida/bench/envs.py +351 -0
  14. src/nida/bench/expresso.py +301 -0
  15. src/nida/bench/fetch.py +704 -0
  16. src/nida/bench/panel.py +817 -0
  17. src/nida/bench/phase0.py +358 -0
  18. src/nida/bench/prepare.py +450 -0
  19. src/nida/bench/protocol.py +325 -0
  20. src/nida/bench/registry.py +166 -0
  21. src/nida/bench/report.py +276 -0
  22. src/nida/bench/textmetrics.py +464 -0
  23. src/nida/bench/workers/__init__.py +7 -0
  24. src/nida/bench/workers/asr_hf.py +103 -0
  25. src/nida/bench/workers/asr_omni.py +42 -0
  26. src/nida/bench/workers/asr_qwen.py +55 -0
  27. src/nida/bench/workers/common.py +74 -0
  28. src/nida/bench/workers/quality_audiobox.py +46 -0
  29. src/nida/bench/workers/runner.py +340 -0
  30. src/nida/bench/workers/vae_voxcpm.py +146 -0
  31. src/nida/cli.py +929 -0
  32. src/nida/data/__init__.py +5 -0
  33. src/nida/data/acoustic.py +1002 -0
  34. src/nida/data/align.py +261 -0
  35. src/nida/data/asr_filter.py +283 -0
  36. src/nida/data/ears.py +1307 -0
  37. src/nida/data/enhance.py +593 -0
  38. src/nida/data/envs.py +229 -0
  39. src/nida/data/events.py +183 -0
  40. src/nida/data/fillers_ctc.py +227 -0
  41. src/nida/data/fusion.py +842 -0
  42. src/nida/data/importer/__init__.py +118 -0
  43. src/nida/data/importer/archives.py +296 -0
  44. src/nida/data/importer/audio.py +146 -0
  45. src/nida/data/importer/config.py +474 -0
  46. src/nida/data/importer/corpora.py +801 -0
  47. src/nida/data/importer/hub.py +150 -0
  48. src/nida/data/importer/pipeline.py +1326 -0
  49. src/nida/data/importer/sources.py +776 -0
  50. src/nida/data/importer/text.py +110 -0
.gitattributes CHANGED
@@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ voices/ada.wav filter=lfs diff=lfs merge=lfs -text
37
+ voices/asli.wav filter=lfs diff=lfs merge=lfs -text
38
+ voices/can.wav filter=lfs diff=lfs merge=lfs -text
39
+ voices/defne.wav filter=lfs diff=lfs merge=lfs -text
40
+ voices/duru.wav filter=lfs diff=lfs merge=lfs -text
41
+ voices/efe.wav filter=lfs diff=lfs merge=lfs -text
42
+ voices/elif.wav filter=lfs diff=lfs merge=lfs -text
43
+ voices/emre.wav filter=lfs diff=lfs merge=lfs -text
44
+ voices/haluk.wav filter=lfs diff=lfs merge=lfs -text
45
+ voices/leyla.wav filter=lfs diff=lfs merge=lfs -text
46
+ voices/mert.wav filter=lfs diff=lfs merge=lfs -text
47
+ voices/nermin.wav filter=lfs diff=lfs merge=lfs -text
48
+ voices/nihat.wav filter=lfs diff=lfs merge=lfs -text
49
+ voices/orhan.wav filter=lfs diff=lfs merge=lfs -text
50
+ voices/selim.wav filter=lfs diff=lfs merge=lfs -text
51
+ voices/su.wav filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
README.md ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: cc-by-nc-4.0
3
+ language:
4
+ - tr
5
+ pipeline_tag: text-to-speech
6
+ tags:
7
+ - text-to-speech
8
+ - turkish
9
+ - flow-matching
10
+ - zero-shot
11
+ ---
12
+
13
+ # Nida v0.5 — Turkish text-to-speech (research preview)
14
+
15
+ **Nida** (*nida*: an exclamation, a call) is an open, non-commercial Turkish text-to-speech model
16
+ trained **from scratch**. v0.5 is an early research preview: it reads Turkish clearly and stably,
17
+ including numbers, dates and abbreviations, with sixteen bundled voices. Its prosody is still flat
18
+ ("reads rather than speaks"). Expressive speech (emotion, laughter, sighs) and better voice
19
+ cloning are the goals of later versions.
20
+
21
+ > Türkçe özet: Nida, sıfırdan eğitilmiş, açık ve ticari olmayan bir Türkçe metinden-sese modelidir.
22
+ > v0.5 bir araştırma önizlemesidir: sayı, tarih ve kısaltmalar dahil Türkçeyi net ve kararlı okur,
23
+ > 16 hazır sesle gelir. Tonlaması henüz düzdür. Duygu, sözsüz sesler ve daha iyi ses klonlama
24
+ > sonraki sürümlerin hedefidir.
25
+
26
+ ## What it is
27
+
28
+ | | |
29
+ |---|---|
30
+ | Architecture | Diffusion transformer (DiT), flow matching on audio latents, explicit per-character durations with windowed Gaussian alignment |
31
+ | Parameters | 201 M |
32
+ | Audio latents | VoxCPM2 AudioVAE V2 (25 Hz × 64), decoded to 48 kHz |
33
+ | Training data | ~380 h: ~210 h Turkish YouTube speech (YODAS, CC-BY-3.0) + ~170 h synthetic multi-voice Turkish (alania, CC-BY-4.0) |
34
+ | Training | 200k steps, single GPU |
35
+ | Voices | sixteen bundled voices; a custom 3–8 s reference clip also works but is **experimental** |
36
+
37
+ ## Measured quality (v0.5)
38
+
39
+ - Intelligibility: character error rate of an ASR read-back (Qwen3-ASR) **0.1–1.2 %** on narration
40
+ and long-form text, **4.5–7 %** on short assistant sentences (numbers, proper names).
41
+ - Speaker similarity with a custom reference clip (experimental; ECAPA-TDNN cosine, generated vs.
42
+ reference): **~0.55** (the same real speaker scores ~0.8, different speakers ≤ 0.36; the audio
43
+ codec alone keeps 0.955). The voice is recognisably close but not a faithful copy.
44
+ - Long-form: stable over 2-minute paragraphs (no drift between sentences in our tests).
45
+ - **TR-Zor** (155 hard Turkish sentences: numbers, dates, times, money, units, abbreviations,
46
+ acronyms, ordinals; [ahenk-turkish-tts](https://github.com/ozguradmin/ahenk-turkish-tts),
47
+ Apache-2.0), judge Whisper large-v3-turbo, word error rate with the set's spoken form as input:
48
+ **2.36 %** (voice *deniz*, an earlier voice not in the final set; 141/155 sentences perfect),
49
+ **3.15 %** (voice *emre*, 133/155). For
50
+ reference, the same set reports Ahenk 2.54 % and Trendyol-TTS 3.98 % (both 2B VoxCPM2 fine-tunes,
51
+ measured by their authors with their own text normaliser; our evaluation normaliser and Whisper
52
+ backend differ, so treat the comparison as indicative).
53
+
54
+ ## Known limitations
55
+
56
+ - Flat, read-like prosody; little emotion even when requested.
57
+ - Custom voices (your own reference clip) are experimental: similarity is moderate and depends a lot
58
+ on the clip. The bundled voices are the supported way to use v0.5.
59
+ - Input text is normalised automatically (numbers, ordinals, dates, clock times, money, units,
60
+ common abbreviations, acronyms spelled with Turkish letter names, phone/ID numbers). Ambiguous
61
+ cases (e.g. `8.30` without context, unknown all-caps words) are reported; dangerous ones stop
62
+ generation instead of producing wrong speech. `--no-normalize` turns it off.
63
+ - Each sentence plus the reference must fit 20 seconds; sentences longer than ~96 spoken
64
+ characters are split at a comma, semicolon or colon (never inside quotes or tags).
65
+ - Trained mostly on Turkish; other languages are not supported.
66
+
67
+ ## Bundled voices
68
+
69
+ Sixteen synthetic voices ship in `voices/`: `voices.tsv` lists all of them (`<wav>\t<transcript>`),
70
+ and `voices/<name>.tsv` selects one. None of them is a recording of a real person.
71
+
72
+ | Voice | | Notes |
73
+ |---|---|---|
74
+ | leyla, duru | female | the cleanest voices; a good default |
75
+ | ada, defne, elif, nermin, su, aslı (`asli`) | female | |
76
+ | nihat | male | warm and calm |
77
+ | emre | male | lively |
78
+ | mert | male | sounds like a phone line; suits an assistant |
79
+ | can, haluk, selim, orhan, efe | male | |
80
+
81
+ can, elif, emre, haluk, nermin, nihat, selim, leyla, defne and ada come from the alania dataset
82
+ (`voice_type: designed-identity`, generated with VoxCPM2 from text descriptions). su, aslı, duru,
83
+ mert, orhan and efe were designed for this release the same way, with VoxCPM2 voice design from
84
+ a text description (no reference audio).
85
+
86
+ ## Usage
87
+
88
+ ```bash
89
+ pip install -r requirements.txt # torch (CUDA build recommended), transformers, soundfile, ...
90
+ pip install -e . # the nida package
91
+ nida say --checkpoint nida-v05-ema.pt --prompt-list voices/leyla.tsv \
92
+ --text "Merhaba! Toplantı saat 14:30'da, 3. katta." --out out --single-env --no-asr
93
+ ```
94
+
95
+ Run it from the release folder: the voice lists use paths relative to the working directory.
96
+ The audio is written to `out/ext-*.wav`, and `out/say.html` plays it. `--prompt-list
97
+ voices/voices.tsv` reads the text with all sixteen voices; `--text-file` reads a text file.
98
+
99
+ Script syntax inside `--text` / `--text-file`:
100
+
101
+ - Sentences are split on `. ! ? …`; long ones also at `, ; :`.
102
+ - `[duraksama:0.3]` inserts exactly 0.3 s of silence; `[kısa duraksama]` / `[duraksama]` let the
103
+ model pause.
104
+
105
+ The first run downloads the VoxCPM2 AudioVAE and a Turkish CTC aligner from Hugging Face. Later
106
+ runs can skip the Hub's network checks with `HF_HUB_OFFLINE=1` (useful on slow or IPv6-broken
107
+ networks). Each `nida say` call loads the models again (about a minute on an RTX 5070); the
108
+ synthesis itself takes a few seconds.
109
+
110
+ ## Related work
111
+
112
+ [FreyaTTS](https://arxiv.org/abs/2607.09530) (2026) is a Turkish text-to-speech model with a
113
+ similar recipe: a non-autoregressive flow-matching DiT on frozen AudioVAE2 latents with a Turkish
114
+ character vocabulary, 183 M parameters. Its numbers are on a different test set and are not
115
+ directly comparable with ours.
116
+
117
+ ## Responsible use
118
+
119
+ **Do not use Nida to imitate a real person's voice without their explicit consent**, to deceive,
120
+ or to impersonate anyone (including public figures). Generated audio must not be presented as a
121
+ real recording of a person. By using the model you agree to these terms in addition to the
122
+ CC-BY-NC-4.0 license (non-commercial use only).
123
+
124
+ ## Licenses and attribution
125
+
126
+ - Model weights: CC-BY-NC-4.0. Code: Apache-2.0.
127
+ - YODAS (espnet/yodas, CC-BY-3.0), alania synthetic speech (cloud0day3/alania-synthetic-speech-tr,
128
+ CC-BY-4.0), VoxCPM2 AudioVAE and voice design (openbmb/VoxCPM2, Apache-2.0), Turkish CTC aligner
129
+ (mpoyraz/wav2vec2-xls-r-300m-cv7-turkish, CC-BY-4.0).
nida-v05-ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd7ff9d7166f99b7df64a5e4ac30c43bce72868138d80bb97cca4e8ca3b939a3
3
+ size 805276081
pyproject.toml ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["hatchling>=1.24"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "nida"
7
+ version = "0.5.0"
8
+ description = "Nida: expressive Turkish text-to-speech (research, non-commercial weights)"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ # Core carries only the text normaliser base (turkificate, MIT, pure Python). torch is not a
12
+ # core dependency: Colab ships its own CUDA build; the `model` extra (ADR-0005) is for local CPU tests only.
13
+ dependencies = ["turkificate==0.2.0"]
14
+
15
+ [project.optional-dependencies]
16
+ # Phase 0 benchmarks (nida.bench); model libraries live only in isolated worker envs.
17
+ bench = ["numpy>=2.0", "scipy>=1.13", "soundfile>=0.12", "pyyaml>=6.0"]
18
+ # Phase 1 data pipeline (nida.data): Parquet stage manifests.
19
+ data = ["nida[bench]", "pyarrow>=17"]
20
+ # Phase 2 model code, local CPU tests (ADR-0005): same torch as Colab's runtime (2.11.0).
21
+ model = ["torch==2.11.0"]
22
+
23
+ [dependency-groups]
24
+ # PEP 735 group: uv installs it by default for `uv sync` / `uv run`.
25
+ dev = [
26
+ "nida[data,model]",
27
+ "pytest>=8.0",
28
+ "hypothesis>=6.100",
29
+ "ruff>=0.6",
30
+ "mypy>=1.10",
31
+ ]
32
+
33
+ [project.scripts]
34
+ nida = "nida.cli:main"
35
+
36
+ [tool.hatch.build.targets.wheel]
37
+ packages = ["src/nida"]
38
+
39
+ [tool.ruff]
40
+ line-length = 100
41
+ target-version = "py311"
42
+ src = ["src", "tests", "lab", "lab/akan"]
43
+
44
+ [tool.ruff.lint]
45
+ select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF", "PTH", "BLE"]
46
+ # Turkish dotless "ı" is a real letter here, not a confusable.
47
+ allowed-confusables = ["ı", "–", "−", "×"]
48
+
49
+ [tool.mypy]
50
+ python_version = "3.11"
51
+ strict = true
52
+ files = ["src/nida", "tests", "lab/akan"]
53
+ # lab/akan holds the `akan` package (lab/akan/akan); `lab` is listed for later lab experiments.
54
+ mypy_path = ["src", "lab/akan", "lab"]
55
+
56
+ [[tool.mypy.overrides]]
57
+ # turkificate (text normaliser base, MIT) ships no py.typed marker; nida.text.normalize wraps it.
58
+ module = ["turkificate", "turkificate.*"]
59
+ ignore_missing_imports = true
60
+
61
+ [tool.pytest.ini_options]
62
+ testpaths = ["tests"]
63
+ pythonpath = ["src", "lab/akan", "lab"]
64
+ addopts = "-q --strict-markers -m 'not gpu'"
65
+ markers = [
66
+ "gpu: requires a CUDA GPU (deselected by default; run with -m gpu)",
67
+ "slow: long-running test",
68
+ ]
requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Nida v0.5 inference (`nida say --single-env`). Install torch + torchaudio first, a CUDA
2
+ # build for your GPU (https://pytorch.org/get-started/locally/), e.g.:
3
+ # pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
4
+ # then: pip install -r requirements.txt && pip install -e .
5
+ transformers==4.57.6
6
+ huggingface-hub
7
+ numpy
8
+ scipy
9
+ soundfile
10
+ pydantic
11
+ pyyaml
src/nida/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ """Nida: expressive Turkish text-to-speech."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __all__ = ["__version__"]
6
+
7
+ __version__ = "0.0.1"
src/nida/bench/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Phase 0 benchmarks (B1 ASR comparison, B2 VAE reconstruction) and listening panels."""
src/nida/bench/audiometrics.py ADDED
@@ -0,0 +1,570 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Objective audio metrics for comparing an original waveform with a reconstruction (B2).
2
+
3
+ All functions are pure and CPU-only (numpy + scipy). Conventions:
4
+
5
+ * Waveforms are 1-D float arrays of shape ``(num_samples,)``. Any floating dtype is accepted
6
+ and converted to ``float64``; integer and complex dtypes are rejected.
7
+ * Sample rates are integers in Hz (``sample_rate_hz``), frequencies are in Hz (``*_hz``),
8
+ lengths are in samples (``*_samples``).
9
+ * Spectrograms are laid out as ``(frames, bins)``.
10
+ * Inputs are validated at the module boundary and violations raise :class:`ValueError`
11
+ with the offending argument named.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import math
17
+ from typing import TypeAlias
18
+
19
+ import numpy as np
20
+ import numpy.typing as npt
21
+
22
+ # scipy ships no type stubs; the ignore is scoped to this single import.
23
+ from scipy import signal # type: ignore[import-untyped]
24
+
25
+ __all__ = [
26
+ "FloatArray",
27
+ "align",
28
+ "band_energy_ratio",
29
+ "band_log_spectral_distance",
30
+ "band_power_db",
31
+ "estimate_delay_samples",
32
+ "log_mel_l1",
33
+ "mel_filterbank",
34
+ "resample",
35
+ "stft_magnitude",
36
+ ]
37
+
38
+ FloatArray: TypeAlias = npt.NDArray[np.float64]
39
+
40
+ # Relative tolerance under which two cross-correlation values count as tied in
41
+ # ``estimate_delay_samples`` (FFT round-off makes exact float ties unreliable).
42
+ _TIE_RTOL = 1e-9
43
+
44
+
45
+ # --------------------------------------------------------------------------------------
46
+ # Validation helpers
47
+ # --------------------------------------------------------------------------------------
48
+
49
+
50
+ def _as_waveform(x: npt.ArrayLike, name: str) -> FloatArray:
51
+ """Validate ``x`` as a non-empty, finite 1-D float waveform and return it as float64."""
52
+ arr = np.asarray(x)
53
+ if arr.dtype.kind != "f":
54
+ raise ValueError(f"{name} must have a floating dtype, got {arr.dtype}")
55
+ if arr.ndim != 1:
56
+ raise ValueError(f"{name} must be 1-D, got shape {arr.shape}")
57
+ if arr.size == 0:
58
+ raise ValueError(f"{name} must not be empty")
59
+ out = arr.astype(np.float64)
60
+ if not np.isfinite(out).all():
61
+ raise ValueError(f"{name} contains non-finite values (NaN or inf)")
62
+ return out
63
+
64
+
65
+ def _check_int(value: int, name: str) -> int:
66
+ if isinstance(value, bool) or not isinstance(value, int | np.integer):
67
+ raise ValueError(f"{name} must be an int, got {value!r}")
68
+ return int(value)
69
+
70
+
71
+ def _check_positive_int(value: int, name: str) -> int:
72
+ ivalue = _check_int(value, name)
73
+ if ivalue <= 0:
74
+ raise ValueError(f"{name} must be > 0, got {ivalue}")
75
+ return ivalue
76
+
77
+
78
+ def _check_positive_float(value: float, name: str) -> float:
79
+ fvalue = float(value)
80
+ if not (math.isfinite(fvalue) and fvalue > 0.0):
81
+ raise ValueError(f"{name} must be a finite number > 0, got {value!r}")
82
+ return fvalue
83
+
84
+
85
+ def _check_band(sample_rate_hz: int, low_hz: float, high_hz: float) -> None:
86
+ nyquist_hz = sample_rate_hz / 2
87
+ # Written as a negated chain so NaN bounds are rejected too.
88
+ if not (0.0 <= low_hz < high_hz <= nyquist_hz):
89
+ raise ValueError(
90
+ f"frequency band must satisfy 0 <= low_hz < high_hz <= {nyquist_hz} "
91
+ f"(sample_rate_hz/2), got low_hz={low_hz}, high_hz={high_hz}"
92
+ )
93
+
94
+
95
+ def _trim_to_common_length(ref: FloatArray, est: FloatArray) -> tuple[FloatArray, FloatArray]:
96
+ num_samples = min(ref.shape[0], est.shape[0])
97
+ return ref[:num_samples], est[:num_samples]
98
+
99
+
100
+ # --------------------------------------------------------------------------------------
101
+ # Resampling
102
+ # --------------------------------------------------------------------------------------
103
+
104
+
105
+ def resample(x: npt.ArrayLike, sample_rate_hz: int, target_rate_hz: int) -> FloatArray:
106
+ """Resample a waveform with a polyphase filter.
107
+
108
+ Uses :func:`scipy.signal.resample_poly` with the up/down factors reduced by their gcd.
109
+ When the rates are equal a copy of the input is returned.
110
+
111
+ Args:
112
+ x: Waveform, shape ``(num_samples,)``, any float dtype.
113
+ sample_rate_hz: Source sample rate in Hz (> 0).
114
+ target_rate_hz: Target sample rate in Hz (> 0).
115
+
116
+ Returns:
117
+ ``float64`` array of shape ``(ceil(num_samples * target_rate_hz / sample_rate_hz),)``.
118
+
119
+ Raises:
120
+ ValueError: If ``x`` is not a finite non-empty 1-D float array or a rate is <= 0.
121
+ """
122
+ wave = _as_waveform(x, "x")
123
+ source_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
124
+ target_hz = _check_positive_int(target_rate_hz, "target_rate_hz")
125
+ if source_hz == target_hz:
126
+ return wave.copy()
127
+ divisor = math.gcd(source_hz, target_hz)
128
+ up = target_hz // divisor
129
+ down = source_hz // divisor
130
+ out = np.asarray(signal.resample_poly(wave, up, down), dtype=np.float64)
131
+ return out
132
+
133
+
134
+ # --------------------------------------------------------------------------------------
135
+ # Spectral front end
136
+ # --------------------------------------------------------------------------------------
137
+
138
+
139
+ def _hz_to_mel(freq_hz: FloatArray) -> FloatArray:
140
+ return np.asarray(2595.0 * np.log10(1.0 + freq_hz / 700.0), dtype=np.float64)
141
+
142
+
143
+ def _mel_to_hz(mel: FloatArray) -> FloatArray:
144
+ return np.asarray(700.0 * (10.0 ** (mel / 2595.0) - 1.0), dtype=np.float64)
145
+
146
+
147
+ def _bin_frequencies_hz(sample_rate_hz: int, n_fft: int) -> FloatArray:
148
+ return np.arange(n_fft // 2 + 1, dtype=np.float64) * sample_rate_hz / n_fft
149
+
150
+
151
+ def mel_filterbank(
152
+ sample_rate_hz: int,
153
+ n_fft: int,
154
+ n_mels: int,
155
+ fmin_hz: float = 0.0,
156
+ fmax_hz: float | None = None,
157
+ ) -> FloatArray:
158
+ """Build a triangular HTK mel filterbank.
159
+
160
+ The mel scale is ``2595 * log10(1 + f / 700)``. ``n_mels + 2`` points equally spaced on the
161
+ mel axis between ``fmin_hz`` and ``fmax_hz`` define the filters: filter ``i`` rises linearly
162
+ from point ``i`` to point ``i + 1`` and falls linearly to point ``i + 2``. Filters are *not*
163
+ area-normalised, so the continuous peak is 1.0 (a sampled filter reaches 1.0 only when an
164
+ FFT bin sits exactly on its centre).
165
+
166
+ Args:
167
+ sample_rate_hz: Sample rate in Hz (> 0).
168
+ n_fft: FFT size in samples (> 0).
169
+ n_mels: Number of mel filters (> 0).
170
+ fmin_hz: Lowest filter edge in Hz, ``0 <= fmin_hz < fmax_hz``.
171
+ fmax_hz: Highest filter edge in Hz, ``<= sample_rate_hz / 2``. Defaults to
172
+ ``sample_rate_hz / 2``.
173
+
174
+ Returns:
175
+ Array of shape ``(n_mels, n_fft // 2 + 1)``, non-negative, ``float64``.
176
+
177
+ Raises:
178
+ ValueError: On invalid arguments, or if some filter covers no FFT bin (``n_mels`` is
179
+ too large for ``n_fft``), since such a filter would silently output zeros.
180
+ """
181
+ rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
182
+ fft_size = _check_positive_int(n_fft, "n_fft")
183
+ num_mels = _check_positive_int(n_mels, "n_mels")
184
+ top_hz = rate_hz / 2 if fmax_hz is None else float(fmax_hz)
185
+ _check_band(rate_hz, float(fmin_hz), top_hz)
186
+
187
+ mel_points = np.linspace(
188
+ _hz_to_mel(np.asarray(float(fmin_hz))),
189
+ _hz_to_mel(np.asarray(top_hz)),
190
+ num_mels + 2,
191
+ dtype=np.float64,
192
+ )
193
+ edges_hz = _mel_to_hz(mel_points)
194
+ left_hz = edges_hz[:-2, np.newaxis]
195
+ centre_hz = edges_hz[1:-1, np.newaxis]
196
+ right_hz = edges_hz[2:, np.newaxis]
197
+ freqs_hz = _bin_frequencies_hz(rate_hz, fft_size)[np.newaxis, :]
198
+
199
+ rising = (freqs_hz - left_hz) / (centre_hz - left_hz)
200
+ falling = (right_hz - freqs_hz) / (right_hz - centre_hz)
201
+ bank = np.maximum(0.0, np.minimum(rising, falling)).astype(np.float64)
202
+
203
+ empty = np.flatnonzero(bank.max(axis=1) <= 0.0)
204
+ if empty.size > 0:
205
+ raise ValueError(
206
+ f"{empty.size} of {num_mels} mel filters cover no FFT bin "
207
+ f"(first empty index {int(empty[0])}); n_fft={fft_size} is too small for "
208
+ f"n_mels={num_mels} at sample_rate_hz={rate_hz}"
209
+ )
210
+ return np.asarray(bank, dtype=np.float64)
211
+
212
+
213
+ def stft_magnitude(x: npt.ArrayLike, n_fft: int, hop_samples: int) -> FloatArray:
214
+ """Magnitude STFT with a periodic Hann window and no centering.
215
+
216
+ Frame ``t`` covers samples ``[t * hop_samples, t * hop_samples + n_fft)``. The tail is
217
+ zero-padded so the last partial frame is included, and inputs shorter than ``n_fft`` are
218
+ zero-padded to a single frame.
219
+
220
+ Args:
221
+ x: Waveform, shape ``(num_samples,)``, any float dtype.
222
+ n_fft: FFT and window size in samples (> 0).
223
+ hop_samples: Hop between frames in samples (> 0).
224
+
225
+ Returns:
226
+ Array of shape ``(frames, n_fft // 2 + 1)`` with
227
+ ``frames = 1 + ceil(max(0, num_samples - n_fft) / hop_samples)``.
228
+
229
+ Raises:
230
+ ValueError: If ``x`` is invalid or ``n_fft`` / ``hop_samples`` is <= 0.
231
+ """
232
+ wave = _as_waveform(x, "x")
233
+ fft_size = _check_positive_int(n_fft, "n_fft")
234
+ hop = _check_positive_int(hop_samples, "hop_samples")
235
+
236
+ num_samples = wave.shape[0]
237
+ num_frames = 1 + -(-max(0, num_samples - fft_size) // hop)
238
+ padded_samples = (num_frames - 1) * hop + fft_size
239
+ padded = np.zeros(padded_samples, dtype=np.float64)
240
+ padded[:num_samples] = wave
241
+
242
+ window = np.asarray(signal.get_window("hann", fft_size, fftbins=True), dtype=np.float64)
243
+ frames = np.lib.stride_tricks.sliding_window_view(padded, fft_size)[::hop]
244
+ spectrum = np.fft.rfft(frames * window, axis=1)
245
+ return np.abs(spectrum).astype(np.float64)
246
+
247
+
248
+ # --------------------------------------------------------------------------------------
249
+ # Metrics
250
+ # --------------------------------------------------------------------------------------
251
+
252
+
253
+ def log_mel_l1(
254
+ ref: npt.ArrayLike,
255
+ est: npt.ArrayLike,
256
+ sample_rate_hz: int,
257
+ n_fft: int,
258
+ hop_samples: int,
259
+ n_mels: int,
260
+ eps: float = 1e-5,
261
+ ) -> float:
262
+ """Log-mel L1 distance between a reference and an estimate.
263
+
264
+ Both waveforms are trimmed to the shorter length, converted to magnitude STFTs, projected
265
+ through the mel filterbank (magnitude, not power), floored at ``eps`` and compared in
266
+ ``log10``. The result is the mean absolute difference over all frames and mel bands.
267
+
268
+ Args:
269
+ ref: Reference waveform, shape ``(num_samples,)``.
270
+ est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``.
271
+ sample_rate_hz: Sample rate of both signals in Hz.
272
+ n_fft: FFT size in samples.
273
+ hop_samples: STFT hop in samples.
274
+ n_mels: Number of mel bands.
275
+ eps: Floor applied to the mel magnitudes before ``log10`` (> 0).
276
+
277
+ Returns:
278
+ Mean absolute difference in log10 mel magnitude (dimensionless, >= 0).
279
+
280
+ Raises:
281
+ ValueError: On invalid waveforms or arguments (see :func:`mel_filterbank` and
282
+ :func:`stft_magnitude`).
283
+ """
284
+ ref_wave = _as_waveform(ref, "ref")
285
+ est_wave = _as_waveform(est, "est")
286
+ floor = _check_positive_float(eps, "eps")
287
+ bank = mel_filterbank(sample_rate_hz, n_fft, n_mels)
288
+ ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave)
289
+
290
+ ref_mel = stft_magnitude(ref_wave, n_fft, hop_samples) @ bank.T
291
+ est_mel = stft_magnitude(est_wave, n_fft, hop_samples) @ bank.T
292
+ diff = np.log10(np.maximum(ref_mel, floor)) - np.log10(np.maximum(est_mel, floor))
293
+ return float(np.mean(np.abs(diff)))
294
+
295
+
296
+ def _band_mask(
297
+ sample_rate_hz: int, n_fft: int, low_hz: float, high_hz: float
298
+ ) -> npt.NDArray[np.bool_]:
299
+ """Boolean mask over FFT bins belonging to the band (see :func:`band_power_db`)."""
300
+ freqs_hz = _bin_frequencies_hz(sample_rate_hz, n_fft)
301
+ upper = freqs_hz <= high_hz if high_hz == sample_rate_hz / 2 else freqs_hz < high_hz
302
+ mask: npt.NDArray[np.bool_] = (freqs_hz >= low_hz) & upper
303
+ if not mask.any():
304
+ raise ValueError(
305
+ f"no FFT bin falls in [{low_hz}, {high_hz}) Hz with n_fft={n_fft} at "
306
+ f"sample_rate_hz={sample_rate_hz} (bin spacing {sample_rate_hz / n_fft} Hz)"
307
+ )
308
+ return mask
309
+
310
+
311
+ def _band_power(
312
+ x: FloatArray, sample_rate_hz: int, n_fft: int, hop_samples: int, mask: npt.NDArray[np.bool_]
313
+ ) -> FloatArray:
314
+ """Power spectrogram restricted to in-band bins, shape ``(frames, num_band_bins)``."""
315
+ power = stft_magnitude(x, n_fft, hop_samples) ** 2
316
+ return np.asarray(power[:, mask], dtype=np.float64)
317
+
318
+
319
+ def band_power_db(
320
+ x: npt.ArrayLike,
321
+ sample_rate_hz: int,
322
+ low_hz: float,
323
+ high_hz: float,
324
+ n_fft: int = 2048,
325
+ hop_samples: int = 512,
326
+ eps: float = 1e-10,
327
+ ) -> FloatArray:
328
+ """Per-frame mean power in a frequency band, in dB.
329
+
330
+ The band contains the FFT bins ``k`` with centre frequency
331
+ ``f_k = k * sample_rate_hz / n_fft`` satisfying ``low_hz <= f_k < high_hz``. When
332
+ ``high_hz == sample_rate_hz / 2`` the upper edge is inclusive so the Nyquist bin belongs
333
+ to the band. For each frame the result is ``10 * log10(mean_k(|X_k|^2) + eps)`` where
334
+ ``X`` is the Hann-windowed STFT from :func:`stft_magnitude` (unnormalised, so absolute
335
+ values depend on ``n_fft``; compare like with like).
336
+
337
+ Args:
338
+ x: Waveform, shape ``(num_samples,)``.
339
+ sample_rate_hz: Sample rate in Hz.
340
+ low_hz: Lower band edge in Hz, ``0 <= low_hz < high_hz``.
341
+ high_hz: Upper band edge in Hz, ``<= sample_rate_hz / 2``.
342
+ n_fft: FFT size in samples.
343
+ hop_samples: STFT hop in samples.
344
+ eps: Power floor added before the logarithm (> 0).
345
+
346
+ Returns:
347
+ Array of shape ``(frames,)`` in dB.
348
+
349
+ Raises:
350
+ ValueError: On invalid arguments, or if no FFT bin falls in the band.
351
+ """
352
+ wave = _as_waveform(x, "x")
353
+ rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
354
+ fft_size = _check_positive_int(n_fft, "n_fft")
355
+ hop = _check_positive_int(hop_samples, "hop_samples")
356
+ floor = _check_positive_float(eps, "eps")
357
+ _check_band(rate_hz, float(low_hz), float(high_hz))
358
+ mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz))
359
+
360
+ in_band = _band_power(wave, rate_hz, fft_size, hop, mask)
361
+ return np.asarray(10.0 * np.log10(in_band.mean(axis=1) + floor), dtype=np.float64)
362
+
363
+
364
+ def band_log_spectral_distance(
365
+ ref: npt.ArrayLike,
366
+ est: npt.ArrayLike,
367
+ sample_rate_hz: int,
368
+ low_hz: float,
369
+ high_hz: float,
370
+ n_fft: int = 2048,
371
+ hop_samples: int = 512,
372
+ eps: float = 1e-10,
373
+ active_within_db: float | None = None,
374
+ ) -> float:
375
+ """Log-spectral distance (RMS over in-band bins, mean over frames) in dB.
376
+
377
+ With ``active_within_db`` only *active* frames are averaged: frames whose full-band
378
+ reference power is within that many dB of the loudest reference frame. This keeps
379
+ silence and noise-floor frames (where ``eps`` dominates) from swamping the average.
380
+
381
+ Both waveforms are trimmed to the shorter length. Per frame the distance is
382
+ ``sqrt(mean_k((10 log10(P_ref[k] + eps) - 10 log10(P_est[k] + eps))^2))`` over the in-band
383
+ bins ``k`` (band definition: see :func:`band_power_db`); the result is the mean over frames.
384
+
385
+ Args:
386
+ ref: Reference waveform, shape ``(num_samples,)``.
387
+ est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``.
388
+ sample_rate_hz: Sample rate of both signals in Hz.
389
+ low_hz: Lower band edge in Hz.
390
+ high_hz: Upper band edge in Hz.
391
+ n_fft: FFT size in samples.
392
+ hop_samples: STFT hop in samples.
393
+ eps: Power floor added before the logarithm (> 0).
394
+ active_within_db: If set (> 0), average only over active frames (see above).
395
+
396
+ Returns:
397
+ Distance in dB (>= 0; 0 for identical signals).
398
+
399
+ Raises:
400
+ ValueError: On invalid arguments, or if no FFT bin falls in the band.
401
+ """
402
+ ref_wave = _as_waveform(ref, "ref")
403
+ est_wave = _as_waveform(est, "est")
404
+ rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
405
+ fft_size = _check_positive_int(n_fft, "n_fft")
406
+ hop = _check_positive_int(hop_samples, "hop_samples")
407
+ floor = _check_positive_float(eps, "eps")
408
+ _check_band(rate_hz, float(low_hz), float(high_hz))
409
+ mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz))
410
+ ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave)
411
+
412
+ ref_power = _band_power(ref_wave, rate_hz, fft_size, hop, mask)
413
+ est_power = _band_power(est_wave, rate_hz, fft_size, hop, mask)
414
+ diff_db = 10.0 * np.log10(ref_power + floor) - 10.0 * np.log10(est_power + floor)
415
+ per_frame = np.sqrt(np.mean(diff_db**2, axis=1))
416
+ if active_within_db is not None:
417
+ window_db = _check_positive_float(active_within_db, "active_within_db")
418
+ full_power = np.sum(stft_magnitude(ref_wave, fft_size, hop) ** 2, axis=1)
419
+ full_db = 10.0 * np.log10(full_power + floor)
420
+ active = full_db >= float(np.max(full_db)) - window_db
421
+ per_frame = per_frame[active]
422
+ return float(np.mean(per_frame))
423
+
424
+
425
+ def band_energy_ratio(
426
+ ref: npt.ArrayLike,
427
+ est: npt.ArrayLike,
428
+ sample_rate_hz: int,
429
+ low_hz: float,
430
+ high_hz: float,
431
+ n_fft: int = 2048,
432
+ hop_samples: int = 512,
433
+ ) -> float:
434
+ """Ratio of in-band energy of the estimate to that of the reference.
435
+
436
+ Both waveforms are trimmed to the shorter length. The energy of a signal is the sum of
437
+ ``|X_k|^2`` over all frames and all in-band bins ``k`` (band definition: see
438
+ :func:`band_power_db`).
439
+
440
+ Args:
441
+ ref: Reference waveform, shape ``(num_samples,)``.
442
+ est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``.
443
+ sample_rate_hz: Sample rate of both signals in Hz.
444
+ low_hz: Lower band edge in Hz.
445
+ high_hz: Upper band edge in Hz.
446
+ n_fft: FFT size in samples.
447
+ hop_samples: STFT hop in samples.
448
+
449
+ Returns:
450
+ ``E_est / E_ref`` (dimensionless, >= 0; 1 for identical signals).
451
+
452
+ Raises:
453
+ ValueError: On invalid arguments, if no FFT bin falls in the band, or if the reference
454
+ has zero in-band energy (the ratio would be undefined).
455
+ """
456
+ ref_wave = _as_waveform(ref, "ref")
457
+ est_wave = _as_waveform(est, "est")
458
+ rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz")
459
+ fft_size = _check_positive_int(n_fft, "n_fft")
460
+ hop = _check_positive_int(hop_samples, "hop_samples")
461
+ _check_band(rate_hz, float(low_hz), float(high_hz))
462
+ mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz))
463
+ ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave)
464
+
465
+ ref_energy = float(np.sum(_band_power(ref_wave, rate_hz, fft_size, hop, mask)))
466
+ est_energy = float(np.sum(_band_power(est_wave, rate_hz, fft_size, hop, mask)))
467
+ if ref_energy == 0.0:
468
+ raise ValueError(
469
+ f"reference has zero energy in [{low_hz}, {high_hz}) Hz; energy ratio is undefined"
470
+ )
471
+ return est_energy / ref_energy
472
+
473
+
474
+ # --------------------------------------------------------------------------------------
475
+ # Alignment
476
+ # --------------------------------------------------------------------------------------
477
+
478
+
479
+ def estimate_delay_samples(ref: npt.ArrayLike, est: npt.ArrayLike, max_lag_samples: int) -> int:
480
+ """Estimate the delay of ``est`` relative to ``ref`` by cross-correlation.
481
+
482
+ The cross-correlation ``c[lag] = sum_n est[n] * ref[n - lag]`` is computed with FFTs
483
+ (zero-padded, so there is no circular wrap-around) and maximised over
484
+ ``-max_lag_samples <= lag <= +max_lag_samples``, further limited to lags for which the
485
+ signals overlap. A positive result means ``est`` is delayed: ``est[n] ~ ref[n - lag]``.
486
+
487
+ Ties (values within a relative tolerance of ``1e-9`` of the best) resolve to the smallest
488
+ ``|lag|``; between ``+k`` and ``-k`` the positive lag wins. Two all-zero signals yield 0.
489
+
490
+ Args:
491
+ ref: Reference waveform, shape ``(num_ref_samples,)``.
492
+ est: Estimated waveform, shape ``(num_est_samples,)``.
493
+ max_lag_samples: Largest absolute lag to consider (>= 0).
494
+
495
+ Returns:
496
+ Delay in samples (``int``) in ``[-max_lag_samples, +max_lag_samples]``.
497
+
498
+ Raises:
499
+ ValueError: On invalid waveforms or negative / non-integer ``max_lag_samples``.
500
+ """
501
+ ref_wave = _as_waveform(ref, "ref")
502
+ est_wave = _as_waveform(est, "est")
503
+ max_lag = _check_int(max_lag_samples, "max_lag_samples")
504
+ if max_lag < 0:
505
+ raise ValueError(f"max_lag_samples must be >= 0, got {max_lag}")
506
+
507
+ num_ref = ref_wave.shape[0]
508
+ num_est = est_wave.shape[0]
509
+ max_positive = min(max_lag, num_est - 1) # est[n] needs n = lag + m with m >= 0
510
+ max_negative = min(max_lag, num_ref - 1)
511
+
512
+ fft_size = 1 << (num_ref + num_est - 1).bit_length()
513
+ est_spectrum = np.fft.rfft(est_wave, fft_size)
514
+ ref_spectrum = np.fft.rfft(ref_wave, fft_size)
515
+ corr = np.fft.irfft(est_spectrum * np.conj(ref_spectrum), fft_size)
516
+
517
+ # Candidate lags ordered by |lag| (0, +1, -1, +2, -2, ...), so the first maximal entry
518
+ # is the smallest-|lag| tie winner.
519
+ lags: list[int] = [0]
520
+ for k in range(1, max(max_positive, max_negative) + 1):
521
+ if k <= max_positive:
522
+ lags.append(k)
523
+ if k <= max_negative:
524
+ lags.append(-k)
525
+ lag_arr = np.asarray(lags, dtype=np.int64)
526
+ values = corr[lag_arr % fft_size] # negative lags wrap to the end of the circular buffer
527
+
528
+ peak = float(values.max())
529
+ tolerance = _TIE_RTOL * float(np.abs(values).max())
530
+ best = int(np.argmax(values >= peak - tolerance))
531
+ return int(lag_arr[best])
532
+
533
+
534
+ def align(
535
+ ref: npt.ArrayLike, est: npt.ArrayLike, delay_samples: int
536
+ ) -> tuple[FloatArray, FloatArray]:
537
+ """Compensate a known delay of ``est`` and trim both signals to a common length.
538
+
539
+ With ``delay_samples = d > 0`` (``est`` lags ``ref``) the first ``d`` samples of ``est``
540
+ are dropped; with ``d < 0`` (``est`` leads) the first ``|d|`` samples of ``ref`` are
541
+ dropped. Both outputs are then cut to the shorter of the two remaining lengths.
542
+
543
+ Args:
544
+ ref: Reference waveform, shape ``(num_ref_samples,)``.
545
+ est: Estimated waveform, shape ``(num_est_samples,)``.
546
+ delay_samples: Delay of ``est`` relative to ``ref``, as returned by
547
+ :func:`estimate_delay_samples`.
548
+
549
+ Returns:
550
+ ``(ref_aligned, est_aligned)``, two ``float64`` copies of identical length >= 1.
551
+
552
+ Raises:
553
+ ValueError: On invalid waveforms, non-integer ``delay_samples``, or if no samples
554
+ remain after compensating the delay.
555
+ """
556
+ ref_wave = _as_waveform(ref, "ref")
557
+ est_wave = _as_waveform(est, "est")
558
+ delay = _check_int(delay_samples, "delay_samples")
559
+
560
+ if delay >= 0:
561
+ ref_part, est_part = ref_wave, est_wave[delay:]
562
+ else:
563
+ ref_part, est_part = ref_wave[-delay:], est_wave
564
+ ref_part, est_part = _trim_to_common_length(ref_part, est_part)
565
+ if ref_part.shape[0] < 1:
566
+ raise ValueError(
567
+ f"no samples left after compensating delay_samples={delay} "
568
+ f"(ref has {ref_wave.shape[0]}, est has {est_wave.shape[0]} samples)"
569
+ )
570
+ return ref_part.copy(), est_part.copy()
src/nida/bench/b1.py ADDED
@@ -0,0 +1,622 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Benchmark B1: Turkish ASR comparison (docs/benchmarks/B1-B2-tasarim.md §1).
2
+
3
+ Stages (each idempotent and resumable):
4
+
5
+ 1. ``prepare`` — :func:`nida.bench.prepare.prepare_b1` (layers A, B and the C pool).
6
+ 2. ``asr`` — every system transcribes every clip in its own environment.
7
+ 3. ``panel`` — choose the final layer-C clips (filler-rich, once) and build the
8
+ transcription panel; the user's corrected transcripts become C references.
9
+ 4. ``score`` — metrics, paired bootstrap and the pre-registered decision rule.
10
+
11
+ Failed worker items are scored as empty hypotheses (all deletions) and counted, so a
12
+ system can never look better by failing on hard clips.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import itertools
18
+ import json
19
+ import logging
20
+ import subprocess
21
+ from collections.abc import Callable, Mapping, Sequence
22
+ from contextvars import ContextVar
23
+ from dataclasses import dataclass, field
24
+ from pathlib import Path
25
+
26
+ from nida.bench.clips import Clip, select_yodas_final, text_has_filler
27
+ from nida.bench.panel import (
28
+ AudioRef,
29
+ PanelItem,
30
+ PanelResult,
31
+ PanelSpec,
32
+ Question,
33
+ write_panel,
34
+ )
35
+ from nida.bench.prepare import Workspace, clip_audio_path
36
+ from nida.bench.protocol import (
37
+ WorkerSummary,
38
+ WorkRequest,
39
+ WorkResult,
40
+ load_results,
41
+ read_summary,
42
+ write_requests,
43
+ )
44
+ from nida.bench.registry import System
45
+ from nida.bench.textmetrics import (
46
+ FILLERS,
47
+ EditCounts,
48
+ cer_counts,
49
+ corpus_rate,
50
+ filler_score,
51
+ has_digits,
52
+ is_hallucination,
53
+ normalize_for_scoring,
54
+ paired_bootstrap,
55
+ strip_fillers,
56
+ wer_counts,
57
+ )
58
+ from nida.utils.io import atomic_write_json
59
+
60
+ __all__ = [
61
+ "C_PANEL_QUESTIONS",
62
+ "DecisionRule",
63
+ "FillerProvenance",
64
+ "build_c_panel",
65
+ "c_references_from_panel",
66
+ "choose_systems",
67
+ "hypotheses",
68
+ "load_c_selection",
69
+ "run_system",
70
+ "score_b1",
71
+ ]
72
+
73
+ log = logging.getLogger(__name__)
74
+
75
+ MAX_ERROR_FRACTION = 0.05
76
+ N_C_FINAL = 50
77
+ # Pre-registered 25; lowered to 20 before any score was seen (pool had 23 filler clips,
78
+ # run 20261008-122110-bench-phase0-b301; docs/benchmarks/B1-B2-tasarim.md §1.1).
79
+ MIN_C_FILLER_CLIPS = 20
80
+ C_REVIEWED_ANSWERS = ("tamam", "emin değilim")
81
+ C_SKIP_ANSWER = "konuşma değil — atla"
82
+
83
+ C_PANEL_QUESTIONS = (
84
+ Question(
85
+ "durum",
86
+ "Bu klip için durum",
87
+ "choice",
88
+ (*C_REVIEWED_ANSWERS, C_SKIP_ANSWER),
89
+ required=True,
90
+ ),
91
+ )
92
+ C_PANEL_INSTRUCTIONS = (
93
+ "Her klibi dinleyin ve metin kutusundaki taslağı duyduğunuz şekilde, kelimesi kelimesine "
94
+ "düzeltin. Taslak YouTube altyazısıdır, hatalı olabilir. Dolgu seslerini şu yazımlarla "
95
+ "ekleyin: ıı, ee, hmm, ımm, aa (ve of, uf, ah, oh, vay, hıı). Sayıları yazıyla yazın. "
96
+ "Noktalama önemli değil. Her klip için bir durum seçin: düzelttiyseniz 'tamam', "
97
+ "anlaşılmayan yer kaldıysa 'emin değilim', konuşma yoksa ya da müzik baskınsa "
98
+ "'konuşma değil — atla'. Durumu seçilmeyen klipler kullanılmaz."
99
+ )
100
+
101
+ Runner = Callable[[Sequence[str]], int]
102
+
103
+ #: Tag put in front of every line :func:`_stream_runner` logs. Callers that run several
104
+ #: workers at the same time set it (per thread) to the job name so the interleaved lines
105
+ #: stay attributable; empty (the default) logs exactly as before.
106
+ LOG_TAG: ContextVar[str] = ContextVar("nida_worker_log_tag", default="")
107
+
108
+
109
+ def _stream_runner(cmd: Sequence[str]) -> int:
110
+ tag = LOG_TAG.get()
111
+ prefix = f"[{tag}] " if tag else ""
112
+ log.info("%s$ %s", prefix, " ".join(cmd))
113
+ with subprocess.Popen(
114
+ list(cmd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
115
+ ) as proc:
116
+ assert proc.stdout is not None # stdout=PIPE guarantees a stream
117
+ for line in proc.stdout:
118
+ log.info("%s | %s", prefix, line.rstrip())
119
+ return proc.wait()
120
+
121
+
122
+ def _worker_dir(workspace: Workspace, system: System, variant: str) -> Path:
123
+ name = system.system_id if not variant else f"{system.system_id}__{variant}"
124
+ return workspace.workers / name
125
+
126
+
127
+ def run_system(
128
+ workspace: Workspace,
129
+ clips: Sequence[Clip],
130
+ system: System,
131
+ python_exe: Path,
132
+ *,
133
+ variant: str = "",
134
+ context: str = "",
135
+ retry_failed: bool = False,
136
+ runner: Runner = _stream_runner,
137
+ ) -> WorkerSummary:
138
+ """Run one ASR system over ``clips`` (resumable) and return its summary.
139
+
140
+ Raises:
141
+ RuntimeError: if the worker exits non-zero or more than 5% of items failed.
142
+ """
143
+ out_dir = _worker_dir(workspace, system, variant)
144
+ requests = [
145
+ WorkRequest(
146
+ clip.clip_id,
147
+ str(clip_audio_path(workspace, clip)),
148
+ {"language": clip.language, **({"context": context} if context else {})},
149
+ )
150
+ for clip in clips
151
+ ]
152
+ write_requests(out_dir / "requests.jsonl", requests)
153
+ cmd = [
154
+ str(python_exe),
155
+ "-m",
156
+ f"nida.bench.workers.{system.worker}",
157
+ "--requests",
158
+ str(out_dir / "requests.jsonl"),
159
+ "--results",
160
+ str(out_dir / "results.jsonl"),
161
+ "--model-id",
162
+ system.model_id,
163
+ "--revision",
164
+ system.revision,
165
+ "--batch-size",
166
+ str(system.batch_size),
167
+ "--device",
168
+ "cuda",
169
+ *(["--retry-failed"] if retry_failed else []),
170
+ ]
171
+ code = runner(cmd)
172
+ if code != 0:
173
+ raise RuntimeError(f"{system.system_id}: worker exited with code {code}")
174
+ summary = read_summary(out_dir / "results.summary.json")
175
+ total = summary.n_ok + summary.n_error
176
+ if total != len(clips):
177
+ raise RuntimeError(f"{system.system_id}: {total} results for {len(clips)} clips")
178
+ if summary.n_error > MAX_ERROR_FRACTION * total:
179
+ raise RuntimeError(
180
+ f"{system.system_id}: {summary.n_error}/{total} items failed (> 5%); "
181
+ "rerun with --retry-failed after fixing the cause"
182
+ )
183
+ return summary
184
+
185
+
186
+ def hypotheses(workspace: Workspace, system: System, variant: str = "") -> dict[str, WorkResult]:
187
+ return load_results(_worker_dir(workspace, system, variant) / "results.jsonl")
188
+
189
+
190
+ def _hyp_text(result: WorkResult | None) -> str:
191
+ if result is None or result.output is None:
192
+ return ""
193
+ text = result.output.get("text", "")
194
+ return text if isinstance(text, str) else ""
195
+
196
+
197
+ # --------------------------------------------------------------------------- C panel
198
+
199
+
200
+ @dataclass(frozen=True)
201
+ class FillerProvenance:
202
+ """Why a layer-C clip may have been selected (used to de-bias filler recall).
203
+
204
+ Attributes:
205
+ caption: The YouTube caption contains a filler.
206
+ systems: Systems whose hypothesis contains a filler.
207
+ """
208
+
209
+ caption: bool
210
+ systems: frozenset[str] = field(default_factory=frozenset)
211
+
212
+ def counts_for(self, system_id: str) -> bool:
213
+ """True if the clip's selection does not depend solely on ``system_id``'s output.
214
+
215
+ Filler-signal clips are favoured during selection. If the only signal was this
216
+ system's own hypothesis, scoring its filler recall on the clip would reward it for
217
+ having caused the selection, so such clips are left out of its recall.
218
+ """
219
+ return self.caption or not self.systems or bool(self.systems - {system_id})
220
+
221
+
222
+ def _filler_provenance(
223
+ clips: Sequence[Clip], all_hypotheses: Mapping[str, Mapping[str, WorkResult]]
224
+ ) -> dict[str, FillerProvenance]:
225
+ out: dict[str, FillerProvenance] = {}
226
+ for clip in clips:
227
+ systems = frozenset(
228
+ system_id
229
+ for system_id, results in all_hypotheses.items()
230
+ if text_has_filler(_hyp_text(results.get(clip.clip_id)))
231
+ )
232
+ out[clip.clip_id] = FillerProvenance(text_has_filler(clip.meta.get("caption", "")), systems)
233
+ return out
234
+
235
+
236
+ def _selection_path(workspace: Workspace) -> Path:
237
+ return workspace.manifests / "b1_c_final.json"
238
+
239
+
240
+ def load_c_selection(workspace: Workspace, seed: int | None = None) -> dict[str, FillerProvenance]:
241
+ """Read the stored layer-C selection: ``{clip_id: provenance}`` in selection order.
242
+
243
+ Raises:
244
+ ValueError: if ``seed`` is given and differs from the seed the selection used.
245
+ """
246
+ obj = json.loads(_selection_path(workspace).read_text(encoding="utf-8"))
247
+ if seed is not None and obj.get("seed") != seed:
248
+ raise ValueError(f"stored C selection used seed {obj.get('seed')}, not {seed}")
249
+ return {
250
+ row["clip_id"]: FillerProvenance(bool(row["caption_filler"]), frozenset(row["systems"]))
251
+ for row in obj["clips"]
252
+ }
253
+
254
+
255
+ def build_c_panel(
256
+ workspace: Workspace,
257
+ pool: Sequence[Clip],
258
+ all_hypotheses: Mapping[str, Mapping[str, WorkResult]],
259
+ seed: int,
260
+ ) -> tuple[list[Clip], Path]:
261
+ """Select the final layer-C clips (once) and write their transcription panel.
262
+
263
+ The selection is stored once in ``manifests/b1_c_final.json`` together with each
264
+ clip's filler provenance and is never re-drawn: a rerun rebuilds the panel for the same
265
+ clips. Before the first selection every system must have a result for every pool clip.
266
+ The draft text is the YouTube caption, never a candidate system's output, so
267
+ references are not anchored to one of the systems under test.
268
+
269
+ Raises:
270
+ RuntimeError: if some system has not transcribed the whole pool, or the stored
271
+ selection does not belong to this pool.
272
+ """
273
+ import soundfile # type: ignore[import-untyped] # bench extra
274
+
275
+ by_id = {c.clip_id: c for c in pool}
276
+ if _selection_path(workspace).exists():
277
+ selection = load_c_selection(workspace, seed)
278
+ stray = [cid for cid in selection if cid not in by_id]
279
+ if stray:
280
+ raise RuntimeError(f"stored C selection has clips outside the pool: {stray[:3]}")
281
+ final = [by_id[cid] for cid in selection]
282
+ else:
283
+ incomplete = sorted(
284
+ s for s, results in all_hypotheses.items() if not set(by_id) <= set(results)
285
+ )
286
+ if incomplete:
287
+ raise RuntimeError(f"systems have not transcribed the whole C pool: {incomplete}")
288
+ texts: dict[str, list[str]] = {}
289
+ for results in all_hypotheses.values():
290
+ for clip_id, result in results.items():
291
+ texts.setdefault(clip_id, []).append(_hyp_text(result))
292
+ final = list(
293
+ select_yodas_final(pool, texts, N_C_FINAL, seed, min_filler_clips=MIN_C_FILLER_CLIPS)
294
+ )
295
+ provenance = _filler_provenance(final, all_hypotheses)
296
+ atomic_write_json(
297
+ _selection_path(workspace),
298
+ {
299
+ "seed": seed,
300
+ "clips": [
301
+ {
302
+ "clip_id": c.clip_id,
303
+ "caption_filler": provenance[c.clip_id].caption,
304
+ "systems": sorted(provenance[c.clip_id].systems),
305
+ }
306
+ for c in final
307
+ ],
308
+ },
309
+ )
310
+ panel_id = f"b1c-{seed}"
311
+ audio_dir = workspace.cache / "panel_audio" / panel_id
312
+ items: list[PanelItem] = []
313
+ for clip in final:
314
+ flac = audio_dir / f"{clip.clip_id}.flac"
315
+ if not flac.exists():
316
+ audio, rate_hz = soundfile.read(str(clip_audio_path(workspace, clip)), dtype="int16")
317
+ flac.parent.mkdir(parents=True, exist_ok=True)
318
+ soundfile.write(str(flac), audio, rate_hz, subtype="PCM_16")
319
+ items.append(
320
+ PanelItem(
321
+ item_id=clip.clip_id,
322
+ audios=(AudioRef("Klip", flac),),
323
+ text=clip.meta.get("caption", ""),
324
+ editable_text=True,
325
+ info=f"YODAS · {clip.duration_ms / 1000:.1f} s",
326
+ )
327
+ )
328
+ spec = PanelSpec(
329
+ panel_id=panel_id,
330
+ title="B1 — Doğal konuşma transkripsiyonu",
331
+ instructions=C_PANEL_INSTRUCTIONS,
332
+ questions=C_PANEL_QUESTIONS,
333
+ items=tuple(items),
334
+ )
335
+ return final, write_panel(spec, workspace.panels / panel_id)
336
+
337
+
338
+ def c_references_from_panel(results: Sequence[PanelResult]) -> dict[str, str]:
339
+ """Usable C references: items explicitly reviewed (``tamam`` / ``emin değilim``).
340
+
341
+ Unreviewed items are excluded even though their CSV row carries the caption draft: an
342
+ untouched caption is not a reference (glued words, fillers almost never written).
343
+ """
344
+ refs: dict[str, str] = {}
345
+ for result in results:
346
+ if result.answers.get("durum") not in C_REVIEWED_ANSWERS:
347
+ continue
348
+ text = (result.text or "").strip()
349
+ if text:
350
+ refs[result.item_id] = text
351
+ return refs
352
+
353
+
354
+ # --------------------------------------------------------------------------- scoring
355
+
356
+
357
+ @dataclass(frozen=True)
358
+ class DecisionRule:
359
+ """Pre-registered rule (design note §1.4), fixed before results are seen.
360
+
361
+ Attributes:
362
+ min_filler_recall: Primary must reach this C filler recall (leave-one-out).
363
+ min_reviewed_c: With fewer reviewed C references the rule is not applied.
364
+ min_filler_ref_tokens: With fewer reference fillers in C the rule is not applied.
365
+ min_loo_ref_tokens: Per-system minimum of reference fillers left after the
366
+ leave-one-out exclusion (the denominator of the recall the rule uses).
367
+ max_failed_ab: Most failed worker items allowed per system on A+B; failed items are
368
+ scored as empty hypotheses, so any more would distort the CER ranking.
369
+ max_failed_c: Same for layer C, where a failed item turns its reference fillers into
370
+ misses and would distort the filler recall the rule depends on.
371
+ """
372
+
373
+ min_filler_recall: float = 0.6
374
+ min_reviewed_c: int = 35
375
+ min_filler_ref_tokens: int = 30
376
+ min_loo_ref_tokens: int = 20
377
+ max_failed_ab: int = 0
378
+ max_failed_c: int = 0
379
+
380
+
381
+ def _layer_scores(
382
+ clips: Sequence[Clip],
383
+ refs: Mapping[str, str],
384
+ hyps: Mapping[str, WorkResult],
385
+ eligible_for_fillers: frozenset[str] | None = None,
386
+ ) -> tuple[dict[str, object], list[EditCounts]]:
387
+ content_cer: list[EditCounts] = []
388
+ content_wer: list[EditCounts] = []
389
+ full_cer: list[EditCounts] = []
390
+ full_wer: list[EditCounts] = []
391
+ pairs: list[tuple[list[str], list[str]]] = []
392
+ loo_pairs: list[tuple[list[str], list[str]]] = []
393
+ n_failed = n_halluc = n_digits = 0
394
+ infer_s = audio_s = 0.0
395
+ for clip in clips:
396
+ ref_text = refs[clip.clip_id]
397
+ result = hyps.get(clip.clip_id)
398
+ hyp_text = _hyp_text(result)
399
+ if result is None or not result.ok:
400
+ n_failed += 1
401
+ else:
402
+ infer_s += result.infer_seconds
403
+ audio_s += clip.duration_ms / 1000
404
+ n_halluc += is_hallucination(ref_text, hyp_text)
405
+ n_digits += has_digits(hyp_text)
406
+ ref_tokens = normalize_for_scoring(ref_text)
407
+ hyp_tokens = normalize_for_scoring(hyp_text)
408
+ content_cer.append(cer_counts(strip_fillers(ref_tokens), strip_fillers(hyp_tokens)))
409
+ content_wer.append(wer_counts(strip_fillers(ref_tokens), strip_fillers(hyp_tokens)))
410
+ full_cer.append(cer_counts(ref_tokens, hyp_tokens))
411
+ full_wer.append(wer_counts(ref_tokens, hyp_tokens))
412
+ pairs.append((ref_tokens, hyp_tokens))
413
+ if eligible_for_fillers is None or clip.clip_id in eligible_for_fillers:
414
+ loo_pairs.append((ref_tokens, hyp_tokens))
415
+ fillers = filler_score(pairs)
416
+ loo = filler_score(loo_pairs)
417
+ n_ok = len(clips) - n_failed
418
+ scores: dict[str, object] = {
419
+ "n": len(clips),
420
+ "n_failed": n_failed,
421
+ "content_cer": corpus_rate(content_cer),
422
+ "content_wer": corpus_rate(content_wer),
423
+ "full_cer": corpus_rate(full_cer),
424
+ "full_wer": corpus_rate(full_wer),
425
+ "hallucination_rate": n_halluc / n_ok if n_ok else None,
426
+ "hyp_with_digits": n_digits,
427
+ "rtf": infer_s / audio_s if audio_s else None,
428
+ "filler_ref_total": fillers.ref_total,
429
+ "filler_hyp_total": fillers.hyp_total,
430
+ "filler_recall": fillers.recall(),
431
+ "filler_precision": fillers.precision(),
432
+ "filler_recall_loo": loo.recall(),
433
+ "filler_loo_ref_total": loo.ref_total,
434
+ "filler_recall_by_type": {
435
+ kind: filler_score(pairs, kind).recall()
436
+ for kind in sorted(FILLERS)
437
+ if filler_score(pairs, kind).ref_total
438
+ },
439
+ }
440
+ return scores, content_cer
441
+
442
+
443
+ def score_b1(
444
+ clips: Sequence[Clip],
445
+ systems: Sequence[System],
446
+ all_hypotheses: Mapping[str, Mapping[str, WorkResult]],
447
+ c_references: Mapping[str, str] | None,
448
+ *,
449
+ seed: int,
450
+ summaries: Mapping[str, WorkerSummary] | None = None,
451
+ c_provenance: Mapping[str, FillerProvenance] | None = None,
452
+ ) -> dict[str, object]:
453
+ """Compute B1 metrics per system and layer, plus paired bootstrap on content CER.
454
+
455
+ Layers: ``A``, ``B``, ``AB`` (A+B together) and ``C`` when ``c_references`` is given
456
+ (only reviewed clips present in it are scored). With ``c_provenance`` the C layer's
457
+ ``filler_recall_loo`` excludes, per system, clips selected only because of that
458
+ system's own hypothesis (see :class:`FillerProvenance`).
459
+ """
460
+ refs: dict[str, str] = {}
461
+ for clip in clips:
462
+ if clip.layer in ("A", "B") and clip.reference is not None:
463
+ refs[clip.clip_id] = clip.reference
464
+ if c_references:
465
+ refs.update(c_references)
466
+ layers: dict[str, list[Clip]] = {
467
+ "A": [c for c in clips if c.layer == "A"],
468
+ "B": [c for c in clips if c.layer == "B"],
469
+ }
470
+ layers["AB"] = layers["A"] + layers["B"]
471
+ if c_references:
472
+ layers["C"] = [c for c in clips if c.layer == "C" and c.clip_id in c_references]
473
+
474
+ per_system: dict[str, dict[str, dict[str, object]]] = {}
475
+ counts_for_bootstrap: dict[str, dict[str, list[EditCounts]]] = {}
476
+ for system in systems:
477
+ hyps = all_hypotheses[system.system_id]
478
+ per_layer: dict[str, dict[str, object]] = {}
479
+ for layer, layer_clips in layers.items():
480
+ eligible: frozenset[str] | None = None
481
+ if layer == "C" and c_provenance is not None:
482
+ eligible = frozenset(
483
+ cid for cid, prov in c_provenance.items() if prov.counts_for(system.system_id)
484
+ )
485
+ scores, counts = _layer_scores(layer_clips, refs, hyps, eligible)
486
+ counts_for_bootstrap.setdefault(layer, {})[system.system_id] = counts
487
+ per_layer[layer] = scores
488
+ worker: dict[str, object] = {"batch_size": system.batch_size, "worker": system.worker}
489
+ if summaries and system.system_id in summaries:
490
+ summary = summaries[system.system_id]
491
+ worker |= {
492
+ "peak_vram_gb": summary.peak_vram_gb,
493
+ "load_seconds": summary.load_seconds,
494
+ "n_error": summary.n_error,
495
+ "packages": dict(summary.packages),
496
+ }
497
+ per_layer["_worker"] = worker
498
+ per_system[system.system_id] = per_layer
499
+
500
+ bootstrap: dict[str, list[dict[str, object]]] = {}
501
+ for layer in ("AB", "B", "C"):
502
+ if layer not in counts_for_bootstrap or not layers[layer]:
503
+ continue
504
+ rows: list[dict[str, object]] = []
505
+ for a, b in itertools.combinations(sorted(counts_for_bootstrap[layer]), 2):
506
+ result = paired_bootstrap(
507
+ counts_for_bootstrap[layer][a], counts_for_bootstrap[layer][b], seed=seed
508
+ )
509
+ rows.append(
510
+ {"a": a, "b": b, "delta": result.delta, "low": result.low, "high": result.high}
511
+ )
512
+ bootstrap[layer] = rows
513
+ return {
514
+ "systems": per_system,
515
+ "bootstrap_content_cer": bootstrap,
516
+ "n_c_reviewed": len(layers.get("C", [])),
517
+ }
518
+
519
+
520
+ def _as_float(value: object, missing: float) -> float:
521
+ return float(value) if isinstance(value, int | float) else missing
522
+
523
+
524
+ def _decide(
525
+ systems: Mapping[str, Mapping[str, Mapping[str, object]]],
526
+ families: Mapping[str, str],
527
+ rule: DecisionRule,
528
+ cer_layer: str,
529
+ ) -> dict[str, object]:
530
+ def cer(system_id: str) -> float:
531
+ return _as_float(systems[system_id][cer_layer]["content_cer"], float("inf"))
532
+
533
+ def recall(system_id: str) -> float:
534
+ c_layer = systems[system_id]["C"]
535
+ return _as_float(c_layer.get("filler_recall_loo", c_layer["filler_recall"]), -1.0)
536
+
537
+ ranked = sorted(systems, key=cer)
538
+ qualified = [s for s in ranked if recall(s) >= rule.min_filler_recall]
539
+ primary = qualified[0] if qualified else ranked[0]
540
+ others = [s for s in ranked if families[s] != families[primary]]
541
+ return {
542
+ "cer_layer": cer_layer,
543
+ "primary": primary,
544
+ "secondary": others[0] if others else None,
545
+ "needs_filler_method": not qualified,
546
+ "qualified_by_filler_recall": qualified,
547
+ }
548
+
549
+
550
+ def choose_systems(
551
+ scores: Mapping[str, object],
552
+ families: Mapping[str, str],
553
+ rule: DecisionRule | None = None,
554
+ ) -> dict[str, object]:
555
+ """Apply the pre-registered rule; returns the suggestion and its reasoning.
556
+
557
+ Primary: among systems whose C filler recall (leave-one-out when available) is at least
558
+ ``rule.min_filler_recall``, the lowest A+B content CER. If none qualifies: the lowest
559
+ A+B content CER and ``needs_filler_method``. Secondary: the lowest A+B content CER among
560
+ systems of a *different family*. The same rule is also evaluated on layer B alone
561
+ (``without_fleurs``) because one candidate (BuzzASR) was trained on FLEURS; the result
562
+ says whether the pick changes. Not applied (``ready=False``) without enough reviewed C
563
+ references, reference fillers, a system whose leave-one-out filler denominator is below
564
+ ``rule.min_loo_ref_tokens``, or a system with failed items beyond
565
+ ``rule.max_failed_ab`` (A+B) / ``rule.max_failed_c`` (C) (the reason names them).
566
+ """
567
+ rule = rule if rule is not None else DecisionRule()
568
+ systems = scores["systems"]
569
+ assert isinstance(systems, dict)
570
+ if not all("C" in layers for layers in systems.values()):
571
+ return {"ready": False, "reason": "C references (user panel) not available yet"}
572
+ n_reviewed = scores.get("n_c_reviewed", 0)
573
+ if not isinstance(n_reviewed, int) or n_reviewed < rule.min_reviewed_c:
574
+ return {
575
+ "ready": False,
576
+ "reason": f"only {n_reviewed} reviewed C clips (< {rule.min_reviewed_c})",
577
+ }
578
+ ref_fillers = min(int(_as_float(v["C"]["filler_ref_total"], 0)) for v in systems.values())
579
+ if ref_fillers < rule.min_filler_ref_tokens:
580
+ return {
581
+ "ready": False,
582
+ "reason": f"only {ref_fillers} reference fillers in C (< {rule.min_filler_ref_tokens})",
583
+ }
584
+ failed = {
585
+ f"{system_id}/{layer}": int(_as_float(v[layer].get("n_failed"), 0))
586
+ for system_id, v in systems.items()
587
+ for layer, limit in (("AB", rule.max_failed_ab), ("C", rule.max_failed_c))
588
+ if _as_float(v[layer].get("n_failed"), 0) > limit
589
+ }
590
+ if failed:
591
+ return {
592
+ "ready": False,
593
+ "reason": f"failed worker items (rerun with --retry-failed): {failed}",
594
+ }
595
+ thin = {
596
+ system_id: int(_as_float(v["C"].get("filler_loo_ref_total"), 0))
597
+ for system_id, v in systems.items()
598
+ if "filler_loo_ref_total" in v["C"]
599
+ and _as_float(v["C"]["filler_loo_ref_total"], 0) < rule.min_loo_ref_tokens
600
+ }
601
+ if thin:
602
+ return {
603
+ "ready": False,
604
+ "reason": f"leave-one-out filler denominators below {rule.min_loo_ref_tokens}: {thin}",
605
+ }
606
+ main = _decide(systems, families, rule, "AB")
607
+ check = _decide(systems, families, rule, "B")
608
+ return {
609
+ "ready": True,
610
+ **main,
611
+ "without_fleurs": check,
612
+ "pick_changes_without_fleurs": (main["primary"], main["secondary"])
613
+ != (check["primary"], check["secondary"]),
614
+ "rule": {
615
+ "min_filler_recall": rule.min_filler_recall,
616
+ "min_reviewed_c": rule.min_reviewed_c,
617
+ "min_filler_ref_tokens": rule.min_filler_ref_tokens,
618
+ "min_loo_ref_tokens": rule.min_loo_ref_tokens,
619
+ "max_failed_ab": rule.max_failed_ab,
620
+ "max_failed_c": rule.max_failed_c,
621
+ },
622
+ }
src/nida/bench/b2.py ADDED
@@ -0,0 +1,684 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Benchmark B2: audio-VAE reconstruction (docs/benchmarks/B1-B2-tasarim.md §2).
2
+
3
+ Stages (idempotent, resumable):
4
+
5
+ 1. ``prepare`` — Turkish subsets from the B1 manifest + Expresso subsets streamed from
6
+ the official 36 GB tar; originals written to ``<workspace>/b2/orig`` (16 kHz Turkish,
7
+ 48 kHz Expresso, 24-bit). They live in the durable workspace (Drive), not the session
8
+ cache, so a new Colab session resumes without re-streaming 36 GB and without results
9
+ that point at vanished files.
10
+ 2. ``vae`` — VoxCPM2 AudioVAE V2 round trip (worker) → ``<workspace>/b2/vae`` (48 kHz).
11
+ 3. ``asr`` / ``quality`` — Whisper on 16 kHz copies of original and reconstruction
12
+ (clips with references), Audiobox Aesthetics on both.
13
+ 4. ``score`` — objective metrics per subset, incl. the "band-16k" baseline
14
+ (48 kHz → 16 kHz → 48 kHz) that isolates what the VAE's super-resolution adds.
15
+ 5. ``panel`` — 40 *blind* A/B pairs (random order per item; key stored outside the
16
+ HTML in ``manifests/b2_panel_key-<seed>.json``) and the scoring of the user's answers.
17
+
18
+ Audiobox resamples to 16 kHz internally, so its scores say nothing about the band above
19
+ 8 kHz; the high-band LSD / energy ratio cover that.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import json
25
+ import logging
26
+ import random
27
+ from collections.abc import Mapping, Sequence
28
+ from dataclasses import asdict, dataclass
29
+ from pathlib import Path
30
+ from typing import Any
31
+
32
+ from nida.bench import fetch
33
+ from nida.bench.audiometrics import (
34
+ align,
35
+ band_energy_ratio,
36
+ band_log_spectral_distance,
37
+ estimate_delay_samples,
38
+ log_mel_l1,
39
+ resample,
40
+ )
41
+ from nida.bench.clips import Clip, clip_from_obj, clip_to_obj
42
+ from nida.bench.expresso import (
43
+ is_wanted_member,
44
+ parse_member,
45
+ parse_read_transcriptions,
46
+ parse_vad_segments,
47
+ select_expresso,
48
+ )
49
+ from nida.bench.panel import AudioRef, PanelItem, PanelResult, PanelSpec, Question, write_panel
50
+ from nida.bench.prepare import Workspace
51
+ from nida.bench.protocol import WorkRequest, WorkResult
52
+ from nida.bench.registry import DATA_PINS
53
+ from nida.bench.textmetrics import (
54
+ cer_counts,
55
+ corpus_rate,
56
+ normalize_english,
57
+ normalize_for_scoring,
58
+ strip_fillers,
59
+ )
60
+ from nida.utils.io import atomic_write_json, atomic_write_text
61
+
62
+ __all__ = [
63
+ "B2Plan",
64
+ "asr_requests",
65
+ "b2_paths",
66
+ "build_b2_panel",
67
+ "clip_metrics",
68
+ "load_b2_manifest",
69
+ "missing_vae_outputs",
70
+ "prepare_b2",
71
+ "quality_requests",
72
+ "score_b2",
73
+ "score_b2_panel",
74
+ "vae_requests",
75
+ ]
76
+
77
+ log = logging.getLogger(__name__)
78
+
79
+ TR_SUBSETS = {"A": "tr-temiz", "B": "tr-medya"}
80
+ HIGH_BAND_HZ = (8000.0, 24000.0)
81
+ MAX_DELAY_MS = 50
82
+ #: High-band LSD is averaged over frames within this many dB of the loudest frame.
83
+ ACTIVE_WITHIN_DB = 40.0
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class B2Plan:
88
+ seed: int = 20261008
89
+ n_tr_per_layer: int = 30
90
+ n_whisper: int = 25
91
+ n_laughing: int = 25
92
+ n_projected: int = 25
93
+ n_nonverbal: int = 15
94
+ n_default: int = 10
95
+ n_panel_per_subset: int = 8
96
+ expresso_url: str = DATA_PINS["expresso_url"]
97
+
98
+ def expresso_counts(self) -> dict[str, int]:
99
+ return {
100
+ "whisper": self.n_whisper,
101
+ "laughing": self.n_laughing,
102
+ "projected": self.n_projected,
103
+ "nonverbal": self.n_nonverbal,
104
+ "default": self.n_default,
105
+ }
106
+
107
+
108
+ @dataclass(frozen=True)
109
+ class B2Paths:
110
+ orig: Path
111
+ vae: Path
112
+ asr16k: Path
113
+
114
+
115
+ def b2_paths(workspace: Workspace) -> B2Paths:
116
+ base = workspace.root / "b2"
117
+ return B2Paths(base / "orig", base / "vae", base / "asr16k")
118
+
119
+
120
+ def _orig_path(workspace: Workspace, clip: Clip) -> Path:
121
+ return b2_paths(workspace).orig / f"{clip.clip_id}.wav"
122
+
123
+
124
+ def _vae_path(workspace: Workspace, clip: Clip) -> Path:
125
+ return b2_paths(workspace).vae / f"{clip.clip_id}.wav"
126
+
127
+
128
+ def _write_expresso_original(source: Path, clip: Clip, dest: Path) -> None:
129
+ import soundfile # type: ignore[import-untyped] # bench extra
130
+
131
+ if dest.exists():
132
+ return
133
+ audio: Any
134
+ audio, rate_hz = soundfile.read(str(source), dtype="float64", always_2d=True)
135
+ if rate_hz != 48000:
136
+ raise ValueError(f"{source}: expected 48 kHz, got {rate_hz}")
137
+ channel_text = clip.meta.get("channel", "0")
138
+ if channel_text not in ("0", "1", "2"):
139
+ raise ValueError(f"{clip.clip_id}: unknown channel {channel_text!r} (expected 0, 1 or 2)")
140
+ channel = int(channel_text)
141
+ column = 0 if channel in (0, 1) else 1
142
+ if channel == 0 and audio.shape[1] != 1:
143
+ raise ValueError(f"{source}: read speech should be mono, got {audio.shape[1]} channels")
144
+ if channel in (1, 2) and audio.shape[1] != 2:
145
+ raise ValueError(f"{source}: conversational audio should be stereo")
146
+ start = round(clip.start_ms * 48)
147
+ stop = round(clip.end_ms * 48)
148
+ if not 0 <= start < stop <= audio.shape[0]:
149
+ raise ValueError(f"{source}: span {clip.start_ms}..{clip.end_ms} ms outside audio")
150
+ dest.parent.mkdir(parents=True, exist_ok=True)
151
+ tmp = dest.with_name(f".{dest.name}.tmp.wav")
152
+ soundfile.write(str(tmp), audio[start:stop, column], 48000, subtype="PCM_24")
153
+ tmp.replace(dest)
154
+
155
+
156
+ def prepare_b2(
157
+ workspace: Workspace,
158
+ b1_clips: Sequence[Clip],
159
+ plan: B2Plan | None = None,
160
+ *,
161
+ opener: Any = None, # Any: urllib-compatible opener (see nida.bench.fetch)
162
+ ) -> list[Clip]:
163
+ """Select B2 clips and write originals; the manifest is written once and reused.
164
+
165
+ Plan and clips are stored together in one atomically written ``b2_manifest.json``
166
+ after everything succeeded, so a stored plan and a stored clip list can never
167
+ disagree, and a failed run leaves no manifest behind.
168
+ """
169
+ import shutil
170
+
171
+ import soundfile
172
+
173
+ plan = plan if plan is not None else B2Plan()
174
+ manifest = _manifest_path(workspace)
175
+ stored_clips: list[Clip] | None = None
176
+ if manifest.exists():
177
+ stored_plan, stored_clips = load_b2_manifest(workspace)
178
+ if stored_plan != plan:
179
+ raise ValueError(f"existing B2 manifest was built with {stored_plan}, not {plan}")
180
+ rng = random.Random(plan.seed)
181
+ tr_clips: list[Clip] = []
182
+ for layer, subset in TR_SUBSETS.items():
183
+ pool = sorted((c for c in b1_clips if c.layer == layer), key=lambda c: c.clip_id)
184
+ if len(pool) < plan.n_tr_per_layer:
185
+ raise ValueError(f"B1 layer {layer} has {len(pool)} clips, need {plan.n_tr_per_layer}")
186
+ for clip in rng.sample(pool, plan.n_tr_per_layer):
187
+ tr_clips.append(
188
+ Clip.build(
189
+ source_id=clip.source_id,
190
+ layer=subset,
191
+ file_id=clip.file_id,
192
+ start_ms=clip.start_ms,
193
+ end_ms=clip.end_ms,
194
+ reference=clip.reference,
195
+ language="tr",
196
+ meta={**clip.meta, "b1_clip_id": clip.clip_id},
197
+ )
198
+ )
199
+
200
+ if stored_clips is not None and all(_orig_path(workspace, c).exists() for c in stored_clips):
201
+ return stored_clips
202
+
203
+ exp_dir = workspace.cache / "expresso"
204
+ extracted = fetch.stream_extract(
205
+ plan.expresso_url,
206
+ exp_dir,
207
+ is_wanted_member,
208
+ opener=opener if opener is not None else fetch.default_opener,
209
+ )
210
+ by_name = {p.relative_to(exp_dir).as_posix(): p for p in extracted}
211
+ vad_file = next((p for n, p in by_name.items() if n.endswith("VAD_segments.txt")), None)
212
+ tr_file = next((p for n, p in by_name.items() if n.endswith("read_transcriptions.txt")), None)
213
+ if vad_file is None or tr_file is None:
214
+ raise FileNotFoundError(
215
+ "Expresso archive lacks VAD_segments.txt or read_transcriptions.txt"
216
+ )
217
+ files: list[tuple[Any, int]] = []
218
+ for name, path in sorted(by_name.items()):
219
+ parsed = parse_member(name)
220
+ if parsed is not None:
221
+ info = soundfile.info(str(path))
222
+ # Floor, never round up: a length past the last sample makes spans invalid.
223
+ files.append((parsed, (1000 * info.frames) // info.samplerate))
224
+ selection = select_expresso(
225
+ files,
226
+ parse_vad_segments(vad_file.read_text(encoding="utf-8")),
227
+ parse_read_transcriptions(tr_file.read_text(encoding="utf-8")),
228
+ plan.expresso_counts(),
229
+ plan.seed,
230
+ )
231
+ for clip in selection.clips:
232
+ member = clip.file_id.split("#", 1)[0]
233
+ _write_expresso_original(by_name[member], clip, _orig_path(workspace, clip))
234
+ for clip in tr_clips:
235
+ dest = _orig_path(workspace, clip)
236
+ if not dest.exists():
237
+ source_clip_id = clip.meta["b1_clip_id"]
238
+ dest.parent.mkdir(parents=True, exist_ok=True)
239
+ shutil.copyfile(workspace.clips16k / f"{source_clip_id}.wav", dest)
240
+ clips = sorted([*tr_clips, *selection.clips], key=lambda c: (c.layer, c.clip_id))
241
+ if stored_clips is not None:
242
+ if [c.clip_id for c in stored_clips] != [c.clip_id for c in clips]:
243
+ raise ValueError("re-derived B2 clips differ from the stored manifest")
244
+ return stored_clips
245
+ atomic_write_text(
246
+ workspace.manifests / "b2_quarantine.jsonl",
247
+ "".join(json.dumps(asdict(q), ensure_ascii=False) + "\n" for q in selection.quarantined),
248
+ )
249
+ atomic_write_json(manifest, {"plan": asdict(plan), "clips": [clip_to_obj(c) for c in clips]})
250
+ return clips
251
+
252
+
253
+ def _manifest_path(workspace: Workspace) -> Path:
254
+ return workspace.manifests / "b2_manifest.json"
255
+
256
+
257
+ def load_b2_manifest(workspace: Workspace) -> tuple[B2Plan, list[Clip]]:
258
+ """Read ``manifests/b2_manifest.json`` (plan and clips, written together)."""
259
+ obj = json.loads(_manifest_path(workspace).read_text(encoding="utf-8"))
260
+ return B2Plan(**obj["plan"]), [clip_from_obj(c) for c in obj["clips"]]
261
+
262
+
263
+ # ------------------------------------------------------------------------ worker inputs
264
+
265
+
266
+ def vae_requests(workspace: Workspace, clips: Sequence[Clip]) -> list[WorkRequest]:
267
+ return [
268
+ WorkRequest(
269
+ c.clip_id, str(_orig_path(workspace, c)), {"out_path": str(_vae_path(workspace, c))}
270
+ )
271
+ for c in clips
272
+ ]
273
+
274
+
275
+ def missing_vae_outputs(workspace: Workspace, results: Mapping[str, WorkResult]) -> list[str]:
276
+ """Ids of successful VAE results whose output file no longer exists (to be redone)."""
277
+ missing: list[str] = []
278
+ for clip_id, result in results.items():
279
+ out = result.output.get("out_path") if result.output is not None else None
280
+ if result.ok and (not isinstance(out, str) or not Path(out).exists()):
281
+ missing.append(clip_id)
282
+ return missing
283
+
284
+
285
+ def _write_16k(source: Path, dest: Path) -> None:
286
+ import soundfile
287
+
288
+ if dest.exists():
289
+ return
290
+ audio: Any
291
+ audio, rate_hz = soundfile.read(str(source), dtype="float64")
292
+ if rate_hz != 16000:
293
+ audio = resample(audio, int(rate_hz), 16000)
294
+ peak = float(abs(audio).max())
295
+ if peak > 1.0:
296
+ audio = audio / peak
297
+ dest.parent.mkdir(parents=True, exist_ok=True)
298
+ tmp = dest.with_name(f".{dest.name}.tmp.wav")
299
+ soundfile.write(str(tmp), audio, 16000, subtype="PCM_16")
300
+ tmp.replace(dest)
301
+
302
+
303
+ def asr_requests(
304
+ workspace: Workspace,
305
+ clips: Sequence[Clip],
306
+ language: str,
307
+ vae_results: Mapping[str, WorkResult] | None = None,
308
+ ) -> list[WorkRequest]:
309
+ """Whisper requests for originals and reconstructions of clips with references.
310
+
311
+ Clips without a usable reconstruction (failed VAE item or missing file) get no
312
+ requests at all: the CER comparison is paired, and scoring counts them as skipped.
313
+ """
314
+ paths = b2_paths(workspace)
315
+ requests: list[WorkRequest] = []
316
+ for clip in clips:
317
+ if clip.language != language or not clip.reference:
318
+ continue
319
+ if not _usable_reconstruction(workspace, clip, vae_results):
320
+ continue
321
+ for kind, source in (
322
+ ("orig", _orig_path(workspace, clip)),
323
+ ("vae", _vae_path(workspace, clip)),
324
+ ):
325
+ dest = paths.asr16k / f"{clip.clip_id}__{kind}.wav"
326
+ _write_16k(source, dest)
327
+ requests.append(
328
+ WorkRequest(f"{clip.clip_id}__{kind}", str(dest), {"language": language})
329
+ )
330
+ return requests
331
+
332
+
333
+ def quality_requests(
334
+ workspace: Workspace,
335
+ clips: Sequence[Clip],
336
+ vae_results: Mapping[str, WorkResult] | None = None,
337
+ ) -> list[WorkRequest]:
338
+ """Audiobox requests for both members of every clip with a usable reconstruction."""
339
+ requests: list[WorkRequest] = []
340
+ for clip in clips:
341
+ if not _usable_reconstruction(workspace, clip, vae_results):
342
+ continue
343
+ requests.append(WorkRequest(f"{clip.clip_id}__orig", str(_orig_path(workspace, clip))))
344
+ requests.append(WorkRequest(f"{clip.clip_id}__vae", str(_vae_path(workspace, clip))))
345
+ return requests
346
+
347
+
348
+ # ------------------------------------------------------------------------------ scoring
349
+
350
+
351
+ def clip_metrics(orig: Any, orig_rate_hz: int, recon: Any, recon_rate_hz: int) -> dict[str, float]:
352
+ """Objective metrics for one original / reconstruction pair.
353
+
354
+ ``orig`` and ``recon`` are 1-D float arrays (Any: numpy arrays, numpy is a bench extra).
355
+ Both are compared at 16 kHz (mel @16k); when the original is 48 kHz they are also
356
+ compared at 48 kHz (mel @48k, high-band LSD and energy ratio). The reconstruction
357
+ is delay-aligned to the original first (cross-correlation at 16 kHz, ±50 ms).
358
+ """
359
+ o16 = resample(orig, orig_rate_hz, 16000)
360
+ r16 = resample(recon, recon_rate_hz, 16000)
361
+ delay = estimate_delay_samples(o16, r16, MAX_DELAY_MS * 16)
362
+ a16, b16 = align(o16, r16, delay)
363
+ out: dict[str, float] = {
364
+ "delay_ms": delay / 16,
365
+ "mel16_l1": log_mel_l1(a16, b16, 16000, 1024, 256, 80),
366
+ }
367
+ if orig_rate_hz == 48000:
368
+ r48 = resample(recon, recon_rate_hz, 48000)
369
+ a48, b48 = align(orig, r48, delay * 3)
370
+ out["mel48_l1"] = log_mel_l1(a48, b48, 48000, 2048, 480, 128)
371
+ out["hb_lsd_db"] = band_log_spectral_distance(
372
+ a48, b48, 48000, *HIGH_BAND_HZ, active_within_db=ACTIVE_WITHIN_DB
373
+ )
374
+ out["hb_energy_ratio"] = band_energy_ratio(a48, b48, 48000, *HIGH_BAND_HZ)
375
+ return out
376
+
377
+
378
+ def _mean(values: Sequence[float]) -> float | None:
379
+ return sum(values) / len(values) if values else None
380
+
381
+
382
+ def _result_text(result: WorkResult | None) -> str | None:
383
+ if result is None or result.output is None:
384
+ return None
385
+ raw = result.output.get("text")
386
+ return raw if isinstance(raw, str) else None
387
+
388
+
389
+ def _scoring_tokens(text: str, language: str) -> list[str]:
390
+ """Content tokens: Turkish pipeline without fillers, or the English normaliser."""
391
+ if language == "tr":
392
+ return strip_fillers(normalize_for_scoring(text))
393
+ return normalize_english(text)
394
+
395
+
396
+ def _usable_reconstruction(
397
+ workspace: Workspace, clip: Clip, vae_results: Mapping[str, WorkResult] | None
398
+ ) -> bool:
399
+ """A reconstruction counts only if its worker result is ok *and* the file exists."""
400
+ if vae_results is not None:
401
+ result = vae_results.get(clip.clip_id)
402
+ if result is None or not result.ok:
403
+ return False
404
+ return _vae_path(workspace, clip).exists()
405
+
406
+
407
+ def score_b2(
408
+ workspace: Workspace,
409
+ clips: Sequence[Clip],
410
+ asr_results: Mapping[str, WorkResult],
411
+ quality_results: Mapping[str, WorkResult],
412
+ vae_results: Mapping[str, WorkResult] | None = None,
413
+ ) -> dict[str, object]:
414
+ """Aggregate per-subset metrics for the VAE and the band-16k baseline.
415
+
416
+ Clips without a usable reconstruction (failed worker result or missing file) are
417
+ skipped and counted (``skipped``); every subset still gets a row (``n_scored`` may be 0).
418
+ Quality deltas are averaged over *paired* clips (both members scored); CER compares
419
+ original vs reconstruction only on clips where both transcriptions exist.
420
+ """
421
+ import soundfile
422
+
423
+ per_subset: dict[str, dict[str, list[float]]] = {}
424
+ cer_pairs: dict[str, dict[str, list[Any]]] = {}
425
+ skipped: dict[str, int] = {}
426
+ for clip in clips:
427
+ vae_file = _vae_path(workspace, clip)
428
+ if not _usable_reconstruction(workspace, clip, vae_results):
429
+ skipped["missing_reconstruction"] = skipped.get("missing_reconstruction", 0) + 1
430
+ continue
431
+ bucket = per_subset.setdefault(clip.layer, {})
432
+ bucket.setdefault("_n_scored", []).append(1.0)
433
+ orig, orig_rate = soundfile.read(str(_orig_path(workspace, clip)), dtype="float64")
434
+ recon, recon_rate = soundfile.read(str(vae_file), dtype="float64")
435
+ for key, value in clip_metrics(orig, int(orig_rate), recon, int(recon_rate)).items():
436
+ bucket.setdefault(f"vae_{key}", []).append(value)
437
+ if int(orig_rate) == 48000:
438
+ band = resample(resample(orig, 48000, 16000), 16000, 48000)
439
+ for key, value in clip_metrics(orig, 48000, band, 48000).items():
440
+ bucket.setdefault(f"band16k_{key}", []).append(value)
441
+ pair = [quality_results.get(f"{clip.clip_id}__{kind}") for kind in ("orig", "vae")]
442
+ if all(q is not None and q.output is not None for q in pair):
443
+ non_numeric = False
444
+ for axis in ("PQ", "CE"):
445
+ scored = [q.output.get(axis) for q in pair if q is not None and q.output]
446
+ numeric = [
447
+ v for v in scored if isinstance(v, int | float) and not isinstance(v, bool)
448
+ ]
449
+ if len(numeric) != 2:
450
+ non_numeric = True
451
+ else:
452
+ orig_score, vae_score = float(numeric[0]), float(numeric[1])
453
+ bucket.setdefault(f"orig_{axis}", []).append(orig_score)
454
+ bucket.setdefault(f"vae_{axis}", []).append(vae_score)
455
+ bucket.setdefault(f"delta_{axis}", []).append(vae_score - orig_score)
456
+ if non_numeric: # counted once per clip, not per axis
457
+ skipped["non_numeric_quality"] = skipped.get("non_numeric_quality", 0) + 1
458
+ else:
459
+ skipped["missing_quality_pair"] = skipped.get("missing_quality_pair", 0) + 1
460
+ if clip.reference:
461
+ orig_text = _result_text(asr_results.get(f"{clip.clip_id}__orig"))
462
+ vae_text = _result_text(asr_results.get(f"{clip.clip_id}__vae"))
463
+ if orig_text is None or vae_text is None:
464
+ skipped["missing_asr_pair"] = skipped.get("missing_asr_pair", 0) + 1
465
+ else:
466
+ ref_tokens = _scoring_tokens(clip.reference, clip.language)
467
+ for kind, text in (("orig", orig_text), ("vae", vae_text)):
468
+ cer_pairs.setdefault(clip.layer, {}).setdefault(kind, []).append(
469
+ cer_counts(ref_tokens, _scoring_tokens(text, clip.language))
470
+ )
471
+ for clip in clips: # subsets with nothing usable still get an (empty) row
472
+ per_subset.setdefault(clip.layer, {})
473
+ summary: dict[str, dict[str, float | int | None]] = {}
474
+ for subset, values in sorted(per_subset.items()):
475
+ row: dict[str, float | int | None] = {
476
+ "n": sum(1 for c in clips if c.layer == subset),
477
+ "n_scored": len(values.pop("_n_scored", [])),
478
+ }
479
+ row.update({key: _mean(vals) for key, vals in sorted(values.items())})
480
+ pairs = cer_pairs.get(subset)
481
+ if pairs:
482
+ row["cer_orig"] = corpus_rate(pairs["orig"])
483
+ row["cer_vae"] = corpus_rate(pairs["vae"])
484
+ row["n_cer_pairs"] = len(pairs["orig"])
485
+ summary[subset] = row
486
+ return {"subsets": summary, "skipped": skipped}
487
+
488
+
489
+ # -------------------------------------------------------------------------------- panel
490
+
491
+
492
+ B2_QUESTIONS = (
493
+ Question(
494
+ "ayni_ifade",
495
+ "A ve B aynı ifadeyi (fısıltı / gülme / bağırma / sözsüz ses / ton) taşıyor mu?",
496
+ "choice",
497
+ ("evet", "kısmen", "hayır"),
498
+ ),
499
+ Question(
500
+ "kalite_a", "A'nın ses kalitesi (1 kötü, 5 kusursuz)", "scale", ("1", "2", "3", "4", "5")
501
+ ),
502
+ Question(
503
+ "kalite_b", "B'nin ses kalitesi (1 kötü, 5 kusursuz)", "scale", ("1", "2", "3", "4", "5")
504
+ ),
505
+ )
506
+ #: Pre-registered acceptance (design §2.4): per expressive subset, the share of "evet"
507
+ #: ("kısmen" counts as *not* preserved) and the mean quality gap original − VAE.
508
+ MIN_PRESERVED_SHARE = 0.8
509
+ MAX_QUALITY_GAP = 1.0
510
+ PANEL_SUBSETS = ("tr", "whisper", "laughing", "projected", "nonverbal")
511
+
512
+
513
+ def _panel_subset(clip: Clip) -> str:
514
+ return "tr" if clip.layer.startswith("tr-") else clip.layer
515
+
516
+
517
+ def _key_path(workspace: Workspace, plan: B2Plan) -> Path:
518
+ return workspace.manifests / f"b2_panel_key-{plan.seed}.json"
519
+
520
+
521
+ def build_b2_panel(
522
+ workspace: Workspace,
523
+ clips: Sequence[Clip],
524
+ plan: B2Plan,
525
+ vae_results: Mapping[str, WorkResult] | None = None,
526
+ ) -> Path:
527
+ """Write the blind 40-pair listening panel and its answer key.
528
+
529
+ Per item the order of original and reconstruction is random and both are labelled
530
+ neutrally ("A", "B"); the key lives in ``manifests/`` and never in the HTML. When
531
+ either member peaks above full scale, the *same* gain is applied to both so clipping
532
+ does not bias the comparison; the gain is recorded in the key.
533
+ """
534
+ import numpy as np
535
+ import soundfile
536
+
537
+ rng = random.Random(plan.seed + 2)
538
+ chosen: list[Clip] = []
539
+ excluded: dict[str, int] = {}
540
+ for subset in PANEL_SUBSETS:
541
+ everything = sorted(
542
+ (c for c in clips if _panel_subset(c) == subset), key=lambda c: c.clip_id
543
+ )
544
+ pool = [c for c in everything if _usable_reconstruction(workspace, c, vae_results)]
545
+ if len(pool) < len(everything):
546
+ excluded[subset] = len(everything) - len(pool)
547
+ log.warning(
548
+ "B2 panel: %d %s clips lack a usable reconstruction", excluded[subset], subset
549
+ )
550
+ chosen.extend(rng.sample(pool, min(plan.n_panel_per_subset, len(pool))))
551
+ rng.shuffle(chosen)
552
+ panel_id = f"b2-{plan.seed}"
553
+ audio_dir = workspace.cache / "panel_audio" / panel_id
554
+ items: list[PanelItem] = []
555
+ key: dict[str, dict[str, object]] = {}
556
+ for clip in chosen:
557
+ orig, orig_rate = soundfile.read(str(_orig_path(workspace, clip)), dtype="float64")
558
+ recon, recon_rate = soundfile.read(str(_vae_path(workspace, clip)), dtype="float64")
559
+ peak = max(float(np.max(np.abs(orig))), float(np.max(np.abs(recon))), 1e-12)
560
+ gain = 1.0 / peak if peak > 1.0 else 1.0
561
+ first_is_orig = rng.random() < 0.5
562
+ members = [("orig", orig, orig_rate), ("vae", recon, recon_rate)]
563
+ if not first_is_orig:
564
+ members.reverse()
565
+ refs: list[AudioRef] = []
566
+ for label, (_kind, audio, rate_hz) in zip(("A", "B"), members, strict=True):
567
+ # Always rewritten: a cached A/B file could disagree with a re-drawn key.
568
+ dest = audio_dir / f"{clip.clip_id}__{label}.flac"
569
+ dest.parent.mkdir(parents=True, exist_ok=True)
570
+ soundfile.write(str(dest), audio * gain, int(rate_hz), subtype="PCM_16")
571
+ refs.append(AudioRef(label, dest))
572
+ key[clip.clip_id] = {
573
+ "subset": _panel_subset(clip),
574
+ "a_is": "orig" if first_is_orig else "vae",
575
+ "gain": gain,
576
+ }
577
+ items.append(
578
+ PanelItem(
579
+ item_id=clip.clip_id,
580
+ audios=tuple(refs),
581
+ text=clip.reference or "",
582
+ info=f"{clip.layer} · {clip.duration_ms / 1000:.1f} s",
583
+ )
584
+ )
585
+ spec = PanelSpec(
586
+ panel_id=panel_id,
587
+ title="B2 — Ses kodlayıcı dinleme testi",
588
+ instructions=(
589
+ "Her satırda A ve B'yi dinleyin. Biri orijinal kayıt, diğeri modelden geçirilmiş "
590
+ "hâli; hangisinin hangisi olduğu rastgele ve gizli. İkisinin aynı ifadeyi "
591
+ "(fısıltı, gülme, bağırma, iç çekme vb.) taşıyıp taşımadığını ve her birinin "
592
+ "kalitesini işaretleyin. Kulaklıkla dinlemeniz önerilir."
593
+ ),
594
+ questions=B2_QUESTIONS,
595
+ items=tuple(items),
596
+ )
597
+ atomic_write_json(
598
+ _key_path(workspace, plan),
599
+ {
600
+ "panel_id": panel_id,
601
+ "items": key,
602
+ "excluded_missing_reconstruction": excluded,
603
+ "min_per_subset": plan.n_panel_per_subset,
604
+ },
605
+ )
606
+ return write_panel(spec, workspace.panels / panel_id)
607
+
608
+
609
+ def score_b2_panel(
610
+ results: Sequence[PanelResult],
611
+ key: Mapping[str, Mapping[str, object]],
612
+ *,
613
+ min_per_subset: int,
614
+ ) -> dict[str, object]:
615
+ """Un-blind the panel answers and apply the pre-registered acceptance rule.
616
+
617
+ Returns per subset: answered items, share of "evet" (preserved), share of "kısmen",
618
+ mean quality of original and reconstruction, the mean gap (orig − vae) and whether the
619
+ subset passes (``preserved_share >= 0.8`` and ``quality_gap <= 1``; the ``tr`` subset is
620
+ reported but carries no expressive-preservation requirement). ``all_pass`` requires all
621
+ four expressive subsets to be present, *every* key item of every subset to be fully
622
+ answered, at least ``min_per_subset`` items per expressive subset (pre-registered:
623
+ the panel size per subset, 8), and every subset to pass — thin or partial evidence
624
+ never passes.
625
+
626
+ Raises:
627
+ ValueError: if an answered item is missing from the key.
628
+ """
629
+ rows: dict[str, dict[str, list[float]]] = {}
630
+ for result in results:
631
+ if result.item_id not in key:
632
+ raise ValueError(f"panel item {result.item_id} not in the answer key")
633
+ entry = key[result.item_id]
634
+ same = result.answers.get("ayni_ifade", "")
635
+ qa, qb = result.answers.get("kalite_a", ""), result.answers.get("kalite_b", "")
636
+ if not same or not qa or not qb:
637
+ continue # unanswered rows do not count either way
638
+ orig_q, vae_q = (
639
+ (float(qa), float(qb)) if entry["a_is"] == "orig" else (float(qb), float(qa))
640
+ )
641
+ bucket = rows.setdefault(str(entry["subset"]), {})
642
+ bucket.setdefault("preserved", []).append(1.0 if same == "evet" else 0.0)
643
+ bucket.setdefault("partial", []).append(1.0 if same == "kısmen" else 0.0)
644
+ bucket.setdefault("orig_q", []).append(orig_q)
645
+ bucket.setdefault("vae_q", []).append(vae_q)
646
+ out: dict[str, dict[str, object]] = {}
647
+ for subset, bucket in sorted(rows.items()):
648
+ preserved = _mean(bucket["preserved"]) or 0.0
649
+ orig_q = _mean(bucket["orig_q"]) or 0.0
650
+ vae_q = _mean(bucket["vae_q"]) or 0.0
651
+ gap = orig_q - vae_q
652
+ passes = gap <= MAX_QUALITY_GAP and (subset == "tr" or preserved >= MIN_PRESERVED_SHARE)
653
+ out[subset] = {
654
+ "n_answered": len(bucket["preserved"]),
655
+ "preserved_share": preserved,
656
+ "partial_share": _mean(bucket["partial"]),
657
+ "quality_orig": orig_q,
658
+ "quality_vae": vae_q,
659
+ "quality_gap": gap,
660
+ "passes": passes,
661
+ }
662
+ in_key: dict[str, int] = {}
663
+ for entry in key.values():
664
+ in_key[str(entry["subset"])] = in_key.get(str(entry["subset"]), 0) + 1
665
+ unanswered = {
666
+ subset: count - int(str(out.get(subset, {}).get("n_answered", 0)))
667
+ for subset, count in sorted(in_key.items())
668
+ if count > int(str(out.get(subset, {}).get("n_answered", 0)))
669
+ }
670
+ expressive = [s for s in PANEL_SUBSETS if s != "tr"]
671
+ missing_subsets = [s for s in expressive if s not in out]
672
+ too_few = {s: in_key.get(s, 0) for s in expressive if in_key.get(s, 0) < min_per_subset}
673
+ return {
674
+ "subsets": out,
675
+ "rule": {"min_preserved_share": MIN_PRESERVED_SHARE, "max_quality_gap": MAX_QUALITY_GAP},
676
+ "unanswered": unanswered,
677
+ "missing_subsets": missing_subsets,
678
+ "too_few_items": too_few,
679
+ "min_per_subset": min_per_subset,
680
+ "all_pass": not missing_subsets
681
+ and not unanswered
682
+ and not too_few
683
+ and all(bool(v["passes"]) for v in out.values()),
684
+ }
src/nida/bench/clips.py ADDED
@@ -0,0 +1,510 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Clip records and deterministic clip selection for benchmarks B1 / B2.
2
+
3
+ Selection functions are pure: they take already-parsed source metadata and a seed, and
4
+ return the chosen clips plus a quarantine list with a machine-readable reason for every
5
+ candidate that was rejected (docs/03-data-pipeline.md: nothing is dropped silently).
6
+ Candidates that are merely *not sampled* are not quarantined; only rule violations are.
7
+
8
+ Ids follow the pipeline convention ``sha1(source_id, file_id, start_ms, end_ms)[:16]``.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import hashlib
14
+ import random
15
+ from collections import defaultdict
16
+ from collections.abc import Iterable, Mapping, Sequence
17
+ from dataclasses import dataclass, field
18
+ from typing import Literal
19
+
20
+ from nida.bench.textmetrics import FILLERS, has_digits, normalize_for_scoring
21
+ from nida.text.turkish import nfc
22
+
23
+ __all__ = [
24
+ "Clip",
25
+ "FleursRow",
26
+ "MediaSpeechItem",
27
+ "Quarantined",
28
+ "Selection",
29
+ "YodasUtt",
30
+ "clip_from_obj",
31
+ "clip_to_obj",
32
+ "make_clip_id",
33
+ "parse_yodas_utt_id",
34
+ "select_fleurs",
35
+ "select_mediaspeech",
36
+ "select_yodas_final",
37
+ "select_yodas_pool",
38
+ "text_has_filler",
39
+ "yodas_video_id",
40
+ ]
41
+
42
+ Language = Literal["tr", "en"]
43
+
44
+ #: Gender labels in the FLEURS TSV; anything else is quarantined, never guessed.
45
+ FLEURS_GENDERS = frozenset({"MALE", "FEMALE"})
46
+ #: Allowed mismatch between a YODAS duration and the span encoded in its utterance id.
47
+ YODAS_DURATION_TOLERANCE_MS = 50
48
+
49
+
50
+ def make_clip_id(source_id: str, file_id: str, start_ms: int, end_ms: int) -> str:
51
+ """Content-addressed clip id (first 16 hex chars of SHA-1)."""
52
+ if start_ms < 0 or end_ms <= start_ms:
53
+ raise ValueError(f"invalid span {start_ms}..{end_ms} ms for {source_id}/{file_id}")
54
+ key = "\x1f".join((source_id, file_id, str(start_ms), str(end_ms)))
55
+ return hashlib.sha1(key.encode("utf-8")).hexdigest()[:16]
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class Clip:
60
+ """One benchmark clip.
61
+
62
+ Attributes:
63
+ clip_id: ``make_clip_id(source_id, file_id, start_ms, end_ms)``.
64
+ source_id: Id in ``configs/sources.yaml``.
65
+ layer: Benchmark layer / subset name (e.g. ``"A"``, ``"whisper"``).
66
+ file_id: File or utterance id inside the source.
67
+ start_ms: Start inside the source file, milliseconds.
68
+ end_ms: End inside the source file, milliseconds (exclusive).
69
+ reference: Reference transcript (raw), or ``None`` when not yet available.
70
+ language: ISO 639-1 code.
71
+ meta: Free string metadata (gender, video id, style, ...).
72
+ """
73
+
74
+ clip_id: str
75
+ source_id: str
76
+ layer: str
77
+ file_id: str
78
+ start_ms: int
79
+ end_ms: int
80
+ reference: str | None
81
+ language: Language
82
+ meta: Mapping[str, str] = field(default_factory=dict)
83
+
84
+ @property
85
+ def duration_ms(self) -> int:
86
+ return self.end_ms - self.start_ms
87
+
88
+ @staticmethod
89
+ def build(
90
+ *,
91
+ source_id: str,
92
+ layer: str,
93
+ file_id: str,
94
+ start_ms: int,
95
+ end_ms: int,
96
+ reference: str | None,
97
+ language: Language,
98
+ meta: Mapping[str, str] | None = None,
99
+ ) -> Clip:
100
+ return Clip(
101
+ clip_id=make_clip_id(source_id, file_id, start_ms, end_ms),
102
+ source_id=source_id,
103
+ layer=layer,
104
+ file_id=file_id,
105
+ start_ms=start_ms,
106
+ end_ms=end_ms,
107
+ reference=None if reference is None else nfc(reference),
108
+ language=language,
109
+ meta=dict(meta or {}),
110
+ )
111
+
112
+
113
+ @dataclass(frozen=True)
114
+ class Quarantined:
115
+ """A rejected candidate (docs/03-data-pipeline.md quarantine record)."""
116
+
117
+ item_id: str
118
+ stage: str
119
+ reason: str
120
+ details: str = ""
121
+
122
+
123
+ @dataclass(frozen=True)
124
+ class Selection:
125
+ clips: tuple[Clip, ...]
126
+ quarantined: tuple[Quarantined, ...]
127
+
128
+
129
+ def _check_count(n: int) -> None:
130
+ if n < 1:
131
+ raise ValueError(f"requested clip count must be >= 1, got {n}")
132
+
133
+
134
+ def _check_duration_window(min_s: float, max_s: float) -> None:
135
+ if not 0 < min_s < max_s:
136
+ raise ValueError(f"invalid duration window {min_s}..{max_s} s")
137
+
138
+
139
+ # ------------------------------------------------------------------------------- FLEURS
140
+
141
+
142
+ @dataclass(frozen=True)
143
+ class FleursRow:
144
+ """One row of FLEURS ``<split>.tsv`` (16 kHz audio)."""
145
+
146
+ sentence_id: str
147
+ file_name: str
148
+ raw_transcription: str
149
+ num_samples: int
150
+ gender: str
151
+
152
+
153
+ def select_fleurs(
154
+ rows: Iterable[FleursRow],
155
+ n: int,
156
+ seed: int,
157
+ *,
158
+ min_s: float = 2.0,
159
+ max_s: float = 25.0,
160
+ source_id: str = "fleurs_tr",
161
+ layer: str = "A",
162
+ ) -> Selection:
163
+ """Pick ``n`` FLEURS recordings: one per sentence id, gender-balanced.
164
+
165
+ Rules: duration in ``[min_s, max_s]``; reference non-empty and without digits (number
166
+ verbalisation is not decided yet, R6); at most one recording per sentence id (several
167
+ speakers read the same sentence and the TSV has no speaker id). Gender balance: half
168
+ from ``MALE`` and half from ``FEMALE`` where possible, the remainder from whatever is left.
169
+
170
+ Raises:
171
+ ValueError: on invalid arguments or if fewer than ``n`` eligible sentences exist.
172
+ """
173
+ _check_count(n)
174
+ _check_duration_window(min_s, max_s)
175
+ quarantined: list[Quarantined] = []
176
+ by_sentence: dict[str, list[FleursRow]] = defaultdict(list)
177
+ for row in rows:
178
+ duration_s = row.num_samples / 16000
179
+ if row.gender not in FLEURS_GENDERS:
180
+ quarantined.append(
181
+ Quarantined(row.file_name, "select", "unexpected_gender", row.gender)
182
+ )
183
+ elif not min_s <= duration_s <= max_s:
184
+ quarantined.append(
185
+ Quarantined(row.file_name, "select", "duration_out_of_range", f"{duration_s:.2f}s")
186
+ )
187
+ elif not normalize_for_scoring(row.raw_transcription):
188
+ quarantined.append(Quarantined(row.file_name, "select", "empty_reference"))
189
+ elif has_digits(row.raw_transcription):
190
+ quarantined.append(Quarantined(row.file_name, "select", "reference_has_digits"))
191
+ else:
192
+ by_sentence[row.sentence_id].append(row)
193
+ rng = random.Random(seed)
194
+ one_per_sentence: list[FleursRow] = []
195
+ for sentence_id in sorted(by_sentence):
196
+ recordings = sorted(by_sentence[sentence_id], key=lambda r: r.file_name)
197
+ one_per_sentence.append(rng.choice(recordings))
198
+ if len(one_per_sentence) < n:
199
+ raise ValueError(f"only {len(one_per_sentence)} eligible FLEURS sentences, need {n}")
200
+ pools: dict[str, list[FleursRow]] = defaultdict(list)
201
+ for row in one_per_sentence:
202
+ pools[row.gender].append(row)
203
+ for pool in pools.values():
204
+ rng.shuffle(pool)
205
+ chosen: list[FleursRow] = []
206
+ half = n // 2
207
+ for gender in ("MALE", "FEMALE"):
208
+ chosen.extend(pools[gender][:half])
209
+ del pools[gender][:half]
210
+ rest = [row for gender in sorted(pools) for row in pools[gender]]
211
+ rng.shuffle(rest)
212
+ chosen.extend(rest[: n - len(chosen)])
213
+ clips = tuple(
214
+ Clip.build(
215
+ source_id=source_id,
216
+ layer=layer,
217
+ file_id=row.file_name,
218
+ start_ms=0,
219
+ end_ms=round(row.num_samples / 16),
220
+ reference=row.raw_transcription,
221
+ language="tr",
222
+ meta={"gender": row.gender, "sentence_id": row.sentence_id},
223
+ )
224
+ for row in sorted(chosen, key=lambda r: r.file_name)
225
+ )
226
+ return Selection(clips, tuple(quarantined))
227
+
228
+
229
+ # --------------------------------------------------------------------------- MediaSpeech
230
+
231
+
232
+ @dataclass(frozen=True)
233
+ class MediaSpeechItem:
234
+ file_id: str # uuid stem of TR/<uuid>.flac
235
+ text: str
236
+ duration_ms: int
237
+
238
+
239
+ def select_mediaspeech(
240
+ items: Iterable[MediaSpeechItem],
241
+ n: int,
242
+ seed: int,
243
+ *,
244
+ min_s: float = 2.0,
245
+ max_s: float = 20.0,
246
+ source_id: str = "mediaspeech_tr",
247
+ layer: str = "B",
248
+ ) -> Selection:
249
+ """Pick ``n`` MediaSpeech segments with the same reference rules as FLEURS."""
250
+ _check_count(n)
251
+ _check_duration_window(min_s, max_s)
252
+ quarantined: list[Quarantined] = []
253
+ eligible: list[MediaSpeechItem] = []
254
+ for item in sorted(items, key=lambda i: i.file_id):
255
+ duration_s = item.duration_ms / 1000
256
+ if not min_s <= duration_s <= max_s:
257
+ quarantined.append(
258
+ Quarantined(item.file_id, "select", "duration_out_of_range", f"{duration_s:.2f}s")
259
+ )
260
+ elif not normalize_for_scoring(item.text):
261
+ quarantined.append(Quarantined(item.file_id, "select", "empty_reference"))
262
+ elif has_digits(item.text):
263
+ quarantined.append(Quarantined(item.file_id, "select", "reference_has_digits"))
264
+ else:
265
+ eligible.append(item)
266
+ if len(eligible) < n:
267
+ raise ValueError(f"only {len(eligible)} eligible MediaSpeech items, need {n}")
268
+ chosen = random.Random(seed).sample(eligible, n)
269
+ clips = tuple(
270
+ Clip.build(
271
+ source_id=source_id,
272
+ layer=layer,
273
+ file_id=item.file_id,
274
+ start_ms=0,
275
+ end_ms=item.duration_ms,
276
+ reference=item.text,
277
+ language="tr",
278
+ )
279
+ for item in sorted(chosen, key=lambda i: i.file_id)
280
+ )
281
+ return Selection(clips, tuple(quarantined))
282
+
283
+
284
+ # --------------------------------------------------------------------------------- YODAS
285
+
286
+
287
+ @dataclass(frozen=True)
288
+ class YodasUtt:
289
+ utt_id: str
290
+ caption: str
291
+ duration_ms: int
292
+ shard: str = "" # archive shard holding the audio ("" = unknown / single shard)
293
+
294
+
295
+ def yodas_video_id(utt_id: str) -> str:
296
+ """Video id of a ``<video>-<seg>-<start_cs>-<end_cs>`` utterance id.
297
+
298
+ Only the shape is checked, not the span: a listing line with an empty span still
299
+ names its video (de-duplication must see it; ingest quarantines the utterance).
300
+
301
+ Raises:
302
+ ValueError: if the id does not have the expected shape.
303
+ """
304
+ parts = utt_id.rsplit("-", 3)
305
+ if len(parts) != 4 or not all(p.isdigit() for p in parts[1:]) or not parts[0]:
306
+ raise ValueError(f"unexpected YODAS utterance id {utt_id!r}")
307
+ return parts[0]
308
+
309
+
310
+ def parse_yodas_utt_id(utt_id: str) -> tuple[str, int, int]:
311
+ """Split ``<video>-<seg>-<start_cs>-<end_cs>`` into (video_id, start_ms, end_ms).
312
+
313
+ YouTube ids may contain ``-``, so the id is split from the right. Times are in
314
+ centiseconds in the id.
315
+
316
+ Raises:
317
+ ValueError: if the id does not have the expected shape.
318
+ """
319
+ video_id = yodas_video_id(utt_id)
320
+ parts = utt_id.rsplit("-", 3)
321
+ start_ms, end_ms = int(parts[2]) * 10, int(parts[3]) * 10
322
+ if end_ms <= start_ms:
323
+ raise ValueError(f"empty span in YODAS utterance id {utt_id!r}")
324
+ return video_id, start_ms, end_ms
325
+
326
+
327
+ def select_yodas_pool(
328
+ utts: Iterable[YodasUtt],
329
+ n: int,
330
+ seed: int,
331
+ *,
332
+ min_s: float = 3.0,
333
+ max_s: float = 15.0,
334
+ max_per_video: int = 3,
335
+ source_id: str = "yodas_tr",
336
+ layer: str = "C",
337
+ ) -> Selection:
338
+ """Pick a candidate pool for layer C (to be transcribed by every ASR first).
339
+
340
+ Rules: duration in range; caption non-empty and digit-free (captions become the
341
+ draft the user corrects, so they must not contain digits either); at most
342
+ ``max_per_video`` clips per video (the video id stands in for the speaker).
343
+ ``reference`` is ``None``: captions are not references; the caption is stored in
344
+ ``meta["caption"]``.
345
+ """
346
+ _check_count(n)
347
+ _check_duration_window(min_s, max_s)
348
+ if max_per_video < 1:
349
+ raise ValueError(f"max_per_video must be >= 1, got {max_per_video}")
350
+ quarantined: list[Quarantined] = []
351
+ by_video: dict[str, list[tuple[YodasUtt, int, int]]] = defaultdict(list)
352
+ for utt in sorted(utts, key=lambda u: u.utt_id):
353
+ try:
354
+ video_id, start_ms, end_ms = parse_yodas_utt_id(utt.utt_id)
355
+ except ValueError as err:
356
+ quarantined.append(Quarantined(utt.utt_id, "select", "bad_utt_id", str(err)))
357
+ continue
358
+ duration_s = utt.duration_ms / 1000
359
+ span_ms = end_ms - start_ms
360
+ if abs(utt.duration_ms - span_ms) > YODAS_DURATION_TOLERANCE_MS:
361
+ quarantined.append(
362
+ Quarantined(
363
+ utt.utt_id, "select", "duration_mismatch", f"{utt.duration_ms} vs {span_ms} ms"
364
+ )
365
+ )
366
+ elif not min_s <= duration_s <= max_s:
367
+ quarantined.append(
368
+ Quarantined(utt.utt_id, "select", "duration_out_of_range", f"{duration_s:.2f}s")
369
+ )
370
+ elif not normalize_for_scoring(utt.caption):
371
+ quarantined.append(Quarantined(utt.utt_id, "select", "empty_caption"))
372
+ elif has_digits(utt.caption):
373
+ quarantined.append(Quarantined(utt.utt_id, "select", "caption_has_digits"))
374
+ else:
375
+ by_video[video_id].append((utt, start_ms, end_ms))
376
+ rng = random.Random(seed)
377
+ eligible: list[tuple[str, YodasUtt]] = []
378
+ for video_id in sorted(by_video):
379
+ candidates = by_video[video_id]
380
+ picked = rng.sample(candidates, min(max_per_video, len(candidates)))
381
+ eligible.extend((video_id, utt) for utt, _, _ in picked)
382
+ if len(eligible) < n:
383
+ raise ValueError(f"only {len(eligible)} eligible YODAS utterances, need {n}")
384
+ chosen = rng.sample(eligible, n)
385
+ clips = tuple(
386
+ Clip.build(
387
+ source_id=source_id,
388
+ layer=layer,
389
+ file_id=utt.utt_id,
390
+ start_ms=0,
391
+ end_ms=utt.duration_ms,
392
+ reference=None,
393
+ language="tr",
394
+ meta={"video_id": video_id, "caption": nfc(utt.caption)}
395
+ | ({"shard": utt.shard} if utt.shard else {}),
396
+ )
397
+ for video_id, utt in sorted(chosen, key=lambda pair: pair[1].utt_id)
398
+ )
399
+ return Selection(clips, tuple(quarantined))
400
+
401
+
402
+ def text_has_filler(text: str) -> bool:
403
+ """True if ``text`` contains a canonical filler (tag-spec section 4).
404
+
405
+ The single definition of the layer-C "filler signal": used both to select clips and
406
+ to record why they were selected (``nida.bench.b1.FillerProvenance``).
407
+ """
408
+ return any(token in FILLERS for token in normalize_for_scoring(text))
409
+
410
+
411
+ def select_yodas_final(
412
+ pool: Sequence[Clip],
413
+ hypotheses: Mapping[str, Sequence[str]],
414
+ n: int,
415
+ seed: int,
416
+ *,
417
+ min_filler_clips: int,
418
+ ) -> tuple[Clip, ...]:
419
+ """Choose the ``n`` layer-C clips the user will transcribe, favouring fillers.
420
+
421
+ A clip has a *filler signal* if its caption or any model hypothesis contains a
422
+ canonical filler (tag-spec section 4). Filler-signal clips are taken first (seeded
423
+ order) up to ``n``; the rest is filled with other pool clips. Hypotheses only steer
424
+ *which* clips are reviewed, never the reference text itself.
425
+
426
+ Args:
427
+ pool: Layer-C pool from :func:`select_yodas_pool`.
428
+ hypotheses: ``clip_id`` → raw hypothesis texts from every ASR.
429
+ n: Number of clips to return.
430
+ seed: RNG seed.
431
+ min_filler_clips: Required number of filler-signal clips (ValueError if the pool
432
+ has fewer, so a filler-poor pool is reported instead of silently accepted).
433
+ """
434
+ _check_count(n)
435
+ if not 0 <= min_filler_clips <= n:
436
+ raise ValueError(f"min_filler_clips must be in [0, {n}], got {min_filler_clips}")
437
+ if len(pool) < n:
438
+ raise ValueError(f"pool has {len(pool)} clips, need {n}")
439
+
440
+ def has_filler_signal(clip: Clip) -> bool:
441
+ texts = [clip.meta.get("caption", ""), *hypotheses.get(clip.clip_id, ())]
442
+ return any(text_has_filler(text) for text in texts)
443
+
444
+ ordered = sorted(pool, key=lambda c: c.clip_id)
445
+ with_filler = [c for c in ordered if has_filler_signal(c)]
446
+ without = [c for c in ordered if not has_filler_signal(c)]
447
+ if len(with_filler) < min_filler_clips:
448
+ raise ValueError(
449
+ f"only {len(with_filler)} pool clips show a filler, need {min_filler_clips}"
450
+ )
451
+ rng = random.Random(seed)
452
+ rng.shuffle(with_filler)
453
+ rng.shuffle(without)
454
+ chosen = (with_filler + without)[:n]
455
+ return tuple(sorted(chosen, key=lambda c: c.clip_id))
456
+
457
+
458
+ # ------------------------------------------------------------------------- serialisation
459
+
460
+
461
+ def clip_to_obj(clip: Clip) -> dict[str, object]:
462
+ """JSON-compatible representation of a clip."""
463
+ return {
464
+ "clip_id": clip.clip_id,
465
+ "source_id": clip.source_id,
466
+ "layer": clip.layer,
467
+ "file_id": clip.file_id,
468
+ "start_ms": clip.start_ms,
469
+ "end_ms": clip.end_ms,
470
+ "reference": clip.reference,
471
+ "language": clip.language,
472
+ "meta": dict(clip.meta),
473
+ }
474
+
475
+
476
+ def clip_from_obj(obj: object) -> Clip:
477
+ """Inverse of :func:`clip_to_obj`; re-derives and checks the content-addressed id.
478
+
479
+ Raises:
480
+ ValueError: on missing/mistyped fields or an id that does not match the content.
481
+ """
482
+ if not isinstance(obj, dict):
483
+ raise ValueError("clip must be a JSON object")
484
+ try:
485
+ language = obj["language"]
486
+ if language not in ("tr", "en"):
487
+ raise ValueError(f"unsupported language {language!r}")
488
+ meta = obj.get("meta", {})
489
+ if not isinstance(meta, dict) or not all(
490
+ isinstance(k, str) and isinstance(v, str) for k, v in meta.items()
491
+ ):
492
+ raise ValueError("meta must map strings to strings")
493
+ reference = obj["reference"]
494
+ if reference is not None and not isinstance(reference, str):
495
+ raise ValueError("reference must be a string or null")
496
+ clip = Clip.build(
497
+ source_id=str(obj["source_id"]),
498
+ layer=str(obj["layer"]),
499
+ file_id=str(obj["file_id"]),
500
+ start_ms=int(obj["start_ms"]),
501
+ end_ms=int(obj["end_ms"]),
502
+ reference=reference,
503
+ language=language,
504
+ meta=meta,
505
+ )
506
+ except KeyError as err:
507
+ raise ValueError(f"clip is missing field {err}") from err
508
+ if clip.clip_id != obj.get("clip_id"):
509
+ raise ValueError(f"clip_id {obj.get('clip_id')!r} does not match content ({clip.clip_id})")
510
+ return clip
src/nida/bench/envs.py ADDED
@@ -0,0 +1,351 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Isolated ``uv`` virtual environments for the benchmark workers.
2
+
3
+ The benchmark workers have conflicting dependency pins (``omnilingual-asr`` needs
4
+ Python <= 3.12 and ``fairseq2n`` pins ``torch==2.8.0``; ``qwen-asr`` pins
5
+ ``transformers==4.57.6``), so each worker family runs in its own environment built by
6
+ ``uv``. This module only *builds and describes* those environments; it never imports the
7
+ heavy packages itself and uses the standard library only.
8
+
9
+ All subprocess calls go through an injectable :data:`CommandRunner` so tests never
10
+ execute ``uv``. An environment is described by an :class:`EnvSpec`; a marker file
11
+ (``.nida-env.json``) is written last so a half-built environment is always rebuilt.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import hashlib
17
+ import json
18
+ import logging
19
+ import re
20
+ import shutil
21
+ import subprocess
22
+ import sys
23
+ from collections.abc import Callable, Mapping, Sequence
24
+ from dataclasses import asdict, dataclass
25
+ from datetime import UTC, datetime
26
+ from pathlib import Path
27
+ from types import MappingProxyType
28
+
29
+ from nida.utils.io import atomic_write_json, atomic_write_text
30
+
31
+ __all__ = [
32
+ "ENV_SPECS",
33
+ "MARKER_NAME",
34
+ "CommandRunner",
35
+ "EnvBuildError",
36
+ "EnvSpec",
37
+ "ensure_env",
38
+ "env_for_worker",
39
+ "find_uv",
40
+ "freeze_text",
41
+ "subprocess_runner",
42
+ ]
43
+
44
+ _LOG = logging.getLogger(__name__)
45
+
46
+ MARKER_NAME = ".nida-env.json"
47
+ FREEZE_NAME = "freeze.txt"
48
+ UV_VERSION = "0.10.4"
49
+ _SUBPROCESS_TIMEOUT_S = 3600
50
+ _STDERR_TAIL_CHARS = 2000
51
+ _NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$")
52
+
53
+
54
+ class EnvBuildError(RuntimeError):
55
+ """Raised when an environment cannot be built or a command fails."""
56
+
57
+
58
+ CommandRunner = Callable[[Sequence[str]], str]
59
+ """Runs a command and returns its stdout; raises :class:`EnvBuildError` on failure."""
60
+
61
+
62
+ @dataclass(frozen=True)
63
+ class EnvSpec:
64
+ """Description of one worker environment.
65
+
66
+ Attributes:
67
+ name: Environment directory name (lowercase ``[a-z][a-z0-9_-]*``).
68
+ python: Python version passed to ``uv venv --python``.
69
+ torch_packages: Pinned torch packages installed first from ``torch_index_url``. Empty
70
+ = the packages bring their own torch (or none): the torch step and the torch
71
+ version guard of :func:`ensure_env` are skipped.
72
+ torch_index_url: Package index serving the matching CUDA wheels (unused when
73
+ ``torch_packages`` is empty).
74
+ packages: Pinned packages installed from the default index after torch.
75
+ workers: Worker module names (under ``nida.bench.workers``) run in this env.
76
+ """
77
+
78
+ name: str
79
+ python: str
80
+ torch_packages: tuple[str, ...]
81
+ torch_index_url: str
82
+ packages: tuple[str, ...]
83
+ workers: tuple[str, ...]
84
+
85
+ def __post_init__(self) -> None:
86
+ if not _NAME_RE.fullmatch(self.name):
87
+ raise ValueError(f"invalid environment name {self.name!r}; expected {_NAME_RE.pattern}")
88
+
89
+ def spec_hash(self) -> str:
90
+ """Return the sha256 hex digest of a canonical JSON of all fields.
91
+
92
+ The digest is stable across runs and processes (sorted keys, fixed separators).
93
+ """
94
+ canonical = json.dumps(asdict(self), sort_keys=True, separators=(",", ":"))
95
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
96
+
97
+
98
+ _PYTHON = "3.12"
99
+ _TORCH_PACKAGES = ("torch==2.8.0", "torchaudio==2.8.0")
100
+ _TORCH_INDEX_URL = "https://download.pytorch.org/whl/cu128"
101
+
102
+ ENV_SPECS: Mapping[str, EnvSpec] = MappingProxyType(
103
+ {
104
+ "hf": EnvSpec(
105
+ name="hf",
106
+ python=_PYTHON,
107
+ torch_packages=_TORCH_PACKAGES,
108
+ torch_index_url=_TORCH_INDEX_URL,
109
+ packages=("transformers==4.57.6", "huggingface-hub", "soundfile", "numpy"),
110
+ workers=("asr_hf",),
111
+ ),
112
+ "qwen": EnvSpec(
113
+ name="qwen",
114
+ python=_PYTHON,
115
+ torch_packages=_TORCH_PACKAGES,
116
+ torch_index_url=_TORCH_INDEX_URL,
117
+ packages=("qwen-asr==0.0.6", "huggingface-hub"),
118
+ workers=("asr_qwen",),
119
+ ),
120
+ "omni": EnvSpec(
121
+ name="omni",
122
+ python=_PYTHON,
123
+ torch_packages=_TORCH_PACKAGES,
124
+ torch_index_url=_TORCH_INDEX_URL,
125
+ # omnilingual-asr needs pyarrow>=20 but keeps NumPy 1.26, and pyarrow 26 needs
126
+ # NumPy 2 ("pyarrow requires NumPy 2.0 or newer, found 1.26.4", fresh env
127
+ # 2026-10-09). 21.0.0 resolves and imports with NumPy 1.26 (checked on Colab).
128
+ packages=("omnilingual-asr==0.2.0", "pyarrow==21.0.0"),
129
+ workers=("asr_omni",),
130
+ ),
131
+ "audio": EnvSpec(
132
+ name="audio",
133
+ python=_PYTHON,
134
+ torch_packages=_TORCH_PACKAGES,
135
+ torch_index_url=_TORCH_INDEX_URL,
136
+ packages=(
137
+ "audiobox-aesthetics==0.0.4",
138
+ "numpy",
139
+ "pydantic",
140
+ "soundfile",
141
+ "huggingface-hub",
142
+ # audiobox_aesthetics.utils imports requests without declaring it
143
+ # (run 20261008-124740-bench-phase0-8c6a failed on the import).
144
+ "requests",
145
+ ),
146
+ workers=("vae_voxcpm", "quality_audiobox"),
147
+ ),
148
+ }
149
+ )
150
+
151
+
152
+ def env_for_worker(worker: str) -> EnvSpec:
153
+ """Return the environment spec that runs ``worker``.
154
+
155
+ Args:
156
+ worker: Worker module name, e.g. ``"asr_qwen"``.
157
+
158
+ Raises:
159
+ ValueError: if no environment lists ``worker``.
160
+ """
161
+ for spec in ENV_SPECS.values():
162
+ if worker in spec.workers:
163
+ return spec
164
+ known = sorted(w for spec in ENV_SPECS.values() for w in spec.workers)
165
+ raise ValueError(f"unknown worker {worker!r}; known workers: {known}")
166
+
167
+
168
+ def subprocess_runner(cmd: Sequence[str]) -> str:
169
+ """Run ``cmd`` with :func:`subprocess.run` and return its stdout.
170
+
171
+ Args:
172
+ cmd: Command and arguments (no shell).
173
+
174
+ Returns:
175
+ Captured stdout text.
176
+
177
+ Raises:
178
+ EnvBuildError: on a non-zero exit code, a timeout, or if the executable is missing.
179
+ The message carries the command, the exit code and the stderr tail.
180
+ """
181
+ command = list(cmd)
182
+ try:
183
+ result = subprocess.run(
184
+ command,
185
+ capture_output=True,
186
+ text=True,
187
+ check=False,
188
+ timeout=_SUBPROCESS_TIMEOUT_S,
189
+ )
190
+ except subprocess.TimeoutExpired as exc:
191
+ raise EnvBuildError(f"command timed out after {_SUBPROCESS_TIMEOUT_S}s: {command}") from exc
192
+ except OSError as exc:
193
+ raise EnvBuildError(f"command could not be started: {command}: {exc}") from exc
194
+ if result.returncode != 0:
195
+ tail = result.stderr[-_STDERR_TAIL_CHARS:]
196
+ raise EnvBuildError(
197
+ f"command failed with exit code {result.returncode}: {command}\nstderr tail:\n{tail}"
198
+ )
199
+ return result.stdout
200
+
201
+
202
+ def find_uv(
203
+ runner: CommandRunner = subprocess_runner,
204
+ which: Callable[[str], str | None] = shutil.which,
205
+ ) -> str:
206
+ """Locate ``uv``, installing it with pip when it is not on ``PATH``.
207
+
208
+ Args:
209
+ runner: Command runner used for the pip install.
210
+ which: Executable lookup (injectable for tests).
211
+
212
+ Returns:
213
+ ``"uv"`` if it is already on ``PATH``, otherwise the path found after installing.
214
+
215
+ Raises:
216
+ EnvBuildError: if ``uv`` is still missing after the install.
217
+ """
218
+ if which("uv"):
219
+ return "uv"
220
+ runner([sys.executable, "-m", "pip", "install", "-q", f"uv=={UV_VERSION}"])
221
+ found = which("uv")
222
+ if not found:
223
+ raise EnvBuildError(f"uv=={UV_VERSION} was installed with pip but is still not on PATH")
224
+ return found
225
+
226
+
227
+ def _python_path(env_dir: Path) -> Path:
228
+ return env_dir / "bin" / "python"
229
+
230
+
231
+ def _remove_env_dir(env_dir: Path, envs_root: Path) -> None:
232
+ """Delete ``env_dir`` if it exists, refusing anything not strictly inside ``envs_root``.
233
+
234
+ Raises:
235
+ EnvBuildError: if ``env_dir`` resolves outside (or equal to) ``envs_root``.
236
+ """
237
+ root = envs_root.resolve()
238
+ target = env_dir.resolve()
239
+ if target == root or not target.is_relative_to(root):
240
+ raise EnvBuildError(f"refusing to delete {env_dir}: not strictly inside {envs_root}")
241
+ if env_dir.is_symlink():
242
+ env_dir.unlink()
243
+ elif env_dir.exists():
244
+ shutil.rmtree(env_dir)
245
+
246
+
247
+ def _is_current(spec: EnvSpec, env_dir: Path) -> bool:
248
+ """Return True if a complete, up-to-date environment exists at ``env_dir``."""
249
+ marker = env_dir / MARKER_NAME
250
+ if not marker.is_file() or not _python_path(env_dir).exists():
251
+ return False
252
+ try:
253
+ data = json.loads(marker.read_text(encoding="utf-8"))
254
+ except (OSError, json.JSONDecodeError) as exc:
255
+ _LOG.warning("unreadable env marker %s (%s); rebuilding", marker, exc)
256
+ return False
257
+ return isinstance(data, dict) and data.get("spec_hash") == spec.spec_hash()
258
+
259
+
260
+ def _expected_torch_version(spec: EnvSpec) -> str:
261
+ for pkg in spec.torch_packages:
262
+ if pkg.startswith("torch=="):
263
+ return pkg.removeprefix("torch==")
264
+ raise EnvBuildError(f"env {spec.name!r} pins no 'torch==X' in torch_packages")
265
+
266
+
267
+ def ensure_env(
268
+ spec: EnvSpec,
269
+ envs_root: Path,
270
+ repo_dir: Path,
271
+ runner: CommandRunner = subprocess_runner,
272
+ uv: str = "uv",
273
+ ) -> Path:
274
+ """Build the environment for ``spec`` unless an up-to-date one already exists.
275
+
276
+ The marker ``<envs_root>/<name>/.nida-env.json`` is written last, so any failure
277
+ leaves no marker and the next call rebuilds from scratch.
278
+
279
+ Args:
280
+ spec: Environment description.
281
+ envs_root: Directory holding all environments.
282
+ repo_dir: Repository root, installed with ``--no-deps -e`` (nida has no runtime deps).
283
+ runner: Command runner (injectable).
284
+ uv: ``uv`` executable.
285
+
286
+ Returns:
287
+ Path of the environment's python executable.
288
+
289
+ Raises:
290
+ EnvBuildError: if a command fails, the torch version guard trips, or the delete
291
+ guard refuses the environment directory.
292
+ """
293
+ env_dir = envs_root / spec.name
294
+ py = _python_path(env_dir)
295
+ if _is_current(spec, env_dir):
296
+ _LOG.info("env %s is up to date at %s", spec.name, env_dir)
297
+ return py
298
+
299
+ envs_root.mkdir(parents=True, exist_ok=True)
300
+ _remove_env_dir(env_dir, envs_root)
301
+
302
+ runner([uv, "venv", "--python", spec.python, str(env_dir)])
303
+ if spec.torch_packages:
304
+ runner(
305
+ [
306
+ uv,
307
+ "pip",
308
+ "install",
309
+ "--python",
310
+ str(py),
311
+ "--index-url",
312
+ spec.torch_index_url,
313
+ *spec.torch_packages,
314
+ ]
315
+ )
316
+ runner([uv, "pip", "install", "--python", str(py), *spec.packages])
317
+ runner([uv, "pip", "install", "--python", str(py), "--no-deps", "-e", str(repo_dir)])
318
+
319
+ if spec.torch_packages:
320
+ expected = _expected_torch_version(spec)
321
+ torch_version = runner(
322
+ [str(py), "-c", "import torch, sys; print(torch.__version__)"]
323
+ ).strip()
324
+ if not torch_version.startswith(expected):
325
+ raise EnvBuildError(
326
+ f"env {spec.name!r}: torch version is {torch_version!r}, expected {expected!r} "
327
+ "(a later install step probably replaced torch)"
328
+ )
329
+
330
+ frozen = runner([uv, "pip", "freeze", "--python", str(py)])
331
+ atomic_write_text(env_dir / FREEZE_NAME, frozen)
332
+
333
+ atomic_write_json(
334
+ env_dir / MARKER_NAME,
335
+ {
336
+ "name": spec.name,
337
+ "spec_hash": spec.spec_hash(),
338
+ "spec": asdict(spec),
339
+ "created_at": datetime.now(UTC).isoformat(),
340
+ },
341
+ )
342
+ return py
343
+
344
+
345
+ def freeze_text(spec: EnvSpec, envs_root: Path) -> str:
346
+ """Return the ``pip freeze`` output recorded when ``spec`` was built.
347
+
348
+ Raises:
349
+ FileNotFoundError: if the environment has not been built (no ``freeze.txt``).
350
+ """
351
+ return (envs_root / spec.name / FREEZE_NAME).read_text(encoding="utf-8")
src/nida/bench/expresso.py ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Expresso layout parsing and clip selection for B2 (docs/benchmarks/B1-B2-tasarim.md §2).
2
+
3
+ Archive layout (textlesslib ``examples/expresso/dataset/README.md``)::
4
+
5
+ expresso/read_transcriptions.txt
6
+ expresso/VAD_segments.txt "{stem}/channel{1|2}\t(s, e) (s, e) ..."
7
+ (mono long-form lines: bare "{stem}", kept as
8
+ channel 0 and not used by B2)
9
+ expresso/audio_48khz/read/{spk}/{style}/{base|longform}/{spk}_{style...}_{id}.wav mono
10
+ expresso/audio_48khz/conversational/{spk1}-{spk2}/{style1[-style2]}/{stem}.wav stereo
11
+
12
+ In a conversational style folder without a hyphen both channels carry that style; with a
13
+ hyphen the left style belongs to channel 1 (left speaker) and the right one to channel 2.
14
+ Subsets used by B2: ``whisper``, ``laughing`` and ``default`` from read speech (they have
15
+ transcripts), ``projected`` and ``nonverbal`` from conversational VAD segments of a
16
+ channel that carries the style. Folder spellings ``non_verbal`` / ``nonverbal`` are both
17
+ accepted.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import hashlib
23
+ import random
24
+ import re
25
+ from collections import defaultdict
26
+ from collections.abc import Iterable, Mapping, Sequence
27
+ from dataclasses import dataclass
28
+
29
+ from nida.bench.clips import Clip, Quarantined, Selection
30
+
31
+ __all__ = [
32
+ "CONVERSATIONAL_SUBSETS",
33
+ "READ_SUBSETS",
34
+ "ExpressoFile",
35
+ "is_wanted_member",
36
+ "parse_member",
37
+ "parse_read_transcriptions",
38
+ "parse_vad_segments",
39
+ "select_expresso",
40
+ ]
41
+
42
+ SOURCE_ID = "expresso"
43
+ READ_SUBSETS = ("whisper", "laughing", "default")
44
+ CONVERSATIONAL_SUBSETS = ("projected", "nonverbal")
45
+ _STYLE_ALIASES = {"non_verbal": "nonverbal"}
46
+ _AUDIO_ROOT = "audio_48khz/"
47
+ #: Keep 1 in N default-style read files (default read speech is large; B2 needs ~10).
48
+ DEFAULT_KEEP_ONE_IN = 8
49
+
50
+
51
+ def _style(name: str) -> str:
52
+ return _STYLE_ALIASES.get(name, name)
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class ExpressoFile:
57
+ """One audio member of the archive.
58
+
59
+ Attributes:
60
+ member: Archive member name (as stored in the tar).
61
+ kind: ``"read"`` or ``"conversational"``.
62
+ stem: File name without extension (key of VAD segments and transcripts).
63
+ speakers: One speaker (read) or (channel1, channel2) speakers.
64
+ styles: One style (read) or (channel1, channel2) styles.
65
+ corpus: ``"base"`` or ``"longform"`` for read speech, ``""`` otherwise.
66
+ """
67
+
68
+ member: str
69
+ kind: str
70
+ stem: str
71
+ speakers: tuple[str, ...]
72
+ styles: tuple[str, ...]
73
+ corpus: str
74
+
75
+
76
+ def parse_member(name: str) -> ExpressoFile | None:
77
+ """Parse an archive member name; ``None`` for anything that is not a 48 kHz WAV."""
78
+ if not name.endswith(".wav") or _AUDIO_ROOT not in name:
79
+ return None
80
+ rel = name.split(_AUDIO_ROOT, 1)[1].split("/")
81
+ stem = rel[-1][: -len(".wav")]
82
+ if rel[0] == "read" and len(rel) == 5:
83
+ _, speaker, style, corpus, _ = rel
84
+ return ExpressoFile(name, "read", stem, (speaker,), (_style(style),), corpus)
85
+ if rel[0] == "conversational" and len(rel) == 4:
86
+ _, pair, style_dir, _ = rel
87
+ speakers = tuple(pair.split("-"))
88
+ styles = tuple(_style(s) for s in style_dir.split("-"))
89
+ if len(speakers) != 2 or len(styles) not in (1, 2):
90
+ return None
91
+ return ExpressoFile(
92
+ name, "conversational", stem, speakers, styles if len(styles) == 2 else styles * 2, ""
93
+ )
94
+ return None
95
+
96
+
97
+ def _stable_fraction(text: str) -> float:
98
+ return int(hashlib.sha1(text.encode("utf-8")).hexdigest()[:8], 16) / 0xFFFFFFFF
99
+
100
+
101
+ def is_wanted_member(name: str) -> bool:
102
+ """Archive filter for streaming extraction (metadata files + B2 subsets only)."""
103
+ if name.endswith(("read_transcriptions.txt", "VAD_segments.txt")):
104
+ return True
105
+ parsed = parse_member(name)
106
+ if parsed is None:
107
+ return False
108
+ if parsed.kind == "read":
109
+ if parsed.corpus != "base":
110
+ return False
111
+ style = parsed.styles[0]
112
+ if style == "default":
113
+ return _stable_fraction(parsed.stem) < 1 / DEFAULT_KEEP_ONE_IN
114
+ return style in ("whisper", "laughing")
115
+ return any(style in CONVERSATIONAL_SUBSETS for style in parsed.styles)
116
+
117
+
118
+ _NUMBER = r"\d+(?:\.\d+)?"
119
+ _SEGMENT_RE = re.compile(rf"\(\s*({_NUMBER})\s*,\s*({_NUMBER})\s*\)")
120
+ _SEGMENTS_RE = re.compile(rf"\s*(?:\(\s*{_NUMBER}\s*,\s*{_NUMBER}\s*\)\s*)*")
121
+
122
+
123
+ def parse_vad_segments(text: str) -> dict[tuple[str, int], list[tuple[float, float]]]:
124
+ """Parse ``VAD_segments.txt`` into ``{(stem, channel): [(start_s, end_s), ...]}``.
125
+
126
+ Conversational keys are ``<stem>/channel1|2``; the official file also lists mono
127
+ long-form read recordings with a bare ``<stem>`` key, stored here as channel 0.
128
+
129
+ Raises:
130
+ ValueError: on a malformed key or a segment with end <= start.
131
+ """
132
+ out: dict[tuple[str, int], list[tuple[float, float]]] = {}
133
+ for number, line in enumerate(text.splitlines(), start=1):
134
+ if not line.strip() or line.startswith("#"):
135
+ continue
136
+ key, _, segments = line.partition("\t")
137
+ stem, sep, channel = key.rpartition("/channel")
138
+ if not sep:
139
+ stem, channel = key, "0" # mono long-form recording (bare key)
140
+ elif channel not in ("1", "2"):
141
+ raise ValueError(f"VAD line {number}: bad key {key!r}")
142
+ if not stem or "/" in stem or any(ch.isspace() for ch in stem):
143
+ raise ValueError(f"VAD line {number}: bad key {key!r}")
144
+ if not _SEGMENTS_RE.fullmatch(segments):
145
+ raise ValueError(f"VAD line {number}: unparseable segments {segments[:80]!r}")
146
+ if (stem, int(channel)) in out:
147
+ raise ValueError(f"VAD line {number}: duplicate key {key!r}")
148
+ spans: list[tuple[float, float]] = []
149
+ for match in _SEGMENT_RE.finditer(segments):
150
+ start_s, end_s = float(match.group(1)), float(match.group(2))
151
+ if end_s <= start_s:
152
+ raise ValueError(f"VAD line {number}: empty segment ({start_s}, {end_s})")
153
+ spans.append((start_s, end_s))
154
+ out[(stem, int(channel))] = spans
155
+ return out
156
+
157
+
158
+ def parse_read_transcriptions(text: str) -> dict[str, str]:
159
+ """Parse ``read_transcriptions.txt`` as ``<stem><TAB or space><text>`` lines.
160
+
161
+ Raises:
162
+ ValueError: on duplicate stems.
163
+ """
164
+ out: dict[str, str] = {}
165
+ for line in text.splitlines():
166
+ if not line.strip() or line.startswith("#"):
167
+ continue
168
+ sep = "\t" if "\t" in line else " "
169
+ stem, _, transcript = line.partition(sep)
170
+ stem = stem.strip()
171
+ if stem in out:
172
+ raise ValueError(f"duplicate transcript id {stem}")
173
+ out[stem] = transcript.strip()
174
+ return out
175
+
176
+
177
+ def _round_robin_by_speaker(
178
+ candidates: Sequence[tuple[str, Clip]], n: int, rng: random.Random
179
+ ) -> list[Clip]:
180
+ by_speaker: dict[str, list[Clip]] = defaultdict(list)
181
+ for speaker, clip in sorted(candidates, key=lambda pair: pair[1].clip_id):
182
+ by_speaker[speaker].append(clip)
183
+ for clips in by_speaker.values():
184
+ rng.shuffle(clips)
185
+ order = sorted(by_speaker)
186
+ chosen: list[Clip] = []
187
+ while len(chosen) < n and any(by_speaker[s] for s in order):
188
+ for speaker in order:
189
+ if by_speaker[speaker] and len(chosen) < n:
190
+ chosen.append(by_speaker[speaker].pop())
191
+ return chosen
192
+
193
+
194
+ def select_expresso(
195
+ files: Iterable[tuple[ExpressoFile, int]],
196
+ vad: Mapping[tuple[str, int], Sequence[tuple[float, float]]],
197
+ transcripts: Mapping[str, str],
198
+ counts: Mapping[str, int],
199
+ seed: int,
200
+ *,
201
+ min_s: float = 3.0,
202
+ max_s: float = 15.0,
203
+ long_segment_cut_s: float = 12.0,
204
+ ) -> Selection:
205
+ """Choose B2 Expresso clips per subset, balanced across speakers.
206
+
207
+ Args:
208
+ files: Extracted files with their length in milliseconds.
209
+ vad: Parsed VAD segments.
210
+ transcripts: Parsed read transcriptions.
211
+ counts: Wanted clips per subset name (keys from READ/CONVERSATIONAL_SUBSETS).
212
+ seed: RNG seed.
213
+ min_s: Minimum clip duration.
214
+ max_s: Maximum duration of a read file / VAD segment taken whole.
215
+ long_segment_cut_s: Longer VAD segments contribute their first this-many seconds.
216
+
217
+ Raises:
218
+ ValueError: on unknown subsets or if a subset has fewer candidates than requested.
219
+ """
220
+ unknown = set(counts) - set(READ_SUBSETS) - set(CONVERSATIONAL_SUBSETS)
221
+ if unknown:
222
+ raise ValueError(f"unknown Expresso subsets {sorted(unknown)}")
223
+ if not 0 < min_s < long_segment_cut_s <= max_s:
224
+ raise ValueError("need 0 < min_s < long_segment_cut_s <= max_s")
225
+ min_ms, max_ms, cut_ms = (round(v * 1000) for v in (min_s, max_s, long_segment_cut_s))
226
+ quarantined: list[Quarantined] = []
227
+ candidates: dict[str, list[tuple[str, Clip]]] = defaultdict(list)
228
+ for item, length_ms in sorted(files, key=lambda pair: pair[0].member):
229
+ if item.kind == "read":
230
+ subset = item.styles[0]
231
+ if subset not in counts:
232
+ continue
233
+ if not min_ms <= length_ms <= max_ms:
234
+ quarantined.append(Quarantined(item.stem, "select", "duration_out_of_range"))
235
+ continue
236
+ reference = transcripts.get(item.stem)
237
+ if not reference:
238
+ quarantined.append(Quarantined(item.stem, "select", "no_transcript"))
239
+ continue
240
+ clip = Clip.build(
241
+ source_id=SOURCE_ID,
242
+ layer=subset,
243
+ file_id=item.member,
244
+ start_ms=0,
245
+ end_ms=length_ms,
246
+ reference=reference,
247
+ language="en",
248
+ meta={"speaker": item.speakers[0], "channel": "0"},
249
+ )
250
+ candidates[subset].append((item.speakers[0], clip))
251
+ continue
252
+ for channel in (1, 2):
253
+ subset = item.styles[channel - 1]
254
+ if subset not in counts:
255
+ continue
256
+ spans = vad.get((item.stem, channel))
257
+ if spans is None:
258
+ quarantined.append(
259
+ Quarantined(f"{item.stem}/channel{channel}", "select", "no_vad_segments")
260
+ )
261
+ continue
262
+ for start_s, end_s in spans:
263
+ # Integer milliseconds: float seconds would drop exact-length segments
264
+ # (4.02 - 1.02 < 3.0 in binary floating point).
265
+ start_ms, end_ms = round(start_s * 1000), round(end_s * 1000)
266
+ if end_ms - start_ms < min_ms:
267
+ continue # short VAD islands are expected; not a rule violation
268
+ if end_ms - start_ms > max_ms:
269
+ end_ms = start_ms + cut_ms
270
+ if end_ms > length_ms:
271
+ if end_ms - length_ms > 1:
272
+ quarantined.append(
273
+ Quarantined(
274
+ item.stem, "select", "vad_beyond_audio", f"{end_ms}>{length_ms}"
275
+ )
276
+ )
277
+ continue
278
+ end_ms = length_ms # ≤ 1 ms of rounding at the very end of the file
279
+ if end_ms - start_ms < min_ms:
280
+ continue
281
+ clip = Clip.build(
282
+ source_id=SOURCE_ID,
283
+ layer=subset,
284
+ file_id=f"{item.member}#ch{channel}",
285
+ start_ms=start_ms,
286
+ end_ms=end_ms,
287
+ reference=None,
288
+ language="en",
289
+ meta={"speaker": item.speakers[channel - 1], "channel": str(channel)},
290
+ )
291
+ candidates[subset].append((item.speakers[channel - 1], clip))
292
+ rng = random.Random(seed)
293
+ chosen: list[Clip] = []
294
+ for subset in sorted(counts):
295
+ wanted = counts[subset]
296
+ if len(candidates[subset]) < wanted:
297
+ raise ValueError(
298
+ f"Expresso subset {subset}: {len(candidates[subset])} candidates, need {wanted}"
299
+ )
300
+ chosen.extend(_round_robin_by_speaker(candidates[subset], wanted, rng))
301
+ return Selection(tuple(sorted(chosen, key=lambda c: c.clip_id)), tuple(quarantined))
src/nida/bench/fetch.py ADDED
@@ -0,0 +1,704 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Reliable, resumable downloads and streaming tar extraction (stdlib only).
2
+
3
+ Benchmark data lives on public hosts (Hugging Face and others) and ranges from a few
4
+ hundred KB to a 36 GB tar. Colab sessions drop connections, so two tools are provided:
5
+
6
+ * :func:`download` fetches one file to disk. It resumes from a ``<name>.part`` file with an
7
+ HTTP ``Range`` request, retries transient network failures, verifies size and SHA-256 and
8
+ only then atomically renames the ``.part`` file to its final name. A completed file is
9
+ never overwritten and a partial file never looks complete.
10
+ * :func:`stream_extract` reads a (possibly huge, possibly compressed) tar archive as a
11
+ stream and writes only the selected regular files to disk, without ever storing the
12
+ archive itself.
13
+
14
+ All network access goes through an injectable :data:`Opener`, so tests run offline.
15
+ :func:`hf_resolve_url` builds revision-pinned Hugging Face download URLs.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import hashlib
21
+ import http.client
22
+ import logging
23
+ import os
24
+ import re
25
+ import shutil
26
+ import ssl
27
+ import tarfile
28
+ import tempfile
29
+ import time
30
+ import urllib.error
31
+ import urllib.parse
32
+ import urllib.request
33
+ import zlib
34
+ from collections.abc import Callable, Iterator
35
+ from contextlib import contextmanager
36
+ from dataclasses import dataclass
37
+ from pathlib import Path
38
+ from types import TracebackType
39
+ from typing import IO, Literal, NoReturn, Protocol, cast
40
+
41
+ __all__ = [
42
+ "Downloaded",
43
+ "FetchError",
44
+ "HeadersLike",
45
+ "Opener",
46
+ "Response",
47
+ "default_opener",
48
+ "download",
49
+ "hf_resolve_url",
50
+ "stream_extract",
51
+ ]
52
+
53
+ logger = logging.getLogger(__name__)
54
+
55
+ _USER_AGENT = "nida-bench/0"
56
+ _SHA256_RE = re.compile(r"[0-9a-f]{64}")
57
+ _COMMIT_SHA_RE = re.compile(r"[0-9a-f]{40}")
58
+ _CONTENT_RANGE_RE = re.compile(r"bytes (\d+)-(\d+)/(\d+|\*)")
59
+ _DRIVE_LETTER_RE = re.compile(r"[A-Za-z]:")
60
+ _MAX_BACKOFF_S = 30.0
61
+ _COPY_CHUNK_BYTES = 1 << 20
62
+ # HTTP 4xx codes that are worth retrying; every other 4xx is a permanent client error.
63
+ _RETRYABLE_4XX = frozenset({408, 429})
64
+
65
+ # Transient network failures. ``HTTPError`` is a ``URLError`` and is classified separately.
66
+ _NETWORK_ERRORS: tuple[type[BaseException], ...] = (
67
+ urllib.error.URLError,
68
+ ConnectionError,
69
+ TimeoutError,
70
+ http.client.HTTPException, # includes IncompleteRead
71
+ ssl.SSLError,
72
+ )
73
+ # Everything that can go wrong while reading an archive stream (network or corrupt data).
74
+ _STREAM_ERRORS: tuple[type[BaseException], ...] = (
75
+ *_NETWORK_ERRORS,
76
+ tarfile.TarError,
77
+ EOFError,
78
+ zlib.error,
79
+ )
80
+
81
+
82
+ class FetchError(RuntimeError):
83
+ """A download or extraction failed; the message carries URL/path and byte counts."""
84
+
85
+
86
+ class _TransientError(Exception):
87
+ """Internal: this attempt failed but a resumed attempt may succeed."""
88
+
89
+ def __init__(self, message: str, *, expected_bytes: int | None = None) -> None:
90
+ super().__init__(message)
91
+ self.expected_bytes = expected_bytes
92
+
93
+
94
+ class HeadersLike(Protocol):
95
+ """Response headers: only ``get`` is used (``http.client.HTTPMessage`` satisfies it)."""
96
+
97
+ def get(self, name: str, default: str | None = None, /) -> str | None:
98
+ """Return the header value, or ``default`` when absent."""
99
+ ...
100
+
101
+
102
+ class Response(Protocol):
103
+ """The slice of an HTTP response this module relies on.
104
+
105
+ ``http.client.HTTPResponse`` (what ``urllib.request.urlopen`` returns) satisfies it.
106
+ """
107
+
108
+ @property
109
+ def status(self) -> int:
110
+ """HTTP status code."""
111
+ ...
112
+
113
+ @property
114
+ def headers(self) -> HeadersLike:
115
+ """Response headers."""
116
+ ...
117
+
118
+ def read(self, amt: int = -1, /) -> bytes:
119
+ """Read up to ``amt`` bytes; ``b""`` means end of stream."""
120
+ ...
121
+
122
+ def __enter__(self) -> Response:
123
+ """Enter the context."""
124
+ ...
125
+
126
+ def __exit__(
127
+ self,
128
+ exc_type: type[BaseException] | None,
129
+ exc: BaseException | None,
130
+ tb: TracebackType | None,
131
+ /,
132
+ ) -> object:
133
+ """Close the response."""
134
+ ...
135
+
136
+
137
+ Opener = Callable[[urllib.request.Request, float], Response]
138
+ """Callable performing a request with a timeout in seconds and returning a response."""
139
+
140
+
141
+ def default_opener(request: urllib.request.Request, timeout_s: float) -> Response:
142
+ """Open ``request`` with :func:`urllib.request.urlopen`.
143
+
144
+ Args:
145
+ request: The prepared request.
146
+ timeout_s: Socket timeout in seconds.
147
+
148
+ Returns:
149
+ The open HTTP response (a context manager).
150
+ """
151
+ response: Response = urllib.request.urlopen(request, timeout=timeout_s)
152
+ return response
153
+
154
+
155
+ @dataclass(frozen=True)
156
+ class Downloaded:
157
+ """A completed, verified download.
158
+
159
+ Attributes:
160
+ path: Final file path.
161
+ size_bytes: File size in bytes.
162
+ sha256: Lowercase hex SHA-256 computed over the final file.
163
+ url: Source URL.
164
+ """
165
+
166
+ path: Path
167
+ size_bytes: int
168
+ sha256: str
169
+ url: str
170
+
171
+
172
+ # --------------------------------------------------------------------------------------
173
+ # download
174
+ # --------------------------------------------------------------------------------------
175
+
176
+
177
+ def download(
178
+ url: str,
179
+ dest: Path,
180
+ *,
181
+ expected_sha256: str | None = None,
182
+ expected_size: int | None = None,
183
+ opener: Opener = default_opener,
184
+ chunk_bytes: int = 1 << 20,
185
+ timeout_s: float = 60.0,
186
+ max_attempts: int = 5,
187
+ sleep: Callable[[float], None] = time.sleep,
188
+ ) -> Downloaded:
189
+ """Download ``url`` to ``dest``, resuming and retrying as needed.
190
+
191
+ If ``dest`` already exists it is verified (size and/or SHA-256 when given) and returned
192
+ without any network access; it is never overwritten. Otherwise the data is written to
193
+ ``<dest>.part`` and resumed with ``Range: bytes=<n>-`` on every retry. A server that
194
+ answers 200 instead of 206 to a ranged request makes the download restart from zero.
195
+ Transient failures (``URLError``, ``ConnectionError``, ``TimeoutError``, short reads,
196
+ HTTP 5xx/408/429) are retried with backoff of 1, 2, 4, ... seconds (capped at 30). Other
197
+ HTTP 4xx errors fail immediately. When all attempts are exhausted the ``.part`` file is
198
+ kept so a later call can resume it.
199
+
200
+ The SHA-256 is computed over the final file (re-read once at the end), so it also
201
+ covers bytes written by earlier, interrupted attempts.
202
+
203
+ Concurrent downloads of the same ``dest`` are not supported.
204
+
205
+ Args:
206
+ url: ``http://`` or ``https://`` URL.
207
+ dest: Final file path; parent directories are created.
208
+ expected_sha256: Expected lowercase/uppercase hex SHA-256 (64 chars), if known.
209
+ expected_size: Expected size in bytes, if known.
210
+ opener: Network access function (injectable for tests).
211
+ chunk_bytes: Read chunk size in bytes.
212
+ timeout_s: Socket timeout per request, seconds.
213
+ max_attempts: Total number of attempts (first try included), at least 1.
214
+ sleep: Sleep function used for backoff between attempts.
215
+
216
+ Returns:
217
+ The verified :class:`Downloaded` record.
218
+
219
+ Raises:
220
+ ValueError: On invalid arguments (bad URL scheme, malformed hash, ...).
221
+ FetchError: On verification failure, permanent HTTP errors or exhausted attempts.
222
+ OSError: On local filesystem failures (e.g. disk full).
223
+ """
224
+ _require_http_url(url)
225
+ sha_expected = _normalise_sha256(expected_sha256)
226
+ if expected_size is not None and expected_size < 0:
227
+ raise ValueError(f"expected_size must be >= 0, got {expected_size}")
228
+ if chunk_bytes < 1:
229
+ raise ValueError(f"chunk_bytes must be >= 1, got {chunk_bytes}")
230
+ if max_attempts < 1:
231
+ raise ValueError(f"max_attempts must be >= 1, got {max_attempts}")
232
+ if timeout_s <= 0:
233
+ raise ValueError(f"timeout_s must be > 0, got {timeout_s}")
234
+
235
+ if dest.exists():
236
+ return _verify_existing(url, dest, sha_expected, expected_size)
237
+
238
+ dest.parent.mkdir(parents=True, exist_ok=True)
239
+ part = dest.with_name(dest.name + ".part")
240
+
241
+ last_error: _TransientError | None = None
242
+ known_total = expected_size # best knowledge of the final size, for error messages
243
+ for attempt in range(1, max_attempts + 1):
244
+ try:
245
+ _attempt(url, part, expected_size, opener, chunk_bytes, timeout_s)
246
+ except _TransientError as exc:
247
+ last_error = exc
248
+ if exc.expected_bytes is not None:
249
+ known_total = exc.expected_bytes
250
+ part_bytes = part.stat().st_size if part.exists() else 0
251
+ logger.warning(
252
+ "download %s attempt %d/%d failed after %d bytes: %s",
253
+ url,
254
+ attempt,
255
+ max_attempts,
256
+ part_bytes,
257
+ exc,
258
+ )
259
+ if attempt < max_attempts:
260
+ sleep(min(2.0 ** (attempt - 1), _MAX_BACKOFF_S))
261
+ continue
262
+ last_error = None
263
+ break
264
+
265
+ if last_error is not None:
266
+ part_bytes = part.stat().st_size if part.exists() else 0
267
+ raise FetchError(
268
+ f"download of {url} failed after {max_attempts} attempts: {part_bytes} bytes "
269
+ f"received in {part} (expected "
270
+ f"{known_total if known_total is not None else 'unknown'}); last error: {last_error}"
271
+ ) from last_error
272
+
273
+ size_bytes = part.stat().st_size
274
+ digest = _sha256_file(part)
275
+ if sha_expected is not None and digest != sha_expected:
276
+ part.unlink()
277
+ raise FetchError(
278
+ f"sha256 mismatch for {url}: got {digest}, expected {sha_expected} "
279
+ f"({size_bytes} bytes; partial file deleted)"
280
+ )
281
+ part.replace(dest)
282
+ logger.info("downloaded %s -> %s (%d bytes, sha256 %s)", url, dest, size_bytes, digest)
283
+ return Downloaded(path=dest, size_bytes=size_bytes, sha256=digest, url=url)
284
+
285
+
286
+ def _require_http_url(url: str) -> None:
287
+ if not url.lower().startswith(("http://", "https://")):
288
+ raise ValueError(f"only http(s) URLs are supported, got {url!r}")
289
+
290
+
291
+ def _normalise_sha256(value: str | None) -> str | None:
292
+ if value is None:
293
+ return None
294
+ lowered = value.strip().lower()
295
+ if _SHA256_RE.fullmatch(lowered) is None:
296
+ raise ValueError(f"expected_sha256 must be 64 hex characters, got {value!r}")
297
+ return lowered
298
+
299
+
300
+ def _sha256_file(path: Path) -> str:
301
+ digest = hashlib.sha256()
302
+ with path.open("rb") as handle:
303
+ while chunk := handle.read(_COPY_CHUNK_BYTES):
304
+ digest.update(chunk)
305
+ return digest.hexdigest()
306
+
307
+
308
+ def _verify_existing(
309
+ url: str, dest: Path, expected_sha256: str | None, expected_size: int | None
310
+ ) -> Downloaded:
311
+ if not dest.is_file():
312
+ raise FetchError(f"destination {dest} exists but is not a regular file (url {url})")
313
+ size_bytes = dest.stat().st_size
314
+ if expected_size is not None and size_bytes != expected_size:
315
+ raise FetchError(
316
+ f"existing file {dest} has {size_bytes} bytes, expected {expected_size} "
317
+ f"(url {url}); refusing to overwrite, delete it manually to re-download"
318
+ )
319
+ digest = _sha256_file(dest)
320
+ if expected_sha256 is not None and digest != expected_sha256:
321
+ raise FetchError(
322
+ f"existing file {dest} has sha256 {digest}, expected {expected_sha256} "
323
+ f"(url {url}); refusing to overwrite, delete it manually to re-download"
324
+ )
325
+ logger.info("download skipped, %s already present (%d bytes)", dest, size_bytes)
326
+ return Downloaded(path=dest, size_bytes=size_bytes, sha256=digest, url=url)
327
+
328
+
329
+ def _attempt(
330
+ url: str,
331
+ part: Path,
332
+ expected_size: int | None,
333
+ opener: Opener,
334
+ chunk_bytes: int,
335
+ timeout_s: float,
336
+ ) -> None:
337
+ """Run one (possibly resumed) transfer into ``part``.
338
+
339
+ Returns normally when ``part`` holds the complete body. Raises ``_TransientError`` when
340
+ a resumed attempt may help and ``FetchError`` for permanent problems.
341
+ """
342
+ offset = part.stat().st_size if part.exists() else 0
343
+ if expected_size is not None:
344
+ if offset > expected_size:
345
+ logger.warning("%s is larger than expected size %d, restarting", part, expected_size)
346
+ part.unlink()
347
+ offset = 0
348
+ elif offset == expected_size:
349
+ return # everything is already on disk; only verification is left
350
+
351
+ headers = {"User-Agent": _USER_AGENT}
352
+ if offset > 0:
353
+ headers["Range"] = f"bytes={offset}-"
354
+ request = urllib.request.Request(url, headers=headers)
355
+
356
+ try:
357
+ response = opener(request, timeout_s)
358
+ except urllib.error.HTTPError as exc:
359
+ code = exc.code
360
+ exc.close()
361
+ _raise_for_http_status(url, part, code, offset)
362
+ except _NETWORK_ERRORS as exc:
363
+ raise _TransientError(f"connecting to {url} failed: {exc!r}") from exc
364
+
365
+ with response:
366
+ status = response.status
367
+ if status not in (200, 206):
368
+ raise FetchError(f"unexpected HTTP status {status} from {url}")
369
+ if status == 200 and offset > 0:
370
+ logger.warning("server ignored Range for %s, restarting from byte 0", url)
371
+ offset = 0
372
+ total = _expected_total(url, response, status, offset)
373
+ if total is not None and expected_size is not None and total != expected_size:
374
+ part.unlink(missing_ok=True)
375
+ raise FetchError(
376
+ f"{url} is {total} bytes according to the server, expected {expected_size}"
377
+ )
378
+
379
+ position = offset
380
+ mode = "ab" if (status == 206 and offset > 0) else "wb"
381
+ with part.open(mode) as handle:
382
+ try:
383
+ while chunk := response.read(chunk_bytes):
384
+ handle.write(chunk)
385
+ position += len(chunk)
386
+ if total is not None and position > total:
387
+ break
388
+ except _NETWORK_ERRORS as exc:
389
+ raise _TransientError(
390
+ f"reading {url} failed at byte {position}: {exc!r}",
391
+ expected_bytes=total,
392
+ ) from exc
393
+ handle.flush()
394
+ os.fsync(handle.fileno())
395
+
396
+ if total is not None and position != total:
397
+ if position > total:
398
+ part.unlink()
399
+ raise _TransientError(
400
+ f"{url} sent more than its Content-Length: {position} bytes, expected {total}",
401
+ expected_bytes=total,
402
+ )
403
+ raise _TransientError(
404
+ f"incomplete body from {url}: {position} bytes, expected {total}",
405
+ expected_bytes=total,
406
+ )
407
+ if expected_size is not None and position != expected_size:
408
+ if position > expected_size:
409
+ part.unlink()
410
+ raise FetchError(
411
+ f"{url} delivered {position} bytes, expected {expected_size} (partial file deleted)"
412
+ )
413
+ raise _TransientError(
414
+ f"stream from {url} ended early: {position} bytes, expected {expected_size}",
415
+ expected_bytes=expected_size,
416
+ )
417
+
418
+
419
+ def _raise_for_http_status(url: str, part: Path, code: int, offset: int) -> NoReturn:
420
+ if code == 416 and offset > 0:
421
+ # The stale .part is at or beyond the end of the remote file: start over.
422
+ part.unlink(missing_ok=True)
423
+ raise _TransientError(f"HTTP 416 for ranged request at byte {offset}; restarting")
424
+ if 400 <= code < 500 and code not in _RETRYABLE_4XX:
425
+ raise FetchError(f"HTTP {code} for {url}")
426
+ raise _TransientError(f"HTTP {code} for {url}")
427
+
428
+
429
+ def _parse_length(url: str, name: str, value: str) -> int:
430
+ try:
431
+ parsed = int(value)
432
+ except ValueError:
433
+ raise FetchError(f"malformed {name} header from {url}: {value!r}") from None
434
+ if parsed < 0:
435
+ raise FetchError(f"malformed {name} header from {url}: {value!r}")
436
+ return parsed
437
+
438
+
439
+ def _expected_total(url: str, response: Response, status: int, offset: int) -> int | None:
440
+ """Return the total file size implied by the response headers, if the server says."""
441
+ length_header = response.headers.get("Content-Length")
442
+ length = _parse_length(url, "Content-Length", length_header) if length_header else None
443
+ if status == 200:
444
+ return length
445
+ range_header = response.headers.get("Content-Range")
446
+ if range_header is None:
447
+ raise FetchError(f"206 response from {url} has no Content-Range header")
448
+ match = _CONTENT_RANGE_RE.fullmatch(range_header.strip())
449
+ if match is None:
450
+ raise FetchError(f"malformed Content-Range from {url}: {range_header!r}")
451
+ start = int(match.group(1))
452
+ if start != offset:
453
+ raise FetchError(
454
+ f"server returned bytes starting at {start} from {url}, requested {offset}"
455
+ )
456
+ if match.group(3) != "*":
457
+ return int(match.group(3))
458
+ return offset + length if length is not None else None
459
+
460
+
461
+ # --------------------------------------------------------------------------------------
462
+ # stream_extract
463
+ # --------------------------------------------------------------------------------------
464
+
465
+
466
+ def stream_extract(
467
+ url_or_path: str | Path,
468
+ dest_dir: Path,
469
+ select: Callable[[str], bool],
470
+ *,
471
+ opener: Opener = default_opener,
472
+ timeout_s: float = 60.0,
473
+ ) -> list[Path]:
474
+ """Extract selected regular files from a tar archive while streaming it.
475
+
476
+ The archive (plain, gzip, bz2 or xz; auto-detected) is read sequentially from a URL or a
477
+ local path and is never stored. A stream cannot be resumed mid-archive: if reading fails
478
+ the call raises :class:`FetchError` and must simply be run again. Files extracted before
479
+ the failure are kept and skipped quickly on the rerun (the archive is read again from
480
+ the start, so the whole transfer is repeated).
481
+
482
+ For every regular-file member the name is normalised (leading ``./`` and empty or ``.``
483
+ components removed) and validated. Unsafe names (empty, absolute, containing ``..``
484
+ components, backslashes, NUL bytes or a Windows drive prefix) raise
485
+ :class:`FetchError` even when ``select`` would reject them: an archive with such members
486
+ is treated as hostile. Selected members are written to ``dest_dir / name`` through a
487
+ temporary file in the same directory and an atomic rename. A target that already exists
488
+ with the member's size is left untouched. Non-regular members (directories, symlinks,
489
+ hard links, devices) are never written; those that ``select`` accepts are counted in the
490
+ log message. If a name occurs twice in the archive the last occurrence wins, as with
491
+ ``tar``.
492
+
493
+ Args:
494
+ url_or_path: ``http(s)://`` URL (fetched with ``opener``) or a local file path.
495
+ dest_dir: Output directory; created if missing.
496
+ select: Predicate on the normalised member name (``/``-separated, relative).
497
+ opener: Network access function (injectable for tests); unused for local paths.
498
+ timeout_s: Socket timeout per request, seconds.
499
+
500
+ Returns:
501
+ Paths of the extracted or already present files, in archive order, without
502
+ duplicates.
503
+
504
+ Raises:
505
+ FetchError: On unsafe member names, unreadable/corrupt/truncated archives and network
506
+ failures.
507
+ OSError: On local filesystem failures (e.g. disk full).
508
+ """
509
+ if timeout_s <= 0:
510
+ raise ValueError(f"timeout_s must be > 0, got {timeout_s}")
511
+ dest_dir.mkdir(parents=True, exist_ok=True)
512
+ root = dest_dir.resolve()
513
+ label = str(url_or_path)
514
+
515
+ result: dict[str, Path] = {} # normalised name -> path; insertion order == archive order
516
+ written_this_run: set[str] = set()
517
+ scanned = 0
518
+ extracted = 0
519
+ already_present = 0
520
+ ignored_non_regular = 0
521
+
522
+ with _open_source(url_or_path, opener, timeout_s) as stream:
523
+ try:
524
+ # tarfile in "r|*" mode only ever calls ``stream.read``; urllib responses are not
525
+ # nominally IO[bytes], hence the cast.
526
+ with tarfile.open(fileobj=cast(IO[bytes], stream), mode="r|*") as archive:
527
+ for member in archive:
528
+ scanned += 1
529
+ if not member.isreg():
530
+ if select(_collapse_name(member.name)):
531
+ ignored_non_regular += 1
532
+ _forget_members(archive)
533
+ continue
534
+ name = _safe_member_name(member.name)
535
+ if select(name):
536
+ target = _resolve_target(root, dest_dir, name)
537
+ if (
538
+ name not in written_this_run
539
+ and target.is_file()
540
+ and target.stat().st_size == member.size
541
+ ):
542
+ already_present += 1
543
+ else:
544
+ source = archive.extractfile(member)
545
+ if source is None:
546
+ raise FetchError(f"cannot read member {name!r} of archive {label}")
547
+ _write_member_atomically(source, target, member.size, name, label)
548
+ written_this_run.add(name)
549
+ extracted += 1
550
+ result.setdefault(name, target)
551
+ # Streaming mode would otherwise keep every TarInfo alive.
552
+ _forget_members(archive)
553
+ except _STREAM_ERRORS as exc:
554
+ raise FetchError(
555
+ f"reading archive {label} failed after {scanned} members "
556
+ f"({extracted} extracted, {already_present} already present): {exc!r}; "
557
+ "re-run to continue (archives cannot be resumed mid-stream)"
558
+ ) from exc
559
+
560
+ logger.info(
561
+ "stream_extract %s: scanned %d members, extracted %d, already present %d, "
562
+ "ignored %d selected non-regular members (directories/links/devices)",
563
+ label,
564
+ scanned,
565
+ extracted,
566
+ already_present,
567
+ ignored_non_regular,
568
+ )
569
+ return list(result.values())
570
+
571
+
572
+ def _forget_members(archive: tarfile.TarFile) -> None:
573
+ """Drop ``TarFile``'s per-member cache; streaming a huge archive would otherwise grow it."""
574
+ # ``members`` is a runtime attribute that typeshed does not declare.
575
+ archive.members = [] # type: ignore[attr-defined]
576
+
577
+
578
+ @contextmanager
579
+ def _open_source(
580
+ url_or_path: str | Path, opener: Opener, timeout_s: float
581
+ ) -> Iterator[Response | IO[bytes]]:
582
+ if isinstance(url_or_path, str) and url_or_path.lower().startswith(("http://", "https://")):
583
+ request = urllib.request.Request(url_or_path, headers={"User-Agent": _USER_AGENT})
584
+ try:
585
+ response = opener(request, timeout_s)
586
+ except _NETWORK_ERRORS as exc: # includes HTTPError
587
+ raise FetchError(f"opening {url_or_path} failed: {exc!r}") from exc
588
+ with response:
589
+ if response.status != 200:
590
+ raise FetchError(f"unexpected HTTP status {response.status} from {url_or_path}")
591
+ yield response
592
+ else:
593
+ path = Path(url_or_path)
594
+ if not path.is_file():
595
+ raise FetchError(f"archive {path} does not exist or is not a file")
596
+ with path.open("rb") as handle:
597
+ yield handle
598
+
599
+
600
+ def _collapse_name(name: str) -> str:
601
+ """Drop leading ``./`` and empty/``.`` components; performs no safety checks."""
602
+ return "/".join(part for part in name.split("/") if part not in ("", "."))
603
+
604
+
605
+ def _safe_member_name(raw_name: str) -> str:
606
+ """Normalise a regular-file member name or raise ``FetchError`` if it is unsafe."""
607
+ reason: str | None = None
608
+ if "\x00" in raw_name:
609
+ reason = "contains a NUL byte"
610
+ elif "\\" in raw_name:
611
+ reason = "contains a backslash"
612
+ elif raw_name.startswith("/"):
613
+ reason = "is an absolute path"
614
+ elif _DRIVE_LETTER_RE.match(raw_name) is not None:
615
+ reason = "starts with a drive letter"
616
+ elif ".." in raw_name.split("/"):
617
+ reason = "contains a '..' component"
618
+ else:
619
+ name = _collapse_name(raw_name)
620
+ if name:
621
+ return name
622
+ reason = "is empty"
623
+ raise FetchError(f"unsafe tar member name {raw_name!r}: {reason}")
624
+
625
+
626
+ def _resolve_target(root: Path, dest_dir: Path, name: str) -> Path:
627
+ """Return ``dest_dir / name`` after checking that it stays inside ``dest_dir``."""
628
+ target = dest_dir.joinpath(*name.split("/"))
629
+ try:
630
+ target.resolve().relative_to(root)
631
+ except ValueError:
632
+ raise FetchError(f"tar member {name!r} would be written outside {dest_dir}") from None
633
+ return target
634
+
635
+
636
+ def _write_member_atomically(
637
+ source: IO[bytes], target: Path, expected_bytes: int, name: str, label: str
638
+ ) -> None:
639
+ """Copy ``source`` to ``target`` via a temp file in the same directory and a rename."""
640
+ target.parent.mkdir(parents=True, exist_ok=True)
641
+ fd, tmp_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".tmp", dir=target.parent)
642
+ tmp_path = Path(tmp_name)
643
+ try:
644
+ with os.fdopen(fd, "wb") as handle:
645
+ shutil.copyfileobj(source, handle, _COPY_CHUNK_BYTES)
646
+ written = handle.tell()
647
+ if written != expected_bytes:
648
+ raise FetchError(
649
+ f"member {name!r} of {label}: wrote {written} bytes, expected {expected_bytes}"
650
+ )
651
+ handle.flush()
652
+ os.fsync(handle.fileno())
653
+ tmp_path.replace(target)
654
+ except BaseException:
655
+ tmp_path.unlink(missing_ok=True)
656
+ raise
657
+
658
+
659
+ # --------------------------------------------------------------------------------------
660
+ # Hugging Face URLs
661
+ # --------------------------------------------------------------------------------------
662
+
663
+
664
+ def hf_resolve_url(
665
+ repo_id: str,
666
+ revision: str,
667
+ path_in_repo: str,
668
+ repo_type: Literal["model", "dataset"],
669
+ ) -> str:
670
+ """Build a revision-pinned Hugging Face ``resolve`` URL.
671
+
672
+ Args:
673
+ repo_id: Repository id such as ``"google/fleurs"``.
674
+ revision: Full 40-character lowercase hex commit SHA (branches and tags are refused
675
+ so the downloaded data is pinned).
676
+ path_in_repo: File path inside the repository (``/``-separated, relative).
677
+ repo_type: ``"dataset"`` or ``"model"``.
678
+
679
+ Returns:
680
+ ``https://huggingface.co/[datasets/]<repo_id>/resolve/<revision>/<path>`` with every
681
+ path component percent-encoded (``/`` kept).
682
+
683
+ Raises:
684
+ ValueError: On a malformed revision, repo id or path, or an unknown repo type.
685
+ """
686
+ if repo_type not in ("model", "dataset"):
687
+ raise ValueError(f"repo_type must be 'model' or 'dataset', got {repo_type!r}")
688
+ if _COMMIT_SHA_RE.fullmatch(revision) is None:
689
+ raise ValueError(f"revision must be a 40-character lowercase hex commit SHA: {revision!r}")
690
+ _require_clean_path("repo_id", repo_id)
691
+ _require_clean_path("path_in_repo", path_in_repo)
692
+ prefix = "datasets/" if repo_type == "dataset" else ""
693
+ quoted_repo = urllib.parse.quote(repo_id, safe="/")
694
+ quoted_path = urllib.parse.quote(path_in_repo, safe="/")
695
+ return f"https://huggingface.co/{prefix}{quoted_repo}/resolve/{revision}/{quoted_path}"
696
+
697
+
698
+ def _require_clean_path(what: str, value: str) -> None:
699
+ parts = value.split("/")
700
+ if not value or any(part in ("", ".", "..") for part in parts) or "\\" in value:
701
+ raise ValueError(
702
+ f"{what} must be a relative '/'-separated path without empty, '.' or '..' "
703
+ f"components: {value!r}"
704
+ )
src/nida/bench/panel.py ADDED
@@ -0,0 +1,817 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Self-contained HTML listening panels and a strict parser for their CSV export.
2
+
3
+ A panel is a single HTML file that a human opens locally in a browser. It embeds the audio
4
+ (base64 data URIs), shows one card per item with per-item questions, keeps answers in
5
+ ``localStorage`` and exports a CSV through a "CSV indir" button. The panel makes no network
6
+ requests of any kind.
7
+
8
+ The CSV layout is a contract between the in-page JavaScript (``buildCsv``) and
9
+ :func:`read_panel_results`::
10
+
11
+ panel_id,item_id,<question ids in spec order>,[text,]note
12
+
13
+ ``text`` is present only if at least one item has ``editable_text``. The file is UTF-8 with
14
+ a BOM, fields are quoted per RFC 4180 and lines end with CRLF. Unanswered questions are
15
+ empty fields.
16
+
17
+ User-visible UI strings are Turkish; everything else is English.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import base64
23
+ import csv
24
+ import html
25
+ import io
26
+ import json
27
+ import re
28
+ from collections.abc import Mapping
29
+ from dataclasses import dataclass
30
+ from pathlib import Path
31
+ from typing import Literal
32
+
33
+ from nida.text.turkish import nfc
34
+ from nida.utils.io import atomic_write_json, atomic_write_text
35
+
36
+ __all__ = [
37
+ "AudioRef",
38
+ "PanelItem",
39
+ "PanelResult",
40
+ "PanelSpec",
41
+ "Question",
42
+ "build_panel_html",
43
+ "csv_header",
44
+ "has_text_column",
45
+ "load_panel_spec",
46
+ "missing_required",
47
+ "read_panel_results",
48
+ "write_panel",
49
+ ]
50
+
51
+ QuestionKind = Literal["choice", "scale", "text"]
52
+
53
+ _QUESTION_KINDS: tuple[str, ...] = ("choice", "scale", "text")
54
+ _QUESTION_ID_RE = re.compile(r"[a-z][a-z0-9_]{0,31}")
55
+ _ITEM_ID_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}")
56
+ _RESERVED_QUESTION_IDS = frozenset({"panel_id", "item_id", "text", "note"})
57
+ _AUDIO_MIME_BY_SUFFIX: Mapping[str, str] = {
58
+ ".wav": "audio/wav",
59
+ ".flac": "audio/flac",
60
+ ".mp3": "audio/mpeg",
61
+ ".ogg": "audio/ogg",
62
+ ".opus": "audio/ogg",
63
+ }
64
+ _DEFAULT_MAX_BYTES = 60_000_000
65
+ _LINE_SEPARATOR = chr(0x2028)
66
+ _PARAGRAPH_SEPARATOR = chr(0x2029)
67
+
68
+
69
+ def audio_mime(path: Path) -> str:
70
+ """Return the MIME type for an audio file, chosen by (case-insensitive) suffix.
71
+
72
+ Raises:
73
+ ValueError: if the suffix is not one of .wav, .flac, .mp3, .ogg, .opus.
74
+ """
75
+ mime = _AUDIO_MIME_BY_SUFFIX.get(path.suffix.lower())
76
+ if mime is None:
77
+ supported = ", ".join(sorted(_AUDIO_MIME_BY_SUFFIX))
78
+ raise ValueError(
79
+ f"unsupported audio suffix {path.suffix!r} for {path} (supported: {supported})"
80
+ )
81
+ return mime
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class AudioRef:
86
+ """One audio clip shown on a card, with the label the listener sees."""
87
+
88
+ label: str
89
+ path: Path
90
+
91
+ def __post_init__(self) -> None:
92
+ if not self.label.strip():
93
+ raise ValueError(f"AudioRef for {self.path} has an empty label")
94
+ audio_mime(self.path) # validates the suffix
95
+
96
+
97
+ @dataclass(frozen=True)
98
+ class Question:
99
+ """A per-item question.
100
+
101
+ ``choice`` renders radio buttons from ``options``; ``scale`` does the same with the
102
+ options being the ordered allowed values (e.g. ``("1", "2", "3", "4", "5")``); ``text``
103
+ is a free-text box and takes no options.
104
+ """
105
+
106
+ id: str
107
+ prompt: str
108
+ kind: QuestionKind
109
+ options: tuple[str, ...] = ()
110
+ required: bool = True
111
+
112
+ def __post_init__(self) -> None:
113
+ if _QUESTION_ID_RE.fullmatch(self.id) is None:
114
+ raise ValueError(f"question id {self.id!r} does not match {_QUESTION_ID_RE.pattern}")
115
+ if self.id in _RESERVED_QUESTION_IDS:
116
+ raise ValueError(f"question id {self.id!r} is reserved")
117
+ if not self.prompt.strip():
118
+ raise ValueError(f"question {self.id!r} has an empty prompt")
119
+ if self.kind not in _QUESTION_KINDS:
120
+ raise ValueError(f"question {self.id!r} has unknown kind {self.kind!r}")
121
+ if self.kind == "text":
122
+ if self.options:
123
+ raise ValueError(f"text question {self.id!r} must not have options")
124
+ return
125
+ if len(self.options) < 2:
126
+ raise ValueError(f"{self.kind} question {self.id!r} needs at least 2 options")
127
+ if any(option == "" for option in self.options):
128
+ raise ValueError(f"{self.kind} question {self.id!r} has an empty option")
129
+ if len(set(self.options)) != len(self.options):
130
+ raise ValueError(f"{self.kind} question {self.id!r} has duplicate options")
131
+
132
+
133
+ @dataclass(frozen=True)
134
+ class PanelItem:
135
+ """One card: audio clips plus optional (editable) text and a caption."""
136
+
137
+ item_id: str
138
+ audios: tuple[AudioRef, ...]
139
+ text: str = ""
140
+ editable_text: bool = False
141
+ info: str = ""
142
+
143
+ def __post_init__(self) -> None:
144
+ if _ITEM_ID_RE.fullmatch(self.item_id) is None:
145
+ raise ValueError(f"item id {self.item_id!r} does not match {_ITEM_ID_RE.pattern}")
146
+ if not self.audios:
147
+ raise ValueError(f"item {self.item_id!r} has no audio")
148
+
149
+
150
+ @dataclass(frozen=True)
151
+ class PanelSpec:
152
+ """A whole panel: title, instructions, questions and items."""
153
+
154
+ panel_id: str
155
+ title: str
156
+ instructions: str
157
+ questions: tuple[Question, ...]
158
+ items: tuple[PanelItem, ...]
159
+
160
+ def __post_init__(self) -> None:
161
+ if _ITEM_ID_RE.fullmatch(self.panel_id) is None:
162
+ raise ValueError(f"panel id {self.panel_id!r} does not match {_ITEM_ID_RE.pattern}")
163
+ if not self.title.strip():
164
+ raise ValueError(f"panel {self.panel_id!r} has an empty title")
165
+ if not self.items:
166
+ raise ValueError(f"panel {self.panel_id!r} has no items")
167
+ _raise_on_duplicates([q.id for q in self.questions], "question id", self.panel_id)
168
+ _raise_on_duplicates([i.item_id for i in self.items], "item id", self.panel_id)
169
+
170
+
171
+ def _raise_on_duplicates(values: list[str], what: str, panel_id: str) -> None:
172
+ seen: set[str] = set()
173
+ duplicates: list[str] = []
174
+ for value in values:
175
+ if value in seen and value not in duplicates:
176
+ duplicates.append(value)
177
+ seen.add(value)
178
+ if duplicates:
179
+ raise ValueError(f"panel {panel_id!r} has duplicate {what}s: {duplicates}")
180
+
181
+
182
+ def has_text_column(spec: PanelSpec) -> bool:
183
+ """Whether the CSV has a ``text`` column (any item has ``editable_text``)."""
184
+ return any(item.editable_text for item in spec.items)
185
+
186
+
187
+ def csv_header(spec: PanelSpec) -> list[str]:
188
+ """The exact CSV header row produced by the panel for ``spec``."""
189
+ header = ["panel_id", "item_id", *(q.id for q in spec.questions)]
190
+ if has_text_column(spec):
191
+ header.append("text")
192
+ header.append("note")
193
+ return header
194
+
195
+
196
+ # --------------------------------------------------------------------------------------
197
+ # HTML generation
198
+ # --------------------------------------------------------------------------------------
199
+
200
+ _CSS = r"""
201
+ :root {
202
+ color-scheme: light dark;
203
+ --bg: #f6f6f3; --fg: #1b1b1a; --muted: #5d5d58; --card: #ffffff; --line: #d7d6cf;
204
+ --accent: #1d5fd0; --accent-fg: #ffffff; --ok: #1a7f3c; --warn: #b45309;
205
+ --field: #ffffff;
206
+ }
207
+ @media (prefers-color-scheme: dark) {
208
+ :root {
209
+ --bg: #151515; --fg: #ececea; --muted: #a3a39d; --card: #1f1f1e; --line: #3a3a37;
210
+ --accent: #6ea0ff; --accent-fg: #0b0b0b; --ok: #5fcf85; --warn: #f0b05a;
211
+ --field: #161616;
212
+ }
213
+ }
214
+ * { box-sizing: border-box; }
215
+ html { -webkit-text-size-adjust: 100%; }
216
+ body {
217
+ margin: 0; background: var(--bg); color: var(--fg);
218
+ font: 16px/1.5 system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
219
+ }
220
+ header.top { padding: 20px 16px 8px; max-width: 760px; margin: 0 auto; }
221
+ h1 { font-size: 1.4rem; margin: 0 0 8px; }
222
+ .instructions { white-space: pre-wrap; color: var(--muted); margin: 0 0 8px; }
223
+ .bar {
224
+ position: sticky; top: 0; z-index: 5; background: var(--bg);
225
+ border-bottom: 1px solid var(--line);
226
+ }
227
+ .bar-inner {
228
+ max-width: 760px; margin: 0 auto; padding: 10px 16px; display: flex; gap: 12px;
229
+ align-items: center; justify-content: space-between; flex-wrap: wrap;
230
+ }
231
+ #progress { font-weight: 600; }
232
+ button.primary {
233
+ min-height: 44px; padding: 0 18px; border: 0; border-radius: 10px; cursor: pointer;
234
+ background: var(--accent); color: var(--accent-fg); font: inherit; font-weight: 600;
235
+ }
236
+ main { max-width: 760px; margin: 0 auto; padding: 8px 16px 64px; }
237
+ .card {
238
+ background: var(--card); border: 1px solid var(--line); border-radius: 14px;
239
+ padding: 14px; margin: 14px 0;
240
+ }
241
+ .card.done { border-color: var(--ok); }
242
+ .card-head { display: flex; gap: 10px; align-items: baseline; flex-wrap: wrap; }
243
+ .num { font-weight: 700; font-size: 1.1rem; }
244
+ .item-id { color: var(--muted); font-size: 0.85rem; word-break: break-all; }
245
+ .info { color: var(--muted); font-size: 0.9rem; margin: 2px 0 8px; white-space: pre-wrap; }
246
+ .audio { margin: 10px 0; }
247
+ .alabel { font-size: 0.9rem; font-weight: 600; margin-bottom: 4px; }
248
+ audio { width: 100%; }
249
+ .text {
250
+ white-space: pre-wrap; margin: 10px 0; padding: 10px 12px; border-radius: 10px;
251
+ background: var(--bg); border: 1px solid var(--line);
252
+ }
253
+ fieldset.q { border: 0; margin: 14px 0 0; padding: 0; min-width: 0; }
254
+ legend, label.lbl { font-weight: 600; padding: 0; margin-bottom: 6px; display: block; }
255
+ .req { color: var(--warn); margin-left: 4px; }
256
+ .opts { display: flex; flex-wrap: wrap; gap: 8px; }
257
+ .opt { position: relative; }
258
+ .opt input {
259
+ position: absolute; opacity: 0; width: 100%; height: 100%; margin: 0; cursor: pointer;
260
+ }
261
+ .opt span {
262
+ display: inline-flex; align-items: center; justify-content: center; min-width: 44px;
263
+ min-height: 44px; padding: 0 14px; border: 1px solid var(--line); border-radius: 10px;
264
+ background: var(--field);
265
+ }
266
+ .opt input:checked + span {
267
+ background: var(--accent); color: var(--accent-fg); border-color: var(--accent);
268
+ }
269
+ .opt input:focus-visible + span { outline: 2px solid var(--accent); outline-offset: 2px; }
270
+ textarea {
271
+ width: 100%; font: inherit; color: var(--fg); background: var(--field);
272
+ border: 1px solid var(--line); border-radius: 10px; padding: 8px 10px; resize: vertical;
273
+ }
274
+ .field { margin-top: 14px; }
275
+ """
276
+
277
+ _JS = r"""
278
+ (function () {
279
+ "use strict";
280
+ var spec = JSON.parse(document.getElementById("panel-data").textContent);
281
+ var STORAGE_KEY = "nida-panel:" + spec.panel_id;
282
+
283
+ /* BEGIN buildCsv */
284
+ function own(obj, key) {
285
+ return obj !== null && obj !== undefined &&
286
+ Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : undefined;
287
+ }
288
+
289
+ function buildCsv(spec, answers) {
290
+ function field(value) {
291
+ var s = value === undefined || value === null ? "" : String(value);
292
+ return /[",\r\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
293
+ }
294
+ var hasText = spec.items.some(function (it) { return it.editable_text; });
295
+ var header = ["panel_id", "item_id"];
296
+ spec.questions.forEach(function (q) { header.push(q.id); });
297
+ if (hasText) { header.push("text"); }
298
+ header.push("note");
299
+ var lines = [header.map(field).join(",")];
300
+ spec.items.forEach(function (it) {
301
+ var a = own(answers, it.item_id);
302
+ var row = [spec.panel_id, it.item_id];
303
+ spec.questions.forEach(function (q) { row.push(own(a, q.id)); });
304
+ if (hasText) {
305
+ var text = own(a, "text");
306
+ row.push(it.editable_text ? (text === undefined ? it.text : text) : "");
307
+ }
308
+ row.push(own(a, "note"));
309
+ lines.push(row.map(field).join(","));
310
+ });
311
+ return lines.join("\r\n") + "\r\n";
312
+ }
313
+ /* END buildCsv */
314
+
315
+ /* BEGIN missingRequired */
316
+ function missingRequired(spec, answers) {
317
+ var missing = [];
318
+ spec.items.forEach(function (it) {
319
+ var a = own(answers, it.item_id);
320
+ var absent = spec.questions.some(function (q) {
321
+ if (!q.required) { return false; }
322
+ var v = own(a, q.id);
323
+ return v === undefined || String(v).trim() === "";
324
+ });
325
+ if (absent) { missing.push(it.item_id); }
326
+ });
327
+ return missing;
328
+ }
329
+ /* END missingRequired */
330
+
331
+ var cards = Array.prototype.slice.call(document.querySelectorAll(".card"));
332
+ var hasRequired = spec.questions.some(function (q) { return q.required; });
333
+
334
+ function collect() {
335
+ var out = Object.create(null);
336
+ cards.forEach(function (card) {
337
+ var id = card.getAttribute("data-item");
338
+ var a = Object.create(null);
339
+ spec.questions.forEach(function (q) {
340
+ if (q.kind === "text") {
341
+ var ta = card.querySelector('textarea[data-q="' + q.id + '"]');
342
+ a[q.id] = ta ? ta.value : "";
343
+ } else {
344
+ var r = card.querySelector('input[data-q="' + q.id + '"]:checked');
345
+ a[q.id] = r ? r.value : "";
346
+ }
347
+ });
348
+ var t = card.querySelector('textarea[data-field="text"]');
349
+ if (t) { a.text = t.value; }
350
+ var n = card.querySelector('textarea[data-field="note"]');
351
+ a.note = n ? n.value : "";
352
+ out[id] = a;
353
+ });
354
+ return out;
355
+ }
356
+
357
+ function restore(saved) {
358
+ if (saved === null || typeof saved !== "object") { return; }
359
+ cards.forEach(function (card) {
360
+ var a = own(saved, card.getAttribute("data-item"));
361
+ if (a === null || typeof a !== "object") { return; }
362
+ spec.questions.forEach(function (q) {
363
+ var v = own(a, q.id);
364
+ if (typeof v !== "string") { return; }
365
+ if (q.kind === "text") {
366
+ var ta = card.querySelector('textarea[data-q="' + q.id + '"]');
367
+ if (ta) { ta.value = v; }
368
+ } else {
369
+ var radios = card.querySelectorAll('input[data-q="' + q.id + '"]');
370
+ Array.prototype.forEach.call(radios, function (r) { r.checked = (r.value === v); });
371
+ }
372
+ });
373
+ var t = card.querySelector('textarea[data-field="text"]');
374
+ if (t && typeof own(a, "text") === "string") { t.value = a.text; }
375
+ var n = card.querySelector('textarea[data-field="note"]');
376
+ if (n && typeof own(a, "note") === "string") { n.value = a.note; }
377
+ });
378
+ }
379
+
380
+ function isComplete(item, a) {
381
+ if (missingRequired({questions: spec.questions, items: [item]}, {[item.item_id]: a}).length) {
382
+ return false;
383
+ }
384
+ if (hasRequired) { return true; }
385
+ var touched = spec.questions.some(function (q) { return String(own(a, q.id) || "") !== ""; });
386
+ return touched || String(own(a, "note") || "") !== "" ||
387
+ (item.editable_text && own(a, "text") !== item.text);
388
+ }
389
+
390
+ function refresh() {
391
+ var answers = collect();
392
+ var done = 0;
393
+ spec.items.forEach(function (item, i) {
394
+ var ok = isComplete(item, own(answers, item.item_id));
395
+ cards[i].classList.toggle("done", ok);
396
+ if (ok) { done += 1; }
397
+ });
398
+ document.getElementById("progress").textContent =
399
+ done + " / " + spec.items.length + " tamamlandı";
400
+ try {
401
+ window.localStorage.setItem(STORAGE_KEY, JSON.stringify(answers));
402
+ } catch (e) { /* storage unavailable */ }
403
+ }
404
+
405
+ function download() {
406
+ var answers = collect();
407
+ var missing = missingRequired(spec, answers);
408
+ if (missing.length > 0) {
409
+ var msg = missing.length + " öğede zorunlu sorular yanıtsız. Yine de CSV indirilsin mi?";
410
+ if (!window.confirm(msg)) { return; }
411
+ }
412
+ var blob = new Blob(["" + buildCsv(spec, answers)], {type: "text/csv;charset=utf-8"});
413
+ var url = URL.createObjectURL(blob);
414
+ var link = document.createElement("a");
415
+ link.href = url;
416
+ link.download = spec.panel_id + "-results.csv";
417
+ document.body.appendChild(link);
418
+ link.click();
419
+ document.body.removeChild(link);
420
+ setTimeout(function () { URL.revokeObjectURL(url); }, 1000);
421
+ }
422
+
423
+ try {
424
+ var raw = window.localStorage.getItem(STORAGE_KEY);
425
+ if (raw) { restore(JSON.parse(raw)); }
426
+ } catch (e) { /* storage unavailable or corrupt: start fresh */ }
427
+
428
+ document.addEventListener("input", refresh);
429
+ document.addEventListener("change", refresh);
430
+ document.addEventListener("play", function (ev) {
431
+ Array.prototype.forEach.call(document.querySelectorAll("audio"), function (el) {
432
+ if (el !== ev.target) { el.pause(); }
433
+ });
434
+ }, true);
435
+ document.getElementById("download").addEventListener("click", download);
436
+ refresh();
437
+ })();
438
+ """
439
+
440
+ _HTML_TEMPLATE = """<!DOCTYPE html>
441
+ <html lang="tr">
442
+ <head>
443
+ <meta charset="utf-8">
444
+ <meta name="viewport" content="width=device-width, initial-scale=1">
445
+ <title>@@TITLE@@</title>
446
+ <style>@@CSS@@</style>
447
+ </head>
448
+ <body>
449
+ <header class="top">
450
+ <h1>@@TITLE@@</h1>
451
+ <p class="instructions">@@INSTRUCTIONS@@</p>
452
+ <p class="instructions">* ile işaretli sorular zorunludur.
453
+ Yanıtlarınız bu tarayıcıda otomatik kaydedilir.</p>
454
+ </header>
455
+ <div class="bar"><div class="bar-inner">
456
+ <span id="progress" role="status">0 / @@COUNT@@ tamamlandı</span>
457
+ <button id="download" class="primary" type="button">CSV indir</button>
458
+ </div></div>
459
+ <main>
460
+ @@CARDS@@
461
+ </main>
462
+ <script type="application/json" id="panel-data">@@DATA@@</script>
463
+ <script>@@JS@@</script>
464
+ </body>
465
+ </html>
466
+ """
467
+
468
+
469
+ def _esc(value: str) -> str:
470
+ return html.escape(value, quote=True)
471
+
472
+
473
+ def _embed_json(obj: object) -> str:
474
+ """Serialise ``obj`` for a ``<script type="application/json">`` block.
475
+
476
+ ``<``, ``>``, ``&`` and the JS line separators are escaped so that no text can close the
477
+ script element or be parsed as markup; the JSON stays valid and decodes identically.
478
+ """
479
+ text = json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
480
+ return (
481
+ text.replace("<", "\\u003c")
482
+ .replace(">", "\\u003e")
483
+ .replace("&", "\\u0026")
484
+ .replace(_LINE_SEPARATOR, "\\u2028")
485
+ .replace(_PARAGRAPH_SEPARATOR, "\\u2029")
486
+ )
487
+
488
+
489
+ def _question_to_dict(question: Question) -> dict[str, object]:
490
+ return {
491
+ "id": question.id,
492
+ "prompt": question.prompt,
493
+ "kind": question.kind,
494
+ "options": list(question.options),
495
+ "required": question.required,
496
+ }
497
+
498
+
499
+ def _spec_to_dict(spec: PanelSpec, *, embed: bool) -> dict[str, object]:
500
+ """Machine-readable spec. ``embed=True`` is the in-page form (audio MIME types);
501
+ otherwise audio file names are listed, never bytes."""
502
+ items: list[dict[str, object]] = []
503
+ for item in spec.items:
504
+ audios: list[dict[str, str]] = [
505
+ {"label": a.label, "mime": audio_mime(a.path)}
506
+ if embed
507
+ else {"label": a.label, "file_name": a.path.name}
508
+ for a in item.audios
509
+ ]
510
+ items.append(
511
+ {
512
+ "item_id": item.item_id,
513
+ "audios": audios,
514
+ "text": item.text,
515
+ "editable_text": item.editable_text,
516
+ "info": item.info,
517
+ }
518
+ )
519
+ return {
520
+ "panel_id": spec.panel_id,
521
+ "title": spec.title,
522
+ "instructions": spec.instructions,
523
+ "questions": [_question_to_dict(q) for q in spec.questions],
524
+ "items": items,
525
+ }
526
+
527
+
528
+ def _audio_html(audio: AudioRef) -> str:
529
+ payload = base64.b64encode(audio.path.read_bytes()).decode("ascii")
530
+ src = f"data:{audio_mime(audio.path)};base64,{payload}"
531
+ return (
532
+ f'<div class="audio"><div class="alabel">{_esc(audio.label)}</div>'
533
+ f'<audio controls preload="none" src="{src}"></audio></div>'
534
+ )
535
+
536
+
537
+ def _question_html(question: Question, item_index: int) -> str:
538
+ star = '<span class="req" aria-hidden="true">*</span>' if question.required else ""
539
+ prompt = _esc(question.prompt)
540
+ qid = _esc(question.id)
541
+ if question.kind == "text":
542
+ field_id = f"t{item_index}_{qid}"
543
+ return (
544
+ f'<div class="field"><label class="lbl" for="{field_id}">{prompt}{star}</label>'
545
+ f'<textarea id="{field_id}" data-q="{qid}" rows="2"></textarea></div>'
546
+ )
547
+ name = f"r{item_index}_{qid}"
548
+ options = "".join(
549
+ f'<label class="opt"><input type="radio" name="{name}" data-q="{qid}" '
550
+ f'value="{_esc(option)}"><span>{_esc(option)}</span></label>'
551
+ for option in question.options
552
+ )
553
+ return (
554
+ f'<fieldset class="q"><legend>{prompt}{star}</legend>'
555
+ f'<div class="opts">{options}</div></fieldset>'
556
+ )
557
+
558
+
559
+ def _card_html(item: PanelItem, index: int, questions: tuple[Question, ...]) -> str:
560
+ parts = [
561
+ f'<section class="card" data-item="{_esc(item.item_id)}">',
562
+ f'<div class="card-head"><span class="num">{index + 1}</span>'
563
+ f'<span class="item-id">{_esc(item.item_id)}</span></div>',
564
+ ]
565
+ if item.info:
566
+ parts.append(f'<div class="info">{_esc(item.info)}</div>')
567
+ parts.extend(_audio_html(a) for a in item.audios)
568
+ if item.editable_text:
569
+ # A single leading newline inside <textarea> is dropped by the HTML parser, so add one
570
+ # to keep text that itself starts with a newline intact.
571
+ parts.append(
572
+ f'<div class="field"><label class="lbl" for="e{index}">'
573
+ "Metin (düzenleyebilirsiniz)</label>"
574
+ f'<textarea id="e{index}" data-field="text" rows="3">\n{_esc(item.text)}</textarea>'
575
+ "</div>"
576
+ )
577
+ elif item.text:
578
+ parts.append(f'<div class="text">{_esc(item.text)}</div>')
579
+ parts.extend(_question_html(q, index) for q in questions)
580
+ parts.append(
581
+ f'<div class="field"><label class="lbl" for="n{index}">Not (isteğe bağlı)</label>'
582
+ f'<textarea id="n{index}" data-field="note" rows="2"></textarea></div>'
583
+ )
584
+ parts.append("</section>")
585
+ return "\n".join(parts)
586
+
587
+
588
+ def build_panel_html(spec: PanelSpec, max_bytes: int = _DEFAULT_MAX_BYTES) -> str:
589
+ """Render ``spec`` as one self-contained HTML5 document.
590
+
591
+ Args:
592
+ spec: The panel definition.
593
+ max_bytes: Upper bound for the UTF-8 size of the document.
594
+
595
+ Returns:
596
+ The HTML text. It references no external resource; audio is embedded as data URIs.
597
+
598
+ Raises:
599
+ ValueError: if the document exceeds ``max_bytes`` (the message reports the size).
600
+ FileNotFoundError: if an audio file is missing.
601
+ """
602
+ cards = "\n".join(_card_html(item, i, spec.questions) for i, item in enumerate(spec.items))
603
+ values = {
604
+ "TITLE": _esc(spec.title),
605
+ "INSTRUCTIONS": _esc(spec.instructions),
606
+ "COUNT": str(len(spec.items)),
607
+ "CSS": _CSS,
608
+ "CARDS": cards,
609
+ "DATA": _embed_json(_spec_to_dict(spec, embed=True)),
610
+ "JS": _JS,
611
+ }
612
+ # Single-pass substitution: inserted values are never rescanned for placeholders.
613
+ document = re.sub(r"@@([A-Z]+)@@", lambda m: values[m.group(1)], _HTML_TEMPLATE)
614
+ size_bytes = len(document.encode("utf-8"))
615
+ if size_bytes > max_bytes:
616
+ raise ValueError(
617
+ f"panel {spec.panel_id!r} is {size_bytes} bytes, over the limit of {max_bytes} bytes; "
618
+ "use fewer items or shorter clips"
619
+ )
620
+ return document
621
+
622
+
623
+ def write_panel(spec: PanelSpec, out_dir: Path) -> Path:
624
+ """Write ``panel.html`` and ``panel.json`` into ``out_dir`` and return the HTML path."""
625
+ document = build_panel_html(spec)
626
+ html_path = out_dir / "panel.html"
627
+ atomic_write_text(html_path, document)
628
+ atomic_write_json(out_dir / "panel.json", _spec_to_dict(spec, embed=False))
629
+ return html_path
630
+
631
+
632
+ # --------------------------------------------------------------------------------------
633
+ # CSV parsing
634
+ # --------------------------------------------------------------------------------------
635
+
636
+
637
+ @dataclass(frozen=True)
638
+ class PanelResult:
639
+ """One parsed CSV row.
640
+
641
+ ``answers`` maps every question id to its answer ("" if unanswered). ``text`` is the
642
+ edited text for items with ``editable_text`` and ``None`` otherwise.
643
+ """
644
+
645
+ item_id: str
646
+ answers: Mapping[str, str]
647
+ text: str | None
648
+ note: str
649
+
650
+
651
+ def _describe_header_difference(expected: list[str], actual: list[str]) -> str:
652
+ missing = [c for c in expected if c not in actual]
653
+ unexpected = [c for c in actual if c not in expected]
654
+ parts: list[str] = []
655
+ if missing:
656
+ parts.append(f"missing columns {missing}")
657
+ if unexpected:
658
+ parts.append(f"unexpected columns {unexpected}")
659
+ if not parts:
660
+ parts.append("columns are the same but in a different order")
661
+ return "; ".join(parts)
662
+
663
+
664
+ def _decode_csv_bytes(data: bytes, csv_path: Path) -> str:
665
+ try:
666
+ text = data.decode("utf-8")
667
+ except UnicodeDecodeError as exc:
668
+ raise ValueError(f"{csv_path} is not valid UTF-8: {exc}") from exc
669
+ return text.removeprefix("")
670
+
671
+
672
+ def read_panel_results(csv_path: Path, spec: PanelSpec) -> list[PanelResult]:
673
+ """Parse a CSV exported by the panel for ``spec``, strictly.
674
+
675
+ Free-text fields (``text``, ``note`` and ``text`` questions) are NFC-normalised.
676
+
677
+ Returns:
678
+ One :class:`PanelResult` per row, in file order.
679
+
680
+ Raises:
681
+ ValueError: on a malformed file, header mismatch, wrong ``panel_id``, unknown or
682
+ repeated item, an answer that is not one of the options, text in the ``text``
683
+ column of a non-editable item, or items of ``spec`` absent from the file.
684
+ """
685
+ text = _decode_csv_bytes(csv_path.read_bytes(), csv_path)
686
+ reader = csv.reader(io.StringIO(text, newline=""), strict=True)
687
+ expected = csv_header(spec)
688
+ questions = {q.id: q for q in spec.questions}
689
+ items = {item.item_id: item for item in spec.items}
690
+ with_text = has_text_column(spec)
691
+
692
+ try:
693
+ header = next(reader, None)
694
+ if header is None:
695
+ raise ValueError(f"{csv_path} is empty; expected header {expected}")
696
+ if header != expected:
697
+ raise ValueError(
698
+ f"{csv_path}: header mismatch ({_describe_header_difference(expected, header)}); "
699
+ f"expected {expected}, got {header}"
700
+ )
701
+ results: list[PanelResult] = []
702
+ seen: set[str] = set()
703
+ for row in reader:
704
+ if not row: # blank line (e.g. trailing newline added by an editor)
705
+ continue
706
+ where = f"{csv_path} line {reader.line_num}"
707
+ if len(row) != len(expected):
708
+ raise ValueError(f"{where}: expected {len(expected)} fields, got {len(row)}")
709
+ cells = dict(zip(expected, row, strict=True))
710
+ if cells["panel_id"] != spec.panel_id:
711
+ raise ValueError(
712
+ f"{where}: panel_id {cells['panel_id']!r} does not match {spec.panel_id!r}"
713
+ )
714
+ item_id = cells["item_id"]
715
+ item = items.get(item_id)
716
+ if item is None:
717
+ raise ValueError(f"{where}: unknown item_id {item_id!r}")
718
+ if item_id in seen:
719
+ raise ValueError(f"{where}: duplicate item_id {item_id!r}")
720
+ seen.add(item_id)
721
+ answers: dict[str, str] = {}
722
+ for qid, question in questions.items():
723
+ value = cells[qid]
724
+ if question.kind == "text":
725
+ value = nfc(value)
726
+ elif value != "" and value not in question.options:
727
+ raise ValueError(
728
+ f"{where}: item {item_id!r} question {qid!r}: {value!r} is not one of "
729
+ f"{list(question.options)}"
730
+ )
731
+ answers[qid] = value
732
+ edited: str | None = None
733
+ if with_text:
734
+ if item.editable_text:
735
+ edited = nfc(cells["text"])
736
+ elif cells["text"] != "":
737
+ raise ValueError(
738
+ f"{where}: item {item_id!r} is not editable but has text {cells['text']!r}"
739
+ )
740
+ results.append(
741
+ PanelResult(item_id=item_id, answers=answers, text=edited, note=nfc(cells["note"]))
742
+ )
743
+ except csv.Error as exc:
744
+ raise ValueError(f"{csv_path}: malformed CSV near line {reader.line_num}: {exc}") from exc
745
+
746
+ absent = [item.item_id for item in spec.items if item.item_id not in seen]
747
+ if absent:
748
+ raise ValueError(f"{csv_path}: {len(absent)} item(s) missing from the CSV: {absent}")
749
+ return results
750
+
751
+
752
+ def missing_required(results: list[PanelResult], spec: PanelSpec) -> list[str]:
753
+ """Item ids (in ``results`` order) whose required questions lack a non-blank answer."""
754
+ required = [q.id for q in spec.questions if q.required]
755
+ return [
756
+ result.item_id
757
+ for result in results
758
+ if any(not result.answers.get(qid, "").strip() for qid in required)
759
+ ]
760
+
761
+
762
+ def _bare_name(name: object) -> str:
763
+ """A stored audio file name must be a plain name inside the panel directory."""
764
+ if (
765
+ not isinstance(name, str)
766
+ or name in ("", ".", "..")
767
+ or "/" in name
768
+ or "\\" in name
769
+ or Path(name).name != name
770
+ ):
771
+ raise ValueError(f"panel audio file name must be a bare name, got {name!r}")
772
+ return name
773
+
774
+
775
+ def load_panel_spec(path: Path) -> PanelSpec:
776
+ """Rebuild a :class:`PanelSpec` from the ``panel.json`` written by :func:`write_panel`.
777
+
778
+ Audio paths point next to ``panel.json`` (file names only are stored); they are needed
779
+ for validating results, not for reading audio, so they need not exist.
780
+
781
+ Raises:
782
+ ValueError: if the file does not have the expected structure.
783
+ """
784
+ obj = json.loads(path.read_text(encoding="utf-8"))
785
+ try:
786
+ questions = tuple(
787
+ Question(
788
+ id=q["id"],
789
+ prompt=q["prompt"],
790
+ kind=q["kind"],
791
+ options=tuple(q["options"]),
792
+ required=bool(q["required"]),
793
+ )
794
+ for q in obj["questions"]
795
+ )
796
+ items = tuple(
797
+ PanelItem(
798
+ item_id=i["item_id"],
799
+ audios=tuple(
800
+ AudioRef(label=a["label"], path=path.parent / _bare_name(a["file_name"]))
801
+ for a in i["audios"]
802
+ ),
803
+ text=i["text"],
804
+ editable_text=bool(i["editable_text"]),
805
+ info=i["info"],
806
+ )
807
+ for i in obj["items"]
808
+ )
809
+ return PanelSpec(
810
+ panel_id=obj["panel_id"],
811
+ title=obj["title"],
812
+ instructions=obj["instructions"],
813
+ questions=questions,
814
+ items=items,
815
+ )
816
+ except (KeyError, TypeError) as err:
817
+ raise ValueError(f"{path}: malformed panel spec ({err})") from err
src/nida/bench/phase0.py ADDED
@@ -0,0 +1,358 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Phase 0 benchmark stage runner behind ``nida bench-phase0`` (B1 + B2 on Colab).
2
+
3
+ Every stage first checks that all sources it touches are ``approved`` in
4
+ ``configs/sources.yaml``. Stages are idempotent and resumable; ``all`` runs the whole
5
+ GPU part in one session (one Drive permission), in this order:
6
+
7
+ check → prepare-b1 → asr-b1 → panel-b1c → score-b1 → prepare-b2 → vae-b2 → asr-b2
8
+ → quality-b2 → score-b2 → panel-b2
9
+
10
+ After the user has answered the panels (no GPU needed):
11
+
12
+ * ``score-b1`` with ``--c-results <csv>`` adds layer C and applies the decision rule;
13
+ * ``score-b2-panel`` with ``--b2-panel-results <csv>`` un-blinds and scores the B2 panel.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import importlib.metadata
19
+ import json
20
+ import logging
21
+ import shutil
22
+ from collections.abc import Sequence
23
+ from dataclasses import dataclass, field
24
+ from pathlib import Path
25
+
26
+ from nida.bench import b1, b2, envs
27
+ from nida.bench.clips import Clip
28
+ from nida.bench.panel import load_panel_spec, read_panel_results
29
+ from nida.bench.prepare import Workspace, load_manifest, prepare_b1
30
+ from nida.bench.protocol import (
31
+ WorkerSummary,
32
+ WorkRequest,
33
+ WorkResult,
34
+ drop_results,
35
+ load_results,
36
+ read_summary,
37
+ write_requests,
38
+ )
39
+ from nida.bench.registry import (
40
+ ASR_SYSTEMS,
41
+ QUALITY_SYSTEM,
42
+ VAE_SYSTEMS,
43
+ System,
44
+ load_sources,
45
+ require_approved,
46
+ )
47
+ from nida.bench.report import render_b1, render_b2, render_b2_panel
48
+ from nida.utils.io import atomic_write_json, atomic_write_text
49
+
50
+ __all__ = ["ALL_STAGES", "STAGES", "Phase0"]
51
+
52
+ log = logging.getLogger(__name__)
53
+
54
+ ALL_STAGES = (
55
+ "check",
56
+ "prepare-b1",
57
+ "asr-b1",
58
+ "panel-b1c",
59
+ "score-b1",
60
+ "prepare-b2",
61
+ "vae-b2",
62
+ "asr-b2",
63
+ "quality-b2",
64
+ "score-b2",
65
+ "panel-b2",
66
+ )
67
+ STAGES = (*ALL_STAGES, "score-b2-panel")
68
+
69
+ B1_DATA = ("fleurs_tr", "mediaspeech_tr", "yodas_tr")
70
+ B2_DATA = ("expresso",)
71
+ FAMILIES = {
72
+ "qwen3_asr_1_7b": "qwen",
73
+ "qwen3_asr_0_6b": "qwen",
74
+ "omniasr_llm_7b": "omni",
75
+ "omniasr_ctc_1b": "omni",
76
+ "whisper_large_v3": "whisper",
77
+ "buzzasr_turkish": "whisper",
78
+ }
79
+ WHISPER_FOR_B2 = ASR_SYSTEMS["whisper_large_v3"]
80
+ MAIN_PACKAGES = ("numpy", "scipy", "soundfile", "PyYAML", "torch")
81
+
82
+
83
+ def _main_package_versions() -> dict[str, str]:
84
+ versions: dict[str, str] = {}
85
+ for name in MAIN_PACKAGES:
86
+ try:
87
+ versions[name] = importlib.metadata.version(name)
88
+ except importlib.metadata.PackageNotFoundError:
89
+ versions[name] = "not-installed"
90
+ return versions
91
+
92
+
93
+ @dataclass
94
+ class Phase0:
95
+ workspace: Workspace
96
+ envs_root: Path
97
+ repo_dir: Path
98
+ sources_path: Path
99
+ systems: tuple[System, ...] = tuple(ASR_SYSTEMS.values())
100
+ seed: int = 20261008
101
+ uv: str | None = None # located lazily, after the approval gate
102
+ retry_failed: bool = False
103
+ _c_results: Path | None = field(default=None, init=False, repr=False)
104
+ _b2_panel_results: Path | None = field(default=None, init=False, repr=False)
105
+
106
+ # ------------------------------------------------------------------ helpers
107
+
108
+ def _require(self, stage: str) -> None:
109
+ sources = load_sources(self.sources_path)
110
+ needed: set[str] = set()
111
+ if stage.endswith("b1") or stage in ("panel-b1c", "check"):
112
+ needed |= set(B1_DATA) | {s.source_id for s in self.systems}
113
+ if "b2" in stage or stage == "check":
114
+ # B2 reuses the Turkish B1 clips (FLEURS, MediaSpeech) as its tr subsets.
115
+ needed |= set(B2_DATA) | {"fleurs_tr", "mediaspeech_tr"}
116
+ needed |= {s.source_id for s in VAE_SYSTEMS.values()}
117
+ needed |= {QUALITY_SYSTEM.source_id, WHISPER_FOR_B2.source_id}
118
+ require_approved(needed, sources)
119
+
120
+ def _python(self, worker: str) -> Path:
121
+ spec = envs.env_for_worker(worker)
122
+ if self.uv is None:
123
+ self.uv = envs.find_uv(envs.subprocess_runner)
124
+ python = envs.ensure_env(spec, self.envs_root, self.repo_dir, uv=self.uv)
125
+ freeze = self.envs_root / spec.name / "freeze.txt"
126
+ if freeze.exists(): # keep the exact package set next to the results (durable)
127
+ dest = self.workspace.root / "envs" / f"{spec.name}-freeze.txt"
128
+ dest.parent.mkdir(parents=True, exist_ok=True)
129
+ shutil.copyfile(freeze, dest)
130
+ return python
131
+
132
+ def _vae_results(self) -> dict[str, WorkResult]:
133
+ return load_results(self.workspace.workers / "b2_vae" / "results.jsonl")
134
+
135
+ def _b2_plan(self) -> b2.B2Plan:
136
+ return b2.B2Plan(seed=self.seed)
137
+
138
+ def _b1_clips(self) -> list[Clip]:
139
+ return load_manifest(self.workspace)[1]
140
+
141
+ def _b2_clips(self) -> list[Clip]:
142
+ return b2.load_b2_manifest(self.workspace)[1]
143
+
144
+ def _summaries(self) -> dict[str, WorkerSummary]:
145
+ out: dict[str, WorkerSummary] = {}
146
+ for system in self.systems:
147
+ path = self.workspace.workers / system.system_id / "results.summary.json"
148
+ if path.exists():
149
+ out[system.system_id] = read_summary(path)
150
+ return out
151
+
152
+ def _run_aux(self, name: str, system: System, requests: Sequence[WorkRequest]) -> None:
153
+ """Run a non-B1 worker (VAE, Whisper on B2, Audiobox) over prepared requests."""
154
+ if not requests:
155
+ # Never prune a results file down to nothing: an empty request set means an
156
+ # upstream input is missing (e.g. the VAE results file), not "no work".
157
+ raise RuntimeError(f"{name}: no requests; is an upstream stage's output missing?")
158
+ out_dir = self.workspace.workers / name
159
+ results_path = out_dir / "results.jsonl"
160
+ if results_path.exists():
161
+ # The request set is the source of truth: results for items that are no longer
162
+ # requested (e.g. a reconstruction that failed on redo) are removed, not trusted.
163
+ wanted = {r.clip_id for r in requests}
164
+ stray = [cid for cid in load_results(results_path, repair=True) if cid not in wanted]
165
+ if stray:
166
+ log.warning("%s: removing %d results no longer requested", name, len(stray))
167
+ drop_results(results_path, stray)
168
+ write_requests(out_dir / "requests.jsonl", requests)
169
+ cmd = [
170
+ str(self._python(system.worker)),
171
+ "-m",
172
+ f"nida.bench.workers.{system.worker}",
173
+ "--requests",
174
+ str(out_dir / "requests.jsonl"),
175
+ "--results",
176
+ str(out_dir / "results.jsonl"),
177
+ "--model-id",
178
+ system.model_id,
179
+ "--revision",
180
+ system.revision,
181
+ "--batch-size",
182
+ str(system.batch_size),
183
+ *(["--retry-failed"] if self.retry_failed else []),
184
+ ]
185
+ code = b1._stream_runner(cmd)
186
+ if code != 0:
187
+ raise RuntimeError(f"{name}: worker exited with code {code}")
188
+ summary = read_summary(out_dir / "results.summary.json")
189
+ if summary.n_ok + summary.n_error != len(requests):
190
+ raise RuntimeError(
191
+ f"{name}: {summary.n_ok + summary.n_error} results for {len(requests)} requests"
192
+ )
193
+ if summary.n_error > b1.MAX_ERROR_FRACTION * (summary.n_ok + summary.n_error):
194
+ raise RuntimeError(
195
+ f"{name}: {summary.n_error} items failed (> 5%); rerun with --retry-failed"
196
+ )
197
+
198
+ # ------------------------------------------------------------------- stages
199
+
200
+ def run(
201
+ self,
202
+ stage: str,
203
+ *,
204
+ c_results: Path | None = None,
205
+ b2_panel_results: Path | None = None,
206
+ ) -> dict[str, object]:
207
+ """Run one stage (or ``all``); returns a JSON-compatible summary for metrics.json."""
208
+ stages = ALL_STAGES if stage == "all" else (stage,)
209
+ unknown = [s for s in stages if s not in STAGES]
210
+ if unknown:
211
+ raise ValueError(f"unknown stage(s) {unknown}; choose from {STAGES} or 'all'")
212
+ self._c_results = c_results
213
+ self._b2_panel_results = b2_panel_results
214
+ result: dict[str, object] = {"main_packages": _main_package_versions()}
215
+ for name in stages:
216
+ log.info("=== stage %s", name)
217
+ self._require(name)
218
+ output = getattr(self, "_stage_" + name.replace("-", "_"))()
219
+ if output is not None:
220
+ result[name] = output
221
+ return result
222
+
223
+ def _stage_check(self) -> dict[str, object]:
224
+ return {"approved": True}
225
+
226
+ def _stage_prepare_b1(self) -> dict[str, object]:
227
+ clips = prepare_b1(self.workspace)
228
+ counts: dict[str, int] = {}
229
+ for clip in clips:
230
+ counts[clip.layer] = counts.get(clip.layer, 0) + 1
231
+ return {"clips_per_layer": counts}
232
+
233
+ def _stage_asr_b1(self) -> dict[str, object]:
234
+ clips = self._b1_clips()
235
+ done: dict[str, object] = {}
236
+ for system in self.systems:
237
+ summary = b1.run_system(
238
+ self.workspace,
239
+ clips,
240
+ system,
241
+ self._python(system.worker),
242
+ retry_failed=self.retry_failed,
243
+ )
244
+ done[system.system_id] = {"n_ok": summary.n_ok, "n_error": summary.n_error}
245
+ return done
246
+
247
+ def _stage_panel_b1c(self) -> dict[str, object]:
248
+ clips = self._b1_clips()
249
+ pool = [c for c in clips if c.layer == "C"]
250
+ hyps = {s.system_id: b1.hypotheses(self.workspace, s) for s in self.systems}
251
+ final, html = b1.build_c_panel(self.workspace, pool, hyps, self.seed)
252
+ return {"panel": str(html), "n_items": len(final)}
253
+
254
+ def _stage_score_b1(self) -> dict[str, object]:
255
+ clips = self._b1_clips()
256
+ hyps = {s.system_id: b1.hypotheses(self.workspace, s) for s in self.systems}
257
+ c_refs: dict[str, str] | None = None
258
+ provenance = None
259
+ if self._c_results is not None:
260
+ spec = load_panel_spec(self.workspace.panels / f"b1c-{self.seed}" / "panel.json")
261
+ c_refs = b1.c_references_from_panel(read_panel_results(self._c_results, spec))
262
+ provenance = b1.load_c_selection(self.workspace, self.seed)
263
+ scores = b1.score_b1(
264
+ clips,
265
+ self.systems,
266
+ hyps,
267
+ c_refs,
268
+ seed=self.seed,
269
+ summaries=self._summaries(),
270
+ c_provenance=provenance,
271
+ )
272
+ choice = b1.choose_systems(scores, FAMILIES)
273
+ report_dir = self.workspace.root / "reports"
274
+ atomic_write_json(report_dir / "b1_metrics.json", {"scores": scores, "choice": choice})
275
+ atomic_write_text(report_dir / "b1_report.md", render_b1(scores, choice))
276
+ return {"choice": choice}
277
+
278
+ def _stage_prepare_b2(self) -> dict[str, object]:
279
+ clips = b2.prepare_b2(self.workspace, self._b1_clips(), self._b2_plan())
280
+ counts: dict[str, int] = {}
281
+ for clip in clips:
282
+ counts[clip.layer] = counts.get(clip.layer, 0) + 1
283
+ return {"clips_per_subset": counts}
284
+
285
+ def _stage_vae_b2(self) -> dict[str, object]:
286
+ system = VAE_SYSTEMS["voxcpm2_audiovae_v2"]
287
+ results_path = self.workspace.workers / "b2_vae" / "results.jsonl"
288
+ redo = b2.missing_vae_outputs(self.workspace, load_results(results_path, repair=True))
289
+ if redo:
290
+ log.warning("b2_vae: %d reconstructions missing on disk; redoing them", len(redo))
291
+ # Invalidate derived data FIRST and the VAE results LAST: if the process dies in
292
+ # between, the rerun still sees the missing reconstruction and redoes all of it.
293
+ stale = [f"{clip_id}__vae" for clip_id in redo]
294
+ for name in ("b2_asr_tr", "b2_asr_en", "b2_quality"):
295
+ downstream = self.workspace.workers / name / "results.jsonl"
296
+ if downstream.exists():
297
+ drop_results(downstream, stale)
298
+ for clip_id in redo:
299
+ (b2.b2_paths(self.workspace).asr16k / f"{clip_id}__vae.wav").unlink(missing_ok=True)
300
+ drop_results(results_path, redo)
301
+ self._run_aux("b2_vae", system, b2.vae_requests(self.workspace, self._b2_clips()))
302
+ return {"redone_missing_outputs": len(redo)}
303
+
304
+ def _stage_asr_b2(self) -> None:
305
+ clips = self._b2_clips()
306
+ for language in ("tr", "en"):
307
+ requests = b2.asr_requests(self.workspace, clips, language, self._vae_results())
308
+ if requests:
309
+ self._run_aux(f"b2_asr_{language}", WHISPER_FOR_B2, requests)
310
+
311
+ def _stage_quality_b2(self) -> None:
312
+ self._run_aux(
313
+ "b2_quality",
314
+ QUALITY_SYSTEM,
315
+ b2.quality_requests(self.workspace, self._b2_clips(), self._vae_results()),
316
+ )
317
+
318
+ def _stage_score_b2(self) -> dict[str, object]:
319
+ asr: dict[str, WorkResult] = {}
320
+ for language in ("tr", "en"):
321
+ asr.update(
322
+ load_results(self.workspace.workers / f"b2_asr_{language}" / "results.jsonl")
323
+ )
324
+ quality = load_results(self.workspace.workers / "b2_quality" / "results.jsonl")
325
+ scores = b2.score_b2(self.workspace, self._b2_clips(), asr, quality, self._vae_results())
326
+ report_dir = self.workspace.root / "reports"
327
+ atomic_write_json(report_dir / "b2_metrics.json", scores)
328
+ atomic_write_text(report_dir / "b2_report.md", render_b2(scores))
329
+ return scores
330
+
331
+ def _stage_panel_b2(self) -> dict[str, object]:
332
+ html = b2.build_b2_panel(
333
+ self.workspace, self._b2_clips(), self._b2_plan(), self._vae_results()
334
+ )
335
+ return {"panel": str(html)}
336
+
337
+ def _stage_score_b2_panel(self) -> dict[str, object]:
338
+ if self._b2_panel_results is None:
339
+ raise ValueError("score-b2-panel needs --b2-panel-results <csv>")
340
+ plan = self._b2_plan()
341
+ spec = load_panel_spec(self.workspace.panels / f"b2-{plan.seed}" / "panel.json")
342
+ key_obj = json.loads(
343
+ (self.workspace.manifests / f"b2_panel_key-{plan.seed}.json").read_text(
344
+ encoding="utf-8"
345
+ )
346
+ )
347
+ scores = b2.score_b2_panel(
348
+ read_panel_results(self._b2_panel_results, spec),
349
+ key_obj["items"],
350
+ min_per_subset=int(key_obj.get("min_per_subset", plan.n_panel_per_subset)),
351
+ )
352
+ scores["excluded_missing_reconstruction"] = key_obj.get(
353
+ "excluded_missing_reconstruction", {}
354
+ )
355
+ report_dir = self.workspace.root / "reports"
356
+ atomic_write_json(report_dir / "b2_panel_metrics.json", scores)
357
+ atomic_write_text(report_dir / "b2_panel_report.md", render_b2_panel(scores))
358
+ return scores
src/nida/bench/prepare.py ADDED
@@ -0,0 +1,450 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """B1 data preparation: fetch pinned sources, select clips, write uniform 16 kHz clips.
2
+
3
+ Layout (``Workspace``):
4
+
5
+ * ``root`` — durable directory on Google Drive (manifests, worker results, panels).
6
+ * ``cache`` — local scratch on the Colab disk (downloads, extracted audio, clip WAVs).
7
+ It may disappear between sessions; everything in it is re-derived deterministically.
8
+
9
+ Every ASR clip is written as 16 kHz mono 16-bit PCM WAV at ``cache/clips16k/<clip_id>.wav``
10
+ so all systems see byte-identical input. Selection is deterministic for a given seed and
11
+ pinned source revision; the resulting manifest is written once to ``root`` and reused.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import csv
17
+ import json
18
+ import logging
19
+ import random
20
+ import re
21
+ from collections.abc import Callable, Iterable, Sequence
22
+ from dataclasses import asdict, dataclass
23
+ from pathlib import Path
24
+ from typing import Any
25
+
26
+ from nida.bench import fetch
27
+ from nida.bench.audiometrics import resample
28
+ from nida.bench.clips import (
29
+ YODAS_DURATION_TOLERANCE_MS,
30
+ Clip,
31
+ FleursRow,
32
+ MediaSpeechItem,
33
+ Quarantined,
34
+ Selection,
35
+ YodasUtt,
36
+ clip_from_obj,
37
+ clip_to_obj,
38
+ select_fleurs,
39
+ select_mediaspeech,
40
+ select_yodas_pool,
41
+ )
42
+ from nida.bench.registry import DATA_PINS
43
+ from nida.utils.io import atomic_write_text
44
+
45
+ __all__ = [
46
+ "B1Plan",
47
+ "Workspace",
48
+ "clip_audio_path",
49
+ "load_manifest",
50
+ "parse_fleurs_tsv",
51
+ "parse_yodas_listing",
52
+ "prepare_b1",
53
+ "write_clip_audio",
54
+ ]
55
+
56
+ log = logging.getLogger(__name__)
57
+
58
+ ASR_SAMPLE_RATE_HZ = 16000
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class Workspace:
63
+ root: Path
64
+ cache: Path
65
+
66
+ @property
67
+ def manifests(self) -> Path:
68
+ return self.root / "manifests"
69
+
70
+ @property
71
+ def workers(self) -> Path:
72
+ return self.root / "workers"
73
+
74
+ @property
75
+ def panels(self) -> Path:
76
+ return self.root / "panels"
77
+
78
+ @property
79
+ def clips16k(self) -> Path:
80
+ return self.cache / "clips16k"
81
+
82
+
83
+ @dataclass(frozen=True)
84
+ class B1Plan:
85
+ """Parameters that fully determine the B1 clip manifest."""
86
+
87
+ seed: int = 20261008
88
+ n_fleurs: int = 100
89
+ n_mediaspeech: int = 75
90
+ n_yodas_pool: int = 300
91
+ fleurs_revision: str = DATA_PINS["fleurs_revision"]
92
+ yodas_revision: str = DATA_PINS["yodas_revision"]
93
+ mediaspeech_url: str = DATA_PINS["mediaspeech_url"]
94
+ # Comma-separated tr000 shards. One shard (00000000) has only 289 eligible
95
+ # utterances at <= 3 per video (run 20261008-120354-bench-phase0-69dc), so two.
96
+ yodas_shards: str = "00000000,00000001"
97
+ # Extra pool candidates drawn so that clips with broken audio (YODAS has empty wavs,
98
+ # run 20261008-121509-bench-phase0-1347) can be quarantined without a short pool.
99
+ yodas_spare: int = 30
100
+
101
+
102
+ def clip_audio_path(workspace: Workspace, clip: Clip) -> Path:
103
+ return workspace.clips16k / f"{clip.clip_id}.wav"
104
+
105
+
106
+ # ------------------------------------------------------------------------------ parsing
107
+
108
+
109
+ def parse_fleurs_tsv(text: str) -> list[FleursRow]:
110
+ """Parse FLEURS ``<split>.tsv`` (7 tab-separated columns, no header, no quoting).
111
+
112
+ Columns: sentence id, file name, raw transcription, normalized transcription,
113
+ characters, number of samples (16 kHz), gender.
114
+
115
+ Raises:
116
+ ValueError: on a row with the wrong number of columns or a non-integer sample count.
117
+ """
118
+ rows: list[FleursRow] = []
119
+ reader = csv.reader(text.splitlines(), delimiter="\t", quoting=csv.QUOTE_NONE)
120
+ for number, cols in enumerate(reader, start=1):
121
+ if not cols:
122
+ continue
123
+ if len(cols) != 7:
124
+ raise ValueError(f"FLEURS tsv line {number}: expected 7 columns, got {len(cols)}")
125
+ try:
126
+ num_samples = int(cols[5])
127
+ except ValueError as err:
128
+ raise ValueError(f"FLEURS tsv line {number}: bad sample count {cols[5]!r}") from err
129
+ rows.append(FleursRow(cols[0], cols[1], cols[2], num_samples, cols[6]))
130
+ return rows
131
+
132
+
133
+ def parse_yodas_listing(text_file: str, duration_file: str, shard: str = "") -> list[YodasUtt]:
134
+ """Join YODAS ``text/<shard>.txt`` (``utt_id caption``) with ``duration/<shard>.txt``.
135
+
136
+ Raises:
137
+ ValueError: on malformed lines, duplicate ids or ids missing from either file.
138
+ """
139
+
140
+ def split(content: str, what: str) -> dict[str, str]:
141
+ out: dict[str, str] = {}
142
+ for number, line in enumerate(content.splitlines(), start=1):
143
+ if not line.strip():
144
+ continue
145
+ parts = line.split(" ", 1)
146
+ utt_id = parts[0]
147
+ value = parts[1] if len(parts) == 2 else ""
148
+ if utt_id in out:
149
+ raise ValueError(f"YODAS {what} line {number}: duplicate id {utt_id}")
150
+ out[utt_id] = value
151
+ return out
152
+
153
+ captions = split(text_file, "text")
154
+ durations = split(duration_file, "duration")
155
+ if set(captions) != set(durations):
156
+ diff = sorted(set(captions) ^ set(durations))
157
+ raise ValueError(f"YODAS text/duration ids differ, e.g. {diff[:3]}")
158
+ utts: list[YodasUtt] = []
159
+ for utt_id in sorted(captions):
160
+ try:
161
+ seconds = float(durations[utt_id])
162
+ except ValueError as err:
163
+ raise ValueError(f"YODAS duration for {utt_id} is not a number") from err
164
+ utts.append(YodasUtt(utt_id, captions[utt_id].strip(), round(seconds * 1000), shard))
165
+ return utts
166
+
167
+
168
+ # -------------------------------------------------------------------------------- audio
169
+
170
+
171
+ def write_clip_audio(source: Path, dest: Path, start_ms: int, end_ms: int | None) -> int:
172
+ """Write ``source[start_ms:end_ms]`` as 16 kHz mono PCM16 WAV at ``dest``.
173
+
174
+ Multi-channel input is rejected (callers pick a channel explicitly). Returns the
175
+ number of samples written. Idempotent: an existing ``dest`` is left untouched.
176
+
177
+ Raises:
178
+ ValueError: on multi-channel audio or an empty / out-of-range span.
179
+ """
180
+ import numpy as np
181
+ import soundfile # type: ignore[import-untyped] # bench extra, untyped
182
+
183
+ if dest.exists():
184
+ return int(soundfile.info(str(dest)).frames)
185
+ audio: Any
186
+ audio, rate_hz = soundfile.read(str(source), dtype="float64", always_2d=True)
187
+ if audio.shape[1] != 1:
188
+ raise ValueError(f"{source}: expected mono, got {audio.shape[1]} channels")
189
+ mono = audio[:, 0]
190
+ start = round(start_ms * rate_hz / 1000)
191
+ stop = mono.shape[0] if end_ms is None else min(mono.shape[0], round(end_ms * rate_hz / 1000))
192
+ if not 0 <= start < stop:
193
+ raise ValueError(f"{source}: empty span {start_ms}..{end_ms} ms ({mono.shape[0]} samples)")
194
+ segment = mono[start:stop]
195
+ if rate_hz != ASR_SAMPLE_RATE_HZ:
196
+ segment = resample(segment, int(rate_hz), ASR_SAMPLE_RATE_HZ)
197
+ peak = float(np.max(np.abs(segment)))
198
+ if peak > 1.0: # resampling overshoot; never wrap around in PCM16
199
+ segment = segment / peak
200
+ dest.parent.mkdir(parents=True, exist_ok=True)
201
+ tmp = dest.with_name(f".{dest.name}.tmp.wav")
202
+ soundfile.write(str(tmp), segment, ASR_SAMPLE_RATE_HZ, subtype="PCM_16")
203
+ tmp.replace(dest)
204
+ return int(segment.shape[0])
205
+
206
+
207
+ # ----------------------------------------------------------------------------- manifest
208
+
209
+
210
+ def _write_jsonl(path: Path, rows: Iterable[dict[str, object]]) -> None:
211
+ atomic_write_text(
212
+ path,
213
+ "".join(json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in rows),
214
+ )
215
+
216
+
217
+ def load_manifest(workspace: Workspace) -> tuple[B1Plan, list[Clip]]:
218
+ """Read the B1 manifest written by :func:`prepare_b1`."""
219
+ plan_obj = json.loads((workspace.manifests / "b1_plan.json").read_text(encoding="utf-8"))
220
+ clips = [
221
+ clip_from_obj(json.loads(line))
222
+ for line in (workspace.manifests / "b1_clips.jsonl")
223
+ .read_text(encoding="utf-8")
224
+ .splitlines()
225
+ if line.strip()
226
+ ]
227
+ return B1Plan(**plan_obj), clips
228
+
229
+
230
+ # ---------------------------------------------------------------------------- prepare
231
+
232
+
233
+ Opener = Callable[..., Any] # Any: urllib-compatible opener, see nida.bench.fetch
234
+
235
+
236
+ def _require_extracted(wanted: set[str], extracted: Sequence[Path], base: Path, what: str) -> None:
237
+ """Fail loudly when an archive did not contain every member we selected."""
238
+ got = {p.relative_to(base).as_posix() for p in extracted}
239
+ missing = sorted(wanted - got)
240
+ if missing:
241
+ raise FileNotFoundError(
242
+ f"{what}: {len(missing)} selected members missing from the archive, e.g. {missing[:3]}"
243
+ )
244
+
245
+
246
+ def _fleurs(plan: B1Plan, ws: Workspace, opener: Opener) -> tuple[Selection, Path]:
247
+ base = ws.cache / "fleurs"
248
+ tsv = fetch.download(
249
+ fetch.hf_resolve_url(
250
+ "google/fleurs", plan.fleurs_revision, "data/tr_tr/test.tsv", "dataset"
251
+ ),
252
+ base / "test.tsv",
253
+ opener=opener,
254
+ )
255
+ selection = select_fleurs(
256
+ parse_fleurs_tsv(tsv.path.read_text(encoding="utf-8")), plan.n_fleurs, plan.seed
257
+ )
258
+ wanted = {f"test/{c.file_id}" for c in selection.clips}
259
+ extracted = fetch.stream_extract(
260
+ fetch.hf_resolve_url(
261
+ "google/fleurs", plan.fleurs_revision, "data/tr_tr/audio/test.tar.gz", "dataset"
262
+ ),
263
+ base,
264
+ wanted.__contains__,
265
+ opener=opener,
266
+ )
267
+ _require_extracted(wanted, extracted, base, "FLEURS")
268
+ return selection, base / "test"
269
+
270
+
271
+ def _mediaspeech(plan: B1Plan, ws: Workspace, opener: Opener) -> tuple[Selection, Path]:
272
+ import soundfile
273
+
274
+ base = ws.cache / "mediaspeech"
275
+ fetch.stream_extract(
276
+ plan.mediaspeech_url, base, lambda name: name.startswith("TR/"), opener=opener
277
+ )
278
+ audio_dir = base / "TR"
279
+ items: list[MediaSpeechItem] = []
280
+ for txt in sorted(audio_dir.glob("*.txt")):
281
+ flac = txt.with_suffix(".flac")
282
+ if not flac.exists():
283
+ raise FileNotFoundError(f"MediaSpeech transcript without audio: {txt.name}")
284
+ info = soundfile.info(str(flac))
285
+ items.append(
286
+ MediaSpeechItem(
287
+ txt.stem,
288
+ txt.read_text(encoding="utf-8").strip(),
289
+ round(1000 * info.frames / info.samplerate),
290
+ )
291
+ )
292
+ if not items:
293
+ raise FileNotFoundError(f"no MediaSpeech transcripts under {audio_dir}")
294
+ return select_mediaspeech(items, plan.n_mediaspeech, plan.seed), audio_dir
295
+
296
+
297
+ def _yodas(plan: B1Plan, ws: Workspace, opener: Opener) -> tuple[Selection, Path]:
298
+ base = ws.cache / "yodas"
299
+ shards = yodas_shards(plan)
300
+
301
+ def get(kind: str, shard: str, ext: str) -> Path:
302
+ name = f"data/tr000/{kind}/{shard}.{ext}"
303
+ url = fetch.hf_resolve_url("espnet/yodas", plan.yodas_revision, name, "dataset")
304
+ return fetch.download(url, base / kind / f"{shard}.{ext}", opener=opener).path
305
+
306
+ utts: list[YodasUtt] = []
307
+ seen: set[str] = set()
308
+ for shard in shards:
309
+ shard_utts = parse_yodas_listing(
310
+ get("text", shard, "txt").read_text(encoding="utf-8"),
311
+ get("duration", shard, "txt").read_text(encoding="utf-8"),
312
+ shard,
313
+ )
314
+ duplicates = seen & {u.utt_id for u in shard_utts}
315
+ if duplicates:
316
+ raise ValueError(f"YODAS utterance ids repeat across shards: {sorted(duplicates)[:3]}")
317
+ seen |= {u.utt_id for u in shard_utts}
318
+ utts.extend(shard_utts)
319
+ if plan.yodas_spare < 0:
320
+ raise ValueError(f"yodas_spare must be >= 0, got {plan.yodas_spare}")
321
+ selection = select_yodas_pool(utts, plan.n_yodas_pool + plan.yodas_spare, plan.seed)
322
+ for shard in shards:
323
+ wanted = {
324
+ f"{shard}/{c.file_id}.wav" for c in selection.clips if c.meta.get("shard") == shard
325
+ }
326
+ if not wanted:
327
+ continue
328
+ audio_url = fetch.hf_resolve_url(
329
+ "espnet/yodas", plan.yodas_revision, f"data/tr000/audio/{shard}.tar.gz", "dataset"
330
+ )
331
+ extracted = fetch.stream_extract(
332
+ audio_url, base / "audio", wanted.__contains__, opener=opener
333
+ )
334
+ _require_extracted(wanted, extracted, base / "audio", "YODAS")
335
+ good: list[Clip] = []
336
+ bad: list[Quarantined] = []
337
+ for clip in selection.clips:
338
+ problem = yodas_audio_problem(_source_file(clip, base / "audio"), clip.end_ms)
339
+ if problem is None:
340
+ good.append(clip)
341
+ else:
342
+ bad.append(Quarantined(clip.file_id, "select", problem[0], problem[1]))
343
+ if len(good) < plan.n_yodas_pool:
344
+ raise ValueError(
345
+ f"only {len(good)} YODAS clips with usable audio, need {plan.n_yodas_pool} "
346
+ f"({len(bad)} quarantined; raise yodas_spare)"
347
+ )
348
+ chosen = random.Random(plan.seed).sample(good, plan.n_yodas_pool)
349
+ log.info("YODAS pool: %d usable, %d broken audio, %d chosen", len(good), len(bad), len(chosen))
350
+ return (
351
+ Selection(tuple(sorted(chosen, key=lambda c: c.file_id)), (*selection.quarantined, *bad)),
352
+ base / "audio",
353
+ )
354
+
355
+
356
+ def yodas_audio_problem(path: Path, expected_ms: int) -> tuple[str, str] | None:
357
+ """``(reason, details)`` if a YODAS wav is unreadable, empty or of the wrong length."""
358
+ import soundfile
359
+
360
+ try:
361
+ info = soundfile.info(str(path))
362
+ except (RuntimeError, soundfile.LibsndfileError) as err:
363
+ return "unreadable_audio", str(err)
364
+ if info.frames == 0:
365
+ return "empty_audio", f"0 frames at {info.samplerate} Hz"
366
+ actual_ms = round(1000 * info.frames / info.samplerate)
367
+ if abs(actual_ms - expected_ms) > YODAS_DURATION_TOLERANCE_MS:
368
+ return "audio_duration_mismatch", f"{actual_ms} vs {expected_ms} ms"
369
+ return None
370
+
371
+
372
+ def yodas_shards(plan: B1Plan) -> list[str]:
373
+ """The plan's YODAS shards; each must be an 8-digit shard name, no repeats."""
374
+ shards = [s.strip() for s in plan.yodas_shards.split(",")]
375
+ if not shards or any(re.fullmatch(r"[0-9]{8}", s) is None for s in shards):
376
+ raise ValueError(f"invalid yodas_shards {plan.yodas_shards!r}")
377
+ if len(set(shards)) != len(shards):
378
+ raise ValueError(f"repeated YODAS shard in {plan.yodas_shards!r}")
379
+ return shards
380
+
381
+
382
+ def _source_file(clip: Clip, audio_dir: Path) -> Path:
383
+ if clip.source_id == "fleurs_tr":
384
+ return audio_dir / clip.file_id
385
+ if clip.source_id == "mediaspeech_tr":
386
+ return audio_dir / f"{clip.file_id}.flac"
387
+ if clip.source_id == "yodas_tr":
388
+ shard = clip.meta.get("shard")
389
+ if not shard:
390
+ raise ValueError(f"YODAS clip {clip.clip_id} has no shard in meta")
391
+ return audio_dir / shard / f"{clip.file_id}.wav"
392
+ raise ValueError(f"unknown B1 source {clip.source_id}")
393
+
394
+
395
+ def prepare_b1(
396
+ workspace: Workspace,
397
+ plan: B1Plan | None = None,
398
+ *,
399
+ opener: Opener | None = None,
400
+ ) -> list[Clip]:
401
+ """Fetch, select and convert all B1 clips; write the manifest once.
402
+
403
+ If a manifest already exists it must have been produced with the same plan
404
+ (otherwise ``ValueError``); its clips are reused and only missing clip audio is
405
+ regenerated (the cache may have been wiped by a new Colab session).
406
+ """
407
+ plan = plan if plan is not None else B1Plan()
408
+ plan_path = workspace.manifests / "b1_plan.json"
409
+ if plan_path.exists():
410
+ existing_plan, existing = load_manifest(workspace)
411
+ if existing_plan != plan:
412
+ raise ValueError(f"existing B1 manifest was built with {existing_plan}, not {plan}")
413
+ if all(clip_audio_path(workspace, c).exists() for c in existing):
414
+ log.info("B1 manifest and clip audio present: %d clips", len(existing))
415
+ return existing
416
+ use_opener = opener if opener is not None else fetch.default_opener
417
+ selections: list[tuple[Selection, Path]] = [
418
+ _fleurs(plan, workspace, use_opener),
419
+ _mediaspeech(plan, workspace, use_opener),
420
+ _yodas(plan, workspace, use_opener),
421
+ ]
422
+ clips: list[Clip] = []
423
+ quarantined: list[Quarantined] = []
424
+ for selection, audio_dir in selections:
425
+ quarantined.extend(selection.quarantined)
426
+ for clip in selection.clips:
427
+ write_clip_audio(
428
+ _source_file(clip, audio_dir),
429
+ clip_audio_path(workspace, clip),
430
+ clip.start_ms,
431
+ clip.end_ms,
432
+ )
433
+ clips.append(clip)
434
+ if plan_path.exists():
435
+ _, existing = load_manifest(workspace)
436
+ if [c.clip_id for c in existing] != [c.clip_id for c in clips]:
437
+ raise ValueError("re-derived B1 clips differ from the stored manifest")
438
+ return existing
439
+ _write_jsonl(workspace.manifests / "b1_clips.jsonl", (clip_to_obj(c) for c in clips))
440
+ _write_jsonl(workspace.manifests / "b1_quarantine.jsonl", (asdict(q) for q in quarantined))
441
+ atomic_write_text(plan_path, json.dumps(asdict(plan), indent=2, sort_keys=True) + "\n")
442
+ log.info("B1 manifest written: %d clips, %d quarantined", len(clips), len(quarantined))
443
+ return clips
444
+
445
+
446
+ def layer_counts(clips: Sequence[Clip]) -> dict[str, int]:
447
+ counts: dict[str, int] = {}
448
+ for clip in clips:
449
+ counts[clip.layer] = counts.get(clip.layer, 0) + 1
450
+ return counts
src/nida/bench/protocol.py ADDED
@@ -0,0 +1,325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JSONL protocol between the benchmark orchestrator and model workers.
2
+
3
+ Each model family runs in its own virtual environment (dependency pins conflict, see
4
+ docs/benchmarks/B1-B2-tasarim.md §1.3), so the orchestrator and the workers only share
5
+ files. This module is the single definition of those files and must stay stdlib-only:
6
+ it is imported inside every worker environment.
7
+
8
+ Layout for one worker invocation::
9
+
10
+ requests.jsonl one ``WorkRequest`` per line (written by the orchestrator)
11
+ results.jsonl one ``WorkResult`` per line (appended by the worker)
12
+ results.summary.json one ``WorkerSummary`` (written by the worker at the end)
13
+
14
+ Results are appended one line at a time and flushed + fsynced, so an interrupted worker
15
+ leaves a valid prefix plus at most one partial trailing line. On restart the worker
16
+ calls :func:`load_results` with ``repair=True``, which drops the partial line, and
17
+ skips every request whose ``clip_id`` already has a result. Failed items are recorded
18
+ with an ``error`` and are never retried silently: callers decide via ``retry_failed``.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import json
24
+ import os
25
+ from collections.abc import Iterable, Iterator, Mapping
26
+ from dataclasses import asdict, dataclass, field
27
+ from pathlib import Path
28
+ from typing import IO
29
+
30
+ __all__ = [
31
+ "ProtocolError",
32
+ "WorkRequest",
33
+ "WorkResult",
34
+ "WorkerSummary",
35
+ "append_result",
36
+ "drop_results",
37
+ "load_requests",
38
+ "load_results",
39
+ "pending_requests",
40
+ "read_summary",
41
+ "write_requests",
42
+ "write_summary",
43
+ ]
44
+
45
+
46
+ class ProtocolError(ValueError):
47
+ """A protocol file is malformed or inconsistent."""
48
+
49
+
50
+ @dataclass(frozen=True)
51
+ class WorkRequest:
52
+ """One unit of work for a worker.
53
+
54
+ Attributes:
55
+ clip_id: Content-addressed clip id; unique within a requests file.
56
+ audio_path: Absolute path of the input audio file.
57
+ params: Worker-specific string parameters (e.g. ``language``, ``context``,
58
+ ``out_path``). Values are strings so the format stays trivially portable.
59
+ """
60
+
61
+ clip_id: str
62
+ audio_path: str
63
+ params: Mapping[str, str] = field(default_factory=dict)
64
+
65
+ def to_json(self) -> str:
66
+ return json.dumps(
67
+ {"clip_id": self.clip_id, "audio_path": self.audio_path, "params": dict(self.params)},
68
+ ensure_ascii=False,
69
+ sort_keys=True,
70
+ )
71
+
72
+ @staticmethod
73
+ def from_obj(obj: object, where: str) -> WorkRequest:
74
+ if not isinstance(obj, dict):
75
+ raise ProtocolError(f"{where}: request must be a JSON object")
76
+ clip_id = obj.get("clip_id")
77
+ audio_path = obj.get("audio_path")
78
+ params = obj.get("params", {})
79
+ if not isinstance(clip_id, str) or not clip_id:
80
+ raise ProtocolError(f"{where}: clip_id must be a non-empty string")
81
+ if not isinstance(audio_path, str) or not audio_path:
82
+ raise ProtocolError(f"{where}: audio_path must be a non-empty string")
83
+ if not isinstance(params, dict) or not all(
84
+ isinstance(k, str) and isinstance(v, str) for k, v in params.items()
85
+ ):
86
+ raise ProtocolError(f"{where}: params must map strings to strings")
87
+ return WorkRequest(clip_id=clip_id, audio_path=audio_path, params=dict(params))
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class WorkResult:
92
+ """Outcome for one request: exactly one of ``output`` or ``error`` is set.
93
+
94
+ Attributes:
95
+ clip_id: Id of the request this answers.
96
+ output: Worker-specific JSON-compatible payload (e.g. ``{"text": ...}``).
97
+ error: ``"<ExceptionType>: <message>"`` when the item failed.
98
+ infer_seconds: Wall-clock inference time attributed to this item (batch time
99
+ divided by batch size), excluding model loading.
100
+ """
101
+
102
+ clip_id: str
103
+ output: Mapping[str, object] | None
104
+ error: str | None
105
+ infer_seconds: float
106
+
107
+ def __post_init__(self) -> None:
108
+ if (self.output is None) == (self.error is None):
109
+ raise ProtocolError(f"{self.clip_id}: exactly one of output/error must be set")
110
+ if self.infer_seconds < 0:
111
+ raise ProtocolError(f"{self.clip_id}: infer_seconds must be >= 0")
112
+
113
+ @property
114
+ def ok(self) -> bool:
115
+ return self.error is None
116
+
117
+ def to_json(self) -> str:
118
+ return json.dumps(
119
+ {
120
+ "clip_id": self.clip_id,
121
+ "output": None if self.output is None else dict(self.output),
122
+ "error": self.error,
123
+ "infer_seconds": self.infer_seconds,
124
+ },
125
+ ensure_ascii=False,
126
+ sort_keys=True,
127
+ )
128
+
129
+ @staticmethod
130
+ def from_obj(obj: object, where: str) -> WorkResult:
131
+ if not isinstance(obj, dict):
132
+ raise ProtocolError(f"{where}: result must be a JSON object")
133
+ clip_id = obj.get("clip_id")
134
+ output = obj.get("output")
135
+ error = obj.get("error")
136
+ seconds = obj.get("infer_seconds")
137
+ if not isinstance(clip_id, str) or not clip_id:
138
+ raise ProtocolError(f"{where}: clip_id must be a non-empty string")
139
+ if output is not None and not isinstance(output, dict):
140
+ raise ProtocolError(f"{where}: output must be an object or null")
141
+ if error is not None and not isinstance(error, str):
142
+ raise ProtocolError(f"{where}: error must be a string or null")
143
+ if isinstance(seconds, bool) or not isinstance(seconds, int | float):
144
+ raise ProtocolError(f"{where}: infer_seconds must be a number")
145
+ return WorkResult(clip_id=clip_id, output=output, error=error, infer_seconds=float(seconds))
146
+
147
+
148
+ @dataclass(frozen=True)
149
+ class WorkerSummary:
150
+ """Facts about one worker invocation, written once at the end."""
151
+
152
+ worker: str
153
+ model_id: str
154
+ revision: str
155
+ load_seconds: float
156
+ peak_vram_gb: float | None
157
+ n_ok: int
158
+ n_error: int
159
+ started_at: str
160
+ ended_at: str
161
+ packages: Mapping[str, str] = field(default_factory=dict)
162
+ extra: Mapping[str, str] = field(default_factory=dict)
163
+
164
+
165
+ def _fsync_line(handle: IO[str], line: str) -> None:
166
+ handle.write(line + "\n")
167
+ handle.flush()
168
+ os.fsync(handle.fileno())
169
+
170
+
171
+ def write_requests(path: Path, requests: Iterable[WorkRequest]) -> int:
172
+ """Write a requests file atomically and return the number of requests.
173
+
174
+ Raises:
175
+ ProtocolError: on duplicate clip ids.
176
+ """
177
+ from nida.utils.io import atomic_write_text # local import: workers never call this
178
+
179
+ seen: set[str] = set()
180
+ lines: list[str] = []
181
+ for request in requests:
182
+ if request.clip_id in seen:
183
+ raise ProtocolError(f"duplicate clip_id in requests: {request.clip_id}")
184
+ seen.add(request.clip_id)
185
+ lines.append(request.to_json())
186
+ atomic_write_text(path, "".join(line + "\n" for line in lines))
187
+ return len(lines)
188
+
189
+
190
+ def _iter_json_lines(path: Path) -> Iterator[tuple[int, str]]:
191
+ with path.open("r", encoding="utf-8") as handle:
192
+ yield from enumerate(handle, start=1)
193
+
194
+
195
+ def load_requests(path: Path) -> list[WorkRequest]:
196
+ """Read and validate a requests file (no partial lines tolerated)."""
197
+ requests: list[WorkRequest] = []
198
+ seen: set[str] = set()
199
+ for number, line in _iter_json_lines(path):
200
+ where = f"{path}:{number}"
201
+ if not line.endswith("\n"):
202
+ raise ProtocolError(f"{where}: truncated request line")
203
+ try:
204
+ obj = json.loads(line)
205
+ except json.JSONDecodeError as err:
206
+ raise ProtocolError(f"{where}: invalid JSON ({err.msg})") from err
207
+ request = WorkRequest.from_obj(obj, where)
208
+ if request.clip_id in seen:
209
+ raise ProtocolError(f"{where}: duplicate clip_id {request.clip_id}")
210
+ seen.add(request.clip_id)
211
+ requests.append(request)
212
+ return requests
213
+
214
+
215
+ def load_results(path: Path, *, repair: bool = False) -> dict[str, WorkResult]:
216
+ """Read a results file into ``{clip_id: result}``.
217
+
218
+ A missing file yields an empty mapping. Only the *last* line may be incomplete (an
219
+ interrupted append): with ``repair=True`` it is removed from the file, otherwise a
220
+ ``ProtocolError`` is raised. Any malformed line before the last one, and any
221
+ duplicate clip id, is always an error.
222
+ """
223
+ if not path.exists():
224
+ return {}
225
+ raw = path.read_bytes()
226
+ # Split at the last newline *in bytes*: an interrupted append may end in the middle of
227
+ # a multi-byte UTF-8 character, so the trailing fragment must not be decoded.
228
+ keep = raw.rfind(b"\n") + 1 # 0 when there is no complete line
229
+ body, trailing = raw[:keep], raw[keep:]
230
+ try:
231
+ text = body.decode("utf-8")
232
+ except UnicodeDecodeError as err:
233
+ raise ProtocolError(f"{path}: complete lines are not valid UTF-8 ({err})") from err
234
+ lines = text.split("\n")[:-1] if text else []
235
+ results: dict[str, WorkResult] = {}
236
+ for number, line in enumerate(lines, start=1):
237
+ where = f"{path}:{number}"
238
+ try:
239
+ obj = json.loads(line)
240
+ except json.JSONDecodeError as err:
241
+ raise ProtocolError(f"{where}: invalid JSON ({err.msg})") from err
242
+ result = WorkResult.from_obj(obj, where)
243
+ if result.clip_id in results:
244
+ raise ProtocolError(f"{where}: duplicate result for {result.clip_id}")
245
+ results[result.clip_id] = result
246
+ if trailing:
247
+ if not repair:
248
+ raise ProtocolError(f"{path}: incomplete trailing line ({len(trailing)} bytes)")
249
+ with path.open("r+b") as handle:
250
+ handle.truncate(keep)
251
+ handle.flush()
252
+ os.fsync(handle.fileno())
253
+ return results
254
+
255
+
256
+ def pending_requests(
257
+ requests: Iterable[WorkRequest],
258
+ results: Mapping[str, WorkResult],
259
+ *,
260
+ retry_failed: bool = False,
261
+ ) -> list[WorkRequest]:
262
+ """Requests that still need work, in input order.
263
+
264
+ Raises:
265
+ ProtocolError: if ``results`` contains clip ids that are not in ``requests``
266
+ (the results file belongs to a different request set).
267
+ """
268
+ request_list = list(requests)
269
+ known = {r.clip_id for r in request_list}
270
+ stray = sorted(set(results) - known)
271
+ if stray:
272
+ raise ProtocolError(f"results contain unknown clip ids, e.g. {stray[:3]}")
273
+ pending: list[WorkRequest] = []
274
+ for request in request_list:
275
+ done = results.get(request.clip_id)
276
+ if done is None or (retry_failed and not done.ok):
277
+ pending.append(request)
278
+ return pending
279
+
280
+
281
+ def append_result(handle: IO[str], result: WorkResult) -> None:
282
+ """Append one result line and fsync, so an interruption loses at most this line."""
283
+ _fsync_line(handle, result.to_json())
284
+
285
+
286
+ def write_summary(path: Path, summary: WorkerSummary) -> None:
287
+ """Write the summary JSON next to the results (stdlib-only atomic replace)."""
288
+ tmp = path.with_name(f".{path.name}.tmp")
289
+ with tmp.open("w", encoding="utf-8") as handle:
290
+ json.dump(asdict(summary), handle, ensure_ascii=False, indent=2, sort_keys=True)
291
+ handle.write("\n")
292
+ handle.flush()
293
+ os.fsync(handle.fileno())
294
+ tmp.replace(path)
295
+
296
+
297
+ def read_summary(path: Path) -> WorkerSummary:
298
+ """Read and validate a summary file."""
299
+ obj = json.loads(path.read_text(encoding="utf-8"))
300
+ if not isinstance(obj, dict):
301
+ raise ProtocolError(f"{path}: summary must be a JSON object")
302
+ try:
303
+ return WorkerSummary(**obj)
304
+ except TypeError as err:
305
+ raise ProtocolError(f"{path}: invalid summary fields ({err})") from err
306
+
307
+
308
+ def drop_results(path: Path, clip_ids: Iterable[str]) -> int:
309
+ """Remove the given clip ids from a results file (atomic rewrite); returns how many.
310
+
311
+ Used when a result's side effect is gone (e.g. its output file was lost), so the
312
+ item becomes pending again instead of being trusted.
313
+ """
314
+ doomed = set(clip_ids)
315
+ results = load_results(path, repair=True)
316
+ kept = [r.to_json() + "\n" for cid, r in results.items() if cid not in doomed]
317
+ removed = len(results) - len(kept)
318
+ if removed:
319
+ tmp = path.with_name(f".{path.name}.tmp")
320
+ with tmp.open("w", encoding="utf-8") as handle:
321
+ handle.writelines(kept)
322
+ handle.flush()
323
+ os.fsync(handle.fileno())
324
+ tmp.replace(path)
325
+ return removed
src/nida/bench/registry.py ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Benchmarked systems and their pinned revisions (single source of truth).
2
+
3
+ Every system maps to a ``configs/sources.yaml`` id; :func:`require_approved` refuses to
4
+ run anything whose source is not ``approved`` (golden rule 6). Revisions are the
5
+ Hugging Face commit SHAs checked on 2026-10-08; Omnilingual weights are addressed by
6
+ model card through fairseq2 (no hub revision).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Iterable, Mapping
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from types import MappingProxyType
15
+ from typing import Any
16
+
17
+ __all__ = [
18
+ "ASR_SYSTEMS",
19
+ "DATA_PINS",
20
+ "QUALITY_SYSTEM",
21
+ "VAE_SYSTEMS",
22
+ "SourceNotApprovedError",
23
+ "System",
24
+ "load_sources",
25
+ "require_approved",
26
+ ]
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class System:
31
+ """One model under test.
32
+
33
+ Attributes:
34
+ system_id: Short id used in file names and reports.
35
+ source_id: Id in ``configs/sources.yaml``.
36
+ worker: Worker module name in ``nida.bench.workers``.
37
+ model_id: Hub id / model card passed to the worker.
38
+ revision: Pinned revision (``""`` when the worker fetches by model card).
39
+ batch_size: Worker batch size on an L4 (22 GB).
40
+ """
41
+
42
+ system_id: str
43
+ source_id: str
44
+ worker: str
45
+ model_id: str
46
+ revision: str
47
+ batch_size: int
48
+
49
+
50
+ def _index(systems: Iterable[System]) -> Mapping[str, System]:
51
+ out: dict[str, System] = {}
52
+ for system in systems:
53
+ if system.system_id in out:
54
+ raise ValueError(f"duplicate system id {system.system_id}")
55
+ out[system.system_id] = system
56
+ return MappingProxyType(out)
57
+
58
+
59
+ ASR_SYSTEMS: Mapping[str, System] = _index(
60
+ (
61
+ System(
62
+ "qwen3_asr_1_7b",
63
+ "qwen3_asr_1_7b",
64
+ "asr_qwen",
65
+ "Qwen/Qwen3-ASR-1.7B",
66
+ "7278e1e70fe206f11671096ffdd38061171dd6e5",
67
+ 16,
68
+ ),
69
+ System(
70
+ "qwen3_asr_0_6b",
71
+ "qwen3_asr_0_6b",
72
+ "asr_qwen",
73
+ "Qwen/Qwen3-ASR-0.6B",
74
+ "5eb144179a02acc5e5ba31e748d22b0cf3e303b0",
75
+ 16,
76
+ ),
77
+ System("omniasr_llm_7b", "omniasr_llm_7b", "asr_omni", "omniASR_LLM_7B_v2", "", 4),
78
+ System("omniasr_ctc_1b", "omniasr_ctc_1b", "asr_omni", "omniASR_CTC_1B_v2", "", 16),
79
+ System(
80
+ "whisper_large_v3",
81
+ "whisper_large_v3",
82
+ "asr_hf",
83
+ "openai/whisper-large-v3",
84
+ "06f233fe06e710322aca913c1bc4249a0d71fce1",
85
+ 8,
86
+ ),
87
+ System(
88
+ "buzzasr_turkish",
89
+ "buzzasr_turkish",
90
+ "asr_hf",
91
+ "BuzzASR/turkish",
92
+ "928b47596e31776dab9daf1e606ecf76234b9ecd",
93
+ 8,
94
+ ),
95
+ )
96
+ )
97
+
98
+ VAE_SYSTEMS: Mapping[str, System] = _index(
99
+ (
100
+ System(
101
+ "voxcpm2_audiovae_v2",
102
+ "voxcpm2_audiovae_v2",
103
+ "vae_voxcpm",
104
+ "openbmb/VoxCPM2",
105
+ "32279effe8c19989596f05d353d1447f51d9e915",
106
+ 1,
107
+ ),
108
+ )
109
+ )
110
+
111
+ QUALITY_SYSTEM = System(
112
+ "audiobox_aesthetics",
113
+ "audiobox_aesthetics",
114
+ "quality_audiobox",
115
+ "facebook/audiobox-aesthetics",
116
+ "9b1dd8e5df9af7216e836a98974fe3b82c56ded6",
117
+ 16,
118
+ )
119
+
120
+ #: Pinned dataset revisions / URLs (checked 2026-10-08).
121
+ DATA_PINS: Mapping[str, str] = MappingProxyType(
122
+ {
123
+ "fleurs_revision": "70bb2e84b976b7e960aa89f1c648e09c59f894dd",
124
+ "yodas_revision": "52c5a1b9730a136bfd7c4513d4962c70d4e50530",
125
+ "mediaspeech_url": "https://www.openslr.org/resources/108/TR.tgz",
126
+ "expresso_url": "https://dl.fbaipublicfiles.com/textless_nlp/expresso/data/expresso.tar",
127
+ }
128
+ )
129
+
130
+
131
+ class SourceNotApprovedError(RuntimeError):
132
+ """A source is missing from the registry or not approved by the user."""
133
+
134
+
135
+ def load_sources(path: Path) -> dict[str, Mapping[str, Any]]:
136
+ """Read ``configs/sources.yaml`` into ``{id: record}``.
137
+
138
+ Raises:
139
+ ValueError: on a malformed file or duplicate ids.
140
+ """
141
+ import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
142
+
143
+ data: Any = yaml.safe_load(path.read_text(encoding="utf-8")) # Any: untyped YAML tree
144
+ if not isinstance(data, dict) or not isinstance(data.get("sources"), list):
145
+ raise ValueError(f"{path}: expected a mapping with a 'sources' list")
146
+ records: dict[str, Mapping[str, Any]] = {}
147
+ for record in data["sources"]:
148
+ if not isinstance(record, dict) or not isinstance(record.get("id"), str):
149
+ raise ValueError(f"{path}: every source needs a string 'id'")
150
+ if record["id"] in records:
151
+ raise ValueError(f"{path}: duplicate source id {record['id']}")
152
+ records[record["id"]] = record
153
+ return records
154
+
155
+
156
+ def require_approved(source_ids: Iterable[str], sources: Mapping[str, Mapping[str, Any]]) -> None:
157
+ """Raise unless every id exists in the registry with ``status: approved``."""
158
+ problems: list[str] = []
159
+ for source_id in sorted(set(source_ids)):
160
+ record = sources.get(source_id)
161
+ if record is None:
162
+ problems.append(f"{source_id}: not in sources.yaml")
163
+ elif record.get("status") != "approved":
164
+ problems.append(f"{source_id}: status is {record.get('status')!r}")
165
+ if problems:
166
+ raise SourceNotApprovedError("sources not approved: " + "; ".join(problems))
src/nida/bench/report.py ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Markdown rendering of B1 / B2 results (Turkish, for the user and the ADRs)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping, Sequence
6
+
7
+ __all__ = ["render_b1", "render_b2", "render_b2_panel"]
8
+
9
+ _B1_LAYER_TITLES = {
10
+ "A": "A — temiz okuma (FLEURS)",
11
+ "B": "B — medya (MediaSpeech)",
12
+ "AB": "A+B",
13
+ "C": "C — doğal + dolgu (YODAS, kullanıcı referansı)",
14
+ }
15
+
16
+
17
+ def _fmt(value: object, digits: int = 3, percent: bool = False) -> str:
18
+ if value is None:
19
+ return "—"
20
+ if isinstance(value, bool):
21
+ return "evet" if value else "hayır"
22
+ if isinstance(value, int):
23
+ return str(value)
24
+ if isinstance(value, float):
25
+ return f"{100 * value:.1f}%" if percent else f"{value:.{digits}f}"
26
+ return str(value)
27
+
28
+
29
+ def _pct(value: object) -> str:
30
+ return _fmt(value, percent=True)
31
+
32
+
33
+ def _row(cells: Sequence[str]) -> str:
34
+ return "| " + " | ".join(cells) + " |"
35
+
36
+
37
+ def _header(cells: Sequence[str]) -> list[str]:
38
+ return [_row(cells), _row(["---"] * len(cells))]
39
+
40
+
41
+ def render_b1(scores: Mapping[str, object], choice: Mapping[str, object]) -> str:
42
+ systems = scores["systems"]
43
+ assert isinstance(systems, Mapping)
44
+ lines = [
45
+ "# B1 — Türkçe ASR karşılaştırması",
46
+ "",
47
+ "Notlar: (1) `buzzasr_turkish` FLEURS ile eğitildi; A katmanında (FLEURS test) iyimser "
48
+ "olabilir, bu yüzden karar kuralı ayrıca yalnız B katmanıyla da uygulanır. "
49
+ "(2) Omnilingual CTC dil koşulu almaz (lang yok sayılır). (3) Başarısız öğeler boş "
50
+ "hipotez (tamamı silme) sayılır; halüsinasyon oranına girmez. (4) RTF, sistemin kendi "
51
+ "toplu iş (batch) boyutuyla ölçüldü; doğrudan hız karşılaştırması değildir.",
52
+ "",
53
+ ]
54
+ if "n_c_reviewed" in scores:
55
+ lines += [f"C katmanında incelenmiş (referanslı) klip: {scores['n_c_reviewed']}", ""]
56
+ for layer, title in _B1_LAYER_TITLES.items():
57
+ if not all(layer in v for v in systems.values()):
58
+ continue
59
+ lines += [f"## {title}", ""]
60
+ lines += _header(
61
+ [
62
+ "Sistem",
63
+ "n",
64
+ "başarısız",
65
+ "CER içerik",
66
+ "WER içerik",
67
+ "CER tam",
68
+ "WER tam",
69
+ "halüsinasyon",
70
+ "dolgu geri çağırma",
71
+ "dolgu geri çağırma (LOO)",
72
+ "LOO payda",
73
+ "dolgu kesinlik",
74
+ "rakamlı hipotez",
75
+ "RTF",
76
+ ]
77
+ )
78
+ for system_id in sorted(systems):
79
+ r = systems[system_id][layer]
80
+ lines.append(
81
+ _row(
82
+ [
83
+ system_id,
84
+ _fmt(r["n"]),
85
+ _fmt(r["n_failed"]),
86
+ _pct(r["content_cer"]),
87
+ _pct(r["content_wer"]),
88
+ _pct(r["full_cer"]),
89
+ _pct(r["full_wer"]),
90
+ _pct(r["hallucination_rate"]),
91
+ _pct(r["filler_recall"]),
92
+ _pct(r.get("filler_recall_loo")),
93
+ _fmt(r.get("filler_loo_ref_total")),
94
+ _pct(r["filler_precision"]),
95
+ _fmt(r.get("hyp_with_digits")),
96
+ _fmt(r["rtf"]),
97
+ ]
98
+ )
99
+ )
100
+ lines.append("")
101
+ workers = {k: v["_worker"] for k, v in systems.items() if "_worker" in v}
102
+ if workers:
103
+ lines += ["## Kaynak kullanımı", ""]
104
+ lines += _header(["Sistem", "batch", "tepe VRAM (GB)", "yükleme (s)", "hata"])
105
+ for system_id, w in sorted(workers.items()):
106
+ lines.append(
107
+ _row(
108
+ [
109
+ system_id,
110
+ _fmt(w.get("batch_size")),
111
+ _fmt(w.get("peak_vram_gb"), 1),
112
+ _fmt(w.get("load_seconds"), 0),
113
+ _fmt(w.get("n_error")),
114
+ ]
115
+ )
116
+ )
117
+ lines.append("")
118
+ boot = scores.get("bootstrap_content_cer")
119
+ if isinstance(boot, Mapping):
120
+ for layer, rows in boot.items():
121
+ lines += [f"## Eşleştirilmiş bootstrap — içerik CER farkı ({layer}, a − b, %95 GA)", ""]
122
+ lines += _header(["a", "b", "fark", "alt", "üst", "anlamlı mı"])
123
+ for r in rows:
124
+ significant = r["low"] > 0 or r["high"] < 0
125
+ lines.append(
126
+ _row(
127
+ [
128
+ r["a"],
129
+ r["b"],
130
+ _pct(r["delta"]),
131
+ _pct(r["low"]),
132
+ _pct(r["high"]),
133
+ "evet" if significant else "hayır",
134
+ ]
135
+ )
136
+ )
137
+ lines.append("")
138
+ lines += ["## Önceden sabitlenmiş karar kuralına göre öneri", ""]
139
+ if not choice.get("ready"):
140
+ lines.append(f"Henüz uygulanamaz: {choice.get('reason')}.")
141
+ else:
142
+ qualified = choice.get("qualified_by_filler_recall")
143
+ qualified_text = ", ".join(qualified) if isinstance(qualified, list) else ""
144
+ lines += [
145
+ f"- Birincil: **{choice['primary']}**",
146
+ f"- İkincil (farklı aile): **{choice['secondary'] or 'yok (başka aile yok)'}**",
147
+ f"- Dolgu için ayrı yöntem gerekir mi: {_fmt(choice['needs_filler_method'])}",
148
+ f"- Dolgu geri çağırma eşiğini geçenler: {qualified_text or '—'}",
149
+ ]
150
+ without = choice.get("without_fleurs")
151
+ if isinstance(without, Mapping):
152
+ changed = choice.get("pick_changes_without_fleurs")
153
+ lines += [
154
+ "",
155
+ "Aynı kural yalnız B katmanıyla (FLEURS hariç): "
156
+ f"birincil **{without['primary']}**, "
157
+ f"ikincil **{without['secondary'] or 'yok'}** — "
158
+ f"seçim {'DEĞİŞİYOR' if changed else 'değişmiyor'}.",
159
+ ]
160
+ lines.append("")
161
+ return "\n".join(lines)
162
+
163
+
164
+ def render_b2(scores: Mapping[str, object]) -> str:
165
+ subsets = scores["subsets"]
166
+ assert isinstance(subsets, Mapping)
167
+ lines = [
168
+ "# B2 — VAE yeniden kurma (VoxCPM2 AudioVAE V2)",
169
+ "",
170
+ "Mel L1: log10 mel genliği farkı (düşük = iyi). HB: 8–24 kHz bandı. "
171
+ "`band16k`: 48→16→48 kHz yeniden örnekleme taban çizgisi (8 kHz üstü yok).",
172
+ "Audiobox 16 kHz'de çalışır; yüksek bant kalitesini ölçmez. HB LSD yalnız etkin "
173
+ "karelerde (en yüksek karenin 40 dB altına kadar) ortalanır. ΔPQ/ΔCE eşleştirilmiş "
174
+ "kliplerin farklarının ortalamasıdır (VAE − orijinal).",
175
+ "",
176
+ ]
177
+ skipped = scores.get("skipped")
178
+ if isinstance(skipped, Mapping) and skipped:
179
+ lines += ["Atlanan: " + ", ".join(f"{k}={v}" for k, v in sorted(skipped.items())), ""]
180
+ lines += _header(
181
+ [
182
+ "Alt küme",
183
+ "n",
184
+ "puanlanan",
185
+ "mel16 VAE",
186
+ "mel48 VAE",
187
+ "mel48 band16k",
188
+ "HB LSD VAE (dB)",
189
+ "HB LSD band16k (dB)",
190
+ "HB enerji oranı VAE",
191
+ "ΔPQ",
192
+ "ΔCE",
193
+ "CER orijinal",
194
+ "CER VAE",
195
+ "gecikme (ms)",
196
+ ]
197
+ )
198
+ for subset, row in subsets.items():
199
+ assert isinstance(row, Mapping)
200
+ lines.append(
201
+ _row(
202
+ [
203
+ subset,
204
+ _fmt(row["n"]),
205
+ _fmt(row.get("n_scored")),
206
+ _fmt(row.get("vae_mel16_l1")),
207
+ _fmt(row.get("vae_mel48_l1")),
208
+ _fmt(row.get("band16k_mel48_l1")),
209
+ _fmt(row.get("vae_hb_lsd_db"), 1),
210
+ _fmt(row.get("band16k_hb_lsd_db"), 1),
211
+ _fmt(row.get("vae_hb_energy_ratio"), 2),
212
+ _fmt(row.get("delta_PQ"), 2),
213
+ _fmt(row.get("delta_CE"), 2),
214
+ _pct(row.get("cer_orig")),
215
+ _pct(row.get("cer_vae")),
216
+ _fmt(row.get("vae_delay_ms"), 1),
217
+ ]
218
+ )
219
+ )
220
+ lines.append("")
221
+ return "\n".join(lines)
222
+
223
+
224
+ def render_b2_panel(scores: Mapping[str, object]) -> str:
225
+ """Un-blinded B2 listening-panel results with the pre-registered pass/fail per subset."""
226
+ subsets = scores["subsets"]
227
+ rule = scores.get("rule")
228
+ assert isinstance(subsets, Mapping) and isinstance(rule, Mapping)
229
+ lines = [
230
+ "# B2 — Dinleme panosu (kör A/B, çözülmüş)",
231
+ "",
232
+ f"Kural: ifade korunma oranı ≥ {_pct(rule['min_preserved_share'])} "
233
+ '("kısmen" korunmamış sayılır; tr hariç) ve kalite farkı (orijinal − VAE) '
234
+ f"≤ {_fmt(rule['max_quality_gap'], 1)}.",
235
+ "",
236
+ ]
237
+ lines += _header(
238
+ [
239
+ "Alt küme",
240
+ "yanıt",
241
+ "korunmuş",
242
+ "kısmen",
243
+ "kalite orijinal",
244
+ "kalite VAE",
245
+ "fark",
246
+ "geçti",
247
+ ]
248
+ )
249
+ for subset, row in subsets.items():
250
+ assert isinstance(row, Mapping)
251
+ lines.append(
252
+ _row(
253
+ [
254
+ subset,
255
+ _fmt(row["n_answered"]),
256
+ _pct(row["preserved_share"]),
257
+ _pct(row["partial_share"]),
258
+ _fmt(row["quality_orig"], 2),
259
+ _fmt(row["quality_vae"], 2),
260
+ _fmt(row["quality_gap"], 2),
261
+ _fmt(row["passes"]),
262
+ ]
263
+ )
264
+ )
265
+ lines += ["", f"Tümü geçti mi: **{_fmt(scores.get('all_pass'))}**", ""]
266
+ for key, label in (
267
+ ("missing_subsets", "Panoda hiç olmayan ifadeli alt kümeler"),
268
+ ("unanswered", "Yanıtlanmamış satırlar (alt küme: adet)"),
269
+ ("too_few_items", "En az adedin altında kalan alt kümeler"),
270
+ ("excluded_missing_reconstruction", "Yeniden kurulamadığı için panoya alınmayan"),
271
+ ):
272
+ value = scores.get(key)
273
+ if value:
274
+ lines.append(f"- {label}: {value}")
275
+ lines.append("")
276
+ return "\n".join(lines)
src/nida/bench/textmetrics.py ADDED
@@ -0,0 +1,464 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Text metrics for scoring ASR hypotheses against Turkish references (benchmark B1).
2
+
3
+ Pure standard-library implementation. The pipeline is:
4
+
5
+ 1. :func:`normalize_for_scoring` turns a raw string into scoring tokens (NFC, Turkish
6
+ lowercasing, punctuation removal, filler canonicalisation per tag-spec section 4).
7
+ 2. :func:`cer_counts` / :func:`wer_counts` return :class:`EditCounts` per utterance.
8
+ 3. :func:`corpus_rate` aggregates counts over a corpus (sum of errors over sum of reference
9
+ lengths, *not* a mean of per-utterance rates).
10
+ 4. :func:`filler_score` measures filler recall / precision, and :func:`paired_bootstrap`
11
+ gives a confidence interval for the difference between two systems.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import math
17
+ import random
18
+ import unicodedata
19
+ from collections import Counter
20
+ from collections.abc import Hashable, Iterable, Sequence
21
+ from dataclasses import dataclass
22
+ from typing import TypeVar
23
+
24
+ from nida.text.fillers import FILLERS, canonical_filler
25
+ from nida.text.turkish import nfc, tr_lower
26
+
27
+ __all__ = [
28
+ "FILLERS",
29
+ "BootstrapResult",
30
+ "EditCounts",
31
+ "FillerScore",
32
+ "canonical_filler",
33
+ "cer_counts",
34
+ "corpus_rate",
35
+ "edit_counts",
36
+ "filler_counts",
37
+ "filler_score",
38
+ "has_digits",
39
+ "is_hallucination",
40
+ "normalize_english",
41
+ "normalize_for_scoring",
42
+ "paired_bootstrap",
43
+ "strip_fillers",
44
+ "wer_counts",
45
+ ]
46
+
47
+ T = TypeVar("T", bound=Hashable)
48
+
49
+ # Apostrophe-like characters that are deleted (not replaced by a space), so that
50
+ # "Ankara'da" becomes "ankarada": U+0027, U+2019 and U+02BC (a modifier letter, category Lm).
51
+ _APOSTROPHES: frozenset[str] = frozenset({"'", "\u2019", "\u02bc"})
52
+
53
+ # Circumflex letters are optional in modern Turkish spelling ("hala"/"hâlâ"); systems and
54
+ # references use them inconsistently, so scoring folds them to the plain vowel.
55
+ _CIRCUMFLEX_FOLD = str.maketrans({"â": "a", "î": "i", "û": "u"})
56
+
57
+ # --------------------------------------------------------------------------------------
58
+ # Normalisation and fillers
59
+ # --------------------------------------------------------------------------------------
60
+
61
+
62
+ def normalize_for_scoring(text: str) -> list[str]:
63
+ """Normalise ``text`` into scoring tokens.
64
+
65
+ Steps: NFC, Turkish lowercase, fold circumflexes (â/î/û → a/i/u), delete apostrophes
66
+ (U+0027, U+2019, U+02BC), turn whitespace and every other punctuation (``P*``) or
67
+ symbol (``S*``) character into a space, split on whitespace, and map each token
68
+ through :func:`canonical_filler`. Letters and digits are kept as-is. The cleaned
69
+ string is re-composed to NFC before splitting, because deleting a character can leave
70
+ a base letter next to a combining mark.
71
+
72
+ Args:
73
+ text: Raw reference or hypothesis text.
74
+
75
+ Returns:
76
+ List of tokens (possibly empty).
77
+
78
+ Examples:
79
+ >>> normalize_for_scoring("Ankara'da, ııı... Hımm!")
80
+ ['ankarada', 'ıı', 'hmm']
81
+ """
82
+ chars: list[str] = []
83
+ for ch in tr_lower(text).translate(_CIRCUMFLEX_FOLD):
84
+ if ch in _APOSTROPHES:
85
+ continue
86
+ if ch.isspace():
87
+ chars.append(" ")
88
+ continue
89
+ # Punctuation and symbols become word boundaries, never glue: "merhaba,dünya" must
90
+ # score as two words, as in common ASR normalisers.
91
+ if unicodedata.category(ch)[0] in ("P", "S"):
92
+ chars.append(" ")
93
+ else:
94
+ chars.append(ch)
95
+ tokens = nfc("".join(chars)).split()
96
+ result: list[str] = []
97
+ for token in tokens:
98
+ canonical = canonical_filler(token)
99
+ result.append(token if canonical is None else canonical)
100
+ return result
101
+
102
+
103
+ def strip_fillers(tokens: Sequence[str]) -> list[str]:
104
+ """Return ``tokens`` without canonical fillers.
105
+
106
+ Tokens must come from :func:`normalize_for_scoring` (already canonical); membership in
107
+ :data:`FILLERS` is tested literally.
108
+ """
109
+ return [token for token in tokens if token not in FILLERS]
110
+
111
+
112
+ def has_digits(text: str) -> bool:
113
+ """Return True if ``text`` (after NFC) contains any Unicode decimal digit."""
114
+ return any(ch.isdecimal() for ch in nfc(text))
115
+
116
+
117
+ # --------------------------------------------------------------------------------------
118
+ # Edit distance
119
+ # --------------------------------------------------------------------------------------
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class EditCounts:
124
+ """Substitution / deletion / insertion counts against a reference of ``ref_len`` units.
125
+
126
+ Attributes:
127
+ substitutions: Reference units replaced by a different hypothesis unit.
128
+ deletions: Reference units missing from the hypothesis.
129
+ insertions: Extra hypothesis units not in the reference.
130
+ ref_len: Number of reference units (characters or tokens).
131
+ """
132
+
133
+ substitutions: int
134
+ deletions: int
135
+ insertions: int
136
+ ref_len: int
137
+
138
+ def __post_init__(self) -> None:
139
+ for name in ("substitutions", "deletions", "insertions", "ref_len"):
140
+ value = getattr(self, name)
141
+ if value < 0:
142
+ raise ValueError(f"EditCounts.{name} must be >= 0, got {value}")
143
+
144
+ @property
145
+ def errors(self) -> int:
146
+ """Total number of edits (S + D + I)."""
147
+ return self.substitutions + self.deletions + self.insertions
148
+
149
+ def rate(self) -> float | None:
150
+ """Error rate ``errors / ref_len``; ``None`` when ``ref_len`` is 0."""
151
+ if self.ref_len == 0:
152
+ return None
153
+ return self.errors / self.ref_len
154
+
155
+ def __add__(self, other: object) -> EditCounts:
156
+ """Component-wise sum, for corpus aggregation."""
157
+ if not isinstance(other, EditCounts):
158
+ return NotImplemented
159
+ return EditCounts(
160
+ substitutions=self.substitutions + other.substitutions,
161
+ deletions=self.deletions + other.deletions,
162
+ insertions=self.insertions + other.insertions,
163
+ ref_len=self.ref_len + other.ref_len,
164
+ )
165
+
166
+
167
+ def edit_counts(ref: Sequence[T], hyp: Sequence[T]) -> EditCounts:
168
+ """Levenshtein alignment with unit costs, returning the S/D/I split.
169
+
170
+ Time is O(len(ref) * len(hyp)); memory is O(len(hyp)) because each DP cell carries its
171
+ own (cost, S, D, I) tuple instead of keeping the full matrix for backtracking.
172
+
173
+ Tie-breaking (deterministic): walking back from the end of both sequences, whenever
174
+ several predecessors give the same minimal total cost the alignment prefers, in order,
175
+ a substitution/match (diagonal), then a deletion (a reference unit with no hypothesis
176
+ unit), then an insertion. Carrying counts forward along the per-cell choice is
177
+ equivalent to that backtracking, because each cell's choice depends only on that cell.
178
+
179
+ Args:
180
+ ref: Reference units (characters or tokens).
181
+ hyp: Hypothesis units.
182
+
183
+ Returns:
184
+ Counts with ``ref_len == len(ref)``.
185
+ """
186
+ num_hyp = len(hyp)
187
+ # Each cell: (cost, substitutions, deletions, insertions).
188
+ prev: list[tuple[int, int, int, int]] = [(j, 0, 0, j) for j in range(num_hyp + 1)]
189
+ for i in range(1, len(ref) + 1):
190
+ ref_unit = ref[i - 1]
191
+ row: list[tuple[int, int, int, int]] = [(i, 0, i, 0)]
192
+ for j in range(1, num_hyp + 1):
193
+ diag = prev[j - 1]
194
+ up = prev[j]
195
+ left = row[j - 1]
196
+ diag_cost = diag[0] + (0 if ref_unit == hyp[j - 1] else 1)
197
+ up_cost = up[0] + 1
198
+ left_cost = left[0] + 1
199
+ if diag_cost <= up_cost and diag_cost <= left_cost:
200
+ mismatch = diag_cost - diag[0]
201
+ row.append((diag_cost, diag[1] + mismatch, diag[2], diag[3]))
202
+ elif up_cost <= left_cost:
203
+ row.append((up_cost, up[1], up[2] + 1, up[3]))
204
+ else:
205
+ row.append((left_cost, left[1], left[2], left[3] + 1))
206
+ prev = row
207
+ _, subs, dels, ins = prev[num_hyp]
208
+ return EditCounts(substitutions=subs, deletions=dels, insertions=ins, ref_len=len(ref))
209
+
210
+
211
+ def cer_counts(ref_tokens: Sequence[str], hyp_tokens: Sequence[str]) -> EditCounts:
212
+ """Character-level edit counts over ``" ".join(tokens)`` (spaces count as characters)."""
213
+ return edit_counts(" ".join(ref_tokens), " ".join(hyp_tokens))
214
+
215
+
216
+ def wer_counts(ref_tokens: Sequence[str], hyp_tokens: Sequence[str]) -> EditCounts:
217
+ """Token-level (word) edit counts."""
218
+ return edit_counts(ref_tokens, hyp_tokens)
219
+
220
+
221
+ def corpus_rate(counts: Iterable[EditCounts]) -> float | None:
222
+ """Corpus-level error rate: ``sum(errors) / sum(ref_len)``.
223
+
224
+ Returns ``None`` if the total reference length is 0.
225
+ """
226
+ total_errors = 0
227
+ total_ref = 0
228
+ for item in counts:
229
+ total_errors += item.errors
230
+ total_ref += item.ref_len
231
+ if total_ref == 0:
232
+ return None
233
+ return total_errors / total_ref
234
+
235
+
236
+ # --------------------------------------------------------------------------------------
237
+ # Fillers
238
+ # --------------------------------------------------------------------------------------
239
+
240
+
241
+ def filler_counts(tokens: Sequence[str]) -> Counter[str]:
242
+ """Count canonical fillers in ``tokens`` (tokens from :func:`normalize_for_scoring`)."""
243
+ return Counter(token for token in tokens if token in FILLERS)
244
+
245
+
246
+ @dataclass(frozen=True)
247
+ class FillerScore:
248
+ """Filler matching totals.
249
+
250
+ Attributes:
251
+ matched: Sum over utterances and types of ``min(n_ref, n_hyp)``.
252
+ ref_total: Number of reference fillers.
253
+ hyp_total: Number of hypothesis fillers.
254
+ """
255
+
256
+ matched: int
257
+ ref_total: int
258
+ hyp_total: int
259
+
260
+ def recall(self) -> float | None:
261
+ """``matched / ref_total``; ``None`` when there are no reference fillers."""
262
+ if self.ref_total == 0:
263
+ return None
264
+ return self.matched / self.ref_total
265
+
266
+ def precision(self) -> float | None:
267
+ """``matched / hyp_total``; ``None`` when there are no hypothesis fillers."""
268
+ if self.hyp_total == 0:
269
+ return None
270
+ return self.matched / self.hyp_total
271
+
272
+
273
+ def filler_score(
274
+ pairs: Iterable[tuple[Sequence[str], Sequence[str]]], filler: str | None = None
275
+ ) -> FillerScore:
276
+ """Aggregate filler matches over ``(ref_tokens, hyp_tokens)`` pairs.
277
+
278
+ Per utterance, ``matched = sum over types of min(n_ref, n_hyp)``.
279
+
280
+ Args:
281
+ pairs: Normalised (reference tokens, hypothesis tokens) per utterance.
282
+ filler: If given, restrict scoring to this canonical filler type.
283
+
284
+ Raises:
285
+ ValueError: If ``filler`` is not a member of :data:`FILLERS`.
286
+ """
287
+ if filler is not None and filler not in FILLERS:
288
+ raise ValueError(f"unknown filler {filler!r}; expected one of {sorted(FILLERS)}")
289
+ matched = 0
290
+ ref_total = 0
291
+ hyp_total = 0
292
+ for ref_tokens, hyp_tokens in pairs:
293
+ ref_counter = filler_counts(ref_tokens)
294
+ hyp_counter = filler_counts(hyp_tokens)
295
+ types = FILLERS if filler is None else frozenset({filler})
296
+ for kind in types:
297
+ n_ref = ref_counter[kind]
298
+ n_hyp = hyp_counter[kind]
299
+ matched += min(n_ref, n_hyp)
300
+ ref_total += n_ref
301
+ hyp_total += n_hyp
302
+ return FillerScore(matched=matched, ref_total=ref_total, hyp_total=hyp_total)
303
+
304
+
305
+ # --------------------------------------------------------------------------------------
306
+ # Hallucination
307
+ # --------------------------------------------------------------------------------------
308
+
309
+
310
+ def is_hallucination(ref_text: str, hyp_text: str, max_ratio: float = 1.5) -> bool:
311
+ """Flag a hypothesis that is empty or far longer than the reference.
312
+
313
+ Lengths are character counts of ``" ".join(normalize_for_scoring(text))``.
314
+
315
+ Returns:
316
+ True if the hypothesis is empty while the reference is not, or if
317
+ ``len(hyp) > max_ratio * len(ref)`` (reference non-empty). If the reference is
318
+ empty, True iff the hypothesis is non-empty.
319
+
320
+ Raises:
321
+ ValueError: If ``max_ratio`` is not > 0.
322
+ """
323
+ if not max_ratio > 0:
324
+ raise ValueError(f"max_ratio must be > 0, got {max_ratio}")
325
+ ref_len = len(" ".join(normalize_for_scoring(ref_text)))
326
+ hyp_len = len(" ".join(normalize_for_scoring(hyp_text)))
327
+ if ref_len == 0:
328
+ return hyp_len > 0
329
+ if hyp_len == 0:
330
+ return True
331
+ return hyp_len > max_ratio * ref_len
332
+
333
+
334
+ # --------------------------------------------------------------------------------------
335
+ # Paired bootstrap
336
+ # --------------------------------------------------------------------------------------
337
+
338
+
339
+ @dataclass(frozen=True)
340
+ class BootstrapResult:
341
+ """Result of :func:`paired_bootstrap`.
342
+
343
+ Attributes:
344
+ delta: ``corpus_rate(a) - corpus_rate(b)`` on the full set.
345
+ low: Lower percentile bound of the resampled deltas.
346
+ high: Upper percentile bound of the resampled deltas.
347
+ n_resamples: Number of resamples requested.
348
+ n_skipped: Resamples skipped because their total reference length was 0; the
349
+ interval is computed from ``n_resamples - n_skipped`` resamples.
350
+ """
351
+
352
+ delta: float
353
+ low: float
354
+ high: float
355
+ n_resamples: int
356
+ n_skipped: int = 0
357
+
358
+
359
+ def _quantile(sorted_values: Sequence[float], q: float) -> float:
360
+ """Linear-interpolation quantile of an already sorted, non-empty sequence."""
361
+ position = q * (len(sorted_values) - 1)
362
+ lower = math.floor(position)
363
+ upper = math.ceil(position)
364
+ if lower == upper:
365
+ return sorted_values[lower]
366
+ fraction = position - lower
367
+ return sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * fraction
368
+
369
+
370
+ def paired_bootstrap(
371
+ a: Sequence[EditCounts],
372
+ b: Sequence[EditCounts],
373
+ n_resamples: int = 10000,
374
+ seed: int = 0,
375
+ confidence: float = 0.95,
376
+ ) -> BootstrapResult:
377
+ """Paired bootstrap confidence interval for the corpus error-rate difference a - b.
378
+
379
+ Utterance indices are resampled with replacement (the same indices for both systems)
380
+ using ``random.Random(seed)``; the interval is the percentile interval of the resampled
381
+ ``corpus_rate(a) - corpus_rate(b)``.
382
+
383
+ Args:
384
+ a: Per-utterance counts for system A.
385
+ b: Per-utterance counts for system B, scored against the same references.
386
+ n_resamples: Number of bootstrap resamples (>= 1).
387
+ seed: Seed for the resampling RNG.
388
+ confidence: Confidence level in (0, 1).
389
+
390
+ Raises:
391
+ ValueError: On length mismatch, empty input, differing per-index ``ref_len``,
392
+ invalid ``n_resamples`` / ``confidence``, zero total reference length, or if
393
+ more than half of the resamples have zero total reference length.
394
+ """
395
+ if len(a) != len(b):
396
+ raise ValueError(f"a and b must have the same length, got {len(a)} and {len(b)}")
397
+ if len(a) == 0:
398
+ raise ValueError("a and b must be non-empty")
399
+ for index, (item_a, item_b) in enumerate(zip(a, b, strict=True)):
400
+ if item_a.ref_len != item_b.ref_len:
401
+ raise ValueError(
402
+ f"ref_len mismatch at index {index}: {item_a.ref_len} vs {item_b.ref_len}; "
403
+ "both systems must be scored against the same references"
404
+ )
405
+ if n_resamples < 1:
406
+ raise ValueError(f"n_resamples must be >= 1, got {n_resamples}")
407
+ if not 0.0 < confidence < 1.0:
408
+ raise ValueError(f"confidence must be in (0, 1), got {confidence}")
409
+ ref_lens = [item.ref_len for item in a]
410
+ total_ref = sum(ref_lens)
411
+ if total_ref == 0:
412
+ raise ValueError("total reference length is 0; the error rate is undefined")
413
+
414
+ errors_a = [item.errors for item in a]
415
+ errors_b = [item.errors for item in b]
416
+ rate_a = corpus_rate(a)
417
+ rate_b = corpus_rate(b)
418
+ if rate_a is None or rate_b is None: # unreachable: total_ref > 0 and refs are equal
419
+ raise ValueError("corpus rate undefined")
420
+ delta = rate_a - rate_b
421
+
422
+ num_items = len(a)
423
+ rng = random.Random(seed)
424
+ population = range(num_items)
425
+ deltas: list[float] = []
426
+ skipped = 0
427
+ for _ in range(n_resamples):
428
+ indices = rng.choices(population, k=num_items)
429
+ sample_ref = sum(ref_lens[i] for i in indices)
430
+ if sample_ref == 0:
431
+ skipped += 1
432
+ continue
433
+ sample_a = sum(errors_a[i] for i in indices)
434
+ sample_b = sum(errors_b[i] for i in indices)
435
+ deltas.append((sample_a - sample_b) / sample_ref)
436
+ if skipped * 2 > n_resamples:
437
+ raise ValueError(
438
+ f"{skipped} of {n_resamples} resamples had zero total reference length "
439
+ "(more than half); the data has too few non-empty references to bootstrap"
440
+ )
441
+ deltas.sort()
442
+ alpha = (1.0 - confidence) / 2.0
443
+ return BootstrapResult(
444
+ delta=delta,
445
+ low=_quantile(deltas, alpha),
446
+ high=_quantile(deltas, 1.0 - alpha),
447
+ n_resamples=n_resamples,
448
+ n_skipped=skipped,
449
+ )
450
+
451
+
452
+ def normalize_english(text: str) -> list[str]:
453
+ """Scoring tokens for *English* text (B2 Expresso clips).
454
+
455
+ NFC, ``str.casefold`` (correct for English; never use it on Turkish text), punctuation
456
+ and symbols become spaces, apostrophes are deleted, split on whitespace. No filler
457
+ canonicalisation: Turkish filler spellings ("of", "ah", "oh") are English words here.
458
+ """
459
+ chars: list[str] = []
460
+ for ch in nfc(text).casefold():
461
+ if ch in _APOSTROPHES:
462
+ continue
463
+ chars.append(" " if ch.isspace() or unicodedata.category(ch)[0] in ("P", "S") else ch)
464
+ return "".join(chars).split()
src/nida/bench/workers/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ """Model workers for Phase 0 benchmarks.
2
+
3
+ Each module here runs inside its own virtual environment (see ``nida.bench.envs``) and is
4
+ started as ``python -m nida.bench.workers.<name> ...``. Workers depend only on the
5
+ standard library, ``nida.bench.protocol`` and their model library; heavy imports happen
6
+ inside ``Adapter.load`` so this package stays importable everywhere.
7
+ """
src/nida/bench/workers/asr_hf.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Whisper-family ASR via the transformers pipeline (whisper-large-v3, BuzzASR/turkish).
2
+
3
+ Decoding: beam search 5, no sampling, task ``transcribe`` with the language forced.
4
+
5
+ An optional per-request ``params["prompt"]`` is passed to Whisper as a decoder prompt
6
+ (``generate_kwargs["prompt_ids"]``, from ``tokenizer.get_prompt_ids``). With transformers
7
+ 4.57 the tokenizer drops the prompt tokens when it decodes (``_strip_prompt``), so the pipeline
8
+ output normally holds the transcript only; a literal leading copy of the prompt is stripped
9
+ here as a safeguard and flagged with ``"prompt_stripped": true``. Without the parameter the
10
+ behaviour (and the output) is exactly the unprompted decoding, so earlier results stay
11
+ comparable.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import importlib
17
+ from collections.abc import Mapping, Sequence
18
+ from typing import Any
19
+
20
+ from nida.bench.protocol import WorkRequest
21
+ from nida.bench.workers.common import language_for, load_mono_16k, snapshot
22
+ from nida.bench.workers.runner import Adapter, ModelRef, main_for
23
+
24
+ NUM_BEAMS = 5
25
+ PROMPT_PARAM = "prompt"
26
+
27
+
28
+ def prompt_for(batch: Sequence[WorkRequest]) -> str | None:
29
+ """The prompt shared by every request of ``batch`` (``None`` = unprompted decoding).
30
+
31
+ Raises:
32
+ ValueError: if requests disagree (a prompt and no prompt count as different) or the
33
+ prompt is blank.
34
+ """
35
+ prompts = {r.params.get(PROMPT_PARAM) for r in batch}
36
+ if len(prompts) != 1:
37
+ shown = sorted("<none>" if p is None else repr(p) for p in prompts)
38
+ raise ValueError(f"mixed prompts in one batch: {shown}")
39
+ prompt = next(iter(prompts))
40
+ if prompt is not None and not prompt.strip():
41
+ raise ValueError("params.prompt is blank; omit it for unprompted decoding")
42
+ return prompt
43
+
44
+
45
+ def strip_prompt_prefix(text: str, prompt: str) -> tuple[str, bool]:
46
+ """Remove a literal leading copy of ``prompt`` from ``text``.
47
+
48
+ Returns:
49
+ ``(text without the prompt, whether a prefix was removed)``.
50
+ """
51
+ text, prompt = text.strip(), prompt.strip()
52
+ if text.startswith(prompt):
53
+ return text[len(prompt) :].strip(), True
54
+ return text, False
55
+
56
+
57
+ class WhisperAdapter(Adapter):
58
+ name = "asr_hf"
59
+ distributions = ("torch", "transformers", "huggingface-hub", "soundfile", "numpy")
60
+
61
+ def __init__(self) -> None:
62
+ self._pipe: Any = None # Any: transformers pipeline object (untyped here)
63
+ self._batch_size = 1
64
+ self._device = "cpu"
65
+
66
+ def load(self, model: ModelRef, device: str, batch_size: int) -> None:
67
+ torch = importlib.import_module("torch")
68
+ transformers = importlib.import_module("transformers")
69
+ self._batch_size = batch_size
70
+ self._device = device
71
+ self._pipe = transformers.pipeline(
72
+ "automatic-speech-recognition",
73
+ model=snapshot(model.model_id, model.revision),
74
+ torch_dtype=torch.float16,
75
+ device=device,
76
+ )
77
+
78
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
79
+ languages = {language_for(r, "whisper") for r in batch}
80
+ if len(languages) != 1:
81
+ raise ValueError(f"mixed languages in one batch: {sorted(languages)}")
82
+ prompt = prompt_for(batch)
83
+ generate_kwargs: dict[str, Any] = { # Any: tensors / strings handed to generate()
84
+ "language": languages.pop(),
85
+ "task": "transcribe",
86
+ "num_beams": NUM_BEAMS,
87
+ "do_sample": False,
88
+ }
89
+ if prompt is not None:
90
+ # rank-1 tensor (<|startofprev|> + prompt tokens); generate() repeats it per item
91
+ prompt_ids = self._pipe.tokenizer.get_prompt_ids(prompt, return_tensors="pt")
92
+ generate_kwargs["prompt_ids"] = prompt_ids.to(self._device)
93
+ inputs = [{"raw": load_mono_16k(r.audio_path), "sampling_rate": 16000} for r in batch]
94
+ outputs = self._pipe(inputs, batch_size=self._batch_size, generate_kwargs=generate_kwargs)
95
+ texts = [str(o["text"]).strip() for o in outputs]
96
+ if prompt is None:
97
+ return [{"text": text} for text in texts]
98
+ stripped = [strip_prompt_prefix(text, prompt) for text in texts]
99
+ return [{"text": text, "prompt_stripped": was} for text, was in stripped]
100
+
101
+
102
+ if __name__ == "__main__":
103
+ raise SystemExit(main_for(WhisperAdapter()))
src/nida/bench/workers/asr_omni.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Meta Omnilingual ASR via its inference pipeline (CTC and LLM model cards).
2
+
3
+ ``model_id`` is the model card name (e.g. ``omniASR_LLM_7B_v2``); weights come from
4
+ Meta's CDN through fairseq2's asset store, so ``revision`` is recorded but unused.
5
+ The pipeline rejects audio longer than 40 s; B1 clips are at most 25 s.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import importlib
11
+ from collections.abc import Mapping, Sequence
12
+ from typing import Any
13
+
14
+ from nida.bench.protocol import WorkRequest
15
+ from nida.bench.workers.common import language_for
16
+ from nida.bench.workers.runner import Adapter, ModelRef, main_for
17
+
18
+
19
+ class OmniAdapter(Adapter):
20
+ name = "asr_omni"
21
+ distributions = ("torch", "omnilingual-asr", "fairseq2", "fairseq2n")
22
+
23
+ def __init__(self) -> None:
24
+ self._pipeline: Any = None # Any: ASRInferencePipeline (untyped)
25
+ self._batch_size = 1
26
+
27
+ def load(self, model: ModelRef, device: str, batch_size: int) -> None:
28
+ module = importlib.import_module("omnilingual_asr.models.inference.pipeline")
29
+ self._batch_size = batch_size
30
+ self._pipeline = module.ASRInferencePipeline(model_card=model.model_id, device=device)
31
+
32
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
33
+ texts = self._pipeline.transcribe(
34
+ [r.audio_path for r in batch],
35
+ lang=[language_for(r, "omni") for r in batch],
36
+ batch_size=self._batch_size,
37
+ )
38
+ return [{"text": str(t).strip()} for t in texts]
39
+
40
+
41
+ if __name__ == "__main__":
42
+ raise SystemExit(main_for(OmniAdapter()))
src/nida/bench/workers/asr_qwen.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Qwen3-ASR via the official ``qwen-asr`` package (transformers backend).
2
+
3
+ ``params["context"]`` (optional) is passed as the system context; B1 uses no context by
4
+ default and a separate, explicitly labelled run for the filler-context variant.
5
+ ``params["language"] == "auto"`` (all items of a batch) lets the model identify the
6
+ language itself (``language=None``, Qwen3-ASR README); the detected name is returned.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import importlib
12
+ from collections.abc import Mapping, Sequence
13
+ from typing import Any
14
+
15
+ from nida.bench.protocol import WorkRequest
16
+ from nida.bench.workers.common import language_for, snapshot
17
+ from nida.bench.workers.runner import Adapter, ModelRef, main_for
18
+
19
+ MAX_NEW_TOKENS = 256
20
+ #: ``params["language"]`` value requesting automatic language identification.
21
+ AUTO_LANGUAGE = "auto"
22
+
23
+
24
+ class QwenAdapter(Adapter):
25
+ name = "asr_qwen"
26
+ distributions = ("torch", "qwen-asr", "transformers", "huggingface-hub")
27
+
28
+ def __init__(self) -> None:
29
+ self._model: Any = None # Any: qwen_asr.Qwen3ASRModel (untyped)
30
+
31
+ def load(self, model: ModelRef, device: str, batch_size: int) -> None:
32
+ torch = importlib.import_module("torch")
33
+ qwen_asr = importlib.import_module("qwen_asr")
34
+ self._model = qwen_asr.Qwen3ASRModel.from_pretrained(
35
+ snapshot(model.model_id, model.revision),
36
+ dtype=torch.bfloat16,
37
+ device_map=device,
38
+ max_inference_batch_size=batch_size,
39
+ max_new_tokens=MAX_NEW_TOKENS,
40
+ )
41
+
42
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
43
+ auto = [r.params.get("language") == AUTO_LANGUAGE for r in batch]
44
+ if any(auto) and not all(auto):
45
+ raise ValueError("a batch mixes automatic and forced languages")
46
+ results = self._model.transcribe(
47
+ audio=[r.audio_path for r in batch],
48
+ context=[r.params.get("context", "") for r in batch],
49
+ language=None if all(auto) else [language_for(r, "qwen") for r in batch],
50
+ )
51
+ return [{"text": str(res.text).strip(), "language": str(res.language)} for res in results]
52
+
53
+
54
+ if __name__ == "__main__":
55
+ raise SystemExit(main_for(QwenAdapter()))
src/nida/bench/workers/common.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Small helpers shared by adapters (stdlib only)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib
6
+ from collections.abc import Mapping, Sequence
7
+ from typing import Any
8
+
9
+ from nida.bench.protocol import WorkRequest
10
+
11
+ __all__ = ["HF_WEIGHT_PATTERNS", "LANGUAGES", "language_for", "load_mono_16k", "snapshot"]
12
+
13
+ #: Files needed to load a transformers checkpoint in safetensors form. Excludes duplicate
14
+ #: formats (flax, pytorch .bin, fp32 shards) that some repos also ship.
15
+ HF_WEIGHT_PATTERNS: tuple[str, ...] = (
16
+ "*.json",
17
+ "*.txt",
18
+ "*.model",
19
+ "*.tiktoken",
20
+ "model.safetensors",
21
+ "model-*.safetensors",
22
+ )
23
+
24
+ #: ISO 639-1 code → spelling each model family expects.
25
+ LANGUAGES: Mapping[str, Mapping[str, str]] = {
26
+ "tr": {"whisper": "turkish", "qwen": "Turkish", "omni": "tur_Latn"},
27
+ "en": {"whisper": "english", "qwen": "English", "omni": "eng_Latn"},
28
+ }
29
+
30
+
31
+ def language_for(request: WorkRequest, family: str) -> str:
32
+ """Model-specific language name for ``request.params["language"]`` (required).
33
+
34
+ Raises:
35
+ ValueError: if the language parameter is missing or unsupported.
36
+ """
37
+ code = request.params.get("language")
38
+ if code is None:
39
+ raise ValueError(f"{request.clip_id}: missing params.language")
40
+ try:
41
+ return LANGUAGES[code][family]
42
+ except KeyError as err:
43
+ raise ValueError(f"{request.clip_id}: unsupported language {code!r} for {family}") from err
44
+
45
+
46
+ def snapshot(
47
+ model_id: str, revision: str, allow_patterns: Sequence[str] = HF_WEIGHT_PATTERNS
48
+ ) -> str:
49
+ """Download (or reuse) a pinned Hugging Face snapshot and return its local path.
50
+
51
+ An empty ``revision`` is refused: benchmark runs must be reproducible. Only files
52
+ matching ``allow_patterns`` are fetched.
53
+ """
54
+ if not revision:
55
+ raise ValueError(f"{model_id}: a pinned revision is required")
56
+ hub = importlib.import_module("huggingface_hub")
57
+ path: str = hub.snapshot_download(
58
+ repo_id=model_id, revision=revision, allow_patterns=list(allow_patterns)
59
+ )
60
+ return path
61
+
62
+
63
+ # Any: numpy is only present inside worker environments and has no stub here.
64
+ def load_mono_16k(path: str) -> Any:
65
+ """Read a 16 kHz mono file as float32 (the orchestrator guarantees this format).
66
+
67
+ Raises:
68
+ ValueError: if the file is not 16 kHz mono.
69
+ """
70
+ soundfile = importlib.import_module("soundfile")
71
+ audio, sample_rate_hz = soundfile.read(path, dtype="float32", always_2d=False)
72
+ if sample_rate_hz != 16000 or audio.ndim != 1:
73
+ raise ValueError(f"{path}: expected 16 kHz mono, got {sample_rate_hz} Hz, {audio.ndim}-D")
74
+ return audio
src/nida/bench/workers/quality_audiobox.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Meta Audiobox Aesthetics quality scores (CE, CU, PC, PQ) for B2.
2
+
3
+ The checkpoint (``checkpoint.pt``, CC-BY-4.0) is taken from the pinned
4
+ ``facebook/audiobox-aesthetics`` revision and passed explicitly, so the package never
5
+ fetches an unpinned "latest" model. The predictor resamples input to 16 kHz.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import importlib
11
+ from collections.abc import Mapping, Sequence
12
+ from typing import Any
13
+
14
+ from nida.bench.protocol import WorkRequest
15
+ from nida.bench.workers.common import snapshot
16
+ from nida.bench.workers.runner import Adapter, ModelRef, main_for
17
+
18
+ AXES = ("CE", "CU", "PC", "PQ")
19
+ CHECKPOINT_FILE = "checkpoint.pt"
20
+
21
+
22
+ class AudioboxAdapter(Adapter):
23
+ name = "quality_audiobox"
24
+ distributions = ("torch", "torchaudio", "audiobox-aesthetics")
25
+
26
+ def __init__(self) -> None:
27
+ self._predictor: Any = None # Any: audiobox predictor (untyped)
28
+
29
+ def load(self, model: ModelRef, device: str, batch_size: int) -> None:
30
+ infer = importlib.import_module("audiobox_aesthetics.infer")
31
+ local = snapshot(model.model_id, model.revision, (CHECKPOINT_FILE,))
32
+ self._predictor = infer.initialize_predictor(ckpt=f"{local}/{CHECKPOINT_FILE}")
33
+
34
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
35
+ scores = self._predictor.forward([{"path": r.audio_path} for r in batch])
36
+ outputs: list[Mapping[str, object]] = []
37
+ for score in scores:
38
+ missing = [axis for axis in AXES if axis not in score]
39
+ if missing:
40
+ raise ValueError(f"audiobox output lacks axes {missing}: {score}")
41
+ outputs.append({axis: float(score[axis]) for axis in AXES})
42
+ return outputs
43
+
44
+
45
+ if __name__ == "__main__":
46
+ raise SystemExit(main_for(AudioboxAdapter()))
src/nida/bench/workers/runner.py ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Generic worker loop shared by every model adapter.
2
+
3
+ The loop is model-agnostic: it reads the requests file, resumes from the results file,
4
+ calls ``Adapter.run_batch`` in fixed-size batches, and appends one fsynced result line per
5
+ request. A failing batch is retried item by item so a single bad clip cannot take its
6
+ neighbours down; every failure is recorded with its exception type and message (never
7
+ dropped) and counted in the summary. Model loading failures are fatal: they propagate and
8
+ the process exits non-zero.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import abc
14
+ import argparse
15
+ import hashlib
16
+ import importlib
17
+ import importlib.metadata
18
+ import json
19
+ import logging
20
+ import sys
21
+ import time
22
+ import traceback
23
+ from collections.abc import Mapping, Sequence
24
+ from dataclasses import dataclass, replace
25
+ from datetime import UTC, datetime
26
+ from pathlib import Path
27
+ from typing import ClassVar
28
+
29
+ from nida.bench.protocol import (
30
+ ProtocolError,
31
+ WorkerSummary,
32
+ WorkRequest,
33
+ WorkResult,
34
+ append_result,
35
+ drop_results,
36
+ load_requests,
37
+ load_results,
38
+ pending_requests,
39
+ read_summary,
40
+ write_summary,
41
+ )
42
+
43
+ __all__ = [
44
+ "MAX_CONSECUTIVE_FAILURES",
45
+ "Adapter",
46
+ "ModelRef",
47
+ "TooManyFailuresError",
48
+ "WorkerOptions",
49
+ "main_for",
50
+ "run_worker",
51
+ ]
52
+
53
+ log = logging.getLogger(__name__)
54
+
55
+ #: Abort when this many items in a row fail: the model/device is broken, not the clips.
56
+ MAX_CONSECUTIVE_FAILURES = 10
57
+ #: Progress is logged when this many more items are done (and at the end).
58
+ PROGRESS_EVERY = 200
59
+ #: When the *same* failure run (same clip ids) aborts more than this many times in a row,
60
+ #: it is evidently repeatable rather than transient and is kept as recorded failures so a
61
+ #: rerun can move past it. Unrelated aborts never accumulate (see ``_abort_count``).
62
+ MAX_ABORTS_BEFORE_KEEPING = 2
63
+
64
+
65
+ class TooManyFailuresError(RuntimeError):
66
+ """Consecutive item failures suggest a broken model or device (e.g. CUDA context)."""
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class ModelRef:
71
+ """Which model to load.
72
+
73
+ Attributes:
74
+ model_id: Hub id, model card name or local path, interpreted by the adapter.
75
+ revision: Pinned revision (commit SHA) or ``""`` when the source has none.
76
+ """
77
+
78
+ model_id: str
79
+ revision: str
80
+
81
+
82
+ class Adapter(abc.ABC):
83
+ """One model family. Subclasses keep heavy imports inside their methods."""
84
+
85
+ name: ClassVar[str]
86
+ #: Distribution names whose installed versions are recorded in the summary.
87
+ distributions: ClassVar[tuple[str, ...]] = ()
88
+
89
+ @abc.abstractmethod
90
+ def load(self, model: ModelRef, device: str, batch_size: int) -> None:
91
+ """Load the model onto ``device``. Exceptions here abort the worker."""
92
+
93
+ @abc.abstractmethod
94
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
95
+ """Return one JSON-compatible output mapping per request, in the same order."""
96
+
97
+ def peak_vram_gb(self) -> float | None:
98
+ """Peak CUDA memory allocated by this process, if torch with CUDA is present."""
99
+ try:
100
+ torch = importlib.import_module("torch")
101
+ except ImportError:
102
+ return None
103
+ if not torch.cuda.is_available():
104
+ return None
105
+ return round(float(torch.cuda.max_memory_allocated()) / 1024**3, 2)
106
+
107
+
108
+ @dataclass(frozen=True)
109
+ class WorkerOptions:
110
+ requests: Path
111
+ results: Path
112
+ summary: Path
113
+ model: ModelRef
114
+ device: str
115
+ batch_size: int
116
+ retry_failed: bool
117
+
118
+
119
+ def _now() -> str:
120
+ return datetime.now(UTC).isoformat(timespec="seconds")
121
+
122
+
123
+ def _describe(err: BaseException) -> str:
124
+ return f"{type(err).__name__}: {err}"
125
+
126
+
127
+ def _package_versions(names: Sequence[str]) -> dict[str, str]:
128
+ versions: dict[str, str] = {}
129
+ for name in names:
130
+ try:
131
+ versions[name] = importlib.metadata.version(name)
132
+ except importlib.metadata.PackageNotFoundError:
133
+ versions[name] = "not-installed"
134
+ return versions
135
+
136
+
137
+ def _abort_counter_path(results: Path) -> Path:
138
+ return results.with_name(results.name + ".aborts.json")
139
+
140
+
141
+ def _abort_count(results: Path, failure_ids: Sequence[str]) -> int:
142
+ """Record an abort and return how often *this* failure run has aborted in a row.
143
+
144
+ The counter stores a signature of the failing clip ids: a different failure run
145
+ restarts at 1, and :func:`_clear_abort_count` removes it after any run that finishes
146
+ without aborting, so transient incidents in different sessions never add up.
147
+ """
148
+ signature = hashlib.sha1("\n".join(sorted(failure_ids)).encode("utf-8")).hexdigest()
149
+ path = _abort_counter_path(results)
150
+ count = 1
151
+ if path.exists():
152
+ try:
153
+ previous = json.loads(path.read_text(encoding="utf-8"))
154
+ if previous.get("signature") == signature:
155
+ count = int(previous.get("count", 0)) + 1
156
+ except (ValueError, AttributeError, TypeError) as err:
157
+ # A damaged counter must not wedge the worker; recount from this abort.
158
+ log.warning("ignoring unreadable abort counter %s (%s); restarting at 1", path, err)
159
+ tmp = path.with_name(f".{path.name}.tmp")
160
+ tmp.write_text(json.dumps({"signature": signature, "count": count}), encoding="utf-8")
161
+ tmp.replace(path)
162
+ return count
163
+
164
+
165
+ def _clear_abort_count(results: Path) -> None:
166
+ _abort_counter_path(results).unlink(missing_ok=True)
167
+
168
+
169
+ def _drop_failed(path: Path) -> None:
170
+ """Rewrite the results file without failed entries (for ``--retry-failed``)."""
171
+ results = load_results(path, repair=True)
172
+ kept = [r.to_json() + "\n" for r in results.values() if r.ok]
173
+ tmp = path.with_name(f".{path.name}.tmp")
174
+ tmp.write_text("".join(kept), encoding="utf-8")
175
+ tmp.replace(path)
176
+
177
+
178
+ def _validated_outputs(batch: Sequence[WorkRequest], outputs: object) -> list[Mapping[str, object]]:
179
+ if not isinstance(outputs, list) or len(outputs) != len(batch):
180
+ got = len(outputs) if isinstance(outputs, list) else type(outputs).__name__
181
+ raise ProtocolError(f"adapter returned {got} outputs for a batch of {len(batch)}")
182
+ for output in outputs:
183
+ if not isinstance(output, Mapping):
184
+ raise ProtocolError(f"adapter output must be a mapping, got {type(output).__name__}")
185
+ return outputs
186
+
187
+
188
+ def _run_one_batch(adapter: Adapter, batch: Sequence[WorkRequest]) -> list[WorkResult]:
189
+ started = time.perf_counter()
190
+ try:
191
+ outputs = _validated_outputs(batch, adapter.run_batch(batch))
192
+ except Exception as err: # noqa: BLE001 - recorded per item below, never swallowed
193
+ if len(batch) == 1:
194
+ log.warning("item %s failed: %s", batch[0].clip_id, _describe(err))
195
+ return [
196
+ WorkResult(
197
+ clip_id=batch[0].clip_id,
198
+ output=None,
199
+ error=_describe(err),
200
+ infer_seconds=time.perf_counter() - started,
201
+ )
202
+ ]
203
+ log.warning("batch of %d failed (%s); retrying items one by one", len(batch), err)
204
+ results: list[WorkResult] = []
205
+ for request in batch:
206
+ results.extend(_run_one_batch(adapter, [request]))
207
+ return results
208
+ per_item = (time.perf_counter() - started) / len(batch)
209
+ return [
210
+ WorkResult(clip_id=r.clip_id, output=dict(o), error=None, infer_seconds=per_item)
211
+ for r, o in zip(batch, outputs, strict=True)
212
+ ]
213
+
214
+
215
+ def run_worker(adapter: Adapter, options: WorkerOptions) -> WorkerSummary:
216
+ """Process every pending request and write the summary. Returns the summary."""
217
+ if options.batch_size < 1:
218
+ raise ValueError(f"batch_size must be >= 1, got {options.batch_size}")
219
+ started_at = _now()
220
+ requests = load_requests(options.requests)
221
+ if options.retry_failed and options.results.exists():
222
+ _drop_failed(options.results)
223
+ done = load_results(options.results, repair=True)
224
+ pending = pending_requests(requests, done)
225
+ log.info(
226
+ "%s: %d requests, %d already done, %d pending",
227
+ adapter.name,
228
+ len(requests),
229
+ len(done),
230
+ len(pending),
231
+ )
232
+
233
+ previous = read_summary(options.summary) if options.summary.exists() else None
234
+ load_seconds = 0.0
235
+ if pending:
236
+ load_start = time.perf_counter()
237
+ adapter.load(options.model, options.device, options.batch_size)
238
+ load_seconds = time.perf_counter() - load_start
239
+ options.results.parent.mkdir(parents=True, exist_ok=True)
240
+ failure_run: list[WorkResult] = []
241
+ aborted = False
242
+ with options.results.open("a", encoding="utf-8") as handle:
243
+ for start in range(0, len(pending), options.batch_size):
244
+ batch = pending[start : start + options.batch_size]
245
+ for result in _run_one_batch(adapter, batch):
246
+ append_result(handle, result)
247
+ failure_run = [] if result.ok else [*failure_run, result]
248
+ if len(failure_run) >= MAX_CONSECUTIVE_FAILURES:
249
+ aborted = True
250
+ break
251
+ if aborted:
252
+ break
253
+ finished = start + len(batch)
254
+ if (
255
+ finished == len(pending)
256
+ or finished // PROGRESS_EVERY != start // PROGRESS_EVERY
257
+ ):
258
+ log.info("%s: %d/%d", adapter.name, finished, len(pending))
259
+ if aborted:
260
+ aborts = _abort_count(options.results, [r.clip_id for r in failure_run])
261
+ if aborts <= MAX_ABORTS_BEFORE_KEEPING:
262
+ # Most likely infrastructure (e.g. a broken CUDA context), not bad clips: make
263
+ # them pending again so a plain rerun retries them instead of scoring them as
264
+ # empty hypotheses.
265
+ drop_results(options.results, [r.clip_id for r in failure_run])
266
+ outcome = "results removed, they will be retried"
267
+ else:
268
+ outcome = (
269
+ f"same failure run aborted {aborts} times: kept as recorded failures so "
270
+ "the next run moves on; use --retry-failed to try them again"
271
+ )
272
+ raise TooManyFailuresError(
273
+ f"{adapter.name}: {len(failure_run)} consecutive failures ({outcome}), "
274
+ f"last: {failure_run[-1].error}"
275
+ )
276
+
277
+ _clear_abort_count(options.results) # finished without aborting
278
+ final = load_results(options.results)
279
+ n_ok = sum(1 for r in final.values() if r.ok)
280
+ if not pending and previous is not None:
281
+ # No-op resume: keep the measurements of the invocation that did the work.
282
+ summary = replace(previous, n_ok=n_ok, n_error=len(final) - n_ok)
283
+ else:
284
+ peaks = [v for v in (adapter.peak_vram_gb(), previous and previous.peak_vram_gb) if v]
285
+ invocations = int(previous.extra.get("invocations", "1")) + 1 if previous else 1
286
+ summary = WorkerSummary(
287
+ worker=adapter.name,
288
+ model_id=options.model.model_id,
289
+ revision=options.model.revision,
290
+ load_seconds=round(load_seconds, 3),
291
+ peak_vram_gb=max(peaks) if peaks else None,
292
+ n_ok=n_ok,
293
+ n_error=len(final) - n_ok,
294
+ started_at=previous.started_at if previous else started_at,
295
+ ended_at=_now(),
296
+ packages=_package_versions(adapter.distributions),
297
+ extra={
298
+ "python": sys.version.split()[0],
299
+ "device": options.device,
300
+ "invocations": str(invocations),
301
+ },
302
+ )
303
+ write_summary(options.summary, summary)
304
+ return summary
305
+
306
+
307
+ def _parse(argv: Sequence[str] | None) -> WorkerOptions:
308
+ parser = argparse.ArgumentParser(description="Run a Nida benchmark worker.")
309
+ parser.add_argument("--requests", type=Path, required=True)
310
+ parser.add_argument("--results", type=Path, required=True)
311
+ parser.add_argument("--summary", type=Path, default=None)
312
+ parser.add_argument("--model-id", required=True)
313
+ parser.add_argument("--revision", default="")
314
+ parser.add_argument("--device", default="cuda")
315
+ parser.add_argument("--batch-size", type=int, default=8)
316
+ parser.add_argument("--retry-failed", action="store_true")
317
+ args = parser.parse_args(argv)
318
+ summary = args.summary or args.results.with_name(args.results.stem + ".summary.json")
319
+ return WorkerOptions(
320
+ requests=args.requests,
321
+ results=args.results,
322
+ summary=summary,
323
+ model=ModelRef(model_id=args.model_id, revision=args.revision),
324
+ device=args.device,
325
+ batch_size=args.batch_size,
326
+ retry_failed=args.retry_failed,
327
+ )
328
+
329
+
330
+ def main_for(adapter: Adapter, argv: Sequence[str] | None = None) -> int:
331
+ """CLI entry point used by each worker module's ``__main__`` block."""
332
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
333
+ options = _parse(argv)
334
+ try:
335
+ summary = run_worker(adapter, options)
336
+ except (ProtocolError, OSError, ValueError, TooManyFailuresError):
337
+ traceback.print_exc()
338
+ return 2
339
+ print(f"{adapter.name}: ok={summary.n_ok} error={summary.n_error}")
340
+ return 0
src/nida/bench/workers/vae_voxcpm.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """VoxCPM2 AudioVAE V2 encode → decode round trip (benchmark B2).
2
+
3
+ Only the VAE is used. Its model code is a single file in the Apache-2.0 VoxCPM repository
4
+ that needs just torch, numpy and pydantic, so it is fetched at a pinned commit and
5
+ verified by SHA-256 instead of installing the ``voxcpm`` package (whose dependencies
6
+ include gradio, funasr and modelscope). Weights and config come from the pinned
7
+ ``openbmb/VoxCPM2`` Hugging Face revision.
8
+
9
+ Per request: read audio (mono, any rate) → resample to the VAE input rate (16 kHz, as
10
+ VoxCPM does with ``librosa.load(sr=16000)``) → ``encode`` (posterior mean, deterministic)
11
+ → ``decode`` at the default output rate (48 kHz) → trim to ``ceil(n_16k * 48000/16000)``
12
+ samples → write ``params["out_path"]`` as 32-bit float WAV.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import hashlib
18
+ import importlib
19
+ import importlib.util
20
+ import json
21
+ import math
22
+ import urllib.request
23
+ from collections.abc import Mapping, Sequence
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+ from nida.bench.protocol import WorkRequest
28
+ from nida.bench.workers.common import snapshot
29
+ from nida.bench.workers.runner import Adapter, ModelRef, main_for
30
+
31
+ VOXCPM_COMMIT = "9d95dd132ad4c54b42ed50561ff878b18e4c75de"
32
+ VAE_SOURCE_URL = (
33
+ "https://raw.githubusercontent.com/OpenBMB/VoxCPM/"
34
+ f"{VOXCPM_COMMIT}/src/voxcpm/modules/audiovae/audio_vae_v2.py"
35
+ )
36
+ VAE_SOURCE_SHA256 = "2efdff1708d8ec1471624aae6f232d0f933de26b788b6901c434246847e2d3a8"
37
+ WEIGHTS_FILE = "audiovae.pth"
38
+ CONFIG_FILE = "config.json"
39
+
40
+
41
+ def _sha256(data: bytes) -> str:
42
+ return hashlib.sha256(data).hexdigest()
43
+
44
+
45
+ def fetch_vae_source(cache_dir: Path) -> Path:
46
+ """Download the pinned ``audio_vae_v2.py`` once and verify its SHA-256.
47
+
48
+ Raises:
49
+ ValueError: if the downloaded or cached file does not match the pinned hash.
50
+ """
51
+ path = cache_dir / f"audio_vae_v2-{VOXCPM_COMMIT[:12]}.py"
52
+ if not path.exists():
53
+ with urllib.request.urlopen(VAE_SOURCE_URL, timeout=60) as response:
54
+ data = response.read()
55
+ if _sha256(data) != VAE_SOURCE_SHA256:
56
+ raise ValueError(f"audio_vae_v2.py hash mismatch: {_sha256(data)}")
57
+ cache_dir.mkdir(parents=True, exist_ok=True)
58
+ tmp = path.with_suffix(".tmp")
59
+ tmp.write_bytes(data)
60
+ tmp.replace(path)
61
+ if _sha256(path.read_bytes()) != VAE_SOURCE_SHA256:
62
+ raise ValueError(f"{path}: cached file does not match the pinned hash")
63
+ return path
64
+
65
+
66
+ def _import_file(path: Path, module_name: str) -> Any: # Any: dynamically loaded module
67
+ spec = importlib.util.spec_from_file_location(module_name, path)
68
+ if spec is None or spec.loader is None:
69
+ raise ImportError(f"cannot import {path}")
70
+ module = importlib.util.module_from_spec(spec)
71
+ spec.loader.exec_module(module)
72
+ return module
73
+
74
+
75
+ class VoxCPMVAEAdapter(Adapter):
76
+ name = "vae_voxcpm"
77
+ distributions = ("torch", "torchaudio", "numpy", "pydantic", "soundfile", "huggingface-hub")
78
+
79
+ def __init__(self, cache_dir: Path | None = None) -> None:
80
+ self._cache_dir = cache_dir or Path.home() / ".cache" / "nida"
81
+ self._vae: Any = None # Any: AudioVAE V2 torch module
82
+ self._torch: Any = None
83
+ self._device = "cpu"
84
+
85
+ def load(self, model: ModelRef, device: str, batch_size: int) -> None:
86
+ torch = importlib.import_module("torch")
87
+ source = _import_file(fetch_vae_source(self._cache_dir), "nida_audio_vae_v2")
88
+ local = Path(snapshot(model.model_id, model.revision, (CONFIG_FILE, WEIGHTS_FILE)))
89
+ config = json.loads((local / CONFIG_FILE).read_text(encoding="utf-8"))
90
+ vae = source.AudioVAE(config=source.AudioVAEConfig(**config["audio_vae_config"]))
91
+ checkpoint = torch.load(local / WEIGHTS_FILE, map_location="cpu", weights_only=True)
92
+ state = checkpoint.get("state_dict", checkpoint)
93
+ incompatible = vae.load_state_dict(state, strict=False)
94
+ if incompatible.missing_keys or incompatible.unexpected_keys:
95
+ raise ValueError(
96
+ f"state dict mismatch: missing={incompatible.missing_keys[:5]} "
97
+ f"unexpected={incompatible.unexpected_keys[:5]}"
98
+ )
99
+ self._vae = vae.to(device=device, dtype=torch.float32).eval()
100
+ self._torch = torch
101
+ self._device = device
102
+
103
+ def _round_trip(self, request: WorkRequest) -> Mapping[str, object]:
104
+ out_path = request.params.get("out_path")
105
+ if not out_path:
106
+ raise ValueError(f"{request.clip_id}: missing params.out_path")
107
+ torch = self._torch
108
+ soundfile = importlib.import_module("soundfile")
109
+ functional = importlib.import_module("torchaudio.functional")
110
+ audio, source_rate_hz = soundfile.read(request.audio_path, dtype="float32")
111
+ if audio.ndim != 1:
112
+ raise ValueError(f"{request.clip_id}: expected mono audio, got shape {audio.shape}")
113
+ in_rate_hz = int(self._vae.sample_rate)
114
+ out_rate_hz = int(self._vae.out_sample_rate)
115
+ wave = torch.from_numpy(audio).unsqueeze(0)
116
+ if source_rate_hz != in_rate_hz:
117
+ wave = functional.resample(wave, source_rate_hz, in_rate_hz)
118
+ num_in_samples = int(wave.shape[-1])
119
+ with torch.inference_mode():
120
+ latents = self._vae.encode(wave.to(self._device), in_rate_hz) # (1, D, frames)
121
+ decoded = self._vae.decode(latents) # (1, 1, samples) at out_rate_hz
122
+ num_out_samples = math.ceil(num_in_samples * out_rate_hz / in_rate_hz)
123
+ out = decoded.reshape(-1)[:num_out_samples].float().cpu().numpy()
124
+ if out.shape[0] < num_out_samples:
125
+ raise ValueError(f"{request.clip_id}: decoder returned {out.shape[0]} samples")
126
+ target = Path(out_path)
127
+ target.parent.mkdir(parents=True, exist_ok=True)
128
+ tmp = target.with_name(f".{target.name}.tmp.wav") # never leave a partial output
129
+ soundfile.write(str(tmp), out, out_rate_hz, subtype="FLOAT")
130
+ tmp.replace(target)
131
+ return {
132
+ "out_path": out_path,
133
+ "out_sample_rate_hz": out_rate_hz,
134
+ "num_latent_frames": int(latents.shape[-1]),
135
+ "latent_dim": int(latents.shape[1]),
136
+ "latent_mean": float(latents.mean()),
137
+ "latent_std": float(latents.std()),
138
+ }
139
+
140
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
141
+ # Clips have different lengths and the VAE is cheap: process one at a time.
142
+ return [self._round_trip(r) for r in batch]
143
+
144
+
145
+ if __name__ == "__main__":
146
+ raise SystemExit(main_for(VoxCPMVAEAdapter()))
src/nida/cli.py ADDED
@@ -0,0 +1,929 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Nida command-line interface.
2
+
3
+ Every subcommand accepts the common run flags (``--runs-root``, ``--run-id``,
4
+ ``--git-commit``) so the Colab runner can call any command the same way.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ import logging
12
+ import sys
13
+ from collections.abc import Sequence
14
+ from pathlib import Path
15
+
16
+ from nida.envreport import collect_env, summarize_env
17
+ from nida.runs import RunContext
18
+ from nida.utils.io import atomic_write_json, atomic_write_text
19
+
20
+ __all__ = ["main"]
21
+
22
+ # src/nida/cli.py → repository root (editable installs, as on Colab).
23
+ _REPO_DIR = Path(__file__).resolve().parents[2]
24
+
25
+
26
+ def _common_parser() -> argparse.ArgumentParser:
27
+ common = argparse.ArgumentParser(add_help=False)
28
+ common.add_argument(
29
+ "--runs-root",
30
+ type=Path,
31
+ default=None,
32
+ help="Directory holding run folders. If omitted, nothing is written to disk.",
33
+ )
34
+ common.add_argument("--run-id", default=None, help="Explicit run id (default: generated).")
35
+ common.add_argument("--git-commit", default=None, help="Commit SHA of the code being run.")
36
+ return common
37
+
38
+
39
+ def _positive_int(text: str) -> int:
40
+ """argparse type: an integer >= 1 (clear error otherwise)."""
41
+ try:
42
+ value = int(text)
43
+ except ValueError:
44
+ raise argparse.ArgumentTypeError(f"expected an integer >= 1, got {text!r}") from None
45
+ if value < 1:
46
+ raise argparse.ArgumentTypeError(f"expected an integer >= 1, got {value}")
47
+ return value
48
+
49
+
50
+ def _build_parser() -> argparse.ArgumentParser:
51
+ parser = argparse.ArgumentParser(prog="nida", description="Nida expressive Turkish TTS.")
52
+ sub = parser.add_subparsers(dest="command", required=True)
53
+ sub.add_parser(
54
+ "env-report",
55
+ parents=[_common_parser()],
56
+ help="Print (and optionally record) Python/torch/GPU facts.",
57
+ )
58
+ bench = sub.add_parser(
59
+ "bench-phase0",
60
+ parents=[_common_parser()],
61
+ help="Phase 0 benchmarks B1 (ASR) and B2 (VAE); see docs/benchmarks/B1-B2-tasarim.md.",
62
+ )
63
+ bench.add_argument("--stage", default="all", help="A stage name or 'all'.")
64
+ bench.add_argument(
65
+ "--workspace",
66
+ type=Path,
67
+ default=None,
68
+ help="Durable benchmark directory (default: <runs-root>/../bench/phase0).",
69
+ )
70
+ bench.add_argument("--cache", type=Path, default=Path("/content/nida_cache"))
71
+ bench.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
72
+ bench.add_argument(
73
+ "--repo", type=Path, default=_REPO_DIR, help="Repository root (for configs and envs)."
74
+ )
75
+ bench.add_argument("--c-results", type=Path, default=None, help="B1 layer-C panel results CSV.")
76
+ bench.add_argument(
77
+ "--b2-panel-results", type=Path, default=None, help="B2 listening panel results CSV."
78
+ )
79
+ bench.add_argument(
80
+ "--retry-failed", action="store_true", help="Retry worker items that failed before."
81
+ )
82
+ pilot = sub.add_parser(
83
+ "data-pilot",
84
+ parents=[_common_parser()],
85
+ help="Phase 1 data pipeline pilot on YODAS (docs/plans/faz-1.md).",
86
+ )
87
+ pilot.add_argument("--stage", default="all", help="A stage name or 'all'.")
88
+ pilot.add_argument(
89
+ "--root",
90
+ type=Path,
91
+ default=None,
92
+ help="Durable pilot directory (default: <runs-root>/../data/pilot).",
93
+ )
94
+ pilot.add_argument("--cache", type=Path, default=Path("/content/nida_cache/pilot"))
95
+ pilot.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
96
+ pilot.add_argument("--repo", type=Path, default=_REPO_DIR)
97
+ pilot.add_argument(
98
+ "--subset",
99
+ choices=("tr000", "tr100"), # keep in sync with nida.data.pilot.YODAS_SUBSETS
100
+ default="tr000",
101
+ help=(
102
+ "YODAS Turkish subset: tr000 (manual subtitles, 64 shards) or tr100 (automatic "
103
+ "subtitles, 430 shards; videos that occur in tr000 are dropped)."
104
+ ),
105
+ )
106
+ pilot.add_argument(
107
+ "--shards", default="00000002", help="Comma-separated shards of the chosen subset."
108
+ )
109
+ pilot.add_argument("--max-hours", type=float, default=10.0)
110
+ pilot.add_argument("--asr-primary", default="omniasr_llm_7b")
111
+ pilot.add_argument("--asr-secondary", default="qwen3_asr_1_7b")
112
+ pilot.add_argument("--retry-failed", action="store_true")
113
+ pilot.add_argument(
114
+ "--asr-batch-size",
115
+ type=int,
116
+ default=None,
117
+ help="Override the ASR workers' batch size (bench sizes fit a 22 GB L4).",
118
+ )
119
+ pilot.add_argument(
120
+ "--parallel-workers",
121
+ type=_positive_int,
122
+ default=1,
123
+ help=(
124
+ "Run up to N of the independent GPU worker stages (sed, diar, both ASR systems, "
125
+ "ctc, emotion) at the same time in 'all' (default 1 = one after another)."
126
+ ),
127
+ )
128
+ ears = sub.add_parser(
129
+ "data-ears",
130
+ parents=[_common_parser()],
131
+ help="EARS -> latent dataset (ADR-0007; docs/03-data-pipeline.md).",
132
+ )
133
+ ears.add_argument(
134
+ "--root",
135
+ type=Path,
136
+ default=None,
137
+ help="Durable directory (default: <runs-root>/../data/ears); dataset is <root>/dataset.",
138
+ )
139
+ ears.add_argument(
140
+ "--speakers", default="all", help="Comma-separated speakers (p001,p002) or 'all'."
141
+ )
142
+ ears.add_argument("--cache", type=Path, default=Path("/content/nida_cache/ears"))
143
+ ears.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
144
+ ears.add_argument("--repo", type=Path, default=_REPO_DIR)
145
+ ears.add_argument("--seed", type=int, default=20261009, help="Speaker split seed.")
146
+ ears.add_argument("--config-dir", type=Path, default=None, help="Default: <repo>/configs/data.")
147
+ imp = sub.add_parser(
148
+ "data-import",
149
+ parents=[_common_parser()],
150
+ help="External transcribed corpus -> latent dataset (nida.data.importer).",
151
+ )
152
+ imp.add_argument(
153
+ "--source",
154
+ # keep in sync with nida.data.importer.sources.SOURCES
155
+ choices=("alania", "tr_full", "wavdir", "issai_tsc", "mediaspeech_tr", "common_voice_tr"),
156
+ required=True,
157
+ help=(
158
+ "alania (HF parquet), tr_full (HF data.jsonl + wavs), wavdir (--source-dir), "
159
+ "issai_tsc (HF tar.gz), mediaspeech_tr (OpenSLR TR.tgz) or common_voice_tr "
160
+ "(--source-dir: a Common Voice release folder or tarball you downloaded)."
161
+ ),
162
+ )
163
+ imp.add_argument(
164
+ "--stage",
165
+ default="all",
166
+ help="select | audio | latents | ctc | timings | dataset | report | all.",
167
+ )
168
+ imp.add_argument(
169
+ "--root",
170
+ type=Path,
171
+ default=None,
172
+ help="Durable directory (default: <runs-root>/../data/import-<source>).",
173
+ )
174
+ imp.add_argument(
175
+ "--cache",
176
+ type=Path,
177
+ default=None,
178
+ help="Local scratch (default: /content/nida_cache/import/<root name>).",
179
+ )
180
+ imp.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
181
+ imp.add_argument("--repo", type=Path, default=_REPO_DIR)
182
+ imp.add_argument("--config-dir", type=Path, default=None, help="Default: <repo>/configs/data.")
183
+ imp.add_argument("--max-hours", type=float, default=10.0, help="Stop selecting at this many h.")
184
+ imp.add_argument(
185
+ "--shard-offset",
186
+ type=int,
187
+ default=0,
188
+ help="First work unit (parquet shard / row chunk); roots take disjoint ranges.",
189
+ )
190
+ imp.add_argument(
191
+ "--shards", type=_positive_int, default=None, help="Number of work units (default: rest)."
192
+ )
193
+ imp.add_argument(
194
+ "--source-dir",
195
+ type=Path,
196
+ default=None,
197
+ help="wavdir: folder with manifest.jsonl; common_voice_tr: release folder or tarball.",
198
+ )
199
+ imp.add_argument(
200
+ "--revision",
201
+ default=None,
202
+ help=(
203
+ "Hub commit sha to pin (default: current head, recorded); mediaspeech_tr: sha256 of "
204
+ "TR.tgz (default: the one in configs/data/import_corpora.yaml)."
205
+ ),
206
+ )
207
+ imp.add_argument("--retry-failed", action="store_true")
208
+ imp.add_argument(
209
+ "--audio-threads",
210
+ type=_positive_int,
211
+ default=4,
212
+ help="Threads converting raw audio to 16 kHz wavs (does not change the outputs).",
213
+ )
214
+ train = sub.add_parser(
215
+ "train",
216
+ parents=[_common_parser()],
217
+ help="Phase 2 flow-matching training on a latent dataset (docs/04-architecture.md).",
218
+ )
219
+ train.add_argument("--config", type=Path, required=True, help="YAML under configs/train/.")
220
+ train.add_argument(
221
+ "--data",
222
+ type=Path,
223
+ required=True,
224
+ action="append",
225
+ help="Latent dataset directory; repeat the flag to train on the union of several.",
226
+ )
227
+ train.add_argument(
228
+ "--out", type=Path, required=True, help="Training workspace (checkpoints, metrics.jsonl)."
229
+ )
230
+ train.add_argument(
231
+ "--resume", action="store_true", help="Continue from the newest checkpoint in --out."
232
+ )
233
+ train.add_argument("--device", default=None, help="cpu | cuda | auto (default: auto).")
234
+ train.add_argument(
235
+ "--init-from",
236
+ type=Path,
237
+ default=None,
238
+ help="Fine-tune: start from this checkpoint's weights and latent stats (same model "
239
+ "config, except dur_cond none -> emotion) when --out has no checkpoint yet; "
240
+ "optimizer and schedule start fresh.",
241
+ )
242
+ samples = sub.add_parser(
243
+ "synth-samples",
244
+ parents=[_common_parser()],
245
+ help="Listening page: validation items re-spoken by a trained checkpoint.",
246
+ )
247
+ samples.add_argument(
248
+ "--checkpoint", type=Path, required=True, help="ckpt-*.pt or a training workspace."
249
+ )
250
+ samples.add_argument("--data", type=Path, required=True, help="Latent dataset directory.")
251
+ samples.add_argument("--out", type=Path, required=True, help="Output directory.")
252
+ samples.add_argument("--n", type=int, default=12, help="Validation items to generate.")
253
+ samples.add_argument("--steps", type=int, default=32, help="ODE steps.")
254
+ samples.add_argument("--w-text", type=float, default=2.0)
255
+ # Keeps 1.0 on purpose (sample CER stays comparable with earlier runs); `say` uses 2.0.
256
+ samples.add_argument("--w-emo", type=float, default=1.0)
257
+ samples.add_argument("--seed", type=int, default=0)
258
+ samples.add_argument("--method", choices=["euler", "midpoint"], default="euler")
259
+ samples.add_argument(
260
+ "--sway", type=float, default=-1.0, help="Sway time-step warp (0 = uniform steps)."
261
+ )
262
+ samples.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
263
+ samples.add_argument("--repo", type=Path, default=_REPO_DIR)
264
+ samples.add_argument("--device", default=None, help="cpu | cuda | auto (default: auto).")
265
+ samples.add_argument(
266
+ "--no-asr", action="store_true", help="Skip the Qwen3-ASR read-back (CER) check."
267
+ )
268
+ say = sub.add_parser(
269
+ "say",
270
+ parents=[_common_parser()],
271
+ help="Speak free text after a validation speaker prompt (aligned checkpoints).",
272
+ )
273
+ say.add_argument("--checkpoint", type=Path, required=True, help="ckpt-*.pt or a workspace.")
274
+ say.add_argument(
275
+ "--data",
276
+ type=Path,
277
+ default=None,
278
+ help="Latent dataset (prompt source); not needed with only --prompt-list prompts.",
279
+ )
280
+ say.add_argument("--out", type=Path, required=True, help="Output directory.")
281
+ text = say.add_mutually_exclusive_group(required=True)
282
+ text.add_argument(
283
+ "--text",
284
+ help=(
285
+ "Text to speak (sentences split on . ! ? …; long ones also at , ; :). Numbers, "
286
+ "%%, TL, clock times, abbreviations and acronyms need not be written out: they are "
287
+ "normalised to spoken Turkish first (--no-normalize disables this). "
288
+ "'@mutlu:0.8' sets the emotion of the text after it; '[duraksama:0.3]' inserts "
289
+ "exactly 0.3 s of silence (it ends the sentence); '[duraksama]' lets the model pause."
290
+ ),
291
+ )
292
+ text.add_argument("--text-file", type=Path, help="UTF-8 file with the text to speak.")
293
+ say.add_argument(
294
+ "--prompt-id", action="append", default=None, help="Validation segment (repeatable)."
295
+ )
296
+ say.add_argument(
297
+ "--n-prompts", type=int, default=3, help="Prompts to pick when no --prompt-id is given."
298
+ )
299
+ say.add_argument(
300
+ "--prompt-list",
301
+ type=Path,
302
+ default=None,
303
+ help="TSV '<wav>\t<transcript>' of prompts from outside the datasets.",
304
+ )
305
+ say.add_argument(
306
+ "--rank-prompts",
307
+ type=int,
308
+ default=0,
309
+ help="Decode this many prompt candidates and keep the cleanest --n-prompts (0 = off).",
310
+ )
311
+ say.add_argument("--steps", type=int, default=32, help="ODE steps.")
312
+ say.add_argument("--w-text", type=float, default=3.0)
313
+ say.add_argument("--w-emo", type=float, default=2.0)
314
+ say.add_argument("--seed", type=int, default=0)
315
+ say.add_argument("--method", choices=["euler", "midpoint"], default="euler")
316
+ say.add_argument("--sway", type=float, default=-1.0)
317
+ say.add_argument("--speed", type=float, default=1.0, help=">1 faster, <1 slower.")
318
+ say.add_argument("--envs", type=Path, default=Path("/content/nida_envs"))
319
+ say.add_argument("--repo", type=Path, default=_REPO_DIR)
320
+ say.add_argument("--device", default=None, help="cpu | cuda | auto (default: auto).")
321
+ say.add_argument("--no-asr", action="store_true", help="Skip the ASR read-back check.")
322
+ say.add_argument(
323
+ "--no-normalize",
324
+ action="store_true",
325
+ help=(
326
+ "Use the text as written: no expansion of digits, symbols, abbreviations or "
327
+ "acronyms (they must then be spelled out, or the text is rejected)."
328
+ ),
329
+ )
330
+ say.add_argument(
331
+ "--strict-normalize",
332
+ action="store_true",
333
+ help=(
334
+ "Stop on every text-normalisation issue (a guessed acronym, an ambiguous time or "
335
+ "ordinal, ...); by default only issues that would be misread stop the run."
336
+ ),
337
+ )
338
+ say.add_argument(
339
+ "--single-env",
340
+ action="store_true",
341
+ help=(
342
+ "Run every worker (VAE, CTC, ASR) with this interpreter instead of building one "
343
+ "venv per worker (a local install with all packages, e.g. a release user)."
344
+ ),
345
+ )
346
+ queue = sub.add_parser(
347
+ "queue",
348
+ parents=[_common_parser()],
349
+ help="Run nida.cli commands listed in a file, one after another (nida.jobqueue).",
350
+ )
351
+ queue.add_argument("--file", type=Path, required=True, help="Queue file (one job a line).")
352
+ queue.add_argument(
353
+ "--wait-minutes",
354
+ type=float,
355
+ default=0.0,
356
+ help="Keep polling the file this long after the last job before returning.",
357
+ )
358
+ queue.add_argument(
359
+ "--bundle-dir", type=Path, default=None, help="Default: <runs-root>/../code."
360
+ )
361
+ queue.add_argument("--log-dir", type=Path, default=Path("/content/nida_logs"))
362
+ queue.add_argument("--repo", type=Path, default=_REPO_DIR)
363
+ return parser
364
+
365
+
366
+ def _env_report_md(metrics: dict[str, object]) -> str:
367
+ def show(value: object) -> str:
368
+ return "—" if value is None else str(value)
369
+
370
+ lines = ["# env-report", ""]
371
+ if metrics["gpu_name"] is None:
372
+ lines.append("- device: CPU only")
373
+ else:
374
+ lines.append(f"- device: {metrics['gpu_name']} (count {metrics['gpu_count']})")
375
+ for key in (
376
+ "gpu_memory_gb",
377
+ "compute_capability",
378
+ "bf16_supported",
379
+ "torch_version",
380
+ "cuda_version",
381
+ "torchaudio_version",
382
+ "python",
383
+ "cpu_count",
384
+ "ram_total_gb",
385
+ "disk_free_gb",
386
+ ):
387
+ lines.append(f"- {key}: {show(metrics[key])}")
388
+ return "\n".join(lines) + "\n"
389
+
390
+
391
+ def _resolve_device(choice: str | None, cuda_available: bool) -> str:
392
+ """``--device``: ``None`` or ``auto`` → cuda when available, else cpu; other values as is."""
393
+ if choice in (None, "auto"):
394
+ return "cuda" if cuda_available else "cpu"
395
+ return str(choice)
396
+
397
+
398
+ def _cmd_env_report(args: argparse.Namespace) -> int:
399
+ env = collect_env()
400
+ print(json.dumps(env, ensure_ascii=False, indent=2, sort_keys=True))
401
+ if args.runs_root is None:
402
+ return 0
403
+ with RunContext(
404
+ args.runs_root, "env-report", run_id=args.run_id, git_commit=args.git_commit
405
+ ) as run:
406
+ atomic_write_json(run.dir / "env.json", env)
407
+ metrics = summarize_env(env)
408
+ atomic_write_json(run.dir / "metrics.json", metrics)
409
+ atomic_write_text(run.dir / "report.md", _env_report_md(metrics))
410
+ print(f"run_id={run.run_id}")
411
+ return 0
412
+
413
+
414
+ def _cmd_bench_phase0(args: argparse.Namespace) -> int:
415
+ from nida.bench.phase0 import Phase0
416
+ from nida.bench.prepare import Workspace
417
+
418
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
419
+ workspace_root = args.workspace
420
+ if workspace_root is None:
421
+ if args.runs_root is None:
422
+ raise SystemExit("bench-phase0 needs --workspace or --runs-root")
423
+ workspace_root = args.runs_root.parent / "bench" / "phase0"
424
+ phase0 = Phase0(
425
+ workspace=Workspace(root=workspace_root, cache=args.cache),
426
+ envs_root=args.envs,
427
+ repo_dir=args.repo,
428
+ sources_path=args.repo / "configs" / "sources.yaml",
429
+ retry_failed=args.retry_failed,
430
+ )
431
+ if args.runs_root is None:
432
+ result = phase0.run(
433
+ args.stage, c_results=args.c_results, b2_panel_results=args.b2_panel_results
434
+ )
435
+ print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
436
+ return 0
437
+ with RunContext(
438
+ args.runs_root, f"bench-phase0-{args.stage}", run_id=args.run_id, git_commit=args.git_commit
439
+ ) as run:
440
+ result = phase0.run(
441
+ args.stage, c_results=args.c_results, b2_panel_results=args.b2_panel_results
442
+ )
443
+ atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
444
+ reports = [
445
+ (workspace_root / "reports" / name)
446
+ for name in ("b1_report.md", "b2_report.md", "b2_panel_report.md")
447
+ if (workspace_root / "reports" / name).exists()
448
+ ]
449
+ body = "\n\n".join(r.read_text(encoding="utf-8") for r in reports)
450
+ atomic_write_text(
451
+ run.dir / "report.md",
452
+ f"# bench-phase0 ({args.stage})\n\nworkspace: {workspace_root}\n\n{body}\n",
453
+ )
454
+ print(f"run_id={run.run_id}")
455
+ return 0
456
+
457
+
458
+ def _cmd_data_pilot(args: argparse.Namespace) -> int:
459
+ from nida.data.pilot import Pilot, PilotPlan
460
+
461
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
462
+ root = args.root
463
+ if root is None:
464
+ if args.runs_root is None:
465
+ raise SystemExit("data-pilot needs --root or --runs-root")
466
+ root = args.runs_root.parent / "data" / "pilot"
467
+ pilot = Pilot(
468
+ root=root,
469
+ cache=args.cache,
470
+ envs_root=args.envs,
471
+ repo_dir=args.repo,
472
+ sources_path=args.repo / "configs" / "sources.yaml",
473
+ config_dir=args.repo / "configs" / "data",
474
+ plan=PilotPlan(
475
+ subset=args.subset,
476
+ shards=args.shards,
477
+ max_hours=args.max_hours,
478
+ asr_primary=args.asr_primary,
479
+ asr_secondary=args.asr_secondary,
480
+ ),
481
+ retry_failed=args.retry_failed,
482
+ asr_batch_size=args.asr_batch_size,
483
+ parallel_workers=args.parallel_workers,
484
+ )
485
+ if args.runs_root is None:
486
+ print(json.dumps(pilot.run(args.stage), ensure_ascii=False, indent=2, default=str))
487
+ return 0
488
+ with RunContext(
489
+ args.runs_root, f"data-pilot-{args.stage}", run_id=args.run_id, git_commit=args.git_commit
490
+ ) as run:
491
+ result = pilot.run(args.stage)
492
+ atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
493
+ report = root / "reports" / "pilot.md"
494
+ body = report.read_text(encoding="utf-8") if report.exists() else ""
495
+ result_json = json.dumps(result, ensure_ascii=False, indent=2, default=str)
496
+ lines = [
497
+ f"# data-pilot ({args.stage})",
498
+ "",
499
+ f"root: {root}",
500
+ "",
501
+ "```json",
502
+ result_json,
503
+ "```",
504
+ "",
505
+ body,
506
+ ]
507
+ atomic_write_text(run.dir / "report.md", "\n".join(lines) + "\n")
508
+ print(f"run_id={run.run_id}")
509
+ return 0
510
+
511
+
512
+ def _cmd_data_ears(args: argparse.Namespace) -> int:
513
+ from nida.bench import fetch
514
+ from nida.bench.b1 import _stream_runner
515
+ from nida.data.ears import Ears, load_settings, parse_speakers
516
+
517
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
518
+ root = args.root
519
+ if root is None:
520
+ if args.runs_root is None:
521
+ raise SystemExit("data-ears needs --root or --runs-root")
522
+ root = args.runs_root.parent / "data" / "ears"
523
+ config_dir = args.config_dir or args.repo / "configs" / "data"
524
+ speakers = parse_speakers(args.speakers, load_settings(config_dir).ears.n_speakers)
525
+ ears = Ears(
526
+ root=root,
527
+ cache=args.cache,
528
+ envs_root=args.envs,
529
+ repo_dir=args.repo,
530
+ sources_path=args.repo / "configs" / "sources.yaml",
531
+ config_dir=config_dir,
532
+ seed=args.seed,
533
+ runner=_stream_runner,
534
+ opener=fetch.default_opener,
535
+ )
536
+ if args.runs_root is None:
537
+ print(json.dumps(ears.run(speakers), ensure_ascii=False, indent=2, default=str))
538
+ return 0
539
+ with RunContext(
540
+ args.runs_root, "data-ears", run_id=args.run_id, git_commit=args.git_commit
541
+ ) as run:
542
+ result = ears.run(speakers)
543
+ atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
544
+ report = root / "report.md"
545
+ body = report.read_text(encoding="utf-8") if report.exists() else ""
546
+ result_json = json.dumps(result, ensure_ascii=False, indent=2, default=str)
547
+ lines = ["# data-ears", "", f"root: {root}", "", "```json", result_json, "```", "", body]
548
+ atomic_write_text(run.dir / "report.md", "\n".join(lines) + "\n")
549
+ print(f"run_id={run.run_id}")
550
+ return 0
551
+
552
+
553
+ def _cmd_data_import(args: argparse.Namespace) -> int:
554
+ from nida.data.importer import Importer, ImportRequest
555
+
556
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
557
+ root = args.root
558
+ if root is None:
559
+ if args.runs_root is None:
560
+ raise SystemExit("data-import needs --root or --runs-root")
561
+ root = args.runs_root.parent / "data" / f"import-{args.source}"
562
+ cache = args.cache or Path("/content/nida_cache/import") / root.name
563
+ importer = Importer(
564
+ root=root,
565
+ cache=cache,
566
+ envs_root=args.envs,
567
+ repo_dir=args.repo,
568
+ request=ImportRequest(
569
+ source=args.source,
570
+ max_hours=args.max_hours,
571
+ unit_offset=args.shard_offset,
572
+ unit_count=args.shards,
573
+ revision=args.revision,
574
+ ),
575
+ config_dir=args.config_dir,
576
+ wavdir=args.source_dir,
577
+ retry_failed=args.retry_failed,
578
+ audio_threads=args.audio_threads,
579
+ )
580
+ if args.runs_root is None:
581
+ print(json.dumps(importer.run(args.stage), ensure_ascii=False, indent=2, default=str))
582
+ return 0
583
+ with RunContext(
584
+ args.runs_root, f"data-import-{args.stage}", run_id=args.run_id, git_commit=args.git_commit
585
+ ) as run:
586
+ result = importer.run(args.stage)
587
+ atomic_write_json(run.dir / "metrics.json", json.loads(json.dumps(result, default=str)))
588
+ report = root / "report.md"
589
+ body = report.read_text(encoding="utf-8") if report.exists() else ""
590
+ result_json = json.dumps(result, ensure_ascii=False, indent=2, default=str)
591
+ lines = [
592
+ f"# data-import {args.source} ({args.stage})",
593
+ "",
594
+ f"root: {root}",
595
+ "",
596
+ "```json",
597
+ result_json,
598
+ "```",
599
+ "",
600
+ body,
601
+ ]
602
+ atomic_write_text(run.dir / "report.md", "\n".join(lines) + "\n")
603
+ print(f"run_id={run.run_id}")
604
+ return 0
605
+
606
+
607
+ #: Exit code of ``nida train`` when a SIGTERM stopped it after saving a checkpoint.
608
+ EXIT_TRAIN_INTERRUPTED = 3
609
+
610
+
611
+ def _cmd_train(args: argparse.Namespace) -> int:
612
+ from nida.train.runner import TrainOutcome, run_training
613
+
614
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
615
+
616
+ def execute() -> TrainOutcome:
617
+ return run_training(
618
+ args.config,
619
+ args.data,
620
+ args.out,
621
+ resume=args.resume,
622
+ device=args.device,
623
+ git_commit=args.git_commit,
624
+ repo_dir=_REPO_DIR,
625
+ init_from=args.init_from,
626
+ )
627
+
628
+ if args.runs_root is None:
629
+ outcome = execute()
630
+ atomic_write_json(args.out / "metrics.json", outcome.metrics)
631
+ atomic_write_text(args.out / "report.md", outcome.report_md)
632
+ else:
633
+ with RunContext(
634
+ args.runs_root, "train", run_id=args.run_id, git_commit=args.git_commit
635
+ ) as run:
636
+ outcome = execute()
637
+ if outcome.result.stopped_by_signal:
638
+ run.status.extra["interrupted"] = "sigterm"
639
+ atomic_write_json(run.dir / "metrics.json", outcome.metrics)
640
+ atomic_write_text(run.dir / "report.md", outcome.report_md)
641
+ print(f"run_id={run.run_id}")
642
+ return EXIT_TRAIN_INTERRUPTED if outcome.result.stopped_by_signal else 0
643
+
644
+
645
+ def _cmd_synth_samples(args: argparse.Namespace) -> int:
646
+ import torch
647
+
648
+ from nida.bench import envs as bench_envs
649
+ from nida.bench.b1 import _stream_runner
650
+ from nida.data.envs import data_env_for_worker
651
+ from nida.infer.samples import SampleSettings, make_listening_page
652
+
653
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
654
+ settings = SampleSettings(
655
+ n_items=args.n,
656
+ steps=args.steps,
657
+ w_text=args.w_text,
658
+ w_emo=args.w_emo,
659
+ seed=args.seed,
660
+ method=args.method,
661
+ sway=args.sway,
662
+ )
663
+ device = _resolve_device(args.device, torch.cuda.is_available())
664
+
665
+ def execute() -> Path:
666
+ python = bench_envs.ensure_env(
667
+ data_env_for_worker("vae_decode"),
668
+ args.envs,
669
+ args.repo,
670
+ uv=bench_envs.find_uv(bench_envs.subprocess_runner),
671
+ )
672
+ asr_python = None
673
+ if not args.no_asr:
674
+ asr_python = bench_envs.ensure_env(
675
+ bench_envs.env_for_worker("asr_qwen"),
676
+ args.envs,
677
+ args.repo,
678
+ uv=bench_envs.find_uv(bench_envs.subprocess_runner),
679
+ )
680
+ return make_listening_page(
681
+ args.checkpoint,
682
+ args.data,
683
+ args.out,
684
+ settings,
685
+ decoder_python=python,
686
+ runner=_stream_runner,
687
+ device=torch.device(device),
688
+ asr_python=asr_python,
689
+ )
690
+
691
+ if args.runs_root is None:
692
+ print(execute())
693
+ return 0
694
+ with RunContext(
695
+ args.runs_root, "synth-samples", run_id=args.run_id, git_commit=args.git_commit
696
+ ) as run:
697
+ page = execute()
698
+ settings_json = (args.out / "settings.json").read_text(encoding="utf-8")
699
+ atomic_write_json(run.dir / "metrics.json", {"page": str(page)})
700
+ atomic_write_text(
701
+ run.dir / "report.md",
702
+ f"# synth-samples\n\npage: {page}\n\n```json\n{settings_json}\n```\n",
703
+ )
704
+ print(f"run_id={run.run_id}")
705
+ return 0
706
+
707
+
708
+ #: Package pins of the isolated worker envs that ``say --single-env`` replaces
709
+ #: (``nida.data.envs`` / ``nida.bench.envs``): checked, never installed.
710
+ _SINGLE_ENV_PINS = {"transformers": "4.57.6"}
711
+ _SINGLE_ENV_ASR_PINS = {"qwen-asr": "0.0.6"}
712
+
713
+
714
+ def _single_env_report(with_asr: bool) -> dict[str, object]:
715
+ """Interpreter and package versions a ``--single-env`` run uses, with every difference
716
+ from the worker envs' pins logged as a warning and listed (results may then differ
717
+ from Colab runs)."""
718
+ from importlib import metadata
719
+
720
+ pins = {**_SINGLE_ENV_PINS, **(_SINGLE_ENV_ASR_PINS if with_asr else {})}
721
+ versions: dict[str, str | None] = {}
722
+ for name in ("torch", "torchaudio", *pins):
723
+ try:
724
+ versions[name] = metadata.version(name)
725
+ except metadata.PackageNotFoundError:
726
+ versions[name] = None
727
+ mismatches = {
728
+ name: {"pinned": pin, "installed": versions[name]}
729
+ for name, pin in pins.items()
730
+ if versions[name] != pin
731
+ }
732
+ for name, diff in mismatches.items():
733
+ logging.getLogger(__name__).warning(
734
+ "say --single-env: %s %s installed, worker env pins %s",
735
+ name,
736
+ diff["installed"],
737
+ diff["pinned"],
738
+ )
739
+ return {
740
+ "single_env": True,
741
+ "python": sys.executable,
742
+ "versions": versions,
743
+ "pin_mismatches": mismatches,
744
+ }
745
+
746
+
747
+ def _cmd_say(args: argparse.Namespace) -> int:
748
+ import torch
749
+
750
+ from nida.bench import envs as bench_envs
751
+ from nida.bench.b1 import _stream_runner
752
+ from nida.data.envs import data_env_for_worker
753
+ from nida.infer.say import (
754
+ SaySettings,
755
+ make_say_page,
756
+ pick_prompts,
757
+ prompt_candidates,
758
+ rank_prompts,
759
+ )
760
+
761
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
762
+ settings = SaySettings(
763
+ steps=args.steps,
764
+ w_text=args.w_text,
765
+ w_emo=args.w_emo,
766
+ seed=args.seed,
767
+ method=args.method,
768
+ sway=args.sway,
769
+ speed=args.speed,
770
+ normalize_text=not args.no_normalize,
771
+ strict_normalize=args.strict_normalize,
772
+ )
773
+ text = args.text if args.text is not None else args.text_file.read_text(encoding="utf-8")
774
+ device = _resolve_device(args.device, torch.cuda.is_available())
775
+ if args.data is None and (args.prompt_list is None or args.prompt_id or args.rank_prompts):
776
+ raise SystemExit("say: --data is required unless every prompt comes from --prompt-list")
777
+ if args.rank_prompts and (args.prompt_list is not None or args.prompt_id):
778
+ logging.getLogger(__name__).warning(
779
+ "say: --rank-prompts is ignored when --prompt-list or --prompt-id gives the prompts"
780
+ )
781
+ # make_say_page reads data_dir only for dataset prompt ids, which the guard above rules
782
+ # out when --data is missing.
783
+ data_dir: Path = args.data if args.data is not None else args.out / "no-dataset"
784
+ worker_device = "cuda" if torch.device(device).type == "cuda" else "cpu"
785
+ if args.single_env:
786
+ args.out.mkdir(parents=True, exist_ok=True)
787
+ atomic_write_json(args.out / "environment.json", _single_env_report(not args.no_asr))
788
+ else:
789
+ (args.out / "environment.json").unlink(missing_ok=True) # never a stale single-env report
790
+
791
+ def env_python(spec: bench_envs.EnvSpec, uv: str | None) -> Path:
792
+ if args.single_env:
793
+ return Path(sys.executable)
794
+ assert uv is not None
795
+ return bench_envs.ensure_env(spec, args.envs, args.repo, uv=uv)
796
+
797
+ def execute() -> Path:
798
+ uv = None if args.single_env else bench_envs.find_uv(bench_envs.subprocess_runner)
799
+ python = env_python(data_env_for_worker("vae_decode"), uv)
800
+ external = []
801
+ if args.prompt_list is not None:
802
+ from nida.data.registry import DATA_MODELS
803
+ from nida.infer.external_prompt import build_external_prompts, read_prompt_list
804
+
805
+ ctc_python = env_python(data_env_for_worker(DATA_MODELS["mpoyraz_ctc_tr"].worker), uv)
806
+ external = build_external_prompts(
807
+ read_prompt_list(args.prompt_list),
808
+ args.out / "external",
809
+ vae_python=env_python(data_env_for_worker("vae_encode"), uv),
810
+ ctc_python=ctc_python,
811
+ runner=_stream_runner,
812
+ device=worker_device,
813
+ )
814
+ if args.prompt_id:
815
+ prompt_ids = args.prompt_id
816
+ elif external:
817
+ prompt_ids = []
818
+ elif args.rank_prompts > 0:
819
+ ranked = rank_prompts(
820
+ args.data,
821
+ prompt_candidates(args.data, args.rank_prompts),
822
+ args.out,
823
+ decoder_python=python,
824
+ runner=_stream_runner,
825
+ device=worker_device,
826
+ )
827
+ prompt_ids = [str(r["segment_id"]) for r in ranked[: args.n_prompts]]
828
+ else:
829
+ prompt_ids = pick_prompts(args.data, args.n_prompts)
830
+ asr_python = None
831
+ if not args.no_asr:
832
+ asr_python = env_python(bench_envs.env_for_worker("asr_qwen"), uv)
833
+ return make_say_page(
834
+ args.checkpoint,
835
+ data_dir,
836
+ prompt_ids,
837
+ text,
838
+ args.out,
839
+ settings,
840
+ decoder_python=python,
841
+ runner=_stream_runner,
842
+ device=torch.device(device),
843
+ asr_python=asr_python,
844
+ external=external,
845
+ )
846
+
847
+ if args.runs_root is None:
848
+ print(execute())
849
+ return 0
850
+ with RunContext(args.runs_root, "say", run_id=args.run_id, git_commit=args.git_commit) as run:
851
+ page = execute()
852
+ settings_json = (args.out / "settings.json").read_text(encoding="utf-8")
853
+ atomic_write_json(run.dir / "metrics.json", {"page": str(page)})
854
+ atomic_write_text(
855
+ run.dir / "report.md", f"# say\n\npage: {page}\n\n```json\n{settings_json}\n```\n"
856
+ )
857
+ print(f"run_id={run.run_id}")
858
+ return 0
859
+
860
+
861
+ def _cmd_queue(args: argparse.Namespace) -> int:
862
+ from nida.jobqueue import (
863
+ run_git,
864
+ run_queue,
865
+ state_path_for,
866
+ stream_to_log,
867
+ summary_markdown,
868
+ )
869
+
870
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
871
+ if args.runs_root is None or args.git_commit is None:
872
+ print("queue needs --runs-root and --git-commit (the Colab runner passes both)")
873
+ return 2
874
+ bundle_dir = args.bundle_dir or args.runs_root.parent / "code"
875
+ with RunContext(args.runs_root, "queue", run_id=args.run_id, git_commit=args.git_commit) as run:
876
+ summary = run_queue(
877
+ args.file,
878
+ runs_root=args.runs_root,
879
+ git_commit=args.git_commit,
880
+ repo_dir=args.repo,
881
+ bundle_dir=bundle_dir,
882
+ log_dir=args.log_dir,
883
+ runner=stream_to_log,
884
+ git=run_git,
885
+ wait_minutes=args.wait_minutes,
886
+ )
887
+ state_file = state_path_for(args.file)
888
+ state = json.loads(state_file.read_text(encoding="utf-8")) if state_file.exists() else {}
889
+ atomic_write_json(
890
+ run.dir / "metrics.json",
891
+ {
892
+ "jobs": len(summary.outcomes),
893
+ "failed": len(summary.failed),
894
+ "interrupted": summary.interrupted,
895
+ },
896
+ )
897
+ atomic_write_text(run.dir / "report.md", summary_markdown(summary, state))
898
+ if summary.interrupted:
899
+ run.status.extra["interrupted"] = str(summary.interrupted_key)
900
+ print(f"run_id={run.run_id}")
901
+ return EXIT_TRAIN_INTERRUPTED if summary.interrupted else 0
902
+
903
+
904
+ def main(argv: Sequence[str] | None = None) -> int:
905
+ """Entry point. Returns a process exit code."""
906
+ args = _build_parser().parse_args(argv)
907
+ if args.command == "env-report":
908
+ return _cmd_env_report(args)
909
+ if args.command == "bench-phase0":
910
+ return _cmd_bench_phase0(args)
911
+ if args.command == "data-pilot":
912
+ return _cmd_data_pilot(args)
913
+ if args.command == "data-ears":
914
+ return _cmd_data_ears(args)
915
+ if args.command == "data-import":
916
+ return _cmd_data_import(args)
917
+ if args.command == "train":
918
+ return _cmd_train(args)
919
+ if args.command == "synth-samples":
920
+ return _cmd_synth_samples(args)
921
+ if args.command == "say":
922
+ return _cmd_say(args)
923
+ if args.command == "queue":
924
+ return _cmd_queue(args)
925
+ raise AssertionError(f"unhandled command {args.command!r}") # argparse prevents this
926
+
927
+
928
+ if __name__ == "__main__":
929
+ raise SystemExit(main())
src/nida/data/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """Phase 1 data pipeline (docs/03-data-pipeline.md).
2
+
3
+ Stage logic lives here as pure, CPU-testable functions over the records in
4
+ :mod:`nida.data.types`; model inference runs in isolated worker environments.
5
+ """
src/nida/data/acoustic.py ADDED
@@ -0,0 +1,1002 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Acoustic measurements for data-pipeline stages S1, S2 and S6 (docs/03-data-pipeline.md).
2
+
3
+ Everything here is pure, CPU-only numpy/scipy over mono waveforms:
4
+
5
+ * **S1** :func:`estimate_bandwidth_hz` - effective bandwidth of a file (detects content that
6
+ was low-passed or upsampled from a lower rate).
7
+ * **S2** :func:`clipping_ratio`, :func:`silence_ratio`, :func:`loudness_dbfs`.
8
+ * **S6** :func:`track_pitch` (numpy YIN), :func:`word_features` (per-word voicing, F0, HNR,
9
+ RMS, spectral tilt), :func:`speaker_baseline`, :func:`whisper_scores`, :func:`shout_scores`.
10
+
11
+ Conventions: waveforms are 1-D floating arrays (float32 or float64) with samples in
12
+ ``[-1, 1]`` (``1.0`` = full scale); ``sample_rate_hz`` is an int; times are integer
13
+ milliseconds (``*_ms``) relative to the start of the waveform. All decision parameters come
14
+ from :class:`AcousticConfig` (``configs/data/acoustic.yaml``); the few literals left in
15
+ this file are numerical guards (division/log floors), not thresholds.
16
+
17
+ Invalid input (non-float dtype, not 1-D, empty, NaN/inf, bad rate) raises
18
+ :class:`ValueError` naming the argument; nothing is silently repaired or dropped.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import math
24
+ import typing
25
+ from collections.abc import Mapping, Sequence
26
+ from dataclasses import dataclass
27
+ from itertools import pairwise
28
+ from pathlib import Path
29
+ from typing import Any, TypeAlias
30
+
31
+ import numpy as np
32
+ import numpy.typing as npt
33
+
34
+ # scipy ships no type stubs; the ignore is scoped to this single import.
35
+ from scipy import signal # type: ignore[import-untyped]
36
+
37
+ from nida.bench.audiometrics import stft_magnitude
38
+ from nida.data.types import TimedWord, WordStyleScore
39
+
40
+ __all__ = [
41
+ "AcousticConfig",
42
+ "BandwidthParams",
43
+ "BaselineError",
44
+ "FloatArray",
45
+ "PitchTrack",
46
+ "SilentAudioError",
47
+ "SpeakerBaseline",
48
+ "WordFeatures",
49
+ "clipping_ratio",
50
+ "estimate_bandwidth_hz",
51
+ "load_acoustic_config",
52
+ "loudness_dbfs",
53
+ "phrase_support",
54
+ "shout_scores",
55
+ "silence_ratio",
56
+ "speaker_baseline",
57
+ "track_pitch",
58
+ "whisper_scores",
59
+ "word_features",
60
+ ]
61
+
62
+ FloatArray: TypeAlias = npt.NDArray[np.float64]
63
+ BoolArray: TypeAlias = npt.NDArray[np.bool_]
64
+ IntArray: TypeAlias = npt.NDArray[np.int64]
65
+
66
+ WHISPER_SOURCE = "acoustic.whisper.v0"
67
+ SHOUT_SOURCE = "acoustic.shout.v0"
68
+
69
+ # Numerical guards (not thresholds): floor under power before log10 / before division.
70
+ _POWER_FLOOR = 1e-24 # -240 dB; makes digital silence a finite number
71
+ _REL_EPS = 1e-9 # relative floor for "this energy is effectively zero"
72
+ _LOGISTIC_CLIP = 60.0 # |z| beyond this saturates to 0/1 and would overflow exp()
73
+ # Frames are processed in chunks to bound the memory of the batched FFTs (an
74
+ # implementation detail with no effect on results).
75
+ _CHUNK_FRAMES = 256
76
+
77
+
78
+ class SilentAudioError(ValueError):
79
+ """The waveform carries no energy, so the requested measurement is undefined."""
80
+
81
+
82
+ class BaselineError(ValueError):
83
+ """Too few voiced words to estimate a speaker baseline (shout not measurable)."""
84
+
85
+
86
+ # --------------------------------------------------------------------------------------
87
+ # Config
88
+ # --------------------------------------------------------------------------------------
89
+
90
+
91
+ def _check_finite(value: float, name: str) -> None:
92
+ if not math.isfinite(value):
93
+ raise ValueError(f"{name} must be finite, got {value!r}")
94
+
95
+
96
+ def _check_pos(value: float, name: str) -> None:
97
+ if not (math.isfinite(value) and value > 0.0):
98
+ raise ValueError(f"{name} must be a finite number > 0, got {value!r}")
99
+
100
+
101
+ @dataclass(frozen=True)
102
+ class BandwidthParams:
103
+ """Analysis parameters of :func:`estimate_bandwidth_hz` (S1).
104
+
105
+ The field defaults are used only when a caller passes no config; they equal the
106
+ values in ``configs/data/acoustic.yaml`` at the time of writing.
107
+
108
+ Attributes:
109
+ n_fft_samples: FFT size of the long-term average spectrum (LTAS).
110
+ active_within_db: A frame is *active* when its energy is within this many dB of the
111
+ loudest frame; only active frames enter the LTAS.
112
+ ref_low_hz: Lower edge of the reference band.
113
+ ref_high_hz: Upper edge of the reference band (clipped to Nyquist at run time).
114
+ The reference level is the median smoothed LTAS level inside this band.
115
+ smooth_hz: Width of the median filter run over the LTAS; rejects isolated
116
+ narrow-band spikes (hum, pilot tones) above the true cut-off. A tone is
117
+ rejected when its spectral footprint (Hann main lobe plus side lobes above the
118
+ ``floor_db`` threshold) is narrower than half this width; a tone far louder
119
+ than the in-band level leaks wider than that and counts as bandwidth.
120
+ """
121
+
122
+ n_fft_samples: int = 2048
123
+ active_within_db: float = 30.0
124
+ ref_low_hz: float = 300.0
125
+ ref_high_hz: float = 3400.0
126
+ smooth_hz: float = 600.0
127
+
128
+ def __post_init__(self) -> None:
129
+ if self.n_fft_samples < 16:
130
+ raise ValueError(f"n_fft_samples must be >= 16, got {self.n_fft_samples}")
131
+ _check_pos(self.active_within_db, "active_within_db")
132
+ _check_pos(self.ref_low_hz, "ref_low_hz")
133
+ _check_pos(self.ref_high_hz, "ref_high_hz")
134
+ _check_pos(self.smooth_hz, "smooth_hz")
135
+ if self.ref_low_hz >= self.ref_high_hz:
136
+ raise ValueError(
137
+ f"need ref_low_hz < ref_high_hz, got {self.ref_low_hz} >= {self.ref_high_hz}"
138
+ )
139
+
140
+ @classmethod
141
+ def from_mapping(cls, raw: object) -> BandwidthParams:
142
+ """Build from a mapping; unknown or missing keys raise :class:`ValueError`."""
143
+ return cls(**_coerce_fields(cls, raw, "bandwidth")) # type: ignore[arg-type]
144
+
145
+
146
+ @dataclass(frozen=True)
147
+ class AcousticConfig:
148
+ """All decision parameters of this module. Values: ``configs/data/acoustic.yaml``.
149
+
150
+ S1 / S2 measurement parameters (consumed by the stage code, which passes them to the
151
+ functions below):
152
+ bandwidth_floor_db: ``floor_db`` for :func:`estimate_bandwidth_hz`.
153
+ bandwidth: Other bandwidth-estimation parameters (:class:`BandwidthParams`).
154
+ clipping_threshold: ``threshold`` for :func:`clipping_ratio` (in (0, 1]).
155
+ silence_frame_ms: ``frame_ms`` for :func:`silence_ratio`.
156
+ silence_dbfs: ``silence_dbfs`` for :func:`silence_ratio`.
157
+
158
+ S6 word features:
159
+ min_word_ms: Words shorter than this get ``None``-marked features.
160
+ word_end_tolerance_ms: A word may overshoot the end of the audio by at most this
161
+ much (alignment rounding); it is clipped. More raises ``ValueError``.
162
+ f0_min_hz, f0_max_hz: Search range of the pitch tracker.
163
+ f0_frame_ms, f0_hop_ms: YIN analysis window and hop. The window must span at least
164
+ two periods of ``f0_min_hz``.
165
+ yin_threshold: Absolute threshold on the cumulative-mean-normalised difference
166
+ function below which a frame counts as voiced.
167
+ f0_energy_floor_dbfs: Frames quieter than this (RMS dBFS) are never voiced.
168
+ hnr_max_db: Per-frame HNR is clipped to ``[-hnr_max_db, +hnr_max_db]``.
169
+ tilt_low_hz, tilt_high_hz: Frequency range of the spectral-tilt regression (the
170
+ upper edge is clipped to Nyquist at run time).
171
+ tilt_num_bands: Number of log-spaced bands the regression is fitted on (>= 3).
172
+ tilt_n_fft_samples: FFT size for the tilt spectrum.
173
+
174
+ Whisper score (soft AND of three logistic factors, see :func:`whisper_scores`):
175
+ whisper_unvoiced_center, whisper_unvoiced_scale: on ``1 - voicing_ratio``.
176
+ whisper_hnr_center_db, whisper_hnr_scale_db: on ``-hnr_db`` (low HNR -> whisper).
177
+ whisper_energy_center_dbfs, whisper_energy_scale_db: on ``rms_dbfs`` (energy must
178
+ be present, otherwise silence would look like a whisper).
179
+
180
+ Shout score (see :func:`shout_scores`):
181
+ baseline_min_voicing_ratio: A word is *voiced* for the speaker baseline when its
182
+ ``voicing_ratio`` is at least this.
183
+ baseline_min_voiced_words: Fewer voiced words than this -> no baseline.
184
+ shout_weight_db, shout_weight_f0_semitones, shout_weight_tilt: Weights of the
185
+ deviations from the speaker baseline in the linear combination.
186
+ shout_center, shout_scale: Logistic centre/scale on that combination.
187
+ """
188
+
189
+ bandwidth_floor_db: float
190
+ bandwidth: BandwidthParams
191
+ clipping_threshold: float
192
+ silence_frame_ms: float
193
+ silence_dbfs: float
194
+
195
+ min_word_ms: int
196
+ word_end_tolerance_ms: int
197
+ f0_min_hz: float
198
+ f0_max_hz: float
199
+ f0_frame_ms: float
200
+ f0_hop_ms: float
201
+ yin_threshold: float
202
+ f0_energy_floor_dbfs: float
203
+ hnr_max_db: float
204
+ tilt_low_hz: float
205
+ tilt_high_hz: float
206
+ tilt_num_bands: int
207
+ tilt_n_fft_samples: int
208
+
209
+ whisper_unvoiced_center: float
210
+ whisper_unvoiced_scale: float
211
+ whisper_hnr_center_db: float
212
+ whisper_hnr_scale_db: float
213
+ whisper_energy_center_dbfs: float
214
+ whisper_energy_scale_db: float
215
+
216
+ baseline_min_voicing_ratio: float
217
+ baseline_min_voiced_words: int
218
+ shout_weight_db: float
219
+ shout_weight_f0_semitones: float
220
+ shout_weight_tilt: float
221
+ shout_center: float
222
+ shout_scale: float
223
+
224
+ def __post_init__(self) -> None:
225
+ _check_pos(self.bandwidth_floor_db, "bandwidth_floor_db")
226
+ if not (math.isfinite(self.clipping_threshold) and 0.0 < self.clipping_threshold <= 1.0):
227
+ raise ValueError(f"clipping_threshold must be in (0, 1], got {self.clipping_threshold}")
228
+ _check_pos(self.silence_frame_ms, "silence_frame_ms")
229
+ _check_finite(self.silence_dbfs, "silence_dbfs")
230
+
231
+ if self.min_word_ms <= 0:
232
+ raise ValueError(f"min_word_ms must be > 0, got {self.min_word_ms}")
233
+ if self.word_end_tolerance_ms < 0:
234
+ raise ValueError(
235
+ f"word_end_tolerance_ms must be >= 0, got {self.word_end_tolerance_ms}"
236
+ )
237
+ _check_pos(self.f0_min_hz, "f0_min_hz")
238
+ _check_pos(self.f0_max_hz, "f0_max_hz")
239
+ if self.f0_min_hz >= self.f0_max_hz:
240
+ raise ValueError(f"need f0_min_hz < f0_max_hz, got {self.f0_min_hz}..{self.f0_max_hz}")
241
+ _check_pos(self.f0_frame_ms, "f0_frame_ms")
242
+ _check_pos(self.f0_hop_ms, "f0_hop_ms")
243
+ if not (math.isfinite(self.yin_threshold) and 0.0 < self.yin_threshold < 1.0):
244
+ raise ValueError(f"yin_threshold must be in (0, 1), got {self.yin_threshold}")
245
+ _check_finite(self.f0_energy_floor_dbfs, "f0_energy_floor_dbfs")
246
+ _check_pos(self.hnr_max_db, "hnr_max_db")
247
+ _check_pos(self.tilt_low_hz, "tilt_low_hz")
248
+ _check_pos(self.tilt_high_hz, "tilt_high_hz")
249
+ if self.tilt_low_hz >= self.tilt_high_hz:
250
+ raise ValueError(
251
+ f"need tilt_low_hz < tilt_high_hz, got {self.tilt_low_hz}..{self.tilt_high_hz}"
252
+ )
253
+ if self.tilt_num_bands < 3:
254
+ raise ValueError(f"tilt_num_bands must be >= 3, got {self.tilt_num_bands}")
255
+ if self.tilt_n_fft_samples < 16:
256
+ raise ValueError(f"tilt_n_fft_samples must be >= 16, got {self.tilt_n_fft_samples}")
257
+ if self.min_word_ms < self.f0_hop_ms:
258
+ raise ValueError(
259
+ f"min_word_ms ({self.min_word_ms}) must be >= f0_hop_ms ({self.f0_hop_ms}) so "
260
+ "that every measurable word contains at least one pitch frame"
261
+ )
262
+
263
+ for name in (
264
+ "whisper_unvoiced_scale",
265
+ "whisper_hnr_scale_db",
266
+ "whisper_energy_scale_db",
267
+ "shout_scale",
268
+ ):
269
+ _check_pos(getattr(self, name), name)
270
+ for name in (
271
+ "whisper_unvoiced_center",
272
+ "whisper_hnr_center_db",
273
+ "whisper_energy_center_dbfs",
274
+ "shout_weight_db",
275
+ "shout_weight_f0_semitones",
276
+ "shout_weight_tilt",
277
+ "shout_center",
278
+ ):
279
+ _check_finite(getattr(self, name), name)
280
+ if not (
281
+ math.isfinite(self.baseline_min_voicing_ratio)
282
+ and 0.0 < self.baseline_min_voicing_ratio <= 1.0
283
+ ):
284
+ raise ValueError(
285
+ "baseline_min_voicing_ratio must be in (0, 1], "
286
+ f"got {self.baseline_min_voicing_ratio}"
287
+ )
288
+ if self.baseline_min_voiced_words < 1:
289
+ raise ValueError(
290
+ f"baseline_min_voiced_words must be >= 1, got {self.baseline_min_voiced_words}"
291
+ )
292
+
293
+ @classmethod
294
+ def from_mapping(cls, raw: object) -> AcousticConfig:
295
+ """Build from a mapping (e.g. parsed YAML); unknown or missing keys are rejected."""
296
+ return cls(**_coerce_fields(cls, raw, "acoustic config")) # type: ignore[arg-type]
297
+
298
+
299
+ def _coerce_fields(cls: type, raw: object, where: str) -> dict[str, object]:
300
+ """Check ``raw`` against the dataclass ``cls`` and coerce each value to its field type.
301
+
302
+ Rejects non-mappings, non-string keys, unknown keys, missing keys, bools in numeric
303
+ fields and non-integral values in int fields. Nested dataclass fields are built
304
+ recursively.
305
+ """
306
+ if not isinstance(raw, Mapping):
307
+ raise ValueError(f"{where}: expected a mapping, got {type(raw).__name__}")
308
+ hints = typing.get_type_hints(cls)
309
+ bad_keys = [k for k in raw if not isinstance(k, str)]
310
+ if bad_keys:
311
+ raise ValueError(f"{where}: non-string keys {bad_keys!r}")
312
+ unknown = sorted(set(raw) - set(hints))
313
+ missing = sorted(set(hints) - set(raw))
314
+ if unknown or missing:
315
+ raise ValueError(f"{where}: unknown keys {unknown}, missing keys {missing}")
316
+
317
+ out: dict[str, object] = {}
318
+ for name, hint in hints.items():
319
+ value = raw[name]
320
+ path = f"{where}.{name}"
321
+ if hint is float:
322
+ if isinstance(value, bool) or not isinstance(value, int | float):
323
+ raise ValueError(f"{path}: expected a number, got {value!r}")
324
+ out[name] = float(value)
325
+ elif hint is int:
326
+ if isinstance(value, bool) or not isinstance(value, int):
327
+ raise ValueError(f"{path}: expected an integer, got {value!r}")
328
+ out[name] = int(value)
329
+ elif hint is BandwidthParams:
330
+ out[name] = BandwidthParams(**_coerce_fields(BandwidthParams, value, path)) # type: ignore[arg-type]
331
+ else: # pragma: no cover - guards future field types
332
+ raise TypeError(f"{path}: unsupported field type {hint!r}")
333
+ return out
334
+
335
+
336
+ def load_acoustic_config(path: Path) -> AcousticConfig:
337
+ """Read an :class:`AcousticConfig` from a YAML file (``configs/data/acoustic.yaml``)."""
338
+ import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
339
+
340
+ data: Any = yaml.safe_load(path.read_text(encoding="utf-8")) # Any: untyped YAML tree
341
+ try:
342
+ return AcousticConfig.from_mapping(data)
343
+ except ValueError as exc:
344
+ raise ValueError(f"{path}: {exc}") from exc
345
+
346
+
347
+ # --------------------------------------------------------------------------------------
348
+ # Input validation
349
+ # --------------------------------------------------------------------------------------
350
+
351
+
352
+ def _as_audio(audio: npt.ArrayLike, name: str = "audio") -> FloatArray:
353
+ """Validate a mono float waveform and return it as float64."""
354
+ arr = np.asarray(audio)
355
+ if arr.dtype.kind != "f":
356
+ raise ValueError(f"{name} must have a floating dtype, got {arr.dtype}")
357
+ if arr.ndim != 1:
358
+ raise ValueError(f"{name} must be 1-D mono, got shape {arr.shape}")
359
+ if arr.size == 0:
360
+ raise ValueError(f"{name} must not be empty")
361
+ out = arr.astype(np.float64)
362
+ if not np.isfinite(out).all():
363
+ raise ValueError(f"{name} contains non-finite values (NaN or inf)")
364
+ return out
365
+
366
+
367
+ def _as_rate(sample_rate_hz: int) -> int:
368
+ if isinstance(sample_rate_hz, bool) or not isinstance(sample_rate_hz, int | np.integer):
369
+ raise ValueError(f"sample_rate_hz must be an int, got {sample_rate_hz!r}")
370
+ rate = int(sample_rate_hz)
371
+ if rate <= 0:
372
+ raise ValueError(f"sample_rate_hz must be > 0, got {rate}")
373
+ return rate
374
+
375
+
376
+ def _db_power(power: float) -> float:
377
+ return 10.0 * math.log10(max(power, _POWER_FLOOR))
378
+
379
+
380
+ # --------------------------------------------------------------------------------------
381
+ # S1: bandwidth
382
+ # --------------------------------------------------------------------------------------
383
+
384
+
385
+ def estimate_bandwidth_hz(
386
+ audio: npt.ArrayLike,
387
+ sample_rate_hz: int,
388
+ *,
389
+ floor_db: float,
390
+ params: BandwidthParams | None = None,
391
+ ) -> float:
392
+ """Effective bandwidth: the highest frequency still within ``floor_db`` of the in-band level.
393
+
394
+ Method: Hann STFT (hop = ``n_fft/2``); keep *active* frames (within
395
+ ``params.active_within_db`` of the loudest); average their power into a long-term
396
+ average spectrum (LTAS) in dB; median-filter it over ``params.smooth_hz`` so isolated
397
+ narrow spikes cannot masquerade as bandwidth; take the reference level as the median of
398
+ the smoothed LTAS over ``[ref_low_hz, min(ref_high_hz, Nyquist)]``; return the centre
399
+ frequency of the highest bin whose smoothed level is ``>= reference - floor_db``.
400
+
401
+ The result is quantised to the FFT bin spacing ``sample_rate_hz / n_fft_samples``.
402
+ Audio upsampled from 16 kHz to 48 kHz reports about 8 kHz *provided* ``floor_db`` is
403
+ smaller than the resampler's stop-band attenuation (choose it per resampler/ source).
404
+
405
+ Args:
406
+ audio: Mono waveform, shape ``(num_samples,)``.
407
+ sample_rate_hz: Sample rate in Hz.
408
+ floor_db: How far below the reference level still counts as "content" (> 0).
409
+ params: Analysis parameters; defaults to :class:`BandwidthParams` defaults.
410
+
411
+ Returns:
412
+ Bandwidth in Hz, in ``(0, sample_rate_hz / 2]``.
413
+
414
+ Raises:
415
+ SilentAudioError: if the waveform is digitally silent.
416
+ ValueError: on invalid input or if the reference band lies above Nyquist.
417
+ """
418
+ x = _as_audio(audio)
419
+ rate = _as_rate(sample_rate_hz)
420
+ _check_pos(floor_db, "floor_db")
421
+ p = BandwidthParams() if params is None else params
422
+
423
+ n_fft = p.n_fft_samples
424
+ nyquist_hz = rate / 2
425
+ bin_hz = rate / n_fft
426
+ if p.ref_low_hz >= nyquist_hz:
427
+ raise ValueError(f"ref_low_hz {p.ref_low_hz} is not below Nyquist {nyquist_hz} Hz")
428
+
429
+ power = stft_magnitude(x, n_fft, n_fft // 2) ** 2 # (frames, bins)
430
+ frame_energy = power.sum(axis=1)
431
+ loudest = float(np.max(frame_energy))
432
+ if loudest <= 0.0:
433
+ raise SilentAudioError("audio is digitally silent; bandwidth is undefined")
434
+ active = frame_energy >= loudest * 10.0 ** (-p.active_within_db / 10.0)
435
+ ltas_db = 10.0 * np.log10(power[active].mean(axis=0) + _POWER_FLOOR)
436
+
437
+ kernel = max(1, round(p.smooth_hz / bin_hz)) | 1 # medfilt needs an odd length
438
+ half = kernel // 2
439
+ padded = np.pad(ltas_db, half, mode="edge") # medfilt zero-pads; edge-pad instead
440
+ smooth = np.asarray(signal.medfilt(padded, kernel_size=kernel), dtype=np.float64)[
441
+ half : half + ltas_db.shape[0]
442
+ ]
443
+
444
+ freqs_hz = np.arange(ltas_db.shape[0], dtype=np.float64) * bin_hz
445
+ ref_band = (freqs_hz >= p.ref_low_hz) & (freqs_hz <= min(p.ref_high_hz, nyquist_hz))
446
+ if not ref_band.any():
447
+ raise ValueError(
448
+ f"no FFT bin in the reference band [{p.ref_low_hz}, {p.ref_high_hz}] Hz "
449
+ f"(n_fft_samples={n_fft}, sample_rate_hz={rate})"
450
+ )
451
+ reference_db = float(np.median(smooth[ref_band]))
452
+ passing = np.flatnonzero(smooth >= reference_db - floor_db)
453
+ return float(freqs_hz[int(passing[-1])])
454
+
455
+
456
+ # --------------------------------------------------------------------------------------
457
+ # S2: clipping, silence, loudness
458
+ # --------------------------------------------------------------------------------------
459
+
460
+
461
+ def clipping_ratio(audio: npt.ArrayLike, *, threshold: float) -> float:
462
+ """Fraction of samples with ``|x| >= threshold``.
463
+
464
+ Args:
465
+ audio: Mono waveform.
466
+ threshold: Magnitude counted as clipped, in ``(0, 1]``.
467
+
468
+ Returns:
469
+ Ratio in ``[0, 1]``.
470
+ """
471
+ x = _as_audio(audio)
472
+ if not (math.isfinite(threshold) and 0.0 < threshold <= 1.0):
473
+ raise ValueError(f"threshold must be in (0, 1], got {threshold!r}")
474
+ return float(np.mean(np.abs(x) >= threshold))
475
+
476
+
477
+ def silence_ratio(
478
+ audio: npt.ArrayLike, sample_rate_hz: int, *, frame_ms: float, silence_dbfs: float
479
+ ) -> float:
480
+ """Fraction of non-overlapping frames whose RMS is below ``silence_dbfs``.
481
+
482
+ The last, possibly shorter frame is included with its own RMS; audio shorter than one
483
+ frame forms a single frame.
484
+
485
+ Args:
486
+ audio: Mono waveform.
487
+ sample_rate_hz: Sample rate in Hz.
488
+ frame_ms: Frame length in ms (> 0).
489
+ silence_dbfs: RMS level (dBFS) below which a frame is silent.
490
+
491
+ Returns:
492
+ Ratio in ``[0, 1]``.
493
+ """
494
+ x = _as_audio(audio)
495
+ rate = _as_rate(sample_rate_hz)
496
+ _check_pos(frame_ms, "frame_ms")
497
+ _check_finite(silence_dbfs, "silence_dbfs")
498
+ frame_len = max(1, round(rate * frame_ms / 1000.0))
499
+ starts = np.arange(0, x.shape[0], frame_len)
500
+ sums = np.add.reduceat(x * x, starts)
501
+ counts = np.diff(np.append(starts, x.shape[0]))
502
+ frame_db = 10.0 * np.log10(sums / counts + _POWER_FLOOR)
503
+ return float(np.mean(frame_db < silence_dbfs))
504
+
505
+
506
+ def loudness_dbfs(audio: npt.ArrayLike) -> float:
507
+ """Whole-signal RMS level in dBFS (``20 log10(rms)``, full-scale amplitude 1.0).
508
+
509
+ This is an unweighted RMS level, *not* LUFS (no K-weighting, no gating). Digital
510
+ silence returns -240.0 instead of ``-inf``.
511
+ """
512
+ x = _as_audio(audio)
513
+ return _db_power(float(np.mean(x * x)))
514
+
515
+
516
+ # --------------------------------------------------------------------------------------
517
+ # S6: pitch tracking (YIN)
518
+ # --------------------------------------------------------------------------------------
519
+
520
+
521
+ @dataclass(frozen=True)
522
+ class PitchTrack:
523
+ """Frame-wise pitch track on a regular grid.
524
+
525
+ Frame ``t`` is centred on sample ``center_samples[t] = t * hop_samples`` (the signal is
526
+ zero-padded by half a window at both ends so that the file edges are covered).
527
+
528
+ Attributes:
529
+ center_samples: Centre sample of each frame, shape ``(num_frames,)``.
530
+ voiced: Whether F0 was detected, shape ``(num_frames,)``.
531
+ f0_hz: F0 in Hz, ``nan`` where unvoiced.
532
+ hnr_db: Harmonics-to-noise ratio in dB (clipped to +-``hnr_max_db``), ``nan``
533
+ where unvoiced.
534
+ """
535
+
536
+ center_samples: IntArray
537
+ voiced: BoolArray
538
+ f0_hz: FloatArray
539
+ hnr_db: FloatArray
540
+
541
+
542
+ def _yin_chunk(
543
+ frames: FloatArray,
544
+ *,
545
+ tau_min: int,
546
+ tau_max: int,
547
+ threshold: float,
548
+ energy_floor_dbfs: float,
549
+ hnr_max_db: float,
550
+ ) -> tuple[BoolArray, FloatArray, FloatArray]:
551
+ """YIN on a batch of frames ``(F, W)``; returns ``(voiced, f0_samples, hnr_db)``.
552
+
553
+ ``f0_samples`` is the (parabolically refined) period in samples, ``nan`` if unvoiced.
554
+ """
555
+ num_frames, width = frames.shape
556
+ n_lags = tau_max + 2 # lags 0..tau_max+1 (tau_max+1 is needed for the interpolation)
557
+ integ = width - n_lags + 1 # integration length so that j + tau stays inside the frame
558
+ nfft = 1 << (integ + width - 1).bit_length()
559
+
560
+ head = frames[:, :integ]
561
+ corr = np.fft.irfft(
562
+ np.conj(np.fft.rfft(head, nfft, axis=1)) * np.fft.rfft(frames, nfft, axis=1),
563
+ nfft,
564
+ axis=1,
565
+ )[:, :n_lags]
566
+ csum = np.concatenate(
567
+ [np.zeros((num_frames, 1)), np.cumsum(frames * frames, axis=1)], axis=1
568
+ ) # (F, W + 1)
569
+ energy0 = csum[:, integ : integ + 1] # (F, 1)
570
+ taus = np.arange(n_lags)
571
+ energy1 = csum[:, integ + taus] - csum[:, taus] # (F, n_lags)
572
+ diff = np.maximum(energy0 + energy1 - 2.0 * corr, 0.0)
573
+ diff[:, 0] = 0.0
574
+
575
+ # Cumulative-mean-normalised difference function (YIN step 3).
576
+ cum = np.cumsum(diff[:, 1:], axis=1)
577
+ cmnd = np.ones((num_frames, n_lags))
578
+ np.divide(diff[:, 1:] * taus[1:], cum, out=cmnd[:, 1:], where=cum > _REL_EPS * energy0)
579
+
580
+ # Absolute threshold (step 4): first lag below it, then walk down to the local minimum.
581
+ below = cmnd[:, tau_min : tau_max + 1] < threshold
582
+ has_dip = below.any(axis=1)
583
+ idx = np.argmax(below, axis=1) + tau_min
584
+ rows = np.arange(num_frames)
585
+ while True:
586
+ advance = has_dip & (idx < tau_max) & (cmnd[rows, idx + 1] < cmnd[rows, idx])
587
+ if not advance.any():
588
+ break
589
+ idx = idx + advance
590
+
591
+ # Parabolic interpolation (step 5).
592
+ c_prev, c_mid, c_next = cmnd[rows, idx - 1], cmnd[rows, idx], cmnd[rows, idx + 1]
593
+ curvature = c_prev - 2.0 * c_mid + c_next
594
+ shift = np.zeros(num_frames)
595
+ np.divide(0.5 * (c_prev - c_next), curvature, out=shift, where=np.abs(curvature) > 1e-12)
596
+ period = idx + np.clip(shift, -1.0, 1.0)
597
+
598
+ # HNR from the normalised autocorrelation at the period (best of idx-1, idx, idx+1).
599
+ best_r = np.zeros(num_frames)
600
+ for offset in (-1, 0, 1):
601
+ lag = idx + offset
602
+ denom = np.sqrt(energy0[:, 0] * energy1[rows, lag])
603
+ r_norm = np.zeros(num_frames)
604
+ np.divide(corr[rows, lag], denom, out=r_norm, where=denom > 0.0)
605
+ best_r = np.maximum(best_r, r_norm)
606
+ best_r = np.clip(best_r, _REL_EPS, 1.0 - _REL_EPS)
607
+ hnr = np.clip(10.0 * np.log10(best_r / (1.0 - best_r)), -hnr_max_db, hnr_max_db)
608
+
609
+ frame_db = 10.0 * np.log10(np.mean(frames * frames, axis=1) + _POWER_FLOOR)
610
+ voiced = has_dip & (frame_db >= energy_floor_dbfs)
611
+ return (
612
+ voiced,
613
+ np.where(voiced, period, np.nan),
614
+ np.where(voiced, hnr, np.nan),
615
+ )
616
+
617
+
618
+ def track_pitch(audio: npt.ArrayLike, sample_rate_hz: int, cfg: AcousticConfig) -> PitchTrack:
619
+ """Track F0 and HNR with a numpy YIN implementation.
620
+
621
+ Per frame (``cfg.f0_frame_ms`` window, ``cfg.f0_hop_ms`` hop): difference function via
622
+ FFT cross-correlation, cumulative-mean normalisation, first dip below
623
+ ``cfg.yin_threshold`` within the lag range of ``[f0_min_hz, f0_max_hz]``, descent to
624
+ the local minimum, parabolic refinement. A frame is voiced when a dip exists *and* its
625
+ RMS is at least ``cfg.f0_energy_floor_dbfs``. HNR is
626
+ ``10 log10(r / (1 - r))`` with ``r`` the normalised autocorrelation at the period,
627
+ clipped to ``+-cfg.hnr_max_db``.
628
+
629
+ Args:
630
+ audio: Mono waveform.
631
+ sample_rate_hz: Sample rate in Hz; must satisfy ``sample_rate_hz >= 4 * f0_max_hz``.
632
+ cfg: Configuration.
633
+
634
+ Raises:
635
+ ValueError: on invalid input, or if the window is shorter than two periods of
636
+ ``f0_min_hz`` at this sample rate, or the rate is too low for ``f0_max_hz``.
637
+ """
638
+ x = _as_audio(audio)
639
+ rate = _as_rate(sample_rate_hz)
640
+ window = round(rate * cfg.f0_frame_ms / 1000.0)
641
+ hop = round(rate * cfg.f0_hop_ms / 1000.0)
642
+ tau_max = math.ceil(rate / cfg.f0_min_hz)
643
+ tau_min = math.ceil(rate / cfg.f0_max_hz)
644
+ if rate < 4 * cfg.f0_max_hz:
645
+ raise ValueError(
646
+ f"sample_rate_hz={rate} is too low for f0_max_hz={cfg.f0_max_hz} "
647
+ "(need sample_rate_hz >= 4 * f0_max_hz)"
648
+ )
649
+ if hop < 1 or window < 2 * tau_max + 2:
650
+ raise ValueError(
651
+ f"f0_frame_ms={cfg.f0_frame_ms} is too short for f0_min_hz={cfg.f0_min_hz} at "
652
+ f"sample_rate_hz={rate}: need a window of >= {2 * tau_max + 2} samples, "
653
+ f"got {window}"
654
+ )
655
+
656
+ num_samples = x.shape[0]
657
+ num_frames = (num_samples - 1) // hop + 1
658
+ pad = window // 2
659
+ padded = np.zeros(num_samples + 2 * pad + window)
660
+ padded[pad : pad + num_samples] = x
661
+ windows = np.lib.stride_tricks.sliding_window_view(padded, window)
662
+
663
+ voiced = np.zeros(num_frames, dtype=np.bool_)
664
+ f0 = np.full(num_frames, np.nan)
665
+ hnr = np.full(num_frames, np.nan)
666
+ for first in range(0, num_frames, _CHUNK_FRAMES):
667
+ stop = min(first + _CHUNK_FRAMES, num_frames)
668
+ # Frame t spans padded[t*hop : t*hop + window], i.e. is centred on sample t*hop.
669
+ chunk = np.ascontiguousarray(windows[first * hop : stop * hop : hop])
670
+ v, period, h = _yin_chunk(
671
+ chunk,
672
+ tau_min=tau_min,
673
+ tau_max=tau_max,
674
+ threshold=cfg.yin_threshold,
675
+ energy_floor_dbfs=cfg.f0_energy_floor_dbfs,
676
+ hnr_max_db=cfg.hnr_max_db,
677
+ )
678
+ voiced[first:stop] = v
679
+ f0[first:stop] = rate / period
680
+ hnr[first:stop] = h
681
+ centers = np.arange(num_frames, dtype=np.int64) * hop
682
+ return PitchTrack(center_samples=centers, voiced=voiced, f0_hz=f0, hnr_db=hnr)
683
+
684
+
685
+ # --------------------------------------------------------------------------------------
686
+ # S6: per-word features
687
+ # --------------------------------------------------------------------------------------
688
+
689
+
690
+ @dataclass(frozen=True)
691
+ class WordFeatures:
692
+ """Acoustic features of one word. ``None`` means *not measurable*, never zero.
693
+
694
+ * Word shorter than ``cfg.min_word_ms``: every field except ``duration_ms`` is ``None``.
695
+ * Word measured but without a single voiced frame: ``voicing_ratio`` is ``0.0`` and
696
+ ``median_f0_hz`` / ``hnr_db`` are ``None``.
697
+
698
+ Attributes:
699
+ duration_ms: Analysed duration (after clipping to the audio end).
700
+ voicing_ratio: Fraction of the pitch frames centred inside the word that are voiced.
701
+ median_f0_hz: Median F0 over voiced frames.
702
+ hnr_db: Median per-frame HNR over voiced frames.
703
+ rms_dbfs: RMS level of the word's samples.
704
+ spectral_tilt_db_per_octave: Slope of log power versus log2 frequency (negative =
705
+ energy falls with frequency; shouted speech is flatter, i.e. less negative).
706
+ """
707
+
708
+ duration_ms: int
709
+ voicing_ratio: float | None
710
+ median_f0_hz: float | None
711
+ hnr_db: float | None
712
+ rms_dbfs: float | None
713
+ spectral_tilt_db_per_octave: float | None
714
+
715
+ @property
716
+ def measurable(self) -> bool:
717
+ """False when the word was too short to measure."""
718
+ return self.voicing_ratio is not None
719
+
720
+
721
+ def _spectral_tilt(word: FloatArray, rate: int, cfg: AcousticConfig) -> float | None:
722
+ """Slope (dB per octave) of band power versus log2 frequency, or ``None`` if undefined.
723
+
724
+ The Hann-windowed power spectrum (averaged over ``tilt_n_fft_samples``-long frames for
725
+ long words; a word shorter than the FFT size gets one window of its own length,
726
+ zero-padded) is averaged into ``cfg.tilt_num_bands`` bands equally spaced in log2
727
+ frequency; a straight line is fitted to band dB versus band-centre log2 frequency.
728
+ Binning in log-frequency stops the dense high-frequency FFT bins from dominating.
729
+ """
730
+ n_fft = cfg.tilt_n_fft_samples
731
+ if word.shape[0] <= n_fft:
732
+ frames = (word * np.hanning(word.shape[0]))[np.newaxis, :]
733
+ else:
734
+ win = np.hanning(n_fft)
735
+ frames = np.lib.stride_tricks.sliding_window_view(word, n_fft)[:: n_fft // 2] * win
736
+ power = np.mean(np.abs(np.fft.rfft(frames, n_fft, axis=1)) ** 2, axis=0)
737
+ if float(power.sum()) <= 0.0:
738
+ return None
739
+
740
+ freqs_hz = np.arange(power.shape[0]) * rate / n_fft
741
+ high_hz = min(cfg.tilt_high_hz, rate / 2)
742
+ if cfg.tilt_low_hz >= high_hz:
743
+ return None
744
+ edges = np.linspace(math.log2(cfg.tilt_low_hz), math.log2(high_hz), cfg.tilt_num_bands + 1)
745
+ centers: list[float] = []
746
+ levels_db: list[float] = []
747
+ for lo, hi in pairwise(edges):
748
+ mask = (freqs_hz >= 2.0**lo) & (freqs_hz < 2.0**hi)
749
+ if mask.any():
750
+ centers.append(float(lo + hi) / 2.0)
751
+ levels_db.append(_db_power(float(power[mask].mean())))
752
+ if len(centers) < 3:
753
+ return None
754
+ slope = np.polyfit(np.asarray(centers), np.asarray(levels_db), 1)[0]
755
+ return float(slope)
756
+
757
+
758
+ def word_features(
759
+ audio: npt.ArrayLike,
760
+ sample_rate_hz: int,
761
+ words: Sequence[TimedWord],
762
+ cfg: AcousticConfig,
763
+ ) -> list[WordFeatures]:
764
+ """Per-word voicing, F0, HNR, RMS and spectral tilt (S6).
765
+
766
+ The pitch track is computed once over the whole waveform; a word owns the frames whose
767
+ centre lies in ``[start, end)`` (so frames at a boundary see a little context from the
768
+ neighbouring word). RMS and tilt use the word's own samples only.
769
+
770
+ Args:
771
+ audio: Mono waveform of the segment.
772
+ sample_rate_hz: Sample rate in Hz.
773
+ words: Words with segment-relative ``start_ms`` / ``end_ms``; need not be sorted.
774
+ cfg: Configuration.
775
+
776
+ Returns:
777
+ One :class:`WordFeatures` per word, in input order; ``[]`` for no words.
778
+
779
+ Raises:
780
+ ValueError: on invalid audio; if a word starts at or after the end of the audio;
781
+ or if a word ends more than ``cfg.word_end_tolerance_ms`` after it (smaller
782
+ overshoots are clipped to the audio end).
783
+ """
784
+ x = _as_audio(audio)
785
+ rate = _as_rate(sample_rate_hz)
786
+ if not words:
787
+ return []
788
+
789
+ num_samples = x.shape[0]
790
+ audio_ms = num_samples * 1000.0 / rate
791
+ track = track_pitch(x, rate, cfg)
792
+
793
+ out: list[WordFeatures] = []
794
+ for word in words:
795
+ if word.start_ms >= audio_ms:
796
+ raise ValueError(
797
+ f"word {word.text!r} starts at {word.start_ms} ms, at or after the end of the "
798
+ f"audio ({audio_ms:.1f} ms)"
799
+ )
800
+ if word.end_ms > audio_ms + cfg.word_end_tolerance_ms:
801
+ raise ValueError(
802
+ f"word {word.text!r} ends at {word.end_ms} ms, more than "
803
+ f"{cfg.word_end_tolerance_ms} ms after the end of the audio ({audio_ms:.1f} ms)"
804
+ )
805
+ end_ms = min(word.end_ms, math.floor(audio_ms))
806
+ duration_ms = max(0, end_ms - word.start_ms)
807
+ if duration_ms < cfg.min_word_ms:
808
+ out.append(WordFeatures(duration_ms, None, None, None, None, None))
809
+ continue
810
+
811
+ start = round(word.start_ms * rate / 1000.0)
812
+ stop = min(round(end_ms * rate / 1000.0), num_samples)
813
+ samples = x[start:stop]
814
+ in_word = (track.center_samples >= start) & (track.center_samples < stop)
815
+ num_frames = int(in_word.sum())
816
+ if samples.shape[0] == 0 or num_frames == 0:
817
+ out.append(WordFeatures(duration_ms, None, None, None, None, None))
818
+ continue
819
+
820
+ voiced = in_word & track.voiced
821
+ num_voiced = int(voiced.sum())
822
+ out.append(
823
+ WordFeatures(
824
+ duration_ms=duration_ms,
825
+ voicing_ratio=num_voiced / num_frames,
826
+ median_f0_hz=float(np.median(track.f0_hz[voiced])) if num_voiced else None,
827
+ hnr_db=float(np.median(track.hnr_db[voiced])) if num_voiced else None,
828
+ rms_dbfs=_db_power(float(np.mean(samples * samples))),
829
+ spectral_tilt_db_per_octave=_spectral_tilt(samples, rate, cfg),
830
+ )
831
+ )
832
+ return out
833
+
834
+
835
+ # --------------------------------------------------------------------------------------
836
+ # S6: speaker-relative style scores
837
+ # --------------------------------------------------------------------------------------
838
+
839
+
840
+ def _logistic(value: float, center: float, scale: float) -> float:
841
+ """``sigmoid((value - center) / scale)``, numerically safe."""
842
+ z = max(-_LOGISTIC_CLIP, min(_LOGISTIC_CLIP, (value - center) / scale))
843
+ return 1.0 / (1.0 + math.exp(-z))
844
+
845
+
846
+ def whisper_scores(features: Sequence[WordFeatures], cfg: AcousticConfig) -> WordStyleScore:
847
+ """Per-word whisper probability: unvoiced, noise-like, but with energy present.
848
+
849
+ ``score = f_unvoiced * f_noise * f_energy`` (a soft AND), each factor a logistic
850
+ ``sigmoid((x - center) / scale)`` with centre/scale from ``cfg``:
851
+
852
+ * ``f_unvoiced`` on ``x = 1 - voicing_ratio`` (``whisper_unvoiced_*``);
853
+ * ``f_noise`` on ``x = -hnr_db`` with centre ``-whisper_hnr_center_db`` (low HNR is
854
+ whisper-like); words with no voiced frame have no HNR and get factor 1.0;
855
+ * ``f_energy`` on ``x = rms_dbfs`` (``whisper_energy_*``), so silence is not a whisper.
856
+
857
+ Words that are not measurable (too short) score ``None``.
858
+ """
859
+ scores: list[float | None] = []
860
+ for feat in features:
861
+ if feat.voicing_ratio is None or feat.rms_dbfs is None:
862
+ scores.append(None)
863
+ continue
864
+ f_unvoiced = _logistic(
865
+ 1.0 - feat.voicing_ratio, cfg.whisper_unvoiced_center, cfg.whisper_unvoiced_scale
866
+ )
867
+ f_noise = (
868
+ 1.0
869
+ if feat.hnr_db is None
870
+ else _logistic(-feat.hnr_db, -cfg.whisper_hnr_center_db, cfg.whisper_hnr_scale_db)
871
+ )
872
+ f_energy = _logistic(
873
+ feat.rms_dbfs, cfg.whisper_energy_center_dbfs, cfg.whisper_energy_scale_db
874
+ )
875
+ scores.append(f_unvoiced * f_noise * f_energy)
876
+ return WordStyleScore(style="whisper", scores=tuple(scores), source=WHISPER_SOURCE)
877
+
878
+
879
+ @dataclass(frozen=True)
880
+ class SpeakerBaseline:
881
+ """A speaker's own reference levels (medians over their voiced words).
882
+
883
+ Attributes:
884
+ median_rms_dbfs: Median word RMS level.
885
+ median_f0_hz: Median of the words' median F0.
886
+ median_tilt_db_per_octave: Median spectral tilt.
887
+ num_voiced_words: How many words the medians are based on.
888
+ """
889
+
890
+ median_rms_dbfs: float
891
+ median_f0_hz: float
892
+ median_tilt_db_per_octave: float
893
+ num_voiced_words: int
894
+
895
+
896
+ def speaker_baseline(features: Sequence[WordFeatures], cfg: AcousticConfig) -> SpeakerBaseline:
897
+ """Medians over the speaker's *voiced* words (``voicing_ratio >= baseline_min_voicing_ratio``).
898
+
899
+ Pass the features of **all** of one speaker's words (many segments), not of a single
900
+ segment: the median is only a baseline if shouted words are a minority.
901
+
902
+ Raises:
903
+ ValueError: if fewer than ``cfg.baseline_min_voiced_words`` voiced words exist.
904
+ """
905
+ voiced = [
906
+ f
907
+ for f in features
908
+ if f.voicing_ratio is not None
909
+ and f.voicing_ratio >= cfg.baseline_min_voicing_ratio
910
+ and f.median_f0_hz is not None
911
+ and f.rms_dbfs is not None
912
+ and f.spectral_tilt_db_per_octave is not None
913
+ ]
914
+ if len(voiced) < cfg.baseline_min_voiced_words:
915
+ raise BaselineError(
916
+ f"speaker baseline needs >= {cfg.baseline_min_voiced_words} voiced words, "
917
+ f"got {len(voiced)} of {len(features)}"
918
+ )
919
+
920
+ def median_of(values: list[float]) -> float:
921
+ return float(np.median(np.asarray(values)))
922
+
923
+ return SpeakerBaseline(
924
+ median_rms_dbfs=median_of([f.rms_dbfs for f in voiced if f.rms_dbfs is not None]),
925
+ median_f0_hz=median_of([f.median_f0_hz for f in voiced if f.median_f0_hz is not None]),
926
+ median_tilt_db_per_octave=median_of(
927
+ [
928
+ f.spectral_tilt_db_per_octave
929
+ for f in voiced
930
+ if f.spectral_tilt_db_per_octave is not None
931
+ ]
932
+ ),
933
+ num_voiced_words=len(voiced),
934
+ )
935
+
936
+
937
+ def shout_scores(
938
+ features: Sequence[WordFeatures],
939
+ speaker_baseline: SpeakerBaseline,
940
+ cfg: AcousticConfig,
941
+ ) -> WordStyleScore:
942
+ """Per-word shout probability relative to the speaker's own baseline (no absolute levels).
943
+
944
+ With deviations from the baseline
945
+
946
+ * ``delta_db = rms_dbfs - baseline.median_rms_dbfs``,
947
+ * ``f0_st = 12 * log2(median_f0_hz / baseline.median_f0_hz)`` (F0 ratio in semitones),
948
+ * ``delta_tilt = tilt - baseline.median_tilt`` (dB/octave; positive = flatter),
949
+
950
+ the score is ``sigmoid((z - shout_center) / shout_scale)`` with
951
+ ``z = w_db * delta_db + w_f0 * f0_st + w_tilt * delta_tilt``.
952
+
953
+ A word that is too short, or has no F0 (unvoiced), cannot be judged and scores ``None``
954
+ ("not marked"), not 0.
955
+ """
956
+ scores: list[float | None] = []
957
+ for feat in features:
958
+ if (
959
+ feat.median_f0_hz is None
960
+ or feat.rms_dbfs is None
961
+ or feat.spectral_tilt_db_per_octave is None
962
+ ):
963
+ scores.append(None)
964
+ continue
965
+ delta_db = feat.rms_dbfs - speaker_baseline.median_rms_dbfs
966
+ f0_st = 12.0 * math.log2(feat.median_f0_hz / speaker_baseline.median_f0_hz)
967
+ delta_tilt = feat.spectral_tilt_db_per_octave - speaker_baseline.median_tilt_db_per_octave
968
+ z = (
969
+ cfg.shout_weight_db * delta_db
970
+ + cfg.shout_weight_f0_semitones * f0_st
971
+ + cfg.shout_weight_tilt * delta_tilt
972
+ )
973
+ scores.append(_logistic(z, cfg.shout_center, cfg.shout_scale))
974
+ return WordStyleScore(style="shout", scores=tuple(scores), source=SHOUT_SOURCE)
975
+
976
+
977
+ def phrase_support(style: WordStyleScore) -> WordStyleScore:
978
+ """Cap each word's style score by its strongest measured neighbour.
979
+
980
+ Whispering and shouting are properties of a phrase, not of one word. A lone short word
981
+ ("de", "ve", "bir") scores high easily: its aligned window is mostly a voiceless
982
+ consonant, or it carries ordinary lexical stress. The pilot smoke run (2026-10-08) lost
983
+ 42 of 258 segments that way (``short_style_run`` / ``ambiguous_style`` on single words).
984
+ So ``s'[i] = min(s[i], max(s[i-1], s[i+1]))`` over the neighbours that have a score.
985
+ A word with no measured neighbour gets ``None`` (not judged). A real style spanning two
986
+ or more words keeps its scores; two adjacent middle-band words stay ambiguous.
987
+
988
+ Returns:
989
+ Same style, scores capped, source suffixed with ``+phrase``.
990
+ """
991
+ raw = style.scores
992
+ out: list[float | None] = []
993
+ for index, score in enumerate(raw):
994
+ neighbours = [
995
+ raw[j] for j in (index - 1, index + 1) if 0 <= j < len(raw) and raw[j] is not None
996
+ ]
997
+ if score is None or not neighbours:
998
+ out.append(None)
999
+ continue
1000
+ support = max(n for n in neighbours if n is not None)
1001
+ out.append(min(score, support))
1002
+ return WordStyleScore(style=style.style, scores=tuple(out), source=style.source + "+phrase")
src/nida/data/align.py ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """S5 — CTC forced alignment of a transcript to character CTC emissions (Viterbi).
2
+
3
+ Given log-softmax emissions ``[num_frames, vocab]`` from a character CTC model (e.g.
4
+ ``mpoyraz_ctc_tr``) and the primary transcript, find the single most likely CTC path that
5
+ emits exactly the transcript's characters (words separated by the model's word-delimiter
6
+ token) and read word intervals off that path. Plain numpy, no torchaudio dependency
7
+ (``torchaudio.functional.forced_align`` is deprecated in the 2.8 line, R8).
8
+
9
+ Text handling: each transcript token is NFC + ``tr_lower``-ed, circumflexes are folded
10
+ (â/î/û → a/i/u: the model has no such letters) and characters outside the vocabulary that
11
+ are punctuation are dropped for alignment only. A token with no alignable character
12
+ (``…``, ``-``) is glued to the previous word (or the next one at the start), so the
13
+ returned words keep their original spelling and punctuation. Any other character
14
+ outside the vocabulary (digits, foreign letters) makes the segment unalignable.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import itertools
20
+ import math
21
+ import unicodedata
22
+ from collections.abc import Mapping, Sequence
23
+ from dataclasses import dataclass, field
24
+ from typing import TYPE_CHECKING
25
+
26
+ from nida.data.types import TimedWord
27
+ from nida.text.turkish import nfc, tr_lower
28
+
29
+ if TYPE_CHECKING:
30
+ import numpy as np
31
+ from numpy.typing import NDArray
32
+
33
+ __all__ = ["AlignmentResult", "align_words", "alignment_units"]
34
+
35
+ _CIRCUMFLEX_FOLD = str.maketrans({"â": "a", "î": "i", "û": "u"})
36
+ # Typographic apostrophes (U+2019, U+02BC) are the same letter-joining apostrophe as U+0027.
37
+ _APOSTROPHES = str.maketrans({"\u2019": "'", "\u02bc": "'"})
38
+ #: Allowed difference between ``num_frames * hop`` and the segment duration, in frames
39
+ #: (wav2vec2 yields ``floor((samples - 400) / 320) + 1`` frames, i.e. up to ~1 frame short).
40
+ FRAME_SLACK = 2
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class AlignmentResult:
45
+ """Either ``words`` (aligned) or ``failure`` (a machine-readable reason) is set."""
46
+
47
+ words: tuple[TimedWord, ...] | None
48
+ failure: str | None
49
+ details: Mapping[str, object] = field(default_factory=dict)
50
+
51
+ def __post_init__(self) -> None:
52
+ if (self.words is None) == (self.failure is None):
53
+ raise ValueError("AlignmentResult needs exactly one of words / failure")
54
+ if self.words is not None and self.details:
55
+ raise ValueError("an aligned result carries no failure details")
56
+
57
+
58
+ def alignment_units(
59
+ tokens: Sequence[str], vocab: Mapping[str, int]
60
+ ) -> tuple[list[str], list[str]] | str:
61
+ """Group transcript tokens into alignable words.
62
+
63
+ Returns:
64
+ ``(display_words, alignment_strings)`` of equal length, or a failure string
65
+ ``"unalignable_chars:<chars>"`` / ``"no_alignable_text"``. Apostrophes join
66
+ letters only: leading/trailing ones are not aligned, and a token made only of
67
+ punctuation (``…``, ``—``, ``'``) is glued to the previous word (or the next one at
68
+ the start) *without* the space between them (``a — b`` → ``a—``, ``b``), because a
69
+ returned word cannot contain whitespace.
70
+ """
71
+ display: list[str] = []
72
+ targets: list[str] = []
73
+ pending_prefix = ""
74
+ bad: set[str] = set()
75
+ for raw in tokens:
76
+ token = nfc(raw)
77
+ if not token or any(ch.isspace() for ch in token):
78
+ raise ValueError(f"invalid transcript token {raw!r}")
79
+ chars: list[str] = []
80
+ for ch in tr_lower(token).translate(_CIRCUMFLEX_FOLD).translate(_APOSTROPHES):
81
+ if ch in vocab:
82
+ chars.append(ch)
83
+ elif unicodedata.category(ch).startswith("P"):
84
+ continue
85
+ else:
86
+ bad.add(ch)
87
+ chars = list("".join(chars).strip("'"))
88
+ if not chars:
89
+ if display:
90
+ display[-1] += token
91
+ else:
92
+ pending_prefix += token
93
+ continue
94
+ display.append(pending_prefix + token)
95
+ pending_prefix = ""
96
+ targets.append("".join(chars))
97
+ if bad:
98
+ return "unalignable_chars:" + "".join(sorted(bad))
99
+ if not targets:
100
+ return "no_alignable_text"
101
+ return display, targets
102
+
103
+
104
+ def _viterbi(log_probs: NDArray[np.float64], labels: Sequence[int], blank: int) -> list[int] | None:
105
+ """Best CTC state path; ``None`` if the label sequence cannot fit in the frames.
106
+
107
+ States are the blank-extended sequence ``b l1 b l2 b … lN b`` (index ``2k+1`` is
108
+ label ``k``). Returns the state index for every frame.
109
+ """
110
+ import numpy as np
111
+
112
+ num_frames = log_probs.shape[0]
113
+ ext = np.full(2 * len(labels) + 1, blank, dtype=np.int64)
114
+ ext[1::2] = labels
115
+ num_states = ext.shape[0]
116
+ # A skip s-2 -> s is allowed into a label that differs from the label two back.
117
+ can_skip = np.zeros(num_states, dtype=bool)
118
+ can_skip[3::2] = ext[3::2] != ext[1:-2:2]
119
+ neg_inf = -np.inf
120
+ score = np.full(num_states, neg_inf)
121
+ score[0] = log_probs[0, ext[0]]
122
+ if num_states > 1:
123
+ score[1] = log_probs[0, ext[1]]
124
+ back = np.zeros((num_frames, num_states), dtype=np.int8) # 0 stay, 1 step, 2 skip
125
+ for t in range(1, num_frames):
126
+ stay = score
127
+ step = np.concatenate(([neg_inf], score[:-1]))
128
+ skip = np.concatenate(([neg_inf, neg_inf], score[:-2]))
129
+ skip = np.where(can_skip, skip, neg_inf)
130
+ stacked = np.stack([stay, step, skip])
131
+ choice = np.argmax(stacked, axis=0)
132
+ best = np.take_along_axis(stacked, choice[None, :], axis=0)[0]
133
+ score = best + log_probs[t, ext]
134
+ back[t] = choice
135
+ final_candidates = [num_states - 1] + ([num_states - 2] if num_states > 1 else [])
136
+ end = max(final_candidates, key=lambda s: score[s])
137
+ if not math.isfinite(float(score[end])):
138
+ return None
139
+ path = [0] * num_frames
140
+ state = end
141
+ for t in range(num_frames - 1, -1, -1):
142
+ path[t] = state
143
+ state -= int(back[t, state])
144
+ return path
145
+
146
+
147
+ def align_words(
148
+ tokens: Sequence[str],
149
+ log_probs: NDArray[np.float32],
150
+ vocab: Mapping[str, int],
151
+ *,
152
+ blank_token: str,
153
+ word_delimiter: str,
154
+ frame_hop_ms: float,
155
+ duration_ms: int,
156
+ ) -> AlignmentResult:
157
+ """Align transcript ``tokens`` (whitespace-split words) to CTC ``log_probs``.
158
+
159
+ Args:
160
+ tokens: Transcript words in order.
161
+ log_probs: ``[num_frames, vocab_size]`` emissions. Log-softmax is re-applied
162
+ here (idempotent on log-probabilities), so raw logits cannot inflate
163
+ ``align_score``. ``-inf`` entries are allowed; NaN and ``+inf`` are not.
164
+ vocab: Token → id (characters, ``blank_token``, ``word_delimiter``).
165
+ blank_token: CTC blank (``[PAD]`` for wav2vec2 HF models).
166
+ word_delimiter: Word separator token (``|``).
167
+ frame_hop_ms: Frame stride in ms (20 for wav2vec2).
168
+ duration_ms: Segment length. ``num_frames * frame_hop_ms`` must be within
169
+ :data:`FRAME_SLACK` frames of it (catches unresampled audio); word ends are
170
+ clamped to it.
171
+
172
+ Returns:
173
+ :class:`AlignmentResult` with words carrying ``align_score`` = mean emission
174
+ probability of the word's character frames, or a failure reason
175
+ (``unalignable_chars``, ``no_alignable_text``, ``too_few_frames``, ``no_path``,
176
+ ``words_beyond_audio``).
177
+
178
+ Raises:
179
+ ValueError: malformed inputs (shape, vocab ids not ``0..V-1``, NaN/+inf, bad hop,
180
+ frame count inconsistent with the duration).
181
+ """
182
+ import numpy as np
183
+
184
+ if log_probs.ndim != 2 or log_probs.shape[1] != len(vocab):
185
+ raise ValueError(f"log_probs shape {log_probs.shape} does not match vocab {len(vocab)}")
186
+ if sorted(vocab.values()) != list(range(len(vocab))):
187
+ raise ValueError("vocab ids must be exactly 0..V-1")
188
+ if np.any(np.isnan(log_probs)) or np.any(np.isposinf(log_probs)):
189
+ raise ValueError("log_probs contain NaN or +inf")
190
+ if blank_token not in vocab or word_delimiter not in vocab:
191
+ raise ValueError("blank or word delimiter missing from vocab")
192
+ if not (math.isfinite(frame_hop_ms) and frame_hop_ms >= 1):
193
+ raise ValueError(f"frame_hop_ms must be a finite number >= 1 ms, got {frame_hop_ms!r}")
194
+ if type(duration_ms) is not int or duration_ms <= 0:
195
+ raise ValueError(f"invalid duration_ms {duration_ms!r}")
196
+ if abs(log_probs.shape[0] * frame_hop_ms - duration_ms) > FRAME_SLACK * frame_hop_ms:
197
+ raise ValueError(
198
+ f"{log_probs.shape[0]} frames x {frame_hop_ms} ms do not match {duration_ms} ms"
199
+ )
200
+ with np.errstate(invalid="ignore"):
201
+ lp = log_probs.astype(np.float64)
202
+ row_max = np.max(lp, axis=1, keepdims=True)
203
+ if not np.all(np.isfinite(row_max)):
204
+ raise ValueError("a frame has no finite log-probability")
205
+ lp = lp - (row_max + np.log(np.sum(np.exp(lp - row_max), axis=1, keepdims=True)))
206
+ special = {blank_token, word_delimiter}
207
+ char_vocab = {tok: idx for tok, idx in vocab.items() if len(tok) == 1 and tok not in special}
208
+ units = alignment_units(tokens, char_vocab)
209
+ if isinstance(units, str):
210
+ reason, _, chars = units.partition(":")
211
+ return AlignmentResult(None, reason, {"chars": chars} if chars else {})
212
+ display, targets = units
213
+
214
+ labels: list[int] = []
215
+ owner: list[int] = [] # word index for each label, -1 for delimiters
216
+ for index, target in enumerate(targets):
217
+ if index:
218
+ labels.append(vocab[word_delimiter])
219
+ owner.append(-1)
220
+ for ch in target:
221
+ labels.append(char_vocab[ch])
222
+ owner.append(index)
223
+ repeats = sum(1 for a, b in itertools.pairwise(labels) if a == b)
224
+ if log_probs.shape[0] < len(labels) + repeats:
225
+ return AlignmentResult(
226
+ None,
227
+ "too_few_frames",
228
+ {"frames": int(log_probs.shape[0]), "needed": len(labels) + repeats},
229
+ )
230
+ path = _viterbi(lp, labels, vocab[blank_token])
231
+ if path is None:
232
+ return AlignmentResult(None, "no_path")
233
+
234
+ first: list[int | None] = [None] * len(targets)
235
+ last: list[int | None] = [None] * len(targets)
236
+ probs: list[list[float]] = [[] for _ in targets]
237
+ for frame, state in enumerate(path):
238
+ if state % 2 == 0:
239
+ continue
240
+ word = owner[state // 2]
241
+ if word < 0:
242
+ continue
243
+ if first[word] is None:
244
+ first[word] = frame
245
+ last[word] = frame
246
+ probs[word].append(math.exp(float(lp[frame, labels[state // 2]])))
247
+ words: list[TimedWord] = []
248
+ for index, text in enumerate(display):
249
+ start_frame, end_frame = first[index], last[index]
250
+ if start_frame is None or end_frame is None:
251
+ return AlignmentResult(None, "no_path", {"word_index": index})
252
+ start_ms = min(round(start_frame * frame_hop_ms), duration_ms - 1)
253
+ end_ms = min(round((end_frame + 1) * frame_hop_ms), duration_ms)
254
+ end_ms = max(end_ms, start_ms + 1)
255
+ if words and start_ms < words[-1].end_ms:
256
+ # Only reachable through the end clamp (emissions slightly longer than the
257
+ # audio): words squeezed past the end cannot be timed, never let them overlap.
258
+ return AlignmentResult(None, "words_beyond_audio", {"word_index": index})
259
+ score = sum(probs[index]) / len(probs[index])
260
+ words.append(TimedWord(text, start_ms, end_ms, score))
261
+ return AlignmentResult(tuple(words), None)
src/nida/data/asr_filter.py ADDED
@@ -0,0 +1,283 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """S4 / S5 filters: dual-ASR agreement and forced-alignment sanity checks.
2
+
3
+ Pure functions; every threshold comes from :class:`AsrFilterConfig` (``configs/data/
4
+ asr_filter.yaml``). A check returns ``None`` when the segment passes and otherwise a
5
+ machine-readable reason string from a fixed set, which the caller writes to quarantine.
6
+
7
+ The agreement metric is computed on transcripts **with filler words removed**
8
+ (docs/03-data-pipeline.md S4): the two ASR models legitimately disagree on fillers, and
9
+ filtering on them would throw away exactly the expressive segments we want.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ from collections.abc import Mapping, Sequence
16
+ from dataclasses import dataclass
17
+ from itertools import pairwise
18
+ from typing import Literal, get_args
19
+
20
+ from nida.bench.textmetrics import cer_counts, normalize_for_scoring, strip_fillers
21
+ from nida.data.types import TimedWord
22
+
23
+ __all__ = [
24
+ "ALIGNMENT_REASONS",
25
+ "TRANSCRIPT_REASONS",
26
+ "AlignmentReason",
27
+ "AsrFilterConfig",
28
+ "TranscriptReason",
29
+ "asr_agreement",
30
+ "check_alignment",
31
+ "check_transcripts",
32
+ ]
33
+
34
+ TranscriptReason = Literal["language", "asr_disagreement", "empty_transcript"]
35
+ AlignmentReason = Literal[
36
+ "align_score",
37
+ "chars_per_s_low",
38
+ "chars_per_s_high",
39
+ "word_outside_segment",
40
+ "words_not_monotonic",
41
+ ]
42
+ TRANSCRIPT_REASONS: tuple[str, ...] = get_args(TranscriptReason)
43
+ ALIGNMENT_REASONS: tuple[str, ...] = get_args(AlignmentReason)
44
+
45
+ _MS_PER_S = 1000 # unit conversion, not a threshold
46
+
47
+ # ASCII-only case folding for language codes. Built-in case mapping is avoided: "ID" must
48
+ # not be mapped through Turkish rules ("ID" -> "ıd"), nor through locale-dependent ones.
49
+ _ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
50
+
51
+ _CONFIG_FIELDS: tuple[str, ...] = (
52
+ "max_cer",
53
+ "allowed_languages",
54
+ "min_chars_per_s",
55
+ "max_chars_per_s",
56
+ "min_align_score",
57
+ )
58
+
59
+
60
+ def _norm_language(code: str) -> str:
61
+ return code.strip().translate(_ASCII_LOWER)
62
+
63
+
64
+ def _real_number(mapping: Mapping[str, object], name: str) -> float:
65
+ value = mapping[name]
66
+ if isinstance(value, bool) or not isinstance(value, int | float):
67
+ raise ValueError(f"asr filter config: {name} must be a number, got {value!r}")
68
+ number = float(value)
69
+ if not math.isfinite(number):
70
+ raise ValueError(f"asr filter config: {name} must be finite, got {value!r}")
71
+ return number
72
+
73
+
74
+ @dataclass(frozen=True)
75
+ class AsrFilterConfig:
76
+ """Thresholds of the S4 / S5 filters; load with :meth:`from_mapping`.
77
+
78
+ Attributes:
79
+ max_cer: Largest accepted CER between the filler-stripped transcripts, in [0, 1].
80
+ A CER exactly equal to it passes.
81
+ allowed_languages: Accepted detected-language codes (ASCII, stored lowercase).
82
+ min_chars_per_s: Smallest accepted letters+digits per second of speech span.
83
+ max_chars_per_s: Largest accepted letters+digits per second of speech span.
84
+ min_align_score: Smallest accepted mean per-word alignment score; the scale depends
85
+ on the aligner.
86
+ """
87
+
88
+ max_cer: float
89
+ allowed_languages: tuple[str, ...]
90
+ min_chars_per_s: float
91
+ max_chars_per_s: float
92
+ min_align_score: float
93
+
94
+ def __post_init__(self) -> None:
95
+ for name in ("max_cer", "min_chars_per_s", "max_chars_per_s", "min_align_score"):
96
+ if not math.isfinite(getattr(self, name)):
97
+ raise ValueError(f"{name} must be finite, got {getattr(self, name)!r}")
98
+ if not 0.0 <= self.max_cer <= 1.0:
99
+ raise ValueError(f"max_cer must be in [0, 1], got {self.max_cer}")
100
+ if self.min_chars_per_s < 0.0:
101
+ raise ValueError(f"min_chars_per_s must be >= 0, got {self.min_chars_per_s}")
102
+ if self.max_chars_per_s <= 0.0 or self.max_chars_per_s < self.min_chars_per_s:
103
+ raise ValueError(
104
+ f"max_chars_per_s ({self.max_chars_per_s}) must be > 0 and >= "
105
+ f"min_chars_per_s ({self.min_chars_per_s})"
106
+ )
107
+ if not self.allowed_languages:
108
+ raise ValueError("allowed_languages must not be empty")
109
+ for code in self.allowed_languages:
110
+ if not code or not code.isascii() or not code.replace("-", "").isalpha():
111
+ raise ValueError(f"invalid language code {code!r}: expected ASCII letters/'-'")
112
+ if code != _norm_language(code):
113
+ raise ValueError(f"language code {code!r} must be lowercase")
114
+ if len(set(self.allowed_languages)) != len(self.allowed_languages):
115
+ raise ValueError(f"duplicate entries in allowed_languages {self.allowed_languages}")
116
+
117
+ @classmethod
118
+ def from_mapping(cls, mapping: Mapping[str, object]) -> AsrFilterConfig:
119
+ """Build a config from a parsed YAML mapping.
120
+
121
+ Language codes are stripped and lowercased (ASCII only).
122
+
123
+ Raises:
124
+ ValueError: If ``mapping`` is not a mapping, has unknown or missing keys, a
125
+ number is not a finite int/float (bools are rejected), or a bound or the
126
+ language list is invalid.
127
+ """
128
+ if not isinstance(mapping, Mapping):
129
+ raise ValueError(f"asr filter config must be a mapping, got {type(mapping).__name__}")
130
+ keys = set(mapping)
131
+ unknown = sorted(str(key) for key in keys - set(_CONFIG_FIELDS))
132
+ missing = sorted(set(_CONFIG_FIELDS) - keys)
133
+ if unknown or missing:
134
+ raise ValueError(f"asr filter config: unknown keys {unknown}, missing keys {missing}")
135
+ raw_languages = mapping["allowed_languages"]
136
+ if isinstance(raw_languages, str) or not isinstance(raw_languages, Sequence):
137
+ raise ValueError(
138
+ f"asr filter config: allowed_languages must be a list, got {raw_languages!r}"
139
+ )
140
+ languages: list[str] = []
141
+ for code in raw_languages:
142
+ if not isinstance(code, str):
143
+ raise ValueError(f"asr filter config: language code {code!r} is not a string")
144
+ languages.append(_norm_language(code))
145
+ return cls(
146
+ max_cer=_real_number(mapping, "max_cer"),
147
+ allowed_languages=tuple(languages),
148
+ min_chars_per_s=_real_number(mapping, "min_chars_per_s"),
149
+ max_chars_per_s=_real_number(mapping, "max_chars_per_s"),
150
+ min_align_score=_real_number(mapping, "min_align_score"),
151
+ )
152
+
153
+
154
+ # --------------------------------------------------------------------------------------
155
+ # S4: transcripts
156
+ # --------------------------------------------------------------------------------------
157
+
158
+
159
+ def asr_agreement(primary: str, secondary: str) -> float:
160
+ """Character error rate between two transcripts, ignoring fillers.
161
+
162
+ Both texts go through :func:`~nida.bench.textmetrics.normalize_for_scoring` (NFC,
163
+ Turkish lowercase, punctuation removed, fillers canonicalised), fillers are removed,
164
+ and the CER is taken over the space-joined tokens with ``primary`` as the reference.
165
+
166
+ Returns:
167
+ 0.0 if both are empty after filler removal (e.g. ``"ıı"`` vs ``""``), 1.0 if
168
+ exactly one is empty, otherwise the CER capped at 1.0 (lower = better agreement).
169
+ """
170
+ ref = strip_fillers(normalize_for_scoring(primary))
171
+ hyp = strip_fillers(normalize_for_scoring(secondary))
172
+ if not ref and not hyp:
173
+ return 0.0
174
+ if not ref or not hyp:
175
+ return 1.0
176
+ rate = cer_counts(ref, hyp).rate()
177
+ if rate is None: # unreachable: ref is non-empty
178
+ raise ValueError("CER undefined for a non-empty reference")
179
+ return min(rate, 1.0)
180
+
181
+
182
+ def check_transcripts(
183
+ primary_text: str,
184
+ secondary_text: str,
185
+ language: str | None,
186
+ cfg: AsrFilterConfig,
187
+ ) -> TranscriptReason | None:
188
+ """S4 filter: empty transcript, language and dual-ASR agreement.
189
+
190
+ Checks run in this order and the first failure is returned:
191
+
192
+ 1. ``"empty_transcript"``: the primary transcript has no token at all after
193
+ normalisation. A fillers-only primary (``"hmm"``) is *not* empty: it is a valid
194
+ sound-only segment.
195
+ 2. ``"language"``: ``language`` is ``None`` or not in ``cfg.allowed_languages``
196
+ (case-insensitive exact match on the code).
197
+ 3. ``"asr_disagreement"``: :func:`asr_agreement` is greater than ``cfg.max_cer``
198
+ (equal passes).
199
+
200
+ Args:
201
+ primary_text: Primary ASR transcript (the label source).
202
+ secondary_text: Secondary ASR transcript.
203
+ language: Detected language code of the segment, or ``None`` if unknown.
204
+ cfg: Thresholds.
205
+
206
+ Returns:
207
+ ``None`` if the segment passes, else the reason.
208
+ """
209
+ if not normalize_for_scoring(primary_text):
210
+ return "empty_transcript"
211
+ if language is None or _norm_language(language) not in cfg.allowed_languages:
212
+ return "language"
213
+ if asr_agreement(primary_text, secondary_text) > cfg.max_cer:
214
+ return "asr_disagreement"
215
+ return None
216
+
217
+
218
+ # --------------------------------------------------------------------------------------
219
+ # S5: alignment
220
+ # --------------------------------------------------------------------------------------
221
+
222
+
223
+ def _num_chars(text: str) -> int:
224
+ """Letters and digits of the normalised text (fillers included, spaces excluded)."""
225
+ return sum(1 for token in normalize_for_scoring(text) for ch in token if ch.isalnum())
226
+
227
+
228
+ def check_alignment(
229
+ words: Sequence[TimedWord], duration_ms: int, cfg: AsrFilterConfig
230
+ ) -> AlignmentReason | None:
231
+ """S5 filter: sanity of the forced-alignment output of one segment.
232
+
233
+ Word times are relative to the segment start. Checks run in this order and the first
234
+ failure is returned:
235
+
236
+ 1. ``"words_not_monotonic"``: a word starts before the previous one ends (touching is
237
+ fine).
238
+ 2. ``"word_outside_segment"``: a word ends after ``duration_ms``.
239
+ 3. ``"align_score"``: the mean word alignment score is below ``cfg.min_align_score``,
240
+ or any word has no (or a non-finite) score: a segment whose alignment cannot be
241
+ verified is not accepted. The mean, not the minimum, is used so that a single hard-to-align
242
+ filler or laugh does not discard an expressive segment.
243
+ 4. ``"chars_per_s_low"`` / ``"chars_per_s_high"``: letters+digits of the normalised
244
+ words (fillers included) per second of speech span (first word start to last word
245
+ end) is below ``cfg.min_chars_per_s`` / above ``cfg.max_chars_per_s``. An empty word
246
+ list (or words without any letter or digit) has rate 0 and is reported as
247
+ ``"chars_per_s_low"`` rather than crashing the pipeline.
248
+
249
+ Thresholds are inclusive: values exactly equal to a limit pass.
250
+
251
+ Args:
252
+ words: Aligned words of the primary transcript, in order.
253
+ duration_ms: Segment duration in milliseconds, must be > 0.
254
+ cfg: Thresholds.
255
+
256
+ Returns:
257
+ ``None`` if the alignment looks sane, else the reason.
258
+
259
+ Raises:
260
+ ValueError: If ``duration_ms`` is not a positive int.
261
+ """
262
+ if isinstance(duration_ms, bool) or not isinstance(duration_ms, int) or duration_ms <= 0:
263
+ raise ValueError(f"duration_ms must be a positive int, got {duration_ms!r}")
264
+ if not words:
265
+ return "chars_per_s_low"
266
+ for previous, word in pairwise(words):
267
+ if word.start_ms < previous.end_ms:
268
+ return "words_not_monotonic"
269
+ if any(word.end_ms > duration_ms for word in words):
270
+ return "word_outside_segment"
271
+ scores = [word.align_score for word in words]
272
+ known = [score for score in scores if score is not None]
273
+ if len(known) != len(scores) or not all(math.isfinite(score) for score in known):
274
+ return "align_score"
275
+ if math.fsum(known) / len(known) < cfg.min_align_score:
276
+ return "align_score"
277
+ span_ms = words[-1].end_ms - words[0].start_ms
278
+ chars_per_s = sum(_num_chars(word.text) for word in words) * _MS_PER_S / span_ms
279
+ if chars_per_s < cfg.min_chars_per_s:
280
+ return "chars_per_s_low"
281
+ if chars_per_s > cfg.max_chars_per_s:
282
+ return "chars_per_s_high"
283
+ return None
src/nida/data/ears.py ADDED
@@ -0,0 +1,1307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """EARS -> latent dataset builder (ADR-0007, ``nida data-ears``).
2
+
3
+ EARS (Expressive Anechoic Recordings of Speech, CC BY-NC 4.0) provides English emotional,
4
+ whispered and non-verbal speech. This module turns the selected files into a *latent dataset
5
+ directory* in exactly the format of :mod:`nida.train.data` (records carry the optional key
6
+ ``"language": "en"``), so it can be trained together with Turkish datasets
7
+ (``train --data A --data B``).
8
+
9
+ What is used (rules and mappings live in ``configs/data/ears.yaml``):
10
+
11
+ * ``emo_<emotion>_sentences``: reading transcript, emotion mapped (unmapped -> ``null``);
12
+ * ``rainbow_NN_<style>`` / ``sentences_NN_<style>``: ``whisper`` -> the whole text wrapped in a
13
+ whisper span (rendered by :mod:`nida.text.tags`), the plain styles -> plain text;
14
+ * laughter/crying/screaming/coughing/throat-clearing files: just the event tag;
15
+ * everything else is skipped and counted by reason.
16
+
17
+ Per speaker (one remote zip of ~564 MB, read with HTTP Range requests, see
18
+ :mod:`nida.data.remote_zip`): list the archive -> decide on every member (skip / text
19
+ quarantine, before any download) -> extract the selected members, convert 48 kHz -> 16 kHz mono
20
+ -> measure. The speakers are handled in *waves* of ``speakers_per_wave``: one AudioVAE run
21
+ (the same ``vae_encode`` worker and environment as the pilot, so the model is loaded once per
22
+ wave, not once per speaker) encodes the wave's audio, then every speaker of the wave gets its
23
+ float16 shard, index records and ``done.json``. Layout::
24
+
25
+ <root>/plan.json seed, config digest, VAE pin, transcripts SHA-256
26
+ <root>/source/transcripts.json pinned reading transcripts
27
+ <root>/speakers/pNNN/records.jsonl index rows of the speaker
28
+ <root>/speakers/pNNN/quarantine.jsonl every quarantined item with its reason
29
+ <root>/speakers/pNNN/done.json written last: the speaker is finished
30
+ <root>/workers/vae/ worker requests/results of the current wave
31
+ <root>/dataset/ latent dataset: index.jsonl, meta.json, shard-pNNN.npy
32
+ <root>/report.md, <root>/stats.json
33
+ <cache>/... raw members, 16 kHz wavs and latent arrays (local disk)
34
+
35
+ Resumable and idempotent: a speaker with ``done.json`` is skipped without any network access, an
36
+ interrupted wave continues (worker results are kept, already converted audio is reused), and
37
+ the dataset directory is reassembled from all finished speakers at the end of every run, so
38
+ it is usable after any run. A run on a root made with another seed, config digest or VAE pin,
39
+ or with different transcripts, is refused.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import hashlib
45
+ import io
46
+ import json
47
+ import logging
48
+ import math
49
+ import re
50
+ import shutil
51
+ import string
52
+ import time
53
+ from bisect import bisect_right
54
+ from collections import Counter
55
+ from collections.abc import Callable, Mapping, Sequence
56
+ from dataclasses import asdict, dataclass, field
57
+ from pathlib import Path
58
+ from typing import Literal
59
+
60
+ import numpy as np
61
+ import numpy.typing as npt
62
+
63
+ # scipy ships no type stubs; the ignore is scoped to this single import (as in acoustic.py).
64
+ from scipy import signal # type: ignore[import-untyped]
65
+
66
+ from nida.bench import envs as bench_envs
67
+ from nida.bench import fetch
68
+ from nida.bench.b1 import MAX_ERROR_FRACTION, _stream_runner
69
+ from nida.bench.protocol import (
70
+ WorkRequest,
71
+ WorkResult,
72
+ drop_results,
73
+ load_results,
74
+ read_summary,
75
+ write_requests,
76
+ )
77
+ from nida.bench.registry import load_sources, require_approved
78
+ from nida.data import acoustic, envs
79
+ from nida.data.ingest import IngestConfig
80
+ from nida.data.pilot import DatasetConfig
81
+ from nida.data.quarantine import QuarantineLog
82
+ from nida.data.registry import DATA_MODELS
83
+ from nida.data.remote_zip import RemoteZip, ZipMember
84
+ from nida.data.splits import assign_splits
85
+ from nida.data.types import QuarantineRecord, segment_id
86
+ from nida.model.vocab import encode_text
87
+ from nida.text import tags
88
+ from nida.text.turkish import nfc
89
+ from nida.train.data import (
90
+ EMOTIONS,
91
+ REJECT_TEXT_TOO_LONG,
92
+ SPLITS,
93
+ DataFormatError,
94
+ load_latent_dataset,
95
+ )
96
+ from nida.utils.io import atomic_write_bytes, atomic_write_json, atomic_write_text
97
+
98
+ __all__ = [
99
+ "SOURCE_ID",
100
+ "AudioFacts",
101
+ "Ears",
102
+ "EarsConfig",
103
+ "EarsSettings",
104
+ "ItemRejected",
105
+ "PlannedItem",
106
+ "Selection",
107
+ "SelectionPlan",
108
+ "Skip",
109
+ "TextRejected",
110
+ "bucket_of",
111
+ "build_text",
112
+ "classify_stem",
113
+ "convert_to_target_rate",
114
+ "en_lower",
115
+ "load_settings",
116
+ "measure_audio",
117
+ "normalise_english",
118
+ "parse_speakers",
119
+ "rate_chars_per_s",
120
+ "resample",
121
+ "select_members",
122
+ "speaker_split",
123
+ ]
124
+
125
+ log = logging.getLogger(__name__)
126
+
127
+ SOURCE_ID = "ears"
128
+ STAGE = "ears"
129
+ LANGUAGE = "en"
130
+ VAE_KEY = "voxcpm2_audiovae_v2"
131
+ DEFAULT_SEED = 20261009
132
+ FLOAT16_MAX_ABS = 6.0e4 # latents beyond this would overflow the float16 shards (as in pilot.py)
133
+ CONFIG_FILE = "ears.yaml"
134
+ INGEST_FILE = "ingest.yaml" # duration limits shared with the Turkish pilot
135
+ DATASET_FILE = "dataset.yaml" # fixed condition-bucket edges
136
+ _ASCII_LOWER = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)
137
+ _SHA256_RE = re.compile(r"[0-9a-f]{64}")
138
+ _SPEAKER_RE = re.compile(r"p(\d{3})")
139
+ _WORD_RE = re.compile(r"[a-z]+")
140
+ _STEM_RE = re.compile(r"[a-z0-9_]+")
141
+ _EMO_RE = re.compile(r"emo_([a-z]+)_(sentences|freeform)")
142
+ _READ_RE = re.compile(r"([a-z]+)_(\d+)_([a-z]+)")
143
+ _TAG_CHARS = frozenset("[]<>")
144
+
145
+ Kind = Literal["emotion", "read", "whisper", "event"]
146
+ FloatArray = npt.NDArray[np.float64]
147
+
148
+
149
+ # --------------------------------------------------------------------------------------
150
+ # Config
151
+ # --------------------------------------------------------------------------------------
152
+
153
+
154
+ def _load_yaml(path: Path) -> Mapping[str, object]:
155
+ import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
156
+
157
+ obj = yaml.safe_load(path.read_text(encoding="utf-8"))
158
+ if not isinstance(obj, dict):
159
+ raise ValueError(f"{path}: expected a mapping")
160
+ return obj
161
+
162
+
163
+ def _non_negative_int(raw: Mapping[str, object], key: str) -> int:
164
+ value = raw[key]
165
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
166
+ raise ValueError(f"{key} must be a non-negative int, got {value!r}")
167
+ return value
168
+
169
+
170
+ def _positive_int(raw: Mapping[str, object], key: str) -> int:
171
+ value = raw[key]
172
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
173
+ raise ValueError(f"{key} must be a positive int, got {value!r}")
174
+ return value
175
+
176
+
177
+ def _fraction(raw: Mapping[str, object], key: str) -> float:
178
+ value = raw[key]
179
+ if isinstance(value, bool) or not isinstance(value, int | float):
180
+ raise ValueError(f"{key} must be a number, got {value!r}")
181
+ if not (math.isfinite(value) and 0.0 <= value < 1.0):
182
+ raise ValueError(f"{key} must be in [0, 1), got {value!r}")
183
+ return float(value)
184
+
185
+
186
+ def _words(raw: Mapping[str, object], key: str) -> tuple[str, ...]:
187
+ value = raw[key]
188
+ if (
189
+ not isinstance(value, list)
190
+ or not value
191
+ or not all(isinstance(x, str) and _WORD_RE.fullmatch(x) for x in value)
192
+ or len(set(value)) != len(value)
193
+ ):
194
+ raise ValueError(f"{key} must be a non-empty list of distinct lowercase words")
195
+ return tuple(value)
196
+
197
+
198
+ def _str_map(raw: Mapping[str, object], key: str) -> dict[str, str]:
199
+ value = raw[key]
200
+ if not isinstance(value, dict) or not all(
201
+ isinstance(k, str) and isinstance(v, str) for k, v in value.items()
202
+ ):
203
+ raise ValueError(f"{key} must be a mapping of strings")
204
+ return dict(value)
205
+
206
+
207
+ @dataclass(frozen=True)
208
+ class EarsConfig:
209
+ """``configs/data/ears.yaml`` (all keys required; see the file for the meaning)."""
210
+
211
+ transcripts_url: str
212
+ transcripts_sha256: str | None
213
+ archive_url_template: str
214
+ n_speakers: int
215
+ expected_sample_rate_hz: int
216
+ target_sample_rate_hz: int
217
+ val_speaker_fraction: float
218
+ test_speaker_fraction: float
219
+ speakers_per_wave: int
220
+ read_prefixes: tuple[str, ...]
221
+ whisper_styles: tuple[str, ...]
222
+ plain_styles: tuple[str, ...]
223
+ max_read_per_speaker: int
224
+ emotion_map: dict[str, str] # plain dicts (not frozen views): ``asdict`` feeds the digest
225
+ event_map: dict[str, str]
226
+ char_map: dict[str, str]
227
+
228
+ def __post_init__(self) -> None:
229
+ for name in ("transcripts_url", "archive_url_template"):
230
+ if not getattr(self, name).lower().startswith(("http://", "https://")):
231
+ raise ValueError(f"{name} must be an http(s) URL")
232
+ if "{speaker}" not in self.archive_url_template:
233
+ raise ValueError("archive_url_template must contain {speaker}")
234
+ if self.transcripts_sha256 is not None and not _SHA256_RE.fullmatch(
235
+ self.transcripts_sha256
236
+ ):
237
+ raise ValueError("transcripts_sha256 must be null or 64 lowercase hex characters")
238
+ if self.val_speaker_fraction + self.test_speaker_fraction >= 1.0:
239
+ raise ValueError("val and test speaker fractions must leave room for train")
240
+ if set(self.whisper_styles) & set(self.plain_styles):
241
+ raise ValueError("a style cannot be both whisper and plain")
242
+ for emotion, mapped in self.emotion_map.items():
243
+ if not _WORD_RE.fullmatch(emotion) or mapped not in EMOTIONS:
244
+ raise ValueError(f"emotion_map {emotion!r}: {mapped!r} is not one of {EMOTIONS}")
245
+ for stem, event in self.event_map.items():
246
+ if not _STEM_RE.fullmatch(stem) or event not in tags.EVENTS:
247
+ raise ValueError(f"event_map {stem!r}: {event!r} is not a tag-spec event id")
248
+ for char, replacement in self.char_map.items():
249
+ if len(char) != 1 or not replacement:
250
+ raise ValueError(f"char_map entries map one character to a string: {char!r}")
251
+
252
+ @classmethod
253
+ def from_mapping(cls, raw: Mapping[str, object]) -> EarsConfig:
254
+ expected = set(cls.__dataclass_fields__)
255
+ if set(raw) != expected:
256
+ raise ValueError(
257
+ f"ears config keys: missing={sorted(expected - set(raw))} "
258
+ f"unknown={sorted(set(raw) - expected)}"
259
+ )
260
+ for key in ("transcripts_url", "archive_url_template"):
261
+ if not isinstance(raw[key], str):
262
+ raise ValueError(f"{key} must be a string")
263
+ sha = raw["transcripts_sha256"]
264
+ if sha is not None and not isinstance(sha, str):
265
+ raise ValueError("transcripts_sha256 must be null or a string")
266
+ return cls(
267
+ transcripts_url=str(raw["transcripts_url"]),
268
+ transcripts_sha256=sha,
269
+ archive_url_template=str(raw["archive_url_template"]),
270
+ n_speakers=_positive_int(raw, "n_speakers"),
271
+ expected_sample_rate_hz=_positive_int(raw, "expected_sample_rate_hz"),
272
+ target_sample_rate_hz=_positive_int(raw, "target_sample_rate_hz"),
273
+ val_speaker_fraction=_fraction(raw, "val_speaker_fraction"),
274
+ test_speaker_fraction=_fraction(raw, "test_speaker_fraction"),
275
+ speakers_per_wave=_positive_int(raw, "speakers_per_wave"),
276
+ read_prefixes=_words(raw, "read_prefixes"),
277
+ whisper_styles=_words(raw, "whisper_styles"),
278
+ plain_styles=_words(raw, "plain_styles"),
279
+ max_read_per_speaker=_non_negative_int(raw, "max_read_per_speaker"),
280
+ emotion_map=_str_map(raw, "emotion_map"),
281
+ event_map=_str_map(raw, "event_map"),
282
+ char_map=_str_map(raw, "char_map"),
283
+ )
284
+
285
+ def archive_url(self, speaker: str) -> str:
286
+ return self.archive_url_template.replace("{speaker}", speaker)
287
+
288
+
289
+ @dataclass(frozen=True)
290
+ class EarsSettings:
291
+ """Everything the builder reads from ``configs/data``."""
292
+
293
+ ears: EarsConfig
294
+ min_duration_ms: int
295
+ max_duration_ms: int
296
+ dataset: DatasetConfig
297
+ digest: str
298
+
299
+
300
+ def load_settings(config_dir: Path) -> EarsSettings:
301
+ """Load ``ears.yaml`` plus the duration limits (``ingest.yaml``) and bucket edges
302
+ (``dataset.yaml``) it shares with the Turkish pilot.
303
+
304
+ ``digest`` names everything a finished speaker depends on; it ignores
305
+ ``transcripts_sha256`` so that pinning the hash in the config later is allowed.
306
+ """
307
+ ears = EarsConfig.from_mapping(_load_yaml(config_dir / CONFIG_FILE))
308
+ ingest_cfg = IngestConfig.from_mapping(_load_yaml(config_dir / INGEST_FILE))
309
+ dataset = DatasetConfig.from_mapping(_load_yaml(config_dir / DATASET_FILE))
310
+ body = asdict(ears)
311
+ del body["transcripts_sha256"]
312
+ del body["speakers_per_wave"] # operational: does not change any output
313
+ canonical = json.dumps(
314
+ {
315
+ "ears": body,
316
+ "min_duration_ms": ingest_cfg.min_duration_ms,
317
+ "max_duration_ms": ingest_cfg.max_duration_ms,
318
+ "dataset": asdict(dataset),
319
+ },
320
+ sort_keys=True,
321
+ ensure_ascii=False,
322
+ )
323
+ return EarsSettings(
324
+ ears=ears,
325
+ min_duration_ms=ingest_cfg.min_duration_ms,
326
+ max_duration_ms=ingest_cfg.max_duration_ms,
327
+ dataset=dataset,
328
+ digest=hashlib.sha1(canonical.encode("utf-8")).hexdigest()[:12],
329
+ )
330
+
331
+
332
+ def parse_speakers(spec: str, n_speakers: int) -> list[str]:
333
+ """``"all"`` or ``"p001,p007"`` -> speaker ids.
334
+
335
+ Raises:
336
+ ValueError: on an empty list, a malformed or out-of-range id, or a repeated id.
337
+ """
338
+ if spec.strip() == "all":
339
+ return [f"p{i:03d}" for i in range(1, n_speakers + 1)]
340
+ ids = [part.strip() for part in spec.split(",")]
341
+ for speaker in ids:
342
+ match = _SPEAKER_RE.fullmatch(speaker)
343
+ if match is None or not 1 <= int(match.group(1)) <= n_speakers:
344
+ raise ValueError(f"invalid speaker {speaker!r}: use p001..p{n_speakers:03d} or 'all'")
345
+ if len(set(ids)) != len(ids):
346
+ raise ValueError(f"repeated speaker in {spec!r}")
347
+ return ids
348
+
349
+
350
+ # --------------------------------------------------------------------------------------
351
+ # Text
352
+ # --------------------------------------------------------------------------------------
353
+
354
+
355
+ class TextRejected(ValueError):
356
+ """The text of an item cannot be used; ``reason`` is the quarantine reason."""
357
+
358
+ def __init__(self, reason: str, detail: str) -> None:
359
+ super().__init__(f"{reason}: {detail}")
360
+ self.reason = reason
361
+ self.detail = detail
362
+
363
+
364
+ def en_lower(text: str) -> str:
365
+ """Lower-case ``A``-``Z`` only (English). Never use ``tr_lower`` (``I`` -> ``ı``) or
366
+ ``str.lower`` (``İ`` -> ``i`` + combining dot) on English text: every other character is
367
+ returned unchanged."""
368
+ return text.translate(_ASCII_LOWER)
369
+
370
+
371
+ def normalise_english(text: str, char_map: Mapping[str, str]) -> str:
372
+ """NFC, ``char_map`` replacements, ASCII lower-case and collapsed whitespace.
373
+
374
+ Returns ``""`` for text without any non-space character.
375
+ """
376
+ mapped = nfc(text).translate({ord(k): v for k, v in char_map.items()})
377
+ return " ".join(en_lower(mapped).split())
378
+
379
+
380
+ @dataclass(frozen=True)
381
+ class BuiltText:
382
+ """Model text (tag-spec surface) and the plain words the speaker says (may be empty)."""
383
+
384
+ tagged: str
385
+ plain: str
386
+
387
+
388
+ def build_text(selection: Selection, transcript: str | None, cfg: EarsConfig) -> BuiltText:
389
+ """Build the training text of one selected file.
390
+
391
+ Raises:
392
+ TextRejected: with reason ``no_transcript``, ``empty_text``, ``digits``,
393
+ ``tag_characters``, ``non_ascii_letter`` or ``text_encoding`` (the model
394
+ vocabulary cannot encode the result).
395
+ """
396
+ if selection.kind == "event":
397
+ assert selection.event is not None # classify_stem always sets it for events
398
+ tagged = tags.render([tags.Event(selection.event)])
399
+ plain = ""
400
+ else:
401
+ if transcript is None:
402
+ raise TextRejected("no_transcript", f"no entry for {selection.stem!r}")
403
+ plain = normalise_english(transcript, cfg.char_map)
404
+ if not plain:
405
+ raise TextRejected("empty_text", "transcript has no characters")
406
+ if any(ch.isdigit() for ch in plain):
407
+ raise TextRejected("digits", plain[:80])
408
+ if any(ch in _TAG_CHARS for ch in plain):
409
+ raise TextRejected("tag_characters", plain[:80])
410
+ if any(ch.isalpha() and not ch.isascii() for ch in plain):
411
+ raise TextRejected("non_ascii_letter", plain[:80])
412
+ if selection.kind == "whisper":
413
+ try:
414
+ span = tags.Span("whisper", tuple(tags.Word(w) for w in plain.split()))
415
+ except ValueError as err:
416
+ raise TextRejected("text_encoding", str(err)) from err
417
+ tagged = tags.render([span])
418
+ else:
419
+ tagged = plain
420
+ try:
421
+ ids = encode_text(tagged)
422
+ except ValueError as err: # unknown character, bad tag
423
+ raise TextRejected("text_encoding", str(err)) from err
424
+ if not ids:
425
+ raise TextRejected("empty_text", "text encodes to no tokens")
426
+ return BuiltText(tagged=tagged, plain=plain)
427
+
428
+
429
+ # --------------------------------------------------------------------------------------
430
+ # Selection
431
+ # --------------------------------------------------------------------------------------
432
+
433
+
434
+ @dataclass(frozen=True)
435
+ class Selection:
436
+ """A file that is used. ``emotion`` is set for ``emotion`` items (may be ``None``),
437
+ ``event`` for ``event`` items."""
438
+
439
+ stem: str
440
+ kind: Kind
441
+ emotion: str | None = None
442
+ event: str | None = None
443
+
444
+ @property
445
+ def label(self) -> str:
446
+ """Report category: ``emotion``, ``read``, ``whisper`` or ``event:<id>``."""
447
+ return f"event:{self.event}" if self.kind == "event" else self.kind
448
+
449
+
450
+ @dataclass(frozen=True)
451
+ class Skip:
452
+ """A file that is not used, with the counted reason."""
453
+
454
+ stem: str
455
+ reason: str
456
+
457
+
458
+ def classify_stem(stem: str, cfg: EarsConfig) -> Selection | Skip:
459
+ """Decide on one file stem (ADR-0007 table). Pure; thresholds and maps come from ``cfg``.
460
+
461
+ Skip reasons: ``freeform``, ``interjection``, ``melodic``, ``event_unmapped``,
462
+ ``style_unsupported``, ``unknown_stem``.
463
+ """
464
+ emo = _EMO_RE.fullmatch(stem)
465
+ if emo is not None:
466
+ if emo.group(2) != "sentences":
467
+ return Skip(stem, "freeform")
468
+ return Selection(stem, "emotion", emotion=cfg.emotion_map.get(emo.group(1)))
469
+ event = cfg.event_map.get(stem)
470
+ if event is not None:
471
+ return Selection(stem, "event", event=event)
472
+ read = _READ_RE.fullmatch(stem)
473
+ if read is not None and read.group(1) in cfg.read_prefixes:
474
+ style = read.group(3)
475
+ if style in cfg.whisper_styles:
476
+ return Selection(stem, "whisper")
477
+ if style in cfg.plain_styles:
478
+ return Selection(stem, "read")
479
+ return Skip(stem, "style_unsupported")
480
+ if stem.startswith("freeform_speech"):
481
+ return Skip(stem, "freeform")
482
+ for prefix, reason in (
483
+ ("interjection_", "interjection"),
484
+ ("melodic_", "melodic"),
485
+ ("nonverbal_", "event_unmapped"),
486
+ ("vegetative_", "event_unmapped"),
487
+ ):
488
+ if stem.startswith(prefix):
489
+ return Skip(stem, reason)
490
+ return Skip(stem, "unknown_stem")
491
+
492
+
493
+ @dataclass(frozen=True)
494
+ class PlannedItem:
495
+ """A selected archive member with its final text."""
496
+
497
+ stem: str
498
+ member: ZipMember
499
+ selection: Selection
500
+ text: BuiltText
501
+
502
+
503
+ @dataclass
504
+ class SelectionPlan:
505
+ """Result of :func:`select_members`: nothing is lost, every member is in one of three places."""
506
+
507
+ items: list[PlannedItem] = field(default_factory=list)
508
+ skipped: Counter[str] = field(default_factory=Counter)
509
+ quarantine: list[QuarantineRecord] = field(default_factory=list)
510
+
511
+
512
+ def select_members(
513
+ speaker: str,
514
+ members: Sequence[ZipMember],
515
+ transcripts: Mapping[str, str],
516
+ cfg: EarsConfig,
517
+ ) -> SelectionPlan:
518
+ """Decide on every archive member before anything is downloaded.
519
+
520
+ ``items`` come back in archive order (ascending ``header_offset``) so extraction profits
521
+ from the read-ahead window. Skipped members are counted by reason; text problems become
522
+ quarantine records (stage ``ears``, item id ``<speaker>/<stem>``).
523
+ """
524
+ plan = SelectionPlan()
525
+ seen: set[str] = set()
526
+ for member in sorted(members, key=lambda m: m.header_offset):
527
+ if member.is_dir:
528
+ plan.skipped["directory"] += 1
529
+ continue
530
+ parts = member.name.split("/")
531
+ if len(parts) != 2 or parts[0] != speaker:
532
+ plan.skipped["outside_speaker_folder"] += 1
533
+ continue
534
+ if not parts[1].endswith(".wav"):
535
+ plan.skipped["not_wav"] += 1
536
+ continue
537
+ stem = parts[1][: -len(".wav")]
538
+ decision = classify_stem(stem, cfg)
539
+ if isinstance(decision, Skip):
540
+ plan.skipped[decision.reason] += 1
541
+ continue
542
+ item_id = f"{speaker}/{stem}"
543
+ if stem in seen:
544
+ plan.quarantine.append(
545
+ QuarantineRecord(item_id, STAGE, "duplicate_member", {"name": member.name})
546
+ )
547
+ continue
548
+ seen.add(stem)
549
+ try:
550
+ text = build_text(decision, transcripts.get(stem), cfg)
551
+ except TextRejected as err:
552
+ plan.quarantine.append(
553
+ QuarantineRecord(item_id, STAGE, err.reason, {"detail": err.detail})
554
+ )
555
+ continue
556
+ plan.items.append(PlannedItem(stem, member, decision, text))
557
+ _cap_reads(plan, speaker, cfg.max_read_per_speaker)
558
+ return plan
559
+
560
+
561
+ def _cap_reads(plan: SelectionPlan, speaker: str, cap: int) -> None:
562
+ """Keep at most ``cap`` plain reading items per speaker (0 = no limit; ADR-0007: plain
563
+ reading is a low share). The kept ones are a deterministic hash choice of the stems;
564
+ the rest are counted as skipped ``read_cap``."""
565
+ reads = [item for item in plan.items if item.selection.kind == "read"]
566
+ if cap == 0 or len(reads) <= cap:
567
+ return
568
+ ranked = sorted(reads, key=lambda it: hashlib.sha1(f"{speaker}/{it.stem}".encode()).hexdigest())
569
+ dropped = {id(item) for item in ranked[cap:]}
570
+ plan.items = [item for item in plan.items if id(item) not in dropped]
571
+ plan.skipped["read_cap"] += len(dropped)
572
+
573
+
574
+ # --------------------------------------------------------------------------------------
575
+ # Audio
576
+ # --------------------------------------------------------------------------------------
577
+
578
+
579
+ class ItemRejected(Exception):
580
+ """An audio file cannot be used; ``reason`` is the quarantine reason."""
581
+
582
+ def __init__(self, reason: str, **details: object) -> None:
583
+ super().__init__(f"{reason}: {details}")
584
+ self.reason = reason
585
+ self.details = details
586
+
587
+
588
+ @dataclass(frozen=True)
589
+ class AudioFacts:
590
+ """Measurements of a converted file: ``duration_ms`` of the 16 kHz audio, RMS level."""
591
+
592
+ duration_ms: int
593
+ loudness_dbfs: float
594
+
595
+
596
+ def resample(audio: FloatArray, rate_hz: int, target_hz: int) -> FloatArray:
597
+ """Polyphase resampling of a mono signal (anti-aliased) from ``rate_hz`` to ``target_hz``.
598
+
599
+ Args:
600
+ audio: ``(num_samples,)`` float signal.
601
+ rate_hz: Source rate.
602
+ target_hz: Target rate.
603
+
604
+ Returns:
605
+ ``(ceil(num_samples * target_hz / rate_hz),)`` float64 signal.
606
+ """
607
+ if rate_hz == target_hz:
608
+ return np.asarray(audio, dtype=np.float64)
609
+ divisor = math.gcd(rate_hz, target_hz)
610
+ out = signal.resample_poly(audio, target_hz // divisor, rate_hz // divisor)
611
+ return np.asarray(out, dtype=np.float64)
612
+
613
+
614
+ def convert_to_target_rate(raw: Path, dest: Path, cfg: EarsConfig) -> None:
615
+ """Read a raw EARS wav, check it, resample to ``target_sample_rate_hz`` mono and write a
616
+ 32-bit float wav atomically to ``dest``.
617
+
618
+ Raises:
619
+ ItemRejected: ``unreadable_audio``, ``not_mono``, ``unexpected_sample_rate``,
620
+ ``empty_audio`` or ``non_finite_audio``.
621
+ """
622
+ import soundfile # type: ignore[import-untyped] # bench extra, untyped
623
+
624
+ try:
625
+ audio, rate_hz = soundfile.read(str(raw), dtype="float64", always_2d=True)
626
+ except (RuntimeError, soundfile.LibsndfileError) as err:
627
+ raise ItemRejected("unreadable_audio", error=str(err)) from err
628
+ if audio.shape[1] != 1:
629
+ raise ItemRejected("not_mono", channels=int(audio.shape[1]))
630
+ if rate_hz != cfg.expected_sample_rate_hz:
631
+ raise ItemRejected("unexpected_sample_rate", sample_rate_hz=int(rate_hz))
632
+ mono = np.ascontiguousarray(audio[:, 0])
633
+ if mono.shape[0] == 0:
634
+ raise ItemRejected("empty_audio")
635
+ if not np.all(np.isfinite(mono)):
636
+ raise ItemRejected("non_finite_audio")
637
+ converted = resample(mono, int(rate_hz), cfg.target_sample_rate_hz)
638
+ buffer = io.BytesIO()
639
+ soundfile.write(buffer, converted, cfg.target_sample_rate_hz, format="WAV", subtype="FLOAT")
640
+ atomic_write_bytes(dest, buffer.getvalue())
641
+
642
+
643
+ def measure_audio(
644
+ path: Path, cfg: EarsConfig, min_duration_ms: int, max_duration_ms: int
645
+ ) -> AudioFacts:
646
+ """Duration and loudness of a converted wav; enforces the duration limits.
647
+
648
+ The loudness is :func:`nida.data.acoustic.loudness_dbfs`, the measure the pilot ingest
649
+ stores, here on the 16 kHz signal the VAE hears.
650
+
651
+ Raises:
652
+ ItemRejected: ``unreadable_audio``, ``too_short``, ``too_long`` or ``silent``.
653
+ ValueError: if the file is not at ``target_sample_rate_hz`` (the cache is inconsistent
654
+ with the config).
655
+ """
656
+ import soundfile
657
+
658
+ try:
659
+ audio, rate_hz = soundfile.read(str(path), dtype="float64", always_2d=True)
660
+ except (RuntimeError, soundfile.LibsndfileError) as err:
661
+ raise ItemRejected("unreadable_audio", error=str(err)) from err
662
+ if rate_hz != cfg.target_sample_rate_hz or audio.shape[1] != 1:
663
+ raise ValueError(
664
+ f"{path}: expected {cfg.target_sample_rate_hz} Hz mono, got {rate_hz} Hz x "
665
+ f"{audio.shape[1]}; delete the cache folder"
666
+ )
667
+ mono = np.ascontiguousarray(audio[:, 0])
668
+ duration_ms = round(1000 * mono.shape[0] / rate_hz)
669
+ if duration_ms < min_duration_ms:
670
+ raise ItemRejected("too_short", duration_ms=duration_ms)
671
+ if duration_ms > max_duration_ms:
672
+ raise ItemRejected("too_long", duration_ms=duration_ms)
673
+ if float(np.max(np.abs(mono))) == 0.0:
674
+ raise ItemRejected("silent")
675
+ return AudioFacts(duration_ms=duration_ms, loudness_dbfs=float(acoustic.loudness_dbfs(mono)))
676
+
677
+
678
+ # --------------------------------------------------------------------------------------
679
+ # Buckets and splits
680
+ # --------------------------------------------------------------------------------------
681
+
682
+
683
+ def bucket_of(value: float, edges: Sequence[float]) -> int:
684
+ """Bucket index of ``value``: the number of ``edges`` that are ``<= value``
685
+ (``numpy.searchsorted(edges, value, side="right")``, as the pilot dataset stage)."""
686
+ return bisect_right(edges, value)
687
+
688
+
689
+ def rate_chars_per_s(plain_text: str, duration_ms: int) -> float:
690
+ """Alphanumeric characters of the plain text per second of audio (pilot formula)."""
691
+ return sum(ch.isalnum() for ch in plain_text) / (duration_ms / 1000)
692
+
693
+
694
+ def speaker_split(group: str, val_fraction: float, test_fraction: float, seed: int) -> str:
695
+ """Split of a speaker group: a pure function of ``(seed, group)`` (:mod:`nida.data.splits`)."""
696
+ fractions = {
697
+ "train": 1.0 - val_fraction - test_fraction,
698
+ "val": val_fraction,
699
+ "test": test_fraction,
700
+ }
701
+ return assign_splits({"speaker": group}, fractions, str(seed))["speaker"]
702
+
703
+
704
+ # --------------------------------------------------------------------------------------
705
+ # The builder
706
+ # --------------------------------------------------------------------------------------
707
+
708
+
709
+ @dataclass(frozen=True)
710
+ class _Measured:
711
+ item: PlannedItem
712
+ segment_id: str
713
+ wav: Path
714
+ facts: AudioFacts
715
+
716
+
717
+ @dataclass
718
+ class _Prepared:
719
+ """A speaker whose audio is ready for the VAE: nothing is written for it yet."""
720
+
721
+ plan: SelectionPlan
722
+ quarantine: list[QuarantineRecord]
723
+ measured: list[_Measured]
724
+
725
+
726
+ @dataclass
727
+ class Ears:
728
+ """EARS -> latent dataset builder (see the module docstring).
729
+
730
+ Attributes:
731
+ root: Durable directory (Drive on Colab).
732
+ cache: Local scratch for raw members, 16 kHz audio and latent arrays.
733
+ envs_root: Where the worker virtual environments live.
734
+ repo_dir: Repository root (environment builds need it).
735
+ sources_path: ``configs/sources.yaml``.
736
+ config_dir: ``configs/data``.
737
+ seed: Split seed (recorded in ``plan.json``).
738
+ runner: Worker command runner (injectable).
739
+ opener: Network access (injectable).
740
+ sleep: Backoff sleep for retries (injectable).
741
+ """
742
+
743
+ root: Path
744
+ cache: Path
745
+ envs_root: Path
746
+ repo_dir: Path
747
+ sources_path: Path
748
+ config_dir: Path
749
+ seed: int = DEFAULT_SEED
750
+ uv: str | None = None
751
+ runner: Callable[[Sequence[str]], int] = _stream_runner
752
+ opener: fetch.Opener = fetch.default_opener
753
+ sleep: Callable[[float], None] = time.sleep
754
+ _python: Path | None = field(default=None, init=False, repr=False)
755
+
756
+ # ---------------------------------------------------------------- paths
757
+
758
+ @property
759
+ def dataset_dir(self) -> Path:
760
+ return self.root / "dataset"
761
+
762
+ def _speaker_dir(self, speaker: str) -> Path:
763
+ return self.root / "speakers" / speaker
764
+
765
+ def _npy(self, segment: str) -> Path:
766
+ return self.cache / "arrays" / "vae" / f"{segment}.npy"
767
+
768
+ # ---------------------------------------------------------------- run
769
+
770
+ def run(self, speakers: Sequence[str]) -> dict[str, object]:
771
+ """Build (or continue) the dataset for ``speakers``.
772
+
773
+ Finished speakers are skipped. The dataset directory, ``report.md`` and
774
+ ``stats.json`` are rewritten from *all* finished speakers of the root at the end.
775
+
776
+ Returns:
777
+ Summary with ``dataset``, ``speakers_done``, ``segments``, ``hours``,
778
+ ``skipped`` and ``quarantined`` counts, and ``failed_speakers``.
779
+
780
+ Raises:
781
+ SourceNotApprovedError: before any download, if a source is not approved.
782
+ ValueError: on a plan/config/transcripts mismatch with the root.
783
+ RuntimeError: if a worker fails, a speaker could not be fetched (after all other
784
+ speakers were processed and the dataset was assembled), or nothing was built.
785
+ """
786
+ settings = load_settings(self.config_dir)
787
+ model = DATA_MODELS[VAE_KEY]
788
+ require_approved({SOURCE_ID, model.source_id}, load_sources(self.sources_path))
789
+ wanted = list(speakers)
790
+ for speaker in wanted:
791
+ parse_speakers(speaker, settings.ears.n_speakers)
792
+ transcripts, transcripts_sha = self._prepare_plan(settings)
793
+ failed: dict[str, str] = {}
794
+ pending: list[str] = []
795
+ for speaker in dict.fromkeys(wanted):
796
+ if (self._speaker_dir(speaker) / "done.json").is_file():
797
+ log.info("EARS %s already done", speaker)
798
+ else:
799
+ pending.append(speaker)
800
+ wave_size = settings.ears.speakers_per_wave
801
+ for start in range(0, len(pending), wave_size):
802
+ wave = pending[start : start + wave_size]
803
+ prepared: dict[str, _Prepared] = {}
804
+ for speaker in wave:
805
+ log.info("EARS %s (%d/%d)", speaker, pending.index(speaker) + 1, len(pending))
806
+ try:
807
+ prepared[speaker] = self._prepare_speaker(speaker, settings, transcripts)
808
+ except fetch.FetchError as err: # network or archive problem: others still run
809
+ log.error("EARS %s failed: %s", speaker, err)
810
+ failed[speaker] = str(err)
811
+ results = self._encode([m for entry in prepared.values() for m in entry.measured])
812
+ for speaker, entry in prepared.items():
813
+ self._finish_speaker(speaker, settings, entry, results)
814
+ summary = self._assemble(settings, transcripts_sha, failed)
815
+ if failed:
816
+ raise RuntimeError(
817
+ f"{len(failed)} speaker(s) failed and can be retried by running again: "
818
+ + "; ".join(f"{s}: {e[:200]}" for s, e in sorted(failed.items()))
819
+ )
820
+ return summary
821
+
822
+ # ---------------------------------------------------------------- plan and transcripts
823
+
824
+ def _prepare_plan(self, settings: EarsSettings) -> tuple[dict[str, str], str]:
825
+ """Check/write ``plan.json`` and fetch the pinned transcripts."""
826
+ model = DATA_MODELS[VAE_KEY]
827
+ core: dict[str, object] = {
828
+ "seed": self.seed,
829
+ "config_digest": settings.digest,
830
+ "archive_url_template": settings.ears.archive_url_template,
831
+ "transcripts_url": settings.ears.transcripts_url,
832
+ "vae": [model.ref.model_id, model.ref.revision],
833
+ }
834
+ plan_path = self.root / "plan.json"
835
+ previous: dict[str, object] | None = None
836
+ if plan_path.is_file():
837
+ loaded = json.loads(plan_path.read_text(encoding="utf-8"))
838
+ previous = dict(loaded)
839
+ recorded_sha = previous.pop("transcripts_sha256", None)
840
+ if previous != core:
841
+ raise ValueError(
842
+ f"{plan_path} was written for a different plan: {previous} != {core}"
843
+ )
844
+ if (
845
+ settings.ears.transcripts_sha256 is not None
846
+ and recorded_sha is not None
847
+ and settings.ears.transcripts_sha256 != recorded_sha
848
+ ):
849
+ raise ValueError(
850
+ f"config pins transcripts {settings.ears.transcripts_sha256} but the root "
851
+ f"was built with {recorded_sha}"
852
+ )
853
+ expected = recorded_sha if isinstance(recorded_sha, str) else None
854
+ else:
855
+ expected = None
856
+ expected = settings.ears.transcripts_sha256 or expected
857
+ downloaded = fetch.download(
858
+ settings.ears.transcripts_url,
859
+ self.root / "source" / "transcripts.json",
860
+ expected_sha256=expected,
861
+ opener=self.opener,
862
+ sleep=self.sleep,
863
+ )
864
+ if previous is None:
865
+ atomic_write_json(plan_path, {**core, "transcripts_sha256": downloaded.sha256})
866
+ return _parse_transcripts(downloaded.path), downloaded.sha256
867
+
868
+ # ---------------------------------------------------------------- speaker
869
+
870
+ def _python_exe(self) -> Path:
871
+ if self._python is None:
872
+ if self.uv is None:
873
+ self.uv = bench_envs.find_uv(bench_envs.subprocess_runner)
874
+ spec = envs.data_env_for_worker(DATA_MODELS[VAE_KEY].worker)
875
+ self._python = bench_envs.ensure_env(spec, self.envs_root, self.repo_dir, uv=self.uv)
876
+ return self._python
877
+
878
+ def _prepare_speaker(
879
+ self, speaker: str, settings: EarsSettings, transcripts: Mapping[str, str]
880
+ ) -> _Prepared:
881
+ """List the archive, decide on every member, fetch + convert + measure the selected."""
882
+ cfg = settings.ears
883
+ QuarantineLog(self._speaker_dir(speaker)).path.unlink(missing_ok=True) # rebuilt below
884
+ archive = RemoteZip(cfg.archive_url(speaker), opener=self.opener, sleep=self.sleep)
885
+ members = archive.members()
886
+ plan = select_members(speaker, members, transcripts, cfg)
887
+ quarantine = list(plan.quarantine)
888
+ log.info(
889
+ "EARS %s: %d members, %d selected, %d quarantined, skipped %s",
890
+ speaker,
891
+ len(members),
892
+ len(plan.items),
893
+ len(quarantine),
894
+ dict(plan.skipped),
895
+ )
896
+ measured: list[_Measured] = []
897
+ for item in plan.items: # archive order: one read-ahead window serves neighbours
898
+ item_id = f"{speaker}/{item.stem}"
899
+ wav = self.cache / "audio16k" / speaker / f"{item.stem}.wav"
900
+ try:
901
+ if not wav.is_file():
902
+ raw = archive.extract(
903
+ item.member, self.cache / "raw" / speaker / f"{item.stem}.wav"
904
+ )
905
+ convert_to_target_rate(raw, wav, cfg)
906
+ raw.unlink()
907
+ facts = measure_audio(wav, cfg, settings.min_duration_ms, settings.max_duration_ms)
908
+ except ItemRejected as err:
909
+ quarantine.append(QuarantineRecord(item_id, STAGE, err.reason, dict(err.details)))
910
+ continue
911
+ sid = segment_id(SOURCE_ID, item_id, 0, facts.duration_ms)
912
+ measured.append(_Measured(item, sid, wav, facts))
913
+ return _Prepared(plan, quarantine, measured)
914
+
915
+ def _finish_speaker(
916
+ self,
917
+ speaker: str,
918
+ settings: EarsSettings,
919
+ prepared: _Prepared,
920
+ results: Mapping[str, WorkResult],
921
+ ) -> None:
922
+ """Write the speaker's outputs, then drop its scratch files."""
923
+ self._write_speaker(
924
+ speaker, settings, prepared.measured, results, prepared.plan, prepared.quarantine
925
+ )
926
+ for sub in ("raw", "audio16k"):
927
+ shutil.rmtree(self.cache / sub / speaker, ignore_errors=True)
928
+ for entry in prepared.measured:
929
+ self._npy(entry.segment_id).unlink(missing_ok=True)
930
+
931
+ def _encode(self, measured: Sequence[_Measured]) -> dict[str, WorkResult]:
932
+ """Run the pilot's ``vae_encode`` worker once over the audio of a wave (resumable)."""
933
+ if not measured:
934
+ return {}
935
+ model = DATA_MODELS[VAE_KEY]
936
+ python = self._python_exe()
937
+ out_dir = self.root / "workers" / "vae"
938
+ results_path = out_dir / "results.jsonl"
939
+ wanted = {m.segment_id for m in measured}
940
+ summary_path = out_dir / "results.summary.json"
941
+ if results_path.exists() and summary_path.exists():
942
+ previous = read_summary(summary_path)
943
+ if (previous.model_id, previous.revision) != (model.ref.model_id, model.ref.revision):
944
+ raise RuntimeError(
945
+ f"existing latents come from {previous.model_id}@"
946
+ f"{previous.revision}, not {model.ref.model_id}@{model.ref.revision}; "
947
+ "move them away first"
948
+ )
949
+ if results_path.exists():
950
+ existing = load_results(results_path, repair=True)
951
+ stale = [
952
+ cid
953
+ for cid, res in existing.items()
954
+ if cid not in wanted
955
+ or res.output is None # failed before: try again (a stuck item fails again)
956
+ or not Path(str(res.output.get("out_path", ""))).is_file()
957
+ ]
958
+ if stale:
959
+ log.info("dropping %d stray, failed or lost worker results", len(stale))
960
+ drop_results(results_path, stale)
961
+ requests = [
962
+ WorkRequest(m.segment_id, str(m.wav), {"out_path": str(self._npy(m.segment_id))})
963
+ for m in measured
964
+ ]
965
+ write_requests(out_dir / "requests.jsonl", requests)
966
+ command = [
967
+ str(python),
968
+ "-m",
969
+ envs.worker_module_name(model.worker),
970
+ "--requests",
971
+ str(out_dir / "requests.jsonl"),
972
+ "--results",
973
+ str(results_path),
974
+ "--model-id",
975
+ model.ref.model_id,
976
+ "--revision",
977
+ model.ref.revision,
978
+ "--batch-size",
979
+ str(model.batch_size),
980
+ "--device",
981
+ "cuda",
982
+ ]
983
+ code = self.runner(command)
984
+ if code != 0:
985
+ raise RuntimeError(f"vae_encode worker exited with code {code}")
986
+ summary = read_summary(summary_path)
987
+ total = summary.n_ok + summary.n_error
988
+ if total != len(requests):
989
+ raise RuntimeError(f"{total} latent results for {len(requests)} requests")
990
+ if summary.n_error > MAX_ERROR_FRACTION * total:
991
+ raise RuntimeError(f"{summary.n_error}/{total} items failed in the worker")
992
+ return load_results(results_path)
993
+
994
+ def _write_speaker(
995
+ self,
996
+ speaker: str,
997
+ settings: EarsSettings,
998
+ measured: Sequence[_Measured],
999
+ results: Mapping[str, WorkResult],
1000
+ plan: SelectionPlan,
1001
+ quarantine: list[QuarantineRecord],
1002
+ ) -> dict[str, object]:
1003
+ """Shard, records, quarantine log and finally the done marker of one speaker."""
1004
+ cfg = settings.ears
1005
+ group = f"ears-{speaker}"
1006
+ split = speaker_split(group, cfg.val_speaker_fraction, cfg.test_speaker_fraction, self.seed)
1007
+ rate_edges = list(settings.dataset.rate_bucket_edges_chars_per_s)
1008
+ loud_edges = list(settings.dataset.loudness_bucket_edges_dbfs)
1009
+ shard_name = f"shard-{speaker}.npy"
1010
+ rows: list[dict[str, object]] = []
1011
+ arrays: list[npt.NDArray[np.float32]] = []
1012
+ kinds: dict[str, dict[str, int]] = {}
1013
+ latent_dim: int | None = None
1014
+ frame_hop_ms: float | None = None
1015
+ frames = 0
1016
+ for entry in sorted(measured, key=lambda m: m.item.stem):
1017
+ item_id = f"{speaker}/{entry.item.stem}"
1018
+ result = results.get(entry.segment_id)
1019
+ if result is None:
1020
+ raise RuntimeError(f"{speaker}: the worker returned no result for {item_id}")
1021
+ if result.output is None:
1022
+ quarantine.append(
1023
+ QuarantineRecord(item_id, STAGE, "vae_error", {"error": str(result.error)})
1024
+ )
1025
+ continue
1026
+ array = np.load(str(result.output["out_path"]), allow_pickle=False)
1027
+ hop = float(str(result.output["frame_hop_ms"]))
1028
+ if array.ndim != 2 or array.shape[0] < 1:
1029
+ raise ValueError(f"{item_id}: latents must be 2-D with frames, got {array.shape}")
1030
+ if frame_hop_ms is not None and hop != frame_hop_ms:
1031
+ raise ValueError(f"{item_id}: frame hop {hop} != {frame_hop_ms}")
1032
+ if latent_dim is not None and int(array.shape[1]) != latent_dim:
1033
+ raise ValueError(f"{item_id}: latent_dim {array.shape[1]} != {latent_dim}")
1034
+ frame_hop_ms, latent_dim = hop, int(array.shape[1])
1035
+ if not np.all(np.isfinite(array)) or float(np.max(np.abs(array))) > FLOAT16_MAX_ABS:
1036
+ quarantine.append(QuarantineRecord(item_id, STAGE, "latents_not_float16", {}))
1037
+ continue
1038
+ num_frames = int(array.shape[0])
1039
+ n_tokens = len(encode_text(entry.item.text.tagged))
1040
+ if n_tokens > num_frames:
1041
+ quarantine.append(
1042
+ QuarantineRecord(
1043
+ item_id,
1044
+ STAGE,
1045
+ REJECT_TEXT_TOO_LONG,
1046
+ {"text_tokens": n_tokens, "num_frames": num_frames},
1047
+ )
1048
+ )
1049
+ continue
1050
+ plain = entry.item.text.plain
1051
+ rows.append(
1052
+ {
1053
+ "segment_id": entry.segment_id,
1054
+ "shard": shard_name,
1055
+ "offset": frames,
1056
+ "num_frames": num_frames,
1057
+ "text": entry.item.text.tagged,
1058
+ "emotion": entry.item.selection.emotion,
1059
+ "intensity": None,
1060
+ "rate_bucket": (
1061
+ bucket_of(rate_chars_per_s(plain, entry.facts.duration_ms), rate_edges)
1062
+ if plain
1063
+ else None
1064
+ ),
1065
+ "loudness_bucket": bucket_of(entry.facts.loudness_dbfs, loud_edges),
1066
+ "group": group,
1067
+ "split": split,
1068
+ "language": LANGUAGE,
1069
+ }
1070
+ )
1071
+ arrays.append(array.astype(np.float32))
1072
+ frames += num_frames
1073
+ stat = kinds.setdefault(entry.item.selection.label, {"segments": 0, "audio_ms": 0})
1074
+ stat["segments"] += 1
1075
+ stat["audio_ms"] += entry.facts.duration_ms
1076
+
1077
+ spk_dir = self._speaker_dir(speaker)
1078
+ if arrays:
1079
+ buffer = io.BytesIO()
1080
+ np.save(buffer, np.concatenate(arrays).astype(np.float16), allow_pickle=False)
1081
+ atomic_write_bytes(self.dataset_dir / shard_name, buffer.getvalue())
1082
+ atomic_write_text(
1083
+ spk_dir / "records.jsonl",
1084
+ "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
1085
+ )
1086
+ QuarantineLog(spk_dir).append(quarantine)
1087
+ marker: dict[str, object] = {
1088
+ "speaker": speaker,
1089
+ "group": group,
1090
+ "split": split,
1091
+ "segments": len(rows),
1092
+ "frames": frames,
1093
+ "audio_ms": sum(s["audio_ms"] for s in kinds.values()),
1094
+ "latent_dim": latent_dim if rows else None,
1095
+ "frame_hop_ms": frame_hop_ms if rows else None,
1096
+ "shard": shard_name if rows else None,
1097
+ "kinds": kinds,
1098
+ "skipped": dict(sorted(plan.skipped.items())),
1099
+ "quarantined": dict(sorted(Counter(q.reason for q in quarantine).items())),
1100
+ }
1101
+ atomic_write_json(spk_dir / "done.json", marker) # written last: the speaker is done
1102
+ return marker
1103
+
1104
+ # ---------------------------------------------------------------- assembly
1105
+
1106
+ def _assemble(
1107
+ self, settings: EarsSettings, transcripts_sha: str, failed: Mapping[str, str]
1108
+ ) -> dict[str, object]:
1109
+ """Index, meta, report and stats from every finished speaker of the root."""
1110
+ markers: list[dict[str, object]] = []
1111
+ speakers_dir = self.root / "speakers"
1112
+ for done_path in sorted(speakers_dir.glob("p*/done.json")) if speakers_dir.is_dir() else []:
1113
+ markers.append(json.loads(done_path.read_text(encoding="utf-8")))
1114
+ with_data = [m for m in markers if m["segments"]]
1115
+ if not with_data:
1116
+ raise RuntimeError(
1117
+ f"no EARS segment built (speakers done: {len(markers)}, failed: {sorted(failed)})"
1118
+ )
1119
+ hops = {float(str(m["frame_hop_ms"])) for m in with_data}
1120
+ dims = {int(str(m["latent_dim"])) for m in with_data}
1121
+ if len(hops) != 1 or len(dims) != 1:
1122
+ raise ValueError(
1123
+ f"speakers disagree on frame hop {sorted(hops)} or latent_dim {sorted(dims)}"
1124
+ )
1125
+ index_lines: list[str] = []
1126
+ for marker in with_data:
1127
+ lines = (
1128
+ (self._speaker_dir(str(marker["speaker"])) / "records.jsonl")
1129
+ .read_text(encoding="utf-8")
1130
+ .splitlines(keepends=True)
1131
+ )
1132
+ if len(lines) != marker["segments"]:
1133
+ raise ValueError(f"{marker['speaker']}: records.jsonl and done.json disagree")
1134
+ if not (self.dataset_dir / str(marker["shard"])).is_file():
1135
+ raise ValueError(f"{marker['speaker']}: shard {marker['shard']} is missing")
1136
+ index_lines.extend(lines)
1137
+ total_segments = len(index_lines)
1138
+ total_frames = sum(int(str(m["frames"])) for m in with_data)
1139
+ atomic_write_text(self.dataset_dir / "index.jsonl", "".join(index_lines))
1140
+ source = load_sources(self.sources_path)[SOURCE_ID]
1141
+ model = DATA_MODELS[VAE_KEY]
1142
+ meta: dict[str, object] = {
1143
+ "segments": total_segments,
1144
+ "frames": total_frames,
1145
+ "latent_dim": dims.pop(),
1146
+ "frame_hop_ms": hops.pop(),
1147
+ "rate_bucket_edges_chars_per_s": list(settings.dataset.rate_bucket_edges_chars_per_s),
1148
+ "loudness_bucket_edges_dbfs": list(settings.dataset.loudness_bucket_edges_dbfs),
1149
+ "source": SOURCE_ID,
1150
+ "license": source.get("license"),
1151
+ "language": LANGUAGE,
1152
+ "transcripts_sha256": transcripts_sha,
1153
+ "transcripts_url": settings.ears.transcripts_url,
1154
+ "config_digest": settings.digest,
1155
+ "seed": self.seed,
1156
+ "vae": [model.ref.model_id, model.ref.revision],
1157
+ "speakers": {
1158
+ str(m["speaker"]): {"split": m["split"], "segments": m["segments"]} for m in markers
1159
+ },
1160
+ }
1161
+ atomic_write_json(self.dataset_dir / "meta.json", meta)
1162
+
1163
+ loader = self._check_with_loader()
1164
+ stats = _stats(markers, loader, settings, failed, meta)
1165
+ atomic_write_json(self.root / "stats.json", stats)
1166
+ atomic_write_text(self.root / "report.md", _render_report(stats))
1167
+ rejected = {s: c for s, c in loader.items() if c["rejected"]}
1168
+ if rejected:
1169
+ raise RuntimeError(f"nida.train.data rejects records the builder accepted: {rejected}")
1170
+ return {
1171
+ "dataset": str(self.dataset_dir),
1172
+ "speakers_done": len(markers),
1173
+ "segments": total_segments,
1174
+ "hours": stats["hours"],
1175
+ "skipped": stats["skipped"],
1176
+ "quarantined": stats["quarantined"],
1177
+ "failed_speakers": dict(failed),
1178
+ }
1179
+
1180
+ def _check_with_loader(self) -> dict[str, dict[str, object]]:
1181
+ """Load every split with the training loader; counts per split (must not reject)."""
1182
+ out: dict[str, dict[str, object]] = {}
1183
+ for split in SPLITS:
1184
+ try:
1185
+ dataset, report = load_latent_dataset(self.dataset_dir, split)
1186
+ except DataFormatError as err:
1187
+ raise RuntimeError(f"the assembled dataset is invalid: {err}") from err
1188
+ dataset.close()
1189
+ out[split] = {
1190
+ "records": report.num_in_split,
1191
+ "accepted": report.num_accepted,
1192
+ "rejected": report.num_rejected,
1193
+ "rejection_counts": dict(report.rejection_counts),
1194
+ }
1195
+ return out
1196
+
1197
+
1198
+ # --------------------------------------------------------------------------------------
1199
+ # Reporting
1200
+ # --------------------------------------------------------------------------------------
1201
+
1202
+
1203
+ def _parse_transcripts(path: Path) -> dict[str, str]:
1204
+ obj = json.loads(path.read_text(encoding="utf-8"))
1205
+ if not isinstance(obj, dict) or not obj:
1206
+ raise ValueError(f"{path}: expected a non-empty mapping stem -> text")
1207
+ bad = [k for k, v in obj.items() if not isinstance(k, str) or not isinstance(v, str)]
1208
+ if bad:
1209
+ raise ValueError(f"{path}: entries must map strings to strings, e.g. {bad[:3]}")
1210
+ return dict(obj)
1211
+
1212
+
1213
+ def _sum_counts(markers: Sequence[Mapping[str, object]], key: str) -> dict[str, int]:
1214
+ total: Counter[str] = Counter()
1215
+ for marker in markers:
1216
+ value = marker[key]
1217
+ assert isinstance(value, dict)
1218
+ total.update({str(k): int(v) for k, v in value.items()})
1219
+ return dict(sorted(total.items(), key=lambda kv: (-kv[1], kv[0])))
1220
+
1221
+
1222
+ def _stats(
1223
+ markers: Sequence[Mapping[str, object]],
1224
+ loader: Mapping[str, Mapping[str, object]],
1225
+ settings: EarsSettings,
1226
+ failed: Mapping[str, str],
1227
+ meta: Mapping[str, object],
1228
+ ) -> dict[str, object]:
1229
+ kinds: dict[str, dict[str, int]] = {}
1230
+ for marker in markers:
1231
+ value = marker["kinds"]
1232
+ assert isinstance(value, dict)
1233
+ for label, stat in value.items():
1234
+ into = kinds.setdefault(label, {"segments": 0, "audio_ms": 0})
1235
+ into["segments"] += int(stat["segments"])
1236
+ into["audio_ms"] += int(stat["audio_ms"])
1237
+ splits: dict[str, dict[str, int]] = {s: {"speakers": 0, "segments": 0} for s in SPLITS}
1238
+ for marker in markers:
1239
+ entry = splits[str(marker["split"])]
1240
+ entry["speakers"] += 1
1241
+ entry["segments"] += int(str(marker["segments"]))
1242
+ audio_ms = sum(int(str(m["audio_ms"])) for m in markers)
1243
+ return {
1244
+ "speakers_done": len(markers),
1245
+ "speakers_total": settings.ears.n_speakers,
1246
+ "segments": meta["segments"],
1247
+ "frames": meta["frames"],
1248
+ "hours": round(audio_ms / 3_600_000, 3),
1249
+ "kinds": dict(sorted(kinds.items())),
1250
+ "splits": splits,
1251
+ "skipped": _sum_counts(markers, "skipped"),
1252
+ "quarantined": _sum_counts(markers, "quarantined"),
1253
+ "loader": {k: dict(v) for k, v in loader.items()},
1254
+ "failed_speakers": dict(failed),
1255
+ "transcripts_sha256": meta["transcripts_sha256"],
1256
+ "config_digest": settings.digest,
1257
+ }
1258
+
1259
+
1260
+ def _render_report(stats: Mapping[str, object]) -> str:
1261
+ kinds = stats["kinds"]
1262
+ splits = stats["splits"]
1263
+ loader = stats["loader"]
1264
+ assert isinstance(kinds, dict) and isinstance(splits, dict) and isinstance(loader, dict)
1265
+ lines = [
1266
+ "# EARS latent veri seti raporu",
1267
+ "",
1268
+ f"- Konuşmacı: {stats['speakers_done']}/{stats['speakers_total']} tamamlandı",
1269
+ f"- Segment: {stats['segments']}, süre: {stats['hours']} sa, kare: {stats['frames']}",
1270
+ f"- Yapılandırma özeti: `{stats['config_digest']}`; transkript SHA-256: "
1271
+ f"`{stats['transcripts_sha256']}`",
1272
+ "",
1273
+ "## Seçilen türler",
1274
+ "",
1275
+ *(
1276
+ f"- {label}: {v['segments']} segment, {v['audio_ms'] / 3_600_000:.3f} sa"
1277
+ for label, v in kinds.items()
1278
+ ),
1279
+ "",
1280
+ "## Bölmeler (konuşmacı-ayrık)",
1281
+ "",
1282
+ *(
1283
+ f"- {name}: {v['speakers']} konuşmacı, {v['segments']} segment"
1284
+ for name, v in splits.items()
1285
+ ),
1286
+ "",
1287
+ "## Atlananlar (neden: sayı)",
1288
+ "",
1289
+ json.dumps(stats["skipped"], ensure_ascii=False),
1290
+ "",
1291
+ "## Karantina (neden: sayı)",
1292
+ "",
1293
+ json.dumps(stats["quarantined"], ensure_ascii=False),
1294
+ "",
1295
+ "## Eğitim yükleyicisi doğrulaması (`nida.train.data`)",
1296
+ "",
1297
+ *(
1298
+ f"- {name}: {v['accepted']}/{v['records']} kabul, ret: {v['rejection_counts']}"
1299
+ for name, v in loader.items()
1300
+ ),
1301
+ ]
1302
+ failed = stats["failed_speakers"]
1303
+ if failed:
1304
+ lines += ["", "## Başarısız konuşmacılar (yeniden çalıştırın)", ""]
1305
+ assert isinstance(failed, dict)
1306
+ lines += [f"- {speaker}: {error[:300]}" for speaker, error in failed.items()]
1307
+ return "\n".join(lines) + "\n"
src/nida/data/enhance.py ADDED
@@ -0,0 +1,593 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Optional speech-enhancement step of the data pilot (docs/03-data-pipeline.md).
2
+
3
+ Everything about enhancement that does not depend on one arm lives here:
4
+
5
+ * :class:`EnhanceConfig` (``configs/data/enhance.yaml``): which arm (``none`` | ``uvr`` |
6
+ ``sidon`` | ``dfn``) and how much length difference between the input and the enhanced
7
+ segment is tolerated;
8
+ * naming of the arm-specific audio root / dataset suffix;
9
+ * the pure audio functions shared by the three arm workers (:func:`fit_to_length`,
10
+ :func:`finish_enhanced_wav`) and the :class:`EnhanceAdapter` template they plug into;
11
+ * the small checks the arms share (device resolution, pinned-file SHA-256).
12
+
13
+ The arms themselves are in ``nida.data.workers.enhance_<arm>``; each runs in its own isolated
14
+ environment (:data:`nida.data.envs.ENHANCE_ENV_SPECS`), because their dependency pins
15
+ conflict (numpy 1 vs 2, Python 3.11 vs 3.12). The worker modules and this module are imported
16
+ inside those environments, so this module imports only the standard library and the stdlib-only
17
+ protocol/worker helpers at import time; numpy, scipy and soundfile are imported inside the
18
+ functions that need them (all three arm environments install them).
19
+
20
+ Contract of a worker request: ``audio_path`` is a 16 kHz mono wav; ``params["out_path"]`` is
21
+ the ``.wav`` to write; ``params["max_length_diff_ms"]`` is the largest length difference the
22
+ worker accepts at all. The output is a 16 kHz mono PCM16 wav with **exactly** as many samples as
23
+ the input. The difference between the arm's output length (after resampling to 16 kHz) and the
24
+ input length is recorded as ``length_diff_samples`` before it is fixed by zero-padding or
25
+ trimming; the payload also carries ``peak`` (before clipping) and ``loudness_dbfs`` (RMS level of
26
+ the PCM16 samples actually written, the same measure as the ingest ``loudness_dbfs``).
27
+
28
+ Who judges the length: the worker only refuses an absurd difference (above
29
+ ``max_length_diff_ms``: a misaligned or truncated output, :class:`EnhanceLengthError`, a failed
30
+ item). Everything between the tolerance and that limit is written and recorded, and the pilot
31
+ alone quarantines it by ``length_tolerance_ms`` (:mod:`nida.data.pilot`), so the tolerance can be
32
+ changed without recomputing and tolerated-but-large differences do not count toward the worker
33
+ error budget.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import abc
39
+ import hashlib
40
+ import io
41
+ import math
42
+ import os
43
+ from collections.abc import Mapping, Sequence
44
+ from dataclasses import dataclass
45
+ from pathlib import Path
46
+ from typing import TYPE_CHECKING
47
+
48
+ from nida.bench.protocol import WorkRequest
49
+ from nida.bench.workers.runner import Adapter
50
+ from nida.data.workers import REQUIRED_SAMPLE_RATE_HZ, require_16k_mono
51
+ from nida.utils.io import atomic_write_bytes
52
+
53
+ if TYPE_CHECKING:
54
+ import numpy as np
55
+ import numpy.typing as npt
56
+
57
+ Waveform = npt.NDArray[np.float32]
58
+
59
+ __all__ = [
60
+ "ARMS",
61
+ "ARM_NONE",
62
+ "CONFIG_FILENAME",
63
+ "ENHANCED_ARMS",
64
+ "NO_ENHANCE",
65
+ "PAYLOAD_KEYS",
66
+ "REASON_CLIPPED",
67
+ "REASON_FAILED",
68
+ "REASON_LENGTH",
69
+ "ArmUnavailableError",
70
+ "EnhanceAdapter",
71
+ "EnhanceConfig",
72
+ "EnhanceError",
73
+ "EnhanceLengthError",
74
+ "FittedAudio",
75
+ "arm_suffix",
76
+ "audio_dir_name",
77
+ "clipping_threshold_param",
78
+ "encode_wav_pcm16",
79
+ "finish_enhanced_wav",
80
+ "fit_to_length",
81
+ "is_length_error",
82
+ "max_length_diff_param",
83
+ "resample_mono",
84
+ "resolve_device",
85
+ "sha256_file",
86
+ "verify_pinned_sha256",
87
+ "wav_loudness_dbfs",
88
+ "wav_out_path_param",
89
+ "wav_stats",
90
+ ]
91
+
92
+ ARM_NONE = "none"
93
+ #: Selectable arms, ``none`` first. The other three have a worker, an environment and a pin.
94
+ ARMS = (ARM_NONE, "uvr", "sidon", "dfn")
95
+ ENHANCED_ARMS = ARMS[1:]
96
+ #: Name of the optional config file in ``configs/data/``; absent = no enhancement.
97
+ CONFIG_FILENAME = "enhance.yaml"
98
+ #: Prefix of the local-cache folder (sibling of ``audio``) holding one arm's enhanced wavs.
99
+ AUDIO_DIR_PREFIX = "audio-enh"
100
+ #: Quarantine reasons of the pilot ``enhance`` stage.
101
+ REASON_LENGTH = "enhance_length_mismatch"
102
+ REASON_FAILED = "enhance_failed"
103
+ REASON_CLIPPED = "enhance_clipped"
104
+ #: Prefix of a pinned file hash in ``ModelRef.revision`` (``sha256:<64 hex digits>``).
105
+ SHA256_REVISION_PREFIX = "sha256:"
106
+ #: Keys of every successful worker payload (:func:`finish_enhanced_wav`); anything else in a
107
+ #: payload came from :meth:`EnhanceAdapter.payload_extras`.
108
+ PAYLOAD_KEYS = frozenset(
109
+ {
110
+ "out_path",
111
+ "sample_rate_hz",
112
+ "num_samples",
113
+ "input_num_samples",
114
+ "length_diff_samples",
115
+ "peak",
116
+ "loudness_dbfs",
117
+ "clipping_ratio",
118
+ "clipping_threshold",
119
+ }
120
+ )
121
+ _SHA256_HEX_LENGTH = 64
122
+ _HASH_BLOCK_BYTES = 1 << 20
123
+
124
+
125
+ class EnhanceError(ValueError):
126
+ """An arm returned unusable audio (not 1-D mono, empty or non-finite)."""
127
+
128
+
129
+ class EnhanceLengthError(EnhanceError):
130
+ """The arm's output length differs from the input by more than the tolerance."""
131
+
132
+
133
+ class ArmUnavailableError(RuntimeError):
134
+ """The arm cannot start at all (missing library, no GPU, silent CPU fallback)."""
135
+
136
+
137
+ def is_length_error(error: str | None) -> bool:
138
+ """True if a worker result's ``error`` string (``"<Type>: <message>"``) is a length error."""
139
+ return error is not None and error.startswith(f"{EnhanceLengthError.__name__}:")
140
+
141
+
142
+ @dataclass(frozen=True)
143
+ class EnhanceConfig:
144
+ """``configs/data/enhance.yaml``.
145
+
146
+ Attributes:
147
+ arm: ``none`` (no enhancement, the baseline), ``uvr``, ``sidon`` or ``dfn``.
148
+ length_tolerance_ms: Largest tolerated difference, in milliseconds, between the
149
+ enhanced and the input segment length (checked in 16 kHz samples). Larger
150
+ differences make the segment an ``enhance_length_mismatch`` quarantine record
151
+ (decided by the pilot from the recorded ``length_diff_samples``).
152
+ max_length_diff_ms: Differences above this make the *worker* fail the item (also
153
+ ``enhance_length_mismatch``); between the two limits the wav is still written.
154
+ Must be >= ``length_tolerance_ms``.
155
+ max_enhanced_clipping_ratio: Largest fraction of samples of the written (PCM16) wav
156
+ that may sit at ``|x| >= acoustic.clipping_threshold`` (the S2 definition; the
157
+ threshold is the one of ``acoustic.yaml``); more makes the segment an
158
+ ``enhance_clipped`` quarantine record. In ``[0, 1]``.
159
+ """
160
+
161
+ arm: str
162
+ length_tolerance_ms: int
163
+ max_length_diff_ms: int
164
+ max_enhanced_clipping_ratio: float
165
+
166
+ def __post_init__(self) -> None:
167
+ if self.arm not in ARMS:
168
+ raise ValueError(f"enhance arm must be one of {ARMS}, got {self.arm!r}")
169
+ for name in ("length_tolerance_ms", "max_length_diff_ms"):
170
+ value = getattr(self, name)
171
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
172
+ raise ValueError(f"{name} must be an integer >= 0, got {value!r}")
173
+ if self.max_length_diff_ms < self.length_tolerance_ms:
174
+ raise ValueError(
175
+ f"max_length_diff_ms ({self.max_length_diff_ms}) must be >= "
176
+ f"length_tolerance_ms ({self.length_tolerance_ms})"
177
+ )
178
+ ratio = self.max_enhanced_clipping_ratio
179
+ if isinstance(ratio, bool) or not isinstance(ratio, int | float):
180
+ raise ValueError(f"max_enhanced_clipping_ratio must be a number, got {ratio!r}")
181
+ if not (math.isfinite(ratio) and 0.0 <= ratio <= 1.0):
182
+ raise ValueError(f"max_enhanced_clipping_ratio must be in [0, 1], got {ratio!r}")
183
+
184
+ @property
185
+ def enabled(self) -> bool:
186
+ return self.arm != ARM_NONE
187
+
188
+ @property
189
+ def length_tolerance_samples(self) -> int:
190
+ """The tolerance in 16 kHz samples (exact: 16 samples per millisecond)."""
191
+ return self.length_tolerance_ms * REQUIRED_SAMPLE_RATE_HZ // 1000
192
+
193
+ @property
194
+ def max_length_diff_samples(self) -> int:
195
+ """The worker's refusal limit in 16 kHz samples."""
196
+ return self.max_length_diff_ms * REQUIRED_SAMPLE_RATE_HZ // 1000
197
+
198
+ @classmethod
199
+ def from_mapping(cls, raw: Mapping[str, object]) -> EnhanceConfig:
200
+ expected = {
201
+ "arm",
202
+ "length_tolerance_ms",
203
+ "max_length_diff_ms",
204
+ "max_enhanced_clipping_ratio",
205
+ }
206
+ if set(raw) != expected:
207
+ raise ValueError(
208
+ f"enhance config keys: missing={sorted(expected - set(raw))} "
209
+ f"unknown={sorted(set(raw) - expected)}"
210
+ )
211
+ arm = raw["arm"]
212
+ if not isinstance(arm, str):
213
+ raise ValueError(f"enhance arm must be a string, got {arm!r}")
214
+ limits: dict[str, int] = {}
215
+ for name in ("length_tolerance_ms", "max_length_diff_ms"):
216
+ value = raw[name]
217
+ if not isinstance(value, int): # bool is rejected by __post_init__
218
+ raise ValueError(f"{name} must be an integer, got {value!r}")
219
+ limits[name] = value
220
+ ratio = raw["max_enhanced_clipping_ratio"]
221
+ if isinstance(ratio, bool) or not isinstance(ratio, int | float):
222
+ raise ValueError(f"max_enhanced_clipping_ratio must be a number, got {ratio!r}")
223
+ return cls(arm=arm, max_enhanced_clipping_ratio=float(ratio), **limits)
224
+
225
+
226
+ #: What a config directory without ``enhance.yaml`` means: the baseline pipeline. The limits are
227
+ #: never read while the arm is ``none``.
228
+ NO_ENHANCE = EnhanceConfig(
229
+ arm=ARM_NONE, length_tolerance_ms=0, max_length_diff_ms=0, max_enhanced_clipping_ratio=0.0
230
+ )
231
+
232
+
233
+ def audio_dir_name(arm: str) -> str:
234
+ """Local-cache folder name of ``arm``'s enhanced wavs (``audio-enh-<arm>``).
235
+
236
+ Raises:
237
+ ValueError: if ``arm`` is ``none`` or unknown.
238
+ """
239
+ if arm not in ENHANCED_ARMS:
240
+ raise ValueError(f"no enhanced audio for arm {arm!r}; arms: {ENHANCED_ARMS}")
241
+ return f"{AUDIO_DIR_PREFIX}-{arm}"
242
+
243
+
244
+ def arm_suffix(arm: str) -> str:
245
+ """Suffix of the arm-specific names (dataset directory, latent results, arrays)."""
246
+ audio_dir_name(arm) # validates
247
+ return f"-enh-{arm}"
248
+
249
+
250
+ # ------------------------------------------------------------------------ audio functions
251
+
252
+
253
+ @dataclass(frozen=True)
254
+ class FittedAudio:
255
+ """An arm output forced to the input length.
256
+
257
+ Attributes:
258
+ samples: float32 mono at 16 kHz, exactly the expected number of samples.
259
+ length_diff_samples: ``len(output after resampling) - len(input)`` *before* the
260
+ pad/trim (positive = the arm returned more samples than the input had).
261
+ peak: Largest absolute sample value before clipping to ``[-1, 1]``.
262
+ """
263
+
264
+ samples: Waveform
265
+ length_diff_samples: int
266
+ peak: float
267
+
268
+
269
+ def resample_mono(samples: Waveform, src_hz: int, dst_hz: int) -> Waveform:
270
+ """Polyphase resample (scipy); the input itself when the rates match.
271
+
272
+ Raises:
273
+ ValueError: on a non-positive rate.
274
+ """
275
+ import numpy as np
276
+ from scipy.signal import resample_poly # type: ignore[import-untyped] # no stubs
277
+
278
+ if src_hz <= 0 or dst_hz <= 0:
279
+ raise ValueError(f"sample rates must be positive, got {src_hz} -> {dst_hz}")
280
+ if src_hz == dst_hz:
281
+ return samples
282
+ divisor = math.gcd(src_hz, dst_hz)
283
+ out = resample_poly(samples.astype(np.float64), dst_hz // divisor, src_hz // divisor)
284
+ return np.ascontiguousarray(out, dtype=np.float32)
285
+
286
+
287
+ def fit_to_length(
288
+ samples: npt.ArrayLike,
289
+ *,
290
+ source_rate_hz: int,
291
+ expected_num_samples: int,
292
+ max_diff_samples: int,
293
+ what: str,
294
+ ) -> FittedAudio:
295
+ """Resample an arm output to 16 kHz and force it to ``expected_num_samples``.
296
+
297
+ Raises:
298
+ EnhanceError: for non-1-D, empty or non-finite output.
299
+ EnhanceLengthError: if the resampled length differs from ``expected_num_samples`` by
300
+ more than ``max_diff_samples`` (nothing is padded or trimmed in that case).
301
+ ValueError: on a non-positive ``expected_num_samples``, negative limit or bad rate.
302
+ """
303
+ import numpy as np
304
+
305
+ if expected_num_samples < 1 or max_diff_samples < 0:
306
+ raise ValueError(
307
+ f"{what}: invalid expected length / limit: {expected_num_samples} / {max_diff_samples}"
308
+ )
309
+ array = np.asarray(samples)
310
+ if array.ndim != 1:
311
+ raise EnhanceError(f"{what}: enhancer returned shape {array.shape}, expected 1-D mono")
312
+ if array.size == 0:
313
+ raise EnhanceError(f"{what}: enhancer returned no samples")
314
+ if not np.isfinite(array).all():
315
+ raise EnhanceError(f"{what}: enhancer returned non-finite samples")
316
+ resampled = resample_mono(
317
+ array.astype(np.float32, copy=False), source_rate_hz, REQUIRED_SAMPLE_RATE_HZ
318
+ )
319
+ diff = int(resampled.shape[0]) - expected_num_samples
320
+ if abs(diff) > max_diff_samples:
321
+ raise EnhanceLengthError(
322
+ f"{what}: output has {resampled.shape[0]} samples at {REQUIRED_SAMPLE_RATE_HZ} Hz, "
323
+ f"expected {expected_num_samples} (difference {diff} samples = "
324
+ f"{1000.0 * diff / REQUIRED_SAMPLE_RATE_HZ:.1f} ms, limit {max_diff_samples} "
325
+ "samples)"
326
+ )
327
+ if diff >= 0:
328
+ fitted = np.ascontiguousarray(resampled[:expected_num_samples])
329
+ else:
330
+ fitted = np.pad(resampled, (0, -diff))
331
+ return FittedAudio(fitted, diff, float(np.abs(fitted).max()))
332
+
333
+
334
+ def encode_wav_pcm16(samples: Waveform) -> bytes:
335
+ """The bytes of a 16 kHz mono PCM16 wav; ``samples`` are clipped to ``[-1, 1]`` first."""
336
+ import numpy as np
337
+ import soundfile # type: ignore[import-untyped] # untyped
338
+
339
+ buffer = io.BytesIO()
340
+ soundfile.write(
341
+ buffer,
342
+ np.clip(samples, -1.0, 1.0),
343
+ REQUIRED_SAMPLE_RATE_HZ,
344
+ subtype="PCM_16",
345
+ format="WAV",
346
+ )
347
+ return buffer.getvalue()
348
+
349
+
350
+ def wav_stats(data: bytes, *, clipping_threshold: float) -> tuple[float, float]:
351
+ """``(loudness_dbfs, clipping_ratio)`` of the samples a wav file really holds.
352
+
353
+ Decoded from the bytes, so the numbers describe what was written (PCM16, clipped to
354
+ ``[-1, 1]``). Both are the ingest (S2) measures, :func:`nida.data.acoustic.loudness_dbfs`
355
+ and :func:`nida.data.acoustic.clipping_ratio` on the float64 samples, so enhanced and
356
+ original audio are comparable.
357
+ """
358
+ import soundfile
359
+
360
+ # numpy + scipy; kept out of the import time of this module
361
+ from nida.data.acoustic import clipping_ratio, loudness_dbfs
362
+
363
+ decoded, _ = soundfile.read(io.BytesIO(data), dtype="float64", always_2d=False)
364
+ return (
365
+ float(loudness_dbfs(decoded)),
366
+ float(clipping_ratio(decoded, threshold=clipping_threshold)),
367
+ )
368
+
369
+
370
+ def wav_loudness_dbfs(data: bytes) -> float:
371
+ """RMS level in dBFS of the samples a wav file really holds (see :func:`wav_stats`)."""
372
+ return wav_stats(data, clipping_threshold=1.0)[0]
373
+
374
+
375
+ def finish_enhanced_wav(
376
+ *,
377
+ out_path: Path,
378
+ samples: npt.ArrayLike,
379
+ source_rate_hz: int,
380
+ input_num_samples: int,
381
+ max_diff_samples: int,
382
+ clipping_threshold: float,
383
+ what: str,
384
+ ) -> dict[str, object]:
385
+ """Fit an arm output to the input length, write it atomically, describe it.
386
+
387
+ The checks run before the write, so a failing segment leaves no output file.
388
+
389
+ Returns:
390
+ Payload: ``out_path``, ``sample_rate_hz``, ``num_samples`` (== ``input_num_samples``),
391
+ ``input_num_samples``, ``length_diff_samples`` (before the fix), ``peak`` (before
392
+ clipping to the PCM16 range), ``loudness_dbfs`` and ``clipping_ratio`` (fraction of the
393
+ PCM16 samples written at ``|x| >= clipping_threshold``, which is echoed back).
394
+
395
+ Raises:
396
+ EnhanceError, EnhanceLengthError, ValueError: see :func:`fit_to_length`.
397
+ """
398
+ fitted = fit_to_length(
399
+ samples,
400
+ source_rate_hz=source_rate_hz,
401
+ expected_num_samples=input_num_samples,
402
+ max_diff_samples=max_diff_samples,
403
+ what=what,
404
+ )
405
+ data = encode_wav_pcm16(fitted.samples)
406
+ atomic_write_bytes(out_path, data)
407
+ loudness, clipping = wav_stats(data, clipping_threshold=clipping_threshold)
408
+ return {
409
+ "out_path": str(out_path),
410
+ "sample_rate_hz": REQUIRED_SAMPLE_RATE_HZ,
411
+ "num_samples": int(fitted.samples.shape[0]),
412
+ "input_num_samples": input_num_samples,
413
+ "length_diff_samples": fitted.length_diff_samples,
414
+ "peak": round(fitted.peak, 6),
415
+ "loudness_dbfs": loudness,
416
+ "clipping_ratio": clipping,
417
+ "clipping_threshold": clipping_threshold,
418
+ }
419
+
420
+
421
+ def wav_out_path_param(request: WorkRequest) -> Path:
422
+ """The ``.wav`` output path named by ``request.params["out_path"]``.
423
+
424
+ Raises:
425
+ ValueError: if the parameter is missing or does not end in ``.wav``.
426
+ """
427
+ raw = request.params.get("out_path")
428
+ if not raw:
429
+ raise ValueError(f"{request.clip_id}: missing params.out_path")
430
+ path = Path(raw)
431
+ if path.suffix != ".wav":
432
+ raise ValueError(f"{request.clip_id}: params.out_path must end in .wav, got {raw!r}")
433
+ return path
434
+
435
+
436
+ def max_length_diff_param(request: WorkRequest) -> int:
437
+ """``params["max_length_diff_ms"]`` converted to 16 kHz samples.
438
+
439
+ Raises:
440
+ ValueError: if the parameter is missing, not a non-negative integer, or unreasonable.
441
+ """
442
+ raw = request.params.get("max_length_diff_ms")
443
+ if raw is None:
444
+ raise ValueError(f"{request.clip_id}: missing params.max_length_diff_ms")
445
+ try:
446
+ milliseconds = int(raw)
447
+ except ValueError:
448
+ raise ValueError(
449
+ f"{request.clip_id}: params.max_length_diff_ms must be an integer, got {raw!r}"
450
+ ) from None
451
+ if milliseconds < 0:
452
+ raise ValueError(f"{request.clip_id}: params.max_length_diff_ms must be >= 0")
453
+ return milliseconds * REQUIRED_SAMPLE_RATE_HZ // 1000
454
+
455
+
456
+ def clipping_threshold_param(request: WorkRequest) -> float:
457
+ """``params["clipping_threshold"]``: the magnitude counted as clipped, in ``(0, 1]``.
458
+
459
+ Raises:
460
+ ValueError: if the parameter is missing or not a number in ``(0, 1]``.
461
+ """
462
+ raw = request.params.get("clipping_threshold")
463
+ if raw is None:
464
+ raise ValueError(f"{request.clip_id}: missing params.clipping_threshold")
465
+ try:
466
+ value = float(raw)
467
+ except ValueError:
468
+ raise ValueError(
469
+ f"{request.clip_id}: params.clipping_threshold must be a number, got {raw!r}"
470
+ ) from None
471
+ if not (math.isfinite(value) and 0.0 < value <= 1.0):
472
+ raise ValueError(f"{request.clip_id}: params.clipping_threshold must be in (0, 1]")
473
+ return value
474
+
475
+
476
+ # ----------------------------------------------------------------------- worker template
477
+
478
+
479
+ class EnhanceAdapter(Adapter):
480
+ """Template of the three arm workers: subclasses load a model and implement :meth:`enhance`.
481
+
482
+ Clips are processed one at a time (the arms are not batched), so a failing clip is an
483
+ error of that clip only: the runner records it and carries on.
484
+ """
485
+
486
+ @abc.abstractmethod
487
+ def enhance(self, request: WorkRequest, wav: Waveform) -> tuple[Waveform, int]:
488
+ """Enhance one clip.
489
+
490
+ Args:
491
+ request: The request (``audio_path`` is the same file ``wav`` was read from).
492
+ wav: float32 mono samples at 16 kHz, shape ``(num_samples,)``.
493
+
494
+ Returns:
495
+ ``(samples, sample_rate_hz)``: a 1-D float waveform at any rate; the caller
496
+ resamples it to 16 kHz and checks its length.
497
+ """
498
+
499
+ def payload_extras(self) -> Mapping[str, object]:
500
+ """Facts about the loaded model/runtime added to every payload (default: none).
501
+
502
+ They let the pilot record, e.g., the execution provider that really ran or the digests
503
+ of model files that are recorded rather than pinned. Keys must not clash with the
504
+ standard payload keys.
505
+ """
506
+ return {}
507
+
508
+ def _one(self, request: WorkRequest) -> Mapping[str, object]:
509
+ import soundfile
510
+
511
+ info = require_16k_mono(request)
512
+ out_path = wav_out_path_param(request)
513
+ max_diff_samples = max_length_diff_param(request)
514
+ clipping_threshold = clipping_threshold_param(request)
515
+ wav, _ = soundfile.read(request.audio_path, dtype="float32", always_2d=False)
516
+ if wav.ndim != 1 or int(wav.shape[0]) != info.num_samples:
517
+ raise ValueError(f"{request.clip_id}: read {wav.shape}, header says {info.num_samples}")
518
+ samples, rate_hz = self.enhance(request, wav)
519
+ extras = self.payload_extras()
520
+ clash = sorted(set(extras) & PAYLOAD_KEYS)
521
+ if clash:
522
+ raise ValueError(f"{request.clip_id}: payload extras clash with {clash}")
523
+ payload = finish_enhanced_wav(
524
+ out_path=out_path,
525
+ samples=samples,
526
+ source_rate_hz=rate_hz,
527
+ input_num_samples=info.num_samples,
528
+ max_diff_samples=max_diff_samples,
529
+ clipping_threshold=clipping_threshold,
530
+ what=request.clip_id,
531
+ )
532
+ return {**payload, **extras}
533
+
534
+ def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
535
+ return [self._one(request) for request in batch]
536
+
537
+
538
+ # ----------------------------------------------------------------------- shared arm checks
539
+
540
+
541
+ def resolve_device(device: str) -> str:
542
+ """Validate ``device`` for this process; ``cpu`` hides CUDA before any library loads.
543
+
544
+ Raises:
545
+ ArmUnavailableError: for CUDA requested but unavailable, torch missing, or a device
546
+ string other than ``cpu`` / ``cuda*``.
547
+ """
548
+ if device == "cpu":
549
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
550
+ return "cpu"
551
+ if device.startswith("cuda"):
552
+ try:
553
+ import torch
554
+ except ImportError as exc:
555
+ raise ArmUnavailableError("torch is not installed in this environment") from exc
556
+ if not torch.cuda.is_available():
557
+ raise ArmUnavailableError(f"device {device!r} requested but CUDA is not available here")
558
+ return device
559
+ raise ArmUnavailableError(f"unsupported device {device!r} (use cpu or cuda)")
560
+
561
+
562
+ def sha256_file(path: Path) -> str:
563
+ """Hex SHA-256 of a file, read in 1 MiB blocks."""
564
+ digest = hashlib.sha256()
565
+ with path.open("rb") as handle:
566
+ for block in iter(lambda: handle.read(_HASH_BLOCK_BYTES), b""):
567
+ digest.update(block)
568
+ return digest.hexdigest()
569
+
570
+
571
+ def verify_pinned_sha256(path: Path, revision: str, *, what: str) -> str:
572
+ """Check that ``path`` hashes to the digest pinned in ``revision`` (``sha256:<hex>``).
573
+
574
+ Returns:
575
+ The verified digest.
576
+
577
+ Raises:
578
+ ValueError: if ``revision`` is not a ``sha256:`` pin, the file is missing, or the
579
+ digest differs (the model file is not the one the pipeline was validated with).
580
+ """
581
+ if not revision.startswith(SHA256_REVISION_PREFIX):
582
+ raise ValueError(
583
+ f"{what}: revision must be '{SHA256_REVISION_PREFIX}<hex>', got {revision!r}"
584
+ )
585
+ expected = revision.removeprefix(SHA256_REVISION_PREFIX)
586
+ if len(expected) != _SHA256_HEX_LENGTH or set(expected) - set("0123456789abcdef"):
587
+ raise ValueError(f"{what}: {revision!r} is not a lowercase SHA-256 hex digest")
588
+ if not path.is_file():
589
+ raise ValueError(f"{what}: expected model file {path} does not exist")
590
+ actual = sha256_file(path)
591
+ if actual != expected:
592
+ raise ValueError(f"{what}: {path.name} has sha256 {actual}, the pinned value is {expected}")
593
+ return actual
src/nida/data/envs.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Isolated ``uv`` environments for the Phase 1 data-pipeline model workers.
2
+
3
+ Same mechanism as :mod:`nida.bench.envs` (this module only *describes* environments; build
4
+ them with :func:`nida.bench.envs.ensure_env`, which writes the freeze file and the marker):
5
+ one environment per model family because their dependency stacks conflict (NeMo,
6
+ FunASR/ModelScope, transformers, panns-inference/librosa). Python and the torch pins are
7
+ taken from the Phase 0 ``hf`` environment, so every worker environment runs the same
8
+ ``torch==2.8.0`` (cu128) build.
9
+
10
+ ``EnvSpec.workers`` here are module names under ``nida.data.workers`` (see
11
+ :func:`worker_module_name`), not under ``nida.bench.workers``.
12
+
13
+ Version pins (PyPI, looked up 2026-10-08):
14
+
15
+ * ``panns-inference==0.1.1`` (latest, 2023-03-26) with ``torchlibrosa==0.1.0`` (latest).
16
+ * ``nemo_toolkit[asr]==3.0.0`` (latest, 2026-08-07; ``torch>=2.6.0`` so the torch 2.8.0 wheel
17
+ is kept; the build's torch guard would catch a replacement). The model card asks for
18
+ ``Cython`` and ``packaging`` to be installed first. System packages ``libsndfile1`` and
19
+ ``ffmpeg`` must exist (present on Colab).
20
+ * ``funasr==1.4.16`` (latest, 2026-09-18).
21
+ * ``speechbrain==1.1.1`` (2026-08-27; the latest, 1.1.2, was published on 2026-10-14 and is not
22
+ pinned yet: one day old). It requires ``torch>=2.1.0`` (the 2.8.0 wheel is kept), and its
23
+ dependency ``hyperpyyaml==1.2.3`` (pinned explicitly) caps ``ruamel.yaml<0.19`` (0.19.x breaks
24
+ HyperPyYAML with ``'Loader' object has no attribute 'max_depth'``), which the resolver
25
+ respects. Importing ``speechbrain`` needs ``torchaudio`` (in the torch pins). Checked locally
26
+ on CPU with torch and torchaudio 2.11.0 (not with the cu128 2.8.0 pair: unverified).
27
+ * ``transformers==4.57.6``: **not** the latest (5.19.0, 2026-10-06). That is a new major
28
+ version never exercised here; 4.57.6 is what the Phase 0 ``hf`` environment already ran
29
+ with torch 2.8.0.
30
+
31
+ Unpinned helpers (numpy, soundfile, huggingface-hub, librosa, matplotlib) follow the bench
32
+ environments; the exact resolved versions are recorded in each environment's ``freeze.txt``.
33
+
34
+ The speech-enhancement arms (:mod:`nida.data.enhance`) have their own mapping,
35
+ :data:`ENHANCE_ENV_SPECS`, because their pins deliberately differ from the shared stack above
36
+ (Python 3.11 for ``dfn``, numpy 1 vs 2, no torch pin for ``uvr``). They are the pins of
37
+ ``scripts/enhance_envs.sh`` (exercised for the A/B listening test, 2026-10-13), copied
38
+ unchanged; ``tests/test_data_enhance.py`` compares both. Same ``EnvSpec`` mechanism, so
39
+ ``ensure_env`` builds them; :func:`data_env_for_worker` looks in both mappings.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ from collections.abc import Mapping
45
+ from types import MappingProxyType
46
+
47
+ from nida.bench.envs import ENV_SPECS as BENCH_ENV_SPECS
48
+ from nida.bench.envs import EnvSpec
49
+
50
+ __all__ = [
51
+ "DATA_ENV_SPECS",
52
+ "DATA_WORKER_PACKAGE",
53
+ "ENHANCE_ENV_SPECS",
54
+ "data_env_for_worker",
55
+ "worker_module_name",
56
+ ]
57
+
58
+ #: Package whose modules are the Phase 1 workers.
59
+ DATA_WORKER_PACKAGE = "nida.data.workers"
60
+
61
+ _BASE = BENCH_ENV_SPECS["hf"] # python + torch pins shared with Phase 0
62
+
63
+ DATA_ENV_SPECS: Mapping[str, EnvSpec] = MappingProxyType(
64
+ {
65
+ "data_sed": EnvSpec(
66
+ name="data_sed",
67
+ python=_BASE.python,
68
+ torch_packages=_BASE.torch_packages,
69
+ torch_index_url=_BASE.torch_index_url,
70
+ packages=(
71
+ "panns-inference==0.1.1",
72
+ "torchlibrosa==0.1.0",
73
+ "librosa",
74
+ "matplotlib",
75
+ "numpy",
76
+ "soundfile",
77
+ ),
78
+ workers=("sed_panns",),
79
+ ),
80
+ "data_diar": EnvSpec(
81
+ name="data_diar",
82
+ python=_BASE.python,
83
+ torch_packages=_BASE.torch_packages,
84
+ torch_index_url=_BASE.torch_index_url,
85
+ packages=(
86
+ "Cython",
87
+ "packaging",
88
+ "nemo_toolkit[asr]==3.0.0",
89
+ # Unpinned, the resolver picks transformers 4.12 + tokenizers 0.10 (no wheel
90
+ # for py3.12, Rust build fails on Colab); pin the Phase 0 transformers.
91
+ "transformers==4.57.6",
92
+ "huggingface-hub<1.0",
93
+ "numpy",
94
+ "soundfile",
95
+ ),
96
+ workers=("diar_sortformer",),
97
+ ),
98
+ "data_ctc": EnvSpec(
99
+ name="data_ctc",
100
+ python=_BASE.python,
101
+ torch_packages=_BASE.torch_packages,
102
+ torch_index_url=_BASE.torch_index_url,
103
+ packages=("transformers==4.57.6", "huggingface-hub", "numpy", "soundfile"),
104
+ workers=("ctc_emissions",),
105
+ ),
106
+ "data_emotion": EnvSpec(
107
+ name="data_emotion",
108
+ python=_BASE.python,
109
+ torch_packages=_BASE.torch_packages,
110
+ torch_index_url=_BASE.torch_index_url,
111
+ packages=(
112
+ "funasr==1.4.16",
113
+ "transformers==4.57.6", # same resolver trap as data_diar
114
+ "huggingface-hub<1.0",
115
+ "numpy",
116
+ "soundfile",
117
+ ),
118
+ workers=("emotion_e2v",),
119
+ ),
120
+ # Speaker similarity (evaluation, scripts/speaker_sim.py): SpeechBrain ECAPA-TDNN.
121
+ # scipy: SpeechBrain depends on it and the script resamples with it.
122
+ "data_spk": EnvSpec(
123
+ name="data_spk",
124
+ python=_BASE.python,
125
+ torch_packages=_BASE.torch_packages,
126
+ torch_index_url=_BASE.torch_index_url,
127
+ packages=(
128
+ "speechbrain==1.1.1",
129
+ # The version SpeechBrain 1.1.1 was checked with; it caps ruamel.yaml<0.19.
130
+ "hyperpyyaml==1.2.3",
131
+ "huggingface-hub",
132
+ "numpy",
133
+ "scipy",
134
+ "soundfile",
135
+ ),
136
+ workers=("spk_ecapa",),
137
+ ),
138
+ # S9 latents: the B2 VAE stack (torch + torchaudio for resampling), without the
139
+ # Audiobox quality model.
140
+ "data_vae": EnvSpec(
141
+ name="data_vae",
142
+ python=_BASE.python,
143
+ torch_packages=BENCH_ENV_SPECS["audio"].torch_packages,
144
+ torch_index_url=_BASE.torch_index_url,
145
+ packages=("numpy", "pydantic", "soundfile", "huggingface-hub"),
146
+ workers=("vae_encode", "vae_decode"),
147
+ ),
148
+ }
149
+ )
150
+
151
+
152
+ #: scripts/enhance_envs.sh, copied: ``torch==2.8.0`` + ``torchaudio==2.8.0`` is the pair of the
153
+ #: sidon/dfn venvs (PyPI's Linux 2.8.0 wheel is the CUDA 12.8 build, as is the cu128 index
154
+ #: wheel installed here). ``uvr`` pins no torch: ``audio-separator[gpu]==0.47.0`` resolves its
155
+ #: own (torch 2.14.1 and onnxruntime-gpu 1.31.0, CUDA 13 builds, NVIDIA driver >= 580; this
156
+ #: combination was NOT exercised on a GPU: unverified, and to be pinned (torch, onnxruntime-gpu)
157
+ #: from the first GPU run's freeze file) and ``ensure_env`` skips its torch step for an empty
158
+ #: ``torch_packages``. ``[gpu]`` is the extra the script picks when ``nvidia-smi`` exists; if
159
+ #: onnxruntime cannot activate the CUDA provider the worker aborts (it never runs on the CPU
160
+ #: silently) and the A/B script's ``ENH_UVR_EXTRA=cpu`` variant is the fallback.
161
+ ENHANCE_ENV_SPECS: Mapping[str, EnvSpec] = MappingProxyType(
162
+ {
163
+ "data_enh_uvr": EnvSpec(
164
+ name="data_enh_uvr",
165
+ python="3.12",
166
+ torch_packages=(),
167
+ torch_index_url="",
168
+ packages=(
169
+ "audio-separator[gpu]==0.47.0",
170
+ "audioread==3.1.0",
171
+ "numpy==2.5.4",
172
+ "scipy==1.18.1",
173
+ "soundfile==0.14.0",
174
+ ),
175
+ workers=("enhance_uvr",),
176
+ ),
177
+ "data_enh_sidon": EnvSpec(
178
+ name="data_enh_sidon",
179
+ python="3.12",
180
+ torch_packages=_BASE.torch_packages,
181
+ torch_index_url=_BASE.torch_index_url,
182
+ packages=(
183
+ "transformers==5.19.0",
184
+ "huggingface_hub==2.2.0",
185
+ "numpy==2.5.4",
186
+ "scipy==1.18.1",
187
+ "soundfile==0.14.0",
188
+ ),
189
+ workers=("enhance_sidon",),
190
+ ),
191
+ "data_enh_dfn": EnvSpec(
192
+ name="data_enh_dfn",
193
+ python="3.11", # deepfilterlib 0.5.6 has wheels for cp38..cp311 only
194
+ torch_packages=_BASE.torch_packages,
195
+ torch_index_url=_BASE.torch_index_url,
196
+ packages=(
197
+ "deepfilternet==0.5.6",
198
+ "numpy==1.26.4",
199
+ "scipy==1.17.1",
200
+ "soundfile==0.14.0",
201
+ ),
202
+ workers=("enhance_dfn",),
203
+ ),
204
+ }
205
+ )
206
+
207
+
208
+ def data_env_for_worker(worker: str) -> EnvSpec:
209
+ """Return the environment spec that runs the Phase 1 ``worker``.
210
+
211
+ Raises:
212
+ ValueError: if no data (or enhancement) environment lists ``worker``.
213
+ """
214
+ specs = (*DATA_ENV_SPECS.values(), *ENHANCE_ENV_SPECS.values())
215
+ for spec in specs:
216
+ if worker in spec.workers:
217
+ return spec
218
+ known = sorted(w for spec in specs for w in spec.workers)
219
+ raise ValueError(f"unknown data worker {worker!r}; known workers: {known}")
220
+
221
+
222
+ def worker_module_name(worker: str) -> str:
223
+ """Dotted module to run with ``python -m`` for ``worker``.
224
+
225
+ Raises:
226
+ ValueError: if ``worker`` is not a known data worker.
227
+ """
228
+ data_env_for_worker(worker) # validates the name
229
+ return f"{DATA_WORKER_PACKAGE}.{worker}"
src/nida/data/events.py ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """S6 — frame-level sound-event probabilities → :class:`EventDetection` intervals.
2
+
3
+ The tagger (PANNs ``Cnn14_DecisionLevelMax``) gives per-frame probabilities for AudioSet
4
+ classes. Each Nida event kind maps to one or more AudioSet classes (tag-spec §2, R3); the
5
+ kind's frame probability is the maximum over its classes. A detection is a maximal run of
6
+ frames at or above ``onset`` (gaps up to ``merge_gap_ms`` are bridged); its confidence is
7
+ the maximum frame probability inside the run. Fusion (S7) then applies the per-kind
8
+ confidence bands, so ``onset`` must not exceed any band's ``low`` — otherwise a
9
+ middle-band event could vanish here instead of dropping its segment. Runs shorter than
10
+ the kind's ``min_ms`` are not detections; for each kind the highest confidence among such
11
+ blips is returned (``too_short``), never silently discarded — the caller drops a segment
12
+ whose blip reaches the fusion band ``low`` (precision before recall, tag-spec §7).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import math
18
+ from collections.abc import Mapping, Sequence
19
+ from dataclasses import dataclass
20
+ from typing import TYPE_CHECKING
21
+
22
+ from nida.data.types import EventDetection
23
+ from nida.text.tags import EVENTS
24
+
25
+ if TYPE_CHECKING:
26
+ import numpy as np
27
+ from numpy.typing import NDArray
28
+
29
+ __all__ = ["EventConfig", "FrameDetections", "detect_events"]
30
+
31
+ _NON_DETECTABLE = frozenset({"pause_short", "pause_long"})
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class EventConfig:
36
+ """configs/data/events.yaml. ``classes[kind]`` lists AudioSet display names."""
37
+
38
+ classes: Mapping[str, tuple[str, ...]]
39
+ onset: float
40
+ merge_gap_ms: int
41
+ min_ms: Mapping[str, int]
42
+
43
+ def __post_init__(self) -> None:
44
+ kinds = set(EVENTS) - _NON_DETECTABLE
45
+ if set(self.classes) != kinds:
46
+ raise ValueError(
47
+ f"classes must map exactly {sorted(kinds)}; "
48
+ f"missing={sorted(kinds - set(self.classes))} "
49
+ f"extra={sorted(set(self.classes) - kinds)}"
50
+ )
51
+ if set(self.min_ms) != kinds:
52
+ raise ValueError("min_ms must have exactly one entry per detectable event kind")
53
+ seen: dict[str, str] = {}
54
+ for kind, names in self.classes.items():
55
+ if not names:
56
+ raise ValueError(f"event kind {kind!r} maps to no AudioSet class")
57
+ for name in names:
58
+ if name in seen:
59
+ raise ValueError(
60
+ f"AudioSet class {name!r} mapped to {seen[name]!r} and {kind!r}"
61
+ )
62
+ seen[name] = kind
63
+ if not (math.isfinite(self.onset) and 0.0 < self.onset <= 1.0):
64
+ raise ValueError(f"onset must be in (0, 1], got {self.onset!r}")
65
+ for name, value in [("merge_gap_ms", self.merge_gap_ms), *self.min_ms.items()]:
66
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
67
+ raise ValueError(f"{name} must be a non-negative int, got {value!r}")
68
+
69
+ @classmethod
70
+ def from_mapping(cls, raw: Mapping[str, object]) -> EventConfig:
71
+ expected = {"classes", "onset", "merge_gap_ms", "min_ms"}
72
+ if set(raw) != expected:
73
+ raise ValueError(
74
+ f"event config keys: missing={sorted(expected - set(raw))} "
75
+ f"unknown={sorted(set(raw) - expected)}"
76
+ )
77
+ classes, min_ms, onset, gap = (
78
+ raw["classes"],
79
+ raw["min_ms"],
80
+ raw["onset"],
81
+ raw["merge_gap_ms"],
82
+ )
83
+ if not isinstance(classes, Mapping) or not all(
84
+ isinstance(v, list) and all(isinstance(n, str) for n in v) for v in classes.values()
85
+ ):
86
+ raise ValueError("classes must map kinds to lists of class names")
87
+ if not isinstance(min_ms, Mapping):
88
+ raise ValueError("min_ms must be a mapping")
89
+ if isinstance(onset, bool) or not isinstance(onset, int | float):
90
+ raise ValueError("onset must be a number")
91
+ if isinstance(gap, bool) or not isinstance(gap, int):
92
+ raise ValueError("merge_gap_ms must be an int")
93
+ return cls(
94
+ classes={str(k): tuple(v) for k, v in classes.items()},
95
+ onset=float(onset),
96
+ merge_gap_ms=gap,
97
+ min_ms={str(k): v for k, v in min_ms.items()}, # validated in __post_init__
98
+ )
99
+
100
+ def check_onset_against(self, band_lows: Mapping[str, float]) -> None:
101
+ """Raise if ``onset`` exceeds a fusion band's ``low`` for some event kind."""
102
+ missing = sorted(set(self.classes) - set(band_lows))
103
+ if missing:
104
+ raise ValueError(f"no fusion band for event kinds {missing}")
105
+ too_high = sorted(k for k in self.classes if band_lows[k] < self.onset)
106
+ if too_high:
107
+ raise ValueError(f"onset {self.onset} above the fusion band low of {too_high}")
108
+
109
+
110
+ @dataclass(frozen=True)
111
+ class FrameDetections:
112
+ detections: tuple[EventDetection, ...]
113
+ too_short: Mapping[str, float] # kind -> max confidence among too-short runs
114
+
115
+
116
+ def detect_events(
117
+ probs: NDArray[np.float32],
118
+ class_names: Sequence[str],
119
+ *,
120
+ frame_hop_ms: float,
121
+ duration_ms: int,
122
+ cfg: EventConfig,
123
+ source: str,
124
+ ) -> FrameDetections:
125
+ """Turn framewise class probabilities ``[num_frames, len(class_names)]`` into detections.
126
+
127
+ Frame ``t`` covers ``[t * hop, (t + 1) * hop)`` ms, clamped to ``duration_ms``.
128
+
129
+ Raises:
130
+ ValueError: shape mismatch, values outside [0, 1] or non-finite, a configured
131
+ class missing from ``class_names``, a bad hop or duration, or a frame count
132
+ inconsistent with ``duration_ms`` (more than one frame of slack).
133
+ """
134
+ import numpy as np
135
+
136
+ if probs.ndim != 2 or probs.shape[1] != len(class_names):
137
+ raise ValueError(f"probs shape {probs.shape} does not match {len(class_names)} classes")
138
+ if len(set(class_names)) != len(class_names):
139
+ raise ValueError("duplicate class names")
140
+ in_range = probs.size == 0 or (float(np.min(probs)) >= 0.0 and float(np.max(probs)) <= 1.0)
141
+ if not np.all(np.isfinite(probs)) or not in_range:
142
+ raise ValueError("probabilities must be finite and in [0, 1]")
143
+ if not (math.isfinite(frame_hop_ms) and frame_hop_ms >= 1):
144
+ raise ValueError(f"frame_hop_ms must be a finite number >= 1 ms, got {frame_hop_ms!r}")
145
+ if type(duration_ms) is not int or duration_ms <= 0:
146
+ raise ValueError(f"invalid duration_ms {duration_ms!r}")
147
+ expected_frames = duration_ms / frame_hop_ms
148
+ if abs(probs.shape[0] - expected_frames) > 1 + 1e-9:
149
+ raise ValueError(
150
+ f"{probs.shape[0]} frames for {duration_ms} ms at {frame_hop_ms} ms hop "
151
+ f"(expected about {expected_frames:.1f})"
152
+ )
153
+ column = {name: i for i, name in enumerate(class_names)}
154
+ missing = sorted({n for names in cfg.classes.values() for n in names} - set(column))
155
+ if missing:
156
+ raise ValueError(f"tagger output lacks configured classes {missing}")
157
+ gap_frames = math.floor(cfg.merge_gap_ms / frame_hop_ms)
158
+
159
+ detections: list[EventDetection] = []
160
+ too_short: dict[str, float] = {}
161
+ for kind in sorted(cfg.classes):
162
+ kind_probs = probs[:, [column[n] for n in cfg.classes[kind]]].max(axis=1)
163
+ active = np.flatnonzero(kind_probs >= cfg.onset)
164
+ if active.size == 0:
165
+ continue
166
+ runs: list[tuple[int, int]] = [] # inclusive frame ranges
167
+ start = prev = int(active[0])
168
+ for frame in active[1:].tolist():
169
+ if frame - prev - 1 > gap_frames:
170
+ runs.append((start, prev))
171
+ start = frame
172
+ prev = frame
173
+ runs.append((start, prev))
174
+ for first, last in runs:
175
+ start_ms = min(round(first * frame_hop_ms), duration_ms - 1)
176
+ end_ms = min(round((last + 1) * frame_hop_ms), duration_ms)
177
+ conf = float(np.max(kind_probs[first : last + 1]))
178
+ if end_ms - start_ms < cfg.min_ms[kind]:
179
+ too_short[kind] = max(too_short.get(kind, 0.0), conf)
180
+ continue
181
+ detections.append(EventDetection(kind, start_ms, end_ms, conf, source))
182
+ detections.sort(key=lambda d: (d.start_ms, d.end_ms, d.kind))
183
+ return FrameDetections(tuple(detections), too_short)
src/nida/data/fillers_ctc.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Filler candidates from CTC emissions in the gaps between aligned words (prototype).
2
+
3
+ docs/research/dolgu-tespiti-oneri.md, option A. Neither the primary nor the secondary ASR
4
+ writes fillers (B1-C), but the character CTC model may still emit their letters where the
5
+ speaker hesitates. For every silent-in-the-transcript gap (segment start → first word,
6
+ between words, last word → end) of at least ``min_gap_ms``, the emissions inside the gap
7
+ are greedily decoded (argmax, collapse repeats, drop blanks and word delimiters). If the
8
+ decoded string is a filler spelling, the gap yields a candidate.
9
+
10
+ Option A was accepted by the user (2026-10-09). :func:`insert_fillers` turns accepted
11
+ candidates into extra :class:`TimedWord` items before fusion (S7); whether that happens is
12
+ ``configs/data/fillers.yaml`` ``enabled`` (off until the filler listening panel shows the
13
+ precision, docs/research/dolgu-tespiti-oneri.md).
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import math
19
+ from collections.abc import Mapping, Sequence
20
+ from dataclasses import dataclass
21
+ from itertools import pairwise
22
+ from typing import TYPE_CHECKING
23
+
24
+ from nida.data.types import TimedWord
25
+ from nida.text.fillers import canonical_filler
26
+
27
+ if TYPE_CHECKING:
28
+ import numpy as np
29
+ from numpy.typing import NDArray
30
+
31
+ __all__ = [
32
+ "CTC_ONLY_FILLERS",
33
+ "FillerCandidate",
34
+ "FillerConfig",
35
+ "gap_filler_candidates",
36
+ "insert_fillers",
37
+ ]
38
+
39
+ #: Greedy CTC collapses doubled letters without a blank between them, so a drawn-out
40
+ #: "ıı" or "mm" often decodes as a single letter. These single-letter forms are accepted
41
+ #: *only* inside a gap (never in running text, where "e" is a conjunction).
42
+ CTC_ONLY_FILLERS: Mapping[str, str] = {"e": "ee", "m": "hmm", "a": "aa"}
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class FillerCandidate:
47
+ """A proposed filler: canonical text, time span (ms) and evidence.
48
+
49
+ ``start_ms``/``end_ms`` is the examined gap; ``speech_start_ms``/``speech_end_ms`` the
50
+ part of it from the first to the last non-blank frame (where the filler is voiced).
51
+ ``conf`` is the mean argmax probability over the gap's non-blank frames; ``decoded`` is
52
+ the raw greedy string.
53
+ """
54
+
55
+ text: str
56
+ start_ms: int
57
+ end_ms: int
58
+ conf: float
59
+ decoded: str
60
+ speech_start_ms: int
61
+ speech_end_ms: int
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class FillerConfig:
66
+ """``configs/data/fillers.yaml``. All keys required."""
67
+
68
+ enabled: bool
69
+ min_gap_ms: int
70
+ min_conf: float
71
+ fillers: frozenset[str]
72
+
73
+ def __post_init__(self) -> None:
74
+ if not isinstance(self.enabled, bool):
75
+ raise ValueError("enabled must be a bool")
76
+ if isinstance(self.min_gap_ms, bool) or not isinstance(self.min_gap_ms, int):
77
+ raise ValueError("min_gap_ms must be an int")
78
+ if self.min_gap_ms < 1:
79
+ raise ValueError("min_gap_ms must be >= 1")
80
+ if isinstance(self.min_conf, bool) or not isinstance(self.min_conf, int | float):
81
+ raise ValueError("min_conf must be a number")
82
+ if not (math.isfinite(self.min_conf) and 0.0 <= self.min_conf <= 1.0):
83
+ raise ValueError("min_conf must be in [0, 1]")
84
+ unknown = sorted(f for f in self.fillers if canonical_filler(f) != f)
85
+ if unknown or not self.fillers:
86
+ raise ValueError(f"fillers must be non-empty canonical filler forms, bad: {unknown}")
87
+
88
+ @classmethod
89
+ def from_mapping(cls, raw: Mapping[str, object]) -> FillerConfig:
90
+ expected = {"enabled", "min_gap_ms", "min_conf", "fillers"}
91
+ if set(raw) != expected:
92
+ raise ValueError(
93
+ f"fillers config keys: missing={sorted(expected - set(raw))} "
94
+ f"unknown={sorted(set(raw) - expected)}"
95
+ )
96
+ fillers = raw["fillers"]
97
+ if not isinstance(fillers, list) or not all(isinstance(f, str) for f in fillers):
98
+ raise ValueError("fillers must be a list of strings")
99
+ return cls(
100
+ enabled=raw["enabled"], # type: ignore[arg-type] # validated in __post_init__
101
+ min_gap_ms=raw["min_gap_ms"], # type: ignore[arg-type]
102
+ min_conf=raw["min_conf"], # type: ignore[arg-type]
103
+ fillers=frozenset(fillers),
104
+ )
105
+
106
+
107
+ def _greedy(
108
+ log_probs: NDArray[np.float64], skip: set[int], id_to_token: Sequence[str]
109
+ ) -> tuple[str, float, int, int]:
110
+ """Collapsed argmax string of ``log_probs``, mean prob of its non-skipped frames, and
111
+ the first / one-past-last non-skipped frame (``0, 0`` when there is none)."""
112
+ import numpy as np
113
+
114
+ best = np.argmax(log_probs, axis=1)
115
+ chars: list[str] = []
116
+ probs: list[float] = []
117
+ frames: list[int] = []
118
+ previous = -1
119
+ for frame, token in enumerate(best.tolist()):
120
+ if token not in skip:
121
+ probs.append(math.exp(float(log_probs[frame, token])))
122
+ frames.append(frame)
123
+ if token != previous:
124
+ chars.append(id_to_token[token])
125
+ previous = token
126
+ if not frames:
127
+ return "", 0.0, 0, 0
128
+ return "".join(chars), sum(probs) / len(probs), frames[0], frames[-1] + 1
129
+
130
+
131
+ def gap_filler_candidates(
132
+ words: Sequence[TimedWord],
133
+ log_probs: NDArray[np.float32],
134
+ vocab: Mapping[str, int],
135
+ *,
136
+ blank_token: str,
137
+ word_delimiter: str,
138
+ frame_hop_ms: float,
139
+ duration_ms: int,
140
+ min_gap_ms: int = 150,
141
+ ) -> list[FillerCandidate]:
142
+ """Filler candidates in the gaps of an aligned segment.
143
+
144
+ Args:
145
+ words: Aligned words (relative ms, in order, non-overlapping), possibly empty.
146
+ log_probs: ``[num_frames, vocab_size]`` CTC emissions (log-softmax re-applied here).
147
+ vocab: Token → id, ids exactly ``0..V-1``.
148
+ blank_token, word_delimiter: Special tokens (``[PAD]``, ``|`` for wav2vec2 HF).
149
+ frame_hop_ms: Frame stride (20 ms for wav2vec2).
150
+ duration_ms: Segment length.
151
+ min_gap_ms: Shorter gaps are not examined.
152
+
153
+ Returns:
154
+ Candidates in time order.
155
+
156
+ Raises:
157
+ ValueError: malformed inputs.
158
+ """
159
+ import numpy as np
160
+
161
+ if log_probs.ndim != 2 or log_probs.shape[1] != len(vocab):
162
+ raise ValueError(f"log_probs shape {log_probs.shape} does not match vocab {len(vocab)}")
163
+ if sorted(vocab.values()) != list(range(len(vocab))):
164
+ raise ValueError("vocab ids must be exactly 0..V-1")
165
+ if blank_token not in vocab or word_delimiter not in vocab:
166
+ raise ValueError("blank or word delimiter missing from vocab")
167
+ if not (math.isfinite(frame_hop_ms) and frame_hop_ms >= 1):
168
+ raise ValueError(f"frame_hop_ms must be finite and >= 1, got {frame_hop_ms!r}")
169
+ if min_gap_ms < 1 or duration_ms <= 0:
170
+ raise ValueError("min_gap_ms and duration_ms must be positive")
171
+ lp = log_probs.astype(np.float64)
172
+ row_max = np.max(lp, axis=1, keepdims=True)
173
+ if not np.all(np.isfinite(row_max)):
174
+ raise ValueError("a frame has no finite log-probability")
175
+ lp = lp - (row_max + np.log(np.sum(np.exp(lp - row_max), axis=1, keepdims=True)))
176
+ id_to_token = [""] * len(vocab)
177
+ for token, index in vocab.items():
178
+ id_to_token[index] = token
179
+ skip = {vocab[blank_token], vocab[word_delimiter]}
180
+
181
+ edges = [0, *[x for w in words for x in (w.start_ms, w.end_ms)], duration_ms]
182
+ candidates: list[FillerCandidate] = []
183
+ for start_ms, end_ms in zip(edges[0::2], edges[1::2], strict=True):
184
+ if end_ms - start_ms < min_gap_ms:
185
+ continue
186
+ first = math.ceil(start_ms / frame_hop_ms)
187
+ last = min(math.floor(end_ms / frame_hop_ms), lp.shape[0])
188
+ if last <= first:
189
+ continue
190
+ decoded, conf, voiced_first, voiced_last = _greedy(lp[first:last], skip, id_to_token)
191
+ if not decoded:
192
+ continue
193
+ text = canonical_filler(decoded) or CTC_ONLY_FILLERS.get(decoded)
194
+ if text is not None:
195
+ speech_start = max(start_ms, round((first + voiced_first) * frame_hop_ms))
196
+ speech_end = min(end_ms, round((first + voiced_last) * frame_hop_ms))
197
+ if speech_end <= speech_start: # rounding at a gap edge: keep one ms
198
+ speech_start = min(speech_start, end_ms - 1)
199
+ speech_end = speech_start + 1
200
+ candidates.append(
201
+ FillerCandidate(text, start_ms, end_ms, conf, decoded, speech_start, speech_end)
202
+ )
203
+ return candidates
204
+
205
+
206
+ def insert_fillers(
207
+ words: Sequence[TimedWord], candidates: Sequence[FillerCandidate], cfg: FillerConfig
208
+ ) -> tuple[tuple[TimedWord, ...], list[FillerCandidate]]:
209
+ """Words with the accepted candidates added as filler words, in time order.
210
+
211
+ A candidate is accepted when its text is in ``cfg.fillers`` and ``conf >= min_conf``;
212
+ it becomes ``TimedWord(text, speech_start_ms, speech_end_ms)`` (no alignment score).
213
+ Candidates come from the gaps of ``words``, so they never overlap a word.
214
+
215
+ Returns:
216
+ ``(words_with_fillers, accepted_candidates)``.
217
+
218
+ Raises:
219
+ ValueError: a candidate overlaps a word or another candidate (a caller bug).
220
+ """
221
+ accepted = [c for c in candidates if c.text in cfg.fillers and c.conf >= cfg.min_conf]
222
+ added = [TimedWord(c.text, c.speech_start_ms, c.speech_end_ms) for c in accepted]
223
+ merged = sorted([*words, *added], key=lambda w: (w.start_ms, w.end_ms))
224
+ for a, b in pairwise(merged):
225
+ if b.start_ms < a.end_ms:
226
+ raise ValueError(f"filler overlaps a word: {a} / {b}")
227
+ return tuple(merged), accepted
src/nida/data/fusion.py ADDED
@@ -0,0 +1,842 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """S7 — deterministic fusion of transcript, alignment and paralinguistic detections.
2
+
3
+ Turns one segment's aligned words (S5), event detections and per-word style scores (S6)
4
+ into tagged text following docs/02-tag-spec.md, or into a quarantine record with a
5
+ machine-readable reason. No LLM, no randomness: the same inputs always give the same
6
+ output. Rules (docs/03-data-pipeline.md S7, tag-spec §7):
7
+
8
+ 1. Bands: a detection below ``low`` is ignored (and counted); at or above ``high`` it is
9
+ used; in the middle band ``[low, high)`` it drops the whole segment
10
+ (``ambiguous_event`` / ``ambiguous_style``) — unless every millisecond of it is
11
+ covered by confident detections of the same family (then the event is there anyway).
12
+ 2. Families: ``laugh`` and ``laugh_long`` are one family, every other kind its own.
13
+ Confident detections of one family that overlap or are at most ``merge_gap_ms`` apart
14
+ are merged.
15
+ 3. Speech overlap: a laugh/cry detection covering more than ``style_word_overlap`` of a
16
+ word's duration marks the word ``laughing`` / ``crying``. If its longest non-speech
17
+ part is at least ``min_residual_event_ms`` it is *also* placed as an event. Any other
18
+ detection whose speech overlap exceeds ``max_event_speech_overlap`` of its length
19
+ drops the segment (``event_overlaps_speech``: false positive or overlapped audio).
20
+ 4. Placement: an event is placed at the word gap nearest the middle of its longest
21
+ non-speech part. A laugh becomes ``laugh_long`` if a detector said so or that placed
22
+ part lasts at least ``laugh_long_min_ms``. Placed events of different kinds whose
23
+ parts overlap by more than ``max_cross_kind_overlap`` of the shorter one drop the
24
+ segment (``conflicting_events``): taggers often fire related classes on one sound.
25
+ 5. Spans: consecutive words marked with the same style form a span if they are at least
26
+ ``min_span_words`` words or ``min_span_ms`` long; a shorter run drops the segment
27
+ (``short_style_run``: audible but untaggable). A word with two styles drops the
28
+ segment (``conflicting_styles``). Per-word score tracks of an ``ignored_styles``
29
+ style are discarded before any of this: the detector is known to be unreliable, so
30
+ it neither tags nor drops anything (unlike ``disabled_tags``, which drop the segment
31
+ whenever the detector fires at or above ``low``).
32
+ 6. Pauses: measured on the longest part of an inter-word gap not covered by a confident
33
+ detection. Longer than ``pause_long_ms`` → ``pause_long``; at least ``pause_short_ms``
34
+ → ``pause_short`` unless the previous word ends in punctuation. None at segment edges.
35
+ 7. Text: words are NFC; standalone fillers are canonicalised to their lowercase canonical
36
+ form (tag-spec §4); words with digits drop the segment (``unnormalised_number``) until
37
+ number normalisation exists; ``[ ] < >`` in a word drops it
38
+ (``reserved_char_in_text``). The output is validated by round-tripping through
39
+ :func:`nida.text.tags.parse` (``invalid_output``).
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import math
45
+ import unicodedata
46
+ from collections.abc import Mapping, Sequence
47
+ from dataclasses import dataclass, field
48
+
49
+ from nida.data.types import EventDetection, QuarantineRecord, TimedWord, WordStyleScore
50
+ from nida.text.fillers import canonical_filler
51
+ from nida.text.tags import EVENTS, SPANS, Event, Item, Span, TagSyntaxError, Word, parse, render
52
+ from nida.text.turkish import nfc, tr_lower
53
+
54
+ __all__ = [
55
+ "EMOTION_LABELS",
56
+ "STAGE",
57
+ "STYLE_FROM_EVENT",
58
+ "Band",
59
+ "EmotionCondition",
60
+ "FusionConfig",
61
+ "FusionResult",
62
+ "TagRecord",
63
+ "emotion_condition",
64
+ "fuse_segment",
65
+ "normalise_word",
66
+ ]
67
+
68
+ STAGE = "fusion"
69
+
70
+ #: Event families that, when they overlap words, mean the words carry a style (rule 3).
71
+ STYLE_FROM_EVENT: Mapping[str, str] = {"laugh": "laughing", "cry": "crying"}
72
+
73
+ #: Output classes of the emotion model (emotion2vec+); ``other``/``unknown`` map to null.
74
+ EMOTION_LABELS: frozenset[str] = frozenset(
75
+ {"angry", "disgusted", "fearful", "happy", "neutral", "other", "sad", "surprised", "unknown"}
76
+ )
77
+ _NULL_EMOTIONS = frozenset({"other", "unknown"})
78
+ _PAUSE_EVENTS = frozenset({"pause_short", "pause_long"})
79
+ _RESERVED_CHARS = frozenset("[]<>")
80
+
81
+
82
+ def _family(kind: str) -> str:
83
+ return "laugh" if kind in {"laugh", "laugh_long"} else kind
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class Band:
88
+ """Confidence band: ``< low`` ignored, ``[low, high)`` ambiguous, ``>= high`` used."""
89
+
90
+ low: float
91
+ high: float
92
+
93
+ def __post_init__(self) -> None:
94
+ if not (0.0 <= self.low <= self.high <= 1.0):
95
+ raise ValueError(f"invalid band low={self.low} high={self.high}")
96
+
97
+
98
+ _INT_FIELDS = (
99
+ "min_span_words",
100
+ "min_span_ms",
101
+ "min_residual_event_ms",
102
+ "laugh_long_min_ms",
103
+ "merge_gap_ms",
104
+ "pause_short_ms",
105
+ "pause_long_ms",
106
+ )
107
+ _FRACTION_FIELDS = (
108
+ "style_word_overlap",
109
+ "max_event_speech_overlap",
110
+ "max_cross_kind_overlap",
111
+ "emotion_min_conf",
112
+ )
113
+
114
+
115
+ @dataclass(frozen=True)
116
+ class FusionConfig:
117
+ """Fusion thresholds (configs/data/fusion.yaml). All fields are required."""
118
+
119
+ bands: Mapping[str, Band]
120
+ disabled_tags: frozenset[str]
121
+ ignored_styles: frozenset[str]
122
+ min_span_words: int
123
+ min_span_ms: int
124
+ style_word_overlap: float
125
+ max_event_speech_overlap: float
126
+ max_cross_kind_overlap: float
127
+ min_residual_event_ms: int
128
+ laugh_long_min_ms: int
129
+ merge_gap_ms: int
130
+ pause_short_ms: int
131
+ pause_long_ms: int
132
+ emotion_min_conf: float
133
+
134
+ def __post_init__(self) -> None:
135
+ required = (set(EVENTS) - _PAUSE_EVENTS) | set(SPANS)
136
+ if set(self.bands) != required:
137
+ missing = sorted(required - set(self.bands))
138
+ extra = sorted(set(self.bands) - required)
139
+ raise ValueError(f"bands must cover exactly the detectable tags: {missing=} {extra=}")
140
+ unknown = self.disabled_tags - (set(EVENTS) | set(SPANS))
141
+ if unknown:
142
+ raise ValueError(f"unknown disabled tags {sorted(unknown)}")
143
+ # Only styles that come as per-word score tracks can be ignored; laughing/crying
144
+ # come from events (STYLE_FROM_EVENT) and have no track to discard.
145
+ scored = set(SPANS) - set(STYLE_FROM_EVENT.values())
146
+ not_styles = self.ignored_styles - scored
147
+ if not_styles:
148
+ raise ValueError(
149
+ f"ignored_styles must be span styles with score tracks {sorted(scored)}, "
150
+ f"got {sorted(not_styles)}"
151
+ )
152
+ both = self.ignored_styles & self.disabled_tags
153
+ if both:
154
+ raise ValueError(f"tags both disabled and ignored: {sorted(both)}")
155
+ for name in _INT_FIELDS:
156
+ value = getattr(self, name)
157
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
158
+ raise ValueError(f"{name} must be a non-negative int, got {value!r}")
159
+ if self.min_span_words < 1:
160
+ raise ValueError("min_span_words must be >= 1")
161
+ if self.min_residual_event_ms < 1:
162
+ raise ValueError("min_residual_event_ms must be >= 1")
163
+ for name in _FRACTION_FIELDS:
164
+ value = getattr(self, name)
165
+ if isinstance(value, bool) or not isinstance(value, int | float):
166
+ raise ValueError(f"{name} must be a number, got {value!r}")
167
+ if not (math.isfinite(value) and 0.0 <= value <= 1.0):
168
+ raise ValueError(f"{name} must be in [0, 1], got {value!r}")
169
+ if not 0 < self.pause_short_ms <= self.pause_long_ms:
170
+ raise ValueError("need 0 < pause_short_ms <= pause_long_ms")
171
+
172
+ @classmethod
173
+ def from_mapping(cls, raw: Mapping[str, object]) -> FusionConfig:
174
+ """Build from a parsed YAML mapping; unknown or missing keys are errors."""
175
+ expected = {f.name for f in cls.__dataclass_fields__.values()}
176
+ if set(raw) != expected:
177
+ raise ValueError(
178
+ f"fusion config keys: missing={sorted(expected - set(raw))} "
179
+ f"unknown={sorted(set(raw) - expected)}"
180
+ )
181
+ bands_raw = raw["bands"]
182
+ if not isinstance(bands_raw, Mapping):
183
+ raise ValueError("bands must be a mapping")
184
+ bands: dict[str, Band] = {}
185
+ for kind, band in bands_raw.items():
186
+ if not isinstance(band, Mapping) or set(band) != {"low", "high"}:
187
+ raise ValueError(f"band {kind!r} must be a mapping with exactly low/high")
188
+ low, high = band["low"], band["high"]
189
+ if not all(isinstance(v, int | float) and not isinstance(v, bool) for v in (low, high)):
190
+ raise ValueError(f"band {kind!r} values must be numbers")
191
+ bands[str(kind)] = Band(float(low), float(high))
192
+ disabled = raw["disabled_tags"]
193
+ if not isinstance(disabled, list) or not all(isinstance(t, str) for t in disabled):
194
+ raise ValueError("disabled_tags must be a list of tag ids")
195
+ ignored = raw["ignored_styles"]
196
+ if not isinstance(ignored, list) or not all(isinstance(t, str) for t in ignored):
197
+ raise ValueError("ignored_styles must be a list of style ids")
198
+ lists = {"bands", "disabled_tags", "ignored_styles"}
199
+ values = {k: v for k, v in raw.items() if k not in lists}
200
+ # The remaining values are typed `object` here; __post_init__ validates each one.
201
+ return cls(
202
+ bands=bands,
203
+ disabled_tags=frozenset(disabled),
204
+ ignored_styles=frozenset(ignored),
205
+ **values, # type: ignore[arg-type]
206
+ )
207
+
208
+
209
+ @dataclass(frozen=True)
210
+ class TagRecord:
211
+ """Provenance of one emitted tag (tag-spec §7.3): ``{id, start, end, conf, source}``.
212
+
213
+ ``start_ms``/``end_ms`` is what the tag stands for (the placed non-speech part of an
214
+ event, the words of a span, the silence of a pause); ``det_start_ms``/``det_end_ms``
215
+ is the merged detector interval it came from, so it can be checked against the
216
+ detector output.
217
+ """
218
+
219
+ tag: str
220
+ start_ms: int
221
+ end_ms: int
222
+ conf: float
223
+ source: str
224
+ det_start_ms: int
225
+ det_end_ms: int
226
+
227
+
228
+ @dataclass(frozen=True)
229
+ class FusionResult:
230
+ """Either ``items``/``text`` (accepted) or ``quarantine`` (dropped), never both."""
231
+
232
+ segment_id: str
233
+ items: tuple[Item, ...] | None
234
+ text: str | None
235
+ tags: tuple[TagRecord, ...]
236
+ quarantine: QuarantineRecord | None
237
+ ignored_detections: int = 0
238
+ #: The below-band detections themselves (``len == ignored_detections``), so a caller can
239
+ #: record what was heard but left untagged (ADR-0010: uncertain breaths keep the segment).
240
+ ignored_events: tuple[EventDetection, ...] = ()
241
+
242
+ def __post_init__(self) -> None:
243
+ accepted = self.items is not None and self.text is not None
244
+ dropped = self.items is None and self.text is None and not self.tags
245
+ if self.quarantine is None and not accepted:
246
+ raise ValueError("accepted result needs items and text")
247
+ if self.quarantine is not None and not dropped:
248
+ raise ValueError("quarantined result must not carry items, text or tags")
249
+
250
+ @property
251
+ def accepted(self) -> bool:
252
+ return self.quarantine is None
253
+
254
+
255
+ @dataclass(frozen=True)
256
+ class EmotionCondition:
257
+ emotion: str | None
258
+ intensity: float | None
259
+
260
+
261
+ @dataclass
262
+ class _Merged:
263
+ family: str
264
+ kinds: set[str]
265
+ start_ms: int
266
+ end_ms: int
267
+ conf: float
268
+ sources: set[str] = field(default_factory=set)
269
+
270
+ @property
271
+ def source(self) -> str:
272
+ return "+".join(sorted(self.sources))
273
+
274
+
275
+ @dataclass(frozen=True)
276
+ class _Placed:
277
+ gap: int
278
+ kind: str
279
+ start_ms: int
280
+ end_ms: int
281
+ event: _Merged
282
+
283
+
284
+ class _Drop(Exception):
285
+ """Internal: the segment is quarantined with ``reason``."""
286
+
287
+ def __init__(self, reason: str, **details: object) -> None:
288
+ super().__init__(reason)
289
+ self.reason = reason
290
+ self.details = details
291
+
292
+
293
+ def _overlap_ms(a_start: int, a_end: int, b_start: int, b_end: int) -> int:
294
+ return max(0, min(a_end, b_end) - max(a_start, b_start))
295
+
296
+
297
+ def _union(intervals: Sequence[tuple[int, int]]) -> list[tuple[int, int]]:
298
+ out: list[tuple[int, int]] = []
299
+ for start, end in sorted(intervals):
300
+ if out and start <= out[-1][1]:
301
+ out[-1] = (out[-1][0], max(out[-1][1], end))
302
+ else:
303
+ out.append((start, end))
304
+ return out
305
+
306
+
307
+ def _validate_inputs(
308
+ words: Sequence[TimedWord],
309
+ duration_ms: int,
310
+ events: Sequence[EventDetection],
311
+ styles: Sequence[WordStyleScore],
312
+ ) -> None:
313
+ if type(duration_ms) is not int or duration_ms <= 0:
314
+ raise ValueError(f"duration_ms must be a positive int, got {duration_ms!r}")
315
+ previous_end = 0
316
+ for word in words:
317
+ if word.start_ms < previous_end:
318
+ raise ValueError(f"words overlap or are unsorted at {word.text!r} ({word.start_ms} ms)")
319
+ if word.end_ms > duration_ms:
320
+ raise ValueError(f"word {word.text!r} ends after the segment ({word.end_ms} ms)")
321
+ previous_end = word.end_ms
322
+ for event in events:
323
+ if event.kind not in EVENTS or event.kind in _PAUSE_EVENTS:
324
+ raise ValueError(f"not a detectable event kind: {event.kind!r}")
325
+ if event.end_ms > duration_ms:
326
+ raise ValueError(f"event {event.kind!r} ends after the segment")
327
+ seen: set[str] = set()
328
+ for style in styles:
329
+ if style.style not in SPANS:
330
+ raise ValueError(f"unknown style {style.style!r}")
331
+ if style.style in seen:
332
+ raise ValueError(f"duplicate style scores for {style.style!r}")
333
+ seen.add(style.style)
334
+ if len(style.scores) != len(words):
335
+ raise ValueError(
336
+ f"style {style.style!r} has {len(style.scores)} scores for {len(words)} words"
337
+ )
338
+
339
+
340
+ def _split_by_band(
341
+ events: Sequence[EventDetection], cfg: FusionConfig
342
+ ) -> tuple[list[EventDetection], list[EventDetection], list[EventDetection]]:
343
+ """(confident, middle band, ignored = below ``low``)."""
344
+ confident: list[EventDetection] = []
345
+ middle: list[EventDetection] = []
346
+ ignored: list[EventDetection] = []
347
+ for event in events:
348
+ band = cfg.bands[event.kind]
349
+ if event.conf < band.low:
350
+ ignored.append(event)
351
+ elif event.conf < band.high:
352
+ middle.append(event)
353
+ else:
354
+ confident.append(event)
355
+ return confident, middle, ignored
356
+
357
+
358
+ def _merge_confident(
359
+ confident: Sequence[EventDetection], middle: Sequence[EventDetection], cfg: FusionConfig
360
+ ) -> list[_Merged]:
361
+ """Rules 1–2: ambiguity check against confident cover, then per-family merging."""
362
+ cover: dict[str, list[tuple[int, int]]] = {}
363
+ for event in confident:
364
+ cover.setdefault(_family(event.kind), []).append((event.start_ms, event.end_ms))
365
+ union = {family: _union(spans) for family, spans in cover.items()}
366
+ for event in sorted(middle, key=lambda e: (e.start_ms, e.end_ms, e.kind, e.source, e.conf)):
367
+ if event.kind in cfg.disabled_tags:
368
+ raise _Drop("disabled_tag", tag=event.kind, start_ms=event.start_ms, conf=event.conf)
369
+ covered = sum(
370
+ _overlap_ms(event.start_ms, event.end_ms, a, b)
371
+ for a, b in union.get(_family(event.kind), [])
372
+ )
373
+ if covered < event.duration_ms:
374
+ raise _Drop(
375
+ "ambiguous_event",
376
+ kind=event.kind,
377
+ conf=event.conf,
378
+ start_ms=event.start_ms,
379
+ end_ms=event.end_ms,
380
+ source=event.source,
381
+ )
382
+ for event in sorted(confident, key=lambda e: (e.start_ms, e.end_ms, e.kind, e.source)):
383
+ if event.kind in cfg.disabled_tags:
384
+ # Checked on the raw kind too: a disabled `laugh` must not survive by being
385
+ # relabelled `laugh_long` after merging.
386
+ raise _Drop("disabled_tag", tag=event.kind, start_ms=event.start_ms, conf=event.conf)
387
+ merged: list[_Merged] = []
388
+ ordered = sorted(
389
+ confident, key=lambda e: (_family(e.kind), e.start_ms, e.end_ms, e.kind, e.source)
390
+ )
391
+ for event in ordered:
392
+ family = _family(event.kind)
393
+ last = merged[-1] if merged else None
394
+ if (
395
+ last is not None
396
+ and last.family == family
397
+ and event.start_ms <= last.end_ms + cfg.merge_gap_ms
398
+ ):
399
+ last.end_ms = max(last.end_ms, event.end_ms)
400
+ last.conf = max(last.conf, event.conf)
401
+ last.kinds.add(event.kind)
402
+ last.sources.add(event.source)
403
+ else:
404
+ merged.append(
405
+ _Merged(
406
+ family, {event.kind}, event.start_ms, event.end_ms, event.conf, {event.source}
407
+ )
408
+ )
409
+ return sorted(merged, key=lambda m: (m.start_ms, m.end_ms, m.family))
410
+
411
+
412
+ def _non_speech_pieces(
413
+ start_ms: int, end_ms: int, words: Sequence[TimedWord]
414
+ ) -> list[tuple[int, int]]:
415
+ """Parts of ``[start_ms, end_ms)`` not covered by any word (words are sorted, disjoint)."""
416
+ pieces: list[tuple[int, int]] = []
417
+ cursor = start_ms
418
+ for word in words:
419
+ if word.end_ms <= cursor:
420
+ continue
421
+ if word.start_ms >= end_ms:
422
+ break
423
+ if word.start_ms > cursor:
424
+ pieces.append((cursor, word.start_ms))
425
+ cursor = max(cursor, word.end_ms)
426
+ if cursor < end_ms:
427
+ pieces.append((cursor, end_ms))
428
+ return pieces
429
+
430
+
431
+ def _nearest_gap(time_ms: float, words: Sequence[TimedWord]) -> int:
432
+ """Gap ``g`` lies before word ``g`` (``g == len(words)``: after the last word)."""
433
+ for index, word in enumerate(words):
434
+ if time_ms <= word.start_ms:
435
+ return index
436
+ if time_ms < word.end_ms:
437
+ # Inside a word (defensive: callers pass the middle of a non-speech piece, which
438
+ # never lies inside a word): the closer boundary wins; a tie goes after it.
439
+ return index if time_ms - word.start_ms < word.end_ms - time_ms else index + 1
440
+ return len(words)
441
+
442
+
443
+ def _event_kind(event: _Merged, placed_ms: int, cfg: FusionConfig) -> str:
444
+ if event.family != "laugh":
445
+ return event.family
446
+ if "laugh_long" in event.kinds or placed_ms >= cfg.laugh_long_min_ms:
447
+ return "laugh_long"
448
+ return "laugh"
449
+
450
+
451
+ def _place_events(
452
+ merged: Sequence[_Merged], words: Sequence[TimedWord], cfg: FusionConfig
453
+ ) -> tuple[list[_Placed], list[tuple[str, list[int], _Merged]]]:
454
+ """Rules 3–4. Returns (placed events, style evidence ``(style, word indices, event)``)."""
455
+ placed: list[_Placed] = []
456
+ style_events: list[tuple[str, list[int], _Merged]] = []
457
+ for event in merged:
458
+ length = event.end_ms - event.start_ms
459
+ style_id = STYLE_FROM_EVENT.get(event.family)
460
+ styled_words = [
461
+ i
462
+ for i, w in enumerate(words)
463
+ if style_id is not None
464
+ and _overlap_ms(event.start_ms, event.end_ms, w.start_ms, w.end_ms)
465
+ > cfg.style_word_overlap * w.duration_ms
466
+ ]
467
+ pieces = _non_speech_pieces(event.start_ms, event.end_ms, words)
468
+ longest = max(pieces, key=lambda p: (p[1] - p[0], -p[0])) if pieces else None
469
+ longest_ms = longest[1] - longest[0] if longest is not None else 0
470
+ if styled_words:
471
+ assert style_id is not None
472
+ # The detection may also cover words it did not mark (overlap <= the style
473
+ # threshold); that speech is laughed/cried over untagged, so bound it like any
474
+ # other event's speech overlap.
475
+ unmarked_ms = sum(
476
+ _overlap_ms(event.start_ms, event.end_ms, w.start_ms, w.end_ms)
477
+ for i, w in enumerate(words)
478
+ if i not in styled_words
479
+ )
480
+ if unmarked_ms > cfg.max_event_speech_overlap * length:
481
+ raise _Drop(
482
+ "event_overlaps_speech",
483
+ kind=sorted(event.kinds),
484
+ start_ms=event.start_ms,
485
+ end_ms=event.end_ms,
486
+ speech_ms=unmarked_ms,
487
+ )
488
+ style_events.append((style_id, styled_words, event))
489
+ if longest_ms < cfg.min_residual_event_ms:
490
+ continue
491
+ else:
492
+ speech_ms = length - sum(b - a for a, b in pieces)
493
+ if longest is None or speech_ms > cfg.max_event_speech_overlap * length:
494
+ raise _Drop(
495
+ "event_overlaps_speech",
496
+ kind=sorted(event.kinds),
497
+ start_ms=event.start_ms,
498
+ end_ms=event.end_ms,
499
+ speech_ms=speech_ms,
500
+ )
501
+ assert longest is not None
502
+ kind = _event_kind(event, longest_ms, cfg)
503
+ if kind in cfg.disabled_tags:
504
+ raise _Drop("disabled_tag", tag=kind, start_ms=event.start_ms, conf=event.conf)
505
+ placed.append(
506
+ _Placed(_nearest_gap((longest[0] + longest[1]) / 2, words), kind, *longest, event)
507
+ )
508
+ for i, first in enumerate(placed):
509
+ for second in placed[i + 1 :]:
510
+ if first.kind == second.kind:
511
+ continue
512
+ shorter = min(first.end_ms - first.start_ms, second.end_ms - second.start_ms)
513
+ shared = _overlap_ms(first.start_ms, first.end_ms, second.start_ms, second.end_ms)
514
+ if shared > cfg.max_cross_kind_overlap * shorter:
515
+ raise _Drop(
516
+ "conflicting_events",
517
+ kinds=sorted({first.kind, second.kind}),
518
+ start_ms=max(first.start_ms, second.start_ms),
519
+ shared_ms=shared,
520
+ )
521
+ return placed, style_events
522
+
523
+
524
+ def _style_marks(
525
+ words: Sequence[TimedWord],
526
+ styles: Sequence[WordStyleScore],
527
+ style_events: Sequence[tuple[str, list[int], _Merged]],
528
+ cfg: FusionConfig,
529
+ ) -> tuple[list[str | None], dict[int, tuple[float, str, int, int]]]:
530
+ """Per-word style (or None) and, per marked word, its evidence
531
+ ``(conf, source, det_start_ms, det_end_ms)`` (the word itself for acoustic scores)."""
532
+ marks: list[set[str]] = [set() for _ in words]
533
+ evidence: list[dict[str, tuple[float, str, int, int]]] = [{} for _ in words]
534
+ for style in sorted(styles, key=lambda s: s.style):
535
+ if style.style in cfg.ignored_styles:
536
+ continue
537
+ band = cfg.bands[style.style]
538
+ for index, score in enumerate(style.scores):
539
+ if score is None or score < band.low:
540
+ continue
541
+ if score < band.high:
542
+ raise _Drop(
543
+ "ambiguous_style",
544
+ style=style.style,
545
+ word_index=index,
546
+ word=words[index].text,
547
+ score=score,
548
+ source=style.source,
549
+ )
550
+ if style.style in cfg.disabled_tags:
551
+ raise _Drop("disabled_tag", tag=style.style, word_index=index, score=score)
552
+ marks[index].add(style.style)
553
+ word = words[index]
554
+ evidence[index][style.style] = (score, style.source, word.start_ms, word.end_ms)
555
+ for style_id, indices, event in style_events:
556
+ if style_id in cfg.disabled_tags:
557
+ raise _Drop("disabled_tag", tag=style_id, start_ms=event.start_ms, conf=event.conf)
558
+ for index in indices:
559
+ marks[index].add(style_id)
560
+ previous = evidence[index].get(style_id)
561
+ if previous is None or event.conf > previous[0]:
562
+ evidence[index][style_id] = (
563
+ event.conf,
564
+ event.source,
565
+ event.start_ms,
566
+ event.end_ms,
567
+ )
568
+ resolved: list[str | None] = []
569
+ for index, word_marks in enumerate(marks):
570
+ if len(word_marks) > 1:
571
+ raise _Drop(
572
+ "conflicting_styles",
573
+ word_index=index,
574
+ word=words[index].text,
575
+ styles=sorted(word_marks),
576
+ )
577
+ resolved.append(next(iter(word_marks)) if word_marks else None)
578
+ chosen = {
579
+ index: evidence[index][style_id]
580
+ for index, style_id in enumerate(resolved)
581
+ if style_id is not None
582
+ }
583
+ return resolved, chosen
584
+
585
+
586
+ def _style_runs(
587
+ words: Sequence[TimedWord], marks: Sequence[str | None], cfg: FusionConfig
588
+ ) -> list[tuple[str, int, int]]:
589
+ """Rule 5: maximal runs ``(style, first, last)`` of equally marked consecutive words."""
590
+ runs: list[tuple[str, int, int]] = []
591
+ index = 0
592
+ while index < len(marks):
593
+ style_id = marks[index]
594
+ if style_id is None:
595
+ index += 1
596
+ continue
597
+ last = index
598
+ while last + 1 < len(marks) and marks[last + 1] == style_id:
599
+ last += 1
600
+ count = last - index + 1
601
+ span_ms = words[last].end_ms - words[index].start_ms
602
+ if count < cfg.min_span_words and span_ms < cfg.min_span_ms:
603
+ raise _Drop(
604
+ "short_style_run",
605
+ style=style_id,
606
+ words=[w.text for w in words[index : last + 1]],
607
+ span_ms=span_ms,
608
+ )
609
+ runs.append((style_id, index, last))
610
+ index = last + 1
611
+ return runs
612
+
613
+
614
+ def _normalise_word(text: str) -> str:
615
+ """Rule 7 for one word."""
616
+ text = nfc(text)
617
+ if any(ch in _RESERVED_CHARS for ch in text):
618
+ raise _Drop("reserved_char_in_text", word=text)
619
+ if any(unicodedata.category(ch).startswith("N") for ch in text): # 3, ², ½, Ⅳ
620
+ raise _Drop("unnormalised_number", word=text)
621
+ start, end = 0, len(text)
622
+ while start < end and unicodedata.category(text[start]).startswith("P"):
623
+ start += 1
624
+ while end > start and unicodedata.category(text[end - 1]).startswith("P"):
625
+ end -= 1
626
+ core = text[start:end]
627
+ if not core:
628
+ return text
629
+ canonical = canonical_filler(tr_lower(core))
630
+ return text if canonical is None else text[:start] + canonical + text[end:]
631
+
632
+
633
+ def normalise_word(text: str) -> str:
634
+ """Rule 7 for one word, as :func:`fuse_segment` applies it to every aligned word.
635
+
636
+ Lets later stages (``timings``) compare the words of a stored tagged text with the
637
+ aligned words they came from.
638
+
639
+ Raises:
640
+ ValueError: if fusion would drop the segment for this word (reserved tag character
641
+ or a digit-like character), with fusion's reason in the message.
642
+ """
643
+ try:
644
+ return _normalise_word(text)
645
+ except _Drop as drop:
646
+ raise ValueError(f"{drop.reason}: {drop.details}") from drop
647
+
648
+
649
+ def _ends_with_punctuation(text: str) -> bool:
650
+ return unicodedata.category(text[-1]).startswith("P")
651
+
652
+
653
+ def _pauses(
654
+ words: Sequence[TimedWord],
655
+ texts: Sequence[str],
656
+ merged: Sequence[_Merged],
657
+ cfg: FusionConfig,
658
+ ) -> list[_Placed]:
659
+ """Rule 6, on the longest part of each gap that no confident detection covers.
660
+
661
+ A short breath inside a 5 s silence must not hide the pause; the pause tag record
662
+ spans that uncovered part, and tags in one gap are later ordered by time.
663
+ """
664
+ out: list[_Placed] = []
665
+ for gap in range(1, len(words)):
666
+ gap_start, gap_end = words[gap - 1].end_ms, words[gap].start_ms
667
+ covered = _union(
668
+ [
669
+ (max(gap_start, e.start_ms), min(gap_end, e.end_ms))
670
+ for e in merged
671
+ if _overlap_ms(gap_start, gap_end, e.start_ms, e.end_ms) > 0
672
+ ]
673
+ )
674
+ free: list[tuple[int, int]] = []
675
+ cursor = gap_start
676
+ for a, b in covered:
677
+ if a > cursor:
678
+ free.append((cursor, a))
679
+ cursor = max(cursor, b)
680
+ if cursor < gap_end:
681
+ free.append((cursor, gap_end))
682
+ if not free:
683
+ continue
684
+ start, end = max(free, key=lambda p: (p[1] - p[0], -p[0]))
685
+ silence = end - start
686
+ kind: str | None = None
687
+ if silence > cfg.pause_long_ms:
688
+ kind = "pause_long"
689
+ elif silence >= cfg.pause_short_ms and not _ends_with_punctuation(texts[gap - 1]):
690
+ kind = "pause_short"
691
+ if kind is not None:
692
+ if kind in cfg.disabled_tags:
693
+ raise _Drop("disabled_tag", tag=kind, start_ms=start)
694
+ silence_event = _Merged(kind, {kind}, start, end, 1.0, {"alignment"})
695
+ out.append(_Placed(gap, kind, start, end, silence_event))
696
+ return out
697
+
698
+
699
+ def _build_items(
700
+ texts: Sequence[str],
701
+ events_by_gap: Mapping[int, Sequence[_Placed]],
702
+ runs: Sequence[tuple[str, int, int]],
703
+ ) -> tuple[Item, ...]:
704
+ run_at = {first: (style_id, last) for style_id, first, last in runs}
705
+ items: list[Item] = []
706
+ index = 0
707
+ while True:
708
+ items.extend(Event(p.kind) for p in events_by_gap.get(index, ()))
709
+ if index == len(texts):
710
+ break
711
+ if index in run_at:
712
+ style_id, last = run_at[index]
713
+ children: list[Word | Event] = []
714
+ for inner in range(index, last + 1):
715
+ if inner > index:
716
+ children.extend(Event(p.kind) for p in events_by_gap.get(inner, ()))
717
+ children.append(Word(texts[inner]))
718
+ items.append(Span(style_id, tuple(children)))
719
+ index = last + 1
720
+ else:
721
+ items.append(Word(texts[index]))
722
+ index += 1
723
+ return tuple(items)
724
+
725
+
726
+ def fuse_segment(
727
+ segment_id: str,
728
+ words: Sequence[TimedWord],
729
+ duration_ms: int,
730
+ events: Sequence[EventDetection],
731
+ styles: Sequence[WordStyleScore],
732
+ cfg: FusionConfig,
733
+ ) -> FusionResult:
734
+ """Fuse one segment's annotations into tagged text (see module docstring).
735
+
736
+ Args:
737
+ segment_id: Content-addressed id (:func:`nida.data.types.segment_id`).
738
+ words: Aligned words, sorted and non-overlapping, segment-relative ms.
739
+ duration_ms: Segment length in ms.
740
+ events: Raw event detections (any confidence; bands are applied here).
741
+ styles: At most one per-word score track per span style.
742
+ cfg: Thresholds.
743
+
744
+ Returns:
745
+ A :class:`FusionResult`; when ``quarantine`` is set, ``items``/``text`` are None.
746
+ ``ignored_detections`` counts below-band detections in both cases and
747
+ ``ignored_events`` holds them.
748
+
749
+ Raises:
750
+ ValueError: on inconsistent inputs (a caller bug, not a data property).
751
+ """
752
+ _validate_inputs(words, duration_ms, events, styles)
753
+ confident, middle, ignored = _split_by_band(events, cfg)
754
+ try:
755
+ if not words:
756
+ raise _Drop("no_words")
757
+ texts = [_normalise_word(w.text) for w in words]
758
+ merged = _merge_confident(confident, middle, cfg)
759
+ placed, style_events = _place_events(merged, words, cfg)
760
+ marks, evidence = _style_marks(words, styles, style_events, cfg)
761
+ runs = _style_runs(words, marks, cfg)
762
+ placed += _pauses(words, texts, merged, cfg)
763
+
764
+ events_by_gap: dict[int, list[_Placed]] = {}
765
+ tags: list[TagRecord] = []
766
+ for p in sorted(placed, key=lambda p: (p.start_ms, p.end_ms, p.kind)):
767
+ events_by_gap.setdefault(p.gap, []).append(p)
768
+ tags.append(
769
+ TagRecord(
770
+ p.kind,
771
+ p.start_ms,
772
+ p.end_ms,
773
+ p.event.conf,
774
+ p.event.source,
775
+ p.event.start_ms,
776
+ p.event.end_ms,
777
+ )
778
+ )
779
+ for style_id, first, last in runs:
780
+ run_evidence = [evidence[i] for i in range(first, last + 1)]
781
+ tags.append(
782
+ TagRecord(
783
+ style_id,
784
+ words[first].start_ms,
785
+ words[last].end_ms,
786
+ min(e[0] for e in run_evidence),
787
+ "+".join(sorted({e[1] for e in run_evidence})),
788
+ min(e[2] for e in run_evidence),
789
+ max(e[3] for e in run_evidence),
790
+ )
791
+ )
792
+ try:
793
+ items = _build_items(texts, events_by_gap, runs)
794
+ text = render(items)
795
+ if parse(text) != items:
796
+ raise _Drop("invalid_output", text=text, detail="round-trip mismatch")
797
+ except (TagSyntaxError, ValueError) as err:
798
+ raise _Drop("invalid_output", error=str(err)) from err
799
+ except _Drop as drop:
800
+ return FusionResult(
801
+ segment_id=segment_id,
802
+ items=None,
803
+ text=None,
804
+ tags=(),
805
+ quarantine=QuarantineRecord(segment_id, STAGE, drop.reason, dict(drop.details)),
806
+ ignored_detections=len(ignored),
807
+ ignored_events=tuple(ignored),
808
+ )
809
+ return FusionResult(
810
+ segment_id=segment_id,
811
+ items=items,
812
+ text=text,
813
+ tags=tuple(sorted(tags, key=lambda t: (t.start_ms, t.end_ms, t.tag))),
814
+ quarantine=None,
815
+ ignored_detections=len(ignored),
816
+ ignored_events=tuple(ignored),
817
+ )
818
+
819
+
820
+ def emotion_condition(scores: Mapping[str, float], cfg: FusionConfig) -> EmotionCondition:
821
+ """Utterance-level emotion condition from emotion-model class probabilities.
822
+
823
+ The top class becomes the condition with its probability as intensity. ``other`` /
824
+ ``unknown`` as top class, or a top probability below ``emotion_min_conf``, give a
825
+ null condition (not "neutral"; tag-spec §5, ADR-0003). Ties resolve to the
826
+ alphabetically first label so the result is deterministic.
827
+
828
+ Raises:
829
+ ValueError: unknown labels, empty scores, or probabilities outside [0, 1].
830
+ """
831
+ if not scores:
832
+ raise ValueError("empty emotion scores")
833
+ unknown = set(scores) - EMOTION_LABELS
834
+ if unknown:
835
+ raise ValueError(f"unknown emotion labels {sorted(unknown)}")
836
+ for label, prob in scores.items():
837
+ if not (math.isfinite(prob) and 0.0 <= prob <= 1.0):
838
+ raise ValueError(f"emotion {label!r} probability {prob!r} not in [0, 1]")
839
+ label, prob = min(scores.items(), key=lambda kv: (-kv[1], kv[0]))
840
+ if label in _NULL_EMOTIONS or prob < cfg.emotion_min_conf:
841
+ return EmotionCondition(None, None)
842
+ return EmotionCondition(label, prob)
src/nida/data/importer/__init__.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Import of an already transcribed external Turkish speech corpus (``nida data-import``).
2
+
3
+ The result is a *latent dataset directory* in exactly the format of :mod:`nida.train.data`
4
+ (``index.jsonl``, ``shard-NNNNN.npy`` float16, ``meta.json``) plus ``timings.jsonl``
5
+ (:mod:`nida.data.timings`), so ``load_latent_dataset(dir, split, timings=True)`` accepts it and
6
+ ``train --data A --data B ...`` can mix it with the YODAS pilot datasets. There is no ASR, no
7
+ fusion and no tag detection: the transcript is trusted, aligned and encoded.
8
+
9
+ Sources (adapters, :mod:`nida.data.importer.sources`; all must be ``approved`` in
10
+ ``configs/sources.yaml``)::
11
+
12
+ alania HF cloud0day3/alania-synthetic-speech-tr, config cc-by only (parquet shards,
13
+ 48 kHz FLAC), QC-filtered, text = ``text_spoken``, group = voice id
14
+ tr_full HF Taklaxbr/tr-full-dataset (``data.jsonl`` + wavs), language tr only, emotion
15
+ neutral/angry/sad/happy, group = speaker id
16
+ wavdir a folder with ``manifest.jsonl`` {file, text, group, source, emotion?}: our own
17
+ synthetic speech (e.g. ema-lightning); each line's ``source`` must be approved
18
+ issai_tsc HF issai/Turkish_Speech_Corpus: one 21 GB ``.tar.gz``
19
+ (``ISSAI_TSC_218/{Train,Dev,Test}/<n>.wav|.txt``, 16 kHz); no speaker in the
20
+ corpus, so group = block of running utterance numbers (see import_corpora.yaml)
21
+ mediaspeech_tr
22
+ OpenSLR SLR108 ``TR.tgz`` (``TR/<uuid>.flac|.txt``), pinned by the sha256 of the
23
+ archive (recorded as the plan's revision); the whole corpus is one group
24
+ common_voice_tr
25
+ a Common Voice release the user downloaded (``--source-dir``: release folder,
26
+ its ``tr`` folder, or the tarball): ``validated.tsv`` rows with
27
+ ``down_votes < up_votes``, MP3 clips, group = hash of ``client_id``
28
+
29
+ A corpus is cut into deterministic *work units* (a parquet shard, a chunk of rows). A root
30
+ (``--root``) is bound to one plan (source, pinned Hub commit sha / archive sha256 / sha1 of the
31
+ local manifest, ``--max-hours``, ``--shard-offset``/``--shards`` unit range, config digest)
32
+ recorded in ``plan.json``; several roots
33
+ with disjoint unit ranges select disjoint data, and the split of a group is a pure function of
34
+ the group id, so groups stay disjoint across roots.
35
+
36
+ Stages (each idempotent and resumable; ``all`` runs them in order)::
37
+
38
+ select -> audio -> latents -> ctc -> timings -> dataset -> report
39
+
40
+ * ``select`` (CPU, network): per unit, metadata filters and transcript normalisation
41
+ (:mod:`nida.data.importer.text`: NFC, text with tag characters rejected, digits spelled out,
42
+ every character must be in the model vocabulary; fillers are not canonicalised); the raw
43
+ audio of the selected rows is put in the local cache; stops at ``--max-hours``. Finished
44
+ units are stored, so an interruption loses at most one unit.
45
+ * ``audio`` (CPU): 16 kHz mono PCM wav per item in the cache, duration and loudness (dBFS)
46
+ measured, duration/silence checks. Lost cache files are rebuilt (the unit is fetched again).
47
+ * ``latents`` / ``ctc`` (GPU): the pilot's ``vae_encode`` and ``ctc_emissions`` workers
48
+ (same isolated environments, same resumable results protocol,
49
+ :func:`nida.data.pilot.run_worker_job`).
50
+ * ``timings`` (CPU): forced alignment of ``text.split()``, the pilot's alignment sanity check
51
+ (:func:`nida.data.asr_filter.check_alignment`, thresholds in ``import.yaml``) and the unit
52
+ timeline (:func:`nida.data.timings.build_timings`, no event tags).
53
+ * ``dataset`` (CPU): ``dataset-<config digest>/`` with only the segments that got a timeline;
54
+ group-disjoint deterministic splits (a pure function of seed and group id: ~3 % of the
55
+ groups val, ~2 % test; the same group is in the same split in every root); rate/loudness
56
+ buckets with the fixed edges of ``configs/data/dataset.yaml``, exactly like the pilot;
57
+ ``language`` ``tr``; ``segment_id = sha1(source id, item id)[:16]``. Verified with the
58
+ training loader. ``meta.json`` is written last.
59
+ * ``report``: ``report.md`` and ``metrics.json``.
60
+
61
+ Every item a stage drops is written to ``<root>/<stage>/quarantine.jsonl`` with a
62
+ machine-readable reason (:mod:`nida.data.quarantine`); counts are in the stage results and in
63
+ the report. Layout::
64
+
65
+ <root>/plan.json
66
+ <root>/select/{manifest.jsonl, quarantine.jsonl, stage.json, units/<digest>.json}
67
+ <root>/{audio,latents,ctc}/{manifest.jsonl, quarantine.jsonl, stage.json}
68
+ <root>/timings/{timings.jsonl, quarantine.jsonl, stage.json}
69
+ <root>/workers/{voxcpm2_audiovae_v2,mpoyraz_ctc_tr}/ worker requests/results/summary
70
+ <root>/dataset-<digest>/{index.jsonl, shard-NNNNN.npy, timings.jsonl, meta.json}
71
+ <root>/{report.md, metrics.json}
72
+ <cache>/hub, raw, audio16k, arrays/ bulky intermediates on local disk, rebuilt when lost
73
+
74
+ Durable outputs live under ``root`` (Drive), bulky intermediates under ``cache`` (local).
75
+ Thresholds and mappings are in ``configs/data/import.yaml`` (shared, and the three first
76
+ sources) and ``configs/data/import_corpora.yaml`` (the three corpora above; its sections have
77
+ their own config digest, so adding them did not move the digest of the first three sources).
78
+ """
79
+
80
+ from __future__ import annotations
81
+
82
+ from nida.data.importer.config import ImportConfig, load_import_config
83
+ from nida.data.importer.corpora import CommonVoiceAdapter, IssaiTscAdapter, MediaSpeechAdapter
84
+ from nida.data.importer.hub import HubClient, HuggingFaceHub
85
+ from nida.data.importer.pipeline import STAGES, Importer, ImportPlan, ImportRequest
86
+ from nida.data.importer.sources import (
87
+ AlaniaAdapter,
88
+ Item,
89
+ SourceAdapter,
90
+ TrFullAdapter,
91
+ WavdirAdapter,
92
+ item_segment_id,
93
+ make_adapter,
94
+ )
95
+ from nida.data.importer.text import TextRejected, normalise_transcript
96
+
97
+ __all__ = [
98
+ "STAGES",
99
+ "AlaniaAdapter",
100
+ "CommonVoiceAdapter",
101
+ "HubClient",
102
+ "HuggingFaceHub",
103
+ "ImportConfig",
104
+ "ImportPlan",
105
+ "ImportRequest",
106
+ "Importer",
107
+ "IssaiTscAdapter",
108
+ "Item",
109
+ "MediaSpeechAdapter",
110
+ "SourceAdapter",
111
+ "TextRejected",
112
+ "TrFullAdapter",
113
+ "WavdirAdapter",
114
+ "item_segment_id",
115
+ "load_import_config",
116
+ "make_adapter",
117
+ "normalise_transcript",
118
+ ]
src/nida/data/importer/archives.py ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """One streaming pass over a tar archive, for the importer sources that ship as one.
2
+
3
+ Three sources arrive as an archive (the ISSAI corpus: a 21 GB ``.tar.gz``; MediaSpeech:
4
+ ``TR.tgz``; a Common Voice release tarball). A stream cannot be searched, and the audio and
5
+ the transcript of an utterance are far apart in it, so the adapters work in *passes*:
6
+
7
+ * an *index* pass reads the small transcript members into memory (and notes which audio
8
+ members exist);
9
+ * an *extract* pass writes the audio members a work unit needs to the cache.
10
+
11
+ Both are :func:`scan_archive`. A member is never written to a path derived from its name: a
12
+ caller-supplied classifier maps a (normalised) member name to a ``(key, kind)`` pair or rejects
13
+ it, and audio goes to the path the caller registered for that key. Anything the classifier
14
+ does not know is an error, not a silent skip (the archives are pinned, so a surprise means
15
+ the format changed).
16
+
17
+ A pass that does not stop early must see the archive *end*: the decompressor is a file object
18
+ that raises on a truncated stream (gzip/bz2/xz trailer), and the tar stream must finish with
19
+ its zero blocks. A cut inside a header, or exactly between members, is an :class:`ArchiveError`,
20
+ never a smaller corpus. (``tarfile`` alone ends silently in both cases and does not check the
21
+ gzip trailer in stream mode.)
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import bz2
27
+ import gzip
28
+ import hashlib
29
+ import json
30
+ import lzma
31
+ import tarfile
32
+ import zlib
33
+ from collections.abc import Callable, Mapping
34
+ from dataclasses import dataclass, field
35
+ from pathlib import Path
36
+ from typing import IO, Literal, Protocol, cast
37
+
38
+ from nida.utils.io import atomic_write_bytes, atomic_write_json
39
+
40
+ __all__ = [
41
+ "MAX_AUDIO_BYTES",
42
+ "MAX_TEXT_BYTES",
43
+ "ArchiveError",
44
+ "ArchiveScan",
45
+ "Classifier",
46
+ "Kind",
47
+ "scan_archive",
48
+ "verify_sha256_once",
49
+ ]
50
+
51
+ #: A transcript member larger than this is not a transcript (the biggest real text member is
52
+ #: a Common Voice ``validated.tsv``, tens of MB).
53
+ MAX_TEXT_BYTES = 512 * 1024 * 1024
54
+ #: An audio member larger than this is refused instead of being read into memory.
55
+ MAX_AUDIO_BYTES = 512 * 1024 * 1024
56
+ #: Names of unexpected members listed in the error message.
57
+ _SHOWN_UNEXPECTED = 5
58
+ #: A tar archive ends with (at least) two zero blocks of this many bytes.
59
+ _BLOCK_BYTES = 512
60
+ _END_BYTES = 2 * _BLOCK_BYTES
61
+ _DRAIN_BYTES = 1 << 20
62
+ _HASH_CHUNK_BYTES = 1 << 20
63
+
64
+ Kind = Literal["text", "audio", "ignore"]
65
+ #: Maps a normalised member name (``/``-separated, no leading ``./``) to ``(key, kind)``, or
66
+ #: ``None`` for a member that must not exist in the archive.
67
+ Classifier = Callable[[str], tuple[str, Kind] | None]
68
+
69
+
70
+ class ArchiveError(RuntimeError):
71
+ """The archive is unreadable, truncated, corrupt or not laid out as expected."""
72
+
73
+
74
+ @dataclass
75
+ class ArchiveScan:
76
+ """What one pass found.
77
+
78
+ Attributes:
79
+ texts: ``key -> bytes`` of the ``text`` members (empty unless ``read_text``).
80
+ audio_heads: ``key -> first bytes`` of the ``audio`` members (see ``audio_head``).
81
+ audio_keys: Keys of every ``audio`` member seen (an early stop leaves it partial).
82
+ extracted: Keys of the audio members written to their registered paths.
83
+ members: Regular-file members visited.
84
+ complete: ``True`` if the whole archive was read, up to its end blocks (no early stop).
85
+ """
86
+
87
+ texts: dict[str, bytes] = field(default_factory=dict)
88
+ audio_heads: dict[str, bytes] = field(default_factory=dict)
89
+ audio_keys: set[str] = field(default_factory=set)
90
+ extracted: set[str] = field(default_factory=set)
91
+ members: int = 0
92
+ complete: bool = True
93
+
94
+
95
+ def _normalise(name: str) -> str:
96
+ while name.startswith("./"):
97
+ name = name[2:]
98
+ return name.rstrip("/")
99
+
100
+
101
+ class _ByteStream(Protocol):
102
+ def read(self, size: int = -1, /) -> bytes: ...
103
+
104
+
105
+ class _Reader:
106
+ """The decompressed byte stream ``tarfile`` reads.
107
+
108
+ Counts the bytes handed out and remembers where the last non-zero byte ended (to check the
109
+ tar end blocks), and turns every error of the decompressor into :class:`ArchiveError`.
110
+ Errors of *writing* extracted files are not touched: they happen elsewhere.
111
+ """
112
+
113
+ def __init__(self, stream: _ByteStream, archive: Path) -> None:
114
+ self._stream = stream
115
+ self._archive = archive
116
+ self.pos = 0
117
+ self.last_nonzero_end = 0
118
+
119
+ def read(self, size: int = -1) -> bytes:
120
+ try:
121
+ data = self._stream.read(size)
122
+ except (OSError, EOFError, zlib.error, lzma.LZMAError) as err:
123
+ raise ArchiveError(
124
+ f"{self._archive}: cannot be read (truncated or corrupt? after "
125
+ f"{self.pos} bytes): {err!r}"
126
+ ) from err
127
+ if data:
128
+ stripped = data.rstrip(b"\x00")
129
+ if stripped:
130
+ self.last_nonzero_end = self.pos + len(stripped)
131
+ self.pos += len(data)
132
+ return data
133
+
134
+
135
+ def _decompressed(raw: IO[bytes]) -> _ByteStream:
136
+ """``raw`` wrapped in the file object of its compression (by magic number), or itself."""
137
+ magic = raw.read(6)
138
+ raw.seek(0)
139
+ if magic.startswith(b"\x1f\x8b"):
140
+ return gzip.GzipFile(fileobj=raw)
141
+ if magic.startswith(b"BZh"):
142
+ return bz2.BZ2File(raw)
143
+ if magic.startswith(b"\xfd7zXZ\x00"):
144
+ return lzma.LZMAFile(raw)
145
+ return raw
146
+
147
+
148
+ def scan_archive(
149
+ archive: Path,
150
+ classify: Classifier,
151
+ *,
152
+ read_text: bool,
153
+ extract: Mapping[str, Path] | None = None,
154
+ audio_head: int = 0,
155
+ ) -> ArchiveScan:
156
+ """Read ``archive`` once (plain, gzip, bz2 or xz; auto-detected).
157
+
158
+ Args:
159
+ archive: Local tar file.
160
+ classify: Member classifier (see :data:`Classifier`).
161
+ read_text: Keep the bytes of every ``text`` member.
162
+ extract: ``key -> destination`` of the ``audio`` members to write (atomically). When
163
+ ``read_text`` is false the pass stops as soon as all of them are written.
164
+ audio_head: Keep the first this many bytes of every ``audio`` member (e.g. a FLAC
165
+ header, to learn the duration without reading the audio).
166
+
167
+ Raises:
168
+ ArchiveError: unreadable/truncated/corrupt archive (including one that ends without
169
+ its end blocks, or whose compressed stream has no trailer); a member that is
170
+ neither a regular file nor a directory, or that ``classify`` rejects; a key seen
171
+ twice; an oversized member.
172
+ FileNotFoundError: ``archive`` does not exist.
173
+ OSError: a destination cannot be written (disk full ...).
174
+ """
175
+ scan = ArchiveScan()
176
+ unexpected: list[str] = []
177
+ seen: set[str] = set()
178
+ wanted = extract or {}
179
+ try:
180
+ with archive.open("rb") as raw:
181
+ reader = _Reader(_decompressed(raw), archive)
182
+ with tarfile.open(
183
+ fileobj=cast(IO[bytes], reader), # stream mode only calls ``read``
184
+ mode="r|",
185
+ ) as tar:
186
+ for member in tar:
187
+ # Streaming mode keeps every TarInfo alive otherwise (hundreds of thousands).
188
+ tar.members = [] # type: ignore[attr-defined] # runtime attribute, not typed
189
+ if member.isdir():
190
+ continue
191
+ name = _normalise(member.name)
192
+ classified = classify(name) if member.isfile() else None
193
+ if classified is None:
194
+ unexpected.append(name)
195
+ continue
196
+ key, kind = classified
197
+ if kind == "ignore":
198
+ continue
199
+ if key in seen:
200
+ raise ArchiveError(f"{archive}: member {name!r} occurs twice")
201
+ seen.add(key)
202
+ scan.members += 1
203
+ if kind == "text":
204
+ _check_size(archive, name, member.size, MAX_TEXT_BYTES)
205
+ if read_text:
206
+ scan.texts[key] = _read(tar, member, archive)
207
+ elif key in wanted:
208
+ _write(tar, member, archive, wanted[key])
209
+ scan.extracted.add(key)
210
+ else:
211
+ scan.audio_keys.add(key)
212
+ if audio_head:
213
+ scan.audio_heads[key] = _read(tar, member, archive, audio_head)
214
+ if key in wanted:
215
+ _check_size(archive, name, member.size, MAX_AUDIO_BYTES)
216
+ _write(tar, member, archive, wanted[key])
217
+ scan.extracted.add(key)
218
+ if not read_text and wanted and len(scan.extracted) == len(wanted):
219
+ scan.complete = False
220
+ break
221
+ if scan.complete:
222
+ _require_end_blocks(archive, reader, tar.offset)
223
+ except (tarfile.TarError, EOFError, zlib.error, gzip.BadGzipFile, lzma.LZMAError) as err:
224
+ raise ArchiveError(f"{archive}: cannot be read as a tar archive: {err!r}") from err
225
+ if unexpected and scan.complete:
226
+ shown = ", ".join(repr(n) for n in unexpected[:_SHOWN_UNEXPECTED])
227
+ raise ArchiveError(
228
+ f"{archive}: {len(unexpected)} unexpected member(s) (not a directory or a known "
229
+ f"file; the format changed?): {shown}"
230
+ )
231
+ return scan
232
+
233
+
234
+ def _require_end_blocks(archive: Path, reader: _Reader, end_offset: int) -> None:
235
+ """The member loop ended: make sure it ended at the archive's end blocks and read the
236
+ stream to its true end (that is where gzip/bz2/xz check their trailer).
237
+
238
+ ``end_offset`` is where ``tarfile`` stopped looking for headers. From there on the stream
239
+ must be zeros only, at least two blocks of them; a cut inside a header leaves non-zero
240
+ bytes or too few zeros, a cut between members leaves nothing.
241
+ """
242
+ while reader.read(_DRAIN_BYTES):
243
+ pass
244
+ if reader.last_nonzero_end > end_offset or reader.pos - end_offset < _END_BYTES:
245
+ raise ArchiveError(
246
+ f"{archive}: ends without the tar end-of-archive blocks (truncated?): "
247
+ f"{reader.pos - end_offset} bytes after the last member at {end_offset}, "
248
+ f"non-zero data up to {reader.last_nonzero_end}"
249
+ )
250
+
251
+
252
+ def verify_sha256_once(path: Path, expected: str) -> None:
253
+ """Check that the file at ``path`` has the SHA-256 ``expected``, hashing it only once.
254
+
255
+ A marker next to the file (``<name>.verified.json``) records the size, mtime and digest of
256
+ a verified file; while those still match, the (21 GB) file is not read again.
257
+
258
+ Raises:
259
+ ArchiveError: the digest differs (the file is incomplete or not the pinned one).
260
+ """
261
+ marker = path.with_name(path.name + ".verified.json")
262
+ stat = path.stat()
263
+ key = {"size": stat.st_size, "mtime_ns": stat.st_mtime_ns, "sha256": expected}
264
+ if marker.is_file():
265
+ try:
266
+ if json.loads(marker.read_text(encoding="utf-8")) == key:
267
+ return
268
+ except json.JSONDecodeError:
269
+ pass # a damaged marker is only a missing marker: hash again
270
+ digest = hashlib.sha256()
271
+ with path.open("rb") as handle:
272
+ while chunk := handle.read(_HASH_CHUNK_BYTES):
273
+ digest.update(chunk)
274
+ if digest.hexdigest() != expected:
275
+ marker.unlink(missing_ok=True)
276
+ raise ArchiveError(
277
+ f"{path}: sha256 {digest.hexdigest()} (size {stat.st_size}) is not the pinned "
278
+ f"{expected}; the download is incomplete or corrupt - delete it and run again"
279
+ )
280
+ atomic_write_json(marker, key)
281
+
282
+
283
+ def _check_size(archive: Path, name: str, size: int, limit: int) -> None:
284
+ if size > limit:
285
+ raise ArchiveError(f"{archive}: member {name!r} has {size} bytes (limit {limit})")
286
+
287
+
288
+ def _read(tar: tarfile.TarFile, member: tarfile.TarInfo, archive: Path, size: int = -1) -> bytes:
289
+ handle = tar.extractfile(member)
290
+ if handle is None:
291
+ raise ArchiveError(f"{archive}: cannot read member {member.name!r}")
292
+ return handle.read(size)
293
+
294
+
295
+ def _write(tar: tarfile.TarFile, member: tarfile.TarInfo, archive: Path, dest: Path) -> None:
296
+ atomic_write_bytes(dest, _read(tar, member, archive))
src/nida/data/importer/audio.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Audio helpers of the import stages: raw file -> 16 kHz mono PCM wav, plus measurements.
2
+
3
+ The wav is what both model workers read (the CTC model requires 16 kHz mono; the VAE encoder
4
+ resamples to its own rate itself, from 16 kHz exactly like the pilot's YODAS audio). Channels
5
+ are averaged, the signal is resampled with a polyphase filter
6
+ (:func:`nida.data.ears.resample`) and written as 16-bit PCM atomically.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import io
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+
15
+ import numpy as np
16
+ import numpy.typing as npt
17
+
18
+ from nida.data import acoustic
19
+ from nida.data.ears import resample
20
+ from nida.utils.io import atomic_write_bytes
21
+
22
+ __all__ = [
23
+ "TARGET_RATE_HZ",
24
+ "AudioFacts",
25
+ "AudioRejected",
26
+ "convert_to_wav16k",
27
+ "measure_wav16k",
28
+ "raw_duration_ms",
29
+ "require_audio_format",
30
+ ]
31
+
32
+ #: Rate of the working copy (CTC requirement).
33
+ TARGET_RATE_HZ = 16000
34
+
35
+
36
+ class AudioRejected(Exception):
37
+ """An audio file that cannot be used; ``reason`` is the quarantine reason."""
38
+
39
+ def __init__(self, reason: str, **details: object) -> None:
40
+ super().__init__(f"{reason}: {details}")
41
+ self.reason = reason
42
+ self.details = dict(details)
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class AudioFacts:
47
+ """Measurements of a 16 kHz working copy.
48
+
49
+ Attributes:
50
+ duration_ms: ``round(1000 * samples / 16000)``.
51
+ loudness_dbfs: Whole-signal RMS level (:func:`nida.data.acoustic.loudness_dbfs`).
52
+ """
53
+
54
+ duration_ms: int
55
+ loudness_dbfs: float
56
+
57
+
58
+ def require_audio_format(name: str) -> None:
59
+ """Raise unless the installed libsndfile reads the format ``name`` (e.g. ``"MP3"``,
60
+ which needs libsndfile >= 1.1, bundled with ``soundfile`` >= 0.12).
61
+
62
+ Raises:
63
+ RuntimeError: the format is not available (installing a newer ``soundfile`` fixes it).
64
+ """
65
+ import soundfile # type: ignore[import-untyped] # bench extra, untyped
66
+
67
+ if name not in soundfile.available_formats():
68
+ raise RuntimeError(
69
+ f"this soundfile/libsndfile ({soundfile.__libsndfile_version__}) cannot read "
70
+ f"{name}; upgrade soundfile (>= 0.12 bundles libsndfile 1.1+)"
71
+ )
72
+
73
+
74
+ def raw_duration_ms(path: Path) -> int:
75
+ """Duration of an audio file from its header (no decoding).
76
+
77
+ Raises:
78
+ AudioRejected: ``unreadable_audio`` (not an audio file) or ``empty_audio``.
79
+ """
80
+ import soundfile
81
+
82
+ try:
83
+ info = soundfile.info(str(path))
84
+ except (RuntimeError, OSError) as err:
85
+ raise AudioRejected("unreadable_audio", error=str(err)) from err
86
+ if int(info.frames) <= 0 or int(info.samplerate) <= 0:
87
+ raise AudioRejected("empty_audio")
88
+ duration_ms: int = round(1000 * int(info.frames) / int(info.samplerate))
89
+ return duration_ms
90
+
91
+
92
+ def _read_mono(path: Path) -> tuple[npt.NDArray[np.float64], int]:
93
+ """``(samples, rate_hz)`` with channels averaged; rejects unreadable/empty/non-finite."""
94
+ import soundfile
95
+
96
+ try:
97
+ audio, rate_hz = soundfile.read(str(path), dtype="float64", always_2d=True)
98
+ except (RuntimeError, OSError) as err:
99
+ raise AudioRejected("unreadable_audio", error=str(err)) from err
100
+ if audio.shape[0] == 0:
101
+ raise AudioRejected("empty_audio")
102
+ mono = np.ascontiguousarray(audio.mean(axis=1))
103
+ if not np.all(np.isfinite(mono)):
104
+ raise AudioRejected("non_finite_audio")
105
+ return mono, int(rate_hz)
106
+
107
+
108
+ def convert_to_wav16k(raw: Path, wav: Path) -> None:
109
+ """Write the 16 kHz mono 16-bit PCM copy of ``raw`` atomically to ``wav``.
110
+
111
+ Raises:
112
+ AudioRejected: ``unreadable_audio``, ``empty_audio`` or ``non_finite_audio``.
113
+ """
114
+ import soundfile
115
+
116
+ mono, rate_hz = _read_mono(raw)
117
+ converted = resample(mono, rate_hz, TARGET_RATE_HZ)
118
+ buffer = io.BytesIO()
119
+ soundfile.write(
120
+ buffer, np.clip(converted, -1.0, 1.0), TARGET_RATE_HZ, format="WAV", subtype="PCM_16"
121
+ )
122
+ atomic_write_bytes(wav, buffer.getvalue())
123
+
124
+
125
+ def measure_wav16k(wav: Path) -> AudioFacts:
126
+ """Duration and loudness of a working copy.
127
+
128
+ Raises:
129
+ AudioRejected: the file is unreadable/empty, or is not a 16 kHz mono file (a stale
130
+ or foreign cache entry; the caller reconverts it).
131
+ """
132
+ import soundfile
133
+
134
+ try:
135
+ info = soundfile.info(str(wav))
136
+ except (RuntimeError, OSError) as err:
137
+ raise AudioRejected("unreadable_audio", error=str(err)) from err
138
+ if int(info.samplerate) != TARGET_RATE_HZ or int(info.channels) != 1:
139
+ raise AudioRejected(
140
+ "bad_working_copy", sample_rate_hz=int(info.samplerate), channels=int(info.channels)
141
+ )
142
+ mono, _ = _read_mono(wav)
143
+ return AudioFacts(
144
+ duration_ms=round(1000 * mono.shape[0] / TARGET_RATE_HZ),
145
+ loudness_dbfs=float(acoustic.loudness_dbfs(mono)),
146
+ )
src/nida/data/importer/config.py ADDED
@@ -0,0 +1,474 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """``configs/data/import.yaml`` (+ ``dataset.yaml``) as validated, frozen records."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import re
9
+ from collections.abc import Mapping
10
+ from dataclasses import asdict, dataclass
11
+ from pathlib import Path
12
+
13
+ from nida.data.asr_filter import AsrFilterConfig
14
+ from nida.data.ears import en_lower
15
+ from nida.data.pilot import DatasetConfig
16
+ from nida.data.registry import DATA_MODELS
17
+ from nida.train.data import EMOTIONS, SPLITS
18
+
19
+ __all__ = [
20
+ "CORPUS_SOURCES",
21
+ "LEGACY_SOURCES",
22
+ "AlaniaConfig",
23
+ "CommonVoiceConfig",
24
+ "ImportConfig",
25
+ "IssaiTscConfig",
26
+ "MediaSpeechConfig",
27
+ "TrFullConfig",
28
+ "WavdirConfig",
29
+ "load_import_config",
30
+ ]
31
+
32
+ IMPORT_CONFIG_FILE = "import.yaml"
33
+ DATASET_CONFIG_FILE = "dataset.yaml"
34
+ #: Sections of the corpora added after the first three sources (see :func:`load_import_config`).
35
+ CORPORA_CONFIG_FILE = "import_corpora.yaml"
36
+ #: Importer sources whose config lives in ``import.yaml``; their digest is the original one.
37
+ LEGACY_SOURCES = ("alania", "tr_full", "wavdir")
38
+ #: Importer sources configured in ``import_corpora.yaml`` (a digest scoped per source).
39
+ CORPUS_SOURCES = ("issai_tsc", "mediaspeech_tr", "common_voice_tr")
40
+ _SHA256_RE = re.compile(r"[0-9a-f]{64}")
41
+
42
+
43
+ def _load_yaml(path: Path) -> Mapping[str, object]:
44
+ import yaml # type: ignore[import-untyped] # PyYAML: bench extra, no stubs installed
45
+
46
+ obj = yaml.safe_load(path.read_text(encoding="utf-8"))
47
+ if not isinstance(obj, dict):
48
+ raise ValueError(f"{path}: expected a mapping")
49
+ return obj
50
+
51
+
52
+ def _keys(raw: Mapping[str, object], expected: set[str], where: str) -> None:
53
+ if set(raw) != expected:
54
+ raise ValueError(
55
+ f"{where}: keys differ: missing={sorted(expected - set(raw))} "
56
+ f"unknown={sorted(set(raw) - expected)}"
57
+ )
58
+
59
+
60
+ def _mapping(raw: Mapping[str, object], key: str) -> Mapping[str, object]:
61
+ value = raw[key]
62
+ if not isinstance(value, dict) or not all(isinstance(k, str) for k in value):
63
+ raise ValueError(f"{key} must be a mapping")
64
+ return value
65
+
66
+
67
+ def _text(raw: Mapping[str, object], key: str, *, allow_empty: bool = False) -> str:
68
+ value = raw[key]
69
+ if not isinstance(value, str) or not (value or allow_empty):
70
+ raise ValueError(f"{key} must be a non-empty string, got {value!r}")
71
+ return value
72
+
73
+
74
+ def _int(raw: Mapping[str, object], key: str, *, minimum: int) -> int:
75
+ value = raw[key]
76
+ if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
77
+ raise ValueError(f"{key} must be an int >= {minimum}, got {value!r}")
78
+ return value
79
+
80
+
81
+ def _bool(raw: Mapping[str, object], key: str) -> bool:
82
+ value = raw[key]
83
+ if not isinstance(value, bool):
84
+ raise ValueError(f"{key} must be true or false, got {value!r}")
85
+ return value
86
+
87
+
88
+ def _float(raw: Mapping[str, object], key: str, *, lo: float, hi: float) -> float:
89
+ value = raw[key]
90
+ if isinstance(value, bool) or not isinstance(value, int | float):
91
+ raise ValueError(f"{key} must be a number, got {value!r}")
92
+ if not (math.isfinite(value) and lo <= value <= hi):
93
+ raise ValueError(f"{key} must be in [{lo}, {hi}], got {value!r}")
94
+ return float(value)
95
+
96
+
97
+ @dataclass(frozen=True)
98
+ class AlaniaConfig:
99
+ """``alania:`` block (source ``alania_synth_tr``)."""
100
+
101
+ repo_id: str
102
+ parquet_glob: str
103
+ min_utmos: float
104
+ max_cer: float
105
+ license_prefix: str
106
+ batch_rows: int
107
+ group_prefix: str
108
+
109
+ @classmethod
110
+ def from_mapping(cls, raw: Mapping[str, object]) -> AlaniaConfig:
111
+ _keys(raw, set(cls.__dataclass_fields__), "alania")
112
+ return cls(
113
+ repo_id=_text(raw, "repo_id"),
114
+ parquet_glob=_text(raw, "parquet_glob"),
115
+ min_utmos=_float(raw, "min_utmos", lo=0.0, hi=5.0),
116
+ max_cer=_float(raw, "max_cer", lo=0.0, hi=1.0),
117
+ license_prefix=_text(raw, "license_prefix"),
118
+ batch_rows=_int(raw, "batch_rows", minimum=1),
119
+ group_prefix=_text(raw, "group_prefix", allow_empty=True),
120
+ )
121
+
122
+
123
+ @dataclass(frozen=True)
124
+ class TrFullConfig:
125
+ """``tr_full:`` block (source ``tr_full_dataset``)."""
126
+
127
+ repo_id: str
128
+ data_file: str
129
+ language: str
130
+ unit_rows: int
131
+ group_prefix: str
132
+
133
+ @classmethod
134
+ def from_mapping(cls, raw: Mapping[str, object]) -> TrFullConfig:
135
+ _keys(raw, set(cls.__dataclass_fields__), "tr_full")
136
+ return cls(
137
+ repo_id=_text(raw, "repo_id"),
138
+ data_file=_text(raw, "data_file"),
139
+ language=_text(raw, "language"),
140
+ unit_rows=_int(raw, "unit_rows", minimum=1),
141
+ group_prefix=_text(raw, "group_prefix", allow_empty=True),
142
+ )
143
+
144
+
145
+ @dataclass(frozen=True)
146
+ class WavdirConfig:
147
+ """``wavdir:`` block (a folder with ``manifest.jsonl``)."""
148
+
149
+ unit_rows: int
150
+ group_prefix: str
151
+
152
+ @classmethod
153
+ def from_mapping(cls, raw: Mapping[str, object]) -> WavdirConfig:
154
+ _keys(raw, set(cls.__dataclass_fields__), "wavdir")
155
+ return cls(
156
+ unit_rows=_int(raw, "unit_rows", minimum=1),
157
+ group_prefix=_text(raw, "group_prefix", allow_empty=True),
158
+ )
159
+
160
+
161
+ def _str_list(raw: Mapping[str, object], key: str) -> tuple[str, ...]:
162
+ value = raw[key]
163
+ if (
164
+ not isinstance(value, list)
165
+ or not value
166
+ or not all(isinstance(v, str) and v for v in value)
167
+ or len(set(value)) != len(value)
168
+ ):
169
+ raise ValueError(f"{key} must be a non-empty list of distinct non-empty strings")
170
+ return tuple(value)
171
+
172
+
173
+ def _plain_name(raw: Mapping[str, object], key: str) -> str:
174
+ """A single path component (no separators, not ``.``/``..``)."""
175
+ value = _text(raw, key)
176
+ if "/" in value or "\\" in value or "\x00" in value or value in (".", ".."):
177
+ raise ValueError(f"{key} must be one plain file or folder name, got {value!r}")
178
+ return value
179
+
180
+
181
+ def _sha256(raw: Mapping[str, object], key: str) -> str:
182
+ value = _text(raw, key)
183
+ if _SHA256_RE.fullmatch(value) is None:
184
+ raise ValueError(f"{key} must be 64 lower-case hex digits, got {value!r}")
185
+ return value
186
+
187
+
188
+ def _force_split(raw: Mapping[str, object]) -> str | None:
189
+ """``force_split``: ``null`` (split by group hash) or the one split every row goes to."""
190
+ value = raw["force_split"]
191
+ if value is not None and value not in SPLITS:
192
+ raise ValueError(f"force_split must be null or one of {SPLITS}, got {value!r}")
193
+ return value
194
+
195
+
196
+ @dataclass(frozen=True)
197
+ class IssaiTscConfig:
198
+ """``issai_tsc:`` block of ``import_corpora.yaml`` (source ``issai_tsc``).
199
+
200
+ ``group_block_ids``: the corpus names no speaker or session, only a running utterance
201
+ number that follows the recording order, so ``numeric id // group_block_ids`` is the
202
+ group (reporting only while ``force_split`` is set). ``archive_sha256`` and
203
+ ``expected_pairs`` (utterance stems, wav and/or txt) guard against a truncated or changed
204
+ archive. ``force_split``: every row goes to this split instead of the group hash (the
205
+ corpus' recordings are contiguous and no group can be pure).
206
+ """
207
+
208
+ repo_id: str
209
+ archive_file: str
210
+ archive_sha256: str
211
+ expected_pairs: int
212
+ archive_root: str
213
+ splits: tuple[str, ...]
214
+ unit_rows: int
215
+ group_block_ids: int
216
+ group_prefix: str
217
+ force_split: str | None
218
+
219
+ @classmethod
220
+ def from_mapping(cls, raw: Mapping[str, object]) -> IssaiTscConfig:
221
+ _keys(raw, set(cls.__dataclass_fields__), "issai_tsc")
222
+ return cls(
223
+ repo_id=_text(raw, "repo_id"),
224
+ archive_file=_plain_name(raw, "archive_file"),
225
+ archive_sha256=_sha256(raw, "archive_sha256"),
226
+ expected_pairs=_int(raw, "expected_pairs", minimum=1),
227
+ archive_root=_plain_name(raw, "archive_root"),
228
+ splits=_str_list(raw, "splits"),
229
+ unit_rows=_int(raw, "unit_rows", minimum=1),
230
+ group_block_ids=_int(raw, "group_block_ids", minimum=1),
231
+ group_prefix=_text(raw, "group_prefix", allow_empty=True),
232
+ force_split=_force_split(raw),
233
+ )
234
+
235
+
236
+ @dataclass(frozen=True)
237
+ class MediaSpeechConfig:
238
+ """``mediaspeech_tr:`` block of ``import_corpora.yaml`` (OpenSLR SLR108 ``TR.tgz``).
239
+
240
+ ``sha256`` pins the archive (a different file is refused); ``expected_pairs`` is the
241
+ number of utterance stems it holds; ``group`` is the one group of the whole corpus (file
242
+ names are random UUIDs: no speaker, channel or video); ``force_split``: see
243
+ :class:`IssaiTscConfig`; ``reserve_bench_clips``: quarantine the clips the phase-0 B1
244
+ benchmark drew from this corpus (``bench_reserved``), so they stay unseen by training.
245
+ """
246
+
247
+ url: str
248
+ sha256: str
249
+ expected_pairs: int
250
+ archive_root: str
251
+ unit_rows: int
252
+ group: str
253
+ force_split: str | None
254
+ reserve_bench_clips: bool
255
+
256
+ @classmethod
257
+ def from_mapping(cls, raw: Mapping[str, object]) -> MediaSpeechConfig:
258
+ _keys(raw, set(cls.__dataclass_fields__), "mediaspeech_tr")
259
+ url = _text(raw, "url")
260
+ if not url.startswith("https://"):
261
+ raise ValueError(f"url must be an https:// URL, got {url!r}")
262
+ return cls(
263
+ url=url,
264
+ sha256=_sha256(raw, "sha256"),
265
+ expected_pairs=_int(raw, "expected_pairs", minimum=1),
266
+ archive_root=_plain_name(raw, "archive_root"),
267
+ unit_rows=_int(raw, "unit_rows", minimum=1),
268
+ group=_text(raw, "group"),
269
+ force_split=_force_split(raw),
270
+ reserve_bench_clips=_bool(raw, "reserve_bench_clips"),
271
+ )
272
+
273
+
274
+ @dataclass(frozen=True)
275
+ class CommonVoiceConfig:
276
+ """``common_voice_tr:`` block of ``import_corpora.yaml`` (a local Common Voice release).
277
+
278
+ ``min_net_votes``: a clip is used when ``up_votes - down_votes >= min_net_votes`` (1 means
279
+ ``down_votes < up_votes``). The group is a hash of ``client_id`` behind ``group_prefix``.
280
+ """
281
+
282
+ locale: str
283
+ validated_file: str
284
+ clips_dir: str
285
+ min_net_votes: int
286
+ unit_rows: int
287
+ group_prefix: str
288
+
289
+ @classmethod
290
+ def from_mapping(cls, raw: Mapping[str, object]) -> CommonVoiceConfig:
291
+ _keys(raw, set(cls.__dataclass_fields__), "common_voice_tr")
292
+ return cls(
293
+ locale=_text(raw, "locale"),
294
+ validated_file=_plain_name(raw, "validated_file"),
295
+ clips_dir=_plain_name(raw, "clips_dir"),
296
+ min_net_votes=_int(raw, "min_net_votes", minimum=1),
297
+ unit_rows=_int(raw, "unit_rows", minimum=1),
298
+ group_prefix=_text(raw, "group_prefix", allow_empty=True),
299
+ )
300
+
301
+
302
+ @dataclass(frozen=True)
303
+ class ImportConfig:
304
+ """Every threshold and mapping of the import stages.
305
+
306
+ Attributes:
307
+ seed: Split-hash seed (``str(seed)`` is passed to :func:`nida.data.splits.assign_splits`).
308
+ val_fraction: Share of groups in the ``val`` split.
309
+ test_fraction: Share of groups in the ``test`` split.
310
+ min_duration_ms: Shortest accepted segment.
311
+ max_duration_ms: Longest accepted segment.
312
+ min_loudness_dbfs: Segments quieter than this (RMS, dBFS) are ``silent``.
313
+ alignment: Thresholds of the forced-alignment sanity check
314
+ (:func:`nida.data.asr_filter.check_alignment`; ``max_cer`` and the languages of
315
+ that record are unused here).
316
+ char_map: Single-character replacements applied to transcripts.
317
+ emotion_map: Source emotion label -> :data:`nida.train.data.EMOTIONS` id.
318
+ issai_tsc: ``import_corpora.yaml`` section of the ISSAI Turkish Speech Corpus.
319
+ mediaspeech_tr: ``import_corpora.yaml`` section of MediaSpeech (SLR108).
320
+ common_voice_tr: ``import_corpora.yaml`` section of Common Voice.
321
+ dataset: Fixed rate/loudness bucket edges (``dataset.yaml``, shared with the pilot).
322
+ digest: 12 hex digits over the config files and the VAE/CTC model pins; names the
323
+ dataset directory and is recorded in ``plan.json``. For ``alania``, ``tr_full``,
324
+ ``wavdir`` (and when no source is given) it covers ``import.yaml`` and
325
+ ``dataset.yaml`` only - the value those roots were made with. For a source of
326
+ ``import_corpora.yaml`` it is derived from that value and the source's own section.
327
+ """
328
+
329
+ seed: int
330
+ val_fraction: float
331
+ test_fraction: float
332
+ min_duration_ms: int
333
+ max_duration_ms: int
334
+ min_loudness_dbfs: float
335
+ alignment: AsrFilterConfig
336
+ char_map: Mapping[str, str]
337
+ emotion_map: Mapping[str, str]
338
+ alania: AlaniaConfig
339
+ tr_full: TrFullConfig
340
+ wavdir: WavdirConfig
341
+ issai_tsc: IssaiTscConfig
342
+ mediaspeech_tr: MediaSpeechConfig
343
+ common_voice_tr: CommonVoiceConfig
344
+ dataset: DatasetConfig
345
+ digest: str
346
+
347
+ def forced_split(self, source: str) -> str | None:
348
+ """The split every row of ``source`` goes to (``force_split``), or ``None``."""
349
+ if source == "issai_tsc":
350
+ return self.issai_tsc.force_split
351
+ if source == "mediaspeech_tr":
352
+ return self.mediaspeech_tr.force_split
353
+ return None
354
+
355
+ @property
356
+ def split_fractions(self) -> dict[str, float]:
357
+ """``{"train": 1 - val - test, "val": ..., "test": ...}``."""
358
+ return {
359
+ "train": 1.0 - self.val_fraction - self.test_fraction,
360
+ "val": self.val_fraction,
361
+ "test": self.test_fraction,
362
+ }
363
+
364
+
365
+ def load_import_config(config_dir: Path, source: str | None = None) -> ImportConfig:
366
+ """Read and validate ``import.yaml``, ``dataset.yaml`` and ``import_corpora.yaml``.
367
+
368
+ Args:
369
+ config_dir: Folder with the three files.
370
+ source: The importer source the config is for; it scopes :attr:`ImportConfig.digest`
371
+ (``None`` or a legacy source: the digest of ``import.yaml`` + ``dataset.yaml``,
372
+ unchanged by ``import_corpora.yaml``; a corpus source: that digest combined with
373
+ the source's own section, so editing one corpus never moves another's roots).
374
+
375
+ Raises:
376
+ ValueError: on unknown/missing keys, out-of-range numbers, multi-character
377
+ ``char_map`` keys, an ``emotion_map`` target that is not an emotion id, or an
378
+ unknown ``source``.
379
+ FileNotFoundError: a config file is missing.
380
+ """
381
+ if source is not None and source not in (*LEGACY_SOURCES, *CORPUS_SOURCES):
382
+ known = (*LEGACY_SOURCES, *CORPUS_SOURCES)
383
+ raise ValueError(f"unknown source {source!r}; known: {known}")
384
+ import_path = config_dir / IMPORT_CONFIG_FILE
385
+ dataset_path = config_dir / DATASET_CONFIG_FILE
386
+ corpora = _load_yaml(config_dir / CORPORA_CONFIG_FILE)
387
+ _keys(corpora, set(CORPUS_SOURCES), CORPORA_CONFIG_FILE)
388
+ raw = _load_yaml(import_path)
389
+ _keys(
390
+ raw,
391
+ {
392
+ "seed",
393
+ "val_fraction",
394
+ "test_fraction",
395
+ "min_duration_ms",
396
+ "max_duration_ms",
397
+ "min_loudness_dbfs",
398
+ "alignment",
399
+ "char_map",
400
+ "emotion_map",
401
+ "alania",
402
+ "tr_full",
403
+ "wavdir",
404
+ },
405
+ IMPORT_CONFIG_FILE,
406
+ )
407
+ char_map = _mapping(raw, "char_map")
408
+ for key, value in char_map.items():
409
+ if len(key) != 1 or not isinstance(value, str):
410
+ raise ValueError(f"char_map: {key!r} must be one character mapped to a string")
411
+ emotion_map = _mapping(raw, "emotion_map")
412
+ for label, emotion in emotion_map.items():
413
+ if not label or not label.isascii() or label != en_lower(label) or label != label.strip():
414
+ raise ValueError(f"emotion_map: label {label!r} must be ASCII lower-case")
415
+ if emotion not in EMOTIONS:
416
+ raise ValueError(f"emotion_map: {label!r} -> {emotion!r} is not one of {EMOTIONS}")
417
+ val_fraction = _float(raw, "val_fraction", lo=0.0, hi=1.0)
418
+ test_fraction = _float(raw, "test_fraction", lo=0.0, hi=1.0)
419
+ if val_fraction + test_fraction >= 1.0:
420
+ raise ValueError("val_fraction + test_fraction must be < 1")
421
+ min_duration_ms = _int(raw, "min_duration_ms", minimum=1)
422
+ max_duration_ms = _int(raw, "max_duration_ms", minimum=1)
423
+ if max_duration_ms <= min_duration_ms:
424
+ raise ValueError("max_duration_ms must exceed min_duration_ms")
425
+ align_raw = _mapping(raw, "alignment")
426
+ _keys(align_raw, {"min_chars_per_s", "max_chars_per_s", "min_align_score"}, "alignment")
427
+ alignment = AsrFilterConfig(
428
+ max_cer=0.0, # S4 only; not used by the importer
429
+ allowed_languages=("tr",), # S4 only; not used by the importer
430
+ min_chars_per_s=_float(align_raw, "min_chars_per_s", lo=0.0, hi=1000.0),
431
+ max_chars_per_s=_float(align_raw, "max_chars_per_s", lo=0.0, hi=1000.0),
432
+ min_align_score=_float(align_raw, "min_align_score", lo=0.0, hi=1.0),
433
+ )
434
+ digest = hashlib.sha1()
435
+ for path in (import_path, dataset_path):
436
+ digest.update(path.name.encode("utf-8") + b"\x00" + path.read_bytes())
437
+ refs = {
438
+ key: [DATA_MODELS[key].ref.model_id, DATA_MODELS[key].ref.revision]
439
+ for key in ("voxcpm2_audiovae_v2", "mpoyraz_ctc_tr")
440
+ }
441
+ digest.update(json.dumps(refs, sort_keys=True).encode("utf-8"))
442
+ issai_tsc = IssaiTscConfig.from_mapping(_mapping(corpora, "issai_tsc"))
443
+ mediaspeech_tr = MediaSpeechConfig.from_mapping(_mapping(corpora, "mediaspeech_tr"))
444
+ common_voice_tr = CommonVoiceConfig.from_mapping(_mapping(corpora, "common_voice_tr"))
445
+ digest_hex = digest.hexdigest()
446
+ sections: dict[str, IssaiTscConfig | MediaSpeechConfig | CommonVoiceConfig] = {
447
+ "issai_tsc": issai_tsc,
448
+ "mediaspeech_tr": mediaspeech_tr,
449
+ "common_voice_tr": common_voice_tr,
450
+ }
451
+ if source is not None and source in sections:
452
+ scoped = hashlib.sha1(digest_hex.encode("ascii") + b"\x00" + source.encode("ascii"))
453
+ section_json = json.dumps(asdict(sections[source]), sort_keys=True)
454
+ scoped.update(b"\x00" + section_json.encode("utf-8"))
455
+ digest_hex = scoped.hexdigest()
456
+ return ImportConfig(
457
+ seed=_int(raw, "seed", minimum=0),
458
+ val_fraction=val_fraction,
459
+ test_fraction=test_fraction,
460
+ min_duration_ms=min_duration_ms,
461
+ max_duration_ms=max_duration_ms,
462
+ min_loudness_dbfs=_float(raw, "min_loudness_dbfs", lo=-240.0, hi=0.0),
463
+ alignment=alignment,
464
+ char_map={str(k): str(v) for k, v in char_map.items()},
465
+ emotion_map={str(k): str(v) for k, v in emotion_map.items()},
466
+ alania=AlaniaConfig.from_mapping(_mapping(raw, "alania")),
467
+ tr_full=TrFullConfig.from_mapping(_mapping(raw, "tr_full")),
468
+ wavdir=WavdirConfig.from_mapping(_mapping(raw, "wavdir")),
469
+ issai_tsc=issai_tsc,
470
+ mediaspeech_tr=mediaspeech_tr,
471
+ common_voice_tr=common_voice_tr,
472
+ dataset=DatasetConfig.from_mapping(_load_yaml(dataset_path)),
473
+ digest=digest_hex[:12],
474
+ )
src/nida/data/importer/corpora.py ADDED
@@ -0,0 +1,801 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Source adapters for the corpora that ship as an archive or as a Common Voice release.
2
+
3
+ * :class:`IssaiTscAdapter` - ``issai/Turkish_Speech_Corpus`` (HF; one 21 GB ``.tar.gz``).
4
+ * :class:`MediaSpeechAdapter` - OpenSLR SLR108 ``TR.tgz`` (pinned by sha256).
5
+ * :class:`CommonVoiceAdapter` - a Common Voice release directory or tarball the user supplied.
6
+
7
+ The first two are *archive corpora*. Transcript and audio of an utterance are far apart in a
8
+ tar stream, so an *index pass* (:func:`nida.data.importer.archives.scan_archive`) reads all
9
+ transcripts once and notes which audio members exist; the rows are put in a deterministic
10
+ order and cut into work units. A unit then costs one *extract pass* that writes the audio of
11
+ its selected rows to ``<cache>/raw/<source>/`` (the audio stage deletes each file after
12
+ converting it, :attr:`ephemeral_raw`; a lost file is extracted again).
13
+
14
+ Every row the adapter cannot turn into an item is quarantined with a reason: ``audio_missing``
15
+ (no audio member for a transcript), ``text_missing`` (no transcript for an audio member),
16
+ ``bad_encoding`` (transcript is not UTF-8), the text reasons of
17
+ :func:`~nida.data.importer.text.normalise_transcript`, and per source the ones listed on the
18
+ class.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import csv
24
+ import hashlib
25
+ import json
26
+ import re
27
+ from collections.abc import Generator, Mapping, Set
28
+ from dataclasses import dataclass
29
+ from pathlib import Path
30
+
31
+ from nida.bench import fetch
32
+ from nida.bench.clips import MediaSpeechItem, select_mediaspeech
33
+ from nida.bench.prepare import B1Plan
34
+ from nida.data.importer.archives import (
35
+ ArchiveError,
36
+ ArchiveScan,
37
+ Kind,
38
+ scan_archive,
39
+ verify_sha256_once,
40
+ )
41
+ from nida.data.importer.audio import require_audio_format
42
+ from nida.data.importer.config import (
43
+ CommonVoiceConfig,
44
+ ImportConfig,
45
+ IssaiTscConfig,
46
+ MediaSpeechConfig,
47
+ )
48
+ from nida.data.importer.hub import HubClient, HubError
49
+ from nida.data.importer.sources import (
50
+ Item,
51
+ Scalar,
52
+ UnitListing,
53
+ _bad_row,
54
+ _chunk_ids,
55
+ _chunk_slice,
56
+ _drop,
57
+ _first_seen,
58
+ _text_drop,
59
+ item_segment_id,
60
+ safe_relative_path,
61
+ )
62
+ from nida.data.importer.text import TextRejected, normalise_transcript
63
+ from nida.data.types import QuarantineRecord
64
+ from nida.text.turkish import nfc
65
+ from nida.utils.io import atomic_write_text
66
+
67
+ __all__ = [
68
+ "COMMON_VOICE_SOURCE_ID",
69
+ "ISSAI_TSC_SOURCE_ID",
70
+ "MEDIASPEECH_SOURCE_ID",
71
+ "CommonVoiceAdapter",
72
+ "IssaiTscAdapter",
73
+ "MediaSpeechAdapter",
74
+ "find_common_voice_root",
75
+ ]
76
+
77
+ ISSAI_TSC_SOURCE_ID = "issai_tsc"
78
+ MEDIASPEECH_SOURCE_ID = "mediaspeech_tr"
79
+ COMMON_VOICE_SOURCE_ID = "common_voice_tr"
80
+
81
+ STATE_OK = "ok"
82
+ STATE_NO_AUDIO = "no_audio"
83
+ STATE_NO_TEXT = "no_text"
84
+ STATE_BAD_ENCODING = "bad_encoding"
85
+ STATE_BENCH_RESERVED = "bench_reserved"
86
+ _STATES = (STATE_OK, STATE_NO_AUDIO, STATE_NO_TEXT, STATE_BAD_ENCODING, STATE_BENCH_RESERVED)
87
+ #: FLAC: ``fLaC`` + metadata block header (4 bytes) + STREAMINFO; its bytes 10-17 hold the
88
+ #: sample rate (20 bits), channels, bit depth and the total sample count (36 bits).
89
+ _FLAC_HEAD_BYTES = 26
90
+ _FLAC_STREAMINFO_AT = 18
91
+
92
+ _ISSAI_MEMBER_RE = re.compile(r"([^/]+)/([^/]+)/([0-9]+)\.(wav|txt)")
93
+ _MEDIASPEECH_MEMBER_RE = re.compile(r"([^/]+)/([A-Za-z0-9_-]+)\.(flac|txt)")
94
+
95
+
96
+ # ------------------------------------------------------------------------------ rows
97
+
98
+
99
+ @dataclass(frozen=True)
100
+ class _ArchiveRow:
101
+ """One utterance of an archive corpus.
102
+
103
+ Attributes:
104
+ stem: Name of the utterance without extension (``Train/123``, a UUID).
105
+ text: Decoded transcript (``None`` unless ``state`` is ``ok``).
106
+ state: ``ok``, ``no_audio``, ``no_text``, ``bad_encoding`` or ``bench_reserved``.
107
+ """
108
+
109
+ stem: str
110
+ text: str | None
111
+ state: str
112
+
113
+
114
+ def _pair_rows(
115
+ texts: Mapping[str, bytes], audio_keys: Set[str], audio_ext: str, text_ext: str
116
+ ) -> list[_ArchiveRow]:
117
+ """Pair transcripts with audio members by stem (every stem of either kind is a row)."""
118
+ stems = {k[: -len(audio_ext)] for k in audio_keys} | {k[: -len(text_ext)] for k in texts}
119
+ rows: list[_ArchiveRow] = []
120
+ for stem in stems:
121
+ raw = texts.get(stem + text_ext)
122
+ if stem + audio_ext not in audio_keys:
123
+ rows.append(_ArchiveRow(stem, None, STATE_NO_AUDIO))
124
+ elif raw is None:
125
+ rows.append(_ArchiveRow(stem, None, STATE_NO_TEXT))
126
+ else:
127
+ try:
128
+ rows.append(_ArchiveRow(stem, raw.decode("utf-8"), STATE_OK))
129
+ except UnicodeDecodeError:
130
+ rows.append(_ArchiveRow(stem, None, STATE_BAD_ENCODING))
131
+ return rows
132
+
133
+
134
+ def _order_rows(rows: list[_ArchiveRow]) -> list[_ArchiveRow]:
135
+ """Deterministic order that does not depend on the order inside the archive: by the sha1
136
+ of the stem, so a prefix of the corpus (``--max-hours``) is a spread over all of it rather
137
+ than its first recordings."""
138
+ return sorted(rows, key=lambda r: (hashlib.sha1(r.stem.encode("utf-8")).hexdigest(), r.stem))
139
+
140
+
141
+ def _bad_state_record(sid: str, item_id: str, row: _ArchiveRow) -> QuarantineRecord | None:
142
+ if row.state == STATE_NO_AUDIO:
143
+ return _drop(sid, item_id, "audio_missing", where="archive")
144
+ if row.state == STATE_NO_TEXT:
145
+ return _drop(sid, item_id, "text_missing")
146
+ if row.state == STATE_BAD_ENCODING:
147
+ return _drop(sid, item_id, "bad_encoding")
148
+ if row.state == STATE_BENCH_RESERVED:
149
+ return _drop(sid, item_id, "bench_reserved")
150
+ return None
151
+
152
+
153
+ class _ArchiveCorpusAdapter:
154
+ """Shared machinery of the archive corpora (see the module docstring).
155
+
156
+ Subclasses provide the archive, the classifier, the rows and the item mapping.
157
+ """
158
+
159
+ ephemeral_raw = True
160
+ source_id = ""
161
+ audio_ext = ""
162
+ text_ext = ""
163
+
164
+ def __init__(self, cfg: ImportConfig, cache: Path) -> None:
165
+ self._cfg = cfg
166
+ self._cache = cache
167
+ self._rows_cache: list[_ArchiveRow] | None = None
168
+
169
+ # -- provided by the subclass
170
+ def _archive(self) -> Path:
171
+ raise NotImplementedError
172
+
173
+ def _classify(self, name: str) -> tuple[str, Kind] | None:
174
+ raise NotImplementedError
175
+
176
+ def _load_rows(self) -> list[_ArchiveRow]:
177
+ raise NotImplementedError
178
+
179
+ def _unit_rows(self) -> int:
180
+ raise NotImplementedError
181
+
182
+ def _expected_pairs(self) -> int:
183
+ raise NotImplementedError
184
+
185
+ def _group(self, row: _ArchiveRow) -> str:
186
+ raise NotImplementedError
187
+
188
+ def _extra(self, row: _ArchiveRow) -> dict[str, Scalar]:
189
+ return {}
190
+
191
+ # -- shared
192
+ @property
193
+ def source_ids(self) -> frozenset[str]:
194
+ return frozenset({self.source_id})
195
+
196
+ def _rows(self) -> list[_ArchiveRow]:
197
+ if self._rows_cache is None:
198
+ rows = self._load_rows()
199
+ if len(rows) != self._expected_pairs():
200
+ raise ArchiveError(
201
+ f"{self._archive()}: {len(rows)} utterances, expected "
202
+ f"{self._expected_pairs()} (configured): a truncated, changed or wrong "
203
+ "archive"
204
+ )
205
+ self._rows_cache = _order_rows(rows)
206
+ return self._rows_cache
207
+
208
+ def _item_id(self, row: _ArchiveRow) -> str:
209
+ return row.stem + self.audio_ext
210
+
211
+ def raw_path(self, item: Item) -> Path:
212
+ return self._cache / "raw" / self.source_id / item.item_id
213
+
214
+ def _units(self) -> tuple[str, ...]:
215
+ return _chunk_ids(len(self._rows()), self._unit_rows())
216
+
217
+ def iter_unit(
218
+ self,
219
+ unit_id: str,
220
+ only: Set[str] | None = None,
221
+ exclude: Mapping[str, str] | None = None,
222
+ ) -> Generator[Item | QuarantineRecord, None, None]:
223
+ window = _chunk_slice(unit_id, self._unit_rows())
224
+ first = dict(exclude or {})
225
+ candidates: list[Item] = []
226
+ for row in self._rows()[window]:
227
+ made = self._row(unit_id, row, only, first)
228
+ if isinstance(made, QuarantineRecord):
229
+ yield made
230
+ elif made is not None:
231
+ candidates.append(made)
232
+ # The key of an audio member is its item id (see the classifiers).
233
+ wanted = {c.item_id: self.raw_path(c) for c in candidates if not self.raw_path(c).is_file()}
234
+ yielded = 0
235
+ try:
236
+ if wanted:
237
+ scan_archive(self._archive(), self._classify, read_text=False, extract=wanted)
238
+ for number, item in enumerate(candidates):
239
+ yielded = number + 1
240
+ if self.raw_path(item).is_file():
241
+ yield item
242
+ else:
243
+ yield _drop(item.segment_id, item.item_id, "audio_missing")
244
+ finally:
245
+ # Raw files of rows nobody took (budget reached, or an error): nothing else would
246
+ # delete them.
247
+ for item in candidates[yielded:]:
248
+ self.raw_path(item).unlink(missing_ok=True)
249
+
250
+ def _row(
251
+ self,
252
+ unit_id: str,
253
+ row: _ArchiveRow,
254
+ only: Set[str] | None,
255
+ first: dict[str, str],
256
+ ) -> Item | QuarantineRecord | None:
257
+ item_id = self._item_id(row)
258
+ sid = item_segment_id(self.source_id, item_id)
259
+ if only is not None and sid not in only:
260
+ return None
261
+ bad = _bad_state_record(sid, item_id, row)
262
+ if bad is not None:
263
+ return bad
264
+ assert row.text is not None
265
+ try:
266
+ text = normalise_transcript(row.text, self._cfg.char_map)
267
+ except TextRejected as err:
268
+ return _text_drop(sid, item_id, err)
269
+ duplicate = _first_seen(first, sid, item_id, unit_id)
270
+ if duplicate is not None:
271
+ return duplicate
272
+ return Item(
273
+ source_id=self.source_id,
274
+ item_id=item_id,
275
+ segment_id=sid,
276
+ unit_id=unit_id,
277
+ text=text,
278
+ group=nfc(self._group(row)),
279
+ emotion=None,
280
+ extra=self._extra(row),
281
+ )
282
+
283
+
284
+ # ------------------------------------------------------------------------------ ISSAI TSC
285
+
286
+
287
+ class IssaiTscAdapter(_ArchiveCorpusAdapter):
288
+ """``issai/Turkish_Speech_Corpus``: ``ISSAI_TSC_218.tar.gz`` with
289
+ ``ISSAI_TSC_218/{Train,Dev,Test}/<number>.wav|.txt`` (layout and numbers: the comments of
290
+ ``configs/data/import_corpora.yaml``).
291
+
292
+ The archive is downloaded once to ``<cache>/hub/issai_tsc/`` (pinned by the Hub commit,
293
+ which fixes the LFS object), indexed once (the index is kept next to it), and read again
294
+ for every work unit. Rows are ordered by the sha1 of ``<folder>/<number>``; the group is
295
+ ``issai_tsc.group_prefix + number // group_block_ids`` because the corpus names no speaker.
296
+ ``extra`` keeps the corpus folder (``corpus_split``) and the number.
297
+
298
+ Reasons of its own: none beyond the shared ones.
299
+ """
300
+
301
+ source_id = ISSAI_TSC_SOURCE_ID
302
+ audio_ext = ".wav"
303
+ text_ext = ".txt"
304
+
305
+ def __init__(
306
+ self, cfg: ImportConfig, cache: Path, hub: HubClient, revision: str | None
307
+ ) -> None:
308
+ super().__init__(cfg, cache)
309
+ self._hub = hub
310
+ self.revision = revision
311
+ self._archive_path: Path | None = None
312
+
313
+ @property
314
+ def _c(self) -> IssaiTscConfig:
315
+ return self._cfg.issai_tsc
316
+
317
+ def _revision(self) -> str:
318
+ if self.revision is None:
319
+ raise HubError("issai_tsc: the revision is not pinned; call list_units() first")
320
+ return self.revision
321
+
322
+ def _archive(self) -> Path:
323
+ if self._archive_path is None or not self._archive_path.is_file():
324
+ path = self._hub.download(
325
+ self._c.repo_id,
326
+ self._revision(),
327
+ self._c.archive_file,
328
+ self._cache / "hub" / "issai_tsc",
329
+ )
330
+ verify_sha256_once(path, self._c.archive_sha256) # raises on a partial download
331
+ self._archive_path = path
332
+ return self._archive_path
333
+
334
+ def _classify(self, name: str) -> tuple[str, Kind] | None:
335
+ match = _ISSAI_MEMBER_RE.fullmatch(name)
336
+ if match is None or match.group(1) != self._c.archive_root:
337
+ return None
338
+ _, split, number, ext = match.groups()
339
+ if split not in self._c.splits:
340
+ return None
341
+ return f"{split}/{number}.{ext}", ("audio" if ext == "wav" else "text")
342
+
343
+ def _index_path(self) -> Path:
344
+ """The index depends on the revision and on how the archive is read (root, folders)."""
345
+ layout = json.dumps(
346
+ [self._c.archive_file, self._c.archive_sha256, self._c.archive_root, self._c.splits]
347
+ )
348
+ key = hashlib.sha1(layout.encode("utf-8")).hexdigest()[:8]
349
+ name = f"index-{self._revision()[:12]}-{key}.jsonl"
350
+ return self._cache / "hub" / "issai_tsc" / name
351
+
352
+ def _load_rows(self) -> list[_ArchiveRow]:
353
+ index = self._index_path()
354
+ if index.is_file():
355
+ return _read_index(index)
356
+ scan = scan_archive(self._archive(), self._classify, read_text=True)
357
+ rows = _pair_rows(scan.texts, scan.audio_keys, self.audio_ext, self.text_ext)
358
+ _write_index(index, rows)
359
+ return rows
360
+
361
+ def _unit_rows(self) -> int:
362
+ return self._c.unit_rows
363
+
364
+ def _expected_pairs(self) -> int:
365
+ return self._c.expected_pairs
366
+
367
+ def _group(self, row: _ArchiveRow) -> str:
368
+ number = int(row.stem.split("/", 1)[1])
369
+ return f"{self._c.group_prefix}{number // self._c.group_block_ids}"
370
+
371
+ def _extra(self, row: _ArchiveRow) -> dict[str, Scalar]:
372
+ split, number = row.stem.split("/", 1)
373
+ return {"corpus_split": split, "utterance_number": int(number)}
374
+
375
+ def list_units(self) -> UnitListing:
376
+ repo = self._c.repo_id
377
+ if self.revision is None:
378
+ self.revision = self._hub.resolve_revision(repo)
379
+ files = self._hub.list_files(repo, self.revision)
380
+ if self._c.archive_file not in files:
381
+ raise HubError(
382
+ f"{repo}@{self.revision}: no file {self._c.archive_file!r}; "
383
+ f"first files: {sorted(files)[:5]}"
384
+ )
385
+ return UnitListing(self.revision, self._units())
386
+
387
+
388
+ def _write_index(path: Path, rows: list[_ArchiveRow]) -> None:
389
+ lines = [
390
+ json.dumps({"stem": r.stem, "text": r.text, "state": r.state}, ensure_ascii=False)
391
+ for r in sorted(rows, key=lambda r: r.stem)
392
+ ]
393
+ atomic_write_text(path, "\n".join(lines) + "\n")
394
+
395
+
396
+ def _read_index(path: Path) -> list[_ArchiveRow]:
397
+ rows: list[_ArchiveRow] = []
398
+ for number, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
399
+ if not line:
400
+ continue
401
+ try:
402
+ obj = json.loads(line)
403
+ except json.JSONDecodeError as err:
404
+ raise ValueError(f"{path}:{number}: invalid JSON ({err.msg}); delete the file") from err
405
+ ok = (
406
+ isinstance(obj, dict)
407
+ and set(obj) == {"stem", "text", "state"}
408
+ and isinstance(obj["stem"], str)
409
+ and obj["state"] in _STATES
410
+ and (obj["text"] is None or isinstance(obj["text"], str))
411
+ )
412
+ if not ok:
413
+ raise ValueError(f"{path}:{number}: not an index row; delete the file")
414
+ rows.append(_ArchiveRow(obj["stem"], obj["text"], obj["state"]))
415
+ return rows
416
+
417
+
418
+ # ------------------------------------------------------------------------------ MediaSpeech
419
+
420
+
421
+ class MediaSpeechAdapter(_ArchiveCorpusAdapter):
422
+ """OpenSLR SLR108 MediaSpeech Turkish: ``TR.tgz`` with ``TR/<uuid>.flac`` + ``<uuid>.txt``.
423
+
424
+ The archive is downloaded from the primary OpenSLR URL to ``<cache>/hub/mediaspeech_tr/``
425
+ (resumable) and must have the sha256 of the pin: ``revision`` (``--revision``, or the one
426
+ recorded in ``plan.json``) or ``mediaspeech_tr.sha256``. A different file is an error and
427
+ is not used. The corpus is one group (``mediaspeech_tr.group``).
428
+ """
429
+
430
+ source_id = MEDIASPEECH_SOURCE_ID
431
+ audio_ext = ".flac"
432
+ text_ext = ".txt"
433
+
434
+ def __init__(
435
+ self,
436
+ cfg: ImportConfig,
437
+ cache: Path,
438
+ opener: fetch.Opener | None,
439
+ revision: str | None,
440
+ ) -> None:
441
+ super().__init__(cfg, cache)
442
+ self._opener = opener or fetch.default_opener
443
+ self.revision = revision
444
+ self._download: fetch.Downloaded | None = None
445
+
446
+ @property
447
+ def _c(self) -> MediaSpeechConfig:
448
+ return self._cfg.mediaspeech_tr
449
+
450
+ def _archive(self) -> Path:
451
+ if self._download is None or not self._download.path.is_file():
452
+ self._download = fetch.download(
453
+ self._c.url,
454
+ self._cache / "hub" / "mediaspeech_tr" / "TR.tgz",
455
+ expected_sha256=self.revision or self._c.sha256,
456
+ opener=self._opener,
457
+ )
458
+ return self._download.path
459
+
460
+ def _classify(self, name: str) -> tuple[str, Kind] | None:
461
+ match = _MEDIASPEECH_MEMBER_RE.fullmatch(name)
462
+ if match is None or match.group(1) != self._c.archive_root:
463
+ return None
464
+ _, stem, ext = match.groups()
465
+ return f"{stem}.{ext}", ("audio" if ext == "flac" else "text")
466
+
467
+ def _load_rows(self) -> list[_ArchiveRow]:
468
+ head = _FLAC_HEAD_BYTES if self._c.reserve_bench_clips else 0
469
+ scan = scan_archive(self._archive(), self._classify, read_text=True, audio_head=head)
470
+ rows = _pair_rows(scan.texts, scan.audio_keys, self.audio_ext, self.text_ext)
471
+ if not self._c.reserve_bench_clips:
472
+ return rows
473
+ reserved = _bench_reserved_stems(scan)
474
+ return [
475
+ _ArchiveRow(r.stem, None, STATE_BENCH_RESERVED)
476
+ if r.state == STATE_OK and r.stem in reserved
477
+ else r
478
+ for r in rows
479
+ ]
480
+
481
+ def _unit_rows(self) -> int:
482
+ return self._c.unit_rows
483
+
484
+ def _expected_pairs(self) -> int:
485
+ return self._c.expected_pairs
486
+
487
+ def _group(self, row: _ArchiveRow) -> str:
488
+ return self._c.group
489
+
490
+ def list_units(self) -> UnitListing:
491
+ units = self._units() # downloads and verifies the archive
492
+ assert self._download is not None
493
+ self.revision = self._download.sha256
494
+ return UnitListing(self.revision, units)
495
+
496
+
497
+ def flac_duration_ms(head: bytes) -> int:
498
+ """Duration of a FLAC file from its first 26 bytes (the STREAMINFO block).
499
+
500
+ The rounding is the benchmark's (``round(1000 * frames / sample_rate)``).
501
+
502
+ Raises:
503
+ ArchiveError: not FLAC, no STREAMINFO first, or a zero sample rate.
504
+ """
505
+ if len(head) < _FLAC_HEAD_BYTES or head[:4] != b"fLaC" or head[4] & 0x7F != 0:
506
+ raise ArchiveError(f"not a FLAC header with STREAMINFO first: {head[:8]!r}")
507
+ packed = int.from_bytes(head[_FLAC_STREAMINFO_AT : _FLAC_STREAMINFO_AT + 8], "big")
508
+ rate_hz, frames = packed >> 44, packed & ((1 << 36) - 1)
509
+ if rate_hz == 0:
510
+ raise ArchiveError("FLAC STREAMINFO with sample rate 0")
511
+ return round(1000 * frames / rate_hz)
512
+
513
+
514
+ def _bench_reserved_stems(scan: ArchiveScan) -> set[str]:
515
+ """Stems of the clips the phase-0 B1 benchmark drew from MediaSpeech.
516
+
517
+ The benchmark's selection (:func:`nida.bench.clips.select_mediaspeech` with the default
518
+ :class:`~nida.bench.prepare.B1Plan`) is recomputed from the archive, exactly as
519
+ :mod:`nida.bench.prepare` feeds it: every pair, transcript stripped, FLAC duration.
520
+ """
521
+ plan = B1Plan()
522
+ items: list[MediaSpeechItem] = []
523
+ for key, raw in scan.texts.items():
524
+ stem = key[: -len(".txt")]
525
+ head = scan.audio_heads.get(stem + ".flac")
526
+ if head is None:
527
+ continue
528
+ try:
529
+ text = raw.decode("utf-8").strip()
530
+ except UnicodeDecodeError:
531
+ continue
532
+ items.append(MediaSpeechItem(stem, text, flac_duration_ms(head)))
533
+ try:
534
+ selection = select_mediaspeech(items, plan.n_mediaspeech, plan.seed)
535
+ except ValueError as err:
536
+ raise ArchiveError(f"cannot reproduce the B1 benchmark selection: {err}") from err
537
+ return {clip.file_id for clip in selection.clips}
538
+
539
+
540
+ # ------------------------------------------------------------------------------ Common Voice
541
+
542
+
543
+ def find_common_voice_root(source: Path, cfg: CommonVoiceConfig) -> Path:
544
+ """The folder with ``validated.tsv`` of a Common Voice release directory.
545
+
546
+ Tries ``source`` itself (the language folder), ``source/<locale>`` and
547
+ ``source/<release>/<locale>`` (``source`` is a folder of releases).
548
+
549
+ Raises:
550
+ FileNotFoundError: none of them has the validated file.
551
+ """
552
+ tried = [source, source / cfg.locale]
553
+ if source.is_dir():
554
+ tried += [d / cfg.locale for d in sorted(source.iterdir()) if d.is_dir()]
555
+ for folder in tried:
556
+ if (folder / cfg.validated_file).is_file():
557
+ return folder
558
+ raise FileNotFoundError(
559
+ f"no {cfg.validated_file} in {source} (tried {[str(t) for t in tried[:4]]}...); "
560
+ f"--source-dir is the Common Voice release folder or its {cfg.locale!r} folder, "
561
+ "or the release tarball"
562
+ )
563
+
564
+
565
+ @dataclass(frozen=True)
566
+ class _CvRow:
567
+ """A row of ``validated.tsv`` (only the columns the importer uses).
568
+
569
+ ``problem`` is set (and the other fields are empty) for a row that does not match the
570
+ header; ``raw`` is then a description of its shape for the quarantine record (never its
571
+ text: the client id would end up in the quarantine log).
572
+ """
573
+
574
+ path: str
575
+ sentence: str
576
+ client_id: str
577
+ up_votes: str
578
+ down_votes: str
579
+ locale: str
580
+ problem: str = ""
581
+ raw: str = ""
582
+
583
+
584
+ _CV_REQUIRED = ("client_id", "path", "sentence", "up_votes", "down_votes")
585
+
586
+
587
+ def _read_cv_tsv(path: Path) -> list[_CvRow]:
588
+ """Rows of a Common Voice TSV (tab separated, no quoting).
589
+
590
+ Raises:
591
+ ValueError: the header lacks a required column or repeats a column.
592
+ """
593
+ rows: list[_CvRow] = []
594
+ with path.open(encoding="utf-8", newline="") as handle:
595
+ reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE)
596
+ header = next(reader, None)
597
+ if header is None:
598
+ raise ValueError(f"{path}: empty file (no header)")
599
+ header = [h.strip() for h in header]
600
+ if len(set(header)) != len(header):
601
+ raise ValueError(f"{path}: duplicate columns in the header {header}")
602
+ missing = [c for c in _CV_REQUIRED if c not in header]
603
+ if missing:
604
+ raise ValueError(f"{path}: columns missing: {missing}; header is {header}")
605
+ at = {name: header.index(name) for name in (*_CV_REQUIRED, "locale") if name in header}
606
+ for fields in reader:
607
+ if not fields:
608
+ continue
609
+ if len(fields) != len(header):
610
+ # Only the shape of the row is kept: its text may hold the client id.
611
+ shape = f"{len(fields)} fields, the header has {len(header)}"
612
+ rows.append(_CvRow("", "", "", "", "", "", "columns", shape))
613
+ continue
614
+ rows.append(
615
+ _CvRow(
616
+ path=fields[at["path"]],
617
+ sentence=fields[at["sentence"]],
618
+ client_id=fields[at["client_id"]],
619
+ up_votes=fields[at["up_votes"]],
620
+ down_votes=fields[at["down_votes"]],
621
+ locale=fields[at["locale"]] if "locale" in at else "",
622
+ )
623
+ )
624
+ return rows
625
+
626
+
627
+ class CommonVoiceAdapter:
628
+ """A Common Voice release the user downloaded (``--source-dir``): a folder or a tarball.
629
+
630
+ Rows are the lines of ``validated.tsv``; a clip is used when ``up_votes - down_votes >=
631
+ common_voice_tr.min_net_votes`` and its ``locale`` is empty or ``common_voice_tr.locale``.
632
+ The group is ``group_prefix`` + the first 16 hex digits of sha1(``client_id``): the raw
633
+ client id (a pseudonymous speaker id the MDC terms forbid trying to resolve) never reaches
634
+ a manifest or dataset. Votes are kept in ``extra``; nothing demographic is.
635
+
636
+ From a folder the MP3 clips are read in place and never deleted. From a tarball only
637
+ ``validated.tsv`` is extracted up front (to ``<cache>/raw/common_voice_tr/<id>/``); the
638
+ clips of a work unit are extracted per unit and deleted by the audio stage like the other
639
+ archive corpora.
640
+
641
+ Reasons of its own: ``language`` (another non-empty locale), ``low_votes``, ``no_group``,
642
+ ``bad_row`` (wrong column count, bad path, non-integer votes).
643
+ """
644
+
645
+ source_id = COMMON_VOICE_SOURCE_ID
646
+
647
+ def __init__(self, cfg: ImportConfig, cache: Path, source: Path) -> None:
648
+ if not source.exists():
649
+ raise FileNotFoundError(f"{source} does not exist (--source-dir)")
650
+ require_audio_format("MP3")
651
+ self._cfg = cfg
652
+ self._cache = cache
653
+ self._source = source
654
+ self._tarball = source.is_file()
655
+ self.ephemeral_raw = self._tarball
656
+ self._layout_cache: tuple[Path, Path] | None = None
657
+ self._rows_cache: list[_CvRow] | None = None
658
+
659
+ @property
660
+ def _c(self) -> CommonVoiceConfig:
661
+ return self._cfg.common_voice_tr
662
+
663
+ @property
664
+ def source_ids(self) -> frozenset[str]:
665
+ return frozenset({self.source_id})
666
+
667
+ def _classify(self, name: str) -> tuple[str, Kind] | None:
668
+ parts = name.split("/")
669
+ in_locale = len(parts) < 3 or parts[-3] == self._c.locale
670
+ if parts[-1] == self._c.validated_file and (len(parts) == 1 or parts[-2] == self._c.locale):
671
+ return "validated", "text"
672
+ if len(parts) >= 2 and parts[-2] == self._c.clips_dir and in_locale:
673
+ return f"{self._c.clips_dir}/{parts[-1]}", "audio"
674
+ return name, "ignore"
675
+
676
+ def _layout(self) -> tuple[Path, Path]:
677
+ """``(validated.tsv, clips folder)`` on disk (a tarball is partly extracted)."""
678
+ if self._layout_cache is None:
679
+ if not self._tarball:
680
+ root = find_common_voice_root(self._source, self._c)
681
+ else:
682
+ stat = self._source.stat()
683
+ identity = f"{self._source.name}\x1f{stat.st_size}\x1f{stat.st_mtime_ns}"
684
+ ident = hashlib.sha1(identity.encode()).hexdigest()
685
+ root = self._cache / "raw" / self.source_id / ident[:12]
686
+ tsv = root / self._c.validated_file
687
+ if not tsv.is_file():
688
+ done = scan_archive(
689
+ self._source, self._classify, read_text=False, extract={"validated": tsv}
690
+ )
691
+ if "validated" not in done.extracted:
692
+ raise ArchiveError(
693
+ f"{self._source}: no {self._c.locale}/{self._c.validated_file}"
694
+ )
695
+ self._layout_cache = (root / self._c.validated_file, root / self._c.clips_dir)
696
+ return self._layout_cache
697
+
698
+ def _rows(self) -> list[_CvRow]:
699
+ if self._rows_cache is None:
700
+ self._rows_cache = _read_cv_tsv(self._layout()[0])
701
+ return self._rows_cache
702
+
703
+ def list_units(self) -> UnitListing:
704
+ tsv = self._layout()[0]
705
+ digest = hashlib.sha1(tsv.read_bytes()).hexdigest()
706
+ rows = self._rows()
707
+ if not rows:
708
+ raise ValueError(f"{tsv}: no row")
709
+ return UnitListing(None, _chunk_ids(len(rows), self._c.unit_rows), digest)
710
+
711
+ def raw_path(self, item: Item) -> Path:
712
+ return self._clip(item.item_id)
713
+
714
+ def iter_unit(
715
+ self,
716
+ unit_id: str,
717
+ only: Set[str] | None = None,
718
+ exclude: Mapping[str, str] | None = None,
719
+ ) -> Generator[Item | QuarantineRecord, None, None]:
720
+ window = _chunk_slice(unit_id, self._c.unit_rows)
721
+ first = dict(exclude or {})
722
+ candidates: list[Item] = []
723
+ for offset, row in enumerate(self._rows()[window]):
724
+ made = self._row(unit_id, window.start + offset, row, only, first)
725
+ if isinstance(made, QuarantineRecord):
726
+ yield made
727
+ elif made is not None:
728
+ candidates.append(made)
729
+ yielded = 0
730
+ try:
731
+ if self._tarball:
732
+ wanted = {
733
+ f"{self._c.clips_dir}/{c.item_id}": self.raw_path(c)
734
+ for c in candidates
735
+ if not self.raw_path(c).is_file()
736
+ }
737
+ if wanted:
738
+ scan_archive(self._source, self._classify, read_text=False, extract=wanted)
739
+ for number, item in enumerate(candidates):
740
+ yielded = number + 1
741
+ if self.raw_path(item).is_file():
742
+ yield item
743
+ else:
744
+ yield _drop(item.segment_id, item.item_id, "audio_missing")
745
+ finally:
746
+ if self._tarball: # a cache copy; the user's own clips are never touched
747
+ for item in candidates[yielded:]:
748
+ self.raw_path(item).unlink(missing_ok=True)
749
+
750
+ def _row(
751
+ self,
752
+ unit_id: str,
753
+ position: int,
754
+ row: _CvRow,
755
+ only: Set[str] | None,
756
+ first: dict[str, str],
757
+ ) -> Item | QuarantineRecord | None:
758
+ cfg = self._cfg
759
+ if row.problem:
760
+ return _bad_row(unit_id, position, row.raw, "bad_row", problem=row.problem)
761
+ rel = safe_relative_path(row.path)
762
+ if rel is None or "\x1f" in rel:
763
+ return _bad_row(unit_id, position, row.path, "bad_row", problem="path")
764
+ sid = item_segment_id(self.source_id, rel)
765
+ if only is not None and sid not in only:
766
+ return None
767
+ locale = row.locale.strip()
768
+ if locale and locale != self._c.locale:
769
+ return _drop(sid, rel, "language", locale=locale)
770
+ try:
771
+ up, down = int(row.up_votes), int(row.down_votes)
772
+ except ValueError:
773
+ return _drop(sid, rel, "bad_row", problem="votes", up_votes=row.up_votes[:20])
774
+ if up - down < self._c.min_net_votes:
775
+ return _drop(sid, rel, "low_votes", up_votes=up, down_votes=down)
776
+ client = nfc(row.client_id).strip()
777
+ if not client:
778
+ return _drop(sid, rel, "no_group")
779
+ try:
780
+ text = normalise_transcript(row.sentence, cfg.char_map)
781
+ except TextRejected as err:
782
+ return _text_drop(sid, rel, err)
783
+ if not self._tarball and not self._clip(rel).is_file():
784
+ return _drop(sid, rel, "audio_missing")
785
+ duplicate = _first_seen(first, sid, rel, unit_id)
786
+ if duplicate is not None:
787
+ return duplicate
788
+ who = hashlib.sha1(f"common-voice-client\x1f{client}".encode()).hexdigest()[:16]
789
+ return Item(
790
+ source_id=self.source_id,
791
+ item_id=rel,
792
+ segment_id=sid,
793
+ unit_id=unit_id,
794
+ text=text,
795
+ group=nfc(self._c.group_prefix + who),
796
+ emotion=None,
797
+ extra={"up_votes": up, "down_votes": down},
798
+ )
799
+
800
+ def _clip(self, item_id: str) -> Path:
801
+ return self._layout()[1] / item_id
src/nida/data/importer/hub.py ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hugging Face Hub access of the import adapters, behind a small injectable protocol.
2
+
3
+ The production implementation uses ``huggingface_hub`` (imported lazily; Colab ships it, it is
4
+ not a declared dependency of Nida). Tests inject a fake that serves local files. Every
5
+ download is pinned to a full commit sha: :meth:`HubClient.resolve_revision` turns the current
6
+ head of a dataset into one, once, and the sha is recorded in ``plan.json``.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import importlib
12
+ import re
13
+ from collections.abc import Sequence
14
+ from concurrent.futures import ThreadPoolExecutor
15
+ from pathlib import Path
16
+ from typing import Any, Protocol
17
+
18
+ __all__ = ["HubClient", "HubError", "HuggingFaceHub", "require_commit_sha", "require_sha256"]
19
+
20
+ _SHA_RE = re.compile(r"[0-9a-f]{40}")
21
+ _SHA256_RE = re.compile(r"[0-9a-f]{64}")
22
+ #: Parallel file downloads of :meth:`HuggingFaceHub.download_many`.
23
+ DOWNLOAD_THREADS = 8
24
+
25
+
26
+ class HubError(RuntimeError):
27
+ """The Hub answered with something the importer cannot use."""
28
+
29
+
30
+ def require_commit_sha(revision: str) -> str:
31
+ """Return ``revision`` if it is a full 40-character lowercase hex commit sha.
32
+
33
+ Raises:
34
+ ValueError: otherwise (branch and tag names are not pins).
35
+ """
36
+ if _SHA_RE.fullmatch(revision) is None:
37
+ raise ValueError(f"revision must be a 40-character lowercase hex commit sha: {revision!r}")
38
+ return revision
39
+
40
+
41
+ def require_sha256(value: str) -> str:
42
+ """Return ``value`` if it is a 64-character lowercase hex SHA-256 digest (the pin of an
43
+ archive that is not a Hub commit).
44
+
45
+ Raises:
46
+ ValueError: otherwise.
47
+ """
48
+ if _SHA256_RE.fullmatch(value) is None:
49
+ raise ValueError(f"expected a 64-character lowercase hex sha256 digest: {value!r}")
50
+ return value
51
+
52
+
53
+ class HubClient(Protocol):
54
+ """What the adapters need from the Hub (dataset repositories only)."""
55
+
56
+ def resolve_revision(self, repo_id: str) -> str:
57
+ """Commit sha of the current head of ``repo_id``."""
58
+ ...
59
+
60
+ def list_files(self, repo_id: str, revision: str) -> list[str]:
61
+ """All file paths of the repository at ``revision`` (unsorted is fine)."""
62
+ ...
63
+
64
+ def download(self, repo_id: str, revision: str, filename: str, local_dir: Path) -> Path:
65
+ """Download one file to ``local_dir / filename``; returns that path."""
66
+ ...
67
+
68
+ def download_many(
69
+ self, repo_id: str, revision: str, filenames: Sequence[str], local_dir: Path
70
+ ) -> None:
71
+ """Download several files to ``local_dir / <name>``; files that do not exist in the
72
+ repository are skipped silently (the caller checks which ones arrived)."""
73
+ ...
74
+
75
+
76
+ class HuggingFaceHub:
77
+ """:class:`HubClient` over ``huggingface_hub`` (public datasets, no token)."""
78
+
79
+ @staticmethod
80
+ def _module() -> Any: # Any: huggingface_hub is imported lazily and ships no stubs here
81
+ try:
82
+ return importlib.import_module("huggingface_hub")
83
+ except ImportError as err:
84
+ raise HubError(
85
+ "huggingface_hub is required for Hub sources (pip install huggingface_hub)"
86
+ ) from err
87
+
88
+ def resolve_revision(self, repo_id: str) -> str:
89
+ info = self._module().HfApi().dataset_info(repo_id)
90
+ sha = getattr(info, "sha", None)
91
+ if not isinstance(sha, str):
92
+ raise HubError(f"{repo_id}: the Hub returned no commit sha")
93
+ return require_commit_sha(sha)
94
+
95
+ def list_files(self, repo_id: str, revision: str) -> list[str]:
96
+ files = (
97
+ self._module()
98
+ .HfApi()
99
+ .list_repo_files(repo_id, repo_type="dataset", revision=require_commit_sha(revision))
100
+ )
101
+ return [str(name) for name in files]
102
+
103
+ def download(self, repo_id: str, revision: str, filename: str, local_dir: Path) -> Path:
104
+ path = self._module().hf_hub_download(
105
+ repo_id=repo_id,
106
+ filename=filename,
107
+ repo_type="dataset",
108
+ revision=require_commit_sha(revision),
109
+ local_dir=str(local_dir),
110
+ )
111
+ return Path(str(path))
112
+
113
+ @classmethod
114
+ def _not_found_error(cls) -> type[BaseException]:
115
+ """``EntryNotFoundError`` of the installed ``huggingface_hub`` (its module moved)."""
116
+ for name in ("huggingface_hub.errors", "huggingface_hub.utils"):
117
+ try:
118
+ error = getattr(importlib.import_module(name), "EntryNotFoundError") # noqa: B009
119
+ except (ImportError, AttributeError):
120
+ continue
121
+ if isinstance(error, type) and issubclass(error, BaseException):
122
+ return error
123
+ raise HubError("huggingface_hub has no EntryNotFoundError")
124
+
125
+ def download_many(
126
+ self, repo_id: str, revision: str, filenames: Sequence[str], local_dir: Path
127
+ ) -> None:
128
+ """One ``hf_hub_download`` per file on a small thread pool; a file that is not in
129
+ the repository is skipped, any other error is raised."""
130
+ if not filenames:
131
+ return
132
+ module = self._module()
133
+ pinned = require_commit_sha(revision)
134
+ missing = self._not_found_error()
135
+
136
+ def fetch_one(name: str) -> None:
137
+ try:
138
+ module.hf_hub_download(
139
+ repo_id=repo_id,
140
+ filename=name,
141
+ repo_type="dataset",
142
+ revision=pinned,
143
+ local_dir=str(local_dir),
144
+ )
145
+ except missing:
146
+ return
147
+
148
+ with ThreadPoolExecutor(max_workers=DOWNLOAD_THREADS) as pool:
149
+ for _ in pool.map(fetch_one, filenames):
150
+ pass
src/nida/data/importer/pipeline.py ADDED
@@ -0,0 +1,1326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The importer: stages, plan and resumability (layout: see :mod:`nida.data.importer`)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import io
7
+ import json
8
+ import logging
9
+ import math
10
+ from collections import Counter
11
+ from collections.abc import Callable, Mapping, Sequence
12
+ from concurrent.futures import ThreadPoolExecutor
13
+ from dataclasses import dataclass, field
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ import numpy as np
18
+
19
+ from nida.bench import envs as bench_envs
20
+ from nida.bench import fetch
21
+ from nida.bench.b1 import _stream_runner
22
+ from nida.bench.protocol import WorkRequest, WorkResult, load_results
23
+ from nida.bench.registry import load_sources, require_approved
24
+ from nida.data import envs
25
+ from nida.data.align import align_words
26
+ from nida.data.asr_filter import check_alignment
27
+ from nida.data.ears import bucket_of, rate_chars_per_s
28
+ from nida.data.importer.audio import (
29
+ AudioRejected,
30
+ convert_to_wav16k,
31
+ measure_wav16k,
32
+ raw_duration_ms,
33
+ )
34
+ from nida.data.importer.config import ImportConfig, load_import_config
35
+ from nida.data.importer.hub import (
36
+ HubClient,
37
+ HuggingFaceHub,
38
+ require_commit_sha,
39
+ require_sha256,
40
+ )
41
+ from nida.data.importer.sources import (
42
+ HUB_SOURCES,
43
+ LOCAL_SOURCES,
44
+ SOURCES,
45
+ Item,
46
+ SourceAdapter,
47
+ make_adapter,
48
+ )
49
+ from nida.data.pilot import DATASET_SHARD_FRAMES, FLOAT16_MAX_ABS, run_worker_job
50
+ from nida.data.quarantine import QuarantineLog
51
+ from nida.data.registry import DATA_MODELS
52
+ from nida.data.splits import assign_splits
53
+ from nida.data.timings import SegmentTimings, build_timings, write_timings
54
+ from nida.data.types import QuarantineRecord
55
+ from nida.model.vocab import encode_text
56
+ from nida.train.data import SPLITS, DataFormatError, load_latent_dataset
57
+ from nida.utils.io import atomic_write_bytes, atomic_write_json, atomic_write_text
58
+
59
+ __all__ = ["STAGES", "ImportPlan", "ImportRequest", "Importer", "assign_group_splits"]
60
+
61
+ log = logging.getLogger(__name__)
62
+
63
+ #: Stages in run order; ``all`` runs them one after another.
64
+ STAGES = ("select", "audio", "latents", "ctc", "timings", "dataset", "report")
65
+ VAE_KEY = "voxcpm2_audiovae_v2"
66
+ CTC_KEY = "mpoyraz_ctc_tr"
67
+ #: Language written to every index record (the importer reads Turkish corpora only).
68
+ LANGUAGE = "tr"
69
+ #: Items between two progress lines.
70
+ PROGRESS_EVERY = 1000
71
+ #: Default number of audio conversion threads (decoding and resampling release the GIL).
72
+ DEFAULT_AUDIO_THREADS = 4
73
+ SECONDS_PER_HOUR = 3600.0
74
+
75
+
76
+ @dataclass(frozen=True)
77
+ class ImportRequest:
78
+ """What the caller asks for; part of the plan of a root.
79
+
80
+ Attributes:
81
+ source: One of :data:`nida.data.importer.sources.SOURCES` (``alania``, ``tr_full``,
82
+ ``wavdir``, ``issai_tsc``, ``mediaspeech_tr``, ``common_voice_tr``).
83
+ max_hours: Stop selecting once this many hours of audio are selected.
84
+ unit_offset: Index of the first work unit (parquet shard / row chunk) of this root.
85
+ unit_count: Number of work units from ``unit_offset`` (``None`` = all the rest).
86
+ Roots with disjoint ranges select disjoint data.
87
+ revision: Pin (``None`` = resolve it once and record it): a Hub commit sha, or for
88
+ ``mediaspeech_tr`` the sha256 of the ``TR.tgz`` archive (default: the one in
89
+ ``import_corpora.yaml``). Refused for the local sources (``wavdir``,
90
+ ``common_voice_tr``), which pin the sha1 of their manifest / validated file.
91
+ """
92
+
93
+ source: str
94
+ max_hours: float = 10.0
95
+ unit_offset: int = 0
96
+ unit_count: int | None = None
97
+ revision: str | None = None
98
+
99
+ def __post_init__(self) -> None:
100
+ if self.source not in SOURCES:
101
+ raise ValueError(f"unknown source {self.source!r}; known: {SOURCES}")
102
+ if not (math.isfinite(self.max_hours) and self.max_hours > 0):
103
+ raise ValueError(f"max_hours must be a positive finite number, got {self.max_hours!r}")
104
+ if isinstance(self.unit_offset, bool) or self.unit_offset < 0:
105
+ raise ValueError(f"unit_offset must be an integer >= 0, got {self.unit_offset!r}")
106
+ if self.unit_count is not None and (
107
+ isinstance(self.unit_count, bool) or self.unit_count < 1
108
+ ):
109
+ raise ValueError(f"unit_count must be an integer >= 1, got {self.unit_count!r}")
110
+ if self.revision is not None and self.source in LOCAL_SOURCES:
111
+ raise ValueError(
112
+ f"--revision is meaningless for source {self.source!r} (a local folder has no "
113
+ "revision; the plan pins the sha1 of its manifest / validated file)"
114
+ )
115
+ if self.revision is not None:
116
+ if self.source == "mediaspeech_tr":
117
+ require_sha256(self.revision)
118
+ else:
119
+ require_commit_sha(self.revision)
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class ImportPlan:
124
+ """The resolved request of a root (``plan.json``): fixes which units are processed.
125
+
126
+ A root is bound to its plan; asking for something else raises (use a new root).
127
+ """
128
+
129
+ source: str
130
+ revision: str | None
131
+ max_hours: float
132
+ unit_offset: int
133
+ unit_count: int | None
134
+ unit_ids: tuple[str, ...]
135
+ config_digest: str
136
+ content_sha1: str | None
137
+
138
+ def to_obj(self) -> dict[str, object]:
139
+ return {
140
+ "source": self.source,
141
+ "revision": self.revision,
142
+ "max_hours": self.max_hours,
143
+ "unit_offset": self.unit_offset,
144
+ "unit_count": self.unit_count,
145
+ "unit_ids": list(self.unit_ids),
146
+ "config_digest": self.config_digest,
147
+ "content_sha1": self.content_sha1,
148
+ }
149
+
150
+ @classmethod
151
+ def from_obj(cls, obj: object, where: Path) -> ImportPlan:
152
+ keys = {
153
+ "source",
154
+ "revision",
155
+ "max_hours",
156
+ "unit_offset",
157
+ "unit_count",
158
+ "unit_ids",
159
+ "config_digest",
160
+ "content_sha1",
161
+ }
162
+ if not isinstance(obj, dict) or set(obj) != keys or not isinstance(obj["unit_ids"], list):
163
+ raise ValueError(f"{where}: not a plan.json written by the importer")
164
+ return cls(
165
+ source=str(obj["source"]),
166
+ revision=None if obj["revision"] is None else str(obj["revision"]),
167
+ max_hours=float(obj["max_hours"]),
168
+ unit_offset=int(obj["unit_offset"]),
169
+ unit_count=None if obj["unit_count"] is None else int(obj["unit_count"]),
170
+ unit_ids=tuple(str(u) for u in obj["unit_ids"]),
171
+ config_digest=str(obj["config_digest"]),
172
+ content_sha1=None if obj["content_sha1"] is None else str(obj["content_sha1"]),
173
+ )
174
+
175
+ def check(self, request: ImportRequest, config_digest: str, content_sha1: str | None) -> None:
176
+ """Raise unless this plan is what ``request`` (and the current config/input) mean."""
177
+ wanted: dict[str, object] = {
178
+ "source": request.source,
179
+ "max_hours": request.max_hours,
180
+ "unit_offset": request.unit_offset,
181
+ "unit_count": request.unit_count,
182
+ "config_digest": config_digest,
183
+ "content_sha1": content_sha1,
184
+ }
185
+ actual = self.to_obj()
186
+ if request.revision is not None:
187
+ wanted["revision"] = request.revision
188
+ differing = {k: (actual[k], v) for k, v in wanted.items() if actual[k] != v}
189
+ if differing:
190
+ raise ValueError(
191
+ "this root was made for another plan (recorded, requested): "
192
+ f"{differing}; use a new --root"
193
+ )
194
+
195
+
196
+ @dataclass(frozen=True)
197
+ class _UnitResult:
198
+ """Outcome of one work unit, stored as ``select/units/<digest>.json``."""
199
+
200
+ unit_id: str
201
+ truncated: bool
202
+ items: tuple[Item, ...]
203
+ quarantine: tuple[QuarantineRecord, ...]
204
+
205
+ def to_obj(self) -> dict[str, object]:
206
+ return {
207
+ "unit_id": self.unit_id,
208
+ "truncated": self.truncated,
209
+ "items": [item.to_obj() for item in self.items],
210
+ "quarantine": [
211
+ {"item_id": q.item_id, "stage": q.stage, "reason": q.reason, "details": q.details}
212
+ for q in self.quarantine
213
+ ],
214
+ }
215
+
216
+ @classmethod
217
+ def load(cls, path: Path) -> _UnitResult:
218
+ obj = json.loads(path.read_text(encoding="utf-8"))
219
+ if not isinstance(obj, dict) or set(obj) != {"unit_id", "truncated", "items", "quarantine"}:
220
+ raise ValueError(f"{path}: not a unit result")
221
+ return cls(
222
+ unit_id=str(obj["unit_id"]),
223
+ truncated=bool(obj["truncated"]),
224
+ items=tuple(Item.from_obj(i, f"{path}: item") for i in obj["items"]),
225
+ quarantine=tuple(
226
+ QuarantineRecord(q["item_id"], q["stage"], q["reason"], q["details"])
227
+ for q in obj["quarantine"]
228
+ ),
229
+ )
230
+
231
+
232
+ @dataclass(frozen=True)
233
+ class _Seg:
234
+ """A segment on its way through the stages.
235
+
236
+ ``duration_ms`` / ``loudness_dbfs`` come from the audio stage; ``num_frames`` and
237
+ ``frame_hop_ms`` from the latents stage (``0`` before it ran).
238
+ """
239
+
240
+ item: Item
241
+ duration_ms: int
242
+ loudness_dbfs: float
243
+ num_frames: int = 0
244
+ frame_hop_ms: float = 0.0
245
+ latent_dim: int = 0
246
+
247
+ @property
248
+ def sid(self) -> str:
249
+ return self.item.segment_id
250
+
251
+
252
+ @dataclass(frozen=True)
253
+ class _TimelineOutcome:
254
+ """A built timeline (with its mean word alignment score) or the quarantine record."""
255
+
256
+ timings: SegmentTimings | None
257
+ quarantine: QuarantineRecord | None
258
+ clamped_units: int = 0
259
+ mean_align_score: float = 0.0
260
+
261
+
262
+ def _digest(text: str) -> str:
263
+ return hashlib.sha1(text.encode("utf-8")).hexdigest()[:16]
264
+
265
+
266
+ def _file_sha1(path: Path) -> str:
267
+ return hashlib.sha1(path.read_bytes()).hexdigest()
268
+
269
+
270
+ def _write_jsonl(path: Path, rows: Sequence[Mapping[str, object]]) -> None:
271
+ text = "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows)
272
+ atomic_write_text(path, text)
273
+
274
+
275
+ def _read_jsonl(path: Path) -> list[dict[str, object]]:
276
+ rows: list[dict[str, object]] = []
277
+ for number, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
278
+ if not line:
279
+ continue
280
+ try:
281
+ obj = json.loads(line)
282
+ except json.JSONDecodeError as err:
283
+ raise ValueError(f"{path}:{number}: invalid JSON ({err.msg})") from err
284
+ if not isinstance(obj, dict):
285
+ raise ValueError(f"{path}:{number}: expected a JSON object")
286
+ rows.append(obj)
287
+ return rows
288
+
289
+
290
+ def _hours(duration_ms: int) -> float:
291
+ return round(duration_ms / 1000 / SECONDS_PER_HOUR, 3)
292
+
293
+
294
+ @dataclass
295
+ class Importer:
296
+ """Imports one transcribed external corpus into a latent dataset (module docstring of
297
+ :mod:`nida.data.importer`).
298
+
299
+ Attributes:
300
+ root: Durable directory (Drive): plan, manifests, quarantine, worker results, dataset.
301
+ cache: Local scratch: raw audio, 16 kHz wavs and ``.npy`` arrays (rebuilt when lost).
302
+ envs_root: Where the worker virtual environments live.
303
+ repo_dir: Repository root (environments are built from it).
304
+ request: What to import.
305
+ config_dir: ``configs/data`` (default ``repo_dir/configs/data``).
306
+ sources_path: ``configs/sources.yaml`` (default ``repo_dir/configs/sources.yaml``).
307
+ wavdir: ``--source-dir``: the folder of a ``wavdir`` source, or the Common Voice release
308
+ folder / tarball of ``common_voice_tr``.
309
+ hub: Hub client (default: ``huggingface_hub``).
310
+ retry_failed: Pass ``--retry-failed`` to the workers.
311
+ runner: Runs a worker command line and returns its exit code.
312
+ opener: HTTP access of the ``mediaspeech_tr`` download (default: ``urllib``).
313
+ uv: Path of the ``uv`` executable (default: found/installed on demand).
314
+ audio_threads: Threads converting raw audio to 16 kHz wavs (operational: does not
315
+ change the outputs).
316
+ """
317
+
318
+ root: Path
319
+ cache: Path
320
+ envs_root: Path
321
+ repo_dir: Path
322
+ request: ImportRequest
323
+ config_dir: Path | None = None
324
+ sources_path: Path | None = None
325
+ wavdir: Path | None = None
326
+ hub: HubClient | None = None
327
+ retry_failed: bool = False
328
+ runner: Callable[[Sequence[str]], int] = _stream_runner
329
+ opener: fetch.Opener | None = None
330
+ uv: str | None = None
331
+ audio_threads: int = DEFAULT_AUDIO_THREADS
332
+ _ready: bool = field(default=False, init=False, repr=False)
333
+ _cfg: ImportConfig | None = field(default=None, init=False, repr=False)
334
+ _plan: ImportPlan | None = field(default=None, init=False, repr=False)
335
+ _adapter: SourceAdapter | None = field(default=None, init=False, repr=False)
336
+ _pythons: dict[str, Path] = field(default_factory=dict, init=False, repr=False)
337
+
338
+ def __post_init__(self) -> None:
339
+ if isinstance(self.audio_threads, bool) or self.audio_threads < 1:
340
+ raise ValueError(f"audio_threads must be an integer >= 1, got {self.audio_threads!r}")
341
+
342
+ # ------------------------------------------------------------------ setup
343
+
344
+ @property
345
+ def cfg(self) -> ImportConfig:
346
+ if self._cfg is None:
347
+ raise RuntimeError("Importer is not prepared; call run()")
348
+ return self._cfg
349
+
350
+ @property
351
+ def plan(self) -> ImportPlan:
352
+ if self._plan is None:
353
+ raise RuntimeError("Importer is not prepared; call run()")
354
+ return self._plan
355
+
356
+ @property
357
+ def adapter(self) -> SourceAdapter:
358
+ if self._adapter is None:
359
+ raise RuntimeError("Importer is not prepared; call run()")
360
+ return self._adapter
361
+
362
+ def dataset_dir(self) -> Path:
363
+ """``<root>/dataset-<config digest>/``."""
364
+ return self.root / f"dataset-{self.cfg.digest}"
365
+
366
+ def _prepare(self) -> None:
367
+ """Load the config, check the licence gate, load or create the plan (once per run)."""
368
+ if self._ready:
369
+ return
370
+ config_dir = self.config_dir or self.repo_dir / "configs" / "data"
371
+ sources_path = self.sources_path or self.repo_dir / "configs" / "sources.yaml"
372
+ cfg = load_import_config(config_dir, self.request.source)
373
+ plan_path = self.root / "plan.json"
374
+ existing: ImportPlan | None = None
375
+ if plan_path.is_file():
376
+ existing = ImportPlan.from_obj(json.loads(plan_path.read_text("utf-8")), plan_path)
377
+ hub = self.hub
378
+ if hub is None and self.request.source in HUB_SOURCES:
379
+ hub = HuggingFaceHub()
380
+ adapter = make_adapter(
381
+ self.request.source,
382
+ cfg,
383
+ self.cache,
384
+ hub=hub,
385
+ revision=existing.revision if existing else self.request.revision,
386
+ wavdir=self.wavdir,
387
+ opener=self.opener,
388
+ )
389
+ needed = set(adapter.source_ids) | {DATA_MODELS[k].source_id for k in (VAE_KEY, CTC_KEY)}
390
+ require_approved(needed, load_sources(sources_path))
391
+ if existing is not None:
392
+ local = self.request.source in LOCAL_SOURCES
393
+ content = adapter.list_units().content_sha1 if local else None
394
+ existing.check(self.request, cfg.digest, content)
395
+ plan = existing
396
+ else:
397
+ listing = adapter.list_units()
398
+ stop = (
399
+ None
400
+ if self.request.unit_count is None
401
+ else (self.request.unit_offset + self.request.unit_count)
402
+ )
403
+ unit_ids = listing.unit_ids[self.request.unit_offset : stop]
404
+ if not unit_ids:
405
+ raise ValueError(
406
+ f"no work unit in range: offset {self.request.unit_offset}, count "
407
+ f"{self.request.unit_count}, {len(listing.unit_ids)} units exist"
408
+ )
409
+ plan = ImportPlan(
410
+ source=self.request.source,
411
+ revision=listing.revision,
412
+ max_hours=self.request.max_hours,
413
+ unit_offset=self.request.unit_offset,
414
+ unit_count=self.request.unit_count,
415
+ unit_ids=unit_ids,
416
+ config_digest=cfg.digest,
417
+ content_sha1=listing.content_sha1,
418
+ )
419
+ atomic_write_json(plan_path, plan.to_obj())
420
+ log.info(
421
+ "plan: %s revision %s, %d of %d units",
422
+ plan.source,
423
+ plan.revision,
424
+ len(unit_ids),
425
+ len(listing.unit_ids),
426
+ )
427
+ self._cfg, self._plan, self._adapter, self._ready = cfg, plan, adapter, True
428
+
429
+ # -------------------------------------------------------------- stage state
430
+
431
+ def _marker(self, stage: str) -> Path:
432
+ return self.root / stage / "stage.json"
433
+
434
+ def _reuse(self, stage: str, input_sha1: str) -> dict[str, object] | None:
435
+ """The recorded result of a finished stage (``reused: True``), else ``None``."""
436
+ marker = self._marker(stage)
437
+ if not marker.is_file():
438
+ return None
439
+ recorded = json.loads(marker.read_text(encoding="utf-8"))
440
+ if recorded.get("input_sha1") != input_sha1:
441
+ raise RuntimeError(
442
+ f"{stage}: its input changed after it ran; move {marker.parent} away to recompute"
443
+ )
444
+ result: dict[str, object] = dict(recorded["result"])
445
+ result["reused"] = True
446
+ return result
447
+
448
+ def _finish(self, stage: str, input_sha1: str, result: dict[str, object]) -> dict[str, object]:
449
+ atomic_write_json(
450
+ self._marker(stage), {"stage": stage, "input_sha1": input_sha1, "result": result}
451
+ )
452
+ return {**result, "reused": False}
453
+
454
+ def _require_done(self, stage: str) -> None:
455
+ if not self._marker(stage).is_file():
456
+ raise RuntimeError(f"stage {stage!r} has not finished; run it first")
457
+
458
+ def _write_quarantine(self, stage: str, records: Sequence[QuarantineRecord]) -> dict[str, int]:
459
+ """Replace the stage's quarantine log by ``records`` (a stage rerun rebuilds it)."""
460
+ qlog = QuarantineLog(self.root / stage)
461
+ qlog.path.unlink(missing_ok=True)
462
+ qlog.append(records)
463
+ return qlog.counts_by_reason()
464
+
465
+ def _drop(self, stage: str, item: Item, reason: str, **details: object) -> QuarantineRecord:
466
+ return QuarantineRecord(
467
+ item.segment_id, stage, reason, {"item_id": item.item_id, **details}
468
+ )
469
+
470
+ # --------------------------------------------------------------------- run
471
+
472
+ def run(self, stage: str) -> dict[str, object]:
473
+ """Run one stage (or ``all``); returns the stage result(s) as plain JSON data.
474
+
475
+ Raises:
476
+ ValueError: unknown stage, or a root made for another plan/config.
477
+ nida.bench.registry.SourceNotApprovedError: a source or model is not approved.
478
+ """
479
+ if stage == "all":
480
+ return {name: self.run(name) for name in STAGES}
481
+ if stage not in STAGES:
482
+ raise ValueError(f"unknown import stage {stage!r}; known: {', '.join(STAGES)}")
483
+ self._prepare()
484
+ log.info("=== import stage %s", stage)
485
+ handler: Callable[[], dict[str, object]] = getattr(self, "_stage_" + stage)
486
+ return handler()
487
+
488
+ # ------------------------------------------------------------------ select
489
+
490
+ def _stage_select(self) -> dict[str, object]:
491
+ """Walk the planned units in order, stop at ``max_hours``; write the item manifest.
492
+
493
+ Each finished unit is stored (``select/units/<digest>.json``) so an interrupted run
494
+ continues with the next unit. The last unit is cut at the first item that would exceed
495
+ the budget (``truncated``). Duplicate segment ids across units are quarantined.
496
+ """
497
+ cfg, plan = self.cfg, self.plan
498
+ plan_sha = _digest(json.dumps(plan.to_obj(), sort_keys=True))
499
+ reused = self._reuse("select", plan_sha)
500
+ if reused is not None:
501
+ return reused
502
+ units_dir = self.root / "select" / "units"
503
+ budget_ms = round(plan.max_hours * 3_600_000)
504
+ total_ms = 0
505
+ done: list[_UnitResult] = []
506
+ selected_in: dict[str, str] = {} # segment id -> unit that selected it
507
+ stopped = False
508
+ for unit_id in plan.unit_ids:
509
+ path = units_dir / f"{_digest(unit_id)}.json"
510
+ if path.is_file():
511
+ result = _UnitResult.load(path)
512
+ if result.unit_id != unit_id:
513
+ raise RuntimeError(
514
+ f"{path} belongs to unit {result.unit_id!r}, not {unit_id!r}"
515
+ )
516
+ else:
517
+ if budget_ms - total_ms < cfg.min_duration_ms:
518
+ stopped = True
519
+ break
520
+ result = self._select_unit(unit_id, budget_ms - total_ms, selected_in)
521
+ atomic_write_json(path, result.to_obj())
522
+ log.info(
523
+ "select: unit %s -> %d items, %d quarantined",
524
+ unit_id,
525
+ len(result.items),
526
+ len(result.quarantine),
527
+ )
528
+ done.append(result)
529
+ for item in result.items:
530
+ selected_in.setdefault(item.segment_id, result.unit_id)
531
+ total_ms += sum(item.raw_duration_ms or 0 for item in result.items)
532
+ if result.truncated:
533
+ stopped = True
534
+ break
535
+ items: list[Item] = []
536
+ records: list[QuarantineRecord] = []
537
+ first_unit: dict[str, str] = {}
538
+ for result in done:
539
+ records.extend(result.quarantine)
540
+ for item in result.items:
541
+ if item.segment_id in first_unit:
542
+ records.append(
543
+ self._drop(
544
+ "select",
545
+ item,
546
+ "duplicate_item",
547
+ unit_id=item.unit_id,
548
+ first_unit_id=first_unit[item.segment_id],
549
+ )
550
+ )
551
+ continue
552
+ first_unit[item.segment_id] = item.unit_id
553
+ items.append(item)
554
+ counts = self._write_quarantine("select", records)
555
+ if not items:
556
+ raise RuntimeError(f"select kept no item from {len(done)} units; quarantined: {counts}")
557
+ _write_jsonl(self.root / "select" / "manifest.jsonl", [i.to_obj() for i in items])
558
+ selected_ms = sum(item.raw_duration_ms or 0 for item in items)
559
+ return self._finish(
560
+ "select",
561
+ plan_sha,
562
+ {
563
+ "units_planned": len(plan.unit_ids),
564
+ "units_processed": len(done),
565
+ "stopped_by_max_hours": stopped,
566
+ "selected": len(items),
567
+ "selected_hours": _hours(selected_ms),
568
+ "quarantined": counts,
569
+ },
570
+ )
571
+
572
+ def _select_unit(
573
+ self, unit_id: str, remaining_ms: int, selected_in: Mapping[str, str]
574
+ ) -> _UnitResult:
575
+ """One unit: items within ``remaining_ms``; ids in ``selected_in`` (earlier units) are
576
+ quarantined as duplicates by the adapter before their audio is written."""
577
+ cfg, adapter = self.cfg, self.adapter
578
+ items: list[Item] = []
579
+ records: list[QuarantineRecord] = []
580
+ used_ms = 0
581
+ truncated = False
582
+ events = adapter.iter_unit(unit_id, exclude=selected_in)
583
+ try:
584
+ for event in events:
585
+ if isinstance(event, QuarantineRecord):
586
+ records.append(event)
587
+ continue
588
+ raw = adapter.raw_path(event)
589
+ reason: str | None = None
590
+ details: dict[str, object] = {}
591
+ duration_ms = 0
592
+ try:
593
+ duration_ms = raw_duration_ms(raw)
594
+ except AudioRejected as err:
595
+ reason, details = err.reason, err.details
596
+ else:
597
+ if duration_ms < cfg.min_duration_ms:
598
+ reason, details = "too_short", {"duration_ms": duration_ms}
599
+ elif duration_ms > cfg.max_duration_ms:
600
+ reason, details = "too_long", {"duration_ms": duration_ms}
601
+ if reason is not None:
602
+ records.append(self._drop("select", event, reason, **details))
603
+ elif used_ms + duration_ms > remaining_ms:
604
+ truncated = True
605
+ else:
606
+ items.append(_with_duration(event, duration_ms))
607
+ used_ms += duration_ms
608
+ if reason is not None or truncated:
609
+ if adapter.ephemeral_raw:
610
+ raw.unlink(missing_ok=True)
611
+ if truncated:
612
+ break
613
+ finally:
614
+ events.close()
615
+ return _UnitResult(unit_id, truncated, tuple(items), tuple(records))
616
+
617
+ def _select_items(self) -> list[Item]:
618
+ self._require_done("select")
619
+ return [
620
+ Item.from_obj(row, f"select/manifest.jsonl:{number}")
621
+ for number, row in enumerate(_read_jsonl(self.root / "select" / "manifest.jsonl"), 1)
622
+ ]
623
+
624
+ # ------------------------------------------------------------------- audio
625
+
626
+ def _wav(self, sid: str) -> Path:
627
+ return self.cache / "audio16k" / f"{sid}.wav"
628
+
629
+ def _npy(self, model_key: str, sid: str) -> Path:
630
+ return self.cache / "arrays" / model_key / f"{sid}.npy"
631
+
632
+ def _refetch(self, items: Sequence[Item]) -> None:
633
+ """Put the raw audio of ``items`` back into the cache (one pass per unit)."""
634
+ by_unit: dict[str, set[str]] = {}
635
+ for item in items:
636
+ by_unit.setdefault(item.unit_id, set()).add(item.segment_id)
637
+ for unit_id, sids in by_unit.items():
638
+ log.info("refetching raw audio of %d items of unit %s", len(sids), unit_id)
639
+ events = self.adapter.iter_unit(unit_id, only=sids)
640
+ try:
641
+ for _ in events:
642
+ pass
643
+ finally:
644
+ events.close()
645
+
646
+ def _ensure_wavs(self, items: Sequence[Item]) -> dict[str, AudioRejected]:
647
+ """Make sure the 16 kHz wav of every item is in the cache.
648
+
649
+ Missing wavs are converted from the raw audio, which is fetched again first if it is
650
+ gone too (ephemeral sources). Converted ephemeral raw files are deleted.
651
+
652
+ Returns:
653
+ ``{segment_id: reason}`` for the items that could not be given a wav.
654
+ """
655
+ adapter = self.adapter
656
+ todo = [i for i in items if not self._wav(i.segment_id).is_file()]
657
+ if not todo:
658
+ return {}
659
+ if adapter.ephemeral_raw:
660
+ self._refetch([i for i in todo if not adapter.raw_path(i).is_file()])
661
+ failures: dict[str, AudioRejected] = {}
662
+ with ThreadPoolExecutor(max_workers=self.audio_threads) as pool:
663
+ outcomes = pool.map(self._convert_one, todo)
664
+ for number, (item, error) in enumerate(zip(todo, outcomes, strict=True), start=1):
665
+ if error is not None:
666
+ failures[item.segment_id] = error
667
+ if number % PROGRESS_EVERY == 0:
668
+ log.info("audio: converted %d/%d", number, len(todo))
669
+ return failures
670
+
671
+ def _convert_one(self, item: Item) -> AudioRejected | None:
672
+ """Raw audio -> working wav for one item (thread-safe: touches only this item's files)."""
673
+ adapter = self.adapter
674
+ raw = adapter.raw_path(item)
675
+ if not raw.is_file():
676
+ return AudioRejected("raw_audio_missing", path=str(raw))
677
+ error: AudioRejected | None = None
678
+ try:
679
+ convert_to_wav16k(raw, self._wav(item.segment_id))
680
+ except AudioRejected as err:
681
+ error = err
682
+ if adapter.ephemeral_raw:
683
+ raw.unlink(missing_ok=True)
684
+ return error
685
+
686
+ def _measure(self, item: Item) -> tuple[int, float]:
687
+ """``(duration_ms, loudness_dbfs)`` of the item's wav; a bad cached wav is rebuilt once."""
688
+ wav = self._wav(item.segment_id)
689
+ for attempt in (1, 2):
690
+ try:
691
+ facts = measure_wav16k(wav)
692
+ except AudioRejected:
693
+ if attempt == 2:
694
+ raise
695
+ wav.unlink(missing_ok=True)
696
+ failed = self._ensure_wavs([item])
697
+ if item.segment_id in failed:
698
+ raise failed[item.segment_id] from None
699
+ else:
700
+ return facts.duration_ms, facts.loudness_dbfs
701
+ raise AssertionError("unreachable") # pragma: no cover
702
+
703
+ def _stage_audio(self) -> dict[str, object]:
704
+ """16 kHz mono wav of every selected item, measured; unusable audio is quarantined."""
705
+ cfg = self.cfg
706
+ items = self._select_items()
707
+ input_sha = _file_sha1(self.root / "select" / "manifest.jsonl")
708
+ reused = self._reuse("audio", input_sha)
709
+ if reused is not None:
710
+ return reused
711
+ failures = self._ensure_wavs(items)
712
+ rows: list[dict[str, object]] = []
713
+ records: list[QuarantineRecord] = []
714
+ kept_ms = 0
715
+ for number, item in enumerate(items, start=1):
716
+ error = failures.get(item.segment_id)
717
+ reason: str | None = None
718
+ details: dict[str, object] = {}
719
+ duration_ms, loudness = 0, 0.0
720
+ if error is not None:
721
+ reason, details = error.reason, error.details
722
+ else:
723
+ try:
724
+ duration_ms, loudness = self._measure(item)
725
+ except AudioRejected as err:
726
+ reason, details = err.reason, err.details
727
+ else:
728
+ if duration_ms < cfg.min_duration_ms:
729
+ reason, details = "too_short", {"duration_ms": duration_ms}
730
+ elif duration_ms > cfg.max_duration_ms:
731
+ reason, details = "too_long", {"duration_ms": duration_ms}
732
+ elif loudness < cfg.min_loudness_dbfs:
733
+ reason, details = "silent", {"loudness_dbfs": round(loudness, 2)}
734
+ if reason is not None:
735
+ records.append(self._drop("audio", item, reason, **details))
736
+ else:
737
+ rows.append(
738
+ {
739
+ "segment_id": item.segment_id,
740
+ "duration_ms": duration_ms,
741
+ "loudness_dbfs": loudness,
742
+ }
743
+ )
744
+ kept_ms += duration_ms
745
+ if self.adapter.ephemeral_raw:
746
+ self.adapter.raw_path(item).unlink(missing_ok=True)
747
+ if number % PROGRESS_EVERY == 0:
748
+ log.info("audio: measured %d/%d", number, len(items))
749
+ counts = self._write_quarantine("audio", records)
750
+ if not rows:
751
+ raise RuntimeError(f"audio kept no item of {len(items)}; quarantined: {counts}")
752
+ _write_jsonl(self.root / "audio" / "manifest.jsonl", rows)
753
+ return self._finish(
754
+ "audio",
755
+ input_sha,
756
+ {
757
+ "input": len(items),
758
+ "kept": len(rows),
759
+ "kept_hours": _hours(kept_ms),
760
+ "quarantined": counts,
761
+ },
762
+ )
763
+
764
+ # ------------------------------------------------------- shared stage tables
765
+
766
+ def _segs(self, upto: str) -> list[_Seg]:
767
+ """Segments that survived every stage up to and including ``upto``
768
+ (``audio`` | ``latents`` | ``ctc``), in select order."""
769
+ order = ("audio", "latents", "ctc")
770
+ if upto not in order:
771
+ raise ValueError(f"unknown stage {upto!r}")
772
+ for stage in order[: order.index(upto) + 1]:
773
+ self._require_done(stage)
774
+ audio = {
775
+ str(r["segment_id"]): r for r in _read_jsonl(self.root / "audio" / "manifest.jsonl")
776
+ }
777
+ latents: dict[str, dict[str, object]] = {}
778
+ if upto != "audio":
779
+ latents = {
780
+ str(r["segment_id"]): r
781
+ for r in _read_jsonl(self.root / "latents" / "manifest.jsonl")
782
+ }
783
+ ctc: set[str] | None = None
784
+ if upto == "ctc":
785
+ ctc = {str(r["segment_id"]) for r in _read_jsonl(self.root / "ctc" / "manifest.jsonl")}
786
+ segs: list[_Seg] = []
787
+ for item in self._select_items():
788
+ row = audio.get(item.segment_id)
789
+ if row is None:
790
+ continue
791
+ frames, hop, dim = 0, 0.0, 0
792
+ if upto != "audio":
793
+ facts = latents.get(item.segment_id)
794
+ if facts is None:
795
+ continue
796
+ frames = int(str(facts["num_frames"]))
797
+ hop = float(str(facts["frame_hop_ms"]))
798
+ dim = int(str(facts["latent_dim"]))
799
+ if ctc is not None and item.segment_id not in ctc:
800
+ continue
801
+ segs.append(
802
+ _Seg(
803
+ item,
804
+ int(str(row["duration_ms"])),
805
+ float(str(row["loudness_dbfs"])),
806
+ frames,
807
+ hop,
808
+ dim,
809
+ )
810
+ )
811
+ return segs
812
+
813
+ # ----------------------------------------------------------------- workers
814
+
815
+ def _python(self, key: str) -> Path:
816
+ if key not in self._pythons:
817
+ if self.uv is None:
818
+ self.uv = bench_envs.find_uv(bench_envs.subprocess_runner)
819
+ spec = envs.data_env_for_worker(DATA_MODELS[key].worker)
820
+ self._pythons[key] = bench_envs.ensure_env(
821
+ spec, self.envs_root, self.repo_dir, uv=self.uv
822
+ )
823
+ return self._pythons[key]
824
+
825
+ def _pending(self, key: str, sids: Sequence[str]) -> set[str]:
826
+ """Segments whose worker result is missing, failed (with ``retry_failed``) or whose
827
+ array left the cache: these need their wav."""
828
+ path = self.root / "workers" / key / "results.jsonl"
829
+ existing = load_results(path, repair=True) if path.exists() else {}
830
+ pending: set[str] = set()
831
+ for sid in sids:
832
+ result = existing.get(sid)
833
+ if result is None:
834
+ pending.add(sid)
835
+ elif result.output is None:
836
+ if self.retry_failed:
837
+ pending.add(sid)
838
+ elif not Path(str(result.output.get("out_path", ""))).is_file():
839
+ pending.add(sid)
840
+ return pending
841
+
842
+ def _run_model(self, key: str, segs: Sequence[_Seg]) -> dict[str, WorkResult]:
843
+ """Run the data worker ``key`` over ``segs`` (resumable; no work when complete)."""
844
+ model = DATA_MODELS[key]
845
+ pending = self._pending(key, [s.sid for s in segs])
846
+ if pending:
847
+ failed = self._ensure_wavs([s.item for s in segs if s.sid in pending])
848
+ if failed:
849
+ sample = {sid: err.reason for sid, err in list(failed.items())[:3]}
850
+ raise RuntimeError(f"{key}: {len(failed)} audio files cannot be rebuilt: {sample}")
851
+ requests = tuple(
852
+ WorkRequest(s.sid, str(self._wav(s.sid)), {"out_path": str(self._npy(key, s.sid))})
853
+ for s in segs
854
+ )
855
+ return run_worker_job(
856
+ self.root / "workers" / key,
857
+ key,
858
+ lambda: self._python(key),
859
+ envs.worker_module_name(model.worker),
860
+ model.ref.model_id,
861
+ model.ref.revision,
862
+ model.batch_size,
863
+ requests,
864
+ array_outputs=True,
865
+ runner=self.runner,
866
+ retry_failed=self.retry_failed,
867
+ skip_if_complete=True,
868
+ )
869
+
870
+ # ----------------------------------------------------------------- latents
871
+
872
+ def _stage_latents(self) -> dict[str, object]:
873
+ """VAE latents (``vae_encode`` worker) of the segments that passed the audio stage."""
874
+ segs = self._segs("audio")
875
+ input_sha = _file_sha1(self.root / "audio" / "manifest.jsonl")
876
+ reused = self._reuse("latents", input_sha)
877
+ if reused is not None:
878
+ return reused
879
+ results = self._run_model(VAE_KEY, segs)
880
+ rows: list[dict[str, object]] = []
881
+ records: list[QuarantineRecord] = []
882
+ hops: set[float] = set()
883
+ dims: set[int] = set()
884
+ for seg in segs:
885
+ result = results.get(seg.sid)
886
+ if result is None:
887
+ raise RuntimeError(f"{VAE_KEY}: no result for {seg.sid}")
888
+ if result.output is None:
889
+ records.append(
890
+ self._drop("latents", seg.item, "latents_failed", error=result.error)
891
+ )
892
+ continue
893
+ array = np.load(str(result.output["out_path"]), allow_pickle=False)
894
+ problem = _latent_problem(array)
895
+ if problem is not None:
896
+ records.append(self._drop("latents", seg.item, problem[0], **problem[1]))
897
+ continue
898
+ hop = float(str(result.output["frame_hop_ms"]))
899
+ hops.add(hop)
900
+ dims.add(int(array.shape[1]))
901
+ rows.append(
902
+ {
903
+ "segment_id": seg.sid,
904
+ "num_frames": int(array.shape[0]),
905
+ "latent_dim": int(array.shape[1]),
906
+ "frame_hop_ms": hop,
907
+ }
908
+ )
909
+ if len(hops) > 1 or len(dims) > 1:
910
+ raise ValueError(f"latents disagree on frame hop {sorted(hops)} or dim {sorted(dims)}")
911
+ counts = self._write_quarantine("latents", records)
912
+ if not rows:
913
+ raise RuntimeError(f"latents kept no segment of {len(segs)}; quarantined: {counts}")
914
+ _write_jsonl(self.root / "latents" / "manifest.jsonl", rows)
915
+ return self._finish(
916
+ "latents",
917
+ input_sha,
918
+ {
919
+ "input": len(segs),
920
+ "kept": len(rows),
921
+ "latent_dim": dims.pop(),
922
+ "frame_hop_ms": hops.pop(),
923
+ "quarantined": counts,
924
+ },
925
+ )
926
+
927
+ # --------------------------------------------------------------------- ctc
928
+
929
+ def _stage_ctc(self) -> dict[str, object]:
930
+ """CTC emissions (``ctc_emissions`` worker) of the segments that have latents."""
931
+ segs = self._segs("latents")
932
+ input_sha = _file_sha1(self.root / "latents" / "manifest.jsonl")
933
+ reused = self._reuse("ctc", input_sha)
934
+ if reused is not None:
935
+ return reused
936
+ results = self._run_model(CTC_KEY, segs)
937
+ rows: list[dict[str, object]] = []
938
+ records: list[QuarantineRecord] = []
939
+ for seg in segs:
940
+ result = results.get(seg.sid)
941
+ if result is None:
942
+ raise RuntimeError(f"{CTC_KEY}: no result for {seg.sid}")
943
+ if result.output is None:
944
+ records.append(self._drop("ctc", seg.item, "ctc_failed", error=result.error))
945
+ else:
946
+ rows.append({"segment_id": seg.sid})
947
+ counts = self._write_quarantine("ctc", records)
948
+ if not rows:
949
+ raise RuntimeError(f"ctc kept no segment of {len(segs)}; quarantined: {counts}")
950
+ _write_jsonl(self.root / "ctc" / "manifest.jsonl", rows)
951
+ return self._finish(
952
+ "ctc", input_sha, {"input": len(segs), "kept": len(rows), "quarantined": counts}
953
+ )
954
+
955
+ # ----------------------------------------------------------------- timings
956
+
957
+ def _stage_timings(self) -> dict[str, object]:
958
+ """Unit timeline of every segment (CTC forced alignment of the transcript), as the
959
+ pilot's ``timings`` stage does, with no event tags (``tags_json`` is ``[]``)."""
960
+ segs = self._segs("ctc")
961
+ input_sha = _file_sha1(self.root / "ctc" / "manifest.jsonl")
962
+ reused = self._reuse("timings", input_sha)
963
+ if reused is not None:
964
+ return reused
965
+ # The request set of the ctc stage (not just the survivors), so no result is pruned.
966
+ results = self._run_model(CTC_KEY, self._segs("latents"))
967
+ built: list[SegmentTimings] = []
968
+ records: list[QuarantineRecord] = []
969
+ clamped = 0
970
+ scores: list[float] = []
971
+ for number, seg in enumerate(segs, start=1):
972
+ output = results[seg.sid].output
973
+ if output is None:
974
+ raise RuntimeError(f"{CTC_KEY}: result of {seg.sid} failed after the ctc stage")
975
+ outcome = self._timeline(seg, output)
976
+ if outcome.timings is not None:
977
+ built.append(outcome.timings)
978
+ clamped += outcome.clamped_units
979
+ scores.append(outcome.mean_align_score)
980
+ else:
981
+ assert outcome.quarantine is not None
982
+ records.append(outcome.quarantine)
983
+ if number % PROGRESS_EVERY == 0:
984
+ log.info("timings: %d/%d", number, len(segs))
985
+ counts = self._write_quarantine("timings", records)
986
+ if not built:
987
+ raise RuntimeError(f"timings built no timeline of {len(segs)}; quarantined: {counts}")
988
+ write_timings(self.root / "timings" / "timings.jsonl", built)
989
+ return self._finish(
990
+ "timings",
991
+ input_sha,
992
+ {
993
+ "input": len(segs),
994
+ "written": len(built),
995
+ "units": sum(len(t.units) for t in built),
996
+ "zero_length_units": sum(t.zero_length_units for t in built),
997
+ "clamped_units": clamped,
998
+ "mean_align_score": round(math.fsum(scores) / len(scores), 4),
999
+ "min_segment_align_score": round(min(scores), 4),
1000
+ "quarantined": counts,
1001
+ },
1002
+ )
1003
+
1004
+ def _timeline(self, seg: _Seg, ctc: Mapping[str, object]) -> _TimelineOutcome:
1005
+ """Align the transcript, check the alignment, build the timeline (or say why not)."""
1006
+ n_tokens = len(encode_text(seg.item.text))
1007
+ if n_tokens > seg.num_frames:
1008
+ record = self._drop(
1009
+ "timings",
1010
+ seg.item,
1011
+ "text_longer_than_frames",
1012
+ tokens=n_tokens,
1013
+ frames=seg.num_frames,
1014
+ )
1015
+ return _TimelineOutcome(None, record)
1016
+ vocab_list = [str(t) for t in ctc["vocab"]] # type: ignore[attr-defined]
1017
+ vocab = {token: index for index, token in enumerate(vocab_list)}
1018
+ emissions = np.load(str(ctc["out_path"]), allow_pickle=False)
1019
+ try:
1020
+ aligned = align_words(
1021
+ seg.item.text.split(),
1022
+ emissions,
1023
+ vocab,
1024
+ blank_token=vocab_list[int(str(ctc["blank_id"]))],
1025
+ word_delimiter="|",
1026
+ frame_hop_ms=float(str(ctc["frame_hop_ms"])),
1027
+ duration_ms=seg.duration_ms,
1028
+ )
1029
+ except ValueError as err:
1030
+ record = self._drop("timings", seg.item, "timings_align_error", error=str(err))
1031
+ return _TimelineOutcome(None, record)
1032
+ if aligned.words is None:
1033
+ assert aligned.failure is not None
1034
+ reason = "timings_align_" + aligned.failure.split(":", 1)[0]
1035
+ record = self._drop(
1036
+ "timings", seg.item, reason, failure=aligned.failure, **dict(aligned.details)
1037
+ )
1038
+ return _TimelineOutcome(None, record)
1039
+ scores = [w.align_score for w in aligned.words if w.align_score is not None]
1040
+ mean_score = math.fsum(scores) / len(scores) if scores else 0.0
1041
+ problem = check_alignment(aligned.words, seg.duration_ms, self.cfg.alignment)
1042
+ if problem is not None:
1043
+ record = self._drop(
1044
+ "timings",
1045
+ seg.item,
1046
+ problem,
1047
+ mean_align_score=round(mean_score, 4),
1048
+ min_align_score=self.cfg.alignment.min_align_score,
1049
+ )
1050
+ return _TimelineOutcome(None, record)
1051
+ built = build_timings(
1052
+ seg.sid,
1053
+ seg.item.text,
1054
+ json.dumps([]),
1055
+ aligned.words,
1056
+ num_frames=seg.num_frames,
1057
+ frame_hop_ms=seg.frame_hop_ms,
1058
+ )
1059
+ if built.timings is None:
1060
+ assert built.quarantine is not None
1061
+ q = built.quarantine
1062
+ record = self._drop("timings", seg.item, q.reason, **q.details)
1063
+ return _TimelineOutcome(None, record)
1064
+ return _TimelineOutcome(built.timings, None, built.clamped_units, mean_score)
1065
+
1066
+ # ----------------------------------------------------------------- dataset
1067
+
1068
+ def _stage_dataset(self) -> dict[str, object]:
1069
+ """Write the latent dataset directory (format: :mod:`nida.train.data`) + timings.jsonl."""
1070
+ cfg = self.cfg
1071
+ dataset_dir = self.dataset_dir()
1072
+ meta_path = dataset_dir / "meta.json"
1073
+ timings_path = self.root / "timings" / "timings.jsonl"
1074
+ self._require_done("timings")
1075
+ timings_sha = _file_sha1(timings_path)
1076
+ if meta_path.is_file():
1077
+ meta = json.loads(meta_path.read_text(encoding="utf-8"))
1078
+ if meta.get("timings_sha1") != timings_sha:
1079
+ raise RuntimeError(
1080
+ f"{meta_path} was built from other timings; move {dataset_dir} away"
1081
+ )
1082
+ return {"dataset": str(dataset_dir), "segments": meta["segments"], "reused": True}
1083
+ timelines = {
1084
+ t.segment_id: t
1085
+ for t in (
1086
+ SegmentTimings.from_obj(row, f"{timings_path}:{n}")
1087
+ for n, row in enumerate(_read_jsonl(timings_path), 1)
1088
+ )
1089
+ }
1090
+ segs = [s for s in self._segs("ctc") if s.sid in timelines]
1091
+ if len(segs) != len(timelines):
1092
+ raise RuntimeError("timings.jsonl names segments that did not survive the stages")
1093
+ splits = assign_group_splits({s.sid: s.item.group for s in segs}, cfg)
1094
+ forced = cfg.forced_split(self.plan.source)
1095
+ if forced is not None: # no group of this source can be pure: all rows are one split
1096
+ splits = {sid: forced for sid in splits}
1097
+ # The request set of the latents stage; recomputes arrays lost with the cache.
1098
+ latents = self._run_model(VAE_KEY, self._segs("audio"))
1099
+ index: list[dict[str, object]] = []
1100
+ shard: list[np.ndarray[Any, np.dtype[np.float16]]] = []
1101
+ shard_frames = 0
1102
+ shard_no = 0
1103
+ latent_dim: int | None = None
1104
+ frame_hop_ms: float | None = None
1105
+
1106
+ def flush() -> None:
1107
+ nonlocal shard, shard_frames, shard_no
1108
+ if not shard:
1109
+ return
1110
+ buffer = io.BytesIO()
1111
+ np.save(buffer, np.concatenate(shard), allow_pickle=False)
1112
+ atomic_write_bytes(dataset_dir / f"shard-{shard_no:05d}.npy", buffer.getvalue())
1113
+ shard, shard_frames, shard_no = [], 0, shard_no + 1
1114
+
1115
+ for seg in sorted(segs, key=lambda s: s.sid):
1116
+ output = latents[seg.sid].output
1117
+ if output is None:
1118
+ raise RuntimeError(f"{VAE_KEY}: result of {seg.sid} failed after the latents stage")
1119
+ array = np.load(str(output["out_path"]), allow_pickle=False)
1120
+ if _latent_problem(array) is not None or int(array.shape[0]) != seg.num_frames:
1121
+ raise RuntimeError(f"{seg.sid}: cached latents differ from the latents stage")
1122
+ latent_dim, frame_hop_ms = seg.latent_dim, seg.frame_hop_ms
1123
+ index.append(
1124
+ {
1125
+ "segment_id": seg.sid,
1126
+ "shard": f"shard-{shard_no:05d}.npy",
1127
+ "offset": shard_frames,
1128
+ "num_frames": seg.num_frames,
1129
+ "text": seg.item.text,
1130
+ "emotion": seg.item.emotion,
1131
+ "intensity": None,
1132
+ "rate_bucket": bucket_of(
1133
+ rate_chars_per_s(seg.item.text, seg.duration_ms),
1134
+ cfg.dataset.rate_bucket_edges_chars_per_s,
1135
+ ),
1136
+ "loudness_bucket": bucket_of(
1137
+ seg.loudness_dbfs, cfg.dataset.loudness_bucket_edges_dbfs
1138
+ ),
1139
+ "group": seg.item.group,
1140
+ "split": splits[seg.sid],
1141
+ "language": LANGUAGE,
1142
+ }
1143
+ )
1144
+ shard.append(array.astype(np.float16))
1145
+ shard_frames += seg.num_frames
1146
+ if shard_frames >= DATASET_SHARD_FRAMES:
1147
+ flush()
1148
+ flush()
1149
+ _write_jsonl(dataset_dir / "index.jsonl", index)
1150
+ write_timings(dataset_dir / "timings.jsonl", [timelines[s.sid] for s in segs])
1151
+ new_meta: dict[str, object] = {
1152
+ "segments": len(index),
1153
+ "frames": sum(int(str(i["num_frames"])) for i in index),
1154
+ "latent_dim": latent_dim,
1155
+ "frame_hop_ms": frame_hop_ms,
1156
+ "rate_bucket_edges_chars_per_s": list(cfg.dataset.rate_bucket_edges_chars_per_s),
1157
+ "loudness_bucket_edges_dbfs": list(cfg.dataset.loudness_bucket_edges_dbfs),
1158
+ "language": LANGUAGE,
1159
+ "source": self.plan.source,
1160
+ "source_ids": sorted({s.item.source_id for s in segs}),
1161
+ "plan": self.plan.to_obj(),
1162
+ "config_digest": cfg.digest,
1163
+ "seed": cfg.seed,
1164
+ "timings_sha1": timings_sha,
1165
+ "vae": [DATA_MODELS[VAE_KEY].ref.model_id, DATA_MODELS[VAE_KEY].ref.revision],
1166
+ "ctc": [DATA_MODELS[CTC_KEY].ref.model_id, DATA_MODELS[CTC_KEY].ref.revision],
1167
+ "splits": dict(sorted(Counter(splits.values()).items())),
1168
+ }
1169
+ if forced is not None:
1170
+ new_meta["forced_split"] = forced
1171
+ atomic_write_json(meta_path, new_meta) # written last: its presence marks the stage done
1172
+ try:
1173
+ self._check_with_loader(dataset_dir)
1174
+ except (DataFormatError, RuntimeError):
1175
+ meta_path.unlink(missing_ok=True)
1176
+ raise
1177
+ return {
1178
+ "dataset": str(dataset_dir),
1179
+ "segments": len(index),
1180
+ "frames": new_meta["frames"],
1181
+ "splits": new_meta["splits"],
1182
+ "reused": False,
1183
+ }
1184
+
1185
+ @staticmethod
1186
+ def _check_with_loader(dataset_dir: Path) -> None:
1187
+ """Every split must load with timelines and no rejection (else the importer is wrong)."""
1188
+ for split in SPLITS:
1189
+ dataset, report = load_latent_dataset(dataset_dir, split, timings=True)
1190
+ dataset.close()
1191
+ if report.num_rejected:
1192
+ raise RuntimeError(
1193
+ f"nida.train.data rejects records of the imported {split} split: "
1194
+ f"{report.rejection_counts}"
1195
+ )
1196
+
1197
+ # ------------------------------------------------------------------ report
1198
+
1199
+ def _stage_report(self) -> dict[str, object]:
1200
+ """``report.md`` and ``metrics.json`` of the root; returns the metrics."""
1201
+ dataset_dir = self.dataset_dir()
1202
+ meta_path = dataset_dir / "meta.json"
1203
+ if not meta_path.is_file():
1204
+ raise RuntimeError(f"{meta_path} is missing: run the dataset stage first")
1205
+ meta = json.loads(meta_path.read_text(encoding="utf-8"))
1206
+ index = _read_jsonl(dataset_dir / "index.jsonl")
1207
+ kept_ids = {str(r["segment_id"]) for r in index}
1208
+ segs = {s.sid: s for s in self._segs("ctc") if s.sid in kept_ids}
1209
+ selected = self._select_items()
1210
+ selected_ms = sum(i.raw_duration_ms or 0 for i in selected)
1211
+ quarantine = {
1212
+ stage: QuarantineLog(self.root / stage).counts_by_reason()
1213
+ for stage in ("select", "audio", "latents", "ctc", "timings")
1214
+ if (self.root / stage).is_dir()
1215
+ }
1216
+ splits: dict[str, dict[str, object]] = {}
1217
+ for split in SPLITS:
1218
+ members = [segs[str(r["segment_id"])] for r in index if r["split"] == split]
1219
+ splits[split] = {
1220
+ "segments": len(members),
1221
+ "hours": _hours(sum(s.duration_ms for s in members)),
1222
+ "groups": len({s.item.group for s in members}),
1223
+ }
1224
+ emotions = Counter("null" if r["emotion"] is None else str(r["emotion"]) for r in index)
1225
+ unmapped = Counter(
1226
+ str(s.item.extra.get("emotion_label"))
1227
+ for s in segs.values()
1228
+ if s.item.emotion is None and s.item.extra.get("emotion_label") is not None
1229
+ )
1230
+ timings_result = json.loads(self._marker("timings").read_text(encoding="utf-8"))["result"]
1231
+ metrics: dict[str, object] = {
1232
+ "alignment": {
1233
+ key: timings_result[key] for key in ("mean_align_score", "min_segment_align_score")
1234
+ },
1235
+ "source": self.plan.source,
1236
+ "source_ids": meta["source_ids"],
1237
+ "revision": self.plan.revision,
1238
+ "config_digest": self.cfg.digest,
1239
+ "max_hours": self.plan.max_hours,
1240
+ "units_planned": len(self.plan.unit_ids),
1241
+ "selected": {"segments": len(selected), "hours": _hours(selected_ms)},
1242
+ "kept": {
1243
+ "segments": len(index),
1244
+ "hours": _hours(sum(s.duration_ms for s in segs.values())),
1245
+ },
1246
+ "quarantine": quarantine,
1247
+ "emotions": dict(sorted(emotions.items())),
1248
+ "unmapped_emotion_labels": dict(sorted(unmapped.items())),
1249
+ "groups": len({s.item.group for s in segs.values()}),
1250
+ "splits": splits,
1251
+ "dataset": str(dataset_dir),
1252
+ }
1253
+ atomic_write_json(self.root / "metrics.json", metrics)
1254
+ atomic_write_text(self.root / "report.md", _render_report(metrics))
1255
+ return metrics
1256
+
1257
+
1258
+ def assign_group_splits(groups: Mapping[str, str], cfg: ImportConfig) -> dict[str, str]:
1259
+ """Group-disjoint, deterministic ``train``/``val``/``test`` split of ``{segment_id: group}``.
1260
+
1261
+ The split of a group is a pure function of ``(cfg.seed, group)``
1262
+ (:func:`nida.data.splits.assign_splits`; ``val_fraction`` / ``test_fraction`` of the groups),
1263
+ independent of which other groups the root holds: the same group lands in the same split in
1264
+ every root, so datasets of different roots can be merged without a group crossing splits.
1265
+ A root with a single group is therefore wholly one split, not necessarily ``train``.
1266
+ """
1267
+ return assign_splits(groups, cfg.split_fractions, str(cfg.seed))
1268
+
1269
+
1270
+ def _with_duration(item: Item, duration_ms: int) -> Item:
1271
+ return Item(
1272
+ source_id=item.source_id,
1273
+ item_id=item.item_id,
1274
+ segment_id=item.segment_id,
1275
+ unit_id=item.unit_id,
1276
+ text=item.text,
1277
+ group=item.group,
1278
+ emotion=item.emotion,
1279
+ raw_duration_ms=duration_ms,
1280
+ extra=item.extra,
1281
+ )
1282
+
1283
+
1284
+ def _latent_problem(array: np.ndarray[Any, Any]) -> tuple[str, dict[str, object]] | None:
1285
+ """A reason (and details) why latents cannot go into a float16 shard, else ``None``."""
1286
+ if array.ndim != 2 or array.shape[0] < 1 or array.shape[1] < 1:
1287
+ return "latents_bad_shape", {"shape": list(array.shape)}
1288
+ if not np.all(np.isfinite(array)) or float(np.max(np.abs(array))) > FLOAT16_MAX_ABS:
1289
+ return "latents_not_float16", {}
1290
+ return None
1291
+
1292
+
1293
+ def _render_report(m: Mapping[str, object]) -> str:
1294
+ selected, kept = m["selected"], m["kept"]
1295
+ assert isinstance(selected, dict) and isinstance(kept, dict)
1296
+ assert isinstance(m["splits"], dict) and isinstance(m["quarantine"], dict)
1297
+ lines = [
1298
+ f"# data-import: {m['source']}",
1299
+ "",
1300
+ f"- source ids: {m['source_ids']}",
1301
+ f"- revision: `{m['revision']}`",
1302
+ f"- config digest: `{m['config_digest']}`",
1303
+ f"- max hours: {m['max_hours']}; work units planned: {m['units_planned']}",
1304
+ f"- selected: {selected['segments']} segments, {selected['hours']} h",
1305
+ f"- kept (in the dataset): {kept['segments']} segments, {kept['hours']} h",
1306
+ f"- groups: {m['groups']}",
1307
+ f"- alignment (timings): {json.dumps(m['alignment'])}",
1308
+ f"- dataset: `{m['dataset']}`",
1309
+ "",
1310
+ "## Splits",
1311
+ "",
1312
+ *(f"- {name}: {json.dumps(info)}" for name, info in m["splits"].items()),
1313
+ "",
1314
+ "## Emotions",
1315
+ "",
1316
+ f"- {json.dumps(m['emotions'], ensure_ascii=False)}",
1317
+ f"- unmapped source labels (-> null): {json.dumps(m['unmapped_emotion_labels'])}",
1318
+ "",
1319
+ "## Quarantine by stage and reason",
1320
+ "",
1321
+ *(
1322
+ f"- {stage}: {json.dumps(counts, ensure_ascii=False)}"
1323
+ for stage, counts in m["quarantine"].items()
1324
+ ),
1325
+ ]
1326
+ return "\n".join(lines) + "\n"
src/nida/data/importer/sources.py ADDED
@@ -0,0 +1,776 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Source adapters of the importer: which corpus items exist, which are selected, where the
2
+ raw audio is.
3
+
4
+ An adapter splits its corpus into *work units* (a parquet shard, a chunk of manifest rows) in a
5
+ deterministic order and, per unit, yields the selected :class:`Item` objects (metadata filters
6
+ and transcript normalisation applied, raw audio placed in the local cache) and a
7
+ :class:`~nida.data.types.QuarantineRecord` for every row it drops, with a machine-readable
8
+ reason. It does not look at audio samples; the stages do.
9
+
10
+ * :class:`AlaniaAdapter` - HF parquet shards (``alania_synth_tr``), filtered by the QC columns.
11
+ * :class:`TrFullAdapter` - HF ``data.jsonl`` + wavs (``tr_full_dataset``).
12
+ * :class:`WavdirAdapter` - a folder with ``manifest.jsonl`` (our own synthetic speech).
13
+
14
+ The archive and Common Voice corpora (``issai_tsc``, ``mediaspeech_tr``, ``common_voice_tr``)
15
+ are in :mod:`nida.data.importer.corpora`; :func:`make_adapter` builds all six.
16
+
17
+ Item ids: ``segment_id = sha1(source_id U+001F item_id)[:16]`` (:func:`item_segment_id`);
18
+ quarantine records use the segment id as ``item_id`` and keep the source's own id in
19
+ ``details["item_id"]``.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import fnmatch
25
+ import hashlib
26
+ import json
27
+ import math
28
+ import re
29
+ from collections.abc import Generator, Mapping, Set
30
+ from dataclasses import dataclass, field
31
+ from pathlib import Path, PurePosixPath
32
+ from typing import TYPE_CHECKING, Protocol
33
+ from urllib.parse import unquote
34
+
35
+ from nida.bench import fetch
36
+ from nida.data.importer.config import CORPUS_SOURCES, LEGACY_SOURCES, ImportConfig
37
+ from nida.data.importer.hub import HubClient, HubError
38
+ from nida.data.importer.text import (
39
+ DETAIL_TEXT_CHARS,
40
+ TextRejected,
41
+ map_emotion,
42
+ normalise_transcript,
43
+ )
44
+ from nida.data.types import QuarantineRecord
45
+ from nida.text.turkish import nfc
46
+ from nida.utils.io import atomic_write_bytes
47
+
48
+ if TYPE_CHECKING:
49
+ import pyarrow as pa # type: ignore[import-untyped] # no stubs
50
+
51
+ __all__ = [
52
+ "ALANIA_SOURCE_ID",
53
+ "HUB_SOURCES",
54
+ "LOCAL_SOURCES",
55
+ "STAGE",
56
+ "TR_FULL_SOURCE_ID",
57
+ "AlaniaAdapter",
58
+ "Item",
59
+ "SourceAdapter",
60
+ "TrFullAdapter",
61
+ "UnitListing",
62
+ "WavdirAdapter",
63
+ "item_segment_id",
64
+ "make_adapter",
65
+ "safe_relative_path",
66
+ ]
67
+
68
+ STAGE = "select"
69
+ ALANIA_SOURCE_ID = "alania_synth_tr"
70
+ TR_FULL_SOURCE_ID = "tr_full_dataset"
71
+ SOURCES = (*LEGACY_SOURCES, *CORPUS_SOURCES)
72
+ #: Sources read from a Hugging Face dataset repository (they need a ``HubClient``).
73
+ HUB_SOURCES = ("alania", "tr_full", "issai_tsc")
74
+ #: Sources read from a local folder or tarball given as ``--source-dir`` (no Hub revision;
75
+ #: ``plan.json`` pins the sha1 of their manifest / validated file).
76
+ LOCAL_SOURCES = ("wavdir", "common_voice_tr")
77
+
78
+ #: Parquet columns the alania adapter reads.
79
+ ALANIA_COLUMNS = (
80
+ "id",
81
+ "audio",
82
+ "text_spoken",
83
+ "duration",
84
+ "voice_id",
85
+ "gender",
86
+ "channel_preset",
87
+ "qc_cer",
88
+ "qc_utmos",
89
+ "license",
90
+ )
91
+ _CHUNK_RE = re.compile(r"chunk-(\d{5})")
92
+ _DRIVE_RE = re.compile(r"[A-Za-z]:")
93
+
94
+ Scalar = str | int | float | None
95
+
96
+
97
+ def item_segment_id(source_id: str, item_id: str) -> str:
98
+ """Content-addressed id of an item: ``sha1(nfc(source_id) U+001F nfc(item_id))[:16]``.
99
+
100
+ Raises:
101
+ ValueError: an empty id, or one containing U+001F (the separator).
102
+ """
103
+ source_id, item_id = nfc(source_id), nfc(item_id)
104
+ for name, value in (("source_id", source_id), ("item_id", item_id)):
105
+ if not value or "\x1f" in value:
106
+ raise ValueError(f"invalid {name} {value!r}")
107
+ return hashlib.sha1(f"{source_id}\x1f{item_id}".encode()).hexdigest()[:16]
108
+
109
+
110
+ def safe_relative_path(value: object) -> str | None:
111
+ """``value`` if it is a plain relative ``/``-separated path inside a folder, else ``None``
112
+ (not a string, empty, absolute, drive letter, backslash, ``..`` component)."""
113
+ if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
114
+ return None
115
+ path = PurePosixPath(value)
116
+ if path.is_absolute() or _DRIVE_RE.match(value) or ".." in path.parts:
117
+ return None
118
+ return nfc(value)
119
+
120
+
121
+ _HUB_RESOLVE_RE = re.compile(r"^https://huggingface\.co/datasets/[^/]+/[^/]+/resolve/[^/]+/(.+)$")
122
+
123
+
124
+ def hub_url_to_path(value: object) -> object:
125
+ """A Hub ``.../datasets/<owner>/<name>/resolve/<rev>/<path>`` URL -> ``<path>``
126
+ (URL-decoded); anything else is returned unchanged.
127
+
128
+ tr_full's ``data.jsonl`` names its audio by full resolve URLs of the dataset it was
129
+ copied from (Codyfederer/tr-full-dataset); the same relative paths exist in the pinned
130
+ repository, so only the path part is used.
131
+ """
132
+ if isinstance(value, str):
133
+ match = _HUB_RESOLVE_RE.match(value)
134
+ if match:
135
+ return unquote(match.group(1))
136
+ return value
137
+
138
+
139
+ @dataclass(frozen=True)
140
+ class Item:
141
+ """One selected corpus item (a line of ``select/manifest.jsonl``).
142
+
143
+ Attributes:
144
+ source_id: ``configs/sources.yaml`` id of the item's source.
145
+ item_id: The source's own id (alania ``id``, tr_full wav path, wavdir ``file``).
146
+ segment_id: :func:`item_segment_id` of the two.
147
+ unit_id: Work unit the item came from (needed to fetch its raw audio again).
148
+ text: Model text (:func:`~nida.data.importer.text.normalise_transcript`).
149
+ group: Split group (speaker / voice), prefixed per source in the config.
150
+ emotion: Emotion id or ``None``.
151
+ raw_duration_ms: Duration of the raw audio from its header; ``None`` until the
152
+ select stage has measured it.
153
+ extra: JSON scalars worth keeping (QC scores, raw emotion label, ...).
154
+ """
155
+
156
+ source_id: str
157
+ item_id: str
158
+ segment_id: str
159
+ unit_id: str
160
+ text: str
161
+ group: str
162
+ emotion: str | None
163
+ raw_duration_ms: int | None = None
164
+ extra: Mapping[str, Scalar] = field(default_factory=dict)
165
+
166
+ def to_obj(self) -> dict[str, object]:
167
+ return {
168
+ "source_id": self.source_id,
169
+ "item_id": self.item_id,
170
+ "segment_id": self.segment_id,
171
+ "unit_id": self.unit_id,
172
+ "text": self.text,
173
+ "group": self.group,
174
+ "emotion": self.emotion,
175
+ "raw_duration_ms": self.raw_duration_ms,
176
+ "extra": dict(self.extra),
177
+ }
178
+
179
+ @classmethod
180
+ def from_obj(cls, obj: object, where: str) -> Item:
181
+ """Inverse of :meth:`to_obj`.
182
+
183
+ Raises:
184
+ ValueError: on a missing/extra key or a wrong type (the message names ``where``).
185
+ """
186
+ keys = {
187
+ "source_id",
188
+ "item_id",
189
+ "segment_id",
190
+ "unit_id",
191
+ "text",
192
+ "group",
193
+ "emotion",
194
+ "raw_duration_ms",
195
+ "extra",
196
+ }
197
+ if not isinstance(obj, dict) or set(obj) != keys:
198
+ raise ValueError(f"{where}: an item needs exactly the keys {sorted(keys)}")
199
+ for key in ("source_id", "item_id", "segment_id", "unit_id", "text", "group"):
200
+ if not isinstance(obj[key], str) or not obj[key]:
201
+ raise ValueError(f"{where}: {key} must be a non-empty string")
202
+ emotion, duration, extra = obj["emotion"], obj["raw_duration_ms"], obj["extra"]
203
+ if emotion is not None and not isinstance(emotion, str):
204
+ raise ValueError(f"{where}: emotion must be a string or null")
205
+ if duration is not None and (isinstance(duration, bool) or not isinstance(duration, int)):
206
+ raise ValueError(f"{where}: raw_duration_ms must be an int or null")
207
+ if not isinstance(extra, dict) or not all(
208
+ isinstance(k, str) and (v is None or isinstance(v, str | int | float))
209
+ for k, v in extra.items()
210
+ ):
211
+ raise ValueError(f"{where}: extra must map strings to JSON scalars")
212
+ return cls(
213
+ source_id=obj["source_id"],
214
+ item_id=obj["item_id"],
215
+ segment_id=obj["segment_id"],
216
+ unit_id=obj["unit_id"],
217
+ text=obj["text"],
218
+ group=obj["group"],
219
+ emotion=emotion,
220
+ raw_duration_ms=duration,
221
+ extra=extra,
222
+ )
223
+
224
+
225
+ @dataclass(frozen=True)
226
+ class UnitListing:
227
+ """All work units of a corpus at one pinned state.
228
+
229
+ Attributes:
230
+ revision: Pinned commit sha (Hub sources) or ``None`` (wavdir).
231
+ unit_ids: Every unit id in processing order.
232
+ content_sha1: sha1 of the wavdir ``manifest.jsonl`` (``None`` for Hub sources).
233
+ """
234
+
235
+ revision: str | None
236
+ unit_ids: tuple[str, ...]
237
+ content_sha1: str | None = None
238
+
239
+
240
+ class SourceAdapter(Protocol):
241
+ """What the select stage needs from a corpus (see the module docstring)."""
242
+
243
+ #: ``True`` if the raw audio is a cache copy the audio stage may delete after conversion.
244
+ ephemeral_raw: bool
245
+
246
+ @property
247
+ def source_ids(self) -> frozenset[str]:
248
+ """``sources.yaml`` ids this run reads (must all be ``approved``)."""
249
+ ...
250
+
251
+ def list_units(self) -> UnitListing:
252
+ """Pin the corpus state (network access for Hub sources) and list its units."""
253
+ ...
254
+
255
+ def iter_unit(
256
+ self,
257
+ unit_id: str,
258
+ only: Set[str] | None = None,
259
+ exclude: Mapping[str, str] | None = None,
260
+ ) -> Generator[Item | QuarantineRecord, None, None]:
261
+ """Selected items and dropped rows of one unit; raw audio is placed in the cache before
262
+ an item is yielded. With ``only`` (segment ids) just those items are yielded again
263
+ (to restore lost raw audio); everything else is skipped without a record.
264
+
265
+ ``exclude`` maps the segment ids selected by earlier units to their unit id. A row
266
+ whose segment id is in it, or was already yielded by this unit, is quarantined as
267
+ ``duplicate_item`` *before* any raw audio is written, so the first item's audio is
268
+ never overwritten."""
269
+ ...
270
+
271
+ def raw_path(self, item: Item) -> Path:
272
+ """Where the raw audio of ``item`` is (or will be put by :meth:`iter_unit`)."""
273
+ ...
274
+
275
+
276
+ def _drop(segment_id: str, item_id: str, reason: str, **details: object) -> QuarantineRecord:
277
+ return QuarantineRecord(segment_id, STAGE, reason, {"item_id": item_id, **details})
278
+
279
+
280
+ def _bad_row(
281
+ unit_id: str, position: int, raw: str, reason: str, **details: object
282
+ ) -> QuarantineRecord:
283
+ """Quarantine record of a row that has no usable id: the key is derived from the unit id
284
+ and the row's position in the (pinned, immutable) unit."""
285
+ key = hashlib.sha1(f"{unit_id}\x1f{position}".encode()).hexdigest()[:16]
286
+ return _drop(key, f"{unit_id}#{position}", reason, row=raw[:DETAIL_TEXT_CHARS], **details)
287
+
288
+
289
+ def _first_seen(
290
+ first: dict[str, str], sid: str, item_id: str, unit_id: str
291
+ ) -> QuarantineRecord | None:
292
+ """Record ``sid`` as selected by ``unit_id``, or return the ``duplicate_item`` record if
293
+ an earlier row (of this or an earlier unit) already claimed it."""
294
+ earlier = first.get(sid)
295
+ if earlier is not None:
296
+ return _drop(sid, item_id, "duplicate_item", unit_id=unit_id, first_unit_id=earlier)
297
+ first[sid] = unit_id
298
+ return None
299
+
300
+
301
+ def _finite(value: object) -> float | None:
302
+ """``value`` as a finite float, else ``None`` (missing, non-numeric, NaN, inf)."""
303
+ if isinstance(value, bool) or not isinstance(value, int | float):
304
+ return None
305
+ return float(value) if math.isfinite(value) else None
306
+
307
+
308
+ def _text_drop(segment_id: str, item_id: str, err: TextRejected) -> QuarantineRecord:
309
+ return _drop(segment_id, item_id, err.reason, **err.details)
310
+
311
+
312
+ # ------------------------------------------------------------------------------ alania
313
+
314
+
315
+ class AlaniaAdapter:
316
+ """``cloud0day3/alania-synthetic-speech-tr``, config ``cc-by`` only (parquet shards).
317
+
318
+ Units are the parquet files matching ``alania.parquet_glob``, sorted by name; rows are
319
+ read in file order. One shard at a time is downloaded, the raw FLAC bytes of the selected
320
+ rows are written to ``<cache>/raw/alania/<segment_id>.flac`` and the parquet file is deleted
321
+ when the unit is finished or abandoned.
322
+ """
323
+
324
+ ephemeral_raw = True
325
+
326
+ def __init__(
327
+ self, cfg: ImportConfig, cache: Path, hub: HubClient, revision: str | None
328
+ ) -> None:
329
+ self._cfg = cfg
330
+ self._cache = cache
331
+ self._hub = hub
332
+ self.revision = revision
333
+
334
+ @property
335
+ def source_ids(self) -> frozenset[str]:
336
+ return frozenset({ALANIA_SOURCE_ID})
337
+
338
+ def list_units(self) -> UnitListing:
339
+ repo = self._cfg.alania.repo_id
340
+ if self.revision is None:
341
+ self.revision = self._hub.resolve_revision(repo)
342
+ files = self._hub.list_files(repo, self.revision)
343
+ units = sorted(f for f in files if fnmatch.fnmatchcase(f, self._cfg.alania.parquet_glob))
344
+ if not units:
345
+ raise HubError(
346
+ f"{repo}@{self.revision}: no file matches {self._cfg.alania.parquet_glob!r}; "
347
+ f"first files: {sorted(files)[:5]}"
348
+ )
349
+ return UnitListing(self.revision, tuple(units))
350
+
351
+ def raw_path(self, item: Item) -> Path:
352
+ return self._cache / "raw" / "alania" / f"{item.segment_id}.flac"
353
+
354
+ def iter_unit(
355
+ self,
356
+ unit_id: str,
357
+ only: Set[str] | None = None,
358
+ exclude: Mapping[str, str] | None = None,
359
+ ) -> Generator[Item | QuarantineRecord, None, None]:
360
+ import pyarrow.parquet as pq # type: ignore[import-untyped] # no stubs
361
+
362
+ if self.revision is None:
363
+ raise HubError("alania: the revision is not pinned; call list_units() first")
364
+ path = self._hub.download(
365
+ self._cfg.alania.repo_id, self.revision, unit_id, self._cache / "hub" / "alania"
366
+ )
367
+ try:
368
+ with pq.ParquetFile(path) as shard:
369
+ missing = [c for c in ALANIA_COLUMNS if c not in shard.schema_arrow.names]
370
+ if missing:
371
+ raise ValueError(f"{unit_id}: parquet columns missing: {missing}")
372
+ position = 0
373
+ first = dict(exclude or {})
374
+ for batch in shard.iter_batches(
375
+ batch_size=self._cfg.alania.batch_rows, columns=list(ALANIA_COLUMNS)
376
+ ):
377
+ for row in _rows(batch):
378
+ event = self._row(unit_id, position, row, only, first)
379
+ position += 1
380
+ if event is not None:
381
+ yield event
382
+ finally:
383
+ path.unlink(missing_ok=True)
384
+
385
+ def _row(
386
+ self,
387
+ unit_id: str,
388
+ position: int,
389
+ row: Mapping[str, object],
390
+ only: Set[str] | None,
391
+ first: dict[str, str],
392
+ ) -> Item | QuarantineRecord | None:
393
+ cfg = self._cfg
394
+ alania = cfg.alania
395
+ item_id = row["id"]
396
+ if not isinstance(item_id, str) or not item_id.strip() or "\x1f" in item_id:
397
+ return _bad_row(unit_id, position, repr(item_id), "bad_row", problem="id")
398
+ sid = item_segment_id(ALANIA_SOURCE_ID, item_id)
399
+ if only is not None and sid not in only:
400
+ return None
401
+ license_ = row["license"]
402
+ if not isinstance(license_, str) or not license_.startswith(alania.license_prefix):
403
+ return _drop(sid, item_id, "license", license=license_)
404
+ utmos, cer = _finite(row["qc_utmos"]), _finite(row["qc_cer"])
405
+ if utmos is None or cer is None:
406
+ return _drop(sid, item_id, "missing_qc", qc_utmos=utmos, qc_cer=cer)
407
+ if utmos < alania.min_utmos:
408
+ return _drop(sid, item_id, "low_utmos", qc_utmos=utmos)
409
+ if cer > alania.max_cer:
410
+ return _drop(sid, item_id, "high_cer", qc_cer=cer)
411
+ seconds = _finite(row["duration"])
412
+ if seconds is None:
413
+ return _drop(sid, item_id, "bad_duration", duration=repr(row["duration"]))
414
+ if seconds * 1000 < cfg.min_duration_ms:
415
+ return _drop(sid, item_id, "too_short", duration_s=seconds)
416
+ if seconds * 1000 > cfg.max_duration_ms:
417
+ return _drop(sid, item_id, "too_long", duration_s=seconds)
418
+ voice = row["voice_id"]
419
+ if not isinstance(voice, str | int) or isinstance(voice, bool) or not str(voice).strip():
420
+ return _drop(sid, item_id, "no_group", voice_id=repr(voice))
421
+ spoken = row["text_spoken"]
422
+ try:
423
+ text = normalise_transcript(spoken if isinstance(spoken, str) else "", cfg.char_map)
424
+ except TextRejected as err:
425
+ return _text_drop(sid, item_id, err)
426
+ audio = row["audio"]
427
+ data = audio.get("bytes") if isinstance(audio, dict) else None
428
+ if not isinstance(data, bytes) or not data:
429
+ return _drop(sid, item_id, "audio_missing")
430
+ item = Item(
431
+ source_id=ALANIA_SOURCE_ID,
432
+ item_id=item_id,
433
+ segment_id=sid,
434
+ unit_id=unit_id,
435
+ text=text,
436
+ group=nfc(alania.group_prefix + str(voice).strip()),
437
+ emotion=None,
438
+ extra={
439
+ "voice_id": str(voice),
440
+ "gender": _scalar(row["gender"]),
441
+ "channel_preset": _scalar(row["channel_preset"]),
442
+ "qc_utmos": utmos,
443
+ "qc_cer": cer,
444
+ },
445
+ )
446
+ duplicate = _first_seen(first, sid, item_id, unit_id)
447
+ if duplicate is not None:
448
+ return duplicate
449
+ atomic_write_bytes(self.raw_path(item), data)
450
+ return item
451
+
452
+
453
+ def _scalar(value: object) -> Scalar:
454
+ """``value`` if it is a JSON scalar, else its ``str`` (QC metadata only)."""
455
+ if value is None or (isinstance(value, str | int | float) and not isinstance(value, bool)):
456
+ return value
457
+ return str(value)
458
+
459
+
460
+ def _rows(batch: pa.RecordBatch) -> list[dict[str, object]]:
461
+ rows: list[dict[str, object]] = batch.to_pylist()
462
+ return rows
463
+
464
+
465
+ # ------------------------------------------------------------------------------ chunks
466
+
467
+
468
+ def _chunk_ids(num_rows: int, unit_rows: int) -> tuple[str, ...]:
469
+ count = -(-num_rows // unit_rows)
470
+ return tuple(f"chunk-{k:05d}" for k in range(count))
471
+
472
+
473
+ def _chunk_slice(unit_id: str, unit_rows: int) -> slice:
474
+ match = _CHUNK_RE.fullmatch(unit_id)
475
+ if match is None:
476
+ raise ValueError(f"not a chunk unit id: {unit_id!r}")
477
+ start = int(match.group(1)) * unit_rows
478
+ return slice(start, start + unit_rows)
479
+
480
+
481
+ def _json_lines(path: Path) -> list[str]:
482
+ """Non-empty lines of a JSONL file (split on ``\\n`` only: U+2028 stays inside strings)."""
483
+ return [line for line in path.read_text(encoding="utf-8").split("\n") if line.strip()]
484
+
485
+
486
+ def _parse_object(line: str) -> dict[str, object] | None:
487
+ try:
488
+ obj = json.loads(line)
489
+ except json.JSONDecodeError:
490
+ return None
491
+ return obj if isinstance(obj, dict) else None
492
+
493
+
494
+ # ------------------------------------------------------------------------------ tr_full
495
+
496
+
497
+ class TrFullAdapter:
498
+ """``Taklaxbr/tr-full-dataset``: ``data.jsonl`` rows and the wavs they name.
499
+
500
+ Units are chunks of ``tr_full.unit_rows`` rows in file order. A row whose ``language`` is
501
+ not ``tr_full.language`` is quarantined (``language``) before its wav is fetched;
502
+ the wavs of the remaining rows of a unit are downloaded one by one (threaded) into
503
+ ``<cache>/raw/tr_full/``.
504
+ """
505
+
506
+ ephemeral_raw = True
507
+
508
+ def __init__(
509
+ self, cfg: ImportConfig, cache: Path, hub: HubClient, revision: str | None
510
+ ) -> None:
511
+ self._cfg = cfg
512
+ self._cache = cache
513
+ self._hub = hub
514
+ self.revision = revision
515
+ self._lines_cache: list[str] | None = None
516
+
517
+ @property
518
+ def source_ids(self) -> frozenset[str]:
519
+ return frozenset({TR_FULL_SOURCE_ID})
520
+
521
+ def _data_lines(self) -> list[str]:
522
+ """The non-empty lines of ``data.jsonl`` (downloaded and read once per adapter)."""
523
+ if self._lines_cache is None:
524
+ path = self._hub.download(
525
+ self._cfg.tr_full.repo_id,
526
+ self._revision(),
527
+ self._cfg.tr_full.data_file,
528
+ self._cache / "hub" / "tr_full",
529
+ )
530
+ self._lines_cache = _json_lines(path)
531
+ return self._lines_cache
532
+
533
+ def list_units(self) -> UnitListing:
534
+ if self.revision is None:
535
+ self.revision = self._hub.resolve_revision(self._cfg.tr_full.repo_id)
536
+ lines = self._data_lines()
537
+ return UnitListing(self.revision, _chunk_ids(len(lines), self._cfg.tr_full.unit_rows))
538
+
539
+ def raw_path(self, item: Item) -> Path:
540
+ return self._cache / "raw" / "tr_full" / item.item_id
541
+
542
+ def iter_unit(
543
+ self,
544
+ unit_id: str,
545
+ only: Set[str] | None = None,
546
+ exclude: Mapping[str, str] | None = None,
547
+ ) -> Generator[Item | QuarantineRecord, None, None]:
548
+ cfg = self._cfg
549
+ window = _chunk_slice(unit_id, cfg.tr_full.unit_rows)
550
+ lines = self._data_lines()[window]
551
+ base = window.start
552
+ candidates: list[Item] = []
553
+ first = dict(exclude or {})
554
+ for offset, line in enumerate(lines):
555
+ position = base + offset
556
+ row = _parse_object(line)
557
+ if row is None:
558
+ yield _bad_row(unit_id, position, line, "bad_row", problem="not a JSON object")
559
+ continue
560
+ made = self._row(unit_id, position, line, row, only, first)
561
+ if isinstance(made, QuarantineRecord):
562
+ yield made
563
+ elif made is not None:
564
+ candidates.append(made)
565
+ wanted = [c.item_id for c in candidates if not self.raw_path(c).is_file()]
566
+ self._hub.download_many(
567
+ cfg.tr_full.repo_id,
568
+ self._revision(),
569
+ wanted,
570
+ self._cache / "raw" / "tr_full",
571
+ )
572
+ for item in candidates:
573
+ if self.raw_path(item).is_file():
574
+ yield item
575
+ else:
576
+ yield _drop(item.segment_id, item.item_id, "audio_missing")
577
+
578
+ def _revision(self) -> str:
579
+ if self.revision is None:
580
+ raise HubError("tr_full: the revision is not pinned; call list_units() first")
581
+ return self.revision
582
+
583
+ def _row(
584
+ self,
585
+ unit_id: str,
586
+ position: int,
587
+ line: str,
588
+ row: Mapping[str, object],
589
+ only: Set[str] | None,
590
+ first: dict[str, str],
591
+ ) -> Item | QuarantineRecord | None:
592
+ cfg = self._cfg
593
+ rel = safe_relative_path(hub_url_to_path(row.get("audio")))
594
+ if rel is None:
595
+ return _bad_row(unit_id, position, line, "bad_row", problem="audio path")
596
+ sid = item_segment_id(TR_FULL_SOURCE_ID, rel)
597
+ if only is not None and sid not in only:
598
+ return None
599
+ language = row.get("language")
600
+ if not isinstance(language, str) or language.strip() != cfg.tr_full.language:
601
+ return _drop(sid, rel, "language", language=language)
602
+ speaker, text_raw = row.get("speaker_id"), row.get("text")
603
+ if (
604
+ not isinstance(speaker, str | int)
605
+ or isinstance(speaker, bool)
606
+ or not str(speaker).strip()
607
+ ):
608
+ return _drop(sid, rel, "no_group", speaker_id=repr(speaker))
609
+ try:
610
+ text = normalise_transcript(text_raw if isinstance(text_raw, str) else "", cfg.char_map)
611
+ except TextRejected as err:
612
+ return _text_drop(sid, rel, err)
613
+ label = row.get("emotion")
614
+ emotion = map_emotion(label, cfg.emotion_map)
615
+ duplicate = _first_seen(first, sid, rel, unit_id)
616
+ if duplicate is not None:
617
+ return duplicate
618
+ return Item(
619
+ source_id=TR_FULL_SOURCE_ID,
620
+ item_id=rel,
621
+ segment_id=sid,
622
+ unit_id=unit_id,
623
+ text=text,
624
+ group=nfc(cfg.tr_full.group_prefix + str(speaker).strip()),
625
+ emotion=emotion,
626
+ extra={"speaker_id": str(speaker), "emotion_label": _scalar(label)},
627
+ )
628
+
629
+
630
+ # ------------------------------------------------------------------------------ wavdir
631
+
632
+
633
+ class WavdirAdapter:
634
+ """A folder with ``manifest.jsonl`` ``{file, text, group, source, emotion?}`` lines.
635
+
636
+ ``file`` is relative to the folder; ``source`` is the ``sources.yaml`` id of whatever
637
+ produced the audio (it must be ``approved``; :attr:`source_ids` lists every one named in
638
+ the manifest). Units are chunks of ``wavdir.unit_rows`` lines. The wavs stay where they
639
+ are: they are read in place and never deleted.
640
+ """
641
+
642
+ ephemeral_raw = False
643
+
644
+ def __init__(self, cfg: ImportConfig, folder: Path) -> None:
645
+ self._cfg = cfg
646
+ self._folder = folder
647
+ self._manifest = folder / "manifest.jsonl"
648
+ self._lines_cache: list[str] | None = None
649
+
650
+ def _lines(self) -> list[str]:
651
+ if self._lines_cache is None:
652
+ if not self._manifest.is_file():
653
+ raise FileNotFoundError(f"{self._manifest} does not exist")
654
+ self._lines_cache = _json_lines(self._manifest)
655
+ return self._lines_cache
656
+
657
+ @property
658
+ def source_ids(self) -> frozenset[str]:
659
+ found: set[str] = set()
660
+ for line in self._lines():
661
+ row = _parse_object(line)
662
+ source = row.get("source") if row is not None else None
663
+ if isinstance(source, str) and source.strip():
664
+ found.add(source.strip())
665
+ return frozenset(found)
666
+
667
+ def list_units(self) -> UnitListing:
668
+ digest = hashlib.sha1(self._manifest.read_bytes()).hexdigest()
669
+ return UnitListing(None, _chunk_ids(len(self._lines()), self._cfg.wavdir.unit_rows), digest)
670
+
671
+ def raw_path(self, item: Item) -> Path:
672
+ return self._folder / item.item_id
673
+
674
+ def iter_unit(
675
+ self,
676
+ unit_id: str,
677
+ only: Set[str] | None = None,
678
+ exclude: Mapping[str, str] | None = None,
679
+ ) -> Generator[Item | QuarantineRecord, None, None]:
680
+ window = _chunk_slice(unit_id, self._cfg.wavdir.unit_rows)
681
+ first = dict(exclude or {})
682
+ for offset, line in enumerate(self._lines()[window]):
683
+ event = self._row(unit_id, window.start + offset, line, only, first)
684
+ if event is not None:
685
+ yield event
686
+
687
+ def _row(
688
+ self,
689
+ unit_id: str,
690
+ position: int,
691
+ line: str,
692
+ only: Set[str] | None,
693
+ first: dict[str, str],
694
+ ) -> Item | QuarantineRecord | None:
695
+ cfg = self._cfg
696
+ row = _parse_object(line)
697
+ if row is None:
698
+ return _bad_row(unit_id, position, line, "bad_row", problem="not a JSON object")
699
+ rel = safe_relative_path(row.get("file"))
700
+ source, group, text_raw = row.get("source"), row.get("group"), row.get("text")
701
+ if rel is None or not isinstance(source, str) or not source.strip() or "\x1f" in source:
702
+ return _bad_row(unit_id, position, line, "bad_row", problem="file or source")
703
+ source = source.strip()
704
+ sid = item_segment_id(source, rel)
705
+ if only is not None and sid not in only:
706
+ return None
707
+ if not isinstance(group, str) or not group.strip():
708
+ return _drop(sid, rel, "no_group")
709
+ try:
710
+ text = normalise_transcript(text_raw if isinstance(text_raw, str) else "", cfg.char_map)
711
+ except TextRejected as err:
712
+ return _text_drop(sid, rel, err)
713
+ if not (self._folder / rel).is_file():
714
+ return _drop(sid, rel, "audio_missing")
715
+ duplicate = _first_seen(first, sid, rel, unit_id)
716
+ if duplicate is not None:
717
+ return duplicate
718
+ label = row.get("emotion")
719
+ return Item(
720
+ source_id=source,
721
+ item_id=rel,
722
+ segment_id=sid,
723
+ unit_id=unit_id,
724
+ text=text,
725
+ group=nfc(cfg.wavdir.group_prefix + group.strip()),
726
+ emotion=map_emotion(label, cfg.emotion_map),
727
+ extra={"emotion_label": _scalar(label)},
728
+ )
729
+
730
+
731
+ def make_adapter(
732
+ source: str,
733
+ cfg: ImportConfig,
734
+ cache: Path,
735
+ *,
736
+ hub: HubClient | None,
737
+ revision: str | None,
738
+ wavdir: Path | None,
739
+ opener: fetch.Opener | None = None,
740
+ ) -> SourceAdapter:
741
+ """The adapter of ``source`` (one of :data:`SOURCES`).
742
+
743
+ Args:
744
+ source: Importer source name.
745
+ cfg: Import config (loaded for that source).
746
+ cache: Local scratch directory.
747
+ hub: Hub client (``alania``, ``tr_full``, ``issai_tsc``).
748
+ revision: Pin: a Hub commit sha, or the sha256 of the ``mediaspeech_tr`` archive.
749
+ wavdir: ``--source-dir`` (``wavdir``: its folder; ``common_voice_tr``: a Common Voice
750
+ release folder or tarball).
751
+ opener: HTTP access of ``mediaspeech_tr`` (default: ``urllib``); tests inject one.
752
+
753
+ Raises:
754
+ ValueError: unknown source, a Hub source without a ``hub``, a local source without
755
+ ``wavdir``.
756
+ """
757
+ if source in HUB_SOURCES and hub is None:
758
+ raise ValueError(f"source {source!r} needs a Hub client")
759
+ if source in LOCAL_SOURCES and wavdir is None:
760
+ raise ValueError(f"source {source!r} needs --source-dir")
761
+ if source == "alania" and hub is not None:
762
+ return AlaniaAdapter(cfg, cache, hub, revision)
763
+ if source == "tr_full" and hub is not None:
764
+ return TrFullAdapter(cfg, cache, hub, revision)
765
+ if source == "wavdir" and wavdir is not None:
766
+ return WavdirAdapter(cfg, wavdir)
767
+ if source in CORPUS_SOURCES:
768
+ from nida.data.importer import corpora # imports this module: not at module level
769
+
770
+ if source == "issai_tsc" and hub is not None:
771
+ return corpora.IssaiTscAdapter(cfg, cache, hub, revision)
772
+ if source == "mediaspeech_tr":
773
+ return corpora.MediaSpeechAdapter(cfg, cache, opener, revision)
774
+ if source == "common_voice_tr" and wavdir is not None:
775
+ return corpora.CommonVoiceAdapter(cfg, cache, wavdir)
776
+ raise ValueError(f"unknown source {source!r}; known: {SOURCES}")
src/nida/data/importer/text.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Transcript and label normalisation for imported corpora.
2
+
3
+ A transcript becomes *model text*: what :func:`nida.model.vocab.encode_text` encodes and what
4
+ the CTC aligner is given. Rules, in order:
5
+
6
+ 1. NFC (:func:`nida.data.pilot.clean_asr_text`, which also undoes ``i`` + U+0307);
7
+ 2. ``char_map`` replacements (typographic quotes and apostrophes);
8
+ 3. text that contains a tag character ``[ ] < >`` is *rejected* (``tag_chars``): the characters
9
+ are reserved for the tag grammar (docs/02-tag-spec.md), and stripping them would turn an
10
+ annotation such as ``[laugh]`` into the spoken word "laugh". Whitespace is collapsed to
11
+ single spaces;
12
+ 4. digits are written out with :func:`nida.text.numbers.verbalize_digits`;
13
+ 5. every remaining character must be in the model vocabulary.
14
+
15
+ Case is kept (the vocabulary lower-cases with ``tr_lower`` itself). Filler sounds are NOT
16
+ canonicalised (tag-spec §4: ``eee`` -> ``ee`` ...): the transcript is used as the corpus
17
+ writes it, a known deviation for imported text.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from collections.abc import Mapping
23
+
24
+ from nida.data.ears import en_lower
25
+ from nida.data.pilot import clean_asr_text
26
+ from nida.model.vocab import VOCAB, encode_text
27
+ from nida.text.numbers import verbalize_digits
28
+ from nida.text.turkish import tr_lower
29
+
30
+ __all__ = [
31
+ "DETAIL_TEXT_CHARS",
32
+ "REASON_EMPTY",
33
+ "REASON_TAG_CHARS",
34
+ "TAG_CHARS",
35
+ "TextRejected",
36
+ "map_emotion",
37
+ "normalise_transcript",
38
+ ]
39
+
40
+ #: Characters reserved for the tag grammar; removed from raw corpus text.
41
+ TAG_CHARS = "[]<>"
42
+ REASON_EMPTY = "empty_text"
43
+ REASON_TAG_CHARS = "tag_chars"
44
+ REASON_NUMBER = "unverbalizable_number"
45
+ REASON_UNTOKENIZABLE = "untokenizable"
46
+
47
+ #: Characters of a transcript / row kept as an excerpt in a quarantine record.
48
+ DETAIL_TEXT_CHARS = 120
49
+
50
+
51
+ class TextRejected(ValueError):
52
+ """A transcript that cannot become model text; ``reason`` is the quarantine reason."""
53
+
54
+ def __init__(self, reason: str, details: Mapping[str, object]) -> None:
55
+ super().__init__(f"{reason}: {dict(details)}")
56
+ self.reason = reason
57
+ self.details = dict(details)
58
+
59
+
60
+ def normalise_transcript(raw: str, char_map: Mapping[str, str]) -> str:
61
+ """Model text of a raw transcript (rules in the module docstring).
62
+
63
+ Args:
64
+ raw: Transcript as stored in the corpus.
65
+ char_map: Single-character replacements (``configs/data/import.yaml``).
66
+
67
+ Raises:
68
+ TextRejected: ``tag_chars`` (``details["chars"]`` lists them); ``empty_text``;
69
+ ``unverbalizable_number`` (a digit token that :func:`verbalize_digits` cannot
70
+ spell); ``untokenizable`` (``details["chars"]`` lists the characters outside the
71
+ vocabulary).
72
+ """
73
+ text = clean_asr_text(raw).translate({ord(k): v for k, v in char_map.items()})
74
+ tags = sorted({ch for ch in text if ch in TAG_CHARS})
75
+ if tags:
76
+ raise TextRejected(
77
+ REASON_TAG_CHARS, {"chars": "".join(tags), "text": text[:DETAIL_TEXT_CHARS]}
78
+ )
79
+ text = " ".join(text.split())
80
+ if not text:
81
+ raise TextRejected(REASON_EMPTY, {})
82
+ if any(ch.isdigit() for ch in text):
83
+ spoken = verbalize_digits(text)
84
+ if spoken is None:
85
+ raise TextRejected(REASON_NUMBER, {"text": text[:DETAIL_TEXT_CHARS]})
86
+ text = " ".join(spoken.split())
87
+ offending = sorted({ch for ch in text if tr_lower(ch) not in VOCAB})
88
+ if offending:
89
+ raise TextRejected(
90
+ REASON_UNTOKENIZABLE,
91
+ {"chars": "".join(offending), "text": text[:DETAIL_TEXT_CHARS]},
92
+ )
93
+ try:
94
+ ids = encode_text(text)
95
+ except ValueError as err: # defence in depth: something other than a character
96
+ raise TextRejected(REASON_UNTOKENIZABLE, {"error": str(err)}) from err
97
+ if not ids:
98
+ raise TextRejected(REASON_EMPTY, {})
99
+ return text
100
+
101
+
102
+ def map_emotion(label: object, emotion_map: Mapping[str, str]) -> str | None:
103
+ """Emotion id of a source label, or ``None`` when absent or not in ``emotion_map``.
104
+
105
+ Labels are English: matched after stripping and ASCII lower-casing (never ``tr_lower``,
106
+ which would turn ``I`` into ``ı``).
107
+ """
108
+ if not isinstance(label, str):
109
+ return None
110
+ return emotion_map.get(en_lower(label.strip()))