diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..510a4c8097384fa7a653dbf25f9f182ce825164d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +voices/ada.wav filter=lfs diff=lfs merge=lfs -text +voices/asli.wav filter=lfs diff=lfs merge=lfs -text +voices/can.wav filter=lfs diff=lfs merge=lfs -text +voices/defne.wav filter=lfs diff=lfs merge=lfs -text +voices/duru.wav filter=lfs diff=lfs merge=lfs -text +voices/efe.wav filter=lfs diff=lfs merge=lfs -text +voices/elif.wav filter=lfs diff=lfs merge=lfs -text +voices/emre.wav filter=lfs diff=lfs merge=lfs -text +voices/haluk.wav filter=lfs diff=lfs merge=lfs -text +voices/leyla.wav filter=lfs diff=lfs merge=lfs -text +voices/mert.wav filter=lfs diff=lfs merge=lfs -text +voices/nermin.wav filter=lfs diff=lfs merge=lfs -text +voices/nihat.wav filter=lfs diff=lfs merge=lfs -text +voices/orhan.wav filter=lfs diff=lfs merge=lfs -text +voices/selim.wav filter=lfs diff=lfs merge=lfs -text +voices/su.wav filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dc6d9835a54f73333904ceb3f586ae841492a041 --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ +--- +license: cc-by-nc-4.0 +language: +- tr +pipeline_tag: text-to-speech +tags: +- text-to-speech +- turkish +- flow-matching +- zero-shot +--- + +# Nida v0.5 — Turkish text-to-speech (research preview) + +**Nida** (*nida*: an exclamation, a call) is an open, non-commercial Turkish text-to-speech model +trained **from scratch**. v0.5 is an early research preview: it reads Turkish clearly and stably, +including numbers, dates and abbreviations, with sixteen bundled voices. Its prosody is still flat +("reads rather than speaks"). Expressive speech (emotion, laughter, sighs) and better voice +cloning are the goals of later versions. + +> Türkçe özet: Nida, sıfırdan eğitilmiş, açık ve ticari olmayan bir Türkçe metinden-sese modelidir. +> v0.5 bir araştırma önizlemesidir: sayı, tarih ve kısaltmalar dahil Türkçeyi net ve kararlı okur, +> 16 hazır sesle gelir. Tonlaması henüz düzdür. Duygu, sözsüz sesler ve daha iyi ses klonlama +> sonraki sürümlerin hedefidir. + +## What it is + +| | | +|---|---| +| Architecture | Diffusion transformer (DiT), flow matching on audio latents, explicit per-character durations with windowed Gaussian alignment | +| Parameters | 201 M | +| Audio latents | VoxCPM2 AudioVAE V2 (25 Hz × 64), decoded to 48 kHz | +| Training data | ~380 h: ~210 h Turkish YouTube speech (YODAS, CC-BY-3.0) + ~170 h synthetic multi-voice Turkish (alania, CC-BY-4.0) | +| Training | 200k steps, single GPU | +| Voices | sixteen bundled voices; a custom 3–8 s reference clip also works but is **experimental** | + +## Measured quality (v0.5) + +- Intelligibility: character error rate of an ASR read-back (Qwen3-ASR) **0.1–1.2 %** on narration + and long-form text, **4.5–7 %** on short assistant sentences (numbers, proper names). +- Speaker similarity with a custom reference clip (experimental; ECAPA-TDNN cosine, generated vs. + reference): **~0.55** (the same real speaker scores ~0.8, different speakers ≤ 0.36; the audio + codec alone keeps 0.955). The voice is recognisably close but not a faithful copy. +- Long-form: stable over 2-minute paragraphs (no drift between sentences in our tests). +- **TR-Zor** (155 hard Turkish sentences: numbers, dates, times, money, units, abbreviations, + acronyms, ordinals; [ahenk-turkish-tts](https://github.com/ozguradmin/ahenk-turkish-tts), + Apache-2.0), judge Whisper large-v3-turbo, word error rate with the set's spoken form as input: + **2.36 %** (voice *deniz*, an earlier voice not in the final set; 141/155 sentences perfect), + **3.15 %** (voice *emre*, 133/155). For + reference, the same set reports Ahenk 2.54 % and Trendyol-TTS 3.98 % (both 2B VoxCPM2 fine-tunes, + measured by their authors with their own text normaliser; our evaluation normaliser and Whisper + backend differ, so treat the comparison as indicative). + +## Known limitations + +- Flat, read-like prosody; little emotion even when requested. +- Custom voices (your own reference clip) are experimental: similarity is moderate and depends a lot + on the clip. The bundled voices are the supported way to use v0.5. +- Input text is normalised automatically (numbers, ordinals, dates, clock times, money, units, + common abbreviations, acronyms spelled with Turkish letter names, phone/ID numbers). Ambiguous + cases (e.g. `8.30` without context, unknown all-caps words) are reported; dangerous ones stop + generation instead of producing wrong speech. `--no-normalize` turns it off. +- Each sentence plus the reference must fit 20 seconds; sentences longer than ~96 spoken + characters are split at a comma, semicolon or colon (never inside quotes or tags). +- Trained mostly on Turkish; other languages are not supported. + +## Bundled voices + +Sixteen synthetic voices ship in `voices/`: `voices.tsv` lists all of them (`\t`), +and `voices/.tsv` selects one. None of them is a recording of a real person. + +| Voice | | Notes | +|---|---|---| +| leyla, duru | female | the cleanest voices; a good default | +| ada, defne, elif, nermin, su, aslı (`asli`) | female | | +| nihat | male | warm and calm | +| emre | male | lively | +| mert | male | sounds like a phone line; suits an assistant | +| can, haluk, selim, orhan, efe | male | | + +can, elif, emre, haluk, nermin, nihat, selim, leyla, defne and ada come from the alania dataset +(`voice_type: designed-identity`, generated with VoxCPM2 from text descriptions). su, aslı, duru, +mert, orhan and efe were designed for this release the same way, with VoxCPM2 voice design from +a text description (no reference audio). + +## Usage + +```bash +pip install -r requirements.txt # torch (CUDA build recommended), transformers, soundfile, ... +pip install -e . # the nida package +nida say --checkpoint nida-v05-ema.pt --prompt-list voices/leyla.tsv \ + --text "Merhaba! Toplantı saat 14:30'da, 3. katta." --out out --single-env --no-asr +``` + +Run it from the release folder: the voice lists use paths relative to the working directory. +The audio is written to `out/ext-*.wav`, and `out/say.html` plays it. `--prompt-list +voices/voices.tsv` reads the text with all sixteen voices; `--text-file` reads a text file. + +Script syntax inside `--text` / `--text-file`: + +- Sentences are split on `. ! ? …`; long ones also at `, ; :`. +- `[duraksama:0.3]` inserts exactly 0.3 s of silence; `[kısa duraksama]` / `[duraksama]` let the + model pause. + +The first run downloads the VoxCPM2 AudioVAE and a Turkish CTC aligner from Hugging Face. Later +runs can skip the Hub's network checks with `HF_HUB_OFFLINE=1` (useful on slow or IPv6-broken +networks). Each `nida say` call loads the models again (about a minute on an RTX 5070); the +synthesis itself takes a few seconds. + +## Related work + +[FreyaTTS](https://arxiv.org/abs/2607.09530) (2026) is a Turkish text-to-speech model with a +similar recipe: a non-autoregressive flow-matching DiT on frozen AudioVAE2 latents with a Turkish +character vocabulary, 183 M parameters. Its numbers are on a different test set and are not +directly comparable with ours. + +## Responsible use + +**Do not use Nida to imitate a real person's voice without their explicit consent**, to deceive, +or to impersonate anyone (including public figures). Generated audio must not be presented as a +real recording of a person. By using the model you agree to these terms in addition to the +CC-BY-NC-4.0 license (non-commercial use only). + +## Licenses and attribution + +- Model weights: CC-BY-NC-4.0. Code: Apache-2.0. +- YODAS (espnet/yodas, CC-BY-3.0), alania synthetic speech (cloud0day3/alania-synthetic-speech-tr, + CC-BY-4.0), VoxCPM2 AudioVAE and voice design (openbmb/VoxCPM2, Apache-2.0), Turkish CTC aligner + (mpoyraz/wav2vec2-xls-r-300m-cv7-turkish, CC-BY-4.0). diff --git a/nida-v05-ema.pt b/nida-v05-ema.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d0aa34b1487deb82cde208d20c7aed76f1e8fc5 --- /dev/null +++ b/nida-v05-ema.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd7ff9d7166f99b7df64a5e4ac30c43bce72868138d80bb97cca4e8ca3b939a3 +size 805276081 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..43d098f1bac8948485d5ec7c5192081a55c37baf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["hatchling>=1.24"] +build-backend = "hatchling.build" + +[project] +name = "nida" +version = "0.5.0" +description = "Nida: expressive Turkish text-to-speech (research, non-commercial weights)" +readme = "README.md" +requires-python = ">=3.11" +# Core carries only the text normaliser base (turkificate, MIT, pure Python). torch is not a +# core dependency: Colab ships its own CUDA build; the `model` extra (ADR-0005) is for local CPU tests only. +dependencies = ["turkificate==0.2.0"] + +[project.optional-dependencies] +# Phase 0 benchmarks (nida.bench); model libraries live only in isolated worker envs. +bench = ["numpy>=2.0", "scipy>=1.13", "soundfile>=0.12", "pyyaml>=6.0"] +# Phase 1 data pipeline (nida.data): Parquet stage manifests. +data = ["nida[bench]", "pyarrow>=17"] +# Phase 2 model code, local CPU tests (ADR-0005): same torch as Colab's runtime (2.11.0). +model = ["torch==2.11.0"] + +[dependency-groups] +# PEP 735 group: uv installs it by default for `uv sync` / `uv run`. +dev = [ + "nida[data,model]", + "pytest>=8.0", + "hypothesis>=6.100", + "ruff>=0.6", + "mypy>=1.10", +] + +[project.scripts] +nida = "nida.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/nida"] + +[tool.ruff] +line-length = 100 +target-version = "py311" +src = ["src", "tests", "lab", "lab/akan"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF", "PTH", "BLE"] +# Turkish dotless "ı" is a real letter here, not a confusable. +allowed-confusables = ["ı", "–", "−", "×"] + +[tool.mypy] +python_version = "3.11" +strict = true +files = ["src/nida", "tests", "lab/akan"] +# lab/akan holds the `akan` package (lab/akan/akan); `lab` is listed for later lab experiments. +mypy_path = ["src", "lab/akan", "lab"] + +[[tool.mypy.overrides]] +# turkificate (text normaliser base, MIT) ships no py.typed marker; nida.text.normalize wraps it. +module = ["turkificate", "turkificate.*"] +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src", "lab/akan", "lab"] +addopts = "-q --strict-markers -m 'not gpu'" +markers = [ + "gpu: requires a CUDA GPU (deselected by default; run with -m gpu)", + "slow: long-running test", +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..325665cb77126178660afde20aefc20fc88f6d20 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# Nida v0.5 inference (`nida say --single-env`). Install torch + torchaudio first, a CUDA +# build for your GPU (https://pytorch.org/get-started/locally/), e.g.: +# pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128 +# then: pip install -r requirements.txt && pip install -e . +transformers==4.57.6 +huggingface-hub +numpy +scipy +soundfile +pydantic +pyyaml diff --git a/src/nida/__init__.py b/src/nida/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..986752ac74803ca3b0b052afbb4a05e072681a5d --- /dev/null +++ b/src/nida/__init__.py @@ -0,0 +1,7 @@ +"""Nida: expressive Turkish text-to-speech.""" + +from __future__ import annotations + +__all__ = ["__version__"] + +__version__ = "0.0.1" diff --git a/src/nida/bench/__init__.py b/src/nida/bench/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a6bb662512734a233d0c582174608370991164fb --- /dev/null +++ b/src/nida/bench/__init__.py @@ -0,0 +1 @@ +"""Phase 0 benchmarks (B1 ASR comparison, B2 VAE reconstruction) and listening panels.""" diff --git a/src/nida/bench/audiometrics.py b/src/nida/bench/audiometrics.py new file mode 100644 index 0000000000000000000000000000000000000000..9a45bec22addfc8b6fcf9aa241717a9988575c8d --- /dev/null +++ b/src/nida/bench/audiometrics.py @@ -0,0 +1,570 @@ +"""Objective audio metrics for comparing an original waveform with a reconstruction (B2). + +All functions are pure and CPU-only (numpy + scipy). Conventions: + +* Waveforms are 1-D float arrays of shape ``(num_samples,)``. Any floating dtype is accepted + and converted to ``float64``; integer and complex dtypes are rejected. +* Sample rates are integers in Hz (``sample_rate_hz``), frequencies are in Hz (``*_hz``), + lengths are in samples (``*_samples``). +* Spectrograms are laid out as ``(frames, bins)``. +* Inputs are validated at the module boundary and violations raise :class:`ValueError` + with the offending argument named. +""" + +from __future__ import annotations + +import math +from typing import TypeAlias + +import numpy as np +import numpy.typing as npt + +# scipy ships no type stubs; the ignore is scoped to this single import. +from scipy import signal # type: ignore[import-untyped] + +__all__ = [ + "FloatArray", + "align", + "band_energy_ratio", + "band_log_spectral_distance", + "band_power_db", + "estimate_delay_samples", + "log_mel_l1", + "mel_filterbank", + "resample", + "stft_magnitude", +] + +FloatArray: TypeAlias = npt.NDArray[np.float64] + +# Relative tolerance under which two cross-correlation values count as tied in +# ``estimate_delay_samples`` (FFT round-off makes exact float ties unreliable). +_TIE_RTOL = 1e-9 + + +# -------------------------------------------------------------------------------------- +# Validation helpers +# -------------------------------------------------------------------------------------- + + +def _as_waveform(x: npt.ArrayLike, name: str) -> FloatArray: + """Validate ``x`` as a non-empty, finite 1-D float waveform and return it as float64.""" + arr = np.asarray(x) + if arr.dtype.kind != "f": + raise ValueError(f"{name} must have a floating dtype, got {arr.dtype}") + if arr.ndim != 1: + raise ValueError(f"{name} must be 1-D, got shape {arr.shape}") + if arr.size == 0: + raise ValueError(f"{name} must not be empty") + out = arr.astype(np.float64) + if not np.isfinite(out).all(): + raise ValueError(f"{name} contains non-finite values (NaN or inf)") + return out + + +def _check_int(value: int, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int | np.integer): + raise ValueError(f"{name} must be an int, got {value!r}") + return int(value) + + +def _check_positive_int(value: int, name: str) -> int: + ivalue = _check_int(value, name) + if ivalue <= 0: + raise ValueError(f"{name} must be > 0, got {ivalue}") + return ivalue + + +def _check_positive_float(value: float, name: str) -> float: + fvalue = float(value) + if not (math.isfinite(fvalue) and fvalue > 0.0): + raise ValueError(f"{name} must be a finite number > 0, got {value!r}") + return fvalue + + +def _check_band(sample_rate_hz: int, low_hz: float, high_hz: float) -> None: + nyquist_hz = sample_rate_hz / 2 + # Written as a negated chain so NaN bounds are rejected too. + if not (0.0 <= low_hz < high_hz <= nyquist_hz): + raise ValueError( + f"frequency band must satisfy 0 <= low_hz < high_hz <= {nyquist_hz} " + f"(sample_rate_hz/2), got low_hz={low_hz}, high_hz={high_hz}" + ) + + +def _trim_to_common_length(ref: FloatArray, est: FloatArray) -> tuple[FloatArray, FloatArray]: + num_samples = min(ref.shape[0], est.shape[0]) + return ref[:num_samples], est[:num_samples] + + +# -------------------------------------------------------------------------------------- +# Resampling +# -------------------------------------------------------------------------------------- + + +def resample(x: npt.ArrayLike, sample_rate_hz: int, target_rate_hz: int) -> FloatArray: + """Resample a waveform with a polyphase filter. + + Uses :func:`scipy.signal.resample_poly` with the up/down factors reduced by their gcd. + When the rates are equal a copy of the input is returned. + + Args: + x: Waveform, shape ``(num_samples,)``, any float dtype. + sample_rate_hz: Source sample rate in Hz (> 0). + target_rate_hz: Target sample rate in Hz (> 0). + + Returns: + ``float64`` array of shape ``(ceil(num_samples * target_rate_hz / sample_rate_hz),)``. + + Raises: + ValueError: If ``x`` is not a finite non-empty 1-D float array or a rate is <= 0. + """ + wave = _as_waveform(x, "x") + source_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz") + target_hz = _check_positive_int(target_rate_hz, "target_rate_hz") + if source_hz == target_hz: + return wave.copy() + divisor = math.gcd(source_hz, target_hz) + up = target_hz // divisor + down = source_hz // divisor + out = np.asarray(signal.resample_poly(wave, up, down), dtype=np.float64) + return out + + +# -------------------------------------------------------------------------------------- +# Spectral front end +# -------------------------------------------------------------------------------------- + + +def _hz_to_mel(freq_hz: FloatArray) -> FloatArray: + return np.asarray(2595.0 * np.log10(1.0 + freq_hz / 700.0), dtype=np.float64) + + +def _mel_to_hz(mel: FloatArray) -> FloatArray: + return np.asarray(700.0 * (10.0 ** (mel / 2595.0) - 1.0), dtype=np.float64) + + +def _bin_frequencies_hz(sample_rate_hz: int, n_fft: int) -> FloatArray: + return np.arange(n_fft // 2 + 1, dtype=np.float64) * sample_rate_hz / n_fft + + +def mel_filterbank( + sample_rate_hz: int, + n_fft: int, + n_mels: int, + fmin_hz: float = 0.0, + fmax_hz: float | None = None, +) -> FloatArray: + """Build a triangular HTK mel filterbank. + + The mel scale is ``2595 * log10(1 + f / 700)``. ``n_mels + 2`` points equally spaced on the + mel axis between ``fmin_hz`` and ``fmax_hz`` define the filters: filter ``i`` rises linearly + from point ``i`` to point ``i + 1`` and falls linearly to point ``i + 2``. Filters are *not* + area-normalised, so the continuous peak is 1.0 (a sampled filter reaches 1.0 only when an + FFT bin sits exactly on its centre). + + Args: + sample_rate_hz: Sample rate in Hz (> 0). + n_fft: FFT size in samples (> 0). + n_mels: Number of mel filters (> 0). + fmin_hz: Lowest filter edge in Hz, ``0 <= fmin_hz < fmax_hz``. + fmax_hz: Highest filter edge in Hz, ``<= sample_rate_hz / 2``. Defaults to + ``sample_rate_hz / 2``. + + Returns: + Array of shape ``(n_mels, n_fft // 2 + 1)``, non-negative, ``float64``. + + Raises: + ValueError: On invalid arguments, or if some filter covers no FFT bin (``n_mels`` is + too large for ``n_fft``), since such a filter would silently output zeros. + """ + rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz") + fft_size = _check_positive_int(n_fft, "n_fft") + num_mels = _check_positive_int(n_mels, "n_mels") + top_hz = rate_hz / 2 if fmax_hz is None else float(fmax_hz) + _check_band(rate_hz, float(fmin_hz), top_hz) + + mel_points = np.linspace( + _hz_to_mel(np.asarray(float(fmin_hz))), + _hz_to_mel(np.asarray(top_hz)), + num_mels + 2, + dtype=np.float64, + ) + edges_hz = _mel_to_hz(mel_points) + left_hz = edges_hz[:-2, np.newaxis] + centre_hz = edges_hz[1:-1, np.newaxis] + right_hz = edges_hz[2:, np.newaxis] + freqs_hz = _bin_frequencies_hz(rate_hz, fft_size)[np.newaxis, :] + + rising = (freqs_hz - left_hz) / (centre_hz - left_hz) + falling = (right_hz - freqs_hz) / (right_hz - centre_hz) + bank = np.maximum(0.0, np.minimum(rising, falling)).astype(np.float64) + + empty = np.flatnonzero(bank.max(axis=1) <= 0.0) + if empty.size > 0: + raise ValueError( + f"{empty.size} of {num_mels} mel filters cover no FFT bin " + f"(first empty index {int(empty[0])}); n_fft={fft_size} is too small for " + f"n_mels={num_mels} at sample_rate_hz={rate_hz}" + ) + return np.asarray(bank, dtype=np.float64) + + +def stft_magnitude(x: npt.ArrayLike, n_fft: int, hop_samples: int) -> FloatArray: + """Magnitude STFT with a periodic Hann window and no centering. + + Frame ``t`` covers samples ``[t * hop_samples, t * hop_samples + n_fft)``. The tail is + zero-padded so the last partial frame is included, and inputs shorter than ``n_fft`` are + zero-padded to a single frame. + + Args: + x: Waveform, shape ``(num_samples,)``, any float dtype. + n_fft: FFT and window size in samples (> 0). + hop_samples: Hop between frames in samples (> 0). + + Returns: + Array of shape ``(frames, n_fft // 2 + 1)`` with + ``frames = 1 + ceil(max(0, num_samples - n_fft) / hop_samples)``. + + Raises: + ValueError: If ``x`` is invalid or ``n_fft`` / ``hop_samples`` is <= 0. + """ + wave = _as_waveform(x, "x") + fft_size = _check_positive_int(n_fft, "n_fft") + hop = _check_positive_int(hop_samples, "hop_samples") + + num_samples = wave.shape[0] + num_frames = 1 + -(-max(0, num_samples - fft_size) // hop) + padded_samples = (num_frames - 1) * hop + fft_size + padded = np.zeros(padded_samples, dtype=np.float64) + padded[:num_samples] = wave + + window = np.asarray(signal.get_window("hann", fft_size, fftbins=True), dtype=np.float64) + frames = np.lib.stride_tricks.sliding_window_view(padded, fft_size)[::hop] + spectrum = np.fft.rfft(frames * window, axis=1) + return np.abs(spectrum).astype(np.float64) + + +# -------------------------------------------------------------------------------------- +# Metrics +# -------------------------------------------------------------------------------------- + + +def log_mel_l1( + ref: npt.ArrayLike, + est: npt.ArrayLike, + sample_rate_hz: int, + n_fft: int, + hop_samples: int, + n_mels: int, + eps: float = 1e-5, +) -> float: + """Log-mel L1 distance between a reference and an estimate. + + Both waveforms are trimmed to the shorter length, converted to magnitude STFTs, projected + through the mel filterbank (magnitude, not power), floored at ``eps`` and compared in + ``log10``. The result is the mean absolute difference over all frames and mel bands. + + Args: + ref: Reference waveform, shape ``(num_samples,)``. + est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``. + sample_rate_hz: Sample rate of both signals in Hz. + n_fft: FFT size in samples. + hop_samples: STFT hop in samples. + n_mels: Number of mel bands. + eps: Floor applied to the mel magnitudes before ``log10`` (> 0). + + Returns: + Mean absolute difference in log10 mel magnitude (dimensionless, >= 0). + + Raises: + ValueError: On invalid waveforms or arguments (see :func:`mel_filterbank` and + :func:`stft_magnitude`). + """ + ref_wave = _as_waveform(ref, "ref") + est_wave = _as_waveform(est, "est") + floor = _check_positive_float(eps, "eps") + bank = mel_filterbank(sample_rate_hz, n_fft, n_mels) + ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave) + + ref_mel = stft_magnitude(ref_wave, n_fft, hop_samples) @ bank.T + est_mel = stft_magnitude(est_wave, n_fft, hop_samples) @ bank.T + diff = np.log10(np.maximum(ref_mel, floor)) - np.log10(np.maximum(est_mel, floor)) + return float(np.mean(np.abs(diff))) + + +def _band_mask( + sample_rate_hz: int, n_fft: int, low_hz: float, high_hz: float +) -> npt.NDArray[np.bool_]: + """Boolean mask over FFT bins belonging to the band (see :func:`band_power_db`).""" + freqs_hz = _bin_frequencies_hz(sample_rate_hz, n_fft) + upper = freqs_hz <= high_hz if high_hz == sample_rate_hz / 2 else freqs_hz < high_hz + mask: npt.NDArray[np.bool_] = (freqs_hz >= low_hz) & upper + if not mask.any(): + raise ValueError( + f"no FFT bin falls in [{low_hz}, {high_hz}) Hz with n_fft={n_fft} at " + f"sample_rate_hz={sample_rate_hz} (bin spacing {sample_rate_hz / n_fft} Hz)" + ) + return mask + + +def _band_power( + x: FloatArray, sample_rate_hz: int, n_fft: int, hop_samples: int, mask: npt.NDArray[np.bool_] +) -> FloatArray: + """Power spectrogram restricted to in-band bins, shape ``(frames, num_band_bins)``.""" + power = stft_magnitude(x, n_fft, hop_samples) ** 2 + return np.asarray(power[:, mask], dtype=np.float64) + + +def band_power_db( + x: npt.ArrayLike, + sample_rate_hz: int, + low_hz: float, + high_hz: float, + n_fft: int = 2048, + hop_samples: int = 512, + eps: float = 1e-10, +) -> FloatArray: + """Per-frame mean power in a frequency band, in dB. + + The band contains the FFT bins ``k`` with centre frequency + ``f_k = k * sample_rate_hz / n_fft`` satisfying ``low_hz <= f_k < high_hz``. When + ``high_hz == sample_rate_hz / 2`` the upper edge is inclusive so the Nyquist bin belongs + to the band. For each frame the result is ``10 * log10(mean_k(|X_k|^2) + eps)`` where + ``X`` is the Hann-windowed STFT from :func:`stft_magnitude` (unnormalised, so absolute + values depend on ``n_fft``; compare like with like). + + Args: + x: Waveform, shape ``(num_samples,)``. + sample_rate_hz: Sample rate in Hz. + low_hz: Lower band edge in Hz, ``0 <= low_hz < high_hz``. + high_hz: Upper band edge in Hz, ``<= sample_rate_hz / 2``. + n_fft: FFT size in samples. + hop_samples: STFT hop in samples. + eps: Power floor added before the logarithm (> 0). + + Returns: + Array of shape ``(frames,)`` in dB. + + Raises: + ValueError: On invalid arguments, or if no FFT bin falls in the band. + """ + wave = _as_waveform(x, "x") + rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz") + fft_size = _check_positive_int(n_fft, "n_fft") + hop = _check_positive_int(hop_samples, "hop_samples") + floor = _check_positive_float(eps, "eps") + _check_band(rate_hz, float(low_hz), float(high_hz)) + mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz)) + + in_band = _band_power(wave, rate_hz, fft_size, hop, mask) + return np.asarray(10.0 * np.log10(in_band.mean(axis=1) + floor), dtype=np.float64) + + +def band_log_spectral_distance( + ref: npt.ArrayLike, + est: npt.ArrayLike, + sample_rate_hz: int, + low_hz: float, + high_hz: float, + n_fft: int = 2048, + hop_samples: int = 512, + eps: float = 1e-10, + active_within_db: float | None = None, +) -> float: + """Log-spectral distance (RMS over in-band bins, mean over frames) in dB. + + With ``active_within_db`` only *active* frames are averaged: frames whose full-band + reference power is within that many dB of the loudest reference frame. This keeps + silence and noise-floor frames (where ``eps`` dominates) from swamping the average. + + Both waveforms are trimmed to the shorter length. Per frame the distance is + ``sqrt(mean_k((10 log10(P_ref[k] + eps) - 10 log10(P_est[k] + eps))^2))`` over the in-band + bins ``k`` (band definition: see :func:`band_power_db`); the result is the mean over frames. + + Args: + ref: Reference waveform, shape ``(num_samples,)``. + est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``. + sample_rate_hz: Sample rate of both signals in Hz. + low_hz: Lower band edge in Hz. + high_hz: Upper band edge in Hz. + n_fft: FFT size in samples. + hop_samples: STFT hop in samples. + eps: Power floor added before the logarithm (> 0). + active_within_db: If set (> 0), average only over active frames (see above). + + Returns: + Distance in dB (>= 0; 0 for identical signals). + + Raises: + ValueError: On invalid arguments, or if no FFT bin falls in the band. + """ + ref_wave = _as_waveform(ref, "ref") + est_wave = _as_waveform(est, "est") + rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz") + fft_size = _check_positive_int(n_fft, "n_fft") + hop = _check_positive_int(hop_samples, "hop_samples") + floor = _check_positive_float(eps, "eps") + _check_band(rate_hz, float(low_hz), float(high_hz)) + mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz)) + ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave) + + ref_power = _band_power(ref_wave, rate_hz, fft_size, hop, mask) + est_power = _band_power(est_wave, rate_hz, fft_size, hop, mask) + diff_db = 10.0 * np.log10(ref_power + floor) - 10.0 * np.log10(est_power + floor) + per_frame = np.sqrt(np.mean(diff_db**2, axis=1)) + if active_within_db is not None: + window_db = _check_positive_float(active_within_db, "active_within_db") + full_power = np.sum(stft_magnitude(ref_wave, fft_size, hop) ** 2, axis=1) + full_db = 10.0 * np.log10(full_power + floor) + active = full_db >= float(np.max(full_db)) - window_db + per_frame = per_frame[active] + return float(np.mean(per_frame)) + + +def band_energy_ratio( + ref: npt.ArrayLike, + est: npt.ArrayLike, + sample_rate_hz: int, + low_hz: float, + high_hz: float, + n_fft: int = 2048, + hop_samples: int = 512, +) -> float: + """Ratio of in-band energy of the estimate to that of the reference. + + Both waveforms are trimmed to the shorter length. The energy of a signal is the sum of + ``|X_k|^2`` over all frames and all in-band bins ``k`` (band definition: see + :func:`band_power_db`). + + Args: + ref: Reference waveform, shape ``(num_samples,)``. + est: Estimated waveform, shape ``(num_samples,)``; may differ in length from ``ref``. + sample_rate_hz: Sample rate of both signals in Hz. + low_hz: Lower band edge in Hz. + high_hz: Upper band edge in Hz. + n_fft: FFT size in samples. + hop_samples: STFT hop in samples. + + Returns: + ``E_est / E_ref`` (dimensionless, >= 0; 1 for identical signals). + + Raises: + ValueError: On invalid arguments, if no FFT bin falls in the band, or if the reference + has zero in-band energy (the ratio would be undefined). + """ + ref_wave = _as_waveform(ref, "ref") + est_wave = _as_waveform(est, "est") + rate_hz = _check_positive_int(sample_rate_hz, "sample_rate_hz") + fft_size = _check_positive_int(n_fft, "n_fft") + hop = _check_positive_int(hop_samples, "hop_samples") + _check_band(rate_hz, float(low_hz), float(high_hz)) + mask = _band_mask(rate_hz, fft_size, float(low_hz), float(high_hz)) + ref_wave, est_wave = _trim_to_common_length(ref_wave, est_wave) + + ref_energy = float(np.sum(_band_power(ref_wave, rate_hz, fft_size, hop, mask))) + est_energy = float(np.sum(_band_power(est_wave, rate_hz, fft_size, hop, mask))) + if ref_energy == 0.0: + raise ValueError( + f"reference has zero energy in [{low_hz}, {high_hz}) Hz; energy ratio is undefined" + ) + return est_energy / ref_energy + + +# -------------------------------------------------------------------------------------- +# Alignment +# -------------------------------------------------------------------------------------- + + +def estimate_delay_samples(ref: npt.ArrayLike, est: npt.ArrayLike, max_lag_samples: int) -> int: + """Estimate the delay of ``est`` relative to ``ref`` by cross-correlation. + + The cross-correlation ``c[lag] = sum_n est[n] * ref[n - lag]`` is computed with FFTs + (zero-padded, so there is no circular wrap-around) and maximised over + ``-max_lag_samples <= lag <= +max_lag_samples``, further limited to lags for which the + signals overlap. A positive result means ``est`` is delayed: ``est[n] ~ ref[n - lag]``. + + Ties (values within a relative tolerance of ``1e-9`` of the best) resolve to the smallest + ``|lag|``; between ``+k`` and ``-k`` the positive lag wins. Two all-zero signals yield 0. + + Args: + ref: Reference waveform, shape ``(num_ref_samples,)``. + est: Estimated waveform, shape ``(num_est_samples,)``. + max_lag_samples: Largest absolute lag to consider (>= 0). + + Returns: + Delay in samples (``int``) in ``[-max_lag_samples, +max_lag_samples]``. + + Raises: + ValueError: On invalid waveforms or negative / non-integer ``max_lag_samples``. + """ + ref_wave = _as_waveform(ref, "ref") + est_wave = _as_waveform(est, "est") + max_lag = _check_int(max_lag_samples, "max_lag_samples") + if max_lag < 0: + raise ValueError(f"max_lag_samples must be >= 0, got {max_lag}") + + num_ref = ref_wave.shape[0] + num_est = est_wave.shape[0] + max_positive = min(max_lag, num_est - 1) # est[n] needs n = lag + m with m >= 0 + max_negative = min(max_lag, num_ref - 1) + + fft_size = 1 << (num_ref + num_est - 1).bit_length() + est_spectrum = np.fft.rfft(est_wave, fft_size) + ref_spectrum = np.fft.rfft(ref_wave, fft_size) + corr = np.fft.irfft(est_spectrum * np.conj(ref_spectrum), fft_size) + + # Candidate lags ordered by |lag| (0, +1, -1, +2, -2, ...), so the first maximal entry + # is the smallest-|lag| tie winner. + lags: list[int] = [0] + for k in range(1, max(max_positive, max_negative) + 1): + if k <= max_positive: + lags.append(k) + if k <= max_negative: + lags.append(-k) + lag_arr = np.asarray(lags, dtype=np.int64) + values = corr[lag_arr % fft_size] # negative lags wrap to the end of the circular buffer + + peak = float(values.max()) + tolerance = _TIE_RTOL * float(np.abs(values).max()) + best = int(np.argmax(values >= peak - tolerance)) + return int(lag_arr[best]) + + +def align( + ref: npt.ArrayLike, est: npt.ArrayLike, delay_samples: int +) -> tuple[FloatArray, FloatArray]: + """Compensate a known delay of ``est`` and trim both signals to a common length. + + With ``delay_samples = d > 0`` (``est`` lags ``ref``) the first ``d`` samples of ``est`` + are dropped; with ``d < 0`` (``est`` leads) the first ``|d|`` samples of ``ref`` are + dropped. Both outputs are then cut to the shorter of the two remaining lengths. + + Args: + ref: Reference waveform, shape ``(num_ref_samples,)``. + est: Estimated waveform, shape ``(num_est_samples,)``. + delay_samples: Delay of ``est`` relative to ``ref``, as returned by + :func:`estimate_delay_samples`. + + Returns: + ``(ref_aligned, est_aligned)``, two ``float64`` copies of identical length >= 1. + + Raises: + ValueError: On invalid waveforms, non-integer ``delay_samples``, or if no samples + remain after compensating the delay. + """ + ref_wave = _as_waveform(ref, "ref") + est_wave = _as_waveform(est, "est") + delay = _check_int(delay_samples, "delay_samples") + + if delay >= 0: + ref_part, est_part = ref_wave, est_wave[delay:] + else: + ref_part, est_part = ref_wave[-delay:], est_wave + ref_part, est_part = _trim_to_common_length(ref_part, est_part) + if ref_part.shape[0] < 1: + raise ValueError( + f"no samples left after compensating delay_samples={delay} " + f"(ref has {ref_wave.shape[0]}, est has {est_wave.shape[0]} samples)" + ) + return ref_part.copy(), est_part.copy() diff --git a/src/nida/bench/b1.py b/src/nida/bench/b1.py new file mode 100644 index 0000000000000000000000000000000000000000..3ac0af9ce0e557dc143cfecabd10b8bf24efbeda --- /dev/null +++ b/src/nida/bench/b1.py @@ -0,0 +1,622 @@ +"""Benchmark B1: Turkish ASR comparison (docs/benchmarks/B1-B2-tasarim.md §1). + +Stages (each idempotent and resumable): + +1. ``prepare`` — :func:`nida.bench.prepare.prepare_b1` (layers A, B and the C pool). +2. ``asr`` — every system transcribes every clip in its own environment. +3. ``panel`` — choose the final layer-C clips (filler-rich, once) and build the + transcription panel; the user's corrected transcripts become C references. +4. ``score`` — metrics, paired bootstrap and the pre-registered decision rule. + +Failed worker items are scored as empty hypotheses (all deletions) and counted, so a +system can never look better by failing on hard clips. +""" + +from __future__ import annotations + +import itertools +import json +import logging +import subprocess +from collections.abc import Callable, Mapping, Sequence +from contextvars import ContextVar +from dataclasses import dataclass, field +from pathlib import Path + +from nida.bench.clips import Clip, select_yodas_final, text_has_filler +from nida.bench.panel import ( + AudioRef, + PanelItem, + PanelResult, + PanelSpec, + Question, + write_panel, +) +from nida.bench.prepare import Workspace, clip_audio_path +from nida.bench.protocol import ( + WorkerSummary, + WorkRequest, + WorkResult, + load_results, + read_summary, + write_requests, +) +from nida.bench.registry import System +from nida.bench.textmetrics import ( + FILLERS, + EditCounts, + cer_counts, + corpus_rate, + filler_score, + has_digits, + is_hallucination, + normalize_for_scoring, + paired_bootstrap, + strip_fillers, + wer_counts, +) +from nida.utils.io import atomic_write_json + +__all__ = [ + "C_PANEL_QUESTIONS", + "DecisionRule", + "FillerProvenance", + "build_c_panel", + "c_references_from_panel", + "choose_systems", + "hypotheses", + "load_c_selection", + "run_system", + "score_b1", +] + +log = logging.getLogger(__name__) + +MAX_ERROR_FRACTION = 0.05 +N_C_FINAL = 50 +# Pre-registered 25; lowered to 20 before any score was seen (pool had 23 filler clips, +# run 20261008-122110-bench-phase0-b301; docs/benchmarks/B1-B2-tasarim.md §1.1). +MIN_C_FILLER_CLIPS = 20 +C_REVIEWED_ANSWERS = ("tamam", "emin değilim") +C_SKIP_ANSWER = "konuşma değil — atla" + +C_PANEL_QUESTIONS = ( + Question( + "durum", + "Bu klip için durum", + "choice", + (*C_REVIEWED_ANSWERS, C_SKIP_ANSWER), + required=True, + ), +) +C_PANEL_INSTRUCTIONS = ( + "Her klibi dinleyin ve metin kutusundaki taslağı duyduğunuz şekilde, kelimesi kelimesine " + "düzeltin. Taslak YouTube altyazısıdır, hatalı olabilir. Dolgu seslerini şu yazımlarla " + "ekleyin: ıı, ee, hmm, ımm, aa (ve of, uf, ah, oh, vay, hıı). Sayıları yazıyla yazın. " + "Noktalama önemli değil. Her klip için bir durum seçin: düzelttiyseniz 'tamam', " + "anlaşılmayan yer kaldıysa 'emin değilim', konuşma yoksa ya da müzik baskınsa " + "'konuşma değil — atla'. Durumu seçilmeyen klipler kullanılmaz." +) + +Runner = Callable[[Sequence[str]], int] + +#: Tag put in front of every line :func:`_stream_runner` logs. Callers that run several +#: workers at the same time set it (per thread) to the job name so the interleaved lines +#: stay attributable; empty (the default) logs exactly as before. +LOG_TAG: ContextVar[str] = ContextVar("nida_worker_log_tag", default="") + + +def _stream_runner(cmd: Sequence[str]) -> int: + tag = LOG_TAG.get() + prefix = f"[{tag}] " if tag else "" + log.info("%s$ %s", prefix, " ".join(cmd)) + with subprocess.Popen( + list(cmd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1 + ) as proc: + assert proc.stdout is not None # stdout=PIPE guarantees a stream + for line in proc.stdout: + log.info("%s | %s", prefix, line.rstrip()) + return proc.wait() + + +def _worker_dir(workspace: Workspace, system: System, variant: str) -> Path: + name = system.system_id if not variant else f"{system.system_id}__{variant}" + return workspace.workers / name + + +def run_system( + workspace: Workspace, + clips: Sequence[Clip], + system: System, + python_exe: Path, + *, + variant: str = "", + context: str = "", + retry_failed: bool = False, + runner: Runner = _stream_runner, +) -> WorkerSummary: + """Run one ASR system over ``clips`` (resumable) and return its summary. + + Raises: + RuntimeError: if the worker exits non-zero or more than 5% of items failed. + """ + out_dir = _worker_dir(workspace, system, variant) + requests = [ + WorkRequest( + clip.clip_id, + str(clip_audio_path(workspace, clip)), + {"language": clip.language, **({"context": context} if context else {})}, + ) + for clip in clips + ] + write_requests(out_dir / "requests.jsonl", requests) + cmd = [ + str(python_exe), + "-m", + f"nida.bench.workers.{system.worker}", + "--requests", + str(out_dir / "requests.jsonl"), + "--results", + str(out_dir / "results.jsonl"), + "--model-id", + system.model_id, + "--revision", + system.revision, + "--batch-size", + str(system.batch_size), + "--device", + "cuda", + *(["--retry-failed"] if retry_failed else []), + ] + code = runner(cmd) + if code != 0: + raise RuntimeError(f"{system.system_id}: worker exited with code {code}") + summary = read_summary(out_dir / "results.summary.json") + total = summary.n_ok + summary.n_error + if total != len(clips): + raise RuntimeError(f"{system.system_id}: {total} results for {len(clips)} clips") + if summary.n_error > MAX_ERROR_FRACTION * total: + raise RuntimeError( + f"{system.system_id}: {summary.n_error}/{total} items failed (> 5%); " + "rerun with --retry-failed after fixing the cause" + ) + return summary + + +def hypotheses(workspace: Workspace, system: System, variant: str = "") -> dict[str, WorkResult]: + return load_results(_worker_dir(workspace, system, variant) / "results.jsonl") + + +def _hyp_text(result: WorkResult | None) -> str: + if result is None or result.output is None: + return "" + text = result.output.get("text", "") + return text if isinstance(text, str) else "" + + +# --------------------------------------------------------------------------- C panel + + +@dataclass(frozen=True) +class FillerProvenance: + """Why a layer-C clip may have been selected (used to de-bias filler recall). + + Attributes: + caption: The YouTube caption contains a filler. + systems: Systems whose hypothesis contains a filler. + """ + + caption: bool + systems: frozenset[str] = field(default_factory=frozenset) + + def counts_for(self, system_id: str) -> bool: + """True if the clip's selection does not depend solely on ``system_id``'s output. + + Filler-signal clips are favoured during selection. If the only signal was this + system's own hypothesis, scoring its filler recall on the clip would reward it for + having caused the selection, so such clips are left out of its recall. + """ + return self.caption or not self.systems or bool(self.systems - {system_id}) + + +def _filler_provenance( + clips: Sequence[Clip], all_hypotheses: Mapping[str, Mapping[str, WorkResult]] +) -> dict[str, FillerProvenance]: + out: dict[str, FillerProvenance] = {} + for clip in clips: + systems = frozenset( + system_id + for system_id, results in all_hypotheses.items() + if text_has_filler(_hyp_text(results.get(clip.clip_id))) + ) + out[clip.clip_id] = FillerProvenance(text_has_filler(clip.meta.get("caption", "")), systems) + return out + + +def _selection_path(workspace: Workspace) -> Path: + return workspace.manifests / "b1_c_final.json" + + +def load_c_selection(workspace: Workspace, seed: int | None = None) -> dict[str, FillerProvenance]: + """Read the stored layer-C selection: ``{clip_id: provenance}`` in selection order. + + Raises: + ValueError: if ``seed`` is given and differs from the seed the selection used. + """ + obj = json.loads(_selection_path(workspace).read_text(encoding="utf-8")) + if seed is not None and obj.get("seed") != seed: + raise ValueError(f"stored C selection used seed {obj.get('seed')}, not {seed}") + return { + row["clip_id"]: FillerProvenance(bool(row["caption_filler"]), frozenset(row["systems"])) + for row in obj["clips"] + } + + +def build_c_panel( + workspace: Workspace, + pool: Sequence[Clip], + all_hypotheses: Mapping[str, Mapping[str, WorkResult]], + seed: int, +) -> tuple[list[Clip], Path]: + """Select the final layer-C clips (once) and write their transcription panel. + + The selection is stored once in ``manifests/b1_c_final.json`` together with each + clip's filler provenance and is never re-drawn: a rerun rebuilds the panel for the same + clips. Before the first selection every system must have a result for every pool clip. + The draft text is the YouTube caption, never a candidate system's output, so + references are not anchored to one of the systems under test. + + Raises: + RuntimeError: if some system has not transcribed the whole pool, or the stored + selection does not belong to this pool. + """ + import soundfile # type: ignore[import-untyped] # bench extra + + by_id = {c.clip_id: c for c in pool} + if _selection_path(workspace).exists(): + selection = load_c_selection(workspace, seed) + stray = [cid for cid in selection if cid not in by_id] + if stray: + raise RuntimeError(f"stored C selection has clips outside the pool: {stray[:3]}") + final = [by_id[cid] for cid in selection] + else: + incomplete = sorted( + s for s, results in all_hypotheses.items() if not set(by_id) <= set(results) + ) + if incomplete: + raise RuntimeError(f"systems have not transcribed the whole C pool: {incomplete}") + texts: dict[str, list[str]] = {} + for results in all_hypotheses.values(): + for clip_id, result in results.items(): + texts.setdefault(clip_id, []).append(_hyp_text(result)) + final = list( + select_yodas_final(pool, texts, N_C_FINAL, seed, min_filler_clips=MIN_C_FILLER_CLIPS) + ) + provenance = _filler_provenance(final, all_hypotheses) + atomic_write_json( + _selection_path(workspace), + { + "seed": seed, + "clips": [ + { + "clip_id": c.clip_id, + "caption_filler": provenance[c.clip_id].caption, + "systems": sorted(provenance[c.clip_id].systems), + } + for c in final + ], + }, + ) + panel_id = f"b1c-{seed}" + audio_dir = workspace.cache / "panel_audio" / panel_id + items: list[PanelItem] = [] + for clip in final: + flac = audio_dir / f"{clip.clip_id}.flac" + if not flac.exists(): + audio, rate_hz = soundfile.read(str(clip_audio_path(workspace, clip)), dtype="int16") + flac.parent.mkdir(parents=True, exist_ok=True) + soundfile.write(str(flac), audio, rate_hz, subtype="PCM_16") + items.append( + PanelItem( + item_id=clip.clip_id, + audios=(AudioRef("Klip", flac),), + text=clip.meta.get("caption", ""), + editable_text=True, + info=f"YODAS · {clip.duration_ms / 1000:.1f} s", + ) + ) + spec = PanelSpec( + panel_id=panel_id, + title="B1 — Doğal konuşma transkripsiyonu", + instructions=C_PANEL_INSTRUCTIONS, + questions=C_PANEL_QUESTIONS, + items=tuple(items), + ) + return final, write_panel(spec, workspace.panels / panel_id) + + +def c_references_from_panel(results: Sequence[PanelResult]) -> dict[str, str]: + """Usable C references: items explicitly reviewed (``tamam`` / ``emin değilim``). + + Unreviewed items are excluded even though their CSV row carries the caption draft: an + untouched caption is not a reference (glued words, fillers almost never written). + """ + refs: dict[str, str] = {} + for result in results: + if result.answers.get("durum") not in C_REVIEWED_ANSWERS: + continue + text = (result.text or "").strip() + if text: + refs[result.item_id] = text + return refs + + +# --------------------------------------------------------------------------- scoring + + +@dataclass(frozen=True) +class DecisionRule: + """Pre-registered rule (design note §1.4), fixed before results are seen. + + Attributes: + min_filler_recall: Primary must reach this C filler recall (leave-one-out). + min_reviewed_c: With fewer reviewed C references the rule is not applied. + min_filler_ref_tokens: With fewer reference fillers in C the rule is not applied. + min_loo_ref_tokens: Per-system minimum of reference fillers left after the + leave-one-out exclusion (the denominator of the recall the rule uses). + max_failed_ab: Most failed worker items allowed per system on A+B; failed items are + scored as empty hypotheses, so any more would distort the CER ranking. + max_failed_c: Same for layer C, where a failed item turns its reference fillers into + misses and would distort the filler recall the rule depends on. + """ + + min_filler_recall: float = 0.6 + min_reviewed_c: int = 35 + min_filler_ref_tokens: int = 30 + min_loo_ref_tokens: int = 20 + max_failed_ab: int = 0 + max_failed_c: int = 0 + + +def _layer_scores( + clips: Sequence[Clip], + refs: Mapping[str, str], + hyps: Mapping[str, WorkResult], + eligible_for_fillers: frozenset[str] | None = None, +) -> tuple[dict[str, object], list[EditCounts]]: + content_cer: list[EditCounts] = [] + content_wer: list[EditCounts] = [] + full_cer: list[EditCounts] = [] + full_wer: list[EditCounts] = [] + pairs: list[tuple[list[str], list[str]]] = [] + loo_pairs: list[tuple[list[str], list[str]]] = [] + n_failed = n_halluc = n_digits = 0 + infer_s = audio_s = 0.0 + for clip in clips: + ref_text = refs[clip.clip_id] + result = hyps.get(clip.clip_id) + hyp_text = _hyp_text(result) + if result is None or not result.ok: + n_failed += 1 + else: + infer_s += result.infer_seconds + audio_s += clip.duration_ms / 1000 + n_halluc += is_hallucination(ref_text, hyp_text) + n_digits += has_digits(hyp_text) + ref_tokens = normalize_for_scoring(ref_text) + hyp_tokens = normalize_for_scoring(hyp_text) + content_cer.append(cer_counts(strip_fillers(ref_tokens), strip_fillers(hyp_tokens))) + content_wer.append(wer_counts(strip_fillers(ref_tokens), strip_fillers(hyp_tokens))) + full_cer.append(cer_counts(ref_tokens, hyp_tokens)) + full_wer.append(wer_counts(ref_tokens, hyp_tokens)) + pairs.append((ref_tokens, hyp_tokens)) + if eligible_for_fillers is None or clip.clip_id in eligible_for_fillers: + loo_pairs.append((ref_tokens, hyp_tokens)) + fillers = filler_score(pairs) + loo = filler_score(loo_pairs) + n_ok = len(clips) - n_failed + scores: dict[str, object] = { + "n": len(clips), + "n_failed": n_failed, + "content_cer": corpus_rate(content_cer), + "content_wer": corpus_rate(content_wer), + "full_cer": corpus_rate(full_cer), + "full_wer": corpus_rate(full_wer), + "hallucination_rate": n_halluc / n_ok if n_ok else None, + "hyp_with_digits": n_digits, + "rtf": infer_s / audio_s if audio_s else None, + "filler_ref_total": fillers.ref_total, + "filler_hyp_total": fillers.hyp_total, + "filler_recall": fillers.recall(), + "filler_precision": fillers.precision(), + "filler_recall_loo": loo.recall(), + "filler_loo_ref_total": loo.ref_total, + "filler_recall_by_type": { + kind: filler_score(pairs, kind).recall() + for kind in sorted(FILLERS) + if filler_score(pairs, kind).ref_total + }, + } + return scores, content_cer + + +def score_b1( + clips: Sequence[Clip], + systems: Sequence[System], + all_hypotheses: Mapping[str, Mapping[str, WorkResult]], + c_references: Mapping[str, str] | None, + *, + seed: int, + summaries: Mapping[str, WorkerSummary] | None = None, + c_provenance: Mapping[str, FillerProvenance] | None = None, +) -> dict[str, object]: + """Compute B1 metrics per system and layer, plus paired bootstrap on content CER. + + Layers: ``A``, ``B``, ``AB`` (A+B together) and ``C`` when ``c_references`` is given + (only reviewed clips present in it are scored). With ``c_provenance`` the C layer's + ``filler_recall_loo`` excludes, per system, clips selected only because of that + system's own hypothesis (see :class:`FillerProvenance`). + """ + refs: dict[str, str] = {} + for clip in clips: + if clip.layer in ("A", "B") and clip.reference is not None: + refs[clip.clip_id] = clip.reference + if c_references: + refs.update(c_references) + layers: dict[str, list[Clip]] = { + "A": [c for c in clips if c.layer == "A"], + "B": [c for c in clips if c.layer == "B"], + } + layers["AB"] = layers["A"] + layers["B"] + if c_references: + layers["C"] = [c for c in clips if c.layer == "C" and c.clip_id in c_references] + + per_system: dict[str, dict[str, dict[str, object]]] = {} + counts_for_bootstrap: dict[str, dict[str, list[EditCounts]]] = {} + for system in systems: + hyps = all_hypotheses[system.system_id] + per_layer: dict[str, dict[str, object]] = {} + for layer, layer_clips in layers.items(): + eligible: frozenset[str] | None = None + if layer == "C" and c_provenance is not None: + eligible = frozenset( + cid for cid, prov in c_provenance.items() if prov.counts_for(system.system_id) + ) + scores, counts = _layer_scores(layer_clips, refs, hyps, eligible) + counts_for_bootstrap.setdefault(layer, {})[system.system_id] = counts + per_layer[layer] = scores + worker: dict[str, object] = {"batch_size": system.batch_size, "worker": system.worker} + if summaries and system.system_id in summaries: + summary = summaries[system.system_id] + worker |= { + "peak_vram_gb": summary.peak_vram_gb, + "load_seconds": summary.load_seconds, + "n_error": summary.n_error, + "packages": dict(summary.packages), + } + per_layer["_worker"] = worker + per_system[system.system_id] = per_layer + + bootstrap: dict[str, list[dict[str, object]]] = {} + for layer in ("AB", "B", "C"): + if layer not in counts_for_bootstrap or not layers[layer]: + continue + rows: list[dict[str, object]] = [] + for a, b in itertools.combinations(sorted(counts_for_bootstrap[layer]), 2): + result = paired_bootstrap( + counts_for_bootstrap[layer][a], counts_for_bootstrap[layer][b], seed=seed + ) + rows.append( + {"a": a, "b": b, "delta": result.delta, "low": result.low, "high": result.high} + ) + bootstrap[layer] = rows + return { + "systems": per_system, + "bootstrap_content_cer": bootstrap, + "n_c_reviewed": len(layers.get("C", [])), + } + + +def _as_float(value: object, missing: float) -> float: + return float(value) if isinstance(value, int | float) else missing + + +def _decide( + systems: Mapping[str, Mapping[str, Mapping[str, object]]], + families: Mapping[str, str], + rule: DecisionRule, + cer_layer: str, +) -> dict[str, object]: + def cer(system_id: str) -> float: + return _as_float(systems[system_id][cer_layer]["content_cer"], float("inf")) + + def recall(system_id: str) -> float: + c_layer = systems[system_id]["C"] + return _as_float(c_layer.get("filler_recall_loo", c_layer["filler_recall"]), -1.0) + + ranked = sorted(systems, key=cer) + qualified = [s for s in ranked if recall(s) >= rule.min_filler_recall] + primary = qualified[0] if qualified else ranked[0] + others = [s for s in ranked if families[s] != families[primary]] + return { + "cer_layer": cer_layer, + "primary": primary, + "secondary": others[0] if others else None, + "needs_filler_method": not qualified, + "qualified_by_filler_recall": qualified, + } + + +def choose_systems( + scores: Mapping[str, object], + families: Mapping[str, str], + rule: DecisionRule | None = None, +) -> dict[str, object]: + """Apply the pre-registered rule; returns the suggestion and its reasoning. + + Primary: among systems whose C filler recall (leave-one-out when available) is at least + ``rule.min_filler_recall``, the lowest A+B content CER. If none qualifies: the lowest + A+B content CER and ``needs_filler_method``. Secondary: the lowest A+B content CER among + systems of a *different family*. The same rule is also evaluated on layer B alone + (``without_fleurs``) because one candidate (BuzzASR) was trained on FLEURS; the result + says whether the pick changes. Not applied (``ready=False``) without enough reviewed C + references, reference fillers, a system whose leave-one-out filler denominator is below + ``rule.min_loo_ref_tokens``, or a system with failed items beyond + ``rule.max_failed_ab`` (A+B) / ``rule.max_failed_c`` (C) (the reason names them). + """ + rule = rule if rule is not None else DecisionRule() + systems = scores["systems"] + assert isinstance(systems, dict) + if not all("C" in layers for layers in systems.values()): + return {"ready": False, "reason": "C references (user panel) not available yet"} + n_reviewed = scores.get("n_c_reviewed", 0) + if not isinstance(n_reviewed, int) or n_reviewed < rule.min_reviewed_c: + return { + "ready": False, + "reason": f"only {n_reviewed} reviewed C clips (< {rule.min_reviewed_c})", + } + ref_fillers = min(int(_as_float(v["C"]["filler_ref_total"], 0)) for v in systems.values()) + if ref_fillers < rule.min_filler_ref_tokens: + return { + "ready": False, + "reason": f"only {ref_fillers} reference fillers in C (< {rule.min_filler_ref_tokens})", + } + failed = { + f"{system_id}/{layer}": int(_as_float(v[layer].get("n_failed"), 0)) + for system_id, v in systems.items() + for layer, limit in (("AB", rule.max_failed_ab), ("C", rule.max_failed_c)) + if _as_float(v[layer].get("n_failed"), 0) > limit + } + if failed: + return { + "ready": False, + "reason": f"failed worker items (rerun with --retry-failed): {failed}", + } + thin = { + system_id: int(_as_float(v["C"].get("filler_loo_ref_total"), 0)) + for system_id, v in systems.items() + if "filler_loo_ref_total" in v["C"] + and _as_float(v["C"]["filler_loo_ref_total"], 0) < rule.min_loo_ref_tokens + } + if thin: + return { + "ready": False, + "reason": f"leave-one-out filler denominators below {rule.min_loo_ref_tokens}: {thin}", + } + main = _decide(systems, families, rule, "AB") + check = _decide(systems, families, rule, "B") + return { + "ready": True, + **main, + "without_fleurs": check, + "pick_changes_without_fleurs": (main["primary"], main["secondary"]) + != (check["primary"], check["secondary"]), + "rule": { + "min_filler_recall": rule.min_filler_recall, + "min_reviewed_c": rule.min_reviewed_c, + "min_filler_ref_tokens": rule.min_filler_ref_tokens, + "min_loo_ref_tokens": rule.min_loo_ref_tokens, + "max_failed_ab": rule.max_failed_ab, + "max_failed_c": rule.max_failed_c, + }, + } diff --git a/src/nida/bench/b2.py b/src/nida/bench/b2.py new file mode 100644 index 0000000000000000000000000000000000000000..537a93106f55d10b708cad5a57f79dc78df0fdbe --- /dev/null +++ b/src/nida/bench/b2.py @@ -0,0 +1,684 @@ +"""Benchmark B2: audio-VAE reconstruction (docs/benchmarks/B1-B2-tasarim.md §2). + +Stages (idempotent, resumable): + +1. ``prepare`` — Turkish subsets from the B1 manifest + Expresso subsets streamed from + the official 36 GB tar; originals written to ``/b2/orig`` (16 kHz Turkish, + 48 kHz Expresso, 24-bit). They live in the durable workspace (Drive), not the session + cache, so a new Colab session resumes without re-streaming 36 GB and without results + that point at vanished files. +2. ``vae`` — VoxCPM2 AudioVAE V2 round trip (worker) → ``/b2/vae`` (48 kHz). +3. ``asr`` / ``quality`` — Whisper on 16 kHz copies of original and reconstruction + (clips with references), Audiobox Aesthetics on both. +4. ``score`` — objective metrics per subset, incl. the "band-16k" baseline + (48 kHz → 16 kHz → 48 kHz) that isolates what the VAE's super-resolution adds. +5. ``panel`` — 40 *blind* A/B pairs (random order per item; key stored outside the + HTML in ``manifests/b2_panel_key-.json``) and the scoring of the user's answers. + +Audiobox resamples to 16 kHz internally, so its scores say nothing about the band above +8 kHz; the high-band LSD / energy ratio cover that. +""" + +from __future__ import annotations + +import json +import logging +import random +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from nida.bench import fetch +from nida.bench.audiometrics import ( + align, + band_energy_ratio, + band_log_spectral_distance, + estimate_delay_samples, + log_mel_l1, + resample, +) +from nida.bench.clips import Clip, clip_from_obj, clip_to_obj +from nida.bench.expresso import ( + is_wanted_member, + parse_member, + parse_read_transcriptions, + parse_vad_segments, + select_expresso, +) +from nida.bench.panel import AudioRef, PanelItem, PanelResult, PanelSpec, Question, write_panel +from nida.bench.prepare import Workspace +from nida.bench.protocol import WorkRequest, WorkResult +from nida.bench.registry import DATA_PINS +from nida.bench.textmetrics import ( + cer_counts, + corpus_rate, + normalize_english, + normalize_for_scoring, + strip_fillers, +) +from nida.utils.io import atomic_write_json, atomic_write_text + +__all__ = [ + "B2Plan", + "asr_requests", + "b2_paths", + "build_b2_panel", + "clip_metrics", + "load_b2_manifest", + "missing_vae_outputs", + "prepare_b2", + "quality_requests", + "score_b2", + "score_b2_panel", + "vae_requests", +] + +log = logging.getLogger(__name__) + +TR_SUBSETS = {"A": "tr-temiz", "B": "tr-medya"} +HIGH_BAND_HZ = (8000.0, 24000.0) +MAX_DELAY_MS = 50 +#: High-band LSD is averaged over frames within this many dB of the loudest frame. +ACTIVE_WITHIN_DB = 40.0 + + +@dataclass(frozen=True) +class B2Plan: + seed: int = 20261008 + n_tr_per_layer: int = 30 + n_whisper: int = 25 + n_laughing: int = 25 + n_projected: int = 25 + n_nonverbal: int = 15 + n_default: int = 10 + n_panel_per_subset: int = 8 + expresso_url: str = DATA_PINS["expresso_url"] + + def expresso_counts(self) -> dict[str, int]: + return { + "whisper": self.n_whisper, + "laughing": self.n_laughing, + "projected": self.n_projected, + "nonverbal": self.n_nonverbal, + "default": self.n_default, + } + + +@dataclass(frozen=True) +class B2Paths: + orig: Path + vae: Path + asr16k: Path + + +def b2_paths(workspace: Workspace) -> B2Paths: + base = workspace.root / "b2" + return B2Paths(base / "orig", base / "vae", base / "asr16k") + + +def _orig_path(workspace: Workspace, clip: Clip) -> Path: + return b2_paths(workspace).orig / f"{clip.clip_id}.wav" + + +def _vae_path(workspace: Workspace, clip: Clip) -> Path: + return b2_paths(workspace).vae / f"{clip.clip_id}.wav" + + +def _write_expresso_original(source: Path, clip: Clip, dest: Path) -> None: + import soundfile # type: ignore[import-untyped] # bench extra + + if dest.exists(): + return + audio: Any + audio, rate_hz = soundfile.read(str(source), dtype="float64", always_2d=True) + if rate_hz != 48000: + raise ValueError(f"{source}: expected 48 kHz, got {rate_hz}") + channel_text = clip.meta.get("channel", "0") + if channel_text not in ("0", "1", "2"): + raise ValueError(f"{clip.clip_id}: unknown channel {channel_text!r} (expected 0, 1 or 2)") + channel = int(channel_text) + column = 0 if channel in (0, 1) else 1 + if channel == 0 and audio.shape[1] != 1: + raise ValueError(f"{source}: read speech should be mono, got {audio.shape[1]} channels") + if channel in (1, 2) and audio.shape[1] != 2: + raise ValueError(f"{source}: conversational audio should be stereo") + start = round(clip.start_ms * 48) + stop = round(clip.end_ms * 48) + if not 0 <= start < stop <= audio.shape[0]: + raise ValueError(f"{source}: span {clip.start_ms}..{clip.end_ms} ms outside audio") + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(f".{dest.name}.tmp.wav") + soundfile.write(str(tmp), audio[start:stop, column], 48000, subtype="PCM_24") + tmp.replace(dest) + + +def prepare_b2( + workspace: Workspace, + b1_clips: Sequence[Clip], + plan: B2Plan | None = None, + *, + opener: Any = None, # Any: urllib-compatible opener (see nida.bench.fetch) +) -> list[Clip]: + """Select B2 clips and write originals; the manifest is written once and reused. + + Plan and clips are stored together in one atomically written ``b2_manifest.json`` + after everything succeeded, so a stored plan and a stored clip list can never + disagree, and a failed run leaves no manifest behind. + """ + import shutil + + import soundfile + + plan = plan if plan is not None else B2Plan() + manifest = _manifest_path(workspace) + stored_clips: list[Clip] | None = None + if manifest.exists(): + stored_plan, stored_clips = load_b2_manifest(workspace) + if stored_plan != plan: + raise ValueError(f"existing B2 manifest was built with {stored_plan}, not {plan}") + rng = random.Random(plan.seed) + tr_clips: list[Clip] = [] + for layer, subset in TR_SUBSETS.items(): + pool = sorted((c for c in b1_clips if c.layer == layer), key=lambda c: c.clip_id) + if len(pool) < plan.n_tr_per_layer: + raise ValueError(f"B1 layer {layer} has {len(pool)} clips, need {plan.n_tr_per_layer}") + for clip in rng.sample(pool, plan.n_tr_per_layer): + tr_clips.append( + Clip.build( + source_id=clip.source_id, + layer=subset, + file_id=clip.file_id, + start_ms=clip.start_ms, + end_ms=clip.end_ms, + reference=clip.reference, + language="tr", + meta={**clip.meta, "b1_clip_id": clip.clip_id}, + ) + ) + + if stored_clips is not None and all(_orig_path(workspace, c).exists() for c in stored_clips): + return stored_clips + + exp_dir = workspace.cache / "expresso" + extracted = fetch.stream_extract( + plan.expresso_url, + exp_dir, + is_wanted_member, + opener=opener if opener is not None else fetch.default_opener, + ) + by_name = {p.relative_to(exp_dir).as_posix(): p for p in extracted} + vad_file = next((p for n, p in by_name.items() if n.endswith("VAD_segments.txt")), None) + tr_file = next((p for n, p in by_name.items() if n.endswith("read_transcriptions.txt")), None) + if vad_file is None or tr_file is None: + raise FileNotFoundError( + "Expresso archive lacks VAD_segments.txt or read_transcriptions.txt" + ) + files: list[tuple[Any, int]] = [] + for name, path in sorted(by_name.items()): + parsed = parse_member(name) + if parsed is not None: + info = soundfile.info(str(path)) + # Floor, never round up: a length past the last sample makes spans invalid. + files.append((parsed, (1000 * info.frames) // info.samplerate)) + selection = select_expresso( + files, + parse_vad_segments(vad_file.read_text(encoding="utf-8")), + parse_read_transcriptions(tr_file.read_text(encoding="utf-8")), + plan.expresso_counts(), + plan.seed, + ) + for clip in selection.clips: + member = clip.file_id.split("#", 1)[0] + _write_expresso_original(by_name[member], clip, _orig_path(workspace, clip)) + for clip in tr_clips: + dest = _orig_path(workspace, clip) + if not dest.exists(): + source_clip_id = clip.meta["b1_clip_id"] + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(workspace.clips16k / f"{source_clip_id}.wav", dest) + clips = sorted([*tr_clips, *selection.clips], key=lambda c: (c.layer, c.clip_id)) + if stored_clips is not None: + if [c.clip_id for c in stored_clips] != [c.clip_id for c in clips]: + raise ValueError("re-derived B2 clips differ from the stored manifest") + return stored_clips + atomic_write_text( + workspace.manifests / "b2_quarantine.jsonl", + "".join(json.dumps(asdict(q), ensure_ascii=False) + "\n" for q in selection.quarantined), + ) + atomic_write_json(manifest, {"plan": asdict(plan), "clips": [clip_to_obj(c) for c in clips]}) + return clips + + +def _manifest_path(workspace: Workspace) -> Path: + return workspace.manifests / "b2_manifest.json" + + +def load_b2_manifest(workspace: Workspace) -> tuple[B2Plan, list[Clip]]: + """Read ``manifests/b2_manifest.json`` (plan and clips, written together).""" + obj = json.loads(_manifest_path(workspace).read_text(encoding="utf-8")) + return B2Plan(**obj["plan"]), [clip_from_obj(c) for c in obj["clips"]] + + +# ------------------------------------------------------------------------ worker inputs + + +def vae_requests(workspace: Workspace, clips: Sequence[Clip]) -> list[WorkRequest]: + return [ + WorkRequest( + c.clip_id, str(_orig_path(workspace, c)), {"out_path": str(_vae_path(workspace, c))} + ) + for c in clips + ] + + +def missing_vae_outputs(workspace: Workspace, results: Mapping[str, WorkResult]) -> list[str]: + """Ids of successful VAE results whose output file no longer exists (to be redone).""" + missing: list[str] = [] + for clip_id, result in results.items(): + out = result.output.get("out_path") if result.output is not None else None + if result.ok and (not isinstance(out, str) or not Path(out).exists()): + missing.append(clip_id) + return missing + + +def _write_16k(source: Path, dest: Path) -> None: + import soundfile + + if dest.exists(): + return + audio: Any + audio, rate_hz = soundfile.read(str(source), dtype="float64") + if rate_hz != 16000: + audio = resample(audio, int(rate_hz), 16000) + peak = float(abs(audio).max()) + if peak > 1.0: + audio = audio / peak + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(f".{dest.name}.tmp.wav") + soundfile.write(str(tmp), audio, 16000, subtype="PCM_16") + tmp.replace(dest) + + +def asr_requests( + workspace: Workspace, + clips: Sequence[Clip], + language: str, + vae_results: Mapping[str, WorkResult] | None = None, +) -> list[WorkRequest]: + """Whisper requests for originals and reconstructions of clips with references. + + Clips without a usable reconstruction (failed VAE item or missing file) get no + requests at all: the CER comparison is paired, and scoring counts them as skipped. + """ + paths = b2_paths(workspace) + requests: list[WorkRequest] = [] + for clip in clips: + if clip.language != language or not clip.reference: + continue + if not _usable_reconstruction(workspace, clip, vae_results): + continue + for kind, source in ( + ("orig", _orig_path(workspace, clip)), + ("vae", _vae_path(workspace, clip)), + ): + dest = paths.asr16k / f"{clip.clip_id}__{kind}.wav" + _write_16k(source, dest) + requests.append( + WorkRequest(f"{clip.clip_id}__{kind}", str(dest), {"language": language}) + ) + return requests + + +def quality_requests( + workspace: Workspace, + clips: Sequence[Clip], + vae_results: Mapping[str, WorkResult] | None = None, +) -> list[WorkRequest]: + """Audiobox requests for both members of every clip with a usable reconstruction.""" + requests: list[WorkRequest] = [] + for clip in clips: + if not _usable_reconstruction(workspace, clip, vae_results): + continue + requests.append(WorkRequest(f"{clip.clip_id}__orig", str(_orig_path(workspace, clip)))) + requests.append(WorkRequest(f"{clip.clip_id}__vae", str(_vae_path(workspace, clip)))) + return requests + + +# ------------------------------------------------------------------------------ scoring + + +def clip_metrics(orig: Any, orig_rate_hz: int, recon: Any, recon_rate_hz: int) -> dict[str, float]: + """Objective metrics for one original / reconstruction pair. + + ``orig`` and ``recon`` are 1-D float arrays (Any: numpy arrays, numpy is a bench extra). + Both are compared at 16 kHz (mel @16k); when the original is 48 kHz they are also + compared at 48 kHz (mel @48k, high-band LSD and energy ratio). The reconstruction + is delay-aligned to the original first (cross-correlation at 16 kHz, ±50 ms). + """ + o16 = resample(orig, orig_rate_hz, 16000) + r16 = resample(recon, recon_rate_hz, 16000) + delay = estimate_delay_samples(o16, r16, MAX_DELAY_MS * 16) + a16, b16 = align(o16, r16, delay) + out: dict[str, float] = { + "delay_ms": delay / 16, + "mel16_l1": log_mel_l1(a16, b16, 16000, 1024, 256, 80), + } + if orig_rate_hz == 48000: + r48 = resample(recon, recon_rate_hz, 48000) + a48, b48 = align(orig, r48, delay * 3) + out["mel48_l1"] = log_mel_l1(a48, b48, 48000, 2048, 480, 128) + out["hb_lsd_db"] = band_log_spectral_distance( + a48, b48, 48000, *HIGH_BAND_HZ, active_within_db=ACTIVE_WITHIN_DB + ) + out["hb_energy_ratio"] = band_energy_ratio(a48, b48, 48000, *HIGH_BAND_HZ) + return out + + +def _mean(values: Sequence[float]) -> float | None: + return sum(values) / len(values) if values else None + + +def _result_text(result: WorkResult | None) -> str | None: + if result is None or result.output is None: + return None + raw = result.output.get("text") + return raw if isinstance(raw, str) else None + + +def _scoring_tokens(text: str, language: str) -> list[str]: + """Content tokens: Turkish pipeline without fillers, or the English normaliser.""" + if language == "tr": + return strip_fillers(normalize_for_scoring(text)) + return normalize_english(text) + + +def _usable_reconstruction( + workspace: Workspace, clip: Clip, vae_results: Mapping[str, WorkResult] | None +) -> bool: + """A reconstruction counts only if its worker result is ok *and* the file exists.""" + if vae_results is not None: + result = vae_results.get(clip.clip_id) + if result is None or not result.ok: + return False + return _vae_path(workspace, clip).exists() + + +def score_b2( + workspace: Workspace, + clips: Sequence[Clip], + asr_results: Mapping[str, WorkResult], + quality_results: Mapping[str, WorkResult], + vae_results: Mapping[str, WorkResult] | None = None, +) -> dict[str, object]: + """Aggregate per-subset metrics for the VAE and the band-16k baseline. + + Clips without a usable reconstruction (failed worker result or missing file) are + skipped and counted (``skipped``); every subset still gets a row (``n_scored`` may be 0). + Quality deltas are averaged over *paired* clips (both members scored); CER compares + original vs reconstruction only on clips where both transcriptions exist. + """ + import soundfile + + per_subset: dict[str, dict[str, list[float]]] = {} + cer_pairs: dict[str, dict[str, list[Any]]] = {} + skipped: dict[str, int] = {} + for clip in clips: + vae_file = _vae_path(workspace, clip) + if not _usable_reconstruction(workspace, clip, vae_results): + skipped["missing_reconstruction"] = skipped.get("missing_reconstruction", 0) + 1 + continue + bucket = per_subset.setdefault(clip.layer, {}) + bucket.setdefault("_n_scored", []).append(1.0) + orig, orig_rate = soundfile.read(str(_orig_path(workspace, clip)), dtype="float64") + recon, recon_rate = soundfile.read(str(vae_file), dtype="float64") + for key, value in clip_metrics(orig, int(orig_rate), recon, int(recon_rate)).items(): + bucket.setdefault(f"vae_{key}", []).append(value) + if int(orig_rate) == 48000: + band = resample(resample(orig, 48000, 16000), 16000, 48000) + for key, value in clip_metrics(orig, 48000, band, 48000).items(): + bucket.setdefault(f"band16k_{key}", []).append(value) + pair = [quality_results.get(f"{clip.clip_id}__{kind}") for kind in ("orig", "vae")] + if all(q is not None and q.output is not None for q in pair): + non_numeric = False + for axis in ("PQ", "CE"): + scored = [q.output.get(axis) for q in pair if q is not None and q.output] + numeric = [ + v for v in scored if isinstance(v, int | float) and not isinstance(v, bool) + ] + if len(numeric) != 2: + non_numeric = True + else: + orig_score, vae_score = float(numeric[0]), float(numeric[1]) + bucket.setdefault(f"orig_{axis}", []).append(orig_score) + bucket.setdefault(f"vae_{axis}", []).append(vae_score) + bucket.setdefault(f"delta_{axis}", []).append(vae_score - orig_score) + if non_numeric: # counted once per clip, not per axis + skipped["non_numeric_quality"] = skipped.get("non_numeric_quality", 0) + 1 + else: + skipped["missing_quality_pair"] = skipped.get("missing_quality_pair", 0) + 1 + if clip.reference: + orig_text = _result_text(asr_results.get(f"{clip.clip_id}__orig")) + vae_text = _result_text(asr_results.get(f"{clip.clip_id}__vae")) + if orig_text is None or vae_text is None: + skipped["missing_asr_pair"] = skipped.get("missing_asr_pair", 0) + 1 + else: + ref_tokens = _scoring_tokens(clip.reference, clip.language) + for kind, text in (("orig", orig_text), ("vae", vae_text)): + cer_pairs.setdefault(clip.layer, {}).setdefault(kind, []).append( + cer_counts(ref_tokens, _scoring_tokens(text, clip.language)) + ) + for clip in clips: # subsets with nothing usable still get an (empty) row + per_subset.setdefault(clip.layer, {}) + summary: dict[str, dict[str, float | int | None]] = {} + for subset, values in sorted(per_subset.items()): + row: dict[str, float | int | None] = { + "n": sum(1 for c in clips if c.layer == subset), + "n_scored": len(values.pop("_n_scored", [])), + } + row.update({key: _mean(vals) for key, vals in sorted(values.items())}) + pairs = cer_pairs.get(subset) + if pairs: + row["cer_orig"] = corpus_rate(pairs["orig"]) + row["cer_vae"] = corpus_rate(pairs["vae"]) + row["n_cer_pairs"] = len(pairs["orig"]) + summary[subset] = row + return {"subsets": summary, "skipped": skipped} + + +# -------------------------------------------------------------------------------- panel + + +B2_QUESTIONS = ( + Question( + "ayni_ifade", + "A ve B aynı ifadeyi (fısıltı / gülme / bağırma / sözsüz ses / ton) taşıyor mu?", + "choice", + ("evet", "kısmen", "hayır"), + ), + Question( + "kalite_a", "A'nın ses kalitesi (1 kötü, 5 kusursuz)", "scale", ("1", "2", "3", "4", "5") + ), + Question( + "kalite_b", "B'nin ses kalitesi (1 kötü, 5 kusursuz)", "scale", ("1", "2", "3", "4", "5") + ), +) +#: Pre-registered acceptance (design §2.4): per expressive subset, the share of "evet" +#: ("kısmen" counts as *not* preserved) and the mean quality gap original − VAE. +MIN_PRESERVED_SHARE = 0.8 +MAX_QUALITY_GAP = 1.0 +PANEL_SUBSETS = ("tr", "whisper", "laughing", "projected", "nonverbal") + + +def _panel_subset(clip: Clip) -> str: + return "tr" if clip.layer.startswith("tr-") else clip.layer + + +def _key_path(workspace: Workspace, plan: B2Plan) -> Path: + return workspace.manifests / f"b2_panel_key-{plan.seed}.json" + + +def build_b2_panel( + workspace: Workspace, + clips: Sequence[Clip], + plan: B2Plan, + vae_results: Mapping[str, WorkResult] | None = None, +) -> Path: + """Write the blind 40-pair listening panel and its answer key. + + Per item the order of original and reconstruction is random and both are labelled + neutrally ("A", "B"); the key lives in ``manifests/`` and never in the HTML. When + either member peaks above full scale, the *same* gain is applied to both so clipping + does not bias the comparison; the gain is recorded in the key. + """ + import numpy as np + import soundfile + + rng = random.Random(plan.seed + 2) + chosen: list[Clip] = [] + excluded: dict[str, int] = {} + for subset in PANEL_SUBSETS: + everything = sorted( + (c for c in clips if _panel_subset(c) == subset), key=lambda c: c.clip_id + ) + pool = [c for c in everything if _usable_reconstruction(workspace, c, vae_results)] + if len(pool) < len(everything): + excluded[subset] = len(everything) - len(pool) + log.warning( + "B2 panel: %d %s clips lack a usable reconstruction", excluded[subset], subset + ) + chosen.extend(rng.sample(pool, min(plan.n_panel_per_subset, len(pool)))) + rng.shuffle(chosen) + panel_id = f"b2-{plan.seed}" + audio_dir = workspace.cache / "panel_audio" / panel_id + items: list[PanelItem] = [] + key: dict[str, dict[str, object]] = {} + for clip in chosen: + orig, orig_rate = soundfile.read(str(_orig_path(workspace, clip)), dtype="float64") + recon, recon_rate = soundfile.read(str(_vae_path(workspace, clip)), dtype="float64") + peak = max(float(np.max(np.abs(orig))), float(np.max(np.abs(recon))), 1e-12) + gain = 1.0 / peak if peak > 1.0 else 1.0 + first_is_orig = rng.random() < 0.5 + members = [("orig", orig, orig_rate), ("vae", recon, recon_rate)] + if not first_is_orig: + members.reverse() + refs: list[AudioRef] = [] + for label, (_kind, audio, rate_hz) in zip(("A", "B"), members, strict=True): + # Always rewritten: a cached A/B file could disagree with a re-drawn key. + dest = audio_dir / f"{clip.clip_id}__{label}.flac" + dest.parent.mkdir(parents=True, exist_ok=True) + soundfile.write(str(dest), audio * gain, int(rate_hz), subtype="PCM_16") + refs.append(AudioRef(label, dest)) + key[clip.clip_id] = { + "subset": _panel_subset(clip), + "a_is": "orig" if first_is_orig else "vae", + "gain": gain, + } + items.append( + PanelItem( + item_id=clip.clip_id, + audios=tuple(refs), + text=clip.reference or "", + info=f"{clip.layer} · {clip.duration_ms / 1000:.1f} s", + ) + ) + spec = PanelSpec( + panel_id=panel_id, + title="B2 — Ses kodlayıcı dinleme testi", + instructions=( + "Her satırda A ve B'yi dinleyin. Biri orijinal kayıt, diğeri modelden geçirilmiş " + "hâli; hangisinin hangisi olduğu rastgele ve gizli. İkisinin aynı ifadeyi " + "(fısıltı, gülme, bağırma, iç çekme vb.) taşıyıp taşımadığını ve her birinin " + "kalitesini işaretleyin. Kulaklıkla dinlemeniz önerilir." + ), + questions=B2_QUESTIONS, + items=tuple(items), + ) + atomic_write_json( + _key_path(workspace, plan), + { + "panel_id": panel_id, + "items": key, + "excluded_missing_reconstruction": excluded, + "min_per_subset": plan.n_panel_per_subset, + }, + ) + return write_panel(spec, workspace.panels / panel_id) + + +def score_b2_panel( + results: Sequence[PanelResult], + key: Mapping[str, Mapping[str, object]], + *, + min_per_subset: int, +) -> dict[str, object]: + """Un-blind the panel answers and apply the pre-registered acceptance rule. + + Returns per subset: answered items, share of "evet" (preserved), share of "kısmen", + mean quality of original and reconstruction, the mean gap (orig − vae) and whether the + subset passes (``preserved_share >= 0.8`` and ``quality_gap <= 1``; the ``tr`` subset is + reported but carries no expressive-preservation requirement). ``all_pass`` requires all + four expressive subsets to be present, *every* key item of every subset to be fully + answered, at least ``min_per_subset`` items per expressive subset (pre-registered: + the panel size per subset, 8), and every subset to pass — thin or partial evidence + never passes. + + Raises: + ValueError: if an answered item is missing from the key. + """ + rows: dict[str, dict[str, list[float]]] = {} + for result in results: + if result.item_id not in key: + raise ValueError(f"panel item {result.item_id} not in the answer key") + entry = key[result.item_id] + same = result.answers.get("ayni_ifade", "") + qa, qb = result.answers.get("kalite_a", ""), result.answers.get("kalite_b", "") + if not same or not qa or not qb: + continue # unanswered rows do not count either way + orig_q, vae_q = ( + (float(qa), float(qb)) if entry["a_is"] == "orig" else (float(qb), float(qa)) + ) + bucket = rows.setdefault(str(entry["subset"]), {}) + bucket.setdefault("preserved", []).append(1.0 if same == "evet" else 0.0) + bucket.setdefault("partial", []).append(1.0 if same == "kısmen" else 0.0) + bucket.setdefault("orig_q", []).append(orig_q) + bucket.setdefault("vae_q", []).append(vae_q) + out: dict[str, dict[str, object]] = {} + for subset, bucket in sorted(rows.items()): + preserved = _mean(bucket["preserved"]) or 0.0 + orig_q = _mean(bucket["orig_q"]) or 0.0 + vae_q = _mean(bucket["vae_q"]) or 0.0 + gap = orig_q - vae_q + passes = gap <= MAX_QUALITY_GAP and (subset == "tr" or preserved >= MIN_PRESERVED_SHARE) + out[subset] = { + "n_answered": len(bucket["preserved"]), + "preserved_share": preserved, + "partial_share": _mean(bucket["partial"]), + "quality_orig": orig_q, + "quality_vae": vae_q, + "quality_gap": gap, + "passes": passes, + } + in_key: dict[str, int] = {} + for entry in key.values(): + in_key[str(entry["subset"])] = in_key.get(str(entry["subset"]), 0) + 1 + unanswered = { + subset: count - int(str(out.get(subset, {}).get("n_answered", 0))) + for subset, count in sorted(in_key.items()) + if count > int(str(out.get(subset, {}).get("n_answered", 0))) + } + expressive = [s for s in PANEL_SUBSETS if s != "tr"] + missing_subsets = [s for s in expressive if s not in out] + too_few = {s: in_key.get(s, 0) for s in expressive if in_key.get(s, 0) < min_per_subset} + return { + "subsets": out, + "rule": {"min_preserved_share": MIN_PRESERVED_SHARE, "max_quality_gap": MAX_QUALITY_GAP}, + "unanswered": unanswered, + "missing_subsets": missing_subsets, + "too_few_items": too_few, + "min_per_subset": min_per_subset, + "all_pass": not missing_subsets + and not unanswered + and not too_few + and all(bool(v["passes"]) for v in out.values()), + } diff --git a/src/nida/bench/clips.py b/src/nida/bench/clips.py new file mode 100644 index 0000000000000000000000000000000000000000..6b756b1aeb3d497c6688a41c3d2f4127bc36f31d --- /dev/null +++ b/src/nida/bench/clips.py @@ -0,0 +1,510 @@ +"""Clip records and deterministic clip selection for benchmarks B1 / B2. + +Selection functions are pure: they take already-parsed source metadata and a seed, and +return the chosen clips plus a quarantine list with a machine-readable reason for every +candidate that was rejected (docs/03-data-pipeline.md: nothing is dropped silently). +Candidates that are merely *not sampled* are not quarantined; only rule violations are. + +Ids follow the pipeline convention ``sha1(source_id, file_id, start_ms, end_ms)[:16]``. +""" + +from __future__ import annotations + +import hashlib +import random +from collections import defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Literal + +from nida.bench.textmetrics import FILLERS, has_digits, normalize_for_scoring +from nida.text.turkish import nfc + +__all__ = [ + "Clip", + "FleursRow", + "MediaSpeechItem", + "Quarantined", + "Selection", + "YodasUtt", + "clip_from_obj", + "clip_to_obj", + "make_clip_id", + "parse_yodas_utt_id", + "select_fleurs", + "select_mediaspeech", + "select_yodas_final", + "select_yodas_pool", + "text_has_filler", + "yodas_video_id", +] + +Language = Literal["tr", "en"] + +#: Gender labels in the FLEURS TSV; anything else is quarantined, never guessed. +FLEURS_GENDERS = frozenset({"MALE", "FEMALE"}) +#: Allowed mismatch between a YODAS duration and the span encoded in its utterance id. +YODAS_DURATION_TOLERANCE_MS = 50 + + +def make_clip_id(source_id: str, file_id: str, start_ms: int, end_ms: int) -> str: + """Content-addressed clip id (first 16 hex chars of SHA-1).""" + if start_ms < 0 or end_ms <= start_ms: + raise ValueError(f"invalid span {start_ms}..{end_ms} ms for {source_id}/{file_id}") + key = "\x1f".join((source_id, file_id, str(start_ms), str(end_ms))) + return hashlib.sha1(key.encode("utf-8")).hexdigest()[:16] + + +@dataclass(frozen=True) +class Clip: + """One benchmark clip. + + Attributes: + clip_id: ``make_clip_id(source_id, file_id, start_ms, end_ms)``. + source_id: Id in ``configs/sources.yaml``. + layer: Benchmark layer / subset name (e.g. ``"A"``, ``"whisper"``). + file_id: File or utterance id inside the source. + start_ms: Start inside the source file, milliseconds. + end_ms: End inside the source file, milliseconds (exclusive). + reference: Reference transcript (raw), or ``None`` when not yet available. + language: ISO 639-1 code. + meta: Free string metadata (gender, video id, style, ...). + """ + + clip_id: str + source_id: str + layer: str + file_id: str + start_ms: int + end_ms: int + reference: str | None + language: Language + meta: Mapping[str, str] = field(default_factory=dict) + + @property + def duration_ms(self) -> int: + return self.end_ms - self.start_ms + + @staticmethod + def build( + *, + source_id: str, + layer: str, + file_id: str, + start_ms: int, + end_ms: int, + reference: str | None, + language: Language, + meta: Mapping[str, str] | None = None, + ) -> Clip: + return Clip( + clip_id=make_clip_id(source_id, file_id, start_ms, end_ms), + source_id=source_id, + layer=layer, + file_id=file_id, + start_ms=start_ms, + end_ms=end_ms, + reference=None if reference is None else nfc(reference), + language=language, + meta=dict(meta or {}), + ) + + +@dataclass(frozen=True) +class Quarantined: + """A rejected candidate (docs/03-data-pipeline.md quarantine record).""" + + item_id: str + stage: str + reason: str + details: str = "" + + +@dataclass(frozen=True) +class Selection: + clips: tuple[Clip, ...] + quarantined: tuple[Quarantined, ...] + + +def _check_count(n: int) -> None: + if n < 1: + raise ValueError(f"requested clip count must be >= 1, got {n}") + + +def _check_duration_window(min_s: float, max_s: float) -> None: + if not 0 < min_s < max_s: + raise ValueError(f"invalid duration window {min_s}..{max_s} s") + + +# ------------------------------------------------------------------------------- FLEURS + + +@dataclass(frozen=True) +class FleursRow: + """One row of FLEURS ``.tsv`` (16 kHz audio).""" + + sentence_id: str + file_name: str + raw_transcription: str + num_samples: int + gender: str + + +def select_fleurs( + rows: Iterable[FleursRow], + n: int, + seed: int, + *, + min_s: float = 2.0, + max_s: float = 25.0, + source_id: str = "fleurs_tr", + layer: str = "A", +) -> Selection: + """Pick ``n`` FLEURS recordings: one per sentence id, gender-balanced. + + Rules: duration in ``[min_s, max_s]``; reference non-empty and without digits (number + verbalisation is not decided yet, R6); at most one recording per sentence id (several + speakers read the same sentence and the TSV has no speaker id). Gender balance: half + from ``MALE`` and half from ``FEMALE`` where possible, the remainder from whatever is left. + + Raises: + ValueError: on invalid arguments or if fewer than ``n`` eligible sentences exist. + """ + _check_count(n) + _check_duration_window(min_s, max_s) + quarantined: list[Quarantined] = [] + by_sentence: dict[str, list[FleursRow]] = defaultdict(list) + for row in rows: + duration_s = row.num_samples / 16000 + if row.gender not in FLEURS_GENDERS: + quarantined.append( + Quarantined(row.file_name, "select", "unexpected_gender", row.gender) + ) + elif not min_s <= duration_s <= max_s: + quarantined.append( + Quarantined(row.file_name, "select", "duration_out_of_range", f"{duration_s:.2f}s") + ) + elif not normalize_for_scoring(row.raw_transcription): + quarantined.append(Quarantined(row.file_name, "select", "empty_reference")) + elif has_digits(row.raw_transcription): + quarantined.append(Quarantined(row.file_name, "select", "reference_has_digits")) + else: + by_sentence[row.sentence_id].append(row) + rng = random.Random(seed) + one_per_sentence: list[FleursRow] = [] + for sentence_id in sorted(by_sentence): + recordings = sorted(by_sentence[sentence_id], key=lambda r: r.file_name) + one_per_sentence.append(rng.choice(recordings)) + if len(one_per_sentence) < n: + raise ValueError(f"only {len(one_per_sentence)} eligible FLEURS sentences, need {n}") + pools: dict[str, list[FleursRow]] = defaultdict(list) + for row in one_per_sentence: + pools[row.gender].append(row) + for pool in pools.values(): + rng.shuffle(pool) + chosen: list[FleursRow] = [] + half = n // 2 + for gender in ("MALE", "FEMALE"): + chosen.extend(pools[gender][:half]) + del pools[gender][:half] + rest = [row for gender in sorted(pools) for row in pools[gender]] + rng.shuffle(rest) + chosen.extend(rest[: n - len(chosen)]) + clips = tuple( + Clip.build( + source_id=source_id, + layer=layer, + file_id=row.file_name, + start_ms=0, + end_ms=round(row.num_samples / 16), + reference=row.raw_transcription, + language="tr", + meta={"gender": row.gender, "sentence_id": row.sentence_id}, + ) + for row in sorted(chosen, key=lambda r: r.file_name) + ) + return Selection(clips, tuple(quarantined)) + + +# --------------------------------------------------------------------------- MediaSpeech + + +@dataclass(frozen=True) +class MediaSpeechItem: + file_id: str # uuid stem of TR/.flac + text: str + duration_ms: int + + +def select_mediaspeech( + items: Iterable[MediaSpeechItem], + n: int, + seed: int, + *, + min_s: float = 2.0, + max_s: float = 20.0, + source_id: str = "mediaspeech_tr", + layer: str = "B", +) -> Selection: + """Pick ``n`` MediaSpeech segments with the same reference rules as FLEURS.""" + _check_count(n) + _check_duration_window(min_s, max_s) + quarantined: list[Quarantined] = [] + eligible: list[MediaSpeechItem] = [] + for item in sorted(items, key=lambda i: i.file_id): + duration_s = item.duration_ms / 1000 + if not min_s <= duration_s <= max_s: + quarantined.append( + Quarantined(item.file_id, "select", "duration_out_of_range", f"{duration_s:.2f}s") + ) + elif not normalize_for_scoring(item.text): + quarantined.append(Quarantined(item.file_id, "select", "empty_reference")) + elif has_digits(item.text): + quarantined.append(Quarantined(item.file_id, "select", "reference_has_digits")) + else: + eligible.append(item) + if len(eligible) < n: + raise ValueError(f"only {len(eligible)} eligible MediaSpeech items, need {n}") + chosen = random.Random(seed).sample(eligible, n) + clips = tuple( + Clip.build( + source_id=source_id, + layer=layer, + file_id=item.file_id, + start_ms=0, + end_ms=item.duration_ms, + reference=item.text, + language="tr", + ) + for item in sorted(chosen, key=lambda i: i.file_id) + ) + return Selection(clips, tuple(quarantined)) + + +# --------------------------------------------------------------------------------- YODAS + + +@dataclass(frozen=True) +class YodasUtt: + utt_id: str + caption: str + duration_ms: int + shard: str = "" # archive shard holding the audio ("" = unknown / single shard) + + +def yodas_video_id(utt_id: str) -> str: + """Video id of a ``